1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
|
/* SPDX-License-Identifier: GPL-2.0-or-later */
/* Copyright (C) 2026 Richard Qin */
/*
* tlb.cc — TLB shootdown implementation
*
* Uses RISC-V SBI Remote Fence extension to perform cross-hart
* TLB invalidation. SBI firmware handles the IPI delivery and
* synchronization — the call blocks until all target harts have
* completed their local sfence.vma.
*
* When a VmSpace is provided, we use its cpu_mask to target only
* the CPUs that have this address space loaded in satp. This avoids
* unnecessary IPI storms on unrelated cores.
*
* Reference: docs/specs/linux_abi_mm_plan.md §SMP §2.1
*/
#include <lume/tlb.h>
#include <lume/vmm.h>
#include <lume/config.h>
#include <arch/mmu.h>
#include <arch/tlb.h>
#include <arch/cpu.h>
/*
* If a shootdown range exceeds this many pages, do a full flush
* instead of per-page invalidation.
*/
static constexpr uint64 kShootdownPageThreshold = 32;
/*
* Build a cpu mask for all online CPUs (fallback when mm is null).
*/
namespace {
uint64 all_cpus_mask()
{
uint64 mask = 0;
for (uint32 i = 0; i < kMaxCpus; i++) {
mask |= (1ULL << i);
}
return mask;
}
/*
* Get the target cpu mask: use mm->cpu_mask when available,
* fall back to all_cpus_mask for kernel address space (mm == nullptr).
*/
uint64 target_cpu_mask(VmSpace* mm)
{
if (mm) {
uint64 mask = mm->cpu_mask.load(__ATOMIC_RELAXED);
/* Always include our own CPU — we may still have stale TLB */
mask |= (1ULL << arch::cpu::id());
return mask;
}
return all_cpus_mask();
}
} // namespace
void tlb_shootdown_range(VmSpace* mm, uint64 start, uint64 end)
{
if (start >= end) {
return;
}
uint64 num_pages = (end - start) / kPageSize;
uint64 cpu_mask = target_cpu_mask(mm);
if (num_pages > kShootdownPageThreshold) {
/* Full flush on target CPUs via arch backend */
arch::tlb::flush_remote(cpu_mask, 0, ~0ULL);
return;
}
/* Range flush on target CPUs via arch backend */
arch::tlb::flush_remote(cpu_mask, start, end - start);
}
void tlb_shootdown_page(VmSpace* mm, uint64 va)
{
uint64 cpu_mask = target_cpu_mask(mm);
arch::tlb::flush_remote(cpu_mask, va, kPageSize);
}
void tlb_shootdown_all()
{
uint64 cpu_mask = all_cpus_mask();
arch::tlb::flush_remote(cpu_mask, 0, ~0ULL);
}
|
