diff options
Diffstat (limited to 'na/arch/riscv/mm')
| -rw-r--r-- | na/arch/riscv/mm/init.cc | 241 | ||||
| -rw-r--r-- | na/arch/riscv/mm/pmap.cc | 71 | ||||
| -rw-r--r-- | na/arch/riscv/mm/tlb_md.cc | 17 |
3 files changed, 320 insertions, 9 deletions
diff --git a/na/arch/riscv/mm/init.cc b/na/arch/riscv/mm/init.cc new file mode 100644 index 0000000..3cebcd9 --- /dev/null +++ b/na/arch/riscv/mm/init.cc @@ -0,0 +1,241 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* Copyright (C) 2026 Richard Qin */ + +/* + * RISC-V early memory-management bootstrap. + * + * Mirrors Linux arch/riscv/mm/init.c — same file name, same C function + * names (setup_vm / relocate_enable_mmu / relocate_kernel), same global + * state (kernel_map, early_pg_dir, trampoline_pg_dir). head.S calls + * setup_vm immediately after BSS-clear, then later calls + * relocate_enable_mmu, which switches SATP, jumps to higher-half VA, + * and finally calls relocate_kernel to apply .rela.dyn. + * + * Differences from Linux: + * - KASLR offset is hard-wired to 0 for now (no pi/ subsystem) — the + * code path that would consume kernel_map.virt_offset is in place + * so adding pi/-based seed reading later only requires populating + * that one field. + * - Only 1 GiB superpages supported in create_pgd_mapping; sufficient + * for the early page tables. Finer-grained mappings come from + * vmm_init() once slab is up. + */ + +#include <lume/init.h> +#include <lume/setup.h> +#include <lume/elf.h> +#include <lume/kernel_map.h> +#include <lume/types.h> +#include <asm/page.h> +#include <asm/csr.h> +#include <asm/pgtable.h> +#include <asm/sections.h> + +/* + * Global kernel mapping descriptor — single source of truth for the + * runtime-vs-link address relationship. Lives in .data so its initial + * zero contents survive the BSS-clear that runs before setup_vm fills + * the fields. Read by VMM, page-fault handler, ptdump, etc. + * + * Linux equivalent: arch/riscv/mm/init.c::kernel_map + */ +struct kernel_mapping kernel_map __attribute__((section(".data"))); + +/* + * create_pgd_mapping — install one PGD entry covering a 1 GiB region. + * + * Only 1 GiB superpages are supported in this early variant; setup_vm + * needs nothing finer. Identical interface to Linux's same-named + * function in arch/riscv/mm/init.c, restricted in arity. + */ +extern "C" void create_pgd_mapping(pgd_t *pgdir, unsigned long va, + unsigned long pa, unsigned long sz, + unsigned long prot) +{ + /* This early variant only supports 1 GiB superpages. */ + if (sz != PGDIR_SIZE) { + return; + } + + /* SV39 PGD index = (va >> 30) & 0x1FF — see Volume II, §4.4.1. + * Each PGD entry is 8 bytes; pgdir[i] is the slot for that VPN[2]. */ + unsigned long idx = (va >> PGDIR_SHIFT) & (PTRS_PER_PGD - 1); + + /* PTE PPN field: physical page number shifted up by 10 (SV39 PTE + * layout: bits [53:10] = PPN, bits [9:0] = flags). */ + unsigned long ppn = pa >> PAGE_SHIFT; + pgdir[idx].pgd = (ppn << 10) | prot; +} + +/* + * setup_vm — fill kernel_map and build early_pg_dir + trampoline_pg_dir. + * + * Called from head.S after BSS is cleared, with: + * load_pa = runtime physical address of _start + * dtb_pa = physical address of the device tree blob (from a1) + * + * Mirrors Linux arch/riscv/mm/init.c::setup_vm. The actual SATP write + * happens later in relocate_enable_mmu; here we only populate page + * tables and the kernel_map descriptor. + */ +extern "C" void __init setup_vm(uintptr_t load_pa, uintptr_t dtb_pa) +{ + (void)dtb_pa; /* KASLR pi/ subsystem will consume this later. */ + + /* --- 1. Fill kernel_map ----------------------------------------- */ + + /* KASLR offset = 0 in this configuration. */ + kernel_map.virt_offset = 0; + kernel_map.virt_addr = KERNEL_LINK_ADDR + kernel_map.virt_offset; + kernel_map.phys_addr = load_pa; + kernel_map.size = (uintptr_t)_end - (uintptr_t)_start; + kernel_map.page_offset = PAGE_OFFSET; + + /* Mirrors Linux: va_pa_offset is the constant added to a physical + * address to obtain its direct-map VA. */ + kernel_map.va_pa_offset = PAGE_OFFSET - load_pa; + + /* va_kernel_pa_offset is the constant added to a load PA to obtain + * the kernel's link VA (used by relocate_enable_mmu to fix sp/ra). */ + kernel_map.va_kernel_pa_offset = kernel_map.virt_addr - load_pa; + + /* --- 2. Build early_pg_dir ------------------------------------- * + * + * SV39 root page table — 4 entries cover identity + higher-half for + * both MMIO (PA 0x0000_0000, 1 GiB) and RAM (PA 0x8000_0000, 1 GiB): + * + * [0] identity PA 0x0000_0000 → VA 0x0000_0000 + * [2] identity PA 0x8000_0000 → VA 0x8000_0000 + * [256] high-half PA 0x0000_0000 → VA 0xFFFFFFC000000000 + * [258] high-half PA 0x8000_0000 → VA 0xFFFFFFC080000000 + * + * Identity mappings keep PC valid right after csrw satp. High-half + * mappings let the post-MMU `jr` to .Lhigher_half resolve. */ + create_pgd_mapping(early_pg_dir, 0x00000000UL, 0x00000000UL, + PGDIR_SIZE, PAGE_KERNEL_EXEC); + create_pgd_mapping(early_pg_dir, 0x80000000UL, 0x80000000UL, + PGDIR_SIZE, PAGE_KERNEL_EXEC); + create_pgd_mapping(early_pg_dir, PAGE_OFFSET + 0x00000000UL, + 0x00000000UL, PGDIR_SIZE, PAGE_KERNEL_EXEC); + create_pgd_mapping(early_pg_dir, PAGE_OFFSET + 0x80000000UL, + 0x80000000UL, PGDIR_SIZE, PAGE_KERNEL_EXEC); + + /* --- 3. Build trampoline_pg_dir -------------------------------- * + * + * The trampoline maps just the gigapage containing _start, both at + * its identity PA and at its higher-half VA. This is the page + * table SATP holds during the single instruction window between + * "MMU off" and "MMU on with early_pg_dir" — the PC must resolve + * under both, so we re-use the same two entries above. */ + create_pgd_mapping(trampoline_pg_dir, load_pa & PGDIR_MASK, + load_pa & PGDIR_MASK, PGDIR_SIZE, PAGE_KERNEL_EXEC); + create_pgd_mapping(trampoline_pg_dir, + (kernel_map.virt_addr) & PGDIR_MASK, + load_pa & PGDIR_MASK, PGDIR_SIZE, PAGE_KERNEL_EXEC); +} + +/* + * relocate_kernel — walk .rela.dyn and apply R_RISCV_RELATIVE entries. + * + * Called from relocate_enable_mmu *after* the SATP switch and the jump + * to higher-half VA — meaning __rela_dyn_start / __rela_dyn_end resolve + * to valid higher-half pointers we can dereference directly. + * + * Mirrors Linux arch/riscv/mm/init.c::relocate_kernel. Key invariant: + * even when kaslr_offset == 0, the slot store MUST still happen — the + * linker leaves R_RISCV_RELATIVE slots zero-filled at link time, and + * only relocate_kernel writes the link-time addend into them. Without + * this pass, .init_array entries (and every other relocated symbol) + * are NULL pointers. + */ +void __init relocate_kernel(void) +{ + auto *rela = (Elf64_Rela *)__rela_dyn_start; + auto *rela_end = (Elf64_Rela *)__rela_dyn_end; + + const uint64 kaslr_offset = kernel_map.virt_offset; + + for (; rela < rela_end; ++rela) { + if (ELF64_R_TYPE(rela->r_info) != R_RISCV_RELATIVE) { + continue; + } + auto *slot = (Elf64_Addr *)(rela->r_offset + kaslr_offset); + *slot = (Elf64_Addr)(rela->r_addend) + kaslr_offset; + } +} + +/* + * relocate_enable_mmu — switch SATP, jump to higher-half, apply relocations. + * + * Called from head.S after setup_vm has filled kernel_map and built the + * page tables. Equivalent to Linux's same-named asmlinkage routine but + * written in C++ so it can share kernel_map / pgtable definitions with + * setup_vm directly. + * + * Three logical steps: + * + * 1. csrw satp ← trampoline_pg_dir. PC continues running at the + * identity PA, which the trampoline still maps; the high-half PA + * mapping is now also live. + * + * 2. la t0, .Lvirt; add t0, t0, va_kernel_pa_offset; jr t0. + * A PC-relative `la` under medany yields the runtime PA of the + * label; adding va_kernel_pa_offset (= link_va - load_pa) converts + * it to the link-time higher-half VA, which the trampoline maps. + * The `jr` then atomically pushes PC into higher-half space. + * + * 3. csrw satp ← early_pg_dir. We're now executing at higher-half + * VA, so the trampoline can be retired in favour of the full + * early page table. sp / ra / gp must be slid to higher-half too, + * otherwise the next ret-or-call uses physical-low addresses + * whose mapping is about to disappear when vmm_init drops the + * identity entries. + * + * 4. Call relocate_kernel — now safe, because all link-time VA + * pointers in __rela_dyn_start[..] dereference correctly. + */ +extern "C" void __init relocate_enable_mmu(pgd_t *pgdir) +{ + (void)pgdir; /* head.S passes early_pg_dir for documentation; we + * read the global symbol directly inside the asm. */ + + const unsigned long va_kernel_pa_offset = kernel_map.va_kernel_pa_offset; + const unsigned long satp_mode = SATP_MODE_SV39; + + asm volatile ( + /* Step 1: csrw satp <- trampoline */ + "la t0, trampoline_pg_dir\n" + "srli t0, t0, 12\n" + "or t0, t0, %[mode]\n" + "csrw satp, t0\n" + "sfence.vma\n" + + /* Step 2: jump to higher-half VA */ + "la t0, 1f\n" + "add t0, t0, %[off]\n" + "jr t0\n" + "1:\n" + + /* Step 3a: csrw satp <- early_pg_dir (now in higher-half) */ + "la t0, early_pg_dir\n" + "srli t0, t0, 12\n" + "or t0, t0, %[mode]\n" + "csrw satp, t0\n" + "sfence.vma\n" + + /* Step 3b: slide sp / ra / gp to higher-half */ + "add sp, sp, %[off]\n" + "add ra, ra, %[off]\n" + ".option push\n" + ".option norelax\n" + "la gp, __global_pointer$\n" + ".option pop\n" + : + : [off] "r"(va_kernel_pa_offset), + [mode] "r"(satp_mode) + : "t0", "memory" + ); + + /* Step 4: apply .rela.dyn now that VA dereferences work. */ + relocate_kernel(); +} diff --git a/na/arch/riscv/mm/pmap.cc b/na/arch/riscv/mm/pmap.cc index ae21d0e..f0a6848 100644 --- a/na/arch/riscv/mm/pmap.cc +++ b/na/arch/riscv/mm/pmap.cc @@ -21,6 +21,7 @@ #include <lume/memblock.h> #include <arch/mmu.h> #include <lume/klog.h> +#include <lume/errno.h> namespace pmap { @@ -45,6 +46,13 @@ inline constexpr uint64 PTE_COW = 1ULL << 8; inline constexpr int kLevels = 3; inline constexpr int kPtePerPage = 512; +/* SV39: upper half of VA space (kernel) starts at L2 index 256 */ +inline constexpr int kKernelPgdStart = kPtePerPage / 2; + +/* SATP PPN mask (44 bits) and page shift */ +inline constexpr uint64 kSatpPpnMask = 0xFFFFFFFFFFFULL; +inline constexpr int kPageShift = 12; + /* Extract VPN[level] from a virtual address */ static inline uint64 vpn(uint64 va, int level) { @@ -101,13 +109,43 @@ static uint64 alloc_pt_page() uint64 create() { - return alloc_pt_page(); + uint64 new_root = alloc_pt_page(); + if (!new_root) return 0; + + /* Copy kernel mappings from the current kernel page table. + * In SV39, entries 256..511 (VPN[2] >= 256) cover the upper + * half of the 64-bit VA space where the kernel lives. + * Without these, switching to the new page table would + * immediately fault on any kernel VA access. + * + * Only do this after the runtime page table is active — + * during early boot the bootstrap PT doesn't cover all of + * RAM so pa_to_ptr on the kernel root would fault. */ + if (g_runtime_mapping_ready) { + uint64 kernel_satp; + asm volatile("csrr %0, satp" : "=r"(kernel_satp)); + uint64 kernel_root_pa = (kernel_satp & kSatpPpnMask) << kPageShift; + + uint64 *new_l2 = pa_to_ptr(new_root); + uint64 *kern_l2 = pa_to_ptr(kernel_root_pa); + + for (int i = kKernelPgdStart; i < kPtePerPage; i++) { + new_l2[i] = kern_l2[i]; + } + } + + return new_root; } void destroy(uint64 root_pa) { uint64 *l2 = pa_to_ptr(root_pa); - for (int i = 0; i < kPtePerPage; i++) { + + /* Only free the user half (entries 0..255). + * Entries 256..511 are shared kernel PGD pointers copied + * from the kernel page table — freeing them would corrupt + * the kernel's own mappings. */ + for (int i = 0; i < kKernelPgdStart; i++) { if (!(l2[i] & PTE_V)) continue; /* Non-leaf L2 entry? (no R/W/X bits = pointer to L1 table) */ @@ -139,7 +177,7 @@ int map(uint64 root_pa, uint64 va, uint64 pa, uint64 perm) if (!(table[idx] & PTE_V)) { uint64 child_pa = alloc_pt_page(); if (!child_pa) - return -12; /* -ENOMEM */ + return -ENOMEM; /* -ENOMEM */ table[idx] = pa_to_pte(child_pa, PTE_V); } table = pa_to_ptr(pte_to_pa(table[idx])); @@ -156,7 +194,7 @@ int map_2mb(uint64 root_pa, uint64 va, uint64 pa, uint64 perm) /* VA and PA must be 2MB-aligned */ constexpr uint64 k2MB = 2ULL * 1024 * 1024; if ((va & (k2MB - 1)) != 0 || (pa & (k2MB - 1)) != 0) { - return -22; /* -EINVAL */ + return -EINVAL; /* -EINVAL */ } uint64 *table = pa_to_ptr(root_pa); @@ -166,7 +204,7 @@ int map_2mb(uint64 root_pa, uint64 va, uint64 pa, uint64 perm) if (!(table[idx2] & PTE_V)) { uint64 child_pa = alloc_pt_page(); if (!child_pa) { - return -12; /* -ENOMEM */ + return -ENOMEM; /* -ENOMEM */ } table[idx2] = pa_to_pte(child_pa, PTE_V); } @@ -197,7 +235,7 @@ int protect(uint64 root_pa, uint64 va, uint64 perm) uint64 idx = vpn(va, level); uint64 pte = table[idx]; if (!(pte & PTE_V)) - return -14; /* -EFAULT */ + return -EFAULT; /* -EFAULT */ if (pte & (PTE_R | PTE_W | PTE_X)) { uint64 pa = pte_to_pa(pte); table[idx] = pa_to_pte(pa, perm | PTE_V); @@ -205,7 +243,7 @@ int protect(uint64 root_pa, uint64 va, uint64 perm) } table = pa_to_ptr(pte_to_pa(pte)); } - return -14; + return -EFAULT; } int protect_range(uint64 root_pa, uint64 start, uint64 end, uint64 perm) @@ -250,6 +288,21 @@ bool lookup(uint64 root_pa, uint64 va, uint64 *pa_out) return false; } +uint64 lookup_pte(uint64 root_pa, uint64 va) +{ + uint64 *table = pa_to_ptr(root_pa); + for (int level = 2; level >= 0; level--) { + uint64 idx = vpn(va, level); + uint64 pte = table[idx]; + if (!(pte & PTE_V)) + return 0; + if (pte & (PTE_R | PTE_W | PTE_X)) + return pte; // leaf PTE raw value + table = pa_to_ptr(pte_to_pa(pte)); + } + return 0; +} + bool is_cow(uint64 root_pa, uint64 va) { uint64 *table = pa_to_ptr(root_pa); @@ -273,7 +326,7 @@ int make_cow(uint64 root_pa, uint64 va) uint64 idx = vpn(va, level); uint64 pte = table[idx]; if (!(pte & PTE_V)) - return -14; /* -EFAULT */ + return -EFAULT; /* -EFAULT */ if (pte & (PTE_R | PTE_W | PTE_X)) { /* Clear write bit, set COW bit */ pte &= ~PTE_W; @@ -283,7 +336,7 @@ int make_cow(uint64 root_pa, uint64 va) } table = pa_to_ptr(pte_to_pa(pte)); } - return -14; + return -EFAULT; } void activate(uint64 root_pa) diff --git a/na/arch/riscv/mm/tlb_md.cc b/na/arch/riscv/mm/tlb_md.cc new file mode 100644 index 0000000..ed9aa29 --- /dev/null +++ b/na/arch/riscv/mm/tlb_md.cc @@ -0,0 +1,17 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* Copyright (C) 2026 Richard Qin */ + +#include <arch/tlb.h> +#include <arch/sbi.h> + +namespace arch::tlb { + +void flush_remote(uint64 cpu_mask, uint64 start, uint64 size) { + if (size == ~0ULL) { + arch::sbi::sbi_remote_sfence_vma(cpu_mask, 0, 0, ~0UL); + } else { + arch::sbi::sbi_remote_sfence_vma(cpu_mask, 0, start, size); + } +} + +} // namespace arch::tlb |