diff options
Diffstat (limited to 'na/arch/riscv/mm')
| -rw-r--r-- | na/arch/riscv/mm/init.cc | 241 | ||||
| -rw-r--r-- | na/arch/riscv/mm/pmap.cc | 361 | ||||
| -rw-r--r-- | na/arch/riscv/mm/tlb_md.cc | 17 |
3 files changed, 0 insertions, 619 deletions
diff --git a/na/arch/riscv/mm/init.cc b/na/arch/riscv/mm/init.cc deleted file mode 100644 index 3cebcd9..0000000 --- a/na/arch/riscv/mm/init.cc +++ /dev/null @@ -1,241 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-or-later */ -/* Copyright (C) 2026 Richard Qin */ - -/* - * RISC-V early memory-management bootstrap. - * - * Mirrors Linux arch/riscv/mm/init.c — same file name, same C function - * names (setup_vm / relocate_enable_mmu / relocate_kernel), same global - * state (kernel_map, early_pg_dir, trampoline_pg_dir). head.S calls - * setup_vm immediately after BSS-clear, then later calls - * relocate_enable_mmu, which switches SATP, jumps to higher-half VA, - * and finally calls relocate_kernel to apply .rela.dyn. - * - * Differences from Linux: - * - KASLR offset is hard-wired to 0 for now (no pi/ subsystem) — the - * code path that would consume kernel_map.virt_offset is in place - * so adding pi/-based seed reading later only requires populating - * that one field. - * - Only 1 GiB superpages supported in create_pgd_mapping; sufficient - * for the early page tables. Finer-grained mappings come from - * vmm_init() once slab is up. - */ - -#include <lume/init.h> -#include <lume/setup.h> -#include <lume/elf.h> -#include <lume/kernel_map.h> -#include <lume/types.h> -#include <asm/page.h> -#include <asm/csr.h> -#include <asm/pgtable.h> -#include <asm/sections.h> - -/* - * Global kernel mapping descriptor — single source of truth for the - * runtime-vs-link address relationship. Lives in .data so its initial - * zero contents survive the BSS-clear that runs before setup_vm fills - * the fields. Read by VMM, page-fault handler, ptdump, etc. - * - * Linux equivalent: arch/riscv/mm/init.c::kernel_map - */ -struct kernel_mapping kernel_map __attribute__((section(".data"))); - -/* - * create_pgd_mapping — install one PGD entry covering a 1 GiB region. - * - * Only 1 GiB superpages are supported in this early variant; setup_vm - * needs nothing finer. Identical interface to Linux's same-named - * function in arch/riscv/mm/init.c, restricted in arity. - */ -extern "C" void create_pgd_mapping(pgd_t *pgdir, unsigned long va, - unsigned long pa, unsigned long sz, - unsigned long prot) -{ - /* This early variant only supports 1 GiB superpages. */ - if (sz != PGDIR_SIZE) { - return; - } - - /* SV39 PGD index = (va >> 30) & 0x1FF — see Volume II, §4.4.1. - * Each PGD entry is 8 bytes; pgdir[i] is the slot for that VPN[2]. */ - unsigned long idx = (va >> PGDIR_SHIFT) & (PTRS_PER_PGD - 1); - - /* PTE PPN field: physical page number shifted up by 10 (SV39 PTE - * layout: bits [53:10] = PPN, bits [9:0] = flags). */ - unsigned long ppn = pa >> PAGE_SHIFT; - pgdir[idx].pgd = (ppn << 10) | prot; -} - -/* - * setup_vm — fill kernel_map and build early_pg_dir + trampoline_pg_dir. - * - * Called from head.S after BSS is cleared, with: - * load_pa = runtime physical address of _start - * dtb_pa = physical address of the device tree blob (from a1) - * - * Mirrors Linux arch/riscv/mm/init.c::setup_vm. The actual SATP write - * happens later in relocate_enable_mmu; here we only populate page - * tables and the kernel_map descriptor. - */ -extern "C" void __init setup_vm(uintptr_t load_pa, uintptr_t dtb_pa) -{ - (void)dtb_pa; /* KASLR pi/ subsystem will consume this later. */ - - /* --- 1. Fill kernel_map ----------------------------------------- */ - - /* KASLR offset = 0 in this configuration. */ - kernel_map.virt_offset = 0; - kernel_map.virt_addr = KERNEL_LINK_ADDR + kernel_map.virt_offset; - kernel_map.phys_addr = load_pa; - kernel_map.size = (uintptr_t)_end - (uintptr_t)_start; - kernel_map.page_offset = PAGE_OFFSET; - - /* Mirrors Linux: va_pa_offset is the constant added to a physical - * address to obtain its direct-map VA. */ - kernel_map.va_pa_offset = PAGE_OFFSET - load_pa; - - /* va_kernel_pa_offset is the constant added to a load PA to obtain - * the kernel's link VA (used by relocate_enable_mmu to fix sp/ra). */ - kernel_map.va_kernel_pa_offset = kernel_map.virt_addr - load_pa; - - /* --- 2. Build early_pg_dir ------------------------------------- * - * - * SV39 root page table — 4 entries cover identity + higher-half for - * both MMIO (PA 0x0000_0000, 1 GiB) and RAM (PA 0x8000_0000, 1 GiB): - * - * [0] identity PA 0x0000_0000 → VA 0x0000_0000 - * [2] identity PA 0x8000_0000 → VA 0x8000_0000 - * [256] high-half PA 0x0000_0000 → VA 0xFFFFFFC000000000 - * [258] high-half PA 0x8000_0000 → VA 0xFFFFFFC080000000 - * - * Identity mappings keep PC valid right after csrw satp. High-half - * mappings let the post-MMU `jr` to .Lhigher_half resolve. */ - create_pgd_mapping(early_pg_dir, 0x00000000UL, 0x00000000UL, - PGDIR_SIZE, PAGE_KERNEL_EXEC); - create_pgd_mapping(early_pg_dir, 0x80000000UL, 0x80000000UL, - PGDIR_SIZE, PAGE_KERNEL_EXEC); - create_pgd_mapping(early_pg_dir, PAGE_OFFSET + 0x00000000UL, - 0x00000000UL, PGDIR_SIZE, PAGE_KERNEL_EXEC); - create_pgd_mapping(early_pg_dir, PAGE_OFFSET + 0x80000000UL, - 0x80000000UL, PGDIR_SIZE, PAGE_KERNEL_EXEC); - - /* --- 3. Build trampoline_pg_dir -------------------------------- * - * - * The trampoline maps just the gigapage containing _start, both at - * its identity PA and at its higher-half VA. This is the page - * table SATP holds during the single instruction window between - * "MMU off" and "MMU on with early_pg_dir" — the PC must resolve - * under both, so we re-use the same two entries above. */ - create_pgd_mapping(trampoline_pg_dir, load_pa & PGDIR_MASK, - load_pa & PGDIR_MASK, PGDIR_SIZE, PAGE_KERNEL_EXEC); - create_pgd_mapping(trampoline_pg_dir, - (kernel_map.virt_addr) & PGDIR_MASK, - load_pa & PGDIR_MASK, PGDIR_SIZE, PAGE_KERNEL_EXEC); -} - -/* - * relocate_kernel — walk .rela.dyn and apply R_RISCV_RELATIVE entries. - * - * Called from relocate_enable_mmu *after* the SATP switch and the jump - * to higher-half VA — meaning __rela_dyn_start / __rela_dyn_end resolve - * to valid higher-half pointers we can dereference directly. - * - * Mirrors Linux arch/riscv/mm/init.c::relocate_kernel. Key invariant: - * even when kaslr_offset == 0, the slot store MUST still happen — the - * linker leaves R_RISCV_RELATIVE slots zero-filled at link time, and - * only relocate_kernel writes the link-time addend into them. Without - * this pass, .init_array entries (and every other relocated symbol) - * are NULL pointers. - */ -void __init relocate_kernel(void) -{ - auto *rela = (Elf64_Rela *)__rela_dyn_start; - auto *rela_end = (Elf64_Rela *)__rela_dyn_end; - - const uint64 kaslr_offset = kernel_map.virt_offset; - - for (; rela < rela_end; ++rela) { - if (ELF64_R_TYPE(rela->r_info) != R_RISCV_RELATIVE) { - continue; - } - auto *slot = (Elf64_Addr *)(rela->r_offset + kaslr_offset); - *slot = (Elf64_Addr)(rela->r_addend) + kaslr_offset; - } -} - -/* - * relocate_enable_mmu — switch SATP, jump to higher-half, apply relocations. - * - * Called from head.S after setup_vm has filled kernel_map and built the - * page tables. Equivalent to Linux's same-named asmlinkage routine but - * written in C++ so it can share kernel_map / pgtable definitions with - * setup_vm directly. - * - * Three logical steps: - * - * 1. csrw satp ← trampoline_pg_dir. PC continues running at the - * identity PA, which the trampoline still maps; the high-half PA - * mapping is now also live. - * - * 2. la t0, .Lvirt; add t0, t0, va_kernel_pa_offset; jr t0. - * A PC-relative `la` under medany yields the runtime PA of the - * label; adding va_kernel_pa_offset (= link_va - load_pa) converts - * it to the link-time higher-half VA, which the trampoline maps. - * The `jr` then atomically pushes PC into higher-half space. - * - * 3. csrw satp ← early_pg_dir. We're now executing at higher-half - * VA, so the trampoline can be retired in favour of the full - * early page table. sp / ra / gp must be slid to higher-half too, - * otherwise the next ret-or-call uses physical-low addresses - * whose mapping is about to disappear when vmm_init drops the - * identity entries. - * - * 4. Call relocate_kernel — now safe, because all link-time VA - * pointers in __rela_dyn_start[..] dereference correctly. - */ -extern "C" void __init relocate_enable_mmu(pgd_t *pgdir) -{ - (void)pgdir; /* head.S passes early_pg_dir for documentation; we - * read the global symbol directly inside the asm. */ - - const unsigned long va_kernel_pa_offset = kernel_map.va_kernel_pa_offset; - const unsigned long satp_mode = SATP_MODE_SV39; - - asm volatile ( - /* Step 1: csrw satp <- trampoline */ - "la t0, trampoline_pg_dir\n" - "srli t0, t0, 12\n" - "or t0, t0, %[mode]\n" - "csrw satp, t0\n" - "sfence.vma\n" - - /* Step 2: jump to higher-half VA */ - "la t0, 1f\n" - "add t0, t0, %[off]\n" - "jr t0\n" - "1:\n" - - /* Step 3a: csrw satp <- early_pg_dir (now in higher-half) */ - "la t0, early_pg_dir\n" - "srli t0, t0, 12\n" - "or t0, t0, %[mode]\n" - "csrw satp, t0\n" - "sfence.vma\n" - - /* Step 3b: slide sp / ra / gp to higher-half */ - "add sp, sp, %[off]\n" - "add ra, ra, %[off]\n" - ".option push\n" - ".option norelax\n" - "la gp, __global_pointer$\n" - ".option pop\n" - : - : [off] "r"(va_kernel_pa_offset), - [mode] "r"(satp_mode) - : "t0", "memory" - ); - - /* Step 4: apply .rela.dyn now that VA dereferences work. */ - relocate_kernel(); -} diff --git a/na/arch/riscv/mm/pmap.cc b/na/arch/riscv/mm/pmap.cc deleted file mode 100644 index f0a6848..0000000 --- a/na/arch/riscv/mm/pmap.cc +++ /dev/null @@ -1,361 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-or-later */ -/* Copyright (C) 2026 Richard Qin */ - -/* - * pmap.cc — SV39 page table operations (MD layer) - * - * Implements the pmap namespace interface for RISC-V SV39. - * 3-level page table: VPN[2] (9 bits) -> VPN[1] (9 bits) -> VPN[0] (9 bits) - * Each level is a 4KB page containing 512 PTEs of 8 bytes each. - * - * Address conversion: all page table operations work with physical addresses. - * To read/write page table entries we must convert PA to VA via pa_to_va(). - * - * Reference: docs/specs/vmm.md - */ - -#include <lume/types.h> -#include <lume/addr.h> -#include <lume/config.h> -#include <lume/pmm.h> -#include <lume/memblock.h> -#include <arch/mmu.h> -#include <lume/klog.h> -#include <lume/errno.h> - -namespace pmap { - -/* - * Before the first pmap::activate(), CPU runs on entry.S bootstrap page table - * (fixed direct-map base). After activation, runtime (KASLR) base is valid. - */ -static bool g_runtime_mapping_ready = false; - -/* PTE bit definitions */ -inline constexpr uint64 PTE_V = 1ULL << 0; -inline constexpr uint64 PTE_R = 1ULL << 1; -inline constexpr uint64 PTE_W = 1ULL << 2; -inline constexpr uint64 PTE_X = 1ULL << 3; -inline constexpr uint64 PTE_U = 1ULL << 4; -inline constexpr uint64 PTE_G = 1ULL << 5; -inline constexpr uint64 PTE_A = 1ULL << 6; -inline constexpr uint64 PTE_D = 1ULL << 7; -inline constexpr uint64 PTE_COW = 1ULL << 8; - -/* SV39: 3 levels, 9 bits per level, 12-bit page offset */ -inline constexpr int kLevels = 3; -inline constexpr int kPtePerPage = 512; - -/* SV39: upper half of VA space (kernel) starts at L2 index 256 */ -inline constexpr int kKernelPgdStart = kPtePerPage / 2; - -/* SATP PPN mask (44 bits) and page shift */ -inline constexpr uint64 kSatpPpnMask = 0xFFFFFFFFFFFULL; -inline constexpr int kPageShift = 12; - -/* Extract VPN[level] from a virtual address */ -static inline uint64 vpn(uint64 va, int level) -{ - return (va >> (12 + 9 * level)) & 0x1FF; -} - -/* Extract PA from a PTE */ -static inline uint64 pte_to_pa(uint64 pte) -{ - return ((pte >> 10) & 0xFFFFFFFFFFFULL) << 12; -} - -/* Create a PTE from a PA and flags */ -static inline uint64 pa_to_pte(uint64 pa, uint64 flags) -{ - return ((pa >> 12) << 10) | flags; -} - -/* Convert PA to kernel VA for accessing page table contents */ -static inline uint64 *pa_to_ptr(uint64 pa) -{ - uint64 va = g_runtime_mapping_ready - ? pa_to_va(phys_addr(pa)).raw - : boot_pa_to_va(phys_addr(pa)).raw; - return reinterpret_cast<uint64 *>(va); -} - -/* Allocate a zeroed page for a page table level. - * During early boot (before PMM), uses memblock. - * After PMM init, uses the buddy allocator. - * Returns PA of the new page, or 0 on failure. */ -static uint64 alloc_pt_page() -{ - uint64 pa; - if (memblock_is_active()) { - pa = memblock_alloc(kPageSize, kPageSize); - } else { - Page *page = pmm_alloc_page(); - if (!page) { - return 0; - } - pa = page_to_pa(page).raw; - } - if (!pa) { - return 0; - } - /* Zero the page */ - uint64 *ptr = pa_to_ptr(pa); - for (int i = 0; i < kPtePerPage; i++) { - ptr[i] = 0; - } - return pa; -} - -uint64 create() -{ - uint64 new_root = alloc_pt_page(); - if (!new_root) return 0; - - /* Copy kernel mappings from the current kernel page table. - * In SV39, entries 256..511 (VPN[2] >= 256) cover the upper - * half of the 64-bit VA space where the kernel lives. - * Without these, switching to the new page table would - * immediately fault on any kernel VA access. - * - * Only do this after the runtime page table is active — - * during early boot the bootstrap PT doesn't cover all of - * RAM so pa_to_ptr on the kernel root would fault. */ - if (g_runtime_mapping_ready) { - uint64 kernel_satp; - asm volatile("csrr %0, satp" : "=r"(kernel_satp)); - uint64 kernel_root_pa = (kernel_satp & kSatpPpnMask) << kPageShift; - - uint64 *new_l2 = pa_to_ptr(new_root); - uint64 *kern_l2 = pa_to_ptr(kernel_root_pa); - - for (int i = kKernelPgdStart; i < kPtePerPage; i++) { - new_l2[i] = kern_l2[i]; - } - } - - return new_root; -} - -void destroy(uint64 root_pa) -{ - uint64 *l2 = pa_to_ptr(root_pa); - - /* Only free the user half (entries 0..255). - * Entries 256..511 are shared kernel PGD pointers copied - * from the kernel page table — freeing them would corrupt - * the kernel's own mappings. */ - for (int i = 0; i < kKernelPgdStart; i++) { - if (!(l2[i] & PTE_V)) - continue; - /* Non-leaf L2 entry? (no R/W/X bits = pointer to L1 table) */ - if (!(l2[i] & (PTE_R | PTE_W | PTE_X))) { - uint64 l1_pa = pte_to_pa(l2[i]); - uint64 *l1 = pa_to_ptr(l1_pa); - for (int j = 0; j < kPtePerPage; j++) { - if (!(l1[j] & PTE_V)) - continue; - if (!(l1[j] & (PTE_R | PTE_W | PTE_X))) { - /* L0 page table */ - uint64 l0_pa = pte_to_pa(l1[j]); - pmm_free_page(pa_to_page(phys_addr(l0_pa))); - } - } - pmm_free_page(pa_to_page(phys_addr(l1_pa))); - } - } - pmm_free_page(pa_to_page(phys_addr(root_pa))); -} - -int map(uint64 root_pa, uint64 va, uint64 pa, uint64 perm) -{ - uint64 *table = pa_to_ptr(root_pa); - - /* Walk levels 2 -> 1, creating intermediate tables as needed */ - for (int level = 2; level > 0; level--) { - uint64 idx = vpn(va, level); - if (!(table[idx] & PTE_V)) { - uint64 child_pa = alloc_pt_page(); - if (!child_pa) - return -ENOMEM; /* -ENOMEM */ - table[idx] = pa_to_pte(child_pa, PTE_V); - } - table = pa_to_ptr(pte_to_pa(table[idx])); - } - - /* Level 0: install the leaf PTE */ - uint64 idx = vpn(va, 0); - table[idx] = pa_to_pte(pa, perm | PTE_V | PTE_A | PTE_D); - return 0; -} - -int map_2mb(uint64 root_pa, uint64 va, uint64 pa, uint64 perm) -{ - /* VA and PA must be 2MB-aligned */ - constexpr uint64 k2MB = 2ULL * 1024 * 1024; - if ((va & (k2MB - 1)) != 0 || (pa & (k2MB - 1)) != 0) { - return -EINVAL; /* -EINVAL */ - } - - uint64 *table = pa_to_ptr(root_pa); - - /* Walk L2: create intermediate table if needed */ - uint64 idx2 = vpn(va, 2); - if (!(table[idx2] & PTE_V)) { - uint64 child_pa = alloc_pt_page(); - if (!child_pa) { - return -ENOMEM; /* -ENOMEM */ - } - table[idx2] = pa_to_pte(child_pa, PTE_V); - } - table = pa_to_ptr(pte_to_pa(table[idx2])); - - /* L1: install leaf PTE (2MB mega page) */ - uint64 idx1 = vpn(va, 1); - table[idx1] = pa_to_pte(pa, perm | PTE_V | PTE_A | PTE_D); - return 0; -} - -void unmap(uint64 root_pa, uint64 va) -{ - uint64 *table = pa_to_ptr(root_pa); - for (int level = 2; level > 0; level--) { - uint64 idx = vpn(va, level); - if (!(table[idx] & PTE_V)) - return; - table = pa_to_ptr(pte_to_pa(table[idx])); - } - table[vpn(va, 0)] = 0; -} - -int protect(uint64 root_pa, uint64 va, uint64 perm) -{ - uint64 *table = pa_to_ptr(root_pa); - for (int level = 2; level >= 0; level--) { - uint64 idx = vpn(va, level); - uint64 pte = table[idx]; - if (!(pte & PTE_V)) - return -EFAULT; /* -EFAULT */ - if (pte & (PTE_R | PTE_W | PTE_X)) { - uint64 pa = pte_to_pa(pte); - table[idx] = pa_to_pte(pa, perm | PTE_V); - return 0; - } - table = pa_to_ptr(pte_to_pa(pte)); - } - return -EFAULT; -} - -int protect_range(uint64 root_pa, uint64 start, uint64 end, uint64 perm) -{ - if (end <= start) { - return 0; - } - - uint64 va = page_align_down(start); - uint64 va_end = page_align_up(end); - - for (; va < va_end; va += kPageSize) { - int ret = protect(root_pa, va, perm); - if (ret == -14) { - continue; /* unmapped page: skip */ - } - if (ret < 0) { - return ret; - } - } - - arch::mmu::flush_tlb_all(); - return 0; -} - -bool lookup(uint64 root_pa, uint64 va, uint64 *pa_out) -{ - uint64 *table = pa_to_ptr(root_pa); - for (int level = 2; level >= 0; level--) { - uint64 idx = vpn(va, level); - uint64 pte = table[idx]; - if (!(pte & PTE_V)) - return false; - /* Leaf PTE? (has R, W, or X) */ - if (pte & (PTE_R | PTE_W | PTE_X)) { - if (pa_out) - *pa_out = pte_to_pa(pte); - return true; - } - table = pa_to_ptr(pte_to_pa(pte)); - } - return false; -} - -uint64 lookup_pte(uint64 root_pa, uint64 va) -{ - uint64 *table = pa_to_ptr(root_pa); - for (int level = 2; level >= 0; level--) { - uint64 idx = vpn(va, level); - uint64 pte = table[idx]; - if (!(pte & PTE_V)) - return 0; - if (pte & (PTE_R | PTE_W | PTE_X)) - return pte; // leaf PTE raw value - table = pa_to_ptr(pte_to_pa(pte)); - } - return 0; -} - -bool is_cow(uint64 root_pa, uint64 va) -{ - uint64 *table = pa_to_ptr(root_pa); - for (int level = 2; level >= 0; level--) { - uint64 idx = vpn(va, level); - uint64 pte = table[idx]; - if (!(pte & PTE_V)) - return false; - if (pte & (PTE_R | PTE_W | PTE_X)) { - return (pte & PTE_COW) != 0; - } - table = pa_to_ptr(pte_to_pa(pte)); - } - return false; -} - -int make_cow(uint64 root_pa, uint64 va) -{ - uint64 *table = pa_to_ptr(root_pa); - for (int level = 2; level >= 0; level--) { - uint64 idx = vpn(va, level); - uint64 pte = table[idx]; - if (!(pte & PTE_V)) - return -EFAULT; /* -EFAULT */ - if (pte & (PTE_R | PTE_W | PTE_X)) { - /* Clear write bit, set COW bit */ - pte &= ~PTE_W; - pte |= PTE_COW; - table[idx] = pte; - return 0; - } - table = pa_to_ptr(pte_to_pa(pte)); - } - return -EFAULT; -} - -void activate(uint64 root_pa) -{ - arch::mmu::set_page_table(root_pa); - arch::mmu::flush_tlb_all(); - g_runtime_mapping_ready = true; -} - -uint64 vm_perm_to_pte(uint64 vm_perm) -{ - /* Translate MI VM_* flags to SV39 PTE bits. - * The MI flags are defined in <lume/vmm.h>. */ - uint64 pte = PTE_V | PTE_A | PTE_D; // Valid + Accessed + Dirty (avoid HW A/D faults) - if (vm_perm & (1ULL << 0)) pte |= PTE_R; // VM_READ - if (vm_perm & (1ULL << 1)) pte |= PTE_W; // VM_WRITE - if (vm_perm & (1ULL << 2)) pte |= PTE_X; // VM_EXEC - if (vm_perm & (1ULL << 3)) pte |= PTE_U; // VM_USER - return pte; -} - -} // namespace pmap diff --git a/na/arch/riscv/mm/tlb_md.cc b/na/arch/riscv/mm/tlb_md.cc deleted file mode 100644 index ed9aa29..0000000 --- a/na/arch/riscv/mm/tlb_md.cc +++ /dev/null @@ -1,17 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-or-later */ -/* Copyright (C) 2026 Richard Qin */ - -#include <arch/tlb.h> -#include <arch/sbi.h> - -namespace arch::tlb { - -void flush_remote(uint64 cpu_mask, uint64 start, uint64 size) { - if (size == ~0ULL) { - arch::sbi::sbi_remote_sfence_vma(cpu_mask, 0, 0, ~0UL); - } else { - arch::sbi::sbi_remote_sfence_vma(cpu_mask, 0, start, size); - } -} - -} // namespace arch::tlb |