/* SPDX-License-Identifier: GPL-2.0-or-later */ /* Copyright (C) 2026 Richard Qin */ /* * pmap.cc — SV39 page table operations (MD layer) * * Implements the pmap namespace interface for RISC-V SV39. * 3-level page table: VPN[2] (9 bits) -> VPN[1] (9 bits) -> VPN[0] (9 bits) * Each level is a 4KB page containing 512 PTEs of 8 bytes each. * * Address conversion: all page table operations work with physical addresses. * To read/write page table entries we must convert PA to VA via pa_to_va(). * * Reference: docs/specs/vmm.md */ #include #include #include #include #include #include #include #include namespace pmap { /* * Before the first pmap::activate(), CPU runs on entry.S bootstrap page table * (fixed direct-map base). After activation, runtime (KASLR) base is valid. */ static bool g_runtime_mapping_ready = false; /* PTE bit definitions */ inline constexpr uint64 PTE_V = 1ULL << 0; inline constexpr uint64 PTE_R = 1ULL << 1; inline constexpr uint64 PTE_W = 1ULL << 2; inline constexpr uint64 PTE_X = 1ULL << 3; inline constexpr uint64 PTE_U = 1ULL << 4; inline constexpr uint64 PTE_G = 1ULL << 5; inline constexpr uint64 PTE_A = 1ULL << 6; inline constexpr uint64 PTE_D = 1ULL << 7; inline constexpr uint64 PTE_COW = 1ULL << 8; /* SV39: 3 levels, 9 bits per level, 12-bit page offset */ inline constexpr int kLevels = 3; inline constexpr int kPtePerPage = 512; /* SV39: upper half of VA space (kernel) starts at L2 index 256 */ inline constexpr int kKernelPgdStart = kPtePerPage / 2; /* SATP PPN mask (44 bits) and page shift */ inline constexpr uint64 kSatpPpnMask = 0xFFFFFFFFFFFULL; inline constexpr int kPageShift = 12; /* Extract VPN[level] from a virtual address */ static inline uint64 vpn(uint64 va, int level) { return (va >> (12 + 9 * level)) & 0x1FF; } /* Extract PA from a PTE */ static inline uint64 pte_to_pa(uint64 pte) { return ((pte >> 10) & 0xFFFFFFFFFFFULL) << 12; } /* Create a PTE from a PA and flags */ static inline uint64 pa_to_pte(uint64 pa, uint64 flags) { return ((pa >> 12) << 10) | flags; } /* Convert PA to kernel VA for accessing page table contents */ static inline uint64 *pa_to_ptr(uint64 pa) { uint64 va = g_runtime_mapping_ready ? pa_to_va(phys_addr(pa)).raw : boot_pa_to_va(phys_addr(pa)).raw; return reinterpret_cast(va); } /* Allocate a zeroed page for a page table level. * During early boot (before PMM), uses memblock. * After PMM init, uses the buddy allocator. * Returns PA of the new page, or 0 on failure. */ static uint64 alloc_pt_page() { uint64 pa; if (memblock_is_active()) { pa = memblock_alloc(kPageSize, kPageSize); } else { Page *page = pmm_alloc_page(); if (!page) { return 0; } pa = page_to_pa(page).raw; } if (!pa) { return 0; } /* Zero the page */ uint64 *ptr = pa_to_ptr(pa); for (int i = 0; i < kPtePerPage; i++) { ptr[i] = 0; } return pa; } uint64 create() { uint64 new_root = alloc_pt_page(); if (!new_root) return 0; /* Copy kernel mappings from the current kernel page table. * In SV39, entries 256..511 (VPN[2] >= 256) cover the upper * half of the 64-bit VA space where the kernel lives. * Without these, switching to the new page table would * immediately fault on any kernel VA access. * * Only do this after the runtime page table is active — * during early boot the bootstrap PT doesn't cover all of * RAM so pa_to_ptr on the kernel root would fault. */ if (g_runtime_mapping_ready) { uint64 kernel_satp; asm volatile("csrr %0, satp" : "=r"(kernel_satp)); uint64 kernel_root_pa = (kernel_satp & kSatpPpnMask) << kPageShift; uint64 *new_l2 = pa_to_ptr(new_root); uint64 *kern_l2 = pa_to_ptr(kernel_root_pa); for (int i = kKernelPgdStart; i < kPtePerPage; i++) { new_l2[i] = kern_l2[i]; } } return new_root; } void destroy(uint64 root_pa) { uint64 *l2 = pa_to_ptr(root_pa); /* Only free the user half (entries 0..255). * Entries 256..511 are shared kernel PGD pointers copied * from the kernel page table — freeing them would corrupt * the kernel's own mappings. */ for (int i = 0; i < kKernelPgdStart; i++) { if (!(l2[i] & PTE_V)) continue; /* Non-leaf L2 entry? (no R/W/X bits = pointer to L1 table) */ if (!(l2[i] & (PTE_R | PTE_W | PTE_X))) { uint64 l1_pa = pte_to_pa(l2[i]); uint64 *l1 = pa_to_ptr(l1_pa); for (int j = 0; j < kPtePerPage; j++) { if (!(l1[j] & PTE_V)) continue; if (!(l1[j] & (PTE_R | PTE_W | PTE_X))) { /* L0 page table */ uint64 l0_pa = pte_to_pa(l1[j]); pmm_free_page(pa_to_page(phys_addr(l0_pa))); } } pmm_free_page(pa_to_page(phys_addr(l1_pa))); } } pmm_free_page(pa_to_page(phys_addr(root_pa))); } int map(uint64 root_pa, uint64 va, uint64 pa, uint64 perm) { uint64 *table = pa_to_ptr(root_pa); /* Walk levels 2 -> 1, creating intermediate tables as needed */ for (int level = 2; level > 0; level--) { uint64 idx = vpn(va, level); if (!(table[idx] & PTE_V)) { uint64 child_pa = alloc_pt_page(); if (!child_pa) return -ENOMEM; /* -ENOMEM */ table[idx] = pa_to_pte(child_pa, PTE_V); } table = pa_to_ptr(pte_to_pa(table[idx])); } /* Level 0: install the leaf PTE */ uint64 idx = vpn(va, 0); table[idx] = pa_to_pte(pa, perm | PTE_V | PTE_A | PTE_D); return 0; } int map_2mb(uint64 root_pa, uint64 va, uint64 pa, uint64 perm) { /* VA and PA must be 2MB-aligned */ constexpr uint64 k2MB = 2ULL * 1024 * 1024; if ((va & (k2MB - 1)) != 0 || (pa & (k2MB - 1)) != 0) { return -EINVAL; /* -EINVAL */ } uint64 *table = pa_to_ptr(root_pa); /* Walk L2: create intermediate table if needed */ uint64 idx2 = vpn(va, 2); if (!(table[idx2] & PTE_V)) { uint64 child_pa = alloc_pt_page(); if (!child_pa) { return -ENOMEM; /* -ENOMEM */ } table[idx2] = pa_to_pte(child_pa, PTE_V); } table = pa_to_ptr(pte_to_pa(table[idx2])); /* L1: install leaf PTE (2MB mega page) */ uint64 idx1 = vpn(va, 1); table[idx1] = pa_to_pte(pa, perm | PTE_V | PTE_A | PTE_D); return 0; } void unmap(uint64 root_pa, uint64 va) { uint64 *table = pa_to_ptr(root_pa); for (int level = 2; level > 0; level--) { uint64 idx = vpn(va, level); if (!(table[idx] & PTE_V)) return; table = pa_to_ptr(pte_to_pa(table[idx])); } table[vpn(va, 0)] = 0; } int protect(uint64 root_pa, uint64 va, uint64 perm) { uint64 *table = pa_to_ptr(root_pa); for (int level = 2; level >= 0; level--) { uint64 idx = vpn(va, level); uint64 pte = table[idx]; if (!(pte & PTE_V)) return -EFAULT; /* -EFAULT */ if (pte & (PTE_R | PTE_W | PTE_X)) { uint64 pa = pte_to_pa(pte); table[idx] = pa_to_pte(pa, perm | PTE_V); return 0; } table = pa_to_ptr(pte_to_pa(pte)); } return -EFAULT; } int protect_range(uint64 root_pa, uint64 start, uint64 end, uint64 perm) { if (end <= start) { return 0; } uint64 va = page_align_down(start); uint64 va_end = page_align_up(end); for (; va < va_end; va += kPageSize) { int ret = protect(root_pa, va, perm); if (ret == -14) { continue; /* unmapped page: skip */ } if (ret < 0) { return ret; } } arch::mmu::flush_tlb_all(); return 0; } bool lookup(uint64 root_pa, uint64 va, uint64 *pa_out) { uint64 *table = pa_to_ptr(root_pa); for (int level = 2; level >= 0; level--) { uint64 idx = vpn(va, level); uint64 pte = table[idx]; if (!(pte & PTE_V)) return false; /* Leaf PTE? (has R, W, or X) */ if (pte & (PTE_R | PTE_W | PTE_X)) { if (pa_out) *pa_out = pte_to_pa(pte); return true; } table = pa_to_ptr(pte_to_pa(pte)); } return false; } uint64 lookup_pte(uint64 root_pa, uint64 va) { uint64 *table = pa_to_ptr(root_pa); for (int level = 2; level >= 0; level--) { uint64 idx = vpn(va, level); uint64 pte = table[idx]; if (!(pte & PTE_V)) return 0; if (pte & (PTE_R | PTE_W | PTE_X)) return pte; // leaf PTE raw value table = pa_to_ptr(pte_to_pa(pte)); } return 0; } bool is_cow(uint64 root_pa, uint64 va) { uint64 *table = pa_to_ptr(root_pa); for (int level = 2; level >= 0; level--) { uint64 idx = vpn(va, level); uint64 pte = table[idx]; if (!(pte & PTE_V)) return false; if (pte & (PTE_R | PTE_W | PTE_X)) { return (pte & PTE_COW) != 0; } table = pa_to_ptr(pte_to_pa(pte)); } return false; } int make_cow(uint64 root_pa, uint64 va) { uint64 *table = pa_to_ptr(root_pa); for (int level = 2; level >= 0; level--) { uint64 idx = vpn(va, level); uint64 pte = table[idx]; if (!(pte & PTE_V)) return -EFAULT; /* -EFAULT */ if (pte & (PTE_R | PTE_W | PTE_X)) { /* Clear write bit, set COW bit */ pte &= ~PTE_W; pte |= PTE_COW; table[idx] = pte; return 0; } table = pa_to_ptr(pte_to_pa(pte)); } return -EFAULT; } void activate(uint64 root_pa) { arch::mmu::set_page_table(root_pa); arch::mmu::flush_tlb_all(); g_runtime_mapping_ready = true; } uint64 vm_perm_to_pte(uint64 vm_perm) { /* Translate MI VM_* flags to SV39 PTE bits. * The MI flags are defined in . */ uint64 pte = PTE_V | PTE_A | PTE_D; // Valid + Accessed + Dirty (avoid HW A/D faults) if (vm_perm & (1ULL << 0)) pte |= PTE_R; // VM_READ if (vm_perm & (1ULL << 1)) pte |= PTE_W; // VM_WRITE if (vm_perm & (1ULL << 2)) pte |= PTE_X; // VM_EXEC if (vm_perm & (1ULL << 3)) pte |= PTE_U; // VM_USER return pte; } } // namespace pmap