diff options
Diffstat (limited to 'arch/riscv/mm/init.cc')
| -rw-r--r-- | arch/riscv/mm/init.cc | 241 |
1 files changed, 241 insertions, 0 deletions
diff --git a/arch/riscv/mm/init.cc b/arch/riscv/mm/init.cc new file mode 100644 index 0000000..3cebcd9 --- /dev/null +++ b/arch/riscv/mm/init.cc @@ -0,0 +1,241 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* Copyright (C) 2026 Richard Qin */ + +/* + * RISC-V early memory-management bootstrap. + * + * Mirrors Linux arch/riscv/mm/init.c — same file name, same C function + * names (setup_vm / relocate_enable_mmu / relocate_kernel), same global + * state (kernel_map, early_pg_dir, trampoline_pg_dir). head.S calls + * setup_vm immediately after BSS-clear, then later calls + * relocate_enable_mmu, which switches SATP, jumps to higher-half VA, + * and finally calls relocate_kernel to apply .rela.dyn. + * + * Differences from Linux: + * - KASLR offset is hard-wired to 0 for now (no pi/ subsystem) — the + * code path that would consume kernel_map.virt_offset is in place + * so adding pi/-based seed reading later only requires populating + * that one field. + * - Only 1 GiB superpages supported in create_pgd_mapping; sufficient + * for the early page tables. Finer-grained mappings come from + * vmm_init() once slab is up. + */ + +#include <lume/init.h> +#include <lume/setup.h> +#include <lume/elf.h> +#include <lume/kernel_map.h> +#include <lume/types.h> +#include <asm/page.h> +#include <asm/csr.h> +#include <asm/pgtable.h> +#include <asm/sections.h> + +/* + * Global kernel mapping descriptor — single source of truth for the + * runtime-vs-link address relationship. Lives in .data so its initial + * zero contents survive the BSS-clear that runs before setup_vm fills + * the fields. Read by VMM, page-fault handler, ptdump, etc. + * + * Linux equivalent: arch/riscv/mm/init.c::kernel_map + */ +struct kernel_mapping kernel_map __attribute__((section(".data"))); + +/* + * create_pgd_mapping — install one PGD entry covering a 1 GiB region. + * + * Only 1 GiB superpages are supported in this early variant; setup_vm + * needs nothing finer. Identical interface to Linux's same-named + * function in arch/riscv/mm/init.c, restricted in arity. + */ +extern "C" void create_pgd_mapping(pgd_t *pgdir, unsigned long va, + unsigned long pa, unsigned long sz, + unsigned long prot) +{ + /* This early variant only supports 1 GiB superpages. */ + if (sz != PGDIR_SIZE) { + return; + } + + /* SV39 PGD index = (va >> 30) & 0x1FF — see Volume II, §4.4.1. + * Each PGD entry is 8 bytes; pgdir[i] is the slot for that VPN[2]. */ + unsigned long idx = (va >> PGDIR_SHIFT) & (PTRS_PER_PGD - 1); + + /* PTE PPN field: physical page number shifted up by 10 (SV39 PTE + * layout: bits [53:10] = PPN, bits [9:0] = flags). */ + unsigned long ppn = pa >> PAGE_SHIFT; + pgdir[idx].pgd = (ppn << 10) | prot; +} + +/* + * setup_vm — fill kernel_map and build early_pg_dir + trampoline_pg_dir. + * + * Called from head.S after BSS is cleared, with: + * load_pa = runtime physical address of _start + * dtb_pa = physical address of the device tree blob (from a1) + * + * Mirrors Linux arch/riscv/mm/init.c::setup_vm. The actual SATP write + * happens later in relocate_enable_mmu; here we only populate page + * tables and the kernel_map descriptor. + */ +extern "C" void __init setup_vm(uintptr_t load_pa, uintptr_t dtb_pa) +{ + (void)dtb_pa; /* KASLR pi/ subsystem will consume this later. */ + + /* --- 1. Fill kernel_map ----------------------------------------- */ + + /* KASLR offset = 0 in this configuration. */ + kernel_map.virt_offset = 0; + kernel_map.virt_addr = KERNEL_LINK_ADDR + kernel_map.virt_offset; + kernel_map.phys_addr = load_pa; + kernel_map.size = (uintptr_t)_end - (uintptr_t)_start; + kernel_map.page_offset = PAGE_OFFSET; + + /* Mirrors Linux: va_pa_offset is the constant added to a physical + * address to obtain its direct-map VA. */ + kernel_map.va_pa_offset = PAGE_OFFSET - load_pa; + + /* va_kernel_pa_offset is the constant added to a load PA to obtain + * the kernel's link VA (used by relocate_enable_mmu to fix sp/ra). */ + kernel_map.va_kernel_pa_offset = kernel_map.virt_addr - load_pa; + + /* --- 2. Build early_pg_dir ------------------------------------- * + * + * SV39 root page table — 4 entries cover identity + higher-half for + * both MMIO (PA 0x0000_0000, 1 GiB) and RAM (PA 0x8000_0000, 1 GiB): + * + * [0] identity PA 0x0000_0000 → VA 0x0000_0000 + * [2] identity PA 0x8000_0000 → VA 0x8000_0000 + * [256] high-half PA 0x0000_0000 → VA 0xFFFFFFC000000000 + * [258] high-half PA 0x8000_0000 → VA 0xFFFFFFC080000000 + * + * Identity mappings keep PC valid right after csrw satp. High-half + * mappings let the post-MMU `jr` to .Lhigher_half resolve. */ + create_pgd_mapping(early_pg_dir, 0x00000000UL, 0x00000000UL, + PGDIR_SIZE, PAGE_KERNEL_EXEC); + create_pgd_mapping(early_pg_dir, 0x80000000UL, 0x80000000UL, + PGDIR_SIZE, PAGE_KERNEL_EXEC); + create_pgd_mapping(early_pg_dir, PAGE_OFFSET + 0x00000000UL, + 0x00000000UL, PGDIR_SIZE, PAGE_KERNEL_EXEC); + create_pgd_mapping(early_pg_dir, PAGE_OFFSET + 0x80000000UL, + 0x80000000UL, PGDIR_SIZE, PAGE_KERNEL_EXEC); + + /* --- 3. Build trampoline_pg_dir -------------------------------- * + * + * The trampoline maps just the gigapage containing _start, both at + * its identity PA and at its higher-half VA. This is the page + * table SATP holds during the single instruction window between + * "MMU off" and "MMU on with early_pg_dir" — the PC must resolve + * under both, so we re-use the same two entries above. */ + create_pgd_mapping(trampoline_pg_dir, load_pa & PGDIR_MASK, + load_pa & PGDIR_MASK, PGDIR_SIZE, PAGE_KERNEL_EXEC); + create_pgd_mapping(trampoline_pg_dir, + (kernel_map.virt_addr) & PGDIR_MASK, + load_pa & PGDIR_MASK, PGDIR_SIZE, PAGE_KERNEL_EXEC); +} + +/* + * relocate_kernel — walk .rela.dyn and apply R_RISCV_RELATIVE entries. + * + * Called from relocate_enable_mmu *after* the SATP switch and the jump + * to higher-half VA — meaning __rela_dyn_start / __rela_dyn_end resolve + * to valid higher-half pointers we can dereference directly. + * + * Mirrors Linux arch/riscv/mm/init.c::relocate_kernel. Key invariant: + * even when kaslr_offset == 0, the slot store MUST still happen — the + * linker leaves R_RISCV_RELATIVE slots zero-filled at link time, and + * only relocate_kernel writes the link-time addend into them. Without + * this pass, .init_array entries (and every other relocated symbol) + * are NULL pointers. + */ +void __init relocate_kernel(void) +{ + auto *rela = (Elf64_Rela *)__rela_dyn_start; + auto *rela_end = (Elf64_Rela *)__rela_dyn_end; + + const uint64 kaslr_offset = kernel_map.virt_offset; + + for (; rela < rela_end; ++rela) { + if (ELF64_R_TYPE(rela->r_info) != R_RISCV_RELATIVE) { + continue; + } + auto *slot = (Elf64_Addr *)(rela->r_offset + kaslr_offset); + *slot = (Elf64_Addr)(rela->r_addend) + kaslr_offset; + } +} + +/* + * relocate_enable_mmu — switch SATP, jump to higher-half, apply relocations. + * + * Called from head.S after setup_vm has filled kernel_map and built the + * page tables. Equivalent to Linux's same-named asmlinkage routine but + * written in C++ so it can share kernel_map / pgtable definitions with + * setup_vm directly. + * + * Three logical steps: + * + * 1. csrw satp ← trampoline_pg_dir. PC continues running at the + * identity PA, which the trampoline still maps; the high-half PA + * mapping is now also live. + * + * 2. la t0, .Lvirt; add t0, t0, va_kernel_pa_offset; jr t0. + * A PC-relative `la` under medany yields the runtime PA of the + * label; adding va_kernel_pa_offset (= link_va - load_pa) converts + * it to the link-time higher-half VA, which the trampoline maps. + * The `jr` then atomically pushes PC into higher-half space. + * + * 3. csrw satp ← early_pg_dir. We're now executing at higher-half + * VA, so the trampoline can be retired in favour of the full + * early page table. sp / ra / gp must be slid to higher-half too, + * otherwise the next ret-or-call uses physical-low addresses + * whose mapping is about to disappear when vmm_init drops the + * identity entries. + * + * 4. Call relocate_kernel — now safe, because all link-time VA + * pointers in __rela_dyn_start[..] dereference correctly. + */ +extern "C" void __init relocate_enable_mmu(pgd_t *pgdir) +{ + (void)pgdir; /* head.S passes early_pg_dir for documentation; we + * read the global symbol directly inside the asm. */ + + const unsigned long va_kernel_pa_offset = kernel_map.va_kernel_pa_offset; + const unsigned long satp_mode = SATP_MODE_SV39; + + asm volatile ( + /* Step 1: csrw satp <- trampoline */ + "la t0, trampoline_pg_dir\n" + "srli t0, t0, 12\n" + "or t0, t0, %[mode]\n" + "csrw satp, t0\n" + "sfence.vma\n" + + /* Step 2: jump to higher-half VA */ + "la t0, 1f\n" + "add t0, t0, %[off]\n" + "jr t0\n" + "1:\n" + + /* Step 3a: csrw satp <- early_pg_dir (now in higher-half) */ + "la t0, early_pg_dir\n" + "srli t0, t0, 12\n" + "or t0, t0, %[mode]\n" + "csrw satp, t0\n" + "sfence.vma\n" + + /* Step 3b: slide sp / ra / gp to higher-half */ + "add sp, sp, %[off]\n" + "add ra, ra, %[off]\n" + ".option push\n" + ".option norelax\n" + "la gp, __global_pointer$\n" + ".option pop\n" + : + : [off] "r"(va_kernel_pa_offset), + [mode] "r"(satp_mode) + : "t0", "memory" + ); + + /* Step 4: apply .rela.dyn now that VA dereferences work. */ + relocate_kernel(); +} |