diff options
| author | Richard Qin <richard.qin.x@gmail.com> | 2026-07-10 11:24:56 +0800 |
|---|---|---|
| committer | Richard Qin <richard.qin.x@gmail.com> | 2026-07-10 11:24:56 +0800 |
| commit | 1767d390fbf3d0c65389de46caf0b6d43ef3b6ad (patch) | |
| tree | 0f5e01cd7c7a05abb6b1c7472d7f00f49caf02cc /na/arch | |
| parent | 63dc025705f6533b91fb1a7d26e6ed864efd9900 (diff) | |
| download | lume-1767d390fbf3d0c65389de46caf0b6d43ef3b6ad.tar.gz lume-1767d390fbf3d0c65389de46caf0b6d43ef3b6ad.tar.xz | |
feat: establish bootable new architecture checkpoint
Replace the compressed boot path with a relocatable higher-half RISC-V entry, expand memory and SMP support, and add the driver, TTY, VFS, exec, signal, syscall, initramfs, and self-test infrastructure needed for a bootable kernel checkpoint.
Diffstat (limited to 'na/arch')
28 files changed, 1524 insertions, 903 deletions
diff --git a/na/arch/riscv/boot/compressed.cc b/na/arch/riscv/boot/compressed.cc deleted file mode 100644 index 27513bf..0000000 --- a/na/arch/riscv/boot/compressed.cc +++ /dev/null @@ -1,471 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-or-later */ -/* Copyright (C) 2026 Richard Qin */ - -#include <arch/config.h> -#include <lume/boot_params.h> -#include <lume/compressed.h> -#include <lume/console.h> -#include <lume/config.h> -#include <lume/decompress.h> -#include <lume/fdt.h> -#include <lume/klog.h> -#include <lume/relocate.h> -#include <lume/types.h> - -extern "C" const CompressedKernelHeader __lume_image_header; -extern "C" char _start[]; -extern "C" char __lume_image_end[]; -extern "C" char __lume_reloc_start[]; -extern "C" char __lume_reloc_end[]; - -struct FdtHeader { - uint32 magic; - uint32 totalsize; - uint32 off_dt_struct; - uint32 off_dt_strings; - uint32 off_mem_rsvmap; - uint32 version; - uint32 last_comp_version; - uint32 boot_cpuid_phys; - uint32 size_dt_strings; - uint32 size_dt_struct; -}; - -struct ReservedRange { - uint64 start; - uint64 end; -}; - -static inline uint32 bswap32(uint32 x) -{ - return ((x & 0xFF000000u) >> 24) | - ((x & 0x00FF0000u) >> 8) | - ((x & 0x0000FF00u) << 8) | - ((x & 0x000000FFu) << 24); -} - -static inline uint64 read_be64(const uint8* p) -{ - uint64 v = 0; - for (int i = 0; i < 8; ++i) { - v = (v << 8) | p[i]; - } - return v; -} - -static inline uint64 addr_to_phys_early(const void* ptr) -{ - uint64 addr = reinterpret_cast<uint64>(ptr); - if (addr >= arch::kDirectMapBaseDefault) { - return addr - arch::kDirectMapBaseDefault; - } - return addr; -} - -static inline uint64 align_down_page(uint64 v) -{ - return v & ~(kPageSize - 1); -} - -static inline bool ranges_overlap(uint64 base, uint64 size, - const ReservedRange& r) -{ - if (size == 0) { - return false; - } - if (base > ~0ULL - size) { - return true; - } - uint64 end = base + size; - return (base < r.end) && (end > r.start); -} - -static uint32 crc32_init() -{ - return 0xFFFFFFFFu; -} - -static uint32 crc32_update(uint32 crc, const uint8* data, uint64 len) -{ - for (uint64 i = 0; i < len; ++i) { - crc ^= data[i]; - for (int b = 0; b < 8; ++b) { - uint32 mask = static_cast<uint32>(-(crc & 1u)); - crc = (crc >> 1) ^ (0xEDB88320u & mask); - } - } - return crc; -} - -static uint32 crc32_final(uint32 crc) -{ - return crc ^ 0xFFFFFFFFu; -} - -static uint32 crc32_bytes(const uint8* data, uint64 len) -{ - return crc32_final(crc32_update(crc32_init(), data, len)); -} - -static void early_put_hex_u64(const char* label, uint64 value) -{ - static const char kHex[] = "0123456789abcdef"; - early_puts(label); - early_puts("0x"); - for (int i = 60; i >= 0; i -= 4) { - early_putc(kHex[(value >> i) & 0xF]); - } - early_putc('\n'); -} - -/* - * boot_stage3_trace — lightweight early-boot diagnostic hook. - * - * Called from entry.S at three points during the decompress/relocate - * flow. Arguments: a0=fdt_paddr, a1=link_base, a2=load_base, a3=phase. - * - * Phase 0: before decompress (load_base not yet known → 0) - * Phase 1: after decompress, before jump - * Phase 2: after flag-set, before re-entry jump - */ -extern "C" void boot_stage3_trace(uint64 fdt_paddr, uint64 link_base, - uint64 load_base, uint64 phase) -{ - /* Phase 0 may fire before console is initialised — keep it cheap. */ - if (phase == 0) { - console_early_init(fdt_paddr); - early_puts("[boot] stage3 trace: phase 0 (pre-decompress)\n"); - return; - } - - static const char* labels[] = { - "[boot] stage3 trace: phase 0\n", - "[boot] stage3 trace: phase 1 (post-decompress)\n", - "[boot] stage3 trace: phase 2 (pre-reentry)\n", - }; - if (phase < 3) { - early_puts(labels[phase]); - } - early_put_hex_u64(" link_base=", link_base); - early_put_hex_u64(" load_base=", load_base); -} - -extern "C" void boot_stage3_loop_panic(uint64 fdt_paddr, uint64 link_base) -{ - console_early_init(fdt_paddr); - early_puts("[boot] stage3 loop detected\n"); - - auto* hdr = reinterpret_cast<const CompressedKernelHeader*>(link_base + 64); - early_put_hex_u64("link_base=", link_base); - early_put_hex_u64("hdr.flags=", hdr->flags); - early_put_hex_u64("hdr.comp=", hdr->compression); - early_put_hex_u64("hdr.uncomp=", hdr->uncompressed_size); - early_put_hex_u64("hdr.comp_size=", hdr->compressed_size); - early_put_hex_u64("hdr.payload_off=", hdr->payload_offset); - early_put_hex_u64("hdr.load_base=", hdr->load_phys_base); - early_put_hex_u64("hdr.reloc_off=", hdr->reloc_offset); - early_put_hex_u64("hdr.reloc_size=", hdr->reloc_size); - - kernel_panic("boot: stage3 loop"); -} - -static int collect_fdt_reserved(uint64 fdt_paddr, ReservedRange* out, int max) -{ - if (!out || max <= 0 || fdt_paddr == 0) { - return 0; - } - - const FdtHeader* hdr = reinterpret_cast<const FdtHeader*>(fdt_paddr); - if (bswap32(hdr->magic) != 0xd00dfeedu) { - return 0; - } - - int count = 0; - uint64 totalsize = bswap32(hdr->totalsize); - if (totalsize != 0 && count < max) { - out[count++] = ReservedRange{fdt_paddr, fdt_paddr + totalsize}; - } - - uint64 rsv_off = bswap32(hdr->off_mem_rsvmap); - const uint8* p = reinterpret_cast<const uint8*>(fdt_paddr + rsv_off); - while (count < max) { - uint64 addr = read_be64(p); - uint64 size = read_be64(p + 8); - p += 16; - if (addr == 0 && size == 0) { - break; - } - if (size == 0) { - continue; - } - out[count++] = ReservedRange{addr, addr + size}; - } - - return count; -} - -static uint64 xorshift64(uint64* state) -{ - uint64 x = *state; - x ^= x << 13; - x ^= x >> 7; - x ^= x << 17; - *state = x; - return x; -} - -static uint64 find_gap_from(uint64 start, uint64 min_base, - const ReservedRange* ranges, int count, - uint64 image_size) -{ - uint64 candidate = align_down_page(start); - - for (int iter = 0; iter < 64; ++iter) { - if (candidate < min_base) { - return 0; - } - - bool hit = false; - uint64 next = candidate; - for (int i = 0; i < count; ++i) { - if (!ranges_overlap(candidate, image_size, ranges[i])) { - continue; - } - hit = true; - if (ranges[i].start < image_size) { - return 0; - } - uint64 below = ranges[i].start - image_size; - below = align_down_page(below); - if (below < next) { - next = below; - } - } - - if (!hit) { - return candidate; - } - if (next >= candidate) { - return 0; - } - candidate = next; - } - - return 0; -} - -static uint64 choose_load_base(uint64 fdt_paddr, uint64 image_size, - uint64 link_base, uint64 payload_base, - uint64 payload_size) -{ - uint64 mem_base = 0; - uint64 mem_size = 0; - FdtManager::early_scan_mem(fdt_paddr, &mem_base, &mem_size); - if (mem_size == 0) { - return link_base; - } - - uint64 mem_top = mem_base + mem_size; - if (mem_top < mem_base || mem_top < image_size) { - return link_base; - } - - ReservedRange reserved[32] = {}; - int count = 0; - - if (count < static_cast<int>(sizeof(reserved) / sizeof(reserved[0]))) { - reserved[count++] = ReservedRange{link_base, link_base + image_size}; - } - if (payload_size != 0 && - count < static_cast<int>(sizeof(reserved) / sizeof(reserved[0]))) { - reserved[count++] = ReservedRange{payload_base, payload_base + payload_size}; - } - count += collect_fdt_reserved(fdt_paddr, reserved + count, - static_cast<int>(sizeof(reserved) / - sizeof(reserved[0])) - count); - - uint64 min_base = align_down_page(mem_base); - uint64 max_base = align_down_page(mem_top - image_size); - if (max_base < min_base) { - return link_base; - } - - uint64 seed = 0; - FdtManager::early_scan_kaslr(fdt_paddr, &seed); - uint64 candidate = max_base; - if (seed != 0) { - seed ^= link_base; - uint64 slots = (max_base - min_base) / kPageSize; - uint64 pick = xorshift64(&seed) % (slots + 1); - candidate = min_base + pick * kPageSize; - } - - uint64 base = find_gap_from(candidate, min_base, reserved, count, image_size); - if (base != 0) { - return base; - } - if (candidate != max_base) { - base = find_gap_from(max_base, min_base, reserved, count, image_size); - if (base != 0) { - return base; - } - } - - return link_base; -} - -extern "C" uint64 boot_decompress_and_relocate(uint64 fdt_paddr) -{ - console_early_init(fdt_paddr); - early_puts("[boot] stage3 start\n"); - - const uint64 image_va_base = reinterpret_cast<uint64>(_start); - uint64 image_size = reinterpret_cast<uint64>(__lume_image_end) - image_va_base; - uint64 reloc_offset = reinterpret_cast<uint64>(__lume_reloc_start) - image_va_base; - uint64 reloc_size = reinterpret_cast<uint64>(__lume_reloc_end) - - reinterpret_cast<uint64>(__lume_reloc_start); - - const uint64 link_base = addr_to_phys_early(_start); - uint64 load_base = link_base; - - uint32 comp = KERNEL_COMP_NONE; - uint32 flags = 0; - uint64 compressed_size = 0; - uint64 payload_offset = 0; - - const CompressedKernelHeader* hdr = &__lume_image_header; - const uint64 header_offset = reinterpret_cast<uint64>(&__lume_image_header) - - image_va_base; - if (hdr->magic == kCompressedKernelMagic && - hdr->version == kCompressedKernelVersion && - hdr->header_size >= sizeof(CompressedKernelHeader)) { - if (hdr->header_crc32 != 0) { - uint32 crc = crc32_init(); - const uint8* hbytes = reinterpret_cast<const uint8*>(hdr); - constexpr uint64 crc_off = offsetof(CompressedKernelHeader, header_crc32); - uint64 header_size = hdr->header_size; - if (header_size > sizeof(CompressedKernelHeader)) { - header_size = sizeof(CompressedKernelHeader); - } - crc = crc32_update(crc, hbytes, crc_off); - uint32 zero = 0; - crc = crc32_update(crc, reinterpret_cast<const uint8*>(&zero), sizeof(zero)); - uint64 tail_off = crc_off + sizeof(uint32); - if (header_size > tail_off) { - crc = crc32_update(crc, hbytes + tail_off, header_size - tail_off); - } - uint32 calc = crc32_final(crc); - if (calc != hdr->header_crc32) { - early_puts("[boot] header crc mismatch\n"); - kernel_panic("boot: header crc mismatch"); - } - } - comp = hdr->compression; - compressed_size = hdr->compressed_size; - if (hdr->uncompressed_size != 0) { - image_size = hdr->uncompressed_size; - } - if (hdr->load_phys_base != 0) { - load_base = hdr->load_phys_base; - } else if (comp != KERNEL_COMP_NONE) { - uint64 payload_base = link_base + payload_offset; - load_base = choose_load_base(fdt_paddr, image_size, link_base, - payload_base, compressed_size); - } - if (hdr->reloc_size != 0) { - reloc_offset = hdr->reloc_offset; - reloc_size = hdr->reloc_size; - } - payload_offset = (hdr->payload_offset != 0) - ? hdr->payload_offset - : hdr->header_size; - } - - if (comp != KERNEL_COMP_NONE && load_base == link_base) { - early_puts("[boot] no safe relocation base\n"); - flags |= BOOT_IMAGE_RELOC_FAILED; - boot_params_set_image_info(load_base, image_size, reloc_offset, - reloc_size, comp, flags); - kernel_panic("boot: no safe relocation base"); - } - - if (comp != KERNEL_COMP_NONE) { - flags |= BOOT_IMAGE_COMPRESSED; - - if (compressed_size == 0 || payload_offset == 0) { - early_puts("[boot] compressed header invalid\n"); - flags |= BOOT_IMAGE_DECOMP_FAILED; - boot_params_set_image_info(load_base, image_size, reloc_offset, - reloc_size, comp, flags); - kernel_panic("boot: compressed header invalid"); - } - - if (hdr->payload_crc32 != 0) { - const uint8* payload = reinterpret_cast<const uint8*>( - image_va_base + payload_offset); - uint32 calc = crc32_bytes(payload, compressed_size); - if (calc != hdr->payload_crc32) { - early_puts("[boot] payload crc mismatch\n"); - flags |= BOOT_IMAGE_DECOMP_FAILED; - boot_params_set_image_info(load_base, image_size, reloc_offset, - reloc_size, comp, flags); - kernel_panic("boot: payload crc mismatch"); - } - } - - if (comp == KERNEL_COMP_LZ4) { - const uint8* payload = reinterpret_cast<const uint8*>( - image_va_base + payload_offset); - uint8* out = reinterpret_cast<uint8*>(load_base); - int written = lz4_decompress(payload, - static_cast<uint32>(compressed_size), - out, static_cast<uint32>(image_size)); - if (written < 0 || static_cast<uint64>(written) != image_size) { - early_puts("[boot] lz4 decompress failed\n"); - flags |= BOOT_IMAGE_DECOMP_FAILED; - boot_params_set_image_info(load_base, image_size, reloc_offset, - reloc_size, comp, flags); - kernel_panic("boot: lz4 decompress failed"); - } - } else { - early_puts("[boot] unsupported compression\n"); - flags |= BOOT_IMAGE_DECOMP_FAILED; - boot_params_set_image_info(load_base, image_size, reloc_offset, - reloc_size, comp, flags); - kernel_panic("boot: unsupported compression"); - } - } - - if (reloc_size != 0 && load_base != link_base) { - if (reloc_offset + reloc_size > image_size) { - early_puts("[boot] reloc table out of range\n"); - flags |= BOOT_IMAGE_RELOC_FAILED; - boot_params_set_image_info(load_base, image_size, reloc_offset, - reloc_size, comp, flags); - kernel_panic("boot: reloc table out of range"); - } - const void* reloc_blob = reinterpret_cast<const void*>( - image_va_base + reloc_offset); - RelocStatus st = relocate_apply(load_base, image_size, link_base, - reloc_blob, reloc_size); - if (st != RELOC_OK) { - early_puts("[boot] relocation failed\n"); - flags |= BOOT_IMAGE_RELOC_FAILED; - boot_params_set_image_info(load_base, image_size, reloc_offset, - reloc_size, comp, flags); - kernel_panic("boot: relocation failed", reloc_status_str(st)); - } - flags |= BOOT_IMAGE_RELOCATED; - } - - if (load_base != link_base) { - auto* new_hdr = reinterpret_cast<CompressedKernelHeader*>( - load_base + header_offset); - new_hdr->flags |= LUME_HDR_FLAG_STAGE3_DONE; - } - - early_puts("[boot] stage3 done\n"); - boot_params_set_image_info(load_base, image_size, reloc_offset, - reloc_size, comp, flags); - return load_base; -} diff --git a/na/arch/riscv/boot/entry.S b/na/arch/riscv/boot/entry.S deleted file mode 100644 index 9154b7f..0000000 --- a/na/arch/riscv/boot/entry.S +++ /dev/null @@ -1,309 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-or-later */ -/* Copyright (C) 2026 Richard Qin */ - -/* - * LumeOS Kernel Entry Point (RISC-V 64) - * - * This file implements the earliest boot sequence: - * 1. RISC-V Linux Boot Protocol Image Header (64 bytes) - * 2. BSP/AP synchronization - * 3. BSS zeroing (BSP only) - * 4. Early page table setup (1GB superpages for identity + higher-half) - * 5. MMU enable and higher-half trampoline - * 6. C++ global constructor invocation - * 7. Jump to kernel_main() - * - * Register contract on entry (from OpenSBI / GRUB / U-Boot): - * a0 = CPU ID (Hart ID) - * a1 = FDT physical base address - * - * Reference: docs/specs/linker_and_entry.md - */ - -#include <lume/config.h> - -#define PAGE_OFFSET 0xFFFFFFC000000000 -#define BOOT_STACK_SIZE 8192 - -/* SV39 PTE flags */ -#define PTE_V (1 << 0) /* Valid */ -#define PTE_R (1 << 1) /* Readable */ -#define PTE_W (1 << 2) /* Writable */ -#define PTE_X (1 << 3) /* Executable */ -#define PTE_A (1 << 6) /* Accessed */ -#define PTE_D (1 << 7) /* Dirty */ - -/* SATP mode field for SV39 */ -#define SATP_SV39 (8UL << 60) - -/* LumeOS compressed image header constants */ -#define LUME_COMP_NONE 0 -#define LUME_HDR_SIZE 80 -#define LUME_HDR_MAGIC 0x4C554D45434F4D50 /* "LUMECOMP" */ -#define LUME_HDR_OFF 64 -#define LUME_HDR_FLAGS_OFF 14 -#define LUME_HDR_FLAG_STAGE3_DONE 0x1 - -/* ======================================================================== - * Section: .head.text — RISC-V Linux Boot Protocol Image Header - * Must be at byte 0 of the kernel binary. - * ======================================================================== */ - .section .head.text, "ax" - .globl _start -_start: - /* code0 (u32, offset 0): executable jump for OpenSBI bare boot */ - j real_start - /* code1 (u32, offset 4): reserved */ - .word 0 - /* text_offset (u64, offset 8): load offset from RAM base */ - .dword 0x200000 - /* image_size (u64, offset 16): will be filled post-link, 0 = unknown */ - .dword 0 - /* flags (u64, offset 24): bit 0: 0=LE, 1=BE */ - .dword 0 - /* version (u32, offset 32): header version 0.2 */ - .word 0x0002 - /* res1 (u32, offset 36) */ - .word 0 - /* res2 (u64, offset 40) */ - .dword 0 - /* magic (u64, offset 48): RISCV_IMAGE_MAGIC "RISCV\0\0\0" LE */ - .dword 0x5643534952 - /* magic2 (u32, offset 56): RISCV_IMAGE_MAGIC2 "RSC\x05" LE */ - .word 0x05435352 - /* res3 (u32, offset 60): reserved for PE/COFF */ - .word 0 - - /* LumeOS compressed kernel header (80 bytes) */ - .align 3 - .globl __lume_image_header -__lume_image_header: - .dword LUME_HDR_MAGIC /* magic */ - .word 1 /* version */ - .half LUME_HDR_SIZE /* header_size */ - .half 0 /* flags */ - .word LUME_COMP_NONE /* compression */ - .word 0 /* reserved0 */ - .dword __lume_image_end - _start /* uncompressed_size */ - .dword 0 /* compressed_size */ - .dword 0 /* load_phys_base (0 = auto) */ - .dword __lume_reloc_start - _start /* reloc_offset */ - .dword __lume_reloc_end - __lume_reloc_start /* reloc_size */ - .dword _ebss - _start /* payload_offset (when compressed) */ - .word 0 /* header_crc32 */ - .word 0 /* payload_crc32 */ - -/* ======================================================================== - * real_start: Actual boot logic begins here (all CPUs land here) - * ======================================================================== */ - .text - .globl real_start -real_start: - /* ---- Step 1: Disable interrupts ---- */ - csrw sie, zero - csrw sip, zero - - /* ---- Step 2: Set up gp (global pointer) ---- */ - /* Critical for medany model: all global variable access goes through gp */ - .option push - .option norelax - la gp, __global_pointer$ - .option pop - - /* ---- Step 3: Set up per-CPU boot stack ---- */ - /* sp = boot_stack + (cpu_id + 1) * BOOT_STACK_SIZE */ - la t0, boot_stack - li t1, BOOT_STACK_SIZE - addi t2, a0, 1 /* t2 = cpu_id + 1 */ - mul t1, t1, t2 /* t1 = (cpu_id + 1) * BOOT_STACK_SIZE */ - add sp, t0, t1 - - /* ---- Step 4: Store boot args (cpu_id and fdt_paddr) ---- */ - mv tp, a0 /* tp = cpu_id */ - mv s0, a1 /* s0 = fdt_paddr (preserved across all C++ calls) */ - /* ---- Step 5: BSP / AP divergence ---- */ - /* CPU 0 is the BSP; all others spin-wait */ - bnez a0, .Lap_wait - -/* =================================================================== - * BSP-only path: Clear BSS, set up early page table, release APs - * =================================================================== */ -.Lbsp_init: - /* ---- 5.1: Clear BSS ---- */ - la t0, _sbss - la t1, _ebss -.Lbss_loop: - bgeu t0, t1, .Lbss_done - sd zero, 0(t0) - addi t0, t0, 8 - j .Lbss_loop -.Lbss_done: - - - /* ---- 5.2: Build early page table (1GB superpages) ---- */ - /* - * SV39 root page table (512 entries x 8 bytes = 4KB) - * We need 4 entries: - * [0] = Identity map: 0x00000000 -> 0x00000000 (MMIO: UART, PLIC, CLINT) - * [2] = Identity map: 0x80000000 -> 0x80000000 (RAM) - * [256] = Higher-half: 0xFFFFFFC000000000 -> 0x00000000 (MMIO via VA) - * [258] = Higher-half: 0xFFFFFFC080000000 -> 0x80000000 (RAM via VA) - * - * VPN[2] index = (VA >> 30) & 0x1FF - * 0x00000000 -> 0 0xFFFFFFC000000000 -> 256 - * 0x80000000 -> 2 0xFFFFFFC080000000 -> 258 - */ - la t0, early_pgdir - - /* PTE flags: V|R|W|X|A|D (1GB leaf superpage) */ - li t2, (PTE_V | PTE_R | PTE_W | PTE_X | PTE_A | PTE_D) - - /* --- MMIO region: physical 0x00000000 --- */ - /* PPN = 0x00000000 >> 12 = 0, PTE = flags only */ - mv t1, t2 /* PTE = 0 | flags */ - - /* Entry [0]: Identity map MMIO */ - sd t1, (0 * 8)(t0) - - /* Entry [256]: Higher-half MMIO (0xFFFFFFC000000000) */ - li t3, (256 * 8) - add t3, t0, t3 - sd t1, 0(t3) - - /* --- RAM region: physical 0x80000000 --- */ - li t1, 0x80000 /* PPN = 0x80000000 >> 12 */ - slli t1, t1, 10 /* PPN field in PTE */ - or t1, t1, t2 /* t1 = complete PTE */ - - /* Entry [2]: Identity map RAM */ - sd t1, (2 * 8)(t0) - - /* Entry [258]: Higher-half RAM (0xFFFFFFC080000000) */ - li t3, (258 * 8) - add t3, t0, t3 - sd t1, 0(t3) - - - /* ---- 5.3: Memory fence ---- */ - fence rw, rw - - /* ---- 5.4: Release APs ---- */ - la t0, boot_sync_flag - li t1, 1 - amoswap.w.rl zero, t1, (t0) - j .Lenable_mmu - -/* =================================================================== - * AP path: spin-wait until BSP signals readiness - * =================================================================== */ -.Lap_wait: - la t0, boot_sync_flag -.Lap_spin: - lw t1, 0(t0) - beqz t1, .Lap_spin - fence r, rw /* Acquire: ensure we see BSP's writes */ - -/* =================================================================== - * All CPUs: Enable MMU and jump to higher-half virtual addresses - * =================================================================== */ -.Lenable_mmu: - - - /* ---- 6.1: Write satp = SV39 mode | (early_pgdir >> 12) ---- */ - /* NOTE: `la` at this point is PC-relative. Since we are still executing - * at physical addresses, it yields the PHYSICAL address directly. - * No VA-to-PA conversion needed. */ - la t0, early_pgdir - srli t0, t0, 12 /* PPN = PA >> 12 */ - li t1, SATP_SV39 - or t0, t0, t1 /* satp = mode | PPN */ - csrw satp, t0 - - /* ---- 6.2: Flush TLB ---- */ - sfence.vma zero, zero - - /* ---- 6.3: Jump to higher-half virtual address ---- */ - /* Now we are executing via the identity map (PA == VA for 0x8...). - * `la` is PC-relative (medany), so it yields a PHYSICAL address. - * We must add PAGE_OFFSET to convert to the higher-half VA. */ - la t0, .Lhigher_half - li t1, PAGE_OFFSET - add t0, t0, t1 - jr t0 - -.Lhigher_half: - /* We are now running at virtual addresses (0xFFFFFFC0...) ! - * The identity map is still active but will be removed by vmm_init(). */ - - /* ---- 6.4: Fix sp and gp to virtual addresses ---- */ - /* sp was set up before MMU using PC-relative `la`, so it holds a - * physical address. Add PAGE_OFFSET to convert to higher-half VA. - * gp is similarly reloaded — now that PC is at VA, `la` resolves - * correctly via PC-relative addressing. */ - li t0, PAGE_OFFSET - add sp, sp, t0 - .option push - .option norelax - la gp, __global_pointer$ - .option pop - - /* Mark MMU enabled for early console mapping logic */ - call console_set_mmu_enabled - - /* ---- Step 7: BSP calls early_cpp_init ---- */ - bnez tp, .Lskip_cpp_init - - /* BSP only: invoke C++ global constructors */ - call early_cpp_init - -.Lskip_cpp_init: - - /* All CPUs: enter kernel_main(cpu_id, fdt_paddr) */ - mv a0, tp /* a0 = cpu_id (from tp) */ - mv a1, s0 /* a1 = fdt_paddr (from s0) */ - call kernel_main - - /* kernel_main should never return; if it does, halt */ -.Lhalt: - csrc sstatus, 2 /* Clear SIE */ - wfi - j .Lhalt - - -/* ======================================================================== - * early_cpp_init: Traverse .init_array and call global constructors - * Called by BSP only, before kernel_main. - * ======================================================================== */ - .globl early_cpp_init -early_cpp_init: - /* Prologue: save ra and callee-saved regs we use as loop state */ - addi sp, sp, -32 - sd ra, 0(sp) - sd s1, 8(sp) - sd s2, 16(sp) - - la s1, __init_array_start - la s2, __init_array_end -.Linit_loop: - bgeu s1, s2, .Linit_done - ld t0, 0(s1) /* Load function pointer */ - jalr ra, t0 /* Call constructor (sets ra = return addr) */ - addi s1, s1, 8 /* Next entry */ - j .Linit_loop -.Linit_done: - /* Epilogue: restore and return */ - ld ra, 0(sp) - ld s1, 8(sp) - ld s2, 16(sp) - addi sp, sp, 32 - ret - - -/* ======================================================================== - * BSS Data: boot_sync_flag - * ======================================================================== */ - .section .bss - .align 2 - .globl boot_sync_flag -boot_sync_flag: - .word 0 diff --git a/na/arch/riscv/include/arch/config.h b/na/arch/riscv/include/arch/config.h index 0949c41..8ae0502 100644 --- a/na/arch/riscv/include/arch/config.h +++ b/na/arch/riscv/include/arch/config.h @@ -35,6 +35,11 @@ inline constexpr uint64 kVmemmapBaseDefault = 0xFFFFFFD000000000ULL; /* Physical memory base (where RAM starts on RISC-V virt platform) */ inline constexpr uint64 kPhysBase = 0x80000000ULL; +/* Kernel physical load address (= PHYS_BASE in linker script). + * Everything in [kPhysBase, kPhysStart) is firmware territory (PMP-protected). + * Analogous to Linux's CONFIG_PHYSICAL_START. */ +inline constexpr uint64 kPhysStart = 0x80200000ULL; + /* ================================================================ * Runtime (post-KASLR) virtual addresses * diff --git a/na/arch/riscv/include/arch/cpu.h b/na/arch/riscv/include/arch/cpu.h index d16cf2d..8c573ec 100644 --- a/na/arch/riscv/include/arch/cpu.h +++ b/na/arch/riscv/include/arch/cpu.h @@ -76,4 +76,20 @@ static inline void timer_intr_off() { __asm__ volatile("csrc sie, %0" :: "r"(SIE_STIE)); } +// Enable software interrupt in sie (sie.SSIE = 1) — needed for IPI +static inline void software_intr_on() { + __asm__ volatile("csrs sie, %0" :: "r"(SIE_SSIE)); +} + +// Enable external interrupt in sie (sie.SEIE = 1) — needed for PLIC (UART, etc.) +static inline void external_intr_on() { + __asm__ volatile("csrs sie, %0" :: "r"(SIE_SEIE)); +} + +// Clear S-mode software interrupt pending bit (sip.SSIP = 0) +// Must be called in the IPI handler to acknowledge the interrupt. +static inline void clear_sip_ssip() { + __asm__ volatile("csrc sip, %0" :: "r"(1UL << 1)); +} + } // namespace arch::cpu diff --git a/na/arch/riscv/include/arch/fpstate.h b/na/arch/riscv/include/arch/fpstate.h new file mode 100644 index 0000000..4b42a22 --- /dev/null +++ b/na/arch/riscv/include/arch/fpstate.h @@ -0,0 +1,94 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* Copyright (C) 2026 Richard Qin */ + +#pragma once + +/* + * FPU State — Floating-Point Register Context + * + * Pre-reserved structure for RISC-V F/D extension floating-point state. + * MUST be allocated in every TaskControlBlock so that future FPU + * save/restore logic has a stable destination without restructuring TCB. + * + * RISC-V F/D extension state: + * - 32 FP registers (f0-f31), each 64 bits for D extension + * - fcsr (Floating-point Control and Status Register) + * + * sstatus.FS field management: + * - FS=0 (Off): FPU disabled, any FP instruction traps + * - FS=1 (Initial): FPU enabled, registers are initial state + * - FS=2 (Clean): FPU enabled, registers not modified since last save + * - FS=3 (Dirty): FPU enabled, registers modified (need save) + * + * Current status: Structure defined, save/restore NOT implemented. + * When implemented, trap_dispatch and swtch.S will use lazy + * save/restore based on sstatus.FS dirty tracking. + * + * Reference: RISC-V ISA Manual §3.1.6.5 (FS field) + */ + +#include <lume/types.h> + +namespace arch { + +struct FpState { + uint64 f[32]; // f0-f31 (64-bit each for D extension) + uint32 fcsr; // Floating-point CSR (frm + fflags) + uint32 _pad; // Alignment padding +}; + +static_assert(sizeof(FpState) == 32 * 8 + 8, "FpState must be 264 bytes"); + +/* + * sstatus.FS field constants + */ +inline constexpr uint64 kSstatusFS_Off = 0ULL << 13; +inline constexpr uint64 kSstatusFS_Initial = 1ULL << 13; +inline constexpr uint64 kSstatusFS_Clean = 2ULL << 13; +inline constexpr uint64 kSstatusFS_Dirty = 3ULL << 13; +inline constexpr uint64 kSstatusFS_Mask = 3ULL << 13; + +/* + * Check if sstatus indicates FPU state is dirty (needs save). + */ +inline bool fpu_state_dirty(uint64 sstatus) { + return (sstatus & kSstatusFS_Mask) == kSstatusFS_Dirty; +} + +/* + * Check if FPU is enabled (FS != Off). + */ +inline bool fpu_enabled(uint64 sstatus) { + return (sstatus & kSstatusFS_Mask) != kSstatusFS_Off; +} + +} // namespace arch + +/* Assembly routines for FPU register save/restore (fpu.S) */ +extern "C" void fpu_save(arch::FpState* state); +extern "C" void fpu_restore(arch::FpState* state); + +namespace arch { + +/* + * Read sstatus.FS field (current FPU state) + */ +inline uint64 fpu_get_fs() { + uint64 sstatus; + __asm__ volatile("csrr %0, sstatus" : "=r"(sstatus)); + return sstatus & kSstatusFS_Mask; +} + +/* + * Set sstatus.FS field. + * @fs: one of kSstatusFS_Off, kSstatusFS_Initial, kSstatusFS_Clean, kSstatusFS_Dirty + */ +inline void fpu_set_fs(uint64 fs) { + __asm__ volatile( + "csrc sstatus, %0\n" + "csrs sstatus, %1\n" + :: "r"(kSstatusFS_Mask), "r"(fs) + ); +} + +} // namespace arch diff --git a/na/arch/riscv/include/arch/mmu.h b/na/arch/riscv/include/arch/mmu.h index a0ee1ed..f4c9b26 100644 --- a/na/arch/riscv/include/arch/mmu.h +++ b/na/arch/riscv/include/arch/mmu.h @@ -50,6 +50,14 @@ static inline uint64 get_page_table() { return (satp_val & SATP_PPN_MASK) << 12; } +// Read the raw satp CSR value (mode + PPN). +// Used when the caller needs the exact register value (e.g. for TrapFrame metadata). +static inline uint64 read_satp() { + uint64 val; + __asm__ volatile("csrr %0, satp" : "=r"(val)); + return val; +} + // Flush the entire TLB (all ASIDs, all addresses) static inline void flush_tlb_all() { __asm__ volatile("sfence.vma zero, zero" ::: "memory"); @@ -60,4 +68,11 @@ static inline void flush_tlb_page(uint64 vaddr) { __asm__ volatile("sfence.vma %0, zero" :: "r"(vaddr) : "memory"); } +// Flush the instruction cache. +// Required after loading executable code into memory (e.g. ELF loading). +// sfence.vma only flushes the TLB; fence.i is needed for the I-cache. +static inline void flush_icache() { + __asm__ volatile("fence.i" ::: "memory"); +} + } // namespace arch::mmu diff --git a/na/arch/riscv/include/arch/pmap.h b/na/arch/riscv/include/arch/pmap.h index a99825a..0b43b9e 100644 --- a/na/arch/riscv/include/arch/pmap.h +++ b/na/arch/riscv/include/arch/pmap.h @@ -35,6 +35,7 @@ void unmap(uint64 root_pa, uint64 va); int protect(uint64 root_pa, uint64 va, uint64 perm); int protect_range(uint64 root_pa, uint64 start, uint64 end, uint64 perm); bool lookup(uint64 root_pa, uint64 va, uint64* pa_out); +uint64 lookup_pte(uint64 root_pa, uint64 va); bool is_cow(uint64 root_pa, uint64 va); int make_cow(uint64 root_pa, uint64 va); void activate(uint64 root_pa); diff --git a/na/arch/riscv/include/arch/sbi.h b/na/arch/riscv/include/arch/sbi.h index b65c37e..80a6d7c 100644 --- a/na/arch/riscv/include/arch/sbi.h +++ b/na/arch/riscv/include/arch/sbi.h @@ -3,6 +3,15 @@ #pragma once +/* + * sbi.h — RISC-V Supervisor Binary Interface (SBI) wrappers + * + * Provides inline ecall wrappers for SBI calls from S-mode. + * Uses the SBI v1.0+ extension-based calling convention. + * + * Reference: RISC-V SBI Specification v2.0 + */ + #include <lume/types.h> namespace arch::sbi { @@ -12,7 +21,20 @@ struct sbiret { long value; }; -static inline sbiret sbi_ecall(int ext, int fid, uint64 arg0, uint64 arg1, uint64 arg2, uint64 arg3, uint64 arg4, uint64 arg5) { +/* SBI error codes */ +inline constexpr long SBI_SUCCESS = 0; +inline constexpr long SBI_ERR_NOT_SUPPORTED = -2; + +/* Extension IDs */ +inline constexpr int SBI_EXT_TIME = 0x54494D45; /* "TIME" */ +inline constexpr int SBI_EXT_IPI = 0x735049; /* "sPI" */ +inline constexpr int SBI_EXT_RFENCE = 0x52464E43; /* "RFNC" */ +inline constexpr int SBI_EXT_HSM = 0x48534D; /* "HSM" */ + +static inline sbiret sbi_ecall(int ext, int fid, uint64 arg0 = 0, + uint64 arg1 = 0, uint64 arg2 = 0, + uint64 arg3 = 0, uint64 arg4 = 0, + uint64 arg5 = 0) { sbiret ret; register uint64 a0 asm("a0") = arg0; register uint64 a1 asm("a1") = arg1; @@ -23,7 +45,7 @@ static inline sbiret sbi_ecall(int ext, int fid, uint64 arg0, uint64 arg1, uint6 register uint64 a6 asm("a6") = fid; register uint64 a7 asm("a7") = ext; __asm__ volatile("ecall" - : "+r"(a0), "=r"(a1) + : "+r"(a0), "+r"(a1) : "r"(a2), "r"(a3), "r"(a4), "r"(a5), "r"(a6), "r"(a7) : "memory"); ret.error = static_cast<long>(a0); @@ -31,9 +53,59 @@ static inline sbiret sbi_ecall(int ext, int fid, uint64 arg0, uint64 arg1, uint6 return ret; } +/* ---- Timer Extension (EID 0x54494D45) ---- */ + static inline void set_timer(uint64 stime_value) { - // Standard SBI TIME Extension (EID 0x54494D45) - sbi_ecall(0x54494D45, 0, stime_value, 0, 0, 0, 0, 0); + sbi_ecall(SBI_EXT_TIME, 0, stime_value); +} + +/* ---- IPI Extension (EID 0x735049) ---- */ + +/* + * sbi_send_ipi — Send S-mode software interrupt to target harts + * + * @hart_mask: bitmask of target harts (relative to hart_mask_base) + * @hart_mask_base: base hart ID for the mask (0 = harts 0-63) + */ +static inline sbiret sbi_send_ipi(uint64 hart_mask, + uint64 hart_mask_base = 0) { + return sbi_ecall(SBI_EXT_IPI, 0, hart_mask, hart_mask_base); +} + +/* ---- Remote Fence Extension (EID 0x52464E43) ---- */ + +/* + * sbi_remote_sfence_vma — Execute SFENCE.VMA on target harts + * + * @hart_mask/base: target harts + * @start_addr: start of VA range to invalidate + * @size: size of range (use ~0UL for full flush) + */ +static inline sbiret sbi_remote_sfence_vma(uint64 hart_mask, + uint64 hart_mask_base, + uint64 start_addr, + uint64 size) { + return sbi_ecall(SBI_EXT_RFENCE, 1, + hart_mask, hart_mask_base, + start_addr, size); +} + +/* ---- HSM Extension (EID 0x48534D) ---- */ + +/* + * sbi_hart_start — Start (wake) a hart that is in STOPPED state + * + * @hartid: target hart ID to start + * @start_addr: PHYSICAL address where the hart should begin execution + * @opaque: passed in a1 register to the hart (e.g. FDT pointer) + * + * OpenSBI v1.0+ parks non-boot harts; they must be explicitly started. + */ +static inline sbiret sbi_hart_start(uint64 hartid, + uint64 start_addr, + uint64 opaque) { + return sbi_ecall(SBI_EXT_HSM, 0, hartid, start_addr, opaque); } } // namespace arch::sbi + diff --git a/na/arch/riscv/include/arch/tlb.h b/na/arch/riscv/include/arch/tlb.h new file mode 100644 index 0000000..46c6b97 --- /dev/null +++ b/na/arch/riscv/include/arch/tlb.h @@ -0,0 +1,19 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* Copyright (C) 2026 Richard Qin */ + +#pragma once + +#include <lume/types.h> + +namespace arch::tlb { + +/* + * Perform a cross-CPU TLB flush (shootdown). + * + * @param cpu_mask: Bitmask of CPUs to target. + * @param start: Start virtual address, or 0 for full address space. + * @param size: Size in bytes, or ~0ULL for full address space. + */ +void flush_remote(uint64 cpu_mask, uint64 start, uint64 size); + +} // namespace arch::tlb diff --git a/na/arch/riscv/include/arch/trap.h b/na/arch/riscv/include/arch/trap.h index c99d8cb..8e55250 100644 --- a/na/arch/riscv/include/arch/trap.h +++ b/na/arch/riscv/include/arch/trap.h @@ -79,4 +79,16 @@ static inline uint64 cause_code(uint64 cause) { return cause & ~SCAUSE_INTERRUPT; } +// Enable Supervisor User Memory access (set SUM bit in sstatus). +// When SUM=1, S-mode can read/write pages marked PTE_U. +// This is needed for syscall handlers to access user-space buffers. +static inline void enable_sum() { + __asm__ volatile("csrs sstatus, %0" :: "r"(1UL << 18)); +} + +// Disable Supervisor User Memory access (clear SUM bit in sstatus). +static inline void disable_sum() { + __asm__ volatile("csrc sstatus, %0" :: "r"(1UL << 18)); +} + } // namespace arch::trap diff --git a/na/arch/riscv/include/arch/trapframe.h b/na/arch/riscv/include/arch/trapframe.h index de49e9c..b198646 100644 --- a/na/arch/riscv/include/arch/trapframe.h +++ b/na/arch/riscv/include/arch/trapframe.h @@ -61,6 +61,7 @@ #ifndef __ASSEMBLER__ #include <lume/types.h> +#include <arch/cpu.h> namespace arch { @@ -89,4 +90,51 @@ static_assert(sizeof(TrapFrame) == 296, "TrapFrame size mismatch with assembly o } // namespace arch +/* Forward declaration — defined in trap_md.cc. + * Must be after TrapFrame definition. */ +extern "C" [[noreturn]] void trap_dispatch(arch::TrapFrame*); + +namespace arch { + +/* + * trapframe_init_user — prepare a TrapFrame for returning to user mode. + * + * Sets arch-specific sstatus: + * SPP = 0 (return to U-mode on sret) + * SPIE = 1 (enable interrupts in U-mode) + * FS = Initial (allow FP instructions) + * + * Called from MI exec.cc instead of setting sstatus directly. + */ +static inline void trapframe_init_user(TrapFrame* tf, uint64 entry, uint64 sp) +{ + /* Clear all registers */ + for (auto& t : tf->t) { t = 0; } + for (auto& s : tf->s) { s = 0; } + for (auto& a : tf->a) { a = 0; } + tf->ra = 0; + tf->gp = 0; + tf->tp = 0; + + tf->epc = entry; + tf->sp = sp; + + /* sstatus: SPP=0 (U-mode), SPIE=1 (interrupts), FS=Initial (FP ok) */ + constexpr uint64 kSPIE = (1ULL << 5); + constexpr uint64 kFS_Initial = (1ULL << 13); + tf->sstatus = kSPIE | kFS_Initial; + + /* Kernel metadata used by trap_vector.S on every trap from U-mode. + * These are RISC-V specific — trap_vector reads them from the + * TrapFrame to switch to kernel context. */ + uint64 satp; + asm volatile("csrr %0, satp" : "=r"(satp)); + tf->kernel_satp = satp; + tf->kernel_sp = reinterpret_cast<uint64>(tf); + tf->kernel_trap = reinterpret_cast<uint64>(&trap_dispatch); + tf->kernel_cpu_id = cpu::id(); +} + +} // namespace arch + #endif /* __ASSEMBLER__ */ diff --git a/na/arch/riscv/include/asm/csr.h b/na/arch/riscv/include/asm/csr.h new file mode 100644 index 0000000..42f1eb4 --- /dev/null +++ b/na/arch/riscv/include/asm/csr.h @@ -0,0 +1,37 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* Copyright (C) 2026 Richard Qin */ + +#ifndef _ASM_RISCV_CSR_H +#define _ASM_RISCV_CSR_H + +/* + * RISC-V CSR (Control & Status Register) definitions used by the early + * boot / MMU-switch path. + * + * LumeOS-native subset of Linux arch/riscv/include/asm/csr.h. Only the + * symbols head.S and mm/init.cc actually reference are defined — adding + * the rest is straightforward when new code needs them, and skipping the + * Linux ALTERNATIVE / vendor-extension machinery keeps the cascade flat. + */ + +/* SATP (Supervisor Address Translation and Protection) layout — RV64 */ +#define SATP_PPN_MASK 0x00000FFFFFFFFFFFUL /* bits [43:0] = root PPN */ +#define SATP_ASID_SHIFT 44 +#define SATP_MODE_SHIFT 60 + +/* SATP MODE encodings (RV64) */ +#define SATP_MODE_BARE (0UL << SATP_MODE_SHIFT) +#define SATP_MODE_SV39 (8UL << SATP_MODE_SHIFT) +#define SATP_MODE_SV48 (9UL << SATP_MODE_SHIFT) + +/* Compose a SATP value from a page-table physical address. + * Result: MODE | (root_ppn) where root_ppn = pgd_pa >> PAGE_SHIFT. */ +#define MAKE_SATP(mode, pgd_pa) ((mode) | (((pgd_pa) >> 12) & SATP_PPN_MASK)) + +/* SSTATUS bits we touch at boot — disable FP / vector at entry so that an + * accidental FP / V instruction in early code traps cleanly rather than + * silently using dirty state from before reset. */ +#define SR_FS_OFF (0UL << 13) +#define SR_VS_OFF (0UL << 9) + +#endif /* _ASM_RISCV_CSR_H */ diff --git a/na/arch/riscv/include/asm/page.h b/na/arch/riscv/include/asm/page.h new file mode 100644 index 0000000..7b9f155 --- /dev/null +++ b/na/arch/riscv/include/asm/page.h @@ -0,0 +1,62 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* Copyright (C) 2026 Richard Qin */ + +#ifndef _ASM_RISCV_PAGE_H +#define _ASM_RISCV_PAGE_H + +/* + * RISC-V page / virtual-memory layout constants. + * + * Mirrors the subset of Linux arch/riscv/include/asm/page.h that LumeOS's + * early boot path needs: + * - PAGE_SIZE / PAGE_SHIFT / PAGE_MASK + * - PAGE_OFFSET (start of the higher-half direct map) + * - KERNEL_LINK_ADDR (link-time VA of _start) + * - SV39 PTE flag bits + * - PTRS_PER_PGD (512 for SV39) + * + * Values match Linux RISC-V SV39 defaults so that struct kernel_mapping + * (filled by setup_vm in arch/riscv/mm/init.cc) carries the same numeric + * relations Linux's mm/init.c maintains. + */ + +#define PAGE_SHIFT 12 +#define PAGE_SIZE (1UL << PAGE_SHIFT) +#define PAGE_MASK (~(PAGE_SIZE - 1)) + +/* Higher-half base of the kernel direct map. Bottom of the SV39 supervisor + * address space minus the unmapped guard region. Identical to Linux's + * CONFIG_PAGE_OFFSET default on RV64. */ +#define PAGE_OFFSET 0xFFFFFFC000000000UL + +/* Where _start was linked. OpenSBI hands control to PHYS_BASE; the kernel + * runs at KERNEL_LINK_ADDR after relocate_enable_mmu jumps to higher-half. */ +#define PHYS_BASE 0x80200000UL +#define KERNEL_LINK_ADDR 0xFFFFFFC080200000UL + +/* SATP mode field — SV39 = 8, shifted to bits [63:60]. */ +#define SATP_MODE_SV39 (8UL << 60) + +#define PTRS_PER_PGD 512 + +/* 1 GB superpage size — entry.S used to build the early page table with + * these; setup_vm in arch/riscv/mm/init.cc now does the same. */ +#define PGDIR_SHIFT 30 +#define PGDIR_SIZE (1UL << PGDIR_SHIFT) +#define PGDIR_MASK (~(PGDIR_SIZE - 1)) + +/* SV39 PTE flag bits (one byte). */ +#define _PAGE_PRESENT (1UL << 0) /* aka VALID — historic Linux name */ +#define _PAGE_READ (1UL << 1) +#define _PAGE_WRITE (1UL << 2) +#define _PAGE_EXEC (1UL << 3) +#define _PAGE_USER (1UL << 4) +#define _PAGE_GLOBAL (1UL << 5) +#define _PAGE_ACCESSED (1UL << 6) +#define _PAGE_DIRTY (1UL << 7) + +#define PAGE_KERNEL_EXEC \ + (_PAGE_PRESENT | _PAGE_READ | _PAGE_WRITE | _PAGE_EXEC | \ + _PAGE_ACCESSED | _PAGE_DIRTY) + +#endif /* _ASM_RISCV_PAGE_H */ diff --git a/na/arch/riscv/include/asm/pgtable.h b/na/arch/riscv/include/asm/pgtable.h new file mode 100644 index 0000000..28fc4c5 --- /dev/null +++ b/na/arch/riscv/include/asm/pgtable.h @@ -0,0 +1,55 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* Copyright (C) 2026 Richard Qin */ + +#ifndef _ASM_RISCV_PGTABLE_H +#define _ASM_RISCV_PGTABLE_H + +/* + * Early page-table construction API used by arch/riscv/mm/init.cc::setup_vm. + * + * Mirrors the create_pgd_mapping interface from Linux's + * arch/riscv/mm/init.c — same name, same semantics: + * + * create_pgd_mapping(pgdir, va, pa, sz, prot) + * + * pgdir : root of the page table to populate + * va : virtual address to map + * pa : physical address to map to + * sz : mapping size — must be a supported leaf size (PAGE_SIZE, + * 2 MiB megapage, or 1 GiB gigapage) + * prot : PTE flag bits (PAGE_KERNEL_EXEC etc., from <asm/page.h>) + * + * The boot-time variant in this file only supports gigapage (1 GiB) + * mappings — that is all setup_vm needs for the early_pg_dir / trampoline_pg_dir + * built before the slab allocator exists. Finer-grained mappings come + * later via vmm_init(). + */ + +#include <lume/types.h> +#include <asm/page.h> + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct { unsigned long pgd; } pgd_t; + +/* Linker-provided early page-table storage (defined in lumeos.lds.S + * ARCH_BSS_EXTRA, lives in .bss so it's zero-initialized). Allocating + * them in the linker script — not in C — mirrors Linux and keeps the + * symbols visible from head.S without a forward C decl mismatch. */ +extern pgd_t early_pg_dir[PTRS_PER_PGD]; +extern pgd_t trampoline_pg_dir[PTRS_PER_PGD]; +extern pgd_t fixmap_pg_dir[PTRS_PER_PGD]; + +/* Install a single PGD entry mapping `va` → `pa` with size `sz` and PTE + * flags `prot`. Only sz == PGDIR_SIZE (1 GiB) is supported in this early + * variant — vmm_init() takes over with create_pgd_mapping_pmd later. */ +void create_pgd_mapping(pgd_t *pgdir, unsigned long va, unsigned long pa, + unsigned long sz, unsigned long prot); + +#ifdef __cplusplus +} +#endif + +#endif /* _ASM_RISCV_PGTABLE_H */ diff --git a/na/arch/riscv/include/asm/sections.h b/na/arch/riscv/include/asm/sections.h new file mode 100644 index 0000000..454d9de --- /dev/null +++ b/na/arch/riscv/include/asm/sections.h @@ -0,0 +1,50 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* Copyright (C) 2026 Richard Qin */ + +#ifndef _ASM_RISCV_SECTIONS_H +#define _ASM_RISCV_SECTIONS_H + +/* + * Linker-emitted symbols that bracket each section of the kernel image. + * + * The lumeos.lds.S linker script defines these as the start / end + * addresses of contiguous regions. Code that walks a section + * (relocate_kernel walks .rela.dyn, the C++ ctor loop walks + * .init_array, the VMM mapping code walks .text vs .rodata, ...) + * imports the pair via this header. + * + * Mirrors Linux arch/riscv/include/asm/sections.h / asm-generic/sections.h. + * Each name and pair is identical to Linux so that direct ports of mm/init.c + * relocate_kernel logic compile unchanged against LumeOS. + */ + +#ifdef __cplusplus +extern "C" { +#endif + +extern char _start[]; +extern char _end[]; + +extern char _stext[]; +extern char _etext[]; +extern char _srodata[]; +extern char _erodata[]; +extern char _sdata[]; +extern char _edata[]; +extern char _sbss[]; +extern char _ebss[]; + +extern char __init_array_start[]; +extern char __init_array_end[]; + +/* .rela.dyn — array of Elf64_Rela entries emitted by the linker under + * `-shared -Bsymbolic`. relocate_kernel walks this range and patches + * each R_RISCV_RELATIVE slot. */ +extern char __rela_dyn_start[]; +extern char __rela_dyn_end[]; + +#ifdef __cplusplus +} +#endif + +#endif /* _ASM_RISCV_SECTIONS_H */ diff --git a/na/arch/riscv/kernel/asm-offsets.c b/na/arch/riscv/kernel/asm-offsets.c new file mode 100644 index 0000000..edecab7 --- /dev/null +++ b/na/arch/riscv/kernel/asm-offsets.c @@ -0,0 +1,29 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +// Copyright (C) 2026 Richard Qin +// +// arch/riscv/kernel/asm-offsets.c — emit struct field offsets and +// scalar constants for consumption by assembly sources (head.S etc.). +// +// 1:1 with Linux arch/riscv/kernel/asm-offsets.c. The Makefile +// compiles this to an .s file (not .o) and runs +// scripts/extract-asm-offsets.sed over the output to produce +// include/generated/asm-offsets.h. +// +// Adding a new offset: include the struct's header, add an OFFSET() +// or DEFINE() line below, rebuild. head.S can then reference the +// generated #define directly. + +#include <lume/kbuild.h> +#include <lume/kernel_map.h> + +void asm_offsets(void) +{ + COMMENT("LumeOS asm-offsets — struct kernel_mapping field offsets"); + OFFSET(KERNEL_MAP_VIRT_ADDR, kernel_mapping, virt_addr); + OFFSET(KERNEL_MAP_VIRT_OFFSET, kernel_mapping, virt_offset); + OFFSET(KERNEL_MAP_PHYS_ADDR, kernel_mapping, phys_addr); + OFFSET(KERNEL_MAP_SIZE, kernel_mapping, size); + OFFSET(KERNEL_MAP_VA_PA_OFFSET, kernel_mapping, va_pa_offset); + OFFSET(KERNEL_MAP_PAGE_OFFSET, kernel_mapping, page_offset); + OFFSET(KERNEL_MAP_VA_KERNEL_PA_OFFSET, kernel_mapping, va_kernel_pa_offset); +} diff --git a/na/arch/riscv/kernel/head.S b/na/arch/riscv/kernel/head.S new file mode 100644 index 0000000..e04c44c --- /dev/null +++ b/na/arch/riscv/kernel/head.S @@ -0,0 +1,317 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* Copyright (C) 2026 Richard Qin */ + +/* + * RISC-V kernel entry — Linux-aligned layout. + * + * Flow (mirrors arch/riscv/kernel/head.S in Linux): + * + * _start [.head.text, byte 0 of image] + * · 64-byte Boot Protocol Image Header + * · j real_start + * + * real_start [.text] + * · save a0 (hartid) / a1 (dtb_pa) + * · setup gp / per-hart sp + * · BSP/AP election via amoswap + * · BSP: clear BSS, call setup_vm, then relocate_enable_mmu + * · AP: spin-wait, then relocate_enable_mmu + * + * relocate_enable_mmu [C function, arch/riscv/mm/init.cc] + * · csrw satp <- trampoline_pg_dir + * · jump to higher-half VA + * · csrw satp <- early_pg_dir + * · slide sp/ra/gp to higher-half + * · call relocate_kernel (walks .rela.dyn, patches R_RISCV_RELATIVE) + * + * real_start (continued in higher-half) + * · BSP only: call early_cpp_init (walks .init_array) + * · call kernel_main(hartid, dtb_pa) + * + * Register contract from OpenSBI / U-Boot: + * a0 = hart ID + * a1 = FDT physical base + */ + +#include <lume/config.h> + +#define PAGE_OFFSET 0xFFFFFFC000000000 +#define BOOT_STACK_SIZE 8192 + +/* ======================================================================== + * .head.text — RISC-V Boot Protocol Image Header + * + * Layout per Documentation/arch/riscv/boot-image-header.rst: + * code0/code1 (8 B) : executable code, lands at PC after OpenSBI hand-off + * text_offset (8 B) : load offset from RAM base + * image_size (8 B) : effective image size (0 = unknown) + * flags (8 B) : bit 0: endian + * version (4 B) : major.minor + * res1/res2 (4+8) : reserved + * magic (8 B) : "RISCV\0\0\0" + * magic2 (4 B) : "RSC\x05" + * res3 (4 B) : PE/COFF reserved + * ======================================================================== */ + .section .head.text, "ax" + .globl _start +_start: + j real_start /* code0 */ + .word 0 /* code1 */ + .dword 0x200000 /* text_offset */ + .dword 0 /* image_size (post-link) */ + .dword 0 /* flags */ + .word 0x0002 /* version 0.2 */ + .word 0 /* res1 */ + .dword 0 /* res2 */ + .dword 0x5643534952 /* magic "RISCV\0\0\0" */ + .word 0x05435352 /* magic2 "RSC\x05" */ + .word 0 /* res3 */ + +/* ======================================================================== + * real_start — all harts land here at physical PC + * ======================================================================== */ + .text + .globl real_start +real_start: + /* DEBUG: 'R<hartid>\n' — UART direct at 0x10000000, MMU off. */ + li t0, 0x10000000 + li t1, 'R' + sb t1, 0(t0) + addi t1, a0, '0' + sb t1, 0(t0) + li t1, '\n' + sb t1, 0(t0) + + /* Mask interrupts and supervisor pending bits. */ + csrw sie, zero + csrw sip, zero + + /* gp for medany — relax must be disabled across the `la`. */ + .option push + .option norelax + la gp, __global_pointer$ + .option pop + + /* per-hart boot stack: sp = boot_stack + (hartid + 1) * STACK_SIZE */ + la t0, boot_stack + li t1, BOOT_STACK_SIZE + addi t2, a0, 1 + mul t1, t1, t2 + add sp, t0, t1 + + /* Save hartid / dtb_pa in callee-saved regs preserved across calls. */ + mv tp, a0 /* tp = hartid */ + mv s0, a1 /* s0 = dtb_pa */ + + /* BSP/AP election: first hart to swap bsp_elected 0→1 is BSP. */ + la t0, bsp_elected + li t1, 1 + amoswap.w.aq t2, t1, (t0) + bnez t2, .Lap_wait /* prev != 0 → we're an AP */ + + /* === BSP path === */ + la t0, bsp_hart_id + sw a0, 0(t0) + + li t3, 0x10000000 + li t4, 'B' + sb t4, 0(t3) + li t4, '\n' + sb t4, 0(t3) + + /* Clear BSS — must precede setup_vm, which writes kernel_map. */ + la t0, _sbss + la t1, _ebss +.Lbss_loop: + bgeu t0, t1, .Lbss_done + sd zero, 0(t0) + addi t0, t0, 8 + j .Lbss_loop +.Lbss_done: + + li t3, 0x10000000 + li t4, 'V' + sb t4, 0(t3) + li t4, '\n' + sb t4, 0(t3) + + /* ---- BSP-only: pre-MMU PIC fixup of .rela.dyn ---- + * + * The kernel is built with -fPIE -shared. All global-data accesses + * compile to GOT-indirect loads (auipc + ld), where the linker + * emits each .got slot's initial value as the link-time VA and a + * companion R_RISCV_RELATIVE entry in .rela.dyn. At this point + * we're running at the physical load address — those link-time VAs + * are not yet mapped, so any global access would fault. + * + * Walk .rela.dyn in pure assembly (PC-relative only — no globals) + * and patch every R_RISCV_RELATIVE slot to addend + (PA - LINK_VA). + * After this, setup_vm can dereference kernel_map / early_pg_dir + * / etc. via the GOT and get correct PA-side pointers. + * + * Mirrors Linux arch/riscv/kernel/head.S — same idea, same loop. + * + * reloc: + * t0 = pa_offset = PA(_start) - KERNEL_LINK_ADDR (negative) + * t1 = &__rela_dyn_start + * t2 = &__rela_dyn_end + * for each 24-byte rela in [t1, t2): + * t3 = r_offset; t4 = r_info; t5 = r_addend + * if (t4 & 0xFFFFFFFF) != R_RISCV_RELATIVE: skip + * slot_pa = r_offset + pa_offset + * *slot_pa = r_addend + pa_offset + */ +#define R_RISCV_RELATIVE 3 + + la t0, _start /* PA of _start */ + li t1, 0xFFFFFFC080200000 /* KERNEL_LINK_ADDR (asm/page.h) */ + sub t0, t0, t1 /* t0 = pa_offset (PA - LINK_VA) */ + + la t1, __rela_dyn_start + la t2, __rela_dyn_end +.Lrela_loop: + bgeu t1, t2, .Lrela_done + ld t3, 0(t1) /* r_offset */ + ld t4, 8(t1) /* r_info */ + ld t5, 16(t1) /* r_addend */ + /* r_info low 32 bits = type */ + li t6, 0xFFFFFFFF + and t4, t4, t6 + li t6, R_RISCV_RELATIVE + bne t4, t6, .Lrela_next + /* slot_pa = r_offset + pa_offset */ + add t3, t3, t0 + /* value = r_addend + pa_offset */ + add t5, t5, t0 + sd t5, 0(t3) +.Lrela_next: + addi t1, t1, 24 /* sizeof(Elf64_Rela) */ + j .Lrela_loop +.Lrela_done: + + /* Memory fence so subsequent loads see the patched slots. */ + fence w, r + + li t3, 0x10000000 + li t4, 'P' + sb t4, 0(t3) + li t4, '\n' + sb t4, 0(t3) + + /* Call setup_vm(load_pa = PA of _start, dtb_pa = s0). + * Both args physical at this point — MMU still off. */ + la a0, _start + mv a1, s0 + call setup_vm + + li t3, 0x10000000 + li t4, 'v' + sb t4, 0(t3) + li t4, '\n' + sb t4, 0(t3) + + /* Release APs once page tables are built. */ + la t0, boot_sync_flag + li t1, 1 + amoswap.w.rl zero, t1, (t0) + j .Lenable_mmu + +/* ======================================================================== + * AP path — spin until BSP signals readiness. + * ======================================================================== */ +.Lap_wait: + la t0, boot_sync_flag +.Lap_spin: + lw t1, 0(t0) + beqz t1, .Lap_spin + fence r, rw + + /* fall through */ + +/* ======================================================================== + * .Lenable_mmu — all harts: enable MMU, jump to higher-half, run reloc. + * + * relocate_enable_mmu does its own SATP write + higher-half jump in + * C (with inline asm), then calls relocate_kernel. After it returns, + * we are at higher-half VA with sp / ra / gp slid up, and .rela.dyn + * is applied (so .init_array pointers are valid). + * ======================================================================== */ +.Lenable_mmu: + la a0, early_pg_dir + call relocate_enable_mmu + /* On return: PC, sp, ra, gp are all at higher-half VA. */ + + li t3, 0x10000000 + li t4, 'M' + sb t4, 0(t3) + li t4, '\n' + sb t4, 0(t3) + + /* Mark MMU enabled for console / pmap layers that check this. */ + call console_set_mmu_enabled + + /* BSP only: walk __init_array and call each C++ ctor. */ + la t0, bsp_hart_id + lw t0, 0(t0) + bne tp, t0, .Lskip_cpp_init + call early_cpp_init +.Lskip_cpp_init: + + /* Hand off to kernel_main(hartid, dtb_pa). */ + mv a0, tp + mv a1, s0 + call kernel_main + + /* kernel_main should not return. */ +.Lhalt: + csrc sstatus, 2 + wfi + j .Lhalt + +/* ======================================================================== + * early_cpp_init — walk .init_array and call each constructor. + * + * Called only by BSP, after relocate_kernel has patched the function + * pointers stored in __init_array. Before relocate_kernel runs those + * pointers are zero-filled by the linker. + * ======================================================================== */ + .globl early_cpp_init +early_cpp_init: + addi sp, sp, -32 + sd ra, 0(sp) + sd s1, 8(sp) + sd s2, 16(sp) + + la s1, __init_array_start + la s2, __init_array_end +.Linit_loop: + bgeu s1, s2, .Linit_done + ld t0, 0(s1) + jalr ra, t0 + addi s1, s1, 8 + j .Linit_loop +.Linit_done: + ld ra, 0(sp) + ld s1, 8(sp) + ld s2, 16(sp) + addi sp, sp, 32 + ret + +/* ======================================================================== + * Election state — in .data so explicit zero survives the BSP BSS clear + * (these are read before BSS is cleared on the BSP, and concurrently by + * APs while BSP is in the clear loop). + * ======================================================================== */ + .section .data + .align 2 + .globl boot_sync_flag +boot_sync_flag: + .word 0 + + .globl bsp_elected +bsp_elected: + .word 0 + + .globl bsp_hart_id +bsp_hart_id: + .word 0 diff --git a/na/arch/riscv/kernel/lumeos.lds.S b/na/arch/riscv/kernel/lumeos.lds.S index ca5218f..9bd083b 100644 --- a/na/arch/riscv/kernel/lumeos.lds.S +++ b/na/arch/riscv/kernel/lumeos.lds.S @@ -38,12 +38,33 @@ ENTRY(_start) PROVIDE(__global_pointer$ = . + 0x800); \ *(.sdata .sdata.*) -/* ===== RISC-V: early page table + per-CPU boot stacks in .bss ===== */ +/* ===== RISC-V: page-table storage + per-CPU boot stacks in .bss ===== + * + * Linux RISC-V layout (arch/riscv/kernel/vmlinux.lds.S): + * early_pg_dir — first early page table built by setup_vm, + * used for the rest of boot after MMU enable + * trampoline_pg_dir — single-page table used only during the SATP + * switch from physical to higher-half VA; its + * identity + virt mappings of _start must both + * be valid in the same instruction window + * fixmap_pg_dir — covers the FIXADDR_START range used by the + * early console (UART MMIO before kernel page + * table proper is built) + * + * All three are allocated here so head.S can take their addresses without + * a C definition; their PA == VA mapping covers them during setup_vm. + */ #define ARCH_BSS_EXTRA \ - /* Early page table: 4KB aligned, used by entry.S before VMM */ \ . = ALIGN(4096); \ - early_pgdir = .; \ + early_pg_dir = .; \ + . += 4096; \ + trampoline_pg_dir = .; \ + . += 4096; \ + fixmap_pg_dir = .; \ . += 4096; \ + /* Legacy name retained for any caller still referencing it; */ \ + /* points at the same first 4 KiB as early_pg_dir. */ \ + early_pgdir = early_pg_dir; \ /* Per-CPU boot stacks: 8KB per CPU, max 8 CPUs */ \ . = ALIGN(4096); \ boot_stack = .; \ @@ -65,16 +86,27 @@ SECTIONS { . = ALIGN(4096); - RODATA_SECTION + /* .rela.dyn lives between .text and .rodata so it ends up in the + * read-only PT_LOAD segment. Mirrors Linux vmlinux.lds.S where + * .rela.dyn sits at the tail of init data, just before RO_DATA. */ + RELA_DYN_SECTION . = ALIGN(4096); - RELOC_SECTION + RODATA_SECTION . = ALIGN(4096); DATA_SECTION + . = ALIGN(8); + + /* .got lives right after .data in the RW PT_LOAD. All other + * dynamic-linker artefacts (.plt, .dynamic, .dynsym, .dynstr, + * .hash, .gnu.hash, .interp, .dynbss) are discarded by the + * DISCARDS macro — mirrors Linux arch/riscv/kernel/vmlinux.lds.S. */ + GOT_SECTION + . = ALIGN(4096); /* End of file-backed image (BSS follows as NOBITS) */ diff --git a/na/arch/riscv/kernel/signal.cc b/na/arch/riscv/kernel/signal.cc new file mode 100644 index 0000000..39fe34e --- /dev/null +++ b/na/arch/riscv/kernel/signal.cc @@ -0,0 +1,163 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* Copyright (C) 2026 Richard Qin */ + +/* + * arch/riscv/signal.cc — Signal frame construction and sigreturn + * + * Constructs a signal frame on the user stack before returning to + * the user signal handler. On sigreturn, restores the original + * registers from the saved frame. + * + * Frame layout on user stack (grows downward): + * + * +-----------------------+ <- original sp + * | sigset_t saved_mask | 8 bytes + * | TrapFrame saved_regs | 296 bytes + * | kernel_siginfo info | 128 bytes + * +-----------------------+ <- new sp (16-byte aligned) + * + * Reference: Linux arch/riscv/kernel/signal.c + */ + +#include <lume/signal.h> +#include <lume/signal_types.h> +#include <lume/signal_nr.h> +#include <lume/task.h> +#include <lume/uaccess.h> +#include <lume/errno.h> +#include <lume/klog.h> +#include <arch/trapframe.h> + +/* ================================================================== */ +/* Signal frame layout */ +/* ================================================================== */ + +struct rt_sigframe { + kernel_siginfo info; /* siginfo for SA_SIGINFO handlers */ + arch::TrapFrame saved_regs; /* saved register state */ + sigset_t saved_mask; /* blocked mask to restore on sigreturn */ +}; + +/* ================================================================== */ +/* Signal trampoline (written to user stack) */ +/* ================================================================== */ + +/* + * When the user's signal handler returns (via `ret`), ra points here. + * This tiny code sequence invokes sys_rt_sigreturn to restore the + * original execution context. + * + * RISC-V instructions (8 bytes): + * li a7, 139 → addi a7, zero, 139 → 0x08b00893 + * ecall → 0x00000073 + */ +static constexpr uint32 sigreturn_trampoline[2] = { + 0x08b00893, /* li a7, 139 (SYS_rt_sigreturn) */ + 0x00000073, /* ecall */ +}; + +/* ================================================================== */ +/* setup_signal_frame */ +/* ================================================================== */ + +int setup_signal_frame(arch::TrapFrame* tf, int signo, + kernel_sigaction* ka, kernel_siginfo* info) +{ + TaskControlBlock* curr = current_task(); + if (!curr) return -EINVAL; + + /* Calculate frame location on user stack */ + uint64 sp = tf->sp; + + /* Reserve space for the trampoline below the frame */ + sp -= sizeof(sigreturn_trampoline); + sp &= ~15ULL; /* 16-byte align */ + uint64 trampoline_addr = sp; + + /* Reserve space for the signal frame */ + sp -= sizeof(rt_sigframe); + sp &= ~15ULL; /* 16-byte align */ + uint64 frame_addr = sp; + + /* Build the frame in kernel memory, then copy to user stack */ + rt_sigframe frame{}; + if (info) { + frame.info = *info; + } + frame.saved_regs = *tf; + frame.saved_mask = curr->blocked; + + /* Copy frame to user stack */ + int err = copy_to_user(frame_addr, &frame, sizeof(frame)); + if (err < 0) return -EFAULT; + + /* Copy trampoline to user stack */ + err = copy_to_user(trampoline_addr, sigreturn_trampoline, + sizeof(sigreturn_trampoline)); + if (err < 0) return -EFAULT; + + /* Set up the trap frame to execute the signal handler */ + tf->epc = ka->sa_handler; + tf->sp = frame_addr; + + /* Arguments to the handler: handler(int signo, siginfo_t*, ucontext_t*) */ + tf->a[0] = static_cast<uint64>(signo); + + if (ka->sa_flags & SA_SIGINFO) { + /* SA_SIGINFO: pass siginfo and ucontext pointers */ + tf->a[1] = frame_addr; /* &info */ + tf->a[2] = frame_addr + sizeof(kernel_siginfo); /* &saved_regs (as ucontext) */ + } else { + tf->a[1] = 0; + tf->a[2] = 0; + } + + /* Return address: when the handler does `ret`, jump to trampoline */ + if (ka->sa_flags & SA_RESTORER) { + tf->ra = ka->sa_restorer; + } else { + tf->ra = trampoline_addr; + } + + return 0; +} + +/* ================================================================== */ +/* arch_sigreturn — restore state from signal frame */ +/* ================================================================== */ + +int64 arch_sigreturn(arch::TrapFrame* tf) +{ + TaskControlBlock* curr = current_task(); + if (!curr) return -EINVAL; + + /* + * The signal frame is located at the current sp. + * We need to read it back and restore the original state. + */ + uint64 frame_addr = tf->sp; + + rt_sigframe frame{}; + int err = copy_from_user(&frame, frame_addr, sizeof(frame)); + if (err < 0) { + printk(KERN_ERR "sigreturn: failed to read signal frame at %llx\n", + frame_addr); + task_kill_current(SIGSEGV); + return -EFAULT; + } + + /* Restore the saved register state */ + *tf = frame.saved_regs; + + /* Restore the blocked mask */ + curr->blocked = frame.saved_mask; + curr->blocked.del(SIGKILL); + curr->blocked.del(SIGSTOP); + + /* + * Return value: we return the saved a0 from the original context. + * The syscall dispatch will NOT overwrite a0 because sigreturn + * is special — it returns via the restored trap frame directly. + */ + return tf->a[0]; +} diff --git a/na/arch/riscv/mm/init.cc b/na/arch/riscv/mm/init.cc new file mode 100644 index 0000000..3cebcd9 --- /dev/null +++ b/na/arch/riscv/mm/init.cc @@ -0,0 +1,241 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* Copyright (C) 2026 Richard Qin */ + +/* + * RISC-V early memory-management bootstrap. + * + * Mirrors Linux arch/riscv/mm/init.c — same file name, same C function + * names (setup_vm / relocate_enable_mmu / relocate_kernel), same global + * state (kernel_map, early_pg_dir, trampoline_pg_dir). head.S calls + * setup_vm immediately after BSS-clear, then later calls + * relocate_enable_mmu, which switches SATP, jumps to higher-half VA, + * and finally calls relocate_kernel to apply .rela.dyn. + * + * Differences from Linux: + * - KASLR offset is hard-wired to 0 for now (no pi/ subsystem) — the + * code path that would consume kernel_map.virt_offset is in place + * so adding pi/-based seed reading later only requires populating + * that one field. + * - Only 1 GiB superpages supported in create_pgd_mapping; sufficient + * for the early page tables. Finer-grained mappings come from + * vmm_init() once slab is up. + */ + +#include <lume/init.h> +#include <lume/setup.h> +#include <lume/elf.h> +#include <lume/kernel_map.h> +#include <lume/types.h> +#include <asm/page.h> +#include <asm/csr.h> +#include <asm/pgtable.h> +#include <asm/sections.h> + +/* + * Global kernel mapping descriptor — single source of truth for the + * runtime-vs-link address relationship. Lives in .data so its initial + * zero contents survive the BSS-clear that runs before setup_vm fills + * the fields. Read by VMM, page-fault handler, ptdump, etc. + * + * Linux equivalent: arch/riscv/mm/init.c::kernel_map + */ +struct kernel_mapping kernel_map __attribute__((section(".data"))); + +/* + * create_pgd_mapping — install one PGD entry covering a 1 GiB region. + * + * Only 1 GiB superpages are supported in this early variant; setup_vm + * needs nothing finer. Identical interface to Linux's same-named + * function in arch/riscv/mm/init.c, restricted in arity. + */ +extern "C" void create_pgd_mapping(pgd_t *pgdir, unsigned long va, + unsigned long pa, unsigned long sz, + unsigned long prot) +{ + /* This early variant only supports 1 GiB superpages. */ + if (sz != PGDIR_SIZE) { + return; + } + + /* SV39 PGD index = (va >> 30) & 0x1FF — see Volume II, §4.4.1. + * Each PGD entry is 8 bytes; pgdir[i] is the slot for that VPN[2]. */ + unsigned long idx = (va >> PGDIR_SHIFT) & (PTRS_PER_PGD - 1); + + /* PTE PPN field: physical page number shifted up by 10 (SV39 PTE + * layout: bits [53:10] = PPN, bits [9:0] = flags). */ + unsigned long ppn = pa >> PAGE_SHIFT; + pgdir[idx].pgd = (ppn << 10) | prot; +} + +/* + * setup_vm — fill kernel_map and build early_pg_dir + trampoline_pg_dir. + * + * Called from head.S after BSS is cleared, with: + * load_pa = runtime physical address of _start + * dtb_pa = physical address of the device tree blob (from a1) + * + * Mirrors Linux arch/riscv/mm/init.c::setup_vm. The actual SATP write + * happens later in relocate_enable_mmu; here we only populate page + * tables and the kernel_map descriptor. + */ +extern "C" void __init setup_vm(uintptr_t load_pa, uintptr_t dtb_pa) +{ + (void)dtb_pa; /* KASLR pi/ subsystem will consume this later. */ + + /* --- 1. Fill kernel_map ----------------------------------------- */ + + /* KASLR offset = 0 in this configuration. */ + kernel_map.virt_offset = 0; + kernel_map.virt_addr = KERNEL_LINK_ADDR + kernel_map.virt_offset; + kernel_map.phys_addr = load_pa; + kernel_map.size = (uintptr_t)_end - (uintptr_t)_start; + kernel_map.page_offset = PAGE_OFFSET; + + /* Mirrors Linux: va_pa_offset is the constant added to a physical + * address to obtain its direct-map VA. */ + kernel_map.va_pa_offset = PAGE_OFFSET - load_pa; + + /* va_kernel_pa_offset is the constant added to a load PA to obtain + * the kernel's link VA (used by relocate_enable_mmu to fix sp/ra). */ + kernel_map.va_kernel_pa_offset = kernel_map.virt_addr - load_pa; + + /* --- 2. Build early_pg_dir ------------------------------------- * + * + * SV39 root page table — 4 entries cover identity + higher-half for + * both MMIO (PA 0x0000_0000, 1 GiB) and RAM (PA 0x8000_0000, 1 GiB): + * + * [0] identity PA 0x0000_0000 → VA 0x0000_0000 + * [2] identity PA 0x8000_0000 → VA 0x8000_0000 + * [256] high-half PA 0x0000_0000 → VA 0xFFFFFFC000000000 + * [258] high-half PA 0x8000_0000 → VA 0xFFFFFFC080000000 + * + * Identity mappings keep PC valid right after csrw satp. High-half + * mappings let the post-MMU `jr` to .Lhigher_half resolve. */ + create_pgd_mapping(early_pg_dir, 0x00000000UL, 0x00000000UL, + PGDIR_SIZE, PAGE_KERNEL_EXEC); + create_pgd_mapping(early_pg_dir, 0x80000000UL, 0x80000000UL, + PGDIR_SIZE, PAGE_KERNEL_EXEC); + create_pgd_mapping(early_pg_dir, PAGE_OFFSET + 0x00000000UL, + 0x00000000UL, PGDIR_SIZE, PAGE_KERNEL_EXEC); + create_pgd_mapping(early_pg_dir, PAGE_OFFSET + 0x80000000UL, + 0x80000000UL, PGDIR_SIZE, PAGE_KERNEL_EXEC); + + /* --- 3. Build trampoline_pg_dir -------------------------------- * + * + * The trampoline maps just the gigapage containing _start, both at + * its identity PA and at its higher-half VA. This is the page + * table SATP holds during the single instruction window between + * "MMU off" and "MMU on with early_pg_dir" — the PC must resolve + * under both, so we re-use the same two entries above. */ + create_pgd_mapping(trampoline_pg_dir, load_pa & PGDIR_MASK, + load_pa & PGDIR_MASK, PGDIR_SIZE, PAGE_KERNEL_EXEC); + create_pgd_mapping(trampoline_pg_dir, + (kernel_map.virt_addr) & PGDIR_MASK, + load_pa & PGDIR_MASK, PGDIR_SIZE, PAGE_KERNEL_EXEC); +} + +/* + * relocate_kernel — walk .rela.dyn and apply R_RISCV_RELATIVE entries. + * + * Called from relocate_enable_mmu *after* the SATP switch and the jump + * to higher-half VA — meaning __rela_dyn_start / __rela_dyn_end resolve + * to valid higher-half pointers we can dereference directly. + * + * Mirrors Linux arch/riscv/mm/init.c::relocate_kernel. Key invariant: + * even when kaslr_offset == 0, the slot store MUST still happen — the + * linker leaves R_RISCV_RELATIVE slots zero-filled at link time, and + * only relocate_kernel writes the link-time addend into them. Without + * this pass, .init_array entries (and every other relocated symbol) + * are NULL pointers. + */ +void __init relocate_kernel(void) +{ + auto *rela = (Elf64_Rela *)__rela_dyn_start; + auto *rela_end = (Elf64_Rela *)__rela_dyn_end; + + const uint64 kaslr_offset = kernel_map.virt_offset; + + for (; rela < rela_end; ++rela) { + if (ELF64_R_TYPE(rela->r_info) != R_RISCV_RELATIVE) { + continue; + } + auto *slot = (Elf64_Addr *)(rela->r_offset + kaslr_offset); + *slot = (Elf64_Addr)(rela->r_addend) + kaslr_offset; + } +} + +/* + * relocate_enable_mmu — switch SATP, jump to higher-half, apply relocations. + * + * Called from head.S after setup_vm has filled kernel_map and built the + * page tables. Equivalent to Linux's same-named asmlinkage routine but + * written in C++ so it can share kernel_map / pgtable definitions with + * setup_vm directly. + * + * Three logical steps: + * + * 1. csrw satp ← trampoline_pg_dir. PC continues running at the + * identity PA, which the trampoline still maps; the high-half PA + * mapping is now also live. + * + * 2. la t0, .Lvirt; add t0, t0, va_kernel_pa_offset; jr t0. + * A PC-relative `la` under medany yields the runtime PA of the + * label; adding va_kernel_pa_offset (= link_va - load_pa) converts + * it to the link-time higher-half VA, which the trampoline maps. + * The `jr` then atomically pushes PC into higher-half space. + * + * 3. csrw satp ← early_pg_dir. We're now executing at higher-half + * VA, so the trampoline can be retired in favour of the full + * early page table. sp / ra / gp must be slid to higher-half too, + * otherwise the next ret-or-call uses physical-low addresses + * whose mapping is about to disappear when vmm_init drops the + * identity entries. + * + * 4. Call relocate_kernel — now safe, because all link-time VA + * pointers in __rela_dyn_start[..] dereference correctly. + */ +extern "C" void __init relocate_enable_mmu(pgd_t *pgdir) +{ + (void)pgdir; /* head.S passes early_pg_dir for documentation; we + * read the global symbol directly inside the asm. */ + + const unsigned long va_kernel_pa_offset = kernel_map.va_kernel_pa_offset; + const unsigned long satp_mode = SATP_MODE_SV39; + + asm volatile ( + /* Step 1: csrw satp <- trampoline */ + "la t0, trampoline_pg_dir\n" + "srli t0, t0, 12\n" + "or t0, t0, %[mode]\n" + "csrw satp, t0\n" + "sfence.vma\n" + + /* Step 2: jump to higher-half VA */ + "la t0, 1f\n" + "add t0, t0, %[off]\n" + "jr t0\n" + "1:\n" + + /* Step 3a: csrw satp <- early_pg_dir (now in higher-half) */ + "la t0, early_pg_dir\n" + "srli t0, t0, 12\n" + "or t0, t0, %[mode]\n" + "csrw satp, t0\n" + "sfence.vma\n" + + /* Step 3b: slide sp / ra / gp to higher-half */ + "add sp, sp, %[off]\n" + "add ra, ra, %[off]\n" + ".option push\n" + ".option norelax\n" + "la gp, __global_pointer$\n" + ".option pop\n" + : + : [off] "r"(va_kernel_pa_offset), + [mode] "r"(satp_mode) + : "t0", "memory" + ); + + /* Step 4: apply .rela.dyn now that VA dereferences work. */ + relocate_kernel(); +} diff --git a/na/arch/riscv/mm/pmap.cc b/na/arch/riscv/mm/pmap.cc index ae21d0e..f0a6848 100644 --- a/na/arch/riscv/mm/pmap.cc +++ b/na/arch/riscv/mm/pmap.cc @@ -21,6 +21,7 @@ #include <lume/memblock.h> #include <arch/mmu.h> #include <lume/klog.h> +#include <lume/errno.h> namespace pmap { @@ -45,6 +46,13 @@ inline constexpr uint64 PTE_COW = 1ULL << 8; inline constexpr int kLevels = 3; inline constexpr int kPtePerPage = 512; +/* SV39: upper half of VA space (kernel) starts at L2 index 256 */ +inline constexpr int kKernelPgdStart = kPtePerPage / 2; + +/* SATP PPN mask (44 bits) and page shift */ +inline constexpr uint64 kSatpPpnMask = 0xFFFFFFFFFFFULL; +inline constexpr int kPageShift = 12; + /* Extract VPN[level] from a virtual address */ static inline uint64 vpn(uint64 va, int level) { @@ -101,13 +109,43 @@ static uint64 alloc_pt_page() uint64 create() { - return alloc_pt_page(); + uint64 new_root = alloc_pt_page(); + if (!new_root) return 0; + + /* Copy kernel mappings from the current kernel page table. + * In SV39, entries 256..511 (VPN[2] >= 256) cover the upper + * half of the 64-bit VA space where the kernel lives. + * Without these, switching to the new page table would + * immediately fault on any kernel VA access. + * + * Only do this after the runtime page table is active — + * during early boot the bootstrap PT doesn't cover all of + * RAM so pa_to_ptr on the kernel root would fault. */ + if (g_runtime_mapping_ready) { + uint64 kernel_satp; + asm volatile("csrr %0, satp" : "=r"(kernel_satp)); + uint64 kernel_root_pa = (kernel_satp & kSatpPpnMask) << kPageShift; + + uint64 *new_l2 = pa_to_ptr(new_root); + uint64 *kern_l2 = pa_to_ptr(kernel_root_pa); + + for (int i = kKernelPgdStart; i < kPtePerPage; i++) { + new_l2[i] = kern_l2[i]; + } + } + + return new_root; } void destroy(uint64 root_pa) { uint64 *l2 = pa_to_ptr(root_pa); - for (int i = 0; i < kPtePerPage; i++) { + + /* Only free the user half (entries 0..255). + * Entries 256..511 are shared kernel PGD pointers copied + * from the kernel page table — freeing them would corrupt + * the kernel's own mappings. */ + for (int i = 0; i < kKernelPgdStart; i++) { if (!(l2[i] & PTE_V)) continue; /* Non-leaf L2 entry? (no R/W/X bits = pointer to L1 table) */ @@ -139,7 +177,7 @@ int map(uint64 root_pa, uint64 va, uint64 pa, uint64 perm) if (!(table[idx] & PTE_V)) { uint64 child_pa = alloc_pt_page(); if (!child_pa) - return -12; /* -ENOMEM */ + return -ENOMEM; /* -ENOMEM */ table[idx] = pa_to_pte(child_pa, PTE_V); } table = pa_to_ptr(pte_to_pa(table[idx])); @@ -156,7 +194,7 @@ int map_2mb(uint64 root_pa, uint64 va, uint64 pa, uint64 perm) /* VA and PA must be 2MB-aligned */ constexpr uint64 k2MB = 2ULL * 1024 * 1024; if ((va & (k2MB - 1)) != 0 || (pa & (k2MB - 1)) != 0) { - return -22; /* -EINVAL */ + return -EINVAL; /* -EINVAL */ } uint64 *table = pa_to_ptr(root_pa); @@ -166,7 +204,7 @@ int map_2mb(uint64 root_pa, uint64 va, uint64 pa, uint64 perm) if (!(table[idx2] & PTE_V)) { uint64 child_pa = alloc_pt_page(); if (!child_pa) { - return -12; /* -ENOMEM */ + return -ENOMEM; /* -ENOMEM */ } table[idx2] = pa_to_pte(child_pa, PTE_V); } @@ -197,7 +235,7 @@ int protect(uint64 root_pa, uint64 va, uint64 perm) uint64 idx = vpn(va, level); uint64 pte = table[idx]; if (!(pte & PTE_V)) - return -14; /* -EFAULT */ + return -EFAULT; /* -EFAULT */ if (pte & (PTE_R | PTE_W | PTE_X)) { uint64 pa = pte_to_pa(pte); table[idx] = pa_to_pte(pa, perm | PTE_V); @@ -205,7 +243,7 @@ int protect(uint64 root_pa, uint64 va, uint64 perm) } table = pa_to_ptr(pte_to_pa(pte)); } - return -14; + return -EFAULT; } int protect_range(uint64 root_pa, uint64 start, uint64 end, uint64 perm) @@ -250,6 +288,21 @@ bool lookup(uint64 root_pa, uint64 va, uint64 *pa_out) return false; } +uint64 lookup_pte(uint64 root_pa, uint64 va) +{ + uint64 *table = pa_to_ptr(root_pa); + for (int level = 2; level >= 0; level--) { + uint64 idx = vpn(va, level); + uint64 pte = table[idx]; + if (!(pte & PTE_V)) + return 0; + if (pte & (PTE_R | PTE_W | PTE_X)) + return pte; // leaf PTE raw value + table = pa_to_ptr(pte_to_pa(pte)); + } + return 0; +} + bool is_cow(uint64 root_pa, uint64 va) { uint64 *table = pa_to_ptr(root_pa); @@ -273,7 +326,7 @@ int make_cow(uint64 root_pa, uint64 va) uint64 idx = vpn(va, level); uint64 pte = table[idx]; if (!(pte & PTE_V)) - return -14; /* -EFAULT */ + return -EFAULT; /* -EFAULT */ if (pte & (PTE_R | PTE_W | PTE_X)) { /* Clear write bit, set COW bit */ pte &= ~PTE_W; @@ -283,7 +336,7 @@ int make_cow(uint64 root_pa, uint64 va) } table = pa_to_ptr(pte_to_pa(pte)); } - return -14; + return -EFAULT; } void activate(uint64 root_pa) diff --git a/na/arch/riscv/mm/tlb_md.cc b/na/arch/riscv/mm/tlb_md.cc new file mode 100644 index 0000000..ed9aa29 --- /dev/null +++ b/na/arch/riscv/mm/tlb_md.cc @@ -0,0 +1,17 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* Copyright (C) 2026 Richard Qin */ + +#include <arch/tlb.h> +#include <arch/sbi.h> + +namespace arch::tlb { + +void flush_remote(uint64 cpu_mask, uint64 start, uint64 size) { + if (size == ~0ULL) { + arch::sbi::sbi_remote_sfence_vma(cpu_mask, 0, 0, ~0UL); + } else { + arch::sbi::sbi_remote_sfence_vma(cpu_mask, 0, start, size); + } +} + +} // namespace arch::tlb diff --git a/na/arch/riscv/module.mk b/na/arch/riscv/module.mk index af1cc26..97d22de 100644 --- a/na/arch/riscv/module.mk +++ b/na/arch/riscv/module.mk @@ -1,16 +1,18 @@ # arch/riscv/module.mk SRCS_CC += arch/riscv/mm/pmap.cc +SRCS_CC += arch/riscv/mm/init.cc SRCS_CC += arch/riscv/config.cc SRCS_CC += arch/riscv/trap/trap_md.cc SRCS_CC += arch/riscv/syscall/syscall_md.cc SRCS_CC += arch/riscv/lib/random.cc -SRCS_CC += arch/riscv/boot/compressed.cc -SRCS_CC += arch/riscv/relocate.cc -SRCS_S += arch/riscv/boot/entry.S +SRCS_CC += arch/riscv/mm/tlb_md.cc +SRCS_S += arch/riscv/kernel/head.S SRCS_S += arch/riscv/trap/swtch.S +SRCS_S += arch/riscv/trap/fpu.S SRCS_S += arch/riscv/trap/trap_vector.S SRCS_S += arch/riscv/uaccess/uaccess_memcpy.S SRCS_CC += arch/riscv/trap/extable.cc +SRCS_CC += arch/riscv/kernel/signal.cc # Arch-specific headers (added to include path) INCLUDES += -Iarch/riscv/include diff --git a/na/arch/riscv/relocate.cc b/na/arch/riscv/relocate.cc deleted file mode 100644 index 0216827..0000000 --- a/na/arch/riscv/relocate.cc +++ /dev/null @@ -1,100 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-or-later */ -/* Copyright (C) 2026 Richard Qin */ - -#include <lume/relocate.h> - -static inline bool range_ok(uint64 offset, uint64 size, uint64 image_size) -{ - if (offset > image_size) { - return false; - } - if (size > image_size - offset) { - return false; - } - return true; -} - -RelocStatus relocate_apply(uint64 load_base, uint64 image_size, - uint64 link_base, const void* reloc_blob, - uint64 reloc_size) -{ - if (!reloc_blob || reloc_size == 0) { - return RELOC_OK; - } - if (reloc_size < sizeof(RelocTableHeader)) { - return RELOC_ERR_SIZE; - } - - const auto* hdr = reinterpret_cast<const RelocTableHeader*>(reloc_blob); - if (hdr->magic != kRelocTableMagic) { - return RELOC_ERR_MAGIC; - } - if (hdr->version != kRelocTableVersion) { - return RELOC_ERR_VERSION; - } - if (hdr->arch != RELOC_ARCH_RISCV) { - return RELOC_ERR_ARCH; - } - if (hdr->entry_size < sizeof(RelocEntry) || (hdr->entry_size % 8) != 0) { - return RELOC_ERR_SIZE; - } - - uint64 max_entries = (reloc_size - sizeof(RelocTableHeader)) / - static_cast<uint64>(hdr->entry_size); - if (hdr->entry_count > max_entries) { - return RELOC_ERR_SIZE; - } - - if (image_size < sizeof(uint64)) { - return RELOC_ERR_RANGE; - } - - const uint64 delta = load_base - link_base; - const uint8* p = reinterpret_cast<const uint8*>(reloc_blob) + - sizeof(RelocTableHeader); - - for (uint32 i = 0; i < hdr->entry_count; ++i) { - const auto* ent = reinterpret_cast<const RelocEntry*>(p); - if ((ent->offset & 0x7) != 0 || - !range_ok(ent->offset, sizeof(uint64), image_size)) { - return RELOC_ERR_RANGE; - } - - uint64* target = reinterpret_cast<uint64*>(load_base + ent->offset); - switch (ent->type) { - case RELOC_RISCV_RELATIVE: - *target = load_base + ent->addend; - break; - case RELOC_RISCV_64: - *target = ent->addend + delta; - break; - default: - return RELOC_ERR_UNSUPPORTED; - } - p += hdr->entry_size; - } - - return RELOC_OK; -} - -const char* reloc_status_str(RelocStatus st) -{ - switch (st) { - case RELOC_OK: - return "ok"; - case RELOC_ERR_MAGIC: - return "bad magic"; - case RELOC_ERR_VERSION: - return "bad version"; - case RELOC_ERR_ARCH: - return "arch mismatch"; - case RELOC_ERR_SIZE: - return "size mismatch"; - case RELOC_ERR_RANGE: - return "range"; - case RELOC_ERR_UNSUPPORTED: - return "unsupported"; - default: - return "unknown"; - } -} diff --git a/na/arch/riscv/syscall/syscall_md.cc b/na/arch/riscv/syscall/syscall_md.cc index a52eb0e..26320c8 100644 --- a/na/arch/riscv/syscall/syscall_md.cc +++ b/na/arch/riscv/syscall/syscall_md.cc @@ -11,7 +11,9 @@ */ #include <lume/syscall.h> +#include <lume/syscall_nr.h> #include <arch/trapframe.h> +#include <arch/cpu.h> void syscall_dispatch_md(arch::TrapFrame* tf) { @@ -22,7 +24,19 @@ void syscall_dispatch_md(arch::TrapFrame* tf) tf->a[3], tf->a[4], tf->a[5], }; + /* Enable interrupts so timer and preemption work during syscalls */ + arch::cpu::intr_on(); + int64 ret = syscall_dispatch(nr, args); + arch::cpu::intr_off(); + + /* sys_rt_sigreturn restores the entire TrapFrame from the signal + * frame on the user stack. Writing ret into a[0] would clobber + * the restored value, so we must skip the writeback. */ + if (nr == static_cast<uint64>(SYS_rt_sigreturn)){ + return; + } tf->a[0] = static_cast<uint64>(ret); /* Return value in a0 */ } + diff --git a/na/arch/riscv/trap/fpu.S b/na/arch/riscv/trap/fpu.S new file mode 100644 index 0000000..90d96d9 --- /dev/null +++ b/na/arch/riscv/trap/fpu.S @@ -0,0 +1,103 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* Copyright (C) 2026 Richard Qin */ + +/* + * FPU Save / Restore (RISC-V D extension) + * + * void fpu_save(FpState* state); + * Saves f0-f31 and fcsr to the FpState structure. + * + * void fpu_restore(FpState* state); + * Restores f0-f31 and fcsr from the FpState structure. + * + * FpState layout: f[0..31] at offset 0 (each 8 bytes), fcsr at offset 256. + * + * Called from schedule() when switching between tasks that have + * used the FPU (sstatus.FS == Dirty on save path). + */ + + .section .text + +/* ====================================================================== + * void fpu_save(FpState* state); // a0 = state pointer + * ====================================================================== */ + .globl fpu_save + .align 2 +fpu_save: + fsd f0, 0(a0) + fsd f1, 8(a0) + fsd f2, 16(a0) + fsd f3, 24(a0) + fsd f4, 32(a0) + fsd f5, 40(a0) + fsd f6, 48(a0) + fsd f7, 56(a0) + fsd f8, 64(a0) + fsd f9, 72(a0) + fsd f10, 80(a0) + fsd f11, 88(a0) + fsd f12, 96(a0) + fsd f13, 104(a0) + fsd f14, 112(a0) + fsd f15, 120(a0) + fsd f16, 128(a0) + fsd f17, 136(a0) + fsd f18, 144(a0) + fsd f19, 152(a0) + fsd f20, 160(a0) + fsd f21, 168(a0) + fsd f22, 176(a0) + fsd f23, 184(a0) + fsd f24, 192(a0) + fsd f25, 200(a0) + fsd f26, 208(a0) + fsd f27, 216(a0) + fsd f28, 224(a0) + fsd f29, 232(a0) + fsd f30, 240(a0) + fsd f31, 248(a0) + frcsr t0 + sw t0, 256(a0) + ret + +/* ====================================================================== + * void fpu_restore(FpState* state); // a0 = state pointer + * ====================================================================== */ + .globl fpu_restore + .align 2 +fpu_restore: + fld f0, 0(a0) + fld f1, 8(a0) + fld f2, 16(a0) + fld f3, 24(a0) + fld f4, 32(a0) + fld f5, 40(a0) + fld f6, 48(a0) + fld f7, 56(a0) + fld f8, 64(a0) + fld f9, 72(a0) + fld f10, 80(a0) + fld f11, 88(a0) + fld f12, 96(a0) + fld f13, 104(a0) + fld f14, 112(a0) + fld f15, 120(a0) + fld f16, 128(a0) + fld f17, 136(a0) + fld f18, 144(a0) + fld f19, 152(a0) + fld f20, 160(a0) + fld f21, 168(a0) + fld f22, 176(a0) + fld f23, 184(a0) + fld f24, 192(a0) + fld f25, 200(a0) + fld f26, 208(a0) + fld f27, 216(a0) + fld f28, 224(a0) + fld f29, 232(a0) + fld f30, 240(a0) + fld f31, 248(a0) + lw t0, 256(a0) + fscsr t0 + ret diff --git a/na/arch/riscv/trap/trap_md.cc b/na/arch/riscv/trap/trap_md.cc index f910b26..e30aea6 100644 --- a/na/arch/riscv/trap/trap_md.cc +++ b/na/arch/riscv/trap/trap_md.cc @@ -120,16 +120,45 @@ static TrapInfo build_trap_info(arch::TrapFrame* tf, * * Reads RISC-V CSRs, translates to MI TrapCause, applies arch-specific * fixups, then calls MI trap_handle(). + * + * After trap_handle returns, we must return to user mode via userret(). + * The assembly jalr t3 that called us has NO return-to-user code after it, + * so trap_dispatch must never return normally. * ======================================================================== */ -extern "C" void trap_dispatch(arch::TrapFrame* tf) +extern "C" void userret(arch::TrapFrame* tf); + +extern "C" [[noreturn]] void trap_dispatch(arch::TrapFrame* tf) { + /* Switch stvec to kernelvec immediately so that any traps during + * kernel execution (timer IRQs, page faults from copy_from_user) + * go through the kernel trap entry, not the user trap entry. + * This matches Linux which sets stvec to handle_exception (kernel) + * at the top of the user trap path. */ + extern void kernelvec(); + arch::trap::set_vector(reinterpret_cast<uint64>(&kernelvec)); + + /* Enable SUM so S-mode can access user pages (PTE_U). + * Without this, any dereference of user-space pointers in syscall + * handlers (sys_write buf, sys_read buf, etc.) would page fault. */ + arch::trap::enable_sum(); + uint64 scause = arch::trap::get_cause(); uint64 stval = arch::trap::get_value(); TrapOrigin origin = origin_from_sstatus(tf->sstatus); TrapInfo info = build_trap_info(tf, scause, stval, origin); + /* Per-trap trace from user mode. Stored at KLOG_DEBUG / KLOG_SUBSYS_TRAP + * so it lives in the ring buffer for postmortem but stays off the console + * by default. No more in-band 20-trap window — flip console_level or + * dump the ring on demand instead. */ + if (origin == TrapOrigin::FromUser) { + pr_debug_s(KLOG_SUBSYS_TRAP, + "trap: cause=%u epc=%llx val=%llx\n", + static_cast<unsigned>(info.cause), info.epc, info.val); + } + /* Save trapframe in TCB for syscall handlers (like fork) */ TaskControlBlock* curr = current_task(); if (curr) { @@ -152,6 +181,13 @@ extern "C" void trap_dispatch(arch::TrapFrame* tf) /* Call MI-layer dispatcher (zero inline assembly from here on) */ trap_handle(tf, info); + + /* Return to user mode. + * Set stvec back to trap_vector for the next user-mode trap, + * then jump to userret which restores registers and does sret. */ + arch::trap::set_vector(reinterpret_cast<uint64>(&trap_vector)); + userret(tf); + __builtin_unreachable(); } /* ======================================================================== diff --git a/na/arch/riscv/trap/trap_vector.S b/na/arch/riscv/trap/trap_vector.S index 160b122..8e45cae 100644 --- a/na/arch/riscv/trap/trap_vector.S +++ b/na/arch/riscv/trap/trap_vector.S @@ -90,6 +90,15 @@ trap_vector: /* Step 6: flush TLB after page table switch */ sfence.vma zero, zero + /* Step 6.5: restore kernel gp (global pointer). + * The compiler uses gp-relative addressing for global variable access + * (e.g. ld reg, offset(gp)). User mode may have set gp to anything, + * so we MUST reload it before entering any C++ code. */ +.option push +.option norelax + la gp, __global_pointer$ +.option pop + /* Pass TrapFrame* to C++ handler (a0 = tf) */ mv a0, tp @@ -97,7 +106,6 @@ trap_vector: mv tp, t2 /* Step 7: jump to C++ trap_handler */ - /* Step 7: jump to C++ trap_handler */ jalr t3 |
