From e90b72ec3082588b296956730bd2930cf8d5d671 Mon Sep 17 00:00:00 2001 From: Richard Qin Date: Fri, 10 Jul 2026 11:31:46 +0800 Subject: refactor: promote new architecture to repository root Archive the deprecated implementation under lume-legacy, make the new kernel the default root-level build, and update repository entry points without changing kernel behavior. --- arch/riscv/config.cc | 276 +++++++++++++++++++++++++++ arch/riscv/defconfig | 39 ++++ arch/riscv/include/arch/config.h | 106 +++++++++++ arch/riscv/include/arch/context.h | 33 ++++ arch/riscv/include/arch/cpu.h | 95 ++++++++++ arch/riscv/include/arch/extable.h | 55 ++++++ arch/riscv/include/arch/fpstate.h | 94 ++++++++++ arch/riscv/include/arch/mmu.h | 78 ++++++++ arch/riscv/include/arch/pmap.h | 49 +++++ arch/riscv/include/arch/random.h | 27 +++ arch/riscv/include/arch/sbi.h | 111 +++++++++++ arch/riscv/include/arch/tlb.h | 19 ++ arch/riscv/include/arch/trap.h | 94 ++++++++++ arch/riscv/include/arch/trapframe.h | 140 ++++++++++++++ arch/riscv/include/asm/csr.h | 37 ++++ arch/riscv/include/asm/page.h | 62 +++++++ arch/riscv/include/asm/pgtable.h | 55 ++++++ arch/riscv/include/asm/sections.h | 50 +++++ arch/riscv/kernel/asm-offsets.c | 29 +++ arch/riscv/kernel/head.S | 317 +++++++++++++++++++++++++++++++ arch/riscv/kernel/lumeos.lds.S | 120 ++++++++++++ arch/riscv/kernel/signal.cc | 163 ++++++++++++++++ arch/riscv/lib/random.cc | 21 +++ arch/riscv/mm/init.cc | 241 ++++++++++++++++++++++++ arch/riscv/mm/pmap.cc | 361 ++++++++++++++++++++++++++++++++++++ arch/riscv/mm/tlb_md.cc | 17 ++ arch/riscv/module.mk | 18 ++ arch/riscv/syscall/syscall_md.cc | 42 +++++ arch/riscv/trap/extable.cc | 92 +++++++++ arch/riscv/trap/fpu.S | 103 ++++++++++ arch/riscv/trap/swtch.S | 96 ++++++++++ arch/riscv/trap/trap_md.cc | 241 ++++++++++++++++++++++++ arch/riscv/trap/trap_vector.S | 283 ++++++++++++++++++++++++++++ arch/riscv/uaccess/uaccess_memcpy.S | 117 ++++++++++++ 34 files changed, 3681 insertions(+) create mode 100644 arch/riscv/config.cc create mode 100644 arch/riscv/defconfig create mode 100644 arch/riscv/include/arch/config.h create mode 100644 arch/riscv/include/arch/context.h create mode 100644 arch/riscv/include/arch/cpu.h create mode 100644 arch/riscv/include/arch/extable.h create mode 100644 arch/riscv/include/arch/fpstate.h create mode 100644 arch/riscv/include/arch/mmu.h create mode 100644 arch/riscv/include/arch/pmap.h create mode 100644 arch/riscv/include/arch/random.h create mode 100644 arch/riscv/include/arch/sbi.h create mode 100644 arch/riscv/include/arch/tlb.h create mode 100644 arch/riscv/include/arch/trap.h create mode 100644 arch/riscv/include/arch/trapframe.h create mode 100644 arch/riscv/include/asm/csr.h create mode 100644 arch/riscv/include/asm/page.h create mode 100644 arch/riscv/include/asm/pgtable.h create mode 100644 arch/riscv/include/asm/sections.h create mode 100644 arch/riscv/kernel/asm-offsets.c create mode 100644 arch/riscv/kernel/head.S create mode 100644 arch/riscv/kernel/lumeos.lds.S create mode 100644 arch/riscv/kernel/signal.cc create mode 100644 arch/riscv/lib/random.cc create mode 100644 arch/riscv/mm/init.cc create mode 100644 arch/riscv/mm/pmap.cc create mode 100644 arch/riscv/mm/tlb_md.cc create mode 100644 arch/riscv/module.mk create mode 100644 arch/riscv/syscall/syscall_md.cc create mode 100644 arch/riscv/trap/extable.cc create mode 100644 arch/riscv/trap/fpu.S create mode 100644 arch/riscv/trap/swtch.S create mode 100644 arch/riscv/trap/trap_md.cc create mode 100644 arch/riscv/trap/trap_vector.S create mode 100644 arch/riscv/uaccess/uaccess_memcpy.S (limited to 'arch') diff --git a/arch/riscv/config.cc b/arch/riscv/config.cc new file mode 100644 index 0000000..a18a825 --- /dev/null +++ b/arch/riscv/config.cc @@ -0,0 +1,276 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* Copyright (C) 2026 Richard Qin */ + +/* + * RISC-V Architecture Configuration — runtime state and KASLR + * + * Provides the runtime (post-KASLR) values of the higher-half + * virtual address bases. kaslr_init() is called once during + * early boot, before vmm_init(), to randomise the layout and + * fill the unified KaslrLayout structure. + * + * KASLR slide budget: + * The default Direct Map starts at 0xFFFFFFC0_0000_0000 (VPN[2]=256). + * The default Vmemmap starts at 0xFFFFFFD0_0000_0000 (VPN[2]=320). + * A 1GB superpage occupies one VPN[2] slot. We allow a slide of + * 0..63 slots (0..63 GB) upward, which keeps both regions inside + * the upper canonical half and avoids colliding with the kernel + * text region (linked at VPN[2] = 258 + 2 = ~0xFFFFFFC0_8020_0000). + * + * In practice, with 128MB RAM the kernel+direct map only touches + * VPN[2] 258-259, so a slide of up to 63GB is safe. + */ + +#include +#include +#include +#include +#include +#include +#include + +extern "C" char _stext[], _kernel_end[]; + +namespace arch { + +/* Global KASLR Layout Structure*/ +KaslrLayout g_kaslr_layout = {}; + +/* Runtime bases — updated by kaslr_layout_compute() */ +uint64 g_direct_map_base = kDirectMapBaseDefault; +uint64 g_vmemmap_base = kVmemmapBaseDefault; + +/* VA Range for validation */ +struct VaRange { + const char* name; + uint64 start; + uint64 end; // exclusive +}; + +/* ================================================================ + * Utilities + * ================================================================ */ +static inline uint64 align_down_page(uint64 v) +{ + return v & ~(kPageSize - 1); +} + +static inline uint64 align_up_page(uint64 v) +{ + return (v + kPageSize - 1) & ~(kPageSize - 1); +} + +static uint64 checked_add_u64(uint64 a, uint64 b, const char* detail) +{ + if (a > (~0ULL - b)) { + kernel_panic("kaslr: address overflow", detail); + } + return a + b; +} + +/* Splitmix64: non-plaintext hash derived from seed (used for seed_hash) */ +static uint64 splitmix64(uint64 x) +{ + x += 0x9e3779b97f4a7c15ULL; + x = (x ^ (x >> 30)) * 0xbf58476d1ce4e5b9ULL; + x = x ^ (x >> 27); + return x * 0x94d049bb133111ebULL; +} + +/* SV39 canonical VA check: bits [63:39] must equal sign bit 38. */ +static bool is_sv39_canonical(uint64 va) +{ + uint64 sign = (va >> 38) & 1ULL; + uint64 upper = va >> 39; + return sign ? (upper == ((1ULL << 25) - 1ULL)) : (upper == 0); +} + +static bool is_sv39_kernel_half(uint64 va) +{ + return is_sv39_canonical(va) && (((va >> 38) & 1ULL) == 1ULL); +} + +static bool ranges_overlap(const VaRange& a, const VaRange& b) +{ + return (a.start < b.end) && (b.start < a.end); +} + +/* ================================================================ + * KASLR Computation (Phase A: Centralized in single function) + * + * This function: + * 1. Generates or derives random seed + * 2. Selects 1GB-aligned slot [0..63] + * 3. Computes seed_hash via splitmix64 + * 4. Fills KaslrLayout structure completely + * 5. Validates ranges (canonical, overlaps) + * 6. Logs according to CONFIG_KASLR_LOG_LEVEL + * ================================================================ */ +static void kaslr_layout_compute() +{ +#ifdef CONFIG_KASLR + constexpr uint64 k1GB = 1ULL << 30; + constexpr uint64 kMaxSlots = 64; + + /* 1. Get seed and source */ + uint64 seed = random_early_seed_value(); + RandomSeedSource rsrc = random_early_seed_source(); + KaslrSeedSource source; + switch (rsrc) { + case RandomSeedSource::FdtProvided: + source = KaslrSeedSource::kFdt; + break; + case RandomSeedSource::HwRandom: + source = KaslrSeedSource::kHardware; + break; + case RandomSeedSource::CycleFallback: + case RandomSeedSource::Unknown: + default: + source = KaslrSeedSource::kCycle; + break; + } + + /* 2. Select random slot and compute slide */ + uint64 raw = get_random_u64(); + uint64 slot = raw % kMaxSlots; + if (slot == 0) { + slot = 1; /* slot=0 causes direct_map to overlap with kernel image VA */ + } + uint64 slide = slot * k1GB; + + /* 3. Compute non-plaintext hash */ + uint64 seed_hash = splitmix64(seed ^ 0xdeadbeefdeadbeefULL); + + /* 4. Compute randomised bases */ + uint64 direct_map_base = kDirectMapBaseDefault + slide; + uint64 vmemmap_base = kVmemmapBaseDefault + slide; + + /* 5. Fill KaslrLayout */ + g_kaslr_layout.seed = seed; + g_kaslr_layout.seed_hash = seed_hash; + g_kaslr_layout.source = source; + g_kaslr_layout.slot = slot; + g_kaslr_layout.slide_bytes = slide; + g_kaslr_layout.policy_id = kKaslrPolicyId; + g_kaslr_layout.direct_map_base = direct_map_base; + g_kaslr_layout.vmemmap_base = vmemmap_base; + g_kaslr_layout.canonical_ok = false; + g_kaslr_layout.overlap_ok = false; + + /* Update global runtime bases */ + g_direct_map_base = direct_map_base; + g_vmemmap_base = vmemmap_base; + + /* 6. Self-check: canonical VA validation and overlap detection */ + uint64 mem_base = 0, mem_size = 0; + fdt_early_get_mem_info(&mem_base, &mem_size); + if (mem_size == 0) { + kernel_panic("kaslr_layout_compute: invalid memory from FDT"); + } + + /* Construct VA ranges */ + uint64 num_pages = mem_size / kPageSize; + uint64 page_bytes = num_pages * sizeof(Page); + uint64 page_span = align_up_page(page_bytes); + + VaRange direct_map { + "direct-map", + checked_add_u64(g_direct_map_base, mem_base, "direct_map start"), + checked_add_u64(checked_add_u64(g_direct_map_base, mem_base, + "direct_map start"), + mem_size, "direct_map end"), + }; + + VaRange vmemmap { + "vmemmap", + g_vmemmap_base, + checked_add_u64(g_vmemmap_base, page_span, "vmemmap end"), + }; + + VaRange kernel_img { + "kernel-image", + align_down_page(reinterpret_cast(_stext)), + align_up_page(reinterpret_cast(_kernel_end)), + }; + + /* Validate each range is canonical */ + auto check_kernel_half = [](const VaRange& r) { + if (r.start >= r.end) { + kernel_panic("kaslr: invalid VA range", r.name); + } + if (!is_sv39_kernel_half(r.start) || !is_sv39_kernel_half(r.end - 1)) { + kernel_panic("kaslr: non-canonical kernel VA range", r.name); + } + }; + + check_kernel_half(direct_map); + check_kernel_half(vmemmap); + check_kernel_half(kernel_img); + + /* Validate no overlaps */ + if (ranges_overlap(direct_map, kernel_img)) { + kernel_panic("kaslr: direct-map overlaps kernel image"); + } + if (ranges_overlap(vmemmap, kernel_img)) { + kernel_panic("kaslr: vmemmap overlaps kernel image"); + } + if (ranges_overlap(direct_map, vmemmap)) { + kernel_panic("kaslr: direct-map overlaps vmemmap"); + } + + /* Mark validation success */ + g_kaslr_layout.canonical_ok = true; + g_kaslr_layout.overlap_ok = true; + + /* 7. Logging according to CONFIG_KASLR_LOG_LEVEL */ + if (kKaslrLogLevel >= 1) { + /* Minimal logging: just the seed and slot (release mode) */ + kprintf("[boot][kaslr] policy_id=%u source=%s slot=%llu\n", + g_kaslr_layout.policy_id, + random_seed_source_name(rsrc), + slot); + } + + if (kKaslrLogLevel >= 2) { + /* Debug logging: full ranges and hashes */ + kprintf("[boot][kaslr] seed=0x%llx seed_hash=0x%llx\n", + seed, seed_hash); + kprintf("[boot][kaslr] direct-map=[0x%llx,0x%llx)\n", + direct_map.start, direct_map.end); + kprintf("[boot][kaslr] vmemmap=[0x%llx,0x%llx)\n", + vmemmap.start, vmemmap.end); + kprintf("[boot][kaslr] kernel-image=[0x%llx,0x%llx)\n", + kernel_img.start, kernel_img.end); + } + +#else /* CONFIG_KASLR disabled */ + g_kaslr_layout.seed = 0; + g_kaslr_layout.seed_hash = 0; + g_kaslr_layout.source = KaslrSeedSource::kCycle; + g_kaslr_layout.slot = 0; + g_kaslr_layout.slide_bytes = 0; + g_kaslr_layout.policy_id = kKaslrPolicyId; + g_kaslr_layout.direct_map_base = kDirectMapBaseDefault; + g_kaslr_layout.vmemmap_base = kVmemmapBaseDefault; + g_kaslr_layout.canonical_ok = true; + g_kaslr_layout.overlap_ok = true; + + g_direct_map_base = kDirectMapBaseDefault; + g_vmemmap_base = kVmemmapBaseDefault; + + if (kKaslrLogLevel >= 1) { + kprintf("[boot][kaslr] disabled policy_id=%u\n", + g_kaslr_layout.policy_id); + } +#endif +} + +/* ================================================================ + * Public entry point + * ================================================================ */ +void kaslr_init() +{ + kaslr_layout_compute(); +} + +} // namespace arch diff --git a/arch/riscv/defconfig b/arch/riscv/defconfig new file mode 100644 index 0000000..87d3ffd --- /dev/null +++ b/arch/riscv/defconfig @@ -0,0 +1,39 @@ +# SPDX-License-Identifier: GPL-2.0-or-later +# Copyright (C) 2026 Richard Qin +# +# LumeOS Default Kernel Configuration — RISC-V (virt) +# +# This is the "golden" default configuration for the RISC-V virt platform. +# To customise, copy this file to the source root as '.config': +# +# cp arch/riscv/defconfig .config +# $EDITOR .config +# make +# +# Any variable not set here inherits its default from scripts/Kconfig.mk. +# ====================================================================== + +# --- Platform ----------------------------------------------------------- +CONFIG_ARCH = riscv + +# --- CPU / SMP ---------------------------------------------------------- +CONFIG_NR_CPUS = 8 + +# --- Scheduler ---------------------------------------------------------- +CONFIG_HZ = 100 +CONFIG_NR_PRIORITIES = 32 + +# --- Memory Management -------------------------------------------------- +CONFIG_PAGE_SIZE = 4096 +CONFIG_MAX_ORDER = 11 +CONFIG_KERNEL_STACK_SIZE = 8192 + +# --- Security / Hardening ----------------------------------------------- +CONFIG_KASLR = y +CONFIG_STACK_CANARY = y + +# --- Debug --------------------------------------------------------------- +CONFIG_DEBUG = y +CONFIG_DEBUG_LOCKS = y +CONFIG_SELFTEST = y +CONFIG_KLOG = y diff --git a/arch/riscv/include/arch/config.h b/arch/riscv/include/arch/config.h new file mode 100644 index 0000000..8ae0502 --- /dev/null +++ b/arch/riscv/include/arch/config.h @@ -0,0 +1,106 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* Copyright (C) 2026 Richard Qin */ + +#pragma once + +/* + * RISC-V Architecture Configuration + * + * These constants define the architecture-specific memory layout. + * Each arch/ implementation must provide this header with the same symbols. + * + * KASLR: At boot, kaslr_init() randomises g_direct_map_base and + * g_vmemmap_base. All MI code must read these runtime variables + * (via pa_to_va / vmm helpers) rather than the static defaults. + */ + +#include + +namespace arch { + +/* ================================================================ + * Default (compile-time) virtual addresses + * + * SV39 gives a 512GB virtual address space. + * Higher-half layout (defaults before KASLR): + * + * 0xFFFFFFC0_00000000 (256) Direct map (PA+offset) + * 0xFFFFFFD0_00000000 (320) Vmemmap (Page array) + * + * The 64GB gap between them is the KASLR slide range. + * ================================================================ */ +inline constexpr uint64 kDirectMapBaseDefault = 0xFFFFFFC000000000ULL; +inline constexpr uint64 kVmemmapBaseDefault = 0xFFFFFFD000000000ULL; + +/* Physical memory base (where RAM starts on RISC-V virt platform) */ +inline constexpr uint64 kPhysBase = 0x80000000ULL; + +/* Kernel physical load address (= PHYS_BASE in linker script). + * Everything in [kPhysBase, kPhysStart) is firmware territory (PMP-protected). + * Analogous to Linux's CONFIG_PHYSICAL_START. */ +inline constexpr uint64 kPhysStart = 0x80200000ULL; + +/* ================================================================ + * Runtime (post-KASLR) virtual addresses + * + * These start equal to the defaults above. If CONFIG_KASLR is + * enabled, kaslr_init() adds a random, 1GB-aligned offset + * to each, sliding both regions upward in VA space. + * ================================================================ */ +extern uint64 g_direct_map_base; +extern uint64 g_vmemmap_base; + +/* ================================================================ + * KASLR Layout Structure (Phase A: Unified KASLR State) + * + * This struct holds all KASLR results: the random seed, derived + * hash, selected slot, and the randomised virtual address bases. + * It serves as the single source of truth for all KASLR decisions + * and is filled once during boot and read-only thereafter. + * ================================================================ */ + +enum class KaslrSeedSource : uint8 { + kFdt = 0, /* Loaded from FDT /random node */ + kHardware = 1, /* From hardware RNG (e.g., RDRAND) */ + kCycle = 2, /* Fallback: CPU cycle counter */ +}; + +struct KaslrLayout { + /* Seed and provenance */ + uint64 seed; /* 64-bit random seed value */ + uint64 seed_hash; /* Non-plaintext derived hash (splitmix64) */ + KaslrSeedSource source; /* Seed provenance (fdt/hw/cycle) */ + + /* Randomisation parameters */ + uint64 slot; /* Slot index: [0, 63] */ + uint64 slide_bytes; /* Byte offset: slot * 1GB */ + + /* Build-time policy */ + uint32 policy_id; /* CONFIG_KASLR_POLICY_ID at build time */ + + /* Randomised base addresses (post-KASLR) */ + uint64 direct_map_base; /* Direct map VA (kDirectMapBaseDefault + slide) */ + uint64 vmemmap_base; /* Vmemmap VA (kVmemmapBaseDefault + slide) */ + + /* Validation flags (for debug logging) */ + bool canonical_ok; /* All VA ranges passed canonical check */ + bool overlap_ok; /* No overlaps detected between regions */ +}; + +/* ================================================================ + * KASLR entry point + * + * Must be called AFTER random_early_init() and BEFORE vmm_init(). + * Fills g_kaslr_layout with randomised bases and self-check results. + * When CONFIG_KASLR is disabled this is a no-op. + * ================================================================ */ +extern KaslrLayout g_kaslr_layout; + +void kaslr_init(); + +/* Helper to query the KASLR layout safely */ +static inline const KaslrLayout& kaslr_layout_get() { + return g_kaslr_layout; +} + +} // namespace arch diff --git a/arch/riscv/include/arch/context.h b/arch/riscv/include/arch/context.h new file mode 100644 index 0000000..3d935fa --- /dev/null +++ b/arch/riscv/include/arch/context.h @@ -0,0 +1,33 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* Copyright (C) 2026 Richard Qin */ + +#pragma once + +/* + * Context Switch Structure (RISC-V 64) + * + * Holds only callee-saved registers for voluntary (non-preemptive) + * context switches between kernel threads / tasks. + * + * Used by: arch::context_switch() in swtch.S + * Reference: docs/specs/arch_abstraction.md §3.1 + */ + +#include + +namespace arch { + +struct Context { + uint64 ra; // 0: Return address + uint64 sp; // 8: Stack pointer + uint64 s[12]; // 16: s0-s11 (callee-saved registers) +}; + +static_assert(sizeof(Context) == 112); + +// Assembly-implemented context switch. +// Saves current CPU registers to old_ctx, loads next_ctx into CPU. +// Defined in arch/riscv/trap/swtch.S +extern "C" void context_switch(Context* old_ctx, Context* next_ctx); + +} // namespace arch diff --git a/arch/riscv/include/arch/cpu.h b/arch/riscv/include/arch/cpu.h new file mode 100644 index 0000000..8c573ec --- /dev/null +++ b/arch/riscv/include/arch/cpu.h @@ -0,0 +1,95 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* Copyright (C) 2026 Richard Qin */ + +#pragma once + +/* + * CPU Control & Status Primitives (RISC-V 64) + * + * Provides interrupt enable/disable, CPU ID, timer, and halt. + * All functions are static inline — zero call overhead. + * + * Precondition: tp register must be initialized with CPU ID + * by entry.S before any of these functions are called. + * + * Reference: docs/specs/arch_abstraction.md §4.1 + */ + +#include + +/* RISC-V sstatus register bit definitions */ +#define SSTATUS_SIE (1UL << 1) /* Supervisor Interrupt Enable */ +#define SSTATUS_SPIE (1UL << 5) /* Supervisor Previous Interrupt Enable */ +#define SSTATUS_SPP (1UL << 8) /* Supervisor Previous Privilege (0=U, 1=S) */ + +/* RISC-V sie register bit definitions */ +#define SIE_SSIE (1UL << 1) /* Supervisor Software Interrupt Enable */ +#define SIE_STIE (1UL << 5) /* Supervisor Timer Interrupt Enable */ +#define SIE_SEIE (1UL << 9) /* Supervisor External Interrupt Enable */ + +namespace arch::cpu { + +// Get current CPU ID (from tp register, set by entry.S) +static inline uint64 id() { + uint64 val; + __asm__ volatile("mv %0, tp" : "=r"(val)); + return val; +} + +// Enable interrupts (set sstatus.SIE = 1) +static inline void intr_on() { + __asm__ volatile("csrs sstatus, %0" :: "r"(SSTATUS_SIE)); +} + +// Disable interrupts (clear sstatus.SIE = 0) +static inline void intr_off() { + __asm__ volatile("csrc sstatus, %0" :: "r"(SSTATUS_SIE)); +} + +// Check if interrupts are currently enabled +static inline bool intr_enabled() { + uint64 sstatus; + __asm__ volatile("csrr %0, sstatus" : "=r"(sstatus)); + return (sstatus & SSTATUS_SIE) != 0; +} + +// Halt CPU until next interrupt (wfi instruction) +static inline void halt_until_interrupt() { + __asm__ volatile("wfi"); +} + +// Read hardware monotonic timer (rdtime instruction) +// Returns ticks since boot; frequency from FDT timebase-frequency. +static inline uint64 read_time() { + uint64 val; + __asm__ volatile("rdtime %0" : "=r"(val)); + return val; +} + +// Enable timer interrupt in sie (sie.STIE = 1) +static inline void timer_intr_on() { + __asm__ volatile("csrs sie, %0" :: "r"(SIE_STIE)); +} + +// Disable timer interrupt in sie (sie.STIE = 0) +static inline void timer_intr_off() { + __asm__ volatile("csrc sie, %0" :: "r"(SIE_STIE)); +} + +// Enable software interrupt in sie (sie.SSIE = 1) — needed for IPI +static inline void software_intr_on() { + __asm__ volatile("csrs sie, %0" :: "r"(SIE_SSIE)); +} + +// Enable external interrupt in sie (sie.SEIE = 1) — needed for PLIC (UART, etc.) +static inline void external_intr_on() { + __asm__ volatile("csrs sie, %0" :: "r"(SIE_SEIE)); +} + +// Clear S-mode software interrupt pending bit (sip.SSIP = 0) +// Must be called in the IPI handler to acknowledge the interrupt. +static inline void clear_sip_ssip() { + __asm__ volatile("csrc sip, %0" :: "r"(1UL << 1)); +} + +} // namespace arch::cpu diff --git a/arch/riscv/include/arch/extable.h b/arch/riscv/include/arch/extable.h new file mode 100644 index 0000000..eb2daac --- /dev/null +++ b/arch/riscv/include/arch/extable.h @@ -0,0 +1,55 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* Copyright (C) 2026 Richard Qin */ + +#pragma once + +#ifndef __ASSEMBLER__ + +#include + +/* + * Exception table entry. + * insn: offset from this struct to the faulting instruction. + * fixup: offset from this struct to the fixup code. + * + * Using relative offsets avoids absolute relocations and saves space. + */ +struct ExceptionTableEntry { + int32 insn; + int32 fixup; +}; + +/* Search the exception table for a given PC. Returns fixup address, or 0 if not found. */ +uint64 search_extable(uint64 pc); + +/* Sorts the exception table at boot for binary search O(log N). */ +void extable_init(); + +#endif /* __ASSEMBLER__ */ + +/* + * EXTABLE macro for assembly usage. + * Appends an entry to the .extable section. + */ +#ifdef __ASSEMBLER__ + +#define EXTABLE(insn, fixup) \ + .pushsection .extable, "a"; \ + .balign 4; \ + .long insn - .; \ + .long fixup - .; \ + .popsection + +#else /* !__ASSEMBLER__ */ + +/* Stringified macro for inline assembly */ +#define _ASM_EXTABLE(insn, fixup) \ + ".pushsection .extable, \"a\"\n" \ + ".balign 4\n" \ + ".long " #insn " - .\n" \ + ".long " #fixup " - .\n" \ + ".popsection\n" + +#define EXTABLE(insn, fixup) _ASM_EXTABLE(insn, fixup) + +#endif /* __ASSEMBLER__ */ diff --git a/arch/riscv/include/arch/fpstate.h b/arch/riscv/include/arch/fpstate.h new file mode 100644 index 0000000..4b42a22 --- /dev/null +++ b/arch/riscv/include/arch/fpstate.h @@ -0,0 +1,94 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* Copyright (C) 2026 Richard Qin */ + +#pragma once + +/* + * FPU State — Floating-Point Register Context + * + * Pre-reserved structure for RISC-V F/D extension floating-point state. + * MUST be allocated in every TaskControlBlock so that future FPU + * save/restore logic has a stable destination without restructuring TCB. + * + * RISC-V F/D extension state: + * - 32 FP registers (f0-f31), each 64 bits for D extension + * - fcsr (Floating-point Control and Status Register) + * + * sstatus.FS field management: + * - FS=0 (Off): FPU disabled, any FP instruction traps + * - FS=1 (Initial): FPU enabled, registers are initial state + * - FS=2 (Clean): FPU enabled, registers not modified since last save + * - FS=3 (Dirty): FPU enabled, registers modified (need save) + * + * Current status: Structure defined, save/restore NOT implemented. + * When implemented, trap_dispatch and swtch.S will use lazy + * save/restore based on sstatus.FS dirty tracking. + * + * Reference: RISC-V ISA Manual §3.1.6.5 (FS field) + */ + +#include + +namespace arch { + +struct FpState { + uint64 f[32]; // f0-f31 (64-bit each for D extension) + uint32 fcsr; // Floating-point CSR (frm + fflags) + uint32 _pad; // Alignment padding +}; + +static_assert(sizeof(FpState) == 32 * 8 + 8, "FpState must be 264 bytes"); + +/* + * sstatus.FS field constants + */ +inline constexpr uint64 kSstatusFS_Off = 0ULL << 13; +inline constexpr uint64 kSstatusFS_Initial = 1ULL << 13; +inline constexpr uint64 kSstatusFS_Clean = 2ULL << 13; +inline constexpr uint64 kSstatusFS_Dirty = 3ULL << 13; +inline constexpr uint64 kSstatusFS_Mask = 3ULL << 13; + +/* + * Check if sstatus indicates FPU state is dirty (needs save). + */ +inline bool fpu_state_dirty(uint64 sstatus) { + return (sstatus & kSstatusFS_Mask) == kSstatusFS_Dirty; +} + +/* + * Check if FPU is enabled (FS != Off). + */ +inline bool fpu_enabled(uint64 sstatus) { + return (sstatus & kSstatusFS_Mask) != kSstatusFS_Off; +} + +} // namespace arch + +/* Assembly routines for FPU register save/restore (fpu.S) */ +extern "C" void fpu_save(arch::FpState* state); +extern "C" void fpu_restore(arch::FpState* state); + +namespace arch { + +/* + * Read sstatus.FS field (current FPU state) + */ +inline uint64 fpu_get_fs() { + uint64 sstatus; + __asm__ volatile("csrr %0, sstatus" : "=r"(sstatus)); + return sstatus & kSstatusFS_Mask; +} + +/* + * Set sstatus.FS field. + * @fs: one of kSstatusFS_Off, kSstatusFS_Initial, kSstatusFS_Clean, kSstatusFS_Dirty + */ +inline void fpu_set_fs(uint64 fs) { + __asm__ volatile( + "csrc sstatus, %0\n" + "csrs sstatus, %1\n" + :: "r"(kSstatusFS_Mask), "r"(fs) + ); +} + +} // namespace arch diff --git a/arch/riscv/include/arch/mmu.h b/arch/riscv/include/arch/mmu.h new file mode 100644 index 0000000..f4c9b26 --- /dev/null +++ b/arch/riscv/include/arch/mmu.h @@ -0,0 +1,78 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* Copyright (C) 2026 Richard Qin */ + +#pragma once + +/* + * MMU Hardware Primitives (RISC-V SV39) + * + * Page table switching, TLB flushing, and fault address reading. + * All functions are static inline — zero call overhead. + * + * Note on SMP: flush_tlb_page() only flushes the LOCAL CPU's TLB. + * Cross-CPU TLB shootdown via IPI is not yet implemented. + * + * Reference: docs/specs/arch_abstraction.md §4.2 + */ + +#include + +/* SATP register: mode field for SV39 */ +#define SATP_SV39 (8UL << 60) + +/* SATP PPN mask (bits 43:0) */ +#define SATP_PPN_MASK ((1UL << 44) - 1) + +namespace arch::mmu { + +// Read the faulting virtual address from stval (set by HW on page fault) +static inline uint64 get_fault_address() { + uint64 val; + __asm__ volatile("csrr %0, stval" : "=r"(val)); + return val; +} + +// Switch the top-level page table. +// Takes a PHYSICAL ADDRESS (not PPN); internally shifts right by 12 +// and combines with SV39 mode bits before writing to satp. +static inline void set_page_table(uint64 physical_pgdir_addr) { + uint64 ppn = physical_pgdir_addr >> 12; + uint64 satp_val = SATP_SV39 | ppn; + __asm__ volatile("csrw satp, %0" :: "r"(satp_val)); + // Note: caller should call flush_tlb_all() after this +} + +// Read the current page table physical address. +// Extracts PPN from satp and converts back to physical address. +static inline uint64 get_page_table() { + uint64 satp_val; + __asm__ volatile("csrr %0, satp" : "=r"(satp_val)); + return (satp_val & SATP_PPN_MASK) << 12; +} + +// Read the raw satp CSR value (mode + PPN). +// Used when the caller needs the exact register value (e.g. for TrapFrame metadata). +static inline uint64 read_satp() { + uint64 val; + __asm__ volatile("csrr %0, satp" : "=r"(val)); + return val; +} + +// Flush the entire TLB (all ASIDs, all addresses) +static inline void flush_tlb_all() { + __asm__ volatile("sfence.vma zero, zero" ::: "memory"); +} + +// Flush a single TLB entry for the given virtual address +static inline void flush_tlb_page(uint64 vaddr) { + __asm__ volatile("sfence.vma %0, zero" :: "r"(vaddr) : "memory"); +} + +// Flush the instruction cache. +// Required after loading executable code into memory (e.g. ELF loading). +// sfence.vma only flushes the TLB; fence.i is needed for the I-cache. +static inline void flush_icache() { + __asm__ volatile("fence.i" ::: "memory"); +} + +} // namespace arch::mmu diff --git a/arch/riscv/include/arch/pmap.h b/arch/riscv/include/arch/pmap.h new file mode 100644 index 0000000..0b43b9e --- /dev/null +++ b/arch/riscv/include/arch/pmap.h @@ -0,0 +1,49 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* Copyright (C) 2026 Richard Qin */ + +#pragma once + +/* + * pmap — SV39 page table interface (MD layer) + * + * Namespace functions, zero runtime overhead. + * Implementation: arch/riscv/mm/pmap.cc + * + * Reference: docs/specs/vmm.md §3.2 + */ + +#include + +namespace pmap { + +/* PTE permission bits */ +inline constexpr uint64 PTE_V = 1ULL << 0; +inline constexpr uint64 PTE_R = 1ULL << 1; +inline constexpr uint64 PTE_W = 1ULL << 2; +inline constexpr uint64 PTE_X = 1ULL << 3; +inline constexpr uint64 PTE_U = 1ULL << 4; +inline constexpr uint64 PTE_G = 1ULL << 5; +inline constexpr uint64 PTE_A = 1ULL << 6; +inline constexpr uint64 PTE_D = 1ULL << 7; +inline constexpr uint64 PTE_COW = 1ULL << 8; /* Custom RSW bit for Copy-On-Write */ + +uint64 create(); +void destroy(uint64 root_pa); +int map(uint64 root_pa, uint64 va, uint64 pa, uint64 perm); +int map_2mb(uint64 root_pa, uint64 va, uint64 pa, uint64 perm); +void unmap(uint64 root_pa, uint64 va); +int protect(uint64 root_pa, uint64 va, uint64 perm); +int protect_range(uint64 root_pa, uint64 start, uint64 end, uint64 perm); +bool lookup(uint64 root_pa, uint64 va, uint64* pa_out); +uint64 lookup_pte(uint64 root_pa, uint64 va); +bool is_cow(uint64 root_pa, uint64 va); +int make_cow(uint64 root_pa, uint64 va); +void activate(uint64 root_pa); + +/* + * Translate MI-layer VM_* permission flags to hardware PTE bits. + * Called by vma.cc when faulting in a page. + */ +uint64 vm_perm_to_pte(uint64 vm_perm); + +} // namespace pmap diff --git a/arch/riscv/include/arch/random.h b/arch/riscv/include/arch/random.h new file mode 100644 index 0000000..8c8c051 --- /dev/null +++ b/arch/riscv/include/arch/random.h @@ -0,0 +1,27 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* Copyright (C) 2026 Richard Qin */ + +#pragma once + +#include + +namespace arch { + +/* + * Get a random 64-bit value from the hardware TRNG. + * Returns true if supported and successful, false otherwise. + */ +bool get_hw_random(uint64& value); + +/* + * Get the current high-resolution hardware cycle counter. + * Used as an entropy fallback (Jitter/Timer). + */ +inline uint64 get_cycles() +{ + uint64 cycles; + __asm__ volatile("rdcycle %0" : "=r"(cycles)); + return cycles; +} + +} // namespace arch diff --git a/arch/riscv/include/arch/sbi.h b/arch/riscv/include/arch/sbi.h new file mode 100644 index 0000000..80a6d7c --- /dev/null +++ b/arch/riscv/include/arch/sbi.h @@ -0,0 +1,111 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* Copyright (C) 2026 Richard Qin */ + +#pragma once + +/* + * sbi.h — RISC-V Supervisor Binary Interface (SBI) wrappers + * + * Provides inline ecall wrappers for SBI calls from S-mode. + * Uses the SBI v1.0+ extension-based calling convention. + * + * Reference: RISC-V SBI Specification v2.0 + */ + +#include + +namespace arch::sbi { + +struct sbiret { + long error; + long value; +}; + +/* SBI error codes */ +inline constexpr long SBI_SUCCESS = 0; +inline constexpr long SBI_ERR_NOT_SUPPORTED = -2; + +/* Extension IDs */ +inline constexpr int SBI_EXT_TIME = 0x54494D45; /* "TIME" */ +inline constexpr int SBI_EXT_IPI = 0x735049; /* "sPI" */ +inline constexpr int SBI_EXT_RFENCE = 0x52464E43; /* "RFNC" */ +inline constexpr int SBI_EXT_HSM = 0x48534D; /* "HSM" */ + +static inline sbiret sbi_ecall(int ext, int fid, uint64 arg0 = 0, + uint64 arg1 = 0, uint64 arg2 = 0, + uint64 arg3 = 0, uint64 arg4 = 0, + uint64 arg5 = 0) { + sbiret ret; + register uint64 a0 asm("a0") = arg0; + register uint64 a1 asm("a1") = arg1; + register uint64 a2 asm("a2") = arg2; + register uint64 a3 asm("a3") = arg3; + register uint64 a4 asm("a4") = arg4; + register uint64 a5 asm("a5") = arg5; + register uint64 a6 asm("a6") = fid; + register uint64 a7 asm("a7") = ext; + __asm__ volatile("ecall" + : "+r"(a0), "+r"(a1) + : "r"(a2), "r"(a3), "r"(a4), "r"(a5), "r"(a6), "r"(a7) + : "memory"); + ret.error = static_cast(a0); + ret.value = static_cast(a1); + return ret; +} + +/* ---- Timer Extension (EID 0x54494D45) ---- */ + +static inline void set_timer(uint64 stime_value) { + sbi_ecall(SBI_EXT_TIME, 0, stime_value); +} + +/* ---- IPI Extension (EID 0x735049) ---- */ + +/* + * sbi_send_ipi — Send S-mode software interrupt to target harts + * + * @hart_mask: bitmask of target harts (relative to hart_mask_base) + * @hart_mask_base: base hart ID for the mask (0 = harts 0-63) + */ +static inline sbiret sbi_send_ipi(uint64 hart_mask, + uint64 hart_mask_base = 0) { + return sbi_ecall(SBI_EXT_IPI, 0, hart_mask, hart_mask_base); +} + +/* ---- Remote Fence Extension (EID 0x52464E43) ---- */ + +/* + * sbi_remote_sfence_vma — Execute SFENCE.VMA on target harts + * + * @hart_mask/base: target harts + * @start_addr: start of VA range to invalidate + * @size: size of range (use ~0UL for full flush) + */ +static inline sbiret sbi_remote_sfence_vma(uint64 hart_mask, + uint64 hart_mask_base, + uint64 start_addr, + uint64 size) { + return sbi_ecall(SBI_EXT_RFENCE, 1, + hart_mask, hart_mask_base, + start_addr, size); +} + +/* ---- HSM Extension (EID 0x48534D) ---- */ + +/* + * sbi_hart_start — Start (wake) a hart that is in STOPPED state + * + * @hartid: target hart ID to start + * @start_addr: PHYSICAL address where the hart should begin execution + * @opaque: passed in a1 register to the hart (e.g. FDT pointer) + * + * OpenSBI v1.0+ parks non-boot harts; they must be explicitly started. + */ +static inline sbiret sbi_hart_start(uint64 hartid, + uint64 start_addr, + uint64 opaque) { + return sbi_ecall(SBI_EXT_HSM, 0, hartid, start_addr, opaque); +} + +} // namespace arch::sbi + diff --git a/arch/riscv/include/arch/tlb.h b/arch/riscv/include/arch/tlb.h new file mode 100644 index 0000000..46c6b97 --- /dev/null +++ b/arch/riscv/include/arch/tlb.h @@ -0,0 +1,19 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* Copyright (C) 2026 Richard Qin */ + +#pragma once + +#include + +namespace arch::tlb { + +/* + * Perform a cross-CPU TLB flush (shootdown). + * + * @param cpu_mask: Bitmask of CPUs to target. + * @param start: Start virtual address, or 0 for full address space. + * @param size: Size in bytes, or ~0ULL for full address space. + */ +void flush_remote(uint64 cpu_mask, uint64 start, uint64 size); + +} // namespace arch::tlb diff --git a/arch/riscv/include/arch/trap.h b/arch/riscv/include/arch/trap.h new file mode 100644 index 0000000..8e55250 --- /dev/null +++ b/arch/riscv/include/arch/trap.h @@ -0,0 +1,94 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* Copyright (C) 2026 Richard Qin */ + +#pragma once + +/* + * Trap Vector Control Primitives (RISC-V 64) + * + * Set/read the S-mode trap vector (stvec), trap cause (scause), + * and trap value (stval) registers. + * + * Reference: docs/specs/arch_abstraction.md §4.3 + */ + +#include + +/* scause interrupt bit (bit 63): 1 = interrupt, 0 = exception */ +#define SCAUSE_INTERRUPT (1UL << 63) + +/* Exception cause codes (scause with bit 63 = 0) */ +#define EXC_INST_MISALIGNED 0 +#define EXC_INST_ACCESS_FAULT 1 +#define EXC_ILLEGAL_INST 2 +#define EXC_BREAKPOINT 3 +#define EXC_LOAD_MISALIGNED 4 +#define EXC_LOAD_ACCESS_FAULT 5 +#define EXC_STORE_MISALIGNED 6 +#define EXC_STORE_ACCESS_FAULT 7 +#define EXC_ECALL_FROM_U 8 +#define EXC_ECALL_FROM_S 9 +#define EXC_INST_PAGE_FAULT 12 +#define EXC_LOAD_PAGE_FAULT 13 +#define EXC_STORE_PAGE_FAULT 15 + +/* Interrupt cause codes (scause with bit 63 = 1, lower bits) */ +#define IRQ_S_SOFTWARE 1 +#define IRQ_S_TIMER 5 +#define IRQ_S_EXTERNAL 9 + +namespace arch::trap { + +// Set S-mode trap vector base address (writes stvec). +// Mode: Direct (all traps go to handler_addr). +static inline void set_vector(uint64 handler_addr) { + // Lowest 2 bits of stvec = mode. 0 = Direct. + __asm__ volatile("csrw stvec, %0" :: "r"(handler_addr)); +} + +// Read current trap vector base address +static inline uint64 get_vector() { + uint64 val; + __asm__ volatile("csrr %0, stvec" : "=r"(val)); + return val; +} + +// Read trap cause register (scause) +// Bit 63: 1=interrupt, 0=exception. Lower bits: cause code. +static inline uint64 get_cause() { + uint64 val; + __asm__ volatile("csrr %0, scause" : "=r"(val)); + return val; +} + +// Read trap value register (stval) +// Contains faulting address (page faults) or faulting instruction. +static inline uint64 get_value() { + uint64 val; + __asm__ volatile("csrr %0, stval" : "=r"(val)); + return val; +} + +// Check if a scause value represents an interrupt (vs exception) +static inline bool is_interrupt(uint64 cause) { + return (cause & SCAUSE_INTERRUPT) != 0; +} + +// Extract the cause code (strip the interrupt bit) +static inline uint64 cause_code(uint64 cause) { + return cause & ~SCAUSE_INTERRUPT; +} + +// Enable Supervisor User Memory access (set SUM bit in sstatus). +// When SUM=1, S-mode can read/write pages marked PTE_U. +// This is needed for syscall handlers to access user-space buffers. +static inline void enable_sum() { + __asm__ volatile("csrs sstatus, %0" :: "r"(1UL << 18)); +} + +// Disable Supervisor User Memory access (clear SUM bit in sstatus). +static inline void disable_sum() { + __asm__ volatile("csrc sstatus, %0" :: "r"(1UL << 18)); +} + +} // namespace arch::trap diff --git a/arch/riscv/include/arch/trapframe.h b/arch/riscv/include/arch/trapframe.h new file mode 100644 index 0000000..b198646 --- /dev/null +++ b/arch/riscv/include/arch/trapframe.h @@ -0,0 +1,140 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* Copyright (C) 2026 Richard Qin */ + +#pragma once + +/* + * Trap Frame Structure (RISC-V 64) + * + * Saved/restored by the assembly trap_vector on every interrupt, + * exception, or system call. Contains the full CPU snapshot needed + * to resume execution after trap handling. + * + * Layout must match the assembly offsets in trap_vector.S exactly. + * Reference: docs/specs/arch_abstraction.md §3.2 + */ + +/* + * TrapFrame field offsets for use in assembly. + * These must be kept in sync with the struct layout below. + * Placed before any C++ code so assembly can #include this file. + */ +#define TF_KERNEL_SATP 0 +#define TF_KERNEL_SP 8 +#define TF_KERNEL_TRAP 16 +#define TF_KERNEL_CPUID 24 +#define TF_EPC 32 +#define TF_SSTATUS 40 +#define TF_RA 48 +#define TF_SP 56 +#define TF_GP 64 +#define TF_TP 72 +#define TF_T0 80 +#define TF_T1 88 +#define TF_T2 96 +#define TF_T3 104 +#define TF_T4 112 +#define TF_T5 120 +#define TF_T6 128 +#define TF_S0 136 +#define TF_S1 144 +#define TF_S2 152 +#define TF_S3 160 +#define TF_S4 168 +#define TF_S5 176 +#define TF_S6 184 +#define TF_S7 192 +#define TF_S8 200 +#define TF_S9 208 +#define TF_S10 216 +#define TF_S11 224 +#define TF_A0 232 +#define TF_A1 240 +#define TF_A2 248 +#define TF_A3 256 +#define TF_A4 264 +#define TF_A5 272 +#define TF_A6 280 +#define TF_A7 288 +#define TF_SIZE 296 + +#ifndef __ASSEMBLER__ + +#include +#include + +namespace arch { + +struct TrapFrame { + /* ---- Kernel metadata (offset 0-39) ---- */ + uint64 kernel_satp; // 0: Kernel page table (written to satp on trap entry) + uint64 kernel_sp; // 8: Top of kernel stack for this task + uint64 kernel_trap; // 16: Address of C++ trap_handler function + uint64 kernel_cpu_id; // 24: CPU ID (loaded into tp on trap entry) + uint64 epc; // 32: Interrupted instruction address (sepc) + + /* ---- CSR backup (offset 40-47) ---- */ + uint64 sstatus; // 40: Privilege status (SPP, SPIE bits) + + /* ---- General-purpose registers (offset 48-295) ---- */ + uint64 ra; // 48: x1 - Return address + uint64 sp; // 56: x2 - User stack pointer + uint64 gp; // 64: x3 - Global pointer + uint64 tp; // 72: x4 - Thread pointer + uint64 t[7]; // 80: x5-x7, x28-x31 (t0-t6, caller-saved temporaries) + uint64 s[12]; // 136: x8-x9, x18-x27 (s0-s11, callee-saved) + uint64 a[8]; // 232: x10-x17 (a0-a7, arguments / syscall params) +}; + +static_assert(sizeof(TrapFrame) == 296, "TrapFrame size mismatch with assembly offsets"); + +} // namespace arch + +/* Forward declaration — defined in trap_md.cc. + * Must be after TrapFrame definition. */ +extern "C" [[noreturn]] void trap_dispatch(arch::TrapFrame*); + +namespace arch { + +/* + * trapframe_init_user — prepare a TrapFrame for returning to user mode. + * + * Sets arch-specific sstatus: + * SPP = 0 (return to U-mode on sret) + * SPIE = 1 (enable interrupts in U-mode) + * FS = Initial (allow FP instructions) + * + * Called from MI exec.cc instead of setting sstatus directly. + */ +static inline void trapframe_init_user(TrapFrame* tf, uint64 entry, uint64 sp) +{ + /* Clear all registers */ + for (auto& t : tf->t) { t = 0; } + for (auto& s : tf->s) { s = 0; } + for (auto& a : tf->a) { a = 0; } + tf->ra = 0; + tf->gp = 0; + tf->tp = 0; + + tf->epc = entry; + tf->sp = sp; + + /* sstatus: SPP=0 (U-mode), SPIE=1 (interrupts), FS=Initial (FP ok) */ + constexpr uint64 kSPIE = (1ULL << 5); + constexpr uint64 kFS_Initial = (1ULL << 13); + tf->sstatus = kSPIE | kFS_Initial; + + /* Kernel metadata used by trap_vector.S on every trap from U-mode. + * These are RISC-V specific — trap_vector reads them from the + * TrapFrame to switch to kernel context. */ + uint64 satp; + asm volatile("csrr %0, satp" : "=r"(satp)); + tf->kernel_satp = satp; + tf->kernel_sp = reinterpret_cast(tf); + tf->kernel_trap = reinterpret_cast(&trap_dispatch); + tf->kernel_cpu_id = cpu::id(); +} + +} // namespace arch + +#endif /* __ASSEMBLER__ */ diff --git a/arch/riscv/include/asm/csr.h b/arch/riscv/include/asm/csr.h new file mode 100644 index 0000000..42f1eb4 --- /dev/null +++ b/arch/riscv/include/asm/csr.h @@ -0,0 +1,37 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* Copyright (C) 2026 Richard Qin */ + +#ifndef _ASM_RISCV_CSR_H +#define _ASM_RISCV_CSR_H + +/* + * RISC-V CSR (Control & Status Register) definitions used by the early + * boot / MMU-switch path. + * + * LumeOS-native subset of Linux arch/riscv/include/asm/csr.h. Only the + * symbols head.S and mm/init.cc actually reference are defined — adding + * the rest is straightforward when new code needs them, and skipping the + * Linux ALTERNATIVE / vendor-extension machinery keeps the cascade flat. + */ + +/* SATP (Supervisor Address Translation and Protection) layout — RV64 */ +#define SATP_PPN_MASK 0x00000FFFFFFFFFFFUL /* bits [43:0] = root PPN */ +#define SATP_ASID_SHIFT 44 +#define SATP_MODE_SHIFT 60 + +/* SATP MODE encodings (RV64) */ +#define SATP_MODE_BARE (0UL << SATP_MODE_SHIFT) +#define SATP_MODE_SV39 (8UL << SATP_MODE_SHIFT) +#define SATP_MODE_SV48 (9UL << SATP_MODE_SHIFT) + +/* Compose a SATP value from a page-table physical address. + * Result: MODE | (root_ppn) where root_ppn = pgd_pa >> PAGE_SHIFT. */ +#define MAKE_SATP(mode, pgd_pa) ((mode) | (((pgd_pa) >> 12) & SATP_PPN_MASK)) + +/* SSTATUS bits we touch at boot — disable FP / vector at entry so that an + * accidental FP / V instruction in early code traps cleanly rather than + * silently using dirty state from before reset. */ +#define SR_FS_OFF (0UL << 13) +#define SR_VS_OFF (0UL << 9) + +#endif /* _ASM_RISCV_CSR_H */ diff --git a/arch/riscv/include/asm/page.h b/arch/riscv/include/asm/page.h new file mode 100644 index 0000000..7b9f155 --- /dev/null +++ b/arch/riscv/include/asm/page.h @@ -0,0 +1,62 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* Copyright (C) 2026 Richard Qin */ + +#ifndef _ASM_RISCV_PAGE_H +#define _ASM_RISCV_PAGE_H + +/* + * RISC-V page / virtual-memory layout constants. + * + * Mirrors the subset of Linux arch/riscv/include/asm/page.h that LumeOS's + * early boot path needs: + * - PAGE_SIZE / PAGE_SHIFT / PAGE_MASK + * - PAGE_OFFSET (start of the higher-half direct map) + * - KERNEL_LINK_ADDR (link-time VA of _start) + * - SV39 PTE flag bits + * - PTRS_PER_PGD (512 for SV39) + * + * Values match Linux RISC-V SV39 defaults so that struct kernel_mapping + * (filled by setup_vm in arch/riscv/mm/init.cc) carries the same numeric + * relations Linux's mm/init.c maintains. + */ + +#define PAGE_SHIFT 12 +#define PAGE_SIZE (1UL << PAGE_SHIFT) +#define PAGE_MASK (~(PAGE_SIZE - 1)) + +/* Higher-half base of the kernel direct map. Bottom of the SV39 supervisor + * address space minus the unmapped guard region. Identical to Linux's + * CONFIG_PAGE_OFFSET default on RV64. */ +#define PAGE_OFFSET 0xFFFFFFC000000000UL + +/* Where _start was linked. OpenSBI hands control to PHYS_BASE; the kernel + * runs at KERNEL_LINK_ADDR after relocate_enable_mmu jumps to higher-half. */ +#define PHYS_BASE 0x80200000UL +#define KERNEL_LINK_ADDR 0xFFFFFFC080200000UL + +/* SATP mode field — SV39 = 8, shifted to bits [63:60]. */ +#define SATP_MODE_SV39 (8UL << 60) + +#define PTRS_PER_PGD 512 + +/* 1 GB superpage size — entry.S used to build the early page table with + * these; setup_vm in arch/riscv/mm/init.cc now does the same. */ +#define PGDIR_SHIFT 30 +#define PGDIR_SIZE (1UL << PGDIR_SHIFT) +#define PGDIR_MASK (~(PGDIR_SIZE - 1)) + +/* SV39 PTE flag bits (one byte). */ +#define _PAGE_PRESENT (1UL << 0) /* aka VALID — historic Linux name */ +#define _PAGE_READ (1UL << 1) +#define _PAGE_WRITE (1UL << 2) +#define _PAGE_EXEC (1UL << 3) +#define _PAGE_USER (1UL << 4) +#define _PAGE_GLOBAL (1UL << 5) +#define _PAGE_ACCESSED (1UL << 6) +#define _PAGE_DIRTY (1UL << 7) + +#define PAGE_KERNEL_EXEC \ + (_PAGE_PRESENT | _PAGE_READ | _PAGE_WRITE | _PAGE_EXEC | \ + _PAGE_ACCESSED | _PAGE_DIRTY) + +#endif /* _ASM_RISCV_PAGE_H */ diff --git a/arch/riscv/include/asm/pgtable.h b/arch/riscv/include/asm/pgtable.h new file mode 100644 index 0000000..28fc4c5 --- /dev/null +++ b/arch/riscv/include/asm/pgtable.h @@ -0,0 +1,55 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* Copyright (C) 2026 Richard Qin */ + +#ifndef _ASM_RISCV_PGTABLE_H +#define _ASM_RISCV_PGTABLE_H + +/* + * Early page-table construction API used by arch/riscv/mm/init.cc::setup_vm. + * + * Mirrors the create_pgd_mapping interface from Linux's + * arch/riscv/mm/init.c — same name, same semantics: + * + * create_pgd_mapping(pgdir, va, pa, sz, prot) + * + * pgdir : root of the page table to populate + * va : virtual address to map + * pa : physical address to map to + * sz : mapping size — must be a supported leaf size (PAGE_SIZE, + * 2 MiB megapage, or 1 GiB gigapage) + * prot : PTE flag bits (PAGE_KERNEL_EXEC etc., from ) + * + * The boot-time variant in this file only supports gigapage (1 GiB) + * mappings — that is all setup_vm needs for the early_pg_dir / trampoline_pg_dir + * built before the slab allocator exists. Finer-grained mappings come + * later via vmm_init(). + */ + +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct { unsigned long pgd; } pgd_t; + +/* Linker-provided early page-table storage (defined in lumeos.lds.S + * ARCH_BSS_EXTRA, lives in .bss so it's zero-initialized). Allocating + * them in the linker script — not in C — mirrors Linux and keeps the + * symbols visible from head.S without a forward C decl mismatch. */ +extern pgd_t early_pg_dir[PTRS_PER_PGD]; +extern pgd_t trampoline_pg_dir[PTRS_PER_PGD]; +extern pgd_t fixmap_pg_dir[PTRS_PER_PGD]; + +/* Install a single PGD entry mapping `va` → `pa` with size `sz` and PTE + * flags `prot`. Only sz == PGDIR_SIZE (1 GiB) is supported in this early + * variant — vmm_init() takes over with create_pgd_mapping_pmd later. */ +void create_pgd_mapping(pgd_t *pgdir, unsigned long va, unsigned long pa, + unsigned long sz, unsigned long prot); + +#ifdef __cplusplus +} +#endif + +#endif /* _ASM_RISCV_PGTABLE_H */ diff --git a/arch/riscv/include/asm/sections.h b/arch/riscv/include/asm/sections.h new file mode 100644 index 0000000..454d9de --- /dev/null +++ b/arch/riscv/include/asm/sections.h @@ -0,0 +1,50 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* Copyright (C) 2026 Richard Qin */ + +#ifndef _ASM_RISCV_SECTIONS_H +#define _ASM_RISCV_SECTIONS_H + +/* + * Linker-emitted symbols that bracket each section of the kernel image. + * + * The lumeos.lds.S linker script defines these as the start / end + * addresses of contiguous regions. Code that walks a section + * (relocate_kernel walks .rela.dyn, the C++ ctor loop walks + * .init_array, the VMM mapping code walks .text vs .rodata, ...) + * imports the pair via this header. + * + * Mirrors Linux arch/riscv/include/asm/sections.h / asm-generic/sections.h. + * Each name and pair is identical to Linux so that direct ports of mm/init.c + * relocate_kernel logic compile unchanged against LumeOS. + */ + +#ifdef __cplusplus +extern "C" { +#endif + +extern char _start[]; +extern char _end[]; + +extern char _stext[]; +extern char _etext[]; +extern char _srodata[]; +extern char _erodata[]; +extern char _sdata[]; +extern char _edata[]; +extern char _sbss[]; +extern char _ebss[]; + +extern char __init_array_start[]; +extern char __init_array_end[]; + +/* .rela.dyn — array of Elf64_Rela entries emitted by the linker under + * `-shared -Bsymbolic`. relocate_kernel walks this range and patches + * each R_RISCV_RELATIVE slot. */ +extern char __rela_dyn_start[]; +extern char __rela_dyn_end[]; + +#ifdef __cplusplus +} +#endif + +#endif /* _ASM_RISCV_SECTIONS_H */ diff --git a/arch/riscv/kernel/asm-offsets.c b/arch/riscv/kernel/asm-offsets.c new file mode 100644 index 0000000..edecab7 --- /dev/null +++ b/arch/riscv/kernel/asm-offsets.c @@ -0,0 +1,29 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +// Copyright (C) 2026 Richard Qin +// +// arch/riscv/kernel/asm-offsets.c — emit struct field offsets and +// scalar constants for consumption by assembly sources (head.S etc.). +// +// 1:1 with Linux arch/riscv/kernel/asm-offsets.c. The Makefile +// compiles this to an .s file (not .o) and runs +// scripts/extract-asm-offsets.sed over the output to produce +// include/generated/asm-offsets.h. +// +// Adding a new offset: include the struct's header, add an OFFSET() +// or DEFINE() line below, rebuild. head.S can then reference the +// generated #define directly. + +#include +#include + +void asm_offsets(void) +{ + COMMENT("LumeOS asm-offsets — struct kernel_mapping field offsets"); + OFFSET(KERNEL_MAP_VIRT_ADDR, kernel_mapping, virt_addr); + OFFSET(KERNEL_MAP_VIRT_OFFSET, kernel_mapping, virt_offset); + OFFSET(KERNEL_MAP_PHYS_ADDR, kernel_mapping, phys_addr); + OFFSET(KERNEL_MAP_SIZE, kernel_mapping, size); + OFFSET(KERNEL_MAP_VA_PA_OFFSET, kernel_mapping, va_pa_offset); + OFFSET(KERNEL_MAP_PAGE_OFFSET, kernel_mapping, page_offset); + OFFSET(KERNEL_MAP_VA_KERNEL_PA_OFFSET, kernel_mapping, va_kernel_pa_offset); +} diff --git a/arch/riscv/kernel/head.S b/arch/riscv/kernel/head.S new file mode 100644 index 0000000..e04c44c --- /dev/null +++ b/arch/riscv/kernel/head.S @@ -0,0 +1,317 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* Copyright (C) 2026 Richard Qin */ + +/* + * RISC-V kernel entry — Linux-aligned layout. + * + * Flow (mirrors arch/riscv/kernel/head.S in Linux): + * + * _start [.head.text, byte 0 of image] + * · 64-byte Boot Protocol Image Header + * · j real_start + * + * real_start [.text] + * · save a0 (hartid) / a1 (dtb_pa) + * · setup gp / per-hart sp + * · BSP/AP election via amoswap + * · BSP: clear BSS, call setup_vm, then relocate_enable_mmu + * · AP: spin-wait, then relocate_enable_mmu + * + * relocate_enable_mmu [C function, arch/riscv/mm/init.cc] + * · csrw satp <- trampoline_pg_dir + * · jump to higher-half VA + * · csrw satp <- early_pg_dir + * · slide sp/ra/gp to higher-half + * · call relocate_kernel (walks .rela.dyn, patches R_RISCV_RELATIVE) + * + * real_start (continued in higher-half) + * · BSP only: call early_cpp_init (walks .init_array) + * · call kernel_main(hartid, dtb_pa) + * + * Register contract from OpenSBI / U-Boot: + * a0 = hart ID + * a1 = FDT physical base + */ + +#include + +#define PAGE_OFFSET 0xFFFFFFC000000000 +#define BOOT_STACK_SIZE 8192 + +/* ======================================================================== + * .head.text — RISC-V Boot Protocol Image Header + * + * Layout per Documentation/arch/riscv/boot-image-header.rst: + * code0/code1 (8 B) : executable code, lands at PC after OpenSBI hand-off + * text_offset (8 B) : load offset from RAM base + * image_size (8 B) : effective image size (0 = unknown) + * flags (8 B) : bit 0: endian + * version (4 B) : major.minor + * res1/res2 (4+8) : reserved + * magic (8 B) : "RISCV\0\0\0" + * magic2 (4 B) : "RSC\x05" + * res3 (4 B) : PE/COFF reserved + * ======================================================================== */ + .section .head.text, "ax" + .globl _start +_start: + j real_start /* code0 */ + .word 0 /* code1 */ + .dword 0x200000 /* text_offset */ + .dword 0 /* image_size (post-link) */ + .dword 0 /* flags */ + .word 0x0002 /* version 0.2 */ + .word 0 /* res1 */ + .dword 0 /* res2 */ + .dword 0x5643534952 /* magic "RISCV\0\0\0" */ + .word 0x05435352 /* magic2 "RSC\x05" */ + .word 0 /* res3 */ + +/* ======================================================================== + * real_start — all harts land here at physical PC + * ======================================================================== */ + .text + .globl real_start +real_start: + /* DEBUG: 'R\n' — UART direct at 0x10000000, MMU off. */ + li t0, 0x10000000 + li t1, 'R' + sb t1, 0(t0) + addi t1, a0, '0' + sb t1, 0(t0) + li t1, '\n' + sb t1, 0(t0) + + /* Mask interrupts and supervisor pending bits. */ + csrw sie, zero + csrw sip, zero + + /* gp for medany — relax must be disabled across the `la`. */ + .option push + .option norelax + la gp, __global_pointer$ + .option pop + + /* per-hart boot stack: sp = boot_stack + (hartid + 1) * STACK_SIZE */ + la t0, boot_stack + li t1, BOOT_STACK_SIZE + addi t2, a0, 1 + mul t1, t1, t2 + add sp, t0, t1 + + /* Save hartid / dtb_pa in callee-saved regs preserved across calls. */ + mv tp, a0 /* tp = hartid */ + mv s0, a1 /* s0 = dtb_pa */ + + /* BSP/AP election: first hart to swap bsp_elected 0→1 is BSP. */ + la t0, bsp_elected + li t1, 1 + amoswap.w.aq t2, t1, (t0) + bnez t2, .Lap_wait /* prev != 0 → we're an AP */ + + /* === BSP path === */ + la t0, bsp_hart_id + sw a0, 0(t0) + + li t3, 0x10000000 + li t4, 'B' + sb t4, 0(t3) + li t4, '\n' + sb t4, 0(t3) + + /* Clear BSS — must precede setup_vm, which writes kernel_map. */ + la t0, _sbss + la t1, _ebss +.Lbss_loop: + bgeu t0, t1, .Lbss_done + sd zero, 0(t0) + addi t0, t0, 8 + j .Lbss_loop +.Lbss_done: + + li t3, 0x10000000 + li t4, 'V' + sb t4, 0(t3) + li t4, '\n' + sb t4, 0(t3) + + /* ---- BSP-only: pre-MMU PIC fixup of .rela.dyn ---- + * + * The kernel is built with -fPIE -shared. All global-data accesses + * compile to GOT-indirect loads (auipc + ld), where the linker + * emits each .got slot's initial value as the link-time VA and a + * companion R_RISCV_RELATIVE entry in .rela.dyn. At this point + * we're running at the physical load address — those link-time VAs + * are not yet mapped, so any global access would fault. + * + * Walk .rela.dyn in pure assembly (PC-relative only — no globals) + * and patch every R_RISCV_RELATIVE slot to addend + (PA - LINK_VA). + * After this, setup_vm can dereference kernel_map / early_pg_dir + * / etc. via the GOT and get correct PA-side pointers. + * + * Mirrors Linux arch/riscv/kernel/head.S — same idea, same loop. + * + * reloc: + * t0 = pa_offset = PA(_start) - KERNEL_LINK_ADDR (negative) + * t1 = &__rela_dyn_start + * t2 = &__rela_dyn_end + * for each 24-byte rela in [t1, t2): + * t3 = r_offset; t4 = r_info; t5 = r_addend + * if (t4 & 0xFFFFFFFF) != R_RISCV_RELATIVE: skip + * slot_pa = r_offset + pa_offset + * *slot_pa = r_addend + pa_offset + */ +#define R_RISCV_RELATIVE 3 + + la t0, _start /* PA of _start */ + li t1, 0xFFFFFFC080200000 /* KERNEL_LINK_ADDR (asm/page.h) */ + sub t0, t0, t1 /* t0 = pa_offset (PA - LINK_VA) */ + + la t1, __rela_dyn_start + la t2, __rela_dyn_end +.Lrela_loop: + bgeu t1, t2, .Lrela_done + ld t3, 0(t1) /* r_offset */ + ld t4, 8(t1) /* r_info */ + ld t5, 16(t1) /* r_addend */ + /* r_info low 32 bits = type */ + li t6, 0xFFFFFFFF + and t4, t4, t6 + li t6, R_RISCV_RELATIVE + bne t4, t6, .Lrela_next + /* slot_pa = r_offset + pa_offset */ + add t3, t3, t0 + /* value = r_addend + pa_offset */ + add t5, t5, t0 + sd t5, 0(t3) +.Lrela_next: + addi t1, t1, 24 /* sizeof(Elf64_Rela) */ + j .Lrela_loop +.Lrela_done: + + /* Memory fence so subsequent loads see the patched slots. */ + fence w, r + + li t3, 0x10000000 + li t4, 'P' + sb t4, 0(t3) + li t4, '\n' + sb t4, 0(t3) + + /* Call setup_vm(load_pa = PA of _start, dtb_pa = s0). + * Both args physical at this point — MMU still off. */ + la a0, _start + mv a1, s0 + call setup_vm + + li t3, 0x10000000 + li t4, 'v' + sb t4, 0(t3) + li t4, '\n' + sb t4, 0(t3) + + /* Release APs once page tables are built. */ + la t0, boot_sync_flag + li t1, 1 + amoswap.w.rl zero, t1, (t0) + j .Lenable_mmu + +/* ======================================================================== + * AP path — spin until BSP signals readiness. + * ======================================================================== */ +.Lap_wait: + la t0, boot_sync_flag +.Lap_spin: + lw t1, 0(t0) + beqz t1, .Lap_spin + fence r, rw + + /* fall through */ + +/* ======================================================================== + * .Lenable_mmu — all harts: enable MMU, jump to higher-half, run reloc. + * + * relocate_enable_mmu does its own SATP write + higher-half jump in + * C (with inline asm), then calls relocate_kernel. After it returns, + * we are at higher-half VA with sp / ra / gp slid up, and .rela.dyn + * is applied (so .init_array pointers are valid). + * ======================================================================== */ +.Lenable_mmu: + la a0, early_pg_dir + call relocate_enable_mmu + /* On return: PC, sp, ra, gp are all at higher-half VA. */ + + li t3, 0x10000000 + li t4, 'M' + sb t4, 0(t3) + li t4, '\n' + sb t4, 0(t3) + + /* Mark MMU enabled for console / pmap layers that check this. */ + call console_set_mmu_enabled + + /* BSP only: walk __init_array and call each C++ ctor. */ + la t0, bsp_hart_id + lw t0, 0(t0) + bne tp, t0, .Lskip_cpp_init + call early_cpp_init +.Lskip_cpp_init: + + /* Hand off to kernel_main(hartid, dtb_pa). */ + mv a0, tp + mv a1, s0 + call kernel_main + + /* kernel_main should not return. */ +.Lhalt: + csrc sstatus, 2 + wfi + j .Lhalt + +/* ======================================================================== + * early_cpp_init — walk .init_array and call each constructor. + * + * Called only by BSP, after relocate_kernel has patched the function + * pointers stored in __init_array. Before relocate_kernel runs those + * pointers are zero-filled by the linker. + * ======================================================================== */ + .globl early_cpp_init +early_cpp_init: + addi sp, sp, -32 + sd ra, 0(sp) + sd s1, 8(sp) + sd s2, 16(sp) + + la s1, __init_array_start + la s2, __init_array_end +.Linit_loop: + bgeu s1, s2, .Linit_done + ld t0, 0(s1) + jalr ra, t0 + addi s1, s1, 8 + j .Linit_loop +.Linit_done: + ld ra, 0(sp) + ld s1, 8(sp) + ld s2, 16(sp) + addi sp, sp, 32 + ret + +/* ======================================================================== + * Election state — in .data so explicit zero survives the BSP BSS clear + * (these are read before BSS is cleared on the BSP, and concurrently by + * APs while BSP is in the clear loop). + * ======================================================================== */ + .section .data + .align 2 + .globl boot_sync_flag +boot_sync_flag: + .word 0 + + .globl bsp_elected +bsp_elected: + .word 0 + + .globl bsp_hart_id +bsp_hart_id: + .word 0 diff --git a/arch/riscv/kernel/lumeos.lds.S b/arch/riscv/kernel/lumeos.lds.S new file mode 100644 index 0000000..9bd083b --- /dev/null +++ b/arch/riscv/kernel/lumeos.lds.S @@ -0,0 +1,120 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* Copyright (C) 2026 Richard Qin */ + +/* + * RISC-V Kernel Linker Script Template + * + * This is a C-preprocessed linker script (.lds.S). The build system + * runs it through CPP to produce the final vmlinux.lds which is + * passed to the linker. + * + * Architecture-specific definitions: + * - OUTPUT_ARCH, ENTRY + * - PAGE_OFFSET, BASE_ADDRESS, PHYS_BASE + * - .head.text placement + * - GP anchor (__global_pointer$ for RISC-V medany) + * - early_pgdir and boot stacks in .bss + * + * Common section macros come from include/lume/vmlinux.lds.h + * + * Reference: docs/specs/linker_and_entry.md + */ + +#include + +/* ===== RISC-V Architecture Constants ===== */ + +OUTPUT_ARCH(riscv) + +PAGE_OFFSET = 0xFFFFFFC000000000; +BASE_ADDRESS = 0xFFFFFFC080200000; +PHYS_BASE = 0x80200000; + +ENTRY(_start) + +/* ===== RISC-V: GP anchor in .data (medany model) ===== */ +#define ARCH_DATA_EXTRA \ + . = ALIGN(16); \ + PROVIDE(__global_pointer$ = . + 0x800); \ + *(.sdata .sdata.*) + +/* ===== RISC-V: page-table storage + per-CPU boot stacks in .bss ===== + * + * Linux RISC-V layout (arch/riscv/kernel/vmlinux.lds.S): + * early_pg_dir — first early page table built by setup_vm, + * used for the rest of boot after MMU enable + * trampoline_pg_dir — single-page table used only during the SATP + * switch from physical to higher-half VA; its + * identity + virt mappings of _start must both + * be valid in the same instruction window + * fixmap_pg_dir — covers the FIXADDR_START range used by the + * early console (UART MMIO before kernel page + * table proper is built) + * + * All three are allocated here so head.S can take their addresses without + * a C definition; their PA == VA mapping covers them during setup_vm. + */ +#define ARCH_BSS_EXTRA \ + . = ALIGN(4096); \ + early_pg_dir = .; \ + . += 4096; \ + trampoline_pg_dir = .; \ + . += 4096; \ + fixmap_pg_dir = .; \ + . += 4096; \ + /* Legacy name retained for any caller still referencing it; */ \ + /* points at the same first 4 KiB as early_pg_dir. */ \ + early_pgdir = early_pg_dir; \ + /* Per-CPU boot stacks: 8KB per CPU, max 8 CPUs */ \ + . = ALIGN(4096); \ + boot_stack = .; \ + . += 8192 * 8; \ + boot_stack_top = .; + +/* ===== Layout ===== */ + +SECTIONS { + . = BASE_ADDRESS; + + /* .head.text: Image Header (must be at byte 0 of the kernel image) */ + .head.text : AT(PHYS_BASE) { + _start = .; + KEEP(*(.head.text)) + } + + TEXT_SECTION + + . = ALIGN(4096); + + /* .rela.dyn lives between .text and .rodata so it ends up in the + * read-only PT_LOAD segment. Mirrors Linux vmlinux.lds.S where + * .rela.dyn sits at the tail of init data, just before RO_DATA. */ + RELA_DYN_SECTION + + . = ALIGN(4096); + + RODATA_SECTION + + . = ALIGN(4096); + + DATA_SECTION + + . = ALIGN(8); + + /* .got lives right after .data in the RW PT_LOAD. All other + * dynamic-linker artefacts (.plt, .dynamic, .dynsym, .dynstr, + * .hash, .gnu.hash, .interp, .dynbss) are discarded by the + * DISCARDS macro — mirrors Linux arch/riscv/kernel/vmlinux.lds.S. */ + GOT_SECTION + + . = ALIGN(4096); + + /* End of file-backed image (BSS follows as NOBITS) */ + __lume_image_end = .; + + BSS_SECTION + + KERNEL_END + + DISCARDS +} diff --git a/arch/riscv/kernel/signal.cc b/arch/riscv/kernel/signal.cc new file mode 100644 index 0000000..39fe34e --- /dev/null +++ b/arch/riscv/kernel/signal.cc @@ -0,0 +1,163 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* Copyright (C) 2026 Richard Qin */ + +/* + * arch/riscv/signal.cc — Signal frame construction and sigreturn + * + * Constructs a signal frame on the user stack before returning to + * the user signal handler. On sigreturn, restores the original + * registers from the saved frame. + * + * Frame layout on user stack (grows downward): + * + * +-----------------------+ <- original sp + * | sigset_t saved_mask | 8 bytes + * | TrapFrame saved_regs | 296 bytes + * | kernel_siginfo info | 128 bytes + * +-----------------------+ <- new sp (16-byte aligned) + * + * Reference: Linux arch/riscv/kernel/signal.c + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +/* ================================================================== */ +/* Signal frame layout */ +/* ================================================================== */ + +struct rt_sigframe { + kernel_siginfo info; /* siginfo for SA_SIGINFO handlers */ + arch::TrapFrame saved_regs; /* saved register state */ + sigset_t saved_mask; /* blocked mask to restore on sigreturn */ +}; + +/* ================================================================== */ +/* Signal trampoline (written to user stack) */ +/* ================================================================== */ + +/* + * When the user's signal handler returns (via `ret`), ra points here. + * This tiny code sequence invokes sys_rt_sigreturn to restore the + * original execution context. + * + * RISC-V instructions (8 bytes): + * li a7, 139 → addi a7, zero, 139 → 0x08b00893 + * ecall → 0x00000073 + */ +static constexpr uint32 sigreturn_trampoline[2] = { + 0x08b00893, /* li a7, 139 (SYS_rt_sigreturn) */ + 0x00000073, /* ecall */ +}; + +/* ================================================================== */ +/* setup_signal_frame */ +/* ================================================================== */ + +int setup_signal_frame(arch::TrapFrame* tf, int signo, + kernel_sigaction* ka, kernel_siginfo* info) +{ + TaskControlBlock* curr = current_task(); + if (!curr) return -EINVAL; + + /* Calculate frame location on user stack */ + uint64 sp = tf->sp; + + /* Reserve space for the trampoline below the frame */ + sp -= sizeof(sigreturn_trampoline); + sp &= ~15ULL; /* 16-byte align */ + uint64 trampoline_addr = sp; + + /* Reserve space for the signal frame */ + sp -= sizeof(rt_sigframe); + sp &= ~15ULL; /* 16-byte align */ + uint64 frame_addr = sp; + + /* Build the frame in kernel memory, then copy to user stack */ + rt_sigframe frame{}; + if (info) { + frame.info = *info; + } + frame.saved_regs = *tf; + frame.saved_mask = curr->blocked; + + /* Copy frame to user stack */ + int err = copy_to_user(frame_addr, &frame, sizeof(frame)); + if (err < 0) return -EFAULT; + + /* Copy trampoline to user stack */ + err = copy_to_user(trampoline_addr, sigreturn_trampoline, + sizeof(sigreturn_trampoline)); + if (err < 0) return -EFAULT; + + /* Set up the trap frame to execute the signal handler */ + tf->epc = ka->sa_handler; + tf->sp = frame_addr; + + /* Arguments to the handler: handler(int signo, siginfo_t*, ucontext_t*) */ + tf->a[0] = static_cast(signo); + + if (ka->sa_flags & SA_SIGINFO) { + /* SA_SIGINFO: pass siginfo and ucontext pointers */ + tf->a[1] = frame_addr; /* &info */ + tf->a[2] = frame_addr + sizeof(kernel_siginfo); /* &saved_regs (as ucontext) */ + } else { + tf->a[1] = 0; + tf->a[2] = 0; + } + + /* Return address: when the handler does `ret`, jump to trampoline */ + if (ka->sa_flags & SA_RESTORER) { + tf->ra = ka->sa_restorer; + } else { + tf->ra = trampoline_addr; + } + + return 0; +} + +/* ================================================================== */ +/* arch_sigreturn — restore state from signal frame */ +/* ================================================================== */ + +int64 arch_sigreturn(arch::TrapFrame* tf) +{ + TaskControlBlock* curr = current_task(); + if (!curr) return -EINVAL; + + /* + * The signal frame is located at the current sp. + * We need to read it back and restore the original state. + */ + uint64 frame_addr = tf->sp; + + rt_sigframe frame{}; + int err = copy_from_user(&frame, frame_addr, sizeof(frame)); + if (err < 0) { + printk(KERN_ERR "sigreturn: failed to read signal frame at %llx\n", + frame_addr); + task_kill_current(SIGSEGV); + return -EFAULT; + } + + /* Restore the saved register state */ + *tf = frame.saved_regs; + + /* Restore the blocked mask */ + curr->blocked = frame.saved_mask; + curr->blocked.del(SIGKILL); + curr->blocked.del(SIGSTOP); + + /* + * Return value: we return the saved a0 from the original context. + * The syscall dispatch will NOT overwrite a0 because sigreturn + * is special — it returns via the restored trap frame directly. + */ + return tf->a[0]; +} diff --git a/arch/riscv/lib/random.cc b/arch/riscv/lib/random.cc new file mode 100644 index 0000000..afc0078 --- /dev/null +++ b/arch/riscv/lib/random.cc @@ -0,0 +1,21 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* Copyright (C) 2026 Richard Qin */ + +#include +#include + +namespace arch { + +bool get_hw_random(uint64& value) +{ + /* + * Real implementation would check for Zkr extension and execute: + * csrr x, seed + * + * Since QEMU virt doesn't guarantee Zkr, we return false to + * force the MI layer to use the timer fallback. + */ + return false; +} + +} // namespace arch diff --git a/arch/riscv/mm/init.cc b/arch/riscv/mm/init.cc new file mode 100644 index 0000000..3cebcd9 --- /dev/null +++ b/arch/riscv/mm/init.cc @@ -0,0 +1,241 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* Copyright (C) 2026 Richard Qin */ + +/* + * RISC-V early memory-management bootstrap. + * + * Mirrors Linux arch/riscv/mm/init.c — same file name, same C function + * names (setup_vm / relocate_enable_mmu / relocate_kernel), same global + * state (kernel_map, early_pg_dir, trampoline_pg_dir). head.S calls + * setup_vm immediately after BSS-clear, then later calls + * relocate_enable_mmu, which switches SATP, jumps to higher-half VA, + * and finally calls relocate_kernel to apply .rela.dyn. + * + * Differences from Linux: + * - KASLR offset is hard-wired to 0 for now (no pi/ subsystem) — the + * code path that would consume kernel_map.virt_offset is in place + * so adding pi/-based seed reading later only requires populating + * that one field. + * - Only 1 GiB superpages supported in create_pgd_mapping; sufficient + * for the early page tables. Finer-grained mappings come from + * vmm_init() once slab is up. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* + * Global kernel mapping descriptor — single source of truth for the + * runtime-vs-link address relationship. Lives in .data so its initial + * zero contents survive the BSS-clear that runs before setup_vm fills + * the fields. Read by VMM, page-fault handler, ptdump, etc. + * + * Linux equivalent: arch/riscv/mm/init.c::kernel_map + */ +struct kernel_mapping kernel_map __attribute__((section(".data"))); + +/* + * create_pgd_mapping — install one PGD entry covering a 1 GiB region. + * + * Only 1 GiB superpages are supported in this early variant; setup_vm + * needs nothing finer. Identical interface to Linux's same-named + * function in arch/riscv/mm/init.c, restricted in arity. + */ +extern "C" void create_pgd_mapping(pgd_t *pgdir, unsigned long va, + unsigned long pa, unsigned long sz, + unsigned long prot) +{ + /* This early variant only supports 1 GiB superpages. */ + if (sz != PGDIR_SIZE) { + return; + } + + /* SV39 PGD index = (va >> 30) & 0x1FF — see Volume II, §4.4.1. + * Each PGD entry is 8 bytes; pgdir[i] is the slot for that VPN[2]. */ + unsigned long idx = (va >> PGDIR_SHIFT) & (PTRS_PER_PGD - 1); + + /* PTE PPN field: physical page number shifted up by 10 (SV39 PTE + * layout: bits [53:10] = PPN, bits [9:0] = flags). */ + unsigned long ppn = pa >> PAGE_SHIFT; + pgdir[idx].pgd = (ppn << 10) | prot; +} + +/* + * setup_vm — fill kernel_map and build early_pg_dir + trampoline_pg_dir. + * + * Called from head.S after BSS is cleared, with: + * load_pa = runtime physical address of _start + * dtb_pa = physical address of the device tree blob (from a1) + * + * Mirrors Linux arch/riscv/mm/init.c::setup_vm. The actual SATP write + * happens later in relocate_enable_mmu; here we only populate page + * tables and the kernel_map descriptor. + */ +extern "C" void __init setup_vm(uintptr_t load_pa, uintptr_t dtb_pa) +{ + (void)dtb_pa; /* KASLR pi/ subsystem will consume this later. */ + + /* --- 1. Fill kernel_map ----------------------------------------- */ + + /* KASLR offset = 0 in this configuration. */ + kernel_map.virt_offset = 0; + kernel_map.virt_addr = KERNEL_LINK_ADDR + kernel_map.virt_offset; + kernel_map.phys_addr = load_pa; + kernel_map.size = (uintptr_t)_end - (uintptr_t)_start; + kernel_map.page_offset = PAGE_OFFSET; + + /* Mirrors Linux: va_pa_offset is the constant added to a physical + * address to obtain its direct-map VA. */ + kernel_map.va_pa_offset = PAGE_OFFSET - load_pa; + + /* va_kernel_pa_offset is the constant added to a load PA to obtain + * the kernel's link VA (used by relocate_enable_mmu to fix sp/ra). */ + kernel_map.va_kernel_pa_offset = kernel_map.virt_addr - load_pa; + + /* --- 2. Build early_pg_dir ------------------------------------- * + * + * SV39 root page table — 4 entries cover identity + higher-half for + * both MMIO (PA 0x0000_0000, 1 GiB) and RAM (PA 0x8000_0000, 1 GiB): + * + * [0] identity PA 0x0000_0000 → VA 0x0000_0000 + * [2] identity PA 0x8000_0000 → VA 0x8000_0000 + * [256] high-half PA 0x0000_0000 → VA 0xFFFFFFC000000000 + * [258] high-half PA 0x8000_0000 → VA 0xFFFFFFC080000000 + * + * Identity mappings keep PC valid right after csrw satp. High-half + * mappings let the post-MMU `jr` to .Lhigher_half resolve. */ + create_pgd_mapping(early_pg_dir, 0x00000000UL, 0x00000000UL, + PGDIR_SIZE, PAGE_KERNEL_EXEC); + create_pgd_mapping(early_pg_dir, 0x80000000UL, 0x80000000UL, + PGDIR_SIZE, PAGE_KERNEL_EXEC); + create_pgd_mapping(early_pg_dir, PAGE_OFFSET + 0x00000000UL, + 0x00000000UL, PGDIR_SIZE, PAGE_KERNEL_EXEC); + create_pgd_mapping(early_pg_dir, PAGE_OFFSET + 0x80000000UL, + 0x80000000UL, PGDIR_SIZE, PAGE_KERNEL_EXEC); + + /* --- 3. Build trampoline_pg_dir -------------------------------- * + * + * The trampoline maps just the gigapage containing _start, both at + * its identity PA and at its higher-half VA. This is the page + * table SATP holds during the single instruction window between + * "MMU off" and "MMU on with early_pg_dir" — the PC must resolve + * under both, so we re-use the same two entries above. */ + create_pgd_mapping(trampoline_pg_dir, load_pa & PGDIR_MASK, + load_pa & PGDIR_MASK, PGDIR_SIZE, PAGE_KERNEL_EXEC); + create_pgd_mapping(trampoline_pg_dir, + (kernel_map.virt_addr) & PGDIR_MASK, + load_pa & PGDIR_MASK, PGDIR_SIZE, PAGE_KERNEL_EXEC); +} + +/* + * relocate_kernel — walk .rela.dyn and apply R_RISCV_RELATIVE entries. + * + * Called from relocate_enable_mmu *after* the SATP switch and the jump + * to higher-half VA — meaning __rela_dyn_start / __rela_dyn_end resolve + * to valid higher-half pointers we can dereference directly. + * + * Mirrors Linux arch/riscv/mm/init.c::relocate_kernel. Key invariant: + * even when kaslr_offset == 0, the slot store MUST still happen — the + * linker leaves R_RISCV_RELATIVE slots zero-filled at link time, and + * only relocate_kernel writes the link-time addend into them. Without + * this pass, .init_array entries (and every other relocated symbol) + * are NULL pointers. + */ +void __init relocate_kernel(void) +{ + auto *rela = (Elf64_Rela *)__rela_dyn_start; + auto *rela_end = (Elf64_Rela *)__rela_dyn_end; + + const uint64 kaslr_offset = kernel_map.virt_offset; + + for (; rela < rela_end; ++rela) { + if (ELF64_R_TYPE(rela->r_info) != R_RISCV_RELATIVE) { + continue; + } + auto *slot = (Elf64_Addr *)(rela->r_offset + kaslr_offset); + *slot = (Elf64_Addr)(rela->r_addend) + kaslr_offset; + } +} + +/* + * relocate_enable_mmu — switch SATP, jump to higher-half, apply relocations. + * + * Called from head.S after setup_vm has filled kernel_map and built the + * page tables. Equivalent to Linux's same-named asmlinkage routine but + * written in C++ so it can share kernel_map / pgtable definitions with + * setup_vm directly. + * + * Three logical steps: + * + * 1. csrw satp ← trampoline_pg_dir. PC continues running at the + * identity PA, which the trampoline still maps; the high-half PA + * mapping is now also live. + * + * 2. la t0, .Lvirt; add t0, t0, va_kernel_pa_offset; jr t0. + * A PC-relative `la` under medany yields the runtime PA of the + * label; adding va_kernel_pa_offset (= link_va - load_pa) converts + * it to the link-time higher-half VA, which the trampoline maps. + * The `jr` then atomically pushes PC into higher-half space. + * + * 3. csrw satp ← early_pg_dir. We're now executing at higher-half + * VA, so the trampoline can be retired in favour of the full + * early page table. sp / ra / gp must be slid to higher-half too, + * otherwise the next ret-or-call uses physical-low addresses + * whose mapping is about to disappear when vmm_init drops the + * identity entries. + * + * 4. Call relocate_kernel — now safe, because all link-time VA + * pointers in __rela_dyn_start[..] dereference correctly. + */ +extern "C" void __init relocate_enable_mmu(pgd_t *pgdir) +{ + (void)pgdir; /* head.S passes early_pg_dir for documentation; we + * read the global symbol directly inside the asm. */ + + const unsigned long va_kernel_pa_offset = kernel_map.va_kernel_pa_offset; + const unsigned long satp_mode = SATP_MODE_SV39; + + asm volatile ( + /* Step 1: csrw satp <- trampoline */ + "la t0, trampoline_pg_dir\n" + "srli t0, t0, 12\n" + "or t0, t0, %[mode]\n" + "csrw satp, t0\n" + "sfence.vma\n" + + /* Step 2: jump to higher-half VA */ + "la t0, 1f\n" + "add t0, t0, %[off]\n" + "jr t0\n" + "1:\n" + + /* Step 3a: csrw satp <- early_pg_dir (now in higher-half) */ + "la t0, early_pg_dir\n" + "srli t0, t0, 12\n" + "or t0, t0, %[mode]\n" + "csrw satp, t0\n" + "sfence.vma\n" + + /* Step 3b: slide sp / ra / gp to higher-half */ + "add sp, sp, %[off]\n" + "add ra, ra, %[off]\n" + ".option push\n" + ".option norelax\n" + "la gp, __global_pointer$\n" + ".option pop\n" + : + : [off] "r"(va_kernel_pa_offset), + [mode] "r"(satp_mode) + : "t0", "memory" + ); + + /* Step 4: apply .rela.dyn now that VA dereferences work. */ + relocate_kernel(); +} diff --git a/arch/riscv/mm/pmap.cc b/arch/riscv/mm/pmap.cc new file mode 100644 index 0000000..f0a6848 --- /dev/null +++ b/arch/riscv/mm/pmap.cc @@ -0,0 +1,361 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* Copyright (C) 2026 Richard Qin */ + +/* + * pmap.cc — SV39 page table operations (MD layer) + * + * Implements the pmap namespace interface for RISC-V SV39. + * 3-level page table: VPN[2] (9 bits) -> VPN[1] (9 bits) -> VPN[0] (9 bits) + * Each level is a 4KB page containing 512 PTEs of 8 bytes each. + * + * Address conversion: all page table operations work with physical addresses. + * To read/write page table entries we must convert PA to VA via pa_to_va(). + * + * Reference: docs/specs/vmm.md + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace pmap { + +/* + * Before the first pmap::activate(), CPU runs on entry.S bootstrap page table + * (fixed direct-map base). After activation, runtime (KASLR) base is valid. + */ +static bool g_runtime_mapping_ready = false; + +/* PTE bit definitions */ +inline constexpr uint64 PTE_V = 1ULL << 0; +inline constexpr uint64 PTE_R = 1ULL << 1; +inline constexpr uint64 PTE_W = 1ULL << 2; +inline constexpr uint64 PTE_X = 1ULL << 3; +inline constexpr uint64 PTE_U = 1ULL << 4; +inline constexpr uint64 PTE_G = 1ULL << 5; +inline constexpr uint64 PTE_A = 1ULL << 6; +inline constexpr uint64 PTE_D = 1ULL << 7; +inline constexpr uint64 PTE_COW = 1ULL << 8; + +/* SV39: 3 levels, 9 bits per level, 12-bit page offset */ +inline constexpr int kLevels = 3; +inline constexpr int kPtePerPage = 512; + +/* SV39: upper half of VA space (kernel) starts at L2 index 256 */ +inline constexpr int kKernelPgdStart = kPtePerPage / 2; + +/* SATP PPN mask (44 bits) and page shift */ +inline constexpr uint64 kSatpPpnMask = 0xFFFFFFFFFFFULL; +inline constexpr int kPageShift = 12; + +/* Extract VPN[level] from a virtual address */ +static inline uint64 vpn(uint64 va, int level) +{ + return (va >> (12 + 9 * level)) & 0x1FF; +} + +/* Extract PA from a PTE */ +static inline uint64 pte_to_pa(uint64 pte) +{ + return ((pte >> 10) & 0xFFFFFFFFFFFULL) << 12; +} + +/* Create a PTE from a PA and flags */ +static inline uint64 pa_to_pte(uint64 pa, uint64 flags) +{ + return ((pa >> 12) << 10) | flags; +} + +/* Convert PA to kernel VA for accessing page table contents */ +static inline uint64 *pa_to_ptr(uint64 pa) +{ + uint64 va = g_runtime_mapping_ready + ? pa_to_va(phys_addr(pa)).raw + : boot_pa_to_va(phys_addr(pa)).raw; + return reinterpret_cast(va); +} + +/* Allocate a zeroed page for a page table level. + * During early boot (before PMM), uses memblock. + * After PMM init, uses the buddy allocator. + * Returns PA of the new page, or 0 on failure. */ +static uint64 alloc_pt_page() +{ + uint64 pa; + if (memblock_is_active()) { + pa = memblock_alloc(kPageSize, kPageSize); + } else { + Page *page = pmm_alloc_page(); + if (!page) { + return 0; + } + pa = page_to_pa(page).raw; + } + if (!pa) { + return 0; + } + /* Zero the page */ + uint64 *ptr = pa_to_ptr(pa); + for (int i = 0; i < kPtePerPage; i++) { + ptr[i] = 0; + } + return pa; +} + +uint64 create() +{ + uint64 new_root = alloc_pt_page(); + if (!new_root) return 0; + + /* Copy kernel mappings from the current kernel page table. + * In SV39, entries 256..511 (VPN[2] >= 256) cover the upper + * half of the 64-bit VA space where the kernel lives. + * Without these, switching to the new page table would + * immediately fault on any kernel VA access. + * + * Only do this after the runtime page table is active — + * during early boot the bootstrap PT doesn't cover all of + * RAM so pa_to_ptr on the kernel root would fault. */ + if (g_runtime_mapping_ready) { + uint64 kernel_satp; + asm volatile("csrr %0, satp" : "=r"(kernel_satp)); + uint64 kernel_root_pa = (kernel_satp & kSatpPpnMask) << kPageShift; + + uint64 *new_l2 = pa_to_ptr(new_root); + uint64 *kern_l2 = pa_to_ptr(kernel_root_pa); + + for (int i = kKernelPgdStart; i < kPtePerPage; i++) { + new_l2[i] = kern_l2[i]; + } + } + + return new_root; +} + +void destroy(uint64 root_pa) +{ + uint64 *l2 = pa_to_ptr(root_pa); + + /* Only free the user half (entries 0..255). + * Entries 256..511 are shared kernel PGD pointers copied + * from the kernel page table — freeing them would corrupt + * the kernel's own mappings. */ + for (int i = 0; i < kKernelPgdStart; i++) { + if (!(l2[i] & PTE_V)) + continue; + /* Non-leaf L2 entry? (no R/W/X bits = pointer to L1 table) */ + if (!(l2[i] & (PTE_R | PTE_W | PTE_X))) { + uint64 l1_pa = pte_to_pa(l2[i]); + uint64 *l1 = pa_to_ptr(l1_pa); + for (int j = 0; j < kPtePerPage; j++) { + if (!(l1[j] & PTE_V)) + continue; + if (!(l1[j] & (PTE_R | PTE_W | PTE_X))) { + /* L0 page table */ + uint64 l0_pa = pte_to_pa(l1[j]); + pmm_free_page(pa_to_page(phys_addr(l0_pa))); + } + } + pmm_free_page(pa_to_page(phys_addr(l1_pa))); + } + } + pmm_free_page(pa_to_page(phys_addr(root_pa))); +} + +int map(uint64 root_pa, uint64 va, uint64 pa, uint64 perm) +{ + uint64 *table = pa_to_ptr(root_pa); + + /* Walk levels 2 -> 1, creating intermediate tables as needed */ + for (int level = 2; level > 0; level--) { + uint64 idx = vpn(va, level); + if (!(table[idx] & PTE_V)) { + uint64 child_pa = alloc_pt_page(); + if (!child_pa) + return -ENOMEM; /* -ENOMEM */ + table[idx] = pa_to_pte(child_pa, PTE_V); + } + table = pa_to_ptr(pte_to_pa(table[idx])); + } + + /* Level 0: install the leaf PTE */ + uint64 idx = vpn(va, 0); + table[idx] = pa_to_pte(pa, perm | PTE_V | PTE_A | PTE_D); + return 0; +} + +int map_2mb(uint64 root_pa, uint64 va, uint64 pa, uint64 perm) +{ + /* VA and PA must be 2MB-aligned */ + constexpr uint64 k2MB = 2ULL * 1024 * 1024; + if ((va & (k2MB - 1)) != 0 || (pa & (k2MB - 1)) != 0) { + return -EINVAL; /* -EINVAL */ + } + + uint64 *table = pa_to_ptr(root_pa); + + /* Walk L2: create intermediate table if needed */ + uint64 idx2 = vpn(va, 2); + if (!(table[idx2] & PTE_V)) { + uint64 child_pa = alloc_pt_page(); + if (!child_pa) { + return -ENOMEM; /* -ENOMEM */ + } + table[idx2] = pa_to_pte(child_pa, PTE_V); + } + table = pa_to_ptr(pte_to_pa(table[idx2])); + + /* L1: install leaf PTE (2MB mega page) */ + uint64 idx1 = vpn(va, 1); + table[idx1] = pa_to_pte(pa, perm | PTE_V | PTE_A | PTE_D); + return 0; +} + +void unmap(uint64 root_pa, uint64 va) +{ + uint64 *table = pa_to_ptr(root_pa); + for (int level = 2; level > 0; level--) { + uint64 idx = vpn(va, level); + if (!(table[idx] & PTE_V)) + return; + table = pa_to_ptr(pte_to_pa(table[idx])); + } + table[vpn(va, 0)] = 0; +} + +int protect(uint64 root_pa, uint64 va, uint64 perm) +{ + uint64 *table = pa_to_ptr(root_pa); + for (int level = 2; level >= 0; level--) { + uint64 idx = vpn(va, level); + uint64 pte = table[idx]; + if (!(pte & PTE_V)) + return -EFAULT; /* -EFAULT */ + if (pte & (PTE_R | PTE_W | PTE_X)) { + uint64 pa = pte_to_pa(pte); + table[idx] = pa_to_pte(pa, perm | PTE_V); + return 0; + } + table = pa_to_ptr(pte_to_pa(pte)); + } + return -EFAULT; +} + +int protect_range(uint64 root_pa, uint64 start, uint64 end, uint64 perm) +{ + if (end <= start) { + return 0; + } + + uint64 va = page_align_down(start); + uint64 va_end = page_align_up(end); + + for (; va < va_end; va += kPageSize) { + int ret = protect(root_pa, va, perm); + if (ret == -14) { + continue; /* unmapped page: skip */ + } + if (ret < 0) { + return ret; + } + } + + arch::mmu::flush_tlb_all(); + return 0; +} + +bool lookup(uint64 root_pa, uint64 va, uint64 *pa_out) +{ + uint64 *table = pa_to_ptr(root_pa); + for (int level = 2; level >= 0; level--) { + uint64 idx = vpn(va, level); + uint64 pte = table[idx]; + if (!(pte & PTE_V)) + return false; + /* Leaf PTE? (has R, W, or X) */ + if (pte & (PTE_R | PTE_W | PTE_X)) { + if (pa_out) + *pa_out = pte_to_pa(pte); + return true; + } + table = pa_to_ptr(pte_to_pa(pte)); + } + return false; +} + +uint64 lookup_pte(uint64 root_pa, uint64 va) +{ + uint64 *table = pa_to_ptr(root_pa); + for (int level = 2; level >= 0; level--) { + uint64 idx = vpn(va, level); + uint64 pte = table[idx]; + if (!(pte & PTE_V)) + return 0; + if (pte & (PTE_R | PTE_W | PTE_X)) + return pte; // leaf PTE raw value + table = pa_to_ptr(pte_to_pa(pte)); + } + return 0; +} + +bool is_cow(uint64 root_pa, uint64 va) +{ + uint64 *table = pa_to_ptr(root_pa); + for (int level = 2; level >= 0; level--) { + uint64 idx = vpn(va, level); + uint64 pte = table[idx]; + if (!(pte & PTE_V)) + return false; + if (pte & (PTE_R | PTE_W | PTE_X)) { + return (pte & PTE_COW) != 0; + } + table = pa_to_ptr(pte_to_pa(pte)); + } + return false; +} + +int make_cow(uint64 root_pa, uint64 va) +{ + uint64 *table = pa_to_ptr(root_pa); + for (int level = 2; level >= 0; level--) { + uint64 idx = vpn(va, level); + uint64 pte = table[idx]; + if (!(pte & PTE_V)) + return -EFAULT; /* -EFAULT */ + if (pte & (PTE_R | PTE_W | PTE_X)) { + /* Clear write bit, set COW bit */ + pte &= ~PTE_W; + pte |= PTE_COW; + table[idx] = pte; + return 0; + } + table = pa_to_ptr(pte_to_pa(pte)); + } + return -EFAULT; +} + +void activate(uint64 root_pa) +{ + arch::mmu::set_page_table(root_pa); + arch::mmu::flush_tlb_all(); + g_runtime_mapping_ready = true; +} + +uint64 vm_perm_to_pte(uint64 vm_perm) +{ + /* Translate MI VM_* flags to SV39 PTE bits. + * The MI flags are defined in . */ + uint64 pte = PTE_V | PTE_A | PTE_D; // Valid + Accessed + Dirty (avoid HW A/D faults) + if (vm_perm & (1ULL << 0)) pte |= PTE_R; // VM_READ + if (vm_perm & (1ULL << 1)) pte |= PTE_W; // VM_WRITE + if (vm_perm & (1ULL << 2)) pte |= PTE_X; // VM_EXEC + if (vm_perm & (1ULL << 3)) pte |= PTE_U; // VM_USER + return pte; +} + +} // namespace pmap diff --git a/arch/riscv/mm/tlb_md.cc b/arch/riscv/mm/tlb_md.cc new file mode 100644 index 0000000..ed9aa29 --- /dev/null +++ b/arch/riscv/mm/tlb_md.cc @@ -0,0 +1,17 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* Copyright (C) 2026 Richard Qin */ + +#include +#include + +namespace arch::tlb { + +void flush_remote(uint64 cpu_mask, uint64 start, uint64 size) { + if (size == ~0ULL) { + arch::sbi::sbi_remote_sfence_vma(cpu_mask, 0, 0, ~0UL); + } else { + arch::sbi::sbi_remote_sfence_vma(cpu_mask, 0, start, size); + } +} + +} // namespace arch::tlb diff --git a/arch/riscv/module.mk b/arch/riscv/module.mk new file mode 100644 index 0000000..97d22de --- /dev/null +++ b/arch/riscv/module.mk @@ -0,0 +1,18 @@ +# arch/riscv/module.mk +SRCS_CC += arch/riscv/mm/pmap.cc +SRCS_CC += arch/riscv/mm/init.cc +SRCS_CC += arch/riscv/config.cc +SRCS_CC += arch/riscv/trap/trap_md.cc +SRCS_CC += arch/riscv/syscall/syscall_md.cc +SRCS_CC += arch/riscv/lib/random.cc +SRCS_CC += arch/riscv/mm/tlb_md.cc +SRCS_S += arch/riscv/kernel/head.S +SRCS_S += arch/riscv/trap/swtch.S +SRCS_S += arch/riscv/trap/fpu.S +SRCS_S += arch/riscv/trap/trap_vector.S +SRCS_S += arch/riscv/uaccess/uaccess_memcpy.S +SRCS_CC += arch/riscv/trap/extable.cc +SRCS_CC += arch/riscv/kernel/signal.cc + +# Arch-specific headers (added to include path) +INCLUDES += -Iarch/riscv/include diff --git a/arch/riscv/syscall/syscall_md.cc b/arch/riscv/syscall/syscall_md.cc new file mode 100644 index 0000000..26320c8 --- /dev/null +++ b/arch/riscv/syscall/syscall_md.cc @@ -0,0 +1,42 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* Copyright (C) 2026 Richard Qin */ + +/* + * Syscall MD Layer — RISC-V Register Extraction + * + * Extracts syscall number (a7) and arguments (a0-a5) from the TrapFrame, + * calls the MI dispatcher, and writes the return value back to a0. + * + * Reference: docs/specs/syscall.md §5 + */ + +#include +#include +#include +#include + +void syscall_dispatch_md(arch::TrapFrame* tf) +{ + uint64 nr = tf->a[7]; /* RISC-V ABI: syscall number in a7 */ + + uint64 args[6] = { + tf->a[0], tf->a[1], tf->a[2], + tf->a[3], tf->a[4], tf->a[5], + }; + + /* Enable interrupts so timer and preemption work during syscalls */ + arch::cpu::intr_on(); + + int64 ret = syscall_dispatch(nr, args); + + arch::cpu::intr_off(); + + /* sys_rt_sigreturn restores the entire TrapFrame from the signal + * frame on the user stack. Writing ret into a[0] would clobber + * the restored value, so we must skip the writeback. */ + if (nr == static_cast(SYS_rt_sigreturn)){ + return; + } + tf->a[0] = static_cast(ret); /* Return value in a0 */ +} + diff --git a/arch/riscv/trap/extable.cc b/arch/riscv/trap/extable.cc new file mode 100644 index 0000000..57a3dc1 --- /dev/null +++ b/arch/riscv/trap/extable.cc @@ -0,0 +1,92 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* Copyright (C) 2026 Richard Qin */ + +#include +#include + +extern struct ExceptionTableEntry __extable_start[]; +extern struct ExceptionTableEntry __extable_end[]; + +static void swap_extable_entries(struct ExceptionTableEntry *a, struct ExceptionTableEntry *b) { + uint64 a_insn = reinterpret_cast(&a->insn) + a->insn; + uint64 a_fixup = reinterpret_cast(&a->fixup) + a->fixup; + uint64 b_insn = reinterpret_cast(&b->insn) + b->insn; + uint64 b_fixup = reinterpret_cast(&b->fixup) + b->fixup; + + a->insn = static_cast(b_insn - reinterpret_cast(&a->insn)); + a->fixup = static_cast(b_fixup - reinterpret_cast(&a->fixup)); + b->insn = static_cast(a_insn - reinterpret_cast(&b->insn)); + b->fixup = static_cast(a_fixup - reinterpret_cast(&b->fixup)); +} + +static void sort_extable(size_t left, size_t right) { + if (left >= right) return; + + size_t mid_idx = left + (right - left) / 2; + struct ExceptionTableEntry *pivot = &__extable_start[mid_idx]; + uint64 pivot_val = reinterpret_cast(&pivot->insn) + pivot->insn; + + size_t i = left; + size_t j = right; + + while (i <= j) { + while ((reinterpret_cast(&__extable_start[i].insn) + __extable_start[i].insn) < pivot_val) { + i++; + } + while ((reinterpret_cast(&__extable_start[j].insn) + __extable_start[j].insn) > pivot_val) { + if (j == 0) { + break; + } + j--; + } + if (i <= j) { + swap_extable_entries(&__extable_start[i], &__extable_start[j]); + i++; + if (j == 0) { + break; + } + j--; + } + } + + if (j > left) { + sort_extable(left, j); + } + if (i < right) { + sort_extable(i, right); + } +} + +void extable_init() { + size_t count = __extable_end - __extable_start; + if (count <= 1) return; + sort_extable(0, count - 1); +} + +uint64 search_extable(uint64 pc) { + size_t count = __extable_end - __extable_start; + if (count == 0) return 0; + + size_t left = 0; + size_t right = count - 1; + + /* Binary search */ + while (left <= right) { + size_t mid = left + (right - left) / 2; + const struct ExceptionTableEntry *entry = &__extable_start[mid]; + uint64 mid_pc = reinterpret_cast(&entry->insn) + entry->insn; + + if (mid_pc == pc) { + return reinterpret_cast(&entry->fixup) + entry->fixup; + } + + if (mid_pc < pc) { + left = mid + 1; + } else { + if (mid == 0) break; + right = mid - 1; + } + } + + return 0; +} diff --git a/arch/riscv/trap/fpu.S b/arch/riscv/trap/fpu.S new file mode 100644 index 0000000..90d96d9 --- /dev/null +++ b/arch/riscv/trap/fpu.S @@ -0,0 +1,103 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* Copyright (C) 2026 Richard Qin */ + +/* + * FPU Save / Restore (RISC-V D extension) + * + * void fpu_save(FpState* state); + * Saves f0-f31 and fcsr to the FpState structure. + * + * void fpu_restore(FpState* state); + * Restores f0-f31 and fcsr from the FpState structure. + * + * FpState layout: f[0..31] at offset 0 (each 8 bytes), fcsr at offset 256. + * + * Called from schedule() when switching between tasks that have + * used the FPU (sstatus.FS == Dirty on save path). + */ + + .section .text + +/* ====================================================================== + * void fpu_save(FpState* state); // a0 = state pointer + * ====================================================================== */ + .globl fpu_save + .align 2 +fpu_save: + fsd f0, 0(a0) + fsd f1, 8(a0) + fsd f2, 16(a0) + fsd f3, 24(a0) + fsd f4, 32(a0) + fsd f5, 40(a0) + fsd f6, 48(a0) + fsd f7, 56(a0) + fsd f8, 64(a0) + fsd f9, 72(a0) + fsd f10, 80(a0) + fsd f11, 88(a0) + fsd f12, 96(a0) + fsd f13, 104(a0) + fsd f14, 112(a0) + fsd f15, 120(a0) + fsd f16, 128(a0) + fsd f17, 136(a0) + fsd f18, 144(a0) + fsd f19, 152(a0) + fsd f20, 160(a0) + fsd f21, 168(a0) + fsd f22, 176(a0) + fsd f23, 184(a0) + fsd f24, 192(a0) + fsd f25, 200(a0) + fsd f26, 208(a0) + fsd f27, 216(a0) + fsd f28, 224(a0) + fsd f29, 232(a0) + fsd f30, 240(a0) + fsd f31, 248(a0) + frcsr t0 + sw t0, 256(a0) + ret + +/* ====================================================================== + * void fpu_restore(FpState* state); // a0 = state pointer + * ====================================================================== */ + .globl fpu_restore + .align 2 +fpu_restore: + fld f0, 0(a0) + fld f1, 8(a0) + fld f2, 16(a0) + fld f3, 24(a0) + fld f4, 32(a0) + fld f5, 40(a0) + fld f6, 48(a0) + fld f7, 56(a0) + fld f8, 64(a0) + fld f9, 72(a0) + fld f10, 80(a0) + fld f11, 88(a0) + fld f12, 96(a0) + fld f13, 104(a0) + fld f14, 112(a0) + fld f15, 120(a0) + fld f16, 128(a0) + fld f17, 136(a0) + fld f18, 144(a0) + fld f19, 152(a0) + fld f20, 160(a0) + fld f21, 168(a0) + fld f22, 176(a0) + fld f23, 184(a0) + fld f24, 192(a0) + fld f25, 200(a0) + fld f26, 208(a0) + fld f27, 216(a0) + fld f28, 224(a0) + fld f29, 232(a0) + fld f30, 240(a0) + fld f31, 248(a0) + lw t0, 256(a0) + fscsr t0 + ret diff --git a/arch/riscv/trap/swtch.S b/arch/riscv/trap/swtch.S new file mode 100644 index 0000000..dbae43d --- /dev/null +++ b/arch/riscv/trap/swtch.S @@ -0,0 +1,96 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* Copyright (C) 2026 Richard Qin */ + +/* + * Context Switch & Kernel Thread Trampoline (RISC-V 64) + * + * context_switch: saves/restores callee-saved registers. + * kthread_entry_trampoline: entry point for newly created kernel threads. + * After context_switch loads the new Context, execution lands here via + * Context.ra = kthread_entry_trampoline. The entry function and argument + * are in s0 and s1 respectively, set by sched_init/create_tcb. + * + * The trampoline calls the MI-layer kthread_entry(entry, arg). + * + * Reference: docs/specs/scheduler.md §6 + */ + + .section .text + +/* ======================================================================== + * void context_switch(Context* old_ctx, Context* next_ctx); + * a0 = old_ctx_ptr, a1 = next_ctx_ptr + * ======================================================================== */ + .globl context_switch + .align 2 +context_switch: + sd ra, 0(a0) + sd sp, 8(a0) + sd s0, 16(a0) + sd s1, 24(a0) + sd s2, 32(a0) + sd s3, 40(a0) + sd s4, 48(a0) + sd s5, 56(a0) + sd s6, 64(a0) + sd s7, 72(a0) + sd s8, 80(a0) + sd s9, 88(a0) + sd s10, 96(a0) + sd s11, 104(a0) + + ld ra, 0(a1) + ld sp, 8(a1) + ld s0, 16(a1) + ld s1, 24(a1) + ld s2, 32(a1) + ld s3, 40(a1) + ld s4, 48(a1) + ld s5, 56(a1) + ld s6, 64(a1) + ld s7, 72(a1) + ld s8, 80(a1) + ld s9, 88(a1) + ld s10, 96(a1) + ld s11, 104(a1) + + ret + +/* ======================================================================== + * kthread_entry_trampoline + * + * Reached via context_switch when a new kthread is first scheduled. + * Context was set up by create_tcb(): + * s0 = entry function pointer + * s1 = argument pointer + * + * Calls the MI-layer C++ function: + * void kthread_entry(void (*entry)(void*), void* arg); + * ======================================================================== */ + .globl kthread_entry_trampoline + .align 2 +kthread_entry_trampoline: + mv a0, s0 /* a0 = entry function */ + mv a1, s1 /* a1 = argument */ + call kthread_entry /* MI-layer C++ function */ + + /* kthread_entry should never return, but if it does: */ + wfi + j kthread_entry_trampoline + 12 /* loop on wfi */ + +/* ======================================================================== + * ret_from_fork + * + * Trampoline for newly forked processes. + * Context is set up by fork() so that s0 contains the TrapFrame pointer. + * + * Calls the MI-layer C++ function: + * void fork_child_entry(TrapFrame* tf); + * ======================================================================== */ + .globl ret_from_fork + .align 2 +ret_from_fork: + mv a0, s0 /* a0 = TrapFrame pointer */ + call fork_child_entry /* does not return, eventually calls userret */ + wfi + j ret_from_fork + 8 \ No newline at end of file diff --git a/arch/riscv/trap/trap_md.cc b/arch/riscv/trap/trap_md.cc new file mode 100644 index 0000000..e30aea6 --- /dev/null +++ b/arch/riscv/trap/trap_md.cc @@ -0,0 +1,241 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* Copyright (C) 2026 Richard Qin */ + +/* + * Trap MD Layer (RISC-V 64) + * + * All architecture-specific trap logic lives here: + * - scause/stval reading (CSR access) + * - scause → MI TrapCause translation + * - RISC-V specific fixups (ecall epc += 4, ebreak epc += 2) + * - stvec setup (trap_init / trap_init_ap) + * - kernel_trap_handler (traps from S-mode) + * + * MI layer (kernel/trap.cc) contains ZERO arch-specific code. + * + * Reference: docs/specs/trap.md §7 + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* PLIC interface (drivers/plic.cc) */ +uint32 plic_claim(); +void plic_complete(uint32 irq); + +/* trap_vector.S: assembly entry point */ +extern "C" void trap_vector(); + +/* ======================================================================== + * scause → TrapCause translation (RISC-V specific) + * ======================================================================== */ + +static TrapCause scause_to_trap_cause(uint64 scause) +{ + if (arch::trap::is_interrupt(scause)) { + switch (arch::trap::cause_code(scause)) { + case IRQ_S_TIMER: return TrapCause::IrqTimer; + case IRQ_S_SOFTWARE: return TrapCause::IrqSoftware; + case IRQ_S_EXTERNAL: return TrapCause::IrqExternal; + default: return TrapCause::Unknown; + } + } + + /* Exception (synchronous) */ + switch (scause) { + case EXC_INST_PAGE_FAULT: return TrapCause::PageFaultExec; + case EXC_LOAD_PAGE_FAULT: return TrapCause::PageFaultLoad; + case EXC_STORE_PAGE_FAULT: return TrapCause::PageFaultStore; + case EXC_ECALL_FROM_U: return TrapCause::Syscall; + case EXC_ILLEGAL_INST: return TrapCause::IllegalInst; + case EXC_BREAKPOINT: return TrapCause::Breakpoint; + + case EXC_INST_ACCESS_FAULT: + case EXC_LOAD_ACCESS_FAULT: + case EXC_STORE_ACCESS_FAULT: + return TrapCause::AccessFault; + + case EXC_INST_MISALIGNED: + case EXC_LOAD_MISALIGNED: + case EXC_STORE_MISALIGNED: + return TrapCause::AlignFault; + + default: + return TrapCause::Unknown; + } +} + +TrapCause trap_translate_scause(uint64 scause) +{ + return scause_to_trap_cause(scause); +} + +static TrapOrigin origin_from_sstatus(uint64 sstatus) +{ + constexpr uint64 kSstatusSPP = (1ULL << 8); + return (sstatus & kSstatusSPP) ? TrapOrigin::FromKernel + : TrapOrigin::FromUser; +} + +static TrapClass classify_trap(uint64 scause, TrapCause cause) +{ + if (arch::trap::is_interrupt(scause)) { + return TrapClass::Interrupt; + } + if (cause == TrapCause::Syscall) { + return TrapClass::Syscall; + } + if (cause == TrapCause::Unknown) { + return TrapClass::Unknown; + } + return TrapClass::Exception; +} + +static TrapInfo build_trap_info(arch::TrapFrame* tf, + uint64 scause, + uint64 stval, + TrapOrigin origin) +{ + TrapInfo info{}; + info.cause = scause_to_trap_cause(scause); + info.cls = classify_trap(scause, info.cause); + info.origin = origin; + info.val = stval; + info.epc = tf ? tf->epc : 0; + info.scause = scause; + return info; +} + +/* ======================================================================== + * trap_dispatch — user-mode trap entry (called from trap_vector.S) + * + * Reads RISC-V CSRs, translates to MI TrapCause, applies arch-specific + * fixups, then calls MI trap_handle(). + * + * After trap_handle returns, we must return to user mode via userret(). + * The assembly jalr t3 that called us has NO return-to-user code after it, + * so trap_dispatch must never return normally. + * ======================================================================== */ + +extern "C" void userret(arch::TrapFrame* tf); + +extern "C" [[noreturn]] void trap_dispatch(arch::TrapFrame* tf) +{ + /* Switch stvec to kernelvec immediately so that any traps during + * kernel execution (timer IRQs, page faults from copy_from_user) + * go through the kernel trap entry, not the user trap entry. + * This matches Linux which sets stvec to handle_exception (kernel) + * at the top of the user trap path. */ + extern void kernelvec(); + arch::trap::set_vector(reinterpret_cast(&kernelvec)); + + /* Enable SUM so S-mode can access user pages (PTE_U). + * Without this, any dereference of user-space pointers in syscall + * handlers (sys_write buf, sys_read buf, etc.) would page fault. */ + arch::trap::enable_sum(); + + uint64 scause = arch::trap::get_cause(); + uint64 stval = arch::trap::get_value(); + + TrapOrigin origin = origin_from_sstatus(tf->sstatus); + TrapInfo info = build_trap_info(tf, scause, stval, origin); + + /* Per-trap trace from user mode. Stored at KLOG_DEBUG / KLOG_SUBSYS_TRAP + * so it lives in the ring buffer for postmortem but stays off the console + * by default. No more in-band 20-trap window — flip console_level or + * dump the ring on demand instead. */ + if (origin == TrapOrigin::FromUser) { + pr_debug_s(KLOG_SUBSYS_TRAP, + "trap: cause=%u epc=%llx val=%llx\n", + static_cast(info.cause), info.epc, info.val); + } + + /* Save trapframe in TCB for syscall handlers (like fork) */ + TaskControlBlock* curr = current_task(); + if (curr) { + curr->trapframe = tf; + } + + /* RISC-V ecall fix: ecall does not increment sepc. + * Advance epc so sret returns to the instruction AFTER ecall. */ + if (info.cause == TrapCause::Syscall) { + tf->epc += 4; + info.epc = tf->epc; + } + + /* RISC-V ebreak fix: advance past the C.EBREAK instruction (2 bytes). + * This is arch-specific because instruction size depends on the ISA. */ + if (info.cause == TrapCause::Breakpoint) { + tf->epc += 2; + info.epc = tf->epc; + } + + /* Call MI-layer dispatcher (zero inline assembly from here on) */ + trap_handle(tf, info); + + /* Return to user mode. + * Set stvec back to trap_vector for the next user-mode trap, + * then jump to userret which restores registers and does sret. */ + arch::trap::set_vector(reinterpret_cast(&trap_vector)); + userret(tf); + __builtin_unreachable(); +} + +/* ======================================================================== + * kernel_trap_handler — traps taken while already in S-mode + * + * For Phase 2, only timer and external interrupts are expected. + * Exceptions in S-mode are always fatal. + * + * This belongs in MD because it reads scause directly. + * ======================================================================== */ + +extern "C" void kernel_trap_handler(arch::TrapFrame* tf) +{ + uint64 scause = arch::trap::get_cause(); + uint64 stval = arch::trap::get_value(); + + if (scause == EXC_LOAD_PAGE_FAULT || scause == EXC_STORE_PAGE_FAULT) { + uint64 fixup = search_extable(tf->epc); + if (fixup != 0) { + /* Exception table entry found: redirect execution to fixup code */ + tf->epc = fixup; + return; + } + } + + TrapInfo info = build_trap_info(tf, scause, stval, TrapOrigin::FromKernel); + trap_handle(tf, info); +} + +/* kernelvec assembly is in trap_vector.S */ +extern "C" void kernelvec(); + +/* ======================================================================== + * trap_init() — BSP: set stvec, init IrqManager + * ======================================================================== */ + +void trap_init() +{ + g_irq_manager.init(); + arch::trap::set_vector(reinterpret_cast(&kernelvec)); +} + +/* ======================================================================== + * trap_init_ap() — AP: set this CPU's stvec + * ======================================================================== */ + +void trap_init_ap() +{ + arch::trap::set_vector(reinterpret_cast(&kernelvec)); +} + diff --git a/arch/riscv/trap/trap_vector.S b/arch/riscv/trap/trap_vector.S new file mode 100644 index 0000000..8e45cae --- /dev/null +++ b/arch/riscv/trap/trap_vector.S @@ -0,0 +1,283 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* Copyright (C) 2026 Richard Qin */ + +/* + * Trap Vector Entry / Exit (RISC-V 64, S-mode) + * + * On trap entry (interrupt / exception / ecall): + * sscratch holds the TrapFrame pointer (set up by kernel before sret). + * We swap sscratch <-> tp, save all 31 GPRs + sstatus + sepc into TrapFrame, + * then load kernel metadata (sp, satp, tp, trap handler) and jump to C++. + * + * On trap return (userret): + * Restore all GPRs + sstatus + sepc from TrapFrame, swap sscratch <-> tp, + * then sret back to user mode. + * + * Layout must match arch::TrapFrame in arch/riscv/include/arch/trapframe.h + * Reference: docs/specs/arch_abstraction.md §7.2 + */ + +#include + + .section .text + .globl trap_vector + .globl userret + .align 4 + +/* + * trap_vector — unified S-mode trap entry point (Direct mode) + * + * Precondition: sscratch = pointer to current task's TrapFrame + */ +trap_vector: + /* Step 1: swap sscratch <-> tp. Now tp = TrapFrame*, sscratch = old tp */ + csrrw tp, sscratch, tp + + /* Step 2: save all 31 general-purpose registers into TrapFrame */ + sd ra, TF_RA(tp) + sd sp, TF_SP(tp) + sd gp, TF_GP(tp) + /* tp (x4) is in sscratch — save it later */ + sd t0, TF_T0(tp) + sd t1, TF_T1(tp) + sd t2, TF_T2(tp) + sd s0, TF_S0(tp) + sd s1, TF_S1(tp) + sd a0, TF_A0(tp) + sd a1, TF_A1(tp) + sd a2, TF_A2(tp) + sd a3, TF_A3(tp) + sd a4, TF_A4(tp) + sd a5, TF_A5(tp) + sd a6, TF_A6(tp) + sd a7, TF_A7(tp) + sd s2, TF_S2(tp) + sd s3, TF_S3(tp) + sd s4, TF_S4(tp) + sd s5, TF_S5(tp) + sd s6, TF_S6(tp) + sd s7, TF_S7(tp) + sd s8, TF_S8(tp) + sd s9, TF_S9(tp) + sd s10, TF_S10(tp) + sd s11, TF_S11(tp) + sd t3, TF_T3(tp) + sd t4, TF_T4(tp) + sd t5, TF_T5(tp) + sd t6, TF_T6(tp) + + /* Save the real tp (user's tp) which is now in sscratch */ + csrr t0, sscratch + sd t0, TF_TP(tp) + + /* Step 3: save sstatus */ + csrr t0, sstatus + sd t0, TF_SSTATUS(tp) + + /* Step 4: save sepc (interrupted PC) */ + csrr t0, sepc + sd t0, TF_EPC(tp) + + /* Step 5: load kernel context from TrapFrame metadata */ + ld sp, TF_KERNEL_SP(tp) /* switch to kernel stack */ + ld t1, TF_KERNEL_SATP(tp) /* kernel page table */ + ld t2, TF_KERNEL_CPUID(tp) /* CPU ID for tp */ + ld t3, TF_KERNEL_TRAP(tp) /* C++ trap_handler address */ + + /* Switch to kernel page table */ + csrw satp, t1 + + /* Step 6: flush TLB after page table switch */ + sfence.vma zero, zero + + /* Step 6.5: restore kernel gp (global pointer). + * The compiler uses gp-relative addressing for global variable access + * (e.g. ld reg, offset(gp)). User mode may have set gp to anything, + * so we MUST reload it before entering any C++ code. */ +.option push +.option norelax + la gp, __global_pointer$ +.option pop + + /* Pass TrapFrame* to C++ handler (a0 = tf) */ + mv a0, tp + + /* Set tp = CPU ID (kernel convention) */ + mv tp, t2 + + /* Step 7: jump to C++ trap_handler */ + jalr t3 + + +/* ======================================================================== + * kernelvec — unified S-mode trap entry point for traps FROM S-MODE + * + * Traps from S-mode already have a valid kernel stack pointer in sp. + * We just allocate a TrapFrame on the stack and save registers. + * ======================================================================== */ + .globl kernelvec + .align 4 +kernelvec: + /* Allocate TrapFrame on stack */ + addi sp, sp, -296 + + /* Save all 31 general-purpose registers into TrapFrame */ + sd ra, TF_RA(sp) + /* Save the OLD sp (before we subtracted 296) */ + addi t0, sp, 296 + sd t0, TF_SP(sp) + sd gp, TF_GP(sp) + sd tp, TF_TP(sp) + sd t0, TF_T0(sp) + sd t1, TF_T1(sp) + sd t2, TF_T2(sp) + sd s0, TF_S0(sp) + sd s1, TF_S1(sp) + sd a0, TF_A0(sp) + sd a1, TF_A1(sp) + sd a2, TF_A2(sp) + sd a3, TF_A3(sp) + sd a4, TF_A4(sp) + sd a5, TF_A5(sp) + sd a6, TF_A6(sp) + sd a7, TF_A7(sp) + sd s2, TF_S2(sp) + sd s3, TF_S3(sp) + sd s4, TF_S4(sp) + sd s5, TF_S5(sp) + sd s6, TF_S6(sp) + sd s7, TF_S7(sp) + sd s8, TF_S8(sp) + sd s9, TF_S9(sp) + sd s10, TF_S10(sp) + sd s11, TF_S11(sp) + sd t3, TF_T3(sp) + sd t4, TF_T4(sp) + sd t5, TF_T5(sp) + sd t6, TF_T6(sp) + + /* Save sstatus and sepc */ + csrr t0, sstatus + sd t0, TF_SSTATUS(sp) + csrr t0, sepc + sd t0, TF_EPC(sp) + + /* Call C++ kernel_trap_handler(TrapFrame* tf) */ + mv a0, sp + call kernel_trap_handler + + /* Restore sstatus and sepc */ + ld t0, TF_SSTATUS(sp) + csrw sstatus, t0 + ld t0, TF_EPC(sp) + csrw sepc, t0 + + /* Restore all 31 general-purpose registers */ + ld ra, TF_RA(sp) + ld gp, TF_GP(sp) + ld tp, TF_TP(sp) + ld t0, TF_T0(sp) + ld t1, TF_T1(sp) + ld t2, TF_T2(sp) + ld s0, TF_S0(sp) + ld s1, TF_S1(sp) + ld a0, TF_A0(sp) + ld a1, TF_A1(sp) + ld a2, TF_A2(sp) + ld a3, TF_A3(sp) + ld a4, TF_A4(sp) + ld a5, TF_A5(sp) + ld a6, TF_A6(sp) + ld a7, TF_A7(sp) + ld s2, TF_S2(sp) + ld s3, TF_S3(sp) + ld s4, TF_S4(sp) + ld s5, TF_S5(sp) + ld s6, TF_S6(sp) + ld s7, TF_S7(sp) + ld s8, TF_S8(sp) + ld s9, TF_S9(sp) + ld s10, TF_S10(sp) + ld s11, TF_S11(sp) + ld t3, TF_T3(sp) + ld t4, TF_T4(sp) + ld t5, TF_T5(sp) + ld t6, TF_T6(sp) + + /* Restore sp last */ + ld sp, TF_SP(sp) + + sret + +/* + * userret — return to user mode + * + * Called from C++ as: userret(TrapFrame* tf) + * a0 = pointer to TrapFrame (with user satp already written by caller) + * + * Precondition: interrupts must be disabled by caller. + */ +userret: + /* a0 = TrapFrame pointer, use it as base */ + + /* Step 1: write back epc -> sepc */ + ld t0, TF_EPC(a0) + csrw sepc, t0 + + /* Step 2: write back sstatus */ + ld t0, TF_SSTATUS(a0) + csrw sstatus, t0 + + /* Step 3: caller is responsible for switching satp to user page table + * and flushing TLB before calling userret. */ + + /* Store TrapFrame address into sscratch for next trap entry */ + csrw sscratch, a0 + + /* Use a0 as base to restore all registers */ + mv tp, a0 + + /* Step 4: restore all 31 general-purpose registers */ + ld ra, TF_RA(tp) + ld sp, TF_SP(tp) + ld gp, TF_GP(tp) + /* defer tp restore — we're using it as base */ + ld t0, TF_T0(tp) + ld t1, TF_T1(tp) + ld t2, TF_T2(tp) + ld s0, TF_S0(tp) + ld s1, TF_S1(tp) + ld a0, TF_A0(tp) + ld a1, TF_A1(tp) + ld a2, TF_A2(tp) + ld a3, TF_A3(tp) + ld a4, TF_A4(tp) + ld a5, TF_A5(tp) + ld a6, TF_A6(tp) + ld a7, TF_A7(tp) + ld s2, TF_S2(tp) + ld s3, TF_S3(tp) + ld s4, TF_S4(tp) + ld s5, TF_S5(tp) + ld s6, TF_S6(tp) + ld s7, TF_S7(tp) + ld s8, TF_S8(tp) + ld s9, TF_S9(tp) + ld s10, TF_S10(tp) + ld s11, TF_S11(tp) + ld t3, TF_T3(tp) + ld t4, TF_T4(tp) + ld t5, TF_T5(tp) + ld t6, TF_T6(tp) + + /* Step 5: restore user tp (swap sscratch <-> tp) */ + /* sscratch currently = TrapFrame address, tp = TrapFrame address */ + /* We need: tp = user's tp, sscratch = TrapFrame address */ + /* Load user tp from TrapFrame first, but we can't use any temp reg now. + * Use the sscratch swap trick: */ + ld tp, TF_TP(tp) + + /* sscratch already holds TrapFrame address (set above with csrw) */ + + /* Step 6: sret — return to user mode */ + sret diff --git a/arch/riscv/uaccess/uaccess_memcpy.S b/arch/riscv/uaccess/uaccess_memcpy.S new file mode 100644 index 0000000..615680d --- /dev/null +++ b/arch/riscv/uaccess/uaccess_memcpy.S @@ -0,0 +1,117 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* Copyright (C) 2026 Richard Qin */ + +#include + +/* + * extern "C" int __uaccess_memcpy(void* dst, const void* src, uint64 len); + * + * a0: dst + * a1: src + * a2: len + * Returns: 0 on success, -EFAULT (-14) on page fault. + */ +.global __uaccess_memcpy +.type __uaccess_memcpy, @function +__uaccess_memcpy: + /* If len == 0, return 0 */ + beqz a2, .Lmemcpy_done + + /* Enable SUM (Supervisor User Memory) bit: sstatus.SUM is bit 18 */ + li t0, (1 << 18) + csrs sstatus, t0 + +.Lmemcpy_loop: + /* Load byte from src */ +1: lb t1, 0(a1) + /* Store byte to dst */ +2: sb t1, 0(a0) + + addi a0, a0, 1 + addi a1, a1, 1 + addi a2, a2, -1 + bnez a2, .Lmemcpy_loop + +.Lmemcpy_done: + /* Disable SUM */ + li t0, (1 << 18) + csrc sstatus, t0 + li a0, 0 + ret + +.Lmemcpy_fault: + /* Fixup routine on fault */ + /* Disable SUM */ + li t0, (1 << 18) + csrc sstatus, t0 + li a0, -14 /* -EFAULT */ + ret + +/* Exception table entries */ +EXTABLE(1b, .Lmemcpy_fault) +EXTABLE(2b, .Lmemcpy_fault) + + +/* + * extern "C" int64 __uaccess_strncpy(char* dst, const char* src, uint64 max_len); + * + * a0: dst + * a1: src + * a2: max_len + * Returns: number of bytes copied (excluding null), or -EFAULT (-14) on page fault. + */ +.global __uaccess_strncpy +.type __uaccess_strncpy, @function +__uaccess_strncpy: + /* If max_len == 0, return 0 */ + beqz a2, .Lstrncpy_done_zero + + /* Enable SUM */ + li t0, (1 << 18) + csrs sstatus, t0 + + /* t2 will store the length copied */ + li t2, 0 + +.Lstrncpy_loop: + /* Load byte from src */ +3: lb t1, 0(a1) + /* Store byte to dst */ +4: sb t1, 0(a0) + + /* If it was the null terminator, we're done */ + beqz t1, .Lstrncpy_done + + addi a0, a0, 1 + addi a1, a1, 1 + addi t2, t2, 1 + addi a2, a2, -1 + bnez a2, .Lstrncpy_loop + + /* Reached max_len without finding null */ + /* We must null-terminate at max_len - 1 as per typical strncpy_from_user semantics */ + addi a0, a0, -1 + sb zero, 0(a0) + addi t2, t2, -1 /* length of string without the forced null terminator */ + +.Lstrncpy_done: + /* Disable SUM */ + li t0, (1 << 18) + csrc sstatus, t0 + mv a0, t2 + ret + +.Lstrncpy_done_zero: + li a0, 0 + ret + +.Lstrncpy_fault: + /* Fixup routine on fault */ + li t0, (1 << 18) + csrc sstatus, t0 + li a0, -14 /* -EFAULT */ + ret + +/* Exception table entries */ +EXTABLE(3b, .Lstrncpy_fault) +EXTABLE(4b, .Lstrncpy_fault) -- cgit v1.2.3