summaryrefslogtreecommitdiff
path: root/na/mm
diff options
context:
space:
mode:
authorRichard Qin <richard.qin.x@gmail.com>2026-07-10 11:31:46 +0800
committerRichard Qin <richard.qin.x@gmail.com>2026-07-10 11:31:46 +0800
commite90b72ec3082588b296956730bd2930cf8d5d671 (patch)
tree36ff6916af9b4176ea5a9662219d11e721f0b691 /na/mm
parent1767d390fbf3d0c65389de46caf0b6d43ef3b6ad (diff)
downloadlume-e90b72ec3082588b296956730bd2930cf8d5d671.tar.gz
lume-e90b72ec3082588b296956730bd2930cf8d5d671.tar.xz
refactor: promote new architecture to repository root
Archive the deprecated implementation under lume-legacy, make the new kernel the default root-level build, and update repository entry points without changing kernel behavior.
Diffstat (limited to 'na/mm')
-rw-r--r--na/mm/memblock.cc257
-rw-r--r--na/mm/module.mk7
-rw-r--r--na/mm/page_cache.cc252
-rw-r--r--na/mm/pmm.cc528
-rw-r--r--na/mm/slab.cc321
-rw-r--r--na/mm/slab.h83
-rw-r--r--na/mm/tlb.cc90
-rw-r--r--na/mm/vma.cc640
-rw-r--r--na/mm/vmm.cc235
9 files changed, 0 insertions, 2413 deletions
diff --git a/na/mm/memblock.cc b/na/mm/memblock.cc
deleted file mode 100644
index aee0245..0000000
--- a/na/mm/memblock.cc
+++ /dev/null
@@ -1,257 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-or-later */
-/* Copyright (C) 2026 Richard Qin */
-
-/*
- * memblock.cc — Boot-time physical memory allocator
- *
- * Simple region tracker: one memory region (from FDT) plus an array
- * of reserved regions. Allocation searches for gaps between reserved
- * regions. All operations are O(n_reserved), which is fine for the
- * ~20 allocations that happen during early boot.
- *
- * Reference: inspired by Linux kernel's mm/memblock.c
- */
-
-#include <lume/memblock.h>
-#include <lume/fdt.h>
-#include <lume/klog.h>
-#include <lume/kprintf.h>
-#include <lume/config.h>
-#include <lume/addr.h>
-
-extern "C" char _stext[], _kernel_end[];
-
-/* ------------------------------------------------------------------ */
-/* Module state */
-/* ------------------------------------------------------------------ */
-
-namespace {
-
-constexpr uint64 kMB = 1024ULL * 1024ULL;
-
-struct {
- uint64 mem_base;
- uint64 mem_size;
-
- struct {
- uint64 base;
- uint64 size;
- } reserved[kMemblockMaxRegions];
- int reserved_count;
- bool active;
-} g_mb;
-
-/* ------------------------------------------------------------------ */
-/* Internal helpers */
-/* ------------------------------------------------------------------ */
-
-/* Insert a reserved region, keeping the array sorted by base. */
-void insert_reserved(uint64 base, uint64 size)
-{
- if (g_mb.reserved_count >= kMemblockMaxRegions) {
- kernel_panic("memblock: too many reserved regions");
- }
-
- int i = g_mb.reserved_count;
- while (i > 0 && g_mb.reserved[i - 1].base > base) {
- g_mb.reserved[i] = g_mb.reserved[i - 1];
- i--;
- }
- g_mb.reserved[i] = {base, size};
- g_mb.reserved_count++;
-}
-
-/* Check if [base, base+size) overlaps any reserved region. */
-bool overlaps_reserved(uint64 base, uint64 size)
-{
- uint64 end = base + size;
- for (int i = 0; i < g_mb.reserved_count; i++) {
- uint64 rb = g_mb.reserved[i].base;
- uint64 re = rb + g_mb.reserved[i].size;
- if (base < re && end > rb) {
- return true;
- }
- }
- return false;
-}
-
-/* Find the end of the first reserved region that covers `addr`. */
-uint64 reserved_end_covering(uint64 addr)
-{
- for (int i = 0; i < g_mb.reserved_count; i++) {
- uint64 rb = g_mb.reserved[i].base;
- uint64 re = rb + g_mb.reserved[i].size;
- if (addr >= rb && addr < re) {
- return re;
- }
- }
- return addr;
-}
-
-} // namespace
-
-/* ------------------------------------------------------------------ */
-/* Public API */
-/* ------------------------------------------------------------------ */
-
-void memblock_init(uint64 fdt_paddr)
-{
-
- /* 1. Get memory layout from FDT */
- uint64 mem_base = 0;
- uint64 mem_size = 0;
- fdt_early_get_mem_info(&mem_base, &mem_size);
- if (mem_size == 0) {
- kernel_panic("memblock_init: FDT reports zero memory");
- }
-
- g_mb.mem_base = mem_base;
- g_mb.mem_size = mem_size;
- g_mb.reserved_count = 0;
- g_mb.active = true;
-
- /* 2. Reserve firmware region [mem_base, kPhysStart).
- *
- * Everything below the kernel's linked load address is firmware
- * territory (OpenSBI), which is PMP-protected. */
- {
- uint64 fw_end = kPhysStart;
- if (fw_end > mem_base) {
- memblock_reserve(mem_base, fw_end - mem_base);
- }
- }
-
- /* 2b. Reserve stale pre-relocation kernel image.
- *
- * After physical KASLR, the kernel is relocated from its original
- * load address (kPhysStart) to a new PA. The old copy at
- * [kPhysStart, kPhysStart + kernel_size) must not be allocated —
- * it contains stale code/data that would corrupt page tables or
- * other structures if reused. */
- {
- uint64 ks_pa = boot_va_to_pa(virt_addr(
- reinterpret_cast<uint64>(_stext))).raw;
- if (ks_pa != kPhysStart) {
- /* Kernel was relocated — reserve the original location */
- uint64 kernel_size = boot_va_to_pa(virt_addr(
- reinterpret_cast<uint64>(_kernel_end))).raw - ks_pa;
- memblock_reserve(kPhysStart, kernel_size);
- }
- }
-
- /* 3. Reserve kernel image [_stext, _kernel_end) */
- uint64 ks_pa = boot_va_to_pa(virt_addr(
- reinterpret_cast<uint64>(_stext))).raw;
- uint64 ke_pa = boot_va_to_pa(virt_addr(
- reinterpret_cast<uint64>(_kernel_end))).raw;
- memblock_reserve(ks_pa, ke_pa - ks_pa);
-
- /* 4. Reserve FDT blob — it lives in RAM and must not be overwritten
- * by vmemmap/page_map or buddy before fdt_unflatten() copies it. */
- {
- struct fdt_hdr { uint32 magic; uint32 totalsize; };
- const auto* fh = reinterpret_cast<const fdt_hdr*>(fdt_paddr);
- uint32 fdt_size = ((fh->totalsize & 0xFF) << 24) |
- (((fh->totalsize >> 8) & 0xFF) << 16) |
- (((fh->totalsize >> 16) & 0xFF) << 8) |
- ((fh->totalsize >> 24) & 0xFF);
- memblock_reserve(fdt_paddr, fdt_size);
- kprintf("[memblock] FDT reserved [0x%llx, 0x%llx)\n",
- fdt_paddr, fdt_paddr + fdt_size);
- }
-
- kprintf("[memblock] RAM [0x%llx, 0x%llx) = %llu MB\n",
- mem_base, mem_base + mem_size, mem_size / kMB);
- kprintf("[memblock] kernel reserved [0x%llx, 0x%llx)\n", ks_pa, ke_pa);
-}
-
-void memblock_reserve(uint64 base, uint64 size)
-{
- uint64 aligned_base = page_align_down(base);
- uint64 aligned_end = page_align_up(base + size);
- insert_reserved(aligned_base, aligned_end - aligned_base);
-}
-
-uint64 memblock_alloc_top(uint64 size, uint64 align)
-{
- uint64 mem_end = g_mb.mem_base + g_mb.mem_size;
-
- /* Search from top of RAM downward */
- uint64 end = page_align_down(mem_end);
- uint64 start = (end - size) & ~(align - 1);
-
- while (start >= g_mb.mem_base && start + size <= mem_end) {
- if (!overlaps_reserved(start, size)) {
- memblock_reserve(start, size);
- return start;
- }
- if (start < align) {
- break;
- }
- start -= align;
- }
-
- kernel_panic("memblock_alloc_top: out of memory");
- return 0;
-}
-
-uint64 memblock_alloc(uint64 size, uint64 align)
-{
- uint64 mem_end = g_mb.mem_base + g_mb.mem_size;
- uint64 start = page_align_up(g_mb.mem_base);
-
- while (start + size <= mem_end) {
- start = (start + align - 1) & ~(align - 1);
- if (start + size > mem_end) {
- break;
- }
-
- if (!overlaps_reserved(start, size)) {
- memblock_reserve(start, size);
- return start;
- }
-
- /* Skip past the overlapping reserved region */
- uint64 skip = reserved_end_covering(start);
- if (skip <= start) {
- start += align;
- } else {
- start = skip;
- }
- }
-
- kernel_panic("memblock_alloc: out of memory");
- return 0;
-}
-
-uint64 memblock_get_mem_base() { return g_mb.mem_base; }
-uint64 memblock_get_mem_size() { return g_mb.mem_size; }
-
-bool memblock_is_active() { return g_mb.active; }
-void memblock_retire() { g_mb.active = false; }
-
-void memblock_for_each_free(void (*cb)(uint64 base, uint64 size, void* ctx),
- void* ctx)
-{
- uint64 pos = g_mb.mem_base;
- uint64 mem_end = g_mb.mem_base + g_mb.mem_size;
-
- for (int i = 0; i < g_mb.reserved_count && pos < mem_end; i++) {
- uint64 rb = g_mb.reserved[i].base;
- if (rb < pos) {
- uint64 re = g_mb.reserved[i].base + g_mb.reserved[i].size;
- if (re > pos) {
- pos = re;
- }
- continue;
- }
- if (rb > pos) {
- cb(pos, rb - pos, ctx);
- }
- pos = g_mb.reserved[i].base + g_mb.reserved[i].size;
- }
-
- if (pos < mem_end) {
- cb(pos, mem_end - pos, ctx);
- }
-}
diff --git a/na/mm/module.mk b/na/mm/module.mk
deleted file mode 100644
index 93cef80..0000000
--- a/na/mm/module.mk
+++ /dev/null
@@ -1,7 +0,0 @@
-SRCS_CC += mm/memblock.cc
-SRCS_CC += mm/pmm.cc
-SRCS_CC += mm/slab.cc
-SRCS_CC += mm/vmm.cc
-SRCS_CC += mm/vma.cc
-SRCS_CC += mm/page_cache.cc
-SRCS_CC += mm/tlb.cc
diff --git a/na/mm/page_cache.cc b/na/mm/page_cache.cc
deleted file mode 100644
index 25c6a60..0000000
--- a/na/mm/page_cache.cc
+++ /dev/null
@@ -1,252 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-or-later */
-/* Copyright (C) 2026 Richard Qin */
-
-/*
- * page_cache.cc — Minimal page cache implementation
- *
- * Open-addressing hash table indexed by (mapping, index) pairs.
- * Fixed-size bucket array allocated at init time via slab.
- *
- * This is intentionally simple: no LRU, no writeback, no eviction.
- * The interface in page_cache.h is designed so that upgrading to a
- * radix tree + LRU scheme later only changes this file.
- *
- * Concurrency: protected by a single spinlock. Acceptable for
- * initial bring-up; will be replaced by per-bucket or RCU-based
- * locking when contention becomes measurable.
- */
-
-#include <lume/page_cache.h>
-#include <lume/page.h>
-#include <lume/pmm.h>
-#include <lume/errno.h>
-#include <lume/new.h>
-#include <kernel/sync/spinlock.h>
-
-/* ------------------------------------------------------------------ */
-/* Hash table internals */
-/* ------------------------------------------------------------------ */
-
-struct CacheEntry {
- void* mapping; // nullptr = empty slot
- uint64 index;
- Page* page;
-};
-
-namespace {
-
-constexpr uint64 kCacheBuckets = 1024;
-
-CacheEntry g_buckets[kCacheBuckets];
-Spinlock g_cache_lock;
-
-uint64 hash_key(void* mapping, uint64 index)
-{
- uint64 m = reinterpret_cast<uint64>(mapping);
- /* FNV-1a inspired mixing */
- uint64 h = m ^ (index * 0x517cc1b727220a95ULL);
- h ^= h >> 17;
- h *= 0xbf58476d1ce4e5b9ULL;
- h ^= h >> 31;
- return h % kCacheBuckets;
-}
-
-/* Linear probe to find an entry or an empty slot. */
-CacheEntry* find_slot(void* mapping, uint64 index)
-{
- uint64 start = hash_key(mapping, index);
- for (uint64 i = 0; i < kCacheBuckets; i++) {
- uint64 idx = (start + i) % kCacheBuckets;
- CacheEntry* e = &g_buckets[idx];
- if (e->mapping == nullptr) {
- return e; // Empty slot
- }
- if (e->mapping == mapping && e->index == index) {
- return e; // Found
- }
- }
- return nullptr; // Table full
-}
-
-/* Find an existing entry (not an empty slot). */
-CacheEntry* find_entry(void* mapping, uint64 index)
-{
- uint64 start = hash_key(mapping, index);
- for (uint64 i = 0; i < kCacheBuckets; i++) {
- uint64 idx = (start + i) % kCacheBuckets;
- CacheEntry* e = &g_buckets[idx];
- if (e->mapping == nullptr) {
- return nullptr; // Empty = not found
- }
- if (e->mapping == mapping && e->index == index) {
- return e;
- }
- }
- return nullptr;
-}
-
-/*
- * Remove an entry and fix up the linear-probe chain.
- * Standard open-addressing deletion with backward shift.
- */
-void remove_entry(CacheEntry* entry)
-{
- uint64 removed = static_cast<uint64>(entry - g_buckets);
- entry->mapping = nullptr;
- entry->page = nullptr;
-
- /* Backward-shift deletion to maintain probe chain integrity */
- uint64 cur = (removed + 1) % kCacheBuckets;
- while (g_buckets[cur].mapping != nullptr) {
- uint64 ideal = hash_key(g_buckets[cur].mapping, g_buckets[cur].index);
- /* Check if cur's ideal position is "before" removed in probe order */
- bool needs_shift = false;
- if (removed <= cur) {
- needs_shift = (ideal <= removed || ideal > cur);
- } else {
- needs_shift = (ideal <= removed && ideal > cur);
- }
-
- if (needs_shift) {
- g_buckets[removed] = g_buckets[cur];
- g_buckets[cur].mapping = nullptr;
- g_buckets[cur].page = nullptr;
- removed = cur;
- }
- cur = (cur + 1) % kCacheBuckets;
- }
-}
-
-} // namespace
-
-/* ------------------------------------------------------------------ */
-/* Public API */
-/* ------------------------------------------------------------------ */
-
-void page_cache_init()
-{
- g_cache_lock.init("page_cache");
- for (uint64 i = 0; i < kCacheBuckets; i++) {
- g_buckets[i].mapping = nullptr;
- g_buckets[i].index = 0;
- g_buckets[i].page = nullptr;
- }
-}
-
-Page* page_cache_get(void* mapping, uint64 index)
-{
- LockGuard guard(g_cache_lock);
- CacheEntry* e = find_entry(mapping, index);
- if (!e) {
- return nullptr;
- }
- page_incref(e->page);
- return e->page;
-}
-
-int page_cache_insert(void* mapping, uint64 index, Page* page)
-{
- if (!mapping || !page) {
- return -EINVAL;
- }
-
- LockGuard guard(g_cache_lock);
-
- /* Check for existing entry */
- CacheEntry* existing = find_entry(mapping, index);
- if (existing) {
- return -EEXIST;
- }
-
- /* Find an empty slot */
- CacheEntry* slot = find_slot(mapping, index);
- if (!slot || slot->mapping != nullptr) {
- return -ENOMEM; // Table full
- }
-
- slot->mapping = mapping;
- slot->index = index;
- slot->page = page;
- page_incref(page); // Cache holds a reference
-
- return 0;
-}
-
-void page_cache_put(Page* page)
-{
- if (!page) {
- return;
- }
- page_decref(page);
-}
-
-Page* page_cache_remove(void* mapping, uint64 index)
-{
- LockGuard guard(g_cache_lock);
- CacheEntry* e = find_entry(mapping, index);
- if (!e) {
- return nullptr;
- }
-
- Page* page = e->page;
- remove_entry(e);
- page_decref(page); // Release cache's reference
- return page;
-}
-
-void page_cache_invalidate(void* mapping)
-{
- if (!mapping) {
- return;
- }
-
- LockGuard guard(g_cache_lock);
-
- /* Scan all buckets — acceptable for current table size */
- for (uint64 i = 0; i < kCacheBuckets; i++) {
- if (g_buckets[i].mapping == mapping) {
- Page* page = g_buckets[i].page;
- remove_entry(&g_buckets[i]);
- page_decref(page);
- /* After removal, re-check this index since backward shift
- * may have moved an entry into this slot. */
- i--;
- }
- }
-}
-
-Page* page_cache_get_or_alloc(void* mapping, uint64 index)
-{
- /* Fast path: already cached */
- Page* page = page_cache_get(mapping, index);
- if (page) {
- return page;
- }
-
- /* Slow path: allocate and insert */
- page = pmm_alloc_page();
- if (!page) {
- return nullptr;
- }
-
- int ret = page_cache_insert(mapping, index, page);
- if (ret == -EEXIST) {
- /* Race: another thread inserted between our get and insert.
- * Free our allocation and return the existing page. */
- pmm_free_page(page);
- return page_cache_get(mapping, index);
- }
- if (ret < 0) {
- pmm_free_page(page);
- return nullptr;
- }
-
- /* Caller gets a ref (insert already added cache's ref,
- * we need one more for the caller). */
- /* Actually, insert did incref for cache. The page already has
- * refcount=1 from alloc + 1 from insert = 2.
- * We return it to caller who must put() when done.
- * So we incref once more for the caller's reference. */
- page_incref(page);
- return page;
-}
diff --git a/na/mm/pmm.cc b/na/mm/pmm.cc
deleted file mode 100644
index b7cf611..0000000
--- a/na/mm/pmm.cc
+++ /dev/null
@@ -1,528 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-or-later */
-/* Copyright (C) 2026 Richard Qin */
-
-/*
- * pmm.cc — Physical Memory Manager implementation
- *
- * Buddy allocator with Per-CPU single-page caches (PCP).
- *
- * Initialization sequence (called by BSP in single-core period):
- * 1. memblock + vmm_init allocate and map the page_map at VMEMMAP.
- * 2. Zero and mark all Page descriptors reserved by default.
- * 3. Carve free regions from memblock into the buddy system.
- *
- * The page_map is a flat array indexed by physical page frame number (PFN).
- * PFN = (pa - mem_base) / PAGE_SIZE. This keeps address conversion O(1).
- *
- * Reference: docs/specs/pmm.md
- */
-
-#include <arch/config.h>
-#include <arch/cpu.h>
-#include <kernel/sync/spinlock.h>
-#include <lume/addr.h>
-#include <lume/config.h>
-#include <lume/klog.h>
-#include <lume/kprintf.h>
-#include <lume/memblock.h>
-#include <lume/new.h>
-#include <lume/pmm.h>
-#include <lume/types.h>
-#include <lume/vmm.h>
-
-/* ------------------------------------------------------------------ */
-/* Per-CPU page cache */
-/* ------------------------------------------------------------------ */
-
-struct PerCpuCache {
- list_node free_list; // Single-page free list
- uint32 count; // Number of cached pages
-};
-
-/* ------------------------------------------------------------------ */
-/* Module-private state */
-/* ------------------------------------------------------------------ */
-
-namespace {
-
-/* PCP Cache tuning parameters */
-constexpr uint32 kPcpBatchDivisor = 1024;
-constexpr uint32 kPcpMinBatchSize = 16;
-constexpr uint64 kMB = 1024ULL * 1024ULL;
-
-Spinlock g_pmm_lock;
-list_node g_free_areas[kMaxOrder]; // Buddy free lists [0..10]
-PerCpuCache g_pcp[kMaxCpus];
-
-Page *g_page_map = nullptr; // Flat array of Page descriptors
-uint64 g_page_map_pa = 0; // Physical base of page_map storage
-uint64 g_page_map_bytes = 0; // Byte size of page_map storage
-uint64 g_mem_base = 0; // Physical memory base (from FDT)
-uint64 g_mem_size = 0; // Total physical memory size
-uint64 g_num_pages = 0; // Total managed pages
-
-} // namespace
-
-uint32 g_pcp_high_watermark = 0;
-uint32 g_pcp_batch_size = 0;
-
-/* ------------------------------------------------------------------ */
-/* Address conversion helpers */
-/* ------------------------------------------------------------------ */
-
-/*
- * PFN (Page Frame Number) — fundamental conversion unit.
- * When sparse vmemmap is implemented, only these two functions change.
- * All other PA↔Page conversions go through them.
- */
-uint64 page_to_pfn(const Page *page) {
- uintptr base = reinterpret_cast<uintptr>(g_page_map);
- uintptr addr = reinterpret_cast<uintptr>(page);
- uintptr end = base + g_num_pages * sizeof(Page);
-
- if (addr < base || addr >= end) {
- kernel_panic("page_to_pfn: page outside page_map bounds");
- }
-
- uint64 idx = (addr - base) / sizeof(Page);
- return (g_mem_base / kPageSize) + idx;
-}
-
-Page *pfn_to_page(uint64 pfn) {
- uint64 base_pfn = g_mem_base / kPageSize;
- if (pfn < base_pfn || pfn >= base_pfn + g_num_pages) {
- kernel_panic("pfn_to_page: PFN outside managed range");
- }
- return &g_page_map[pfn - base_pfn];
-}
-
-PhysAddr page_to_pa(const Page *page) {
- return pfn_to_pa(Pfn{page_to_pfn(page)});
-}
-
-Page *pa_to_page(PhysAddr pa) {
- return pfn_to_page(pa_to_pfn(pa).raw);
-}
-
-VirtAddr page_to_va(const Page *page) {
- return pa_to_va(page_to_pa(page));
-}
-
-
-/* ------------------------------------------------------------------ */
-/* Buddy system internals (caller must hold g_pmm_lock) */
-/* ------------------------------------------------------------------ */
-
-namespace {
-
-/*
- * Find a page's buddy at a given order.
- * Buddy PFN = PFN ^ (1 << order).
- */
-inline Page *buddy_of(Page *page, uint8 order) {
- uint64 pfn = static_cast<uint64>(page - g_page_map);
- uint64 buddy_pfn = pfn ^ (1ULL << order);
- if (buddy_pfn >= g_num_pages) {
- return nullptr;
- }
- return &g_page_map[buddy_pfn];
-}
-
-/*
- * Insert a free block into the buddy free list at the given order.
- */
-void buddy_list_add(Page *page, uint8 order) {
- page->type = PageType::Free;
- page->order = order;
- list_node *head = &g_free_areas[order];
- list_node *node = &page->free.free_link;
- node->next = head->next;
- node->prev = head;
- head->next->prev = node;
- head->next = node;
-}
-
-/*
- * Remove a block from whatever buddy free list it is on.
- */
-void buddy_list_del(Page *page) {
- list_node *node = &page->free.free_link;
- node->prev->next = node->next;
- node->next->prev = node->prev;
- node->next = node;
- node->prev = node;
-}
-
-/*
- * Allocate a block of 2^order contiguous pages from the buddy system.
- * Splits higher-order blocks if no exact match is available.
- * Returns nullptr if no memory available.
- */
-Page *buddy_alloc(uint8 order) {
- for (int cur = order; cur < kMaxOrder; cur++) {
- list_node *head = &g_free_areas[cur];
- if (head->next == head) {
- continue; // Empty at this order
- }
-
- // Pop the first block from this order's free list
- list_node *node = head->next;
- // Compute Page* from the free_link member offset
- Page *page =
- reinterpret_cast<Page *>(reinterpret_cast<uintptr>(node) -
- __builtin_offsetof(Page, free.free_link));
- buddy_list_del(page);
-
- // Split down to the requested order
- while (cur > order) {
- cur--;
- // The upper half becomes a free buddy
- Page *split = &g_page_map[(page - g_page_map) + (1ULL << cur)];
- buddy_list_add(split, static_cast<uint8>(cur));
- }
-
- page->type = PageType::Allocated;
- page->order = order;
- page->refcount.store(1, __ATOMIC_RELAXED);
- page->mapcount.store(0, __ATOMIC_RELAXED);
- page->flags.store(0, __ATOMIC_RELAXED);
- page->mapping = nullptr;
- page->index = 0;
- return page;
- }
- return nullptr;
-}
-
-/*
- * Return a block of 2^order pages to the buddy system.
- * Coalesces with its buddy recursively up to kMaxOrder-1.
- */
-void buddy_free(Page *page, uint8 order) {
- uint64 pfn = static_cast<uint64>(page - g_page_map);
-
- while (order < kMaxOrder - 1) {
- Page *buddy = buddy_of(page, order);
- if (!buddy) {
- break;
- }
- // Buddy must be free AND at the same order to coalesce
- if (buddy->type != PageType::Free || buddy->order != order) {
- break;
- }
-
- // Remove buddy from its free list and merge
- buddy_list_del(buddy);
-
- // The merged block starts at the lower PFN
- uint64 buddy_pfn = static_cast<uint64>(buddy - g_page_map);
- if (buddy_pfn < pfn) {
- page = buddy;
- pfn = buddy_pfn;
- }
- order++;
- }
-
- buddy_list_add(page, order);
-}
-
-/* ------------------------------------------------------------------ */
-/* Per-CPU cache operations (interrupts must be disabled by caller) */
-/* ------------------------------------------------------------------ */
-
-/*
- * Refill the PCP from the global buddy system.
- * Acquires g_pmm_lock internally.
- */
-void refill_pcp(PerCpuCache *pcp) {
- LockGuard guard(g_pmm_lock);
- for (uint32 i = 0; i < g_pcp_batch_size; i++) {
- Page *page = buddy_alloc(0);
- if (!page) {
- break;
- }
- // Put onto PCP list (reuse the free_link for PCP chain)
- page->type = PageType::PcpCached;
- list_node *node = &page->free.free_link;
- node->next = pcp->free_list.next;
- node->prev = &pcp->free_list;
- pcp->free_list.next->prev = node;
- pcp->free_list.next = node;
- pcp->count++;
- }
-}
-
-/*
- * Drain half the PCP back to the global buddy system.
- * Acquires g_pmm_lock internally.
- */
-void drain_pcp(PerCpuCache *pcp) {
- LockGuard guard(g_pmm_lock);
- int to_drain = pcp->count / 2;
- for (int i = 0; i < to_drain; i++) {
- list_node *head = &pcp->free_list;
- if (head->next == head) {
- break;
- }
- list_node *node = head->next;
- // Unlink from PCP
- node->prev->next = node->next;
- node->next->prev = node->prev;
- pcp->count--;
-
- Page *page =
- reinterpret_cast<Page *>(reinterpret_cast<uintptr>(node) -
- __builtin_offsetof(Page, free.free_link));
- buddy_free(page, 0);
- }
-}
-
-} // namespace
-
-/* ------------------------------------------------------------------ */
-/* Public API: single-page (fast path via PCP) */
-/* ------------------------------------------------------------------ */
-
-Page *pmm_alloc_page() {
- /*
- * Disable interrupts to prevent reentry on the same CPU
- * (e.g. timer IRQ handler calling pmm_alloc_page while we
- * are mid-way through manipulating the PCP list).
- */
- bool was_on = arch::cpu::intr_enabled();
- arch::cpu::intr_off();
-
- uint64 cpu = arch::cpu::id();
- PerCpuCache *pcp = &g_pcp[cpu];
-
- if (pcp->count == 0) {
- refill_pcp(pcp);
- }
-
- Page *page = nullptr;
- if (pcp->count > 0) {
- list_node *node = pcp->free_list.next;
- // Unlink
- node->prev->next = node->next;
- node->next->prev = node->prev;
- pcp->count--;
-
- page =
- reinterpret_cast<Page *>(reinterpret_cast<uintptr>(node) -
- __builtin_offsetof(Page, free.free_link));
-
- // Validate the page is within the page_map bounds
- if (page < g_page_map || page >= g_page_map + g_num_pages) {
- kernel_panic("pmm_alloc_page: PCP returned out-of-bounds page");
- }
-
- page->type = PageType::Allocated;
- page->order = 0;
- page->refcount.store(1, __ATOMIC_RELAXED);
- page->mapcount.store(0, __ATOMIC_RELAXED);
- page->flags.store(0, __ATOMIC_RELAXED);
- page->mapping = nullptr;
- page->index = 0;
- }
-
- if (was_on) {
- arch::cpu::intr_on();
- }
- return page;
-}
-
-void pmm_free_page(Page *page) {
- if (!page) {
- kernel_panic("pmm_free_page: null page");
- }
-
- bool was_on = arch::cpu::intr_enabled();
- arch::cpu::intr_off();
-
- uint64 cpu = arch::cpu::id();
- PerCpuCache *pcp = &g_pcp[cpu];
-
- // Push onto PCP
- page->type = PageType::PcpCached;
- page->order = 0;
- list_node *node = &page->free.free_link;
- node->next = pcp->free_list.next;
- node->prev = &pcp->free_list;
- pcp->free_list.next->prev = node;
- pcp->free_list.next = node;
- pcp->count++;
-
- // Drain if over high watermark
- if (pcp->count >= g_pcp_high_watermark) {
- drain_pcp(pcp);
- }
-
- if (was_on) {
- arch::cpu::intr_on();
- }
-}
-
-/* ------------------------------------------------------------------ */
-/* Public API: multi-page (slow path, direct buddy) */
-/* ------------------------------------------------------------------ */
-
-Page *pmm_alloc_pages(uint8 order) {
- if (order >= kMaxOrder) {
- return nullptr;
- }
- LockGuard guard(g_pmm_lock);
- return buddy_alloc(order);
-}
-
-void pmm_free_pages(Page *page, uint8 order) {
- if (!page) {
- kernel_panic("pmm_free_pages: null page");
- }
- if (order >= kMaxOrder) {
- kernel_panic("pmm_free_pages: order out of range");
- }
- LockGuard guard(g_pmm_lock);
- buddy_free(page, order);
-}
-
-/* ------------------------------------------------------------------ */
-/* Public API: reference counting */
-/* ------------------------------------------------------------------ */
-
-void page_incref(Page *page) { page->refcount.fetch_add(1); }
-
-void page_decref(Page *page) {
- uint32 old = page->refcount.fetch_sub(1);
- if (old == 1) {
- // Refcount dropped to 0 — return to pool based on originally allocated
- // order
- if (page->order == 0) {
- pmm_free_page(page);
- } else {
- pmm_free_pages(page, page->order);
- }
- }
-}
-
-/* ------------------------------------------------------------------ */
-/* Initialization */
-/* ------------------------------------------------------------------ */
-
-/*
- * Helper: zero out a range of bytes. We cannot use memset here
- * because lib/string may not be linked yet.
- */
-namespace {
-void zero_range(void *start, uint64 bytes) {
- uint8 *p = static_cast<uint8 *>(start);
- for (uint64 i = 0; i < bytes; i++) {
- p[i] = 0;
- }
-}
-} // namespace
-
-void pmm_init() {
- /* 1. Get memory layout from memblock (already scanned FDT) */
- g_mem_base = memblock_get_mem_base();
- g_mem_size = memblock_get_mem_size();
- g_num_pages = vmm_get_num_pages();
-
- if (g_mem_size == 0) {
- kernel_panic("pmm_init: memblock reports zero memory");
- }
-
- /* 2. PCP parameters */
- uint32 calc_batch = static_cast<uint32>(g_num_pages / (kMaxCpus * kPcpBatchDivisor));
- g_pcp_batch_size = (calc_batch > kPcpMinBatchSize) ? calc_batch : kPcpMinBatchSize;
- g_pcp_high_watermark = g_pcp_batch_size * 4;
-
- /* 3. page_map was allocated by vmm_init at vmemmap VA.
- * No relocation needed — page_map starts life at the right VA. */
- g_page_map_pa = vmm_get_page_map_pa();
- g_page_map_bytes = vmm_get_page_map_size();
- g_page_map = reinterpret_cast<Page *>(arch::g_vmemmap_base);
-
- /* 4. Zero out the entire page_map */
- zero_range(g_page_map, g_page_map_bytes);
-
- /* 4b. Default every page descriptor to reserved/non-free. */
- for (uint64 i = 0; i < g_num_pages; i++) {
- Page *page = &g_page_map[i];
- page->refcount.store(0, __ATOMIC_RELAXED);
- page->mapcount.store(0, __ATOMIC_RELAXED);
- page->flags.store(0, __ATOMIC_RELAXED);
- page->type = PageType::Allocated;
- page->order = 0;
- page->mapping = nullptr;
- page->index = 0;
- page->lru.init();
- page->free.free_link.init();
- }
-
- /* 5. Initialize buddy free list heads */
- g_pmm_lock.init("pmm");
- for (int i = 0; i < kMaxOrder; i++) {
- g_free_areas[i].init();
- }
-
- /* 6. Initialize PCP list heads */
- for (int i = 0; i < kMaxCpus; i++) {
- g_pcp[i].free_list.init();
- g_pcp[i].count = 0;
- }
-
- /* 7. Feed free regions from memblock into the buddy system.
- * memblock knows about all reserved regions (firmware, kernel,
- * page table pages, page_map). Everything else is free. */
- struct FeedCtx {
- uint64 total_pages;
- };
- FeedCtx ctx{0};
-
- memblock_for_each_free(
- [](uint64 base, uint64 size, void *arg) {
- auto *c = static_cast<FeedCtx *>(arg);
- uint64 start_pfn = (base - memblock_get_mem_base()) / kPageSize;
- uint64 end_pfn = start_pfn + size / kPageSize;
-
- /* Access page_map via the globals directly (they are in anon namespace now) */
-
- uint64 pfn = start_pfn;
- while (pfn < end_pfn) {
- uint8 order = 0;
- for (int o = kMaxOrder - 1; o >= 0; o--) {
- uint64 block_size = 1ULL << o;
- if ((pfn & (block_size - 1)) == 0 && pfn + block_size <= end_pfn) {
- order = static_cast<uint8>(o);
- break;
- }
- }
-
- Page *page = &g_page_map[pfn];
- page->type = PageType::Free;
- page->order = order;
- page->refcount.store(0, __ATOMIC_RELAXED);
- page->free.free_link.init();
-
- /* Inline buddy_list_add since we're inside a lambda */
- list_node *head = &g_free_areas[order];
- list_node *node = &page->free.free_link;
- node->next = head->next;
- node->prev = head;
- head->next->prev = node;
- head->next = node;
-
- pfn += (1ULL << order);
- c->total_pages += (1ULL << order);
- }
- },
- &ctx);
-
- kprintf("[pmm] buddy: %llu free pages (%llu MB)\n", ctx.total_pages,
- (ctx.total_pages * kPageSize) / kMB);
-
- /* 8. Retire memblock — all future allocations go through PMM */
- memblock_retire();
-}
-
-PhysAddr pmm_get_mem_base() { return phys_addr(g_mem_base); }
-uint64 pmm_get_mem_size() { return g_mem_size; }
diff --git a/na/mm/slab.cc b/na/mm/slab.cc
deleted file mode 100644
index 43d8c4a..0000000
--- a/na/mm/slab.cc
+++ /dev/null
@@ -1,321 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-or-later */
-/* Copyright (C) 2026 Richard Qin */
-
-/*
- * slab.cc — SLUB allocator implementation
- *
- * Key design points:
- * - Free object list is embedded in the objects themselves (first 8 bytes).
- * - Per-page metadata lives in Page.slab (cache, obj_count, freelist).
- * - Per-CPU "active page" avoids lock contention on the fast path.
- * - partial_ list protected by per-KmemCache Spinlock.
- * - PMM calls happen outside the lock to avoid lock ordering issues.
- *
- * Address note: the kernel may be running at physical addresses via
- * identity mapping (before vmm_init). We handle PA/VA carefully:
- * PMM gives us Page*, page_to_pa() gives PA, and we access page
- * contents via pa_to_va() (higher-half VA, always mapped).
- *
- * Reference: docs/specs/slab.md
- */
-
-#include "slab.h"
-#include <lume/pmm.h>
-#include <lume/addr.h>
-#include <lume/klog.h>
-#include <lume/new.h>
-#include <arch/cpu.h>
-
-/* ------------------------------------------------------------------ */
-/* Helpers */
-/* ------------------------------------------------------------------ */
-
-/*
- * Convert a Page* to the usable virtual address of the page's contents.
- * Uses the higher-half mapping which is always active.
- */
-static inline void* page_to_data_va(Page* page)
-{
- return reinterpret_cast<void*>(page_to_va(page).raw);
-}
-
-/* Round up x to the next multiple of align (align must be power of 2) */
-static inline uint32 align_up(uint32 x, uint32 align)
-{
- return (x + align - 1) & ~(align - 1);
-}
-
-/* ------------------------------------------------------------------ */
-/* Global size-class caches */
-/* ------------------------------------------------------------------ */
-
-KmemCache g_size_caches[kNumSizeClasses];
-
-/* ------------------------------------------------------------------ */
-/* KmemCache::init */
-/* ------------------------------------------------------------------ */
-
-void KmemCache::init(const char* name, uint32 obj_size, uint32 obj_align)
-{
- name_ = name;
- obj_align_ = (obj_align < 8) ? 8 : obj_align; // minimum 8 for freelist ptr
- obj_size_ = align_up((obj_size < 8) ? 8 : obj_size, obj_align_);
- objs_per_slab_ = kPageSize / obj_size_;
-
- lock_.init(name);
- partial_.init();
-
- for (int i = 0; i < kMaxCpus; i++)
- cpu_slab_[i].active = nullptr;
-}
-
-/* ------------------------------------------------------------------ */
-/* KmemCache::new_slab — allocate and format a fresh slab page */
-/* ------------------------------------------------------------------ */
-
-Page* KmemCache::new_slab()
-{
- Page* page = pmm_alloc_page();
- if (!page)
- return nullptr;
-
- page->type = PageType::Slab;
- page->slab.cache = this;
- page->slab.obj_count = 0;
-
- /* Build the embedded free list across all object slots */
- uint8* base = static_cast<uint8*>(page_to_data_va(page));
- void* head = nullptr;
-
- /* Chain objects from last to first so freelist order = low addr first */
- for (int i = static_cast<int>(objs_per_slab_) - 1; i >= 0; i--) {
- void* obj = base + i * obj_size_;
- *reinterpret_cast<void**>(obj) = head;
- head = obj;
- }
- page->slab.freelist = head;
-
- return page;
-}
-
-/* ------------------------------------------------------------------ */
-/* Partial list management (lock must be held) */
-/* ------------------------------------------------------------------ */
-
-Page* KmemCache::get_partial()
-{
- /*
- * Phase 1 simplification: no partial list tracking.
- * The slow path always allocates a fresh page from PMM.
- * Full pages stay alive (Page.slab.cache is valid for kfree).
- * Empty pages are returned to PMM immediately.
- *
- * TODO: add list_node to Page.slab and implement partial tracking
- * to avoid wasting partially-filled pages.
- */
- (void)partial_;
- return nullptr;
-}
-
-void KmemCache::put_partial(Page* page)
-{
- (void)page;
-}
-
-void KmemCache::remove_partial(Page* page)
-{
- (void)page;
-}
-
-/* ------------------------------------------------------------------ */
-/* KmemCache::alloc */
-/* ------------------------------------------------------------------ */
-
-void* KmemCache::alloc()
-{
- bool was_on = arch::cpu::intr_enabled();
- arch::cpu::intr_off();
-
- uint64 cpu = arch::cpu::id();
- Page* active = cpu_slab_[cpu].active;
-
- /* Fast path: pop from active page's freelist */
- if (active && active->slab.freelist) {
- void* obj = active->slab.freelist;
- active->slab.freelist = *reinterpret_cast<void**>(obj);
- active->slab.obj_count++;
- if (was_on) arch::cpu::intr_on();
- return obj;
- }
-
- /* Slow path: need a new active page */
-
- /* If current active is full, detach it (it stays as Slab state,
- * cache pointer still valid for kfree to find it) */
- if (active && !active->slab.freelist) {
- cpu_slab_[cpu].active = nullptr;
- }
-
- if (was_on) arch::cpu::intr_on();
-
- /* Try to get a partial page or allocate a fresh one.
- * PMM call happens without any lock held. */
- Page* new_page = new_slab();
- if (!new_page)
- return nullptr;
-
- /* Install as new active page */
- was_on = arch::cpu::intr_enabled();
- arch::cpu::intr_off();
-
- /* Another interrupt handler might have set a new active while we
- * were allocating. If so, put ours aside (simplified: just use ours). */
- cpu_slab_[cpu].active = new_page;
-
- void* obj = new_page->slab.freelist;
- new_page->slab.freelist = *reinterpret_cast<void**>(obj);
- new_page->slab.obj_count++;
-
- if (was_on) arch::cpu::intr_on();
- return obj;
-}
-
-/* ------------------------------------------------------------------ */
-/* KmemCache::free */
-/* ------------------------------------------------------------------ */
-
-void KmemCache::free(void* obj)
-{
- if (!obj)
- return;
-
- /* Find the Page for this object's page */
- uint64 obj_addr = reinterpret_cast<uint64>(obj);
-
- /* Convert VA to PA. Like pmm.cc, handle both PA and VA cases. */
- uint64 obj_pa = va_to_pa(virt_addr(obj_addr)).raw;
- uint64 page_pa = obj_pa & ~(kPageSize - 1);
- Page* page = pa_to_page(phys_addr(page_pa));
-
- if (page->type != PageType::Slab || page->slab.cache != this)
- kernel_panic("slab free: object does not belong to this cache");
-
- bool was_on = arch::cpu::intr_enabled();
- arch::cpu::intr_off();
-
- /* Push object back onto the page's freelist */
- *reinterpret_cast<void**>(obj) = page->slab.freelist;
- page->slab.freelist = obj;
- page->slab.obj_count--;
-
- /* If page is now completely empty, return it to PMM */
- if (page->slab.obj_count == 0) {
- /* If this page is our active, clear it */
- uint64 cpu = arch::cpu::id();
- if (cpu_slab_[cpu].active == page)
- cpu_slab_[cpu].active = nullptr;
-
- if (was_on) arch::cpu::intr_on();
-
- /* Return page to PMM */
- page->slab.cache = nullptr;
- page->slab.freelist = nullptr;
- pmm_free_page(page);
- return;
- }
-
- if (was_on) arch::cpu::intr_on();
-}
-
-/* ------------------------------------------------------------------ */
-/* Public free-function API */
-/* ------------------------------------------------------------------ */
-
-/* Find the smallest size class >= size */
-static KmemCache* find_cache(uint32 size)
-{
- for (int i = 0; i < kNumSizeClasses; i++) {
- if (kSizeClasses[i] >= size)
- return &g_size_caches[i];
- }
- return nullptr; // Too large for slab
-}
-
-void* kmalloc(uint32 size)
-{
- if (size == 0)
- return nullptr;
-
- /* Large allocation: fall back to PMM directly */
- if (size > kSizeClasses[kNumSizeClasses - 1]) {
- uint8 order = 0;
- uint64 needed = (size + kPageSize - 1) / kPageSize;
- while ((1ULL << order) < needed)
- order++;
- Page* page = pmm_alloc_pages(order);
- if (!page)
- return nullptr;
- return reinterpret_cast<void*>(pa_to_va(page_to_pa(page)).raw);
- }
-
- KmemCache* cache = find_cache(size);
- if (!cache)
- return nullptr;
- return cache->alloc();
-}
-
-void kfree(void* ptr)
-{
- if (!ptr)
- return;
-
- uint64 addr = reinterpret_cast<uint64>(ptr);
- uint64 pa = va_to_pa(virt_addr(addr)).raw;
- uint64 page_pa = pa & ~(kPageSize - 1);
- Page* page = pa_to_page(phys_addr(page_pa));
-
- if (page->type == PageType::Slab) {
- KmemCache* cache = page->slab.cache;
- if (!cache)
- kernel_panic("kfree: slab page has null cache");
- cache->free(ptr);
- } else if (page->type == PageType::Allocated) {
- /* Large allocation — return directly to PMM */
- pmm_free_pages(page, page->order);
- } else {
- kernel_panic("kfree: invalid page state");
- }
-}
-
-/* ------------------------------------------------------------------ */
-/* C++ operator new / delete */
-/* ------------------------------------------------------------------ */
-
-/* Use decltype(sizeof(0)) instead of size_t to match the compiler's
- * built-in expectation for operator new (unsigned long vs unsigned long long). */
-using __sz = decltype(sizeof(0));
-
-void* operator new(__sz size) { return kmalloc(static_cast<uint32>(size)); }
-void* operator new[](__sz size) { return kmalloc(static_cast<uint32>(size)); }
-
-void operator delete(void* ptr) noexcept { kfree(ptr); }
-void operator delete[](void* ptr) noexcept { kfree(ptr); }
-void operator delete(void* ptr, __sz) noexcept { kfree(ptr); }
-void operator delete[](void* ptr, __sz) noexcept { kfree(ptr); }
-
-/* ------------------------------------------------------------------ */
-/* Initialization */
-/* ------------------------------------------------------------------ */
-
-void slab_init()
-{
- for (int i = 0; i < kNumSizeClasses; i++) {
- /* Build a name string like "kmalloc-32" */
- static const char* names[kNumSizeClasses] = {
- "kmalloc-8", "kmalloc-16", "kmalloc-32",
- "kmalloc-64", "kmalloc-128", "kmalloc-256",
- "kmalloc-512", "kmalloc-1024","kmalloc-2048"
- };
- g_size_caches[i].init(names[i], kSizeClasses[i], 8);
- }
-}
diff --git a/na/mm/slab.h b/na/mm/slab.h
deleted file mode 100644
index 909ee62..0000000
--- a/na/mm/slab.h
+++ /dev/null
@@ -1,83 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-or-later */
-/* Copyright (C) 2026 Richard Qin */
-
-#pragma once
-
-/*
- * SLUB Allocator — module-private header
- *
- * SLUB = Simple List of Unfull Blocks.
- * Metadata lives in Page descriptors (no per-page Slab header).
- * Free object list is embedded inside the objects themselves.
- * Per-CPU "active page" eliminates lock contention on the fast path.
- *
- * Reference: docs/specs/slab.md
- */
-
-#include <lume/types.h>
-#include <lume/config.h>
-#include <lume/page.h>
-#include <lume/list.h>
-#include <kernel/sync/spinlock.h>
-
-/* Size classes: 8, 16, 32, 64, 128, 256, 512, 1024, 2048 */
-inline constexpr int kNumSizeClasses = 9;
-inline constexpr uint32 kSizeClasses[kNumSizeClasses] = {
- 8, 16, 32, 64, 128, 256, 512, 1024, 2048
-};
-
-/* Per-CPU active slab page pointer */
-struct PerCpuSlab {
- Page* active; // Currently used slab page (nullptr if none)
-};
-
-/*
- * KmemCache — manages a pool of fixed-size objects.
- *
- * Lock contract:
- * lock_ protects partial_ list only.
- * Per-CPU cpu_slab_ access requires interrupts disabled.
- * PMM calls happen outside lock_ to avoid lock ordering issues.
- */
-class KmemCache {
-public:
- KmemCache() = default;
-
- // Explicit init (Placement New / BSS pattern)
- void init(const char* name, uint32 obj_size, uint32 obj_align);
-
- // Allocate one object. Returns nullptr on OOM.
- void* alloc();
-
- // Free one object back to its owning page.
- void free(void* obj);
-
- const char* name() const { return name_; }
- uint32 obj_size() const { return obj_size_; }
-
-private:
- // Get a partially-filled page from the partial list (lock must be held)
- Page* get_partial();
-
- // Put a page onto the partial list (lock must be held)
- void put_partial(Page* page);
-
- // Remove a page from the partial list (lock must be held)
- void remove_partial(Page* page);
-
- // Allocate a new page from PMM and format it as a slab
- Page* new_slab();
-
- Spinlock lock_; // Protects partial_ only
- list_node partial_; // Partially-filled slab pages
-
- PerCpuSlab cpu_slab_[kMaxCpus]; // Per-CPU active page
-
- const char* name_ = "uninit";
- uint32 obj_size_ = 0; // Aligned object size (bytes)
- uint32 obj_align_ = 0; // Alignment requirement
- uint32 objs_per_slab_ = 0; // Objects per 4KB page
-};
-
-/* Global size-class caches */
-extern KmemCache g_size_caches[kNumSizeClasses];
diff --git a/na/mm/tlb.cc b/na/mm/tlb.cc
deleted file mode 100644
index b7a8ba7..0000000
--- a/na/mm/tlb.cc
+++ /dev/null
@@ -1,90 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-or-later */
-/* Copyright (C) 2026 Richard Qin */
-
-/*
- * tlb.cc — TLB shootdown implementation
- *
- * Uses RISC-V SBI Remote Fence extension to perform cross-hart
- * TLB invalidation. SBI firmware handles the IPI delivery and
- * synchronization — the call blocks until all target harts have
- * completed their local sfence.vma.
- *
- * When a VmSpace is provided, we use its cpu_mask to target only
- * the CPUs that have this address space loaded in satp. This avoids
- * unnecessary IPI storms on unrelated cores.
- *
- * Reference: docs/specs/linux_abi_mm_plan.md §SMP §2.1
- */
-
-#include <lume/tlb.h>
-#include <lume/vmm.h>
-#include <lume/config.h>
-#include <arch/mmu.h>
-#include <arch/tlb.h>
-#include <arch/cpu.h>
-
-/*
- * If a shootdown range exceeds this many pages, do a full flush
- * instead of per-page invalidation.
- */
-static constexpr uint64 kShootdownPageThreshold = 32;
-
-/*
- * Build a cpu mask for all online CPUs (fallback when mm is null).
- */
-namespace {
-uint64 all_cpus_mask()
-{
- uint64 mask = 0;
- for (uint32 i = 0; i < kMaxCpus; i++) {
- mask |= (1ULL << i);
- }
- return mask;
-}
-
-/*
- * Get the target cpu mask: use mm->cpu_mask when available,
- * fall back to all_cpus_mask for kernel address space (mm == nullptr).
- */
-uint64 target_cpu_mask(VmSpace* mm)
-{
- if (mm) {
- uint64 mask = mm->cpu_mask.load(__ATOMIC_RELAXED);
- /* Always include our own CPU — we may still have stale TLB */
- mask |= (1ULL << arch::cpu::id());
- return mask;
- }
- return all_cpus_mask();
-}
-} // namespace
-
-void tlb_shootdown_range(VmSpace* mm, uint64 start, uint64 end)
-{
- if (start >= end) {
- return;
- }
-
- uint64 num_pages = (end - start) / kPageSize;
- uint64 cpu_mask = target_cpu_mask(mm);
-
- if (num_pages > kShootdownPageThreshold) {
- /* Full flush on target CPUs via arch backend */
- arch::tlb::flush_remote(cpu_mask, 0, ~0ULL);
- return;
- }
-
- /* Range flush on target CPUs via arch backend */
- arch::tlb::flush_remote(cpu_mask, start, end - start);
-}
-
-void tlb_shootdown_page(VmSpace* mm, uint64 va)
-{
- uint64 cpu_mask = target_cpu_mask(mm);
- arch::tlb::flush_remote(cpu_mask, va, kPageSize);
-}
-
-void tlb_shootdown_all()
-{
- uint64 cpu_mask = all_cpus_mask();
- arch::tlb::flush_remote(cpu_mask, 0, ~0ULL);
-}
diff --git a/na/mm/vma.cc b/na/mm/vma.cc
deleted file mode 100644
index da89e08..0000000
--- a/na/mm/vma.cc
+++ /dev/null
@@ -1,640 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-or-later */
-/* Copyright (C) 2026 Richard Qin */
-
-/*
- * vma.cc — Virtual Memory Area management (MI layer)
- *
- * Manages user address spaces using a red-black tree of VMA regions.
- * Implements demand paging: vmm_map_user records intent only;
- * physical pages are allocated on page fault via vmm_handle_page_fault.
- *
- * MI/MD separation: this file NEVER references pmap::PTE_* constants.
- * Permission translation is delegated to pmap::vm_perm_to_pte().
- *
- * Reference: docs/specs/vmm.md
- */
-
-#include <lume/vmm.h>
-#include <lume/addr.h>
-#include <lume/config.h>
-#include <lume/errno.h>
-#include <lume/pmm.h>
-#include <lume/new.h>
-#include <lume/klog.h>
-#include <arch/pmap.h>
-#include <lume/tlb.h>
-#include <lume/mm_layout.h>
-
-/* ------------------------------------------------------------------ */
-/* Rbtree callbacks for VMA interval tree */
-/* ------------------------------------------------------------------ */
-
-static bool vma_less(struct rb_node* node, const struct rb_node* parent) {
- VmAreaStruct* n = reinterpret_cast<VmAreaStruct*>(
- reinterpret_cast<char*>(node) - __builtin_offsetof(VmAreaStruct, rb));
- const VmAreaStruct* p = reinterpret_cast<const VmAreaStruct*>(
- reinterpret_cast<const char*>(parent) - __builtin_offsetof(VmAreaStruct, rb));
- return n->start < p->start;
-}
-
-static int vma_cmp(const void* key, const struct rb_node* node) {
- uint64 addr = *static_cast<const uint64*>(key);
- const VmAreaStruct* n = reinterpret_cast<const VmAreaStruct*>(
- reinterpret_cast<const char*>(node) - __builtin_offsetof(VmAreaStruct, rb));
- if (addr < n->start) return -1;
- if (addr >= n->end) return 1;
- return 0;
-}
-
-static VmAreaStruct* node_to_vma(struct rb_node* node)
-{
- return reinterpret_cast<VmAreaStruct*>(
- reinterpret_cast<char*>(node) - __builtin_offsetof(VmAreaStruct, rb));
-}
-
-static uint64 align_up_any(uint64 value, uint64 align)
-{
- if (align == 0) {
- return value;
- }
- return ((value + align - 1) / align) * align;
-}
-
-static bool range_overlaps(uint64 a_start, uint64 a_end,
- uint64 b_start, uint64 b_end)
-{
- return a_start < b_end && b_start < a_end;
-}
-
-
-
-static bool vma_overlaps_locked(VmSpace* space, uint64 start, uint64 end)
-{
- struct rb_node* node = rb_first(&space->vma_tree);
- while (node) {
- VmAreaStruct* vma = node_to_vma(node);
- if (range_overlaps(start, end, vma->start, vma->end)) {
- return true;
- }
- node = rb_next(node);
- }
- return false;
-}
-
-static bool vma_range_covered_locked(VmSpace* space, uint64 start, uint64 end)
-{
- uint64 cursor = start;
- struct rb_node* node = rb_first(&space->vma_tree);
- while (node && cursor < end) {
- VmAreaStruct* vma = node_to_vma(node);
- if (vma->end <= cursor) {
- node = rb_next(node);
- continue;
- }
- if (vma->start > cursor) {
- return false;
- }
- if (vma->end > cursor) {
- cursor = vma->end;
- }
- node = rb_next(node);
- }
- return cursor >= end;
-}
-
-static void vma_try_merge_neighbors_locked(VmSpace* space, VmAreaStruct* vma)
-{
- struct rb_node* prev_node = rb_prev(&vma->rb);
- if (prev_node) {
- VmAreaStruct* prev = node_to_vma(prev_node);
- if (prev->end == vma->start && prev->perm == vma->perm) {
- prev->end = vma->end;
- rb_erase(&vma->rb, &space->vma_tree);
- delete vma;
- vma = prev;
- }
- }
-
- struct rb_node* next_node = rb_next(&vma->rb);
- if (next_node) {
- VmAreaStruct* next = node_to_vma(next_node);
- if (vma->end == next->start && vma->perm == next->perm) {
- vma->end = next->end;
- rb_erase(&next->rb, &space->vma_tree);
- delete next;
- }
- }
-}
-
-static int vma_split_locked(VmSpace* space, VmAreaStruct* vma, uint64 split)
-{
- if (split <= vma->start || split >= vma->end) {
- return 0;
- }
-
- VmAreaStruct* right = new VmAreaStruct;
- if (!right) {
- return -ENOMEM;
- }
-
- right->start = split;
- right->end = vma->end;
- right->perm = vma->perm;
- right->mm = space;
-
- vma->end = split;
- rb_add(&right->rb, &space->vma_tree, vma_less);
- return 0;
-}
-
-static int vma_split_range_locked(VmSpace* space, uint64 start, uint64 end)
-{
- if (end <= start) {
- return 0;
- }
-
- struct rb_node* start_node = rb_find(&start, &space->vma_tree, vma_cmp);
- if (!start_node) {
- return -ENOMEM;
- }
- VmAreaStruct* start_vma = node_to_vma(start_node);
- int ret = vma_split_locked(space, start_vma, start);
- if (ret < 0) {
- return ret;
- }
-
- uint64 end_key = end - 1;
- struct rb_node* end_node = rb_find(&end_key, &space->vma_tree, vma_cmp);
- if (!end_node) {
- return -ENOMEM;
- }
- VmAreaStruct* end_vma = node_to_vma(end_node);
- return vma_split_locked(space, end_vma, end);
-}
-
-/* ------------------------------------------------------------------ */
-/* VmSpace lifecycle */
-/* ------------------------------------------------------------------ */
-
-VmSpace* VmSpace::create() {
- VmSpace* space = new VmSpace;
- if (!space) return nullptr;
-
- space->root_pa = pmap::create();
- if (!space->root_pa) {
- delete space;
- return nullptr;
- }
-
- space->vma_tree = RB_ROOT;
- space->lock.init("VmSpace");
- space->refcount.store(1, __ATOMIC_RELAXED);
- space->brk_start = 0;
- space->brk_end = 0;
- return space;
-}
-
-void VmSpace::incref() {
- refcount.fetch_add(1, __ATOMIC_RELAXED);
-}
-
-void VmSpace::put() {
- if (refcount.fetch_sub(1, __ATOMIC_ACQ_REL) == 1) {
- destroy();
- }
-}
-
-void VmSpace::destroy() {
- /* Drain VMA tree under lock, then release BEFORE delete this */
- lock.acquire();
- uint64 root = root_pa;
- struct rb_node* node = rb_first(&vma_tree);
- while (node) {
- VmAreaStruct* vma = reinterpret_cast<VmAreaStruct*>(
- reinterpret_cast<char*>(node) - __builtin_offsetof(VmAreaStruct, rb));
-
- /* Decref and dec mapcount for all mapped physical pages */
- for (uint64 addr = vma->start; addr < vma->end; addr += kPageSize) {
- uint64 pa;
- if (pmap::lookup(root, addr, &pa)) {
- Page* pg = pa_to_page(phys_addr(pa));
- page_dec_mapcount(pg);
- page_decref(pg);
- }
- }
-
- node = rb_next(node);
- rb_erase(&vma->rb, &vma_tree);
- delete vma;
- }
- lock.release();
-
- /* Destroy hardware page tables outside the lock */
- pmap::destroy(root);
-
- /* Finally free VmSpace itself */
- delete this;
-}
-
-/* ------------------------------------------------------------------ */
-/* Public API: VMA manipulation */
-/* ------------------------------------------------------------------ */
-
-int vmm_map_user(VmSpace* space, uint64 va, uint64 len, uint64 perm) {
- if (!space) return -EINVAL;
-
- va = page_align_down(va);
- len = page_align_up(len);
- if (len == 0) return 0;
- if (va < kUserVaBase) return -EINVAL;
- if (va >= kUserVaLimit) return -EINVAL;
- if (va + len < va || va + len > kUserVaLimit) return -EINVAL;
-
- VmAreaStruct* vma = new VmAreaStruct;
- if (!vma) return -ENOMEM;
- vma->start = va;
- vma->end = va + len;
- vma->perm = perm | VM_USER; // Enforce user-mode access (MI flag)
- vma->mm = space;
-
- LockGuard guard(space->lock);
- if (vma_overlaps_locked(space, vma->start, vma->end)) {
- delete vma;
- return -EINVAL;
- }
- rb_add(&vma->rb, &space->vma_tree, vma_less);
- vma_try_merge_neighbors_locked(space, vma);
- return 0;
-}
-
-int vmm_unmap_user(VmSpace* space, uint64 va, uint64 len) {
- if (!space) return -EINVAL;
- va = page_align_down(va);
- len = page_align_up(len);
- if (len == 0) return 0;
- if (va < kUserVaBase) return -EINVAL;
- if (va >= kUserVaLimit) return -EINVAL;
- if (va + len < va || va + len > kUserVaLimit) return -EINVAL;
- uint64 end = va + len;
-
- LockGuard guard(space->lock);
- struct rb_node* node = rb_first(&space->vma_tree);
- while (node) {
- VmAreaStruct* vma = node_to_vma(node);
- struct rb_node* next = rb_next(node);
-
- if (vma->end <= va) {
- node = next;
- continue;
- }
- if (vma->start >= end) {
- break;
- }
-
- uint64 unmap_start = (vma->start > va) ? vma->start : va;
- uint64 unmap_end = (vma->end < end) ? vma->end : end;
-
- if (unmap_start < unmap_end) {
- for (uint64 addr = unmap_start; addr < unmap_end; addr += kPageSize) {
- uint64 pa;
- if (pmap::lookup(space->root_pa, addr, &pa)) {
- pmap::unmap(space->root_pa, addr);
- Page* pg = pa_to_page(phys_addr(pa));
- page_dec_mapcount(pg);
- page_decref(pg);
- }
- }
- }
-
- if (unmap_start == vma->start && unmap_end == vma->end) {
- rb_erase(&vma->rb, &space->vma_tree);
- delete vma;
- } else if (unmap_start == vma->start) {
- rb_erase(&vma->rb, &space->vma_tree);
- vma->start = unmap_end;
- rb_add(&vma->rb, &space->vma_tree, vma_less);
- } else if (unmap_end == vma->end) {
- vma->end = unmap_start;
- } else {
- VmAreaStruct* right = new VmAreaStruct;
- if (!right) {
- return -ENOMEM;
- }
- right->start = unmap_end;
- right->end = vma->end;
- right->perm = vma->perm;
- right->mm = space;
-
- vma->end = unmap_start;
- rb_add(&right->rb, &space->vma_tree, vma_less);
- break;
- }
-
- node = next;
- }
- return 0;
-}
-
-int vmm_protect_user(VmSpace* space, uint64 va, uint64 len, uint64 perm)
-{
- if (!space) return -EINVAL;
- va = page_align_down(va);
- len = page_align_up(len);
- if (len == 0) return -EINVAL;
- if (va < kUserVaBase) return -EINVAL;
- if (va >= kUserVaLimit) return -EINVAL;
- if (va + len < va || va + len > kUserVaLimit) return -EINVAL;
-
- uint64 end = va + len;
-
- LockGuard guard(space->lock);
- if (!vma_range_covered_locked(space, va, end)) {
- return -ENOMEM;
- }
- int ret = vma_split_range_locked(space, va, end);
- if (ret < 0) {
- return ret;
- }
-
- bool tlb_flush_needed = false;
- struct rb_node* node = rb_find(&va, &space->vma_tree, vma_cmp);
- while (node) {
- VmAreaStruct* vma = node_to_vma(node);
- if (vma->start >= end) {
- break;
- }
- if (vma->end <= va) {
- node = rb_next(node);
- continue;
- }
-
- vma->perm = (perm | VM_USER);
-
- for (uint64 addr = vma->start; addr < vma->end; addr += kPageSize) {
- uint64 pa;
- if (!pmap::lookup(space->root_pa, addr, &pa)) {
- continue;
- }
-
- if (perm == 0) {
- Page* pg = pa_to_page(phys_addr(pa));
- page_dec_mapcount(pg);
- page_decref(pg);
- pmap::unmap(space->root_pa, addr);
- tlb_flush_needed = true;
- continue;
- }
-
- uint64 pte_perm = pmap::vm_perm_to_pte(vma->perm);
- if ((vma->perm & VM_WRITE) && pmap::is_cow(space->root_pa, addr)) {
- pte_perm &= ~pmap::PTE_W;
- pte_perm |= pmap::PTE_COW;
- }
-
- int ret = pmap::protect(space->root_pa, addr, pte_perm);
- if (ret < 0) {
- return ret;
- }
- tlb_flush_needed = true;
- }
-
- node = rb_next(node);
- }
-
- if (tlb_flush_needed) {
- tlb_shootdown_range(space, va, end);
- }
- return 0;
-}
-
-uint64 mm_find_free_va(VmSpace* space, uint64 length, uint64 align)
-{
- if (!space) return 0;
- length = page_align_up(length);
- if (length == 0) return 0;
- if (align < kPageSize) align = kPageSize;
-
- LockGuard guard(space->lock);
-
- /* Linux: arch_get_unmapped_area() starts from mm->mmap_base.
- * RISC-V: TASK_UNMAPPED_BASE = PAGE_ALIGN(TASK_SIZE / 3).
- * Fall back to kUserVaBase if mmap_base not set (kernel VmSpaces). */
- uint64 base = space->mmap_base ? space->mmap_base : kUserVaBase;
- uint64 cursor = align_up_any(base, align);
- struct rb_node* node = rb_first(&space->vma_tree);
- while (node) {
- VmAreaStruct* vma = node_to_vma(node);
- if (cursor <= vma->start) {
- if (cursor > kUserVaLimit - length) {
- return 0;
- }
- if (cursor + length <= vma->start) {
- return cursor;
- }
- }
- if (cursor < vma->end) {
- cursor = vma->end;
- }
- cursor = align_up_any(cursor, align);
- node = rb_next(node);
- }
-
- if (cursor > kUserVaLimit - length) {
- return 0;
- }
- return cursor;
-}
-
-bool vma_range_covered(VmSpace* space, uint64 start, uint64 end)
-{
- if (!space) return false;
- start = page_align_down(start);
- end = page_align_up(end);
- if (end <= start) return true;
-
- LockGuard guard(space->lock);
- return vma_range_covered_locked(space, start, end);
-}
-
-/* ------------------------------------------------------------------ */
-/* Page fault handler (demand paging) */
-/* ------------------------------------------------------------------ */
-
-static bool vma_allows(const VmAreaStruct* vma, VmFaultCause cause)
-{
- switch (cause) {
- case VmFaultCause::Read:
- return (vma->perm & VM_READ) != 0;
- case VmFaultCause::Write:
- return (vma->perm & VM_WRITE) != 0;
- case VmFaultCause::Exec:
- return (vma->perm & VM_EXEC) != 0;
- default:
- return false;
- }
-}
-
-int vmm_handle_page_fault(VmSpace* space, uint64 fault_addr, VmFaultCause cause) {
- if (!space) return -EINVAL;
-
- uint64 aligned_addr = page_align_down(fault_addr);
-
- LockGuard guard(space->lock);
- struct rb_node* node = rb_find(&fault_addr, &space->vma_tree, vma_cmp);
- if (!node) {
- return -EFAULT; // No VMA covers this address → segfault
- }
-
- VmAreaStruct* vma = reinterpret_cast<VmAreaStruct*>(
- reinterpret_cast<char*>(node) - __builtin_offsetof(VmAreaStruct, rb));
-
- if (!vma_allows(vma, cause)) {
- return -EFAULT;
- }
-
- /* Check if page is already mapped */
- uint64 paddr;
- if (pmap::lookup(space->root_pa, aligned_addr, &paddr)) {
- if (cause == VmFaultCause::Write && pmap::is_cow(space->root_pa, aligned_addr)) {
- /* Store Page Fault on a COW page */
- Page* page = pmm_alloc_page();
- if (!page) return -ENOMEM;
- uint64 new_pa = page_to_pa(page).raw;
-
- /* Copy page contents */
- uint8* dst = reinterpret_cast<uint8*>(pa_to_va(phys_addr(new_pa)).raw);
- const uint8* src = reinterpret_cast<const uint8*>(pa_to_va(phys_addr(paddr)).raw);
- for (uint64 i = 0; i < kPageSize; i++) {
- dst[i] = src[i];
- }
-
- /* Remap with write permission, clearing COW */
- uint64 pte_perm = pmap::vm_perm_to_pte(vma->perm);
- if (pmap::map(space->root_pa, aligned_addr, new_pa, pte_perm) != 0) {
- pmm_free_page(page);
- return -ENOMEM;
- }
-
- /* New page gets mapcount = 1 (we just mapped it) */
- page_inc_mapcount(page);
-
- /* Decrement mapcount and refcount of old page */
- Page* old_page = pa_to_page(phys_addr(paddr));
- page_dec_mapcount(old_page);
- page_decref(old_page);
-
- /* Flush TLB to remove old read-only cached translation */
- tlb_shootdown_page(space, aligned_addr);
- return 0;
- }
- /* DEBUG: page already mapped but still faulting — dump PTE */
- {
- uint64 cur_satp;
- asm volatile("csrr %0, satp" : "=r"(cur_satp));
- printk(KERN_ERR "fault: va=%llx pa=%llx cause=%d pte=%llx satp=%llx root=%llx\n",
- aligned_addr, paddr, static_cast<int>(cause),
- pmap::lookup_pte(space->root_pa, aligned_addr),
- cur_satp, space->root_pa);
- }
- return 0; // Already mapped, maybe read fault on existing page
- }
-
- /* Allocate physical page and zero it.
- * Linux: do_anonymous_page() uses alloc_zeroed_user_highpage_movable().
- * Anonymous pages MUST be zeroed — userspace expects MAP_ANONYMOUS
- * pages to contain all zeros (POSIX requirement). */
- Page* page = pmm_alloc_page();
- if (!page) return -ENOMEM;
-
- uint64 pa = page_to_pa(page).raw;
-
- /* Zero the page before mapping it to userspace */
- /* Zero the page before mapping it to userspace */
- uint8* kva = reinterpret_cast<uint8*>(pa_to_va(PhysAddr{pa}).raw);
- for (uint64 i = 0; i < kPageSize; i++) kva[i] = 0;
-
- /* Translate MI permission flags to hardware PTE bits (MD layer) */
- uint64 pte_perm = pmap::vm_perm_to_pte(vma->perm);
-
- if (pmap::map(space->root_pa, aligned_addr, pa, pte_perm) != 0) {
- pmm_free_page(page);
- return -ENOMEM;
- }
-
- /* Initial mapping: refcount is already 1 from pmm_alloc_page.
- * Set mapcount = 1 to track this PTE mapping. */
- page_inc_mapcount(page);
-
- return 0;
-}
-
-/* ------------------------------------------------------------------ */
-/* Fork / Clone (COW infrastructure) */
-/* ------------------------------------------------------------------ */
-
-VmSpace* vmm_clone(VmSpace* parent) {
- if (!parent) return nullptr;
-
- VmSpace* child = VmSpace::create();
- if (!child) return nullptr;
-
- child->brk_start = parent->brk_start;
- child->brk_end = parent->brk_end;
-
- LockGuard parent_guard(parent->lock);
-
- struct rb_node* node = rb_first(&parent->vma_tree);
- while (node) {
- VmAreaStruct* parent_vma = reinterpret_cast<VmAreaStruct*>(
- reinterpret_cast<char*>(node) - __builtin_offsetof(VmAreaStruct, rb));
-
- VmAreaStruct* child_vma = new VmAreaStruct;
- if (!child_vma) {
- child->destroy();
- return nullptr;
- }
- child_vma->start = parent_vma->start;
- child_vma->end = parent_vma->end;
- child_vma->perm = parent_vma->perm;
- child_vma->mm = child;
-
- /* Copy pages and mark COW if writable */
- for (uint64 va = child_vma->start; va < child_vma->end; va += kPageSize) {
- uint64 pa;
- if (pmap::lookup(parent->root_pa, va, &pa)) {
- /* Make parent COW if it was writable */
- if (child_vma->perm & VM_WRITE) {
- pmap::make_cow(parent->root_pa, va);
- /* Flush TLB for parent since we reduced permissions */
- tlb_shootdown_page(parent, va);
- }
-
- /* Map in child as COW or read-only */
- uint64 pte_perm = pmap::vm_perm_to_pte(child_vma->perm);
- if (child_vma->perm & VM_WRITE) {
- pte_perm &= ~pmap::PTE_W;
- pte_perm |= pmap::PTE_COW;
- }
-
- if (pmap::map(child->root_pa, va, pa, pte_perm) != 0) {
- /* If map fails, we leak what we did in this loop for now,
- * but child->destroy() will clean it up safely since
- * we increment refcounts as we go. */
- delete child_vma;
- child->destroy();
- return nullptr;
- }
-
- /* Increment physical page refcount and mapcount */
- Page* shared_page = pa_to_page(phys_addr(pa));
- page_incref(shared_page);
- page_inc_mapcount(shared_page);
- }
- }
-
- rb_add(&child_vma->rb, &child->vma_tree, vma_less);
- node = rb_next(node);
- }
-
- return child;
-}
diff --git a/na/mm/vmm.cc b/na/mm/vmm.cc
deleted file mode 100644
index 51617e3..0000000
--- a/na/mm/vmm.cc
+++ /dev/null
@@ -1,235 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-or-later */
-/* Copyright (C) 2026 Richard Qin */
-
-/*
- * vmm.cc — Virtual Memory Manager (Phase 1: Kernel Page Table)
- *
- * Builds a fine-grained kernel page table, activates it, and retires
- * the early_pgdir identity mapping from entry.S.
- *
- * With memblock integration, the initialization order is:
- * memblock_init → vmm_init (allocates page_map from memblock,
- * maps vmemmap, activates page table) → pmm_init (buddy system
- * starts with page_map already at vmemmap VA — no relocation).
- *
- * Reference: docs/specs/vmm.md
- */
-
-#include <arch/config.h>
-#include <arch/mmu.h>
-#include <arch/pmap.h>
-#include <lume/tlb.h>
-#include <lume/addr.h>
-#include <lume/console.h>
-#include <lume/fdt.h>
-#include <lume/page.h>
-#include <lume/klog.h>
-#include <lume/kprintf.h>
-#include <lume/memblock.h>
-#include <lume/pmm.h>
-#include <lume/types.h>
-#include <lume/vmm.h>
-#include <lume/config.h>
-
-/* Linker-exported section boundaries (virtual addresses) */
-extern "C" char _stext[], _etext[];
-extern "C" char _srodata[], _erodata[];
-extern "C" char _sdata[], _edata[];
-extern "C" char _sbss[], _ebss[];
-extern "C" char _kernel_end[];
-
-/* Global kernel page table root PA — external linkage for task_exit */
-uint64 g_kernel_pgtbl = 0;
-
-namespace {
-
-/* VMM exports for PMM: page_map location after vmemmap mapping. */
-uint64 g_vmemmap_page_map_pa = 0;
-uint64 g_vmemmap_page_map_size = 0;
-uint64 g_vmemmap_num_pages = 0;
-
-/* Convert a linker symbol to PA (kernel is executing in higher-half VA). */
-inline uint64 sym_to_pa(const void *sym) {
- return boot_va_to_pa(virt_addr(reinterpret_cast<uint64>(sym))).raw;
-}
-
-/* Map a PA range into the kernel page table via the direct map. */
-void map_range(uint64 root, uint64 pa_start, uint64 pa_end,
- uint64 perm) {
- for (uint64 pa = pa_start; pa < pa_end; pa += kPageSize) {
- uint64 va = pa_to_va(phys_addr(pa)).raw;
- int ret = pmap::map(root, va, pa, perm | pmap::PTE_G);
- if (ret != 0) {
- kernel_panic("vmm_init: pmap::map failed");
- }
- }
-}
-
-/* Map a PA range into the kernel page table at the fixed linked VA. */
-void map_range_linked(uint64 root, uint64 pa_start, uint64 pa_end,
- uint64 perm) {
- for (uint64 pa = pa_start; pa < pa_end; pa += kPageSize) {
- uint64 va = boot_pa_to_va(phys_addr(pa)).raw;
- int ret = pmap::map(root, va, pa, perm | pmap::PTE_G);
- if (ret != 0) {
- kernel_panic("vmm_init: pmap::map (linked) failed");
- }
- }
-}
-
-/* Map a PA range to an explicit VA base, using 2MB pages where possible. */
-void map_range_at_huge(uint64 root, uint64 va_base, uint64 pa_start,
- uint64 pa_end, uint64 perm) {
- constexpr uint64 k2MB = 2ULL * 1024 * 1024;
- uint64 pa = pa_start;
- uint64 va = va_base;
-
- /* Map 2MB-aligned chunks with mega pages */
- while (pa + k2MB <= pa_end) {
- if ((va & (k2MB - 1)) == 0 && (pa & (k2MB - 1)) == 0) {
- int ret = pmap::map_2mb(root, va, pa, perm | pmap::PTE_G);
- if (ret != 0) {
- kernel_panic("vmm_init: pmap::map_2mb failed");
- }
- pa += k2MB;
- va += k2MB;
- } else {
- break;
- }
- }
-
- /* Map remaining 4KB tail */
- while (pa < pa_end) {
- int ret = pmap::map(root, va, pa, perm | pmap::PTE_G);
- if (ret != 0) {
- kernel_panic("vmm_init: vmemmap 4KB pmap::map failed");
- }
- pa += kPageSize;
- va += kPageSize;
- }
-}
-
-} // namespace
-
-uint64 vmm_get_page_map_pa() { return g_vmemmap_page_map_pa; }
-uint64 vmm_get_page_map_size() { return g_vmemmap_page_map_size; }
-uint64 vmm_get_num_pages() { return g_vmemmap_num_pages; }
-
-void vmm_init() {
- /* 1. Allocate new root page table (from memblock) */
- uint64 root = pmap::create();
- if (!root) {
- kernel_panic("vmm_init: failed to create root page table");
- }
-
- /* 2. Compute PA boundaries of each kernel section */
- uint64 text_start = page_align_down(sym_to_pa(_stext));
- uint64 text_end = page_align_up(sym_to_pa(_etext));
- uint64 ro_start = page_align_down(sym_to_pa(_srodata));
- uint64 ro_end = page_align_up(sym_to_pa(_erodata));
- uint64 data_start = page_align_down(sym_to_pa(_sdata));
- uint64 bss_end = page_align_up(sym_to_pa(_ebss));
-
- /* Memory extent from memblock */
- uint64 mem_base = memblock_get_mem_base();
- uint64 mem_size = memblock_get_mem_size();
- uint64 mem_end = mem_base + mem_size;
-
- /* 3. Map kernel image at FIXED linked VA.
- *
- * Kernel text/data are linked at the bootstrap higher-half addresses
- * (kDirectMapBaseDefault + PA). They must remain executable/readable at
- * those addresses until full kernel-image relocation is implemented.
- */
- map_range_linked(root, text_start, text_end, pmap::PTE_R | pmap::PTE_X);
-
- /* 4. Map .rodata (R) at fixed linked VA.
- * We map [_srodata, _sdata) so that the .lume.reloc section (which
- * sits in its own padded output section between _erodata and _sdata)
- * is also covered. Relocator only reads it during early boot, but
- * mapping it keeps post-vmm accesses to __lume_reloc_* symbols sane. */
- map_range_linked(root, ro_start, page_align_down(sym_to_pa(_sdata)),
- pmap::PTE_R);
-
- /* 5. Map .data + .bss (RW) at fixed linked VA */
- map_range_linked(root, data_start, bss_end, pmap::PTE_R | pmap::PTE_W);
-
- /* 6. Map usable physical RAM into the KASLR-slid direct map.
- *
- * The direct map provides pa_to_va() / va_to_pa() for the physical
- * address space. We map from kPhysStart (the kernel's linked load
- * address), skipping [mem_base, kPhysStart) because that region is
- * PMP-protected firmware (OpenSBI) — S-mode cannot access it.
- *
- * Linux does the same on RISC-V: the linear map covers only the
- * usable RAM range, not firmware-reserved regions.
- *
- * Uses 2MB huge pages for efficiency (64 entries for 128MB vs 32K).
- */
- uint64 dm_start = kPhysStart;
- if (dm_start < mem_base) {
- dm_start = mem_base;
- }
- uint64 dm_va_base = pa_to_va(phys_addr(dm_start)).raw;
- map_range_at_huge(root, dm_va_base, dm_start, mem_end,
- pmap::PTE_R | pmap::PTE_W);
-
- /* 7. Allocate page_map from memblock (top of RAM) and map at VMEMMAP.
- *
- * Since memblock is active (PMM not yet initialized), pmap::map
- * allocates page table pages from memblock too. page_map starts
- * life at the vmemmap VA — no relocation ever needed.
- */
- uint64 num_pages = mem_size / kPageSize;
- uint64 page_map_bytes = num_pages * sizeof(Page);
- uint64 page_map_span = page_align_up(page_map_bytes);
- uint64 page_map_pa = memblock_alloc_top(page_map_span, kPageSize);
-
- if (!page_map_pa) {
- kernel_panic("vmm_init: memblock failed to allocate page_map");
- }
-
- kprintf("[vmm] vmemmap: %llu pages, %llu KB at PA 0x%llx\n", num_pages,
- page_map_span / 1024, page_map_pa);
-
- /* Map page_map at vmemmap_base from KaslrLayout (randomised by KASLR), using 2MB pages where possible */
- map_range_at_huge(root, arch::kaslr_layout_get().vmemmap_base, page_map_pa,
- page_map_pa + page_map_span, pmap::PTE_R | pmap::PTE_W);
-
- /* Export for pmm_init() */
- g_vmemmap_page_map_pa = page_map_pa;
- g_vmemmap_page_map_size = page_map_bytes;
- g_vmemmap_num_pages = num_pages;
-
- /* 8. Pre-map console MMIO before page table switch */
- g_kernel_pgtbl = root;
- console_init();
-
- /* 9. Activate the new page table */
- pmap::activate(root);
-
- /* Console MMIO access must now use runtime (KASLR) direct-map base. */
- console_use_runtime_mapping();
-
- kprintf("[vmm] vmm_init complete (vmemmap at 0x%llx)\n", arch::kaslr_layout_get().vmemmap_base);
-}
-
-void vmm_init_ap() {
- if (!g_kernel_pgtbl) {
- kernel_panic("vmm_init_ap: kernel page table not ready");
- }
- pmap::activate(g_kernel_pgtbl);
-}
-
-void vmm_map_kernel_mmio(uint64 pa, uint64 size) {
- if (!g_kernel_pgtbl) {
- kernel_panic("vmm_map_kernel_mmio: kernel page table not ready");
- }
-
- uint64 aligned_pa = page_align_down(pa);
- uint64 aligned_size = page_align_up(pa + size) - aligned_pa;
-
- map_range(g_kernel_pgtbl, aligned_pa, aligned_pa + aligned_size,
- pmap::PTE_R | pmap::PTE_W | pmap::PTE_G);
- tlb_shootdown_all();
-}