summaryrefslogtreecommitdiff
path: root/mm
diff options
context:
space:
mode:
Diffstat (limited to 'mm')
-rw-r--r--mm/memblock.cc257
-rw-r--r--mm/module.mk7
-rw-r--r--mm/page_cache.cc252
-rw-r--r--mm/pmm.cc528
-rw-r--r--mm/slab.cc321
-rw-r--r--mm/slab.h83
-rw-r--r--mm/tlb.cc90
-rw-r--r--mm/vma.cc640
-rw-r--r--mm/vmm.cc235
9 files changed, 2413 insertions, 0 deletions
diff --git a/mm/memblock.cc b/mm/memblock.cc
new file mode 100644
index 0000000..aee0245
--- /dev/null
+++ b/mm/memblock.cc
@@ -0,0 +1,257 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/* Copyright (C) 2026 Richard Qin */
+
+/*
+ * memblock.cc — Boot-time physical memory allocator
+ *
+ * Simple region tracker: one memory region (from FDT) plus an array
+ * of reserved regions. Allocation searches for gaps between reserved
+ * regions. All operations are O(n_reserved), which is fine for the
+ * ~20 allocations that happen during early boot.
+ *
+ * Reference: inspired by Linux kernel's mm/memblock.c
+ */
+
+#include <lume/memblock.h>
+#include <lume/fdt.h>
+#include <lume/klog.h>
+#include <lume/kprintf.h>
+#include <lume/config.h>
+#include <lume/addr.h>
+
+extern "C" char _stext[], _kernel_end[];
+
+/* ------------------------------------------------------------------ */
+/* Module state */
+/* ------------------------------------------------------------------ */
+
+namespace {
+
+constexpr uint64 kMB = 1024ULL * 1024ULL;
+
+struct {
+ uint64 mem_base;
+ uint64 mem_size;
+
+ struct {
+ uint64 base;
+ uint64 size;
+ } reserved[kMemblockMaxRegions];
+ int reserved_count;
+ bool active;
+} g_mb;
+
+/* ------------------------------------------------------------------ */
+/* Internal helpers */
+/* ------------------------------------------------------------------ */
+
+/* Insert a reserved region, keeping the array sorted by base. */
+void insert_reserved(uint64 base, uint64 size)
+{
+ if (g_mb.reserved_count >= kMemblockMaxRegions) {
+ kernel_panic("memblock: too many reserved regions");
+ }
+
+ int i = g_mb.reserved_count;
+ while (i > 0 && g_mb.reserved[i - 1].base > base) {
+ g_mb.reserved[i] = g_mb.reserved[i - 1];
+ i--;
+ }
+ g_mb.reserved[i] = {base, size};
+ g_mb.reserved_count++;
+}
+
+/* Check if [base, base+size) overlaps any reserved region. */
+bool overlaps_reserved(uint64 base, uint64 size)
+{
+ uint64 end = base + size;
+ for (int i = 0; i < g_mb.reserved_count; i++) {
+ uint64 rb = g_mb.reserved[i].base;
+ uint64 re = rb + g_mb.reserved[i].size;
+ if (base < re && end > rb) {
+ return true;
+ }
+ }
+ return false;
+}
+
+/* Find the end of the first reserved region that covers `addr`. */
+uint64 reserved_end_covering(uint64 addr)
+{
+ for (int i = 0; i < g_mb.reserved_count; i++) {
+ uint64 rb = g_mb.reserved[i].base;
+ uint64 re = rb + g_mb.reserved[i].size;
+ if (addr >= rb && addr < re) {
+ return re;
+ }
+ }
+ return addr;
+}
+
+} // namespace
+
+/* ------------------------------------------------------------------ */
+/* Public API */
+/* ------------------------------------------------------------------ */
+
+void memblock_init(uint64 fdt_paddr)
+{
+
+ /* 1. Get memory layout from FDT */
+ uint64 mem_base = 0;
+ uint64 mem_size = 0;
+ fdt_early_get_mem_info(&mem_base, &mem_size);
+ if (mem_size == 0) {
+ kernel_panic("memblock_init: FDT reports zero memory");
+ }
+
+ g_mb.mem_base = mem_base;
+ g_mb.mem_size = mem_size;
+ g_mb.reserved_count = 0;
+ g_mb.active = true;
+
+ /* 2. Reserve firmware region [mem_base, kPhysStart).
+ *
+ * Everything below the kernel's linked load address is firmware
+ * territory (OpenSBI), which is PMP-protected. */
+ {
+ uint64 fw_end = kPhysStart;
+ if (fw_end > mem_base) {
+ memblock_reserve(mem_base, fw_end - mem_base);
+ }
+ }
+
+ /* 2b. Reserve stale pre-relocation kernel image.
+ *
+ * After physical KASLR, the kernel is relocated from its original
+ * load address (kPhysStart) to a new PA. The old copy at
+ * [kPhysStart, kPhysStart + kernel_size) must not be allocated —
+ * it contains stale code/data that would corrupt page tables or
+ * other structures if reused. */
+ {
+ uint64 ks_pa = boot_va_to_pa(virt_addr(
+ reinterpret_cast<uint64>(_stext))).raw;
+ if (ks_pa != kPhysStart) {
+ /* Kernel was relocated — reserve the original location */
+ uint64 kernel_size = boot_va_to_pa(virt_addr(
+ reinterpret_cast<uint64>(_kernel_end))).raw - ks_pa;
+ memblock_reserve(kPhysStart, kernel_size);
+ }
+ }
+
+ /* 3. Reserve kernel image [_stext, _kernel_end) */
+ uint64 ks_pa = boot_va_to_pa(virt_addr(
+ reinterpret_cast<uint64>(_stext))).raw;
+ uint64 ke_pa = boot_va_to_pa(virt_addr(
+ reinterpret_cast<uint64>(_kernel_end))).raw;
+ memblock_reserve(ks_pa, ke_pa - ks_pa);
+
+ /* 4. Reserve FDT blob — it lives in RAM and must not be overwritten
+ * by vmemmap/page_map or buddy before fdt_unflatten() copies it. */
+ {
+ struct fdt_hdr { uint32 magic; uint32 totalsize; };
+ const auto* fh = reinterpret_cast<const fdt_hdr*>(fdt_paddr);
+ uint32 fdt_size = ((fh->totalsize & 0xFF) << 24) |
+ (((fh->totalsize >> 8) & 0xFF) << 16) |
+ (((fh->totalsize >> 16) & 0xFF) << 8) |
+ ((fh->totalsize >> 24) & 0xFF);
+ memblock_reserve(fdt_paddr, fdt_size);
+ kprintf("[memblock] FDT reserved [0x%llx, 0x%llx)\n",
+ fdt_paddr, fdt_paddr + fdt_size);
+ }
+
+ kprintf("[memblock] RAM [0x%llx, 0x%llx) = %llu MB\n",
+ mem_base, mem_base + mem_size, mem_size / kMB);
+ kprintf("[memblock] kernel reserved [0x%llx, 0x%llx)\n", ks_pa, ke_pa);
+}
+
+void memblock_reserve(uint64 base, uint64 size)
+{
+ uint64 aligned_base = page_align_down(base);
+ uint64 aligned_end = page_align_up(base + size);
+ insert_reserved(aligned_base, aligned_end - aligned_base);
+}
+
+uint64 memblock_alloc_top(uint64 size, uint64 align)
+{
+ uint64 mem_end = g_mb.mem_base + g_mb.mem_size;
+
+ /* Search from top of RAM downward */
+ uint64 end = page_align_down(mem_end);
+ uint64 start = (end - size) & ~(align - 1);
+
+ while (start >= g_mb.mem_base && start + size <= mem_end) {
+ if (!overlaps_reserved(start, size)) {
+ memblock_reserve(start, size);
+ return start;
+ }
+ if (start < align) {
+ break;
+ }
+ start -= align;
+ }
+
+ kernel_panic("memblock_alloc_top: out of memory");
+ return 0;
+}
+
+uint64 memblock_alloc(uint64 size, uint64 align)
+{
+ uint64 mem_end = g_mb.mem_base + g_mb.mem_size;
+ uint64 start = page_align_up(g_mb.mem_base);
+
+ while (start + size <= mem_end) {
+ start = (start + align - 1) & ~(align - 1);
+ if (start + size > mem_end) {
+ break;
+ }
+
+ if (!overlaps_reserved(start, size)) {
+ memblock_reserve(start, size);
+ return start;
+ }
+
+ /* Skip past the overlapping reserved region */
+ uint64 skip = reserved_end_covering(start);
+ if (skip <= start) {
+ start += align;
+ } else {
+ start = skip;
+ }
+ }
+
+ kernel_panic("memblock_alloc: out of memory");
+ return 0;
+}
+
+uint64 memblock_get_mem_base() { return g_mb.mem_base; }
+uint64 memblock_get_mem_size() { return g_mb.mem_size; }
+
+bool memblock_is_active() { return g_mb.active; }
+void memblock_retire() { g_mb.active = false; }
+
+void memblock_for_each_free(void (*cb)(uint64 base, uint64 size, void* ctx),
+ void* ctx)
+{
+ uint64 pos = g_mb.mem_base;
+ uint64 mem_end = g_mb.mem_base + g_mb.mem_size;
+
+ for (int i = 0; i < g_mb.reserved_count && pos < mem_end; i++) {
+ uint64 rb = g_mb.reserved[i].base;
+ if (rb < pos) {
+ uint64 re = g_mb.reserved[i].base + g_mb.reserved[i].size;
+ if (re > pos) {
+ pos = re;
+ }
+ continue;
+ }
+ if (rb > pos) {
+ cb(pos, rb - pos, ctx);
+ }
+ pos = g_mb.reserved[i].base + g_mb.reserved[i].size;
+ }
+
+ if (pos < mem_end) {
+ cb(pos, mem_end - pos, ctx);
+ }
+}
diff --git a/mm/module.mk b/mm/module.mk
new file mode 100644
index 0000000..93cef80
--- /dev/null
+++ b/mm/module.mk
@@ -0,0 +1,7 @@
+SRCS_CC += mm/memblock.cc
+SRCS_CC += mm/pmm.cc
+SRCS_CC += mm/slab.cc
+SRCS_CC += mm/vmm.cc
+SRCS_CC += mm/vma.cc
+SRCS_CC += mm/page_cache.cc
+SRCS_CC += mm/tlb.cc
diff --git a/mm/page_cache.cc b/mm/page_cache.cc
new file mode 100644
index 0000000..25c6a60
--- /dev/null
+++ b/mm/page_cache.cc
@@ -0,0 +1,252 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/* Copyright (C) 2026 Richard Qin */
+
+/*
+ * page_cache.cc — Minimal page cache implementation
+ *
+ * Open-addressing hash table indexed by (mapping, index) pairs.
+ * Fixed-size bucket array allocated at init time via slab.
+ *
+ * This is intentionally simple: no LRU, no writeback, no eviction.
+ * The interface in page_cache.h is designed so that upgrading to a
+ * radix tree + LRU scheme later only changes this file.
+ *
+ * Concurrency: protected by a single spinlock. Acceptable for
+ * initial bring-up; will be replaced by per-bucket or RCU-based
+ * locking when contention becomes measurable.
+ */
+
+#include <lume/page_cache.h>
+#include <lume/page.h>
+#include <lume/pmm.h>
+#include <lume/errno.h>
+#include <lume/new.h>
+#include <kernel/sync/spinlock.h>
+
+/* ------------------------------------------------------------------ */
+/* Hash table internals */
+/* ------------------------------------------------------------------ */
+
+struct CacheEntry {
+ void* mapping; // nullptr = empty slot
+ uint64 index;
+ Page* page;
+};
+
+namespace {
+
+constexpr uint64 kCacheBuckets = 1024;
+
+CacheEntry g_buckets[kCacheBuckets];
+Spinlock g_cache_lock;
+
+uint64 hash_key(void* mapping, uint64 index)
+{
+ uint64 m = reinterpret_cast<uint64>(mapping);
+ /* FNV-1a inspired mixing */
+ uint64 h = m ^ (index * 0x517cc1b727220a95ULL);
+ h ^= h >> 17;
+ h *= 0xbf58476d1ce4e5b9ULL;
+ h ^= h >> 31;
+ return h % kCacheBuckets;
+}
+
+/* Linear probe to find an entry or an empty slot. */
+CacheEntry* find_slot(void* mapping, uint64 index)
+{
+ uint64 start = hash_key(mapping, index);
+ for (uint64 i = 0; i < kCacheBuckets; i++) {
+ uint64 idx = (start + i) % kCacheBuckets;
+ CacheEntry* e = &g_buckets[idx];
+ if (e->mapping == nullptr) {
+ return e; // Empty slot
+ }
+ if (e->mapping == mapping && e->index == index) {
+ return e; // Found
+ }
+ }
+ return nullptr; // Table full
+}
+
+/* Find an existing entry (not an empty slot). */
+CacheEntry* find_entry(void* mapping, uint64 index)
+{
+ uint64 start = hash_key(mapping, index);
+ for (uint64 i = 0; i < kCacheBuckets; i++) {
+ uint64 idx = (start + i) % kCacheBuckets;
+ CacheEntry* e = &g_buckets[idx];
+ if (e->mapping == nullptr) {
+ return nullptr; // Empty = not found
+ }
+ if (e->mapping == mapping && e->index == index) {
+ return e;
+ }
+ }
+ return nullptr;
+}
+
+/*
+ * Remove an entry and fix up the linear-probe chain.
+ * Standard open-addressing deletion with backward shift.
+ */
+void remove_entry(CacheEntry* entry)
+{
+ uint64 removed = static_cast<uint64>(entry - g_buckets);
+ entry->mapping = nullptr;
+ entry->page = nullptr;
+
+ /* Backward-shift deletion to maintain probe chain integrity */
+ uint64 cur = (removed + 1) % kCacheBuckets;
+ while (g_buckets[cur].mapping != nullptr) {
+ uint64 ideal = hash_key(g_buckets[cur].mapping, g_buckets[cur].index);
+ /* Check if cur's ideal position is "before" removed in probe order */
+ bool needs_shift = false;
+ if (removed <= cur) {
+ needs_shift = (ideal <= removed || ideal > cur);
+ } else {
+ needs_shift = (ideal <= removed && ideal > cur);
+ }
+
+ if (needs_shift) {
+ g_buckets[removed] = g_buckets[cur];
+ g_buckets[cur].mapping = nullptr;
+ g_buckets[cur].page = nullptr;
+ removed = cur;
+ }
+ cur = (cur + 1) % kCacheBuckets;
+ }
+}
+
+} // namespace
+
+/* ------------------------------------------------------------------ */
+/* Public API */
+/* ------------------------------------------------------------------ */
+
+void page_cache_init()
+{
+ g_cache_lock.init("page_cache");
+ for (uint64 i = 0; i < kCacheBuckets; i++) {
+ g_buckets[i].mapping = nullptr;
+ g_buckets[i].index = 0;
+ g_buckets[i].page = nullptr;
+ }
+}
+
+Page* page_cache_get(void* mapping, uint64 index)
+{
+ LockGuard guard(g_cache_lock);
+ CacheEntry* e = find_entry(mapping, index);
+ if (!e) {
+ return nullptr;
+ }
+ page_incref(e->page);
+ return e->page;
+}
+
+int page_cache_insert(void* mapping, uint64 index, Page* page)
+{
+ if (!mapping || !page) {
+ return -EINVAL;
+ }
+
+ LockGuard guard(g_cache_lock);
+
+ /* Check for existing entry */
+ CacheEntry* existing = find_entry(mapping, index);
+ if (existing) {
+ return -EEXIST;
+ }
+
+ /* Find an empty slot */
+ CacheEntry* slot = find_slot(mapping, index);
+ if (!slot || slot->mapping != nullptr) {
+ return -ENOMEM; // Table full
+ }
+
+ slot->mapping = mapping;
+ slot->index = index;
+ slot->page = page;
+ page_incref(page); // Cache holds a reference
+
+ return 0;
+}
+
+void page_cache_put(Page* page)
+{
+ if (!page) {
+ return;
+ }
+ page_decref(page);
+}
+
+Page* page_cache_remove(void* mapping, uint64 index)
+{
+ LockGuard guard(g_cache_lock);
+ CacheEntry* e = find_entry(mapping, index);
+ if (!e) {
+ return nullptr;
+ }
+
+ Page* page = e->page;
+ remove_entry(e);
+ page_decref(page); // Release cache's reference
+ return page;
+}
+
+void page_cache_invalidate(void* mapping)
+{
+ if (!mapping) {
+ return;
+ }
+
+ LockGuard guard(g_cache_lock);
+
+ /* Scan all buckets — acceptable for current table size */
+ for (uint64 i = 0; i < kCacheBuckets; i++) {
+ if (g_buckets[i].mapping == mapping) {
+ Page* page = g_buckets[i].page;
+ remove_entry(&g_buckets[i]);
+ page_decref(page);
+ /* After removal, re-check this index since backward shift
+ * may have moved an entry into this slot. */
+ i--;
+ }
+ }
+}
+
+Page* page_cache_get_or_alloc(void* mapping, uint64 index)
+{
+ /* Fast path: already cached */
+ Page* page = page_cache_get(mapping, index);
+ if (page) {
+ return page;
+ }
+
+ /* Slow path: allocate and insert */
+ page = pmm_alloc_page();
+ if (!page) {
+ return nullptr;
+ }
+
+ int ret = page_cache_insert(mapping, index, page);
+ if (ret == -EEXIST) {
+ /* Race: another thread inserted between our get and insert.
+ * Free our allocation and return the existing page. */
+ pmm_free_page(page);
+ return page_cache_get(mapping, index);
+ }
+ if (ret < 0) {
+ pmm_free_page(page);
+ return nullptr;
+ }
+
+ /* Caller gets a ref (insert already added cache's ref,
+ * we need one more for the caller). */
+ /* Actually, insert did incref for cache. The page already has
+ * refcount=1 from alloc + 1 from insert = 2.
+ * We return it to caller who must put() when done.
+ * So we incref once more for the caller's reference. */
+ page_incref(page);
+ return page;
+}
diff --git a/mm/pmm.cc b/mm/pmm.cc
new file mode 100644
index 0000000..b7cf611
--- /dev/null
+++ b/mm/pmm.cc
@@ -0,0 +1,528 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/* Copyright (C) 2026 Richard Qin */
+
+/*
+ * pmm.cc — Physical Memory Manager implementation
+ *
+ * Buddy allocator with Per-CPU single-page caches (PCP).
+ *
+ * Initialization sequence (called by BSP in single-core period):
+ * 1. memblock + vmm_init allocate and map the page_map at VMEMMAP.
+ * 2. Zero and mark all Page descriptors reserved by default.
+ * 3. Carve free regions from memblock into the buddy system.
+ *
+ * The page_map is a flat array indexed by physical page frame number (PFN).
+ * PFN = (pa - mem_base) / PAGE_SIZE. This keeps address conversion O(1).
+ *
+ * Reference: docs/specs/pmm.md
+ */
+
+#include <arch/config.h>
+#include <arch/cpu.h>
+#include <kernel/sync/spinlock.h>
+#include <lume/addr.h>
+#include <lume/config.h>
+#include <lume/klog.h>
+#include <lume/kprintf.h>
+#include <lume/memblock.h>
+#include <lume/new.h>
+#include <lume/pmm.h>
+#include <lume/types.h>
+#include <lume/vmm.h>
+
+/* ------------------------------------------------------------------ */
+/* Per-CPU page cache */
+/* ------------------------------------------------------------------ */
+
+struct PerCpuCache {
+ list_node free_list; // Single-page free list
+ uint32 count; // Number of cached pages
+};
+
+/* ------------------------------------------------------------------ */
+/* Module-private state */
+/* ------------------------------------------------------------------ */
+
+namespace {
+
+/* PCP Cache tuning parameters */
+constexpr uint32 kPcpBatchDivisor = 1024;
+constexpr uint32 kPcpMinBatchSize = 16;
+constexpr uint64 kMB = 1024ULL * 1024ULL;
+
+Spinlock g_pmm_lock;
+list_node g_free_areas[kMaxOrder]; // Buddy free lists [0..10]
+PerCpuCache g_pcp[kMaxCpus];
+
+Page *g_page_map = nullptr; // Flat array of Page descriptors
+uint64 g_page_map_pa = 0; // Physical base of page_map storage
+uint64 g_page_map_bytes = 0; // Byte size of page_map storage
+uint64 g_mem_base = 0; // Physical memory base (from FDT)
+uint64 g_mem_size = 0; // Total physical memory size
+uint64 g_num_pages = 0; // Total managed pages
+
+} // namespace
+
+uint32 g_pcp_high_watermark = 0;
+uint32 g_pcp_batch_size = 0;
+
+/* ------------------------------------------------------------------ */
+/* Address conversion helpers */
+/* ------------------------------------------------------------------ */
+
+/*
+ * PFN (Page Frame Number) — fundamental conversion unit.
+ * When sparse vmemmap is implemented, only these two functions change.
+ * All other PA↔Page conversions go through them.
+ */
+uint64 page_to_pfn(const Page *page) {
+ uintptr base = reinterpret_cast<uintptr>(g_page_map);
+ uintptr addr = reinterpret_cast<uintptr>(page);
+ uintptr end = base + g_num_pages * sizeof(Page);
+
+ if (addr < base || addr >= end) {
+ kernel_panic("page_to_pfn: page outside page_map bounds");
+ }
+
+ uint64 idx = (addr - base) / sizeof(Page);
+ return (g_mem_base / kPageSize) + idx;
+}
+
+Page *pfn_to_page(uint64 pfn) {
+ uint64 base_pfn = g_mem_base / kPageSize;
+ if (pfn < base_pfn || pfn >= base_pfn + g_num_pages) {
+ kernel_panic("pfn_to_page: PFN outside managed range");
+ }
+ return &g_page_map[pfn - base_pfn];
+}
+
+PhysAddr page_to_pa(const Page *page) {
+ return pfn_to_pa(Pfn{page_to_pfn(page)});
+}
+
+Page *pa_to_page(PhysAddr pa) {
+ return pfn_to_page(pa_to_pfn(pa).raw);
+}
+
+VirtAddr page_to_va(const Page *page) {
+ return pa_to_va(page_to_pa(page));
+}
+
+
+/* ------------------------------------------------------------------ */
+/* Buddy system internals (caller must hold g_pmm_lock) */
+/* ------------------------------------------------------------------ */
+
+namespace {
+
+/*
+ * Find a page's buddy at a given order.
+ * Buddy PFN = PFN ^ (1 << order).
+ */
+inline Page *buddy_of(Page *page, uint8 order) {
+ uint64 pfn = static_cast<uint64>(page - g_page_map);
+ uint64 buddy_pfn = pfn ^ (1ULL << order);
+ if (buddy_pfn >= g_num_pages) {
+ return nullptr;
+ }
+ return &g_page_map[buddy_pfn];
+}
+
+/*
+ * Insert a free block into the buddy free list at the given order.
+ */
+void buddy_list_add(Page *page, uint8 order) {
+ page->type = PageType::Free;
+ page->order = order;
+ list_node *head = &g_free_areas[order];
+ list_node *node = &page->free.free_link;
+ node->next = head->next;
+ node->prev = head;
+ head->next->prev = node;
+ head->next = node;
+}
+
+/*
+ * Remove a block from whatever buddy free list it is on.
+ */
+void buddy_list_del(Page *page) {
+ list_node *node = &page->free.free_link;
+ node->prev->next = node->next;
+ node->next->prev = node->prev;
+ node->next = node;
+ node->prev = node;
+}
+
+/*
+ * Allocate a block of 2^order contiguous pages from the buddy system.
+ * Splits higher-order blocks if no exact match is available.
+ * Returns nullptr if no memory available.
+ */
+Page *buddy_alloc(uint8 order) {
+ for (int cur = order; cur < kMaxOrder; cur++) {
+ list_node *head = &g_free_areas[cur];
+ if (head->next == head) {
+ continue; // Empty at this order
+ }
+
+ // Pop the first block from this order's free list
+ list_node *node = head->next;
+ // Compute Page* from the free_link member offset
+ Page *page =
+ reinterpret_cast<Page *>(reinterpret_cast<uintptr>(node) -
+ __builtin_offsetof(Page, free.free_link));
+ buddy_list_del(page);
+
+ // Split down to the requested order
+ while (cur > order) {
+ cur--;
+ // The upper half becomes a free buddy
+ Page *split = &g_page_map[(page - g_page_map) + (1ULL << cur)];
+ buddy_list_add(split, static_cast<uint8>(cur));
+ }
+
+ page->type = PageType::Allocated;
+ page->order = order;
+ page->refcount.store(1, __ATOMIC_RELAXED);
+ page->mapcount.store(0, __ATOMIC_RELAXED);
+ page->flags.store(0, __ATOMIC_RELAXED);
+ page->mapping = nullptr;
+ page->index = 0;
+ return page;
+ }
+ return nullptr;
+}
+
+/*
+ * Return a block of 2^order pages to the buddy system.
+ * Coalesces with its buddy recursively up to kMaxOrder-1.
+ */
+void buddy_free(Page *page, uint8 order) {
+ uint64 pfn = static_cast<uint64>(page - g_page_map);
+
+ while (order < kMaxOrder - 1) {
+ Page *buddy = buddy_of(page, order);
+ if (!buddy) {
+ break;
+ }
+ // Buddy must be free AND at the same order to coalesce
+ if (buddy->type != PageType::Free || buddy->order != order) {
+ break;
+ }
+
+ // Remove buddy from its free list and merge
+ buddy_list_del(buddy);
+
+ // The merged block starts at the lower PFN
+ uint64 buddy_pfn = static_cast<uint64>(buddy - g_page_map);
+ if (buddy_pfn < pfn) {
+ page = buddy;
+ pfn = buddy_pfn;
+ }
+ order++;
+ }
+
+ buddy_list_add(page, order);
+}
+
+/* ------------------------------------------------------------------ */
+/* Per-CPU cache operations (interrupts must be disabled by caller) */
+/* ------------------------------------------------------------------ */
+
+/*
+ * Refill the PCP from the global buddy system.
+ * Acquires g_pmm_lock internally.
+ */
+void refill_pcp(PerCpuCache *pcp) {
+ LockGuard guard(g_pmm_lock);
+ for (uint32 i = 0; i < g_pcp_batch_size; i++) {
+ Page *page = buddy_alloc(0);
+ if (!page) {
+ break;
+ }
+ // Put onto PCP list (reuse the free_link for PCP chain)
+ page->type = PageType::PcpCached;
+ list_node *node = &page->free.free_link;
+ node->next = pcp->free_list.next;
+ node->prev = &pcp->free_list;
+ pcp->free_list.next->prev = node;
+ pcp->free_list.next = node;
+ pcp->count++;
+ }
+}
+
+/*
+ * Drain half the PCP back to the global buddy system.
+ * Acquires g_pmm_lock internally.
+ */
+void drain_pcp(PerCpuCache *pcp) {
+ LockGuard guard(g_pmm_lock);
+ int to_drain = pcp->count / 2;
+ for (int i = 0; i < to_drain; i++) {
+ list_node *head = &pcp->free_list;
+ if (head->next == head) {
+ break;
+ }
+ list_node *node = head->next;
+ // Unlink from PCP
+ node->prev->next = node->next;
+ node->next->prev = node->prev;
+ pcp->count--;
+
+ Page *page =
+ reinterpret_cast<Page *>(reinterpret_cast<uintptr>(node) -
+ __builtin_offsetof(Page, free.free_link));
+ buddy_free(page, 0);
+ }
+}
+
+} // namespace
+
+/* ------------------------------------------------------------------ */
+/* Public API: single-page (fast path via PCP) */
+/* ------------------------------------------------------------------ */
+
+Page *pmm_alloc_page() {
+ /*
+ * Disable interrupts to prevent reentry on the same CPU
+ * (e.g. timer IRQ handler calling pmm_alloc_page while we
+ * are mid-way through manipulating the PCP list).
+ */
+ bool was_on = arch::cpu::intr_enabled();
+ arch::cpu::intr_off();
+
+ uint64 cpu = arch::cpu::id();
+ PerCpuCache *pcp = &g_pcp[cpu];
+
+ if (pcp->count == 0) {
+ refill_pcp(pcp);
+ }
+
+ Page *page = nullptr;
+ if (pcp->count > 0) {
+ list_node *node = pcp->free_list.next;
+ // Unlink
+ node->prev->next = node->next;
+ node->next->prev = node->prev;
+ pcp->count--;
+
+ page =
+ reinterpret_cast<Page *>(reinterpret_cast<uintptr>(node) -
+ __builtin_offsetof(Page, free.free_link));
+
+ // Validate the page is within the page_map bounds
+ if (page < g_page_map || page >= g_page_map + g_num_pages) {
+ kernel_panic("pmm_alloc_page: PCP returned out-of-bounds page");
+ }
+
+ page->type = PageType::Allocated;
+ page->order = 0;
+ page->refcount.store(1, __ATOMIC_RELAXED);
+ page->mapcount.store(0, __ATOMIC_RELAXED);
+ page->flags.store(0, __ATOMIC_RELAXED);
+ page->mapping = nullptr;
+ page->index = 0;
+ }
+
+ if (was_on) {
+ arch::cpu::intr_on();
+ }
+ return page;
+}
+
+void pmm_free_page(Page *page) {
+ if (!page) {
+ kernel_panic("pmm_free_page: null page");
+ }
+
+ bool was_on = arch::cpu::intr_enabled();
+ arch::cpu::intr_off();
+
+ uint64 cpu = arch::cpu::id();
+ PerCpuCache *pcp = &g_pcp[cpu];
+
+ // Push onto PCP
+ page->type = PageType::PcpCached;
+ page->order = 0;
+ list_node *node = &page->free.free_link;
+ node->next = pcp->free_list.next;
+ node->prev = &pcp->free_list;
+ pcp->free_list.next->prev = node;
+ pcp->free_list.next = node;
+ pcp->count++;
+
+ // Drain if over high watermark
+ if (pcp->count >= g_pcp_high_watermark) {
+ drain_pcp(pcp);
+ }
+
+ if (was_on) {
+ arch::cpu::intr_on();
+ }
+}
+
+/* ------------------------------------------------------------------ */
+/* Public API: multi-page (slow path, direct buddy) */
+/* ------------------------------------------------------------------ */
+
+Page *pmm_alloc_pages(uint8 order) {
+ if (order >= kMaxOrder) {
+ return nullptr;
+ }
+ LockGuard guard(g_pmm_lock);
+ return buddy_alloc(order);
+}
+
+void pmm_free_pages(Page *page, uint8 order) {
+ if (!page) {
+ kernel_panic("pmm_free_pages: null page");
+ }
+ if (order >= kMaxOrder) {
+ kernel_panic("pmm_free_pages: order out of range");
+ }
+ LockGuard guard(g_pmm_lock);
+ buddy_free(page, order);
+}
+
+/* ------------------------------------------------------------------ */
+/* Public API: reference counting */
+/* ------------------------------------------------------------------ */
+
+void page_incref(Page *page) { page->refcount.fetch_add(1); }
+
+void page_decref(Page *page) {
+ uint32 old = page->refcount.fetch_sub(1);
+ if (old == 1) {
+ // Refcount dropped to 0 — return to pool based on originally allocated
+ // order
+ if (page->order == 0) {
+ pmm_free_page(page);
+ } else {
+ pmm_free_pages(page, page->order);
+ }
+ }
+}
+
+/* ------------------------------------------------------------------ */
+/* Initialization */
+/* ------------------------------------------------------------------ */
+
+/*
+ * Helper: zero out a range of bytes. We cannot use memset here
+ * because lib/string may not be linked yet.
+ */
+namespace {
+void zero_range(void *start, uint64 bytes) {
+ uint8 *p = static_cast<uint8 *>(start);
+ for (uint64 i = 0; i < bytes; i++) {
+ p[i] = 0;
+ }
+}
+} // namespace
+
+void pmm_init() {
+ /* 1. Get memory layout from memblock (already scanned FDT) */
+ g_mem_base = memblock_get_mem_base();
+ g_mem_size = memblock_get_mem_size();
+ g_num_pages = vmm_get_num_pages();
+
+ if (g_mem_size == 0) {
+ kernel_panic("pmm_init: memblock reports zero memory");
+ }
+
+ /* 2. PCP parameters */
+ uint32 calc_batch = static_cast<uint32>(g_num_pages / (kMaxCpus * kPcpBatchDivisor));
+ g_pcp_batch_size = (calc_batch > kPcpMinBatchSize) ? calc_batch : kPcpMinBatchSize;
+ g_pcp_high_watermark = g_pcp_batch_size * 4;
+
+ /* 3. page_map was allocated by vmm_init at vmemmap VA.
+ * No relocation needed — page_map starts life at the right VA. */
+ g_page_map_pa = vmm_get_page_map_pa();
+ g_page_map_bytes = vmm_get_page_map_size();
+ g_page_map = reinterpret_cast<Page *>(arch::g_vmemmap_base);
+
+ /* 4. Zero out the entire page_map */
+ zero_range(g_page_map, g_page_map_bytes);
+
+ /* 4b. Default every page descriptor to reserved/non-free. */
+ for (uint64 i = 0; i < g_num_pages; i++) {
+ Page *page = &g_page_map[i];
+ page->refcount.store(0, __ATOMIC_RELAXED);
+ page->mapcount.store(0, __ATOMIC_RELAXED);
+ page->flags.store(0, __ATOMIC_RELAXED);
+ page->type = PageType::Allocated;
+ page->order = 0;
+ page->mapping = nullptr;
+ page->index = 0;
+ page->lru.init();
+ page->free.free_link.init();
+ }
+
+ /* 5. Initialize buddy free list heads */
+ g_pmm_lock.init("pmm");
+ for (int i = 0; i < kMaxOrder; i++) {
+ g_free_areas[i].init();
+ }
+
+ /* 6. Initialize PCP list heads */
+ for (int i = 0; i < kMaxCpus; i++) {
+ g_pcp[i].free_list.init();
+ g_pcp[i].count = 0;
+ }
+
+ /* 7. Feed free regions from memblock into the buddy system.
+ * memblock knows about all reserved regions (firmware, kernel,
+ * page table pages, page_map). Everything else is free. */
+ struct FeedCtx {
+ uint64 total_pages;
+ };
+ FeedCtx ctx{0};
+
+ memblock_for_each_free(
+ [](uint64 base, uint64 size, void *arg) {
+ auto *c = static_cast<FeedCtx *>(arg);
+ uint64 start_pfn = (base - memblock_get_mem_base()) / kPageSize;
+ uint64 end_pfn = start_pfn + size / kPageSize;
+
+ /* Access page_map via the globals directly (they are in anon namespace now) */
+
+ uint64 pfn = start_pfn;
+ while (pfn < end_pfn) {
+ uint8 order = 0;
+ for (int o = kMaxOrder - 1; o >= 0; o--) {
+ uint64 block_size = 1ULL << o;
+ if ((pfn & (block_size - 1)) == 0 && pfn + block_size <= end_pfn) {
+ order = static_cast<uint8>(o);
+ break;
+ }
+ }
+
+ Page *page = &g_page_map[pfn];
+ page->type = PageType::Free;
+ page->order = order;
+ page->refcount.store(0, __ATOMIC_RELAXED);
+ page->free.free_link.init();
+
+ /* Inline buddy_list_add since we're inside a lambda */
+ list_node *head = &g_free_areas[order];
+ list_node *node = &page->free.free_link;
+ node->next = head->next;
+ node->prev = head;
+ head->next->prev = node;
+ head->next = node;
+
+ pfn += (1ULL << order);
+ c->total_pages += (1ULL << order);
+ }
+ },
+ &ctx);
+
+ kprintf("[pmm] buddy: %llu free pages (%llu MB)\n", ctx.total_pages,
+ (ctx.total_pages * kPageSize) / kMB);
+
+ /* 8. Retire memblock — all future allocations go through PMM */
+ memblock_retire();
+}
+
+PhysAddr pmm_get_mem_base() { return phys_addr(g_mem_base); }
+uint64 pmm_get_mem_size() { return g_mem_size; }
diff --git a/mm/slab.cc b/mm/slab.cc
new file mode 100644
index 0000000..43d8c4a
--- /dev/null
+++ b/mm/slab.cc
@@ -0,0 +1,321 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/* Copyright (C) 2026 Richard Qin */
+
+/*
+ * slab.cc — SLUB allocator implementation
+ *
+ * Key design points:
+ * - Free object list is embedded in the objects themselves (first 8 bytes).
+ * - Per-page metadata lives in Page.slab (cache, obj_count, freelist).
+ * - Per-CPU "active page" avoids lock contention on the fast path.
+ * - partial_ list protected by per-KmemCache Spinlock.
+ * - PMM calls happen outside the lock to avoid lock ordering issues.
+ *
+ * Address note: the kernel may be running at physical addresses via
+ * identity mapping (before vmm_init). We handle PA/VA carefully:
+ * PMM gives us Page*, page_to_pa() gives PA, and we access page
+ * contents via pa_to_va() (higher-half VA, always mapped).
+ *
+ * Reference: docs/specs/slab.md
+ */
+
+#include "slab.h"
+#include <lume/pmm.h>
+#include <lume/addr.h>
+#include <lume/klog.h>
+#include <lume/new.h>
+#include <arch/cpu.h>
+
+/* ------------------------------------------------------------------ */
+/* Helpers */
+/* ------------------------------------------------------------------ */
+
+/*
+ * Convert a Page* to the usable virtual address of the page's contents.
+ * Uses the higher-half mapping which is always active.
+ */
+static inline void* page_to_data_va(Page* page)
+{
+ return reinterpret_cast<void*>(page_to_va(page).raw);
+}
+
+/* Round up x to the next multiple of align (align must be power of 2) */
+static inline uint32 align_up(uint32 x, uint32 align)
+{
+ return (x + align - 1) & ~(align - 1);
+}
+
+/* ------------------------------------------------------------------ */
+/* Global size-class caches */
+/* ------------------------------------------------------------------ */
+
+KmemCache g_size_caches[kNumSizeClasses];
+
+/* ------------------------------------------------------------------ */
+/* KmemCache::init */
+/* ------------------------------------------------------------------ */
+
+void KmemCache::init(const char* name, uint32 obj_size, uint32 obj_align)
+{
+ name_ = name;
+ obj_align_ = (obj_align < 8) ? 8 : obj_align; // minimum 8 for freelist ptr
+ obj_size_ = align_up((obj_size < 8) ? 8 : obj_size, obj_align_);
+ objs_per_slab_ = kPageSize / obj_size_;
+
+ lock_.init(name);
+ partial_.init();
+
+ for (int i = 0; i < kMaxCpus; i++)
+ cpu_slab_[i].active = nullptr;
+}
+
+/* ------------------------------------------------------------------ */
+/* KmemCache::new_slab — allocate and format a fresh slab page */
+/* ------------------------------------------------------------------ */
+
+Page* KmemCache::new_slab()
+{
+ Page* page = pmm_alloc_page();
+ if (!page)
+ return nullptr;
+
+ page->type = PageType::Slab;
+ page->slab.cache = this;
+ page->slab.obj_count = 0;
+
+ /* Build the embedded free list across all object slots */
+ uint8* base = static_cast<uint8*>(page_to_data_va(page));
+ void* head = nullptr;
+
+ /* Chain objects from last to first so freelist order = low addr first */
+ for (int i = static_cast<int>(objs_per_slab_) - 1; i >= 0; i--) {
+ void* obj = base + i * obj_size_;
+ *reinterpret_cast<void**>(obj) = head;
+ head = obj;
+ }
+ page->slab.freelist = head;
+
+ return page;
+}
+
+/* ------------------------------------------------------------------ */
+/* Partial list management (lock must be held) */
+/* ------------------------------------------------------------------ */
+
+Page* KmemCache::get_partial()
+{
+ /*
+ * Phase 1 simplification: no partial list tracking.
+ * The slow path always allocates a fresh page from PMM.
+ * Full pages stay alive (Page.slab.cache is valid for kfree).
+ * Empty pages are returned to PMM immediately.
+ *
+ * TODO: add list_node to Page.slab and implement partial tracking
+ * to avoid wasting partially-filled pages.
+ */
+ (void)partial_;
+ return nullptr;
+}
+
+void KmemCache::put_partial(Page* page)
+{
+ (void)page;
+}
+
+void KmemCache::remove_partial(Page* page)
+{
+ (void)page;
+}
+
+/* ------------------------------------------------------------------ */
+/* KmemCache::alloc */
+/* ------------------------------------------------------------------ */
+
+void* KmemCache::alloc()
+{
+ bool was_on = arch::cpu::intr_enabled();
+ arch::cpu::intr_off();
+
+ uint64 cpu = arch::cpu::id();
+ Page* active = cpu_slab_[cpu].active;
+
+ /* Fast path: pop from active page's freelist */
+ if (active && active->slab.freelist) {
+ void* obj = active->slab.freelist;
+ active->slab.freelist = *reinterpret_cast<void**>(obj);
+ active->slab.obj_count++;
+ if (was_on) arch::cpu::intr_on();
+ return obj;
+ }
+
+ /* Slow path: need a new active page */
+
+ /* If current active is full, detach it (it stays as Slab state,
+ * cache pointer still valid for kfree to find it) */
+ if (active && !active->slab.freelist) {
+ cpu_slab_[cpu].active = nullptr;
+ }
+
+ if (was_on) arch::cpu::intr_on();
+
+ /* Try to get a partial page or allocate a fresh one.
+ * PMM call happens without any lock held. */
+ Page* new_page = new_slab();
+ if (!new_page)
+ return nullptr;
+
+ /* Install as new active page */
+ was_on = arch::cpu::intr_enabled();
+ arch::cpu::intr_off();
+
+ /* Another interrupt handler might have set a new active while we
+ * were allocating. If so, put ours aside (simplified: just use ours). */
+ cpu_slab_[cpu].active = new_page;
+
+ void* obj = new_page->slab.freelist;
+ new_page->slab.freelist = *reinterpret_cast<void**>(obj);
+ new_page->slab.obj_count++;
+
+ if (was_on) arch::cpu::intr_on();
+ return obj;
+}
+
+/* ------------------------------------------------------------------ */
+/* KmemCache::free */
+/* ------------------------------------------------------------------ */
+
+void KmemCache::free(void* obj)
+{
+ if (!obj)
+ return;
+
+ /* Find the Page for this object's page */
+ uint64 obj_addr = reinterpret_cast<uint64>(obj);
+
+ /* Convert VA to PA. Like pmm.cc, handle both PA and VA cases. */
+ uint64 obj_pa = va_to_pa(virt_addr(obj_addr)).raw;
+ uint64 page_pa = obj_pa & ~(kPageSize - 1);
+ Page* page = pa_to_page(phys_addr(page_pa));
+
+ if (page->type != PageType::Slab || page->slab.cache != this)
+ kernel_panic("slab free: object does not belong to this cache");
+
+ bool was_on = arch::cpu::intr_enabled();
+ arch::cpu::intr_off();
+
+ /* Push object back onto the page's freelist */
+ *reinterpret_cast<void**>(obj) = page->slab.freelist;
+ page->slab.freelist = obj;
+ page->slab.obj_count--;
+
+ /* If page is now completely empty, return it to PMM */
+ if (page->slab.obj_count == 0) {
+ /* If this page is our active, clear it */
+ uint64 cpu = arch::cpu::id();
+ if (cpu_slab_[cpu].active == page)
+ cpu_slab_[cpu].active = nullptr;
+
+ if (was_on) arch::cpu::intr_on();
+
+ /* Return page to PMM */
+ page->slab.cache = nullptr;
+ page->slab.freelist = nullptr;
+ pmm_free_page(page);
+ return;
+ }
+
+ if (was_on) arch::cpu::intr_on();
+}
+
+/* ------------------------------------------------------------------ */
+/* Public free-function API */
+/* ------------------------------------------------------------------ */
+
+/* Find the smallest size class >= size */
+static KmemCache* find_cache(uint32 size)
+{
+ for (int i = 0; i < kNumSizeClasses; i++) {
+ if (kSizeClasses[i] >= size)
+ return &g_size_caches[i];
+ }
+ return nullptr; // Too large for slab
+}
+
+void* kmalloc(uint32 size)
+{
+ if (size == 0)
+ return nullptr;
+
+ /* Large allocation: fall back to PMM directly */
+ if (size > kSizeClasses[kNumSizeClasses - 1]) {
+ uint8 order = 0;
+ uint64 needed = (size + kPageSize - 1) / kPageSize;
+ while ((1ULL << order) < needed)
+ order++;
+ Page* page = pmm_alloc_pages(order);
+ if (!page)
+ return nullptr;
+ return reinterpret_cast<void*>(pa_to_va(page_to_pa(page)).raw);
+ }
+
+ KmemCache* cache = find_cache(size);
+ if (!cache)
+ return nullptr;
+ return cache->alloc();
+}
+
+void kfree(void* ptr)
+{
+ if (!ptr)
+ return;
+
+ uint64 addr = reinterpret_cast<uint64>(ptr);
+ uint64 pa = va_to_pa(virt_addr(addr)).raw;
+ uint64 page_pa = pa & ~(kPageSize - 1);
+ Page* page = pa_to_page(phys_addr(page_pa));
+
+ if (page->type == PageType::Slab) {
+ KmemCache* cache = page->slab.cache;
+ if (!cache)
+ kernel_panic("kfree: slab page has null cache");
+ cache->free(ptr);
+ } else if (page->type == PageType::Allocated) {
+ /* Large allocation — return directly to PMM */
+ pmm_free_pages(page, page->order);
+ } else {
+ kernel_panic("kfree: invalid page state");
+ }
+}
+
+/* ------------------------------------------------------------------ */
+/* C++ operator new / delete */
+/* ------------------------------------------------------------------ */
+
+/* Use decltype(sizeof(0)) instead of size_t to match the compiler's
+ * built-in expectation for operator new (unsigned long vs unsigned long long). */
+using __sz = decltype(sizeof(0));
+
+void* operator new(__sz size) { return kmalloc(static_cast<uint32>(size)); }
+void* operator new[](__sz size) { return kmalloc(static_cast<uint32>(size)); }
+
+void operator delete(void* ptr) noexcept { kfree(ptr); }
+void operator delete[](void* ptr) noexcept { kfree(ptr); }
+void operator delete(void* ptr, __sz) noexcept { kfree(ptr); }
+void operator delete[](void* ptr, __sz) noexcept { kfree(ptr); }
+
+/* ------------------------------------------------------------------ */
+/* Initialization */
+/* ------------------------------------------------------------------ */
+
+void slab_init()
+{
+ for (int i = 0; i < kNumSizeClasses; i++) {
+ /* Build a name string like "kmalloc-32" */
+ static const char* names[kNumSizeClasses] = {
+ "kmalloc-8", "kmalloc-16", "kmalloc-32",
+ "kmalloc-64", "kmalloc-128", "kmalloc-256",
+ "kmalloc-512", "kmalloc-1024","kmalloc-2048"
+ };
+ g_size_caches[i].init(names[i], kSizeClasses[i], 8);
+ }
+}
diff --git a/mm/slab.h b/mm/slab.h
new file mode 100644
index 0000000..909ee62
--- /dev/null
+++ b/mm/slab.h
@@ -0,0 +1,83 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/* Copyright (C) 2026 Richard Qin */
+
+#pragma once
+
+/*
+ * SLUB Allocator — module-private header
+ *
+ * SLUB = Simple List of Unfull Blocks.
+ * Metadata lives in Page descriptors (no per-page Slab header).
+ * Free object list is embedded inside the objects themselves.
+ * Per-CPU "active page" eliminates lock contention on the fast path.
+ *
+ * Reference: docs/specs/slab.md
+ */
+
+#include <lume/types.h>
+#include <lume/config.h>
+#include <lume/page.h>
+#include <lume/list.h>
+#include <kernel/sync/spinlock.h>
+
+/* Size classes: 8, 16, 32, 64, 128, 256, 512, 1024, 2048 */
+inline constexpr int kNumSizeClasses = 9;
+inline constexpr uint32 kSizeClasses[kNumSizeClasses] = {
+ 8, 16, 32, 64, 128, 256, 512, 1024, 2048
+};
+
+/* Per-CPU active slab page pointer */
+struct PerCpuSlab {
+ Page* active; // Currently used slab page (nullptr if none)
+};
+
+/*
+ * KmemCache — manages a pool of fixed-size objects.
+ *
+ * Lock contract:
+ * lock_ protects partial_ list only.
+ * Per-CPU cpu_slab_ access requires interrupts disabled.
+ * PMM calls happen outside lock_ to avoid lock ordering issues.
+ */
+class KmemCache {
+public:
+ KmemCache() = default;
+
+ // Explicit init (Placement New / BSS pattern)
+ void init(const char* name, uint32 obj_size, uint32 obj_align);
+
+ // Allocate one object. Returns nullptr on OOM.
+ void* alloc();
+
+ // Free one object back to its owning page.
+ void free(void* obj);
+
+ const char* name() const { return name_; }
+ uint32 obj_size() const { return obj_size_; }
+
+private:
+ // Get a partially-filled page from the partial list (lock must be held)
+ Page* get_partial();
+
+ // Put a page onto the partial list (lock must be held)
+ void put_partial(Page* page);
+
+ // Remove a page from the partial list (lock must be held)
+ void remove_partial(Page* page);
+
+ // Allocate a new page from PMM and format it as a slab
+ Page* new_slab();
+
+ Spinlock lock_; // Protects partial_ only
+ list_node partial_; // Partially-filled slab pages
+
+ PerCpuSlab cpu_slab_[kMaxCpus]; // Per-CPU active page
+
+ const char* name_ = "uninit";
+ uint32 obj_size_ = 0; // Aligned object size (bytes)
+ uint32 obj_align_ = 0; // Alignment requirement
+ uint32 objs_per_slab_ = 0; // Objects per 4KB page
+};
+
+/* Global size-class caches */
+extern KmemCache g_size_caches[kNumSizeClasses];
diff --git a/mm/tlb.cc b/mm/tlb.cc
new file mode 100644
index 0000000..b7a8ba7
--- /dev/null
+++ b/mm/tlb.cc
@@ -0,0 +1,90 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/* Copyright (C) 2026 Richard Qin */
+
+/*
+ * tlb.cc — TLB shootdown implementation
+ *
+ * Uses RISC-V SBI Remote Fence extension to perform cross-hart
+ * TLB invalidation. SBI firmware handles the IPI delivery and
+ * synchronization — the call blocks until all target harts have
+ * completed their local sfence.vma.
+ *
+ * When a VmSpace is provided, we use its cpu_mask to target only
+ * the CPUs that have this address space loaded in satp. This avoids
+ * unnecessary IPI storms on unrelated cores.
+ *
+ * Reference: docs/specs/linux_abi_mm_plan.md §SMP §2.1
+ */
+
+#include <lume/tlb.h>
+#include <lume/vmm.h>
+#include <lume/config.h>
+#include <arch/mmu.h>
+#include <arch/tlb.h>
+#include <arch/cpu.h>
+
+/*
+ * If a shootdown range exceeds this many pages, do a full flush
+ * instead of per-page invalidation.
+ */
+static constexpr uint64 kShootdownPageThreshold = 32;
+
+/*
+ * Build a cpu mask for all online CPUs (fallback when mm is null).
+ */
+namespace {
+uint64 all_cpus_mask()
+{
+ uint64 mask = 0;
+ for (uint32 i = 0; i < kMaxCpus; i++) {
+ mask |= (1ULL << i);
+ }
+ return mask;
+}
+
+/*
+ * Get the target cpu mask: use mm->cpu_mask when available,
+ * fall back to all_cpus_mask for kernel address space (mm == nullptr).
+ */
+uint64 target_cpu_mask(VmSpace* mm)
+{
+ if (mm) {
+ uint64 mask = mm->cpu_mask.load(__ATOMIC_RELAXED);
+ /* Always include our own CPU — we may still have stale TLB */
+ mask |= (1ULL << arch::cpu::id());
+ return mask;
+ }
+ return all_cpus_mask();
+}
+} // namespace
+
+void tlb_shootdown_range(VmSpace* mm, uint64 start, uint64 end)
+{
+ if (start >= end) {
+ return;
+ }
+
+ uint64 num_pages = (end - start) / kPageSize;
+ uint64 cpu_mask = target_cpu_mask(mm);
+
+ if (num_pages > kShootdownPageThreshold) {
+ /* Full flush on target CPUs via arch backend */
+ arch::tlb::flush_remote(cpu_mask, 0, ~0ULL);
+ return;
+ }
+
+ /* Range flush on target CPUs via arch backend */
+ arch::tlb::flush_remote(cpu_mask, start, end - start);
+}
+
+void tlb_shootdown_page(VmSpace* mm, uint64 va)
+{
+ uint64 cpu_mask = target_cpu_mask(mm);
+ arch::tlb::flush_remote(cpu_mask, va, kPageSize);
+}
+
+void tlb_shootdown_all()
+{
+ uint64 cpu_mask = all_cpus_mask();
+ arch::tlb::flush_remote(cpu_mask, 0, ~0ULL);
+}
diff --git a/mm/vma.cc b/mm/vma.cc
new file mode 100644
index 0000000..da89e08
--- /dev/null
+++ b/mm/vma.cc
@@ -0,0 +1,640 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/* Copyright (C) 2026 Richard Qin */
+
+/*
+ * vma.cc — Virtual Memory Area management (MI layer)
+ *
+ * Manages user address spaces using a red-black tree of VMA regions.
+ * Implements demand paging: vmm_map_user records intent only;
+ * physical pages are allocated on page fault via vmm_handle_page_fault.
+ *
+ * MI/MD separation: this file NEVER references pmap::PTE_* constants.
+ * Permission translation is delegated to pmap::vm_perm_to_pte().
+ *
+ * Reference: docs/specs/vmm.md
+ */
+
+#include <lume/vmm.h>
+#include <lume/addr.h>
+#include <lume/config.h>
+#include <lume/errno.h>
+#include <lume/pmm.h>
+#include <lume/new.h>
+#include <lume/klog.h>
+#include <arch/pmap.h>
+#include <lume/tlb.h>
+#include <lume/mm_layout.h>
+
+/* ------------------------------------------------------------------ */
+/* Rbtree callbacks for VMA interval tree */
+/* ------------------------------------------------------------------ */
+
+static bool vma_less(struct rb_node* node, const struct rb_node* parent) {
+ VmAreaStruct* n = reinterpret_cast<VmAreaStruct*>(
+ reinterpret_cast<char*>(node) - __builtin_offsetof(VmAreaStruct, rb));
+ const VmAreaStruct* p = reinterpret_cast<const VmAreaStruct*>(
+ reinterpret_cast<const char*>(parent) - __builtin_offsetof(VmAreaStruct, rb));
+ return n->start < p->start;
+}
+
+static int vma_cmp(const void* key, const struct rb_node* node) {
+ uint64 addr = *static_cast<const uint64*>(key);
+ const VmAreaStruct* n = reinterpret_cast<const VmAreaStruct*>(
+ reinterpret_cast<const char*>(node) - __builtin_offsetof(VmAreaStruct, rb));
+ if (addr < n->start) return -1;
+ if (addr >= n->end) return 1;
+ return 0;
+}
+
+static VmAreaStruct* node_to_vma(struct rb_node* node)
+{
+ return reinterpret_cast<VmAreaStruct*>(
+ reinterpret_cast<char*>(node) - __builtin_offsetof(VmAreaStruct, rb));
+}
+
+static uint64 align_up_any(uint64 value, uint64 align)
+{
+ if (align == 0) {
+ return value;
+ }
+ return ((value + align - 1) / align) * align;
+}
+
+static bool range_overlaps(uint64 a_start, uint64 a_end,
+ uint64 b_start, uint64 b_end)
+{
+ return a_start < b_end && b_start < a_end;
+}
+
+
+
+static bool vma_overlaps_locked(VmSpace* space, uint64 start, uint64 end)
+{
+ struct rb_node* node = rb_first(&space->vma_tree);
+ while (node) {
+ VmAreaStruct* vma = node_to_vma(node);
+ if (range_overlaps(start, end, vma->start, vma->end)) {
+ return true;
+ }
+ node = rb_next(node);
+ }
+ return false;
+}
+
+static bool vma_range_covered_locked(VmSpace* space, uint64 start, uint64 end)
+{
+ uint64 cursor = start;
+ struct rb_node* node = rb_first(&space->vma_tree);
+ while (node && cursor < end) {
+ VmAreaStruct* vma = node_to_vma(node);
+ if (vma->end <= cursor) {
+ node = rb_next(node);
+ continue;
+ }
+ if (vma->start > cursor) {
+ return false;
+ }
+ if (vma->end > cursor) {
+ cursor = vma->end;
+ }
+ node = rb_next(node);
+ }
+ return cursor >= end;
+}
+
+static void vma_try_merge_neighbors_locked(VmSpace* space, VmAreaStruct* vma)
+{
+ struct rb_node* prev_node = rb_prev(&vma->rb);
+ if (prev_node) {
+ VmAreaStruct* prev = node_to_vma(prev_node);
+ if (prev->end == vma->start && prev->perm == vma->perm) {
+ prev->end = vma->end;
+ rb_erase(&vma->rb, &space->vma_tree);
+ delete vma;
+ vma = prev;
+ }
+ }
+
+ struct rb_node* next_node = rb_next(&vma->rb);
+ if (next_node) {
+ VmAreaStruct* next = node_to_vma(next_node);
+ if (vma->end == next->start && vma->perm == next->perm) {
+ vma->end = next->end;
+ rb_erase(&next->rb, &space->vma_tree);
+ delete next;
+ }
+ }
+}
+
+static int vma_split_locked(VmSpace* space, VmAreaStruct* vma, uint64 split)
+{
+ if (split <= vma->start || split >= vma->end) {
+ return 0;
+ }
+
+ VmAreaStruct* right = new VmAreaStruct;
+ if (!right) {
+ return -ENOMEM;
+ }
+
+ right->start = split;
+ right->end = vma->end;
+ right->perm = vma->perm;
+ right->mm = space;
+
+ vma->end = split;
+ rb_add(&right->rb, &space->vma_tree, vma_less);
+ return 0;
+}
+
+static int vma_split_range_locked(VmSpace* space, uint64 start, uint64 end)
+{
+ if (end <= start) {
+ return 0;
+ }
+
+ struct rb_node* start_node = rb_find(&start, &space->vma_tree, vma_cmp);
+ if (!start_node) {
+ return -ENOMEM;
+ }
+ VmAreaStruct* start_vma = node_to_vma(start_node);
+ int ret = vma_split_locked(space, start_vma, start);
+ if (ret < 0) {
+ return ret;
+ }
+
+ uint64 end_key = end - 1;
+ struct rb_node* end_node = rb_find(&end_key, &space->vma_tree, vma_cmp);
+ if (!end_node) {
+ return -ENOMEM;
+ }
+ VmAreaStruct* end_vma = node_to_vma(end_node);
+ return vma_split_locked(space, end_vma, end);
+}
+
+/* ------------------------------------------------------------------ */
+/* VmSpace lifecycle */
+/* ------------------------------------------------------------------ */
+
+VmSpace* VmSpace::create() {
+ VmSpace* space = new VmSpace;
+ if (!space) return nullptr;
+
+ space->root_pa = pmap::create();
+ if (!space->root_pa) {
+ delete space;
+ return nullptr;
+ }
+
+ space->vma_tree = RB_ROOT;
+ space->lock.init("VmSpace");
+ space->refcount.store(1, __ATOMIC_RELAXED);
+ space->brk_start = 0;
+ space->brk_end = 0;
+ return space;
+}
+
+void VmSpace::incref() {
+ refcount.fetch_add(1, __ATOMIC_RELAXED);
+}
+
+void VmSpace::put() {
+ if (refcount.fetch_sub(1, __ATOMIC_ACQ_REL) == 1) {
+ destroy();
+ }
+}
+
+void VmSpace::destroy() {
+ /* Drain VMA tree under lock, then release BEFORE delete this */
+ lock.acquire();
+ uint64 root = root_pa;
+ struct rb_node* node = rb_first(&vma_tree);
+ while (node) {
+ VmAreaStruct* vma = reinterpret_cast<VmAreaStruct*>(
+ reinterpret_cast<char*>(node) - __builtin_offsetof(VmAreaStruct, rb));
+
+ /* Decref and dec mapcount for all mapped physical pages */
+ for (uint64 addr = vma->start; addr < vma->end; addr += kPageSize) {
+ uint64 pa;
+ if (pmap::lookup(root, addr, &pa)) {
+ Page* pg = pa_to_page(phys_addr(pa));
+ page_dec_mapcount(pg);
+ page_decref(pg);
+ }
+ }
+
+ node = rb_next(node);
+ rb_erase(&vma->rb, &vma_tree);
+ delete vma;
+ }
+ lock.release();
+
+ /* Destroy hardware page tables outside the lock */
+ pmap::destroy(root);
+
+ /* Finally free VmSpace itself */
+ delete this;
+}
+
+/* ------------------------------------------------------------------ */
+/* Public API: VMA manipulation */
+/* ------------------------------------------------------------------ */
+
+int vmm_map_user(VmSpace* space, uint64 va, uint64 len, uint64 perm) {
+ if (!space) return -EINVAL;
+
+ va = page_align_down(va);
+ len = page_align_up(len);
+ if (len == 0) return 0;
+ if (va < kUserVaBase) return -EINVAL;
+ if (va >= kUserVaLimit) return -EINVAL;
+ if (va + len < va || va + len > kUserVaLimit) return -EINVAL;
+
+ VmAreaStruct* vma = new VmAreaStruct;
+ if (!vma) return -ENOMEM;
+ vma->start = va;
+ vma->end = va + len;
+ vma->perm = perm | VM_USER; // Enforce user-mode access (MI flag)
+ vma->mm = space;
+
+ LockGuard guard(space->lock);
+ if (vma_overlaps_locked(space, vma->start, vma->end)) {
+ delete vma;
+ return -EINVAL;
+ }
+ rb_add(&vma->rb, &space->vma_tree, vma_less);
+ vma_try_merge_neighbors_locked(space, vma);
+ return 0;
+}
+
+int vmm_unmap_user(VmSpace* space, uint64 va, uint64 len) {
+ if (!space) return -EINVAL;
+ va = page_align_down(va);
+ len = page_align_up(len);
+ if (len == 0) return 0;
+ if (va < kUserVaBase) return -EINVAL;
+ if (va >= kUserVaLimit) return -EINVAL;
+ if (va + len < va || va + len > kUserVaLimit) return -EINVAL;
+ uint64 end = va + len;
+
+ LockGuard guard(space->lock);
+ struct rb_node* node = rb_first(&space->vma_tree);
+ while (node) {
+ VmAreaStruct* vma = node_to_vma(node);
+ struct rb_node* next = rb_next(node);
+
+ if (vma->end <= va) {
+ node = next;
+ continue;
+ }
+ if (vma->start >= end) {
+ break;
+ }
+
+ uint64 unmap_start = (vma->start > va) ? vma->start : va;
+ uint64 unmap_end = (vma->end < end) ? vma->end : end;
+
+ if (unmap_start < unmap_end) {
+ for (uint64 addr = unmap_start; addr < unmap_end; addr += kPageSize) {
+ uint64 pa;
+ if (pmap::lookup(space->root_pa, addr, &pa)) {
+ pmap::unmap(space->root_pa, addr);
+ Page* pg = pa_to_page(phys_addr(pa));
+ page_dec_mapcount(pg);
+ page_decref(pg);
+ }
+ }
+ }
+
+ if (unmap_start == vma->start && unmap_end == vma->end) {
+ rb_erase(&vma->rb, &space->vma_tree);
+ delete vma;
+ } else if (unmap_start == vma->start) {
+ rb_erase(&vma->rb, &space->vma_tree);
+ vma->start = unmap_end;
+ rb_add(&vma->rb, &space->vma_tree, vma_less);
+ } else if (unmap_end == vma->end) {
+ vma->end = unmap_start;
+ } else {
+ VmAreaStruct* right = new VmAreaStruct;
+ if (!right) {
+ return -ENOMEM;
+ }
+ right->start = unmap_end;
+ right->end = vma->end;
+ right->perm = vma->perm;
+ right->mm = space;
+
+ vma->end = unmap_start;
+ rb_add(&right->rb, &space->vma_tree, vma_less);
+ break;
+ }
+
+ node = next;
+ }
+ return 0;
+}
+
+int vmm_protect_user(VmSpace* space, uint64 va, uint64 len, uint64 perm)
+{
+ if (!space) return -EINVAL;
+ va = page_align_down(va);
+ len = page_align_up(len);
+ if (len == 0) return -EINVAL;
+ if (va < kUserVaBase) return -EINVAL;
+ if (va >= kUserVaLimit) return -EINVAL;
+ if (va + len < va || va + len > kUserVaLimit) return -EINVAL;
+
+ uint64 end = va + len;
+
+ LockGuard guard(space->lock);
+ if (!vma_range_covered_locked(space, va, end)) {
+ return -ENOMEM;
+ }
+ int ret = vma_split_range_locked(space, va, end);
+ if (ret < 0) {
+ return ret;
+ }
+
+ bool tlb_flush_needed = false;
+ struct rb_node* node = rb_find(&va, &space->vma_tree, vma_cmp);
+ while (node) {
+ VmAreaStruct* vma = node_to_vma(node);
+ if (vma->start >= end) {
+ break;
+ }
+ if (vma->end <= va) {
+ node = rb_next(node);
+ continue;
+ }
+
+ vma->perm = (perm | VM_USER);
+
+ for (uint64 addr = vma->start; addr < vma->end; addr += kPageSize) {
+ uint64 pa;
+ if (!pmap::lookup(space->root_pa, addr, &pa)) {
+ continue;
+ }
+
+ if (perm == 0) {
+ Page* pg = pa_to_page(phys_addr(pa));
+ page_dec_mapcount(pg);
+ page_decref(pg);
+ pmap::unmap(space->root_pa, addr);
+ tlb_flush_needed = true;
+ continue;
+ }
+
+ uint64 pte_perm = pmap::vm_perm_to_pte(vma->perm);
+ if ((vma->perm & VM_WRITE) && pmap::is_cow(space->root_pa, addr)) {
+ pte_perm &= ~pmap::PTE_W;
+ pte_perm |= pmap::PTE_COW;
+ }
+
+ int ret = pmap::protect(space->root_pa, addr, pte_perm);
+ if (ret < 0) {
+ return ret;
+ }
+ tlb_flush_needed = true;
+ }
+
+ node = rb_next(node);
+ }
+
+ if (tlb_flush_needed) {
+ tlb_shootdown_range(space, va, end);
+ }
+ return 0;
+}
+
+uint64 mm_find_free_va(VmSpace* space, uint64 length, uint64 align)
+{
+ if (!space) return 0;
+ length = page_align_up(length);
+ if (length == 0) return 0;
+ if (align < kPageSize) align = kPageSize;
+
+ LockGuard guard(space->lock);
+
+ /* Linux: arch_get_unmapped_area() starts from mm->mmap_base.
+ * RISC-V: TASK_UNMAPPED_BASE = PAGE_ALIGN(TASK_SIZE / 3).
+ * Fall back to kUserVaBase if mmap_base not set (kernel VmSpaces). */
+ uint64 base = space->mmap_base ? space->mmap_base : kUserVaBase;
+ uint64 cursor = align_up_any(base, align);
+ struct rb_node* node = rb_first(&space->vma_tree);
+ while (node) {
+ VmAreaStruct* vma = node_to_vma(node);
+ if (cursor <= vma->start) {
+ if (cursor > kUserVaLimit - length) {
+ return 0;
+ }
+ if (cursor + length <= vma->start) {
+ return cursor;
+ }
+ }
+ if (cursor < vma->end) {
+ cursor = vma->end;
+ }
+ cursor = align_up_any(cursor, align);
+ node = rb_next(node);
+ }
+
+ if (cursor > kUserVaLimit - length) {
+ return 0;
+ }
+ return cursor;
+}
+
+bool vma_range_covered(VmSpace* space, uint64 start, uint64 end)
+{
+ if (!space) return false;
+ start = page_align_down(start);
+ end = page_align_up(end);
+ if (end <= start) return true;
+
+ LockGuard guard(space->lock);
+ return vma_range_covered_locked(space, start, end);
+}
+
+/* ------------------------------------------------------------------ */
+/* Page fault handler (demand paging) */
+/* ------------------------------------------------------------------ */
+
+static bool vma_allows(const VmAreaStruct* vma, VmFaultCause cause)
+{
+ switch (cause) {
+ case VmFaultCause::Read:
+ return (vma->perm & VM_READ) != 0;
+ case VmFaultCause::Write:
+ return (vma->perm & VM_WRITE) != 0;
+ case VmFaultCause::Exec:
+ return (vma->perm & VM_EXEC) != 0;
+ default:
+ return false;
+ }
+}
+
+int vmm_handle_page_fault(VmSpace* space, uint64 fault_addr, VmFaultCause cause) {
+ if (!space) return -EINVAL;
+
+ uint64 aligned_addr = page_align_down(fault_addr);
+
+ LockGuard guard(space->lock);
+ struct rb_node* node = rb_find(&fault_addr, &space->vma_tree, vma_cmp);
+ if (!node) {
+ return -EFAULT; // No VMA covers this address → segfault
+ }
+
+ VmAreaStruct* vma = reinterpret_cast<VmAreaStruct*>(
+ reinterpret_cast<char*>(node) - __builtin_offsetof(VmAreaStruct, rb));
+
+ if (!vma_allows(vma, cause)) {
+ return -EFAULT;
+ }
+
+ /* Check if page is already mapped */
+ uint64 paddr;
+ if (pmap::lookup(space->root_pa, aligned_addr, &paddr)) {
+ if (cause == VmFaultCause::Write && pmap::is_cow(space->root_pa, aligned_addr)) {
+ /* Store Page Fault on a COW page */
+ Page* page = pmm_alloc_page();
+ if (!page) return -ENOMEM;
+ uint64 new_pa = page_to_pa(page).raw;
+
+ /* Copy page contents */
+ uint8* dst = reinterpret_cast<uint8*>(pa_to_va(phys_addr(new_pa)).raw);
+ const uint8* src = reinterpret_cast<const uint8*>(pa_to_va(phys_addr(paddr)).raw);
+ for (uint64 i = 0; i < kPageSize; i++) {
+ dst[i] = src[i];
+ }
+
+ /* Remap with write permission, clearing COW */
+ uint64 pte_perm = pmap::vm_perm_to_pte(vma->perm);
+ if (pmap::map(space->root_pa, aligned_addr, new_pa, pte_perm) != 0) {
+ pmm_free_page(page);
+ return -ENOMEM;
+ }
+
+ /* New page gets mapcount = 1 (we just mapped it) */
+ page_inc_mapcount(page);
+
+ /* Decrement mapcount and refcount of old page */
+ Page* old_page = pa_to_page(phys_addr(paddr));
+ page_dec_mapcount(old_page);
+ page_decref(old_page);
+
+ /* Flush TLB to remove old read-only cached translation */
+ tlb_shootdown_page(space, aligned_addr);
+ return 0;
+ }
+ /* DEBUG: page already mapped but still faulting — dump PTE */
+ {
+ uint64 cur_satp;
+ asm volatile("csrr %0, satp" : "=r"(cur_satp));
+ printk(KERN_ERR "fault: va=%llx pa=%llx cause=%d pte=%llx satp=%llx root=%llx\n",
+ aligned_addr, paddr, static_cast<int>(cause),
+ pmap::lookup_pte(space->root_pa, aligned_addr),
+ cur_satp, space->root_pa);
+ }
+ return 0; // Already mapped, maybe read fault on existing page
+ }
+
+ /* Allocate physical page and zero it.
+ * Linux: do_anonymous_page() uses alloc_zeroed_user_highpage_movable().
+ * Anonymous pages MUST be zeroed — userspace expects MAP_ANONYMOUS
+ * pages to contain all zeros (POSIX requirement). */
+ Page* page = pmm_alloc_page();
+ if (!page) return -ENOMEM;
+
+ uint64 pa = page_to_pa(page).raw;
+
+ /* Zero the page before mapping it to userspace */
+ /* Zero the page before mapping it to userspace */
+ uint8* kva = reinterpret_cast<uint8*>(pa_to_va(PhysAddr{pa}).raw);
+ for (uint64 i = 0; i < kPageSize; i++) kva[i] = 0;
+
+ /* Translate MI permission flags to hardware PTE bits (MD layer) */
+ uint64 pte_perm = pmap::vm_perm_to_pte(vma->perm);
+
+ if (pmap::map(space->root_pa, aligned_addr, pa, pte_perm) != 0) {
+ pmm_free_page(page);
+ return -ENOMEM;
+ }
+
+ /* Initial mapping: refcount is already 1 from pmm_alloc_page.
+ * Set mapcount = 1 to track this PTE mapping. */
+ page_inc_mapcount(page);
+
+ return 0;
+}
+
+/* ------------------------------------------------------------------ */
+/* Fork / Clone (COW infrastructure) */
+/* ------------------------------------------------------------------ */
+
+VmSpace* vmm_clone(VmSpace* parent) {
+ if (!parent) return nullptr;
+
+ VmSpace* child = VmSpace::create();
+ if (!child) return nullptr;
+
+ child->brk_start = parent->brk_start;
+ child->brk_end = parent->brk_end;
+
+ LockGuard parent_guard(parent->lock);
+
+ struct rb_node* node = rb_first(&parent->vma_tree);
+ while (node) {
+ VmAreaStruct* parent_vma = reinterpret_cast<VmAreaStruct*>(
+ reinterpret_cast<char*>(node) - __builtin_offsetof(VmAreaStruct, rb));
+
+ VmAreaStruct* child_vma = new VmAreaStruct;
+ if (!child_vma) {
+ child->destroy();
+ return nullptr;
+ }
+ child_vma->start = parent_vma->start;
+ child_vma->end = parent_vma->end;
+ child_vma->perm = parent_vma->perm;
+ child_vma->mm = child;
+
+ /* Copy pages and mark COW if writable */
+ for (uint64 va = child_vma->start; va < child_vma->end; va += kPageSize) {
+ uint64 pa;
+ if (pmap::lookup(parent->root_pa, va, &pa)) {
+ /* Make parent COW if it was writable */
+ if (child_vma->perm & VM_WRITE) {
+ pmap::make_cow(parent->root_pa, va);
+ /* Flush TLB for parent since we reduced permissions */
+ tlb_shootdown_page(parent, va);
+ }
+
+ /* Map in child as COW or read-only */
+ uint64 pte_perm = pmap::vm_perm_to_pte(child_vma->perm);
+ if (child_vma->perm & VM_WRITE) {
+ pte_perm &= ~pmap::PTE_W;
+ pte_perm |= pmap::PTE_COW;
+ }
+
+ if (pmap::map(child->root_pa, va, pa, pte_perm) != 0) {
+ /* If map fails, we leak what we did in this loop for now,
+ * but child->destroy() will clean it up safely since
+ * we increment refcounts as we go. */
+ delete child_vma;
+ child->destroy();
+ return nullptr;
+ }
+
+ /* Increment physical page refcount and mapcount */
+ Page* shared_page = pa_to_page(phys_addr(pa));
+ page_incref(shared_page);
+ page_inc_mapcount(shared_page);
+ }
+ }
+
+ rb_add(&child_vma->rb, &child->vma_tree, vma_less);
+ node = rb_next(node);
+ }
+
+ return child;
+}
diff --git a/mm/vmm.cc b/mm/vmm.cc
new file mode 100644
index 0000000..51617e3
--- /dev/null
+++ b/mm/vmm.cc
@@ -0,0 +1,235 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/* Copyright (C) 2026 Richard Qin */
+
+/*
+ * vmm.cc — Virtual Memory Manager (Phase 1: Kernel Page Table)
+ *
+ * Builds a fine-grained kernel page table, activates it, and retires
+ * the early_pgdir identity mapping from entry.S.
+ *
+ * With memblock integration, the initialization order is:
+ * memblock_init → vmm_init (allocates page_map from memblock,
+ * maps vmemmap, activates page table) → pmm_init (buddy system
+ * starts with page_map already at vmemmap VA — no relocation).
+ *
+ * Reference: docs/specs/vmm.md
+ */
+
+#include <arch/config.h>
+#include <arch/mmu.h>
+#include <arch/pmap.h>
+#include <lume/tlb.h>
+#include <lume/addr.h>
+#include <lume/console.h>
+#include <lume/fdt.h>
+#include <lume/page.h>
+#include <lume/klog.h>
+#include <lume/kprintf.h>
+#include <lume/memblock.h>
+#include <lume/pmm.h>
+#include <lume/types.h>
+#include <lume/vmm.h>
+#include <lume/config.h>
+
+/* Linker-exported section boundaries (virtual addresses) */
+extern "C" char _stext[], _etext[];
+extern "C" char _srodata[], _erodata[];
+extern "C" char _sdata[], _edata[];
+extern "C" char _sbss[], _ebss[];
+extern "C" char _kernel_end[];
+
+/* Global kernel page table root PA — external linkage for task_exit */
+uint64 g_kernel_pgtbl = 0;
+
+namespace {
+
+/* VMM exports for PMM: page_map location after vmemmap mapping. */
+uint64 g_vmemmap_page_map_pa = 0;
+uint64 g_vmemmap_page_map_size = 0;
+uint64 g_vmemmap_num_pages = 0;
+
+/* Convert a linker symbol to PA (kernel is executing in higher-half VA). */
+inline uint64 sym_to_pa(const void *sym) {
+ return boot_va_to_pa(virt_addr(reinterpret_cast<uint64>(sym))).raw;
+}
+
+/* Map a PA range into the kernel page table via the direct map. */
+void map_range(uint64 root, uint64 pa_start, uint64 pa_end,
+ uint64 perm) {
+ for (uint64 pa = pa_start; pa < pa_end; pa += kPageSize) {
+ uint64 va = pa_to_va(phys_addr(pa)).raw;
+ int ret = pmap::map(root, va, pa, perm | pmap::PTE_G);
+ if (ret != 0) {
+ kernel_panic("vmm_init: pmap::map failed");
+ }
+ }
+}
+
+/* Map a PA range into the kernel page table at the fixed linked VA. */
+void map_range_linked(uint64 root, uint64 pa_start, uint64 pa_end,
+ uint64 perm) {
+ for (uint64 pa = pa_start; pa < pa_end; pa += kPageSize) {
+ uint64 va = boot_pa_to_va(phys_addr(pa)).raw;
+ int ret = pmap::map(root, va, pa, perm | pmap::PTE_G);
+ if (ret != 0) {
+ kernel_panic("vmm_init: pmap::map (linked) failed");
+ }
+ }
+}
+
+/* Map a PA range to an explicit VA base, using 2MB pages where possible. */
+void map_range_at_huge(uint64 root, uint64 va_base, uint64 pa_start,
+ uint64 pa_end, uint64 perm) {
+ constexpr uint64 k2MB = 2ULL * 1024 * 1024;
+ uint64 pa = pa_start;
+ uint64 va = va_base;
+
+ /* Map 2MB-aligned chunks with mega pages */
+ while (pa + k2MB <= pa_end) {
+ if ((va & (k2MB - 1)) == 0 && (pa & (k2MB - 1)) == 0) {
+ int ret = pmap::map_2mb(root, va, pa, perm | pmap::PTE_G);
+ if (ret != 0) {
+ kernel_panic("vmm_init: pmap::map_2mb failed");
+ }
+ pa += k2MB;
+ va += k2MB;
+ } else {
+ break;
+ }
+ }
+
+ /* Map remaining 4KB tail */
+ while (pa < pa_end) {
+ int ret = pmap::map(root, va, pa, perm | pmap::PTE_G);
+ if (ret != 0) {
+ kernel_panic("vmm_init: vmemmap 4KB pmap::map failed");
+ }
+ pa += kPageSize;
+ va += kPageSize;
+ }
+}
+
+} // namespace
+
+uint64 vmm_get_page_map_pa() { return g_vmemmap_page_map_pa; }
+uint64 vmm_get_page_map_size() { return g_vmemmap_page_map_size; }
+uint64 vmm_get_num_pages() { return g_vmemmap_num_pages; }
+
+void vmm_init() {
+ /* 1. Allocate new root page table (from memblock) */
+ uint64 root = pmap::create();
+ if (!root) {
+ kernel_panic("vmm_init: failed to create root page table");
+ }
+
+ /* 2. Compute PA boundaries of each kernel section */
+ uint64 text_start = page_align_down(sym_to_pa(_stext));
+ uint64 text_end = page_align_up(sym_to_pa(_etext));
+ uint64 ro_start = page_align_down(sym_to_pa(_srodata));
+ uint64 ro_end = page_align_up(sym_to_pa(_erodata));
+ uint64 data_start = page_align_down(sym_to_pa(_sdata));
+ uint64 bss_end = page_align_up(sym_to_pa(_ebss));
+
+ /* Memory extent from memblock */
+ uint64 mem_base = memblock_get_mem_base();
+ uint64 mem_size = memblock_get_mem_size();
+ uint64 mem_end = mem_base + mem_size;
+
+ /* 3. Map kernel image at FIXED linked VA.
+ *
+ * Kernel text/data are linked at the bootstrap higher-half addresses
+ * (kDirectMapBaseDefault + PA). They must remain executable/readable at
+ * those addresses until full kernel-image relocation is implemented.
+ */
+ map_range_linked(root, text_start, text_end, pmap::PTE_R | pmap::PTE_X);
+
+ /* 4. Map .rodata (R) at fixed linked VA.
+ * We map [_srodata, _sdata) so that the .lume.reloc section (which
+ * sits in its own padded output section between _erodata and _sdata)
+ * is also covered. Relocator only reads it during early boot, but
+ * mapping it keeps post-vmm accesses to __lume_reloc_* symbols sane. */
+ map_range_linked(root, ro_start, page_align_down(sym_to_pa(_sdata)),
+ pmap::PTE_R);
+
+ /* 5. Map .data + .bss (RW) at fixed linked VA */
+ map_range_linked(root, data_start, bss_end, pmap::PTE_R | pmap::PTE_W);
+
+ /* 6. Map usable physical RAM into the KASLR-slid direct map.
+ *
+ * The direct map provides pa_to_va() / va_to_pa() for the physical
+ * address space. We map from kPhysStart (the kernel's linked load
+ * address), skipping [mem_base, kPhysStart) because that region is
+ * PMP-protected firmware (OpenSBI) — S-mode cannot access it.
+ *
+ * Linux does the same on RISC-V: the linear map covers only the
+ * usable RAM range, not firmware-reserved regions.
+ *
+ * Uses 2MB huge pages for efficiency (64 entries for 128MB vs 32K).
+ */
+ uint64 dm_start = kPhysStart;
+ if (dm_start < mem_base) {
+ dm_start = mem_base;
+ }
+ uint64 dm_va_base = pa_to_va(phys_addr(dm_start)).raw;
+ map_range_at_huge(root, dm_va_base, dm_start, mem_end,
+ pmap::PTE_R | pmap::PTE_W);
+
+ /* 7. Allocate page_map from memblock (top of RAM) and map at VMEMMAP.
+ *
+ * Since memblock is active (PMM not yet initialized), pmap::map
+ * allocates page table pages from memblock too. page_map starts
+ * life at the vmemmap VA — no relocation ever needed.
+ */
+ uint64 num_pages = mem_size / kPageSize;
+ uint64 page_map_bytes = num_pages * sizeof(Page);
+ uint64 page_map_span = page_align_up(page_map_bytes);
+ uint64 page_map_pa = memblock_alloc_top(page_map_span, kPageSize);
+
+ if (!page_map_pa) {
+ kernel_panic("vmm_init: memblock failed to allocate page_map");
+ }
+
+ kprintf("[vmm] vmemmap: %llu pages, %llu KB at PA 0x%llx\n", num_pages,
+ page_map_span / 1024, page_map_pa);
+
+ /* Map page_map at vmemmap_base from KaslrLayout (randomised by KASLR), using 2MB pages where possible */
+ map_range_at_huge(root, arch::kaslr_layout_get().vmemmap_base, page_map_pa,
+ page_map_pa + page_map_span, pmap::PTE_R | pmap::PTE_W);
+
+ /* Export for pmm_init() */
+ g_vmemmap_page_map_pa = page_map_pa;
+ g_vmemmap_page_map_size = page_map_bytes;
+ g_vmemmap_num_pages = num_pages;
+
+ /* 8. Pre-map console MMIO before page table switch */
+ g_kernel_pgtbl = root;
+ console_init();
+
+ /* 9. Activate the new page table */
+ pmap::activate(root);
+
+ /* Console MMIO access must now use runtime (KASLR) direct-map base. */
+ console_use_runtime_mapping();
+
+ kprintf("[vmm] vmm_init complete (vmemmap at 0x%llx)\n", arch::kaslr_layout_get().vmemmap_base);
+}
+
+void vmm_init_ap() {
+ if (!g_kernel_pgtbl) {
+ kernel_panic("vmm_init_ap: kernel page table not ready");
+ }
+ pmap::activate(g_kernel_pgtbl);
+}
+
+void vmm_map_kernel_mmio(uint64 pa, uint64 size) {
+ if (!g_kernel_pgtbl) {
+ kernel_panic("vmm_map_kernel_mmio: kernel page table not ready");
+ }
+
+ uint64 aligned_pa = page_align_down(pa);
+ uint64 aligned_size = page_align_up(pa + size) - aligned_pa;
+
+ map_range(g_kernel_pgtbl, aligned_pa, aligned_pa + aligned_size,
+ pmap::PTE_R | pmap::PTE_W | pmap::PTE_G);
+ tlb_shootdown_all();
+}
generated by cgit v1.2.3 (git 2.46.0) at 2026-10-29 13:03:54 +0000