feat: establish bootable new architecture checkpoint

Replace the compressed boot path with a relocatable higher-half RISC-V entry, expand memory and SMP support, and add the driver, TTY, VFS, exec, signal, syscall, initramfs, and self-test infrastructure needed for a bootable kernel checkpoint.

8 files changed, 624 insertions, 124 deletions

diff --git a/na/mm/memblock.cc b/na/mm/memblock.cc
index 4cd0b16..aee0245 100644
--- a/na/mm/memblock.cc
+++ b/na/mm/memblock.cc
@@ -16,6 +16,7 @@
 #include <lume/fdt.h>
 #include <lume/klog.h>
 #include <lume/kprintf.h>
+#include <lume/config.h>
 #include <lume/addr.h>
 
 extern "C" char _stext[], _kernel_end[];
@@ -24,7 +25,11 @@ extern "C" char _stext[], _kernel_end[];
 /*  Module state                                                       */
 /* ------------------------------------------------------------------ */
 
-static struct {
+namespace {
+
+constexpr uint64 kMB = 1024ULL * 1024ULL;
+
+struct {
     uint64 mem_base;
     uint64 mem_size;
 
@@ -41,7 +46,7 @@ static struct {
 /* ------------------------------------------------------------------ */
 
 /* Insert a reserved region, keeping the array sorted by base. */
-static void insert_reserved(uint64 base, uint64 size)
+void insert_reserved(uint64 base, uint64 size)
 {
     if (g_mb.reserved_count >= kMemblockMaxRegions) {
         kernel_panic("memblock: too many reserved regions");
@@ -57,7 +62,7 @@ static void insert_reserved(uint64 base, uint64 size)
 }
 
 /* Check if [base, base+size) overlaps any reserved region. */
-static bool overlaps_reserved(uint64 base, uint64 size)
+bool overlaps_reserved(uint64 base, uint64 size)
 {
     uint64 end = base + size;
     for (int i = 0; i < g_mb.reserved_count; i++) {
@@ -71,7 +76,7 @@ static bool overlaps_reserved(uint64 base, uint64 size)
 }
 
 /* Find the end of the first reserved region that covers `addr`. */
-static uint64 reserved_end_covering(uint64 addr)
+uint64 reserved_end_covering(uint64 addr)
 {
     for (int i = 0; i < g_mb.reserved_count; i++) {
         uint64 rb = g_mb.reserved[i].base;
@@ -83,13 +88,14 @@ static uint64 reserved_end_covering(uint64 addr)
     return addr;
 }
 
+} // namespace
+
 /* ------------------------------------------------------------------ */
 /*  Public API                                                         */
 /* ------------------------------------------------------------------ */
 
 void memblock_init(uint64 fdt_paddr)
 {
-    (void)fdt_paddr;
 
     /* 1. Get memory layout from FDT */
     uint64 mem_base = 0;
@@ -104,21 +110,58 @@ void memblock_init(uint64 fdt_paddr)
     g_mb.reserved_count = 0;
     g_mb.active = true;
 
-    /* 2. Reserve firmware region [mem_base, kernel_start).
-     *    OpenSBI occupies the first 2MB of RAM on RISC-V virt. */
-    uint64 ks_pa = boot_va_to_pa(virt_addr(
-        reinterpret_cast<uint64>(_stext))).raw;
-    if (ks_pa > mem_base) {
-        memblock_reserve(mem_base, ks_pa - mem_base);
+    /* 2. Reserve firmware region [mem_base, kPhysStart).
+     *
+     * Everything below the kernel's linked load address is firmware
+     * territory (OpenSBI), which is PMP-protected. */
+    {
+        uint64 fw_end = kPhysStart;
+        if (fw_end > mem_base) {
+            memblock_reserve(mem_base, fw_end - mem_base);
+        }
+    }
+
+    /* 2b. Reserve stale pre-relocation kernel image.
+     *
+     * After physical KASLR, the kernel is relocated from its original
+     * load address (kPhysStart) to a new PA.  The old copy at
+     * [kPhysStart, kPhysStart + kernel_size) must not be allocated —
+     * it contains stale code/data that would corrupt page tables or
+     * other structures if reused. */
+    {
+        uint64 ks_pa = boot_va_to_pa(virt_addr(
+            reinterpret_cast<uint64>(_stext))).raw;
+        if (ks_pa != kPhysStart) {
+            /* Kernel was relocated — reserve the original location */
+            uint64 kernel_size = boot_va_to_pa(virt_addr(
+                reinterpret_cast<uint64>(_kernel_end))).raw - ks_pa;
+            memblock_reserve(kPhysStart, kernel_size);
+        }
     }
 
     /* 3. Reserve kernel image [_stext, _kernel_end) */
+    uint64 ks_pa = boot_va_to_pa(virt_addr(
+        reinterpret_cast<uint64>(_stext))).raw;
     uint64 ke_pa = boot_va_to_pa(virt_addr(
         reinterpret_cast<uint64>(_kernel_end))).raw;
     memblock_reserve(ks_pa, ke_pa - ks_pa);
 
+    /* 4. Reserve FDT blob — it lives in RAM and must not be overwritten
+     *    by vmemmap/page_map or buddy before fdt_unflatten() copies it. */
+    {
+        struct fdt_hdr { uint32 magic; uint32 totalsize; };
+        const auto* fh = reinterpret_cast<const fdt_hdr*>(fdt_paddr);
+        uint32 fdt_size = ((fh->totalsize & 0xFF) << 24) |
+                          (((fh->totalsize >> 8) & 0xFF) << 16) |
+                          (((fh->totalsize >> 16) & 0xFF) << 8) |
+                          ((fh->totalsize >> 24) & 0xFF);
+        memblock_reserve(fdt_paddr, fdt_size);
+        kprintf("[memblock] FDT reserved [0x%llx, 0x%llx)\n",
+                fdt_paddr, fdt_paddr + fdt_size);
+    }
+
     kprintf("[memblock] RAM [0x%llx, 0x%llx) = %llu MB\n",
-            mem_base, mem_base + mem_size, mem_size / (1024 * 1024));
+            mem_base, mem_base + mem_size, mem_size / kMB);
     kprintf("[memblock] kernel reserved [0x%llx, 0x%llx)\n", ks_pa, ke_pa);
 }
 
@@ -196,7 +239,6 @@ void memblock_for_each_free(void (*cb)(uint64 base, uint64 size, void* ctx),
     for (int i = 0; i < g_mb.reserved_count && pos < mem_end; i++) {
         uint64 rb = g_mb.reserved[i].base;
         if (rb < pos) {
-            /* Overlapping/adjacent — skip */
             uint64 re = g_mb.reserved[i].base + g_mb.reserved[i].size;
             if (re > pos) {
                 pos = re;
@@ -204,13 +246,11 @@ void memblock_for_each_free(void (*cb)(uint64 base, uint64 size, void* ctx),
             continue;
         }
         if (rb > pos) {
-            /* Free gap [pos, rb) */
             cb(pos, rb - pos, ctx);
         }
         pos = g_mb.reserved[i].base + g_mb.reserved[i].size;
     }
 
-    /* Trailing free region after last reserved */
     if (pos < mem_end) {
         cb(pos, mem_end - pos, ctx);
     }
diff --git a/na/mm/module.mk b/na/mm/module.mk
index d497067..93cef80 100644
--- a/na/mm/module.mk
+++ b/na/mm/module.mk
@@ -3,3 +3,5 @@ SRCS_CC += mm/pmm.cc
 SRCS_CC += mm/slab.cc
 SRCS_CC += mm/vmm.cc
 SRCS_CC += mm/vma.cc
+SRCS_CC += mm/page_cache.cc
+SRCS_CC += mm/tlb.cc
diff --git a/na/mm/page_cache.cc b/na/mm/page_cache.cc
new file mode 100644
index 0000000..25c6a60
--- /dev/null
+++ b/na/mm/page_cache.cc
@@ -0,0 +1,252 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/* Copyright (C) 2026 Richard Qin */
+
+/*
+ * page_cache.cc — Minimal page cache implementation
+ *
+ * Open-addressing hash table indexed by (mapping, index) pairs.
+ * Fixed-size bucket array allocated at init time via slab.
+ *
+ * This is intentionally simple: no LRU, no writeback, no eviction.
+ * The interface in page_cache.h is designed so that upgrading to a
+ * radix tree + LRU scheme later only changes this file.
+ *
+ * Concurrency: protected by a single spinlock. Acceptable for
+ * initial bring-up; will be replaced by per-bucket or RCU-based
+ * locking when contention becomes measurable.
+ */
+
+#include <lume/page_cache.h>
+#include <lume/page.h>
+#include <lume/pmm.h>
+#include <lume/errno.h>
+#include <lume/new.h>
+#include <kernel/sync/spinlock.h>
+
+/* ------------------------------------------------------------------ */
+/*  Hash table internals                                              */
+/* ------------------------------------------------------------------ */
+
+struct CacheEntry {
+    void*  mapping;   // nullptr = empty slot
+    uint64 index;
+    Page*  page;
+};
+
+namespace {
+
+constexpr uint64 kCacheBuckets = 1024;
+
+CacheEntry g_buckets[kCacheBuckets];
+Spinlock   g_cache_lock;
+
+uint64 hash_key(void* mapping, uint64 index)
+{
+    uint64 m = reinterpret_cast<uint64>(mapping);
+    /* FNV-1a inspired mixing */
+    uint64 h = m ^ (index * 0x517cc1b727220a95ULL);
+    h ^= h >> 17;
+    h *= 0xbf58476d1ce4e5b9ULL;
+    h ^= h >> 31;
+    return h % kCacheBuckets;
+}
+
+/* Linear probe to find an entry or an empty slot. */
+CacheEntry* find_slot(void* mapping, uint64 index)
+{
+    uint64 start = hash_key(mapping, index);
+    for (uint64 i = 0; i < kCacheBuckets; i++) {
+        uint64 idx = (start + i) % kCacheBuckets;
+        CacheEntry* e = &g_buckets[idx];
+        if (e->mapping == nullptr) {
+            return e;  // Empty slot
+        }
+        if (e->mapping == mapping && e->index == index) {
+            return e;  // Found
+        }
+    }
+    return nullptr;  // Table full
+}
+
+/* Find an existing entry (not an empty slot). */
+CacheEntry* find_entry(void* mapping, uint64 index)
+{
+    uint64 start = hash_key(mapping, index);
+    for (uint64 i = 0; i < kCacheBuckets; i++) {
+        uint64 idx = (start + i) % kCacheBuckets;
+        CacheEntry* e = &g_buckets[idx];
+        if (e->mapping == nullptr) {
+            return nullptr;  // Empty = not found
+        }
+        if (e->mapping == mapping && e->index == index) {
+            return e;
+        }
+    }
+    return nullptr;
+}
+
+/*
+ * Remove an entry and fix up the linear-probe chain.
+ * Standard open-addressing deletion with backward shift.
+ */
+void remove_entry(CacheEntry* entry)
+{
+    uint64 removed = static_cast<uint64>(entry - g_buckets);
+    entry->mapping = nullptr;
+    entry->page = nullptr;
+
+    /* Backward-shift deletion to maintain probe chain integrity */
+    uint64 cur = (removed + 1) % kCacheBuckets;
+    while (g_buckets[cur].mapping != nullptr) {
+        uint64 ideal = hash_key(g_buckets[cur].mapping, g_buckets[cur].index);
+        /* Check if cur's ideal position is "before" removed in probe order */
+        bool needs_shift = false;
+        if (removed <= cur) {
+            needs_shift = (ideal <= removed || ideal > cur);
+        } else {
+            needs_shift = (ideal <= removed && ideal > cur);
+        }
+
+        if (needs_shift) {
+            g_buckets[removed] = g_buckets[cur];
+            g_buckets[cur].mapping = nullptr;
+            g_buckets[cur].page = nullptr;
+            removed = cur;
+        }
+        cur = (cur + 1) % kCacheBuckets;
+    }
+}
+
+} // namespace
+
+/* ------------------------------------------------------------------ */
+/*  Public API                                                        */
+/* ------------------------------------------------------------------ */
+
+void page_cache_init()
+{
+    g_cache_lock.init("page_cache");
+    for (uint64 i = 0; i < kCacheBuckets; i++) {
+        g_buckets[i].mapping = nullptr;
+        g_buckets[i].index = 0;
+        g_buckets[i].page = nullptr;
+    }
+}
+
+Page* page_cache_get(void* mapping, uint64 index)
+{
+    LockGuard guard(g_cache_lock);
+    CacheEntry* e = find_entry(mapping, index);
+    if (!e) {
+        return nullptr;
+    }
+    page_incref(e->page);
+    return e->page;
+}
+
+int page_cache_insert(void* mapping, uint64 index, Page* page)
+{
+    if (!mapping || !page) {
+        return -EINVAL;
+    }
+
+    LockGuard guard(g_cache_lock);
+
+    /* Check for existing entry */
+    CacheEntry* existing = find_entry(mapping, index);
+    if (existing) {
+        return -EEXIST;
+    }
+
+    /* Find an empty slot */
+    CacheEntry* slot = find_slot(mapping, index);
+    if (!slot || slot->mapping != nullptr) {
+        return -ENOMEM;  // Table full
+    }
+
+    slot->mapping = mapping;
+    slot->index = index;
+    slot->page = page;
+    page_incref(page);  // Cache holds a reference
+
+    return 0;
+}
+
+void page_cache_put(Page* page)
+{
+    if (!page) {
+        return;
+    }
+    page_decref(page);
+}
+
+Page* page_cache_remove(void* mapping, uint64 index)
+{
+    LockGuard guard(g_cache_lock);
+    CacheEntry* e = find_entry(mapping, index);
+    if (!e) {
+        return nullptr;
+    }
+
+    Page* page = e->page;
+    remove_entry(e);
+    page_decref(page);  // Release cache's reference
+    return page;
+}
+
+void page_cache_invalidate(void* mapping)
+{
+    if (!mapping) {
+        return;
+    }
+
+    LockGuard guard(g_cache_lock);
+
+    /* Scan all buckets — acceptable for current table size */
+    for (uint64 i = 0; i < kCacheBuckets; i++) {
+        if (g_buckets[i].mapping == mapping) {
+            Page* page = g_buckets[i].page;
+            remove_entry(&g_buckets[i]);
+            page_decref(page);
+            /* After removal, re-check this index since backward shift
+             * may have moved an entry into this slot. */
+            i--;
+        }
+    }
+}
+
+Page* page_cache_get_or_alloc(void* mapping, uint64 index)
+{
+    /* Fast path: already cached */
+    Page* page = page_cache_get(mapping, index);
+    if (page) {
+        return page;
+    }
+
+    /* Slow path: allocate and insert */
+    page = pmm_alloc_page();
+    if (!page) {
+        return nullptr;
+    }
+
+    int ret = page_cache_insert(mapping, index, page);
+    if (ret == -EEXIST) {
+        /* Race: another thread inserted between our get and insert.
+         * Free our allocation and return the existing page. */
+        pmm_free_page(page);
+        return page_cache_get(mapping, index);
+    }
+    if (ret < 0) {
+        pmm_free_page(page);
+        return nullptr;
+    }
+
+    /* Caller gets a ref (insert already added cache's ref,
+     * we need one more for the caller). */
+    /* Actually, insert did incref for cache. The page already has
+     * refcount=1 from alloc + 1 from insert = 2.
+     * We return it to caller who must put() when done.
+     * So we incref once more for the caller's reference. */
+    page_incref(page);
+    return page;
+}
diff --git a/na/mm/pmm.cc b/na/mm/pmm.cc
index a5a0b0e..b7cf611 100644
--- a/na/mm/pmm.cc
+++ b/na/mm/pmm.cc
@@ -43,16 +43,25 @@ struct PerCpuCache {
 /*  Module-private state                                              */
 /* ------------------------------------------------------------------ */
 
-static Spinlock g_pmm_lock;
-static list_node g_free_areas[kMaxOrder]; // Buddy free lists [0..10]
-static PerCpuCache g_pcp[kMaxCpus];
+namespace {
 
-static Page *g_page_map = nullptr; // Flat array of Page descriptors
-static uint64 g_page_map_pa = 0;   // Physical base of page_map storage
-static uint64 g_page_map_bytes = 0; // Byte size of page_map storage
-static uint64 g_mem_base = 0;        // Physical memory base (from FDT)
-static uint64 g_mem_size = 0;        // Total physical memory size
-static uint64 g_num_pages = 0;       // Total managed pages
+/* PCP Cache tuning parameters */
+constexpr uint32 kPcpBatchDivisor = 1024;
+constexpr uint32 kPcpMinBatchSize = 16;
+constexpr uint64 kMB = 1024ULL * 1024ULL;
+
+Spinlock g_pmm_lock;
+list_node g_free_areas[kMaxOrder]; // Buddy free lists [0..10]
+PerCpuCache g_pcp[kMaxCpus];
+
+Page *g_page_map = nullptr; // Flat array of Page descriptors
+uint64 g_page_map_pa = 0;   // Physical base of page_map storage
+uint64 g_page_map_bytes = 0; // Byte size of page_map storage
+uint64 g_mem_base = 0;        // Physical memory base (from FDT)
+uint64 g_mem_size = 0;        // Total physical memory size
+uint64 g_num_pages = 0;       // Total managed pages
+
+} // namespace
 
 uint32 g_pcp_high_watermark = 0;
 uint32 g_pcp_batch_size = 0;
@@ -62,63 +71,68 @@ uint32 g_pcp_batch_size = 0;
 /* ------------------------------------------------------------------ */
 
 /*
- * Physical Page Number: index into the page_map array.
- * PFN = (pa - g_mem_base) / kPageSize
+ * PFN (Page Frame Number) — fundamental conversion unit.
+ * When sparse vmemmap is implemented, only these two functions change.
+ * All other PA↔Page conversions go through them.
  */
-static inline uint64 pa_to_page_index(uint64 pa) {
-  if (pa < g_mem_base || pa >= g_mem_base + g_num_pages * kPageSize)
-    kernel_panic("pa_to_page: physical address outside managed range");
-  return (pa - g_mem_base) / kPageSize;
-}
-
-static inline uint64 page_index_to_pa(uint64 idx) {
-  if (idx >= g_num_pages)
-    kernel_panic("page_to_pa: page index outside managed range");
-  return g_mem_base + idx * kPageSize;
-}
-
-PhysAddr page_to_pa(const Page *page) {
+uint64 page_to_pfn(const Page *page) {
   uintptr base = reinterpret_cast<uintptr>(g_page_map);
   uintptr addr = reinterpret_cast<uintptr>(page);
   uintptr end = base + g_num_pages * sizeof(Page);
 
-  if (addr < base || addr >= end)
-    kernel_panic("page_to_pa: page outside page_map bounds");
+  if (addr < base || addr >= end) {
+    kernel_panic("page_to_pfn: page outside page_map bounds");
+  }
 
   uint64 idx = (addr - base) / sizeof(Page);
-  return phys_addr(page_index_to_pa(idx));
+  return (g_mem_base / kPageSize) + idx;
+}
+
+Page *pfn_to_page(uint64 pfn) {
+  uint64 base_pfn = g_mem_base / kPageSize;
+  if (pfn < base_pfn || pfn >= base_pfn + g_num_pages) {
+    kernel_panic("pfn_to_page: PFN outside managed range");
+  }
+  return &g_page_map[pfn - base_pfn];
+}
+
+PhysAddr page_to_pa(const Page *page) {
+  return pfn_to_pa(Pfn{page_to_pfn(page)});
 }
 
 Page *pa_to_page(PhysAddr pa) {
-  uint64 idx = pa_to_page_index(pa.raw);
-  return &g_page_map[idx];
+  return pfn_to_page(pa_to_pfn(pa).raw);
 }
 
 VirtAddr page_to_va(const Page *page) {
   return pa_to_va(page_to_pa(page));
 }
 
+
 /* ------------------------------------------------------------------ */
 /*  Buddy system internals (caller must hold g_pmm_lock)              */
 /* ------------------------------------------------------------------ */
 
+namespace {
+
 /*
  * Find a page's buddy at a given order.
  * Buddy PFN = PFN ^ (1 << order).
  */
-static inline Page *buddy_of(Page *page, uint8 order) {
+inline Page *buddy_of(Page *page, uint8 order) {
   uint64 pfn = static_cast<uint64>(page - g_page_map);
   uint64 buddy_pfn = pfn ^ (1ULL << order);
-  if (buddy_pfn >= g_num_pages)
+  if (buddy_pfn >= g_num_pages) {
     return nullptr;
+  }
   return &g_page_map[buddy_pfn];
 }
 
 /*
  * Insert a free block into the buddy free list at the given order.
  */
-static void buddy_list_add(Page *page, uint8 order) {
-  page->state = PageState::Free;
+void buddy_list_add(Page *page, uint8 order) {
+  page->type = PageType::Free;
   page->order = order;
   list_node *head = &g_free_areas[order];
   list_node *node = &page->free.free_link;
@@ -131,7 +145,7 @@ static void buddy_list_add(Page *page, uint8 order) {
 /*
  * Remove a block from whatever buddy free list it is on.
  */
-static void buddy_list_del(Page *page) {
+void buddy_list_del(Page *page) {
   list_node *node = &page->free.free_link;
   node->prev->next = node->next;
   node->next->prev = node->prev;
@@ -144,11 +158,12 @@ static void buddy_list_del(Page *page) {
  * Splits higher-order blocks if no exact match is available.
  * Returns nullptr if no memory available.
  */
-static Page *buddy_alloc(uint8 order) {
+Page *buddy_alloc(uint8 order) {
   for (int cur = order; cur < kMaxOrder; cur++) {
     list_node *head = &g_free_areas[cur];
-    if (head->next == head)
+    if (head->next == head) {
       continue; // Empty at this order
+    }
 
     // Pop the first block from this order's free list
     list_node *node = head->next;
@@ -166,9 +181,13 @@ static Page *buddy_alloc(uint8 order) {
       buddy_list_add(split, static_cast<uint8>(cur));
     }
 
-    page->state = PageState::Allocated;
+    page->type = PageType::Allocated;
     page->order = order;
     page->refcount.store(1, __ATOMIC_RELAXED);
+    page->mapcount.store(0, __ATOMIC_RELAXED);
+    page->flags.store(0, __ATOMIC_RELAXED);
+    page->mapping = nullptr;
+    page->index = 0;
     return page;
   }
   return nullptr;
@@ -178,16 +197,18 @@ static Page *buddy_alloc(uint8 order) {
  * Return a block of 2^order pages to the buddy system.
  * Coalesces with its buddy recursively up to kMaxOrder-1.
  */
-static void buddy_free(Page *page, uint8 order) {
+void buddy_free(Page *page, uint8 order) {
   uint64 pfn = static_cast<uint64>(page - g_page_map);
 
   while (order < kMaxOrder - 1) {
     Page *buddy = buddy_of(page, order);
-    if (!buddy)
+    if (!buddy) {
       break;
+    }
     // Buddy must be free AND at the same order to coalesce
-    if (buddy->state != PageState::Free || buddy->order != order)
+    if (buddy->type != PageType::Free || buddy->order != order) {
       break;
+    }
 
     // Remove buddy from its free list and merge
     buddy_list_del(buddy);
@@ -212,14 +233,15 @@ static void buddy_free(Page *page, uint8 order) {
  * Refill the PCP from the global buddy system.
  * Acquires g_pmm_lock internally.
  */
-static void refill_pcp(PerCpuCache *pcp) {
+void refill_pcp(PerCpuCache *pcp) {
   LockGuard guard(g_pmm_lock);
   for (uint32 i = 0; i < g_pcp_batch_size; i++) {
     Page *page = buddy_alloc(0);
-    if (!page)
+    if (!page) {
       break;
+    }
     // Put onto PCP list (reuse the free_link for PCP chain)
-    page->state = PageState::PcpCached;
+    page->type = PageType::PcpCached;
     list_node *node = &page->free.free_link;
     node->next = pcp->free_list.next;
     node->prev = &pcp->free_list;
@@ -233,13 +255,14 @@ static void refill_pcp(PerCpuCache *pcp) {
  * Drain half the PCP back to the global buddy system.
  * Acquires g_pmm_lock internally.
  */
-static void drain_pcp(PerCpuCache *pcp) {
+void drain_pcp(PerCpuCache *pcp) {
   LockGuard guard(g_pmm_lock);
   int to_drain = pcp->count / 2;
   for (int i = 0; i < to_drain; i++) {
     list_node *head = &pcp->free_list;
-    if (head->next == head)
+    if (head->next == head) {
       break;
+    }
     list_node *node = head->next;
     // Unlink from PCP
     node->prev->next = node->next;
@@ -253,6 +276,8 @@ static void drain_pcp(PerCpuCache *pcp) {
   }
 }
 
+} // namespace
+
 /* ------------------------------------------------------------------ */
 /*  Public API: single-page (fast path via PCP)                       */
 /* ------------------------------------------------------------------ */
@@ -269,8 +294,9 @@ Page *pmm_alloc_page() {
   uint64 cpu = arch::cpu::id();
   PerCpuCache *pcp = &g_pcp[cpu];
 
-  if (pcp->count == 0)
+  if (pcp->count == 0) {
     refill_pcp(pcp);
+  }
 
   Page *page = nullptr;
   if (pcp->count > 0) {
@@ -289,19 +315,25 @@ Page *pmm_alloc_page() {
       kernel_panic("pmm_alloc_page: PCP returned out-of-bounds page");
     }
 
-    page->state = PageState::Allocated;
+    page->type = PageType::Allocated;
     page->order = 0;
     page->refcount.store(1, __ATOMIC_RELAXED);
+    page->mapcount.store(0, __ATOMIC_RELAXED);
+    page->flags.store(0, __ATOMIC_RELAXED);
+    page->mapping = nullptr;
+    page->index = 0;
   }
 
-  if (was_on)
+  if (was_on) {
     arch::cpu::intr_on();
+  }
   return page;
 }
 
 void pmm_free_page(Page *page) {
-  if (!page)
+  if (!page) {
     kernel_panic("pmm_free_page: null page");
+  }
 
   bool was_on = arch::cpu::intr_enabled();
   arch::cpu::intr_off();
@@ -310,7 +342,7 @@ void pmm_free_page(Page *page) {
   PerCpuCache *pcp = &g_pcp[cpu];
 
   // Push onto PCP
-  page->state = PageState::PcpCached;
+  page->type = PageType::PcpCached;
   page->order = 0;
   list_node *node = &page->free.free_link;
   node->next = pcp->free_list.next;
@@ -320,11 +352,13 @@ void pmm_free_page(Page *page) {
   pcp->count++;
 
   // Drain if over high watermark
-  if (pcp->count >= g_pcp_high_watermark)
+  if (pcp->count >= g_pcp_high_watermark) {
     drain_pcp(pcp);
+  }
 
-  if (was_on)
+  if (was_on) {
     arch::cpu::intr_on();
+  }
 }
 
 /* ------------------------------------------------------------------ */
@@ -332,17 +366,20 @@ void pmm_free_page(Page *page) {
 /* ------------------------------------------------------------------ */
 
 Page *pmm_alloc_pages(uint8 order) {
-  if (order >= kMaxOrder)
+  if (order >= kMaxOrder) {
     return nullptr;
+  }
   LockGuard guard(g_pmm_lock);
   return buddy_alloc(order);
 }
 
 void pmm_free_pages(Page *page, uint8 order) {
-  if (!page)
+  if (!page) {
     kernel_panic("pmm_free_pages: null page");
-  if (order >= kMaxOrder)
+  }
+  if (order >= kMaxOrder) {
     kernel_panic("pmm_free_pages: order out of range");
+  }
   LockGuard guard(g_pmm_lock);
   buddy_free(page, order);
 }
@@ -358,10 +395,11 @@ void page_decref(Page *page) {
   if (old == 1) {
     // Refcount dropped to 0 — return to pool based on originally allocated
     // order
-    if (page->order == 0)
+    if (page->order == 0) {
       pmm_free_page(page);
-    else
+    } else {
       pmm_free_pages(page, page->order);
+    }
   }
 }
 
@@ -373,11 +411,14 @@ void page_decref(Page *page) {
  * Helper: zero out a range of bytes.  We cannot use memset here
  * because lib/string may not be linked yet.
  */
-static void zero_range(void *start, uint64 bytes) {
+namespace {
+void zero_range(void *start, uint64 bytes) {
   uint8 *p = static_cast<uint8 *>(start);
-  for (uint64 i = 0; i < bytes; i++)
+  for (uint64 i = 0; i < bytes; i++) {
     p[i] = 0;
+  }
 }
+} // namespace
 
 void pmm_init() {
   /* 1. Get memory layout from memblock (already scanned FDT) */
@@ -390,9 +431,9 @@ void pmm_init() {
   }
 
   /* 2. PCP parameters */
-  uint32 calc_batch = static_cast<uint32>(g_num_pages / (kMaxCpus * 1024));
-  g_pcp_batch_size = (calc_batch > 16) ? calc_batch : 16;
-  g_pcp_high_watermark = g_pcp_batch_size * 2;
+  uint32 calc_batch = static_cast<uint32>(g_num_pages / (kMaxCpus * kPcpBatchDivisor));
+  g_pcp_batch_size = (calc_batch > kPcpMinBatchSize) ? calc_batch : kPcpMinBatchSize;
+  g_pcp_high_watermark = g_pcp_batch_size * 4;
 
   /* 3. page_map was allocated by vmm_init at vmemmap VA.
    *    No relocation needed — page_map starts life at the right VA. */
@@ -407,8 +448,13 @@ void pmm_init() {
   for (uint64 i = 0; i < g_num_pages; i++) {
     Page *page = &g_page_map[i];
     page->refcount.store(0, __ATOMIC_RELAXED);
-    page->state = PageState::Allocated;
+    page->mapcount.store(0, __ATOMIC_RELAXED);
+    page->flags.store(0, __ATOMIC_RELAXED);
+    page->type = PageType::Allocated;
     page->order = 0;
+    page->mapping = nullptr;
+    page->index = 0;
+    page->lru.init();
     page->free.free_link.init();
   }
 
@@ -438,9 +484,7 @@ void pmm_init() {
         uint64 start_pfn = (base - memblock_get_mem_base()) / kPageSize;
         uint64 end_pfn = start_pfn + size / kPageSize;
 
-        /* Access page_map via the extern g_page_map (already at vmemmap VA) */
-        extern Page *g_page_map;
-        extern list_node g_free_areas[];
+        /* Access page_map via the globals directly (they are in anon namespace now) */
 
         uint64 pfn = start_pfn;
         while (pfn < end_pfn) {
@@ -454,7 +498,7 @@ void pmm_init() {
           }
 
           Page *page = &g_page_map[pfn];
-          page->state = PageState::Free;
+          page->type = PageType::Free;
           page->order = order;
           page->refcount.store(0, __ATOMIC_RELAXED);
           page->free.free_link.init();
@@ -474,7 +518,7 @@ void pmm_init() {
       &ctx);
 
   kprintf("[pmm] buddy: %llu free pages (%llu MB)\n", ctx.total_pages,
-          (ctx.total_pages * kPageSize) / (1024 * 1024));
+          (ctx.total_pages * kPageSize) / kMB);
 
   /* 8. Retire memblock — all future allocations go through PMM */
   memblock_retire();
diff --git a/na/mm/slab.cc b/na/mm/slab.cc
index e9b06cc..43d8c4a 100644
--- a/na/mm/slab.cc
+++ b/na/mm/slab.cc
@@ -79,7 +79,7 @@ Page* KmemCache::new_slab()
     if (!page)
         return nullptr;
 
-    page->state = PageState::Slab;
+    page->type = PageType::Slab;
     page->slab.cache = this;
     page->slab.obj_count = 0;
 
@@ -197,7 +197,7 @@ void KmemCache::free(void* obj)
     uint64 page_pa = obj_pa & ~(kPageSize - 1);
     Page* page = pa_to_page(phys_addr(page_pa));
 
-    if (page->state != PageState::Slab || page->slab.cache != this)
+    if (page->type != PageType::Slab || page->slab.cache != this)
         kernel_panic("slab free: object does not belong to this cache");
 
     bool was_on = arch::cpu::intr_enabled();
@@ -274,12 +274,12 @@ void kfree(void* ptr)
     uint64 page_pa = pa & ~(kPageSize - 1);
     Page* page = pa_to_page(phys_addr(page_pa));
 
-    if (page->state == PageState::Slab) {
+    if (page->type == PageType::Slab) {
         KmemCache* cache = page->slab.cache;
         if (!cache)
             kernel_panic("kfree: slab page has null cache");
         cache->free(ptr);
-    } else if (page->state == PageState::Allocated) {
+    } else if (page->type == PageType::Allocated) {
         /* Large allocation — return directly to PMM */
         pmm_free_pages(page, page->order);
     } else {
diff --git a/na/mm/tlb.cc b/na/mm/tlb.cc
new file mode 100644
index 0000000..b7a8ba7
--- /dev/null
+++ b/na/mm/tlb.cc
@@ -0,0 +1,90 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/* Copyright (C) 2026 Richard Qin */
+
+/*
+ * tlb.cc — TLB shootdown implementation
+ *
+ * Uses RISC-V SBI Remote Fence extension to perform cross-hart
+ * TLB invalidation. SBI firmware handles the IPI delivery and
+ * synchronization — the call blocks until all target harts have
+ * completed their local sfence.vma.
+ *
+ * When a VmSpace is provided, we use its cpu_mask to target only
+ * the CPUs that have this address space loaded in satp. This avoids
+ * unnecessary IPI storms on unrelated cores.
+ *
+ * Reference: docs/specs/linux_abi_mm_plan.md §SMP §2.1
+ */
+
+#include <lume/tlb.h>
+#include <lume/vmm.h>
+#include <lume/config.h>
+#include <arch/mmu.h>
+#include <arch/tlb.h>
+#include <arch/cpu.h>
+
+/*
+ * If a shootdown range exceeds this many pages, do a full flush
+ * instead of per-page invalidation.
+ */
+static constexpr uint64 kShootdownPageThreshold = 32;
+
+/*
+ * Build a cpu mask for all online CPUs (fallback when mm is null).
+ */
+namespace {
+uint64 all_cpus_mask()
+{
+    uint64 mask = 0;
+    for (uint32 i = 0; i < kMaxCpus; i++) {
+        mask |= (1ULL << i);
+    }
+    return mask;
+}
+
+/*
+ * Get the target cpu mask: use mm->cpu_mask when available,
+ * fall back to all_cpus_mask for kernel address space (mm == nullptr).
+ */
+uint64 target_cpu_mask(VmSpace* mm)
+{
+    if (mm) {
+        uint64 mask = mm->cpu_mask.load(__ATOMIC_RELAXED);
+        /* Always include our own CPU — we may still have stale TLB */
+        mask |= (1ULL << arch::cpu::id());
+        return mask;
+    }
+    return all_cpus_mask();
+}
+} // namespace
+
+void tlb_shootdown_range(VmSpace* mm, uint64 start, uint64 end)
+{
+    if (start >= end) {
+        return;
+    }
+
+    uint64 num_pages = (end - start) / kPageSize;
+    uint64 cpu_mask = target_cpu_mask(mm);
+
+    if (num_pages > kShootdownPageThreshold) {
+        /* Full flush on target CPUs via arch backend */
+        arch::tlb::flush_remote(cpu_mask, 0, ~0ULL);
+        return;
+    }
+
+    /* Range flush on target CPUs via arch backend */
+    arch::tlb::flush_remote(cpu_mask, start, end - start);
+}
+
+void tlb_shootdown_page(VmSpace* mm, uint64 va)
+{
+    uint64 cpu_mask = target_cpu_mask(mm);
+    arch::tlb::flush_remote(cpu_mask, va, kPageSize);
+}
+
+void tlb_shootdown_all()
+{
+    uint64 cpu_mask = all_cpus_mask();
+    arch::tlb::flush_remote(cpu_mask, 0, ~0ULL);
+}
diff --git a/na/mm/vma.cc b/na/mm/vma.cc
index 10db65d..da89e08 100644
--- a/na/mm/vma.cc
+++ b/na/mm/vma.cc
@@ -22,7 +22,8 @@
 #include <lume/new.h>
 #include <lume/klog.h>
 #include <arch/pmap.h>
-#include <arch/mmu.h>
+#include <lume/tlb.h>
+#include <lume/mm_layout.h>
 
 /* ------------------------------------------------------------------ */
 /*  Rbtree callbacks for VMA interval tree                            */
@@ -65,8 +66,7 @@ static bool range_overlaps(uint64 a_start, uint64 a_end,
     return a_start < b_end && b_start < a_end;
 }
 
-static constexpr uint64 kUserVaBase = kPageSize;  // Avoid mapping the null page
-static constexpr uint64 kUserVaLimit = 0xFFFFFFC000000000ULL;
+
 
 static bool vma_overlaps_locked(VmSpace* space, uint64 start, uint64 end)
 {
@@ -188,11 +188,22 @@ VmSpace* VmSpace::create() {
 
     space->vma_tree = RB_ROOT;
     space->lock.init("VmSpace");
+    space->refcount.store(1, __ATOMIC_RELAXED);
     space->brk_start = 0;
     space->brk_end = 0;
     return space;
 }
 
+void VmSpace::incref() {
+    refcount.fetch_add(1, __ATOMIC_RELAXED);
+}
+
+void VmSpace::put() {
+    if (refcount.fetch_sub(1, __ATOMIC_ACQ_REL) == 1) {
+        destroy();
+    }
+}
+
 void VmSpace::destroy() {
     /* Drain VMA tree under lock, then release BEFORE delete this */
     lock.acquire();
@@ -202,11 +213,13 @@ void VmSpace::destroy() {
         VmAreaStruct* vma = reinterpret_cast<VmAreaStruct*>(
             reinterpret_cast<char*>(node) - __builtin_offsetof(VmAreaStruct, rb));
         
-        /* Decref all mapped physical pages */
+        /* Decref and dec mapcount for all mapped physical pages */
         for (uint64 addr = vma->start; addr < vma->end; addr += kPageSize) {
             uint64 pa;
             if (pmap::lookup(root, addr, &pa)) {
-                page_decref(pa_to_page(phys_addr(pa)));
+                Page* pg = pa_to_page(phys_addr(pa));
+                page_dec_mapcount(pg);
+                page_decref(pg);
             }
         }
 
@@ -285,8 +298,10 @@ int vmm_unmap_user(VmSpace* space, uint64 va, uint64 len) {
             for (uint64 addr = unmap_start; addr < unmap_end; addr += kPageSize) {
                 uint64 pa;
                 if (pmap::lookup(space->root_pa, addr, &pa)) {
-                    page_decref(pa_to_page(phys_addr(pa)));
                     pmap::unmap(space->root_pa, addr);
+                    Page* pg = pa_to_page(phys_addr(pa));
+                    page_dec_mapcount(pg);
+                    page_decref(pg);
                 }
             }
         }
@@ -362,7 +377,9 @@ int vmm_protect_user(VmSpace* space, uint64 va, uint64 len, uint64 perm)
             }
 
             if (perm == 0) {
-                page_decref(pa_to_page(phys_addr(pa)));
+                Page* pg = pa_to_page(phys_addr(pa));
+                page_dec_mapcount(pg);
+                page_decref(pg);
                 pmap::unmap(space->root_pa, addr);
                 tlb_flush_needed = true;
                 continue;
@@ -385,7 +402,7 @@ int vmm_protect_user(VmSpace* space, uint64 va, uint64 len, uint64 perm)
     }
 
     if (tlb_flush_needed) {
-        arch::mmu::flush_tlb_all();
+        tlb_shootdown_range(space, va, end);
     }
     return 0;
 }
@@ -399,7 +416,11 @@ uint64 mm_find_free_va(VmSpace* space, uint64 length, uint64 align)
 
     LockGuard guard(space->lock);
 
-    uint64 cursor = align_up_any(kUserVaBase, align);
+    /* Linux: arch_get_unmapped_area() starts from mm->mmap_base.
+     * RISC-V: TASK_UNMAPPED_BASE = PAGE_ALIGN(TASK_SIZE / 3).
+     * Fall back to kUserVaBase if mmap_base not set (kernel VmSpaces). */
+    uint64 base = space->mmap_base ? space->mmap_base : kUserVaBase;
+    uint64 cursor = align_up_any(base, align);
     struct rb_node* node = rb_first(&space->vma_tree);
     while (node) {
         VmAreaStruct* vma = node_to_vma(node);
@@ -494,22 +515,44 @@ int vmm_handle_page_fault(VmSpace* space, uint64 fault_addr, VmFaultCause cause)
                 return -ENOMEM;
             }
 
-            /* Decrement refcount of old page */
-            page_decref(pa_to_page(phys_addr(paddr)));
+            /* New page gets mapcount = 1 (we just mapped it) */
+            page_inc_mapcount(page);
+
+            /* Decrement mapcount and refcount of old page */
+            Page* old_page = pa_to_page(phys_addr(paddr));
+            page_dec_mapcount(old_page);
+            page_decref(old_page);
             
             /* Flush TLB to remove old read-only cached translation */
-            asm volatile("sfence.vma %0" : : "r"(aligned_addr) : "memory");
+            tlb_shootdown_page(space, aligned_addr);
             return 0;
         }
+        /* DEBUG: page already mapped but still faulting — dump PTE */
+        {
+            uint64 cur_satp;
+            asm volatile("csrr %0, satp" : "=r"(cur_satp));
+            printk(KERN_ERR "fault: va=%llx pa=%llx cause=%d pte=%llx satp=%llx root=%llx\n",
+                    aligned_addr, paddr, static_cast<int>(cause),
+                    pmap::lookup_pte(space->root_pa, aligned_addr),
+                    cur_satp, space->root_pa);
+        }
         return 0;  // Already mapped, maybe read fault on existing page
     }
 
-    /* Allocate physical page */
+    /* Allocate physical page and zero it.
+     * Linux: do_anonymous_page() uses alloc_zeroed_user_highpage_movable().
+     * Anonymous pages MUST be zeroed — userspace expects MAP_ANONYMOUS
+     * pages to contain all zeros (POSIX requirement). */
     Page* page = pmm_alloc_page();
     if (!page) return -ENOMEM;
 
     uint64 pa = page_to_pa(page).raw;
 
+    /* Zero the page before mapping it to userspace */
+    /* Zero the page before mapping it to userspace */
+    uint8* kva = reinterpret_cast<uint8*>(pa_to_va(PhysAddr{pa}).raw);
+    for (uint64 i = 0; i < kPageSize; i++) kva[i] = 0;
+
     /* Translate MI permission flags to hardware PTE bits (MD layer) */
     uint64 pte_perm = pmap::vm_perm_to_pte(vma->perm);
 
@@ -518,7 +561,9 @@ int vmm_handle_page_fault(VmSpace* space, uint64 fault_addr, VmFaultCause cause)
         return -ENOMEM;
     }
 
-    /* Initial mapping, refcount is already 1 from pmm_alloc_page */
+    /* Initial mapping: refcount is already 1 from pmm_alloc_page.
+     * Set mapcount = 1 to track this PTE mapping. */
+    page_inc_mapcount(page);
 
     return 0;
 }
@@ -561,7 +606,7 @@ VmSpace* vmm_clone(VmSpace* parent) {
                 if (child_vma->perm & VM_WRITE) {
                     pmap::make_cow(parent->root_pa, va);
                     /* Flush TLB for parent since we reduced permissions */
-                    asm volatile("sfence.vma %0" : : "r"(va) : "memory");
+                    tlb_shootdown_page(parent, va);
                 }
                 
                 /* Map in child as COW or read-only */
@@ -580,8 +625,10 @@ VmSpace* vmm_clone(VmSpace* parent) {
                     return nullptr;
                 }
                 
-                /* Increment physical page refcount */
-                page_incref(pa_to_page(phys_addr(pa)));
+                /* Increment physical page refcount and mapcount */
+                Page* shared_page = pa_to_page(phys_addr(pa));
+                page_incref(shared_page);
+                page_inc_mapcount(shared_page);
             }
         }
         
diff --git a/na/mm/vmm.cc b/na/mm/vmm.cc
index 0aa2fb0..51617e3 100644
--- a/na/mm/vmm.cc
+++ b/na/mm/vmm.cc
@@ -18,6 +18,7 @@
 #include <arch/config.h>
 #include <arch/mmu.h>
 #include <arch/pmap.h>
+#include <lume/tlb.h>
 #include <lume/addr.h>
 #include <lume/console.h>
 #include <lume/fdt.h>
@@ -28,6 +29,7 @@
 #include <lume/pmm.h>
 #include <lume/types.h>
 #include <lume/vmm.h>
+#include <lume/config.h>
 
 /* Linker-exported section boundaries (virtual addresses) */
 extern "C" char _stext[], _etext[];
@@ -36,25 +38,23 @@ extern "C" char _sdata[], _edata[];
 extern "C" char _sbss[], _ebss[];
 extern "C" char _kernel_end[];
 
-/* Global kernel page table root PA */
-static uint64 g_kernel_pgtbl = 0;
+/* Global kernel page table root PA — external linkage for task_exit */
+uint64 g_kernel_pgtbl = 0;
 
-/* VMM exports for PMM: page_map location after vmemmap mapping. */
-static uint64 g_vmemmap_page_map_pa = 0;
-static uint64 g_vmemmap_page_map_size = 0;
-static uint64 g_vmemmap_num_pages = 0;
+namespace {
 
-uint64 vmm_get_page_map_pa() { return g_vmemmap_page_map_pa; }
-uint64 vmm_get_page_map_size() { return g_vmemmap_page_map_size; }
-uint64 vmm_get_num_pages() { return g_vmemmap_num_pages; }
+/* VMM exports for PMM: page_map location after vmemmap mapping. */
+uint64 g_vmemmap_page_map_pa = 0;
+uint64 g_vmemmap_page_map_size = 0;
+uint64 g_vmemmap_num_pages = 0;
 
 /* Convert a linker symbol to PA (kernel is executing in higher-half VA). */
-static inline uint64 sym_to_pa(const void *sym) {
+inline uint64 sym_to_pa(const void *sym) {
   return boot_va_to_pa(virt_addr(reinterpret_cast<uint64>(sym))).raw;
 }
 
 /* Map a PA range into the kernel page table via the direct map. */
-static void map_range(uint64 root, uint64 pa_start, uint64 pa_end,
+void map_range(uint64 root, uint64 pa_start, uint64 pa_end,
                       uint64 perm) {
   for (uint64 pa = pa_start; pa < pa_end; pa += kPageSize) {
     uint64 va = pa_to_va(phys_addr(pa)).raw;
@@ -66,7 +66,7 @@ static void map_range(uint64 root, uint64 pa_start, uint64 pa_end,
 }
 
 /* Map a PA range into the kernel page table at the fixed linked VA. */
-static void map_range_linked(uint64 root, uint64 pa_start, uint64 pa_end,
+void map_range_linked(uint64 root, uint64 pa_start, uint64 pa_end,
                              uint64 perm) {
   for (uint64 pa = pa_start; pa < pa_end; pa += kPageSize) {
     uint64 va = boot_pa_to_va(phys_addr(pa)).raw;
@@ -78,7 +78,7 @@ static void map_range_linked(uint64 root, uint64 pa_start, uint64 pa_end,
 }
 
 /* Map a PA range to an explicit VA base, using 2MB pages where possible. */
-static void map_range_at_huge(uint64 root, uint64 va_base, uint64 pa_start,
+void map_range_at_huge(uint64 root, uint64 va_base, uint64 pa_start,
                               uint64 pa_end, uint64 perm) {
   constexpr uint64 k2MB = 2ULL * 1024 * 1024;
   uint64 pa = pa_start;
@@ -109,6 +109,12 @@ static void map_range_at_huge(uint64 root, uint64 va_base, uint64 pa_start,
   }
 }
 
+} // namespace
+
+uint64 vmm_get_page_map_pa() { return g_vmemmap_page_map_pa; }
+uint64 vmm_get_page_map_size() { return g_vmemmap_page_map_size; }
+uint64 vmm_get_num_pages() { return g_vmemmap_num_pages; }
+
 void vmm_init() {
   /* 1. Allocate new root page table (from memblock) */
   uint64 root = pmap::create();
@@ -137,17 +143,36 @@ void vmm_init() {
    */
   map_range_linked(root, text_start, text_end, pmap::PTE_R | pmap::PTE_X);
 
-  /* 4. Map .rodata (R) at fixed linked VA */
-  map_range_linked(root, ro_start, ro_end, pmap::PTE_R);
+  /* 4. Map .rodata (R) at fixed linked VA.
+   * We map [_srodata, _sdata) so that the .lume.reloc section (which
+   * sits in its own padded output section between _erodata and _sdata)
+   * is also covered.  Relocator only reads it during early boot, but
+   * mapping it keeps post-vmm accesses to __lume_reloc_* symbols sane. */
+  map_range_linked(root, ro_start, page_align_down(sym_to_pa(_sdata)),
+                   pmap::PTE_R);
 
   /* 5. Map .data + .bss (RW) at fixed linked VA */
   map_range_linked(root, data_start, bss_end, pmap::PTE_R | pmap::PTE_W);
 
-  /* 6. Map remaining physical memory after kernel (RW) */
-  uint64 after_kernel = page_align_up(sym_to_pa(_kernel_end));
-  if (after_kernel < mem_end) {
-    map_range(root, after_kernel, mem_end, pmap::PTE_R | pmap::PTE_W);
+  /* 6. Map usable physical RAM into the KASLR-slid direct map.
+   *
+   * The direct map provides pa_to_va() / va_to_pa() for the physical
+   * address space.  We map from kPhysStart (the kernel's linked load
+   * address), skipping [mem_base, kPhysStart) because that region is
+   * PMP-protected firmware (OpenSBI) — S-mode cannot access it.
+   *
+   * Linux does the same on RISC-V: the linear map covers only the
+   * usable RAM range, not firmware-reserved regions.
+   *
+   * Uses 2MB huge pages for efficiency (64 entries for 128MB vs 32K).
+   */
+  uint64 dm_start = kPhysStart;
+  if (dm_start < mem_base) {
+    dm_start = mem_base;
   }
+  uint64 dm_va_base = pa_to_va(phys_addr(dm_start)).raw;
+  map_range_at_huge(root, dm_va_base, dm_start, mem_end,
+                    pmap::PTE_R | pmap::PTE_W);
 
   /* 7. Allocate page_map from memblock (top of RAM) and map at VMEMMAP.
    *
@@ -206,5 +231,5 @@ void vmm_map_kernel_mmio(uint64 pa, uint64 size) {
 
   map_range(g_kernel_pgtbl, aligned_pa, aligned_pa + aligned_size,
             pmap::PTE_R | pmap::PTE_W | pmap::PTE_G);
-  arch::mmu::flush_tlb_all();
+  tlb_shootdown_all();
 }