diff options
Diffstat (limited to 'mm')
| -rw-r--r-- | mm/Build.mk | 6 | ||||
| -rw-r--r-- | mm/memblock.cc | 257 | ||||
| -rw-r--r-- | mm/page_cache.cc | 252 | ||||
| -rw-r--r-- | mm/pmm.cc | 528 | ||||
| -rw-r--r-- | mm/slab.cc | 321 | ||||
| -rw-r--r-- | mm/slab.h | 83 | ||||
| -rw-r--r-- | mm/tlb.cc | 90 | ||||
| -rw-r--r-- | mm/vma.cc | 640 | ||||
| -rw-r--r-- | mm/vmm.cc | 235 |
9 files changed, 0 insertions, 2412 deletions
diff --git a/mm/Build.mk b/mm/Build.mk deleted file mode 100644 index b6d816d..0000000 --- a/mm/Build.mk +++ /dev/null @@ -1,6 +0,0 @@ -targets += mm/core -target.mm/core.kind := source-set -target.mm/core.sources := \ - mm/memblock.cc mm/pmm.cc mm/slab.cc mm/vmm.cc mm/vma.cc \ - mm/page_cache.cc mm/tlb.cc - diff --git a/mm/memblock.cc b/mm/memblock.cc deleted file mode 100644 index aee0245..0000000 --- a/mm/memblock.cc +++ /dev/null @@ -1,257 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-or-later */ -/* Copyright (C) 2026 Richard Qin */ - -/* - * memblock.cc — Boot-time physical memory allocator - * - * Simple region tracker: one memory region (from FDT) plus an array - * of reserved regions. Allocation searches for gaps between reserved - * regions. All operations are O(n_reserved), which is fine for the - * ~20 allocations that happen during early boot. - * - * Reference: inspired by Linux kernel's mm/memblock.c - */ - -#include <lume/memblock.h> -#include <lume/fdt.h> -#include <lume/klog.h> -#include <lume/kprintf.h> -#include <lume/config.h> -#include <lume/addr.h> - -extern "C" char _stext[], _kernel_end[]; - -/* ------------------------------------------------------------------ */ -/* Module state */ -/* ------------------------------------------------------------------ */ - -namespace { - -constexpr uint64 kMB = 1024ULL * 1024ULL; - -struct { - uint64 mem_base; - uint64 mem_size; - - struct { - uint64 base; - uint64 size; - } reserved[kMemblockMaxRegions]; - int reserved_count; - bool active; -} g_mb; - -/* ------------------------------------------------------------------ */ -/* Internal helpers */ -/* ------------------------------------------------------------------ */ - -/* Insert a reserved region, keeping the array sorted by base. */ -void insert_reserved(uint64 base, uint64 size) -{ - if (g_mb.reserved_count >= kMemblockMaxRegions) { - kernel_panic("memblock: too many reserved regions"); - } - - int i = g_mb.reserved_count; - while (i > 0 && g_mb.reserved[i - 1].base > base) { - g_mb.reserved[i] = g_mb.reserved[i - 1]; - i--; - } - g_mb.reserved[i] = {base, size}; - g_mb.reserved_count++; -} - -/* Check if [base, base+size) overlaps any reserved region. */ -bool overlaps_reserved(uint64 base, uint64 size) -{ - uint64 end = base + size; - for (int i = 0; i < g_mb.reserved_count; i++) { - uint64 rb = g_mb.reserved[i].base; - uint64 re = rb + g_mb.reserved[i].size; - if (base < re && end > rb) { - return true; - } - } - return false; -} - -/* Find the end of the first reserved region that covers `addr`. */ -uint64 reserved_end_covering(uint64 addr) -{ - for (int i = 0; i < g_mb.reserved_count; i++) { - uint64 rb = g_mb.reserved[i].base; - uint64 re = rb + g_mb.reserved[i].size; - if (addr >= rb && addr < re) { - return re; - } - } - return addr; -} - -} // namespace - -/* ------------------------------------------------------------------ */ -/* Public API */ -/* ------------------------------------------------------------------ */ - -void memblock_init(uint64 fdt_paddr) -{ - - /* 1. Get memory layout from FDT */ - uint64 mem_base = 0; - uint64 mem_size = 0; - fdt_early_get_mem_info(&mem_base, &mem_size); - if (mem_size == 0) { - kernel_panic("memblock_init: FDT reports zero memory"); - } - - g_mb.mem_base = mem_base; - g_mb.mem_size = mem_size; - g_mb.reserved_count = 0; - g_mb.active = true; - - /* 2. Reserve firmware region [mem_base, kPhysStart). - * - * Everything below the kernel's linked load address is firmware - * territory (OpenSBI), which is PMP-protected. */ - { - uint64 fw_end = kPhysStart; - if (fw_end > mem_base) { - memblock_reserve(mem_base, fw_end - mem_base); - } - } - - /* 2b. Reserve stale pre-relocation kernel image. - * - * After physical KASLR, the kernel is relocated from its original - * load address (kPhysStart) to a new PA. The old copy at - * [kPhysStart, kPhysStart + kernel_size) must not be allocated — - * it contains stale code/data that would corrupt page tables or - * other structures if reused. */ - { - uint64 ks_pa = boot_va_to_pa(virt_addr( - reinterpret_cast<uint64>(_stext))).raw; - if (ks_pa != kPhysStart) { - /* Kernel was relocated — reserve the original location */ - uint64 kernel_size = boot_va_to_pa(virt_addr( - reinterpret_cast<uint64>(_kernel_end))).raw - ks_pa; - memblock_reserve(kPhysStart, kernel_size); - } - } - - /* 3. Reserve kernel image [_stext, _kernel_end) */ - uint64 ks_pa = boot_va_to_pa(virt_addr( - reinterpret_cast<uint64>(_stext))).raw; - uint64 ke_pa = boot_va_to_pa(virt_addr( - reinterpret_cast<uint64>(_kernel_end))).raw; - memblock_reserve(ks_pa, ke_pa - ks_pa); - - /* 4. Reserve FDT blob — it lives in RAM and must not be overwritten - * by vmemmap/page_map or buddy before fdt_unflatten() copies it. */ - { - struct fdt_hdr { uint32 magic; uint32 totalsize; }; - const auto* fh = reinterpret_cast<const fdt_hdr*>(fdt_paddr); - uint32 fdt_size = ((fh->totalsize & 0xFF) << 24) | - (((fh->totalsize >> 8) & 0xFF) << 16) | - (((fh->totalsize >> 16) & 0xFF) << 8) | - ((fh->totalsize >> 24) & 0xFF); - memblock_reserve(fdt_paddr, fdt_size); - kprintf("[memblock] FDT reserved [0x%llx, 0x%llx)\n", - fdt_paddr, fdt_paddr + fdt_size); - } - - kprintf("[memblock] RAM [0x%llx, 0x%llx) = %llu MB\n", - mem_base, mem_base + mem_size, mem_size / kMB); - kprintf("[memblock] kernel reserved [0x%llx, 0x%llx)\n", ks_pa, ke_pa); -} - -void memblock_reserve(uint64 base, uint64 size) -{ - uint64 aligned_base = page_align_down(base); - uint64 aligned_end = page_align_up(base + size); - insert_reserved(aligned_base, aligned_end - aligned_base); -} - -uint64 memblock_alloc_top(uint64 size, uint64 align) -{ - uint64 mem_end = g_mb.mem_base + g_mb.mem_size; - - /* Search from top of RAM downward */ - uint64 end = page_align_down(mem_end); - uint64 start = (end - size) & ~(align - 1); - - while (start >= g_mb.mem_base && start + size <= mem_end) { - if (!overlaps_reserved(start, size)) { - memblock_reserve(start, size); - return start; - } - if (start < align) { - break; - } - start -= align; - } - - kernel_panic("memblock_alloc_top: out of memory"); - return 0; -} - -uint64 memblock_alloc(uint64 size, uint64 align) -{ - uint64 mem_end = g_mb.mem_base + g_mb.mem_size; - uint64 start = page_align_up(g_mb.mem_base); - - while (start + size <= mem_end) { - start = (start + align - 1) & ~(align - 1); - if (start + size > mem_end) { - break; - } - - if (!overlaps_reserved(start, size)) { - memblock_reserve(start, size); - return start; - } - - /* Skip past the overlapping reserved region */ - uint64 skip = reserved_end_covering(start); - if (skip <= start) { - start += align; - } else { - start = skip; - } - } - - kernel_panic("memblock_alloc: out of memory"); - return 0; -} - -uint64 memblock_get_mem_base() { return g_mb.mem_base; } -uint64 memblock_get_mem_size() { return g_mb.mem_size; } - -bool memblock_is_active() { return g_mb.active; } -void memblock_retire() { g_mb.active = false; } - -void memblock_for_each_free(void (*cb)(uint64 base, uint64 size, void* ctx), - void* ctx) -{ - uint64 pos = g_mb.mem_base; - uint64 mem_end = g_mb.mem_base + g_mb.mem_size; - - for (int i = 0; i < g_mb.reserved_count && pos < mem_end; i++) { - uint64 rb = g_mb.reserved[i].base; - if (rb < pos) { - uint64 re = g_mb.reserved[i].base + g_mb.reserved[i].size; - if (re > pos) { - pos = re; - } - continue; - } - if (rb > pos) { - cb(pos, rb - pos, ctx); - } - pos = g_mb.reserved[i].base + g_mb.reserved[i].size; - } - - if (pos < mem_end) { - cb(pos, mem_end - pos, ctx); - } -} diff --git a/mm/page_cache.cc b/mm/page_cache.cc deleted file mode 100644 index 25c6a60..0000000 --- a/mm/page_cache.cc +++ /dev/null @@ -1,252 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-or-later */ -/* Copyright (C) 2026 Richard Qin */ - -/* - * page_cache.cc — Minimal page cache implementation - * - * Open-addressing hash table indexed by (mapping, index) pairs. - * Fixed-size bucket array allocated at init time via slab. - * - * This is intentionally simple: no LRU, no writeback, no eviction. - * The interface in page_cache.h is designed so that upgrading to a - * radix tree + LRU scheme later only changes this file. - * - * Concurrency: protected by a single spinlock. Acceptable for - * initial bring-up; will be replaced by per-bucket or RCU-based - * locking when contention becomes measurable. - */ - -#include <lume/page_cache.h> -#include <lume/page.h> -#include <lume/pmm.h> -#include <lume/errno.h> -#include <lume/new.h> -#include <kernel/sync/spinlock.h> - -/* ------------------------------------------------------------------ */ -/* Hash table internals */ -/* ------------------------------------------------------------------ */ - -struct CacheEntry { - void* mapping; // nullptr = empty slot - uint64 index; - Page* page; -}; - -namespace { - -constexpr uint64 kCacheBuckets = 1024; - -CacheEntry g_buckets[kCacheBuckets]; -Spinlock g_cache_lock; - -uint64 hash_key(void* mapping, uint64 index) -{ - uint64 m = reinterpret_cast<uint64>(mapping); - /* FNV-1a inspired mixing */ - uint64 h = m ^ (index * 0x517cc1b727220a95ULL); - h ^= h >> 17; - h *= 0xbf58476d1ce4e5b9ULL; - h ^= h >> 31; - return h % kCacheBuckets; -} - -/* Linear probe to find an entry or an empty slot. */ -CacheEntry* find_slot(void* mapping, uint64 index) -{ - uint64 start = hash_key(mapping, index); - for (uint64 i = 0; i < kCacheBuckets; i++) { - uint64 idx = (start + i) % kCacheBuckets; - CacheEntry* e = &g_buckets[idx]; - if (e->mapping == nullptr) { - return e; // Empty slot - } - if (e->mapping == mapping && e->index == index) { - return e; // Found - } - } - return nullptr; // Table full -} - -/* Find an existing entry (not an empty slot). */ -CacheEntry* find_entry(void* mapping, uint64 index) -{ - uint64 start = hash_key(mapping, index); - for (uint64 i = 0; i < kCacheBuckets; i++) { - uint64 idx = (start + i) % kCacheBuckets; - CacheEntry* e = &g_buckets[idx]; - if (e->mapping == nullptr) { - return nullptr; // Empty = not found - } - if (e->mapping == mapping && e->index == index) { - return e; - } - } - return nullptr; -} - -/* - * Remove an entry and fix up the linear-probe chain. - * Standard open-addressing deletion with backward shift. - */ -void remove_entry(CacheEntry* entry) -{ - uint64 removed = static_cast<uint64>(entry - g_buckets); - entry->mapping = nullptr; - entry->page = nullptr; - - /* Backward-shift deletion to maintain probe chain integrity */ - uint64 cur = (removed + 1) % kCacheBuckets; - while (g_buckets[cur].mapping != nullptr) { - uint64 ideal = hash_key(g_buckets[cur].mapping, g_buckets[cur].index); - /* Check if cur's ideal position is "before" removed in probe order */ - bool needs_shift = false; - if (removed <= cur) { - needs_shift = (ideal <= removed || ideal > cur); - } else { - needs_shift = (ideal <= removed && ideal > cur); - } - - if (needs_shift) { - g_buckets[removed] = g_buckets[cur]; - g_buckets[cur].mapping = nullptr; - g_buckets[cur].page = nullptr; - removed = cur; - } - cur = (cur + 1) % kCacheBuckets; - } -} - -} // namespace - -/* ------------------------------------------------------------------ */ -/* Public API */ -/* ------------------------------------------------------------------ */ - -void page_cache_init() -{ - g_cache_lock.init("page_cache"); - for (uint64 i = 0; i < kCacheBuckets; i++) { - g_buckets[i].mapping = nullptr; - g_buckets[i].index = 0; - g_buckets[i].page = nullptr; - } -} - -Page* page_cache_get(void* mapping, uint64 index) -{ - LockGuard guard(g_cache_lock); - CacheEntry* e = find_entry(mapping, index); - if (!e) { - return nullptr; - } - page_incref(e->page); - return e->page; -} - -int page_cache_insert(void* mapping, uint64 index, Page* page) -{ - if (!mapping || !page) { - return -EINVAL; - } - - LockGuard guard(g_cache_lock); - - /* Check for existing entry */ - CacheEntry* existing = find_entry(mapping, index); - if (existing) { - return -EEXIST; - } - - /* Find an empty slot */ - CacheEntry* slot = find_slot(mapping, index); - if (!slot || slot->mapping != nullptr) { - return -ENOMEM; // Table full - } - - slot->mapping = mapping; - slot->index = index; - slot->page = page; - page_incref(page); // Cache holds a reference - - return 0; -} - -void page_cache_put(Page* page) -{ - if (!page) { - return; - } - page_decref(page); -} - -Page* page_cache_remove(void* mapping, uint64 index) -{ - LockGuard guard(g_cache_lock); - CacheEntry* e = find_entry(mapping, index); - if (!e) { - return nullptr; - } - - Page* page = e->page; - remove_entry(e); - page_decref(page); // Release cache's reference - return page; -} - -void page_cache_invalidate(void* mapping) -{ - if (!mapping) { - return; - } - - LockGuard guard(g_cache_lock); - - /* Scan all buckets — acceptable for current table size */ - for (uint64 i = 0; i < kCacheBuckets; i++) { - if (g_buckets[i].mapping == mapping) { - Page* page = g_buckets[i].page; - remove_entry(&g_buckets[i]); - page_decref(page); - /* After removal, re-check this index since backward shift - * may have moved an entry into this slot. */ - i--; - } - } -} - -Page* page_cache_get_or_alloc(void* mapping, uint64 index) -{ - /* Fast path: already cached */ - Page* page = page_cache_get(mapping, index); - if (page) { - return page; - } - - /* Slow path: allocate and insert */ - page = pmm_alloc_page(); - if (!page) { - return nullptr; - } - - int ret = page_cache_insert(mapping, index, page); - if (ret == -EEXIST) { - /* Race: another thread inserted between our get and insert. - * Free our allocation and return the existing page. */ - pmm_free_page(page); - return page_cache_get(mapping, index); - } - if (ret < 0) { - pmm_free_page(page); - return nullptr; - } - - /* Caller gets a ref (insert already added cache's ref, - * we need one more for the caller). */ - /* Actually, insert did incref for cache. The page already has - * refcount=1 from alloc + 1 from insert = 2. - * We return it to caller who must put() when done. - * So we incref once more for the caller's reference. */ - page_incref(page); - return page; -} diff --git a/mm/pmm.cc b/mm/pmm.cc deleted file mode 100644 index b7cf611..0000000 --- a/mm/pmm.cc +++ /dev/null @@ -1,528 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-or-later */ -/* Copyright (C) 2026 Richard Qin */ - -/* - * pmm.cc — Physical Memory Manager implementation - * - * Buddy allocator with Per-CPU single-page caches (PCP). - * - * Initialization sequence (called by BSP in single-core period): - * 1. memblock + vmm_init allocate and map the page_map at VMEMMAP. - * 2. Zero and mark all Page descriptors reserved by default. - * 3. Carve free regions from memblock into the buddy system. - * - * The page_map is a flat array indexed by physical page frame number (PFN). - * PFN = (pa - mem_base) / PAGE_SIZE. This keeps address conversion O(1). - * - * Reference: docs/specs/pmm.md - */ - -#include <arch/config.h> -#include <arch/cpu.h> -#include <kernel/sync/spinlock.h> -#include <lume/addr.h> -#include <lume/config.h> -#include <lume/klog.h> -#include <lume/kprintf.h> -#include <lume/memblock.h> -#include <lume/new.h> -#include <lume/pmm.h> -#include <lume/types.h> -#include <lume/vmm.h> - -/* ------------------------------------------------------------------ */ -/* Per-CPU page cache */ -/* ------------------------------------------------------------------ */ - -struct PerCpuCache { - list_node free_list; // Single-page free list - uint32 count; // Number of cached pages -}; - -/* ------------------------------------------------------------------ */ -/* Module-private state */ -/* ------------------------------------------------------------------ */ - -namespace { - -/* PCP Cache tuning parameters */ -constexpr uint32 kPcpBatchDivisor = 1024; -constexpr uint32 kPcpMinBatchSize = 16; -constexpr uint64 kMB = 1024ULL * 1024ULL; - -Spinlock g_pmm_lock; -list_node g_free_areas[kMaxOrder]; // Buddy free lists [0..10] -PerCpuCache g_pcp[kMaxCpus]; - -Page *g_page_map = nullptr; // Flat array of Page descriptors -uint64 g_page_map_pa = 0; // Physical base of page_map storage -uint64 g_page_map_bytes = 0; // Byte size of page_map storage -uint64 g_mem_base = 0; // Physical memory base (from FDT) -uint64 g_mem_size = 0; // Total physical memory size -uint64 g_num_pages = 0; // Total managed pages - -} // namespace - -uint32 g_pcp_high_watermark = 0; -uint32 g_pcp_batch_size = 0; - -/* ------------------------------------------------------------------ */ -/* Address conversion helpers */ -/* ------------------------------------------------------------------ */ - -/* - * PFN (Page Frame Number) — fundamental conversion unit. - * When sparse vmemmap is implemented, only these two functions change. - * All other PA↔Page conversions go through them. - */ -uint64 page_to_pfn(const Page *page) { - uintptr base = reinterpret_cast<uintptr>(g_page_map); - uintptr addr = reinterpret_cast<uintptr>(page); - uintptr end = base + g_num_pages * sizeof(Page); - - if (addr < base || addr >= end) { - kernel_panic("page_to_pfn: page outside page_map bounds"); - } - - uint64 idx = (addr - base) / sizeof(Page); - return (g_mem_base / kPageSize) + idx; -} - -Page *pfn_to_page(uint64 pfn) { - uint64 base_pfn = g_mem_base / kPageSize; - if (pfn < base_pfn || pfn >= base_pfn + g_num_pages) { - kernel_panic("pfn_to_page: PFN outside managed range"); - } - return &g_page_map[pfn - base_pfn]; -} - -PhysAddr page_to_pa(const Page *page) { - return pfn_to_pa(Pfn{page_to_pfn(page)}); -} - -Page *pa_to_page(PhysAddr pa) { - return pfn_to_page(pa_to_pfn(pa).raw); -} - -VirtAddr page_to_va(const Page *page) { - return pa_to_va(page_to_pa(page)); -} - - -/* ------------------------------------------------------------------ */ -/* Buddy system internals (caller must hold g_pmm_lock) */ -/* ------------------------------------------------------------------ */ - -namespace { - -/* - * Find a page's buddy at a given order. - * Buddy PFN = PFN ^ (1 << order). - */ -inline Page *buddy_of(Page *page, uint8 order) { - uint64 pfn = static_cast<uint64>(page - g_page_map); - uint64 buddy_pfn = pfn ^ (1ULL << order); - if (buddy_pfn >= g_num_pages) { - return nullptr; - } - return &g_page_map[buddy_pfn]; -} - -/* - * Insert a free block into the buddy free list at the given order. - */ -void buddy_list_add(Page *page, uint8 order) { - page->type = PageType::Free; - page->order = order; - list_node *head = &g_free_areas[order]; - list_node *node = &page->free.free_link; - node->next = head->next; - node->prev = head; - head->next->prev = node; - head->next = node; -} - -/* - * Remove a block from whatever buddy free list it is on. - */ -void buddy_list_del(Page *page) { - list_node *node = &page->free.free_link; - node->prev->next = node->next; - node->next->prev = node->prev; - node->next = node; - node->prev = node; -} - -/* - * Allocate a block of 2^order contiguous pages from the buddy system. - * Splits higher-order blocks if no exact match is available. - * Returns nullptr if no memory available. - */ -Page *buddy_alloc(uint8 order) { - for (int cur = order; cur < kMaxOrder; cur++) { - list_node *head = &g_free_areas[cur]; - if (head->next == head) { - continue; // Empty at this order - } - - // Pop the first block from this order's free list - list_node *node = head->next; - // Compute Page* from the free_link member offset - Page *page = - reinterpret_cast<Page *>(reinterpret_cast<uintptr>(node) - - __builtin_offsetof(Page, free.free_link)); - buddy_list_del(page); - - // Split down to the requested order - while (cur > order) { - cur--; - // The upper half becomes a free buddy - Page *split = &g_page_map[(page - g_page_map) + (1ULL << cur)]; - buddy_list_add(split, static_cast<uint8>(cur)); - } - - page->type = PageType::Allocated; - page->order = order; - page->refcount.store(1, __ATOMIC_RELAXED); - page->mapcount.store(0, __ATOMIC_RELAXED); - page->flags.store(0, __ATOMIC_RELAXED); - page->mapping = nullptr; - page->index = 0; - return page; - } - return nullptr; -} - -/* - * Return a block of 2^order pages to the buddy system. - * Coalesces with its buddy recursively up to kMaxOrder-1. - */ -void buddy_free(Page *page, uint8 order) { - uint64 pfn = static_cast<uint64>(page - g_page_map); - - while (order < kMaxOrder - 1) { - Page *buddy = buddy_of(page, order); - if (!buddy) { - break; - } - // Buddy must be free AND at the same order to coalesce - if (buddy->type != PageType::Free || buddy->order != order) { - break; - } - - // Remove buddy from its free list and merge - buddy_list_del(buddy); - - // The merged block starts at the lower PFN - uint64 buddy_pfn = static_cast<uint64>(buddy - g_page_map); - if (buddy_pfn < pfn) { - page = buddy; - pfn = buddy_pfn; - } - order++; - } - - buddy_list_add(page, order); -} - -/* ------------------------------------------------------------------ */ -/* Per-CPU cache operations (interrupts must be disabled by caller) */ -/* ------------------------------------------------------------------ */ - -/* - * Refill the PCP from the global buddy system. - * Acquires g_pmm_lock internally. - */ -void refill_pcp(PerCpuCache *pcp) { - LockGuard guard(g_pmm_lock); - for (uint32 i = 0; i < g_pcp_batch_size; i++) { - Page *page = buddy_alloc(0); - if (!page) { - break; - } - // Put onto PCP list (reuse the free_link for PCP chain) - page->type = PageType::PcpCached; - list_node *node = &page->free.free_link; - node->next = pcp->free_list.next; - node->prev = &pcp->free_list; - pcp->free_list.next->prev = node; - pcp->free_list.next = node; - pcp->count++; - } -} - -/* - * Drain half the PCP back to the global buddy system. - * Acquires g_pmm_lock internally. - */ -void drain_pcp(PerCpuCache *pcp) { - LockGuard guard(g_pmm_lock); - int to_drain = pcp->count / 2; - for (int i = 0; i < to_drain; i++) { - list_node *head = &pcp->free_list; - if (head->next == head) { - break; - } - list_node *node = head->next; - // Unlink from PCP - node->prev->next = node->next; - node->next->prev = node->prev; - pcp->count--; - - Page *page = - reinterpret_cast<Page *>(reinterpret_cast<uintptr>(node) - - __builtin_offsetof(Page, free.free_link)); - buddy_free(page, 0); - } -} - -} // namespace - -/* ------------------------------------------------------------------ */ -/* Public API: single-page (fast path via PCP) */ -/* ------------------------------------------------------------------ */ - -Page *pmm_alloc_page() { - /* - * Disable interrupts to prevent reentry on the same CPU - * (e.g. timer IRQ handler calling pmm_alloc_page while we - * are mid-way through manipulating the PCP list). - */ - bool was_on = arch::cpu::intr_enabled(); - arch::cpu::intr_off(); - - uint64 cpu = arch::cpu::id(); - PerCpuCache *pcp = &g_pcp[cpu]; - - if (pcp->count == 0) { - refill_pcp(pcp); - } - - Page *page = nullptr; - if (pcp->count > 0) { - list_node *node = pcp->free_list.next; - // Unlink - node->prev->next = node->next; - node->next->prev = node->prev; - pcp->count--; - - page = - reinterpret_cast<Page *>(reinterpret_cast<uintptr>(node) - - __builtin_offsetof(Page, free.free_link)); - - // Validate the page is within the page_map bounds - if (page < g_page_map || page >= g_page_map + g_num_pages) { - kernel_panic("pmm_alloc_page: PCP returned out-of-bounds page"); - } - - page->type = PageType::Allocated; - page->order = 0; - page->refcount.store(1, __ATOMIC_RELAXED); - page->mapcount.store(0, __ATOMIC_RELAXED); - page->flags.store(0, __ATOMIC_RELAXED); - page->mapping = nullptr; - page->index = 0; - } - - if (was_on) { - arch::cpu::intr_on(); - } - return page; -} - -void pmm_free_page(Page *page) { - if (!page) { - kernel_panic("pmm_free_page: null page"); - } - - bool was_on = arch::cpu::intr_enabled(); - arch::cpu::intr_off(); - - uint64 cpu = arch::cpu::id(); - PerCpuCache *pcp = &g_pcp[cpu]; - - // Push onto PCP - page->type = PageType::PcpCached; - page->order = 0; - list_node *node = &page->free.free_link; - node->next = pcp->free_list.next; - node->prev = &pcp->free_list; - pcp->free_list.next->prev = node; - pcp->free_list.next = node; - pcp->count++; - - // Drain if over high watermark - if (pcp->count >= g_pcp_high_watermark) { - drain_pcp(pcp); - } - - if (was_on) { - arch::cpu::intr_on(); - } -} - -/* ------------------------------------------------------------------ */ -/* Public API: multi-page (slow path, direct buddy) */ -/* ------------------------------------------------------------------ */ - -Page *pmm_alloc_pages(uint8 order) { - if (order >= kMaxOrder) { - return nullptr; - } - LockGuard guard(g_pmm_lock); - return buddy_alloc(order); -} - -void pmm_free_pages(Page *page, uint8 order) { - if (!page) { - kernel_panic("pmm_free_pages: null page"); - } - if (order >= kMaxOrder) { - kernel_panic("pmm_free_pages: order out of range"); - } - LockGuard guard(g_pmm_lock); - buddy_free(page, order); -} - -/* ------------------------------------------------------------------ */ -/* Public API: reference counting */ -/* ------------------------------------------------------------------ */ - -void page_incref(Page *page) { page->refcount.fetch_add(1); } - -void page_decref(Page *page) { - uint32 old = page->refcount.fetch_sub(1); - if (old == 1) { - // Refcount dropped to 0 — return to pool based on originally allocated - // order - if (page->order == 0) { - pmm_free_page(page); - } else { - pmm_free_pages(page, page->order); - } - } -} - -/* ------------------------------------------------------------------ */ -/* Initialization */ -/* ------------------------------------------------------------------ */ - -/* - * Helper: zero out a range of bytes. We cannot use memset here - * because lib/string may not be linked yet. - */ -namespace { -void zero_range(void *start, uint64 bytes) { - uint8 *p = static_cast<uint8 *>(start); - for (uint64 i = 0; i < bytes; i++) { - p[i] = 0; - } -} -} // namespace - -void pmm_init() { - /* 1. Get memory layout from memblock (already scanned FDT) */ - g_mem_base = memblock_get_mem_base(); - g_mem_size = memblock_get_mem_size(); - g_num_pages = vmm_get_num_pages(); - - if (g_mem_size == 0) { - kernel_panic("pmm_init: memblock reports zero memory"); - } - - /* 2. PCP parameters */ - uint32 calc_batch = static_cast<uint32>(g_num_pages / (kMaxCpus * kPcpBatchDivisor)); - g_pcp_batch_size = (calc_batch > kPcpMinBatchSize) ? calc_batch : kPcpMinBatchSize; - g_pcp_high_watermark = g_pcp_batch_size * 4; - - /* 3. page_map was allocated by vmm_init at vmemmap VA. - * No relocation needed — page_map starts life at the right VA. */ - g_page_map_pa = vmm_get_page_map_pa(); - g_page_map_bytes = vmm_get_page_map_size(); - g_page_map = reinterpret_cast<Page *>(arch::g_vmemmap_base); - - /* 4. Zero out the entire page_map */ - zero_range(g_page_map, g_page_map_bytes); - - /* 4b. Default every page descriptor to reserved/non-free. */ - for (uint64 i = 0; i < g_num_pages; i++) { - Page *page = &g_page_map[i]; - page->refcount.store(0, __ATOMIC_RELAXED); - page->mapcount.store(0, __ATOMIC_RELAXED); - page->flags.store(0, __ATOMIC_RELAXED); - page->type = PageType::Allocated; - page->order = 0; - page->mapping = nullptr; - page->index = 0; - page->lru.init(); - page->free.free_link.init(); - } - - /* 5. Initialize buddy free list heads */ - g_pmm_lock.init("pmm"); - for (int i = 0; i < kMaxOrder; i++) { - g_free_areas[i].init(); - } - - /* 6. Initialize PCP list heads */ - for (int i = 0; i < kMaxCpus; i++) { - g_pcp[i].free_list.init(); - g_pcp[i].count = 0; - } - - /* 7. Feed free regions from memblock into the buddy system. - * memblock knows about all reserved regions (firmware, kernel, - * page table pages, page_map). Everything else is free. */ - struct FeedCtx { - uint64 total_pages; - }; - FeedCtx ctx{0}; - - memblock_for_each_free( - [](uint64 base, uint64 size, void *arg) { - auto *c = static_cast<FeedCtx *>(arg); - uint64 start_pfn = (base - memblock_get_mem_base()) / kPageSize; - uint64 end_pfn = start_pfn + size / kPageSize; - - /* Access page_map via the globals directly (they are in anon namespace now) */ - - uint64 pfn = start_pfn; - while (pfn < end_pfn) { - uint8 order = 0; - for (int o = kMaxOrder - 1; o >= 0; o--) { - uint64 block_size = 1ULL << o; - if ((pfn & (block_size - 1)) == 0 && pfn + block_size <= end_pfn) { - order = static_cast<uint8>(o); - break; - } - } - - Page *page = &g_page_map[pfn]; - page->type = PageType::Free; - page->order = order; - page->refcount.store(0, __ATOMIC_RELAXED); - page->free.free_link.init(); - - /* Inline buddy_list_add since we're inside a lambda */ - list_node *head = &g_free_areas[order]; - list_node *node = &page->free.free_link; - node->next = head->next; - node->prev = head; - head->next->prev = node; - head->next = node; - - pfn += (1ULL << order); - c->total_pages += (1ULL << order); - } - }, - &ctx); - - kprintf("[pmm] buddy: %llu free pages (%llu MB)\n", ctx.total_pages, - (ctx.total_pages * kPageSize) / kMB); - - /* 8. Retire memblock — all future allocations go through PMM */ - memblock_retire(); -} - -PhysAddr pmm_get_mem_base() { return phys_addr(g_mem_base); } -uint64 pmm_get_mem_size() { return g_mem_size; } diff --git a/mm/slab.cc b/mm/slab.cc deleted file mode 100644 index 43d8c4a..0000000 --- a/mm/slab.cc +++ /dev/null @@ -1,321 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-or-later */ -/* Copyright (C) 2026 Richard Qin */ - -/* - * slab.cc — SLUB allocator implementation - * - * Key design points: - * - Free object list is embedded in the objects themselves (first 8 bytes). - * - Per-page metadata lives in Page.slab (cache, obj_count, freelist). - * - Per-CPU "active page" avoids lock contention on the fast path. - * - partial_ list protected by per-KmemCache Spinlock. - * - PMM calls happen outside the lock to avoid lock ordering issues. - * - * Address note: the kernel may be running at physical addresses via - * identity mapping (before vmm_init). We handle PA/VA carefully: - * PMM gives us Page*, page_to_pa() gives PA, and we access page - * contents via pa_to_va() (higher-half VA, always mapped). - * - * Reference: docs/specs/slab.md - */ - -#include "slab.h" -#include <lume/pmm.h> -#include <lume/addr.h> -#include <lume/klog.h> -#include <lume/new.h> -#include <arch/cpu.h> - -/* ------------------------------------------------------------------ */ -/* Helpers */ -/* ------------------------------------------------------------------ */ - -/* - * Convert a Page* to the usable virtual address of the page's contents. - * Uses the higher-half mapping which is always active. - */ -static inline void* page_to_data_va(Page* page) -{ - return reinterpret_cast<void*>(page_to_va(page).raw); -} - -/* Round up x to the next multiple of align (align must be power of 2) */ -static inline uint32 align_up(uint32 x, uint32 align) -{ - return (x + align - 1) & ~(align - 1); -} - -/* ------------------------------------------------------------------ */ -/* Global size-class caches */ -/* ------------------------------------------------------------------ */ - -KmemCache g_size_caches[kNumSizeClasses]; - -/* ------------------------------------------------------------------ */ -/* KmemCache::init */ -/* ------------------------------------------------------------------ */ - -void KmemCache::init(const char* name, uint32 obj_size, uint32 obj_align) -{ - name_ = name; - obj_align_ = (obj_align < 8) ? 8 : obj_align; // minimum 8 for freelist ptr - obj_size_ = align_up((obj_size < 8) ? 8 : obj_size, obj_align_); - objs_per_slab_ = kPageSize / obj_size_; - - lock_.init(name); - partial_.init(); - - for (int i = 0; i < kMaxCpus; i++) - cpu_slab_[i].active = nullptr; -} - -/* ------------------------------------------------------------------ */ -/* KmemCache::new_slab — allocate and format a fresh slab page */ -/* ------------------------------------------------------------------ */ - -Page* KmemCache::new_slab() -{ - Page* page = pmm_alloc_page(); - if (!page) - return nullptr; - - page->type = PageType::Slab; - page->slab.cache = this; - page->slab.obj_count = 0; - - /* Build the embedded free list across all object slots */ - uint8* base = static_cast<uint8*>(page_to_data_va(page)); - void* head = nullptr; - - /* Chain objects from last to first so freelist order = low addr first */ - for (int i = static_cast<int>(objs_per_slab_) - 1; i >= 0; i--) { - void* obj = base + i * obj_size_; - *reinterpret_cast<void**>(obj) = head; - head = obj; - } - page->slab.freelist = head; - - return page; -} - -/* ------------------------------------------------------------------ */ -/* Partial list management (lock must be held) */ -/* ------------------------------------------------------------------ */ - -Page* KmemCache::get_partial() -{ - /* - * Phase 1 simplification: no partial list tracking. - * The slow path always allocates a fresh page from PMM. - * Full pages stay alive (Page.slab.cache is valid for kfree). - * Empty pages are returned to PMM immediately. - * - * TODO: add list_node to Page.slab and implement partial tracking - * to avoid wasting partially-filled pages. - */ - (void)partial_; - return nullptr; -} - -void KmemCache::put_partial(Page* page) -{ - (void)page; -} - -void KmemCache::remove_partial(Page* page) -{ - (void)page; -} - -/* ------------------------------------------------------------------ */ -/* KmemCache::alloc */ -/* ------------------------------------------------------------------ */ - -void* KmemCache::alloc() -{ - bool was_on = arch::cpu::intr_enabled(); - arch::cpu::intr_off(); - - uint64 cpu = arch::cpu::id(); - Page* active = cpu_slab_[cpu].active; - - /* Fast path: pop from active page's freelist */ - if (active && active->slab.freelist) { - void* obj = active->slab.freelist; - active->slab.freelist = *reinterpret_cast<void**>(obj); - active->slab.obj_count++; - if (was_on) arch::cpu::intr_on(); - return obj; - } - - /* Slow path: need a new active page */ - - /* If current active is full, detach it (it stays as Slab state, - * cache pointer still valid for kfree to find it) */ - if (active && !active->slab.freelist) { - cpu_slab_[cpu].active = nullptr; - } - - if (was_on) arch::cpu::intr_on(); - - /* Try to get a partial page or allocate a fresh one. - * PMM call happens without any lock held. */ - Page* new_page = new_slab(); - if (!new_page) - return nullptr; - - /* Install as new active page */ - was_on = arch::cpu::intr_enabled(); - arch::cpu::intr_off(); - - /* Another interrupt handler might have set a new active while we - * were allocating. If so, put ours aside (simplified: just use ours). */ - cpu_slab_[cpu].active = new_page; - - void* obj = new_page->slab.freelist; - new_page->slab.freelist = *reinterpret_cast<void**>(obj); - new_page->slab.obj_count++; - - if (was_on) arch::cpu::intr_on(); - return obj; -} - -/* ------------------------------------------------------------------ */ -/* KmemCache::free */ -/* ------------------------------------------------------------------ */ - -void KmemCache::free(void* obj) -{ - if (!obj) - return; - - /* Find the Page for this object's page */ - uint64 obj_addr = reinterpret_cast<uint64>(obj); - - /* Convert VA to PA. Like pmm.cc, handle both PA and VA cases. */ - uint64 obj_pa = va_to_pa(virt_addr(obj_addr)).raw; - uint64 page_pa = obj_pa & ~(kPageSize - 1); - Page* page = pa_to_page(phys_addr(page_pa)); - - if (page->type != PageType::Slab || page->slab.cache != this) - kernel_panic("slab free: object does not belong to this cache"); - - bool was_on = arch::cpu::intr_enabled(); - arch::cpu::intr_off(); - - /* Push object back onto the page's freelist */ - *reinterpret_cast<void**>(obj) = page->slab.freelist; - page->slab.freelist = obj; - page->slab.obj_count--; - - /* If page is now completely empty, return it to PMM */ - if (page->slab.obj_count == 0) { - /* If this page is our active, clear it */ - uint64 cpu = arch::cpu::id(); - if (cpu_slab_[cpu].active == page) - cpu_slab_[cpu].active = nullptr; - - if (was_on) arch::cpu::intr_on(); - - /* Return page to PMM */ - page->slab.cache = nullptr; - page->slab.freelist = nullptr; - pmm_free_page(page); - return; - } - - if (was_on) arch::cpu::intr_on(); -} - -/* ------------------------------------------------------------------ */ -/* Public free-function API */ -/* ------------------------------------------------------------------ */ - -/* Find the smallest size class >= size */ -static KmemCache* find_cache(uint32 size) -{ - for (int i = 0; i < kNumSizeClasses; i++) { - if (kSizeClasses[i] >= size) - return &g_size_caches[i]; - } - return nullptr; // Too large for slab -} - -void* kmalloc(uint32 size) -{ - if (size == 0) - return nullptr; - - /* Large allocation: fall back to PMM directly */ - if (size > kSizeClasses[kNumSizeClasses - 1]) { - uint8 order = 0; - uint64 needed = (size + kPageSize - 1) / kPageSize; - while ((1ULL << order) < needed) - order++; - Page* page = pmm_alloc_pages(order); - if (!page) - return nullptr; - return reinterpret_cast<void*>(pa_to_va(page_to_pa(page)).raw); - } - - KmemCache* cache = find_cache(size); - if (!cache) - return nullptr; - return cache->alloc(); -} - -void kfree(void* ptr) -{ - if (!ptr) - return; - - uint64 addr = reinterpret_cast<uint64>(ptr); - uint64 pa = va_to_pa(virt_addr(addr)).raw; - uint64 page_pa = pa & ~(kPageSize - 1); - Page* page = pa_to_page(phys_addr(page_pa)); - - if (page->type == PageType::Slab) { - KmemCache* cache = page->slab.cache; - if (!cache) - kernel_panic("kfree: slab page has null cache"); - cache->free(ptr); - } else if (page->type == PageType::Allocated) { - /* Large allocation — return directly to PMM */ - pmm_free_pages(page, page->order); - } else { - kernel_panic("kfree: invalid page state"); - } -} - -/* ------------------------------------------------------------------ */ -/* C++ operator new / delete */ -/* ------------------------------------------------------------------ */ - -/* Use decltype(sizeof(0)) instead of size_t to match the compiler's - * built-in expectation for operator new (unsigned long vs unsigned long long). */ -using __sz = decltype(sizeof(0)); - -void* operator new(__sz size) { return kmalloc(static_cast<uint32>(size)); } -void* operator new[](__sz size) { return kmalloc(static_cast<uint32>(size)); } - -void operator delete(void* ptr) noexcept { kfree(ptr); } -void operator delete[](void* ptr) noexcept { kfree(ptr); } -void operator delete(void* ptr, __sz) noexcept { kfree(ptr); } -void operator delete[](void* ptr, __sz) noexcept { kfree(ptr); } - -/* ------------------------------------------------------------------ */ -/* Initialization */ -/* ------------------------------------------------------------------ */ - -void slab_init() -{ - for (int i = 0; i < kNumSizeClasses; i++) { - /* Build a name string like "kmalloc-32" */ - static const char* names[kNumSizeClasses] = { - "kmalloc-8", "kmalloc-16", "kmalloc-32", - "kmalloc-64", "kmalloc-128", "kmalloc-256", - "kmalloc-512", "kmalloc-1024","kmalloc-2048" - }; - g_size_caches[i].init(names[i], kSizeClasses[i], 8); - } -} diff --git a/mm/slab.h b/mm/slab.h deleted file mode 100644 index 909ee62..0000000 --- a/mm/slab.h +++ /dev/null @@ -1,83 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-or-later */ -/* Copyright (C) 2026 Richard Qin */ - -#pragma once - -/* - * SLUB Allocator — module-private header - * - * SLUB = Simple List of Unfull Blocks. - * Metadata lives in Page descriptors (no per-page Slab header). - * Free object list is embedded inside the objects themselves. - * Per-CPU "active page" eliminates lock contention on the fast path. - * - * Reference: docs/specs/slab.md - */ - -#include <lume/types.h> -#include <lume/config.h> -#include <lume/page.h> -#include <lume/list.h> -#include <kernel/sync/spinlock.h> - -/* Size classes: 8, 16, 32, 64, 128, 256, 512, 1024, 2048 */ -inline constexpr int kNumSizeClasses = 9; -inline constexpr uint32 kSizeClasses[kNumSizeClasses] = { - 8, 16, 32, 64, 128, 256, 512, 1024, 2048 -}; - -/* Per-CPU active slab page pointer */ -struct PerCpuSlab { - Page* active; // Currently used slab page (nullptr if none) -}; - -/* - * KmemCache — manages a pool of fixed-size objects. - * - * Lock contract: - * lock_ protects partial_ list only. - * Per-CPU cpu_slab_ access requires interrupts disabled. - * PMM calls happen outside lock_ to avoid lock ordering issues. - */ -class KmemCache { -public: - KmemCache() = default; - - // Explicit init (Placement New / BSS pattern) - void init(const char* name, uint32 obj_size, uint32 obj_align); - - // Allocate one object. Returns nullptr on OOM. - void* alloc(); - - // Free one object back to its owning page. - void free(void* obj); - - const char* name() const { return name_; } - uint32 obj_size() const { return obj_size_; } - -private: - // Get a partially-filled page from the partial list (lock must be held) - Page* get_partial(); - - // Put a page onto the partial list (lock must be held) - void put_partial(Page* page); - - // Remove a page from the partial list (lock must be held) - void remove_partial(Page* page); - - // Allocate a new page from PMM and format it as a slab - Page* new_slab(); - - Spinlock lock_; // Protects partial_ only - list_node partial_; // Partially-filled slab pages - - PerCpuSlab cpu_slab_[kMaxCpus]; // Per-CPU active page - - const char* name_ = "uninit"; - uint32 obj_size_ = 0; // Aligned object size (bytes) - uint32 obj_align_ = 0; // Alignment requirement - uint32 objs_per_slab_ = 0; // Objects per 4KB page -}; - -/* Global size-class caches */ -extern KmemCache g_size_caches[kNumSizeClasses]; diff --git a/mm/tlb.cc b/mm/tlb.cc deleted file mode 100644 index b7a8ba7..0000000 --- a/mm/tlb.cc +++ /dev/null @@ -1,90 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-or-later */ -/* Copyright (C) 2026 Richard Qin */ - -/* - * tlb.cc — TLB shootdown implementation - * - * Uses RISC-V SBI Remote Fence extension to perform cross-hart - * TLB invalidation. SBI firmware handles the IPI delivery and - * synchronization — the call blocks until all target harts have - * completed their local sfence.vma. - * - * When a VmSpace is provided, we use its cpu_mask to target only - * the CPUs that have this address space loaded in satp. This avoids - * unnecessary IPI storms on unrelated cores. - * - * Reference: docs/specs/linux_abi_mm_plan.md §SMP §2.1 - */ - -#include <lume/tlb.h> -#include <lume/vmm.h> -#include <lume/config.h> -#include <arch/mmu.h> -#include <arch/tlb.h> -#include <arch/cpu.h> - -/* - * If a shootdown range exceeds this many pages, do a full flush - * instead of per-page invalidation. - */ -static constexpr uint64 kShootdownPageThreshold = 32; - -/* - * Build a cpu mask for all online CPUs (fallback when mm is null). - */ -namespace { -uint64 all_cpus_mask() -{ - uint64 mask = 0; - for (uint32 i = 0; i < kMaxCpus; i++) { - mask |= (1ULL << i); - } - return mask; -} - -/* - * Get the target cpu mask: use mm->cpu_mask when available, - * fall back to all_cpus_mask for kernel address space (mm == nullptr). - */ -uint64 target_cpu_mask(VmSpace* mm) -{ - if (mm) { - uint64 mask = mm->cpu_mask.load(__ATOMIC_RELAXED); - /* Always include our own CPU — we may still have stale TLB */ - mask |= (1ULL << arch::cpu::id()); - return mask; - } - return all_cpus_mask(); -} -} // namespace - -void tlb_shootdown_range(VmSpace* mm, uint64 start, uint64 end) -{ - if (start >= end) { - return; - } - - uint64 num_pages = (end - start) / kPageSize; - uint64 cpu_mask = target_cpu_mask(mm); - - if (num_pages > kShootdownPageThreshold) { - /* Full flush on target CPUs via arch backend */ - arch::tlb::flush_remote(cpu_mask, 0, ~0ULL); - return; - } - - /* Range flush on target CPUs via arch backend */ - arch::tlb::flush_remote(cpu_mask, start, end - start); -} - -void tlb_shootdown_page(VmSpace* mm, uint64 va) -{ - uint64 cpu_mask = target_cpu_mask(mm); - arch::tlb::flush_remote(cpu_mask, va, kPageSize); -} - -void tlb_shootdown_all() -{ - uint64 cpu_mask = all_cpus_mask(); - arch::tlb::flush_remote(cpu_mask, 0, ~0ULL); -} diff --git a/mm/vma.cc b/mm/vma.cc deleted file mode 100644 index da89e08..0000000 --- a/mm/vma.cc +++ /dev/null @@ -1,640 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-or-later */ -/* Copyright (C) 2026 Richard Qin */ - -/* - * vma.cc — Virtual Memory Area management (MI layer) - * - * Manages user address spaces using a red-black tree of VMA regions. - * Implements demand paging: vmm_map_user records intent only; - * physical pages are allocated on page fault via vmm_handle_page_fault. - * - * MI/MD separation: this file NEVER references pmap::PTE_* constants. - * Permission translation is delegated to pmap::vm_perm_to_pte(). - * - * Reference: docs/specs/vmm.md - */ - -#include <lume/vmm.h> -#include <lume/addr.h> -#include <lume/config.h> -#include <lume/errno.h> -#include <lume/pmm.h> -#include <lume/new.h> -#include <lume/klog.h> -#include <arch/pmap.h> -#include <lume/tlb.h> -#include <lume/mm_layout.h> - -/* ------------------------------------------------------------------ */ -/* Rbtree callbacks for VMA interval tree */ -/* ------------------------------------------------------------------ */ - -static bool vma_less(struct rb_node* node, const struct rb_node* parent) { - VmAreaStruct* n = reinterpret_cast<VmAreaStruct*>( - reinterpret_cast<char*>(node) - __builtin_offsetof(VmAreaStruct, rb)); - const VmAreaStruct* p = reinterpret_cast<const VmAreaStruct*>( - reinterpret_cast<const char*>(parent) - __builtin_offsetof(VmAreaStruct, rb)); - return n->start < p->start; -} - -static int vma_cmp(const void* key, const struct rb_node* node) { - uint64 addr = *static_cast<const uint64*>(key); - const VmAreaStruct* n = reinterpret_cast<const VmAreaStruct*>( - reinterpret_cast<const char*>(node) - __builtin_offsetof(VmAreaStruct, rb)); - if (addr < n->start) return -1; - if (addr >= n->end) return 1; - return 0; -} - -static VmAreaStruct* node_to_vma(struct rb_node* node) -{ - return reinterpret_cast<VmAreaStruct*>( - reinterpret_cast<char*>(node) - __builtin_offsetof(VmAreaStruct, rb)); -} - -static uint64 align_up_any(uint64 value, uint64 align) -{ - if (align == 0) { - return value; - } - return ((value + align - 1) / align) * align; -} - -static bool range_overlaps(uint64 a_start, uint64 a_end, - uint64 b_start, uint64 b_end) -{ - return a_start < b_end && b_start < a_end; -} - - - -static bool vma_overlaps_locked(VmSpace* space, uint64 start, uint64 end) -{ - struct rb_node* node = rb_first(&space->vma_tree); - while (node) { - VmAreaStruct* vma = node_to_vma(node); - if (range_overlaps(start, end, vma->start, vma->end)) { - return true; - } - node = rb_next(node); - } - return false; -} - -static bool vma_range_covered_locked(VmSpace* space, uint64 start, uint64 end) -{ - uint64 cursor = start; - struct rb_node* node = rb_first(&space->vma_tree); - while (node && cursor < end) { - VmAreaStruct* vma = node_to_vma(node); - if (vma->end <= cursor) { - node = rb_next(node); - continue; - } - if (vma->start > cursor) { - return false; - } - if (vma->end > cursor) { - cursor = vma->end; - } - node = rb_next(node); - } - return cursor >= end; -} - -static void vma_try_merge_neighbors_locked(VmSpace* space, VmAreaStruct* vma) -{ - struct rb_node* prev_node = rb_prev(&vma->rb); - if (prev_node) { - VmAreaStruct* prev = node_to_vma(prev_node); - if (prev->end == vma->start && prev->perm == vma->perm) { - prev->end = vma->end; - rb_erase(&vma->rb, &space->vma_tree); - delete vma; - vma = prev; - } - } - - struct rb_node* next_node = rb_next(&vma->rb); - if (next_node) { - VmAreaStruct* next = node_to_vma(next_node); - if (vma->end == next->start && vma->perm == next->perm) { - vma->end = next->end; - rb_erase(&next->rb, &space->vma_tree); - delete next; - } - } -} - -static int vma_split_locked(VmSpace* space, VmAreaStruct* vma, uint64 split) -{ - if (split <= vma->start || split >= vma->end) { - return 0; - } - - VmAreaStruct* right = new VmAreaStruct; - if (!right) { - return -ENOMEM; - } - - right->start = split; - right->end = vma->end; - right->perm = vma->perm; - right->mm = space; - - vma->end = split; - rb_add(&right->rb, &space->vma_tree, vma_less); - return 0; -} - -static int vma_split_range_locked(VmSpace* space, uint64 start, uint64 end) -{ - if (end <= start) { - return 0; - } - - struct rb_node* start_node = rb_find(&start, &space->vma_tree, vma_cmp); - if (!start_node) { - return -ENOMEM; - } - VmAreaStruct* start_vma = node_to_vma(start_node); - int ret = vma_split_locked(space, start_vma, start); - if (ret < 0) { - return ret; - } - - uint64 end_key = end - 1; - struct rb_node* end_node = rb_find(&end_key, &space->vma_tree, vma_cmp); - if (!end_node) { - return -ENOMEM; - } - VmAreaStruct* end_vma = node_to_vma(end_node); - return vma_split_locked(space, end_vma, end); -} - -/* ------------------------------------------------------------------ */ -/* VmSpace lifecycle */ -/* ------------------------------------------------------------------ */ - -VmSpace* VmSpace::create() { - VmSpace* space = new VmSpace; - if (!space) return nullptr; - - space->root_pa = pmap::create(); - if (!space->root_pa) { - delete space; - return nullptr; - } - - space->vma_tree = RB_ROOT; - space->lock.init("VmSpace"); - space->refcount.store(1, __ATOMIC_RELAXED); - space->brk_start = 0; - space->brk_end = 0; - return space; -} - -void VmSpace::incref() { - refcount.fetch_add(1, __ATOMIC_RELAXED); -} - -void VmSpace::put() { - if (refcount.fetch_sub(1, __ATOMIC_ACQ_REL) == 1) { - destroy(); - } -} - -void VmSpace::destroy() { - /* Drain VMA tree under lock, then release BEFORE delete this */ - lock.acquire(); - uint64 root = root_pa; - struct rb_node* node = rb_first(&vma_tree); - while (node) { - VmAreaStruct* vma = reinterpret_cast<VmAreaStruct*>( - reinterpret_cast<char*>(node) - __builtin_offsetof(VmAreaStruct, rb)); - - /* Decref and dec mapcount for all mapped physical pages */ - for (uint64 addr = vma->start; addr < vma->end; addr += kPageSize) { - uint64 pa; - if (pmap::lookup(root, addr, &pa)) { - Page* pg = pa_to_page(phys_addr(pa)); - page_dec_mapcount(pg); - page_decref(pg); - } - } - - node = rb_next(node); - rb_erase(&vma->rb, &vma_tree); - delete vma; - } - lock.release(); - - /* Destroy hardware page tables outside the lock */ - pmap::destroy(root); - - /* Finally free VmSpace itself */ - delete this; -} - -/* ------------------------------------------------------------------ */ -/* Public API: VMA manipulation */ -/* ------------------------------------------------------------------ */ - -int vmm_map_user(VmSpace* space, uint64 va, uint64 len, uint64 perm) { - if (!space) return -EINVAL; - - va = page_align_down(va); - len = page_align_up(len); - if (len == 0) return 0; - if (va < kUserVaBase) return -EINVAL; - if (va >= kUserVaLimit) return -EINVAL; - if (va + len < va || va + len > kUserVaLimit) return -EINVAL; - - VmAreaStruct* vma = new VmAreaStruct; - if (!vma) return -ENOMEM; - vma->start = va; - vma->end = va + len; - vma->perm = perm | VM_USER; // Enforce user-mode access (MI flag) - vma->mm = space; - - LockGuard guard(space->lock); - if (vma_overlaps_locked(space, vma->start, vma->end)) { - delete vma; - return -EINVAL; - } - rb_add(&vma->rb, &space->vma_tree, vma_less); - vma_try_merge_neighbors_locked(space, vma); - return 0; -} - -int vmm_unmap_user(VmSpace* space, uint64 va, uint64 len) { - if (!space) return -EINVAL; - va = page_align_down(va); - len = page_align_up(len); - if (len == 0) return 0; - if (va < kUserVaBase) return -EINVAL; - if (va >= kUserVaLimit) return -EINVAL; - if (va + len < va || va + len > kUserVaLimit) return -EINVAL; - uint64 end = va + len; - - LockGuard guard(space->lock); - struct rb_node* node = rb_first(&space->vma_tree); - while (node) { - VmAreaStruct* vma = node_to_vma(node); - struct rb_node* next = rb_next(node); - - if (vma->end <= va) { - node = next; - continue; - } - if (vma->start >= end) { - break; - } - - uint64 unmap_start = (vma->start > va) ? vma->start : va; - uint64 unmap_end = (vma->end < end) ? vma->end : end; - - if (unmap_start < unmap_end) { - for (uint64 addr = unmap_start; addr < unmap_end; addr += kPageSize) { - uint64 pa; - if (pmap::lookup(space->root_pa, addr, &pa)) { - pmap::unmap(space->root_pa, addr); - Page* pg = pa_to_page(phys_addr(pa)); - page_dec_mapcount(pg); - page_decref(pg); - } - } - } - - if (unmap_start == vma->start && unmap_end == vma->end) { - rb_erase(&vma->rb, &space->vma_tree); - delete vma; - } else if (unmap_start == vma->start) { - rb_erase(&vma->rb, &space->vma_tree); - vma->start = unmap_end; - rb_add(&vma->rb, &space->vma_tree, vma_less); - } else if (unmap_end == vma->end) { - vma->end = unmap_start; - } else { - VmAreaStruct* right = new VmAreaStruct; - if (!right) { - return -ENOMEM; - } - right->start = unmap_end; - right->end = vma->end; - right->perm = vma->perm; - right->mm = space; - - vma->end = unmap_start; - rb_add(&right->rb, &space->vma_tree, vma_less); - break; - } - - node = next; - } - return 0; -} - -int vmm_protect_user(VmSpace* space, uint64 va, uint64 len, uint64 perm) -{ - if (!space) return -EINVAL; - va = page_align_down(va); - len = page_align_up(len); - if (len == 0) return -EINVAL; - if (va < kUserVaBase) return -EINVAL; - if (va >= kUserVaLimit) return -EINVAL; - if (va + len < va || va + len > kUserVaLimit) return -EINVAL; - - uint64 end = va + len; - - LockGuard guard(space->lock); - if (!vma_range_covered_locked(space, va, end)) { - return -ENOMEM; - } - int ret = vma_split_range_locked(space, va, end); - if (ret < 0) { - return ret; - } - - bool tlb_flush_needed = false; - struct rb_node* node = rb_find(&va, &space->vma_tree, vma_cmp); - while (node) { - VmAreaStruct* vma = node_to_vma(node); - if (vma->start >= end) { - break; - } - if (vma->end <= va) { - node = rb_next(node); - continue; - } - - vma->perm = (perm | VM_USER); - - for (uint64 addr = vma->start; addr < vma->end; addr += kPageSize) { - uint64 pa; - if (!pmap::lookup(space->root_pa, addr, &pa)) { - continue; - } - - if (perm == 0) { - Page* pg = pa_to_page(phys_addr(pa)); - page_dec_mapcount(pg); - page_decref(pg); - pmap::unmap(space->root_pa, addr); - tlb_flush_needed = true; - continue; - } - - uint64 pte_perm = pmap::vm_perm_to_pte(vma->perm); - if ((vma->perm & VM_WRITE) && pmap::is_cow(space->root_pa, addr)) { - pte_perm &= ~pmap::PTE_W; - pte_perm |= pmap::PTE_COW; - } - - int ret = pmap::protect(space->root_pa, addr, pte_perm); - if (ret < 0) { - return ret; - } - tlb_flush_needed = true; - } - - node = rb_next(node); - } - - if (tlb_flush_needed) { - tlb_shootdown_range(space, va, end); - } - return 0; -} - -uint64 mm_find_free_va(VmSpace* space, uint64 length, uint64 align) -{ - if (!space) return 0; - length = page_align_up(length); - if (length == 0) return 0; - if (align < kPageSize) align = kPageSize; - - LockGuard guard(space->lock); - - /* Linux: arch_get_unmapped_area() starts from mm->mmap_base. - * RISC-V: TASK_UNMAPPED_BASE = PAGE_ALIGN(TASK_SIZE / 3). - * Fall back to kUserVaBase if mmap_base not set (kernel VmSpaces). */ - uint64 base = space->mmap_base ? space->mmap_base : kUserVaBase; - uint64 cursor = align_up_any(base, align); - struct rb_node* node = rb_first(&space->vma_tree); - while (node) { - VmAreaStruct* vma = node_to_vma(node); - if (cursor <= vma->start) { - if (cursor > kUserVaLimit - length) { - return 0; - } - if (cursor + length <= vma->start) { - return cursor; - } - } - if (cursor < vma->end) { - cursor = vma->end; - } - cursor = align_up_any(cursor, align); - node = rb_next(node); - } - - if (cursor > kUserVaLimit - length) { - return 0; - } - return cursor; -} - -bool vma_range_covered(VmSpace* space, uint64 start, uint64 end) -{ - if (!space) return false; - start = page_align_down(start); - end = page_align_up(end); - if (end <= start) return true; - - LockGuard guard(space->lock); - return vma_range_covered_locked(space, start, end); -} - -/* ------------------------------------------------------------------ */ -/* Page fault handler (demand paging) */ -/* ------------------------------------------------------------------ */ - -static bool vma_allows(const VmAreaStruct* vma, VmFaultCause cause) -{ - switch (cause) { - case VmFaultCause::Read: - return (vma->perm & VM_READ) != 0; - case VmFaultCause::Write: - return (vma->perm & VM_WRITE) != 0; - case VmFaultCause::Exec: - return (vma->perm & VM_EXEC) != 0; - default: - return false; - } -} - -int vmm_handle_page_fault(VmSpace* space, uint64 fault_addr, VmFaultCause cause) { - if (!space) return -EINVAL; - - uint64 aligned_addr = page_align_down(fault_addr); - - LockGuard guard(space->lock); - struct rb_node* node = rb_find(&fault_addr, &space->vma_tree, vma_cmp); - if (!node) { - return -EFAULT; // No VMA covers this address → segfault - } - - VmAreaStruct* vma = reinterpret_cast<VmAreaStruct*>( - reinterpret_cast<char*>(node) - __builtin_offsetof(VmAreaStruct, rb)); - - if (!vma_allows(vma, cause)) { - return -EFAULT; - } - - /* Check if page is already mapped */ - uint64 paddr; - if (pmap::lookup(space->root_pa, aligned_addr, &paddr)) { - if (cause == VmFaultCause::Write && pmap::is_cow(space->root_pa, aligned_addr)) { - /* Store Page Fault on a COW page */ - Page* page = pmm_alloc_page(); - if (!page) return -ENOMEM; - uint64 new_pa = page_to_pa(page).raw; - - /* Copy page contents */ - uint8* dst = reinterpret_cast<uint8*>(pa_to_va(phys_addr(new_pa)).raw); - const uint8* src = reinterpret_cast<const uint8*>(pa_to_va(phys_addr(paddr)).raw); - for (uint64 i = 0; i < kPageSize; i++) { - dst[i] = src[i]; - } - - /* Remap with write permission, clearing COW */ - uint64 pte_perm = pmap::vm_perm_to_pte(vma->perm); - if (pmap::map(space->root_pa, aligned_addr, new_pa, pte_perm) != 0) { - pmm_free_page(page); - return -ENOMEM; - } - - /* New page gets mapcount = 1 (we just mapped it) */ - page_inc_mapcount(page); - - /* Decrement mapcount and refcount of old page */ - Page* old_page = pa_to_page(phys_addr(paddr)); - page_dec_mapcount(old_page); - page_decref(old_page); - - /* Flush TLB to remove old read-only cached translation */ - tlb_shootdown_page(space, aligned_addr); - return 0; - } - /* DEBUG: page already mapped but still faulting — dump PTE */ - { - uint64 cur_satp; - asm volatile("csrr %0, satp" : "=r"(cur_satp)); - printk(KERN_ERR "fault: va=%llx pa=%llx cause=%d pte=%llx satp=%llx root=%llx\n", - aligned_addr, paddr, static_cast<int>(cause), - pmap::lookup_pte(space->root_pa, aligned_addr), - cur_satp, space->root_pa); - } - return 0; // Already mapped, maybe read fault on existing page - } - - /* Allocate physical page and zero it. - * Linux: do_anonymous_page() uses alloc_zeroed_user_highpage_movable(). - * Anonymous pages MUST be zeroed — userspace expects MAP_ANONYMOUS - * pages to contain all zeros (POSIX requirement). */ - Page* page = pmm_alloc_page(); - if (!page) return -ENOMEM; - - uint64 pa = page_to_pa(page).raw; - - /* Zero the page before mapping it to userspace */ - /* Zero the page before mapping it to userspace */ - uint8* kva = reinterpret_cast<uint8*>(pa_to_va(PhysAddr{pa}).raw); - for (uint64 i = 0; i < kPageSize; i++) kva[i] = 0; - - /* Translate MI permission flags to hardware PTE bits (MD layer) */ - uint64 pte_perm = pmap::vm_perm_to_pte(vma->perm); - - if (pmap::map(space->root_pa, aligned_addr, pa, pte_perm) != 0) { - pmm_free_page(page); - return -ENOMEM; - } - - /* Initial mapping: refcount is already 1 from pmm_alloc_page. - * Set mapcount = 1 to track this PTE mapping. */ - page_inc_mapcount(page); - - return 0; -} - -/* ------------------------------------------------------------------ */ -/* Fork / Clone (COW infrastructure) */ -/* ------------------------------------------------------------------ */ - -VmSpace* vmm_clone(VmSpace* parent) { - if (!parent) return nullptr; - - VmSpace* child = VmSpace::create(); - if (!child) return nullptr; - - child->brk_start = parent->brk_start; - child->brk_end = parent->brk_end; - - LockGuard parent_guard(parent->lock); - - struct rb_node* node = rb_first(&parent->vma_tree); - while (node) { - VmAreaStruct* parent_vma = reinterpret_cast<VmAreaStruct*>( - reinterpret_cast<char*>(node) - __builtin_offsetof(VmAreaStruct, rb)); - - VmAreaStruct* child_vma = new VmAreaStruct; - if (!child_vma) { - child->destroy(); - return nullptr; - } - child_vma->start = parent_vma->start; - child_vma->end = parent_vma->end; - child_vma->perm = parent_vma->perm; - child_vma->mm = child; - - /* Copy pages and mark COW if writable */ - for (uint64 va = child_vma->start; va < child_vma->end; va += kPageSize) { - uint64 pa; - if (pmap::lookup(parent->root_pa, va, &pa)) { - /* Make parent COW if it was writable */ - if (child_vma->perm & VM_WRITE) { - pmap::make_cow(parent->root_pa, va); - /* Flush TLB for parent since we reduced permissions */ - tlb_shootdown_page(parent, va); - } - - /* Map in child as COW or read-only */ - uint64 pte_perm = pmap::vm_perm_to_pte(child_vma->perm); - if (child_vma->perm & VM_WRITE) { - pte_perm &= ~pmap::PTE_W; - pte_perm |= pmap::PTE_COW; - } - - if (pmap::map(child->root_pa, va, pa, pte_perm) != 0) { - /* If map fails, we leak what we did in this loop for now, - * but child->destroy() will clean it up safely since - * we increment refcounts as we go. */ - delete child_vma; - child->destroy(); - return nullptr; - } - - /* Increment physical page refcount and mapcount */ - Page* shared_page = pa_to_page(phys_addr(pa)); - page_incref(shared_page); - page_inc_mapcount(shared_page); - } - } - - rb_add(&child_vma->rb, &child->vma_tree, vma_less); - node = rb_next(node); - } - - return child; -} diff --git a/mm/vmm.cc b/mm/vmm.cc deleted file mode 100644 index 51617e3..0000000 --- a/mm/vmm.cc +++ /dev/null @@ -1,235 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-or-later */ -/* Copyright (C) 2026 Richard Qin */ - -/* - * vmm.cc — Virtual Memory Manager (Phase 1: Kernel Page Table) - * - * Builds a fine-grained kernel page table, activates it, and retires - * the early_pgdir identity mapping from entry.S. - * - * With memblock integration, the initialization order is: - * memblock_init → vmm_init (allocates page_map from memblock, - * maps vmemmap, activates page table) → pmm_init (buddy system - * starts with page_map already at vmemmap VA — no relocation). - * - * Reference: docs/specs/vmm.md - */ - -#include <arch/config.h> -#include <arch/mmu.h> -#include <arch/pmap.h> -#include <lume/tlb.h> -#include <lume/addr.h> -#include <lume/console.h> -#include <lume/fdt.h> -#include <lume/page.h> -#include <lume/klog.h> -#include <lume/kprintf.h> -#include <lume/memblock.h> -#include <lume/pmm.h> -#include <lume/types.h> -#include <lume/vmm.h> -#include <lume/config.h> - -/* Linker-exported section boundaries (virtual addresses) */ -extern "C" char _stext[], _etext[]; -extern "C" char _srodata[], _erodata[]; -extern "C" char _sdata[], _edata[]; -extern "C" char _sbss[], _ebss[]; -extern "C" char _kernel_end[]; - -/* Global kernel page table root PA — external linkage for task_exit */ -uint64 g_kernel_pgtbl = 0; - -namespace { - -/* VMM exports for PMM: page_map location after vmemmap mapping. */ -uint64 g_vmemmap_page_map_pa = 0; -uint64 g_vmemmap_page_map_size = 0; -uint64 g_vmemmap_num_pages = 0; - -/* Convert a linker symbol to PA (kernel is executing in higher-half VA). */ -inline uint64 sym_to_pa(const void *sym) { - return boot_va_to_pa(virt_addr(reinterpret_cast<uint64>(sym))).raw; -} - -/* Map a PA range into the kernel page table via the direct map. */ -void map_range(uint64 root, uint64 pa_start, uint64 pa_end, - uint64 perm) { - for (uint64 pa = pa_start; pa < pa_end; pa += kPageSize) { - uint64 va = pa_to_va(phys_addr(pa)).raw; - int ret = pmap::map(root, va, pa, perm | pmap::PTE_G); - if (ret != 0) { - kernel_panic("vmm_init: pmap::map failed"); - } - } -} - -/* Map a PA range into the kernel page table at the fixed linked VA. */ -void map_range_linked(uint64 root, uint64 pa_start, uint64 pa_end, - uint64 perm) { - for (uint64 pa = pa_start; pa < pa_end; pa += kPageSize) { - uint64 va = boot_pa_to_va(phys_addr(pa)).raw; - int ret = pmap::map(root, va, pa, perm | pmap::PTE_G); - if (ret != 0) { - kernel_panic("vmm_init: pmap::map (linked) failed"); - } - } -} - -/* Map a PA range to an explicit VA base, using 2MB pages where possible. */ -void map_range_at_huge(uint64 root, uint64 va_base, uint64 pa_start, - uint64 pa_end, uint64 perm) { - constexpr uint64 k2MB = 2ULL * 1024 * 1024; - uint64 pa = pa_start; - uint64 va = va_base; - - /* Map 2MB-aligned chunks with mega pages */ - while (pa + k2MB <= pa_end) { - if ((va & (k2MB - 1)) == 0 && (pa & (k2MB - 1)) == 0) { - int ret = pmap::map_2mb(root, va, pa, perm | pmap::PTE_G); - if (ret != 0) { - kernel_panic("vmm_init: pmap::map_2mb failed"); - } - pa += k2MB; - va += k2MB; - } else { - break; - } - } - - /* Map remaining 4KB tail */ - while (pa < pa_end) { - int ret = pmap::map(root, va, pa, perm | pmap::PTE_G); - if (ret != 0) { - kernel_panic("vmm_init: vmemmap 4KB pmap::map failed"); - } - pa += kPageSize; - va += kPageSize; - } -} - -} // namespace - -uint64 vmm_get_page_map_pa() { return g_vmemmap_page_map_pa; } -uint64 vmm_get_page_map_size() { return g_vmemmap_page_map_size; } -uint64 vmm_get_num_pages() { return g_vmemmap_num_pages; } - -void vmm_init() { - /* 1. Allocate new root page table (from memblock) */ - uint64 root = pmap::create(); - if (!root) { - kernel_panic("vmm_init: failed to create root page table"); - } - - /* 2. Compute PA boundaries of each kernel section */ - uint64 text_start = page_align_down(sym_to_pa(_stext)); - uint64 text_end = page_align_up(sym_to_pa(_etext)); - uint64 ro_start = page_align_down(sym_to_pa(_srodata)); - uint64 ro_end = page_align_up(sym_to_pa(_erodata)); - uint64 data_start = page_align_down(sym_to_pa(_sdata)); - uint64 bss_end = page_align_up(sym_to_pa(_ebss)); - - /* Memory extent from memblock */ - uint64 mem_base = memblock_get_mem_base(); - uint64 mem_size = memblock_get_mem_size(); - uint64 mem_end = mem_base + mem_size; - - /* 3. Map kernel image at FIXED linked VA. - * - * Kernel text/data are linked at the bootstrap higher-half addresses - * (kDirectMapBaseDefault + PA). They must remain executable/readable at - * those addresses until full kernel-image relocation is implemented. - */ - map_range_linked(root, text_start, text_end, pmap::PTE_R | pmap::PTE_X); - - /* 4. Map .rodata (R) at fixed linked VA. - * We map [_srodata, _sdata) so that the .lume.reloc section (which - * sits in its own padded output section between _erodata and _sdata) - * is also covered. Relocator only reads it during early boot, but - * mapping it keeps post-vmm accesses to __lume_reloc_* symbols sane. */ - map_range_linked(root, ro_start, page_align_down(sym_to_pa(_sdata)), - pmap::PTE_R); - - /* 5. Map .data + .bss (RW) at fixed linked VA */ - map_range_linked(root, data_start, bss_end, pmap::PTE_R | pmap::PTE_W); - - /* 6. Map usable physical RAM into the KASLR-slid direct map. - * - * The direct map provides pa_to_va() / va_to_pa() for the physical - * address space. We map from kPhysStart (the kernel's linked load - * address), skipping [mem_base, kPhysStart) because that region is - * PMP-protected firmware (OpenSBI) — S-mode cannot access it. - * - * Linux does the same on RISC-V: the linear map covers only the - * usable RAM range, not firmware-reserved regions. - * - * Uses 2MB huge pages for efficiency (64 entries for 128MB vs 32K). - */ - uint64 dm_start = kPhysStart; - if (dm_start < mem_base) { - dm_start = mem_base; - } - uint64 dm_va_base = pa_to_va(phys_addr(dm_start)).raw; - map_range_at_huge(root, dm_va_base, dm_start, mem_end, - pmap::PTE_R | pmap::PTE_W); - - /* 7. Allocate page_map from memblock (top of RAM) and map at VMEMMAP. - * - * Since memblock is active (PMM not yet initialized), pmap::map - * allocates page table pages from memblock too. page_map starts - * life at the vmemmap VA — no relocation ever needed. - */ - uint64 num_pages = mem_size / kPageSize; - uint64 page_map_bytes = num_pages * sizeof(Page); - uint64 page_map_span = page_align_up(page_map_bytes); - uint64 page_map_pa = memblock_alloc_top(page_map_span, kPageSize); - - if (!page_map_pa) { - kernel_panic("vmm_init: memblock failed to allocate page_map"); - } - - kprintf("[vmm] vmemmap: %llu pages, %llu KB at PA 0x%llx\n", num_pages, - page_map_span / 1024, page_map_pa); - - /* Map page_map at vmemmap_base from KaslrLayout (randomised by KASLR), using 2MB pages where possible */ - map_range_at_huge(root, arch::kaslr_layout_get().vmemmap_base, page_map_pa, - page_map_pa + page_map_span, pmap::PTE_R | pmap::PTE_W); - - /* Export for pmm_init() */ - g_vmemmap_page_map_pa = page_map_pa; - g_vmemmap_page_map_size = page_map_bytes; - g_vmemmap_num_pages = num_pages; - - /* 8. Pre-map console MMIO before page table switch */ - g_kernel_pgtbl = root; - console_init(); - - /* 9. Activate the new page table */ - pmap::activate(root); - - /* Console MMIO access must now use runtime (KASLR) direct-map base. */ - console_use_runtime_mapping(); - - kprintf("[vmm] vmm_init complete (vmemmap at 0x%llx)\n", arch::kaslr_layout_get().vmemmap_base); -} - -void vmm_init_ap() { - if (!g_kernel_pgtbl) { - kernel_panic("vmm_init_ap: kernel page table not ready"); - } - pmap::activate(g_kernel_pgtbl); -} - -void vmm_map_kernel_mmio(uint64 pa, uint64 size) { - if (!g_kernel_pgtbl) { - kernel_panic("vmm_map_kernel_mmio: kernel page table not ready"); - } - - uint64 aligned_pa = page_align_down(pa); - uint64 aligned_size = page_align_up(pa + size) - aligned_pa; - - map_range(g_kernel_pgtbl, aligned_pa, aligned_pa + aligned_size, - pmap::PTE_R | pmap::PTE_W | pmap::PTE_G); - tlb_shootdown_all(); -} |