summaryrefslogtreecommitdiff
path: root/src/kernel/mm/vm.cc
diff options
context:
space:
mode:
Diffstat (limited to 'src/kernel/mm/vm.cc')
-rw-r--r--src/kernel/mm/vm.cc462
1 files changed, 300 insertions, 162 deletions
diff --git a/src/kernel/mm/vm.cc b/src/kernel/mm/vm.cc
index 650fcf1..1155d2c 100644
--- a/src/kernel/mm/vm.cc
+++ b/src/kernel/mm/vm.cc
@@ -1,75 +1,85 @@
#include "kernel/mm.h"
#include "kernel/pmm.h"
#include "kernel/riscv.h"
-#include "drivers/uart.h"
#include "common/types.h"
-#include "kernel/config.h"
#include "lib/string.h"
+#include "drivers/uart.h"
+extern "C"
+{
+ extern char kernel_end[];
+ extern char trampoline[];
+}
-extern char text_start[];
-extern char text_end[];
-extern char rodata_start[];
-extern char rodata_end[];
-extern char data_start[];
-extern char data_end[];
-extern char bss_start[];
-extern char bss_end[];
-extern char kernel_end[];
+namespace VM
+{
-extern char uservec[];
-extern char userret[];
-extern char trampoline[];
+ uint64 *kernel_pagetable;
-VM::pagetable_t kernel_pagetable;
+ static void page_ref_inc(uint64 pa)
+ {
+ struct Page *p = PMM::pa_to_page(pa);
+ if (p)
+ {
+ __atomic_add_fetch(&p->refcnt, 1, __ATOMIC_RELAXED);
+ }
+ }
-namespace VM
-{
- // pagetable_t kernel_pagetable;
+ static bool page_ref_dec(uint64 pa)
+ {
+ struct Page *p = PMM::pa_to_page(pa);
+ if (p)
+ {
+ if (__atomic_sub_fetch(&p->refcnt, 1, __ATOMIC_RELEASE) == 0)
+ {
+ return true;
+ }
+ }
+ return false;
+ }
- pte_t *walk(pagetable_t pagetable, uint64 va, int alloc)
+ static uint64 *walk(uint64 *pagetable, uint64 va, int alloc)
{
if (va >= MAXVA)
- return 0;
-
+ return nullptr;
+
for (int level = 2; level > 0; level--)
{
- pte_t *pte = &pagetable[PX(level, va)];
-
+ uint64 *pte = &pagetable[PX(level, va)];
if (*pte & PTE_V)
{
- pagetable = (pagetable_t)PTE2PA(*pte);
+ pagetable = (uint64 *)PTE2PA(*pte);
}
else
{
- if (!alloc || (pagetable = (pagetable_t)PMM::alloc_page()) == 0)
- return 0;
+ if (!alloc || (pagetable = (uint64 *)PMM::alloc_page()) == nullptr)
+ return nullptr;
+
memset(pagetable, 0, PGSIZE);
- // pagetable = (pagetable_t)PMM::alloc_page();
*pte = PA2PTE(pagetable) | PTE_V;
}
}
return &pagetable[PX(0, va)];
}
- int mappages(pagetable_t pagetable, uint64 va, uint64 size, uint64 pa, int perm)
+ int mappages(uint64 *pagetable, uint64 va, uint64 size, uint64 pa, int perm)
{
uint64 a, last;
- pte_t *pte;
+ uint64 *pte;
+
+ if (size == 0)
+ return -1;
a = PGROUNDDOWN(va);
last = PGROUNDDOWN(va + size - 1);
- for(;;)
+ for (;;)
{
- if((pte = walk(pagetable, a, 1)) == 0)
+ if ((pte = walk(pagetable, a, 1)) == nullptr)
return -1;
if (*pte & PTE_V)
{
- Drivers::uart_puts("Remap panic: ");
- Drivers::print_hex(a);
- Drivers::uart_puts("\n");
return -1;
}
@@ -83,228 +93,356 @@ namespace VM
return 0;
}
- void kvmmap(uint64 va, uint64 pa, uint64 sz, int perm)
+ uint64 *uvmcreate()
{
- if (mappages(kernel_pagetable, va, sz, pa, perm) != 0)
+ uint64 *pagetable = (uint64 *)PMM::alloc_page();
+ if (pagetable)
{
- Drivers::uart_puts("kvmmap: failed to map\n");
+ memset(pagetable, 0, PGSIZE);
+
+ if (mappages(pagetable, TRAMPOLINE, PGSIZE, (uint64)trampoline, PTE_R | PTE_X) < 0)
+ {
+ PMM::free_page(pagetable);
+ return nullptr;
+ }
}
+ return pagetable;
}
- void kvminit()
+ void uvminit(uint64 *pagetable, uchar *src, uint64 sz)
{
- kernel_pagetable = (pagetable_t)PMM::alloc_page();
- memset(kernel_pagetable, 0, PGSIZE);
-
- kvmmap(g_uart_base, g_uart_base, PGSIZE, PTE_R | PTE_W);
-
- kvmmap(g_plic_base, g_plic_base, 0x400000, PTE_R | PTE_W);
-
- // kvmmap(g_clint_base, g_clint_base, 0x10000, PTE_R | PTE_W);
-
- kvmmap((uint64)text_start, (uint64)text_start,
- (uint64)text_end - (uint64)text_start, PTE_R | PTE_X);
-
- kvmmap((uint64)rodata_start, (uint64)rodata_start,
- (uint64)rodata_end - (uint64)rodata_start, PTE_R);
+ char *mem;
- kvmmap((uint64)data_start, (uint64)data_start,
- (uint64)kernel_end - (uint64)data_start, PTE_R | PTE_W);
+ if (sz >= PGSIZE)
+ {
+ Drivers::uart_puts("uvminit: initcode too big!\n");
+ return;
+ }
- kvmmap((uint64)kernel_end, (uint64)kernel_end,
- (uint64)PHYSTOP - (uint64)kernel_end, PTE_R | PTE_W);
+ mem = (char *)PMM::alloc_page();
+ memset(mem, 0, PGSIZE);
- kvmmap(TRAMPOLINE, (uint64)trampoline, PGSIZE, PTE_R | PTE_X);
- }
- void kvminithart()
- {
- w_satp(MAKE_SATP(kernel_pagetable));
- sfence_vma();
+ mappages(pagetable, 0, PGSIZE, (uint64)mem, PTE_W | PTE_R | PTE_X | PTE_U);
+ memmove(mem, src, sz);
}
-
- pagetable_t uvmcreate()
+ uint64 uvmalloc(uint64 *pagetable, uint64 oldsz, uint64 newsz, int xperm)
{
- pagetable_t pagetable = (pagetable_t)PMM::alloc_page();
- if (pagetable == 0)
- return 0;
- memset(pagetable, 0, PGSIZE);
+ char *mem;
+ uint64 a;
+
+ if (newsz < oldsz)
+ return oldsz;
- if (mappages(pagetable, TRAMPOLINE, PGSIZE, (uint64)trampoline, PTE_R | PTE_X) < 0)
+ oldsz = PGROUNDUP(oldsz);
+ for (a = oldsz; a < newsz; a += PGSIZE)
{
- return 0;
+ mem = (char *)PMM::alloc_page();
+ if (mem == nullptr)
+ {
+ uvmdealloc(pagetable, a, oldsz);
+ return 0;
+ }
+ memset(mem, 0, PGSIZE);
+ if (mappages(pagetable, a, PGSIZE, (uint64)mem, PTE_R | PTE_U | xperm) != 0)
+ {
+ PMM::free_page(mem);
+ uvmdealloc(pagetable, a, oldsz);
+ return 0;
+ }
}
-
- return pagetable;
+ return newsz;
}
- void uvminit(pagetable_t pagetable, uchar *src, uint sz)
+ uint64 uvmdealloc(uint64 *pagetable, uint64 oldsz, uint64 newsz)
{
- char *mem;
+ if (newsz >= oldsz)
+ return oldsz;
- if (sz >= PGSIZE)
+ if (PGROUNDUP(newsz) < PGROUNDUP(oldsz))
{
- Drivers::uart_puts("uvminit: initcode too big\n");
- return;
+ int npages = (PGROUNDUP(oldsz) - PGROUNDUP(newsz)) / PGSIZE;
+ uvmunmap(pagetable, PGROUNDUP(newsz), npages, 1);
}
- mem = (char*)PMM::alloc_page();
- memset(mem, 0, PGSIZE);
-
- mappages(pagetable, 0, PGSIZE, (uint64)mem, PTE_W|PTE_R|PTE_X|PTE_U);
-
- memmove(mem, src, sz);
-
- uint64 trap_pa = (uint64)uservec;
- uint64 trap_page_start = PGROUNDDOWN(trap_pa);
-
- mappages(pagetable, trap_page_start, PGSIZE, trap_page_start, PTE_R | PTE_X);
-
- mappages(pagetable, trap_page_start + PGSIZE, PGSIZE, trap_page_start + PGSIZE, PTE_R | PTE_X);
+ return newsz;
}
- void uvmunmap(pagetable_t pagetable, uint64 va, uint64 npages, int do_free)
+ void uvmunmap(uint64 *pagetable, uint64 va, uint64 npages, int do_free)
{
uint64 a;
- pte_t *pte;
+ uint64 *pte;
if ((va % PGSIZE) != 0)
- {
- Drivers::uart_puts("uvmunmap: not aligned\n");
return;
- }
- for(a = va; a < va + npages * PGSIZE; a += PGSIZE)
+ for (a = va; a < va + npages * PGSIZE; a += PGSIZE)
{
- if ((pte = walk(pagetable, a, 0)) == 0)
+ if ((pte = walk(pagetable, a, 0)) == nullptr)
continue;
- if((*pte & PTE_V) == 0)
+ if ((*pte & PTE_V) == 0)
continue;
- if (PTE_FLAGS(*pte) == PTE_V)
+ if ((PTE_FLAGS(*pte) & (PTE_R | PTE_W | PTE_X)) == 0)
{
- Drivers::uart_puts("uvmunmap: not a leaf\n");
+ continue;
}
if (do_free)
{
uint64 pa = PTE2PA(*pte);
- PMM::free_page((void *)pa);
+ if (page_ref_dec(pa))
+ {
+ PMM::free_page((void *)pa);
+ }
}
*pte = 0;
}
}
- void uvmfree(pagetable_t pagetable, uint64 sz)
+ static void freewalk(uint64 *pagetable)
{
- if(sz > 0)
- uvmunmap(pagetable, 0, PGROUNDDOWN(sz)/PGSIZE, 1);
+ for (int i = 0; i < 512; i++)
+ {
+ uint64 pte = pagetable[i];
+ if ((pte & PTE_V) && (pte & (PTE_R | PTE_W | PTE_X)) == 0)
+ {
+ uint64 *child = (uint64 *)PTE2PA(pte);
+ freewalk(child);
+ pagetable[i] = 0;
+ }
+ }
+ PMM::free_page(pagetable);
+ }
- PMM::free_page((void *)pagetable);
+ void uvmfree(uint64 *pagetable, uint64 sz)
+ {
+ if (sz > 0)
+ uvmunmap(pagetable, 0, PGROUNDUP(sz) / PGSIZE, 1);
+ freewalk(pagetable);
}
- int uvmcopy(pagetable_t oldpt, pagetable_t newpt, uint64 sz)
+ int uvmcopy(uint64 *oldpt, uint64 *newpt, uint64 sz)
{
- pte_t *pte;
+ uint64 *pte;
uint64 pa, i;
uint flags;
- char *mem;
- for(i = 0; i < sz; i += PGSIZE)
+ for (i = 0; i < sz; i += PGSIZE)
{
- if((pte = walk(oldpt, i, 0)) == 0)
+ if ((pte = walk(oldpt, i, 0)) == nullptr)
continue;
if ((*pte & PTE_V) == 0)
continue;
-
+
pa = PTE2PA(*pte);
flags = PTE_FLAGS(*pte);
- if ((mem = (char *)PMM::alloc_page()) == 0)
- goto err;
-
- memmove(mem, (char *)pa, PGSIZE);
+ if (flags & PTE_W)
+ {
+ flags &= ~PTE_W;
+ flags |= PTE_COW;
+ *pte = PA2PTE(pa) | flags;
+ }
- if (mappages(newpt, i, PGSIZE, (uint64)mem, flags) != 0)
+ if (mappages(newpt, i, PGSIZE, pa, flags) != 0)
{
- PMM::free_page(mem);
- goto err;
+ return -1;
}
+
+ page_ref_inc(pa);
}
+
+ sfence_vma();
return 0;
- err:
- uvmunmap(newpt, 0, i / PGSIZE, 1);
- return -1;
}
- uint64 uvmalloc(pagetable_t pagetable, uint64 oldsz, uint64 newsz)
- {
- char *mem;
- uint64 a;
+ int handle_cow_fault(uint64 *pagetable, uint64 va);
- if (newsz < oldsz)
- return oldsz;
+ int copyout(uint64 *pagetable, uint64 dstva, char *src, uint64 len)
+ {
+ uint64 n, va0, pa0;
+ uint64 *pte;
- oldsz = PGROUNDUP(oldsz);
- for (a = oldsz; a < newsz; a += PGSIZE)
+ while (len > 0)
{
- mem = (char *)PMM::alloc_page();
- if (mem == 0)
- {
- uvmdealloc(pagetable, a, oldsz);
- return 0;
- }
- memset(mem, 0, PGSIZE);
- if (mappages(pagetable, a, PGSIZE, (uint64)mem, PTE_W | PTE_X | PTE_R | PTE_U) != 0)
+ va0 = PGROUNDDOWN(dstva);
+ pte = walk(pagetable, va0, 0);
+ if (pte == nullptr || (*pte & PTE_V) == 0 || (*pte & PTE_U) == 0)
+ return -1;
+
+ // Note: Maybe a bug
+ if (*pte & PTE_COW)
{
- PMM::free_page(mem);
- uvmdealloc(pagetable, a, oldsz);
- return 0;
+ if (handle_cow_fault(pagetable, va0) < 0)
+ return -1;
+ // Re-walk because handle_cow_fault replaced the page mapping
+ pte = walk(pagetable, va0, 0);
+ if (pte == nullptr || (*pte & PTE_V) == 0 || (*pte & PTE_U) == 0)
+ return -1;
}
- }
- return newsz;
- }
- uint64 uvmdealloc(pagetable_t pagetable, uint64 oldsz, uint64 newsz)
- {
- if (newsz >= oldsz)
- return oldsz;
+ pa0 = PTE2PA(*pte);
+ n = PGSIZE - (dstva - va0);
+ if (n > len)
+ n = len;
- if (PGROUNDUP(newsz) < PGROUNDUP(oldsz))
- {
- int npages = (PGROUNDUP(oldsz) - PGROUNDUP(newsz)) / PGSIZE;
- uvmunmap(pagetable, PGROUNDUP(newsz), npages, 1);
- }
+ memmove((void *)(pa0 + (dstva - va0)), src, n);
- return newsz;
+ len -= n;
+ src += n;
+ dstva = va0 + PGSIZE;
+ }
+ return 0;
}
- int copyin(pagetable_t pagetable, char *dst, uint64 srcva, uint64 len)
+ int copyin(uint64 *pagetable, char *dst, uint64 srcva, uint64 len)
{
uint64 n, va0, pa0;
- pte_t *pte;
+ uint64 *pte;
while (len > 0)
{
va0 = PGROUNDDOWN(srcva);
pte = walk(pagetable, va0, 0);
- if(pte == 0 || (*pte & PTE_V) == 0 || (*pte & PTE_U) == 0)
- {
+ if (pte == nullptr || (*pte & PTE_V) == 0 || (*pte & PTE_U) == 0)
return -1;
- }
+
pa0 = PTE2PA(*pte);
n = PGSIZE - (srcva - va0);
- if(n > len)
+ if (n > len)
n = len;
-
- memmove(dst, (void*)(pa0 + (srcva - va0)), n);
+
+ memmove(dst, (void *)(pa0 + (srcva - va0)), n);
len -= n;
dst += n;
srcva = va0 + PGSIZE;
}
return 0;
-
+ }
+
+ int copyinstr(uint64 *pagetable, char *dst, uint64 srcva, uint64 max)
+ {
+ uint64 n, va0, pa0;
+ uint64 *pte;
+ int got_null = 0;
+ uint64 len = 0;
+
+ while (len < max)
+ {
+ va0 = PGROUNDDOWN(srcva);
+ pte = walk(pagetable, va0, 0);
+
+ if (pte == nullptr || (*pte & PTE_V) == 0 || (*pte & PTE_U) == 0)
+ return -1;
+
+ pa0 = PTE2PA(*pte);
+
+ n = PGSIZE - (srcva - va0);
+ if (n > max - len)
+ n = max - len;
+
+ char *p = (char *)(pa0 + (srcva - va0));
+ while (n > 0)
+ {
+ if (*p == '\0')
+ {
+ *dst = '\0';
+ got_null = 1;
+ break;
+ }
+ *dst = *p;
+ n--;
+ srcva++;
+ dst++;
+ p++;
+ len++;
+ }
+ if (got_null)
+ break;
+ }
+
+ if (got_null)
+ return 0;
+
+ return -1;
+ }
+
+ int handle_cow_fault(uint64 *pagetable, uint64 va)
+ {
+ if (va >= MAXVA)
+ return -1;
+
+ uint64 *pte = walk(pagetable, va, 0);
+ if (pte == nullptr || (*pte & PTE_V) == 0 || (*pte & PTE_U) == 0)
+ return -1;
+
+ if ((*pte & PTE_COW) == 0)
+ return -1;
+
+ uint64 pa = PTE2PA(*pte);
+ uint64 flags = PTE_FLAGS(*pte);
+
+ char *mem = (char *)PMM::alloc_page();
+ if (mem == nullptr)
+ return -1;
+
+ memmove(mem, (char *)pa, PGSIZE);
+
+ uint64 new_flags = (flags | PTE_W) & ~PTE_COW;
+ *pte = PA2PTE((uint64)mem) | new_flags | PTE_V;
+
+ if (page_ref_dec(pa))
+ {
+ PMM::free_page((void *)pa);
+ }
+
+ sfence_vma();
+ return 0;
+ }
+
+ static void kvmmap(uint64 va, uint64 pa, uint64 sz, int perm)
+ {
+ if (mappages(kernel_pagetable, va, sz, pa, perm) != 0)
+ {
+ Drivers::uart_puts("kvmmap: failed\n");
+ while (1)
+ ;
+ }
+ }
+
+ void kvminit()
+ {
+ kernel_pagetable = uvmcreate();
+
+ kvmmap(0x10000000, 0x10000000, 0x10000, PTE_R | PTE_W);
+ kvmmap(0x0c000000, 0x0c000000, 0x400000, PTE_R | PTE_W);
+ kvmmap(0x02000000, 0x02000000, 0x10000, PTE_R | PTE_W); // CLINT
+ kvmmap(KERNBASE, KERNBASE, PHYSTOP - KERNBASE, PTE_R | PTE_W | PTE_X);
+ }
+
+ void kvminithart()
+ {
+ w_satp(MAKE_SATP(kernel_pagetable));
+ sfence_vma();
+ Drivers::uart_puts("[Boot] MMU ENABLED! Using virtual addresses.\n");
+ }
+
+ uint64 walkaddr(uint64 *pagetable, uint64 va)
+ {
+ uint64 *pte;
+ uint64 pa;
+
+ if (va >= MAXVA)
+ return 0;
+
+ pte = walk(pagetable, va, 0);
+ if (pte == 0 || (*pte & PTE_V) == 0 || (*pte & PTE_U) == 0)
+ return 0;
+
+ pa = PTE2PA(*pte);
+ return pa;
}
} \ No newline at end of file
generated by cgit v1.2.3 (git 2.46.0) at 2026-10-29 14:38:18 +0000