path: root/lume-legacy/src/kernel/mm/vm.cc

// SPDX-License-Identifier: GPL-2.0-or-later
/*
 * Copyright (C) 2026 Richard QIn
 */
#include "kernel/mm.h"
#include "kernel/pmm.h"
#include "kernel/riscv.h"
#include "common/types.h"
#include "lib/string.h"
#include "drivers/uart.h"
#include "kernel/fdt.h"

extern "C"
{
    extern char kernel_end[];
    extern char trampoline[];
}

namespace VM
{

    uint64 *kernel_pagetable;

    static void page_ref_inc(uint64 pa)
    {
        struct Page *p = PMM::pa_to_page(pa);
        if (p)
        {
            __atomic_add_fetch(&p->refcnt, 1, __ATOMIC_RELAXED);
        }
    }

    static bool page_ref_dec(uint64 pa)
    {
        struct Page *p = PMM::pa_to_page(pa);
        if (p)
        {
            if (__atomic_sub_fetch(&p->refcnt, 1, __ATOMIC_RELEASE) == 0)
            {
                return true;
            }
        }
        return false;
    }

    static uint64 *walk(uint64 *pagetable, uint64 va, int alloc)
    {
        if (va >= MAXVA)
            return nullptr;

        for (int level = 2; level > 0; level--)
        {
            uint64 *pte = &pagetable[PX(level, va)];
            if (*pte & PTE_V)
            {
                pagetable = (uint64 *)PTE2PA(*pte);
            }
            else
            {
                if (!alloc || (pagetable = (uint64 *)PMM::alloc_page()) == nullptr)
                    return nullptr;

                memset(pagetable, 0, PGSIZE);
                *pte = PA2PTE(pagetable) | PTE_V;
            }
        }
        return &pagetable[PX(0, va)];
    }

    int mappages(uint64 *pagetable, uint64 va, uint64 size, uint64 pa, int perm)
    {
        uint64 a, last;
        uint64 *pte;

        if (size == 0)
            return -1;

        a = PGROUNDDOWN(va);
        last = PGROUNDDOWN(va + size - 1);

        for (;;)
        {
            if ((pte = walk(pagetable, a, 1)) == nullptr)
                return -1;

            if (*pte & PTE_V)
            {
                return -1;
            }

            *pte = PA2PTE(pa) | perm | PTE_V;

            if (a == last)
                break;
            a += PGSIZE;
            pa += PGSIZE;
        }
        return 0;
    }

    uint64 *uvmcreate()
    {
        uint64 *pagetable = (uint64 *)PMM::alloc_page();
        if (pagetable)
        {
            memset(pagetable, 0, PGSIZE);

            if (mappages(pagetable, TRAMPOLINE, PGSIZE, (uint64)trampoline, PTE_R | PTE_X) < 0)
            {
                PMM::free_page(pagetable);
                return nullptr;
            }
        }
        return pagetable;
    }

    void uvminit(uint64 *pagetable, uchar *src, uint64 sz)
    {
        char *mem;

        if (sz >= PGSIZE)
        {
            Drivers::uart_puts("uvminit: initcode too big!\n");
            return;
        }

        mem = (char *)PMM::alloc_page();
        memset(mem, 0, PGSIZE);

        mappages(pagetable, 0, PGSIZE, (uint64)mem, PTE_W | PTE_R | PTE_X | PTE_U);
        memmove(mem, src, sz);
    }

    uint64 uvmalloc(uint64 *pagetable, uint64 oldsz, uint64 newsz, int xperm)
    {
        char *mem;
        uint64 a;

        if (newsz < oldsz)
            return oldsz;

        oldsz = PGROUNDUP(oldsz);
        for (a = oldsz; a < newsz; a += PGSIZE)
        {
            mem = (char *)PMM::alloc_page();
            if (mem == nullptr)
            {
                uvmdealloc(pagetable, a, oldsz);
                return 0;
            }
            memset(mem, 0, PGSIZE);
            if (mappages(pagetable, a, PGSIZE, (uint64)mem, PTE_R | PTE_U | xperm) != 0)
            {
                PMM::free_page(mem);
                uvmdealloc(pagetable, a, oldsz);
                return 0;
            }
        }
        return newsz;
    }

    uint64 uvmdealloc(uint64 *pagetable, uint64 oldsz, uint64 newsz)
    {
        if (newsz >= oldsz)
            return oldsz;

        if (PGROUNDUP(newsz) < PGROUNDUP(oldsz))
        {
            int npages = (PGROUNDUP(oldsz) - PGROUNDUP(newsz)) / PGSIZE;
            uvmunmap(pagetable, PGROUNDUP(newsz), npages, 1);
        }

        return newsz;
    }

    void uvmunmap(uint64 *pagetable, uint64 va, uint64 npages, int do_free)
    {
        uint64 a;
        uint64 *pte;

        if ((va % PGSIZE) != 0)
            return;

        for (a = va; a < va + npages * PGSIZE; a += PGSIZE)
        {
            if ((pte = walk(pagetable, a, 0)) == nullptr)
                continue;

            if ((*pte & PTE_V) == 0)
                continue;

            if ((PTE_FLAGS(*pte) & (PTE_R | PTE_W | PTE_X)) == 0)
            {
                continue;
            }

            if (do_free)
            {
                uint64 pa = PTE2PA(*pte);
                if (page_ref_dec(pa))
                {
                    PMM::free_page((void *)pa);
                }
            }
            *pte = 0;
        }
    }

    static void freewalk(uint64 *pagetable)
    {
        for (int i = 0; i < 512; i++)
        {
            uint64 pte = pagetable[i];
            if ((pte & PTE_V) && (pte & (PTE_R | PTE_W | PTE_X)) == 0)
            {
                uint64 *child = (uint64 *)PTE2PA(pte);
                freewalk(child);
                pagetable[i] = 0;
            }
        }
        PMM::free_page(pagetable);
    }

    void uvmfree(uint64 *pagetable, uint64 sz)
    {
        if (sz > 0)
            uvmunmap(pagetable, 0, PGROUNDUP(sz) / PGSIZE, 1);
        freewalk(pagetable);
    }

    int uvmcopy(uint64 *oldpt, uint64 *newpt, uint64 sz)
    {
        uint64 *pte;
        uint64 pa, i;
        uint flags;

        for (i = 0; i < sz; i += PGSIZE)
        {
            if ((pte = walk(oldpt, i, 0)) == nullptr)
                continue;
            if ((*pte & PTE_V) == 0)
                continue;

            pa = PTE2PA(*pte);
            flags = PTE_FLAGS(*pte);

            if (flags & PTE_W)
            {
                flags &= ~PTE_W;
                flags |= PTE_COW;
                *pte = PA2PTE(pa) | flags;
            }

            if (mappages(newpt, i, PGSIZE, pa, flags) != 0)
            {
                return -1;
            }

            page_ref_inc(pa);
        }

        sfence_vma();
        return 0;
    }

    int handle_cow_fault(uint64 *pagetable, uint64 va);

    int copyout(uint64 *pagetable, uint64 dstva, char *src, uint64 len)
    {
        uint64 n, va0, pa0;
        uint64 *pte;

        while (len > 0)
        {
            va0 = PGROUNDDOWN(dstva);
            pte = walk(pagetable, va0, 0);
            if (pte == nullptr || (*pte & PTE_V) == 0 || (*pte & PTE_U) == 0)
                return -1;

            // Note: Maybe a bug
            if (*pte & PTE_COW)
            {
                if (handle_cow_fault(pagetable, va0) < 0)
                    return -1;
                // Re-walk because handle_cow_fault replaced the page mapping
                pte = walk(pagetable, va0, 0);
                if (pte == nullptr || (*pte & PTE_V) == 0 || (*pte & PTE_U) == 0)
                    return -1;
            }

            pa0 = PTE2PA(*pte);
            n = PGSIZE - (dstva - va0);
            if (n > len)
                n = len;

            memmove((void *)(pa0 + (dstva - va0)), src, n);

            len -= n;
            src += n;
            dstva = va0 + PGSIZE;
        }
        return 0;
    }

    int copyin(uint64 *pagetable, char *dst, uint64 srcva, uint64 len)
    {
        uint64 n, va0, pa0;
        uint64 *pte;

        while (len > 0)
        {
            va0 = PGROUNDDOWN(srcva);
            pte = walk(pagetable, va0, 0);
            if (pte == nullptr || (*pte & PTE_V) == 0 || (*pte & PTE_U) == 0)
                return -1;

            pa0 = PTE2PA(*pte);
            n = PGSIZE - (srcva - va0);
            if (n > len)
                n = len;

            memmove(dst, (void *)(pa0 + (srcva - va0)), n);

            len -= n;
            dst += n;
            srcva = va0 + PGSIZE;
        }
        return 0;
    }

    int copyinstr(uint64 *pagetable, char *dst, uint64 srcva, uint64 max)
    {
        uint64 n, va0, pa0;
        uint64 *pte;
        int got_null = 0;
        uint64 len = 0;

        while (len < max)
        {
            va0 = PGROUNDDOWN(srcva);
            pte = walk(pagetable, va0, 0);

            if (pte == nullptr || (*pte & PTE_V) == 0 || (*pte & PTE_U) == 0)
                return -1;

            pa0 = PTE2PA(*pte);

            n = PGSIZE - (srcva - va0);
            if (n > max - len)
                n = max - len;

            char *p = (char *)(pa0 + (srcva - va0));
            while (n > 0)
            {
                if (*p == '\0')
                {
                    *dst = '\0';
                    got_null = 1;
                    break;
                }
                *dst = *p;
                n--;
                srcva++;
                dst++;
                p++;
                len++;
            }
            if (got_null)
                break;
        }

        if (got_null)
            return 0;

        return -1;
    }

    int handle_cow_fault(uint64 *pagetable, uint64 va)
    {
        if (va >= MAXVA)
            return -1;

        uint64 *pte = walk(pagetable, va, 0);
        if (pte == nullptr || (*pte & PTE_V) == 0 || (*pte & PTE_U) == 0)
            return -1;

        if ((*pte & PTE_COW) == 0)
            return -1;

        uint64 pa = PTE2PA(*pte);
        uint64 flags = PTE_FLAGS(*pte);

        char *mem = (char *)PMM::alloc_page();
        if (mem == nullptr)
            return -1;

        memmove(mem, (char *)pa, PGSIZE);

        uint64 new_flags = (flags | PTE_W) & ~PTE_COW;
        *pte = PA2PTE((uint64)mem) | new_flags | PTE_V;

        if (page_ref_dec(pa))
        {
            PMM::free_page((void *)pa);
        }

        sfence_vma();
        return 0;
    }

    static void kvmmap(uint64 va, uint64 pa, uint64 sz, int perm)
    {
        if (mappages(kernel_pagetable, va, sz, pa, perm) != 0)
        {
            Drivers::uart_puts("kvmmap: failed\n");
            while (1)
                ;
        }
    }

    void kvminit()
    {
        kernel_pagetable = uvmcreate();

        // 1. Map all device MMIO regions discovered by FDT
        for (int i = 0; i < g_devices.mmio_count; i++)
        {
            uint64 base = g_devices.mmio_regions[i].base;
            uint64 size = g_devices.mmio_regions[i].size;
            kvmmap(base, base, size, PTE_R | PTE_W);
        }

        // 2. Map Kernel Memory (PHYSTOP is now dynamic based on FDT)
        kvmmap(KERNBASE, KERNBASE, PHYSTOP - KERNBASE, PTE_R | PTE_W | PTE_X);
    }

    void kvminithart()
    {
        w_satp(MAKE_SATP(kernel_pagetable));
        sfence_vma();
        Drivers::uart_puts("[Boot] MMU ENABLED! Using virtual addresses.\n");
    }

    uint64 walkaddr(uint64 *pagetable, uint64 va)
    {
        uint64 *pte;
        uint64 pa;

        if (va >= MAXVA)
            return 0;

        pte = walk(pagetable, va, 0);
        if (pte == 0 || (*pte & PTE_V) == 0 || (*pte & PTE_U) == 0)
            return 0;

        pa = PTE2PA(*pte);
        return pa;
    }
}