diff options
Diffstat (limited to 'kernel/sync')
| -rw-r--r-- | kernel/sync/cpu_state.h | 35 | ||||
| -rw-r--r-- | kernel/sync/mutex.cc | 156 | ||||
| -rw-r--r-- | kernel/sync/spinlock.cc | 120 | ||||
| -rw-r--r-- | kernel/sync/spinlock.h | 61 |
4 files changed, 372 insertions, 0 deletions
diff --git a/kernel/sync/cpu_state.h b/kernel/sync/cpu_state.h new file mode 100644 index 0000000..1f7dde0 --- /dev/null +++ b/kernel/sync/cpu_state.h @@ -0,0 +1,35 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* Copyright (C) 2026 Richard Qin */ + +#pragma once + +/* + * Per-CPU Interrupt Nesting Tracker + * + * Tracks spinlock nesting depth on each CPU to ensure interrupts + * are only restored when the last spinlock is released. + * + * BSS-segment global array; zero-initialized by entry.S BSS clear. + * Reference: docs/specs/spinlock.md §3.1 + */ + +#include <lume/types.h> +#include <lume/config.h> + +namespace sync { + +struct CpuSyncState { + int lock_depth; // Number of spinlocks held by this CPU + bool intr_was_on; // Were interrupts enabled before the first lock? +}; + +inline constexpr uint64 kInvalidCpuId = ~0ULL; + +// BSS-segment global array, zero-initialized by entry.S +extern CpuSyncState g_cpu_sync_states[kMaxCpus]; + +// Internal API — called only by Spinlock::acquire/release +void push_intr(); +void pop_intr(); + +} // namespace sync diff --git a/kernel/sync/mutex.cc b/kernel/sync/mutex.cc new file mode 100644 index 0000000..c4d3e9d --- /dev/null +++ b/kernel/sync/mutex.cc @@ -0,0 +1,156 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* Copyright (C) 2026 Richard Qin */ + +/* + * Mutex — Linux-style sleeping mutex (no PI). + * + * Implementation notes (mirroring kernel/locking/mutex.c, simplified): + * + * - Fast path (uncontended): wait_lock_ -> see owner_==null -> claim -> unlock. + * - Slow path (contended): enqueue MutexWaiter on stack, sleep, race for + * ownership after each wake. unlock() wakes only the head waiter. + * - No optimistic spin, no handoff anti-starvation, no interruptible wait. + * These belong to follow-up work (see mutex.md §5). + * + * Reference: docs/specs/mutex.md + */ + +#include <lume/mutex.h> +#include <lume/task.h> +#include <lume/sched.h> +#include <lume/klog.h> +#include <arch/cpu.h> + +namespace { + +struct MutexWaiter { + list_node link; + TaskControlBlock* task; +}; + +} // namespace + +/* ------------------------------------------------------------------ */ +/* init */ +/* ------------------------------------------------------------------ */ + +void Mutex::init(const char* name) +{ + owner_ = nullptr; + name_ = name; + wait_lock_.init(name != nullptr ? name : "mutex"); + wait_list_.init(); +} + +/* ------------------------------------------------------------------ */ +/* try_lock — non-blocking, safe from any context */ +/* ------------------------------------------------------------------ */ + +bool Mutex::try_lock() +{ + LockGuard g(wait_lock_); + if (owner_ != nullptr) { + return false; + } + owner_ = current_task(); + return true; +} + +/* ------------------------------------------------------------------ */ +/* lock — blocking, must be called in schedulable context */ +/* ------------------------------------------------------------------ */ + +void Mutex::lock() +{ + TaskControlBlock* curr = current_task(); + + if (curr == nullptr) { + kernel_panic("Mutex::lock: no current task"); + } + if (curr->preempt_count > 0) { + kernel_panic("Mutex::lock: preemption disabled (holding spinlock?)"); + } + + wait_lock_.acquire(); + + if (owner_ == nullptr) { + owner_ = curr; + wait_lock_.release(); + return; + } + + if (owner_ == curr) { + wait_lock_.release(); + kernel_panic("Mutex::lock: recursive acquire by same task"); + } + + /* Slow path: enqueue ourselves and sleep until we are head + lock free. */ + MutexWaiter waiter; + waiter.link.init(); + waiter.task = curr; + list_add_tail(&waiter.link, &wait_list_); + + for (;;) { + /* Set state inside wait_lock_ so unlock-side sees a coherent view: + * either we are on the list AND Sleeping (will be woken), or we + * already took ownership above (not in this loop). */ + curr->state = TaskState::Sleeping; + wait_lock_.release(); + + schedule(); + + wait_lock_.acquire(); + + if (owner_ == nullptr && wait_list_.next == &waiter.link) { + owner_ = curr; + list_del_init(&waiter.link); + wait_lock_.release(); + return; + } + /* Spurious wake / lost race — keep waiting. */ + } +} + +/* ------------------------------------------------------------------ */ +/* unlock */ +/* ------------------------------------------------------------------ */ + +void Mutex::unlock() +{ + TaskControlBlock* curr = current_task(); + + wait_lock_.acquire(); + + if (owner_ != curr) { + wait_lock_.release(); + kernel_panic("Mutex::unlock: not the owner"); + } + + owner_ = nullptr; + + if (!wait_list_.is_empty()) { + list_node* first = wait_list_.next; + MutexWaiter* w = list_entry<MutexWaiter, &MutexWaiter::link>(first); + TaskControlBlock* victim = w->task; + + /* Race-tolerant wake: only enqueue if still sleeping. If the + * waiter is already Runnable (e.g. it was already woken by a + * timer-driven schedule reentry and didn't yet retry the lock), + * sched_enqueue would double-link it. */ + if (victim->state == TaskState::Sleeping) { + victim->state = TaskState::Runnable; + sched_enqueue(victim); + } + } + + wait_lock_.release(); +} + +/* ------------------------------------------------------------------ */ +/* Debug accessors */ +/* ------------------------------------------------------------------ */ + +bool Mutex::is_owned_by_current() const +{ + return owner_ == current_task(); +} diff --git a/kernel/sync/spinlock.cc b/kernel/sync/spinlock.cc new file mode 100644 index 0000000..f649fe7 --- /dev/null +++ b/kernel/sync/spinlock.cc @@ -0,0 +1,120 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* Copyright (C) 2026 Richard Qin */ + +/* + * Spinlock Implementation + * + * TTAS (Test-and-Test-and-Set) spinlock with: + * - Acquire: __ATOMIC_ACQUIRE memory ordering + * - Release: __ATOMIC_RELEASE memory ordering + * - Automatic interrupt disable/enable via push_intr/pop_intr + * - Dead-lock and invalid-release detection via kernel_panic + * + * Reference: docs/specs/spinlock.md + */ + +#include "spinlock.h" +#include "cpu_state.h" +#include <arch/cpu.h> +#include <lume/klog.h> +#include <lume/task.h> + +// ============================================================ +// Per-CPU sync state (BSS zero-initialized) +// ============================================================ +namespace sync { + +CpuSyncState g_cpu_sync_states[kMaxCpus]; + +void push_intr() { + bool was_on = arch::cpu::intr_enabled(); + arch::cpu::intr_off(); + + auto& state = g_cpu_sync_states[arch::cpu::id()]; + if (state.lock_depth == 0) { + state.intr_was_on = was_on; + } + state.lock_depth++; +} + +void pop_intr() { + auto& state = g_cpu_sync_states[arch::cpu::id()]; + state.lock_depth--; + + if (state.lock_depth == 0 && state.intr_was_on) { + arch::cpu::intr_on(); + } +} + +} // namespace sync + +// ============================================================ +// Spinlock methods +// ============================================================ + +void Spinlock::init(const char* name) { + name_ = name; + locked_.store(0, __ATOMIC_RELAXED); + cpu_id_ = sync::kInvalidCpuId; +} + +void Spinlock::acquire() { +#ifdef DEBUG + // Optional: detect interrupt-state leakage + if (arch::cpu::intr_enabled()) { + auto& state = sync::g_cpu_sync_states[arch::cpu::id()]; + if (state.lock_depth > 0) { + kernel_panic("spinlock: interrupts enabled while holding lock"); + } + } +#endif + + // Step 1: disable interrupts, track nesting depth + sync::push_intr(); + + // Step 2: dead-lock detection (same CPU re-acquiring) + if (is_held_by_current_cpu()) { + kernel_panic("spinlock: double lock", name_); + } + + // Step 3: TTAS spin loop + while (true) { + // Outer: spin on local cache copy (no bus traffic) + while (locked_.load(__ATOMIC_RELAXED)) + ; + // Inner: attempt atomic exchange with acquire semantics + if (locked_.exchange(1, __ATOMIC_ACQUIRE) == 0) + break; + } + + // Step 4: record owner + cpu_id_ = arch::cpu::id(); + + // Step 5: bump preempt_count — prevents schedule() from preempting + TaskControlBlock* curr = current_task(); + if (curr) curr->preempt_count++; +} + +void Spinlock::release() { + // Step 1: verify caller is the holder + if (!is_held_by_current_cpu()) { + kernel_panic("spinlock: invalid release", name_); + } + + // Step 2: clear owner + cpu_id_ = sync::kInvalidCpuId; + + // Step 3: release the lock with release semantics + locked_.store(0, __ATOMIC_RELEASE); + + // Step 4: drop preempt_count + TaskControlBlock* curr = current_task(); + if (curr) curr->preempt_count--; + + // Step 5: restore interrupt state if this was the last lock + sync::pop_intr(); +} + +bool Spinlock::is_held_by_current_cpu() const { + return cpu_id_ == arch::cpu::id(); +} diff --git a/kernel/sync/spinlock.h b/kernel/sync/spinlock.h new file mode 100644 index 0000000..a1369d1 --- /dev/null +++ b/kernel/sync/spinlock.h @@ -0,0 +1,61 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* Copyright (C) 2026 Richard Qin */ + +#pragma once + +/* + * Spinlock & LockGuard + * + * Provides mutual exclusion for SMP kernels using TTAS + * (Test-and-Test-and-Set) with acquire/release memory ordering. + * Interrupt disable/enable is managed automatically via push_intr/pop_intr. + * + * Reference: docs/specs/spinlock.md + */ + +#include <lume/atomic.h> +#include <lume/types.h> +#include "cpu_state.h" + +class Spinlock { +public: + // Default: BSS zero is safe. Call init() to give the lock a name. + Spinlock() = default; + + // Explicit initialization (for global objects or Placement New) + void init(const char* name); + + // Non-copyable, non-movable (lock identity is its address) + Spinlock(const Spinlock&) = delete; + Spinlock& operator=(const Spinlock&) = delete; + Spinlock(Spinlock&&) = delete; + Spinlock& operator=(Spinlock&&) = delete; + + void acquire(); + void release(); + bool is_held_by_current_cpu() const; + +private: + lume::atomic<uint32> locked_; // 0 = free, 1 = held + const char* name_ = "uninit"; // Lock name (for panic messages) + uint64 cpu_id_ = sync::kInvalidCpuId; // Holder CPU ID +}; + +class LockGuard { +public: + explicit LockGuard(Spinlock& lock) : lock_(lock) { + lock_.acquire(); + } + + ~LockGuard() { + lock_.release(); + } + + LockGuard(const LockGuard&) = delete; + LockGuard& operator=(const LockGuard&) = delete; + LockGuard(LockGuard&&) = delete; + LockGuard& operator=(LockGuard&&) = delete; + +private: + Spinlock& lock_; +}; |
