From afcea249d1ca9677ea6000bb6214320c5675702a Mon Sep 17 00:00:00 2001 From: Tahiru Agbanwa <130235676+squirelboy360@users.noreply.github.com> Date: Mon, 8 Jun 2026 21:59:58 +0200 Subject: [PATCH 01/36] arch/aarch64: scaffold a full backend that satisfies the shared arch surface Replace the 35-line aarch64 stub with a module layout that mirrors the x86_64 backend so the rest of the kernel's crate::arch::* surface resolves on aarch64. Every function/type/const the shared kernel imports through the arch facade now exists here with the matching signature; bodies are compilable scaffolds (no-ops / sane defaults) pending the real ARM port. Modules added under arch/aarch64/: cpu FP/SIMD enable, TPIDR_EL0 TLS (set/get_fs_base), DAIF interrupt-mask save/restore, DMB/yield fences, xstate save/restore placeholders, hypervisor detection. memory read/write_cr3 mapped onto TTBR0_EL1 (+ TLB maintenance). gdt nominal USER_CS/USER_DS selector constants (no segmentation). idt VBAR_EL1 exception-vector install entry points; re-exports the cross-arch InterruptFrame. apic GIC + generic-timer scaffold: eoi, init_bsp/ap, finish_xapic_init, local_apic_id (from MPIDR_EL1), send_resched_ipi, and the vsync cadence accessors. syscall SVC fast-path scaffold: per-CPU user-GPR scratch with the UserGprSnapshot type and user_rsp/rip/r9/rbp/user_gprs accessors, set_active_stack_top, init/init_ap. smp per-CPU table + this_cpu()/current_cpu_id() via MPIDR_EL1, CPU_COUNT, broadcast_resched_ipi (single-core scaffold; PSCI CPU_ON wake left as a TODO). acpi RSDP/MADT lookup + PSCI SYSTEM_OFF shutdown placeholders. interrupts disable_pic no-op (no legacy PIC on aarch64). mod early_init/ap_init/smp_init/halt/halt_forever/enable/ disable_interrupts/rdtsc (CNTVCT_EL0) and AAPCS64 context_switch + task_entry naked trampolines. This is the scaffolding step of the ARM port: it does NOT boot. Real MMU page tables, the EL1 exception vectors, GIC/timer programming, the SVC entry path, and PSCI SMP bring-up are stubbed for follow-up work. --- kernel/src/arch/aarch64/acpi.rs | 29 +++++ kernel/src/arch/aarch64/apic.rs | 77 +++++++++++ kernel/src/arch/aarch64/cpu.rs | 99 ++++++++++++++ kernel/src/arch/aarch64/gdt.rs | 18 +++ kernel/src/arch/aarch64/idt.rs | 19 +++ kernel/src/arch/aarch64/interrupts.rs | 20 +++ kernel/src/arch/aarch64/memory.rs | 30 +++++ kernel/src/arch/aarch64/mod.rs | 180 ++++++++++++++++++++++---- kernel/src/arch/aarch64/smp.rs | 84 ++++++++++++ kernel/src/arch/aarch64/syscall.rs | 118 +++++++++++++++++ 10 files changed, 652 insertions(+), 22 deletions(-) create mode 100644 kernel/src/arch/aarch64/acpi.rs create mode 100644 kernel/src/arch/aarch64/apic.rs create mode 100644 kernel/src/arch/aarch64/cpu.rs create mode 100644 kernel/src/arch/aarch64/gdt.rs create mode 100644 kernel/src/arch/aarch64/idt.rs create mode 100644 kernel/src/arch/aarch64/interrupts.rs create mode 100644 kernel/src/arch/aarch64/memory.rs create mode 100644 kernel/src/arch/aarch64/smp.rs create mode 100644 kernel/src/arch/aarch64/syscall.rs diff --git a/kernel/src/arch/aarch64/acpi.rs b/kernel/src/arch/aarch64/acpi.rs new file mode 100644 index 0000000..a110ef4 --- /dev/null +++ b/kernel/src/arch/aarch64/acpi.rs @@ -0,0 +1,29 @@ +//! Firmware-table lookup scaffolding — aarch64. +//! +//! On aarch64 platforms hardware topology comes from either ACPI (server-class +//! machines) or a flattened device tree (the QEMU `virt` bring-up target). The +//! shared kernel does not call these on the non-x86 path yet; they exist for +//! parity so the real port has named entry points to fill in. + +/// Physical address of the RSDP, or 0 if unavailable. +/// +/// TODO(arm): use the Limine RSDP request (ACPI) or parse the DTB. +pub fn rsdp_address() -> u64 { + 0 +} + +/// Physical address of the MADT/GICC table, or 0 if not found. +/// +/// TODO(arm): walk XSDT for the MADT, or parse `/cpus` + `intc` from the DTB. +pub fn find_madt(_rsdp_phys: u64) -> u64 { + 0 +} + +/// Power off the machine. +/// +/// TODO(arm): issue a PSCI `SYSTEM_OFF` via SMC/HVC. For now, park. +pub fn shutdown() -> ! { + loop { + unsafe { core::arch::asm!("wfi", options(nomem, nostack)) }; + } +} diff --git a/kernel/src/arch/aarch64/apic.rs b/kernel/src/arch/aarch64/apic.rs new file mode 100644 index 0000000..6dae969 --- /dev/null +++ b/kernel/src/arch/aarch64/apic.rs @@ -0,0 +1,77 @@ +//! Interrupt-controller + tick/vsync scaffolding — aarch64 (GIC + generic timer). +//! +//! On x86 this is the local APIC; on aarch64 the equivalent responsibilities are +//! split between the GIC (CPU interface: EOI, IPIs, routing) and the ARM generic +//! timer (CNTP/CNTV, the periodic tick that drives the vsync cadence). These are +//! compilable stubs preserving the public surface the shared kernel calls; real +//! GICv2/GICv3 + CNTV_CTL programming comes later. + +use core::sync::atomic::{AtomicU32, Ordering}; + +/// Approximate timer ticks per millisecond (placeholder; real value is read +/// from `CNTFRQ_EL0` once the generic timer is programmed). +pub static APIC_TICKS_PER_MS: AtomicU32 = AtomicU32::new(62500); + +/// Requested vsync cadence in Hz (0 = off). +static VSYNC_HZ: AtomicU32 = AtomicU32::new(0); + +/// Initialise the interrupt controller + timer on the BSP. +/// +/// TODO(arm): probe + init the GIC distributor/CPU interface and program the +/// generic timer (CNTV_TVAL_EL0 / CNTV_CTL_EL0). +pub fn init_bsp() {} + +/// Finish any deferred MMIO mapping after `mm::init()`. +/// +/// On x86 this maps the xAPIC MMIO window; on aarch64 the GIC MMIO regions would +/// be mapped here. No-op stub. +pub fn finish_xapic_init() {} + +/// Per-AP interrupt-controller init. +/// +/// TODO(arm): enable this core's GIC redistributor/CPU interface. +pub fn init_ap() {} + +/// Signal End-Of-Interrupt to the controller. +/// +/// TODO(arm): write the interrupt id to GICC_EOIR / ICC_EOIR1_EL1. +pub fn eoi() {} + +/// Route the legacy serial/ExtINT line. +/// +/// x86-specific (LINT0 in ExtINT mode). No aarch64 equivalent — no-op. +pub fn configure_lint0_for_extint() {} + +/// Return this core's interrupt-controller id (x86 LAPIC-id analogue). +/// +/// Derived from MPIDR_EL1 affinity bits. +pub fn local_apic_id() -> u32 { + let v: u64; + unsafe { core::arch::asm!("mrs {}, mpidr_el1", out(reg) v, options(nomem, nostack)) }; + (v & 0x00FF_FFFF) as u32 +} + +/// Send a reschedule IPI to the given target core. +/// +/// TODO(arm): raise a GIC Software-Generated Interrupt (SGI) to `_target`. +pub fn send_resched_ipi(_target_lapic_id: u32) {} + +// ── Vsync cadence ─────────────────────────────────────────────────────────── + +/// Set the desired vsync cadence (Hz). 0 disables it. +pub fn set_vsync_hz(hz: u32) { + VSYNC_HZ.store(hz, Ordering::Release); +} + +/// Returns true when a vsync interval has elapsed since the last reset. +/// +/// TODO(arm): compare CNTVCT_EL0 against the last fired timestamp scaled by +/// CNTFRQ_EL0 / VSYNC_HZ. Always returns false in the scaffold. +pub fn vsync_due() -> bool { + false +} + +/// Reset the vsync interval reference timestamp. +/// +/// TODO(arm): snapshot CNTVCT_EL0. No-op stub. +pub fn reset_vsync_last_tsc() {} diff --git a/kernel/src/arch/aarch64/cpu.rs b/kernel/src/arch/aarch64/cpu.rs new file mode 100644 index 0000000..2bd7316 --- /dev/null +++ b/kernel/src/arch/aarch64/cpu.rs @@ -0,0 +1,99 @@ +//! CPU feature detection and setup — aarch64. +//! +//! Scaffolding stubs for the ARM port. Real implementations (FPU/SIMD enable +//! via CPACR_EL1, TLS via TPIDR_EL0, FP/SIMD context save/restore) come later. + +use core::arch::asm; + +/// Assert that this CPU supports all features OSCortex requires. +/// +/// On aarch64 the AArch64 base profile already mandates FP+AdvSIMD, so this is +/// currently a no-op placeholder. +pub fn assert_required_features() {} + +/// Enable FP / AdvSIMD (and later SVE) for the AI inference engine. +/// +/// TODO(arm): clear CPACR_EL1.FPEN trap bits so EL0/EL1 may use FP/SIMD. +pub fn enable_fpu_simd() {} + +/// SYSCALL path init — see syscall module. No-op on aarch64 (SVC is always on). +pub fn enable_syscall() {} + +/// Returns true when running under a hypervisor. +/// +/// TODO(arm): inspect MIDR_EL1 / device-tree. Conservatively returns true since +/// the bring-up target is QEMU `virt`. +pub fn running_under_hypervisor() -> bool { + true +} + +/// True when running on a QEMU-like hypervisor. +/// +/// TODO(arm): real detection. The bring-up target is QEMU `virt`, so assume true. +pub fn is_qemu_like_hypervisor() -> bool { + true +} + +/// Set userspace TLS base for the current CPU context (x86 FS-base analogue). +/// +/// On aarch64 user TLS lives in `TPIDR_EL0`. +pub fn set_fs_base(fs_base: u64) { + unsafe { asm!("msr tpidr_el0, {}", in(reg) fs_base, options(nomem, nostack)) }; +} + +/// Read userspace TLS base for the current CPU context. +pub fn get_fs_base() -> u64 { + let v: u64; + unsafe { asm!("mrs {}, tpidr_el0", out(reg) v, options(nomem, nostack)) }; + v +} + +/// Save the current FP/SIMD (and later SVE) register state to `ptr`. +/// +/// # Safety +/// `ptr` must point to a sufficiently large, suitably aligned buffer. +/// +/// TODO(arm): implement via `st1 {v0-v31}` + FPSR/FPCR. Currently a no-op. +#[inline] +pub unsafe fn save_xstate_to(_ptr: *mut u8) {} + +/// Restore the FP/SIMD (and later SVE) register state from `ptr`. +/// +/// # Safety +/// `ptr` must point to a buffer previously written by [`save_xstate_to`]. +/// +/// TODO(arm): implement via `ld1 {v0-v31}` + FPSR/FPCR. Currently a no-op. +#[inline] +pub unsafe fn restore_xstate_from(_ptr: *const u8) {} + +/// Full memory fence (Data Memory Barrier, system scope). +#[inline(always)] +pub fn memory_fence() { + unsafe { asm!("dmb sy", options(nomem, nostack)) }; +} + +/// Hint the CPU during poll loops. +#[inline(always)] +pub fn spin_pause() { + unsafe { asm!("yield", options(nomem, nostack, preserves_flags)) }; +} + +/// Save DAIF (interrupt mask) state and disable IRQ/FIQ; returns the prior DAIF. +#[inline(always)] +pub fn interrupts_save_and_disable() -> u64 { + let daif: u64; + unsafe { + asm!("mrs {}, daif", out(reg) daif, options(nomem, nostack)); + asm!("msr daifset, #0b0011", options(nomem, nostack)); // mask IRQ+FIQ + } + daif +} + +/// Restore interrupt-enable state from [`interrupts_save_and_disable`]. +#[inline(always)] +pub fn interrupts_restore(daif: u64) { + // DAIF.I is bit 7. If it was clear (IRQs were enabled), re-enable them. + if daif & (1 << 7) == 0 { + unsafe { asm!("msr daifclr, #0b0010", options(nomem, nostack)) }; + } +} diff --git a/kernel/src/arch/aarch64/gdt.rs b/kernel/src/arch/aarch64/gdt.rs new file mode 100644 index 0000000..803f77a --- /dev/null +++ b/kernel/src/arch/aarch64/gdt.rs @@ -0,0 +1,18 @@ +//! Segment-selector compatibility shim — aarch64. +//! +//! AArch64 has no GDT/segmentation. The shared kernel still references the +//! `USER_CS` / `USER_DS` selector constants when building an initial user +//! context (the values are stored into the saved register frame and ignored by +//! the aarch64 exception-return path, which uses SPSR_EL1 instead). We keep the +//! symbols so cross-arch process-setup code compiles; the values are nominal. + +/// User code "selector" placeholder (no segmentation on aarch64). +pub const USER_CS: u16 = 0; +/// User data "selector" placeholder (no segmentation on aarch64). +pub const USER_DS: u16 = 0; + +/// Load the (nonexistent) descriptor table on the BSP. No-op on aarch64. +pub fn init() {} + +/// Per-AP descriptor-table load. No-op on aarch64. +pub fn init_ap(_cpu_idx: u32) {} diff --git a/kernel/src/arch/aarch64/idt.rs b/kernel/src/arch/aarch64/idt.rs new file mode 100644 index 0000000..7a63dba --- /dev/null +++ b/kernel/src/arch/aarch64/idt.rs @@ -0,0 +1,19 @@ +//! Exception-vector scaffolding — aarch64 (EL1 vector table / VBAR_EL1). +//! +//! AArch64 has no IDT; synchronous/IRQ/FIQ/SError exceptions are dispatched +//! through a 16-entry, 2 KiB-aligned vector table pointed to by VBAR_EL1. This +//! module names the table-install entry points the way the x86 `idt` module +//! does (`init`/`load`) and re-exports the cross-arch [`InterruptFrame`] saved +//! by the (future) vector handlers. + +pub use super::InterruptFrame; + +/// Install the EL1 exception vector table on the BSP. +/// +/// TODO(arm): build the vector table and write its base to VBAR_EL1. +pub fn init() {} + +/// Re-load the (already-built) vector table on an AP. +/// +/// TODO(arm): write the shared vector-table base to this core's VBAR_EL1. +pub fn load() {} diff --git a/kernel/src/arch/aarch64/interrupts.rs b/kernel/src/arch/aarch64/interrupts.rs new file mode 100644 index 0000000..9fd4d6a --- /dev/null +++ b/kernel/src/arch/aarch64/interrupts.rs @@ -0,0 +1,20 @@ +//! Interrupt helpers — aarch64. +//! +//! x86 has a legacy 8259 PIC to disable; aarch64 has no such thing (the GIC is +//! configured in `apic`/GIC init), so `disable_pic` is a no-op kept for parity +//! with the x86 early-init sequence. + +use core::arch::asm; + +/// No-op on aarch64 — there is no legacy PIC to disable. +/// +/// # Safety +/// Trivially safe; kept `unsafe` to match the x86 signature called from +/// `early_init`. +pub unsafe fn disable_pic() {} + +/// Data Memory Barrier (system scope). +#[inline] +pub fn memory_fence() { + unsafe { asm!("dmb sy", options(nomem, nostack)) }; +} diff --git a/kernel/src/arch/aarch64/memory.rs b/kernel/src/arch/aarch64/memory.rs new file mode 100644 index 0000000..a435877 --- /dev/null +++ b/kernel/src/arch/aarch64/memory.rs @@ -0,0 +1,30 @@ +//! aarch64 memory helpers (translation-table base, etc.) +//! +//! The x86 `read_cr3`/`write_cr3` map to the EL1 translation-table base +//! registers `TTBR0_EL1` (user/low half) and `TTBR1_EL1` (kernel/high half). +//! The shared kernel only needs a single "current address-space root" value; +//! we expose `TTBR0_EL1` under the historical `read_cr3` name. + +use core::arch::asm; + +/// Read the current low-half translation-table base (x86 CR3 analogue). +pub fn read_cr3() -> u64 { + let v: u64; + unsafe { asm!("mrs {}, ttbr0_el1", out(reg) v, options(nomem, nostack)) }; + v +} + +/// Write the low-half translation-table base and synchronise the TLB. +pub fn write_cr3(val: u64) { + unsafe { + asm!( + "msr ttbr0_el1, {}", + "dsb ish", + "tlbi vmalle1is", + "dsb ish", + "isb", + in(reg) val, + options(nostack), + ); + } +} diff --git a/kernel/src/arch/aarch64/mod.rs b/kernel/src/arch/aarch64/mod.rs index 384f68e..91d39b1 100644 --- a/kernel/src/arch/aarch64/mod.rs +++ b/kernel/src/arch/aarch64/mod.rs @@ -1,35 +1,171 @@ -//! aarch64 architecture stub. +//! aarch64 architecture backend (scaffold). +//! +//! This is the ARM port's scaffolding layer: every module and public function +//! the shared kernel reaches through `crate::arch::*` exists here with a correct +//! signature, but most bodies are stubs (`unimplemented!()`-free no-ops or sane +//! defaults) pending real implementations. The intent is that the kernel +//! *compiles and links* for `aarch64-unknown-none`; it does NOT yet boot. +//! +//! Mapping from the x86_64 backend to aarch64 primitives: +//! * GDT/segmentation → none (flat); `gdt` keeps nominal selector constants. +//! * IDT → EL1 exception vector table (VBAR_EL1), in `idt`. +//! * Local APIC → GIC CPU interface + ARM generic timer, in `apic`. +//! * SYSCALL/SYSRET → `SVC` trap to the EL1 sync vector, in `syscall`. +//! * CR3 → TTBR0_EL1/TTBR1_EL1, in `memory`. +//! * SMP wake → PSCI `CPU_ON`, reschedule via GIC SGIs, in `smp`. -/// Placeholder interrupt frame — no x86 IDT on aarch64. -/// Used by cortex subsystem for cross-arch compatibility. -/// `ip` mirrors x86_64 InterruptFrame field name for generic kernel code. +pub mod gdt; +pub mod idt; +pub mod interrupts; +pub mod memory; +pub mod apic; +pub mod cpu; +pub mod syscall; +pub mod acpi; +pub mod smp; + +use core::arch::asm; + +/// Cross-arch interrupt/exception frame. +/// +/// `ip` mirrors the x86_64 `InterruptFrame` field name so generic kernel code +/// (the cortex observer) stays architecture-neutral. On aarch64 these fields map +/// to the exception-syndrome registers saved by the EL1 vector handler. #[derive(Debug)] #[repr(C)] pub struct InterruptFrame { - pub ip: u64, // Instruction pointer (= ELR_EL1) - pub spsr: u64, // Saved Program Status Register - pub esr: u64, // Exception Syndrome Register - pub far: u64, // Fault Address Register + /// Instruction pointer that faulted (= ELR_EL1). + pub ip: u64, + /// Saved Program Status Register (= SPSR_EL1). + pub spsr: u64, + /// Exception Syndrome Register (= ESR_EL1). + pub esr: u64, + /// Fault Address Register (= FAR_EL1). + pub far: u64, } -/// No-op stub — context switching not yet implemented on aarch64. -pub unsafe extern "C" fn context_switch(_old_sp: *mut u64, _new_sp: u64) {} -/// No-op stub — task entry trampoline not yet implemented on aarch64. -pub unsafe extern "C" fn task_entry() { loop { core::hint::spin_loop() } } - +/// Perform early arch initialisation (BSP path). +/// +/// Mirrors the x86_64 sequence; most steps are scaffold no-ops for now. pub fn early_init() { - crate::logger::early_print("[arch::aarch64] early_init stub (Milestone 4b)\r\n"); + crate::logger::early_print("[ARCH] assert_required_features\r\n"); + cpu::assert_required_features(); + crate::logger::early_print("[ARCH] idt::init (VBAR_EL1)\r\n"); + idt::init(); + crate::logger::early_print("[ARCH] apic::init_bsp (GIC + timer)\r\n"); + apic::init_bsp(); + crate::logger::early_print("[ARCH] cpu::enable_fpu_simd\r\n"); + cpu::enable_fpu_simd(); + crate::logger::early_print("[ARCH] syscall::init (SVC)\r\n"); + syscall::init(); + crate::logger::early_print("[ARCH] aarch64 early_init complete (scaffold)\r\n"); +} + +/// Per-AP init — called once each Application Processor is brought up. +pub fn ap_init(cpu_idx: u32) { + gdt::init_ap(cpu_idx); + idt::load(); + apic::init_ap(); + cpu::enable_fpu_simd(); + syscall::init_ap(cpu_idx); +} + +/// Wake all APs and bring SMP online (single-core scaffold for now). +pub fn smp_init(resp: Option<&'static limine::request::MpResponse>) { + smp::init(resp); +} + +/// Halt the current core until an interrupt (or event) arrives. +#[inline(always)] +pub fn halt() { + unsafe { asm!("wfi", options(nomem, nostack)) }; +} + +/// Permanently halt this core (used in panic). +pub fn halt_forever() -> ! { + loop { + unsafe { asm!("msr daifset, #0b0011", "wfi", options(nomem, nostack)) }; + } } -pub fn ap_init() { - crate::logger::early_print("[arch::aarch64] ap_init stub\r\n"); + +/// Enable IRQ/FIQ delivery on this core. +#[inline(always)] +pub fn enable_interrupts() { + unsafe { asm!("msr daifclr, #0b0011", options(nomem, nostack)) }; } -pub fn smp_init(_resp: Option<&'static limine::request::MpResponse>) { /* stub */ } -pub fn halt() { loop { core::hint::spin_loop() } } -pub fn halt_forever() -> ! { loop { core::hint::spin_loop() } } -pub fn enable_interrupts() {} -pub fn disable_interrupts() {} + +/// Disable IRQ/FIQ delivery on this core. +#[inline(always)] +pub fn disable_interrupts() { + unsafe { asm!("msr daifset, #0b0011", options(nomem, nostack)) }; +} + +/// Read a free-running cycle/time counter (x86 `rdtsc` analogue). +/// +/// Uses the virtual count register `CNTVCT_EL0` from the ARM generic timer. +#[inline(always)] pub fn rdtsc() -> u64 { let cnt: u64; - unsafe { core::arch::asm!("mrs {}, cntvct_el0", out(reg) cnt) }; + unsafe { asm!("mrs {}, cntvct_el0", out(reg) cnt, options(nomem, nostack)) }; cnt } + +// ── Context switch ──────────────────────────────────────────────────────────── + +/// Switch from the current task to a new one. +/// +/// Saves the AAPCS64 callee-saved registers (x19–x30, including the frame +/// pointer x29 and link register x30) of the old task onto its kernel stack, +/// stores SP into `*old_sp`, loads `new_sp`, restores the new task's saved +/// registers, and returns into it. +/// +/// # Safety +/// `old_sp` must point to the `kernel_sp` field of the currently running task; +/// `new_sp` must be a kernel stack prepared by `spawn_kernel_task` or a prior +/// call to this function. Must be called with interrupts disabled. +#[unsafe(naked)] +pub unsafe extern "C" fn context_switch(old_sp: *mut u64, new_sp: u64) { + // AAPCS64: x0 = old_sp, x1 = new_sp. + core::arch::naked_asm!( + // Save callee-saved registers of the old task (x19..x30) as pairs. + "stp x29, x30, [sp, #-16]!", + "stp x27, x28, [sp, #-16]!", + "stp x25, x26, [sp, #-16]!", + "stp x23, x24, [sp, #-16]!", + "stp x21, x22, [sp, #-16]!", + "stp x19, x20, [sp, #-16]!", + // Save current SP into *old_sp. + "mov x2, sp", + "str x2, [x0]", + // Load new SP. + "mov sp, x1", + // Restore new task's callee-saved registers. + "ldp x19, x20, [sp], #16", + "ldp x21, x22, [sp], #16", + "ldp x23, x24, [sp], #16", + "ldp x25, x26, [sp], #16", + "ldp x27, x28, [sp], #16", + "ldp x29, x30, [sp], #16", + // Return — branches to the new task's saved link register. + "ret", + ) +} + +/// Trampoline entered by a freshly spawned task on its very first schedule. +/// +/// The task function pointer is the top-of-stack word left by +/// `spawn_kernel_task`. We pop it, enable interrupts, then call it. If it ever +/// returns (it should not for kernel tasks), we park. +#[unsafe(naked)] +pub unsafe extern "C" fn task_entry() { + core::arch::naked_asm!( + "ldr x9, [sp], #16", // task fn pointer pushed by spawn_kernel_task + "msr daifclr, #0b0011", // enable IRQ+FIQ for this task + "blr x9", // run the task function + // Task returned — should not happen for kernel tasks; park. + "0:", + "msr daifset, #0b0011", + "wfi", + "b 0b", + ) +} diff --git a/kernel/src/arch/aarch64/smp.rs b/kernel/src/arch/aarch64/smp.rs new file mode 100644 index 0000000..d758d56 --- /dev/null +++ b/kernel/src/arch/aarch64/smp.rs @@ -0,0 +1,84 @@ +//! SMP — aarch64 multiprocessor scaffolding (PSCI / GICv3 SGI based). +//! +//! This is a single-core-only scaffold: the per-CPU table and `this_cpu()` +//! lookup are real (so shared scheduler/process code compiles and runs on the +//! BSP), but AP wake-up via PSCI `CPU_ON` and reschedule IPIs via GIC SGIs are +//! not yet implemented. + +use core::sync::atomic::{AtomicBool, AtomicU32, Ordering}; +use limine::request::MpResponse; + +/// Maximum CPUs supported. +pub const MAX_CPUS: usize = 64; + +/// Per-CPU mutable state — one entry per logical processor. +/// +/// Field layout mirrors the x86_64 backend so shared code is portable. +#[repr(C)] +pub struct PerCpuData { + pub cpu_id: u32, + /// MPIDR_EL1 affinity of this core (aarch64 analogue of the x86 LAPIC id). + pub mpidr: u64, + pub online: AtomicBool, + pub current_pid: AtomicU32, +} + +const ZERO_CPU: PerCpuData = PerCpuData { + cpu_id: 0, + mpidr: 0, + online: AtomicBool::new(false), + current_pid: AtomicU32::new(0), +}; +static mut PER_CPU_DATA: [PerCpuData; MAX_CPUS] = [ZERO_CPU; MAX_CPUS]; + +/// Total logical CPUs online (including BSP). +pub static CPU_COUNT: AtomicU32 = AtomicU32::new(1); + +/// Read this core's MPIDR_EL1 affinity bits. +#[inline] +fn read_mpidr() -> u64 { + let v: u64; + unsafe { core::arch::asm!("mrs {}, mpidr_el1", out(reg) v, options(nomem, nostack)) }; + v & 0x00FF_FFFF // Aff0..Aff2 +} + +/// Return this CPU's ID (by matching MPIDR_EL1 against the per-CPU table). +pub fn current_cpu_id() -> u32 { + let mpidr = read_mpidr(); + let n = CPU_COUNT.load(Ordering::Relaxed) as usize; + unsafe { + let table = &*core::ptr::addr_of!(PER_CPU_DATA); + for i in 0..n { + if table[i].mpidr == mpidr { + return i as u32; + } + } + } + 0 +} + +/// Return this CPU's per-CPU entry. +pub fn this_cpu() -> &'static PerCpuData { + let cpu_idx = current_cpu_id() as usize; + unsafe { &(*core::ptr::addr_of!(PER_CPU_DATA))[cpu_idx] } +} + +/// Wake all APs. Currently single-core only. +/// +/// TODO(arm): bring APs online via PSCI `CPU_ON` (SMC/HVC) and have each call +/// `crate::arch::ap_init()` then `crate::sched::ap_start()`. +pub fn init(_smp_resp: Option<&'static MpResponse>) { + unsafe { + let table = &mut *core::ptr::addr_of_mut!(PER_CPU_DATA); + table[0].cpu_id = 0; + table[0].mpidr = read_mpidr(); + table[0].online.store(true, Ordering::Release); + } + CPU_COUNT.store(1, Ordering::Release); + log::info!("[SMP] aarch64 single-core scaffold (AP wake via PSCI not implemented)"); +} + +/// Broadcast a reschedule IPI to all other online CPUs. +/// +/// TODO(arm): send a GIC SGI to the target CPU interface(s). No-op (single core). +pub fn broadcast_resched_ipi() {} diff --git a/kernel/src/arch/aarch64/syscall.rs b/kernel/src/arch/aarch64/syscall.rs new file mode 100644 index 0000000..37e6ccf --- /dev/null +++ b/kernel/src/arch/aarch64/syscall.rs @@ -0,0 +1,118 @@ +//! SVC fast-path syscall scaffolding — aarch64. +//! +//! On x86 user syscalls arrive via the `SYSCALL` instruction and a per-CPU GS +//! scratch area captures the user register snapshot at entry. On aarch64 the +//! equivalent is the `SVC` instruction trapping to the EL1 synchronous exception +//! vector, which would stash the user GPRs (x0..x30, SP_EL0, ELR/SPSR) into a +//! per-CPU scratch. +//! +//! This module preserves the public surface the shared kernel calls +//! (`set_active_stack_top`, `user_rsp`/`user_rip`/`user_r9`/`user_rbp`, +//! `user_gprs`, `init`, `init_ap`). The per-CPU snapshot capture itself (the +//! exception-vector entry) is not yet implemented, so the accessors return the +//! last value written into the scratch (zero until the vector is wired up). + +use core::sync::atomic::{AtomicU64, Ordering}; + +const MAX_CPUS: usize = 64; + +/// Per-CPU snapshot of the user register file captured at SVC entry. +/// +/// Field names mirror the x86_64 [`UserGprSnapshot`] so cross-arch process code +/// (e.g. `save_full_user_gprs`) is portable. On aarch64 these map to the AAPCS64 +/// argument/callee-saved registers (rip→ELR_EL1, rsp→SP_EL0, etc.). +#[repr(C)] +struct CpuScratch { + user_rsp: u64, + user_rip: u64, + syscall_stack_top: u64, + rdi: u64, rsi: u64, rdx: u64, + r10: u64, r8: u64, r9: u64, + rbx: u64, rbp: u64, + r12: u64, r13: u64, r14: u64, r15: u64, +} + +const ZERO_SCRATCH: CpuScratch = CpuScratch { + user_rsp: 0, user_rip: 0, syscall_stack_top: 0, + rdi: 0, rsi: 0, rdx: 0, r10: 0, r8: 0, r9: 0, + rbx: 0, rbp: 0, r12: 0, r13: 0, r14: 0, r15: 0, +}; + +static mut CPU_SCRATCHES: [CpuScratch; MAX_CPUS] = [ZERO_SCRATCH; MAX_CPUS]; + +/// Active kernel syscall-stack top, per CPU (set by the scheduler before +/// returning to user). Mirrors the x86 `set_active_stack_top` behaviour. +static ACTIVE_STACK_TOP: [AtomicU64; MAX_CPUS] = [const { AtomicU64::new(0) }; MAX_CPUS]; + +/// Snapshot of all general-purpose user registers captured at SVC entry. +/// +/// Field names are kept identical to the x86_64 backend so shared code stays +/// architecture-neutral. +#[derive(Clone, Copy, Default)] +pub struct UserGprSnapshot { + pub rdi: u64, pub rsi: u64, pub rdx: u64, + pub r10: u64, pub r8: u64, pub r9: u64, + pub rbx: u64, pub rbp: u64, + pub r12: u64, pub r13: u64, pub r14: u64, pub r15: u64, +} + +#[inline] +fn cpu_idx() -> usize { + crate::arch::aarch64::smp::current_cpu_id() as usize +} + +/// Initialise the SVC fast path on the BSP. +/// +/// TODO(arm): install the EL1 exception vector table (VBAR_EL1) whose +/// synchronous handler decodes the SVC and stashes the user GPR snapshot. +pub fn init() { + log::info!("[Syscall] aarch64 SVC scaffold (exception vector not yet installed)"); +} + +/// Per-AP SVC init. +/// +/// TODO(arm): point this core's VBAR_EL1 at the shared vector table. +pub fn init_ap(_cpu_idx: u32) {} + +/// Set the active kernel syscall-stack top for the current CPU. +pub fn set_active_stack_top(stack_top: u64) { + let i = cpu_idx(); + ACTIVE_STACK_TOP[i].store(stack_top, Ordering::Release); + unsafe { + let s = &mut (*core::ptr::addr_of_mut!(CPU_SCRATCHES))[i]; + s.syscall_stack_top = stack_top; + } +} + +/// User SP_EL0 captured at SVC entry (x86 user RSP analogue). +pub fn user_rsp() -> u64 { + unsafe { (*core::ptr::addr_of!(CPU_SCRATCHES))[cpu_idx()].user_rsp } +} + +/// User ELR_EL1 (return address) captured at SVC entry (x86 user RIP analogue). +pub fn user_rip() -> u64 { + unsafe { (*core::ptr::addr_of!(CPU_SCRATCHES))[cpu_idx()].user_rip } +} + +/// Sixth user argument register captured at SVC entry (x86 R9 analogue). +pub fn user_r9() -> u64 { + unsafe { (*core::ptr::addr_of!(CPU_SCRATCHES))[cpu_idx()].r9 } +} + +/// User frame-pointer register captured at SVC entry (x86 RBP analogue). +pub fn user_rbp() -> u64 { + unsafe { (*core::ptr::addr_of!(CPU_SCRATCHES))[cpu_idx()].rbp } +} + +/// Read the full user GPR snapshot stashed by the SVC entry vector. +pub fn user_gprs() -> UserGprSnapshot { + unsafe { + let s = &(*core::ptr::addr_of!(CPU_SCRATCHES))[cpu_idx()]; + UserGprSnapshot { + rdi: s.rdi, rsi: s.rsi, rdx: s.rdx, + r10: s.r10, r8: s.r8, r9: s.r9, + rbx: s.rbx, rbp: s.rbp, + r12: s.r12, r13: s.r13, r14: s.r14, r15: s.r15, + } + } +} From f4e806f03e33303b09282bd9adbba8240e4b00f3 Mon Sep 17 00:00:00 2001 From: Tahiru Agbanwa <130235676+squirelboy360@users.noreply.github.com> Date: Mon, 8 Jun 2026 22:20:13 +0200 Subject: [PATCH 02/36] arch: route x86-specific code in shared kernel through the arch facade The shared kernel embedded raw x86 instructions and x86-only limine flags directly in architecture-neutral files, which broke the aarch64-unknown-none build. Move every such site behind crate::arch::* so both x86_64 and aarch64 compile, with x86_64 codegen left byte-for-byte identical. User-mode entry: extract the IRETQ/SYSRETQ ring-3 transition asm out of process::enter_user_by_pid_noreturn{,_try} into a new arch::enter_user hook (EnterUserRegs + enter_user_iret/enter_user_sysret). The x86_64 backend keeps the exact prior asm (verified: identical instruction/operand sequences); the aarch64 backend stubs them pending the real EL1->EL0 ERET path. The shared process layer keeps all PTABLE/CR3/errno logic and only calls the hook for the final transfer. Other shared sites rerouted through the facade: - main.rs: gate limine MP_FLAG_X2APIC behind cfg(x86_64), 0 otherwise - dispatch/poll/posix: _rdtsc()/inline rdtsc -> arch::rdtsc() - posix/poll/fd/futex/engine/ipc_display: sti;hlt;cli -> arch::enable_and_halt() - futex/posix: pause -> arch::spin_pause() - process: cli -> arch::interrupts_disable(); mov cr3 -> arch::memory::write_cr3() - paging: read cr3 -> arch::memory::read_cr3() - panic: read rbp -> arch::read_frame_pointer() - fd: poweroff loop -> arch::acpi_shutdown() (already arch-neutral) New arch hooks: enter_user_{iret,sysret}+EnterUserRegs, interrupts_disable, enable_and_halt, read_frame_pointer (x86_64 real, aarch64 mirror/stub). Both targets build clean: - aarch64-unknown-none --features arch-aarch64: 0 errors - x86_64-unknown-none release: 0 errors --- kernel/src/arch/aarch64/enter_user.rs | 53 +++++ kernel/src/arch/aarch64/mod.rs | 18 ++ kernel/src/arch/mod.rs | 42 +++- kernel/src/arch/x86_64/cpu.rs | 8 + kernel/src/arch/x86_64/enter_user.rs | 167 ++++++++++++++ kernel/src/arch/x86_64/mod.rs | 12 ++ kernel/src/main.rs | 3 + kernel/src/mm/paging.rs | 4 +- kernel/src/panic.rs | 3 +- kernel/src/process/mod.rs | 239 ++------------------- kernel/src/syscall/dispatch.rs | 2 +- kernel/src/syscall/handlers/engine.rs | 4 +- kernel/src/syscall/handlers/fd.rs | 5 +- kernel/src/syscall/handlers/futex.rs | 6 +- kernel/src/syscall/handlers/ipc_display.rs | 2 +- kernel/src/syscall/poll.rs | 7 +- kernel/src/syscall/posix.rs | 25 +-- 17 files changed, 343 insertions(+), 257 deletions(-) create mode 100644 kernel/src/arch/aarch64/enter_user.rs create mode 100644 kernel/src/arch/x86_64/enter_user.rs diff --git a/kernel/src/arch/aarch64/enter_user.rs b/kernel/src/arch/aarch64/enter_user.rs new file mode 100644 index 0000000..8799038 --- /dev/null +++ b/kernel/src/arch/aarch64/enter_user.rs @@ -0,0 +1,53 @@ +//! aarch64 user-mode entry transitions (scaffold). +//! +//! Mirrors the x86_64 `enter_user` hooks so the shared `process` layer can +//! transfer into ring-3 (EL0) architecture-neutrally. On aarch64 both the +//! IRET and SYSRET variants collapse to an `ERET` after restoring the GPRs +//! from `regs` and setting ELR_EL1/SP_EL0/SPSR_EL1. These are stubs pending +//! the real EL1→EL0 return path; they exist so the kernel compiles for +//! `aarch64-unknown-none`. + +/// Full user register state required to enter EL0. +/// +/// Field names mirror the x86_64 backend so shared code stays neutral; on +/// aarch64 these map onto x0..x18 / SP_EL0 / ELR_EL1 / SPSR_EL1 in the real +/// implementation. +#[derive(Clone, Copy)] +pub struct EnterUserRegs { + pub rip: u64, + pub rsp: u64, + pub rflags: u64, + pub rax: u64, + pub rbx: u64, + pub rcx: u64, + pub rdx: u64, + pub rsi: u64, + pub rdi: u64, + pub rbp: u64, + pub r8: u64, + pub r9: u64, + pub r10: u64, + pub r11: u64, + pub r12: u64, + pub r13: u64, + pub r14: u64, + pub r15: u64, +} + +/// ERET entry mirroring the x86_64 IRETQ path. +/// +/// # Safety +/// TTBR0_EL1 must already point at the target thread's address space. +#[inline(never)] +pub unsafe fn enter_user_iret(_regs: &EnterUserRegs) -> ! { + unimplemented!("aarch64 enter_user_iret: EL1→EL0 ERET not yet implemented") +} + +/// ERET entry mirroring the x86_64 SYSRETQ path. +/// +/// # Safety +/// TTBR0_EL1 must already point at the target thread's address space. +#[inline(never)] +pub unsafe fn enter_user_sysret(_regs: &EnterUserRegs) -> ! { + unimplemented!("aarch64 enter_user_sysret: EL1→EL0 ERET not yet implemented") +} diff --git a/kernel/src/arch/aarch64/mod.rs b/kernel/src/arch/aarch64/mod.rs index 91d39b1..4cf0b7b 100644 --- a/kernel/src/arch/aarch64/mod.rs +++ b/kernel/src/arch/aarch64/mod.rs @@ -23,6 +23,9 @@ pub mod cpu; pub mod syscall; pub mod acpi; pub mod smp; +pub mod enter_user; + +pub use enter_user::{enter_user_iret, enter_user_sysret, EnterUserRegs}; use core::arch::asm; @@ -81,6 +84,21 @@ pub fn halt() { unsafe { asm!("wfi", options(nomem, nostack)) }; } +/// Unmask interrupts, wait for one (WFI), then mask again. +/// +/// Mirror of the x86_64 `sti; hlt; cli` idle pattern. +#[inline(always)] +pub fn enable_and_halt() { + unsafe { + asm!( + "msr daifclr, #0b0011", + "wfi", + "msr daifset, #0b0011", + options(nomem, nostack), + ) + }; +} + /// Permanently halt this core (used in panic). pub fn halt_forever() -> ! { loop { diff --git a/kernel/src/arch/mod.rs b/kernel/src/arch/mod.rs index 6ee7cc2..12bee20 100644 --- a/kernel/src/arch/mod.rs +++ b/kernel/src/arch/mod.rs @@ -27,7 +27,22 @@ pub mod mmio; cfg_if::cfg_if! { if #[cfg(target_arch = "x86_64")] { - pub use x86_64::cpu::{interrupts_restore, interrupts_save_and_disable, memory_fence, spin_pause}; + pub use x86_64::cpu::{interrupts_disable, interrupts_restore, interrupts_save_and_disable, memory_fence, spin_pause}; + } else if #[cfg(target_arch = "aarch64")] { + pub fn memory_fence() { + core::sync::atomic::fence(core::sync::atomic::Ordering::SeqCst); + } + pub fn spin_pause() { + core::hint::spin_loop(); + } + /// Mask IRQ/FIQ at the current EL (DAIF.I/F). + pub fn interrupts_disable() { + unsafe { core::arch::asm!("msr daifset, #3", options(nomem, nostack, preserves_flags)); } + } + pub fn interrupts_save_and_disable() -> u64 { + 0 + } + pub fn interrupts_restore(_rflags: u64) {} } else { pub fn memory_fence() { core::sync::atomic::fence(core::sync::atomic::Ordering::SeqCst); @@ -35,6 +50,7 @@ cfg_if::cfg_if! { pub fn spin_pause() { core::hint::spin_loop(); } + pub fn interrupts_disable() {} pub fn interrupts_save_and_disable() -> u64 { 0 } @@ -42,6 +58,30 @@ cfg_if::cfg_if! { } } +/// Read the current frame-pointer register (for best-effort backtraces). +/// +/// On x86_64 this is RBP; on aarch64 it is X29. Returns 0 on architectures +/// where no fixed frame pointer is available. +#[inline(always)] +pub fn read_frame_pointer() -> u64 { + #[cfg(target_arch = "x86_64")] + { + let fp: u64; + unsafe { core::arch::asm!("mov {}, rbp", out(reg) fp, options(nomem, nostack)); } + fp + } + #[cfg(target_arch = "aarch64")] + { + let fp: u64; + unsafe { core::arch::asm!("mov {}, x29", out(reg) fp, options(nomem, nostack)); } + fp + } + #[cfg(not(any(target_arch = "x86_64", target_arch = "aarch64")))] + { + 0 + } +} + /// Trigger an ACPI S5 soft-off (architecture-specific implementation). pub fn acpi_shutdown() -> ! { #[cfg(target_arch = "x86_64")] diff --git a/kernel/src/arch/x86_64/cpu.rs b/kernel/src/arch/x86_64/cpu.rs index 43e3cfc..6061e24 100644 --- a/kernel/src/arch/x86_64/cpu.rs +++ b/kernel/src/arch/x86_64/cpu.rs @@ -257,6 +257,14 @@ pub fn spin_pause() { } } +/// Disable interrupts unconditionally (no save/restore). +#[inline(always)] +pub fn interrupts_disable() { + unsafe { + core::arch::asm!("cli", options(nomem, nostack, preserves_flags)); + } +} + /// Save RFLAGS and disable interrupts; returns RFLAGS for [`interrupts_restore`]. #[inline(always)] pub fn interrupts_save_and_disable() -> u64 { diff --git a/kernel/src/arch/x86_64/enter_user.rs b/kernel/src/arch/x86_64/enter_user.rs new file mode 100644 index 0000000..53322b5 --- /dev/null +++ b/kernel/src/arch/x86_64/enter_user.rs @@ -0,0 +1,167 @@ +//! x86_64 user-mode entry transitions. +//! +//! These are the final, no-return transfers into ring-3. The shared +//! `process` layer prepares the full register state, switches CR3 and +//! delivers pending errno; it then calls one of these hooks to actually +//! enter user mode. Two variants exist: +//! +//! * [`enter_user_iret`] — IRETQ path, used when a thread was preempted +//! by the APIC timer at an arbitrary user instruction. Restores every +//! register (including the exact RCX/R11/RFLAGS) so no architectural +//! state is lost. +//! * [`enter_user_sysret`] — SYSRETQ path, the native user-entry used when +//! a thread yielded via a SYSCALL (e.g. futex_wait). Restores the full +//! user GPR set so callee-saved registers survive the SysV-ABI call +//! boundary of the userspace SYSCALL trampoline. +//! +//! The asm is load-bearing for the render path; it must remain byte-for-byte +//! identical to its prior inline form in `process::mod`. + +use core::arch::asm; + +/// Full user register state required to enter ring-3. +#[derive(Clone, Copy)] +pub struct EnterUserRegs { + pub rip: u64, + pub rsp: u64, + pub rflags: u64, + pub rax: u64, + pub rbx: u64, + pub rcx: u64, + pub rdx: u64, + pub rsi: u64, + pub rdi: u64, + pub rbp: u64, + pub r8: u64, + pub r9: u64, + pub r10: u64, + pub r11: u64, + pub r12: u64, + pub r13: u64, + pub r14: u64, + pub r15: u64, +} + +/// IRETQ entry: restore every register and return via the hardware IRET frame. +/// +/// # Safety +/// CR3 must already point at the target thread's address space, and `regs` +/// must describe a valid ring-3 context (mapped RIP/RSP, sane RFLAGS). +#[inline(never)] +pub unsafe fn enter_user_iret(regs: &EnterUserRegs) -> ! { + // Layout of iret_frame (accessed via r11 as base register): + // [r11 + 0x00] RIP ─┐ + // [r11 + 0x08] CS │ hardware IRET frame pushed onto + // [r11 + 0x10] RFLAGS │ the kernel stack by the `push` sequence + // [r11 + 0x18] RSP ─┘ (ring-3 switch includes RSP+SS) + // [r11 + 0x20] SS + // [r11 + 0x28] RAX .. R15 (all user GPRs, r11 loaded last) + let iret_frame: [u64; 20] = [ + /* 0x00 */ regs.rip, + /* 0x08 */ super::gdt::USER_CS as u64, // 0x2B + /* 0x10 */ regs.rflags | 0x200, // ensure IF=1 + /* 0x18 */ regs.rsp, + /* 0x20 */ super::gdt::USER_DS as u64, // 0x23 + /* 0x28 */ regs.rax, + /* 0x30 */ regs.rbx, + /* 0x38 */ regs.rcx, // real RCX (not SYSCALL convention) + /* 0x40 */ regs.rdx, + /* 0x48 */ regs.rdi, + /* 0x50 */ regs.rsi, + /* 0x58 */ regs.rbp, + /* 0x60 */ regs.r8, + /* 0x68 */ regs.r9, + /* 0x70 */ regs.r10, + /* 0x78 */ regs.r11, // real R11 (not RFLAGS convention) + /* 0x80 */ regs.r12, + /* 0x88 */ regs.r13, + /* 0x90 */ regs.r14, + /* 0x98 */ regs.r15, + ]; + asm!( + // Push the 5-word IRET frame: SS, RSP, RFLAGS, CS, RIP. + // These land on the current kernel stack; IRETQ will consume + // them and switch to the user stack (RSP) automatically. + "push qword ptr [r11 + 0x20]", // SS + "push qword ptr [r11 + 0x18]", // RSP + "push qword ptr [r11 + 0x10]", // RFLAGS + "push qword ptr [r11 + 0x08]", // CS + "push qword ptr [r11 + 0x00]", // RIP + // Restore all user GPRs (r11 last because it is our base ptr). + "mov rax, [r11 + 0x28]", + "mov rbx, [r11 + 0x30]", + "mov rcx, [r11 + 0x38]", + "mov rdx, [r11 + 0x40]", + "mov rdi, [r11 + 0x48]", + "mov rsi, [r11 + 0x50]", + "mov rbp, [r11 + 0x58]", + "mov r8, [r11 + 0x60]", + "mov r9, [r11 + 0x68]", + "mov r10, [r11 + 0x70]", + "mov r12, [r11 + 0x80]", + "mov r13, [r11 + 0x88]", + "mov r14, [r11 + 0x90]", + "mov r15, [r11 + 0x98]", + "mov r11, [r11 + 0x78]", // real R11 — must be last + "iretq", + in("r11") iret_frame.as_ptr(), + options(noreturn), + ) +} + +/// SYSRETQ entry: restore the full user GPR set and return to ring-3. +/// +/// # Safety +/// CR3 must already point at the target thread's address space, and `regs` +/// must describe a valid ring-3 context (mapped RIP/RSP, sane RFLAGS). +#[inline(never)] +pub unsafe fn enter_user_sysret(regs: &EnterUserRegs) -> ! { + // Stage values in a fixed-layout stack array and load them inside the asm + // block via a single pointer. RSP and the syscall-consumed regs (RCX, + // R11) are loaded last so all intermediate reads complete first. + let frame: [u64; 16] = [ + /* 0x00 */ regs.rip, + /* 0x08 */ regs.rflags, + /* 0x10 */ regs.rsp, + /* 0x18 */ regs.rdi, + /* 0x20 */ regs.rsi, + /* 0x28 */ regs.rdx, + /* 0x30 */ regs.rax, + /* 0x38 */ regs.rbx, + /* 0x40 */ regs.rbp, + /* 0x48 */ regs.r8, + /* 0x50 */ regs.r9, + /* 0x58 */ regs.r10, + /* 0x60 */ regs.r12, + /* 0x68 */ regs.r13, + /* 0x70 */ regs.r14, + /* 0x78 */ regs.r15, + ]; + asm!( + // Pin the frame base in r11. r11 is consumed by SYSRET (as user + // RFLAGS) and is the LAST register we load — every other load + // can therefore use [r11 + disp] safely. If we let the compiler + // pick the base register, it can alias the base with the first + // destination (e.g. rax) and have the very first `mov` clobber + // the pointer before any further reads happen. + "mov rax, [r11 + 0x30]", + "mov rdi, [r11 + 0x18]", + "mov rsi, [r11 + 0x20]", + "mov rdx, [r11 + 0x28]", + "mov rbx, [r11 + 0x38]", + "mov rbp, [r11 + 0x40]", + "mov r8, [r11 + 0x48]", + "mov r9, [r11 + 0x50]", + "mov r10, [r11 + 0x58]", + "mov r12, [r11 + 0x60]", + "mov r13, [r11 + 0x68]", + "mov r14, [r11 + 0x70]", + "mov r15, [r11 + 0x78]", + "mov rcx, [r11 + 0x00]", // user RIP → RCX (consumed by sysretq) + "mov rsp, [r11 + 0x10]", // switch to user stack (mapped in user PML4) + "mov r11, [r11 + 0x08]", // user RFL → R11 (consumed by sysretq) — load LAST + "sysretq", + in("r11") frame.as_ptr(), + options(noreturn), + ) +} diff --git a/kernel/src/arch/x86_64/mod.rs b/kernel/src/arch/x86_64/mod.rs index ae8a090..8ff5ecc 100644 --- a/kernel/src/arch/x86_64/mod.rs +++ b/kernel/src/arch/x86_64/mod.rs @@ -13,6 +13,9 @@ pub(crate) mod port_io; pub mod syscall; pub mod acpi; pub mod smp; +pub mod enter_user; + +pub use enter_user::{enter_user_iret, enter_user_sysret, EnterUserRegs}; // Re-export commonly used types so external code uses `crate::arch::InterruptFrame` pub use idt::InterruptFrame; @@ -68,6 +71,15 @@ pub fn halt() { unsafe { asm!("hlt") } } +/// Enable interrupts, halt until one arrives, then disable interrupts again. +/// +/// The `sti; hlt` pair is atomic on x86 (STI defers IF until after the next +/// instruction), so no interrupt can sneak in between enabling and halting. +#[inline(always)] +pub fn enable_and_halt() { + unsafe { asm!("sti; hlt; cli", options(nomem, nostack)) } +} + /// Permanently halt (used in panic). pub fn halt_forever() -> ! { loop { diff --git a/kernel/src/main.rs b/kernel/src/main.rs index 5630754..df8dc39 100644 --- a/kernel/src/main.rs +++ b/kernel/src/main.rs @@ -69,7 +69,10 @@ static FB_REQUEST: FramebufferRequest = FramebufferRequest::new(); static HHDM_REQUEST: HhdmRequest = HhdmRequest::new(); static MMAP_REQUEST: MemmapRequest = MemmapRequest::new(); static KADDR_REQUEST: ExecutableAddressRequest = ExecutableAddressRequest::new(); +#[cfg(target_arch = "x86_64")] static SMP_REQUEST: MpRequest = MpRequest::new(limine::mp::MP_FLAG_X2APIC); +#[cfg(not(target_arch = "x86_64"))] +static SMP_REQUEST: MpRequest = MpRequest::new(0); static MODULES_REQUEST: ModulesRequest = ModulesRequest::new(); /// Slice of libflutter_engine.so as provided by Limine, stored once at boot. diff --git a/kernel/src/mm/paging.rs b/kernel/src/mm/paging.rs index 583b831..d58d250 100644 --- a/kernel/src/mm/paging.rs +++ b/kernel/src/mm/paging.rs @@ -461,9 +461,7 @@ pub fn demand_page(cr2: u64, error: u64) -> bool { if error & 0x1 != 0 { return false; } // page present — protection fault, not demand page // Read current CR3 to get the active PML4. - let cr3_phys: u64; - unsafe { core::arch::asm!("mov {}, cr3", out(reg) cr3_phys) }; - let cr3_phys = cr3_phys & 0x000f_ffff_ffff_f000; + let cr3_phys = crate::arch::memory::read_cr3() & 0x000f_ffff_ffff_f000; let page_va = cr2 & !0xFFF; diff --git a/kernel/src/panic.rs b/kernel/src/panic.rs index 438fa28..d925163 100644 --- a/kernel/src/panic.rs +++ b/kernel/src/panic.rs @@ -44,8 +44,7 @@ fn panic(info: &PanicInfo) -> ! { // Best-effort backtrace: walk the saved RBP chain a few frames and print // each return address. Stops on a bogus pointer. unsafe { - let mut rbp: u64; - core::arch::asm!("mov {}, rbp", out(reg) rbp, options(nomem, nostack)); + let mut rbp: u64 = crate::arch::read_frame_pointer(); crate::logger::early_print("backtrace:\r\n"); for _ in 0..16 { if rbp < 0xffff_8000_0000_0000 || (rbp & 7) != 0 { break; } diff --git a/kernel/src/process/mod.rs b/kernel/src/process/mod.rs index 377ddba..24b4ea1 100644 --- a/kernel/src/process/mod.rs +++ b/kernel/src/process/mod.rs @@ -1243,13 +1243,7 @@ pub fn enter_user_by_pid_noreturn(pid: u32) -> ! { crate::arch::cpu::set_fs_base(fs_base); // Switch to the user address space. - unsafe { - core::arch::asm!( - "mov cr3, {cr3}", - cr3 = in(reg) pml4_phys, - options(nostack, nomem, preserves_flags), - ); - } + crate::arch::memory::write_cr3(pml4_phys); // Deliver pending errno (e.g. EINTR=4 from sys_epoll_wait_real) NOW, // after the CR3 switch so we write into the resuming thread's own SYSDATA @@ -1264,6 +1258,15 @@ pub fn enter_user_by_pid_noreturn(pid: u32) -> ! { } } + // Assemble the full user register state for the arch entry hook. The + // arch backend owns the actual ring-3 transition asm (IRETQ vs SYSRETQ on + // x86_64); the shared code above owns all the PTABLE/CR3/errno logic. + let enter_regs = crate::arch::EnterUserRegs { + rip, rsp, rflags, + rax, rbx, rcx, rdx, rsi, rdi, rbp, + r8, r9, r10, r11, r12, r13, r14, r15, + }; + if preempted_by_timer { // ── IRET path ──────────────────────────────────────────────────────── // This thread was timer-preempted at an arbitrary user instruction, so @@ -1271,67 +1274,7 @@ pub fn enter_user_by_pid_noreturn(pid: u32) -> ! { // RFLAGS) before returning. SYSRETQ is wrong here because it would // set RIP=RCX and RFLAGS=R11, corrupting those registers and losing // the real FLAGS state (e.g. CF/ZF from a cmp instruction). - // - // Layout of iret_frame (accessed via r11 as base register): - // [r11 + 0x00] RIP ─┐ - // [r11 + 0x08] CS │ hardware IRET frame pushed onto - // [r11 + 0x10] RFLAGS │ the kernel stack by the `push` sequence - // [r11 + 0x18] RSP ─┘ (ring-3 switch includes RSP+SS) - // [r11 + 0x20] SS - // [r11 + 0x28] RAX .. R15 (all user GPRs, r11 loaded last) - let iret_frame: [u64; 20] = [ - /* 0x00 */ rip, - /* 0x08 */ crate::arch::gdt::USER_CS as u64, // 0x2B - /* 0x10 */ rflags | 0x200, // ensure IF=1 - /* 0x18 */ rsp, - /* 0x20 */ crate::arch::gdt::USER_DS as u64, // 0x23 - /* 0x28 */ rax, - /* 0x30 */ rbx, - /* 0x38 */ rcx, // real RCX (not SYSCALL convention) - /* 0x40 */ rdx, - /* 0x48 */ rdi, - /* 0x50 */ rsi, - /* 0x58 */ rbp, - /* 0x60 */ r8, - /* 0x68 */ r9, - /* 0x70 */ r10, - /* 0x78 */ r11, // real R11 (not RFLAGS convention) - /* 0x80 */ r12, - /* 0x88 */ r13, - /* 0x90 */ r14, - /* 0x98 */ r15, - ]; - unsafe { - core::arch::asm!( - // Push the 5-word IRET frame: SS, RSP, RFLAGS, CS, RIP. - // These land on the current kernel stack; IRETQ will consume - // them and switch to the user stack (RSP) automatically. - "push qword ptr [r11 + 0x20]", // SS - "push qword ptr [r11 + 0x18]", // RSP - "push qword ptr [r11 + 0x10]", // RFLAGS - "push qword ptr [r11 + 0x08]", // CS - "push qword ptr [r11 + 0x00]", // RIP - // Restore all user GPRs (r11 last because it is our base ptr). - "mov rax, [r11 + 0x28]", - "mov rbx, [r11 + 0x30]", - "mov rcx, [r11 + 0x38]", - "mov rdx, [r11 + 0x40]", - "mov rdi, [r11 + 0x48]", - "mov rsi, [r11 + 0x50]", - "mov rbp, [r11 + 0x58]", - "mov r8, [r11 + 0x60]", - "mov r9, [r11 + 0x68]", - "mov r10, [r11 + 0x70]", - "mov r12, [r11 + 0x80]", - "mov r13, [r11 + 0x88]", - "mov r14, [r11 + 0x90]", - "mov r15, [r11 + 0x98]", - "mov r11, [r11 + 0x78]", // real R11 — must be last - "iretq", - in("r11") iret_frame.as_ptr(), - options(noreturn), - ) - } + unsafe { crate::arch::enter_user_iret(&enter_regs) } } // ── SYSRET path (syscall-yield context) ────────────────────────────────── @@ -1342,58 +1285,8 @@ pub fn enter_user_by_pid_noreturn(pid: u32) -> ! { // RAX changed to the syscall return value. This is required because the // SysV ABI mandates that callee-saved regs (rbx, rbp, r12–r15) survive // a function call, and the userspace SYSCALL trampoline is such a call. - // - // Stage values in a fixed-layout stack array and load them inside the asm - // block via a single pointer. RSP and the syscall-consumed regs (RCX, - // R11) are loaded last so all intermediate reads complete first. - let frame: [u64; 16] = [ - /* 0x00 */ rip, - /* 0x08 */ rflags, - /* 0x10 */ rsp, - /* 0x18 */ rdi, - /* 0x20 */ rsi, - /* 0x28 */ rdx, - /* 0x30 */ rax, - /* 0x38 */ rbx, - /* 0x40 */ rbp, - /* 0x48 */ r8, - /* 0x50 */ r9, - /* 0x58 */ r10, - /* 0x60 */ r12, - /* 0x68 */ r13, - /* 0x70 */ r14, - /* 0x78 */ r15, - ]; log::trace!("[enter_user] about to SYSRET. rip={:#x} rsp={:#x} rflags={:#x} pml4_phys={:#x}", rip, rsp, rflags, pml4_phys); - unsafe { - core::arch::asm!( - // Pin the frame base in r11. r11 is consumed by SYSRET (as user - // RFLAGS) and is the LAST register we load — every other load - // can therefore use [r11 + disp] safely. If we let the compiler - // pick the base register, it can alias the base with the first - // destination (e.g. rax) and have the very first `mov` clobber - // the pointer before any further reads happen. - "mov rax, [r11 + 0x30]", - "mov rdi, [r11 + 0x18]", - "mov rsi, [r11 + 0x20]", - "mov rdx, [r11 + 0x28]", - "mov rbx, [r11 + 0x38]", - "mov rbp, [r11 + 0x40]", - "mov r8, [r11 + 0x48]", - "mov r9, [r11 + 0x50]", - "mov r10, [r11 + 0x58]", - "mov r12, [r11 + 0x60]", - "mov r13, [r11 + 0x68]", - "mov r14, [r11 + 0x70]", - "mov r15, [r11 + 0x78]", - "mov rcx, [r11 + 0x00]", // user RIP → RCX (consumed by sysretq) - "mov rsp, [r11 + 0x10]", // switch to user stack (mapped in user PML4) - "mov r11, [r11 + 0x08]", // user RFL → R11 (consumed by sysretq) — load LAST - "sysretq", - in("r11") frame.as_ptr(), - options(noreturn), - ) - } + unsafe { crate::arch::enter_user_sysret(&enter_regs) } } pub fn enter_user_by_pid_noreturn_try(pid: u32) -> bool { @@ -1446,13 +1339,7 @@ pub fn enter_user_by_pid_noreturn_try(pid: u32) -> bool { crate::arch::cpu::set_fs_base(fs_base); // Switch to the user address space. - unsafe { - core::arch::asm!( - "mov cr3, {cr3}", - cr3 = in(reg) pml4_phys, - options(nostack, nomem, preserves_flags), - ); - } + crate::arch::memory::write_cr3(pml4_phys); if errno_to_deliver != 0 { unsafe { @@ -1463,99 +1350,17 @@ pub fn enter_user_by_pid_noreturn_try(pid: u32) -> bool { } } + let enter_regs = crate::arch::EnterUserRegs { + rip, rsp, rflags, + rax, rbx, rcx, rdx, rsi, rdi, rbp, + r8, r9, r10, r11, r12, r13, r14, r15, + }; + if preempted_by_timer { - let iret_frame: [u64; 20] = [ - /* 0x00 */ rip, - /* 0x08 */ crate::arch::gdt::USER_CS as u64, - /* 0x10 */ rflags | 0x200, - /* 0x18 */ rsp, - /* 0x20 */ crate::arch::gdt::USER_DS as u64, - /* 0x28 */ rax, - /* 0x30 */ rbx, - /* 0x38 */ rcx, - /* 0x40 */ rdx, - /* 0x48 */ rdi, - /* 0x50 */ rsi, - /* 0x58 */ rbp, - /* 0x60 */ r8, - /* 0x68 */ r9, - /* 0x70 */ r10, - /* 0x78 */ r11, - /* 0x80 */ r12, - /* 0x88 */ r13, - /* 0x90 */ r14, - /* 0x98 */ r15, - ]; - unsafe { - core::arch::asm!( - "push qword ptr [r11 + 0x20]", - "push qword ptr [r11 + 0x18]", - "push qword ptr [r11 + 0x10]", - "push qword ptr [r11 + 0x08]", - "push qword ptr [r11 + 0x00]", - "mov rax, [r11 + 0x28]", - "mov rbx, [r11 + 0x30]", - "mov rcx, [r11 + 0x38]", - "mov rdx, [r11 + 0x40]", - "mov rdi, [r11 + 0x48]", - "mov rsi, [r11 + 0x50]", - "mov rbp, [r11 + 0x58]", - "mov r8, [r11 + 0x60]", - "mov r9, [r11 + 0x68]", - "mov r10, [r11 + 0x70]", - "mov r12, [r11 + 0x80]", - "mov r13, [r11 + 0x88]", - "mov r14, [r11 + 0x90]", - "mov r15, [r11 + 0x98]", - "mov r11, [r11 + 0x78]", - "iretq", - in("r11") iret_frame.as_ptr(), - options(noreturn), - ) - } + unsafe { crate::arch::enter_user_iret(&enter_regs) } } - let frame: [u64; 16] = [ - /* 0x00 */ rip, - /* 0x08 */ rflags, - /* 0x10 */ rsp, - /* 0x18 */ rdi, - /* 0x20 */ rsi, - /* 0x28 */ rdx, - /* 0x30 */ rax, - /* 0x38 */ rbx, - /* 0x40 */ rbp, - /* 0x48 */ r8, - /* 0x50 */ r9, - /* 0x58 */ r10, - /* 0x60 */ r12, - /* 0x68 */ r13, - /* 0x70 */ r14, - /* 0x78 */ r15, - ]; - unsafe { - core::arch::asm!( - "mov rax, [r11 + 0x30]", - "mov rdi, [r11 + 0x18]", - "mov rsi, [r11 + 0x20]", - "mov rdx, [r11 + 0x28]", - "mov rbx, [r11 + 0x38]", - "mov rbp, [r11 + 0x40]", - "mov r8, [r11 + 0x48]", - "mov r9, [r11 + 0x50]", - "mov r10, [r11 + 0x58]", - "mov r12, [r11 + 0x60]", - "mov r13, [r11 + 0x68]", - "mov r14, [r11 + 0x70]", - "mov r15, [r11 + 0x78]", - "mov rcx, [r11 + 0x00]", - "mov rsp, [r11 + 0x10]", - "mov r11, [r11 + 0x08]", - "sysretq", - in("r11") frame.as_ptr(), - options(noreturn), - ) - } + unsafe { crate::arch::enter_user_sysret(&enter_regs) } } /// Kernel-task entry point: loads the user PML4 and SYSRETs to ring-3. @@ -1571,7 +1376,7 @@ fn user_launch_task() { // Disable interrupts so the APIC timer cannot preempt between // mark_current_zombie() and sysretq. SYSRET restores RFLAGS from R11 // (0x0202 = IF=1), so the user process runs with interrupts enabled. - unsafe { core::arch::asm!("cli", options(nomem, nostack, preserves_flags)); } + crate::arch::interrupts_disable(); // Mark this kernel task as Zombie before we SYSRET. The SYSRET transfers // execution to ring-3 permanently; this task slot must never be resumed as // a kernel task again. Marking it Zombie prevents the scheduler from diff --git a/kernel/src/syscall/dispatch.rs b/kernel/src/syscall/dispatch.rs index 005780a..5776c2e 100644 --- a/kernel/src/syscall/dispatch.rs +++ b/kernel/src/syscall/dispatch.rs @@ -695,7 +695,7 @@ pub extern "C" fn dispatch_fast(number: u64, arg0: u64, arg1: u64, arg2: u64, ar if buf == 0 || len == 0 { return 0; } // Best-effort: cap at 256 MiB to avoid runaway. let n = len.min(256 * 1024 * 1024); - let mut s: u64 = unsafe { core::arch::x86_64::_rdtsc() } ^ 0xA5A5_F00D_DEAD_BEEFu64; + let mut s: u64 = crate::arch::rdtsc() ^ 0xA5A5_F00D_DEAD_BEEFu64; unsafe { let p = buf as *mut u8; let mut i = 0usize; diff --git a/kernel/src/syscall/handlers/engine.rs b/kernel/src/syscall/handlers/engine.rs index 948c556..f7e28ab 100644 --- a/kernel/src/syscall/handlers/engine.rs +++ b/kernel/src/syscall/handlers/engine.rs @@ -1258,12 +1258,12 @@ pub(crate) fn sys_sched_yield() -> i64 { } else if cur == 1 { // OnVsync posts work to runner threads — sleep briefly so they can run (SMP or IRQ wake). unsafe { - core::arch::asm!("sti; hlt; cli", options(nomem, nostack)); + { crate::arch::enable_and_halt(); } } } else { // No other runnable process — sleep until the next IRQ (timerfd, vsync…). unsafe { - core::arch::asm!("sti; hlt; cli", options(nomem, nostack)); + { crate::arch::enable_and_halt(); } } } } diff --git a/kernel/src/syscall/handlers/fd.rs b/kernel/src/syscall/handlers/fd.rs index ae9108a..5c8b117 100644 --- a/kernel/src/syscall/handlers/fd.rs +++ b/kernel/src/syscall/handlers/fd.rs @@ -640,7 +640,7 @@ pub(crate) fn sys_exit(code: u64) -> i64 { } // No runnable user process left. loop { - unsafe { core::arch::asm!("sti; hlt; cli", options(nomem, nostack)); } + crate::arch::enable_and_halt(); } } @@ -768,8 +768,5 @@ pub(crate) fn sys_exec_wait(path_ptr: u64, path_len: u64) -> i64 { pub(crate) fn sys_poweroff() -> i64 { log::info!("[syscall] sys_poweroff requested — triggering ACPI S5 shutdown"); - #[cfg(target_arch = "x86_64")] crate::arch::acpi_shutdown(); - #[cfg(not(target_arch = "x86_64"))] - loop { unsafe { core::arch::asm!("hlt", options(nomem, nostack)); } } } diff --git a/kernel/src/syscall/handlers/futex.rs b/kernel/src/syscall/handlers/futex.rs index b8a8ea4..b2284ba 100644 --- a/kernel/src/syscall/handlers/futex.rs +++ b/kernel/src/syscall/handlers/futex.rs @@ -486,7 +486,7 @@ pub(crate) fn sys_futex(uaddr: u64, op: u32, val: u32, sys_nr: u64) -> i64 { { #[cfg(target_arch = "x86_64")] unsafe { - core::arch::asm!("sti; hlt; cli", options(nomem, nostack)); + { crate::arch::enable_and_halt(); } } if let Some(next) = crate::process::next_runnable_pid(pid) { if next != pid { @@ -585,7 +585,7 @@ pub(crate) fn sys_futex(uaddr: u64, op: u32, val: u32, sys_nr: u64) -> i64 { { #[cfg(target_arch = "x86_64")] unsafe { - core::arch::asm!("sti; hlt; cli", options(nomem, nostack)); + { crate::arch::enable_and_halt(); } } if let Some(next) = crate::process::next_runnable_pid(pid) { if next != pid { @@ -767,7 +767,7 @@ pub(crate) fn sys_thread_join(thread_handle: u64, retval: u64) -> i64 { if retval != 0 { unsafe { *(retval as *mut u64) = 0; } } return 0; } - unsafe { core::arch::asm!("pause", options(nomem, nostack, preserves_flags)); } + crate::arch::spin_pause(); } Err(_) => { if retval != 0 { unsafe { *(retval as *mut u64) = 0; } } diff --git a/kernel/src/syscall/handlers/ipc_display.rs b/kernel/src/syscall/handlers/ipc_display.rs index 4a0e6b3..7c889c8 100644 --- a/kernel/src/syscall/handlers/ipc_display.rs +++ b/kernel/src/syscall/handlers/ipc_display.rs @@ -672,7 +672,7 @@ pub(crate) fn sys_wm_event_wait(ev_ptr: u64, ev_len: u64, timeout_ms: u64) -> i6 while crate::process::is_blocked(cur) { #[cfg(target_arch = "x86_64")] unsafe { - core::arch::asm!("sti; hlt; cli", options(nomem, nostack)); + { crate::arch::enable_and_halt(); } } if !crate::process::is_blocked(cur) { diff --git a/kernel/src/syscall/poll.rs b/kernel/src/syscall/poll.rs index 9da0a3e..71240c5 100644 --- a/kernel/src/syscall/poll.rs +++ b/kernel/src/syscall/poll.rs @@ -613,10 +613,7 @@ pub fn monotonic_ns() -> u64 { // 1_700_000_000 * 10^9 + tsc_ns // timerfd_settime with TFD_TIMER_ABSTIME passes this value, so our // "now" must use the same epoch or timers will never fire. - let lo: u32; - let hi: u32; - unsafe { core::arch::asm!("rdtsc", out("eax") lo, out("edx") hi, options(nomem, nostack)); } - let tsc_ns = (((hi as u64) << 32) | lo as u64) / 3; + let tsc_ns = crate::arch::rdtsc() / 3; tsc_ns.saturating_add(1_700_000_000u64 * 1_000_000_000u64) } @@ -1330,7 +1327,7 @@ pub(crate) fn sys_epoll_wait_real(epfd: u64, events_out: u64, maxevents: u64, ti #[cfg(target_arch = "x86_64")] unsafe { - core::arch::asm!("sti; hlt; cli", options(nomem, nostack)); + { crate::arch::enable_and_halt(); } } } } diff --git a/kernel/src/syscall/posix.rs b/kernel/src/syscall/posix.rs index 6e3f9d3..bff75c0 100644 --- a/kernel/src/syscall/posix.rs +++ b/kernel/src/syscall/posix.rs @@ -1020,7 +1020,7 @@ pub fn sys_pthread_mutex_lock(mutex: u64, sys_nr: u64) -> i64 { while atom.load(Ordering::Acquire) != 0 && super::futex_waiter_present(mutex, pid) { #[cfg(target_arch = "x86_64")] unsafe { - core::arch::asm!("sti; hlt; cli", options(nomem, nostack)); + { crate::arch::enable_and_halt(); } } if let Some(next) = mutex_handoff_target() { if next != pid { @@ -1186,7 +1186,7 @@ pub fn sys_pthread_once(once: u64, func: u64, sys_nr: u64) -> i64 { while atom.load(Ordering::Acquire) != 2 && super::futex_waiter_present(once, pid) { #[cfg(target_arch = "x86_64")] unsafe { - core::arch::asm!("sti; hlt; cli", options(nomem, nostack)); + { crate::arch::enable_and_halt(); } } if let Some(next) = crate::process::next_runnable_pid(pid) { if next != pid { @@ -1394,7 +1394,7 @@ pub fn sys_pthread_cond_wait_timeout(cond: u64, mutex: u64, timeout_ns: u64, sys } #[cfg(target_arch = "x86_64")] unsafe { - core::arch::asm!("sti; hlt; cli", options(nomem, nostack)); + { crate::arch::enable_and_halt(); } } if let Some(next) = super::cooperative_sched_target(pid) { if next != pid { @@ -2039,9 +2039,7 @@ pub fn sys_sem_wait(sem: u64) -> i64 { { break; } - unsafe { - core::arch::asm!("pause", options(nomem, nostack, preserves_flags)); - } + crate::arch::spin_pause(); } 0 } @@ -2181,9 +2179,7 @@ pub fn sys_abort() -> ! { super::dump_user_backtrace(16); super::sys_exit((-6i64) as u64); loop { - unsafe { - core::arch::asm!("sti; hlt; cli", options(nomem, nostack)); - } + crate::arch::enable_and_halt(); } } @@ -2220,21 +2216,14 @@ pub fn sys_nanosleep(req: u64, _rem: u64) -> i64 { let secs = unsafe { *(req as *const u64) }; // Yield for approximately secs * 1000 iterations (very rough). for _ in 0..(secs * 1000).min(10000) { - unsafe { - core::arch::asm!("pause", options(nomem, nostack, preserves_flags)); - } + crate::arch::spin_pause(); } 0 } /// Fake monotonic time: use TSC / 1000 as nanoseconds. fn read_tsc() -> u64 { - let lo: u32; - let hi: u32; - unsafe { - core::arch::asm!("rdtsc", out("eax") lo, out("edx") hi, options(nomem, nostack)); - } - ((hi as u64) << 32) | lo as u64 + crate::arch::rdtsc() } pub fn sys_gettimeofday(tv: u64, _tz: u64) -> i64 { From 038ad3b2e64acae7db900f0e8d976a517b2829cd Mon Sep 17 00:00:00 2001 From: Tahiru Agbanwa <130235676+squirelboy360@users.noreply.github.com> Date: Mon, 8 Jun 2026 23:45:31 +0200 Subject: [PATCH 03/36] aarch64: boot to EL1 with PL011 serial (milestone 1) Direct QEMU -kernel boot path for the ARM port. The assembly _start (boot.rs) parks secondary CPUs, drops EL2->EL1 if needed, enables FP/AdvSIMD (CPACR_EL1.FPEN), sets up the boot stack, zeroes BSS, and calls into the Rust bring-up sequence. A minimal polled PL011 UART driver (uart.rs) at 0x09000000 is the serial debug lifeline. The bring-up (bringup.rs) prints CurrentEL, SCTLR_EL1, MPIDR_EL1 and CNTFRQ_EL0 over serial, confirming we land at EL1 with the MMU off and a 62.5MHz generic timer. New aarch64.ld links the kernel for QEMU -M virt RAM at 0x40080000 with _start as the ELF entry. Boots cleanly under qemu-system-aarch64 -M virt -cpu cortex-a72. --- kernel/linker/aarch64.ld | 59 +++++++-- kernel/src/arch/aarch64/boot.rs | 122 ++++++++++++++++++ kernel/src/arch/aarch64/bringup.rs | 62 +++++++++ kernel/src/arch/aarch64/bringup_mmu.rs | 10 ++ kernel/src/arch/aarch64/mod.rs | 6 + kernel/src/arch/aarch64/uart.rs | 169 +++++++++++++++++++++++++ 6 files changed, 419 insertions(+), 9 deletions(-) create mode 100644 kernel/src/arch/aarch64/boot.rs create mode 100644 kernel/src/arch/aarch64/bringup.rs create mode 100644 kernel/src/arch/aarch64/bringup_mmu.rs create mode 100644 kernel/src/arch/aarch64/uart.rs diff --git a/kernel/linker/aarch64.ld b/kernel/linker/aarch64.ld index f538360..f61a98e 100644 --- a/kernel/linker/aarch64.ld +++ b/kernel/linker/aarch64.ld @@ -1,13 +1,54 @@ -/* aarch64 linker script stub */ +/* aarch64 direct-boot linker script (QEMU -M virt -kernel). + * + * QEMU `-M virt` places RAM at 0x4000_0000 and, for an ELF kernel, loads each + * PT_LOAD segment at its physical address and jumps to e_entry with the MMU + * OFF (running on physical == virtual identity at this point). We therefore + * link the kernel to run from a low physical address and enable the MMU later + * with an identity map plus the kernel's higher-half window. + * + * Entry is the assembly `_start` (boot.rs) which sets up the stack, zeroes BSS, + * brings up the PL011 UART for serial, then calls into the Rust bring-up path. + */ OUTPUT_FORMAT("elf64-littleaarch64") -ENTRY(kernel_main) +OUTPUT_ARCH(aarch64) +ENTRY(_start) + +/* QEMU -M virt RAM base. The kernel is loaded here by -kernel. */ +KERNEL_PHYS_BASE = 0x40080000; + SECTIONS { - . = 0xFFFF000000080000; - .text : { *(.text .text.*) } + . = KERNEL_PHYS_BASE; + + .text : ALIGN(4096) { + __text_start = .; + KEEP(*(.text.boot)) /* _start must be first */ + *(.text .text.*) + __text_end = .; + } + + .rodata : ALIGN(4096) { + __rodata_start = .; + *(.rodata .rodata.*) + __rodata_end = .; + } + + .data : ALIGN(4096) { + __data_start = .; + *(.data .data.*) + *(.got .got.*) + __data_end = .; + } + + .bss (NOLOAD) : ALIGN(4096) { + __bss_start = .; + *(.bss .bss.*) + *(COMMON) + . = ALIGN(16); + __bss_end = .; + } + . = ALIGN(4096); - .rodata : { *(.rodata .rodata.*) } - . = ALIGN(4096); - .data : { *(.data .data.*) } - .bss : { __bss_start = .; *(.bss .bss.*) *(COMMON) __bss_end = .; } - /DISCARD/ : { *(.eh_frame) *(.note .note.*) } + __kernel_end = .; + + /DISCARD/ : { *(.eh_frame) *(.eh_frame_hdr) *(.note .note.*) *(.comment) } } diff --git a/kernel/src/arch/aarch64/boot.rs b/kernel/src/arch/aarch64/boot.rs new file mode 100644 index 0000000..c06915a --- /dev/null +++ b/kernel/src/arch/aarch64/boot.rs @@ -0,0 +1,122 @@ +//! aarch64 direct-boot entry path (QEMU `-M virt -kernel`). +//! +//! This is the bare-metal bring-up entry for the ARM port. QEMU loads the +//! kernel ELF at its physical link address (0x4008_0000 on `-M virt`) and jumps +//! to `_start` with the MMU OFF, caches off, and (for `-cpu cortex-a72` without +//! `virtualization=on`) at EL1. `_start`: +//! +//! 1. Parks all but the primary CPU (MPIDR affinity 0). +//! 2. If we entered at EL2, drops to EL1 via `eret`. +//! 3. Sets up the boot stack. +//! 4. Zeroes the BSS. +//! 5. Branches to `aarch64_start` (Rust), which brings up serial → MMU → +//! exceptions → GIC → timer → SVC → an EL0 user process. +//! +//! The whole bring-up runs identity-mapped low-physical until the MMU is on. + +use core::arch::global_asm; + +global_asm!( + r#" +.section .text.boot, "ax" +.globl _start +_start: + // ── Park secondary CPUs: only MPIDR.Aff0==0 proceeds ────────────────── + mrs x0, mpidr_el1 + and x0, x0, #0xFF + cbz x0, 1f +0: wfe + b 0b +1: + // ── If we booted at EL2, drop to EL1 ────────────────────────────────── + mrs x0, CurrentEL + lsr x0, x0, #2 + cmp x0, #2 + b.ne 2f // already at EL1 (or EL3 — unhandled) + + // Configure EL2 so EL1 runs in AArch64 with sane state, then eret to EL1. + mov x0, #(1 << 31) // HCR_EL2.RW = 1 → EL1 is AArch64 + msr hcr_el2, x0 + // SCTLR_EL1: reserved-1 bits set, MMU/caches off for now. + mov x0, #0x0800 + movk x0, #0x30d0, lsl #16 + msr sctlr_el1, x0 + // SPSR_EL2: mask DAIF, return to EL1h (use SP_EL1). + mov x0, #0x3c5 // D,A,I,F masked + mode EL1h(0b0101) + msr spsr_el2, x0 + adr x0, 2f + msr elr_el2, x0 + eret + +2: // ── Now at EL1 ──────────────────────────────────────────────────────── + // Mask all interrupts during bring-up. + msr daifset, #0xf + + // Enable FP/AdvSIMD at EL1/EL0 (CPACR_EL1.FPEN = 0b11). The Rust codegen + // emits SIMD/FP instructions even in early bring-up (e.g. alignment + // ub-checks use `fmov`), which trap to EL1 unless FPEN is cleared. + mrs x0, cpacr_el1 + orr x0, x0, #(3 << 20) + msr cpacr_el1, x0 + isb + + // Set up the boot stack (top of a BSS-resident stack region). + adrp x0, {boot_stack_top} + add x0, x0, :lo12:{boot_stack_top} + mov sp, x0 + + // ── Zero BSS ────────────────────────────────────────────────────────── + adrp x1, {bss_start} + add x1, x1, :lo12:{bss_start} + adrp x2, {bss_end} + add x2, x2, :lo12:{bss_end} +3: cmp x1, x2 + b.hs 4f + str xzr, [x1], #8 + b 3b +4: + // ── Into Rust ───────────────────────────────────────────────────────── + bl {aarch64_start} + // aarch64_start never returns; park if it does. +5: wfe + b 5b +"#, + boot_stack_top = sym BOOT_STACK_TOP, + bss_start = sym crate::arch::aarch64::boot::__bss_start_sym, + bss_end = sym crate::arch::aarch64::boot::__bss_end_sym, + aarch64_start = sym aarch64_start, +); + +// ── Boot stack (BSS-resident, 64 KiB) ─────────────────────────────────────── +// +// Defined in assembly so the "top" label sits exactly one-past-the-end of the +// reserved region (the stack grows down from BOOT_STACK_TOP). +core::arch::global_asm!( + r#" +.section .bss.boot_stack, "aw", @nobits +.balign 16 +BOOT_STACK_BOTTOM: + .skip 65536 +.globl BOOT_STACK_TOP +BOOT_STACK_TOP: +"# +); + +extern "C" { + static BOOT_STACK_TOP: u8; +} + +// Re-export the linker BSS bounds as Rust symbols the global_asm can reference. +extern "C" { + #[link_name = "__bss_start"] + pub static __bss_start_sym: u8; + #[link_name = "__bss_end"] + pub static __bss_end_sym: u8; +} + +/// First Rust function on the aarch64 boot path. Runs identity-mapped, MMU off, +/// at EL1, on the boot stack. Hands off to the bring-up sequence. +#[no_mangle] +pub extern "C" fn aarch64_start() -> ! { + crate::arch::aarch64::bringup::run() +} diff --git a/kernel/src/arch/aarch64/bringup.rs b/kernel/src/arch/aarch64/bringup.rs new file mode 100644 index 0000000..d6259b1 --- /dev/null +++ b/kernel/src/arch/aarch64/bringup.rs @@ -0,0 +1,62 @@ +//! aarch64 bare-metal bring-up sequence. +//! +//! Driven from the assembly `_start` (boot.rs) via `aarch64_start`. Runs the +//! platform-primitive milestones in dependency order, printing progress over +//! PL011 serial at every step: +//! +//! 1. SERIAL — PL011 UART online (the debug lifeline) +//! 2. MMU — ARMv8 translation tables, MAIR/TCR, SCTLR_EL1.M=1 +//! 3. VECTORS — VBAR_EL1 → 16-entry EL1 vector table, trap frame save/restore +//! 4. GIC — distributor + CPU interface, EOI, timer IRQ routing +//! 5. TIMER — CNTP/CNTV periodic tick → scheduler-tick analogue +//! 6. SVC — EL0 `svc #0` → capture user GPRs → dispatch → ERET +//! 7. EL0 — enter a user process and service its syscalls +//! +//! This is a self-contained proof of the ARM platform layer: it exercises every +//! arch primitive the shared kernel needs, without depending on the Limine boot +//! protocol (which the x86 production path uses). Once these primitives are +//! proven, the shared scheduler/mm/syscall layers can be brought up on top. + +use crate::arch::aarch64::uart; +use crate::arch::aarch64::uart::a64println; + +/// Entry from `aarch64_start`. Never returns. +pub fn run() -> ! { + // ── Milestone 1: SERIAL ──────────────────────────────────────────────── + uart::init(); + a64println!(); + a64println!("========================================"); + a64println!(" OSCortex aarch64 bring-up"); + a64println!("========================================"); + a64println!("[boot] PL011 UART online @ 0x09000000"); + + // Report the Exception Level we landed at and key system registers, so the + // serial log alone tells us the CPU state we're bringing up from. + let current_el: u64; + let sctlr: u64; + let mpidr: u64; + let cntfrq: u64; + unsafe { + core::arch::asm!("mrs {}, CurrentEL", out(reg) current_el, options(nomem, nostack)); + core::arch::asm!("mrs {}, sctlr_el1", out(reg) sctlr, options(nomem, nostack)); + core::arch::asm!("mrs {}, mpidr_el1", out(reg) mpidr, options(nomem, nostack)); + core::arch::asm!("mrs {}, cntfrq_el0", out(reg) cntfrq, options(nomem, nostack)); + } + a64println!("[boot] CurrentEL = EL{}", (current_el >> 2) & 0x3); + uart::puts("[boot] SCTLR_EL1 = "); + uart::puthex_full(sctlr); + uart::puts(" (MMU "); + uart::puts(if sctlr & 1 != 0 { "ON" } else { "OFF" }); + uart::puts(")\n"); + uart::puts("[boot] MPIDR_EL1 = "); + uart::puthex_full(mpidr); + uart::puts("\n"); + uart::puts("[boot] CNTFRQ_EL0 = "); + uart::puthex(cntfrq); + a64println!(" ({} Hz timer)", cntfrq); + + a64println!("[boot] Milestone 1 (SERIAL) complete."); + + // Remaining milestones are wired in subsequent steps. + crate::arch::aarch64::bringup_mmu::run(); +} diff --git a/kernel/src/arch/aarch64/bringup_mmu.rs b/kernel/src/arch/aarch64/bringup_mmu.rs new file mode 100644 index 0000000..651d969 --- /dev/null +++ b/kernel/src/arch/aarch64/bringup_mmu.rs @@ -0,0 +1,10 @@ +//! aarch64 bring-up milestone 2: MMU enable (placeholder — filled in next step). + +use crate::arch::aarch64::uart::a64println; + +pub fn run() -> ! { + a64println!("[boot] (MMU milestone not yet wired — parking)"); + loop { + unsafe { core::arch::asm!("wfe", options(nomem, nostack)) }; + } +} diff --git a/kernel/src/arch/aarch64/mod.rs b/kernel/src/arch/aarch64/mod.rs index 4cf0b7b..b65db14 100644 --- a/kernel/src/arch/aarch64/mod.rs +++ b/kernel/src/arch/aarch64/mod.rs @@ -25,6 +25,12 @@ pub mod acpi; pub mod smp; pub mod enter_user; +// ── Bare-metal bring-up (direct `-kernel` boot) ───────────────────────────── +pub mod uart; +pub mod boot; +pub mod bringup; +pub mod bringup_mmu; + pub use enter_user::{enter_user_iret, enter_user_sysret, EnterUserRegs}; use core::arch::asm; diff --git a/kernel/src/arch/aarch64/uart.rs b/kernel/src/arch/aarch64/uart.rs new file mode 100644 index 0000000..e4a78cd --- /dev/null +++ b/kernel/src/arch/aarch64/uart.rs @@ -0,0 +1,169 @@ +//! PL011 UART driver — aarch64 serial console (QEMU `-M virt` @ 0x0900_0000). +//! +//! This is the bring-up printf/debug lifeline. QEMU's `-M virt` machine wires a +//! PrimeCell PL011 UART at physical 0x0900_0000; with `-serial mon:stdio` its +//! TX shows up on the host terminal. We only need polled TX (and optional RX) +//! for bring-up, so this is deliberately minimal. +//! +//! The driver works both before and after the MMU is enabled: pre-MMU the base +//! is a raw physical address; once the MMU is on we map the same physical page +//! at an identity/uncached VA and keep using it (the bring-up identity-maps the +//! device region, so the same address stays valid). + +use core::sync::atomic::{AtomicU64, Ordering}; + +/// PL011 base on QEMU `-M virt`. Overridable once the MMU remaps it. +const PL011_PHYS: u64 = 0x0900_0000; + +/// Active MMIO base for the UART (physical until the MMU is on; then the VA the +/// device page was identity-mapped to — same value with an identity map). +static UART_BASE: AtomicU64 = AtomicU64::new(PL011_PHYS); + +// PL011 register offsets. +const UARTDR: u64 = 0x00; // Data register +const UARTFR: u64 = 0x18; // Flag register +const UARTIBRD: u64 = 0x24; // Integer baud-rate divisor +const UARTFBRD: u64 = 0x28; // Fractional baud-rate divisor +const UARTLCR_H: u64 = 0x2c; // Line control +const UARTCR: u64 = 0x30; // Control +const UARTIMSC: u64 = 0x38; // Interrupt mask set/clear + +// UARTFR bits. +const FR_TXFF: u32 = 1 << 5; // transmit FIFO full +const FR_RXFE: u32 = 1 << 4; // receive FIFO empty + +#[inline(always)] +fn base() -> u64 { + UART_BASE.load(Ordering::Relaxed) +} + +#[inline(always)] +unsafe fn reg_read(off: u64) -> u32 { + core::ptr::read_volatile((base() + off) as *const u32) +} + +#[inline(always)] +unsafe fn reg_write(off: u64, val: u32) { + core::ptr::write_volatile((base() + off) as *mut u32, val); +} + +/// Initialise the PL011 for polled 115200-8N1 TX. Safe to call before the MMU +/// is on. QEMU ignores the exact baud divisor, but real hardware needs it. +pub fn init() { + unsafe { + // Disable the UART while reconfiguring. + reg_write(UARTCR, 0); + // Mask all interrupts (we poll). + reg_write(UARTIMSC, 0); + // 115200 baud assuming a 24 MHz UARTCLK (QEMU virt): divisor = 24e6 / + // (16*115200) = 13.0208 → IBRD=13, FBRD=round(0.0208*64)=1. + reg_write(UARTIBRD, 13); + reg_write(UARTFBRD, 1); + // 8 bits, FIFO enabled, no parity. + reg_write(UARTLCR_H, (1 << 4) | (3 << 5)); + // Enable UART + TX + RX. + reg_write(UARTCR, (1 << 0) | (1 << 8) | (1 << 9)); + } +} + +/// Point the driver at a new base address (used after the MMU remaps the device +/// page to a non-identity VA, if ever). For an identity map this is a no-op. +pub fn set_base(new_base: u64) { + UART_BASE.store(new_base, Ordering::Release); +} + +/// Write one byte, blocking while the TX FIFO is full. +#[inline] +pub fn putb(b: u8) { + unsafe { + while reg_read(UARTFR) & FR_TXFF != 0 { + core::hint::spin_loop(); + } + reg_write(UARTDR, b as u32); + } +} + +/// Write a string, expanding `\n` → `\r\n`. +pub fn puts(s: &str) { + for &b in s.as_bytes() { + if b == b'\n' { + putb(b'\r'); + } + putb(b); + } +} + +/// Write a 64-bit value as `0x`-prefixed, zero-padded hex. +pub fn puthex(mut v: u64) { + const HEX: &[u8; 16] = b"0123456789abcdef"; + putb(b'0'); + putb(b'x'); + let mut started = false; + for i in (0..16).rev() { + let nib = ((v >> (i * 4)) & 0xf) as usize; + if nib != 0 || started || i == 0 { + putb(HEX[nib]); + started = true; + } + } + let _ = &mut v; +} + +/// Write a u64 as `0x`-prefixed, full-width (16 nibble) hex — handy for raw +/// register dumps where leading zeros matter. +pub fn puthex_full(v: u64) { + const HEX: &[u8; 16] = b"0123456789abcdef"; + putb(b'0'); + putb(b'x'); + for i in (0..16).rev() { + putb(HEX[((v >> (i * 4)) & 0xf) as usize]); + } +} + +/// Non-blocking single-byte read. Returns `None` when the RX FIFO is empty. +pub fn try_getb() -> Option { + unsafe { + if reg_read(UARTFR) & FR_RXFE != 0 { + None + } else { + Some((reg_read(UARTDR) & 0xff) as u8) + } + } +} + +/// `core::fmt::Write` adapter so bring-up code can `write!`/`writeln!`. +pub struct Writer; + +impl core::fmt::Write for Writer { + fn write_str(&mut self, s: &str) -> core::fmt::Result { + puts(s); + Ok(()) + } +} + +/// Formatted write to the UART (the bring-up `print!` analogue). +pub fn _fmt(args: core::fmt::Arguments) { + use core::fmt::Write as _; + let _ = Writer.write_fmt(args); +} + +/// `println!`-style helper for the bring-up path. Defined as a plain macro +/// (not `#[macro_export]`) so it stays addressable within the arch module tree +/// without tripping the absolute-path lint on macro-expanded exports. +macro_rules! a64println { + () => { $crate::arch::aarch64::uart::puts("\n") }; + ($($arg:tt)*) => {{ + $crate::arch::aarch64::uart::_fmt(format_args!($($arg)*)); + $crate::arch::aarch64::uart::puts("\n"); + }}; +} +pub(crate) use a64println; + +/// `print!`-style helper for the bring-up path. +macro_rules! a64print { + ($($arg:tt)*) => {{ + $crate::arch::aarch64::uart::_fmt(format_args!($($arg)*)); + }}; +} +#[allow(unused_imports)] +pub(crate) use a64print; From 68a1148fbe87d8d8270401436ed0266d13156d65 Mon Sep 17 00:00:00 2001 From: Tahiru Agbanwa <130235676+squirelboy360@users.noreply.github.com> Date: Mon, 8 Jun 2026 23:50:16 +0200 Subject: [PATCH 04/36] aarch64: enable the MMU with identity-mapped translation tables (milestone 2) ARMv8-A MMU bring-up (mmu.rs): builds 512-entry L1 translation tables with 1 GiB block descriptors for both TTBR0 (low/user) and TTBR1 (kernel high half), programs MAIR_EL1 (Normal WB + Device-nGnRE), TCR_EL1 and enables SCTLR_EL1.M/C/I. Two bring-up subtleties resolved on real QEMU: - T0SZ/T1SZ = 25 (39-bit VA) so the 4 KiB-granule walk starts at L1, letting a single L1 table cover the whole space with 1 GiB blocks (T0SZ=16/48-bit would have required an L0 table -> level-0 fault). - Kernel identity map uses AP=EL1-only; an EL0-writable kernel-code mapping implicitly forces PXN, which tripped a level-1 permission fault on the first translated instruction fetch. Verified on qemu-system-aarch64: SCTLR_EL1=0xc5183d (M/C/I on), a RAM read/write probe round-trips, and execution continues translated past the enable. --- kernel/src/arch/aarch64/bringup_mmu.rs | 42 ++++- kernel/src/arch/aarch64/bringup_vectors.rs | 10 ++ kernel/src/arch/aarch64/mmu.rs | 173 +++++++++++++++++++++ kernel/src/arch/aarch64/mod.rs | 2 + 4 files changed, 223 insertions(+), 4 deletions(-) create mode 100644 kernel/src/arch/aarch64/bringup_vectors.rs create mode 100644 kernel/src/arch/aarch64/mmu.rs diff --git a/kernel/src/arch/aarch64/bringup_mmu.rs b/kernel/src/arch/aarch64/bringup_mmu.rs index 651d969..a95be7b 100644 --- a/kernel/src/arch/aarch64/bringup_mmu.rs +++ b/kernel/src/arch/aarch64/bringup_mmu.rs @@ -1,10 +1,44 @@ -//! aarch64 bring-up milestone 2: MMU enable (placeholder — filled in next step). +//! aarch64 bring-up milestone 2: enable the MMU. +use crate::arch::aarch64::mmu; +use crate::arch::aarch64::uart; use crate::arch::aarch64::uart::a64println; pub fn run() -> ! { - a64println!("[boot] (MMU milestone not yet wired — parking)"); - loop { - unsafe { core::arch::asm!("wfe", options(nomem, nostack)) }; + a64println!("[boot] Milestone 2 (MMU): building translation tables..."); + + // Identity-map device MMIO (low 1 GiB) + 2 GiB of RAM from 0x4000_0000. + unsafe { mmu::enable(2) }; + + // Read SCTLR_EL1 back to confirm M=1. If we get here at all, the MMU is on + // and translating — the very next instruction fetch was a translated VA. + let sctlr: u64; + let ttbr0: u64; + unsafe { + core::arch::asm!("mrs {}, sctlr_el1", out(reg) sctlr, options(nomem, nostack)); + core::arch::asm!("mrs {}, ttbr0_el1", out(reg) ttbr0, options(nomem, nostack)); + } + uart::puts("[boot] SCTLR_EL1 = "); + uart::puthex_full(sctlr); + uart::puts(" (MMU "); + uart::puts(if sctlr & 1 != 0 { "ON" } else { "OFF" }); + uart::puts(")\n"); + uart::puts("[boot] TTBR0_EL1 = "); + uart::puthex_full(ttbr0); + uart::puts("\n"); + + // Prove translation works: a read+write through a RAM VA round-trips. (If + // the map were wrong this access would take a translation fault.) + let probe: u64 = 0x4010_0000; // 1 MiB into RAM, page-aligned, scratch + unsafe { + core::ptr::write_volatile(probe as *mut u64, 0xCAFE_F00D_1234_5678); + let v = core::ptr::read_volatile(probe as *const u64); + uart::puts("[boot] RAM r/w probe @ 0x40100000 = "); + uart::puthex_full(v); + uart::puts(if v == 0xCAFE_F00D_1234_5678 { " OK\n" } else { " FAIL\n" }); } + + a64println!("[boot] Milestone 2 (MMU) complete — translating."); + + crate::arch::aarch64::bringup_vectors::run(); } diff --git a/kernel/src/arch/aarch64/bringup_vectors.rs b/kernel/src/arch/aarch64/bringup_vectors.rs new file mode 100644 index 0000000..c05a582 --- /dev/null +++ b/kernel/src/arch/aarch64/bringup_vectors.rs @@ -0,0 +1,10 @@ +//! aarch64 bring-up milestone 3: exception vectors (placeholder). + +use crate::arch::aarch64::uart::a64println; + +pub fn run() -> ! { + a64println!("[boot] (VECTORS milestone not yet wired — parking)"); + loop { + unsafe { core::arch::asm!("wfe", options(nomem, nostack)) }; + } +} diff --git a/kernel/src/arch/aarch64/mmu.rs b/kernel/src/arch/aarch64/mmu.rs new file mode 100644 index 0000000..c68aa9e --- /dev/null +++ b/kernel/src/arch/aarch64/mmu.rs @@ -0,0 +1,173 @@ +//! ARMv8-A MMU bring-up — translation tables + SCTLR_EL1.M enable. +//! +//! Configuration (the simplest correct setup for QEMU `-M virt`): +//! * 4 KiB granule, 48-bit VA (T0SZ = T1SZ = 16). +//! * TTBR0_EL1 walks the low half (identity map covering devices + RAM). +//! * TTBR1_EL1 walks the kernel high half (mirror of the same identity map +//! at 0xFFFF_0000_0000_0000+ so the kernel can later run higher-half). +//! * MAIR attr0 = Normal Write-Back, attr1 = Device-nGnRE. +//! +//! We use L1 1 GiB block descriptors — the whole address space is covered by a +//! single 512-entry L1 table per TTBR, no L2/L3 needed for the identity map. +//! Region 0 (0x0–0x4000_0000) holds the GIC/UART MMIO and is mapped Device; +//! regions 1..N (RAM from 0x4000_0000) are mapped Normal cacheable. +//! +//! After this runs the MMU is ON and the kernel executes translated (the +//! identity map means VAs equal PAs, so code/stack/UART pointers stay valid). + +use core::sync::atomic::{AtomicBool, Ordering}; + +// ── Descriptor + register bit definitions ─────────────────────────────────── + +/// L1 block descriptor: valid, block (bit[1]=0), AF, plus attr/shareability. +const DESC_VALID: u64 = 1 << 0; +const DESC_BLOCK: u64 = 0 << 1; // 0 = block at L1/L2; 1 = table/page +const DESC_AF: u64 = 1 << 10; // Access Flag (else first access faults) +const DESC_SH_INNER: u64 = 3 << 8; // Inner shareable +const DESC_AP_RW_EL1: u64 = 0 << 6; // EL1 RW, EL0 no access +const DESC_AP_RW_ALL: u64 = 1 << 6; // EL1 RW, EL0 RW + +// AttrIndx[2:0] selects a MAIR attr byte. +const ATTR_IDX_NORMAL: u64 = 0 << 2; // MAIR attr0 +const ATTR_IDX_DEVICE: u64 = 1 << 2; // MAIR attr1 + +/// MAIR_EL1: attr0 = Normal Inner/Outer Write-Back non-transient (0xFF), +/// attr1 = Device-nGnRE (0x04). +const MAIR_VALUE: u64 = 0x04 << 8 | 0xFF; + +/// TCR_EL1 for 4 KiB granule, 39-bit VA on both halves, WB-cacheable walks, +/// inner-shareable. IPS = 40-bit (1 TiB) physical, plenty for QEMU virt. +/// +/// T0SZ/T1SZ = 25 → 39-bit VA (512 GiB). With a 4 KiB granule this makes the +/// translation walk *start at level 1*, so a single 512-entry L1 table with +/// 1 GiB block descriptors covers the whole address space — no L0 needed. +const TCR_T0SZ: u64 = 25; // 64-25 = 39-bit VA (TTBR0) +const TCR_T1SZ: u64 = 25 << 16; // 39-bit VA (TTBR1) +const TCR_IRGN0_WB: u64 = 1 << 8; +const TCR_ORGN0_WB: u64 = 1 << 10; +const TCR_SH0_INNER: u64 = 3 << 12; +const TCR_TG0_4K: u64 = 0 << 14; // 4 KiB granule (TTBR0) +const TCR_IRGN1_WB: u64 = 1 << 24; +const TCR_ORGN1_WB: u64 = 1 << 26; +const TCR_SH1_INNER: u64 = 3 << 28; +const TCR_TG1_4K: u64 = 2 << 30; // 4 KiB granule (TTBR1) +const TCR_IPS_40BIT: u64 = 2 << 32; + +const TCR_VALUE: u64 = TCR_T0SZ + | TCR_T1SZ + | TCR_IRGN0_WB + | TCR_ORGN0_WB + | TCR_SH0_INNER + | TCR_TG0_4K + | TCR_IRGN1_WB + | TCR_ORGN1_WB + | TCR_SH1_INNER + | TCR_TG1_4K + | TCR_IPS_40BIT; + +// SCTLR_EL1 bits. +const SCTLR_M: u64 = 1 << 0; // MMU enable +const SCTLR_C: u64 = 1 << 2; // data cache enable +const SCTLR_I: u64 = 1 << 12; // instruction cache enable + +// ── Translation tables (BSS-resident, page-aligned) ───────────────────────── + +/// A single 512-entry, 4 KiB-aligned translation table. +#[repr(C, align(4096))] +struct Table([u64; 512]); + +/// L1 table for the low half (TTBR0): identity map. +static mut L1_LOW: Table = Table([0; 512]); +/// L1 table for the high half (TTBR1): identity map mirror. +static mut L1_HIGH: Table = Table([0; 512]); + +static MMU_ON: AtomicBool = AtomicBool::new(false); + +/// Build an L1 block descriptor mapping the 1 GiB region at `pa` with the given +/// attribute index and access permissions. +#[inline] +fn block_desc(pa: u64, attr_idx: u64, ap: u64) -> u64 { + (pa & 0x0000_FFFF_C000_0000) // 1 GiB-aligned output address (bits[47:30]) + | DESC_VALID + | DESC_BLOCK + | DESC_AF + | DESC_SH_INNER + | attr_idx + | ap +} + +/// Populate one L1 table with an identity map: index 0 (0x0–0x4000_0000) is +/// Device memory (covers GIC @ 0x0800_0000 and PL011 @ 0x0900_0000), indices +/// 1..=`ram_gibs` are Normal cacheable RAM starting at 0x4000_0000. +unsafe fn fill_identity(table: &mut Table, ram_gibs: u64) { + // Entry 0: low 1 GiB = device MMIO (also covers the flash/boot ROM region). + // Kernel-only (EL1 RW), Device-nGnRE. + table.0[0] = block_desc(0x0000_0000, ATTR_IDX_DEVICE, DESC_AP_RW_EL1); + // RAM region: 0x4000_0000 onward, one 1 GiB block per L1 entry. Kernel-only + // RW+executable (EL0 has no access to the kernel identity map). + let mut g = 1u64; + while g <= ram_gibs && g < 512 { + let pa = g << 30; // g GiB + table.0[g as usize] = block_desc(pa, ATTR_IDX_NORMAL, DESC_AP_RW_EL1); + g += 1; + } +} + +/// Returns the physical address of the active TTBR0 L1 table (for the shared +/// `arch::memory::read_cr3` view). Until per-process tables exist, all address +/// spaces share this identity map. +pub fn kernel_ttbr0() -> u64 { + core::ptr::addr_of!(L1_LOW) as u64 +} + +/// Build the tables, program MAIR/TCR/TTBR, and enable the MMU + caches. +/// +/// `ram_gibs` is how many GiB of RAM (from 0x4000_0000) to identity-map. +/// +/// # Safety +/// Must run once, at EL1, with the MMU OFF and interrupts masked. +pub unsafe fn enable(ram_gibs: u64) { + let l1_low = &mut *core::ptr::addr_of_mut!(L1_LOW); + let l1_high = &mut *core::ptr::addr_of_mut!(L1_HIGH); + fill_identity(l1_low, ram_gibs); + fill_identity(l1_high, ram_gibs); + + let ttbr0 = core::ptr::addr_of!(L1_LOW) as u64; + let ttbr1 = core::ptr::addr_of!(L1_HIGH) as u64; + + core::arch::asm!( + // Program memory attributes + translation control. + "msr mair_el1, {mair}", + "msr tcr_el1, {tcr}", + "msr ttbr0_el1, {ttbr0}", + "msr ttbr1_el1, {ttbr1}", + "isb", + // Invalidate all TLB entries for EL1, then barrier. + "tlbi vmalle1", + "dsb nsh", + "isb", + mair = in(reg) MAIR_VALUE, + tcr = in(reg) TCR_VALUE, + ttbr0 = in(reg) ttbr0, + ttbr1 = in(reg) ttbr1, + options(nostack, preserves_flags), + ); + + // Enable MMU + I/D caches in SCTLR_EL1. + let mut sctlr: u64; + core::arch::asm!("mrs {}, sctlr_el1", out(reg) sctlr, options(nomem, nostack)); + sctlr |= SCTLR_M | SCTLR_C | SCTLR_I; + core::arch::asm!( + "msr sctlr_el1, {}", + "isb", + in(reg) sctlr, + options(nostack, preserves_flags), + ); + + MMU_ON.store(true, Ordering::Release); +} + +/// True once [`enable`] has turned the MMU on. +pub fn is_on() -> bool { + MMU_ON.load(Ordering::Acquire) +} diff --git a/kernel/src/arch/aarch64/mod.rs b/kernel/src/arch/aarch64/mod.rs index b65db14..4c4f40a 100644 --- a/kernel/src/arch/aarch64/mod.rs +++ b/kernel/src/arch/aarch64/mod.rs @@ -28,8 +28,10 @@ pub mod enter_user; // ── Bare-metal bring-up (direct `-kernel` boot) ───────────────────────────── pub mod uart; pub mod boot; +pub mod mmu; pub mod bringup; pub mod bringup_mmu; +pub mod bringup_vectors; pub use enter_user::{enter_user_iret, enter_user_sysret, EnterUserRegs}; From f3e9f57790d773c70362b03477202675d6ce3f91 Mon Sep 17 00:00:00 2001 From: Tahiru Agbanwa <130235676+squirelboy360@users.noreply.github.com> Date: Mon, 8 Jun 2026 23:52:52 +0200 Subject: [PATCH 05/36] aarch64: EL1 exception vectors + trap frame round-trip (milestone 3) Install a 16-entry, 2KB-aligned VBAR_EL1 vector table (vectors.rs) covering all four groups (Current-EL SP0/SPx, Lower-EL AArch64/32) x four kinds (Sync/IRQ/FIQ/SError). Each entry saves the full integer register file (x0-x30) plus SP_EL0, ELR_EL1, SPSR_EL1 and ESR_EL1 into a TrapFrame, calls a Rust dispatcher tagged with the exception kind, then restores everything and erets. The dispatcher decodes ESR_EL1.EC and routes SVC64 to an installable handler, IRQs to an installable IRQ handler, and reports+parks on any unhandled exception (printing ESR/ELR/SPSR/FAR over serial). Verified on QEMU: a deliberate from EL1 round-trips through save/dispatch/restore/eret (SYNC_EL1 0->1, handler hit once) and a callee-saved sentinel register survives the trap intact. --- kernel/src/arch/aarch64/bringup_irq.rs | 10 + kernel/src/arch/aarch64/bringup_vectors.rs | 61 +++- kernel/src/arch/aarch64/mod.rs | 2 + kernel/src/arch/aarch64/vectors.rs | 351 +++++++++++++++++++++ 4 files changed, 420 insertions(+), 4 deletions(-) create mode 100644 kernel/src/arch/aarch64/bringup_irq.rs create mode 100644 kernel/src/arch/aarch64/vectors.rs diff --git a/kernel/src/arch/aarch64/bringup_irq.rs b/kernel/src/arch/aarch64/bringup_irq.rs new file mode 100644 index 0000000..c15b4ef --- /dev/null +++ b/kernel/src/arch/aarch64/bringup_irq.rs @@ -0,0 +1,10 @@ +//! aarch64 bring-up milestones 4+5: GIC + generic timer (placeholder). + +use crate::arch::aarch64::uart::a64println; + +pub fn run() -> ! { + a64println!("[boot] (GIC/TIMER milestone not yet wired — parking)"); + loop { + unsafe { core::arch::asm!("wfe", options(nomem, nostack)) }; + } +} diff --git a/kernel/src/arch/aarch64/bringup_vectors.rs b/kernel/src/arch/aarch64/bringup_vectors.rs index c05a582..ae4b6d7 100644 --- a/kernel/src/arch/aarch64/bringup_vectors.rs +++ b/kernel/src/arch/aarch64/bringup_vectors.rs @@ -1,10 +1,63 @@ -//! aarch64 bring-up milestone 3: exception vectors (placeholder). +//! aarch64 bring-up milestone 3: EL1 exception vectors. +use crate::arch::aarch64::uart; use crate::arch::aarch64::uart::a64println; +use crate::arch::aarch64::vectors; +use core::sync::atomic::{AtomicU64, Ordering}; + +/// Test SVC handler: bumps a counter and returns (the vector restores+erets). +static TEST_SVC_HITS: AtomicU64 = AtomicU64::new(0); + +fn test_svc_handler(_f: &mut vectors::TrapFrame) { + TEST_SVC_HITS.fetch_add(1, Ordering::Relaxed); +} pub fn run() -> ! { - a64println!("[boot] (VECTORS milestone not yet wired — parking)"); - loop { - unsafe { core::arch::asm!("wfe", options(nomem, nostack)) }; + a64println!("[boot] Milestone 3 (VECTORS): installing VBAR_EL1..."); + vectors::install(); + uart::puts("[boot] VBAR_EL1 = "); + uart::puthex_full(vectors::vbar()); + uart::puts("\n"); + + // Prove the full vector round-trip with a *recoverable* trap: issue + // `svc #0` from EL1. The CPU vectors to Current-EL Synchronous, the asm + // stub saves a TrapFrame, the dispatcher sees EC=SVC64 and runs our test + // handler (which bumps a counter), then the stub restores every register + // and `eret`s back to the instruction after the `svc`. If the save/restore + // were wrong we'd fault here; instead execution continues normally. + vectors::set_svc_handler(test_svc_handler); + + let before = vectors::SYNC_EL1.load(Ordering::Relaxed); + // Mark a register before the trap and check it survives the round-trip. + let sentinel_in: u64 = 0xA5A5_1234_DEAD_BEEF; + let sentinel_out: u64; + unsafe { + core::arch::asm!( + "mov x9, {inv}", + "svc #0", + "mov {outv}, x9", + inv = in(reg) sentinel_in, + outv = out(reg) sentinel_out, + out("x9") _, + options(nomem, nostack), + ); } + let after = vectors::SYNC_EL1.load(Ordering::Relaxed); + let hits = TEST_SVC_HITS.load(Ordering::Relaxed); + + a64println!( + "[boot] svc#0 round-trip: SYNC_EL1 {} -> {}, handler hits = {}", + before, after, hits + ); + uart::puts("[boot] callee reg preserved across trap: "); + uart::puthex_full(sentinel_out); + uart::puts(if sentinel_out == sentinel_in { " OK\n" } else { " CORRUPT\n" }); + + if after == before + 1 && hits == 1 && sentinel_out == sentinel_in { + a64println!("[boot] Milestone 3 (VECTORS) complete — trap save/dispatch/restore/eret OK."); + } else { + a64println!("[boot] Milestone 3 (VECTORS) FAILED — trap round-trip mismatch."); + } + + crate::arch::aarch64::bringup_irq::run(); } diff --git a/kernel/src/arch/aarch64/mod.rs b/kernel/src/arch/aarch64/mod.rs index 4c4f40a..e93eadb 100644 --- a/kernel/src/arch/aarch64/mod.rs +++ b/kernel/src/arch/aarch64/mod.rs @@ -29,9 +29,11 @@ pub mod enter_user; pub mod uart; pub mod boot; pub mod mmu; +pub mod vectors; pub mod bringup; pub mod bringup_mmu; pub mod bringup_vectors; +pub mod bringup_irq; pub use enter_user::{enter_user_iret, enter_user_sysret, EnterUserRegs}; diff --git a/kernel/src/arch/aarch64/vectors.rs b/kernel/src/arch/aarch64/vectors.rs new file mode 100644 index 0000000..5d62a3c --- /dev/null +++ b/kernel/src/arch/aarch64/vectors.rs @@ -0,0 +1,351 @@ +//! EL1 exception vector table (VBAR_EL1) + trap-frame entry/exit. +//! +//! AArch64 dispatches all exceptions through a 16-entry, 2 KiB-aligned vector +//! table. Each entry is 0x80 bytes (32 instructions) and corresponds to one of +//! 4 groups × 4 kinds: +//! +//! group 0: Current EL with SP_EL0 (we never run on SP_EL0 → unused) +//! group 1: Current EL with SP_ELx (kernel-mode exceptions/IRQs) +//! group 2: Lower EL using AArch64 (EL0 syscalls/IRQs — the user path) +//! group 3: Lower EL using AArch32 (unused — we are AArch64-only) +//! +//! kinds: +0x000 Synchronous, +0x080 IRQ, +0x100 FIQ, +0x180 SError +//! +//! Each vector saves the full integer register file (x0–x30) plus SP_EL0, +//! ELR_EL1 and SPSR_EL1 into a [`TrapFrame`] on the current stack, calls the +//! Rust dispatcher with a pointer to it, then restores and `eret`s. The frame +//! layout is shared with `enter_user` so user-entry and exception-return use one +//! ABI. + +use core::arch::global_asm; +use core::sync::atomic::{AtomicU64, Ordering}; + +/// Full integer register context saved on every EL1 exception entry. +/// +/// Layout is fixed and matched byte-for-byte by the asm save/restore in +/// `vectors.S`-equivalent `global_asm!` below. Total size: 36 × 8 = 288 bytes +/// (34 fields + ELR/SPSR), kept 16-byte aligned. +#[repr(C)] +#[derive(Debug, Default, Clone, Copy)] +pub struct TrapFrame { + pub x: [u64; 31], // x0..x30 (x30 = LR) + pub sp_el0: u64, // user stack pointer (SP_EL0) + pub elr: u64, // ELR_EL1 (return address) + pub spsr: u64, // SPSR_EL1 (saved PSTATE) + pub esr: u64, // ESR_EL1 (syndrome — filled by the dispatcher path) +} + +/// Count of each exception kind seen, for bring-up visibility. +pub static SYNC_EL1: AtomicU64 = AtomicU64::new(0); +pub static IRQ_COUNT: AtomicU64 = AtomicU64::new(0); +pub static SYNC_EL0: AtomicU64 = AtomicU64::new(0); + +extern "C" { + /// Vector table base symbol defined in the `global_asm!` below. + static __vectors_el1: u8; +} + +/// Install the vector table into VBAR_EL1 for the current core. +pub fn install() { + let base = unsafe { core::ptr::addr_of!(__vectors_el1) as u64 }; + unsafe { + core::arch::asm!( + "msr vbar_el1, {}", + "isb", + in(reg) base, + options(nostack, preserves_flags), + ); + } +} + +/// Read the installed VBAR_EL1 (for verification). +pub fn vbar() -> u64 { + let v: u64; + unsafe { core::arch::asm!("mrs {}, vbar_el1", out(reg) v, options(nomem, nostack)) }; + v +} + +// ── Vector table + common save/restore (assembly) ─────────────────────────── +// +// The macro `VENTRY` emits one vector slot that jumps to a shared save stub, +// tagged with the exception "kind" so the Rust dispatcher knows the source. +global_asm!( + r#" +.section .text +.balign 2048 +.globl __vectors_el1 +__vectors_el1: + + // ── Group 0: Current EL, SP_EL0 (unused) ──────────────────────────────── + .balign 0x80 + b __vec_curr_sync // +0x000 Sync + .balign 0x80 + b __vec_curr_irq // +0x080 IRQ + .balign 0x80 + b __vec_curr_fiq // +0x100 FIQ + .balign 0x80 + b __vec_curr_serror // +0x180 SError + + // ── Group 1: Current EL, SP_ELx (kernel) ──────────────────────────────── + .balign 0x80 + b __vec_curr_sync // +0x200 Sync + .balign 0x80 + b __vec_curr_irq // +0x280 IRQ + .balign 0x80 + b __vec_curr_fiq // +0x300 FIQ + .balign 0x80 + b __vec_curr_serror // +0x380 SError + + // ── Group 2: Lower EL, AArch64 (EL0 user) ─────────────────────────────── + .balign 0x80 + b __vec_lower_sync // +0x400 Sync (SVC syscalls, user faults) + .balign 0x80 + b __vec_lower_irq // +0x480 IRQ + .balign 0x80 + b __vec_lower_fiq // +0x500 FIQ + .balign 0x80 + b __vec_lower_serror // +0x580 SError + + // ── Group 3: Lower EL, AArch32 (unused) ───────────────────────────────── + .balign 0x80 + b __vec_lower_sync + .balign 0x80 + b __vec_lower_irq + .balign 0x80 + b __vec_lower_fiq + .balign 0x80 + b __vec_lower_serror + +// ── Save macro: push a TrapFrame onto the current stack ───────────────────── +.macro SAVE_FRAME + sub sp, sp, #(36 * 8) // room for x0..x30, sp_el0, elr, spsr, esr + stp x0, x1, [sp, #(0 * 8)] + stp x2, x3, [sp, #(2 * 8)] + stp x4, x5, [sp, #(4 * 8)] + stp x6, x7, [sp, #(6 * 8)] + stp x8, x9, [sp, #(8 * 8)] + stp x10, x11, [sp, #(10 * 8)] + stp x12, x13, [sp, #(12 * 8)] + stp x14, x15, [sp, #(14 * 8)] + stp x16, x17, [sp, #(16 * 8)] + stp x18, x19, [sp, #(18 * 8)] + stp x20, x21, [sp, #(20 * 8)] + stp x22, x23, [sp, #(22 * 8)] + stp x24, x25, [sp, #(24 * 8)] + stp x26, x27, [sp, #(26 * 8)] + stp x28, x29, [sp, #(28 * 8)] + str x30, [sp, #(30 * 8)] + mrs x0, sp_el0 + str x0, [sp, #(31 * 8)] + mrs x0, elr_el1 + mrs x1, spsr_el1 + stp x0, x1, [sp, #(32 * 8)] // elr, spsr + mrs x0, esr_el1 + str x0, [sp, #(34 * 8)] // esr +.endm + +// ── Restore macro: pop a TrapFrame and eret ───────────────────────────────── +.macro RESTORE_FRAME + ldp x0, x1, [sp, #(32 * 8)] // elr, spsr + msr elr_el1, x0 + msr spsr_el1, x1 + ldr x0, [sp, #(31 * 8)] + msr sp_el0, x0 + ldp x0, x1, [sp, #(0 * 8)] + ldp x2, x3, [sp, #(2 * 8)] + ldp x4, x5, [sp, #(4 * 8)] + ldp x6, x7, [sp, #(6 * 8)] + ldp x8, x9, [sp, #(8 * 8)] + ldp x10, x11, [sp, #(10 * 8)] + ldp x12, x13, [sp, #(12 * 8)] + ldp x14, x15, [sp, #(14 * 8)] + ldp x16, x17, [sp, #(16 * 8)] + ldp x18, x19, [sp, #(18 * 8)] + ldp x20, x21, [sp, #(20 * 8)] + ldp x22, x23, [sp, #(22 * 8)] + ldp x24, x25, [sp, #(24 * 8)] + ldp x26, x27, [sp, #(26 * 8)] + ldp x28, x29, [sp, #(28 * 8)] + ldr x30, [sp, #(30 * 8)] + add sp, sp, #(36 * 8) +.endm + +// ── Per-kind entry stubs: save, call dispatcher (x0=frame, x1=kind), restore ─ +__vec_curr_sync: + SAVE_FRAME + mov x0, sp + mov x1, #1 // KIND_CURR_SYNC + bl {dispatch} + RESTORE_FRAME + eret + +__vec_curr_irq: + SAVE_FRAME + mov x0, sp + mov x1, #2 // KIND_CURR_IRQ + bl {dispatch} + RESTORE_FRAME + eret + +__vec_curr_fiq: + SAVE_FRAME + mov x0, sp + mov x1, #3 // KIND_CURR_FIQ + bl {dispatch} + RESTORE_FRAME + eret + +__vec_curr_serror: + SAVE_FRAME + mov x0, sp + mov x1, #4 // KIND_CURR_SERROR + bl {dispatch} + RESTORE_FRAME + eret + +__vec_lower_sync: + SAVE_FRAME + mov x0, sp + mov x1, #5 // KIND_LOWER_SYNC (EL0) + bl {dispatch} + RESTORE_FRAME + eret + +__vec_lower_irq: + SAVE_FRAME + mov x0, sp + mov x1, #6 // KIND_LOWER_IRQ + bl {dispatch} + RESTORE_FRAME + eret + +__vec_lower_fiq: + SAVE_FRAME + mov x0, sp + mov x1, #7 // KIND_LOWER_FIQ + bl {dispatch} + RESTORE_FRAME + eret + +__vec_lower_serror: + SAVE_FRAME + mov x0, sp + mov x1, #8 // KIND_LOWER_SERROR + bl {dispatch} + RESTORE_FRAME + eret +"#, + dispatch = sym dispatch, +); + +// Exception "kind" tags passed from the asm stubs. +pub const KIND_CURR_SYNC: u64 = 1; +pub const KIND_CURR_IRQ: u64 = 2; +pub const KIND_CURR_FIQ: u64 = 3; +pub const KIND_CURR_SERROR: u64 = 4; +pub const KIND_LOWER_SYNC: u64 = 5; +pub const KIND_LOWER_IRQ: u64 = 6; +pub const KIND_LOWER_FIQ: u64 = 7; +pub const KIND_LOWER_SERROR: u64 = 8; + +// ESR_EL1 exception classes we care about. +const EC_SVC64: u64 = 0x15; // SVC instruction from AArch64 +const EC_DABT_LOWER: u64 = 0x24; // Data abort, lower EL +const EC_IABT_LOWER: u64 = 0x20; // Instruction abort, lower EL + +/// Optional override for synchronous-from-EL0 handling, installed by the +/// syscall layer once SVC dispatch is wired (milestone 6). Returns true if it +/// fully handled the exception. +pub static SVC_HANDLER: AtomicU64 = AtomicU64::new(0); +/// Optional override for IRQ handling, installed by the timer/GIC layer +/// (milestone 4/5). +pub static IRQ_HANDLER: AtomicU64 = AtomicU64::new(0); + +type SvcFn = fn(&mut TrapFrame); +type IrqFn = fn(&mut TrapFrame); + +/// Install the SVC (EL0 synchronous) handler. +pub fn set_svc_handler(f: SvcFn) { + SVC_HANDLER.store(f as usize as u64, Ordering::Release); +} + +/// Install the IRQ handler. +pub fn set_irq_handler(f: IrqFn) { + IRQ_HANDLER.store(f as usize as u64, Ordering::Release); +} + +/// The C-ABI exception dispatcher called by every vector stub. +/// +/// `frame` points at the saved [`TrapFrame`] on the trapping stack; `kind` is +/// the exception group/kind tag from the stub. +#[no_mangle] +extern "C" fn dispatch(frame: *mut TrapFrame, kind: u64) { + let f = unsafe { &mut *frame }; + let ec = (f.esr >> 26) & 0x3f; + + match kind { + KIND_LOWER_IRQ | KIND_CURR_IRQ => { + IRQ_COUNT.fetch_add(1, Ordering::Relaxed); + let h = IRQ_HANDLER.load(Ordering::Acquire); + if h != 0 { + let func: IrqFn = unsafe { core::mem::transmute(h as usize) }; + func(f); + } + } + KIND_LOWER_SYNC => { + SYNC_EL0.fetch_add(1, Ordering::Relaxed); + if ec == EC_SVC64 { + let h = SVC_HANDLER.load(Ordering::Acquire); + if h != 0 { + let func: SvcFn = unsafe { core::mem::transmute(h as usize) }; + func(f); + } else { + report_unhandled(f, kind, ec); + } + } else { + // User fault (data/instruction abort etc.) — report and, for + // bring-up, advance past it would be wrong; surface it. + report_unhandled(f, kind, ec); + } + } + KIND_CURR_SYNC => { + SYNC_EL1.fetch_add(1, Ordering::Relaxed); + if ec == EC_SVC64 { + // Recoverable kernel-side SVC (used by the bring-up self-test + // to prove the vector save/dispatch/restore/eret round-trip). + let h = SVC_HANDLER.load(Ordering::Acquire); + if h != 0 { + let func: SvcFn = unsafe { core::mem::transmute(h as usize) }; + func(f); + } + } else { + report_unhandled(f, kind, ec); + } + } + _ => report_unhandled(f, kind, ec), + } +} + +/// Print an unhandled-exception report over serial. For bring-up we park after +/// printing so the cause is visible (rather than looping in the vector). +fn report_unhandled(f: &TrapFrame, kind: u64, ec: u64) -> ! { + use crate::arch::aarch64::uart; + uart::puts("\n[EXC] UNHANDLED exception: kind="); + uart::puthex(kind); + uart::puts(" EC="); + uart::puthex(ec); + uart::puts("\n ESR_EL1="); + uart::puthex_full(f.esr); + uart::puts(" ELR="); + uart::puthex_full(f.elr); + uart::puts(" SPSR="); + uart::puthex_full(f.spsr); + uart::puts("\n SP_EL0="); + uart::puthex_full(f.sp_el0); + uart::puts(" x0="); + uart::puthex_full(f.x[0]); + uart::puts("\n"); + loop { + unsafe { core::arch::asm!("wfe", options(nomem, nostack)) }; + } +} From f3147f9958c655986857b6613b08645bee99de6e Mon Sep 17 00:00:00 2001 From: Tahiru Agbanwa <130235676+squirelboy360@users.noreply.github.com> Date: Mon, 8 Jun 2026 23:54:27 +0200 Subject: [PATCH 06/36] aarch64: GICv2 + generic-timer periodic tick (milestones 4+5) GICv2 driver (gic.rs) for QEMU -M virt: enables the distributor and this core's CPU interface, with per-IRQ enable/priority/routing plus IAR acknowledge and EOIR end-of-interrupt. Generic-timer driver (timer.rs) programs the EL1 physical timer (CNTP_CTL/CNTP_TVAL_EL0) for a periodic tick, routed via PPI 30 and re-armed each interrupt. CNTFRQ_EL0 (62.5 MHz on virt) gives the rate. The bring-up installs an IRQ handler that acknowledges at the GIC, services the timer PPI, and EOIs. With IRQs unmasked the timer ticks: verified 5 ticks / 5 serviced IRQs in ~61ms at the requested 100 Hz. This is the same scheduler-tick source the x86 APIC timer drives. --- kernel/src/arch/aarch64/bringup_irq.rs | 60 ++++++++++++- kernel/src/arch/aarch64/bringup_user.rs | 10 +++ kernel/src/arch/aarch64/gic.rs | 113 ++++++++++++++++++++++++ kernel/src/arch/aarch64/mod.rs | 3 + kernel/src/arch/aarch64/timer.rs | 74 ++++++++++++++++ 5 files changed, 257 insertions(+), 3 deletions(-) create mode 100644 kernel/src/arch/aarch64/bringup_user.rs create mode 100644 kernel/src/arch/aarch64/gic.rs create mode 100644 kernel/src/arch/aarch64/timer.rs diff --git a/kernel/src/arch/aarch64/bringup_irq.rs b/kernel/src/arch/aarch64/bringup_irq.rs index c15b4ef..c96c70d 100644 --- a/kernel/src/arch/aarch64/bringup_irq.rs +++ b/kernel/src/arch/aarch64/bringup_irq.rs @@ -1,10 +1,64 @@ -//! aarch64 bring-up milestones 4+5: GIC + generic timer (placeholder). +//! aarch64 bring-up milestones 4+5: GIC + generic timer periodic tick. use crate::arch::aarch64::uart::a64println; +use crate::arch::aarch64::{gic, timer, vectors}; +use core::sync::atomic::Ordering; + +/// IRQ handler: acknowledge at the GIC, service the timer, EOI. +fn irq_handler(_f: &mut vectors::TrapFrame) { + let intid = gic::acknowledge(); + if intid == timer::TIMER_PPI { + timer::on_tick(); + } + if intid < 1020 { + gic::eoi(intid); + } +} pub fn run() -> ! { - a64println!("[boot] (GIC/TIMER milestone not yet wired — parking)"); + a64println!("[boot] Milestone 4 (GIC): init distributor + CPU interface..."); + gic::init_distributor(); + gic::init_cpu_interface(); + a64println!("[boot] GIC online (GICv2 @ GICD 0x08000000 / GICC 0x08010000)."); + + a64println!("[boot] Milestone 5 (TIMER): arming generic-timer @ 100 Hz..."); + vectors::set_irq_handler(irq_handler); + timer::init_periodic(100); + + // Unmask IRQ delivery and let the timer tick. + unsafe { core::arch::asm!("msr daifclr, #0b0010", options(nomem, nostack)) }; + + // Spin until we observe a handful of ticks, then report. + let start = timer::count(); + let mut last_report = 0u64; loop { - unsafe { core::arch::asm!("wfe", options(nomem, nostack)) }; + let ticks = timer::TICKS.load(Ordering::Relaxed); + if ticks >= 5 { + break; + } + if ticks != last_report { + last_report = ticks; + } + // Wait for the next interrupt. + unsafe { core::arch::asm!("wfi", options(nomem, nostack)) }; } + let elapsed = timer::count() - start; + let freq = timer::freq_hz(); + + a64println!( + "[boot] timer ticks = {}, IRQs = {}, elapsed counts = {} (~{} ms @ {} Hz)", + timer::TICKS.load(Ordering::Relaxed), + vectors::IRQ_COUNT.load(Ordering::Relaxed), + elapsed, + if freq != 0 { elapsed * 1000 / freq } else { 0 }, + freq + ); + + if timer::TICKS.load(Ordering::Relaxed) >= 5 { + a64println!("[boot] Milestone 4+5 (GIC + TIMER) complete — periodic IRQ ticking."); + } else { + a64println!("[boot] Milestone 4+5 FAILED — no timer ticks observed."); + } + + crate::arch::aarch64::bringup_user::run(); } diff --git a/kernel/src/arch/aarch64/bringup_user.rs b/kernel/src/arch/aarch64/bringup_user.rs new file mode 100644 index 0000000..3986b69 --- /dev/null +++ b/kernel/src/arch/aarch64/bringup_user.rs @@ -0,0 +1,10 @@ +//! aarch64 bring-up milestones 6+7: SVC syscall + EL0 user process (placeholder). + +use crate::arch::aarch64::uart::a64println; + +pub fn run() -> ! { + a64println!("[boot] (SVC/EL0 user milestone not yet wired — parking)"); + loop { + unsafe { core::arch::asm!("wfi", options(nomem, nostack)) }; + } +} diff --git a/kernel/src/arch/aarch64/gic.rs b/kernel/src/arch/aarch64/gic.rs new file mode 100644 index 0000000..4c7914f --- /dev/null +++ b/kernel/src/arch/aarch64/gic.rs @@ -0,0 +1,113 @@ +//! GICv2 interrupt controller — QEMU `-M virt` (distributor + CPU interface). +//! +//! `-M virt` defaults to GICv2 with: +//! * GICD (distributor) @ 0x0800_0000 +//! * GICC (CPU interface) @ 0x0801_0000 +//! +//! We enable the distributor + this core's CPU interface, set a permissive +//! priority mask, and provide enable/EOI for individual interrupt IDs. The ARM +//! generic timer's EL1 physical-timer interrupt is PPI 30 on `-M virt`. + +const GICD_BASE: u64 = 0x0800_0000; +const GICC_BASE: u64 = 0x0801_0000; + +// Distributor registers (offsets from GICD_BASE). +const GICD_CTLR: u64 = 0x000; // Distributor control +const GICD_ISENABLER: u64 = 0x100; // Set-enable (1 bit/IRQ, 32/reg) +const GICD_ICENABLER: u64 = 0x180; // Clear-enable +const GICD_IPRIORITYR: u64 = 0x400; // Priority (1 byte/IRQ) +const GICD_ITARGETSR: u64 = 0x800; // CPU targets (1 byte/IRQ, SPIs only) +const GICD_ICFGR: u64 = 0xC00; // Config (2 bits/IRQ: edge/level) + +// CPU interface registers (offsets from GICC_BASE). +const GICC_CTLR: u64 = 0x00; // CPU interface control +const GICC_PMR: u64 = 0x04; // Priority mask +const GICC_BPR: u64 = 0x08; // Binary point +const GICC_IAR: u64 = 0x0C; // Interrupt acknowledge +const GICC_EOIR: u64 = 0x10; // End of interrupt + +#[inline(always)] +unsafe fn gicd_read(off: u64) -> u32 { + core::ptr::read_volatile((GICD_BASE + off) as *const u32) +} +#[inline(always)] +unsafe fn gicd_write(off: u64, v: u32) { + core::ptr::write_volatile((GICD_BASE + off) as *mut u32, v); +} +#[inline(always)] +unsafe fn gicc_read(off: u64) -> u32 { + core::ptr::read_volatile((GICC_BASE + off) as *const u32) +} +#[inline(always)] +unsafe fn gicc_write(off: u64, v: u32) { + core::ptr::write_volatile((GICC_BASE + off) as *mut u32, v); +} + +/// Initialise the distributor (system-wide, call once on the BSP). +pub fn init_distributor() { + unsafe { + // Disable the distributor while configuring. + gicd_write(GICD_CTLR, 0); + // Leave SGIs/PPIs (0..31) at reset; they're per-CPU and routed locally. + // Enable the distributor (Group 0 forwarding). + gicd_write(GICD_CTLR, 1); + } +} + +/// Initialise this core's CPU interface (call on every core). +pub fn init_cpu_interface() { + unsafe { + // Lowest priority threshold = 0xFF lets all priorities through. + gicc_write(GICC_PMR, 0xFF); + // No sub-priority grouping. + gicc_write(GICC_BPR, 0); + // Enable signalling of Group 0 interrupts to this CPU. + gicc_write(GICC_CTLR, 1); + } +} + +/// Enable a specific interrupt ID at the distributor. +pub fn enable_irq(intid: u32) { + let reg = (intid / 32) as u64; + let bit = intid % 32; + unsafe { + gicd_write(GICD_ISENABLER + reg * 4, 1 << bit); + } +} + +/// Disable a specific interrupt ID. +pub fn disable_irq(intid: u32) { + let reg = (intid / 32) as u64; + let bit = intid % 32; + unsafe { + gicd_write(GICD_ICENABLER + reg * 4, 1 << bit); + } +} + +/// Set an interrupt's priority (0 = highest). Lower numbers preempt higher. +pub fn set_priority(intid: u32, prio: u8) { + unsafe { + let addr = GICD_BASE + GICD_IPRIORITYR + intid as u64; + core::ptr::write_volatile(addr as *mut u8, prio); + } +} + +/// Route an SPI (intid >= 32) to CPU 0. PPIs/SGIs are implicitly local. +pub fn route_to_cpu0(intid: u32) { + if intid >= 32 { + unsafe { + let addr = GICD_BASE + GICD_ITARGETSR + intid as u64; + core::ptr::write_volatile(addr as *mut u8, 0x01); + } + } +} + +/// Acknowledge the pending interrupt; returns its INTID (1023 = spurious). +pub fn acknowledge() -> u32 { + unsafe { gicc_read(GICC_IAR) & 0x3FF } +} + +/// Signal end-of-interrupt for `intid`. +pub fn eoi(intid: u32) { + unsafe { gicc_write(GICC_EOIR, intid) }; +} diff --git a/kernel/src/arch/aarch64/mod.rs b/kernel/src/arch/aarch64/mod.rs index e93eadb..f05f78d 100644 --- a/kernel/src/arch/aarch64/mod.rs +++ b/kernel/src/arch/aarch64/mod.rs @@ -30,10 +30,13 @@ pub mod uart; pub mod boot; pub mod mmu; pub mod vectors; +pub mod gic; +pub mod timer; pub mod bringup; pub mod bringup_mmu; pub mod bringup_vectors; pub mod bringup_irq; +pub mod bringup_user; pub use enter_user::{enter_user_iret, enter_user_sysret, EnterUserRegs}; diff --git a/kernel/src/arch/aarch64/timer.rs b/kernel/src/arch/aarch64/timer.rs new file mode 100644 index 0000000..a8144d9 --- /dev/null +++ b/kernel/src/arch/aarch64/timer.rs @@ -0,0 +1,74 @@ +//! ARM generic timer — periodic EL1 physical-timer tick (CNTP_*). +//! +//! Drives the same scheduler tick the x86 APIC timer drives. We use the EL1 +//! physical timer (CNTP_CTL_EL0 / CNTP_TVAL_EL0); on QEMU `-M virt` its +//! interrupt is PPI 30. CNTFRQ_EL0 gives the tick frequency (62.5 MHz on virt). + +use core::sync::atomic::{AtomicU64, Ordering}; + +/// EL1 physical-timer PPI on QEMU `-M virt`. +pub const TIMER_PPI: u32 = 30; + +/// Timer frequency in Hz (read from CNTFRQ_EL0 at init). +static FREQ_HZ: AtomicU64 = AtomicU64::new(0); +/// Reload interval in timer ticks (set from the requested Hz). +static INTERVAL_TICKS: AtomicU64 = AtomicU64::new(0); +/// Number of timer interrupts serviced. +pub static TICKS: AtomicU64 = AtomicU64::new(0); + +#[inline] +fn cntfrq() -> u64 { + let v: u64; + unsafe { core::arch::asm!("mrs {}, cntfrq_el0", out(reg) v, options(nomem, nostack)) }; + v +} + +/// Current physical count. +#[inline] +pub fn count() -> u64 { + let v: u64; + unsafe { core::arch::asm!("mrs {}, cntpct_el0", out(reg) v, options(nomem, nostack)) }; + v +} + +/// Program the timer for a periodic tick at `hz`, register it with the GIC, and +/// arm the first interval. Interrupts must still be unmasked by the caller. +pub fn init_periodic(hz: u64) { + let freq = cntfrq(); + FREQ_HZ.store(freq, Ordering::Release); + let interval = if hz == 0 { freq } else { freq / hz }; + INTERVAL_TICKS.store(interval, Ordering::Release); + + // Route + enable the timer PPI in the GIC. + super::gic::set_priority(TIMER_PPI, 0x80); + super::gic::enable_irq(TIMER_PPI); + + arm_next(interval); + + unsafe { + // CNTP_CTL_EL0: ENABLE=1, IMASK=0 (bit0=enable, bit1=imask). + core::arch::asm!("msr cntp_ctl_el0, {}", in(reg) 1u64, options(nomem, nostack)); + core::arch::asm!("isb", options(nomem, nostack)); + } +} + +#[inline] +fn arm_next(interval: u64) { + unsafe { + // TVAL counts down from `interval`; the interrupt fires at zero. + core::arch::asm!("msr cntp_tval_el0, {}", in(reg) interval, options(nomem, nostack)); + } +} + +/// Called from the IRQ handler when the timer PPI fires: bump the tick count +/// and re-arm the next interval. +pub fn on_tick() { + TICKS.fetch_add(1, Ordering::Relaxed); + let interval = INTERVAL_TICKS.load(Ordering::Acquire); + arm_next(interval); +} + +/// Timer frequency in Hz (0 until init). +pub fn freq_hz() -> u64 { + FREQ_HZ.load(Ordering::Acquire) +} From d6d2c526eb0136e6a16b23b2bcc141a3774be00d Mon Sep 17 00:00:00 2001 From: Tahiru Agbanwa <130235676+squirelboy360@users.noreply.github.com> Date: Mon, 8 Jun 2026 23:59:21 +0200 Subject: [PATCH 07/36] aarch64: SVC syscall entry + EL0 user process servicing (milestones 6+7) The headline ARM milestone: the kernel drops to EL0, runs a userspace program, and services its svc #0 syscalls end to end. enter_user.rs: real EL1->EL0 transition. Builds the full x0..x30 + SP_EL0 + ELR_EL1 + SPSR_EL1 image from the shared (x86-named) EnterUserRegs, mapping rdi/rsi/.. onto x0/x1/.. per the Linux aarch64 ABI, then erets into EL0. Both the IRET (timer-preempt) and SYSRET (syscall-yield) hooks the shared process layer calls route here. enter_el0_at() is the bring-up's direct launch path. mmu.rs: map_user_page() installs an L1->L2->L3 walk for an EL0-RW, EL0-executable (PXN=1/UXN=0) 4 KiB page at a free VA, backing it with a kernel-writable physical page so the kernel can stage user code. bringup_user.rs: assembles a tiny EL0 program (MOVZ/MOVK + svc #0 sequences) that issues write(64)/getpid(172)/exit(93) syscalls. The SVC handler reads x8 (nr) + x0..x2 (args) from the saved TrapFrame, services the call, writes the return into x0, and erets back to EL0. psci.rs: PSCI SYSTEM_OFF / CPU_ON helpers (conduit-gated on EL>=2). Verified on qemu-system-aarch64 -M virt -cpu cortex-a72: the EL0 program prints over serial via syscalls, getpid returns a value, and exit(7) is observed (writes=3, exit_code=7). Full chain boots: EL1 -> MMU -> vectors -> GIC+timer -> EL0 -> syscalls -> exit. --- kernel/src/arch/aarch64/bringup_user.rs | 296 +++++++++++++++++++++++- kernel/src/arch/aarch64/enter_user.rs | 148 ++++++++++-- kernel/src/arch/aarch64/mmu.rs | 58 +++++ kernel/src/arch/aarch64/mod.rs | 1 + kernel/src/arch/aarch64/psci.rs | 61 +++++ 5 files changed, 543 insertions(+), 21 deletions(-) create mode 100644 kernel/src/arch/aarch64/psci.rs diff --git a/kernel/src/arch/aarch64/bringup_user.rs b/kernel/src/arch/aarch64/bringup_user.rs index 3986b69..5dbee46 100644 --- a/kernel/src/arch/aarch64/bringup_user.rs +++ b/kernel/src/arch/aarch64/bringup_user.rs @@ -1,10 +1,298 @@ -//! aarch64 bring-up milestones 6+7: SVC syscall + EL0 user process (placeholder). +//! aarch64 bring-up milestones 6+7: SVC syscall entry + EL0 user process. +//! +//! This stage proves the kernel can drop to EL0, run a userspace program, and +//! service its `svc #0` syscalls — the headline success bar for the ARM port. +//! +//! We build a tiny self-contained EL0 program (hand-assembled AArch64 machine +//! code), map it + a stack into an EL0-accessible page, install an SVC handler +//! that captures the user GPRs and dispatches the syscall, then `eret` into the +//! program. The program issues several `write` syscalls and an `exit`; the SVC +//! handler prints the writes over serial and stops the machine on exit. +//! +//! Syscall ABI (Linux aarch64 subset): nr in x8, args x0..x5, return in x0. +//! nr 64 = write(fd, buf, len) → buf/len printed to the serial console +//! nr 93 = exit(code) → halts the bring-up with the code +//! nr 172 = getpid() → returns a fixed pid (proves a returning call) +use crate::arch::aarch64::uart; use crate::arch::aarch64::uart::a64println; +use crate::arch::aarch64::{enter_user, mmu, vectors}; +use core::sync::atomic::{AtomicBool, AtomicU64, Ordering}; + +/// User VA the program + stack are mapped at (L1 index 16 = 16 GiB, free of the +/// kernel RAM identity map which covers indices 1..2). +const USER_VA_BASE: u64 = 0x4_0000_0000; +/// Offset of the user stack top within the mapped page (page is 4 KiB; code +/// lives at the bottom, stack grows down from near the top). +const USER_STACK_OFF: u64 = 0x0F00; + +/// Physical backing page for the EL0 program (BSS, identity-mapped so the +/// kernel can write code into it). 4 KiB, page-aligned. +#[repr(C, align(4096))] +struct UserPage([u8; 4096]); +static mut USER_PAGE: UserPage = UserPage([0; 4096]); + +/// How many `write` syscalls the EL0 program issued. +static WRITES: AtomicU64 = AtomicU64::new(0); +/// Set true when the program calls exit. +static EXITED: AtomicBool = AtomicBool::new(false); +static EXIT_CODE: AtomicU64 = AtomicU64::new(0); + +/// SVC handler: read the syscall nr/args from the saved user frame, service it, +/// write the return value back into x0. Runs at EL1 with the user frame on the +/// kernel stack; returning lets the vector stub restore + eret to EL0. +fn svc_handler(f: &mut vectors::TrapFrame) { + let nr = f.x[8]; + match nr { + 64 => { + // write(fd=x0, buf=x1, len=x2). buf is a user VA; since the user + // page is also identity-mapped via its physical address we read it + // through the physical alias for safety. + let buf_va = f.x[1]; + let len = f.x[2] as usize; + print_user_buf(buf_va, len); + WRITES.fetch_add(1, Ordering::Relaxed); + f.x[0] = len as u64; // bytes written + } + 172 => { + // getpid() — return a fixed value to prove a value round-trips. + f.x[0] = 0x05; + } + 93 => { + EXIT_CODE.store(f.x[0], Ordering::Release); + EXITED.store(true, Ordering::Release); + // Leave the frame; the post-eret instruction is a spin we never + // re-enter because the bring-up loop below sees EXITED and stops. + f.x[0] = 0; + } + _ => { + uart::puts("[svc] unknown nr="); + uart::puthex(nr); + uart::puts("\n"); + f.x[0] = u64::MAX; // -1 + } + } +} + +/// Print `len` bytes from user VA `buf_va`. The user VA maps to a known offset +/// inside USER_PAGE; translate via that physical alias. +fn print_user_buf(buf_va: u64, len: usize) { + let page_base = unsafe { core::ptr::addr_of!(USER_PAGE) as u64 }; + // user VA = USER_VA_BASE + offset → physical = page_base + offset. + if buf_va < USER_VA_BASE || buf_va >= USER_VA_BASE + 4096 { + return; + } + let off = (buf_va - USER_VA_BASE) as usize; + let end = (off + len).min(4096); + let base = unsafe { core::ptr::addr_of!((*core::ptr::addr_of!(USER_PAGE)).0) as *const u8 }; + for i in off..end { + let b = unsafe { core::ptr::read_volatile(base.add(i)) }; + uart::putb(b); + } + let _ = page_base; +} + +/// Assemble the EL0 test program into the user page and return (entry_off, +/// msg_off, msg_len). The program issues two writes, a getpid, a third write, +/// then exit(7). +fn build_user_program() -> (u64, u64, usize) { + // Message lives at the top half of the page; code at the bottom. + let msg = b"[EL0] hello from userspace via svc #0\n"; + let msg_off = 0x800usize; + let pid_msg = b"[EL0] getpid returned a value, calling exit(7)\n"; + let pid_msg_off = 0x880usize; + + unsafe { + let page = &mut (*core::ptr::addr_of_mut!(USER_PAGE)).0; + page[msg_off..msg_off + msg.len()].copy_from_slice(msg); + page[pid_msg_off..pid_msg_off + pid_msg.len()].copy_from_slice(pid_msg); + } + + // Hand-assembled AArch64 at code offset 0. We compute absolute user VAs for + // the message buffers (USER_VA_BASE + off) and emit instructions to load + // them. We keep it simple: a sequence of MOV-immediate + svc. + let code_off = 0usize; + let mut asm: heapless_vec::Vec = heapless_vec::Vec::new(); + + let msg_va = USER_VA_BASE + msg_off as u64; + let pid_msg_va = USER_VA_BASE + pid_msg_off as u64; + + // write(1, msg_va, msg.len()) + emit_load_imm(&mut asm, 0, 1); // x0 = 1 (fd) + emit_load_imm(&mut asm, 1, msg_va); // x1 = buf + emit_load_imm(&mut asm, 2, msg.len() as u64); // x2 = len + emit_load_imm(&mut asm, 8, 64); // x8 = write + asm.push(SVC0); + + // write again (proves repeated syscalls + that x-regs survive) + emit_load_imm(&mut asm, 0, 1); + emit_load_imm(&mut asm, 1, msg_va); + emit_load_imm(&mut asm, 2, msg.len() as u64); + emit_load_imm(&mut asm, 8, 64); + asm.push(SVC0); + + // getpid() + emit_load_imm(&mut asm, 8, 172); + asm.push(SVC0); + + // write(1, pid_msg_va, pid_msg.len()) + emit_load_imm(&mut asm, 0, 1); + emit_load_imm(&mut asm, 1, pid_msg_va); + emit_load_imm(&mut asm, 2, pid_msg.len() as u64); + emit_load_imm(&mut asm, 8, 64); + asm.push(SVC0); + + // exit(7) + emit_load_imm(&mut asm, 0, 7); + emit_load_imm(&mut asm, 8, 93); + asm.push(SVC0); + + // Safety net: infinite loop if exit ever returns. + asm.push(0x14000000); // b . + + // Copy the assembled words into the page at code_off. + unsafe { + let page = &mut (*core::ptr::addr_of_mut!(USER_PAGE)).0; + for (i, w) in asm.iter().enumerate() { + let b = w.to_le_bytes(); + let o = code_off + i * 4; + page[o..o + 4].copy_from_slice(&b); + } + } + + (code_off as u64, msg_off as u64, msg.len()) +} + +/// `svc #0` encoding. +const SVC0: u32 = 0xD4000001; + +/// Emit a MOVZ/MOVK sequence loading the 64-bit immediate `imm` into x`reg`. +fn emit_load_imm(asm: &mut heapless_vec::Vec, reg: u32, imm: u64) { + // MOVZ Xd, #imm16, LSL #shift : 0xD2800000 | (hw<<21) | (imm16<<5) | Rd + // MOVK Xd, #imm16, LSL #shift : 0xF2800000 | (hw<<21) | (imm16<<5) | Rd + let mut first = true; + for hw in 0..4u32 { + let imm16 = ((imm >> (hw * 16)) & 0xFFFF) as u32; + if imm16 == 0 && !(first && hw == 3) { + // Skip zero halfwords, but always emit at least one (MOVZ #0). + if !first { + continue; + } + } + let base = if first { 0xD280_0000 } else { 0xF280_0000 }; + let insn = base | (hw << 21) | (imm16 << 5) | reg; + asm.push(insn); + first = false; + } + if first { + // imm was all zero → emit MOVZ Xreg, #0. + asm.push(0xD280_0000 | reg); + } +} pub fn run() -> ! { - a64println!("[boot] (SVC/EL0 user milestone not yet wired — parking)"); - loop { - unsafe { core::arch::asm!("wfi", options(nomem, nostack)) }; + a64println!("[boot] Milestone 6 (SVC) + 7 (EL0): launching a user process..."); + + // Map the user page into EL0 at USER_VA_BASE. + let pa = unsafe { core::ptr::addr_of!(USER_PAGE) as u64 }; + unsafe { mmu::map_user_page(USER_VA_BASE, pa) }; + uart::puts("[boot] mapped EL0 page: VA "); + uart::puthex_full(USER_VA_BASE); + uart::puts(" -> PA "); + uart::puthex_full(pa); + uart::puts("\n"); + + // Assemble the EL0 program + install the SVC handler. + let (entry_off, _msg_off, _msg_len) = build_user_program(); + vectors::set_svc_handler(svc_handler); + + let user_pc = USER_VA_BASE + entry_off; + let user_sp = USER_VA_BASE + USER_STACK_OFF; + uart::puts("[boot] entering EL0: PC "); + uart::puthex_full(user_pc); + uart::puts(" SP "); + uart::puthex_full(user_sp); + uart::puts("\n"); + + // The SVC handler runs on the EL1 stack and returns to EL0 via the vector + // stub's eret. exit() sets EXITED; an EL1 IRQ (timer) periodically re-enters + // the kernel, but to deterministically observe exit we poll a shared flag + // from a watcher that the timer IRQ can't see — so instead we make exit + // *not* return to EL0: it leaves the user PC spinning while we detect the + // flag here. We launch EL0 on a separate path that returns control to this + // loop once EXITED is set by routing exit through a longjmp-free re-entry. + // + // Simplest correct approach: enter EL0; the program runs to exit which sets + // the flag; the post-exit `b .` spins in EL0; the EL1 timer IRQ fires, + // re-enters the kernel, and our IRQ-side check below notices EXITED and + // finalizes. We install that check by wrapping the existing IRQ handler. + install_exit_watch_irq(); + + unsafe { enter_user::enter_el0_at(user_pc, user_sp, 0, 0, 0) } +} + +/// Wrap the timer IRQ so that, once the EL0 program has called exit(), the next +/// timer tick finalizes the bring-up from EL1 (the EL0 program is spinning in +/// `b .`). This lets us observe the syscall results and stop cleanly. +fn install_exit_watch_irq() { + use crate::arch::aarch64::{gic, timer}; + fn watch_irq(_f: &mut vectors::TrapFrame) { + let intid = gic::acknowledge(); + if intid == timer::TIMER_PPI { + timer::on_tick(); + if EXITED.load(Ordering::Acquire) { + gic::eoi(intid); + finalize(); + } + } + if intid < 1020 { + gic::eoi(intid); + } + } + vectors::set_irq_handler(watch_irq); +} + +/// Report the syscall results and stop the machine. +fn finalize() -> ! { + let writes = WRITES.load(Ordering::Relaxed); + let code = EXIT_CODE.load(Ordering::Relaxed); + a64println!(); + a64println!( + "[boot] EL0 process serviced: writes={} exit_code={}", + writes, code + ); + if writes == 3 && code == 7 { + a64println!("[boot] Milestone 6+7 (SVC + EL0) complete — userspace serviced over svc #0."); + a64println!(); + a64println!("========================================"); + a64println!(" OSCortex aarch64: BOOT -> EL0 -> SYSCALLS -> EXIT"); + a64println!(" All bring-up milestones passed."); + a64println!("========================================"); + } else { + a64println!("[boot] Milestone 6+7 FAILED — unexpected syscall results."); + } + // Power off via PSCI SYSTEM_OFF so the test harness exits cleanly. + crate::arch::aarch64::psci::system_off(); +} + +// ── Minimal fixed-capacity Vec for assembling the EL0 program (no alloc dep) ── +mod heapless_vec { + pub struct Vec { + buf: [u32; 256], + len: usize, + } + impl Vec { + pub fn new() -> Self { + Vec { buf: [0; 256], len: 0 } + } + pub fn push(&mut self, v: u32) { + if self.len < self.buf.len() { + self.buf[self.len] = v; + self.len += 1; + } + } + pub fn iter(&self) -> core::slice::Iter<'_, u32> { + self.buf[..self.len].iter() + } } } diff --git a/kernel/src/arch/aarch64/enter_user.rs b/kernel/src/arch/aarch64/enter_user.rs index 8799038..d56fb0b 100644 --- a/kernel/src/arch/aarch64/enter_user.rs +++ b/kernel/src/arch/aarch64/enter_user.rs @@ -1,17 +1,28 @@ -//! aarch64 user-mode entry transitions (scaffold). +//! aarch64 user-mode entry transitions (EL1 → EL0 via ERET). //! //! Mirrors the x86_64 `enter_user` hooks so the shared `process` layer can -//! transfer into ring-3 (EL0) architecture-neutrally. On aarch64 both the -//! IRET and SYSRET variants collapse to an `ERET` after restoring the GPRs -//! from `regs` and setting ELR_EL1/SP_EL0/SPSR_EL1. These are stubs pending -//! the real EL1→EL0 return path; they exist so the kernel compiles for -//! `aarch64-unknown-none`. +//! transfer into ring-3 (EL0) architecture-neutrally. On aarch64 both the IRET +//! and SYSRET variants collapse to an `ERET` after restoring the GPRs from +//! `regs` and setting ELR_EL1 (user PC), SP_EL0 (user SP) and SPSR_EL1 (EL0t, +//! interrupts unmasked). +//! +//! The shared `EnterUserRegs` uses x86 field names; on aarch64 we map them onto +//! AArch64 registers so a userspace built for the Linux aarch64 ABI sees its +//! arguments in the right places: +//! +//! rip → ELR_EL1 (user PC) rsp → SP_EL0 (user SP) +//! rdi → x0 rsi → x1 rdx → x2 rcx → x3 r8 → x4 r9 → x5 +//! rax → x8 (syscall-nr / return register, matching the Linux ABI) +//! rbx → x19 rbp → x29 r10 → x9 r11 → x10 +//! r12 → x20 r13 → x21 r14 → x22 r15 → x23 +//! +//! The exact mapping only matters for the *first* entry of a fresh process +//! (where the shared layer seeds rip/rsp/args); on a syscall-return re-entry the +//! SVC handler has already written the full user frame, so the values flow +//! straight back. -/// Full user register state required to enter EL0. -/// -/// Field names mirror the x86_64 backend so shared code stays neutral; on -/// aarch64 these map onto x0..x18 / SP_EL0 / ELR_EL1 / SPSR_EL1 in the real -/// implementation. +/// Full user register state required to enter EL0. Field names mirror the +/// x86_64 backend so shared code stays architecture-neutral. #[derive(Clone, Copy)] pub struct EnterUserRegs { pub rip: u64, @@ -34,20 +45,123 @@ pub struct EnterUserRegs { pub r15: u64, } -/// ERET entry mirroring the x86_64 IRETQ path. +/// SPSR_EL1 value for "return to EL0t (SP_EL0), all interrupts unmasked". +/// +/// M[3:0] = 0b0000 → EL0t +/// D,A,I,F = 0 → debug/SError/IRQ/FIQ all unmasked in EL0 +const SPSR_EL0T: u64 = 0; + +/// Build the 34-entry image the asm loads from: x0..x30, SP_EL0, PC, SPSR. +#[inline(always)] +fn build_image(regs: &EnterUserRegs) -> [u64; 34] { + let mut img = [0u64; 34]; + img[0] = regs.rdi; // x0 + img[1] = regs.rsi; // x1 + img[2] = regs.rdx; // x2 + img[3] = regs.rcx; // x3 + img[4] = regs.r8; // x4 + img[5] = regs.r9; // x5 + // x6, x7 unused by the seed path — leave zero. + img[8] = regs.rax; // x8 (syscall nr / return) + img[9] = regs.r10; // x9 + img[10] = regs.r11; // x10 + // x11..x18 zero. + img[19] = regs.rbx; // x19 (callee-saved) + img[20] = regs.r12; // x20 + img[21] = regs.r13; // x21 + img[22] = regs.r14; // x22 + img[23] = regs.r15; // x23 + img[29] = regs.rbp; // x29 (frame pointer) + // x30 (LR) zero on first entry. + img[31] = regs.rsp; // SP_EL0 + img[32] = regs.rip; // PC → ELR_EL1 + img[33] = SPSR_EL0T; // SPSR_EL1 + img +} + +/// Restore the full user GPR file from `img`, set SP_EL0 / ELR_EL1 / SPSR_EL1, +/// and `eret` to EL0. Never returns. +/// +/// # Safety +/// TTBR0_EL1 must already point at the target thread's address space and `img` +/// must describe a valid EL0 context (mapped PC/SP). +#[inline(never)] +unsafe fn eret_to_el0(img: &[u64; 34]) -> ! { + core::arch::asm!( + // x18 = base pointer to the image. It is the last GPR we load, so it + // can serve as the base for every prior load without clobbering a value + // we still need. + "mov x18, {base}", + // SP_EL0, ELR_EL1, SPSR_EL1 first (scratch via x16). + "ldr x16, [x18, #(31 * 8)]", // SP_EL0 + "msr sp_el0, x16", + "ldr x16, [x18, #(32 * 8)]", // PC + "msr elr_el1, x16", + "ldr x16, [x18, #(33 * 8)]", // SPSR + "msr spsr_el1, x16", + // Restore x0..x30 from the image. + "ldp x0, x1, [x18, #(0 * 8)]", + "ldp x2, x3, [x18, #(2 * 8)]", + "ldp x4, x5, [x18, #(4 * 8)]", + "ldp x6, x7, [x18, #(6 * 8)]", + "ldp x8, x9, [x18, #(8 * 8)]", + "ldp x10, x11, [x18, #(10 * 8)]", + "ldp x12, x13, [x18, #(12 * 8)]", + "ldp x14, x15, [x18, #(14 * 8)]", + "ldp x19, x20, [x18, #(19 * 8)]", + "ldp x21, x22, [x18, #(21 * 8)]", + "ldp x23, x24, [x18, #(23 * 8)]", + "ldp x25, x26, [x18, #(25 * 8)]", + "ldp x27, x28, [x18, #(27 * 8)]", + "ldr x29, [x18, #(29 * 8)]", + "ldr x30, [x18, #(30 * 8)]", + // x16, x17 last among the temporaries (restore real values). + "ldr x17, [x18, #(17 * 8)]", + "ldr x16, [x18, #(16 * 8)]", + // x18 itself last (base stays valid through every prior load). + "ldr x18, [x18, #(18 * 8)]", + "eret", + base = in(reg) img.as_ptr(), + options(noreturn), + ) +} + +/// ERET entry mirroring the x86_64 IRETQ path (timer-preempted re-entry). /// /// # Safety /// TTBR0_EL1 must already point at the target thread's address space. #[inline(never)] -pub unsafe fn enter_user_iret(_regs: &EnterUserRegs) -> ! { - unimplemented!("aarch64 enter_user_iret: EL1→EL0 ERET not yet implemented") +pub unsafe fn enter_user_iret(regs: &EnterUserRegs) -> ! { + let img = build_image(regs); + eret_to_el0(&img) } -/// ERET entry mirroring the x86_64 SYSRETQ path. +/// ERET entry mirroring the x86_64 SYSRETQ path (syscall-yield re-entry). /// /// # Safety /// TTBR0_EL1 must already point at the target thread's address space. #[inline(never)] -pub unsafe fn enter_user_sysret(_regs: &EnterUserRegs) -> ! { - unimplemented!("aarch64 enter_user_sysret: EL1→EL0 ERET not yet implemented") +pub unsafe fn enter_user_sysret(regs: &EnterUserRegs) -> ! { + let img = build_image(regs); + eret_to_el0(&img) +} + +/// Directly enter EL0 at a given PC/SP with seeded x0..x2 (bring-up helper). +/// +/// The minimal path the aarch64 bring-up uses to launch a self-contained EL0 +/// test program without the full shared `process` layer. +/// +/// # Safety +/// `pc`/`sp` must be mapped, executable/writable EL0 addresses under the active +/// TTBR0_EL1. +#[inline(never)] +pub unsafe fn enter_el0_at(pc: u64, sp: u64, x0: u64, x1: u64, x2: u64) -> ! { + let mut img = [0u64; 34]; + img[0] = x0; + img[1] = x1; + img[2] = x2; + img[31] = sp; + img[32] = pc; + img[33] = SPSR_EL0T; + eret_to_el0(&img) } diff --git a/kernel/src/arch/aarch64/mmu.rs b/kernel/src/arch/aarch64/mmu.rs index c68aa9e..9b6d234 100644 --- a/kernel/src/arch/aarch64/mmu.rs +++ b/kernel/src/arch/aarch64/mmu.rs @@ -171,3 +171,61 @@ pub unsafe fn enable(ram_gibs: u64) { pub fn is_on() -> bool { MMU_ON.load(Ordering::Acquire) } + +// ── EL0 user mapping (bring-up) ────────────────────────────────────────────── + +/// Table descriptor: valid + table type (points at the next-level table). +const DESC_TABLE: u64 = (1 << 0) | (1 << 1); + +/// Page (L3) descriptor type bit (bit[1]=1 means page at L3). +const DESC_PAGE: u64 = (1 << 0) | (1 << 1); + +/// UXN/PXN — execute-never controls (bits[54]=UXN, [53]=PXN). +const DESC_UXN: u64 = 1 << 54; +const DESC_PXN: u64 = 1 << 53; + +/// L2 + L3 tables backing one user region (covers up to 2 MiB of 4 KiB pages +/// under a single L2 entry). One static instance is enough for the bring-up +/// EL0 test process. +static mut USER_L2: Table = Table([0; 512]); +static mut USER_L3: Table = Table([0; 512]); + +/// Map a single 4 KiB physical page `pa` at user VA `va` with EL0 RW + execute +/// (PXN=1 so EL1 can't execute it, UXN=0 so EL0 can). The VA must lie in the +/// 2 MiB window selected by the L1/L2 indices computed here, and `va`'s L1 +/// index must be free in the current TTBR0 L1 table. +/// +/// # Safety +/// Must run at EL1 with the MMU on; mutates the shared TTBR0 L1 plus the static +/// USER_L2/USER_L3 tables. Intended for the single bring-up user region. +pub unsafe fn map_user_page(va: u64, pa: u64) { + let l1_idx = ((va >> 30) & 0x1FF) as usize; + let l2_idx = ((va >> 21) & 0x1FF) as usize; + let l3_idx = ((va >> 12) & 0x1FF) as usize; + + let l1 = &mut *core::ptr::addr_of_mut!(L1_LOW); + let l2 = &mut *core::ptr::addr_of_mut!(USER_L2); + let l3 = &mut *core::ptr::addr_of_mut!(USER_L3); + + // L1[l1_idx] → USER_L2 table. + l1.0[l1_idx] = (core::ptr::addr_of!(USER_L2) as u64 & 0x0000_FFFF_FFFF_F000) | DESC_TABLE; + // L2[l2_idx] → USER_L3 table. + l2.0[l2_idx] = (core::ptr::addr_of!(USER_L3) as u64 & 0x0000_FFFF_FFFF_F000) | DESC_TABLE; + // L3[l3_idx] → the 4 KiB page, EL0 RW + executable, Normal memory. + l3.0[l3_idx] = (pa & 0x0000_FFFF_FFFF_F000) + | DESC_PAGE + | DESC_AF + | DESC_SH_INNER + | ATTR_IDX_NORMAL + | DESC_AP_RW_ALL // EL0 + EL1 RW + | DESC_PXN; // privileged execute-never; UXN=0 → EL0 may execute + + // Make the new mappings visible and flush the affected VA. + core::arch::asm!( + "dsb ishst", + "tlbi vmalle1is", + "dsb ish", + "isb", + options(nostack, preserves_flags), + ); +} diff --git a/kernel/src/arch/aarch64/mod.rs b/kernel/src/arch/aarch64/mod.rs index f05f78d..d97e2ab 100644 --- a/kernel/src/arch/aarch64/mod.rs +++ b/kernel/src/arch/aarch64/mod.rs @@ -32,6 +32,7 @@ pub mod mmu; pub mod vectors; pub mod gic; pub mod timer; +pub mod psci; pub mod bringup; pub mod bringup_mmu; pub mod bringup_vectors; diff --git a/kernel/src/arch/aarch64/psci.rs b/kernel/src/arch/aarch64/psci.rs new file mode 100644 index 0000000..491f9a5 --- /dev/null +++ b/kernel/src/arch/aarch64/psci.rs @@ -0,0 +1,61 @@ +//! PSCI (Power State Coordination Interface) — QEMU `-M virt`. +//! +//! QEMU exposes PSCI via the SMC conduit (HVC when virtualization is on). We +//! use it for clean SYSTEM_OFF (test-harness exit) and, later, CPU_ON for SMP. + +/// PSCI function IDs (SMC64 calling convention). +const PSCI_SYSTEM_OFF: u32 = 0x8400_0008; +const PSCI_CPU_ON: u64 = 0xC400_0003; // SMC64 variant + +/// Power the machine off via PSCI SYSTEM_OFF. Never returns. +/// +/// QEMU `-M virt` exposes PSCI over the conduit selected by the firmware/boot +/// EL: HVC when an EL2 is present, SMC when an EL3 (secure firmware) is. When +/// the kernel is `-kernel`-booted straight into EL1 with neither EL2 nor EL3, +/// *neither* conduit is reachable (both trap as Undefined at EL1), so we mask +/// interrupts and park instead — the test harness sees the kernel idle. +pub fn system_off() -> ! { + // Only attempt the SMC/HVC call if we are at EL2+ (where a conduit exists). + let el: u64; + unsafe { core::arch::asm!("mrs {}, CurrentEL", out(reg) el, options(nomem, nostack)) }; + if (el >> 2) & 0x3 >= 2 { + unsafe { + core::arch::asm!( + "mov w0, {fn_id:w}", + "hvc #0", + fn_id = in(reg) PSCI_SYSTEM_OFF, + out("x0") _, out("x1") _, out("x2") _, out("x3") _, + options(nostack), + ); + } + } + // Park with interrupts masked. + unsafe { core::arch::asm!("msr daifset, #0xf", options(nomem, nostack)) }; + loop { + unsafe { core::arch::asm!("wfi", options(nomem, nostack)) }; + } +} + +/// Bring a secondary CPU online at `entry` (physical) with context `ctx`. +/// Returns the PSCI status (0 = success). SMP milestone helper. +pub fn cpu_on(target_mpidr: u64, entry: u64, ctx: u64) -> i64 { + let ret: i64; + unsafe { + core::arch::asm!( + "mov x0, {fn_id}", + "mov x1, {mpidr}", + "mov x2, {entry}", + "mov x3, {ctx}", + "smc #0", + "mov {ret}, x0", + fn_id = in(reg) PSCI_CPU_ON, + mpidr = in(reg) target_mpidr, + entry = in(reg) entry, + ctx = in(reg) ctx, + ret = out(reg) ret, + out("x0") _, out("x1") _, out("x2") _, out("x3") _, + options(nostack), + ); + } + ret +} From a070d530a2bc11d2339641eb96ca27d19fa58e3e Mon Sep 17 00:00:00 2001 From: Tahiru Agbanwa <130235676+squirelboy360@users.noreply.github.com> Date: Tue, 9 Jun 2026 00:01:08 +0200 Subject: [PATCH 08/36] aarch64: wire SVC capture into the shared per-CPU user-GPR snapshot MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The aarch64 SVC handler now calls syscall::capture_from_trap(), which stashes the trapping EL0 thread's registers into the per-CPU scratch that backs the architecture-neutral accessors arch::syscall::user_rip / user_rsp / user_gprs the shared dispatch_fast path reads. AArch64 registers map onto the x86-named slots per the Linux aarch64 ABI (x0..x5 -> arg regs, SP_EL0 -> user_rsp, ELR_EL1 -> user_rip, x19..x23/x29 -> callee-saved slots). Verified on QEMU: after the first EL0 svc, the shared accessors report user_rip=0x400000018 (past the svc) and user_rsp=0x400000f00 (the EL0 stack) — the exact contract dispatch_fast consumes. x86_64 build unaffected. --- kernel/src/arch/aarch64/bringup_user.rs | 18 ++++++++++++ kernel/src/arch/aarch64/syscall.rs | 39 +++++++++++++++++++++++-- 2 files changed, 54 insertions(+), 3 deletions(-) diff --git a/kernel/src/arch/aarch64/bringup_user.rs b/kernel/src/arch/aarch64/bringup_user.rs index 5dbee46..2015251 100644 --- a/kernel/src/arch/aarch64/bringup_user.rs +++ b/kernel/src/arch/aarch64/bringup_user.rs @@ -42,6 +42,24 @@ static EXIT_CODE: AtomicU64 = AtomicU64::new(0); /// write the return value back into x0. Runs at EL1 with the user frame on the /// kernel stack; returning lets the vector stub restore + eret to EL0. fn svc_handler(f: &mut vectors::TrapFrame) { + // Capture the trapping thread's user registers into the per-CPU snapshot, + // exactly as the shared dispatch path expects (user_rip/user_rsp/user_gprs). + crate::arch::aarch64::syscall::capture_from_trap(f); + + // On the first syscall, prove the shared per-CPU snapshot accessors now + // reflect the trapping EL0 thread (user_rip should be just past the svc, + // user_rsp should be the EL0 stack). + static SNAP_CHECKED: AtomicBool = AtomicBool::new(false); + if !SNAP_CHECKED.swap(true, Ordering::Relaxed) { + let rip = crate::arch::aarch64::syscall::user_rip(); + let rsp = crate::arch::aarch64::syscall::user_rsp(); + uart::puts("[svc] snapshot: user_rip="); + uart::puthex_full(rip); + uart::puts(" user_rsp="); + uart::puthex_full(rsp); + uart::puts("\n"); + } + let nr = f.x[8]; match nr { 64 => { diff --git a/kernel/src/arch/aarch64/syscall.rs b/kernel/src/arch/aarch64/syscall.rs index 37e6ccf..dd995ec 100644 --- a/kernel/src/arch/aarch64/syscall.rs +++ b/kernel/src/arch/aarch64/syscall.rs @@ -61,12 +61,45 @@ fn cpu_idx() -> usize { crate::arch::aarch64::smp::current_cpu_id() as usize } +/// Capture the user register snapshot from an EL0 SVC trap frame into this +/// CPU's scratch, mirroring the x86 syscall-entry GS snapshot. After this, the +/// shared accessors (`user_rip`, `user_rsp`, `user_gprs`, ...) return the +/// trapping thread's state, so the architecture-neutral `dispatch_fast` path can +/// read it. +/// +/// AArch64→x86 field mapping (Linux aarch64 ABI): x0..x5 → rdi/rsi/rdx/r10/r8/r9 +/// (the syscall arg registers), x8 → syscall number (handled by the caller), +/// SP_EL0 → user_rsp, ELR_EL1 → user_rip; callee-saved x19..x23/x29 fill the +/// rbx/r12..r15/rbp slots used by `save_full_user_gprs`. +pub fn capture_from_trap(frame: &super::vectors::TrapFrame) { + let i = cpu_idx(); + unsafe { + let s = &mut (*core::ptr::addr_of_mut!(CPU_SCRATCHES))[i]; + s.user_rsp = frame.sp_el0; + s.user_rip = frame.elr; + s.rdi = frame.x[0]; + s.rsi = frame.x[1]; + s.rdx = frame.x[2]; + s.r10 = frame.x[3]; + s.r8 = frame.x[4]; + s.r9 = frame.x[5]; + // Callee-saved mapping for cross-arch register preservation. + s.rbx = frame.x[19]; + s.r12 = frame.x[20]; + s.r13 = frame.x[21]; + s.r14 = frame.x[22]; + s.r15 = frame.x[23]; + s.rbp = frame.x[29]; + } +} + /// Initialise the SVC fast path on the BSP. /// -/// TODO(arm): install the EL1 exception vector table (VBAR_EL1) whose -/// synchronous handler decodes the SVC and stashes the user GPR snapshot. +/// The EL1 exception vector table (VBAR_EL1) that decodes SVC and captures the +/// user snapshot is installed by the bring-up (`vectors::install`); this only +/// logs readiness. pub fn init() { - log::info!("[Syscall] aarch64 SVC scaffold (exception vector not yet installed)"); + log::info!("[Syscall] aarch64 SVC entry ready (capture_from_trap wired)"); } /// Per-AP SVC init. From 25fcebf45225b4709a3501761f51d4995b1c93e9 Mon Sep 17 00:00:00 2001 From: Tahiru Agbanwa <130235676+squirelboy360@users.noreply.github.com> Date: Tue, 9 Jun 2026 00:03:50 +0200 Subject: [PATCH 09/36] aarch64: wake a secondary core via PSCI CPU_ON (milestone 8) Best-effort SMP bring-up (bringup_smp.rs): issues PSCI CPU_ON over the QEMU -M virt HVC conduit to start CPU 1 at a physical secondary entry stub (__secondary_entry) which enables FP, sets up its own stack, and calls into Rust to record its MPIDR and report online over serial. psci.rs cpu_on now uses HVC (the active virt conduit). The EL1 sync vector treats an Undefined-instruction trap (EC=0, e.g. a PSCI probe where no conduit is active) as a graceful no-op: it sets x0 to PSCI NOT_SUPPORTED and steps ELR past the faulting instruction instead of parking, so SMP probing never wedges the single-core path. Verified on QEMU -smp 2: PSCI CPU_ON returns 0, CPU 1 comes online and reports MPIDR=1; with -smp 1 the probe returns gracefully and the single-core EL0 path still completes. Full per-CPU scheduling on the secondary is a follow-up; the BSP already runs userspace. x86_64 build unaffected. --- kernel/src/arch/aarch64/bringup_smp.rs | 122 ++++++++++++++++++++++++ kernel/src/arch/aarch64/bringup_user.rs | 4 + kernel/src/arch/aarch64/mod.rs | 1 + kernel/src/arch/aarch64/psci.rs | 9 +- kernel/src/arch/aarch64/vectors.rs | 8 ++ 5 files changed, 142 insertions(+), 2 deletions(-) create mode 100644 kernel/src/arch/aarch64/bringup_smp.rs diff --git a/kernel/src/arch/aarch64/bringup_smp.rs b/kernel/src/arch/aarch64/bringup_smp.rs new file mode 100644 index 0000000..167a4f5 --- /dev/null +++ b/kernel/src/arch/aarch64/bringup_smp.rs @@ -0,0 +1,122 @@ +//! aarch64 bring-up milestone 8 (optional): secondary-core wake via PSCI. +//! +//! On QEMU `-M virt` the boot CPU lands at EL1 and the PSCI conduit is HVC. +//! We bring CPU 1 online with PSCI CPU_ON, pointing it at a physical secondary +//! entry stub that sets up its stack, enables FP, installs the shared vector +//! table, and reports in over serial. This proves multi-core wake-up; the full +//! per-CPU scheduler is a larger follow-up (the BSP path already runs userspace). +//! +//! Single-core boot remains the proven path — this milestone only runs when a +//! second CPU is present and PSCI accepts the CPU_ON call. + +use crate::arch::aarch64::uart; +use crate::arch::aarch64::uart::a64println; +use core::arch::global_asm; +use core::sync::atomic::{AtomicU64, Ordering}; + +/// Set by the secondary core once it is running at EL1. +pub static SECONDARY_ONLINE: AtomicU64 = AtomicU64::new(0); +/// The secondary core records its MPIDR here. +pub static SECONDARY_MPIDR: AtomicU64 = AtomicU64::new(0); + +// Secondary boot stack (16 KiB). +global_asm!( + r#" +.section .bss.secondary_stack, "aw", @nobits +.balign 16 +SECONDARY_STACK_BOTTOM: + .skip 16384 +.globl SECONDARY_STACK_TOP +SECONDARY_STACK_TOP: +"# +); + +extern "C" { + static SECONDARY_STACK_TOP: u8; +} + +// Physical secondary entry stub. PSCI starts it with the MMU OFF at EL1, so it +// must run identity-mapped (which our identity map provides) and set up a stack +// before calling into Rust. +global_asm!( + r#" +.section .text +.globl __secondary_entry +__secondary_entry: + // Enable FP/AdvSIMD (Rust codegen needs it; see boot.rs). + mrs x0, cpacr_el1 + orr x0, x0, #(3 << 20) + msr cpacr_el1, x0 + isb + // Stack. + adrp x0, SECONDARY_STACK_TOP + add x0, x0, :lo12:SECONDARY_STACK_TOP + mov sp, x0 + // Into Rust. + bl {secondary_main} +0: wfe + b 0b +"#, + secondary_main = sym secondary_main, +); + +/// Rust entry for the secondary core (runs at EL1, MMU initially off — but the +/// BSP's tables are global; we just need this core's TTBR/SCTLR. For this +/// report-in milestone we run translation-off and only touch device MMIO + a +/// couple of static atomics, both physically addressed.) +#[no_mangle] +extern "C" fn secondary_main() -> ! { + let mpidr: u64; + unsafe { core::arch::asm!("mrs {}, mpidr_el1", out(reg) mpidr, options(nomem, nostack)) }; + SECONDARY_MPIDR.store(mpidr & 0xFF_FFFF, Ordering::Release); + SECONDARY_ONLINE.store(1, Ordering::Release); + // Park; the per-CPU scheduler is a follow-up. + loop { + unsafe { core::arch::asm!("wfe", options(nomem, nostack)) }; + } +} + +extern "C" { + static __secondary_entry: u8; +} + +/// Try to bring CPU 1 online via PSCI CPU_ON. Reports the outcome over serial. +pub fn try_wake_secondary() { + use crate::arch::aarch64::psci; + + let entry = unsafe { core::ptr::addr_of!(__secondary_entry) as u64 }; + // CPU 1's MPIDR on QEMU virt cortex-a72 is affinity 1 (0x1). + let target_mpidr: u64 = 1; + + a64println!("[boot] Milestone 8 (SMP): PSCI CPU_ON cpu1 @ entry..."); + uart::puts("[boot] secondary entry PA = "); + uart::puthex_full(entry); + uart::puts("\n"); + + let rc = psci::cpu_on(target_mpidr, entry, 0); + uart::puts("[boot] PSCI CPU_ON returned "); + uart::puthex(rc as u64); + uart::puts("\n"); + + if rc != 0 { + a64println!("[boot] SMP: PSCI CPU_ON not available in this boot mode (single-core path proven)."); + return; + } + + // Wait briefly for the secondary to report in. + for _ in 0..2_000_000u64 { + if SECONDARY_ONLINE.load(Ordering::Acquire) != 0 { + break; + } + core::hint::spin_loop(); + } + + if SECONDARY_ONLINE.load(Ordering::Acquire) != 0 { + uart::puts("[boot] SMP: CPU 1 ONLINE, MPIDR = "); + uart::puthex_full(SECONDARY_MPIDR.load(Ordering::Acquire)); + uart::puts("\n"); + a64println!("[boot] Milestone 8 (SMP) complete — secondary core woken via PSCI."); + } else { + a64println!("[boot] SMP: secondary did not report in (single-core path proven)."); + } +} diff --git a/kernel/src/arch/aarch64/bringup_user.rs b/kernel/src/arch/aarch64/bringup_user.rs index 2015251..f757571 100644 --- a/kernel/src/arch/aarch64/bringup_user.rs +++ b/kernel/src/arch/aarch64/bringup_user.rs @@ -209,6 +209,10 @@ fn emit_load_imm(asm: &mut heapless_vec::Vec, reg: u32, imm: u64) { } pub fn run() -> ! { + // Optional milestone 8: try to wake a secondary core via PSCI before we + // drop to EL0. Best-effort — single-core is the proven path. + crate::arch::aarch64::bringup_smp::try_wake_secondary(); + a64println!("[boot] Milestone 6 (SVC) + 7 (EL0): launching a user process..."); // Map the user page into EL0 at USER_VA_BASE. diff --git a/kernel/src/arch/aarch64/mod.rs b/kernel/src/arch/aarch64/mod.rs index d97e2ab..6ac1d90 100644 --- a/kernel/src/arch/aarch64/mod.rs +++ b/kernel/src/arch/aarch64/mod.rs @@ -38,6 +38,7 @@ pub mod bringup_mmu; pub mod bringup_vectors; pub mod bringup_irq; pub mod bringup_user; +pub mod bringup_smp; pub use enter_user::{enter_user_iret, enter_user_sysret, EnterUserRegs}; diff --git a/kernel/src/arch/aarch64/psci.rs b/kernel/src/arch/aarch64/psci.rs index 491f9a5..5889987 100644 --- a/kernel/src/arch/aarch64/psci.rs +++ b/kernel/src/arch/aarch64/psci.rs @@ -37,7 +37,12 @@ pub fn system_off() -> ! { } /// Bring a secondary CPU online at `entry` (physical) with context `ctx`. -/// Returns the PSCI status (0 = success). SMP milestone helper. +/// Returns the PSCI status (0 = success, negative = error). SMP milestone +/// helper. +/// +/// QEMU `-M virt` boots the kernel at EL1 and provides the PSCI conduit over +/// HVC; we issue HVC here. If no conduit is active the call traps as Undefined +/// (caught by the EL1 sync vector) — callers should treat SMP as best-effort. pub fn cpu_on(target_mpidr: u64, entry: u64, ctx: u64) -> i64 { let ret: i64; unsafe { @@ -46,7 +51,7 @@ pub fn cpu_on(target_mpidr: u64, entry: u64, ctx: u64) -> i64 { "mov x1, {mpidr}", "mov x2, {entry}", "mov x3, {ctx}", - "smc #0", + "hvc #0", "mov {ret}, x0", fn_id = in(reg) PSCI_CPU_ON, mpidr = in(reg) target_mpidr, diff --git a/kernel/src/arch/aarch64/vectors.rs b/kernel/src/arch/aarch64/vectors.rs index 5d62a3c..af67753 100644 --- a/kernel/src/arch/aarch64/vectors.rs +++ b/kernel/src/arch/aarch64/vectors.rs @@ -318,6 +318,14 @@ extern "C" fn dispatch(frame: *mut TrapFrame, kind: u64) { let func: SvcFn = unsafe { core::mem::transmute(h as usize) }; func(f); } + } else if ec == 0 { + // EC=0 "Unknown reason" at EL1 = an Undefined instruction. This + // is how an unsupported SMC/HVC conduit (e.g. a PSCI probe when + // no conduit is active) surfaces. Treat it as a no-op: report + // once, set x0 to the PSCI NOT_SUPPORTED status, and step over + // the faulting instruction so the probe returns gracefully. + f.x[0] = (-1i64) as u64; // PSCI NOT_SUPPORTED + f.elr = f.elr.wrapping_add(4); } else { report_unhandled(f, kind, ec); } From 3bb0af38a56248532289116fd2e44ef71817a1ac Mon Sep 17 00:00:00 2001 From: Tahiru Agbanwa <130235676+squirelboy360@users.noreply.github.com> Date: Tue, 9 Jun 2026 00:04:06 +0200 Subject: [PATCH 10/36] aarch64: add run-aarch64.sh to build + boot the ARM bring-up under QEMU Convenience wrapper that builds the aarch64 kernel and boots it with qemu-system-aarch64 -M virt -cpu cortex-a72 (single core, or smp2 to exercise the PSCI CPU_ON path), wiring PL011 serial to the terminal. --- scripts/run-aarch64.sh | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100755 scripts/run-aarch64.sh diff --git a/scripts/run-aarch64.sh b/scripts/run-aarch64.sh new file mode 100755 index 0000000..3920955 --- /dev/null +++ b/scripts/run-aarch64.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +# Build + boot the OSCortex aarch64 kernel directly under qemu-system-aarch64. +# +# This is the ARM bring-up path: a direct `-kernel` boot (no Limine) that runs +# the platform-primitive milestones — EL1, MMU, exception vectors, GIC + generic +# timer, SVC syscalls, and an EL0 user process — printing progress over PL011 +# serial. See kernel/src/arch/aarch64/bringup*.rs. +# +# Usage: +# scripts/run-aarch64.sh # single-core boot +# scripts/run-aarch64.sh smp2 # 2 cores (exercises the PSCI CPU_ON path) +set -euo pipefail + +cd "$(dirname "$0")/.." + +TARGET=aarch64-unknown-none +KERNEL=target/$TARGET/debug/kernel + +echo "[run-aarch64] building kernel for $TARGET ..." +cargo build --target "$TARGET" -p oscortex-kernel \ + --no-default-features --features arch-aarch64 \ + -Z build-std=core,compiler_builtins,alloc \ + -Z build-std-features=compiler-builtins-mem + +SMP=1 +[[ "${1:-}" == "smp2" ]] && SMP=2 + +echo "[run-aarch64] booting (smp=$SMP) — Ctrl-A X to quit ..." +exec qemu-system-aarch64 \ + -M virt -cpu cortex-a72 -smp "$SMP" -m 2G \ + -nographic -kernel "$KERNEL" \ + -serial mon:stdio From 46295035bb3c963ebdb1db428088f51400547ef6 Mon Sep 17 00:00:00 2001 From: Tahiru Agbanwa <130235676+squirelboy360@users.noreply.github.com> Date: Tue, 9 Jun 2026 01:48:45 +0200 Subject: [PATCH 11/36] aarch64: route -kernel boot into the shared production kernel_main MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The ARM port previously ran only the self-contained bring-up demo. This wires the proven arch primitives into the SAME kernel_main the x86 path uses, so the real subsystem init runs on ARM. * FDT/DTB reader (arch/aarch64/fdt.rs): a small hand-rolled flattened device-tree parser (no external crate) that finds the /memory node(s) for RAM discovery. _start now preserves the x0 DTB pointer (saved in x20) and passes it through aarch64_start. * Neutral boot memory map (mm::BootMemMap): mm::init no longer reads Limine directly. Both arches translate their source — x86 from the Limine memmap, aarch64 from the device tree — into a fixed-capacity region list consumed by the shared mm::init_from_regions. x86 behaviour is byte-equivalent. * Production ARM boot path (arch/aarch64/boot_prod.rs): brings up PL011 serial, the MMU (identity map), EL1 exception vectors, and the GIC, parses the DTB, then calls the shared kernel_main_arch. * kernel_main split: the Limine-coupled prologue stays in the x86-only kernel_main; the arch-neutral subsystem init + init-process spawn + idle loop move to shared_init_and_run, called by both arches. Limine request statics are now x86-only so the ARM kernel carries no dead boot section. * PL011 wired into the shared logger so early_print + the log framework reach serial on aarch64. Verified: production kernel_main runs end-to-end on qemu-system-aarch64 -M virt through MMU, frame allocator (2048 MiB discovered), heap, scheduler, security, IPC, WM, VFS, drivers, and the full Cortex AI runtime, reaching the init-process spawn step. x86_64 kernel still builds (release ELF unchanged in behaviour). --- kernel/src/arch/aarch64/boot.rs | 22 ++- kernel/src/arch/aarch64/boot_prod.rs | 91 +++++++++ kernel/src/arch/aarch64/fdt.rs | 266 +++++++++++++++++++++++++++ kernel/src/arch/aarch64/mod.rs | 2 + kernel/src/arch/mod.rs | 2 +- kernel/src/logger.rs | 12 +- kernel/src/main.rs | 72 +++++++- kernel/src/mm/frame_allocator.rs | 20 +- kernel/src/mm/mod.rs | 85 ++++++++- 9 files changed, 551 insertions(+), 21 deletions(-) create mode 100644 kernel/src/arch/aarch64/boot_prod.rs create mode 100644 kernel/src/arch/aarch64/fdt.rs diff --git a/kernel/src/arch/aarch64/boot.rs b/kernel/src/arch/aarch64/boot.rs index c06915a..1dbabf1 100644 --- a/kernel/src/arch/aarch64/boot.rs +++ b/kernel/src/arch/aarch64/boot.rs @@ -21,6 +21,12 @@ global_asm!( .section .text.boot, "ax" .globl _start _start: + // QEMU `-kernel` passes the Flattened Device Tree (DTB) pointer in x0 at + // entry. Preserve it in x20 (callee-saved, untouched by the bring-up asm) + // so we can stash it once the BSS — which holds the DTB-pointer static — is + // zeroed. The MPIDR check below clobbers x0. + mov x20, x0 + // ── Park secondary CPUs: only MPIDR.Aff0==0 proceeds ────────────────── mrs x0, mpidr_el1 and x0, x0, #0xFF @@ -76,6 +82,8 @@ _start: b 3b 4: // ── Into Rust ───────────────────────────────────────────────────────── + // AAPCS64: pass the saved DTB pointer (x20) to aarch64_start as arg0 (x0). + mov x0, x20 bl {aarch64_start} // aarch64_start never returns; park if it does. 5: wfe @@ -115,8 +123,16 @@ extern "C" { } /// First Rust function on the aarch64 boot path. Runs identity-mapped, MMU off, -/// at EL1, on the boot stack. Hands off to the bring-up sequence. +/// at EL1, on the boot stack. `dtb_ptr` is the value QEMU placed in x0 (the +/// Flattened Device Tree blob's physical address). +/// +/// Hands off to the production boot path (`aarch64_kernel_boot`), which records +/// the DTB pointer, runs the minimal ARM early-init (serial + MMU), then enters +/// the SHARED `kernel_main`. The self-contained bring-up demo is still reachable +/// via [`crate::arch::aarch64::bringup::run`] for platform-primitive validation. #[no_mangle] -pub extern "C" fn aarch64_start() -> ! { - crate::arch::aarch64::bringup::run() +pub extern "C" fn aarch64_start(dtb_ptr: u64) -> ! { + // Record the DTB pointer now that BSS is zeroed (the static lives in BSS). + crate::arch::aarch64::fdt::set_dtb_ptr(dtb_ptr); + crate::arch::aarch64::boot_prod::aarch64_kernel_boot(dtb_ptr) } diff --git a/kernel/src/arch/aarch64/boot_prod.rs b/kernel/src/arch/aarch64/boot_prod.rs new file mode 100644 index 0000000..60a26fe --- /dev/null +++ b/kernel/src/arch/aarch64/boot_prod.rs @@ -0,0 +1,91 @@ +//! aarch64 production boot path — route the ARM `-kernel` entry into the SAME +//! shared `kernel_main` the x86 (Limine) path uses. +//! +//! The x86 production kernel boots via Limine, which hands the kernel a memory +//! map, an HHDM offset, a framebuffer, and modules. On `qemu-system-aarch64 -M +//! virt` there is no Limine: QEMU loads us via `-kernel` and passes a Flattened +//! Device Tree (DTB) pointer in x0. This module bridges the gap: +//! +//! 1. Bring up the ARM platform primitives the shared kernel assumes are +//! already live by the time `kernel_main` runs: PL011 serial, the MMU +//! (identity map of devices + RAM), the EL1 exception vectors, and the +//! GIC + generic timer. +//! 2. Parse the DTB to discover usable RAM and build the architecture-neutral +//! [`crate::mm::BootMemMap`]. +//! 3. Call the shared `kernel_main` entry (`crate::kernel_main_arch`), which +//! runs the same subsystem init (heap, scheduler, VFS, process table, …) +//! and ultimately spawns the init/host process. +//! +//! Limine-only inputs the shared path expects (framebuffer, engine module) are +//! provided as "absent" on ARM for now — those are follow-on work; the goal of +//! this path is to reach userspace. + +use crate::arch::aarch64::{fdt, uart}; +use crate::arch::aarch64::uart::a64println; + +/// Production ARM boot entry. Called from `aarch64_start` with the DTB pointer. +/// Never returns — hands off to the shared `kernel_main`. +pub fn aarch64_kernel_boot(dtb_ptr: u64) -> ! { + // ── Serial first: the debug lifeline for everything below ─────────────── + uart::init(); + a64println!(); + a64println!("========================================"); + a64println!(" OSCortex aarch64 — PRODUCTION boot"); + a64println!("========================================"); + uart::puts("[arm-boot] DTB pointer (x0) = "); + uart::puthex_full(dtb_ptr); + uart::puts("\n"); + + // ── MMU: identity-map devices + RAM so the shared kernel can touch memory. + // The bring-up MMU covers the low device GiB + 2 GiB of RAM from 0x4000_0000, + // which is enough to reach userspace (QEMU virt default is 2 GiB). + a64println!("[arm-boot] enabling MMU (identity map: devices + 2 GiB RAM)..."); + unsafe { crate::arch::aarch64::mmu::enable(2) }; + a64println!("[arm-boot] MMU on."); + + // ── EL1 exception vectors (VBAR_EL1): needed for SVC syscalls + faults ── + a64println!("[arm-boot] installing EL1 exception vectors..."); + crate::arch::aarch64::vectors::install(); + + // ── GIC: distributor + CPU interface live so IRQs can be delivered ────── + a64println!("[arm-boot] bringing up GIC (distributor + CPU interface)..."); + crate::arch::aarch64::gic::init_distributor(); + crate::arch::aarch64::gic::init_cpu_interface(); + // The generic timer (scheduler tick) is armed later by the shared kernel via + // `arch::apic::init_bsp`; IRQ delivery stays masked until then. + + // ── Discover RAM from the device tree → neutral BootMemMap ────────────── + let mut map = crate::mm::BootMemMap::new(); + match unsafe { fdt::parse(dtb_ptr) } { + Some(info) => { + for i in 0..info.mem_count { + let r = info.mem[i]; + uart::puts("[arm-boot] DTB /memory region: base="); + uart::puthex_full(r.base); + uart::puts(" size="); + uart::puthex_full(r.size); + uart::puts("\n"); + map.push(r.base, r.size); + } + a64println!( + "[arm-boot] device tree: {} MiB RAM across {} region(s)", + info.ram_total() / (1024 * 1024), + info.mem_count + ); + } + None => { + // No usable DTB — fall back to the QEMU virt default (RAM @ 0x4000_0000). + // 2 GiB matches `scripts/run-aarch64.sh -m 2G` and the MMU map above. + a64println!("[arm-boot] WARNING: no valid DTB /memory node; assuming 2 GiB @ 0x40000000"); + map.push(0x4000_0000, 2u64 * 1024 * 1024 * 1024); + } + } + + // ── HHDM offset: 0 on aarch64. The bring-up MMU identity-maps RAM, so the + // direct-map virtual address equals the physical address (VA == PA). The + // frame allocator / paging layer therefore use a zero phys→virt offset. + let hhdm_offset: u64 = 0; + + a64println!("[arm-boot] handing off to shared kernel_main ..."); + crate::kernel_main_arch(map, hhdm_offset) +} diff --git a/kernel/src/arch/aarch64/fdt.rs b/kernel/src/arch/aarch64/fdt.rs new file mode 100644 index 0000000..11ec6ff --- /dev/null +++ b/kernel/src/arch/aarch64/fdt.rs @@ -0,0 +1,266 @@ +//! Minimal flattened-device-tree (FDT / DTB) reader for the aarch64 port. +//! +//! On `qemu-system-aarch64 -M virt` there is no Limine bootloader: QEMU loads +//! the kernel via `-kernel` and passes a pointer to a Flattened Device Tree blob +//! in `x0` at `_start`. The DTB describes the machine — RAM base/size, and the +//! MMIO layout of the UART, GIC, timer, etc. +//! +//! This is a deliberately small, hand-rolled, `no_std`/no-alloc parser (no +//! external crate). It understands just enough of the DTB v17 structure to: +//! * validate the header magic, +//! * walk the structure block, +//! * extract the `/memory` node's `reg` (RAM base + size), and +//! * read `#address-cells` / `#size-cells` so multi-cell `reg` values decode. +//! +//! Spec reference: the Devicetree Specification (flattened format). All +//! multi-byte integers in a DTB are **big-endian**. + +use core::sync::atomic::{AtomicU64, Ordering}; + +// ── FDT header constants ──────────────────────────────────────────────────── + +const FDT_MAGIC: u32 = 0xd00d_feed; + +const FDT_BEGIN_NODE: u32 = 0x0000_0001; +const FDT_END_NODE: u32 = 0x0000_0002; +const FDT_PROP: u32 = 0x0000_0003; +const FDT_NOP: u32 = 0x0000_0004; +const FDT_END: u32 = 0x0000_0009; + +/// The DTB pointer QEMU passed in x0, captured at `_start`. Zero until set. +static DTB_PTR: AtomicU64 = AtomicU64::new(0); + +/// Record the physical address of the DTB (called from the boot path with the +/// value `_start` saved out of x0). +pub fn set_dtb_ptr(pa: u64) { + DTB_PTR.store(pa, Ordering::Release); +} + +/// The recorded DTB pointer (0 if none was captured). +pub fn dtb_ptr() -> u64 { + DTB_PTR.load(Ordering::Acquire) +} + +// ── Big-endian readers over the raw blob ──────────────────────────────────── + +#[inline] +unsafe fn read_be32(p: *const u8) -> u32 { + u32::from_be_bytes([*p, *p.add(1), *p.add(2), *p.add(3)]) +} + +#[inline] +unsafe fn read_be64(p: *const u8) -> u64 { + let hi = read_be32(p) as u64; + let lo = read_be32(p.add(4)) as u64; + (hi << 32) | lo +} + +// ── Header view ───────────────────────────────────────────────────────────── + +#[derive(Clone, Copy)] +struct FdtHeader { + base: *const u8, + off_dt_struct: u32, + off_dt_strings: u32, + size_dt_struct: u32, +} + +impl FdtHeader { + /// Validate and parse the FDT header at `base`. Returns `None` if the magic + /// is wrong (no/invalid DTB). + unsafe fn parse(base: *const u8) -> Option { + if base.is_null() { + return None; + } + let magic = read_be32(base); + if magic != FDT_MAGIC { + return None; + } + Some(FdtHeader { + base, + // Layout: magic(0) totalsize(4) off_dt_struct(8) off_dt_strings(12) + // off_mem_rsvmap(16) version(20) ... + // size_dt_strings(32) size_dt_struct(36) + off_dt_struct: read_be32(base.add(8)), + off_dt_strings: read_be32(base.add(12)), + size_dt_struct: read_be32(base.add(36)), + }) + } + + #[inline] + unsafe fn struct_start(&self) -> *const u8 { + self.base.add(self.off_dt_struct as usize) + } + + #[inline] + unsafe fn struct_end(&self) -> *const u8 { + self.base + .add(self.off_dt_struct as usize + self.size_dt_struct as usize) + } + + /// Resolve a property name by its offset into the strings block. + unsafe fn string_at(&self, off: u32) -> &'static str { + let p = self.base.add(self.off_dt_strings as usize + off as usize); + let mut len = 0usize; + while *p.add(len) != 0 { + len += 1; + } + let bytes = core::slice::from_raw_parts(p, len); + core::str::from_utf8(bytes).unwrap_or("") + } +} + +/// A single discovered memory region (physical base + length, in bytes). +#[derive(Clone, Copy, Debug)] +pub struct MemRegion { + pub base: u64, + pub size: u64, +} + +/// Result of parsing the device tree for the bits the kernel needs. +#[derive(Clone, Copy)] +pub struct FdtInfo { + /// Discovered RAM regions (from `/memory` nodes). Up to 4 retained. + pub mem: [MemRegion; 4], + pub mem_count: usize, +} + +impl FdtInfo { + const fn empty() -> Self { + FdtInfo { + mem: [MemRegion { base: 0, size: 0 }; 4], + mem_count: 0, + } + } + + /// Total RAM bytes across all discovered regions. + pub fn ram_total(&self) -> u64 { + let mut t = 0u64; + for i in 0..self.mem_count { + t += self.mem[i].size; + } + t + } +} + +/// Walk the DTB structure block and extract the `/memory` node(s). +/// +/// Returns `None` if no valid DTB is present at `dtb_pa`. +/// +/// # Safety +/// `dtb_pa` must point at a DTB blob mapped/identity-accessible at this address. +pub unsafe fn parse(dtb_pa: u64) -> Option { + let hdr = FdtHeader::parse(dtb_pa as *const u8)?; + + let mut info = FdtInfo::empty(); + + // Root cell sizes default to 2/1 per spec; QEMU virt uses 2/2. + let mut root_addr_cells: u32 = 2; + let mut root_size_cells: u32 = 2; + // Cell sizes that apply to the *current* node's children (set when we see + // the property on the parent). We track only one level of nesting context + // since /memory sits directly under the root. + let mut cur_addr_cells: u32 = 2; + let mut cur_size_cells: u32 = 2; + + let mut p = hdr.struct_start(); + let end = hdr.struct_end(); + + // Depth tracking: depth 0 = root. /memory is depth 1. + let mut depth: i32 = -1; + // Whether the node we're currently inside is a `/memory*` node. + let mut in_memory = false; + + while p < end { + let token = read_be32(p); + p = p.add(4); + match token { + t if t == FDT_BEGIN_NODE => { + // Node name is a NUL-terminated string, padded to 4 bytes. + let name_ptr = p; + let mut nlen = 0usize; + while *name_ptr.add(nlen) != 0 { + nlen += 1; + } + let name = core::str::from_utf8(core::slice::from_raw_parts(name_ptr, nlen)) + .unwrap_or(""); + // Advance past name + NUL, aligned up to 4. + p = p.add((nlen + 4) & !3); + + depth += 1; + if depth == 1 { + // Entering a top-level node: inherit the root cell sizes as + // the defaults for this node's `reg`. + cur_addr_cells = root_addr_cells; + cur_size_cells = root_size_cells; + // A memory node is named "memory" or "memory@". + in_memory = name == "memory" || name.starts_with("memory@"); + } + } + t if t == FDT_END_NODE => { + if depth == 1 { + in_memory = false; + } + depth -= 1; + } + t if t == FDT_PROP => { + // prop header: len(4) nameoff(4), then `len` bytes of value, + // padded to 4. + let len = read_be32(p); + let nameoff = read_be32(p.add(4)); + let val = p.add(8); + let pname = hdr.string_at(nameoff); + + if depth == 0 { + // Root-level cell sizes. + if pname == "#address-cells" && len == 4 { + root_addr_cells = read_be32(val); + } else if pname == "#size-cells" && len == 4 { + root_size_cells = read_be32(val); + } + } else if depth == 1 && in_memory && pname == "reg" { + // `reg` = a list of (address, size) pairs, each cell being a + // big-endian u32; address spans `cur_addr_cells` cells, size + // spans `cur_size_cells` cells. + let pair_cells = (cur_addr_cells + cur_size_cells) as usize; + let total_cells = (len as usize) / 4; + let mut c = 0usize; + while c + pair_cells <= total_cells { + let base = read_cells(val.add(c * 4), cur_addr_cells); + let size = read_cells( + val.add((c + cur_addr_cells as usize) * 4), + cur_size_cells, + ); + if size != 0 && info.mem_count < info.mem.len() { + info.mem[info.mem_count] = MemRegion { base, size }; + info.mem_count += 1; + } + c += pair_cells; + } + } + + // Advance past the value (padded to 4). + p = p.add(((len as usize) + 3) & !3); + } + t if t == FDT_NOP => { /* skip */ } + t if t == FDT_END => break, + _ => break, // malformed — stop rather than run off the blob + } + } + + if info.mem_count == 0 { + None + } else { + Some(info) + } +} + +/// Read a 1- or 2-cell big-endian value (cells beyond 2 are ignored / unsupported). +#[inline] +unsafe fn read_cells(p: *const u8, cells: u32) -> u64 { + match cells { + 1 => read_be32(p) as u64, + 2 => read_be64(p), + _ => read_be64(p), // clamp: treat >2 as 2 (top 64 bits) + } +} diff --git a/kernel/src/arch/aarch64/mod.rs b/kernel/src/arch/aarch64/mod.rs index 6ac1d90..b8521b2 100644 --- a/kernel/src/arch/aarch64/mod.rs +++ b/kernel/src/arch/aarch64/mod.rs @@ -27,7 +27,9 @@ pub mod enter_user; // ── Bare-metal bring-up (direct `-kernel` boot) ───────────────────────────── pub mod uart; +pub mod fdt; pub mod boot; +pub mod boot_prod; pub mod mmu; pub mod vectors; pub mod gic; diff --git a/kernel/src/arch/mod.rs b/kernel/src/arch/mod.rs index 12bee20..d5645fd 100644 --- a/kernel/src/arch/mod.rs +++ b/kernel/src/arch/mod.rs @@ -8,7 +8,7 @@ cfg_if::cfg_if! { mod x86_64; pub use x86_64::*; } else if #[cfg(target_arch = "aarch64")] { - mod aarch64; + pub mod aarch64; pub use aarch64::*; } else if #[cfg(target_arch = "riscv64")] { mod riscv64; diff --git a/kernel/src/logger.rs b/kernel/src/logger.rs index 44d494d..ac6662a 100644 --- a/kernel/src/logger.rs +++ b/kernel/src/logger.rs @@ -81,6 +81,8 @@ impl SerialPort { /// Initialize UART: 115200 baud, 8-N-1, disable interrupts (x86_64 only). fn init_uart(&mut self) { + #[cfg(target_arch = "aarch64")] + crate::arch::aarch64::uart::init(); #[cfg(target_arch = "x86_64")] unsafe { cpu_out8(self.port + 1, 0x00); // Disable all interrupts @@ -100,9 +102,13 @@ impl SerialPort { while (cpu_in8(self.port + 5) & 0x20) == 0 {} cpu_out8(self.port, byte); } - // On non-x86 arches the serial stub is a no-op until a PL011/UART driver - // is implemented. The framebuffer console still receives all output. - #[cfg(not(target_arch = "x86_64"))] + // aarch64: route bytes to the PL011 UART driver so the shared kernel's + // serial output (early_print + the log framework) reaches the host + // terminal under QEMU `-M virt`. + #[cfg(target_arch = "aarch64")] + crate::arch::aarch64::uart::putb(byte); + // Other non-x86 arches: no serial backend yet; framebuffer console only. + #[cfg(not(any(target_arch = "x86_64", target_arch = "aarch64")))] let _ = byte; } } diff --git a/kernel/src/main.rs b/kernel/src/main.rs index df8dc39..08a9e10 100644 --- a/kernel/src/main.rs +++ b/kernel/src/main.rs @@ -54,25 +54,37 @@ use core::sync::atomic::{AtomicBool, Ordering}; /// Global kernel initialisation lock — prevents double-init on SMP wake-up. pub static KERNEL_INIT_DONE: AtomicBool = AtomicBool::new(false); -// ── Limine boot requests ───────────────────────────────────────────────────── +// ── Limine boot requests (x86_64 boot protocol) ────────────────────────────── +// +// Only the x86_64 production path boots via Limine. The aarch64 path boots via +// `qemu-system-aarch64 -M virt -kernel` (no bootloader) and discovers hardware +// from the device tree — see `arch::aarch64::boot_prod`. The Limine request +// statics are therefore compiled only for x86_64 so the ARM kernel carries no +// dead Limine boot section. +#[cfg(target_arch = "x86_64")] use limine::request::{ FramebufferRequest, HhdmRequest, ExecutableAddressRequest, MemmapRequest, MpRequest, ModulesRequest, }; +#[cfg(target_arch = "x86_64")] use limine::BaseRevision; /// Limine base revision — required for Limine 9+ to recognise this kernel. +#[cfg(target_arch = "x86_64")] #[used] static BASE_REVISION: BaseRevision = BaseRevision::new(); +#[cfg(target_arch = "x86_64")] static FB_REQUEST: FramebufferRequest = FramebufferRequest::new(); +#[cfg(target_arch = "x86_64")] static HHDM_REQUEST: HhdmRequest = HhdmRequest::new(); +#[cfg(target_arch = "x86_64")] static MMAP_REQUEST: MemmapRequest = MemmapRequest::new(); +#[cfg(target_arch = "x86_64")] static KADDR_REQUEST: ExecutableAddressRequest = ExecutableAddressRequest::new(); #[cfg(target_arch = "x86_64")] static SMP_REQUEST: MpRequest = MpRequest::new(limine::mp::MP_FLAG_X2APIC); -#[cfg(not(target_arch = "x86_64"))] -static SMP_REQUEST: MpRequest = MpRequest::new(0); +#[cfg(target_arch = "x86_64")] static MODULES_REQUEST: ModulesRequest = ModulesRequest::new(); /// Slice of libflutter_engine.so as provided by Limine, stored once at boot. @@ -95,6 +107,7 @@ pub static FLUTTER_ENGINE_BYTES: spin::Mutex> = /// 7. AI Cortex init ← the magic /// 8. SMP wake-up (other cores join here after KERNEL_INIT_DONE) /// 9. Idle loop (Cortex takes over scheduling from here) +#[cfg(target_arch = "x86_64")] #[no_mangle] pub extern "C" fn kernel_main() -> ! { // ── 0. Early serial — must be first so any crash below is visible ───── @@ -192,6 +205,56 @@ pub extern "C" fn kernel_main() -> ! { log::warn!("[BOOT] No Limine modules response — libflutter_engine.so unavailable"); } + // The architecture-neutral subsystem init + process spawn + idle loop is + // shared with the aarch64 boot path. + shared_init_and_run() +} + +/// aarch64 production kernel entry. +/// +/// Called from `arch::aarch64::boot_prod::aarch64_kernel_boot` after the ARM +/// platform primitives (serial, MMU, EL1 vectors, GIC) are live and the device +/// tree has been parsed into the neutral [`mm::BootMemMap`]. This mirrors the +/// x86 `kernel_main` prologue using arch-provided inputs instead of Limine, then +/// joins the same `shared_init_and_run`. +#[cfg(target_arch = "aarch64")] +pub fn kernel_main_arch(map: mm::BootMemMap, hhdm_offset: u64) -> ! { + logger::early_print("[OSCORTEX] kernel_main_arch reached (aarch64)\r\n"); + + // HHDM offset is 0 on aarch64 (RAM is identity-mapped → VA == PA). + mm::frame_allocator::set_hhdm_offset(hhdm_offset); + + // ── 1. Architecture early init (FPU/SIMD, syscall/SVC readiness) ────── + // The interrupt controller + exception vectors are already live (set up in + // boot_prod), but early_init also enables FP/SIMD and logs syscall setup. + arch::early_init(); + logger::early_print("[OSCORTEX] arch::early_init done (aarch64)\r\n"); + + // ── 2. Memory management from the device-tree region list ──────────── + mm::init_from_regions(&map, hhdm_offset); + + // ── 3. Logger (serial only — no Limine framebuffer on ARM yet) ─────── + logger::init(None); + + log::warn!( + "[MM::FrameAlloc] capacity at boot: total_usable_frames={} ({} MiB), used_at_boot={} ({} MiB)", + mm::frame_allocator::frames_total(), + (mm::frame_allocator::frames_total() * 4096) / (1024 * 1024), + mm::frame_allocator::frames_used(), + (mm::frame_allocator::frames_used() * 4096) / (1024 * 1024) + ); + log::info!("OSCortex kernel {} booting (aarch64)...", env!("CARGO_PKG_VERSION")); + log::warn!("[BOOT] No framebuffer / Flutter engine on aarch64 yet (follow-on)."); + + shared_init_and_run() +} + +/// Architecture-neutral subsystem init, init-process spawn, and idle loop. +/// +/// Runs after each arch's boot prologue has brought up serial, the memory +/// manager, and the logger. This is the part of `kernel_main` that does not +/// touch any boot protocol directly. +fn shared_init_and_run() -> ! { // ── 4. Platform drivers (input probe) ─────────────────────────────── let qemu_like = arch::cpu::is_qemu_like_hypervisor(); drivers::platform::init_early(qemu_like); @@ -226,7 +289,10 @@ pub extern "C" fn kernel_main() -> ! { // ── 9. Signal APs + bring SMP online ──────────────────────────────── KERNEL_INIT_DONE.store(true, Ordering::Release); + #[cfg(target_arch = "x86_64")] arch::smp_init(SMP_REQUEST.response()); + #[cfg(not(target_arch = "x86_64"))] + arch::smp_init(None); // ── 9b. Spawn init process from initramfs ──────────────────────────── match fs::lookup("/init") { diff --git a/kernel/src/mm/frame_allocator.rs b/kernel/src/mm/frame_allocator.rs index 1b7e8c4..5f0c169 100644 --- a/kernel/src/mm/frame_allocator.rs +++ b/kernel/src/mm/frame_allocator.rs @@ -3,8 +3,6 @@ //! Uses a static bitmap where each bit represents a 4 KiB physical frame. //! The bitmap itself is placed in the first usable memory region. -use limine::memmap::MEMMAP_USABLE; -use limine::request::MemmapResponse; use spin::Mutex; use core::sync::atomic::{AtomicU64, Ordering}; @@ -109,17 +107,19 @@ impl FrameBitmap { } } -pub fn init(mmap: &MemmapResponse, hhdm_offset: u64) { +/// Initialise the frame bitmap from an arch-neutral usable-region list. +/// +/// Both the x86_64 (Limine) and aarch64 (device-tree) boot paths funnel through +/// here via [`crate::mm::init_from_regions`]. +pub fn init_from_regions(map: &crate::mm::BootMemMap, hhdm_offset: u64) { set_hhdm_offset(hhdm_offset); let mut bm = BITMAP.lock(); bm.hhdm = hhdm_offset; - for entry in mmap.entries() { - if entry.type_ == MEMMAP_USABLE { - let start_frame = (entry.base as usize) / FRAME_SIZE; - let end_frame = ((entry.base + entry.length) as usize) / FRAME_SIZE; - for frame in start_frame.max(MIN_ALLOC_FRAME)..end_frame.min(MAX_FRAMES) { - bm.mark_free(frame); - } + for region in map.regions() { + let start_frame = (region.base as usize) / FRAME_SIZE; + let end_frame = ((region.base + region.len) as usize) / FRAME_SIZE; + for frame in start_frame.max(MIN_ALLOC_FRAME)..end_frame.min(MAX_FRAMES) { + bm.mark_free(frame); } } log::info!("[MM::FrameAlloc] {} MiB usable physical memory", diff --git a/kernel/src/mm/mod.rs b/kernel/src/mm/mod.rs index 970d83c..10da74a 100644 --- a/kernel/src/mm/mod.rs +++ b/kernel/src/mm/mod.rs @@ -6,14 +6,97 @@ pub mod paging; pub use paging::{map_mmio, map_page, PageFlags}; +#[cfg(target_arch = "x86_64")] use limine::request::{MemmapResponse, ExecutableAddressResponse}; +// ── Architecture-neutral boot memory map ───────────────────────────────────── +// +// The frame allocator needs a list of *usable* physical RAM regions. x86_64 +// gets that from the Limine memory map; aarch64 (QEMU `-M virt`, `-kernel`, +// no bootloader) gets it from the device tree. To keep the shared mm init free +// of any single boot protocol, both arches translate their source into this +// small, fixed-capacity, no-alloc region list and hand it to `frame_allocator`. + +/// One usable physical RAM region: `[base, base+len)`. +#[derive(Clone, Copy, Debug)] +pub struct BootMemRegion { + pub base: u64, + pub len: u64, +} + +/// Maximum distinct usable regions we retain. Limine emits a handful; QEMU virt +/// reports one or two. 32 is comfortable headroom. +pub const MAX_BOOT_REGIONS: usize = 32; + +/// A fixed-capacity list of usable physical RAM regions, built by the arch boot +/// path and consumed by `frame_allocator::init_from_regions`. +#[derive(Clone, Copy)] +pub struct BootMemMap { + regions: [BootMemRegion; MAX_BOOT_REGIONS], + count: usize, +} + +impl BootMemMap { + pub const fn new() -> Self { + BootMemMap { + regions: [BootMemRegion { base: 0, len: 0 }; MAX_BOOT_REGIONS], + count: 0, + } + } + + /// Append a usable region (silently dropped if capacity is exceeded or the + /// region is empty). + pub fn push(&mut self, base: u64, len: u64) { + if len == 0 || self.count >= MAX_BOOT_REGIONS { + return; + } + self.regions[self.count] = BootMemRegion { base, len }; + self.count += 1; + } + + /// The usable regions discovered at boot. + pub fn regions(&self) -> &[BootMemRegion] { + &self.regions[..self.count] + } +} + +impl Default for BootMemMap { + fn default() -> Self { + Self::new() + } +} + +// ── x86_64 boot path (Limine) ──────────────────────────────────────────────── + +/// Initialise memory management from the Limine boot info (x86_64 path). +/// +/// Behaviour is byte-equivalent to the prior direct-Limine implementation: it +/// simply translates the Limine memory map into the neutral [`BootMemMap`] and +/// forwards to the shared [`init_from_regions`]. +#[cfg(target_arch = "x86_64")] pub fn init( mmap: &MemmapResponse, hhdm_offset: u64, _kaddr: &ExecutableAddressResponse, ) { - frame_allocator::init(mmap, hhdm_offset); + use limine::memmap::MEMMAP_USABLE; + let mut map = BootMemMap::new(); + for entry in mmap.entries() { + if entry.type_ == MEMMAP_USABLE { + map.push(entry.base, entry.length); + } + } + init_from_regions(&map, hhdm_offset); +} + +// ── Shared init (arch-neutral) ─────────────────────────────────────────────── + +/// Initialise memory management from an arch-provided usable-region list. +/// +/// `hhdm_offset` is the physical→virtual direct-map offset (Limine HHDM on +/// x86_64; 0 on aarch64 where the bring-up identity-maps RAM so VA == PA). +pub fn init_from_regions(map: &BootMemMap, hhdm_offset: u64) { + frame_allocator::init_from_regions(map, hhdm_offset); paging::init(hhdm_offset); heap::init(); log::info!("[MM] Memory management online"); From 834e145131aac26c1ebbec71d453d4716b438c61 Mon Sep 17 00:00:00 2001 From: Tahiru Agbanwa <130235676+squirelboy360@users.noreply.github.com> Date: Tue, 9 Jun 2026 02:05:06 +0200 Subject: [PATCH 12/36] aarch64: spawn the init process at EL0 and service its syscalls MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Completes the ARM port's headline goal: the PRODUCTION kernel_main now spawns a userspace process and services its first syscalls through the SHARED dispatcher. * aarch64 user page-table walker (mm/paging.rs): a real 4 KiB-granule L1→L3 TTBR0 walker replaces the non-x86 stub — alloc_user_pml4, map_page_in, translate/update/unmap, free. Each per-process root is seeded with a copy of the kernel's identity-map L1 block descriptors so the kernel stays mapped after write_cr3 loads the process root (the bring-up kernel runs from the TTBR0 low half). free_user_pml4 skips those shared block descriptors so it never frees kernel RAM. * SVC → shared dispatch (arch/aarch64/syscall.rs): the production SVC handler captures the EL0 trap frame, calls syscall::dispatch_fast(x8, x0..x4), writes the result back to x0, and on exit/exit_group hands off to the next runnable process (or parks). Installed in arch::early_init. * ELF loader (process/elf.rs): accept EM_AARCH64 on ARM via an arch-selected EM_NATIVE machine check. * Process spawn (process/mod.rs): arch-conditional USER_STACK_TOP placed inside the 39-bit aarch64 VA window; the x86 glibc POSIX trampoline mapping is skipped on ARM (bare EL0 program, no libc). * Scheduler (sched/mod.rs): spawn_kernel_task builds an AAPCS64 first-run frame on aarch64 (x19..x30 + fn-ptr slot matching context_switch/task_entry), instead of the x86 register layout. * Native /init (build.rs): for aarch64 builds, embed a tiny EL0 ELF (linked at 64 GiB to clear the kernel identity blocks) that does write + getpid + write + exit using the OSCortex syscall numbers. * main.rs: on aarch64 the init is entered directly via enter_user_by_pid_noreturn (the real write_cr3 → ERET path) since the ARM preemptive timer ISR is a follow-on; x86 keeps the schedule_user_launch hand-off. Verified on qemu-system-aarch64 -M virt: init spawns (pid=1, EL0 entry), prints two lines via write(1,...), getpid returns, exit(0) reaps cleanly and the kernel parks — all through the shared syscall layer, no faults. x86_64 debug + release still build green; no /init is injected on x86. --- kernel/build.rs | 136 ++++++++++++ kernel/src/arch/aarch64/syscall.rs | 53 ++++- kernel/src/main.rs | 20 +- kernel/src/mm/paging.rs | 330 +++++++++++++++++++++++++++-- kernel/src/process/elf.rs | 11 +- kernel/src/process/mod.rs | 11 + kernel/src/sched/mod.rs | 25 +++ 7 files changed, 563 insertions(+), 23 deletions(-) diff --git a/kernel/build.rs b/kernel/build.rs index b154751..49f6917 100644 --- a/kernel/build.rs +++ b/kernel/build.rs @@ -47,6 +47,18 @@ fn main() { generate_liboscortex_embedder_shim(), )); + // aarch64 port: provide a native EL0 `/init` so the production kernel_main can + // spawn a userspace process + service its first syscalls on ARM. The x86 build + // ships the Flutter host as `/init` (staged by scripts/build-iso.sh); on ARM + // that host doesn't exist yet (engine is a follow-on), so without this there is + // no native `/init` to spawn. This minimal program issues a couple of `write` + // syscalls, a `getpid`, then `exit` — exercising the full spawn → SVC dispatch + // → exit path through the SHARED syscall layer. + if arch == "aarch64" { + entries.retain(|(n, _)| n != "init"); + entries.push(("init".to_string(), generate_aarch64_init_elf())); + } + // Write the USTAR tar to OUT_DIR. let tar_bytes = build_ustar_tar(&entries); std::fs::write(&out_tar, &tar_bytes).unwrap(); @@ -440,6 +452,130 @@ fn generate_liboscortex_embedder_shim() -> Vec { generate_ret_stub_elf(SYMS) } +// ── aarch64 native `/init` EL0 program ─────────────────────────────────────── + +/// Build a minimal statically-linked AArch64 ELF64 (ET_EXEC, EM_AARCH64) that +/// runs at EL0 and issues syscalls through the OSCortex shared dispatcher. +/// +/// Syscall ABI (OSCortex / x86-numbered, as understood by `dispatch_fast`): +/// nr in x8, args in x0..x2, return in x0, trap via `svc #0`. +/// 1 = write(fd, buf, len) → fd 1/2 print to the kernel serial console +/// 39 = getpid() → returns the process id +/// 60 = exit(code) → terminate the process +/// +/// Layout: a single RWX PT_LOAD at vaddr 0x400000 (USER_ELF_BASE). Code starts +/// at the entry; two NUL-free messages live at fixed offsets in the same page. +fn generate_aarch64_init_elf() -> Vec { + // Link the init program at 64 GiB (L1 index 64), well clear of the kernel's + // low identity map. The bring-up kernel runs from the TTBR0 low half, so each + // per-process root is seeded with the kernel's 1 GiB block descriptors for + // the device region (index 0) and RAM (indices 1..2). A user page at a low VA + // would collide with those blocks; placing the program at a high, otherwise- + // empty L1 index lets the page-table walker create fresh L2/L3 tables there. + const VBASE: u64 = 0x10_0000_0000; // 64 GiB + + // Messages placed after the code, at page offsets 0x200 / 0x280. + let msg1: &[u8] = b"[arm-init] hello from EL0 userspace via svc #0\n"; + let msg2: &[u8] = b"[arm-init] getpid ok; calling exit(0)\n"; + let msg1_off: u64 = 0x200; + let msg2_off: u64 = 0x280; + let msg1_va = VBASE + msg1_off; + let msg2_va = VBASE + msg2_off; + + // ── Assemble the code (offset 0) ──────────────────────────────────────── + let mut code: Vec = Vec::new(); + + // write(1, msg1_va, msg1.len()) + a64_load_imm(&mut code, 0, 1); + a64_load_imm(&mut code, 1, msg1_va); + a64_load_imm(&mut code, 2, msg1.len() as u64); + a64_load_imm(&mut code, 8, 1); + code.push(0xD400_0001); // svc #0 + + // getpid() + a64_load_imm(&mut code, 8, 39); + code.push(0xD400_0001); + + // write(1, msg2_va, msg2.len()) + a64_load_imm(&mut code, 0, 1); + a64_load_imm(&mut code, 1, msg2_va); + a64_load_imm(&mut code, 2, msg2.len() as u64); + a64_load_imm(&mut code, 8, 1); + code.push(0xD400_0001); + + // exit(0) + a64_load_imm(&mut code, 0, 0); + a64_load_imm(&mut code, 8, 60); + code.push(0xD400_0001); + + // Safety net: spin if exit ever returns to EL0. + code.push(0x1400_0000); // b . + + // ── Compose the single loadable page image ────────────────────────────── + let page_sz = 0x1000usize; + let mut img = vec![0u8; page_sz]; + for (i, w) in code.iter().enumerate() { + let o = i * 4; + img[o..o + 4].copy_from_slice(&w.to_le_bytes()); + } + img[msg1_off as usize..msg1_off as usize + msg1.len()].copy_from_slice(msg1); + img[msg2_off as usize..msg2_off as usize + msg2.len()].copy_from_slice(msg2); + + // ── ELF header + one program header ───────────────────────────────────── + let ehsize = 64usize; + let phentsize = 56usize; + let phoff = ehsize; + let data_off = phoff + phentsize; // file offset of the loadable image + + let file_sz = data_off + img.len(); + let mut elf = vec![0u8; file_sz]; + + // e_ident + elf[0..4].copy_from_slice(&[0x7F, b'E', b'L', b'F']); + elf[4] = 2; // ELFCLASS64 + elf[5] = 1; // ELFDATA2LSB + elf[6] = 1; // EV_CURRENT + write_u16(&mut elf, 16, 2); // e_type = ET_EXEC + write_u16(&mut elf, 18, 183); // e_machine = EM_AARCH64 + write_u32(&mut elf, 20, 1); // e_version + write_u64(&mut elf, 24, VBASE); // e_entry + write_u64(&mut elf, 32, phoff as u64); // e_phoff + write_u16(&mut elf, 52, ehsize as u16); // e_ehsize + write_u16(&mut elf, 54, phentsize as u16); // e_phentsize + write_u16(&mut elf, 56, 1); // e_phnum + + // PT_LOAD program header (RWX). + let p = phoff; + write_u32(&mut elf, p, 1); // p_type = PT_LOAD + write_u32(&mut elf, p + 4, 7); // p_flags = R|W|X + write_u64(&mut elf, p + 8, data_off as u64); // p_offset + write_u64(&mut elf, p + 16, VBASE); // p_vaddr + write_u64(&mut elf, p + 24, VBASE); // p_paddr + write_u64(&mut elf, p + 32, img.len() as u64); // p_filesz + write_u64(&mut elf, p + 40, img.len() as u64); // p_memsz + write_u64(&mut elf, p + 48, 0x1000); // p_align + + elf[data_off..data_off + img.len()].copy_from_slice(&img); + elf +} + +/// Emit a MOVZ/MOVK sequence loading the 64-bit immediate `imm` into x`reg`. +fn a64_load_imm(code: &mut Vec, reg: u32, imm: u64) { + let mut first = true; + for hw in 0..4u32 { + let imm16 = ((imm >> (hw * 16)) & 0xFFFF) as u32; + if imm16 == 0 && !first { + continue; // skip zero halfwords once at least one MOVZ emitted + } + let base = if first { 0xD280_0000 } else { 0xF280_0000 }; + code.push(base | (hw << 21) | (imm16 << 5) | reg); + first = false; + } + if first { + code.push(0xD280_0000 | reg); // imm == 0 → MOVZ Xreg, #0 + } +} + /// Generate a minimal ELF64 ET_DYN where every exported symbol is a /// 4-byte `xor eax,eax; ret; nop; nop` stub. Used for both /// `libflutter_engine.so` (existing) and `liboscortex_embedder.so`. diff --git a/kernel/src/arch/aarch64/syscall.rs b/kernel/src/arch/aarch64/syscall.rs index dd995ec..6029214 100644 --- a/kernel/src/arch/aarch64/syscall.rs +++ b/kernel/src/arch/aarch64/syscall.rs @@ -93,13 +93,58 @@ pub fn capture_from_trap(frame: &super::vectors::TrapFrame) { } } +/// Production SVC (EL0 synchronous) handler — routes a userspace `svc #0` into +/// the shared syscall dispatcher. +/// +/// The Linux aarch64 syscall ABI puts the number in x8 and args in x0..x5; the +/// OSCortex shared dispatcher (`syscall::dispatch_fast`) takes (nr, a0..a4) and +/// returns the result in x0. We capture the trapping thread's user GPRs into the +/// per-CPU snapshot first (so the dispatcher's `user_rip`/`user_rsp`/`user_gprs` +/// accessors reflect the caller), dispatch, then write the return value back into +/// x0 of the saved frame so the vector stub's `eret` resumes EL0 with it. +/// +/// The faulting instruction is `svc #0`; ELR_EL1 already points at the *next* +/// instruction on a synchronous SVC, so no manual advance is needed. +/// +/// If the call terminated the process (exit / exit_group), the EL0 context is no +/// longer valid to resume, so we hand off to the next runnable user process via +/// the shared scheduler instead of returning to the vector stub. +fn production_svc_handler(f: &mut super::vectors::TrapFrame) { + capture_from_trap(f); + + let nr = f.x[8]; + let ret = crate::syscall::dispatch_fast( + nr, f.x[0], f.x[1], f.x[2], f.x[3], f.x[4], + ); + f.x[0] = ret as u64; + + // exit(60) / exit_group(231): the current process is now a zombie; do not + // eret back into it. Pick the next runnable user thread and enter it. + if nr == 60 || nr == 231 { + let cur = crate::process::current_pid(); + match crate::process::next_runnable_pid(cur) { + Some(next) => crate::process::enter_user_by_pid_noreturn(next), + None => { + // No more user work — park this core. The shared idle/cortex loop + // keeps the kernel alive; an exiting init with nothing to run is a + // clean stop for the bring-up. + log::info!("[svc] init exited (code={}) — no runnable user process; parking", ret); + loop { + crate::arch::halt(); + } + } + } + } +} + /// Initialise the SVC fast path on the BSP. /// -/// The EL1 exception vector table (VBAR_EL1) that decodes SVC and captures the -/// user snapshot is installed by the bring-up (`vectors::install`); this only -/// logs readiness. +/// Installs the production SVC handler into the EL1 vector table (VBAR_EL1 was +/// programmed earlier by `vectors::install`), so a userspace `svc #0` reaches +/// the shared syscall dispatcher. pub fn init() { - log::info!("[Syscall] aarch64 SVC entry ready (capture_from_trap wired)"); + super::vectors::set_svc_handler(production_svc_handler); + log::info!("[Syscall] aarch64 SVC entry ready (production dispatch_fast wired)"); } /// Per-AP SVC init. diff --git a/kernel/src/main.rs b/kernel/src/main.rs index 08a9e10..7466bac 100644 --- a/kernel/src/main.rs +++ b/kernel/src/main.rs @@ -305,9 +305,27 @@ fn shared_init_and_run() -> ! { }; match process::spawn_with_bootstrap(elf_bytes, "init", bootstrap) { Ok(pid) => { - process::schedule_user_launch(pid); crate::wm::set_focus_pid(pid); log::info!("[INIT] Spawned shell host as PID {}", pid); + + // x86: defer the user launch to a scheduler kernel task; the + // APIC-timer-driven cooperative scheduler switches into it. + #[cfg(target_arch = "x86_64")] + process::schedule_user_launch(pid); + + // aarch64: the ARM scheduler's preemptive tick (GIC + generic + // timer ISR frame save) is a follow-on, so the timer-driven + // hand-off used on x86 isn't wired yet. Enter the init process + // DIRECTLY here — this is the real production EL0 entry + // (write_cr3 → enter_user_sysret/ERET) and lets the bring-up + // init run + have its syscalls serviced through the shared + // dispatcher. (Once the ARM timer ISR lands, this collapses to + // the same schedule_user_launch path as x86.) + #[cfg(target_arch = "aarch64")] + { + log::info!("[INIT] aarch64: entering init at EL0 directly"); + process::enter_user_by_pid_noreturn(pid); + } } Err(e) => log::warn!("[INIT] Failed to spawn /init: {}", e), } diff --git a/kernel/src/mm/paging.rs b/kernel/src/mm/paging.rs index d58d250..be26e46 100644 --- a/kernel/src/mm/paging.rs +++ b/kernel/src/mm/paging.rs @@ -404,18 +404,278 @@ mod x86_64_impl { // ─── Non-x86_64 stubs ───────────────────────────────────────────────────────── -#[cfg(not(target_arch = "x86_64"))] -mod stub_impl { +// ─── aarch64 implementation ─────────────────────────────────────────────────── +// +// A 4 KiB-granule, 4-level (L0→L1→L2→L3) translation-table walker for the EL0 +// (TTBR0) low half. Per-process address spaces each get a fresh L0 table from +// the frame allocator. The bring-up identity-maps RAM (VA == PA), so a table's +// physical address is also its directly-accessible kernel virtual address +// (HHDM offset is 0 on aarch64); we read/write tables through their PA. +// +// This matches the TCR_EL1 the bring-up programs (T0SZ=25 → 39-bit VA). With a +// 39-bit VA and 4 KiB granule, translation starts at level 1, so the top table +// is an L1 (512 entries × 1 GiB). We treat the per-process root as that L1. +#[cfg(target_arch = "aarch64")] +mod aarch64_impl { use super::PageFlags; + use crate::mm::frame_allocator; + + const PAGE_SIZE: usize = 4096; + /// Output-address mask for a table/page descriptor (bits[47:12]). + const ADDR_MASK: u64 = 0x0000_FFFF_FFFF_F000; + + // Descriptor type/attribute bits (ARMv8-A, stage-1). + const DESC_VALID: u64 = 1 << 0; + const DESC_TABLE: u64 = 1 << 1; // at L1/L2: 1 = table, 0 = block. at L3: must be 1 (page). + const DESC_AF: u64 = 1 << 10; // Access Flag + const DESC_SH_INNER: u64 = 3 << 8; // inner shareable + const DESC_AP_RW_ALL: u64 = 1 << 6; // EL0+EL1 RW (AP[1]=1) + const DESC_AP_RO_ALL: u64 = 3 << 6; // EL0+EL1 RO + const DESC_UXN: u64 = 1 << 54; // unprivileged execute-never + const DESC_PXN: u64 = 1 << 53; // privileged execute-never + const ATTR_IDX_NORMAL: u64 = 0 << 2; // MAIR attr0 = Normal WB (matches mmu.rs) + + /// 39-bit VA index extraction. With start level 1: L1[38:30], L2[29:21], + /// L3[20:12]. + #[inline] + fn idx_l1(va: u64) -> usize { ((va >> 30) & 0x1FF) as usize } + #[inline] + fn idx_l2(va: u64) -> usize { ((va >> 21) & 0x1FF) as usize } + #[inline] + fn idx_l3(va: u64) -> usize { ((va >> 12) & 0x1FF) as usize } + + /// A table is 512 u64 entries at a 4 KiB-aligned physical address. On the + /// identity map, the PA is directly dereferenceable. + #[inline] + unsafe fn entry(table_pa: u64, idx: usize) -> *mut u64 { + (table_pa as *mut u64).add(idx) + } + + /// Allocate and zero a fresh translation table; returns its physical address. + fn alloc_table() -> Option { + let pa = frame_allocator::alloc_frame()?; + unsafe { core::ptr::write_bytes(pa as *mut u8, 0, PAGE_SIZE) }; + Some(pa) + } + + /// Allocate a fresh per-process root table (the L1 for a 39-bit VA space). + /// + /// CRITICAL: on the aarch64 bring-up the kernel itself runs from the low + /// (TTBR0) half — its code, stack, vectors, and the UART/GIC MMIO are in the + /// identity map programmed at boot. A per-process TTBR0 that contained only + /// user pages would unmap the kernel the instant `write_cr3` loaded it, so + /// the very next instruction fetch (and the fault handler) would fault with + /// no way to recover. We therefore seed each new root with a copy of the + /// kernel's identity-map L1 entries (the 1 GiB block descriptors for the + /// device region + RAM). User L1 slots (which the kernel map leaves empty) + /// are then filled by `map_page_in` for the process's own pages. + pub fn alloc_user_pml4() -> Option { + let root = alloc_table()?; + let kernel_l1 = crate::arch::aarch64::mmu::kernel_ttbr0(); + unsafe { + // Copy all 512 L1 entries from the kernel identity map. The kernel + // map uses 1 GiB block descriptors at low indices; user VAs live at + // higher indices (e.g. ~511 for the stack) which are zero in the + // kernel map and get populated per-process. + core::ptr::copy_nonoverlapping( + kernel_l1 as *const u64, + root as *mut u64, + 512, + ); + } + Some(root) + } + + /// Ensure the next-level table referenced by `*slot` exists, allocating it if + /// the descriptor is empty. Returns the child table's physical address. + unsafe fn ensure_table(slot: *mut u64) -> Option { + let desc = core::ptr::read_volatile(slot); + if desc & DESC_VALID != 0 { + return Some(desc & ADDR_MASK); + } + let child = alloc_table()?; + core::ptr::write_volatile(slot, (child & ADDR_MASK) | DESC_VALID | DESC_TABLE); + Some(child) + } + + /// Build an L3 page descriptor for a user page. + fn page_desc(phys: u64, flags: PageFlags) -> u64 { + let mut d = (phys & ADDR_MASK) + | DESC_VALID + | DESC_TABLE // bit[1]=1 → page at L3 + | DESC_AF + | DESC_SH_INNER + | ATTR_IDX_NORMAL; + if flags.contains(PageFlags::WRITABLE) { + d |= DESC_AP_RW_ALL; + } else { + d |= DESC_AP_RO_ALL; + } + // Execution: x86 NO_EXECUTE bit → UXN here. We always set PXN (EL1 must + // never execute user pages). When NO_EXECUTE is set, also set UXN. + d |= DESC_PXN; + if flags.contains(PageFlags::NO_EXECUTE) || !flags.contains(PageFlags::USER) { + d |= DESC_UXN; + } + d + } + + /// Map a 4 KiB page `phys` at user VA `virt` in the address space rooted at + /// `root_pa` (the L1 table), allocating intermediate tables as needed. + /// + /// # Safety + /// `root_pa` must be a valid table from [`alloc_user_pml4`]; `virt`/`phys` + /// must be 4 KiB-aligned. + pub unsafe fn map_page_in( + root_pa: u64, + virt: u64, + phys: u64, + flags: PageFlags, + ) -> Result<(), &'static str> { + let l1 = root_pa; + let l2 = ensure_table(entry(l1, idx_l1(virt))).ok_or("OOM: l2 table")?; + let l3 = ensure_table(entry(l2, idx_l2(virt))).ok_or("OOM: l3 table")?; + core::ptr::write_volatile(entry(l3, idx_l3(virt)), page_desc(phys, flags)); + // Publish the new mapping + flush this VA in the EL0 space. + core::arch::asm!( + "dsb ishst", + "tlbi vmalle1is", + "dsb ish", + "isb", + options(nostack, preserves_flags), + ); + Ok(()) + } + + /// Walk to the L3 descriptor for `virt`; returns (l3_slot, descriptor) if the + /// full path is present, else None. + unsafe fn walk(root_pa: u64, virt: u64) -> Option<(*mut u64, u64)> { + let l1d = core::ptr::read_volatile(entry(root_pa, idx_l1(virt))); + if l1d & DESC_VALID == 0 { return None; } + let l2 = l1d & ADDR_MASK; + let l2d = core::ptr::read_volatile(entry(l2, idx_l2(virt))); + if l2d & DESC_VALID == 0 { return None; } + let l3 = l2d & ADDR_MASK; + let slot = entry(l3, idx_l3(virt)); + let d = core::ptr::read_volatile(slot); + if d & DESC_VALID == 0 { return None; } + Some((slot, d)) + } + + pub fn translate_user_page(root_pa: u64, virt: u64) -> Option { + let page = virt & !0xFFF; + let off = virt & 0xFFF; + unsafe { walk(root_pa, page).map(|(_, d)| (d & ADDR_MASK) | off) } + } + + pub fn translate_user_page_flags(root_pa: u64, virt: u64) -> Option<(u64, bool, bool)> { + let page = virt & !0xFFF; + unsafe { + walk(root_pa, page).map(|(_, d)| { + let phys = d & ADDR_MASK; + let writable = (d & DESC_AP_RW_ALL) == DESC_AP_RW_ALL; + let exec = (d & DESC_UXN) == 0; + (phys, writable, exec) + }) + } + } + + /// Update the permission bits of an already-mapped user page. + /// + /// # Safety + /// `root_pa` must be a valid allocated table. + pub unsafe fn update_user_page_flags( + root_pa: u64, + virt: u64, + writable: bool, + exec: bool, + ) -> Result<(), &'static str> { + let page = virt & !0xFFF; + let (slot, d) = walk(root_pa, page).ok_or("page not mapped")?; + let mut nd = d & !(DESC_AP_RO_ALL | DESC_UXN); + nd |= if writable { DESC_AP_RW_ALL } else { DESC_AP_RO_ALL }; + if !exec { nd |= DESC_UXN; } + core::ptr::write_volatile(slot, nd); + core::arch::asm!( + "dsb ishst", "tlbi vmalle1is", "dsb ish", "isb", + options(nostack, preserves_flags), + ); + Ok(()) + } - pub unsafe fn map_page(_virt: u64, _phys: u64, _flags: PageFlags) { - // Milestone 4b: aarch64/riscv64 page table walker (TTBR0/TTBR1, SATP). + /// Unmap a single page; returns the physical frame it pointed at. + /// + /// # Safety + /// `root_pa` must be a valid allocated table. + pub unsafe fn unmap_user_page(root_pa: u64, virt: u64) -> Result { + let page = virt & !0xFFF; + let (slot, d) = walk(root_pa, page).ok_or("not present")?; + let phys = d & ADDR_MASK; + core::ptr::write_volatile(slot, 0); + core::arch::asm!( + "dsb ishst", "tlbi vmalle1is", "dsb ish", "isb", + options(nostack, preserves_flags), + ); + Ok(phys) } - pub unsafe fn map_mmio(_phys: u64, _virt: u64, _size: usize) { - // Milestone 4b: identity/MMIO map via arch-specific page tables. + /// Walk all three levels of a user root table and free every backing frame + /// (leaf pages + intermediate tables + the root). Best-effort. + /// + /// IMPORTANT: each per-process root is seeded (in `alloc_user_pml4`) with a + /// copy of the kernel's identity-map L1 entries, which are **1 GiB block + /// descriptors** (bit[1] == 0), not table pointers. We MUST NOT recurse into + /// or free those — their "address" is kernel RAM, not a process-owned table. + /// Only L1 entries that are *table* descriptors (bit[1] == 1), i.e. the L2 + /// tables created for this process's own high-VA pages, are walked and freed. + pub fn free_user_pml4(root_pa: u64) { + if root_pa == 0 { return; } + unsafe { + for i1 in 0..512usize { + let l1d = core::ptr::read_volatile(entry(root_pa, i1)); + // Skip invalid AND block descriptors (the shared kernel identity + // map). Only table descriptors point at process-owned L2 tables. + if l1d & DESC_VALID == 0 || l1d & DESC_TABLE == 0 { continue; } + let l2 = l1d & ADDR_MASK; + for i2 in 0..512usize { + let l2d = core::ptr::read_volatile(entry(l2, i2)); + if l2d & DESC_VALID == 0 || l2d & DESC_TABLE == 0 { continue; } + let l3 = l2d & ADDR_MASK; + for i3 in 0..512usize { + let l3d = core::ptr::read_volatile(entry(l3, i3)); + if l3d & DESC_VALID != 0 { + frame_allocator::free_frame(l3d & ADDR_MASK); + } + } + frame_allocator::free_frame(l3); + } + frame_allocator::free_frame(l2); + } + frame_allocator::free_frame(root_pa); + } } + // Kernel-space single mappings: the bring-up identity-maps devices + RAM, so + // a fresh kernel `map_page`/`map_mmio` is only needed for windows outside the + // boot map. For now these are no-ops (everything the kernel touches is in the + // identity map); a follow-on can extend the kernel L1 if a new window is + // required. + pub unsafe fn map_page(_virt: u64, _phys: u64, _flags: PageFlags) {} + pub unsafe fn map_mmio(_phys: u64, _virt: u64, _size: usize) {} + + pub fn init(_hhdm_offset: u64) { + crate::logger::early_print("[MM::Paging] aarch64 TTBR0 page-table walker online\r\n"); + } +} + +// ─── non-x86, non-aarch64 stub ──────────────────────────────────────────────── +#[cfg(not(any(target_arch = "x86_64", target_arch = "aarch64")))] +mod stub_impl { + use super::PageFlags; + + pub unsafe fn map_page(_virt: u64, _phys: u64, _flags: PageFlags) {} + pub unsafe fn map_mmio(_phys: u64, _virt: u64, _size: usize) {} + pub fn init(_hhdm_offset: u64) { crate::logger::early_print("[MM::Paging] Virtual memory manager online (stub — non-x86)\r\n"); } @@ -427,7 +687,9 @@ mod stub_impl { pub fn init(hhdm_offset: u64) { #[cfg(target_arch = "x86_64")] x86_64_impl::init(hhdm_offset); - #[cfg(not(target_arch = "x86_64"))] + #[cfg(target_arch = "aarch64")] + aarch64_impl::init(hhdm_offset); + #[cfg(not(any(target_arch = "x86_64", target_arch = "aarch64")))] stub_impl::init(hhdm_offset); } @@ -440,7 +702,9 @@ pub unsafe fn map_page(virt: u64, phys: u64, flags: PageFlags) { let _lock = PAGE_TABLE_LOCK.lock(); #[cfg(target_arch = "x86_64")] unsafe { x86_64_impl::map_page(virt, phys, flags) } - #[cfg(not(target_arch = "x86_64"))] + #[cfg(target_arch = "aarch64")] + unsafe { aarch64_impl::map_page(virt, phys, flags) } + #[cfg(not(any(target_arch = "x86_64", target_arch = "aarch64")))] unsafe { stub_impl::map_page(virt, phys, flags) } } @@ -452,7 +716,9 @@ pub unsafe fn map_mmio(phys: u64, virt: u64, size: usize) { let _lock = PAGE_TABLE_LOCK.lock(); #[cfg(target_arch = "x86_64")] unsafe { x86_64_impl::map_mmio(phys, virt, size) } - #[cfg(not(target_arch = "x86_64"))] + #[cfg(target_arch = "aarch64")] + unsafe { aarch64_impl::map_mmio(phys, virt, size) } + #[cfg(not(any(target_arch = "x86_64", target_arch = "aarch64")))] unsafe { stub_impl::map_mmio(phys, virt, size) } } @@ -524,7 +790,9 @@ pub fn demand_page(cr2: u64, error: u64) -> bool { pub fn alloc_user_pml4() -> Option { #[cfg(target_arch = "x86_64")] return x86_64_impl::alloc_user_pml4(); - #[cfg(not(target_arch = "x86_64"))] + #[cfg(target_arch = "aarch64")] + return aarch64_impl::alloc_user_pml4(); + #[cfg(not(any(target_arch = "x86_64", target_arch = "aarch64")))] return None; } @@ -543,7 +811,14 @@ pub unsafe fn map_user_page(pml4_phys: u64, virt: u64, phys: u64) PageFlags::PRESENT | PageFlags::WRITABLE | PageFlags::USER | PageFlags::NO_EXECUTE, ) }; - #[cfg(not(target_arch = "x86_64"))] + #[cfg(target_arch = "aarch64")] + return unsafe { + aarch64_impl::map_page_in( + pml4_phys, virt, phys, + PageFlags::PRESENT | PageFlags::WRITABLE | PageFlags::USER | PageFlags::NO_EXECUTE, + ) + }; + #[cfg(not(any(target_arch = "x86_64", target_arch = "aarch64")))] { let _ = (pml4_phys, virt, phys); Ok(()) } } @@ -563,7 +838,14 @@ pub unsafe fn map_user_page_with_flags( if !exec { flags |= PageFlags::NO_EXECUTE; } return unsafe { x86_64_impl::map_page_in(pml4_phys, virt, phys, flags) }; } - #[cfg(not(target_arch = "x86_64"))] + #[cfg(target_arch = "aarch64")] + { + let mut flags = PageFlags::PRESENT | PageFlags::USER; + if writable { flags |= PageFlags::WRITABLE; } + if !exec { flags |= PageFlags::NO_EXECUTE; } + return unsafe { aarch64_impl::map_page_in(pml4_phys, virt, phys, flags) }; + } + #[cfg(not(any(target_arch = "x86_64", target_arch = "aarch64")))] { let _ = (pml4_phys, virt, phys, writable, exec); Ok(()) } } @@ -573,7 +855,9 @@ pub fn translate_user_page(pml4_phys: u64, virt: u64) -> Option { let _lock = PAGE_TABLE_LOCK.lock(); #[cfg(target_arch = "x86_64")] { x86_64_impl::translate_user_page(pml4_phys, virt) } - #[cfg(not(target_arch = "x86_64"))] + #[cfg(target_arch = "aarch64")] + { aarch64_impl::translate_user_page(pml4_phys, virt) } + #[cfg(not(any(target_arch = "x86_64", target_arch = "aarch64")))] { let _ = (pml4_phys, virt); None } } @@ -583,7 +867,9 @@ pub fn translate_user_page_flags(pml4_phys: u64, virt: u64) -> Option<(u64, bool let _lock = PAGE_TABLE_LOCK.lock(); #[cfg(target_arch = "x86_64")] { x86_64_impl::translate_user_page_flags(pml4_phys, virt) } - #[cfg(not(target_arch = "x86_64"))] + #[cfg(target_arch = "aarch64")] + { aarch64_impl::translate_user_page_flags(pml4_phys, virt) } + #[cfg(not(any(target_arch = "x86_64", target_arch = "aarch64")))] { let _ = (pml4_phys, virt); None } } @@ -592,7 +878,9 @@ pub fn free_user_pml4(pml4_phys: u64) { let _lock = PAGE_TABLE_LOCK.lock(); #[cfg(target_arch = "x86_64")] x86_64_impl::free_user_pml4(pml4_phys); - #[cfg(not(target_arch = "x86_64"))] + #[cfg(target_arch = "aarch64")] + aarch64_impl::free_user_pml4(pml4_phys); + #[cfg(not(any(target_arch = "x86_64", target_arch = "aarch64")))] let _ = pml4_phys; } @@ -611,7 +899,11 @@ pub unsafe fn update_user_page( { unsafe { x86_64_impl::update_user_page_flags(pml4_phys, virt, writable, exec) } } - #[cfg(not(target_arch = "x86_64"))] + #[cfg(target_arch = "aarch64")] + { + unsafe { aarch64_impl::update_user_page_flags(pml4_phys, virt, writable, exec) } + } + #[cfg(not(any(target_arch = "x86_64", target_arch = "aarch64")))] { let _ = (pml4_phys, virt, writable, exec); Ok(()) @@ -628,7 +920,11 @@ pub unsafe fn unmap_user_page(pml4_phys: u64, virt: u64) -> Result Result { if hdr.e_ident[0..4] != ELF_MAGIC { return Err("bad ELF magic"); } if hdr.e_ident[4] != ELFCLASS64 { return Err("not ELF64"); } if hdr.e_ident[5] != ELFDATA2LSB { return Err("not little-endian"); } - if hdr.e_machine != EM_X86_64 { return Err("not x86_64"); } + if hdr.e_machine != EM_NATIVE { return Err("wrong ELF machine for arch"); } if hdr.e_type != ET_EXEC && hdr.e_type != ET_DYN { return Err("not executable ELF"); } diff --git a/kernel/src/process/mod.rs b/kernel/src/process/mod.rs index 24b4ea1..71b7ae2 100644 --- a/kernel/src/process/mod.rs +++ b/kernel/src/process/mod.rs @@ -44,7 +44,14 @@ pub const MAX_PROCS: usize = 256; pub const USER_ELF_BASE: u64 = 0x0000_0000_0040_0000; // 4 MiB /// User stack top (grows downward; bottom = TOP − STACK_SIZE). +/// +/// x86_64 uses a 47-bit canonical-low address. aarch64's bring-up MMU configures +/// a 39-bit TTBR0 VA window (T0SZ=25 → max user VA 512 GiB), so the ARM stack top +/// is placed just under that boundary instead. +#[cfg(not(target_arch = "aarch64"))] pub const USER_STACK_TOP: u64 = 0x0000_7FFF_FFFF_0000; +#[cfg(target_arch = "aarch64")] +pub const USER_STACK_TOP: u64 = 0x0000_007F_FFF0_0000; // ~512 GiB − 1 MiB (39-bit VA) pub const USER_STACK_SIZE: usize = 4 * 1024 * 1024; // 4 MiB const SYSCALL_STACK_SIZE: usize = 64 * 1024; const XSTATE_SIZE: usize = 4096; @@ -510,6 +517,10 @@ pub fn spawn_with_bootstrap( } // Map POSIX trampoline + sysdata pages so glibc symbols resolve correctly. + // These are x86_64 glibc/musl shims; the aarch64 bring-up init is a bare EL0 + // program with no libc, so they are skipped on ARM (follow-on once an ARM + // userland links a C library). + #[cfg(not(target_arch = "aarch64"))] posix_trampolines::map_system_pages(pml4_phys)?; let idx = idx_of(pid); diff --git a/kernel/src/sched/mod.rs b/kernel/src/sched/mod.rs index e04ac6b..ccfb8c7 100644 --- a/kernel/src/sched/mod.rs +++ b/kernel/src/sched/mod.rs @@ -144,6 +144,7 @@ pub fn spawn_kernel_task(name: &'static str, f: fn()) -> u64 { // [r13 = 0] // [r14 = 0] // [r15 = 0] ← kernel_sp points here + #[cfg(not(target_arch = "aarch64"))] let kernel_sp = unsafe { let stack_top = stack_base.add(STACK_SIZE) as *mut u64; let mut sp = stack_top; @@ -158,6 +159,30 @@ pub fn spawn_kernel_task(name: &'static str, f: fn()) -> u64 { sp as u64 }; + // aarch64 uses a different context_switch/task_entry ABI (AAPCS64 callee-saved + // pairs x19..x30, then `ret` to x30; task_entry pops the fn ptr via + // `ldr x9,[sp],#16` then `blr x9`). Build a matching first-run frame. + // + // Layout (low → high address): + // sp+0 x19 sp+8 x20 + // sp+16 x21 sp+24 x22 + // sp+32 x23 sp+40 x24 + // sp+48 x25 sp+56 x26 + // sp+64 x27 sp+72 x28 + // sp+80 x29 sp+88 x30 ← x30 = task_entry (context_switch `ret`s here) + // sp+96 fn ptr ← read by task_entry's `ldr x9,[sp],#16` + #[cfg(target_arch = "aarch64")] + let kernel_sp = unsafe { + let stack_top = stack_base.add(STACK_SIZE) as *mut u64; + // Reserve the fn-ptr slot at the very top, then the 12-word save area. + let fn_slot = stack_top.sub(1); + fn_slot.write(f as u64); + let frame = fn_slot.sub(12); // 12 u64 = x19..x30 + for i in 0..12 { frame.add(i).write(0); } + frame.add(11).write(crate::arch::task_entry as unsafe extern "C" fn() as u64); // x30 + frame as u64 + }; + let task = Task { pid, state: TaskState::Ready, priority: 128, name, kernel_sp, stack_base }; let mut s = SCHED.lock(); From 2f4b601cdeb025800c1b2d6c5a002f14e49c971c Mon Sep 17 00:00:00 2001 From: Tahiru Agbanwa <130235676+squirelboy360@users.noreply.github.com> Date: Tue, 9 Jun 2026 02:10:57 +0200 Subject: [PATCH 13/36] =?UTF-8?q?aarch64:=20robust=20DTB=20discovery=20?= =?UTF-8?q?=E2=80=94=20x0,=20RAM=20probe,=20then=20arch=20default?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit QEMU's -M virt passes the DTB pointer in x0 for a flat/Image boot, but for an ELF -kernel it may leave x0 = 0 and place the blob at an image-dependent address. boot_prod now: (1) uses x0 when it points at a valid FDT (the spec mechanism, also correct on real hardware), (2) probes the RAM base + a bounded low window for the FDT magic, (3) falls back to the QEMU virt default (2 GiB @ 0x40000000, matching -m) when neither yields a /memory node. Adds fdt::scan_for_dtb for the magic-scan path. The parser itself is unchanged and validated against the machine's real device tree. --- kernel/src/arch/aarch64/boot_prod.rs | 32 +++++++++++++++++++++++++++- kernel/src/arch/aarch64/fdt.rs | 22 +++++++++++++++++++ 2 files changed, 53 insertions(+), 1 deletion(-) diff --git a/kernel/src/arch/aarch64/boot_prod.rs b/kernel/src/arch/aarch64/boot_prod.rs index 60a26fe..735e057 100644 --- a/kernel/src/arch/aarch64/boot_prod.rs +++ b/kernel/src/arch/aarch64/boot_prod.rs @@ -54,9 +54,39 @@ pub fn aarch64_kernel_boot(dtb_ptr: u64) -> ! { // The generic timer (scheduler tick) is armed later by the shared kernel via // `arch::apic::init_bsp`; IRQ delivery stays masked until then. + // ── Locate the DTB ────────────────────────────────────────────────────── + // QEMU passes the DTB pointer in x0 when booting a flat image, but when it + // loads an *ELF* kernel on `-M virt` it instead places the generated DTB at + // the base of RAM (0x4000_0000) and may leave x0 = 0. If x0 didn't point at a + // valid DTB, probe the RAM base for the FDT magic before falling back. + let dtb_at = if dtb_ptr != 0 && unsafe { fdt::parse(dtb_ptr).is_some() } { + dtb_ptr + } else { + const QEMU_VIRT_DTB: u64 = 0x4000_0000; + if unsafe { fdt::parse(QEMU_VIRT_DTB).is_some() } { + uart::puts("[arm-boot] DTB not in x0; found FDT at RAM base 0x40000000\n"); + QEMU_VIRT_DTB + } else if let Some(found) = + // QEMU's exact DTB placement for an ELF `-kernel` depends on the image; + // probe a small bounded window just past the RAM base (cheap under TCG). + // When the bootloader DOES pass a valid x0 (flat-image boot, real HW), + // the first branch above already used it. Otherwise we fall through to + // the architecture default below — equivalent to `-m` on QEMU virt. + unsafe { fdt::scan_for_dtb(0x4000_0000, 0x4400_0000) } + { + uart::puts("[arm-boot] DTB located by RAM scan @ "); + uart::puthex_full(found); + uart::puts("\n"); + found + } else { + dtb_ptr // keep original; parse() below will fail → fallback map + } + }; + fdt::set_dtb_ptr(dtb_at); + // ── Discover RAM from the device tree → neutral BootMemMap ────────────── let mut map = crate::mm::BootMemMap::new(); - match unsafe { fdt::parse(dtb_ptr) } { + match unsafe { fdt::parse(dtb_at) } { Some(info) => { for i in 0..info.mem_count { let r = info.mem[i]; diff --git a/kernel/src/arch/aarch64/fdt.rs b/kernel/src/arch/aarch64/fdt.rs index 11ec6ff..b857a4d 100644 --- a/kernel/src/arch/aarch64/fdt.rs +++ b/kernel/src/arch/aarch64/fdt.rs @@ -255,6 +255,28 @@ pub unsafe fn parse(dtb_pa: u64) -> Option { } } +/// Scan a physical address range for the FDT magic on an 8-byte stride and +/// return the first address whose blob parses to a valid `/memory` node. +/// +/// QEMU's `-M virt` ELF-kernel boot does not reliably pass the DTB pointer in x0 +/// and may place the generated blob at a position that depends on the kernel +/// image size. As a robust last resort, we scan a bounded window of RAM for it. +/// +/// # Safety +/// `[start, end)` must be readable (identity-mapped) physical RAM. +pub unsafe fn scan_for_dtb(start: u64, end: u64) -> Option { + let mut p = start & !0x7; + while p < end { + if read_be32(p as *const u8) == FDT_MAGIC { + if parse(p).is_some() { + return Some(p); + } + } + p += 8; + } + None +} + /// Read a 1- or 2-cell big-endian value (cells beyond 2 are ignored / unsupported). #[inline] unsafe fn read_cells(p: *const u8, cells: u32) -> u64 { From 4fc41b9942a64b22b18469ee99e16431ec6b9c12 Mon Sep 17 00:00:00 2001 From: Tahiru Agbanwa <130235676+squirelboy360@users.noreply.github.com> Date: Tue, 9 Jun 2026 02:12:59 +0200 Subject: [PATCH 14/36] aarch64: document the production boot path in run-aarch64.sh The ARM `-kernel` boot now routes into the shared kernel_main and spawns userspace; update the script header to describe that instead of only the self-contained bring-up demo. --- scripts/run-aarch64.sh | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/scripts/run-aarch64.sh b/scripts/run-aarch64.sh index 3920955..0cb7beb 100755 --- a/scripts/run-aarch64.sh +++ b/scripts/run-aarch64.sh @@ -1,10 +1,17 @@ #!/usr/bin/env bash # Build + boot the OSCortex aarch64 kernel directly under qemu-system-aarch64. # -# This is the ARM bring-up path: a direct `-kernel` boot (no Limine) that runs -# the platform-primitive milestones — EL1, MMU, exception vectors, GIC + generic -# timer, SVC syscalls, and an EL0 user process — printing progress over PL011 -# serial. See kernel/src/arch/aarch64/bringup*.rs. +# Direct `-kernel` boot (no Limine — that is the x86_64 boot protocol). The ARM +# `_start` (kernel/src/arch/aarch64/boot.rs) saves the device-tree pointer from +# x0 and routes into the PRODUCTION boot path (boot_prod.rs): PL011 serial → MMU +# → EL1 exception vectors → GIC → device-tree RAM discovery → the SHARED +# `kernel_main`. That runs the real subsystem init (heap, scheduler, VFS, process +# table, Cortex runtime, …) and spawns the `/init` process at EL0, whose syscalls +# are serviced through the shared dispatcher — all over PL011 serial. +# +# The self-contained platform-primitive bring-up (EL1/MMU/vectors/GIC/timer/SVC/ +# EL0 self-test, kernel/src/arch/aarch64/bringup*.rs) is still present and was the +# proving ground for these primitives; the production path above now reuses them. # # Usage: # scripts/run-aarch64.sh # single-core boot From 078d0898c1796a6b5b5c42943a52c83cf0557377 Mon Sep 17 00:00:00 2001 From: Tahiru Agbanwa <130235676+squirelboy360@users.noreply.github.com> Date: Tue, 9 Jun 2026 02:13:31 +0200 Subject: [PATCH 15/36] mm: widen BootMemMap capacity to 128 regions Ensures a fragmented x86 Limine memory map never drops usable RAM when translating into the architecture-neutral region list. --- kernel/src/mm/mod.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/kernel/src/mm/mod.rs b/kernel/src/mm/mod.rs index 10da74a..a4de534 100644 --- a/kernel/src/mm/mod.rs +++ b/kernel/src/mm/mod.rs @@ -24,9 +24,10 @@ pub struct BootMemRegion { pub len: u64, } -/// Maximum distinct usable regions we retain. Limine emits a handful; QEMU virt -/// reports one or two. 32 is comfortable headroom. -pub const MAX_BOOT_REGIONS: usize = 32; +/// Maximum distinct usable regions we retain. Limine typically emits well under +/// a dozen usable entries (and QEMU virt one or two), but a fragmented memory map +/// can have more; 128 is generous headroom so no usable RAM is ever dropped. +pub const MAX_BOOT_REGIONS: usize = 128; /// A fixed-capacity list of usable physical RAM regions, built by the arch boot /// path and consumed by `frame_allocator::init_from_regions`. From d2de014ded952ec19ddf5e014a3fde91617271be Mon Sep 17 00:00:00 2001 From: Tahiru Agbanwa Date: Tue, 9 Jun 2026 02:25:48 +0200 Subject: [PATCH 16/36] =?UTF-8?q?aarch64:=20ramfb=20framebuffer=20via=20fw?= =?UTF-8?q?=5Fcfg=20=E2=80=94=20display-capable=20on=20-M=20virt?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit QEMU -M virt ships no Limine framebuffer (Limine is the x86 boot protocol), so the ARM port had no display. Add a fw_cfg DMA reader/writer and use it to configure QEMU's -device ramfb: * kernel/src/arch/aarch64/ramfb.rs: walk the fw_cfg file directory to find the etc/ramfb selector, allocate a 1280x800 XRGB8888 framebuffer in identity-mapped RAM, and DMA-write the big-endian RAMFBCfg into it. Two gotchas handled: SELECT and WRITE must be separate DMAs (a combined control word is rejected), and RamfbCfg must be repr(C, packed) so the write length is exactly the 28-byte file size QEMU expects. * drivers/fb.rs: add init_raw(addr,w,h,pitch) — an arch-neutral entry point so the existing fb console/compositor (fed from Limine on x86) works unchanged on ARM from a raw buffer. * main.rs (kernel_main_arch): bring ramfb up after the frame allocator is online, publish it to the shared fb, and paint a boot fill. * scripts/run-aarch64.sh: add -device ramfb; DISPLAY_MODE=none|cocoa|gtk|sdl with a headless monitor socket for screendump capture. Validated on qemu-system-aarch64: monitor screendump is now 1280x800 (was the 640x480 default), with the painted teal top band (20,184,166), navy body (26,26,46) and white boot-console glyphs all present — pixels the kernel wrote are scanned out by ramfb. --- kernel/src/arch/aarch64/mod.rs | 1 + kernel/src/arch/aarch64/ramfb.rs | 297 +++++++++++++++++++++++++++++++ kernel/src/drivers/fb.rs | 23 +++ kernel/src/main.rs | 23 ++- scripts/run-aarch64.sh | 37 +++- 5 files changed, 374 insertions(+), 7 deletions(-) create mode 100644 kernel/src/arch/aarch64/ramfb.rs diff --git a/kernel/src/arch/aarch64/mod.rs b/kernel/src/arch/aarch64/mod.rs index b8521b2..4e141a0 100644 --- a/kernel/src/arch/aarch64/mod.rs +++ b/kernel/src/arch/aarch64/mod.rs @@ -28,6 +28,7 @@ pub mod enter_user; // ── Bare-metal bring-up (direct `-kernel` boot) ───────────────────────────── pub mod uart; pub mod fdt; +pub mod ramfb; pub mod boot; pub mod boot_prod; pub mod mmu; diff --git a/kernel/src/arch/aarch64/ramfb.rs b/kernel/src/arch/aarch64/ramfb.rs new file mode 100644 index 0000000..00463de --- /dev/null +++ b/kernel/src/arch/aarch64/ramfb.rs @@ -0,0 +1,297 @@ +//! QEMU `ramfb` display via the `fw_cfg` interface — aarch64 `-M virt`. +//! +//! `qemu-system-aarch64 -M virt` provides no Limine framebuffer (Limine is the +//! x86_64 boot protocol). QEMU's `-device ramfb` instead exposes a software +//! framebuffer that the guest configures through **fw_cfg**: the guest writes a +//! big-endian [`RamfbCfg`] describing a guest-RAM buffer (physical address, +//! DRM fourcc pixel format, dimensions, stride) to the `etc/ramfb` fw_cfg file, +//! and QEMU thereafter scans that buffer out to the display every refresh. +//! +//! ## fw_cfg on `-M virt` +//! The MMIO fw_cfg device sits at a fixed base (QEMU virt: `0x0902_0000`). Its +//! register block (big-endian, "DMA" interface) is: +//! +//! * `+0x00` (8 bytes) `data` — legacy data port (unused here). +//! * `+0x08` (2 bytes) `selector` — write the 16-bit file/key selector (BE). +//! * `+0x10` (8 bytes) `dma` — write the **physical** address of a +//! [`FwCfgDmaAccess`] control block (BE) to +//! kick a DMA transfer. +//! +//! To configure ramfb we: +//! 1. Read the fw_cfg file directory (`FW_CFG_FILE_DIR`, selector `0x0019`) +//! and find the `etc/ramfb` entry → its write-only selector key. +//! 2. Allocate a framebuffer in guest RAM (identity-mapped, so VA == PA). +//! 3. DMA-write a big-endian [`RamfbCfg`] into that file selector. +//! +//! After step 3 QEMU latches the configuration and the framebuffer is live; any +//! pixels the kernel writes into the buffer appear on screen at the next refresh. +//! +//! The framebuffer is then published to the shared [`crate::drivers::fb`] console +//! through its arch-neutral `init_raw` hook, so the existing compositor / fb text +//! console (which x86 feeds from Limine) works unchanged on ARM. + +use core::sync::atomic::{AtomicU64, Ordering}; + +/// fw_cfg MMIO base on QEMU `-M virt`. +const FW_CFG_BASE: u64 = 0x0902_0000; + +/// MMIO register offset for the fw_cfg DMA control-block pointer (big-endian). +/// All fw_cfg access here goes through the DMA interface, which folds item +/// selection into the DMA control word, so the legacy selector port is unused. +const FW_CFG_REG_DMA: u64 = 0x10; + +/// Well-known fw_cfg selector for the file directory. +const FW_CFG_FILE_DIR: u16 = 0x0019; + +/// fw_cfg DMA control bits (the `control` field of [`FwCfgDmaAccess`], BE). +const FW_CFG_DMA_CTL_ERROR: u32 = 0x01; +const FW_CFG_DMA_CTL_READ: u32 = 0x02; +const FW_CFG_DMA_CTL_SKIP: u32 = 0x04; +const FW_CFG_DMA_CTL_SELECT: u32 = 0x08; +const FW_CFG_DMA_CTL_WRITE: u32 = 0x10; + +/// DRM fourcc for `XRGB8888` (little-endian: byte0=B, byte1=G, byte2=R, byte3=X) +/// — the format the shared fb console expects (32 bpp, X in the high byte). +const DRM_FORMAT_XRGB8888: u32 = 0x3432_5258; // 'XR24' + +/// Default display geometry. QEMU ramfb is fully software-defined, so any size +/// the guest picks is honoured; 1280×800 matches the x86 default shell surface. +const FB_WIDTH: u32 = 1280; +const FB_HEIGHT: u32 = 800; +const FB_BPP: u32 = 4; // bytes per pixel (XRGB8888) + +/// fw_cfg DMA access control block. All fields big-endian per the fw_cfg spec. +#[repr(C)] +struct FwCfgDmaAccess { + control: u32, // BE + length: u32, // BE + address: u64, // BE (physical address of the data buffer) +} + +/// The `etc/ramfb` configuration record. **All fields big-endian.** +/// +/// `#[repr(C, packed)]` is required: the on-wire record is exactly 28 bytes +/// (8+4+4+4+4+4), but a naturally-aligned `repr(C)` would pad the leading `u64` +/// out to 32, and QEMU rejects an `etc/ramfb` write whose length exceeds the +/// file's declared 28-byte size (the cause of the earlier DMA error bit). +#[repr(C, packed)] +struct RamfbCfg { + addr: u64, // BE — physical address of the framebuffer + fourcc: u32, // BE — DRM fourcc pixel format + flags: u32, // BE — reserved (0) + width: u32, // BE + height: u32, // BE + stride: u32, // BE — bytes per row +} + +/// One fw_cfg file-directory entry (big-endian fields), spec layout. +#[repr(C)] +struct FwCfgFile { + size: u32, // BE — file size in bytes + select: u16, // BE — selector key + reserved: u16, // BE + name: [u8; 56], // NUL-padded ASCII name +} + +/// Published framebuffer physical base once ramfb is configured (0 = not up). +static FB_PHYS: AtomicU64 = AtomicU64::new(0); + +#[inline(always)] +unsafe fn mmio_write_u64(off: u64, v: u64) { + core::ptr::write_volatile((FW_CFG_BASE + off) as *mut u64, v); +} + +/// Kick a fw_cfg DMA transfer described by `dma` (which must be a populated, +/// big-endian [`FwCfgDmaAccess`] in guest RAM) and spin until QEMU clears the +/// busy control bits. Returns `false` if QEMU flagged an error. +unsafe fn fw_cfg_dma(dma: &mut FwCfgDmaAccess) -> bool { + // The DMA address register is big-endian; write the *physical* address of the + // control block (identity map → VA == PA). + let dma_pa = dma as *mut FwCfgDmaAccess as u64; + mmio_write_u64(FW_CFG_REG_DMA, dma_pa.to_be()); + + // QEMU processes the transfer synchronously under TCG, but per spec we must + // poll `control` until all bits except a possible ERROR are cleared. + loop { + let ctl = core::ptr::read_volatile(&dma.control as *const u32); + let ctl = u32::from_be(ctl); + if ctl & FW_CFG_DMA_CTL_ERROR != 0 { + log::warn!("[ramfb] fw_cfg DMA error: control={:#x}", ctl); + return false; + } + // All action bits cleared → transfer complete. + if ctl + & (FW_CFG_DMA_CTL_READ + | FW_CFG_DMA_CTL_WRITE + | FW_CFG_DMA_CTL_SELECT + | FW_CFG_DMA_CTL_SKIP) + == 0 + { + return true; + } + core::hint::spin_loop(); + } +} + +/// Select a fw_cfg item and DMA-read `len` bytes of it into `buf` (physical). +unsafe fn fw_cfg_read(selector: u16, buf: *mut u8, len: u32) -> bool { + let mut dma = FwCfgDmaAccess { + control: ((FW_CFG_DMA_CTL_SELECT | FW_CFG_DMA_CTL_READ) + | ((selector as u32) << 16)) + .to_be(), + length: len.to_be(), + address: (buf as u64).to_be(), + }; + fw_cfg_dma(&mut dma) +} + +/// Select a fw_cfg item by key and DMA-write `len` bytes from `buf` into it. +/// +/// QEMU rejects a combined SELECT+WRITE in one DMA control word (it returns the +/// error bit), so we issue a SELECT-only DMA first to point fw_cfg at the file, +/// then a WRITE-only DMA carrying the payload. +unsafe fn fw_cfg_write(selector: u16, buf: *const u8, len: u32) -> bool { + // 1. SELECT-only: point fw_cfg at the target file (resets its offset to 0). + let mut sel = FwCfgDmaAccess { + control: (FW_CFG_DMA_CTL_SELECT | ((selector as u32) << 16)).to_be(), + length: 0u32.to_be(), + address: 0u64.to_be(), + }; + if !fw_cfg_dma(&mut sel) { + return false; + } + // 2. WRITE-only: stream the payload into the now-selected file. + let mut dma = FwCfgDmaAccess { + control: FW_CFG_DMA_CTL_WRITE.to_be(), + length: len.to_be(), + address: (buf as u64).to_be(), + }; + fw_cfg_dma(&mut dma) +} + +/// Walk the fw_cfg file directory and return the selector key of `etc/ramfb`. +unsafe fn find_ramfb_selector() -> Option { + // The directory begins with a big-endian u32 count, then `count` FwCfgFile + // records. Read the count first. + let mut count_be: u32 = 0; + if !fw_cfg_read( + FW_CFG_FILE_DIR, + &mut count_be as *mut u32 as *mut u8, + 4, + ) { + return None; + } + let count = u32::from_be(count_be); + if count == 0 || count > 1024 { + return None; + } + + // Re-select + re-read so the directory stream restarts at the count, then + // skip the 4-byte count and stream the entries. + let entries_len = count + .saturating_mul(core::mem::size_of::() as u32) + .saturating_add(4); + let buf_pa = crate::mm::frame_allocator::alloc_contiguous_frames( + ((entries_len as usize) + 4095) / 4096, + )?; + let ok = fw_cfg_read(FW_CFG_FILE_DIR, buf_pa as *mut u8, entries_len); + if !ok { + crate::mm::frame_allocator::free_frame(buf_pa); + return None; + } + + let mut found: Option = None; + let base = (buf_pa + 4) as *const FwCfgFile; // skip the leading count + for i in 0..count as usize { + let f = &*base.add(i); + // Match the NUL-terminated name against "etc/ramfb". + let name = &f.name; + let target = b"etc/ramfb"; + let mut matches = true; + for (j, &c) in target.iter().enumerate() { + if name[j] != c { + matches = false; + break; + } + } + if matches && name[target.len()] == 0 { + found = Some(u16::from_be(f.select)); + log::debug!( + "[ramfb] fw_cfg dir: etc/ramfb sel={:#x} size={} (of {} files)", + u16::from_be(f.select), + u32::from_be(f.size), + count + ); + break; + } + } + + crate::mm::frame_allocator::free_frame(buf_pa); + found +} + +/// Bring up the QEMU ramfb display and publish it to the shared fb console. +/// +/// Returns `(phys_addr, width, height, stride_bytes)` on success. Must be called +/// after the frame allocator is online (so we can reserve the framebuffer) and +/// with the low-1 GiB device MMIO identity-mapped (so fw_cfg is reachable). +pub fn init() -> Option<(u64, u32, u32, u32)> { + unsafe { + let selector = match find_ramfb_selector() { + Some(s) => s, + None => { + log::warn!("[ramfb] etc/ramfb not found in fw_cfg directory"); + return None; + } + }; + + let stride = FB_WIDTH * FB_BPP; + let fb_bytes = (stride as usize) * (FB_HEIGHT as usize); + let pages = (fb_bytes + 4095) / 4096; + let fb_phys = crate::mm::frame_allocator::alloc_contiguous_frames(pages)?; + + // Zero the framebuffer (identity-mapped → write directly). + { + let p = fb_phys as *mut u32; + for i in 0..(fb_bytes / 4) { + core::ptr::write_volatile(p.add(i), 0x0000_0000); + } + } + + // Build + DMA-write the big-endian RAMFBCfg into etc/ramfb. + let cfg = RamfbCfg { + addr: fb_phys.to_be(), + fourcc: DRM_FORMAT_XRGB8888.to_be(), + flags: 0u32.to_be(), + width: FB_WIDTH.to_be(), + height: FB_HEIGHT.to_be(), + stride: stride.to_be(), + }; + let ok = fw_cfg_write( + selector, + &cfg as *const RamfbCfg as *const u8, + core::mem::size_of::() as u32, + ); + if !ok { + log::warn!("[ramfb] fw_cfg write of etc/ramfb failed"); + return None; + } + + FB_PHYS.store(fb_phys, Ordering::Release); + log::info!( + "[ramfb] up: {}x{} stride={} @ phys {:#x} (sel={:#x})", + FB_WIDTH, + FB_HEIGHT, + stride, + fb_phys, + selector + ); + + Some((fb_phys, FB_WIDTH, FB_HEIGHT, stride)) + } +} + +/// Physical base of the ramfb framebuffer (0 until [`init`] succeeds). +pub fn fb_phys() -> u64 { + FB_PHYS.load(Ordering::Acquire) +} diff --git a/kernel/src/drivers/fb.rs b/kernel/src/drivers/fb.rs index c833a21..e7ce2f6 100644 --- a/kernel/src/drivers/fb.rs +++ b/kernel/src/drivers/fb.rs @@ -208,6 +208,29 @@ pub fn init(fb_resp: &limine::request::FramebufferResponse) { FB_READY.store(true, Ordering::Release); } +/// Initialise the framebuffer console from raw parameters (arch-neutral). +/// +/// The x86 path feeds the framebuffer from the Limine response (`init` above); +/// architectures without a bootloader framebuffer (e.g. aarch64 QEMU ramfb, set +/// up by `arch::aarch64::ramfb`) call this with the buffer they configured. The +/// `addr` must be a directly-writable virtual address (identity-mapped on ARM, +/// so VA == PA) and `pitch_bytes` is the number of bytes per row. Only 32 bpp +/// (XRGB) is supported, matching the rest of this console. +pub fn init_raw(addr: u64, width: u32, height: u32, pitch_bytes: u32) { + if addr == 0 || width == 0 || height == 0 || pitch_bytes < width * 4 { + return; + } + FB_ADDR .store(addr, Ordering::Release); + FB_PITCH_PX.store(pitch_bytes / 4, Ordering::Release); + FB_WIDTH .store(width, Ordering::Release); + FB_HEIGHT .store(height, Ordering::Release); + FB_COLS .store(width / CHAR_W, Ordering::Release); + FB_ROWS .store(height / CHAR_H, Ordering::Release); + + clear(); + FB_READY.store(true, Ordering::Release); +} + /// Write a string to the framebuffer console. /// /// Handles `\n` (advance to next row, scroll if needed) and diff --git a/kernel/src/main.rs b/kernel/src/main.rs index 7466bac..c24abe7 100644 --- a/kernel/src/main.rs +++ b/kernel/src/main.rs @@ -244,7 +244,28 @@ pub fn kernel_main_arch(map: mm::BootMemMap, hhdm_offset: u64) -> ! { (mm::frame_allocator::frames_used() * 4096) / (1024 * 1024) ); log::info!("OSCortex kernel {} booting (aarch64)...", env!("CARGO_PKG_VERSION")); - log::warn!("[BOOT] No framebuffer / Flutter engine on aarch64 yet (follow-on)."); + + // ── 3b. Framebuffer via QEMU ramfb (fw_cfg) ────────────────────────── + // `-M virt` has no Limine framebuffer; configure QEMU's `-device ramfb` + // through fw_cfg and publish the buffer to the shared fb console so the + // compositor path works unchanged. Requires the frame allocator (online + // after `mm::init_from_regions`). + match arch::aarch64::ramfb::init() { + Some((addr, w, h, pitch)) => { + // Identity-mapped RAM → the physical framebuffer base is directly + // writable as a virtual address (VA == PA on aarch64). + drivers::fb::init_raw(addr, w, h, pitch); + log::info!("[BOOT] aarch64 framebuffer online: {}x{} via ramfb", w, h); + // Paint a visible boot fill so the display is provably driven even + // before any userspace renders (top band teal, body dark navy). + drivers::fb::fill_rect(0, 0, w, 48, 0x0014_B8A6); + drivers::fb::fill_rect(0, 48, w, h - 48, 0x001A_1A2E); + drivers::fb::write_str("\n OSCortex aarch64 — ramfb framebuffer up\n"); + } + None => { + log::warn!("[BOOT] aarch64 ramfb unavailable (need -device ramfb); serial-only."); + } + } shared_init_and_run() } diff --git a/scripts/run-aarch64.sh b/scripts/run-aarch64.sh index 0cb7beb..fa478e9 100755 --- a/scripts/run-aarch64.sh +++ b/scripts/run-aarch64.sh @@ -30,10 +30,35 @@ cargo build --target "$TARGET" -p oscortex-kernel \ -Z build-std-features=compiler-builtins-mem SMP=1 -[[ "${1:-}" == "smp2" ]] && SMP=2 +DISPLAY_MODE="${DISPLAY_MODE:-none}" # none | gtk | cocoa | sdl +for arg in "$@"; do + case "$arg" in + smp2) SMP=2 ;; + gtk|cocoa|sdl) DISPLAY_MODE="$arg" ;; + graphical) DISPLAY_MODE="cocoa" ;; # macOS default backend + esac +done -echo "[run-aarch64] booting (smp=$SMP) — Ctrl-A X to quit ..." -exec qemu-system-aarch64 \ - -M virt -cpu cortex-a72 -smp "$SMP" -m 2G \ - -nographic -kernel "$KERNEL" \ - -serial mon:stdio +# `-device ramfb`: QEMU's software framebuffer, configured by the guest through +# fw_cfg (kernel/src/arch/aarch64/ramfb.rs). With DISPLAY_MODE=none the kernel +# still drives the buffer (capture it via the QEMU monitor `screendump`); a real +# display backend (cocoa/gtk/sdl) shows it on screen. +COMMON=( + -M virt -cpu cortex-a72 -smp "$SMP" -m 2G + -kernel "$KERNEL" + -device ramfb +) + +if [[ "$DISPLAY_MODE" == "none" ]]; then + echo "[run-aarch64] booting (smp=$SMP, headless ramfb) — Ctrl-A X to quit ..." + echo "[run-aarch64] monitor: /tmp/qemu-arm-monitor.sock (screendump to capture the fb)" + exec qemu-system-aarch64 "${COMMON[@]}" \ + -display none \ + -serial mon:stdio \ + -monitor unix:/tmp/qemu-arm-monitor.sock,server,nowait +else + echo "[run-aarch64] booting (smp=$SMP, display=$DISPLAY_MODE) — close window or Ctrl-A X to quit ..." + exec qemu-system-aarch64 "${COMMON[@]}" \ + -display "$DISPLAY_MODE" \ + -serial mon:stdio +fi From 85c80345572c459e6dd25816d2c5ac48a0d1dbb6 Mon Sep 17 00:00:00 2001 From: Tahiru Agbanwa Date: Tue, 9 Jun 2026 02:57:55 +0200 Subject: [PATCH 17/36] =?UTF-8?q?aarch64:=20timer-driven=20preemption=20?= =?UTF-8?q?=E2=80=94=20shared=20cooperative=20scheduler=20hand-off?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Wire the ARM generic-timer IRQ to the SAME shared cooperative-scheduler hand-off the x86 APIC-timer ISR uses, so EL0 threads (the Flutter engine's threads, ultimately) are timer-preempted and round-robined — collapsing the ARM 'enter init directly' path onto x86's timer-driven model. * arch/aarch64/apic.rs: production_irq_handler — on a generic-timer PPI taken from EL0, apply the same quantum + input/focus preempt policy as x86, map the live EL1 vector TrapFrame to the shared UserRegs, call the shared process::timer_preempt_switch_try (arch-neutral: PTABLE bookkeeping, CR3/ TTBR0, fs_base, xstate, current_pid), then write the next thread's frame back into the TrapFrame so the vector stub's RESTORE_FRAME+eret enters it. init_bsp only installs the handler; start_scheduler_tick arms the timer. * Full-fidelity context: the shared UserRegs carries only the x86-named register subset, so each thread's complete ARM frame (x0..x30/SP_EL0/ELR/ SPSR) round-trips through a new per-process arch_trapframe slot (process::arch_store/take_trapframe, try_lock — ISR-safe). Scratch registers (x6/x7, x11–x18, x24–x28, x30) survive arbitrary-instruction preemption. * IRQ enablement: IRQs are NOT unmasked at EL1; the eret into EL0 (SPSR=EL0t, I/F clear) enables them at the userspace boundary. This closes a sporadic boot hang where a timer fired mid enter_user_by_pid_noreturn (during the CR3 switch / image build) — boots went from ~50% hanging to 5/5 clean. * mm: reserve the kernel image in the frame allocator on ARM. QEMU -kernel loads the image (0x40080000) into RAM the device tree reports usable from 0x40000000; the allocator marked it all free, so the first large contiguous alloc (the 64 MiB heap, then the compositor's 4 MiB double-buffer) overran live kernel code — a latent corruption that hung the first big alloc once a framebuffer made the compositor enable double-buffering. frame_allocator:: reserve_range carves [RAM base..__kernel_end] before the heap is built. * build.rs + main.rs: the ARM /init program is now a compute-bound loop (busy-spins with NO syscall between ticks, prints a per-instance tag for a bounded budget, then exits). main.rs spawns TWO instances (A=x0:0, B=x0:1); since neither yields cooperatively, interleaved A/B output on serial can ONLY come from the timer preempting + switching between them. Validated on qemu-system-aarch64 (5/5 boots): both EL0 threads make progress with fair, interleaved ticks (e.g. A=12 B=12, 7-8 A<->B switches) then exit cleanly. The longer compute-budget variant showed thousands of balanced ticks (A=3341/B=3350, 2148 switches). x86 builds clean (no regression). --- kernel/build.rs | 104 ++++++++++++---- kernel/src/arch/aarch64/apic.rs | 197 ++++++++++++++++++++++++++++++- kernel/src/main.rs | 54 +++++++-- kernel/src/mm/frame_allocator.rs | 28 +++++ kernel/src/mm/mod.rs | 25 ++++ kernel/src/process/mod.rs | 60 ++++++++++ 6 files changed, 428 insertions(+), 40 deletions(-) diff --git a/kernel/build.rs b/kernel/build.rs index 49f6917..9138c12 100644 --- a/kernel/build.rs +++ b/kernel/build.rs @@ -474,41 +474,94 @@ fn generate_aarch64_init_elf() -> Vec { // empty L1 index lets the page-table walker create fresh L2/L3 tables there. const VBASE: u64 = 0x10_0000_0000; // 64 GiB - // Messages placed after the code, at page offsets 0x200 / 0x280. - let msg1: &[u8] = b"[arm-init] hello from EL0 userspace via svc #0\n"; - let msg2: &[u8] = b"[arm-init] getpid ok; calling exit(0)\n"; - let msg1_off: u64 = 0x200; - let msg2_off: u64 = 0x280; - let msg1_va = VBASE + msg1_off; - let msg2_va = VBASE + msg2_off; + // Two messages selected by the bootstrap value the kernel seeds in x0 at + // entry: process A (x0==0) prints msg_a, any other value prints msg_b. This + // lets two instances of the SAME program emit distinguishable output so an + // observer can see the timer round-robining between them. + let msg_a: &[u8] = b"[arm-A] tick (EL0 compute loop A)\n"; + let msg_b: &[u8] = b"[arm-B] tick (EL0 compute loop B)\n"; + let hello: &[u8] = b"[arm-init] EL0 up; entering compute loop (timer preemption test)\n"; + let msg_a_off: u64 = 0x200; + let msg_b_off: u64 = 0x280; + let hello_off: u64 = 0x300; + let msg_a_va = VBASE + msg_a_off; + let msg_b_va = VBASE + msg_b_off; + let hello_va = VBASE + hello_off; // ── Assemble the code (offset 0) ──────────────────────────────────────── + // + // Pseudocode: + // x19 = (x0 == 0) ? msg_a_va : msg_b_va // pick this instance's msg + // x20 = (x0 == 0) ? len_a : len_b + // write(1, hello, hello.len()) // announce once + // loop: + // x21 = SPIN_ITERS // pure-compute busy spin — + // spin: subs x21,x21,#1; b.ne spin // NO syscall, so only the + // // timer can switch us out + // write(1, x19, x20) // print this instance's tick + // b loop + // + // Because the spin does no syscalls, the kernel cannot cooperatively yield + // here; if both A and B make progress (interleaved ticks on serial) the + // generic-timer ISR MUST be preempting and switching between them. + const SPIN_ITERS: u64 = 4_000_000; + // Each thread prints this many ticks, then exits — enough to demonstrate the + // timer round-robining the two compute-bound threads, after which the system + // settles into the kernel idle/cortex loop (rather than spamming forever). + const TICK_BUDGET: u64 = 12; let mut code: Vec = Vec::new(); - // write(1, msg1_va, msg1.len()) + // Select message VA (x19) and length (x20) from x0. + a64_load_imm(&mut code, 19, msg_a_va); + a64_load_imm(&mut code, 20, msg_a.len() as u64); + a64_load_imm(&mut code, 9, msg_b_va); + a64_load_imm(&mut code, 10, msg_b.len() as u64); + // cmp x0, #0 ; csel x19, x19, x9, eq ; csel x20, x20, x10, eq + code.push(0xF100_001F); // subs xzr, x0, #0 (cmp x0,#0) + code.push(0x9A89_0273); // csel x19, x19, x9, eq + code.push(0x9A8A_0294); // csel x20, x20, x10, eq + + // x22 = TICK_BUDGET (remaining ticks before this thread exits) + a64_load_imm(&mut code, 22, TICK_BUDGET); + + // write(1, hello, hello.len()) a64_load_imm(&mut code, 0, 1); - a64_load_imm(&mut code, 1, msg1_va); - a64_load_imm(&mut code, 2, msg1.len() as u64); + a64_load_imm(&mut code, 1, hello_va); + a64_load_imm(&mut code, 2, hello.len() as u64); a64_load_imm(&mut code, 8, 1); code.push(0xD400_0001); // svc #0 - // getpid() - a64_load_imm(&mut code, 8, 39); - code.push(0xD400_0001); - - // write(1, msg2_va, msg2.len()) + // loop: + let loop_idx = code.len(); + // x21 = SPIN_ITERS + a64_load_imm(&mut code, 21, SPIN_ITERS); + // spin: subs x21, x21, #1 ; b.ne spin (pure compute — no syscall) + let spin_idx = code.len(); + code.push(0xF100_06B5); // subs x21, x21, #1 + { + let off = (spin_idx as i64 - code.len() as i64) as i32; // negative + let imm19 = (off as u32) & 0x7FFFF; + code.push(0x5400_0001 | (imm19 << 5)); // b.ne spin + } + // write(1, x19, x20) a64_load_imm(&mut code, 0, 1); - a64_load_imm(&mut code, 1, msg2_va); - a64_load_imm(&mut code, 2, msg2.len() as u64); + code.push(0xAA13_03E1); // mov x1, x19 + code.push(0xAA14_03E2); // mov x2, x20 a64_load_imm(&mut code, 8, 1); - code.push(0xD400_0001); - - // exit(0) + code.push(0xD400_0001); // svc #0 + // subs x22, x22, #1 ; b.ne loop (loop until the tick budget is spent) + code.push(0xF100_06D6); // subs x22, x22, #1 + { + let off = (loop_idx as i64 - code.len() as i64) as i32; // negative + let imm19 = (off as u32) & 0x7FFFF; + code.push(0x5400_0001 | (imm19 << 5)); // b.ne loop + } + // exit(0) — lets the system settle into the kernel idle/cortex loop after + // the preemption demonstration rather than spinning forever. a64_load_imm(&mut code, 0, 0); a64_load_imm(&mut code, 8, 60); - code.push(0xD400_0001); - - // Safety net: spin if exit ever returns to EL0. + code.push(0xD400_0001); // svc #0 + // safety net: spin if exit ever returns to EL0. code.push(0x1400_0000); // b . // ── Compose the single loadable page image ────────────────────────────── @@ -518,8 +571,9 @@ fn generate_aarch64_init_elf() -> Vec { let o = i * 4; img[o..o + 4].copy_from_slice(&w.to_le_bytes()); } - img[msg1_off as usize..msg1_off as usize + msg1.len()].copy_from_slice(msg1); - img[msg2_off as usize..msg2_off as usize + msg2.len()].copy_from_slice(msg2); + img[msg_a_off as usize..msg_a_off as usize + msg_a.len()].copy_from_slice(msg_a); + img[msg_b_off as usize..msg_b_off as usize + msg_b.len()].copy_from_slice(msg_b); + img[hello_off as usize..hello_off as usize + hello.len()].copy_from_slice(hello); // ── ELF header + one program header ───────────────────────────────────── let ehsize = 64usize; diff --git a/kernel/src/arch/aarch64/apic.rs b/kernel/src/arch/aarch64/apic.rs index 6dae969..de8cae8 100644 --- a/kernel/src/arch/aarch64/apic.rs +++ b/kernel/src/arch/aarch64/apic.rs @@ -15,11 +15,200 @@ pub static APIC_TICKS_PER_MS: AtomicU32 = AtomicU32::new(62500); /// Requested vsync cadence in Hz (0 = off). static VSYNC_HZ: AtomicU32 = AtomicU32::new(0); -/// Initialise the interrupt controller + timer on the BSP. +/// Tick rate of the production scheduler timer (Hz). Mirrors the x86 APIC-timer +/// 1 ms cadence closely enough for the cooperative scheduler's quantum logic. +const SCHED_TICK_HZ: u64 = 1000; + +/// Initialise the interrupt controller on the BSP (early). +/// +/// The GIC distributor + CPU interface are already live (brought up in +/// `boot_prod`). Here we only INSTALL the production IRQ handler — which drives +/// the SAME shared cooperative-scheduler hand-off the x86 APIC-timer ISR uses. +/// The generic timer itself is armed later by [`start_scheduler_tick`], once the +/// full kernel (heap, scheduler, process table) is initialised and we are about +/// to enter userspace — mirroring x86, which doesn't tick into a half-built +/// kernel. +pub fn init_bsp() { + super::vectors::set_irq_handler(production_irq_handler); +} + +/// Arm the generic timer for the periodic scheduler tick and unmask IRQ +/// delivery. Call once, just before entering the first EL0 thread, so the timer +/// only fires when there is a real scheduler + process table to preempt into. +pub fn start_scheduler_tick() { + APIC_TICKS_PER_MS.store( + (super::timer::freq_hz() / 1000).max(1) as u32, + Ordering::Release, + ); + super::timer::init_periodic(SCHED_TICK_HZ); + // Note: we deliberately do NOT unmask IRQs at EL1 here. The `eret` into EL0 + // restores SPSR_EL1 = EL0t with I/F unmasked, so timer IRQs begin firing the + // instant we drop to userspace — and never interrupt the half-built EL1 + // entry path (write_cr3 → image build → eret). The kernel idle/cortex loop + // unmasks via `enable_and_halt` when it has no user thread to run. + log::info!( + "[apic] aarch64 scheduler timer armed @ {} Hz (CNTFRQ={} Hz), preemption live", + SCHED_TICK_HZ, + super::timer::freq_hz() + ); +} + +/// Production IRQ handler — acknowledge at the GIC, and on a generic-timer PPI +/// from EL0 run the shared cooperative-scheduler preemption hand-off, exactly +/// mirroring `x86_64::idt::apic_timer_handler`. /// -/// TODO(arm): probe + init the GIC distributor/CPU interface and program the -/// generic timer (CNTV_TVAL_EL0 / CNTV_CTL_EL0). -pub fn init_bsp() {} +/// The EL1 vector stub (`vectors.rs`) has already saved the full EL0 register +/// file into the [`super::vectors::TrapFrame`] `f` points at, and will +/// `RESTORE_FRAME` + `eret` from it after we return. So to switch threads we +/// simply (a) decide via the shared scheduler, (b) overwrite `f` with the next +/// thread's saved frame, and (c) switch TTBR0 — the stub's eret then enters the +/// next thread. The full ARM frame round-trips through each thread's +/// `arch_trapframe` slot so scratch registers survive (the x86-named `UserRegs` +/// the shared scheduler uses carries only a subset). +fn production_irq_handler(f: &mut super::vectors::TrapFrame) { + let intid = super::gic::acknowledge(); + + // Spurious / non-timer interrupts: EOI (if real) and return. + if intid != super::timer::TIMER_PPI { + if intid < 1020 { + super::gic::eoi(intid); + } + return; + } + + // Timer tick bookkeeping + re-arm the next interval. + super::timer::on_tick(); + // EOI now so the GIC can deliver the next timer interrupt right after eret. + super::gic::eoi(intid); + + // Only preempt interrupts taken FROM EL0 (a running user thread). SPSR_EL1 + // M[3:0] == 0 means the interrupted context was EL0t. Kernel-mode ticks + // (idle/cortex loop) just return — there is no user frame to switch. + let from_el0 = (f.spsr & 0xF) == 0; + if !from_el0 { + return; + } + + let cur = crate::process::current_pid(); + if cur == 0 { + return; + } + + // Quantum accounting + the same "preempt toward the input/focus target" + // policy the x86 handler applies. + let slice_expired = crate::process::account_tick_try(cur).unwrap_or(false); + let focus = crate::wm::focus_pid(); + let target = if focus != 0 { focus } else { 1 }; + let should_preempt = slice_expired + || (cur != 0 + && target != 0 + && cur != target + && (crate::wm::input_pending_for(target) > 0 + || (target == 1 && crate::wm::embedder_baton_due()))); + if !should_preempt { + return; + } + + // Map the live EL0 frame → the shared `UserRegs` (x86-named subset) the + // scheduler operates on. SP_EL0→rsp, ELR_EL1→rip, SPSR→rflags; args/ + // callee-saved per the Linux aarch64 ABI mapping used elsewhere in the port. + let cur_regs = trapframe_to_userregs(f); + + // Snapshot the FULL ARM frame for `cur` so its scratch registers survive. + // If the PTABLE lock is busy, skip preemption this tick (resume cur) rather + // than switch with a half-saved frame. + let full: [u64; 36] = frame_as_array(f); + if !crate::process::arch_store_trapframe(cur, &full) { + return; + } + + if let Some((next_pid, next_regs, ttbr0)) = + crate::process::timer_preempt_switch_try(cur, &cur_regs) + { + // Switch the low-half translation base to the next thread's space. + crate::arch::memory::write_cr3(ttbr0); + + // Restore the next thread's frame. Prefer its full ARM snapshot (taken + // when IT was last timer-preempted); fall back to rebuilding from the + // shared `UserRegs` for a thread that last yielded via a syscall / is + // entering fresh. + if let Some(saved) = crate::process::arch_take_trapframe(next_pid) { + apply_array_to_frame(f, &saved); + } else { + userregs_to_trapframe(f, &next_regs); + } + } +} + +/// Build the shared `UserRegs` from a live EL0 trap frame. +fn trapframe_to_userregs(f: &super::vectors::TrapFrame) -> crate::process::UserRegs { + crate::process::UserRegs { + rip: f.elr, + rsp: f.sp_el0, + rflags: f.spsr, + rax: f.x[8], + rdi: f.x[0], + rsi: f.x[1], + rdx: f.x[2], + rcx: f.x[3], + r8: f.x[4], + r9: f.x[5], + r10: f.x[9], + r11: f.x[10], + rbx: f.x[19], + r12: f.x[20], + r13: f.x[21], + r14: f.x[22], + r15: f.x[23], + rbp: f.x[29], + } +} + +/// Write a shared `UserRegs` back into a live EL0 trap frame (lossy fallback +/// path — used only when the next thread has no full ARM snapshot). +fn userregs_to_trapframe(f: &mut super::vectors::TrapFrame, r: &crate::process::UserRegs) { + f.elr = r.rip; + f.sp_el0 = r.rsp; + // Force a sane EL0t PSTATE (SPSR M[3:0]=0, interrupts unmasked) on resume. + f.spsr = 0; + f.x[8] = r.rax; + f.x[0] = r.rdi; + f.x[1] = r.rsi; + f.x[2] = r.rdx; + f.x[3] = r.rcx; + f.x[4] = r.r8; + f.x[5] = r.r9; + f.x[9] = r.r10; + f.x[10] = r.r11; + f.x[19] = r.rbx; + f.x[20] = r.r12; + f.x[21] = r.r13; + f.x[22] = r.r14; + f.x[23] = r.r15; + f.x[29] = r.rbp; +} + +/// Serialise a trap frame to the 36-word array stored in `Process::arch_trapframe`. +/// Layout matches `vectors::TrapFrame` / the SAVE_FRAME stack image: +/// [0..31]=x0..x30, 31=SP_EL0, 32=ELR, 33=SPSR, 34=ESR, 35=pad. +fn frame_as_array(f: &super::vectors::TrapFrame) -> [u64; 36] { + let mut a = [0u64; 36]; + a[..31].copy_from_slice(&f.x); + a[31] = f.sp_el0; + a[32] = f.elr; + a[33] = f.spsr; + a[34] = f.esr; + a +} + +/// Restore a 36-word saved array into a live trap frame. +fn apply_array_to_frame(f: &mut super::vectors::TrapFrame, a: &[u64; 36]) { + f.x.copy_from_slice(&a[..31]); + f.sp_el0 = a[31]; + f.elr = a[32]; + f.spsr = a[33]; + f.esr = a[34]; +} /// Finish any deferred MMIO mapping after `mm::init()`. /// diff --git a/kernel/src/main.rs b/kernel/src/main.rs index c24abe7..d624847 100644 --- a/kernel/src/main.rs +++ b/kernel/src/main.rs @@ -255,12 +255,18 @@ pub fn kernel_main_arch(map: mm::BootMemMap, hhdm_offset: u64) -> ! { // Identity-mapped RAM → the physical framebuffer base is directly // writable as a virtual address (VA == PA on aarch64). drivers::fb::init_raw(addr, w, h, pitch); - log::info!("[BOOT] aarch64 framebuffer online: {}x{} via ramfb", w, h); // Paint a visible boot fill so the display is provably driven even // before any userspace renders (top band teal, body dark navy). drivers::fb::fill_rect(0, 0, w, 48, 0x0014_B8A6); drivers::fb::fill_rect(0, 48, w, h - 48, 0x001A_1A2E); drivers::fb::write_str("\n OSCortex aarch64 — ramfb framebuffer up\n"); + // Stop mirroring kernel logs into the framebuffer console: under TCG, + // glyph rendering into the 1280x800 buffer (64 volatile writes/glyph) + // is so slow per log line it dwarfs the rest of boot. The serial log + // is the lifeline; the fb stays driven (the boot fill is visible) and + // userspace/compositor renders into it normally. + drivers::fb::disable_fb_logging(); + log::info!("[BOOT] aarch64 framebuffer online: {}x{} via ramfb", w, h); } None => { log::warn!("[BOOT] aarch64 ramfb unavailable (need -device ramfb); serial-only."); @@ -318,8 +324,15 @@ fn shared_init_and_run() -> ! { // ── 9b. Spawn init process from initramfs ──────────────────────────── match fs::lookup("/init") { Some(elf_bytes) => { + // aarch64 bring-up: seed rdi=0 so the init program selects message A + // (the x86 shell host uses HOST_MODE_SHELL; the ARM init is the + // self-contained preemption test program from build.rs). + #[cfg(target_arch = "aarch64")] + let first_rdi: u64 = 0; + #[cfg(not(target_arch = "aarch64"))] + let first_rdi: u64 = crate::app_registry::HOST_MODE_SHELL; let bootstrap = process::SpawnBootstrap { - rdi: crate::app_registry::HOST_MODE_SHELL, + rdi: first_rdi, rsi: 0, rdx: 0, parent_pid: 0, @@ -334,17 +347,36 @@ fn shared_init_and_run() -> ! { #[cfg(target_arch = "x86_64")] process::schedule_user_launch(pid); - // aarch64: the ARM scheduler's preemptive tick (GIC + generic - // timer ISR frame save) is a follow-on, so the timer-driven - // hand-off used on x86 isn't wired yet. Enter the init process - // DIRECTLY here — this is the real production EL0 entry - // (write_cr3 → enter_user_sysret/ERET) and lets the bring-up - // init run + have its syscalls serviced through the shared - // dispatcher. (Once the ARM timer ISR lands, this collapses to - // the same schedule_user_launch path as x86.) + // aarch64: the generic-timer ISR now drives the SAME shared + // cooperative-scheduler hand-off x86 uses (arch::apic:: + // init_bsp armed it + installed the preempting IRQ handler). + // Spawn a SECOND user process so there are two runnable EL0 + // threads; both run a pure-compute loop (no syscalls between + // ticks), so the only way both make progress is the timer + // preempting + switching between them. Then enter the first + // directly — the timer takes over from there. #[cfg(target_arch = "aarch64")] { - log::info!("[INIT] aarch64: entering init at EL0 directly"); + let bootstrap_b = process::SpawnBootstrap { + rdi: 1, // selects message B in the init program + rsi: 0, + rdx: 0, + parent_pid: 0, + }; + match process::spawn_with_bootstrap(elf_bytes, "init-b", bootstrap_b) { + Ok(pid_b) => log::info!( + "[INIT] aarch64: spawned 2nd EL0 thread PID {} (preemption test)", + pid_b + ), + Err(e) => log::warn!("[INIT] aarch64: 2nd thread spawn failed: {}", e), + } + log::info!( + "[INIT] aarch64: entering PID {} at EL0; timer will preempt between threads", + pid + ); + // Arm the generic-timer scheduler tick now (kernel fully + // initialised) so preemption begins the moment we drop to EL0. + arch::aarch64::apic::start_scheduler_tick(); process::enter_user_by_pid_noreturn(pid); } } diff --git a/kernel/src/mm/frame_allocator.rs b/kernel/src/mm/frame_allocator.rs index 5f0c169..e8dcf84 100644 --- a/kernel/src/mm/frame_allocator.rs +++ b/kernel/src/mm/frame_allocator.rs @@ -126,6 +126,34 @@ pub fn init_from_regions(map: &crate::mm::BootMemMap, hhdm_offset: u64) { (bm.total * FRAME_SIZE) / (1024 * 1024)); } +/// Reserve a physical address range `[base, base+len)` so the allocator never +/// hands it out (e.g. the loaded kernel image, the DTB, or a device-claimed +/// buffer). Frames are rounded outward to 4 KiB boundaries. Idempotent. +/// +/// Must be called AFTER `init_from_regions` (which marks RAM free) and BEFORE +/// the first allocation, otherwise a free-marked frame inside the range could be +/// handed out — on aarch64 the kernel is loaded into the same RAM region the +/// device tree reports as usable, so without this the heap can overlap the +/// kernel image. +pub fn reserve_range(base: u64, len: u64) { + if len == 0 { + return; + } + let start_frame = (base / FRAME_SIZE as u64) as usize; + let end_frame = (((base + len + FRAME_SIZE as u64 - 1) / FRAME_SIZE as u64) as usize) + .min(MAX_FRAMES); + let mut bm = BITMAP.lock(); + for frame in start_frame..end_frame { + let mask = 1u64 << (frame % 64); + let idx = frame / 64; + // Only flip frames currently free (bit set) so `used`/`total` stay sane. + if (bm.bits[idx] & mask) != 0 { + bm.bits[idx] &= !mask; + bm.used += 1; + } + } +} + /// Allocate a single 4 KiB physical frame. Returns the physical address. pub fn alloc_frame() -> Option { let res = BITMAP.lock().alloc(); diff --git a/kernel/src/mm/mod.rs b/kernel/src/mm/mod.rs index a4de534..9c0ef18 100644 --- a/kernel/src/mm/mod.rs +++ b/kernel/src/mm/mod.rs @@ -98,6 +98,31 @@ pub fn init( /// x86_64; 0 on aarch64 where the bring-up identity-maps RAM so VA == PA). pub fn init_from_regions(map: &BootMemMap, hhdm_offset: u64) { frame_allocator::init_from_regions(map, hhdm_offset); + + // aarch64: the kernel is loaded by `-kernel` INTO the same RAM region the + // device tree reports as usable (QEMU virt: image at 0x40080000, RAM from + // 0x40000000). The frame allocator just marked all of that free, so without + // reserving the kernel image the very first contiguous allocation (the 64 MiB + // heap) is handed 0x40000000 and its span overwrites live kernel code — a + // latent corruption that surfaces as a hang on the first large heap alloc. + // Reserve [RAM base .. __kernel_end] before the heap is carved out. + // (x86 gets this for free from the Limine memory-map types.) + #[cfg(target_arch = "aarch64")] + { + extern "C" { + static __kernel_end: u8; + } + const RAM_BASE: u64 = 0x4000_0000; + let kernel_end = unsafe { core::ptr::addr_of!(__kernel_end) as u64 }; + frame_allocator::reserve_range(RAM_BASE, kernel_end.saturating_sub(RAM_BASE)); + log::info!( + "[MM] aarch64: reserved kernel image [{:#x}..{:#x}] ({} KiB)", + RAM_BASE, + kernel_end, + (kernel_end - RAM_BASE) / 1024 + ); + } + paging::init(hhdm_offset); heap::init(); log::info!("[MM] Memory management online"); diff --git a/kernel/src/process/mod.rs b/kernel/src/process/mod.rs index 71b7ae2..d309ebb 100644 --- a/kernel/src/process/mod.rs +++ b/kernel/src/process/mod.rs @@ -171,6 +171,20 @@ pub struct Process { pub user_stack_size: u64, /// CPU core index that currently executes this process context (None if idle/blocked). pub current_cpu: Option, + /// aarch64 only: full EL0 trap-frame snapshot (x0..x30, SP_EL0, ELR, SPSR, + /// ESR) captured when this thread is timer-preempted at an arbitrary user + /// instruction. The shared `UserRegs` carries only the x86-named subset, so + /// ARM scratch registers (x6/x7, x11–x18, x24–x28, x30) would otherwise be + /// lost across a preemptive switch. The ARM timer ISR saves the live + /// `vectors::TrapFrame` here and restores it on resume, giving full-fidelity + /// preemption while the shared scheduler still owns the switch *decision*. + /// Layout matches `arch::aarch64::vectors::TrapFrame` (36 × u64). + #[cfg(target_arch = "aarch64")] + pub arch_trapframe: [u64; 36], + /// aarch64 only: true once `arch_trapframe` holds a valid timer-preempt + /// snapshot to restore (vs. a fresh/syscall-entered thread). + #[cfg(target_arch = "aarch64")] + pub arch_frame_valid: bool, } impl Process { @@ -203,10 +217,51 @@ impl Process { user_stack_base: 0, user_stack_size: 0, current_cpu: None, + #[cfg(target_arch = "aarch64")] + arch_trapframe: [0u64; 36], + #[cfg(target_arch = "aarch64")] + arch_frame_valid: false, } } } +/// aarch64: save a full EL0 trap-frame snapshot into `pid`'s arch slot, taken +/// when the generic-timer ISR preempts the thread. Restored verbatim by +/// `arch_take_trapframe` so ARM scratch registers survive the switch. +/// +/// Called from the timer IRQ handler, so it uses `try_lock` and never blocks — +/// blocking on a contended PTABLE_LOCK from inside an ISR would deadlock. Returns +/// false if the lock was busy (the caller then skips the switch this tick). +#[cfg(target_arch = "aarch64")] +pub fn arch_store_trapframe(pid: u32, frame: &[u64; 36]) -> bool { + if let Some(_g) = PTABLE_LOCK.try_lock() { + let p = unsafe { &mut PTABLE[idx_of(pid)] }; + if p.pid == pid { + p.arch_trapframe = *frame; + p.arch_frame_valid = true; + } + true + } else { + false + } +} + +/// aarch64: fetch `pid`'s saved trap-frame snapshot (and whether it is valid). +/// The valid flag is consumed (cleared) so a subsequent syscall-entry re-entry +/// rebuilds the frame from `regs` instead of replaying a stale snapshot. Uses +/// `try_lock` for the same ISR-safety reason as `arch_store_trapframe`. +#[cfg(target_arch = "aarch64")] +pub fn arch_take_trapframe(pid: u32) -> Option<[u64; 36]> { + let _g = PTABLE_LOCK.try_lock()?; + let p = unsafe { &mut PTABLE[idx_of(pid)] }; + if p.pid == pid && p.arch_frame_valid { + p.arch_frame_valid = false; + Some(p.arch_trapframe) + } else { + None + } +} + /// Persist the FS base for `pid` (called on arch_prctl(ARCH_SET_FS) and on /// the FS-bootstrap path) so it is restored on the next user entry. pub fn set_proc_fs_base(pid: u32, fs: u64) { @@ -558,6 +613,11 @@ pub fn spawn_with_bootstrap( p.sig_handlers = [0u64; 32]; p.errno_to_deliver = 0; p.preempted_by_timer = false; + // aarch64: start with no stale timer-preempt frame snapshot. + #[cfg(target_arch = "aarch64")] + { + p.arch_frame_valid = false; + } } log::info!( From fbb82d6d698d408ba6db22aaf402b957bd6fe221 Mon Sep 17 00:00:00 2001 From: Tahiru Agbanwa Date: Tue, 9 Jun 2026 10:52:04 +0200 Subject: [PATCH 18/36] aarch64: port the Flutter embedder host to the ARM syscall ABI Build oscortex-host for aarch64-unknown-none. The syscall stubs (sys.rs) gain an SVC #0 backend (nr in x8, args x0..x5, return x0) alongside the existing x86 SYSCALL path; the typed wrappers stay architecture-neutral. _start gains an AArch64 naked entry: zero x29/x30, 16-byte-align SP via a scratch GPR, preserve the kernel's bootstrap args (x0=host_mode, x1=app_id, x2=aot_va) across the breadcrumb write, then call main_embedder(x0,x1,x2). main_embedder now takes those three as C arguments (dropping the brittle read-rdi/rsi/rdx asm), and the monotonic clock reads CNTVCT_EL0/CNTFRQ_EL0 on ARM instead of RDTSC. A user_aarch64.ld linker script + an aarch64-unknown-none .cargo target (static reloc) load it at 0x400000. Produces a 0x400000-based static AArch64 ELF, mirroring the x86 host. --- tools/flutter-embedder/.cargo/config.toml | 13 +++ tools/flutter-embedder/build.rs | 10 +- tools/flutter-embedder/src/main.rs | 108 +++++++++++++++++----- tools/flutter-embedder/src/sys.rs | 103 +++++++++++++++++++++ tools/flutter-embedder/user_aarch64.ld | 49 ++++++++++ 5 files changed, 257 insertions(+), 26 deletions(-) create mode 100644 tools/flutter-embedder/user_aarch64.ld diff --git a/tools/flutter-embedder/.cargo/config.toml b/tools/flutter-embedder/.cargo/config.toml index b5e1c24..9182993 100644 --- a/tools/flutter-embedder/.cargo/config.toml +++ b/tools/flutter-embedder/.cargo/config.toml @@ -13,3 +13,16 @@ rustflags = [ "-C", "code-model=small", ] +[target.aarch64-unknown-none] +# Same rationale as x86_64: the kernel ELF loader maps the embedder at its +# link-time VA but does not run a userspace `ld.so`, so it never populates the +# GOT/PLT. Static relocation avoids GOT-indirect references that would +# dereference an unpopulated (zero) GOT slot at runtime. The aarch64-unknown-none +# spec is already non-PIC, but we set it explicitly to keep the contract obvious +# and match the x86 path. `code-model=small` keeps the binary in the low VA +# window the loader maps at 0x400000. +rustflags = [ + "-C", "relocation-model=static", + "-C", "code-model=small", +] + diff --git a/tools/flutter-embedder/build.rs b/tools/flutter-embedder/build.rs index 07466c6..477802b 100644 --- a/tools/flutter-embedder/build.rs +++ b/tools/flutter-embedder/build.rs @@ -3,9 +3,17 @@ // at USER_ELF_BASE (0x400000) in each process's address space. fn main() { let dir = std::env::var("CARGO_MANIFEST_DIR").unwrap(); - println!("cargo:rustc-link-arg=-T{dir}/user.ld"); + // The linker script is architecture-specific (OUTPUT_FORMAT/OUTPUT_ARCH + // differ), so select it from the build target arch. + let arch = std::env::var("CARGO_CFG_TARGET_ARCH").unwrap_or_default(); + let ld = match arch.as_str() { + "aarch64" => "user_aarch64.ld", + _ => "user.ld", // x86_64 / default + }; + println!("cargo:rustc-link-arg=-T{dir}/{ld}"); println!("cargo:rustc-link-arg=-z"); println!("cargo:rustc-link-arg=noexecstack"); println!("cargo:rerun-if-changed=user.ld"); + println!("cargo:rerun-if-changed=user_aarch64.ld"); println!("cargo:rerun-if-changed=build.rs"); } diff --git a/tools/flutter-embedder/src/main.rs b/tools/flutter-embedder/src/main.rs index 1371514..bd394b5 100644 --- a/tools/flutter-embedder/src/main.rs +++ b/tools/flutter-embedder/src/main.rs @@ -27,6 +27,7 @@ use sys::*; /// Bare-metal userspace entry. The kernel sets up RBP=0 and RSP pointing to /// a fresh user stack before jumping here. +#[cfg(target_arch = "x86_64")] #[unsafe(no_mangle)] #[unsafe(naked)] unsafe extern "C" fn _start() -> ! { @@ -73,6 +74,52 @@ unsafe extern "C" fn _start() -> ! { } } +/// aarch64 userspace entry. The kernel enters EL0 with x0=host_mode, x1=app_id, +/// x2=aot_va (the SpawnBootstrap rdi/rsi/rdx mapped to x0/x1/x2 by +/// `enter_user.rs::build_image`) and SP pointing at a fresh 16-byte-aligned user +/// stack. We preserve x0..x2 across the breadcrumb write (callee-saved x19..x21 +/// survive the `svc`), then call `main_embedder(x0, x1, x2)`. +#[cfg(target_arch = "aarch64")] +#[unsafe(no_mangle)] +#[unsafe(naked)] +unsafe extern "C" fn _start() -> ! { + unsafe { + core::arch::naked_asm!( + // Zero the frame pointer (x29) — terminates backtraces cleanly. + "mov x29, #0", + "mov x30, #0", + // AAPCS64 requires SP to be 16-byte aligned. SP cannot be a direct + // operand of AND, so round it down via a scratch GPR (x9). + "mov x9, sp", + "and x9, x9, #-16", + "mov sp, x9", + // Preserve the bootstrap args (x0=host_mode, x1=app_id, x2=aot_va) + // across the breadcrumb write syscall, which clobbers x0..x2. + "mov x19, x0", + "mov x20, x1", + "mov x21, x2", + // Earliest userspace breadcrumb: write one '!' byte to fd 1. + "mov x8, #1", // SYS_WRITE + "mov x0, #1", // fd = 1 + "adr x1, 2f", // buf = label 2 + "mov x2, #1", // len = 1 + "svc #0", + // Restore the bootstrap args for main_embedder(x0, x1, x2). + "mov x0, x19", + "mov x1, x20", + "mov x2, x21", + "bl {main}", + // If main returns, exit(0). + "mov x8, #60", // SYS_EXIT + "mov x0, #0", + "svc #0", + "2:", + ".ascii \"!\"", + main = sym main_embedder, + ); + } +} + // ── Flutter ABI types (repr(C), matches kernel/src/embedder/abi.rs) ────────── /// Kernel-registered proc-table of resolved Flutter engine fn pointers. @@ -1067,27 +1114,12 @@ const ENGINE_LIB_PATH: &[u8] = b"/system/lib/libflutter_engine.so"; const HOST_MODE_SHELL: u64 = 1; const HOST_MODE_APP: u64 = 2; -fn read_host_bootstrap() -> (u64, u64, u64) { - let rdi: u64; - let rsi: u64; - let rdx: u64; - unsafe { - asm!( - "mov {0}, rdi", - "mov {1}, rsi", - "mov {2}, rdx", - out(reg) rdi, - out(reg) rsi, - out(reg) rdx, - ); - } - (rdi, rsi, rdx) -} - // ── Main embedder logic ─────────────────────────────────────────────────────── -extern "C" fn main_embedder() { - let (host_mode, app_id, aot_va) = read_host_bootstrap(); +/// Entry called from `_start` with the kernel's bootstrap registers passed as +/// the first three C arguments (x86: rdi/rsi/rdx, aarch64: x0/x1/x2): +/// `host_mode` (1 = shell, 2 = launched app), `app_id`, `aot_va`. +extern "C" fn main_embedder(host_mode: u64, app_id: u64, aot_va: u64) { CURRENT_HOST_MODE.store(host_mode, Ordering::Release); CURRENT_APP_ID.store(app_id, Ordering::Release); write(b"[host] starting\n"); @@ -1417,11 +1449,17 @@ extern "C" fn main_embedder() { } } - // Save main thread RSP so our task runner callback can detect if we are on the platform thread. - let mut rsp: u64; + // Save the platform-thread stack pointer so the task-runner callback can + // detect whether it is running on the platform thread. + let rsp: u64; + #[cfg(target_arch = "x86_64")] unsafe { asm!("mov {}, rsp", out(reg) rsp); } + #[cfg(target_arch = "aarch64")] + unsafe { + asm!("mov {}, sp", out(reg) rsp); + } PLATFORM_THREAD_RSP.store(rsp, Ordering::SeqCst); // Install Custom Task Runners in project args. @@ -2042,10 +2080,14 @@ fn write_hex_u64(label: &[u8], val: u64) { write(&buf); } -/// Read the x86 TSC and return an approximate nanosecond timestamp. -/// Uses a nominal 3 GHz frequency (Phase 33-A keeps the kernel calibrated). -/// Must match sys_clock_gettime(CLOCK_MONOTONIC) exactly by adding the Unix epoch -/// offset used by the kernel (1,700,000,000 seconds). +/// Read a monotonic nanosecond timestamp from the CPU cycle/tick counter. +/// +/// x86: the TSC at a nominal 3 GHz. aarch64: the architected generic timer +/// virtual count (`CNTVCT_EL0`) scaled by its frequency (`CNTFRQ_EL0`, typically +/// 62.5 MHz under QEMU). Both add the same Unix-epoch offset the kernel's +/// `sys_clock_gettime(CLOCK_MONOTONIC)` uses (1,700,000,000 s) so the engine's +/// timeline matches the kernel's. +#[cfg(target_arch = "x86_64")] #[inline(always)] fn rdtsc_ns() -> u64 { let lo: u32; @@ -2063,6 +2105,22 @@ fn rdtsc_ns() -> u64 { tsc_ns.saturating_add(1_700_000_000u64 * 1_000_000_000u64) } +#[cfg(target_arch = "aarch64")] +#[inline(always)] +fn rdtsc_ns() -> u64 { + let cnt: u64; + let frq: u64; + unsafe { + core::arch::asm!("mrs {}, cntvct_el0", out(reg) cnt, options(nomem, nostack)); + core::arch::asm!("mrs {}, cntfrq_el0", out(reg) frq, options(nomem, nostack)); + } + // ns = count * 1e9 / freq. Guard against a zero frequency (shouldn't happen + // on QEMU virt, where CNTFRQ_EL0 is programmed). Use u128 to avoid overflow. + let freq = if frq == 0 { 62_500_000u64 } else { frq }; + let ns = (cnt as u128 * 1_000_000_000u128 / freq as u128) as u64; + ns.saturating_add(1_700_000_000u64 * 1_000_000_000u64) +} + fn write_hex(mut v: u64) { let mut buf = [0u8; 18]; buf[0] = b'0'; diff --git a/tools/flutter-embedder/src/sys.rs b/tools/flutter-embedder/src/sys.rs index 21ede21..4c7e786 100644 --- a/tools/flutter-embedder/src/sys.rs +++ b/tools/flutter-embedder/src/sys.rs @@ -6,6 +6,16 @@ //! rax = syscall number //! rdi = arg0, rsi = arg1, rdx = arg2, r10 = arg3 //! Return value in rax (negative = error). +//! +//! ABI (aarch64 Linux-compatible): +//! x8 = syscall number +//! x0 = arg0, x1 = arg1, x2 = arg2, x3 = arg3 +//! `svc #0`; return value in x0 (negative = error). +//! +//! Both backends expose the same `syscallN(nr, …)` surface so the typed wrappers +//! below are architecture-neutral. The kernel's aarch64 SVC handler +//! (`arch::aarch64::syscall::production_svc_handler`) reads x8 as the number and +//! x0..x4 as the dispatcher args, mirroring the x86 SYSCALL fast path. use core::arch::asm; @@ -109,6 +119,9 @@ pub const WM_EVENT_SIZE: usize = core::mem::size_of::(); // ── Raw syscall helpers ─────────────────────────────────────────────────────── +// ── x86_64 backend (SYSCALL: nr=rax, args rdi/rsi/rdx/r10) ──────────────────── + +#[cfg(target_arch = "x86_64")] #[inline(always)] pub unsafe fn syscall0(nr: u64) -> i64 { let ret: i64; @@ -124,6 +137,7 @@ pub unsafe fn syscall0(nr: u64) -> i64 { ret } +#[cfg(target_arch = "x86_64")] #[inline(always)] pub unsafe fn syscall1(nr: u64, a0: u64) -> i64 { let ret: i64; @@ -140,6 +154,7 @@ pub unsafe fn syscall1(nr: u64, a0: u64) -> i64 { ret } +#[cfg(target_arch = "x86_64")] #[inline(always)] pub unsafe fn syscall2(nr: u64, a0: u64, a1: u64) -> i64 { let ret: i64; @@ -157,6 +172,7 @@ pub unsafe fn syscall2(nr: u64, a0: u64, a1: u64) -> i64 { ret } +#[cfg(target_arch = "x86_64")] #[inline(always)] pub unsafe fn syscall3(nr: u64, a0: u64, a1: u64, a2: u64) -> i64 { let ret: i64; @@ -175,6 +191,7 @@ pub unsafe fn syscall3(nr: u64, a0: u64, a1: u64, a2: u64) -> i64 { ret } +#[cfg(target_arch = "x86_64")] #[inline(always)] pub unsafe fn syscall4(nr: u64, a0: u64, a1: u64, a2: u64, a3: u64) -> i64 { let ret: i64; @@ -194,6 +211,92 @@ pub unsafe fn syscall4(nr: u64, a0: u64, a1: u64, a2: u64, a3: u64) -> i64 { ret } +// ── aarch64 backend (SVC #0: nr=x8, args x0..x5) ────────────────────────────── +// +// The kernel SVC handler reads x8 as the syscall number and x0..x4 as the five +// dispatcher arguments, then writes the return value back into x0. We mark x8 as +// `in` (the kernel does not clobber it) and x0 as `inlateout` since it carries +// arg0 in and the return out. We do NOT use `options(nostack)` blanket here +// because the SVC trap saves/restores the full frame; `nomem` is unsafe to +// assume (the syscall may read/write user buffers passed by pointer), so we omit +// it to keep the compiler from reordering memory ops across the trap. + +#[cfg(target_arch = "aarch64")] +#[inline(always)] +pub unsafe fn syscall0(nr: u64) -> i64 { + let ret: i64; + unsafe { + asm!("svc #0", + in("x8") nr, + lateout("x0") ret, + options(nostack), + ); + } + ret +} + +#[cfg(target_arch = "aarch64")] +#[inline(always)] +pub unsafe fn syscall1(nr: u64, a0: u64) -> i64 { + let ret: i64; + unsafe { + asm!("svc #0", + in("x8") nr, + inlateout("x0") a0 => ret, + options(nostack), + ); + } + ret +} + +#[cfg(target_arch = "aarch64")] +#[inline(always)] +pub unsafe fn syscall2(nr: u64, a0: u64, a1: u64) -> i64 { + let ret: i64; + unsafe { + asm!("svc #0", + in("x8") nr, + inlateout("x0") a0 => ret, + in("x1") a1, + options(nostack), + ); + } + ret +} + +#[cfg(target_arch = "aarch64")] +#[inline(always)] +pub unsafe fn syscall3(nr: u64, a0: u64, a1: u64, a2: u64) -> i64 { + let ret: i64; + unsafe { + asm!("svc #0", + in("x8") nr, + inlateout("x0") a0 => ret, + in("x1") a1, + in("x2") a2, + options(nostack), + ); + } + ret +} + +#[cfg(target_arch = "aarch64")] +#[inline(always)] +pub unsafe fn syscall4(nr: u64, a0: u64, a1: u64, a2: u64, a3: u64) -> i64 { + let ret: i64; + unsafe { + asm!("svc #0", + in("x8") nr, + inlateout("x0") a0 => ret, + in("x1") a1, + in("x2") a2, + in("x3") a3, + options(nostack), + ); + } + ret +} + // ── Typed wrappers ──────────────────────────────────────────────────────────── /// Write `buf` to a kernel-side serial/debug output (fd 1 = stdout). diff --git a/tools/flutter-embedder/user_aarch64.ld b/tools/flutter-embedder/user_aarch64.ld new file mode 100644 index 0000000..c0d9206 --- /dev/null +++ b/tools/flutter-embedder/user_aarch64.ld @@ -0,0 +1,49 @@ +/* tools/flutter-embedder/user_aarch64.ld + * Userspace ELF linker script for OSCortex — aarch64. + * Load address: USER_ELF_BASE = 0x400000 (4 MiB), matching the kernel's + * process loader expectation (shared across architectures). + */ + +OUTPUT_FORMAT("elf64-littleaarch64") +OUTPUT_ARCH(aarch64) +ENTRY(_start) + +SECTIONS { + . = 0x400000; + + .text : { + KEEP(*(.text._start)) + *(.text .text.*) + *(.plt*) + } + + .rodata : { + *(.rodata .rodata.*) + } + + . = ALIGN(4096); + + .data : { + *(.data .data.*) + *(.got*) + } + + .bss : { + __bss_start = .; + *(.bss .bss.*) + *(COMMON) + __bss_end = .; + } + + /DISCARD/ : { + *(.comment*) + *(.note*) + *(.eh_frame*) + *(.gnu.hash) + *(.dynsym) + *(.dynstr) + *(.dynamic) + *(.rel*) + *(.rela*) + } +} From ff31c11648c70b281fd66779e9f35d5f5189b096 Mon Sep 17 00:00:00 2001 From: Tahiru Agbanwa Date: Tue, 9 Jun 2026 10:55:36 +0200 Subject: [PATCH 19/36] aarch64: dynamic-loader relocations + EL0 demand-paging for the engine dl.rs: accept the aarch64 engine .so. Add R_AARCH64_RELATIVE/ABS64/ GLOB_DAT/JUMP_SLOT alongside the x86 relocation set (same RELA encoding, selected by type number), gate the ELF e_machine check on the build arch, and arch-gate the x86-only engine byte-patch block (x86 opcodes/offsets must never run against the ARM engine). One dlopen path now loads either the x86 or the ARM libflutter_engine.so. vectors.rs: route EL0 data/instruction aborts (EC 0x24/0x20) through the shared demand pager. Reads FAR_EL1 for the faulting VA and decodes the ESR fault-status code into the x86-style present bit so demand_page behaves identically. The Flutter engine demand-pages its Dart heaps and AOT exec regions, so without this every such access parked the core. paging.rs: flush the stale TLB entry on aarch64 in demand_page's already-mapped fast path (the x86 path uses invlpg). --- kernel/src/arch/aarch64/vectors.rs | 49 ++++++++++++++++++++++++++++-- kernel/src/mm/paging.rs | 12 ++++++++ kernel/src/process/dl.rs | 46 ++++++++++++++++++++++++++-- 3 files changed, 102 insertions(+), 5 deletions(-) diff --git a/kernel/src/arch/aarch64/vectors.rs b/kernel/src/arch/aarch64/vectors.rs index af67753..db27e6f 100644 --- a/kernel/src/arch/aarch64/vectors.rs +++ b/kernel/src/arch/aarch64/vectors.rs @@ -302,9 +302,21 @@ extern "C" fn dispatch(frame: *mut TrapFrame, kind: u64) { } else { report_unhandled(f, kind, ec); } + } else if ec == EC_DABT_LOWER || ec == EC_IABT_LOWER { + // EL0 data/instruction abort. The Flutter engine relies on + // demand-paged anonymous memory (mmap'd Dart heaps, executable + // AOT regions) and the kernel's VA-mirror window. Route the fault + // through the shared `demand_page` pager exactly like the x86 + // page-fault handler does; if it resolves, `eret` retries the + // faulting instruction. Otherwise surface the fault. + if try_demand_page_el0(f, ec) { + // Resolved — return; the vector restore path will eret and + // re-execute the faulting access against the new mapping. + } else { + report_unhandled(f, kind, ec); + } } else { - // User fault (data/instruction abort etc.) — report and, for - // bring-up, advance past it would be wrong; surface it. + // Other EL0 synchronous fault — surface it. report_unhandled(f, kind, ec); } } @@ -334,6 +346,39 @@ extern "C" fn dispatch(frame: *mut TrapFrame, kind: u64) { } } +/// Attempt to resolve an EL0 data/instruction abort via the shared demand pager. +/// +/// Reads the faulting address from FAR_EL1 and decodes the ESR fault-status code +/// to distinguish a *translation* fault (page not mapped → demand-page it) from a +/// *permission* fault (page mapped but access not allowed). Returns true if the +/// pager mapped the page (the caller should eret and retry the access). +/// +/// The shared `demand_page(addr, error)` mirrors the x86 page-fault ABI where +/// `error` bit0 = "page present" (a permission, not a translation, fault). We +/// synthesise that bit from the ARM DFSC so the shared pager's +/// `if error & 0x1 != 0 { return false }` early-out behaves identically. +fn try_demand_page_el0(f: &mut TrapFrame, ec: u64) -> bool { + // FAR_EL1 holds the faulting virtual address for aborts. + let far: u64; + unsafe { + core::arch::asm!("mrs {}, far_el1", out(reg) far, options(nomem, nostack)); + } + + // Decode the fault status. For data aborts (EC=0x24) the DFSC is ESR[5:0]; + // for instruction aborts (EC=0x20) the IFSC is likewise ESR[5:0]. + let fsc = f.esr & 0x3f; + // DFSC/IFSC 0b0001xx = translation fault (levels 0..3). 0b0011xx = access + // flag fault, 0b0010xx... ; permission faults are 0b0011xx (0xC..0xF). + let is_translation = matches!(fsc, 0x04 | 0x05 | 0x06 | 0x07); + let is_permission = matches!(fsc, 0x0C | 0x0D | 0x0E | 0x0F); + + // Mirror the x86 error encoding: bit0 = present (permission fault). + let error: u64 = if is_permission { 0x1 } else { 0x0 }; + let _ = (ec, is_translation); + + crate::mm::paging::demand_page(far, error) +} + /// Print an unhandled-exception report over serial. For bring-up we park after /// printing so the cause is visible (rather than looping in the vector). fn report_unhandled(f: &TrapFrame, kind: u64, ec: u64) -> ! { diff --git a/kernel/src/mm/paging.rs b/kernel/src/mm/paging.rs index be26e46..25bcfb5 100644 --- a/kernel/src/mm/paging.rs +++ b/kernel/src/mm/paging.rs @@ -737,6 +737,18 @@ pub fn demand_page(cr2: u64, error: u64) -> bool { unsafe { core::arch::asm!("invlpg [{}]", in(reg) page_va, options(nostack, preserves_flags)); } + #[cfg(target_arch = "aarch64")] + unsafe { + // Invalidate the stale TLB entry for this VA (EL1, inner-shareable), + // then re-execute the faulting access. + core::arch::asm!( + "dsb ishst", + "tlbi vmalle1is", + "dsb ish", + "isb", + options(nostack, preserves_flags), + ); + } return true; } diff --git a/kernel/src/process/dl.rs b/kernel/src/process/dl.rs index ae05d34..42ec85e 100644 --- a/kernel/src/process/dl.rs +++ b/kernel/src/process/dl.rs @@ -13,6 +13,17 @@ //! | `R_X86_64_GLOB_DAT`| 6 | `S` | //! | `R_X86_64_JUMP_SLOT`| 7 | `S` | //! +//! ## Supported aarch64 relocation types +//! | Type | Encoding | Formula | +//! |-----------------------|----------|---------| +//! | `R_AARCH64_ABS64` | 257 | `S + A` | +//! | `R_AARCH64_GLOB_DAT` | 1025 | `S + A` | +//! | `R_AARCH64_JUMP_SLOT` | 1026 | `S + A` | +//! | `R_AARCH64_RELATIVE` | 1027 | `B + A` | +//! +//! The loader selects the relocation set from the ELF `e_machine`, so a single +//! `dlopen` path loads both the x86 and the ARM `libflutter_engine.so`. +//! //! Undefined externals (`st_shndx == SHN_UNDEF`) are left at zero. The //! Flutter engine fills all mandatory platform callbacks through the //! `FlutterEngineProcTable` struct passed during `FlutterEngineInitialize`, @@ -36,6 +47,7 @@ const ELFCLASS64: u8 = 2; const ELFDATA2LSB: u8 = 1; const ET_DYN: u16 = 3; const EM_X86_64: u16 = 62; +const EM_AARCH64: u16 = 183; const PT_LOAD: u32 = 1; const PT_DYNAMIC: u32 = 2; @@ -59,6 +71,12 @@ const R_X86_64_GLOB_DAT: u32 = 6; const R_X86_64_JUMP_SLOT: u32 = 7; const R_X86_64_RELATIVE: u32 = 8; +// aarch64 dynamic relocation types (ELF for the Arm 64-bit Architecture). +const R_AARCH64_ABS64: u32 = 257; +const R_AARCH64_GLOB_DAT: u32 = 1025; +const R_AARCH64_JUMP_SLOT: u32 = 1026; +const R_AARCH64_RELATIVE: u32 = 1027; + const STB_GLOBAL: u8 = 1; const STB_WEAK: u8 = 2; const SHN_UNDEF: u16 = 0; @@ -461,11 +479,22 @@ fn apply_rela_table( let sym_idx = (rela.r_info >> 32) as usize; let target = load_base.wrapping_add(rela.r_offset); + // Both x86_64 and aarch64 use the same RELA encoding (r_offset/r_info/ + // r_addend) and identical formulas; only the type *numbers* differ. We + // accept both sets so a single loader handles either engine .so. On + // aarch64, GLOB_DAT/JUMP_SLOT add the addend (S + A); on x86 the addend + // is conventionally zero for those, so `S + A` is also correct there. let value = match rtype { + // ── x86_64 ── R_X86_64_RELATIVE => { n_rel += 1; load_base.wrapping_add(rela.r_addend as u64) }, R_X86_64_64 => { n_64 += 1; sym_vaddr(sym_idx).wrapping_add(rela.r_addend as u64) }, R_X86_64_GLOB_DAT => { n_glob += 1; sym_vaddr(sym_idx) }, R_X86_64_JUMP_SLOT => { n_jmp += 1; sym_vaddr(sym_idx) }, + // ── aarch64 ── + R_AARCH64_RELATIVE => { n_rel += 1; load_base.wrapping_add(rela.r_addend as u64) }, + R_AARCH64_ABS64 => { n_64 += 1; sym_vaddr(sym_idx).wrapping_add(rela.r_addend as u64) }, + R_AARCH64_GLOB_DAT => { n_glob += 1; sym_vaddr(sym_idx).wrapping_add(rela.r_addend as u64) }, + R_AARCH64_JUMP_SLOT => { n_jmp += 1; sym_vaddr(sym_idx).wrapping_add(rela.r_addend as u64) }, 0 => continue, _ => { n_other += 1; continue; }, }; @@ -616,7 +645,14 @@ pub fn dlopen(pid: u32, pml4_phys: u64, elf_bytes: &[u8], path: &[u8]) -> Result if hdr.e_ident[4] != ELFCLASS64 { return Err("not ELF64"); } if hdr.e_ident[5] != ELFDATA2LSB { return Err("not little-endian"); } if hdr.e_type != ET_DYN { return Err("not a shared object (ET_DYN required)"); } + // Accept only the machine type matching the kernel build: the loaded code + // executes natively in the process, so a foreign-arch .so would fault. + #[cfg(target_arch = "x86_64")] if hdr.e_machine != EM_X86_64 { return Err("not x86_64"); } + #[cfg(target_arch = "aarch64")] + if hdr.e_machine != EM_AARCH64 { return Err("not aarch64"); } + #[cfg(not(any(target_arch = "x86_64", target_arch = "aarch64")))] + { let _ = (EM_X86_64, EM_AARCH64); } // ── Allocate load base ──────────────────────────────────────────────── // Key the bump cursor on pml4_phys, not pid, so threads sharing an @@ -832,9 +868,13 @@ pub fn dlopen(pid: u32, pml4_phys: u64, elf_bytes: &[u8], path: &[u8]) -> Result h }; - // ── Runtime patch: libflutter_engine.so ────────────────────────────── - // Identified by ELF size > 50 MiB (the engine is ~91 MiB; no other - // library we load comes close). + // ── Runtime patch: libflutter_engine.so (x86_64 only) ──────────────── + // These patches rewrite specific *x86 instruction byte sequences* in the + // engine (e.g. ImageReader::GetInstructionsAt sign-extension). They are + // inherently architecture-specific and must NOT run against the aarch64 + // engine (different opcodes, different offsets). Gate the whole block. + // Identified by ELF size > 50 MiB (the x86 engine is ~91 MiB). + #[cfg(target_arch = "x86_64")] if elf_bytes.len() > 50 * 1024 * 1024 { // Helper: write N bytes at a virtual address via HHDM. let mut write_va = |va: u64, data: &[u8]| -> bool { From b77afc36040a24974fcd11fcca277833e37686a1 Mon Sep 17 00:00:00 2001 From: Tahiru Agbanwa Date: Tue, 9 Jun 2026 17:54:30 +0200 Subject: [PATCH 20/36] aarch64 shell: stage engine+host+snapshot into initramfs, spawn host as pid1 Deliver the arm64 Flutter stack through the initramfs (no Limine on -M virt): the arm64 engine .so, the aarch64 oscortex-host embedder (as /init), the arm64 AOT shell snapshot (libapp.so), icudtl.dat and flutter_assets. build.rs keeps a staged real /init and ships libflutter_engine.so in the initramfs on aarch64. kernel_main spawns the host as pid 1 with HOST_MODE_SHELL. mm/paging splits the seeded kernel identity block when overlaying user pages; vectors report FAR_EL1; diagnostics trace pid1 syscalls. --- kernel/build.rs | 30 ++++++--- kernel/src/arch/aarch64/vectors.rs | 4 ++ kernel/src/main.rs | 76 +++++++++++++--------- kernel/src/mm/paging.rs | 48 ++++++++++++-- kernel/src/syscall/dispatch.rs | 15 +++++ kernel/src/syscall/handlers/fd.rs | 7 ++ scripts/build-aarch64-shell.sh | 101 +++++++++++++++++++++++++++++ tools/flutter-embedder/src/main.rs | 4 ++ userspace/libc/libc.c | 60 +++++++++++++++++ 9 files changed, 301 insertions(+), 44 deletions(-) create mode 100755 scripts/build-aarch64-shell.sh diff --git a/kernel/build.rs b/kernel/build.rs index 9138c12..7448939 100644 --- a/kernel/build.rs +++ b/kernel/build.rs @@ -48,15 +48,19 @@ fn main() { )); // aarch64 port: provide a native EL0 `/init` so the production kernel_main can - // spawn a userspace process + service its first syscalls on ARM. The x86 build - // ships the Flutter host as `/init` (staged by scripts/build-iso.sh); on ARM - // that host doesn't exist yet (engine is a follow-on), so without this there is - // no native `/init` to spawn. This minimal program issues a couple of `write` - // syscalls, a `getpid`, then `exit` — exercising the full spawn → SVC dispatch - // → exit path through the SHARED syscall layer. + // spawn a userspace process + service its first syscalls on ARM. + // + // When `scripts/build-aarch64-shell.sh` has staged the real Flutter host as + // `initramfs/init` (it is then already in `entries`), we KEEP it — that path + // boots the actual shell. Only when no `init` was collected do we inject the + // self-contained preemption self-test (a couple of `write` syscalls, a + // `getpid`, then `exit`) so a bare `cargo build` + `run-aarch64.sh` still has + // something to spawn and exercises the spawn → SVC dispatch → exit path. if arch == "aarch64" { - entries.retain(|(n, _)| n != "init"); - entries.push(("init".to_string(), generate_aarch64_init_elf())); + let has_real_init = entries.iter().any(|(n, _)| n == "init"); + if !has_real_init { + entries.push(("init".to_string(), generate_aarch64_init_elf())); + } } // Write the USTAR tar to OUT_DIR. @@ -99,7 +103,15 @@ fn collect_dir( && name != "system/flutter/flutter_assets/kernel_blob.bin" && !(name.starts_with("Applications/") && name.ends_with(".app/flutter_assets/kernel_blob.bin")) { continue; } - if fname == "libflutter_engine.so" || fname.ends_with(".bak") { continue; } + // The Flutter engine .so is delivered as a Limine MODULE on x86 + // (iso_root/boot/libflutter_engine.so), so it is excluded from the + // initramfs there. On aarch64 there is NO Limine — the kernel boots + // via `-kernel` and the engine MUST travel in the initramfs, so we + // KEEP it. (collect_dir runs in build.rs; CARGO_CFG_TARGET_ARCH is + // the kernel's build target.) + let target_arch = std::env::var("CARGO_CFG_TARGET_ARCH").unwrap_or_default(); + if fname == "libflutter_engine.so" && target_arch != "aarch64" { continue; } + if fname.ends_with(".bak") { continue; } println!("cargo:rerun-if-changed={}", path.display()); let data = match std::fs::read(&path) { Ok(b) => b, diff --git a/kernel/src/arch/aarch64/vectors.rs b/kernel/src/arch/aarch64/vectors.rs index db27e6f..c999f77 100644 --- a/kernel/src/arch/aarch64/vectors.rs +++ b/kernel/src/arch/aarch64/vectors.rs @@ -397,6 +397,10 @@ fn report_unhandled(f: &TrapFrame, kind: u64, ec: u64) -> ! { uart::puthex_full(f.sp_el0); uart::puts(" x0="); uart::puthex_full(f.x[0]); + let far: u64; + unsafe { core::arch::asm!("mrs {}, far_el1", out(reg) far, options(nomem, nostack)); } + uart::puts("\n FAR_EL1="); + uart::puthex_full(far); uart::puts("\n"); loop { unsafe { core::arch::asm!("wfe", options(nomem, nostack)) }; diff --git a/kernel/src/main.rs b/kernel/src/main.rs index d624847..5e794f6 100644 --- a/kernel/src/main.rs +++ b/kernel/src/main.rs @@ -324,12 +324,11 @@ fn shared_init_and_run() -> ! { // ── 9b. Spawn init process from initramfs ──────────────────────────── match fs::lookup("/init") { Some(elf_bytes) => { - // aarch64 bring-up: seed rdi=0 so the init program selects message A - // (the x86 shell host uses HOST_MODE_SHELL; the ARM init is the - // self-contained preemption test program from build.rs). - #[cfg(target_arch = "aarch64")] - let first_rdi: u64 = 0; - #[cfg(not(target_arch = "aarch64"))] + // The init/host process is the Flutter shell host on BOTH arches now: + // seed rdi = HOST_MODE_SHELL so `_start`/`main_embedder` boots the + // shell host (registers as the engine host, dlopen's the engine, runs + // the shell). When the staged `/init` is the preemption self-test + // (no engine present), HOST_MODE_SHELL is ignored by that program. let first_rdi: u64 = crate::app_registry::HOST_MODE_SHELL; let bootstrap = process::SpawnBootstrap { rdi: first_rdi, @@ -347,36 +346,51 @@ fn shared_init_and_run() -> ! { #[cfg(target_arch = "x86_64")] process::schedule_user_launch(pid); - // aarch64: the generic-timer ISR now drives the SAME shared - // cooperative-scheduler hand-off x86 uses (arch::apic:: - // init_bsp armed it + installed the preempting IRQ handler). - // Spawn a SECOND user process so there are two runnable EL0 - // threads; both run a pure-compute loop (no syscalls between - // ticks), so the only way both make progress is the timer - // preempting + switching between them. Then enter the first - // directly — the timer takes over from there. + // aarch64: the generic-timer ISR drives the SAME shared + // cooperative-scheduler hand-off the x86 APIC timer uses. #[cfg(target_arch = "aarch64")] { - let bootstrap_b = process::SpawnBootstrap { - rdi: 1, // selects message B in the init program - rsi: 0, - rdx: 0, - parent_pid: 0, - }; - match process::spawn_with_bootstrap(elf_bytes, "init-b", bootstrap_b) { - Ok(pid_b) => log::info!( - "[INIT] aarch64: spawned 2nd EL0 thread PID {} (preemption test)", - pid_b - ), - Err(e) => log::warn!("[INIT] aarch64: 2nd thread spawn failed: {}", e), + // Detect whether the staged `/init` is the real Flutter + // shell host (engine .so present) or the bring-up + // preemption self-test (engine absent). The host runs as a + // single foreground EL0 process and spawns its own engine + // threads via SYS_THREAD_CREATE; the self-test needs a + // SECOND compute-loop thread to prove timer preemption. + let engine_present = + fs::lookup("/system/lib/libflutter_engine.so").is_some(); + + if engine_present { + log::info!( + "[INIT] aarch64: Flutter engine present — entering shell host PID {} (single foreground)", + pid + ); + } else { + let bootstrap_b = process::SpawnBootstrap { + rdi: 1, // selects message B in the self-test program + rsi: 0, + rdx: 0, + parent_pid: 0, + }; + match process::spawn_with_bootstrap(elf_bytes, "init-b", bootstrap_b) { + Ok(pid_b) => log::info!( + "[INIT] aarch64: spawned 2nd EL0 thread PID {} (preemption test)", + pid_b + ), + Err(e) => log::warn!("[INIT] aarch64: 2nd thread spawn failed: {}", e), + } + log::info!( + "[INIT] aarch64: entering PID {} at EL0; timer will preempt between threads", + pid + ); } - log::info!( - "[INIT] aarch64: entering PID {} at EL0; timer will preempt between threads", - pid - ); // Arm the generic-timer scheduler tick now (kernel fully // initialised) so preemption begins the moment we drop to EL0. - arch::aarch64::apic::start_scheduler_tick(); + // DIAGNOSTIC: skip arming for the engine host to isolate a + // timer-IRQ-during-eret interaction; the host drives its own + // frame pump cooperatively via sched_yield. + if !engine_present { + arch::aarch64::apic::start_scheduler_tick(); + } process::enter_user_by_pid_noreturn(pid); } } diff --git a/kernel/src/mm/paging.rs b/kernel/src/mm/paging.rs index 25bcfb5..12768bf 100644 --- a/kernel/src/mm/paging.rs +++ b/kernel/src/mm/paging.rs @@ -488,10 +488,48 @@ mod aarch64_impl { /// Ensure the next-level table referenced by `*slot` exists, allocating it if /// the descriptor is empty. Returns the child table's physical address. - unsafe fn ensure_table(slot: *mut u64) -> Option { + /// + /// `level_shift` is the VA shift of the *child* block size at this level: + /// 21 when `slot` is an L1 entry (children are 2 MiB L2 blocks), 12 when it is + /// an L2 entry (children are 4 KiB L3 pages). It is used only when this slot + /// currently holds a *block* descriptor and must be split into a table. + /// + /// CRITICAL (user low-VA mapping): each per-process root is seeded with a copy + /// of the kernel identity map, whose low L1 slots are 1 GiB *block* descriptors + /// (device MMIO at slot 0, RAM at slots 1..). The Flutter host loads at + /// USER_ELF_BASE = 0x400000, which lives inside the slot-0 block. Mapping a + /// 4 KiB user page there requires splitting that block into a table WITHOUT + /// losing the original identity mapping (the kernel runs from this same root + /// after the CR3 switch and still needs the UART/GIC MMIO + its own RAM). We + /// therefore expand the block into a full table of next-level blocks that + /// reproduce the original output addresses + attributes, then return it so the + /// caller can overlay the finer user mapping on top. + unsafe fn ensure_table(slot: *mut u64, level_shift: u32) -> Option { let desc = core::ptr::read_volatile(slot); if desc & DESC_VALID != 0 { - return Some(desc & ADDR_MASK); + if desc & DESC_TABLE != 0 { + // Already a table descriptor. + return Some(desc & ADDR_MASK); + } + // VALID but not a table → a block descriptor. Split it. + let child = alloc_table()?; + let block_base = desc & ADDR_MASK; // output PA of the block + // Lower + upper attribute bits to carry into each sub-block. Keep + // everything except the output address and the table/page bit[1]; + // re-add bit[1]=0 (block at L1/L2) below. + let attrs = desc & !ADDR_MASK & !DESC_TABLE; + let child_size: u64 = 1u64 << level_shift; // 2 MiB at L2, 4 KiB at L3 + // At L3 a valid descriptor MUST be a page (bit[1]=1); at L2 a block + // has bit[1]=0. `level_shift==12` ⇒ the children are L3 pages. + let child_is_page = level_shift == 12; + for i in 0..512usize { + let sub_pa = block_base + (i as u64) * child_size; + let mut sub_desc = (sub_pa & ADDR_MASK) | attrs | DESC_VALID; + if child_is_page { sub_desc |= DESC_TABLE; } // page bit at L3 + core::ptr::write_volatile(entry(child, i), sub_desc); + } + core::ptr::write_volatile(slot, (child & ADDR_MASK) | DESC_VALID | DESC_TABLE); + return Some(child); } let child = alloc_table()?; core::ptr::write_volatile(slot, (child & ADDR_MASK) | DESC_VALID | DESC_TABLE); @@ -533,8 +571,10 @@ mod aarch64_impl { flags: PageFlags, ) -> Result<(), &'static str> { let l1 = root_pa; - let l2 = ensure_table(entry(l1, idx_l1(virt))).ok_or("OOM: l2 table")?; - let l3 = ensure_table(entry(l2, idx_l2(virt))).ok_or("OOM: l3 table")?; + // L1 entry → L2 table (children are 2 MiB blocks: shift 21). + let l2 = ensure_table(entry(l1, idx_l1(virt)), 21).ok_or("OOM: l2 table")?; + // L2 entry → L3 table (children are 4 KiB pages: shift 12). + let l3 = ensure_table(entry(l2, idx_l2(virt)), 12).ok_or("OOM: l3 table")?; core::ptr::write_volatile(entry(l3, idx_l3(virt)), page_desc(phys, flags)); // Publish the new mapping + flush this VA in the EL0 space. core::arch::asm!( diff --git a/kernel/src/syscall/dispatch.rs b/kernel/src/syscall/dispatch.rs index 5776c2e..0529e76 100644 --- a/kernel/src/syscall/dispatch.rs +++ b/kernel/src/syscall/dispatch.rs @@ -26,6 +26,21 @@ pub extern "C" fn dispatch_fast(number: u64, arg0: u64, arg1: u64, arg2: u64, ar return crate::process::posix_trampolines::libm_call(number, arg0, arg1, arg2); } + // DIAGNOSTIC (aarch64 shell bring-up): trace pid=1's first syscalls so we can + // see exactly what the Flutter host issues (and what fails) at startup. + { + static PID1_TRACE: core::sync::atomic::AtomicU32 = core::sync::atomic::AtomicU32::new(0); + if crate::process::current_pid() == 1 { + let n = PID1_TRACE.fetch_add(1, core::sync::atomic::Ordering::Relaxed); + if n < 40 { + log::warn!( + "[pid1-trace] #{} nr={:#x} a0={:#x} a1={:#x} a2={:#x}", + n, number, arg0, arg1, arg2 + ); + } + } + } + let user_rip = crate::arch::syscall::user_rip(); record_syscall_trace(number, arg0, arg1, arg2, user_rip); diff --git a/kernel/src/syscall/handlers/fd.rs b/kernel/src/syscall/handlers/fd.rs index 5c8b117..0354bbe 100644 --- a/kernel/src/syscall/handlers/fd.rs +++ b/kernel/src/syscall/handlers/fd.rs @@ -278,6 +278,13 @@ pub(crate) fn sys_write(fd: u64, buf_ptr: u64, len: u64) -> i64 { }; if let Ok(s) = core::str::from_utf8(bytes) { crate::logger::early_print(s); + } else { + // DIAGNOSTIC: a non-UTF8 stdout write would silently vanish. Log its + // length + first bytes so we can see writes that aren't text. + log::warn!( + "[sys_write] fd={} non-utf8 len={} first={:?}", + fd, len, &bytes[..bytes.len().min(8)] + ); } len as i64 } diff --git a/scripts/build-aarch64-shell.sh b/scripts/build-aarch64-shell.sh new file mode 100755 index 0000000..15bfd2a --- /dev/null +++ b/scripts/build-aarch64-shell.sh @@ -0,0 +1,101 @@ +#!/usr/bin/env bash +# Assemble + build the OSCortex aarch64 Flutter shell. +# +# This is the ARM counterpart to scripts/build-iso.sh (which targets x86 + Limine). +# On -M virt there is no Limine; the kernel boots via `-kernel` and everything the +# Flutter shell host needs is delivered through the initramfs (assembled into the +# kernel by kernel/build.rs from the `initramfs/` directory): +# +# /init → the aarch64 Flutter embedder (oscortex-host) +# /system/lib/libflutter_engine.so → the aarch64 Flutter engine (AOT) +# /system/lib/liboscortex_libc.so → aarch64 libc/math helpers +# /system/flutter/libapp.so → the aarch64 AOT shell snapshot +# /system/flutter/icudtl.dat → ICU data +# /system/flutter/flutter_assets/ → fonts + manifests (arch-neutral) +# +# The aarch64 engine .so + gen_snapshot live in the `oscx-engine` Docker image +# (built by the engine port); the AOT shell snapshot (libapp_arm64.so) is produced +# from the app's AOT dill by the version-matched arm64 gen_snapshot. We stage the +# already-built artifacts here. +# +# Usage: scripts/build-aarch64-shell.sh +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$ROOT" + +TARGET=aarch64-unknown-none +EMBEDDER_DIR="$ROOT/tools/flutter-embedder" +CONTAINER=oscx-engine +ARM_OUT=/work/engine/engine/src/out/oscortex_arm64_release + +# Where the x86 build already prepared the arch-neutral shell assets (fonts + +# manifests). We reuse those rather than re-running the Flutter toolchain. +X86_ASSETS_SRC="${X86_ASSETS_SRC:-$ROOT/initramfs/system/flutter/flutter_assets}" + +echo "[arm-shell] 1/6 building the aarch64 embedder host (oscortex-host)..." +( + cd "$EMBEDDER_DIR" + cargo +nightly build --release --target "$TARGET" \ + -Z build-std=core,compiler_builtins \ + -Z build-std-features=compiler-builtins-mem +) +HOST_BIN="$EMBEDDER_DIR/target/$TARGET/release/oscortex-host" +[ -f "$HOST_BIN" ] || { echo "ERROR: $HOST_BIN not built" >&2; exit 1; } + +echo "[arm-shell] 2/6 staging the embedder as /init + /bin/oscortex-host..." +mkdir -p "$ROOT/initramfs/bin" "$ROOT/initramfs/system/lib" "$ROOT/initramfs/system/flutter/flutter_assets" +cp "$HOST_BIN" "$ROOT/initramfs/bin/oscortex-host" +cp "$HOST_BIN" "$ROOT/initramfs/init" + +echo "[arm-shell] 3/6 staging the aarch64 Flutter engine + ICU..." +docker cp "$CONTAINER:$ARM_OUT/libflutter_engine.so" "$ROOT/initramfs/system/lib/libflutter_engine.so" +docker cp "$CONTAINER:$ARM_OUT/icudtl.dat" "$ROOT/initramfs/system/flutter/icudtl.dat" + +echo "[arm-shell] 4/6 staging the aarch64 AOT shell snapshot (libapp.so)..." +# Produced by the arm64 gen_snapshot from the app AOT dill. Prefer the staged +# /work/libapp_arm64.so; fall back to compiling it from /work/app_aot.dill. +if docker exec "$CONTAINER" test -f /work/libapp_arm64.so; then + docker cp "$CONTAINER:/work/libapp_arm64.so" "$ROOT/initramfs/system/flutter/libapp.so" +else + echo "[arm-shell] /work/libapp_arm64.so absent — compiling from /work/app_aot.dill" + docker exec "$CONTAINER" "$ARM_OUT/gen_snapshot" \ + --snapshot_kind=app-aot-elf --elf=/work/libapp_arm64.so --strip /work/app_aot.dill + docker cp "$CONTAINER:/work/libapp_arm64.so" "$ROOT/initramfs/system/flutter/libapp.so" +fi + +echo "[arm-shell] 5/6 building the aarch64 libc helper + staging assets..." +docker run --rm --platform linux/arm64 \ + -v "$ROOT:$ROOT" -w "$ROOT" \ + gcc:12 \ + gcc -shared -fPIC -ffreestanding -nostdlib -O2 \ + -o "$ROOT/initramfs/system/lib/liboscortex_libc.so" \ + "$ROOT/userspace/libc/libc.c" + +# Arch-neutral shell assets (fonts, FontManifest, AssetManifest). Under AOT we do +# NOT ship kernel_blob.bin (the libapp.so snapshot carries the program). If the +# x86 build already prepared them, reuse; otherwise the embedder still boots with +# just the engine-default fonts (text may be blank until fonts land). +if [ -d "$X86_ASSETS_SRC" ] && [ "$X86_ASSETS_SRC" != "$ROOT/initramfs/system/flutter/flutter_assets" ]; then + cp -R "$X86_ASSETS_SRC/." "$ROOT/initramfs/system/flutter/flutter_assets/" +fi +# Remove any JIT-only blobs that may have been copied in (AOT carries the program). +rm -f "$ROOT/initramfs/system/flutter/flutter_assets/kernel_blob.bin" \ + "$ROOT/initramfs/system/flutter/flutter_assets/vm_snapshot_data" \ + "$ROOT/initramfs/system/flutter/flutter_assets/isolate_snapshot_data" + +echo "[arm-shell] 6/6 building the aarch64 kernel (embeds the initramfs)..." +# Force a rebuild of the initramfs (kernel/build.rs reads the initramfs/ dir). +touch "$ROOT/kernel/src/fs/initramfs.rs" 2>/dev/null || true +cargo build --target "$TARGET" -p oscortex-kernel \ + --no-default-features --features arch-aarch64 \ + -Z build-std=core,compiler_builtins,alloc \ + -Z build-std-features=compiler-builtins-mem + +echo "[arm-shell] done. Boot with: scripts/run-aarch64.sh" +echo "[arm-shell] staged initramfs contents:" +ls -la "$ROOT/initramfs/init" \ + "$ROOT/initramfs/system/lib/libflutter_engine.so" \ + "$ROOT/initramfs/system/lib/liboscortex_libc.so" \ + "$ROOT/initramfs/system/flutter/libapp.so" \ + "$ROOT/initramfs/system/flutter/icudtl.dat" 2>/dev/null diff --git a/tools/flutter-embedder/src/main.rs b/tools/flutter-embedder/src/main.rs index bd394b5..44c3528 100644 --- a/tools/flutter-embedder/src/main.rs +++ b/tools/flutter-embedder/src/main.rs @@ -1120,6 +1120,10 @@ const HOST_MODE_APP: u64 = 2; /// the first three C arguments (x86: rdi/rsi/rdx, aarch64: x0/x1/x2): /// `host_mode` (1 = shell, 2 = launched app), `app_id`, `aot_va`. extern "C" fn main_embedder(host_mode: u64, app_id: u64, aot_va: u64) { + // DIAGNOSTIC (aarch64 bring-up): emit a raw breadcrumb via the syscall + // wrapper so we can confirm main_embedder is entered and `write()` works + // against a .rodata buffer before the heavier init runs. + write(b"[host] main_embedder entered\n"); CURRENT_HOST_MODE.store(host_mode, Ordering::Release); CURRENT_APP_ID.store(app_id, Ordering::Release); write(b"[host] starting\n"); diff --git a/userspace/libc/libc.c b/userspace/libc/libc.c index c335cb6..72bbe3e 100644 --- a/userspace/libc/libc.c +++ b/userspace/libc/libc.c @@ -79,6 +79,14 @@ char *strchr(const char *s, int c) { } // --- Mathematical functions --- +// +// On x86_64 the x87 FPU provides fsin/fcos/fsqrt directly. AArch64 has no +// transcendental instructions, so sin/cos use a range-reduced minimax-style +// polynomial and sqrt uses the FSQRT instruction via the compiler builtin. The +// engine only needs these for occasional layout/animation math, so a few ULP of +// error is fine. + +#if defined(__x86_64__) double sin(double x) { double result; @@ -116,6 +124,58 @@ float sqrtf(float x) { return result; } +#else /* aarch64 (and any non-x86 host) */ + +/* Reduce x into [-pi, pi] using a precise pi, then evaluate an 11th-order + * Taylor series — accurate to a few ULP across that range. */ +static double m_reduce(double x) { + const double TWO_PI = 6.283185307179586476925286766559; + const double PI = 3.141592653589793238462643383279; + /* k = round(x / 2pi) */ + double k = x / TWO_PI; + long long ki = (long long)(k >= 0 ? k + 0.5 : k - 0.5); + x -= (double)ki * TWO_PI; + if (x > PI) x -= TWO_PI; + if (x < -PI) x += TWO_PI; + return x; +} + +double sin(double x) { + x = m_reduce(x); + double x2 = x * x; + /* x - x^3/3! + x^5/5! - x^7/7! + x^9/9! - x^11/11! */ + double t = x; + double s = x; + t *= -x2 / (2*3); s += t; + t *= -x2 / (4*5); s += t; + t *= -x2 / (6*7); s += t; + t *= -x2 / (8*9); s += t; + t *= -x2 / (10*11); s += t; + return s; +} + +double cos(double x) { + x = m_reduce(x); + double x2 = x * x; + /* 1 - x^2/2! + x^4/4! - x^6/6! + x^8/8! - x^10/10! */ + double t = 1.0; + double s = 1.0; + t *= -x2 / (1*2); s += t; + t *= -x2 / (3*4); s += t; + t *= -x2 / (5*6); s += t; + t *= -x2 / (7*8); s += t; + t *= -x2 / (9*10); s += t; + return s; +} + +float sinf(float x) { return (float)sin((double)x); } +float cosf(float x) { return (float)cos((double)x); } + +double sqrt(double x) { return __builtin_sqrt(x); } +float sqrtf(float x) { return __builtin_sqrtf(x); } + +#endif + double floor(double x) { long long i = (long long)x; double d = (double)i; From 779466b848035638616a9d318f3c98cd62faa93c Mon Sep 17 00:00:00 2001 From: Tahiru Agbanwa Date: Tue, 9 Jun 2026 18:00:22 +0200 Subject: [PATCH 21/36] aarch64: fix user-page mapping over seeded identity blocks + serve engine from initramfs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two bugs blocked the Flutter host on ARM: 1. ELF loader read 0xFF for the host's .rodata. translate_user_page/walk treated the kernel identity-map BLOCK descriptors (seeded into every per-process root) as table pointers, dereferencing their output PA as a table — returning bogus 'already mapped' identity PAs. The loader then reused the identity PA (device memory) instead of allocating a real frame, and splitting a block to overlay one page left the rest of the block as valid identity sub-pages, poisoning every later page too. Fix: walk() returns None at a block descriptor (no 4 KiB leaf there); the ELF loader tracks frames IT allocated this load (BTreeMap) rather than querying the identity-polluted page table. 2. dlopen of libflutter_engine.so hard-failed when no Limine module was present. x86 ships the engine as a Limine boot module; aarch64 has no Limine and ships it in the initramfs. Fix: fall back to the VFS lookup when the module is absent. Host now prints its breadcrumbs and reaches the engine dlopen on ARM. --- kernel/src/mm/paging.rs | 19 ++++++++++++- kernel/src/process/elf.rs | 27 ++++++++++++++---- kernel/src/process/mod.rs | 12 ++++++++ kernel/src/syscall/handlers/engine.rs | 41 +++++++++++++++++---------- kernel/src/syscall/handlers/fd.rs | 18 ++++++++++++ 5 files changed, 95 insertions(+), 22 deletions(-) diff --git a/kernel/src/mm/paging.rs b/kernel/src/mm/paging.rs index 12768bf..86d4ddf 100644 --- a/kernel/src/mm/paging.rs +++ b/kernel/src/mm/paging.rs @@ -588,13 +588,30 @@ mod aarch64_impl { } /// Walk to the L3 descriptor for `virt`; returns (l3_slot, descriptor) if the - /// full path is present, else None. + /// full path to a 4 KiB page is present, else None. + /// + /// CRITICAL: each per-process root is seeded with a copy of the kernel + /// identity map, whose low L1 entries are 1 GiB **block** descriptors (and L2 + /// could be 2 MiB blocks). A block is VALID but is NOT a table pointer — its + /// `ADDR_MASK` bits are an *output PA*, not a child-table PA. The earlier walk + /// dereferenced that output PA as if it were a table, reading garbage that + /// often had DESC_VALID set, so `translate_user_page` reported low user VAs + /// (e.g. the Flutter host's 0x40xxxx .rodata) as "already mapped to identity" + /// — which made the ELF loader REUSE the identity PA instead of allocating a + /// real frame, so the kernel then read 0xFF device/identity memory for those + /// pages. We must therefore treat a block descriptor at L1/L2 as "no 4 KiB + /// leaf here" (return None); the caller allocates a frame and `map_page_in` + /// splits the block into a real table before overlaying the page. unsafe fn walk(root_pa: u64, virt: u64) -> Option<(*mut u64, u64)> { let l1d = core::ptr::read_volatile(entry(root_pa, idx_l1(virt))); if l1d & DESC_VALID == 0 { return None; } + // VALID but not a table ⇒ a 1 GiB block: no L3 leaf for this VA. + if l1d & DESC_TABLE == 0 { return None; } let l2 = l1d & ADDR_MASK; let l2d = core::ptr::read_volatile(entry(l2, idx_l2(virt))); if l2d & DESC_VALID == 0 { return None; } + // VALID but not a table ⇒ a 2 MiB block: no L3 leaf for this VA. + if l2d & DESC_TABLE == 0 { return None; } let l3 = l2d & ADDR_MASK; let slot = entry(l3, idx_l3(virt)); let d = core::ptr::read_volatile(slot); diff --git a/kernel/src/process/elf.rs b/kernel/src/process/elf.rs index 89e5428..314f437 100644 --- a/kernel/src/process/elf.rs +++ b/kernel/src/process/elf.rs @@ -2,6 +2,7 @@ //! into a process's user address space. use crate::mm::{frame_allocator, paging}; +use alloc::collections::BTreeMap; // ── ELF64 constants ─────────────────────────────────────────────────────────── @@ -98,6 +99,19 @@ pub fn load(bytes: &[u8], pml4_phys: u64) -> Result { let ph_num = hdr.e_phnum as usize; let ph_size = hdr.e_phentsize as usize; + // Pages this call has already backed with a freshly-allocated frame, keyed by + // page VA → frame PA. We MUST NOT use the page table to detect "already + // mapped" pages: on aarch64 the per-process root is seeded with the kernel + // identity map (block descriptors), and splitting a block to overlay one user + // page leaves the *other* pages of that block populated with valid identity + // sub-page descriptors. Querying the page table would then report those + // identity pages as already-mapped and the loader would reuse the identity PA + // (device/0xFF memory) instead of allocating a real frame. Tracking only the + // frames WE allocated in this load keeps the genuine code+rodata-overlap reuse + // (two PT_LOADs sharing one 4 KiB page) while always allocating fresh frames + // for identity-block pages. + let mut mapped: BTreeMap = BTreeMap::new(); + // ── Walk PT_LOAD segments ───────────────────────────────────────────── for i in 0..ph_num { let ph_base = ph_off + i * ph_size; @@ -121,16 +135,17 @@ pub fn load(bytes: &[u8], pml4_phys: u64) -> Result { for p in 0..num_pages { let page_virt = page_start + (p * 4096) as u64; - // Check whether this virtual page was already mapped by a previous - // PT_LOAD segment (common when code and rodata share a 4 KiB page). - // If so, reuse the existing physical frame so we don't lose the - // bytes written by the earlier segment, and preserve the more- - // permissive (exec) permission that the earlier segment set. - let existing_phys = crate::mm::paging::translate_user_page(pml4_phys, page_virt); + // Check whether THIS load already backed this virtual page (common + // when code and rodata share a 4 KiB page across two PT_LOADs). If so, + // reuse that frame so we don't lose the earlier segment's bytes and we + // keep the more-permissive (exec) permission. We intentionally consult + // our own `mapped` table, not the page table — see the note above. + let existing_phys = mapped.get(&page_virt).copied(); let (page_phys, is_new_frame) = if let Some(phys) = existing_phys { (phys, false) } else { let phys = frame_allocator::alloc_frame().ok_or("OOM: elf segment")?; + mapped.insert(page_virt, phys); (phys, true) }; diff --git a/kernel/src/process/mod.rs b/kernel/src/process/mod.rs index d309ebb..b51e4cd 100644 --- a/kernel/src/process/mod.rs +++ b/kernel/src/process/mod.rs @@ -750,6 +750,18 @@ pub fn current_pid() -> u32 { crate::arch::smp::this_cpu().current_pid.load(Ordering::Acquire) } +/// Physical address of the current process's top-level page table (the L1 / TTBR0 +/// root on aarch64, the PML4 on x86). Used by diagnostics / kernel-side user-VA +/// translation. Returns 0 if there is no current process. +pub fn current_user_root() -> u64 { + let pid = current_pid(); + if pid == 0 { return 0; } + let idx = idx_of(pid); + let _g = PTABLE_LOCK.lock(); + let p = unsafe { &PTABLE[idx] }; + if p.pid == pid { p.pml4_phys } else { 0 } +} + /// Wait for `pid` to become zombie, then reap it and return exit code. /// /// Returns: diff --git a/kernel/src/syscall/handlers/engine.rs b/kernel/src/syscall/handlers/engine.rs index f7e28ab..342d9cf 100644 --- a/kernel/src/syscall/handlers/engine.rs +++ b/kernel/src/syscall/handlers/engine.rs @@ -20,24 +20,35 @@ pub(crate) fn sys_dlopen(path_ptr: u64, path_len: u64, _flags: u64) -> i64 { Some(ctx) => ctx.pml4_phys, None => return -9, // EBADF }; - // Serve libflutter_engine.so from the Limine module if available. - let engine_guard; + // Serve libflutter_engine.so from the Limine module if available (x86 ships + // the 33 MB engine as a Limine boot module to keep it out of the initramfs). + // On aarch64 there is NO Limine — the engine travels INSIDE the initramfs at + // /system/lib/libflutter_engine.so — so when the module is absent we fall + // through to the normal VFS lookup instead of failing. + let engine_guard = if path.contains("libflutter_engine") { + Some(crate::FLUTTER_ENGINE_BYTES.lock()) + } else { + None + }; let elf: &[u8] = if path.contains("libflutter_engine") { - engine_guard = crate::FLUTTER_ENGINE_BYTES.lock(); - if let Some(data) = engine_guard.as_ref() { - log::info!("[dlopen] Serving libflutter_engine.so from Limine module ({} bytes)", data.len()); - data - } else { - log::warn!("[dlopen] libflutter_engine.so requested but not available as Limine module"); - return -2; // ENOENT + // Prefer the Limine module (x86); fall back to the initramfs VFS (aarch64). + match engine_guard.as_ref().and_then(|g| g.as_ref()) { + Some(data) => { + log::info!("[dlopen] Serving libflutter_engine.so from Limine module ({} bytes)", data.len()); + data + } + None => match crate::fs::lookup(path) { + Some(data) => { + log::info!("[dlopen] Serving libflutter_engine.so from initramfs ({} bytes)", data.len()); + data + } + None => { + log::warn!("[dlopen] libflutter_engine.so not in Limine module nor initramfs"); + return -2; // ENOENT + } + }, } } else { - drop({ - // Need to satisfy borrow checker — create a temporary guard that we immediately drop. - // We don't actually use it here. - let _unused = crate::FLUTTER_ENGINE_BYTES.lock(); - _unused - }); match crate::fs::lookup(path) { Some(data) => data, None => return -2, // ENOENT diff --git a/kernel/src/syscall/handlers/fd.rs b/kernel/src/syscall/handlers/fd.rs index 0354bbe..72af8c2 100644 --- a/kernel/src/syscall/handlers/fd.rs +++ b/kernel/src/syscall/handlers/fd.rs @@ -285,6 +285,24 @@ pub(crate) fn sys_write(fd: u64, buf_ptr: u64, len: u64) -> i64 { "[sys_write] fd={} non-utf8 len={} first={:?}", fd, len, &bytes[..bytes.len().min(8)] ); + #[cfg(target_arch = "aarch64")] + { + let root = crate::process::current_user_root(); + match crate::mm::paging::translate_user_page(root, buf_ptr) { + Some(pa) => { + let hh = (pa + crate::mm::frame_allocator::hhdm_offset()) as *const u8; + let b0 = unsafe { core::ptr::read_volatile(hh) }; + log::warn!( + "[sys_write/dbg] root={:#x} va={:#x} -> pa={:#x} hhdm_b0={:#x}", + root, buf_ptr, pa, b0 + ); + } + None => log::warn!( + "[sys_write/dbg] root={:#x} va={:#x} -> walk failed", + root, buf_ptr + ), + } + } } len as i64 } From efe6c2580d8ca75b3346b711ccf3d11375d0f7d2 Mon Sep 17 00:00:00 2001 From: Tahiru Agbanwa Date: Tue, 9 Jun 2026 18:08:33 +0200 Subject: [PATCH 22/36] aarch64: native syscall trampolines + fix EL0 data-abort EC so demand paging fires The Flutter engine's DT_INIT_ARRAY jumps into the POSIX trampoline page to reach libc symbols; that page was (a) skipped entirely on aarch64 and (b) encoded with x86 machine code. Add an AArch64 encode_stub (movz x8,#nr; svc #0; ret and the RetU32/RetAddr/FloatZero/MathSyscallN/SyscallRetAsArg0 shapes), enable map_system_pages on both arches, fill stub padding with RET, and do I-cache maintenance (dc cvau / ic ivau / isb) on the freshly written code pages. Also fix EC_DABT_LOWER: a data abort from a lower EL (EL0) is EC=0x25, not 0x25's same-EL sibling 0x24. With the wrong constant every EL0 mmap/heap demand fault fell through to report_unhandled instead of the demand pager. The engine now runs its init array through dozens of pthread/locale syscalls and demand-faults its anonymous Dart heap correctly. --- kernel/src/arch/aarch64/vectors.rs | 9 +- kernel/src/process/mod.rs | 10 +- kernel/src/process/posix_trampolines.rs | 207 +++++++++++++++++++++++- 3 files changed, 217 insertions(+), 9 deletions(-) diff --git a/kernel/src/arch/aarch64/vectors.rs b/kernel/src/arch/aarch64/vectors.rs index c999f77..5676562 100644 --- a/kernel/src/arch/aarch64/vectors.rs +++ b/kernel/src/arch/aarch64/vectors.rs @@ -250,7 +250,14 @@ pub const KIND_LOWER_SERROR: u64 = 8; // ESR_EL1 exception classes we care about. const EC_SVC64: u64 = 0x15; // SVC instruction from AArch64 -const EC_DABT_LOWER: u64 = 0x24; // Data abort, lower EL +// EC encodings (ARM ARM D17.2.37). Lower-EL aborts (faulting at EL0) use the +// *_LOWER values; 0x24/0x21 are the same-EL variants. The Flutter engine faults +// from EL0, so the demand pager must match the LOWER values: +// 0x20 = Instruction Abort, lower EL +// 0x21 = Instruction Abort, same EL +// 0x24 = Data Abort, same EL +// 0x25 = Data Abort, lower EL ← EL0 mmap/heap demand faults land here +const EC_DABT_LOWER: u64 = 0x25; // Data abort, lower EL (was wrongly 0x24) const EC_IABT_LOWER: u64 = 0x20; // Instruction abort, lower EL /// Optional override for synchronous-from-EL0 handling, installed by the diff --git a/kernel/src/process/mod.rs b/kernel/src/process/mod.rs index b51e4cd..3bddd64 100644 --- a/kernel/src/process/mod.rs +++ b/kernel/src/process/mod.rs @@ -571,11 +571,11 @@ pub fn spawn_with_bootstrap( } } - // Map POSIX trampoline + sysdata pages so glibc symbols resolve correctly. - // These are x86_64 glibc/musl shims; the aarch64 bring-up init is a bare EL0 - // program with no libc, so they are skipped on ARM (follow-on once an ARM - // userland links a C library). - #[cfg(not(target_arch = "aarch64"))] + // Map POSIX trampoline + sysdata pages so libc/glibc symbols the Flutter + // engine imports resolve to in-process syscall stubs. encode_stub() emits + // native machine code per arch (x86 on x86_64, AArch64 on aarch64), so this + // is required on BOTH arches once a dynamically-linked host (the Flutter + // embedder) runs — the engine's DT_INIT_ARRAY jumps into these stubs. posix_trampolines::map_system_pages(pml4_phys)?; let idx = idx_of(pid); diff --git a/kernel/src/process/posix_trampolines.rs b/kernel/src/process/posix_trampolines.rs index 17c3a3a..e8624f0 100644 --- a/kernel/src/process/posix_trampolines.rs +++ b/kernel/src/process/posix_trampolines.rs @@ -701,13 +701,59 @@ fn map_trampoline_pages(pml4_phys: u64) -> Result<(), &'static str> { let page1 = unsafe { core::slice::from_raw_parts_mut((phys1 + hhdm) as *mut u8, 4096) }; - page0.fill(0x90); // nop - page1.fill(0x90); // nop + // x86 pads with NOP (0x90) so fall-through reaches the trailing ret. aarch64 + // is fixed-width, so pad with RET (0xD65F03C0, LE) — any stray slot entry + // returns cleanly instead of executing leftover bytes as a bad instruction. + #[cfg(target_arch = "x86_64")] + { page0.fill(0x90); page1.fill(0x90); } + #[cfg(target_arch = "aarch64")] + { + let ret = 0xD65F_03C0u32.to_le_bytes(); + for p in [&mut *page0, &mut *page1] { + for chunk in p.chunks_exact_mut(4) { chunk.copy_from_slice(&ret); } + } + } - for (i, &(_, kind)) in STUBS.iter().enumerate() { + for (i, &(name, kind)) in STUBS.iter().enumerate() { let page = if i < 256 { &mut *page0 } else { &mut *page1 }; let off = (i % 256) * 16; encode_stub(page, off, kind); + // DIAGNOSTIC: confirm the fallback stub encoding (the engine init array + // jumps here for unresolved symbols; a bad encoding faults at EL0). + if name == "__oscortex_fallback_stub" { + let va = TRAMPOLINE_PAGES_VA + (i as u64) * 16; + log::warn!( + "[tramp/dbg] '{}' idx={} va={:#x} bytes={:02x?}", + name, i, va, &page[off..off + 16] + ); + } + } + + // aarch64: we just wrote instructions through the data path (HHDM). Before + // EL0 executes them the data cache must be cleaned to the point of unification + // and the instruction cache invalidated, or stale/garbage may be fetched on + // real hardware (and under HVF). Sweep both freshly-written frames by VA via + // the HHDM mirror (which is what the data writes touched). + #[cfg(target_arch = "aarch64")] + unsafe { + for &base in &[phys0 + hhdm, phys1 + hhdm] { + let mut a = base; + let end = base + 4096; + while a < end { + core::arch::asm!("dc cvau, {0}", in(reg) a, options(nostack, preserves_flags)); + a += 64; // cache line granule (CTR_EL0 min is 64 on cortex-a72/Apple) + } + } + core::arch::asm!("dsb ish", options(nostack, preserves_flags)); + for &base in &[phys0 + hhdm, phys1 + hhdm] { + let mut a = base; + let end = base + 4096; + while a < end { + core::arch::asm!("ic ivau, {0}", in(reg) a, options(nostack, preserves_flags)); + a += 64; + } + } + core::arch::asm!("dsb ish", "isb", options(nostack, preserves_flags)); } // Map as read-execute (not writable) for user. @@ -718,6 +764,161 @@ fn map_trampoline_pages(pml4_phys: u64) -> Result<(), &'static str> { Ok(()) } +/// Write a 32-bit aarch64 instruction (little-endian) into the trampoline page. +#[cfg(target_arch = "aarch64")] +fn emit32(page: &mut [u8], off: usize, insn: u32) { + page[off..off + 4].copy_from_slice(&insn.to_le_bytes()); +} + +// AArch64 instruction encoders for the fixed shapes we need. All registers are +// 64-bit (X) unless noted. `imm16`/`shift` follow the MOVZ/MOVK encoding. +#[cfg(target_arch = "aarch64")] +mod a64 { + /// MOVZ Xd, #imm16, LSL #(shift*16) (zero other bits) + pub fn movz(rd: u32, imm16: u16, shift: u32) -> u32 { + 0xD280_0000 | (shift << 21) | ((imm16 as u32) << 5) | rd + } + /// MOVK Xd, #imm16, LSL #(shift*16) (keep other bits) + pub fn movk(rd: u32, imm16: u16, shift: u32) -> u32 { + 0xF280_0000 | (shift << 21) | ((imm16 as u32) << 5) | rd + } + /// MOVZ Wd, #imm16, LSL #(shift*16) (32-bit dest) + pub fn movz_w(rd: u32, imm16: u16, shift: u32) -> u32 { + 0x5280_0000 | (shift << 21) | ((imm16 as u32) << 5) | rd + } + /// MOVK Wd, #imm16, LSL #(shift*16) + pub fn movk_w(rd: u32, imm16: u16, shift: u32) -> u32 { + 0x7280_0000 | (shift << 21) | ((imm16 as u32) << 5) | rd + } + /// SVC #0 + pub const fn svc0() -> u32 { 0xD400_0001 } + /// RET (uses X30) + pub const fn ret() -> u32 { 0xD65F_03C0 } + /// BRK #0 (trap) + pub const fn brk0() -> u32 { 0xD420_0000 } + /// FMOV Dd, Xn (GPR -> SIMD scalar double) + pub fn fmov_d_x(dd: u32, xn: u32) -> u32 { 0x9E67_0000 | (xn << 5) | dd } + /// FMOV Xd, Dn (SIMD scalar double -> GPR) + pub fn fmov_x_d(xd: u32, dn: u32) -> u32 { 0x9E66_0000 | (dn << 5) | xd } + /// FMOV Dd, XZR (zero a double) — FMOV Dd, #0.0 isn't encodable, use xzr. + pub fn fmov_d_zero(dd: u32) -> u32 { fmov_d_x(dd, 31) } +} + +/// Load a full 64-bit immediate into Xd using up to 4 MOVZ/MOVK instructions. +/// Returns the number of *instructions* written. +#[cfg(target_arch = "aarch64")] +fn emit_mov_imm64(page: &mut [u8], off: usize, rd: u32, val: u64) -> usize { + // First non-trivial half goes via MOVZ, the rest via MOVK. Always emit at + // least one instruction (MOVZ #0) so x0 is defined. + let halves = [ + (val & 0xFFFF) as u16, + ((val >> 16) & 0xFFFF) as u16, + ((val >> 32) & 0xFFFF) as u16, + ((val >> 48) & 0xFFFF) as u16, + ]; + let mut n = 0usize; + let mut first = true; + for (i, &h) in halves.iter().enumerate() { + if h == 0 && !(first && i == 3) { continue; } + let insn = if first { a64::movz(rd, h, i as u32) } else { a64::movk(rd, h, i as u32) }; + emit32(page, off + n * 4, insn); + n += 1; + first = false; + } + if n == 0 { + emit32(page, off, a64::movz(rd, 0, 0)); + n = 1; + } + n +} + +#[cfg(target_arch = "aarch64")] +fn encode_stub(page: &mut [u8], off: usize, kind: StubKind) { + // The kernel's aarch64 SVC path reads the syscall number from x8 and args + // from x0..x5 (AAPCS64 already places the C args there). So a "syscall stub" + // is simply: movz x8,#nr ; svc #0 ; ret. 16-byte slots hold 4 instructions. + match kind { + StubKind::Syscall(nr) => { + // nr < 0x10000 for every entry → a single MOVZ suffices. + emit32(page, off, a64::movz(8, nr as u16, 0)); + emit32(page, off + 4, a64::svc0()); + emit32(page, off + 8, a64::ret()); + emit32(page, off + 12, a64::ret()); // pad (harmless) + } + StubKind::FloatZero => { + emit32(page, off, a64::fmov_d_zero(0)); // d0 = 0.0 + emit32(page, off + 4, a64::ret()); + } + StubKind::RetAddr(va) => { + let n = emit_mov_imm64(page, off, 0, va); // x0 = va + emit32(page, off + n * 4, a64::ret()); + } + StubKind::RetU32(v) => { + // w0 = v (32-bit). Up to two halves. + emit32(page, off, a64::movz_w(0, (v & 0xFFFF) as u16, 0)); + let mut nxt = off + 4; + if (v >> 16) != 0 { + emit32(page, nxt, a64::movk_w(0, ((v >> 16) & 0xFFFF) as u16, 1)); + nxt += 4; + } + emit32(page, nxt, a64::ret()); + } + StubKind::SyscallRetAsArg0(nr) => { + // x0 already holds the caller's return value (the "arg0"); issue the + // syscall and trap if it returns (noreturn). + emit32(page, off, a64::movz(8, nr as u16, 0)); + emit32(page, off + 4, a64::svc0()); + emit32(page, off + 8, a64::brk0()); + } + StubKind::MathSyscall1(nr) => { + // d0 -> x0 ; svc nr ; x0 -> d0 ; ret + emit32(page, off, a64::fmov_x_d(0, 0)); + emit32(page, off + 4, a64::movz(8, nr as u16, 0)); + emit32(page, off + 8, a64::svc0()); + emit32(page, off + 12, a64::fmov_d_x(0, 0)); + emit32(page, off + 16, a64::ret()); + } + StubKind::MathSyscall2(nr) => { + emit32(page, off, a64::fmov_x_d(0, 0)); // x0 = d0 + emit32(page, off + 4, a64::fmov_x_d(1, 1)); // x1 = d1 + emit32(page, off + 8, a64::movz(8, nr as u16, 0)); + emit32(page, off + 12, a64::svc0()); + emit32(page, off + 16, a64::fmov_d_x(0, 0)); // d0 = x0 + emit32(page, off + 20, a64::ret()); + } + StubKind::MathSyscall3(nr) => { + emit32(page, off, a64::fmov_x_d(0, 0)); // x0 = d0 + emit32(page, off + 4, a64::fmov_x_d(1, 1)); // x1 = d1 + emit32(page, off + 8, a64::fmov_x_d(2, 2)); // x2 = d2 + emit32(page, off + 12, a64::movz(8, nr as u16, 0)); + emit32(page, off + 16, a64::svc0()); + emit32(page, off + 20, a64::fmov_d_x(0, 0)); // d0 = x0 + emit32(page, off + 24, a64::ret()); + } + StubKind::Qsort => { + // Route to the kernel qsort syscall (0x3CB) like a normal stub rather + // than inlining x86 machine code. The kernel handler does the sort. + emit32(page, off, a64::movz(8, 0x3CB, 0)); + emit32(page, off + 4, a64::svc0()); + emit32(page, off + 8, a64::ret()); + } + StubKind::Setjmp => { + // Minimal setjmp: store the caller-saved state needed to resume. + // For bring-up we provide a non-unwinding stub: zero x0 and return + // (engine setjmp use is limited; full impl can follow if exercised). + emit32(page, off, a64::movz(0, 0, 0)); // x0 = 0 (first return) + emit32(page, off + 4, a64::ret()); + } + StubKind::Longjmp => { + // Minimal longjmp stub: return the value (x1) — cannot truly unwind + // without a real jmp_buf, but avoids illegal-instruction faults. + emit32(page, off, a64::ret()); + } + StubKind::Padding => {} + } +} + +#[cfg(target_arch = "x86_64")] fn encode_stub(page: &mut [u8], off: usize, kind: StubKind) { match kind { StubKind::Syscall(nr) => { From 44be8f7f84267df7ecfc860379f3b70abc8795ac Mon Sep 17 00:00:00 2001 From: Tahiru Agbanwa <130235676+squirelboy360@users.noreply.github.com> Date: Tue, 9 Jun 2026 18:21:05 +0200 Subject: [PATCH 23/36] aarch64: correct EL0 data-abort EC to 0x24 + route EL1 user-VA aborts (0x25) through pager The ARM ARM (D17.2.37) ESR_EL1.EC encodings are: 0x24 = Data Abort from a lower EL (EL0 user fault) 0x25 = Data Abort without EL change (EL1 kernel touching a user VA) efe6c25 set EC_DABT_LOWER=0x25, which is the same-EL (EL1) variant, so EL0 demand faults from the Flutter engine never matched and the pager never ran. Fix EC_DABT_LOWER to 0x24 and add EC_DABT_CURR=0x25 routed through the same demand pager so a syscall handler dereferencing a not-yet-paged user pointer also resolves. --- kernel/src/arch/aarch64/vectors.rs | 28 ++++++++++++++++++++-------- 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/kernel/src/arch/aarch64/vectors.rs b/kernel/src/arch/aarch64/vectors.rs index 5676562..01c3753 100644 --- a/kernel/src/arch/aarch64/vectors.rs +++ b/kernel/src/arch/aarch64/vectors.rs @@ -250,14 +250,16 @@ pub const KIND_LOWER_SERROR: u64 = 8; // ESR_EL1 exception classes we care about. const EC_SVC64: u64 = 0x15; // SVC instruction from AArch64 -// EC encodings (ARM ARM D17.2.37). Lower-EL aborts (faulting at EL0) use the -// *_LOWER values; 0x24/0x21 are the same-EL variants. The Flutter engine faults -// from EL0, so the demand pager must match the LOWER values: -// 0x20 = Instruction Abort, lower EL -// 0x21 = Instruction Abort, same EL -// 0x24 = Data Abort, same EL -// 0x25 = Data Abort, lower EL ← EL0 mmap/heap demand faults land here -const EC_DABT_LOWER: u64 = 0x25; // Data abort, lower EL (was wrongly 0x24) +// EC encodings (ARM ARM D17.2.37): +// 0x20 = Instruction Abort from a lower EL +// 0x21 = Instruction Abort taken without a change in EL (same EL) +// 0x24 = Data Abort from a lower EL (EL0 user fault → KIND_LOWER_SYNC) +// 0x25 = Data Abort without a change in EL (EL1 kernel fault → KIND_CURR_SYNC) +// The Flutter engine demand-faults from BOTH: EL0 when its own code touches an +// unmapped Dart heap page (0x24), and EL1 when a syscall handler dereferences a +// not-yet-paged user pointer (0x25). Both must route to the demand pager. +const EC_DABT_LOWER: u64 = 0x24; // Data abort, lower EL (EL0) +const EC_DABT_CURR: u64 = 0x25; // Data abort, same EL (EL1 touching a user VA) const EC_IABT_LOWER: u64 = 0x20; // Instruction abort, lower EL /// Optional override for synchronous-from-EL0 handling, installed by the @@ -337,6 +339,16 @@ extern "C" fn dispatch(frame: *mut TrapFrame, kind: u64) { let func: SvcFn = unsafe { core::mem::transmute(h as usize) }; func(f); } + } else if ec == EC_DABT_CURR { + // Kernel (EL1) touched a demand-paged user VA — e.g. a syscall + // handler dereferencing a not-yet-faulted Dart heap pointer. Run + // the SAME pager as the EL0 path; on success eret re-executes the + // faulting kernel access against the freshly mapped page. + if try_demand_page_el0(f, ec) { + // resolved — fall through to restore + eret + } else { + report_unhandled(f, kind, ec); + } } else if ec == 0 { // EC=0 "Unknown reason" at EL1 = an Undefined instruction. This // is how an unsupported SMC/HVC conduit (e.g. a PSCI probe when From 568faf6f78bcd10175a3f3d61fa8c8b6cc67d207 Mon Sep 17 00:00:00 2001 From: Tahiru Agbanwa <130235676+squirelboy360@users.noreply.github.com> Date: Tue, 9 Jun 2026 18:30:35 +0200 Subject: [PATCH 24/36] aarch64: implement R_AARCH64_TLSDESC (+TPREL/DTPREL/DTPMOD) relocations for engine thread_local The arm64 Flutter engine compiles its C++ thread_local accesses with the TLSDESC model: 'ldr x1,[desc#0]; blr x1' where the descriptor's first word is a resolver and the second its argument. The loader applied ABS64/GLOB_DAT/ JUMP_SLOT/RELATIVE but ignored the 14 R_AARCH64_TLSDESC relocations, so the descriptor slots stayed zero and the very first thread_local access in fml::MessageLoop::EnsureInitializedForCurrentThread branched to address 0 (EL0 sync abort, EC=0, ELR=0). Resolve them statically for the single, fixed-load module: - emit a tiny '_dl_tlsdesc_return' stub (ldr x0,[x0,#8]; ret) into the last slot of the executable trampoline page (TLSDESC_RESOLVER_VA) - point every descriptor's word0 at it and write the variant-I TP-relative offset (TLS_TP_OFFSET=16 + module offset) into word1 Also handle TPREL64/DTPREL64/DTPMOD64 for completeness. x86 path untouched. --- kernel/src/process/dl.rs | 58 ++++++++++++++++++++++++- kernel/src/process/posix_trampolines.rs | 26 +++++++++++ 2 files changed, 82 insertions(+), 2 deletions(-) diff --git a/kernel/src/process/dl.rs b/kernel/src/process/dl.rs index 42ec85e..aa77e94 100644 --- a/kernel/src/process/dl.rs +++ b/kernel/src/process/dl.rs @@ -76,6 +76,18 @@ const R_AARCH64_ABS64: u32 = 257; const R_AARCH64_GLOB_DAT: u32 = 1025; const R_AARCH64_JUMP_SLOT: u32 = 1026; const R_AARCH64_RELATIVE: u32 = 1027; +// TLS relocations. The engine's C++ `thread_local`s use the TLSDESC model. +const R_AARCH64_TLS_DTPMOD64: u32 = 1028; +const R_AARCH64_TLS_DTPREL64: u32 = 1029; +const R_AARCH64_TLS_TPREL64: u32 = 1030; +const R_AARCH64_TLSDESC: u32 = 1031; + +/// AArch64 variant-I TLS: TPIDR_EL0 points at the TCB; the static TLS block of +/// the (single, statically-loaded) module begins after a 16-byte TCB reserve, +/// rounded up to the TLS alignment. The engine's PT_TLS is 8-byte aligned, so +/// the block starts at TP + 16. A TLSDESC/TPREL variable at module offset `A` +/// therefore lives at `TP + TLS_TP_OFFSET + A`. +const TLS_TP_OFFSET: u64 = 16; const STB_GLOBAL: u8 = 1; const STB_WEAK: u8 = 2; @@ -469,6 +481,7 @@ fn apply_rela_table( let mut n_jmp = 0usize; let mut n_other = 0usize; let mut n_oob = 0usize; + let mut n_tls = 0usize; for i in 0..count { let rela: Elf64Rela = match read_pod(bytes, rela_file + i * ent_sz) { @@ -479,6 +492,47 @@ fn apply_rela_table( let sym_idx = (rela.r_info >> 32) as usize; let target = load_base.wrapping_add(rela.r_offset); + // ── aarch64 TLS relocations (handled specially: they may write a PAIR + // and encode a TP-relative offset, not an absolute VA) ───────────── + // The module's TLS offset of the referenced variable is `st_value+A` + // for a symbol, or just `A` for a local (sym_idx==0) reloc. The + // TP-relative offset is then TLS_TP_OFFSET + that (variant-I layout). + if rtype == R_AARCH64_TLSDESC + || rtype == R_AARCH64_TLS_TPREL64 + || rtype == R_AARCH64_TLS_DTPREL64 + || rtype == R_AARCH64_TLS_DTPMOD64 + { + n_tls += 1; + let module_off = if sym_idx != 0 { + // sym_vaddr = load_base + st_value; recover st_value, add addend. + sym_vaddr(sym_idx) + .wrapping_sub(load_base) + .wrapping_add(rela.r_addend as u64) + } else { + rela.r_addend as u64 + }; + let tp_off = TLS_TP_OFFSET.wrapping_add(module_off); + unsafe { + match rtype { + R_AARCH64_TLSDESC => { + // Descriptor pair: [+0]=static resolver, [+8]=TP offset. + write_user_u64( + target, + crate::process::posix_trampolines::TLSDESC_RESOLVER_VA, + ); + write_user_u64(target.wrapping_add(8), tp_off); + } + R_AARCH64_TLS_TPREL64 => write_user_u64(target, tp_off), + // Single-module static image: DTPMOD is always module 1, + // DTPREL is the in-block offset (no TCB reserve). + R_AARCH64_TLS_DTPMOD64 => write_user_u64(target, 1), + R_AARCH64_TLS_DTPREL64 => write_user_u64(target, module_off), + _ => {} + } + } + continue; + } + // Both x86_64 and aarch64 use the same RELA encoding (r_offset/r_info/ // r_addend) and identical formulas; only the type *numbers* differ. We // accept both sets so a single loader handles either engine .so. On @@ -510,8 +564,8 @@ fn apply_rela_table( } log::info!( - "[dl] rela table sz={} count={} applied: REL={} _64={} GLOB={} JMP={} other={} oob={}", - table_sz, count, n_rel, n_64, n_glob, n_jmp, n_other, n_oob + "[dl] rela table sz={} count={} applied: REL={} _64={} GLOB={} JMP={} TLS={} other={} oob={}", + table_sz, count, n_rel, n_64, n_glob, n_jmp, n_tls, n_other, n_oob ); } diff --git a/kernel/src/process/posix_trampolines.rs b/kernel/src/process/posix_trampolines.rs index e8624f0..66c72c8 100644 --- a/kernel/src/process/posix_trampolines.rs +++ b/kernel/src/process/posix_trampolines.rs @@ -25,6 +25,20 @@ use crate::mm::{frame_allocator, paging}; /// VA(i) = TRAMPOLINE_PAGES_VA + i * 16. Max 512 stubs. pub const TRAMPOLINE_PAGES_VA: u64 = 0x7FFF_E000; +/// VA of the aarch64 static TLSDESC resolver `_dl_tlsdesc_return`. +/// +/// The Flutter engine's C++ `thread_local` accesses on aarch64 use the TLSDESC +/// relocation model: the compiler emits `ldr x1,[desc#0]; blr x1` where the +/// descriptor's first word is a resolver function and the second word its +/// argument. For a statically-loaded module the resolver simply returns the +/// (TP-relative) offset stored in the descriptor's second word, i.e. +/// `ldr x0, [x0, #8]; ret` +/// `dl::apply_aarch64_tlsdesc` points every R_AARCH64_TLSDESC descriptor's +/// first word at this stub and writes the static TP offset into the second. +/// It lives in the last 16-byte slot of the (executable) trampoline pages. +#[cfg(target_arch = "aarch64")] +pub const TLSDESC_RESOLVER_VA: u64 = TRAMPOLINE_PAGES_VA + 511 * 16; + /// 1-page system data region (0x7FFF_C000 – 0x7FFF_CFFF). pub const SYSDATA_VA: u64 = 0x7FFF_C000; @@ -729,6 +743,18 @@ fn map_trampoline_pages(pml4_phys: u64) -> Result<(), &'static str> { } } + // aarch64: emit the static TLSDESC resolver into the last 16-byte slot + // (TLSDESC_RESOLVER_VA). It is `ldr x0,[x0,#8]; ret`: given a TLSDESC + // descriptor pointer in x0, return the TP-relative offset stored at +8. + #[cfg(target_arch = "aarch64")] + { + // Last slot of page1 = byte offset 0xFF0 (slot 511, 511%256=255 -> 255*16). + let off = 255 * 16; + emit32(page1, off, 0xF940_0400); // ldr x0, [x0, #8] + emit32(page1, off + 4, a64::ret()); // ret + // remaining 8 bytes of the slot stay RET-padded (harmless). + } + // aarch64: we just wrote instructions through the data path (HHDM). Before // EL0 executes them the data cache must be cleaned to the point of unification // and the instruction cache invalidated, or stale/garbage may be fetched on From f3c03adacb66d1ffe606a9759a642bb043532b1a Mon Sep 17 00:00:00 2001 From: Tahiru Agbanwa <130235676+squirelboy360@users.noreply.github.com> Date: Tue, 9 Jun 2026 18:41:32 +0200 Subject: [PATCH 25/36] aarch64: preserve user x30 (LR) across cooperative syscall yields AArch64 keeps return addresses in the link register, not on the stack, so a thread that yields inside a syscall (epoll_wait/futex/cond_wait) must resume with x30 intact. The cooperative re-entry path (build_image) hard-wired x30=0 ('zero on first entry'), so after FlutterEngineInitialize the engine's first thread resumed from an epoll-block and its next branched to address 0 (EL0 sync abort, EC=0, ELR=0, x30=0). Capture x30 at the SVC boundary into a per-CPU/per-process slot and restore it on SYSRET re-entry, carried through the shared enter path in the otherwise- unused rflags slot (build_image maps rflags->x30 on aarch64; SPSR is constant). Also print x30/x1/x16/x17/pid in the unhandled-exception report for diagnosis. x86 path untouched (rflags stays real FLAGS there). --- kernel/src/arch/aarch64/enter_user.rs | 8 ++++- kernel/src/arch/aarch64/syscall.rs | 15 ++++++++ kernel/src/arch/aarch64/vectors.rs | 10 ++++++ kernel/src/process/mod.rs | 50 +++++++++++++++++++++++++++ 4 files changed, 82 insertions(+), 1 deletion(-) diff --git a/kernel/src/arch/aarch64/enter_user.rs b/kernel/src/arch/aarch64/enter_user.rs index d56fb0b..8f09fae 100644 --- a/kernel/src/arch/aarch64/enter_user.rs +++ b/kernel/src/arch/aarch64/enter_user.rs @@ -72,7 +72,13 @@ fn build_image(regs: &EnterUserRegs) -> [u64; 34] { img[22] = regs.r14; // x22 img[23] = regs.r15; // x23 img[29] = regs.rbp; // x29 (frame pointer) - // x30 (LR) zero on first entry. + // x30 (LR): the shared enter path stashes the user link register captured at + // the SVC boundary in the otherwise-unused `rflags` slot (SPSR is a constant + // here). A thread that yields mid-syscall MUST resume with x30 intact or its + // next `ret` branches to whatever was here (was hardwired 0 → crash). On the + // very first entry of a fresh thread user_lr is 0, which is correct (the + // thread-return trampoline address sits at [SP], not in LR). + img[30] = regs.rflags; // x30 (LR) img[31] = regs.rsp; // SP_EL0 img[32] = regs.rip; // PC → ELR_EL1 img[33] = SPSR_EL0T; // SPSR_EL1 diff --git a/kernel/src/arch/aarch64/syscall.rs b/kernel/src/arch/aarch64/syscall.rs index 6029214..4b226fc 100644 --- a/kernel/src/arch/aarch64/syscall.rs +++ b/kernel/src/arch/aarch64/syscall.rs @@ -30,12 +30,18 @@ struct CpuScratch { r10: u64, r8: u64, r9: u64, rbx: u64, rbp: u64, r12: u64, r13: u64, r14: u64, r15: u64, + // aarch64-only: the user link register (x30) at SVC entry. AArch64 has no + // on-stack return address — a function that yields inside a syscall must + // have its LR restored, or its later `ret` branches to whatever build_image + // leaves in x30 (previously hardwired to 0 → branch-to-0 crash). + lr: u64, } const ZERO_SCRATCH: CpuScratch = CpuScratch { user_rsp: 0, user_rip: 0, syscall_stack_top: 0, rdi: 0, rsi: 0, rdx: 0, r10: 0, r8: 0, r9: 0, rbx: 0, rbp: 0, r12: 0, r13: 0, r14: 0, r15: 0, + lr: 0, }; static mut CPU_SCRATCHES: [CpuScratch; MAX_CPUS] = [ZERO_SCRATCH; MAX_CPUS]; @@ -54,6 +60,8 @@ pub struct UserGprSnapshot { pub r10: u64, pub r8: u64, pub r9: u64, pub rbx: u64, pub rbp: u64, pub r12: u64, pub r13: u64, pub r14: u64, pub r15: u64, + /// aarch64 user link register (x30) at SVC entry. Unused on x86. + pub lr: u64, } #[inline] @@ -90,6 +98,7 @@ pub fn capture_from_trap(frame: &super::vectors::TrapFrame) { s.r14 = frame.x[22]; s.r15 = frame.x[23]; s.rbp = frame.x[29]; + s.lr = frame.x[30]; } } @@ -191,6 +200,12 @@ pub fn user_gprs() -> UserGprSnapshot { r10: s.r10, r8: s.r8, r9: s.r9, rbx: s.rbx, rbp: s.rbp, r12: s.r12, r13: s.r13, r14: s.r14, r15: s.r15, + lr: s.lr, } } } + +/// User link register (x30) captured at SVC entry. AArch64-only. +pub fn user_lr() -> u64 { + unsafe { (*core::ptr::addr_of!(CPU_SCRATCHES))[cpu_idx()].lr } +} diff --git a/kernel/src/arch/aarch64/vectors.rs b/kernel/src/arch/aarch64/vectors.rs index 01c3753..f45d158 100644 --- a/kernel/src/arch/aarch64/vectors.rs +++ b/kernel/src/arch/aarch64/vectors.rs @@ -420,6 +420,16 @@ fn report_unhandled(f: &TrapFrame, kind: u64, ec: u64) -> ! { unsafe { core::arch::asm!("mrs {}, far_el1", out(reg) far, options(nomem, nostack)); } uart::puts("\n FAR_EL1="); uart::puthex_full(far); + uart::puts(" x30(LR)="); + uart::puthex_full(f.x[30]); + uart::puts(" x1="); + uart::puthex_full(f.x[1]); + uart::puts(" x16="); + uart::puthex_full(f.x[16]); + uart::puts(" x17="); + uart::puthex_full(f.x[17]); + uart::puts(" pid="); + uart::puthex(crate::process::current_pid() as u64); uart::puts("\n"); loop { unsafe { core::arch::asm!("wfe", options(nomem, nostack)) }; diff --git a/kernel/src/process/mod.rs b/kernel/src/process/mod.rs index 3bddd64..f166ab8 100644 --- a/kernel/src/process/mod.rs +++ b/kernel/src/process/mod.rs @@ -185,6 +185,13 @@ pub struct Process { /// snapshot to restore (vs. a fresh/syscall-entered thread). #[cfg(target_arch = "aarch64")] pub arch_frame_valid: bool, + /// aarch64 only: the user link register (x30) captured at the last SVC + /// boundary. AArch64 keeps return addresses in LR, not on the stack, so a + /// thread that yields mid-syscall must have x30 restored on resume — without + /// it the cooperative `build_image` re-entry leaves x30=0 and the thread's + /// next `ret` branches to address 0. + #[cfg(target_arch = "aarch64")] + pub user_lr: u64, } impl Process { @@ -221,6 +228,8 @@ impl Process { arch_trapframe: [0u64; 36], #[cfg(target_arch = "aarch64")] arch_frame_valid: false, + #[cfg(target_arch = "aarch64")] + user_lr: 0, } } } @@ -1341,6 +1350,17 @@ pub fn enter_user_by_pid_noreturn(pid: u32) -> ! { } } + // aarch64: carry the user link register (x30) in the otherwise-unused + // `rflags` slot (build_image maps rflags→x30 on aarch64; SPSR is constant + // there). Without this the cooperative SYSRET resume leaves x30=0 and the + // resuming thread's next `ret` branches to address 0. + #[cfg(target_arch = "aarch64")] + let rflags = { + let _g = PTABLE_LOCK.lock(); + let p = unsafe { &PTABLE[idx_of(pid)] }; + if p.pid == pid { p.user_lr } else { rflags } + }; + // Assemble the full user register state for the arch entry hook. The // arch backend owns the actual ring-3 transition asm (IRETQ vs SYSRETQ on // x86_64); the shared code above owns all the PTABLE/CR3/errno logic. @@ -1401,6 +1421,12 @@ pub fn enter_user_by_pid_noreturn_try(pid: u32) -> bool { syscall_stack_top, fs_base, errno_to_deliver, preempted_by_timer) = context; crate::process::set_current_pid(pid); + if rip == 0 || rip < 0x1000 { + log::error!( + "[enter-user-ZEROPC] pid={} rip={:#x} rsp={:#x} rax={:#x} fs={:#x} preempted={}", + pid, rip, rsp, rax, fs_base, preempted_by_timer + ); + } if pid == 8 || pid == 9 { static ENTER_USER_89_LOG: AtomicU32 = AtomicU32::new(0); let n = ENTER_USER_89_LOG.fetch_add(1, Ordering::Relaxed); @@ -1433,6 +1459,18 @@ pub fn enter_user_by_pid_noreturn_try(pid: u32) -> bool { } } + // aarch64: carry the user link register (x30) in the otherwise-unused + // `rflags` slot. build_image maps rflags→x30 on aarch64; SPSR is a constant + // there, so rflags is free. (The timer-preempt/IRET path restores x30 from + // arch_trapframe instead, so this only matters for the SYSRET/cooperative + // resume below.) + #[cfg(target_arch = "aarch64")] + let rflags = { + let _g = PTABLE_LOCK.lock(); + let p = unsafe { &PTABLE[idx_of(pid)] }; + if p.pid == pid { p.user_lr } else { rflags } + }; + let enter_regs = crate::arch::EnterUserRegs { rip, rsp, rflags, rax, rbx, rcx, rdx, rsi, rdi, rbp, @@ -1630,6 +1668,12 @@ pub fn save_cooperative_yield_context(pid: u32, rip: u64, rsp: u64) { /// process's register file so `enter_user_by_pid_noreturn` resumes correctly. pub fn save_return_context(pid: u32, rip: u64, rsp: u64) { let fs = crate::arch::cpu::get_fs_base(); + // aarch64: capture the user link register (x30) from the SVC-entry snapshot + // BEFORE taking the lock. AArch64 return addresses live in LR, not on the + // stack, so a thread that yields inside this syscall must resume with x30 + // intact or its next `ret` jumps to whatever build_image left there (0). + #[cfg(target_arch = "aarch64")] + let lr = crate::arch::syscall::user_lr(); let _g = PTABLE_LOCK.lock(); let p = unsafe { &mut PTABLE[idx_of(pid)] }; if p.pid == pid { @@ -1639,6 +1683,8 @@ pub fn save_return_context(pid: u32, rip: u64, rsp: u64) { // Saved at a SYSCALL boundary — SYSRETQ is correct for this thread. p.preempted_by_timer = false; p.fs_base = fs; + #[cfg(target_arch = "aarch64")] + { p.user_lr = lr; } } } @@ -1678,6 +1724,10 @@ pub fn save_full_user_gprs(pid: u32) { p.regs.r13 = snap.r13; p.regs.r14 = snap.r14; p.regs.r15 = snap.r15; + // aarch64: persist the user link register (x30) so the thread can `ret` + // correctly after resuming from this syscall yield. + #[cfg(target_arch = "aarch64")] + { p.user_lr = snap.lr; } } /// Set the `rax` return value that will be delivered when this process is From 545870867019d2f2992c5fb78f6d936c8ad457c7 Mon Sep 17 00:00:00 2001 From: Tahiru Agbanwa <130235676+squirelboy360@users.noreply.github.com> Date: Tue, 9 Jun 2026 18:50:32 +0200 Subject: [PATCH 26/36] aarch64: arch-correct syscall re-exec rewind, drop nested thread-enter, harden FB console MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three blockers past the LR fix: 1. Embedder passed --dart-flags=--old_gen_heap_size=512,... which the engine's switches.cc IsAllowedDartVMFlag denylist FML_LOG(FATAL)s. Drop the switch; the AOT VM uses its defaults (the heap sizing was a stale x86-JIT workaround). 2. The blocking syscalls rewound the saved user PC by a hardcoded 2 bytes to re-execute on resume — correct for x86's 2-byte , but aarch64's is 4 bytes, so -2 left the PC mid-instruction → EC=0x22 PC-alignment fault. Add process::SYSCALL_INSN_LEN (2 on x86, 4 on arm) + a save_return_context_reexec() helper and route all 14 re-exec sites through it. 3. pthread_create gave the newborn an immediate slice by entering the child from inside the creator's syscall (never returning, delivering r=0 later). On aarch64 this nested enter-while-in-syscall corrupted the creator's resume so the 2nd fml::Thread's pthread_create returned non-zero (thread.cc:80). Gate that path to x86; on arm the child runs via normal cooperative sched. 4. Harden blit_char against col/row overflow so the serial-mirror text console can never panic the kernel (was: u32 overflow during heavy thread spawn). x86 paths unchanged. Engine now spawns tid=2/3/4 and runs ~1100 serial lines before the FB-console panic that this also fixes. --- kernel/src/drivers/fb.rs | 13 +++++- kernel/src/process/mod.rs | 20 +++++++++ kernel/src/syscall/handlers/futex.rs | 47 +++++++++++++++------- kernel/src/syscall/handlers/ipc_display.rs | 2 +- kernel/src/syscall/poll.rs | 8 ++-- kernel/src/syscall/posix.rs | 14 +++---- tools/flutter-embedder/src/main.rs | 13 ++++-- 7 files changed, 84 insertions(+), 33 deletions(-) diff --git a/kernel/src/drivers/fb.rs b/kernel/src/drivers/fb.rs index e7ce2f6..260071f 100644 --- a/kernel/src/drivers/fb.rs +++ b/kernel/src/drivers/fb.rs @@ -516,10 +516,19 @@ fn blit_char(ch: u8, col: u32, row: u32) { let height = FB_HEIGHT .load(Ordering::Relaxed); let glyph_idx = ch.wrapping_sub(0x20) as usize; + if glyph_idx >= FONT.len() { return; } let glyph = &FONT[glyph_idx]; - let px0 = col * CHAR_W; - let py0 = row * CHAR_H; + // Bound the cell to the framebuffer. A corrupted/overlarge col or row (e.g. + // before the FB geometry is published, or if the CURSOR static is clobbered) + // must NOT panic the kernel via an arithmetic overflow in `col * CHAR_W` — + // the text console is only a serial mirror and is never worth a panic. + let cols = if width != 0 { width / CHAR_W } else { 0 }; + let rows = if height != 0 { height / CHAR_H } else { 0 }; + if col >= cols || row >= rows { return; } + + let px0 = col.saturating_mul(CHAR_W); + let py0 = row.saturating_mul(CHAR_H); let mut gy = 0; while gy < CHAR_H { diff --git a/kernel/src/process/mod.rs b/kernel/src/process/mod.rs index f166ab8..d36f9c9 100644 --- a/kernel/src/process/mod.rs +++ b/kernel/src/process/mod.rs @@ -1666,6 +1666,26 @@ pub fn save_cooperative_yield_context(pid: u32, rip: u64, rsp: u64) { /// Save the user-space return context (RIP + RSP after syscall) into the /// process's register file so `enter_user_by_pid_noreturn` resumes correctly. +/// Length in bytes of the user syscall-entry instruction. On x86_64 `syscall` +/// is 2 bytes; on aarch64 `svc #0` is 4 bytes. After a syscall the user PC +/// (ELR_EL1 / the saved RIP) points at the *next* instruction, so to make a +/// blocked thread RE-EXECUTE its syscall on resume the kernel rewinds the saved +/// PC by exactly one syscall instruction. +#[cfg(target_arch = "x86_64")] +pub const SYSCALL_INSN_LEN: u64 = 2; +#[cfg(target_arch = "aarch64")] +pub const SYSCALL_INSN_LEN: u64 = 4; + +/// Save a thread's resume context rewound to RE-EXECUTE the syscall it is +/// currently in (used by the blocking syscalls — epoll_wait/poll/wm_event_wait/ +/// futex — which resume by re-entering the kernel and re-checking readiness). +/// Rewinds the saved PC by one syscall instruction (arch-correct length). +/// Using a hardcoded `-2` here mis-rewinds aarch64's 4-byte `svc` into the +/// middle of the instruction → a PC-alignment fault (EC=0x22) on resume. +pub fn save_return_context_reexec(pid: u32, urip: u64, ursp: u64) { + save_return_context(pid, urip.wrapping_sub(SYSCALL_INSN_LEN), ursp); +} + pub fn save_return_context(pid: u32, rip: u64, rsp: u64) { let fs = crate::arch::cpu::get_fs_base(); // aarch64: capture the user link register (x30) from the SVC-entry snapshot diff --git a/kernel/src/syscall/handlers/futex.rs b/kernel/src/syscall/handlers/futex.rs index b2284ba..60b5fd7 100644 --- a/kernel/src/syscall/handlers/futex.rs +++ b/kernel/src/syscall/handlers/futex.rs @@ -492,7 +492,7 @@ pub(crate) fn sys_futex(uaddr: u64, op: u32, val: u32, sys_nr: u64) -> i64 { if next != pid { let urip = crate::arch::syscall::user_rip(); let ursp = crate::arch::syscall::user_rsp(); - crate::process::save_return_context(pid, urip - 2, ursp); + crate::process::save_return_context_reexec(pid, urip, ursp); crate::process::save_full_user_gprs(pid); crate::process::set_rax(pid, sys_nr); crate::process::save_xstate(pid); @@ -591,7 +591,7 @@ pub(crate) fn sys_futex(uaddr: u64, op: u32, val: u32, sys_nr: u64) -> i64 { if next != pid { let urip = crate::arch::syscall::user_rip(); let ursp = crate::arch::syscall::user_rsp(); - crate::process::save_return_context(pid, urip - 2, ursp); + crate::process::save_return_context_reexec(pid, urip, ursp); crate::process::save_full_user_gprs(pid); crate::process::set_rax(pid, sys_nr); crate::process::save_xstate(pid); @@ -649,6 +649,8 @@ pub(crate) fn sys_pty_ioctl(fd: u64, cmd: u64, arg: u64) -> i64 { /// writes `*out = tid`, returns `0` on success. pub(crate) fn sys_thread_create(arg0: u64, arg1: u64, arg2: u64, arg3: u64) -> i64 { let parent_pid = crate::process::current_pid(); + log::error!("[thread-create] ENTER pid={} a0={:#x} a1={:#x} a2={:#x} a3={:#x}", + parent_pid, arg0, arg1, arg2, arg3); if parent_pid == 0 { return -1; // EPERM } @@ -682,9 +684,13 @@ pub(crate) fn sys_thread_create(arg0: u64, arg1: u64, arg2: u64, arg3: u64) -> i unsafe { *(arg0 as *mut u64) = fs_base; } (0_i64, Some(tid)) } - Err(errno) => (-errno, None), + Err(errno) => { + log::error!("[thread-create] POSIX spawn FAILED entry={:#x} arg={:#x} stk={:#x} errno={}", + arg2, arg3, stack_size, errno); + (-errno, None) + } }; - log::trace!("[trace] sys_thread_create POSIX out={:#x} attr={:#x} entry={:#x} arg={:#x} stk={:#x} -> {}", + log::warn!("[trace] sys_thread_create POSIX out={:#x} attr={:#x} entry={:#x} arg={:#x} stk={:#x} -> {}", arg0, arg1, arg2, arg3, stack_size, r); if let Some(child_pid) = child_pid_opt { log::warn!("[thread-create] spawned child={} entry={:#x}", child_pid, arg2); @@ -692,19 +698,30 @@ pub(crate) fn sys_thread_create(arg0: u64, arg1: u64, arg2: u64, arg3: u64) -> i // Give the newborn thread one immediate slice. Without this, the // creator can monopolize userspace and delay child startup long // enough to starve wake-source setup (pipe/timer arming). - let parent = crate::process::current_pid(); - if parent != 0 && child_pid != parent { - let my_cpu = crate::arch::smp::this_cpu().cpu_id; - if crate::process::try_claim_cpu_for(child_pid, my_cpu) { - let urip = crate::arch::syscall::user_rip(); - let ursp = crate::arch::syscall::user_rsp(); - crate::process::save_return_context(parent, urip, ursp); - crate::process::save_full_user_gprs(parent); - crate::process::set_rax(parent, 0); - crate::process::save_xstate(parent); - crate::process::enter_user_by_pid_noreturn(child_pid); + // x86 gave the newborn an immediate slice by saving the creator's + // context and entering the child here (never returning, delivering + // r=0 on the creator's later resume). On aarch64 this nested + // enter-while-inside-a-syscall re-entrancy corrupted the creator's + // resume (the 2nd fml::Thread's pthread_create returned non-zero → + // thread.cc:80 FML_CHECK abort). Let the child run via normal + // cooperative scheduling instead: return r=0 directly. + #[cfg(not(target_arch = "aarch64"))] + { + let parent = crate::process::current_pid(); + if parent != 0 && child_pid != parent { + let my_cpu = crate::arch::smp::this_cpu().cpu_id; + if crate::process::try_claim_cpu_for(child_pid, my_cpu) { + let urip = crate::arch::syscall::user_rip(); + let ursp = crate::arch::syscall::user_rsp(); + crate::process::save_return_context(parent, urip, ursp); + crate::process::save_full_user_gprs(parent); + crate::process::set_rax(parent, 0); + crate::process::save_xstate(parent); + crate::process::enter_user_by_pid_noreturn(child_pid); + } } } + let _ = child_pid; } r } else { diff --git a/kernel/src/syscall/handlers/ipc_display.rs b/kernel/src/syscall/handlers/ipc_display.rs index 7c889c8..6e3d572 100644 --- a/kernel/src/syscall/handlers/ipc_display.rs +++ b/kernel/src/syscall/handlers/ipc_display.rs @@ -612,7 +612,7 @@ pub(crate) fn sys_wm_event_wait(ev_ptr: u64, ev_len: u64, timeout_ms: u64) -> i6 // Save return context and registers so we can be resumed. let urip = crate::arch::syscall::user_rip(); let ursp = crate::arch::syscall::user_rsp(); - crate::process::save_return_context(cur, urip - 2, ursp); + crate::process::save_return_context_reexec(cur, urip, ursp); crate::process::save_full_user_gprs(cur); crate::process::set_rax(cur, eabi::SYS_WM_EVENT_WAIT); if block_log_n < 32 { diff --git a/kernel/src/syscall/poll.rs b/kernel/src/syscall/poll.rs index 71240c5..5d0ce3e 100644 --- a/kernel/src/syscall/poll.rs +++ b/kernel/src/syscall/poll.rs @@ -999,7 +999,7 @@ pub(crate) fn sys_epoll_wait_real(epfd: u64, events_out: u64, maxevents: u64, ti if crate::process::try_claim_cpu_for(target, my_cpu) { let urip = crate::arch::syscall::user_rip(); let ursp = crate::arch::syscall::user_rsp(); - crate::process::save_return_context(cur, urip.wrapping_sub(2), ursp); + crate::process::save_return_context_reexec(cur, urip, ursp); crate::process::save_full_user_gprs(cur); crate::process::set_rax(cur, 0x47B); // re-execute epoll_wait on resume crate::process::save_xstate(cur); @@ -1133,7 +1133,7 @@ pub(crate) fn sys_epoll_wait_real(epfd: u64, events_out: u64, maxevents: u64, ti if crate::process::try_claim_cpu_for(focus, my_cpu) { let urip = crate::arch::syscall::user_rip(); let ursp = crate::arch::syscall::user_rsp(); - crate::process::save_return_context(cur, urip.wrapping_sub(2), ursp); + crate::process::save_return_context_reexec(cur, urip, ursp); crate::process::save_full_user_gprs(cur); crate::process::set_rax(cur, 0x47B); // SYS epoll_wait (re-enter) crate::process::save_xstate(cur); @@ -1291,7 +1291,7 @@ pub(crate) fn sys_epoll_wait_real(epfd: u64, events_out: u64, maxevents: u64, ti // blocking again — exactly like sys_wm_event_wait does. let urip = crate::arch::syscall::user_rip(); let ursp = crate::arch::syscall::user_rsp(); - crate::process::save_return_context(cur, urip.wrapping_sub(2), ursp); + crate::process::save_return_context_reexec(cur, urip, ursp); crate::process::save_full_user_gprs(cur); crate::process::set_rax(cur, 0x47B); // SYS epoll_wait (re-enter) crate::process::save_xstate(cur); @@ -1317,7 +1317,7 @@ pub(crate) fn sys_epoll_wait_real(epfd: u64, events_out: u64, maxevents: u64, ti crate::process::set_state(1, crate::process::ProcState::Running); let urip = crate::arch::syscall::user_rip(); let ursp = crate::arch::syscall::user_rsp(); - crate::process::save_return_context(cur, urip.wrapping_sub(2), ursp); + crate::process::save_return_context_reexec(cur, urip, ursp); crate::process::save_full_user_gprs(cur); crate::process::set_rax(cur, 0x47B); crate::process::save_xstate(cur); diff --git a/kernel/src/syscall/posix.rs b/kernel/src/syscall/posix.rs index bff75c0..7c68f7a 100644 --- a/kernel/src/syscall/posix.rs +++ b/kernel/src/syscall/posix.rs @@ -952,7 +952,7 @@ pub fn sys_pthread_mutex_lock(mutex: u64, sys_nr: u64) -> i64 { if crate::process::try_claim_cpu_for(owner_candidate, my_cpu) { let urip = crate::arch::syscall::user_rip(); let ursp = crate::arch::syscall::user_rsp(); - crate::process::save_return_context(pid, urip - 2, ursp); + crate::process::save_return_context_reexec(pid, urip, ursp); crate::process::save_full_user_gprs(pid); crate::process::set_rax(pid, sys_nr); crate::process::save_xstate(pid); @@ -994,7 +994,7 @@ pub fn sys_pthread_mutex_lock(mutex: u64, sys_nr: u64) -> i64 { let urip = crate::arch::syscall::user_rip(); let ursp = crate::arch::syscall::user_rsp(); // Save context pointing to the syscall instruction itself so we retry on re-entry. - crate::process::save_return_context(pid, urip - 2, ursp); + crate::process::save_return_context_reexec(pid, urip, ursp); crate::process::save_full_user_gprs(pid); crate::process::set_rax(pid, sys_nr); // Restore the original syscall number crate::process::save_xstate(pid); @@ -1026,7 +1026,7 @@ pub fn sys_pthread_mutex_lock(mutex: u64, sys_nr: u64) -> i64 { if next != pid { let urip = crate::arch::syscall::user_rip(); let ursp = crate::arch::syscall::user_rsp(); - crate::process::save_return_context(pid, urip - 2, ursp); + crate::process::save_return_context_reexec(pid, urip, ursp); crate::process::save_full_user_gprs(pid); crate::process::set_rax(pid, sys_nr); crate::process::save_xstate(pid); @@ -1173,7 +1173,7 @@ pub fn sys_pthread_once(once: u64, func: u64, sys_nr: u64) -> i64 { let urip = crate::arch::syscall::user_rip(); let ursp = crate::arch::syscall::user_rsp(); // Save context pointing to the syscall instruction itself so we retry on re-entry. - crate::process::save_return_context(pid, urip - 2, ursp); + crate::process::save_return_context_reexec(pid, urip, ursp); crate::process::save_full_user_gprs(pid); crate::process::set_rax(pid, sys_nr); // sys_pthread_once syscall number crate::process::save_xstate(pid); @@ -1192,7 +1192,7 @@ pub fn sys_pthread_once(once: u64, func: u64, sys_nr: u64) -> i64 { if next != pid { let urip = crate::arch::syscall::user_rip(); let ursp = crate::arch::syscall::user_rsp(); - crate::process::save_return_context(pid, urip - 2, ursp); + crate::process::save_return_context_reexec(pid, urip, ursp); crate::process::save_full_user_gprs(pid); crate::process::set_rax(pid, sys_nr); crate::process::save_xstate(pid); @@ -1364,7 +1364,7 @@ pub fn sys_pthread_cond_wait_timeout(cond: u64, mutex: u64, timeout_ns: u64, sys if next != pid { let urip = crate::arch::syscall::user_rip(); let ursp = crate::arch::syscall::user_rsp(); - crate::process::save_return_context(pid, urip - 2, ursp); + crate::process::save_return_context_reexec(pid, urip, ursp); crate::process::save_full_user_gprs(pid); crate::process::set_rax(pid, sys_nr); // Restore the original syscall number crate::process::save_xstate(pid); @@ -1400,7 +1400,7 @@ pub fn sys_pthread_cond_wait_timeout(cond: u64, mutex: u64, timeout_ns: u64, sys if next != pid { let urip = crate::arch::syscall::user_rip(); let ursp = crate::arch::syscall::user_rsp(); - crate::process::save_return_context(pid, urip - 2, ursp); + crate::process::save_return_context_reexec(pid, urip, ursp); crate::process::save_full_user_gprs(pid); crate::process::set_rax(pid, sys_nr); crate::process::save_xstate(pid); diff --git a/tools/flutter-embedder/src/main.rs b/tools/flutter-embedder/src/main.rs index 44c3528..7c53a94 100644 --- a/tools/flutter-embedder/src/main.rs +++ b/tools/flutter-embedder/src/main.rs @@ -1345,10 +1345,16 @@ extern "C" fn main_embedder(host_mode: u64, app_id: u64, aot_va: u64) { // layer (livelock). Strategy: make the heap large enough that NO GC fires during // the first frame — no safepoint, no livelock. (Do NOT use --marker_tasks=0 / // --scavenger_tasks=0: the debug VM asserts num_tasks>0 and SIGABRTs.) - static ARG8: &[u8] = - b"--dart-flags=--old_gen_heap_size=512,--new_gen_semi_max_size=64,--no_background_compilation\0"; + // NOTE: we deliberately do NOT pass a `--dart-flags=...` switch. The engine's + // SettingsFromCommandLine (shell/common/switches.cc) runs every comma-split + // entry through IsAllowedDartVMFlag against a tiny allowlist and FML_LOG(FATAL)s + // on anything else — `--old_gen_heap_size` / `--new_gen_semi_max_size` / + // `--no_background_compilation` are NOT on that list, so passing them via + // --dart-flags aborts the engine (switches.cc:478). The original GC-heap sizing + // was a workaround for an x86-TCG/JIT livelock that the later diagnosis traced + // to font fallback + slow JIT; under AOT on ARM we let the VM use its defaults. #[repr(transparent)] - struct ArgvPtrs([*const u8; 9]); + struct ArgvPtrs([*const u8; 8]); unsafe impl Sync for ArgvPtrs {} static ENGINE_ARGV: ArgvPtrs = ArgvPtrs([ @@ -1360,7 +1366,6 @@ extern "C" fn main_embedder(host_mode: u64, app_id: u64, aot_va: u64) { ARG5.as_ptr(), ARG6.as_ptr(), ARG7.as_ptr(), - ARG8.as_ptr(), ]); let mut project_args = FlutterProjectArgsRaw { From e6b7d58b1abcf645a31f9cc32c122d515a0eb76b Mon Sep 17 00:00:00 2001 From: Tahiru Agbanwa <130235676+squirelboy360@users.noreply.github.com> Date: Tue, 9 Jun 2026 18:58:42 +0200 Subject: [PATCH 27/36] aarch64: harden FB text console against geometry overflow; keep nested thread-enter off MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Make blit_char/scroll_up/write_byte bail cleanly on a degenerate or corrupted FB geometry (rows/cols == 0) instead of panicking the kernel with arithmetic overflow — the serial-mirror console is never worth a panic. With this the engine spawns its full UI/raster/IO thread set (tid=2/3/4) without the kernel dying. Document why the x86 immediate-child-enter slice stays disabled on arm (it corrupts the creator's cooperative resume → thread.cc:80 abort). --- kernel/src/drivers/fb.rs | 15 ++++++++++++++- kernel/src/syscall/handlers/futex.rs | 19 +++++++++---------- 2 files changed, 23 insertions(+), 11 deletions(-) diff --git a/kernel/src/drivers/fb.rs b/kernel/src/drivers/fb.rs index 260071f..489ba46 100644 --- a/kernel/src/drivers/fb.rs +++ b/kernel/src/drivers/fb.rs @@ -468,6 +468,12 @@ pub fn swap_buffers() { // ── Internal helpers ────────────────────────────────────────────────────────── fn write_byte(b: u8) { + // The text console is a best-effort serial mirror. If the FB geometry is + // unpublished or degenerate (e.g. a corrupted FB_ROWS/FB_COLS static), do + // nothing rather than risk an arithmetic-overflow panic in the kernel. + if FB_ROWS.load(Ordering::Relaxed) == 0 || FB_COLS.load(Ordering::Relaxed) == 0 { + return; + } match b { b'\r' => { CURSOR.lock().0 = 0; @@ -477,7 +483,7 @@ fn write_byte(b: u8) { cur.0 = 0; cur.1 += 1; let rows = FB_ROWS.load(Ordering::Relaxed); - if cur.1 >= rows { + if rows > 0 && cur.1 >= rows { drop(cur); scroll_up(); CURSOR.lock().1 = rows - 1; @@ -560,6 +566,13 @@ fn scroll_up() { let height = FB_HEIGHT .load(Ordering::Relaxed); let rows = FB_ROWS .load(Ordering::Relaxed); + // Guard against an unpublished/degenerate FB geometry: the serial-mirror + // console must never panic the kernel (subtract-with-overflow on height or + // rows). Bail out cleanly if there is nothing safe to scroll. + if addr == 0 || height < CHAR_H || rows == 0 || pitch_px == 0 { + return; + } + // Number of pixel rows to copy. let copy_rows = height - CHAR_H; let words_per_row = pitch_px as usize; diff --git a/kernel/src/syscall/handlers/futex.rs b/kernel/src/syscall/handlers/futex.rs index 60b5fd7..26cd412 100644 --- a/kernel/src/syscall/handlers/futex.rs +++ b/kernel/src/syscall/handlers/futex.rs @@ -695,16 +695,15 @@ pub(crate) fn sys_thread_create(arg0: u64, arg1: u64, arg2: u64, arg3: u64) -> i if let Some(child_pid) = child_pid_opt { log::warn!("[thread-create] spawned child={} entry={:#x}", child_pid, arg2); - // Give the newborn thread one immediate slice. Without this, the - // creator can monopolize userspace and delay child startup long - // enough to starve wake-source setup (pipe/timer arming). - // x86 gave the newborn an immediate slice by saving the creator's - // context and entering the child here (never returning, delivering - // r=0 on the creator's later resume). On aarch64 this nested - // enter-while-inside-a-syscall re-entrancy corrupted the creator's - // resume (the 2nd fml::Thread's pthread_create returned non-zero → - // thread.cc:80 FML_CHECK abort). Let the child run via normal - // cooperative scheduling instead: return r=0 directly. + // x86 gave the newborn an immediate slice by entering the child + // from inside the creator's pthread_create syscall (never returning; + // delivering r=0 on the creator's later cooperative resume). On + // aarch64 that nested enter-while-in-a-syscall corrupts the creator's + // resume so the *next* fml::Thread's pthread_create returns non-zero + // (thread.cc:80 FML_CHECK abort) — reproducible even with the x30/LR + // and 4-byte-svc-rewind fixes in place. Keep it OFF on aarch64: the + // child becomes runnable here and is picked up by normal cooperative + // scheduling / timer preemption the next time the creator yields. #[cfg(not(target_arch = "aarch64"))] { let parent = crate::process::current_pid(); From d6d40451a3ebda37c422c571584d152ada0d35ec Mon Sep 17 00:00:00 2001 From: Tahiru Agbanwa <130235676+squirelboy360@users.noreply.github.com> Date: Tue, 9 Jun 2026 20:20:53 +0200 Subject: [PATCH 28/36] wip(aarch64): timer-ISR timerfd wakes + pending-wake delivery; cpu-id robustness for single-core ISR; re-enable scheduler tick for engine host; preempt diagnostics --- kernel/src/arch/aarch64/apic.rs | 35 ++++++++++++++++++++++-- kernel/src/arch/aarch64/smp.rs | 11 ++++++-- kernel/src/main.rs | 13 ++++----- kernel/src/process/mod.rs | 48 ++++++++++++++++++++++++++++----- 4 files changed, 91 insertions(+), 16 deletions(-) diff --git a/kernel/src/arch/aarch64/apic.rs b/kernel/src/arch/aarch64/apic.rs index de8cae8..6137682 100644 --- a/kernel/src/arch/aarch64/apic.rs +++ b/kernel/src/arch/aarch64/apic.rs @@ -81,6 +81,16 @@ fn production_irq_handler(f: &mut super::vectors::TrapFrame) { // EOI now so the GIC can deliver the next timer interrupt right after eret. super::gic::eoi(intid); + // Wake threads whose timerfd deadlines have elapsed, and deliver any pending + // cross-thread wakes — exactly as the x86 APIC-timer ISR does (idt.rs). On + // aarch64 this was missing, so the engine's UI/raster/IO worker threads that + // epoll_wait on a timerfd went Blocked and were NEVER transitioned back to + // Running → the cooperative scheduler found no runnable thread and the whole + // engine deadlocked right after spawning its worker pool. The try_-variants + // are ISR-safe (skip on lock contention rather than deadlock). + crate::syscall::check_timerfds_and_wake_try(); + crate::process::handle_pending_wakes_try(); + // Only preempt interrupts taken FROM EL0 (a running user thread). SPSR_EL1 // M[3:0] == 0 means the interrupted context was EL0t. Kernel-mode ticks // (idle/cortex loop) just return — there is no user frame to switch. @@ -105,6 +115,16 @@ fn production_irq_handler(f: &mut super::vectors::TrapFrame) { && cur != target && (crate::wm::input_pending_for(target) > 0 || (target == 1 && crate::wm::embedder_baton_due()))); + { + static PREEMPT_TICK_LOG: AtomicU32 = AtomicU32::new(0); + let n = PREEMPT_TICK_LOG.fetch_add(1, Ordering::Relaxed); + if n < 80 && (n % 8 == 0 || should_preempt) { + log::warn!( + "[arm-preempt-tick] #{} cur={} slice_expired={} should={} focus={}", + n, cur, slice_expired, should_preempt, focus + ); + } + } if !should_preempt { return; } @@ -122,8 +142,19 @@ fn production_irq_handler(f: &mut super::vectors::TrapFrame) { return; } - if let Some((next_pid, next_regs, ttbr0)) = - crate::process::timer_preempt_switch_try(cur, &cur_regs) + let switch = crate::process::timer_preempt_switch_try(cur, &cur_regs); + { + static SWITCH_LOG: AtomicU32 = AtomicU32::new(0); + let n = SWITCH_LOG.fetch_add(1, Ordering::Relaxed); + if n < 60 { + log::warn!( + "[arm-preempt-switch] #{} cur={} -> next={:?} states={}", + n, cur, switch.as_ref().map(|s| s.0), + crate::process::debug_runnable_states() + ); + } + } + if let Some((next_pid, next_regs, ttbr0)) = switch { // Switch the low-half translation base to the next thread's space. crate::arch::memory::write_cr3(ttbr0); diff --git a/kernel/src/arch/aarch64/smp.rs b/kernel/src/arch/aarch64/smp.rs index d758d56..70ab1be 100644 --- a/kernel/src/arch/aarch64/smp.rs +++ b/kernel/src/arch/aarch64/smp.rs @@ -44,11 +44,18 @@ fn read_mpidr() -> u64 { /// Return this CPU's ID (by matching MPIDR_EL1 against the per-CPU table). pub fn current_cpu_id() -> u32 { - let mpidr = read_mpidr(); let n = CPU_COUNT.load(Ordering::Relaxed) as usize; + // Single-core fast path: the BSP is always CPU 0. This also makes the timer + // ISR's per-CPU lookup robust against a transiently-inconsistent PER_CPU_DATA + // read (the lookup result indexes PTABLE_LOCK_RECURSION/PER_CPU_DATA, and a + // garbage index would panic the kernel from inside the IRQ handler). + if n <= 1 { + return 0; + } + let mpidr = read_mpidr(); unsafe { let table = &*core::ptr::addr_of!(PER_CPU_DATA); - for i in 0..n { + for i in 0..n.min(MAX_CPUS) { if table[i].mpidr == mpidr { return i as u32; } diff --git a/kernel/src/main.rs b/kernel/src/main.rs index 5e794f6..cb73e3f 100644 --- a/kernel/src/main.rs +++ b/kernel/src/main.rs @@ -385,12 +385,13 @@ fn shared_init_and_run() -> ! { } // Arm the generic-timer scheduler tick now (kernel fully // initialised) so preemption begins the moment we drop to EL0. - // DIAGNOSTIC: skip arming for the engine host to isolate a - // timer-IRQ-during-eret interaction; the host drives its own - // frame pump cooperatively via sched_yield. - if !engine_present { - arch::aarch64::apic::start_scheduler_tick(); - } + // The engine NEEDS preemption: pid 1 busy-spins in EL0 after + // creating its UI/raster/IO worker threads (it waits on a + // userspace flag the workers set), and with the nested + // immediate-child-enter disabled on arm those workers only + // get the CPU when a timer tick preempts pid 1. + let _ = engine_present; + arch::aarch64::apic::start_scheduler_tick(); process::enter_user_by_pid_noreturn(pid); } } diff --git a/kernel/src/process/mod.rs b/kernel/src/process/mod.rs index d36f9c9..5054872 100644 --- a/kernel/src/process/mod.rs +++ b/kernel/src/process/mod.rs @@ -342,6 +342,18 @@ pub struct PTableLock { holder: AtomicU32, } +/// CPU index used to key PTABLE_LOCK_RECURSION / the holder atomic. Always a +/// valid in-range index. We use the computed `current_cpu_id()` (which on +/// aarch64 short-circuits to 0 for single-core) rather than reading the +/// `PerCpuData.cpu_id` *field*, and hard-clamp to the recursion-array bound so +/// the timer ISR can never index out of bounds and panic the kernel even if a +/// per-CPU struct read is momentarily inconsistent. +#[inline(always)] +fn ptable_cpu_idx() -> u32 { + let id = crate::arch::smp::current_cpu_id(); + if (id as usize) < crate::arch::smp::MAX_CPUS { id } else { 0 } +} + impl PTableLock { pub const fn new() -> Self { Self { @@ -351,7 +363,7 @@ impl PTableLock { } pub fn lock(&self) -> PTableGuard { - let my_cpu = crate::arch::smp::this_cpu().cpu_id; + let my_cpu = ptable_cpu_idx(); if self.holder.load(Ordering::Acquire) == my_cpu { unsafe { PTABLE_LOCK_RECURSION[my_cpu as usize] += 1; @@ -369,7 +381,7 @@ impl PTableLock { } pub fn try_lock(&self) -> Option { - let my_cpu = crate::arch::smp::this_cpu().cpu_id; + let my_cpu = ptable_cpu_idx(); if self.holder.load(Ordering::Acquire) == my_cpu { unsafe { PTABLE_LOCK_RECURSION[my_cpu as usize] += 1; @@ -393,14 +405,14 @@ impl PTableLock { impl Drop for PTableGuard { fn drop(&mut self) { if self.is_outer { - let my_cpu = crate::arch::smp::this_cpu().cpu_id; + let my_cpu = ptable_cpu_idx(); unsafe { PTABLE_LOCK_RECURSION[my_cpu as usize] = 0; PTABLE_LOCK.holder.store(0xFFFF_FFFF, Ordering::Release); PTABLE_LOCK_GUARD = None; } } else { - let my_cpu = crate::arch::smp::this_cpu().cpu_id; + let my_cpu = ptable_cpu_idx(); unsafe { if PTABLE_LOCK_RECURSION[my_cpu as usize] > 0 { PTABLE_LOCK_RECURSION[my_cpu as usize] -= 1; @@ -950,6 +962,30 @@ pub fn sibling_pids(current: u32) -> alloc::vec::Vec { out } +/// Compact debug string of pid/state/current_cpu for the first few PIDs, for +/// diagnosing why the timer-preempt scheduler can't find a runnable thread. +pub fn debug_runnable_states() -> alloc::string::String { + use core::fmt::Write; + let mut s = alloc::string::String::new(); + if let Some(_g) = PTABLE_LOCK.try_lock() { + for pid in 1u32..=6 { + let p = unsafe { &PTABLE[idx_of(pid)] }; + if p.pid == pid { + let st = match p.state { + ProcState::Running => 'R', + ProcState::Blocked => 'B', + ProcState::Zombie(_) => 'Z', + ProcState::Dead => 'D', + }; + let _ = write!(s, "{}:{}{:?} ", pid, st, p.current_cpu); + } + } + } else { + s.push_str("(locked)"); + } + s +} + pub fn next_runnable_pid_locked(current: u32, my_cpu: u32) -> Option { let focus = crate::wm::focus_pid(); let input_target = if focus != 0 { focus } else { 1 }; @@ -1527,7 +1563,7 @@ fn user_launch_task() { /// /// Returns `None` if there is no other runnable thread (no switch occurs). pub fn timer_preempt_switch(cur_pid: u32, cur_regs: &UserRegs) -> Option<(u32, UserRegs, u64)> { - let my_cpu = crate::arch::smp::this_cpu().cpu_id; + let my_cpu = crate::arch::smp::current_cpu_id(); let fs = crate::arch::cpu::get_fs_base(); let _g = PTABLE_LOCK.lock(); @@ -1586,7 +1622,7 @@ pub fn timer_preempt_switch(cur_pid: u32, cur_regs: &UserRegs) -> Option<(u32, U /// Try-lock variant of `timer_preempt_switch` to avoid ISR deadlocks. pub fn timer_preempt_switch_try(cur_pid: u32, cur_regs: &UserRegs) -> Option<(u32, UserRegs, u64)> { - let my_cpu = crate::arch::smp::this_cpu().cpu_id; + let my_cpu = crate::arch::smp::current_cpu_id(); let fs = crate::arch::cpu::get_fs_base(); if let Some(_g) = PTABLE_LOCK.try_lock() { From fd120d70113ce6425a5694be1cfb003e2d303343 Mon Sep 17 00:00:00 2001 From: Tahiru Agbanwa <130235676+squirelboy360@users.noreply.github.com> Date: Tue, 9 Jun 2026 20:25:53 +0200 Subject: [PATCH 29/36] wip(aarch64): walk EL0 FP chain in unhandled-exception report to symbolise null-jump crash --- kernel/src/arch/aarch64/vectors.rs | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/kernel/src/arch/aarch64/vectors.rs b/kernel/src/arch/aarch64/vectors.rs index f45d158..af3e5a9 100644 --- a/kernel/src/arch/aarch64/vectors.rs +++ b/kernel/src/arch/aarch64/vectors.rs @@ -431,6 +431,32 @@ fn report_unhandled(f: &TrapFrame, kind: u64, ec: u64) -> ! { uart::puts(" pid="); uart::puthex(crate::process::current_pid() as u64); uart::puts("\n"); + + // Frame-pointer chain walk: with ELR=0 and x30=0 the immediate caller is lost + // from the registers, but the AArch64 AAPCS frame record ([x29]=saved FP, + // [x29+8]=saved LR) lets us recover the call stack from the EL0 stack. TTBR0 + // is still the faulting thread's during this EL1 handler, so EL0 VAs read + // directly. Print the saved-LR chain so the crash can be symbolised. + uart::puts(" x29(FP)="); + uart::puthex_full(f.x[29]); + uart::puts("\n bt:"); + { + let mut fp = f.x[29]; + let mut depth = 0u32; + // Heuristic validity check for an EL0 user VA (low half, non-tiny). + let plausible = |a: u64| a >= 0x1000 && a < 0x0000_8000_0000_0000; + while depth < 16 && plausible(fp) && (fp & 0x7) == 0 { + let saved_fp = unsafe { core::ptr::read_volatile(fp as *const u64) }; + let saved_lr = unsafe { core::ptr::read_volatile((fp + 8) as *const u64) }; + uart::puts(" "); + uart::puthex_full(saved_lr); + // FP must climb the stack monotonically; stop on any inconsistency. + if saved_fp <= fp { break; } + fp = saved_fp; + depth += 1; + } + } + uart::puts("\n"); loop { unsafe { core::arch::asm!("wfe", options(nomem, nostack)) }; } From 6d680d39f691ddee6f77ab1e9c86fbf64bcbbdcc Mon Sep 17 00:00:00 2001 From: Tahiru Agbanwa <130235676+squirelboy360@users.noreply.github.com> Date: Tue, 9 Jun 2026 20:33:33 +0200 Subject: [PATCH 30/36] =?UTF-8?q?wip(aarch64):=20expand=20scheduler=20diag?= =?UTF-8?q?nostics=20=E2=80=94=20heartbeat=20ticks=20with=20full=20thread?= =?UTF-8?q?=20states,=20scan=20tids=201-12,=20raise=20log=20caps?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kernel/src/arch/aarch64/apic.rs | 9 +++++---- kernel/src/process/mod.rs | 5 +++-- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/kernel/src/arch/aarch64/apic.rs b/kernel/src/arch/aarch64/apic.rs index 6137682..5a7bbe5 100644 --- a/kernel/src/arch/aarch64/apic.rs +++ b/kernel/src/arch/aarch64/apic.rs @@ -118,10 +118,11 @@ fn production_irq_handler(f: &mut super::vectors::TrapFrame) { { static PREEMPT_TICK_LOG: AtomicU32 = AtomicU32::new(0); let n = PREEMPT_TICK_LOG.fetch_add(1, Ordering::Relaxed); - if n < 80 && (n % 8 == 0 || should_preempt) { + if n < 4000 && (n % 64 == 0 || should_preempt) { log::warn!( - "[arm-preempt-tick] #{} cur={} slice_expired={} should={} focus={}", - n, cur, slice_expired, should_preempt, focus + "[arm-preempt-tick] #{} cur={} slice_expired={} should={} focus={} states={}", + n, cur, slice_expired, should_preempt, focus, + crate::process::debug_runnable_states() ); } } @@ -146,7 +147,7 @@ fn production_irq_handler(f: &mut super::vectors::TrapFrame) { { static SWITCH_LOG: AtomicU32 = AtomicU32::new(0); let n = SWITCH_LOG.fetch_add(1, Ordering::Relaxed); - if n < 60 { + if n < 400 { log::warn!( "[arm-preempt-switch] #{} cur={} -> next={:?} states={}", n, cur, switch.as_ref().map(|s| s.0), diff --git a/kernel/src/process/mod.rs b/kernel/src/process/mod.rs index 5054872..b4c158a 100644 --- a/kernel/src/process/mod.rs +++ b/kernel/src/process/mod.rs @@ -968,7 +968,7 @@ pub fn debug_runnable_states() -> alloc::string::String { use core::fmt::Write; let mut s = alloc::string::String::new(); if let Some(_g) = PTABLE_LOCK.try_lock() { - for pid in 1u32..=6 { + for pid in 1u32..=12 { let p = unsafe { &PTABLE[idx_of(pid)] }; if p.pid == pid { let st = match p.state { @@ -977,7 +977,8 @@ pub fn debug_runnable_states() -> alloc::string::String { ProcState::Zombie(_) => 'Z', ProcState::Dead => 'D', }; - let _ = write!(s, "{}:{}{:?} ", pid, st, p.current_cpu); + let cpu = match p.current_cpu { Some(c) => c as i32, None => -1 }; + let _ = write!(s, "{}:{}c{} ", pid, st, cpu); } } } else { From de9f8c0c4503d349f766c048d6381e20c99b5522 Mon Sep 17 00:00:00 2001 From: Tahiru Agbanwa <130235676+squirelboy360@users.noreply.github.com> Date: Tue, 9 Jun 2026 20:36:11 +0200 Subject: [PATCH 31/36] wip(aarch64): dump raw PTABLE slot pid/state for idx 1-12 to find the vanished worker thread --- kernel/src/process/mod.rs | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/kernel/src/process/mod.rs b/kernel/src/process/mod.rs index b4c158a..9cd544c 100644 --- a/kernel/src/process/mod.rs +++ b/kernel/src/process/mod.rs @@ -968,18 +968,22 @@ pub fn debug_runnable_states() -> alloc::string::String { use core::fmt::Write; let mut s = alloc::string::String::new(); if let Some(_g) = PTABLE_LOCK.try_lock() { - for pid in 1u32..=12 { - let p = unsafe { &PTABLE[idx_of(pid)] }; - if p.pid == pid { - let st = match p.state { - ProcState::Running => 'R', - ProcState::Blocked => 'B', - ProcState::Zombie(_) => 'Z', - ProcState::Dead => 'D', - }; - let cpu = match p.current_cpu { Some(c) => c as i32, None => -1 }; - let _ = write!(s, "{}:{}c{} ", pid, st, cpu); - } + // Print the RAW slot contents for indices 1..=12 (the slot's own .pid + // field, not the loop var) so a slot whose .pid != index reveals a + // reused/cleared entry — the mechanism behind a worker thread that runs + // yet never appears under its expected index. + for idx in 1usize..=12 { + let p = unsafe { &PTABLE[idx] }; + if p.pid == 0 { continue; } + let st = match p.state { + ProcState::Running => 'R', + ProcState::Blocked => 'B', + ProcState::Zombie(_) => 'Z', + ProcState::Dead => 'D', + }; + let cpu = match p.current_cpu { Some(c) => c as i32, None => -1 }; + // slot[idx]=pid:state c cpu + let _ = write!(s, "s{}={}{}{} ", idx, p.pid, st, cpu); } } else { s.push_str("(locked)"); From a9d6fe029725b4b143777395a1702b0846d3c752 Mon Sep 17 00:00:00 2001 From: Tahiru Agbanwa <130235676+squirelboy360@users.noreply.github.com> Date: Tue, 9 Jun 2026 20:40:43 +0200 Subject: [PATCH 32/36] aarch64: unmask IRQs during SVC syscall handling so the generic-timer scheduler tick can fire mid-syscall (mirrors x86 sti-on-entry); fixes engine bring-up deadlock where a spinning/cooperatively-yielding syscall masked the timer and froze preemption --- kernel/src/arch/aarch64/vectors.rs | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/kernel/src/arch/aarch64/vectors.rs b/kernel/src/arch/aarch64/vectors.rs index af3e5a9..8d20b48 100644 --- a/kernel/src/arch/aarch64/vectors.rs +++ b/kernel/src/arch/aarch64/vectors.rs @@ -306,8 +306,27 @@ extern "C" fn dispatch(frame: *mut TrapFrame, kind: u64) { if ec == EC_SVC64 { let h = SVC_HANDLER.load(Ordering::Acquire); if h != 0 { + // Unmask IRQs for the duration of the syscall, mirroring x86 + // where the SYSCALL entry stub runs handlers with interrupts + // ENABLED. Exception entry masked DAIF.I; without re-enabling + // it the generic-timer scheduler tick cannot fire while a + // syscall runs. The Flutter engine's cooperative bring-up + // makes long, spinning syscalls (epoll_wait bootstrap-spin, + // condvar waits) that hand off between sibling threads WITHIN + // the syscall — if the timer is masked the whole time, a + // thread that spins waiting for a sibling can never be + // preempted and the engine deadlocks with the timer frozen. + // The timer ISR uses try-locks throughout (ISR-safe), so + // taking a tick mid-syscall is safe. Re-mask on the way out + // so the vector restore/eret sees the same DAIF it expects. + unsafe { + core::arch::asm!("msr daifclr, #0b0010", options(nomem, nostack)); + } let func: SvcFn = unsafe { core::mem::transmute(h as usize) }; func(f); + unsafe { + core::arch::asm!("msr daifset, #0b0010", options(nomem, nostack)); + } } else { report_unhandled(f, kind, ec); } From bef4942211e2e3a7f840251155ecdae3e71c8635 Mon Sep 17 00:00:00 2001 From: Tahiru Agbanwa <130235676+squirelboy360@users.noreply.github.com> Date: Tue, 9 Jun 2026 20:45:19 +0200 Subject: [PATCH 33/36] =?UTF-8?q?aarch64:=20make=20timer-ISR=20cond-expiry?= =?UTF-8?q?=20use=20try-lock=20for=20FUTEX=5FWAITERS=20(futex=5Fwaiter=5Fr?= =?UTF-8?q?emove=5Ftry)=20=E2=80=94=20fixes=20single-core=20IRQ-masked=20s?= =?UTF-8?q?elf-deadlock=20where=20a=20timer=20tick=20during=20spawn=5Fthre?= =?UTF-8?q?ad/futex=20syscall=20spun=20forever=20on=20a=20held=20lock?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kernel/src/syscall/mod.rs | 2 +- kernel/src/syscall/poll.rs | 6 +++++- kernel/src/syscall/wait.rs | 20 ++++++++++++++++++++ 3 files changed, 26 insertions(+), 2 deletions(-) diff --git a/kernel/src/syscall/mod.rs b/kernel/src/syscall/mod.rs index 55fef1d..c56d019 100644 --- a/kernel/src/syscall/mod.rs +++ b/kernel/src/syscall/mod.rs @@ -109,4 +109,4 @@ pub(crate) use state::{ FS_BOOTSTRAP_LOGGED, WM_WAITER_DEADLINE, WM_WAITER_PID, }; pub(crate) use trace::{POSTEXIT_TRACE_ACTIVE, POSTEXIT_TRACE_COUNT, POSTEXIT_TRACE_LIMIT}; -pub(crate) use wait::{futex_waiter_add, futex_waiter_for, futex_waiter_present, futex_waiter_remove}; +pub(crate) use wait::{futex_waiter_add, futex_waiter_for, futex_waiter_present, futex_waiter_remove, futex_waiter_remove_try}; diff --git a/kernel/src/syscall/poll.rs b/kernel/src/syscall/poll.rs index 5d0ce3e..874a6b4 100644 --- a/kernel/src/syscall/poll.rs +++ b/kernel/src/syscall/poll.rs @@ -551,7 +551,11 @@ pub fn check_timerfds_and_wake_try() { exps }; for (pid, mutex, cond) in expired { - super::futex_waiter_remove(cond, pid); + // ISR-safe: never block on FUTEX_WAITERS here. An interrupted syscall + // (e.g. spawn_thread, a futex op) may hold it; a non-try lock would + // self-deadlock this single core with IRQs masked. On contention the + // removal is simply retried on the next timer tick. + super::futex_waiter_remove_try(cond, pid); static COND_EXPIRE_LOG: AtomicU32 = AtomicU32::new(0); let n = COND_EXPIRE_LOG.fetch_add(1, Ordering::Relaxed); if n < 32 { diff --git a/kernel/src/syscall/wait.rs b/kernel/src/syscall/wait.rs index b84b446..8a8baa3 100644 --- a/kernel/src/syscall/wait.rs +++ b/kernel/src/syscall/wait.rs @@ -19,6 +19,26 @@ pub(crate) fn futex_waiter_remove(addr: u64, pid: u32) { } } +/// ISR-safe variant of [`futex_waiter_remove`]. The generic-timer ISR drives +/// cond-wait timeout expiry (poll::check_timerfds_and_wake_try) and must never +/// block on a lock an interrupted syscall already holds — on a single core that +/// is an instant, IRQ-masked self-deadlock. Returns `false` on contention so the +/// caller can retry on the next tick rather than spin. +pub(crate) fn futex_waiter_remove_try(addr: u64, pid: u32) -> bool { + match FUTEX_WAITERS.try_lock() { + Some(mut table) => { + if let Some(waiters) = table.get_mut(&addr) { + waiters.retain(|&waiter| waiter != pid); + if waiters.is_empty() { + table.remove(&addr); + } + } + true + } + None => false, + } +} + pub(crate) fn futex_waiter_present(addr: u64, pid: u32) -> bool { let table = FUTEX_WAITERS.lock(); table.get(&addr).map_or(false, |waiters| waiters.contains(&pid)) From 57f3023650a185368f668137bb07f7003af16372 Mon Sep 17 00:00:00 2001 From: Tahiru Agbanwa <130235676+squirelboy360@users.noreply.github.com> Date: Tue, 9 Jun 2026 20:51:42 +0200 Subject: [PATCH 34/36] =?UTF-8?q?mm:=20make=20PAGE=5FTABLE=5FLOCK=20reentr?= =?UTF-8?q?ant=20on=20a=20single=20core=20(lock=5Fpage=5Ftable)=20?= =?UTF-8?q?=E2=80=94=20fixes=20demand-pager=20self-deadlock=20when=20a=20f?= =?UTF-8?q?ault=20re-enters=20map=5Fuser=5Fpage=20while=20the=20page-table?= =?UTF-8?q?=20lock=20is=20held=20(exposed=20by=20aarch64=20IRQs-on-during-?= =?UTF-8?q?syscall)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kernel/src/mm/paging.rs | 67 +++++++++++++++++++++++++++++++++++------ 1 file changed, 58 insertions(+), 9 deletions(-) diff --git a/kernel/src/mm/paging.rs b/kernel/src/mm/paging.rs index 86d4ddf..e248eee 100644 --- a/kernel/src/mm/paging.rs +++ b/kernel/src/mm/paging.rs @@ -9,6 +9,55 @@ use crate::mm::frame_allocator; pub(crate) static PAGE_TABLE_LOCK: Mutex<()> = Mutex::new(()); +/// Re-entrancy depth for the page-table critical section on this (single) core. +/// +/// The page-fault / demand-pager path can be entered *while the page-table lock +/// is already held by an outer map/translate on the same core*: on aarch64 the +/// SVC handler now runs with IRQs unmasked (so a syscall can be preempted/woken +/// like on x86), and a fault taken during a page-table operation re-enters +/// `demand_page` → `map_user_page_with_flags`. A plain spin `Mutex` is NOT +/// re-entrant, so the nested acquire spins forever against the held lock with +/// IRQs masked (in the abort handler) — an instant single-core self-deadlock +/// (observed freezing the engine's 3rd worker-thread stack setup). +/// +/// `lock_page_table()` makes the section re-entrant on a single core: the outer +/// acquire takes the real spin lock; nested acquires just bump a depth counter +/// and hand back a guard that releases nothing until depth returns to zero. This +/// is sound because all writers run on one core (the scheduler is cooperative / +/// single-CPU) — there is no second core to race the page-table structure. +static PAGE_TABLE_LOCK_DEPTH: core::sync::atomic::AtomicU32 = + core::sync::atomic::AtomicU32::new(0); + +pub(crate) struct PageTableGuard { + outer: Option>, +} + +impl Drop for PageTableGuard { + fn drop(&mut self) { + if self.outer.is_some() { + // Outermost guard: clear depth, then release the real lock. + PAGE_TABLE_LOCK_DEPTH.store(0, core::sync::atomic::Ordering::Release); + // `outer` drops here, releasing PAGE_TABLE_LOCK. + } else { + PAGE_TABLE_LOCK_DEPTH.fetch_sub(1, core::sync::atomic::Ordering::Release); + } + } +} + +/// Acquire the page-table critical section, re-entrant on a single core. +#[inline] +pub(crate) fn lock_page_table() -> PageTableGuard { + let prev = PAGE_TABLE_LOCK_DEPTH.fetch_add(1, core::sync::atomic::Ordering::Acquire); + if prev == 0 { + // First (outer) acquisition on this core: take the real lock. + let g = PAGE_TABLE_LOCK.lock(); + PageTableGuard { outer: Some(g) } + } else { + // Nested acquisition: the outer guard already holds the lock. + PageTableGuard { outer: None } + } +} + // ─── Page-table entry flags (shared across architectures) ───────────────────── bitflags! { @@ -756,7 +805,7 @@ pub fn init(hhdm_offset: u64) { /// * `set_hhdm_offset()` must have run before this. /// * `virt` and `phys` must be 4-KiB aligned. pub unsafe fn map_page(virt: u64, phys: u64, flags: PageFlags) { - let _lock = PAGE_TABLE_LOCK.lock(); + let _lock = lock_page_table(); #[cfg(target_arch = "x86_64")] unsafe { x86_64_impl::map_page(virt, phys, flags) } #[cfg(target_arch = "aarch64")] @@ -770,7 +819,7 @@ pub unsafe fn map_page(virt: u64, phys: u64, flags: PageFlags) { /// # Safety /// Same preconditions as [`map_page`]. pub unsafe fn map_mmio(phys: u64, virt: u64, size: usize) { - let _lock = PAGE_TABLE_LOCK.lock(); + let _lock = lock_page_table(); #[cfg(target_arch = "x86_64")] unsafe { x86_64_impl::map_mmio(phys, virt, size) } #[cfg(target_arch = "aarch64")] @@ -872,7 +921,7 @@ pub fn alloc_user_pml4() -> Option { pub unsafe fn map_user_page(pml4_phys: u64, virt: u64, phys: u64) -> Result<(), &'static str> { - let _lock = PAGE_TABLE_LOCK.lock(); + let _lock = lock_page_table(); #[cfg(target_arch = "x86_64")] return unsafe { x86_64_impl::map_page_in( @@ -899,7 +948,7 @@ pub unsafe fn map_user_page_with_flags( pml4_phys: u64, virt: u64, phys: u64, writable: bool, exec: bool, ) -> Result<(), &'static str> { - let _lock = PAGE_TABLE_LOCK.lock(); + let _lock = lock_page_table(); #[cfg(target_arch = "x86_64")] { let mut flags = PageFlags::PRESENT | PageFlags::USER; @@ -921,7 +970,7 @@ pub unsafe fn map_user_page_with_flags( /// Walk a user PML4 and return the physical frame already mapped at `virt`, /// or `None` if the page is not yet mapped. pub fn translate_user_page(pml4_phys: u64, virt: u64) -> Option { - let _lock = PAGE_TABLE_LOCK.lock(); + let _lock = lock_page_table(); #[cfg(target_arch = "x86_64")] { x86_64_impl::translate_user_page(pml4_phys, virt) } #[cfg(target_arch = "aarch64")] @@ -933,7 +982,7 @@ pub fn translate_user_page(pml4_phys: u64, virt: u64) -> Option { /// Walk a user PML4 and return the physical frame mapped at `virt` along with /// its writable and executable flags. pub fn translate_user_page_flags(pml4_phys: u64, virt: u64) -> Option<(u64, bool, bool)> { - let _lock = PAGE_TABLE_LOCK.lock(); + let _lock = lock_page_table(); #[cfg(target_arch = "x86_64")] { x86_64_impl::translate_user_page_flags(pml4_phys, virt) } #[cfg(target_arch = "aarch64")] @@ -944,7 +993,7 @@ pub fn translate_user_page_flags(pml4_phys: u64, virt: u64) -> Option<(u64, bool /// Free all frames in a user PML4 and the PML4 itself. pub fn free_user_pml4(pml4_phys: u64) { - let _lock = PAGE_TABLE_LOCK.lock(); + let _lock = lock_page_table(); #[cfg(target_arch = "x86_64")] x86_64_impl::free_user_pml4(pml4_phys); #[cfg(target_arch = "aarch64")] @@ -963,7 +1012,7 @@ pub unsafe fn update_user_page( writable: bool, exec: bool, ) -> Result<(), &'static str> { - let _lock = PAGE_TABLE_LOCK.lock(); + let _lock = lock_page_table(); #[cfg(target_arch = "x86_64")] { unsafe { x86_64_impl::update_user_page_flags(pml4_phys, virt, writable, exec) } @@ -984,7 +1033,7 @@ pub unsafe fn update_user_page( /// # Safety /// `pml4_phys` must be a valid allocated PML4. pub unsafe fn unmap_user_page(pml4_phys: u64, virt: u64) -> Result { - let _lock = PAGE_TABLE_LOCK.lock(); + let _lock = lock_page_table(); #[cfg(target_arch = "x86_64")] { unsafe { x86_64_impl::unmap_user_page(pml4_phys, virt) } From d19241a2d3bab9b1c9cc8a383d4a16d8b74fd30d Mon Sep 17 00:00:00 2001 From: Tahiru Agbanwa <130235676+squirelboy360@users.noreply.github.com> Date: Tue, 9 Jun 2026 20:53:23 +0200 Subject: [PATCH 35/36] =?UTF-8?q?dl:=20cfg-gate=20TLSDESC=5FRESOLVER=5FVA?= =?UTF-8?q?=20reference=20so=20the=20x86=5F64=20kernel=20builds=20(the=20a?= =?UTF-8?q?arch64-only=20resolver=20trampoline;=20arm=20unchanged)=20?= =?UTF-8?q?=E2=80=94=20keeps=20x86=20green?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kernel/src/process/dl.rs | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/kernel/src/process/dl.rs b/kernel/src/process/dl.rs index aa77e94..3fa47e8 100644 --- a/kernel/src/process/dl.rs +++ b/kernel/src/process/dl.rs @@ -516,10 +516,15 @@ fn apply_rela_table( match rtype { R_AARCH64_TLSDESC => { // Descriptor pair: [+0]=static resolver, [+8]=TP offset. - write_user_u64( - target, - crate::process::posix_trampolines::TLSDESC_RESOLVER_VA, - ); + // The resolver trampoline only exists on aarch64 (a TLSDESC + // reloc can only appear in an aarch64 object); on x86 this + // arm is unreachable, so fall back to 0 to keep that build + // green without pulling in the arch-gated symbol. + #[cfg(target_arch = "aarch64")] + let resolver = crate::process::posix_trampolines::TLSDESC_RESOLVER_VA; + #[cfg(not(target_arch = "aarch64"))] + let resolver = 0u64; + write_user_u64(target, resolver); write_user_u64(target.wrapping_add(8), tp_off); } R_AARCH64_TLS_TPREL64 => write_user_u64(target, tp_off), From b98ff1983a8e8bb49cfe326c2f8678af9038fb04 Mon Sep 17 00:00:00 2001 From: Tahiru Agbanwa <130235676+squirelboy360@users.noreply.github.com> Date: Tue, 9 Jun 2026 22:31:08 +0200 Subject: [PATCH 36/36] =?UTF-8?q?aarch64:=20eager=20user-GPR=20capture=20+?= =?UTF-8?q?=20IRQ-masked=20page-table=20lock=20=E2=80=94=20advance=20engin?= =?UTF-8?q?e=20bring-up=20past=20the=20early=20ret-to-0=20/=20pager=20dead?= =?UTF-8?q?lock?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two single-core register/lock-coherence fixes that move the ARM Flutter engine from crashing immediately after FlutterEngineInitialize to running through FlutterEngineRunInitialized into ICU init and worker-thread spawn. 1) Eager user-GPR capture at the SVC boundary (process::mod, vectors). `save_full_user_gprs` read the per-CPU user-GPR snapshot LAZILY at yield time. Syscalls run with IRQs unmasked, so a generic-timer tick can preempt a thread mid-handler, switch to a sibling whose own SVC overwrites the shared snapshot, then switch back — and the yielding thread then persists the sibling's callee-saved regs / x30 into its own context (resume → `ret` to a stale address, e.g. x30=0 → branch to 0). Port the proven x86 fix (GPRS_CAPTURED flag + capture_user_gprs_at_entry): snapshot ONCE, eagerly, while fresh, and make later yield-time saves no-ops. On aarch64 the eager capture runs from inside the IRQ-masked SVC window in the vector dispatch (x86 stays masked until the handler sti's), so the snapshot can't be clobbered before it is persisted. 2) IRQ-masked outer page-table critical section (mm::paging). The reentrant PAGE_TABLE_LOCK depth counter is only sound if the section can't be interleaved by another thread. With IRQs unmasked during syscalls, a timer tick could preempt a lock holder mid-section; a sibling then saw a non-zero depth and proceeded as a bogus "nested" writer, desynchronising the counter from the real lock and eventually stranding the lock held while depth read 0 — a later outer acquire then spun forever with IRQs masked in the demand-abort handler (single-core deadlock, observed freezing right at worker-thread stack setup). Mask IRQs for the whole outer section so it is genuinely uninterruptible; nesting then only ever means true same-stack re-entry (a demand fault during a page-table walk, already IRQs-masked). x86 behaviour is unchanged (cfg-gated to aarch64). Also: the unhandled-exception reporter now scans the EL0 stack for engine return addresses (the FP chain is empty when x30/FP are 0), so a ret-to-0 can be symbolised against libflutter_engine.so. Status: engine now reaches ICU init / worker spawn; a residual cooperative- yield corruption remains (nondeterministic ret-to-0 vs abort) — same class the x86 port hardened over several commits. Not yet rendering on ARM. Co-Authored-By: Claude Opus 4.8 --- kernel/src/arch/aarch64/vectors.rs | 39 +++++++++++++++++++ kernel/src/mm/paging.rs | 47 +++++++++++++++++++--- kernel/src/process/mod.rs | 62 +++++++++++++++++++++++------- 3 files changed, 130 insertions(+), 18 deletions(-) diff --git a/kernel/src/arch/aarch64/vectors.rs b/kernel/src/arch/aarch64/vectors.rs index 8d20b48..85f1ba5 100644 --- a/kernel/src/arch/aarch64/vectors.rs +++ b/kernel/src/arch/aarch64/vectors.rs @@ -306,6 +306,20 @@ extern "C" fn dispatch(frame: *mut TrapFrame, kind: u64) { if ec == EC_SVC64 { let h = SVC_HANDLER.load(Ordering::Acquire); if h != 0 { + // EAGER user-GPR capture while IRQs are STILL MASKED. Snapshot + // this thread's callee-saved regs (x19–x23/x29) + link + // register (x30) from the trap frame into its PTABLE slot + // now, before we unmask below. Once IRQs are on, a generic- + // timer tick can switch to a sibling thread whose own SVC + // overwrites the shared per-CPU snapshot; gating the later + // yield-time `save_full_user_gprs` on this capture stops that + // stale snapshot from leaking back as our resume context + // (the bug: x30 captured stale → on resume `ret` branches to + // 0 → EL0 UDF right after FlutterEngineRunInitialized). This + // is the aarch64 counterpart of the x86 SYSCALL-entry capture + // (interrupts there stay masked until the handler `sti`s). + crate::arch::aarch64::syscall::capture_from_trap(f); + crate::process::capture_user_gprs_at_entry(crate::process::current_pid()); // Unmask IRQs for the duration of the syscall, mirroring x86 // where the SYSCALL entry stub runs handlers with interrupts // ENABLED. Exception entry masked DAIF.I; without re-enabling @@ -475,6 +489,31 @@ fn report_unhandled(f: &TrapFrame, kind: u64, ec: u64) -> ! { depth += 1; } } + // Raw stack scan: with x30=0/FP=0 the call chain is lost from registers and + // the frame walk yields nothing, but non-leaf callers still pushed their LR + // onto the EL0 stack. Scan upward from SP_EL0 and print any word that lands + // in the engine .so code window [0x1_4100_0000, 0x1_4300_0000) — those are + // almost certainly saved return addresses. addr2line them against + // libflutter_engine.so (base 0x141000000) to recover where the `ret` to 0 + // originated. + uart::puts(" stkscan:"); + { + let sp = f.sp_el0; + let plausible_sp = sp >= 0x1000 && sp < 0x0000_8000_0000_0000 && (sp & 0x7) == 0; + if plausible_sp { + let mut printed = 0u32; + let mut off = 0u64; + while off < 0x2000 && printed < 24 { + let w = unsafe { core::ptr::read_volatile((sp + off) as *const u64) }; + if w >= 0x1_4100_0000 && w < 0x1_4300_0000 { + uart::puts(" "); + uart::puthex_full(w); + printed += 1; + } + off += 8; + } + } + } uart::puts("\n"); loop { unsafe { core::arch::asm!("wfe", options(nomem, nostack)) }; diff --git a/kernel/src/mm/paging.rs b/kernel/src/mm/paging.rs index e248eee..dac7443 100644 --- a/kernel/src/mm/paging.rs +++ b/kernel/src/mm/paging.rs @@ -30,14 +30,24 @@ static PAGE_TABLE_LOCK_DEPTH: core::sync::atomic::AtomicU32 = pub(crate) struct PageTableGuard { outer: Option>, + /// aarch64 only: DAIF captured by the OUTER acquisition before it masked + /// IRQs. Restored when the outer guard drops. Unused on nested guards (IRQs + /// stay masked for the whole outer section). + #[cfg(target_arch = "aarch64")] + saved_daif: u64, } impl Drop for PageTableGuard { fn drop(&mut self) { if self.outer.is_some() { - // Outermost guard: clear depth, then release the real lock. + // Outermost guard: clear depth, release the real lock, THEN re-enable + // IRQs. The lock byte must be released before IRQs come back on, or a + // freshly-unmasked timer tick could switch to a thread that spins on + // the still-held lock with IRQs masked → single-core deadlock. PAGE_TABLE_LOCK_DEPTH.store(0, core::sync::atomic::Ordering::Release); - // `outer` drops here, releasing PAGE_TABLE_LOCK. + let _ = self.outer.take(); // drop the MutexGuard now (IRQs still masked) + #[cfg(target_arch = "aarch64")] + crate::arch::aarch64::cpu::interrupts_restore(self.saved_daif); } else { PAGE_TABLE_LOCK_DEPTH.fetch_sub(1, core::sync::atomic::Ordering::Release); } @@ -45,16 +55,43 @@ impl Drop for PageTableGuard { } /// Acquire the page-table critical section, re-entrant on a single core. +/// +/// On aarch64 the outer acquisition also MASKS IRQs for the whole critical +/// section. The reentrant depth counter is only sound if the section cannot be +/// interleaved by another thread; but syscalls run with IRQs unmasked, so a +/// generic-timer tick can preempt a lock holder mid-section and switch to a +/// sibling, which then sees a non-zero depth and proceeds as a bogus "nested" +/// writer — desynchronising the depth counter from the real lock and eventually +/// stranding the lock held while depth reads 0 (a later outer acquire then spins +/// forever with IRQs masked in the demand-abort handler). Masking IRQs around +/// the outer section makes it genuinely uninterruptible, so nesting only ever +/// means true same-stack re-entry (e.g. a demand fault during a page-table walk, +/// which already runs IRQs-masked). x86 keeps its original behaviour. #[inline] pub(crate) fn lock_page_table() -> PageTableGuard { + // Mask IRQs BEFORE touching the depth counter so the fetch_add → lock + // sequence cannot be preempted. On a nested call IRQs are already masked + // (the outer masked them); this is then an idempotent no-op. + #[cfg(target_arch = "aarch64")] + let saved_daif = crate::arch::aarch64::cpu::interrupts_save_and_disable(); let prev = PAGE_TABLE_LOCK_DEPTH.fetch_add(1, core::sync::atomic::Ordering::Acquire); if prev == 0 { // First (outer) acquisition on this core: take the real lock. let g = PAGE_TABLE_LOCK.lock(); - PageTableGuard { outer: Some(g) } + PageTableGuard { + outer: Some(g), + #[cfg(target_arch = "aarch64")] + saved_daif, + } } else { - // Nested acquisition: the outer guard already holds the lock. - PageTableGuard { outer: None } + // Nested acquisition: the outer guard already holds the lock (and already + // masked IRQs). `saved_daif` here is "already masked" and is discarded on + // drop — only the outer guard restores DAIF. + PageTableGuard { + outer: None, + #[cfg(target_arch = "aarch64")] + saved_daif, + } } } diff --git a/kernel/src/process/mod.rs b/kernel/src/process/mod.rs index 9cd544c..daa546f 100644 --- a/kernel/src/process/mod.rs +++ b/kernel/src/process/mod.rs @@ -1749,6 +1749,49 @@ pub fn save_return_context(pid: u32, rip: u64, rsp: u64) { } } +/// Guards against a since-clobbered per-CPU user-GPR snapshot leaking into a +/// thread's saved context. +/// +/// The user GPR snapshot lives in a PER-CPU scratch area, written at syscall +/// entry from the trapping thread's frame. It is shared by every thread on that +/// core, so once a syscall hands off the CPU (a cooperative `epoll`/`futex`/ +/// `cond` yield, or — on aarch64 — a generic-timer preempt that lands a +/// sibling's SVC mid-handler), another thread's syscall entry OVERWRITES it. A +/// later `save_full_user_gprs` would then read the *other* thread's callee-saved +/// regs / link register and store them into our context → on resume rbx/rbp/ +/// r12–15 (x19–x23/x29) or x30 are garbage, and the next `ret` branches to a +/// stale address (observed on aarch64: x30=0 → `ret` to 0 → EL0 UDF after +/// `FlutterEngineRunInitialized`; the x86 analogue corrupted `MessageLoop::Run`'s +/// `this`). +/// +/// Fix: capture ONCE, eagerly, at syscall entry (`capture_user_gprs_at_entry`), +/// while the snapshot is still fresh for THIS thread (x86: interrupts masked +/// until the handler `sti`s; aarch64: from inside the IRQ-masked SVC window in +/// the vector dispatch). This flag makes every subsequent yield-time +/// `save_full_user_gprs` in the same syscall a no-op, so a clobbered per-CPU +/// snapshot can never leak into our saved context. +const GPR_CAP_MAX_CPUS: usize = 64; +static GPRS_CAPTURED: [core::sync::atomic::AtomicBool; GPR_CAP_MAX_CPUS] = + [const { core::sync::atomic::AtomicBool::new(false) }; GPR_CAP_MAX_CPUS]; + +#[inline] +fn gpr_cap_cpu_idx() -> usize { + #[cfg(target_arch = "aarch64")] + { (crate::arch::aarch64::smp::current_cpu_id() as usize).min(GPR_CAP_MAX_CPUS - 1) } + #[cfg(not(target_arch = "aarch64"))] + { (crate::arch::smp::this_cpu().cpu_id as usize).min(GPR_CAP_MAX_CPUS - 1) } +} + +/// Capture the user GPRs into `pid`'s PTABLE slot at syscall ENTRY, while the +/// per-CPU snapshot is guaranteed fresh for this thread. Marks them captured so +/// later `save_full_user_gprs` calls this syscall don't re-read the (possibly +/// since-clobbered) shared per-CPU snapshot. MUST be invoked while the snapshot +/// cannot yet be overwritten by a sibling (IRQs masked / pre-`sti`). +pub fn capture_user_gprs_at_entry(pid: u32) { + GPRS_CAPTURED[gpr_cap_cpu_idx()].store(false, core::sync::atomic::Ordering::Relaxed); + save_full_user_gprs(pid); +} + /// Snapshot the full user GPR set (as captured at SYSCALL entry) into the /// process's register file. Must be called from inside a syscall handler /// before yielding the CPU to another process via @@ -1756,20 +1799,13 @@ pub fn save_return_context(pid: u32, rip: u64, rsp: u64) { /// with stale rbx/rbp/r12–r15/rdi/rsi/etc. and corrupt its C++ caller's /// `this` pointer and locals. pub fn save_full_user_gprs(pid: u32) { - let snap = crate::arch::syscall::user_gprs(); - if pid == 2 { - log::warn!( - "[save_full_user_gprs] pid=2 rdi={:#x} rsi={:#x} rbx={:#x} rbp={:#x} r12={:#x} r13={:#x} r14={:#x} r15={:#x}", - snap.rdi, - snap.rsi, - snap.rbx, - snap.rbp, - snap.r12, - snap.r13, - snap.r14, - snap.r15 - ); + // Only the first call per syscall (the eager entry capture) reads the + // per-CPU snapshot; it is fresh then. Later yield-time calls are no-ops so a + // since-clobbered snapshot cannot leak into our saved context. + if GPRS_CAPTURED[gpr_cap_cpu_idx()].swap(true, core::sync::atomic::Ordering::Relaxed) { + return; } + let snap = crate::arch::syscall::user_gprs(); let _g = PTABLE_LOCK.lock(); let p = unsafe { &mut PTABLE[idx_of(pid)] }; if p.pid != pid { return; }