diff --git a/kernel/build.rs b/kernel/build.rs index b154751..7448939 100644 --- a/kernel/build.rs +++ b/kernel/build.rs @@ -47,6 +47,22 @@ fn main() { generate_liboscortex_embedder_shim(), )); + // aarch64 port: provide a native EL0 `/init` so the production kernel_main can + // spawn a userspace process + service its first syscalls on ARM. + // + // When `scripts/build-aarch64-shell.sh` has staged the real Flutter host as + // `initramfs/init` (it is then already in `entries`), we KEEP it — that path + // boots the actual shell. Only when no `init` was collected do we inject the + // self-contained preemption self-test (a couple of `write` syscalls, a + // `getpid`, then `exit`) so a bare `cargo build` + `run-aarch64.sh` still has + // something to spawn and exercises the spawn → SVC dispatch → exit path. + if arch == "aarch64" { + let has_real_init = entries.iter().any(|(n, _)| n == "init"); + if !has_real_init { + entries.push(("init".to_string(), generate_aarch64_init_elf())); + } + } + // Write the USTAR tar to OUT_DIR. let tar_bytes = build_ustar_tar(&entries); std::fs::write(&out_tar, &tar_bytes).unwrap(); @@ -87,7 +103,15 @@ fn collect_dir( && name != "system/flutter/flutter_assets/kernel_blob.bin" && !(name.starts_with("Applications/") && name.ends_with(".app/flutter_assets/kernel_blob.bin")) { continue; } - if fname == "libflutter_engine.so" || fname.ends_with(".bak") { continue; } + // The Flutter engine .so is delivered as a Limine MODULE on x86 + // (iso_root/boot/libflutter_engine.so), so it is excluded from the + // initramfs there. On aarch64 there is NO Limine — the kernel boots + // via `-kernel` and the engine MUST travel in the initramfs, so we + // KEEP it. (collect_dir runs in build.rs; CARGO_CFG_TARGET_ARCH is + // the kernel's build target.) + let target_arch = std::env::var("CARGO_CFG_TARGET_ARCH").unwrap_or_default(); + if fname == "libflutter_engine.so" && target_arch != "aarch64" { continue; } + if fname.ends_with(".bak") { continue; } println!("cargo:rerun-if-changed={}", path.display()); let data = match std::fs::read(&path) { Ok(b) => b, @@ -440,6 +464,184 @@ fn generate_liboscortex_embedder_shim() -> Vec { generate_ret_stub_elf(SYMS) } +// ── aarch64 native `/init` EL0 program ─────────────────────────────────────── + +/// Build a minimal statically-linked AArch64 ELF64 (ET_EXEC, EM_AARCH64) that +/// runs at EL0 and issues syscalls through the OSCortex shared dispatcher. +/// +/// Syscall ABI (OSCortex / x86-numbered, as understood by `dispatch_fast`): +/// nr in x8, args in x0..x2, return in x0, trap via `svc #0`. +/// 1 = write(fd, buf, len) → fd 1/2 print to the kernel serial console +/// 39 = getpid() → returns the process id +/// 60 = exit(code) → terminate the process +/// +/// Layout: a single RWX PT_LOAD at vaddr 0x400000 (USER_ELF_BASE). Code starts +/// at the entry; two NUL-free messages live at fixed offsets in the same page. +fn generate_aarch64_init_elf() -> Vec { + // Link the init program at 64 GiB (L1 index 64), well clear of the kernel's + // low identity map. The bring-up kernel runs from the TTBR0 low half, so each + // per-process root is seeded with the kernel's 1 GiB block descriptors for + // the device region (index 0) and RAM (indices 1..2). A user page at a low VA + // would collide with those blocks; placing the program at a high, otherwise- + // empty L1 index lets the page-table walker create fresh L2/L3 tables there. + const VBASE: u64 = 0x10_0000_0000; // 64 GiB + + // Two messages selected by the bootstrap value the kernel seeds in x0 at + // entry: process A (x0==0) prints msg_a, any other value prints msg_b. This + // lets two instances of the SAME program emit distinguishable output so an + // observer can see the timer round-robining between them. + let msg_a: &[u8] = b"[arm-A] tick (EL0 compute loop A)\n"; + let msg_b: &[u8] = b"[arm-B] tick (EL0 compute loop B)\n"; + let hello: &[u8] = b"[arm-init] EL0 up; entering compute loop (timer preemption test)\n"; + let msg_a_off: u64 = 0x200; + let msg_b_off: u64 = 0x280; + let hello_off: u64 = 0x300; + let msg_a_va = VBASE + msg_a_off; + let msg_b_va = VBASE + msg_b_off; + let hello_va = VBASE + hello_off; + + // ── Assemble the code (offset 0) ──────────────────────────────────────── + // + // Pseudocode: + // x19 = (x0 == 0) ? msg_a_va : msg_b_va // pick this instance's msg + // x20 = (x0 == 0) ? len_a : len_b + // write(1, hello, hello.len()) // announce once + // loop: + // x21 = SPIN_ITERS // pure-compute busy spin — + // spin: subs x21,x21,#1; b.ne spin // NO syscall, so only the + // // timer can switch us out + // write(1, x19, x20) // print this instance's tick + // b loop + // + // Because the spin does no syscalls, the kernel cannot cooperatively yield + // here; if both A and B make progress (interleaved ticks on serial) the + // generic-timer ISR MUST be preempting and switching between them. + const SPIN_ITERS: u64 = 4_000_000; + // Each thread prints this many ticks, then exits — enough to demonstrate the + // timer round-robining the two compute-bound threads, after which the system + // settles into the kernel idle/cortex loop (rather than spamming forever). + const TICK_BUDGET: u64 = 12; + let mut code: Vec = Vec::new(); + + // Select message VA (x19) and length (x20) from x0. + a64_load_imm(&mut code, 19, msg_a_va); + a64_load_imm(&mut code, 20, msg_a.len() as u64); + a64_load_imm(&mut code, 9, msg_b_va); + a64_load_imm(&mut code, 10, msg_b.len() as u64); + // cmp x0, #0 ; csel x19, x19, x9, eq ; csel x20, x20, x10, eq + code.push(0xF100_001F); // subs xzr, x0, #0 (cmp x0,#0) + code.push(0x9A89_0273); // csel x19, x19, x9, eq + code.push(0x9A8A_0294); // csel x20, x20, x10, eq + + // x22 = TICK_BUDGET (remaining ticks before this thread exits) + a64_load_imm(&mut code, 22, TICK_BUDGET); + + // write(1, hello, hello.len()) + a64_load_imm(&mut code, 0, 1); + a64_load_imm(&mut code, 1, hello_va); + a64_load_imm(&mut code, 2, hello.len() as u64); + a64_load_imm(&mut code, 8, 1); + code.push(0xD400_0001); // svc #0 + + // loop: + let loop_idx = code.len(); + // x21 = SPIN_ITERS + a64_load_imm(&mut code, 21, SPIN_ITERS); + // spin: subs x21, x21, #1 ; b.ne spin (pure compute — no syscall) + let spin_idx = code.len(); + code.push(0xF100_06B5); // subs x21, x21, #1 + { + let off = (spin_idx as i64 - code.len() as i64) as i32; // negative + let imm19 = (off as u32) & 0x7FFFF; + code.push(0x5400_0001 | (imm19 << 5)); // b.ne spin + } + // write(1, x19, x20) + a64_load_imm(&mut code, 0, 1); + code.push(0xAA13_03E1); // mov x1, x19 + code.push(0xAA14_03E2); // mov x2, x20 + a64_load_imm(&mut code, 8, 1); + code.push(0xD400_0001); // svc #0 + // subs x22, x22, #1 ; b.ne loop (loop until the tick budget is spent) + code.push(0xF100_06D6); // subs x22, x22, #1 + { + let off = (loop_idx as i64 - code.len() as i64) as i32; // negative + let imm19 = (off as u32) & 0x7FFFF; + code.push(0x5400_0001 | (imm19 << 5)); // b.ne loop + } + // exit(0) — lets the system settle into the kernel idle/cortex loop after + // the preemption demonstration rather than spinning forever. + a64_load_imm(&mut code, 0, 0); + a64_load_imm(&mut code, 8, 60); + code.push(0xD400_0001); // svc #0 + // safety net: spin if exit ever returns to EL0. + code.push(0x1400_0000); // b . + + // ── Compose the single loadable page image ────────────────────────────── + let page_sz = 0x1000usize; + let mut img = vec![0u8; page_sz]; + for (i, w) in code.iter().enumerate() { + let o = i * 4; + img[o..o + 4].copy_from_slice(&w.to_le_bytes()); + } + img[msg_a_off as usize..msg_a_off as usize + msg_a.len()].copy_from_slice(msg_a); + img[msg_b_off as usize..msg_b_off as usize + msg_b.len()].copy_from_slice(msg_b); + img[hello_off as usize..hello_off as usize + hello.len()].copy_from_slice(hello); + + // ── ELF header + one program header ───────────────────────────────────── + let ehsize = 64usize; + let phentsize = 56usize; + let phoff = ehsize; + let data_off = phoff + phentsize; // file offset of the loadable image + + let file_sz = data_off + img.len(); + let mut elf = vec![0u8; file_sz]; + + // e_ident + elf[0..4].copy_from_slice(&[0x7F, b'E', b'L', b'F']); + elf[4] = 2; // ELFCLASS64 + elf[5] = 1; // ELFDATA2LSB + elf[6] = 1; // EV_CURRENT + write_u16(&mut elf, 16, 2); // e_type = ET_EXEC + write_u16(&mut elf, 18, 183); // e_machine = EM_AARCH64 + write_u32(&mut elf, 20, 1); // e_version + write_u64(&mut elf, 24, VBASE); // e_entry + write_u64(&mut elf, 32, phoff as u64); // e_phoff + write_u16(&mut elf, 52, ehsize as u16); // e_ehsize + write_u16(&mut elf, 54, phentsize as u16); // e_phentsize + write_u16(&mut elf, 56, 1); // e_phnum + + // PT_LOAD program header (RWX). + let p = phoff; + write_u32(&mut elf, p, 1); // p_type = PT_LOAD + write_u32(&mut elf, p + 4, 7); // p_flags = R|W|X + write_u64(&mut elf, p + 8, data_off as u64); // p_offset + write_u64(&mut elf, p + 16, VBASE); // p_vaddr + write_u64(&mut elf, p + 24, VBASE); // p_paddr + write_u64(&mut elf, p + 32, img.len() as u64); // p_filesz + write_u64(&mut elf, p + 40, img.len() as u64); // p_memsz + write_u64(&mut elf, p + 48, 0x1000); // p_align + + elf[data_off..data_off + img.len()].copy_from_slice(&img); + elf +} + +/// Emit a MOVZ/MOVK sequence loading the 64-bit immediate `imm` into x`reg`. +fn a64_load_imm(code: &mut Vec, reg: u32, imm: u64) { + let mut first = true; + for hw in 0..4u32 { + let imm16 = ((imm >> (hw * 16)) & 0xFFFF) as u32; + if imm16 == 0 && !first { + continue; // skip zero halfwords once at least one MOVZ emitted + } + let base = if first { 0xD280_0000 } else { 0xF280_0000 }; + code.push(base | (hw << 21) | (imm16 << 5) | reg); + first = false; + } + if first { + code.push(0xD280_0000 | reg); // imm == 0 → MOVZ Xreg, #0 + } +} + /// Generate a minimal ELF64 ET_DYN where every exported symbol is a /// 4-byte `xor eax,eax; ret; nop; nop` stub. Used for both /// `libflutter_engine.so` (existing) and `liboscortex_embedder.so`. diff --git a/kernel/linker/aarch64.ld b/kernel/linker/aarch64.ld index f538360..f61a98e 100644 --- a/kernel/linker/aarch64.ld +++ b/kernel/linker/aarch64.ld @@ -1,13 +1,54 @@ -/* aarch64 linker script stub */ +/* aarch64 direct-boot linker script (QEMU -M virt -kernel). + * + * QEMU `-M virt` places RAM at 0x4000_0000 and, for an ELF kernel, loads each + * PT_LOAD segment at its physical address and jumps to e_entry with the MMU + * OFF (running on physical == virtual identity at this point). We therefore + * link the kernel to run from a low physical address and enable the MMU later + * with an identity map plus the kernel's higher-half window. + * + * Entry is the assembly `_start` (boot.rs) which sets up the stack, zeroes BSS, + * brings up the PL011 UART for serial, then calls into the Rust bring-up path. + */ OUTPUT_FORMAT("elf64-littleaarch64") -ENTRY(kernel_main) +OUTPUT_ARCH(aarch64) +ENTRY(_start) + +/* QEMU -M virt RAM base. The kernel is loaded here by -kernel. */ +KERNEL_PHYS_BASE = 0x40080000; + SECTIONS { - . = 0xFFFF000000080000; - .text : { *(.text .text.*) } + . = KERNEL_PHYS_BASE; + + .text : ALIGN(4096) { + __text_start = .; + KEEP(*(.text.boot)) /* _start must be first */ + *(.text .text.*) + __text_end = .; + } + + .rodata : ALIGN(4096) { + __rodata_start = .; + *(.rodata .rodata.*) + __rodata_end = .; + } + + .data : ALIGN(4096) { + __data_start = .; + *(.data .data.*) + *(.got .got.*) + __data_end = .; + } + + .bss (NOLOAD) : ALIGN(4096) { + __bss_start = .; + *(.bss .bss.*) + *(COMMON) + . = ALIGN(16); + __bss_end = .; + } + . = ALIGN(4096); - .rodata : { *(.rodata .rodata.*) } - . = ALIGN(4096); - .data : { *(.data .data.*) } - .bss : { __bss_start = .; *(.bss .bss.*) *(COMMON) __bss_end = .; } - /DISCARD/ : { *(.eh_frame) *(.note .note.*) } + __kernel_end = .; + + /DISCARD/ : { *(.eh_frame) *(.eh_frame_hdr) *(.note .note.*) *(.comment) } } diff --git a/kernel/src/arch/aarch64/acpi.rs b/kernel/src/arch/aarch64/acpi.rs new file mode 100644 index 0000000..a110ef4 --- /dev/null +++ b/kernel/src/arch/aarch64/acpi.rs @@ -0,0 +1,29 @@ +//! Firmware-table lookup scaffolding — aarch64. +//! +//! On aarch64 platforms hardware topology comes from either ACPI (server-class +//! machines) or a flattened device tree (the QEMU `virt` bring-up target). The +//! shared kernel does not call these on the non-x86 path yet; they exist for +//! parity so the real port has named entry points to fill in. + +/// Physical address of the RSDP, or 0 if unavailable. +/// +/// TODO(arm): use the Limine RSDP request (ACPI) or parse the DTB. +pub fn rsdp_address() -> u64 { + 0 +} + +/// Physical address of the MADT/GICC table, or 0 if not found. +/// +/// TODO(arm): walk XSDT for the MADT, or parse `/cpus` + `intc` from the DTB. +pub fn find_madt(_rsdp_phys: u64) -> u64 { + 0 +} + +/// Power off the machine. +/// +/// TODO(arm): issue a PSCI `SYSTEM_OFF` via SMC/HVC. For now, park. +pub fn shutdown() -> ! { + loop { + unsafe { core::arch::asm!("wfi", options(nomem, nostack)) }; + } +} diff --git a/kernel/src/arch/aarch64/apic.rs b/kernel/src/arch/aarch64/apic.rs new file mode 100644 index 0000000..5a7bbe5 --- /dev/null +++ b/kernel/src/arch/aarch64/apic.rs @@ -0,0 +1,298 @@ +//! Interrupt-controller + tick/vsync scaffolding — aarch64 (GIC + generic timer). +//! +//! On x86 this is the local APIC; on aarch64 the equivalent responsibilities are +//! split between the GIC (CPU interface: EOI, IPIs, routing) and the ARM generic +//! timer (CNTP/CNTV, the periodic tick that drives the vsync cadence). These are +//! compilable stubs preserving the public surface the shared kernel calls; real +//! GICv2/GICv3 + CNTV_CTL programming comes later. + +use core::sync::atomic::{AtomicU32, Ordering}; + +/// Approximate timer ticks per millisecond (placeholder; real value is read +/// from `CNTFRQ_EL0` once the generic timer is programmed). +pub static APIC_TICKS_PER_MS: AtomicU32 = AtomicU32::new(62500); + +/// Requested vsync cadence in Hz (0 = off). +static VSYNC_HZ: AtomicU32 = AtomicU32::new(0); + +/// Tick rate of the production scheduler timer (Hz). Mirrors the x86 APIC-timer +/// 1 ms cadence closely enough for the cooperative scheduler's quantum logic. +const SCHED_TICK_HZ: u64 = 1000; + +/// Initialise the interrupt controller on the BSP (early). +/// +/// The GIC distributor + CPU interface are already live (brought up in +/// `boot_prod`). Here we only INSTALL the production IRQ handler — which drives +/// the SAME shared cooperative-scheduler hand-off the x86 APIC-timer ISR uses. +/// The generic timer itself is armed later by [`start_scheduler_tick`], once the +/// full kernel (heap, scheduler, process table) is initialised and we are about +/// to enter userspace — mirroring x86, which doesn't tick into a half-built +/// kernel. +pub fn init_bsp() { + super::vectors::set_irq_handler(production_irq_handler); +} + +/// Arm the generic timer for the periodic scheduler tick and unmask IRQ +/// delivery. Call once, just before entering the first EL0 thread, so the timer +/// only fires when there is a real scheduler + process table to preempt into. +pub fn start_scheduler_tick() { + APIC_TICKS_PER_MS.store( + (super::timer::freq_hz() / 1000).max(1) as u32, + Ordering::Release, + ); + super::timer::init_periodic(SCHED_TICK_HZ); + // Note: we deliberately do NOT unmask IRQs at EL1 here. The `eret` into EL0 + // restores SPSR_EL1 = EL0t with I/F unmasked, so timer IRQs begin firing the + // instant we drop to userspace — and never interrupt the half-built EL1 + // entry path (write_cr3 → image build → eret). The kernel idle/cortex loop + // unmasks via `enable_and_halt` when it has no user thread to run. + log::info!( + "[apic] aarch64 scheduler timer armed @ {} Hz (CNTFRQ={} Hz), preemption live", + SCHED_TICK_HZ, + super::timer::freq_hz() + ); +} + +/// Production IRQ handler — acknowledge at the GIC, and on a generic-timer PPI +/// from EL0 run the shared cooperative-scheduler preemption hand-off, exactly +/// mirroring `x86_64::idt::apic_timer_handler`. +/// +/// The EL1 vector stub (`vectors.rs`) has already saved the full EL0 register +/// file into the [`super::vectors::TrapFrame`] `f` points at, and will +/// `RESTORE_FRAME` + `eret` from it after we return. So to switch threads we +/// simply (a) decide via the shared scheduler, (b) overwrite `f` with the next +/// thread's saved frame, and (c) switch TTBR0 — the stub's eret then enters the +/// next thread. The full ARM frame round-trips through each thread's +/// `arch_trapframe` slot so scratch registers survive (the x86-named `UserRegs` +/// the shared scheduler uses carries only a subset). +fn production_irq_handler(f: &mut super::vectors::TrapFrame) { + let intid = super::gic::acknowledge(); + + // Spurious / non-timer interrupts: EOI (if real) and return. + if intid != super::timer::TIMER_PPI { + if intid < 1020 { + super::gic::eoi(intid); + } + return; + } + + // Timer tick bookkeeping + re-arm the next interval. + super::timer::on_tick(); + // EOI now so the GIC can deliver the next timer interrupt right after eret. + super::gic::eoi(intid); + + // Wake threads whose timerfd deadlines have elapsed, and deliver any pending + // cross-thread wakes — exactly as the x86 APIC-timer ISR does (idt.rs). On + // aarch64 this was missing, so the engine's UI/raster/IO worker threads that + // epoll_wait on a timerfd went Blocked and were NEVER transitioned back to + // Running → the cooperative scheduler found no runnable thread and the whole + // engine deadlocked right after spawning its worker pool. The try_-variants + // are ISR-safe (skip on lock contention rather than deadlock). + crate::syscall::check_timerfds_and_wake_try(); + crate::process::handle_pending_wakes_try(); + + // Only preempt interrupts taken FROM EL0 (a running user thread). SPSR_EL1 + // M[3:0] == 0 means the interrupted context was EL0t. Kernel-mode ticks + // (idle/cortex loop) just return — there is no user frame to switch. + let from_el0 = (f.spsr & 0xF) == 0; + if !from_el0 { + return; + } + + let cur = crate::process::current_pid(); + if cur == 0 { + return; + } + + // Quantum accounting + the same "preempt toward the input/focus target" + // policy the x86 handler applies. + let slice_expired = crate::process::account_tick_try(cur).unwrap_or(false); + let focus = crate::wm::focus_pid(); + let target = if focus != 0 { focus } else { 1 }; + let should_preempt = slice_expired + || (cur != 0 + && target != 0 + && cur != target + && (crate::wm::input_pending_for(target) > 0 + || (target == 1 && crate::wm::embedder_baton_due()))); + { + static PREEMPT_TICK_LOG: AtomicU32 = AtomicU32::new(0); + let n = PREEMPT_TICK_LOG.fetch_add(1, Ordering::Relaxed); + if n < 4000 && (n % 64 == 0 || should_preempt) { + log::warn!( + "[arm-preempt-tick] #{} cur={} slice_expired={} should={} focus={} states={}", + n, cur, slice_expired, should_preempt, focus, + crate::process::debug_runnable_states() + ); + } + } + if !should_preempt { + return; + } + + // Map the live EL0 frame → the shared `UserRegs` (x86-named subset) the + // scheduler operates on. SP_EL0→rsp, ELR_EL1→rip, SPSR→rflags; args/ + // callee-saved per the Linux aarch64 ABI mapping used elsewhere in the port. + let cur_regs = trapframe_to_userregs(f); + + // Snapshot the FULL ARM frame for `cur` so its scratch registers survive. + // If the PTABLE lock is busy, skip preemption this tick (resume cur) rather + // than switch with a half-saved frame. + let full: [u64; 36] = frame_as_array(f); + if !crate::process::arch_store_trapframe(cur, &full) { + return; + } + + let switch = crate::process::timer_preempt_switch_try(cur, &cur_regs); + { + static SWITCH_LOG: AtomicU32 = AtomicU32::new(0); + let n = SWITCH_LOG.fetch_add(1, Ordering::Relaxed); + if n < 400 { + log::warn!( + "[arm-preempt-switch] #{} cur={} -> next={:?} states={}", + n, cur, switch.as_ref().map(|s| s.0), + crate::process::debug_runnable_states() + ); + } + } + if let Some((next_pid, next_regs, ttbr0)) = switch + { + // Switch the low-half translation base to the next thread's space. + crate::arch::memory::write_cr3(ttbr0); + + // Restore the next thread's frame. Prefer its full ARM snapshot (taken + // when IT was last timer-preempted); fall back to rebuilding from the + // shared `UserRegs` for a thread that last yielded via a syscall / is + // entering fresh. + if let Some(saved) = crate::process::arch_take_trapframe(next_pid) { + apply_array_to_frame(f, &saved); + } else { + userregs_to_trapframe(f, &next_regs); + } + } +} + +/// Build the shared `UserRegs` from a live EL0 trap frame. +fn trapframe_to_userregs(f: &super::vectors::TrapFrame) -> crate::process::UserRegs { + crate::process::UserRegs { + rip: f.elr, + rsp: f.sp_el0, + rflags: f.spsr, + rax: f.x[8], + rdi: f.x[0], + rsi: f.x[1], + rdx: f.x[2], + rcx: f.x[3], + r8: f.x[4], + r9: f.x[5], + r10: f.x[9], + r11: f.x[10], + rbx: f.x[19], + r12: f.x[20], + r13: f.x[21], + r14: f.x[22], + r15: f.x[23], + rbp: f.x[29], + } +} + +/// Write a shared `UserRegs` back into a live EL0 trap frame (lossy fallback +/// path — used only when the next thread has no full ARM snapshot). +fn userregs_to_trapframe(f: &mut super::vectors::TrapFrame, r: &crate::process::UserRegs) { + f.elr = r.rip; + f.sp_el0 = r.rsp; + // Force a sane EL0t PSTATE (SPSR M[3:0]=0, interrupts unmasked) on resume. + f.spsr = 0; + f.x[8] = r.rax; + f.x[0] = r.rdi; + f.x[1] = r.rsi; + f.x[2] = r.rdx; + f.x[3] = r.rcx; + f.x[4] = r.r8; + f.x[5] = r.r9; + f.x[9] = r.r10; + f.x[10] = r.r11; + f.x[19] = r.rbx; + f.x[20] = r.r12; + f.x[21] = r.r13; + f.x[22] = r.r14; + f.x[23] = r.r15; + f.x[29] = r.rbp; +} + +/// Serialise a trap frame to the 36-word array stored in `Process::arch_trapframe`. +/// Layout matches `vectors::TrapFrame` / the SAVE_FRAME stack image: +/// [0..31]=x0..x30, 31=SP_EL0, 32=ELR, 33=SPSR, 34=ESR, 35=pad. +fn frame_as_array(f: &super::vectors::TrapFrame) -> [u64; 36] { + let mut a = [0u64; 36]; + a[..31].copy_from_slice(&f.x); + a[31] = f.sp_el0; + a[32] = f.elr; + a[33] = f.spsr; + a[34] = f.esr; + a +} + +/// Restore a 36-word saved array into a live trap frame. +fn apply_array_to_frame(f: &mut super::vectors::TrapFrame, a: &[u64; 36]) { + f.x.copy_from_slice(&a[..31]); + f.sp_el0 = a[31]; + f.elr = a[32]; + f.spsr = a[33]; + f.esr = a[34]; +} + +/// Finish any deferred MMIO mapping after `mm::init()`. +/// +/// On x86 this maps the xAPIC MMIO window; on aarch64 the GIC MMIO regions would +/// be mapped here. No-op stub. +pub fn finish_xapic_init() {} + +/// Per-AP interrupt-controller init. +/// +/// TODO(arm): enable this core's GIC redistributor/CPU interface. +pub fn init_ap() {} + +/// Signal End-Of-Interrupt to the controller. +/// +/// TODO(arm): write the interrupt id to GICC_EOIR / ICC_EOIR1_EL1. +pub fn eoi() {} + +/// Route the legacy serial/ExtINT line. +/// +/// x86-specific (LINT0 in ExtINT mode). No aarch64 equivalent — no-op. +pub fn configure_lint0_for_extint() {} + +/// Return this core's interrupt-controller id (x86 LAPIC-id analogue). +/// +/// Derived from MPIDR_EL1 affinity bits. +pub fn local_apic_id() -> u32 { + let v: u64; + unsafe { core::arch::asm!("mrs {}, mpidr_el1", out(reg) v, options(nomem, nostack)) }; + (v & 0x00FF_FFFF) as u32 +} + +/// Send a reschedule IPI to the given target core. +/// +/// TODO(arm): raise a GIC Software-Generated Interrupt (SGI) to `_target`. +pub fn send_resched_ipi(_target_lapic_id: u32) {} + +// ── Vsync cadence ─────────────────────────────────────────────────────────── + +/// Set the desired vsync cadence (Hz). 0 disables it. +pub fn set_vsync_hz(hz: u32) { + VSYNC_HZ.store(hz, Ordering::Release); +} + +/// Returns true when a vsync interval has elapsed since the last reset. +/// +/// TODO(arm): compare CNTVCT_EL0 against the last fired timestamp scaled by +/// CNTFRQ_EL0 / VSYNC_HZ. Always returns false in the scaffold. +pub fn vsync_due() -> bool { + false +} + +/// Reset the vsync interval reference timestamp. +/// +/// TODO(arm): snapshot CNTVCT_EL0. No-op stub. +pub fn reset_vsync_last_tsc() {} diff --git a/kernel/src/arch/aarch64/boot.rs b/kernel/src/arch/aarch64/boot.rs new file mode 100644 index 0000000..1dbabf1 --- /dev/null +++ b/kernel/src/arch/aarch64/boot.rs @@ -0,0 +1,138 @@ +//! aarch64 direct-boot entry path (QEMU `-M virt -kernel`). +//! +//! This is the bare-metal bring-up entry for the ARM port. QEMU loads the +//! kernel ELF at its physical link address (0x4008_0000 on `-M virt`) and jumps +//! to `_start` with the MMU OFF, caches off, and (for `-cpu cortex-a72` without +//! `virtualization=on`) at EL1. `_start`: +//! +//! 1. Parks all but the primary CPU (MPIDR affinity 0). +//! 2. If we entered at EL2, drops to EL1 via `eret`. +//! 3. Sets up the boot stack. +//! 4. Zeroes the BSS. +//! 5. Branches to `aarch64_start` (Rust), which brings up serial → MMU → +//! exceptions → GIC → timer → SVC → an EL0 user process. +//! +//! The whole bring-up runs identity-mapped low-physical until the MMU is on. + +use core::arch::global_asm; + +global_asm!( + r#" +.section .text.boot, "ax" +.globl _start +_start: + // QEMU `-kernel` passes the Flattened Device Tree (DTB) pointer in x0 at + // entry. Preserve it in x20 (callee-saved, untouched by the bring-up asm) + // so we can stash it once the BSS — which holds the DTB-pointer static — is + // zeroed. The MPIDR check below clobbers x0. + mov x20, x0 + + // ── Park secondary CPUs: only MPIDR.Aff0==0 proceeds ────────────────── + mrs x0, mpidr_el1 + and x0, x0, #0xFF + cbz x0, 1f +0: wfe + b 0b +1: + // ── If we booted at EL2, drop to EL1 ────────────────────────────────── + mrs x0, CurrentEL + lsr x0, x0, #2 + cmp x0, #2 + b.ne 2f // already at EL1 (or EL3 — unhandled) + + // Configure EL2 so EL1 runs in AArch64 with sane state, then eret to EL1. + mov x0, #(1 << 31) // HCR_EL2.RW = 1 → EL1 is AArch64 + msr hcr_el2, x0 + // SCTLR_EL1: reserved-1 bits set, MMU/caches off for now. + mov x0, #0x0800 + movk x0, #0x30d0, lsl #16 + msr sctlr_el1, x0 + // SPSR_EL2: mask DAIF, return to EL1h (use SP_EL1). + mov x0, #0x3c5 // D,A,I,F masked + mode EL1h(0b0101) + msr spsr_el2, x0 + adr x0, 2f + msr elr_el2, x0 + eret + +2: // ── Now at EL1 ──────────────────────────────────────────────────────── + // Mask all interrupts during bring-up. + msr daifset, #0xf + + // Enable FP/AdvSIMD at EL1/EL0 (CPACR_EL1.FPEN = 0b11). The Rust codegen + // emits SIMD/FP instructions even in early bring-up (e.g. alignment + // ub-checks use `fmov`), which trap to EL1 unless FPEN is cleared. + mrs x0, cpacr_el1 + orr x0, x0, #(3 << 20) + msr cpacr_el1, x0 + isb + + // Set up the boot stack (top of a BSS-resident stack region). + adrp x0, {boot_stack_top} + add x0, x0, :lo12:{boot_stack_top} + mov sp, x0 + + // ── Zero BSS ────────────────────────────────────────────────────────── + adrp x1, {bss_start} + add x1, x1, :lo12:{bss_start} + adrp x2, {bss_end} + add x2, x2, :lo12:{bss_end} +3: cmp x1, x2 + b.hs 4f + str xzr, [x1], #8 + b 3b +4: + // ── Into Rust ───────────────────────────────────────────────────────── + // AAPCS64: pass the saved DTB pointer (x20) to aarch64_start as arg0 (x0). + mov x0, x20 + bl {aarch64_start} + // aarch64_start never returns; park if it does. +5: wfe + b 5b +"#, + boot_stack_top = sym BOOT_STACK_TOP, + bss_start = sym crate::arch::aarch64::boot::__bss_start_sym, + bss_end = sym crate::arch::aarch64::boot::__bss_end_sym, + aarch64_start = sym aarch64_start, +); + +// ── Boot stack (BSS-resident, 64 KiB) ─────────────────────────────────────── +// +// Defined in assembly so the "top" label sits exactly one-past-the-end of the +// reserved region (the stack grows down from BOOT_STACK_TOP). +core::arch::global_asm!( + r#" +.section .bss.boot_stack, "aw", @nobits +.balign 16 +BOOT_STACK_BOTTOM: + .skip 65536 +.globl BOOT_STACK_TOP +BOOT_STACK_TOP: +"# +); + +extern "C" { + static BOOT_STACK_TOP: u8; +} + +// Re-export the linker BSS bounds as Rust symbols the global_asm can reference. +extern "C" { + #[link_name = "__bss_start"] + pub static __bss_start_sym: u8; + #[link_name = "__bss_end"] + pub static __bss_end_sym: u8; +} + +/// First Rust function on the aarch64 boot path. Runs identity-mapped, MMU off, +/// at EL1, on the boot stack. `dtb_ptr` is the value QEMU placed in x0 (the +/// Flattened Device Tree blob's physical address). +/// +/// Hands off to the production boot path (`aarch64_kernel_boot`), which records +/// the DTB pointer, runs the minimal ARM early-init (serial + MMU), then enters +/// the SHARED `kernel_main`. The self-contained bring-up demo is still reachable +/// via [`crate::arch::aarch64::bringup::run`] for platform-primitive validation. +#[no_mangle] +pub extern "C" fn aarch64_start(dtb_ptr: u64) -> ! { + // Record the DTB pointer now that BSS is zeroed (the static lives in BSS). + crate::arch::aarch64::fdt::set_dtb_ptr(dtb_ptr); + crate::arch::aarch64::boot_prod::aarch64_kernel_boot(dtb_ptr) +} diff --git a/kernel/src/arch/aarch64/boot_prod.rs b/kernel/src/arch/aarch64/boot_prod.rs new file mode 100644 index 0000000..735e057 --- /dev/null +++ b/kernel/src/arch/aarch64/boot_prod.rs @@ -0,0 +1,121 @@ +//! aarch64 production boot path — route the ARM `-kernel` entry into the SAME +//! shared `kernel_main` the x86 (Limine) path uses. +//! +//! The x86 production kernel boots via Limine, which hands the kernel a memory +//! map, an HHDM offset, a framebuffer, and modules. On `qemu-system-aarch64 -M +//! virt` there is no Limine: QEMU loads us via `-kernel` and passes a Flattened +//! Device Tree (DTB) pointer in x0. This module bridges the gap: +//! +//! 1. Bring up the ARM platform primitives the shared kernel assumes are +//! already live by the time `kernel_main` runs: PL011 serial, the MMU +//! (identity map of devices + RAM), the EL1 exception vectors, and the +//! GIC + generic timer. +//! 2. Parse the DTB to discover usable RAM and build the architecture-neutral +//! [`crate::mm::BootMemMap`]. +//! 3. Call the shared `kernel_main` entry (`crate::kernel_main_arch`), which +//! runs the same subsystem init (heap, scheduler, VFS, process table, …) +//! and ultimately spawns the init/host process. +//! +//! Limine-only inputs the shared path expects (framebuffer, engine module) are +//! provided as "absent" on ARM for now — those are follow-on work; the goal of +//! this path is to reach userspace. + +use crate::arch::aarch64::{fdt, uart}; +use crate::arch::aarch64::uart::a64println; + +/// Production ARM boot entry. Called from `aarch64_start` with the DTB pointer. +/// Never returns — hands off to the shared `kernel_main`. +pub fn aarch64_kernel_boot(dtb_ptr: u64) -> ! { + // ── Serial first: the debug lifeline for everything below ─────────────── + uart::init(); + a64println!(); + a64println!("========================================"); + a64println!(" OSCortex aarch64 — PRODUCTION boot"); + a64println!("========================================"); + uart::puts("[arm-boot] DTB pointer (x0) = "); + uart::puthex_full(dtb_ptr); + uart::puts("\n"); + + // ── MMU: identity-map devices + RAM so the shared kernel can touch memory. + // The bring-up MMU covers the low device GiB + 2 GiB of RAM from 0x4000_0000, + // which is enough to reach userspace (QEMU virt default is 2 GiB). + a64println!("[arm-boot] enabling MMU (identity map: devices + 2 GiB RAM)..."); + unsafe { crate::arch::aarch64::mmu::enable(2) }; + a64println!("[arm-boot] MMU on."); + + // ── EL1 exception vectors (VBAR_EL1): needed for SVC syscalls + faults ── + a64println!("[arm-boot] installing EL1 exception vectors..."); + crate::arch::aarch64::vectors::install(); + + // ── GIC: distributor + CPU interface live so IRQs can be delivered ────── + a64println!("[arm-boot] bringing up GIC (distributor + CPU interface)..."); + crate::arch::aarch64::gic::init_distributor(); + crate::arch::aarch64::gic::init_cpu_interface(); + // The generic timer (scheduler tick) is armed later by the shared kernel via + // `arch::apic::init_bsp`; IRQ delivery stays masked until then. + + // ── Locate the DTB ────────────────────────────────────────────────────── + // QEMU passes the DTB pointer in x0 when booting a flat image, but when it + // loads an *ELF* kernel on `-M virt` it instead places the generated DTB at + // the base of RAM (0x4000_0000) and may leave x0 = 0. If x0 didn't point at a + // valid DTB, probe the RAM base for the FDT magic before falling back. + let dtb_at = if dtb_ptr != 0 && unsafe { fdt::parse(dtb_ptr).is_some() } { + dtb_ptr + } else { + const QEMU_VIRT_DTB: u64 = 0x4000_0000; + if unsafe { fdt::parse(QEMU_VIRT_DTB).is_some() } { + uart::puts("[arm-boot] DTB not in x0; found FDT at RAM base 0x40000000\n"); + QEMU_VIRT_DTB + } else if let Some(found) = + // QEMU's exact DTB placement for an ELF `-kernel` depends on the image; + // probe a small bounded window just past the RAM base (cheap under TCG). + // When the bootloader DOES pass a valid x0 (flat-image boot, real HW), + // the first branch above already used it. Otherwise we fall through to + // the architecture default below — equivalent to `-m` on QEMU virt. + unsafe { fdt::scan_for_dtb(0x4000_0000, 0x4400_0000) } + { + uart::puts("[arm-boot] DTB located by RAM scan @ "); + uart::puthex_full(found); + uart::puts("\n"); + found + } else { + dtb_ptr // keep original; parse() below will fail → fallback map + } + }; + fdt::set_dtb_ptr(dtb_at); + + // ── Discover RAM from the device tree → neutral BootMemMap ────────────── + let mut map = crate::mm::BootMemMap::new(); + match unsafe { fdt::parse(dtb_at) } { + Some(info) => { + for i in 0..info.mem_count { + let r = info.mem[i]; + uart::puts("[arm-boot] DTB /memory region: base="); + uart::puthex_full(r.base); + uart::puts(" size="); + uart::puthex_full(r.size); + uart::puts("\n"); + map.push(r.base, r.size); + } + a64println!( + "[arm-boot] device tree: {} MiB RAM across {} region(s)", + info.ram_total() / (1024 * 1024), + info.mem_count + ); + } + None => { + // No usable DTB — fall back to the QEMU virt default (RAM @ 0x4000_0000). + // 2 GiB matches `scripts/run-aarch64.sh -m 2G` and the MMU map above. + a64println!("[arm-boot] WARNING: no valid DTB /memory node; assuming 2 GiB @ 0x40000000"); + map.push(0x4000_0000, 2u64 * 1024 * 1024 * 1024); + } + } + + // ── HHDM offset: 0 on aarch64. The bring-up MMU identity-maps RAM, so the + // direct-map virtual address equals the physical address (VA == PA). The + // frame allocator / paging layer therefore use a zero phys→virt offset. + let hhdm_offset: u64 = 0; + + a64println!("[arm-boot] handing off to shared kernel_main ..."); + crate::kernel_main_arch(map, hhdm_offset) +} diff --git a/kernel/src/arch/aarch64/bringup.rs b/kernel/src/arch/aarch64/bringup.rs new file mode 100644 index 0000000..d6259b1 --- /dev/null +++ b/kernel/src/arch/aarch64/bringup.rs @@ -0,0 +1,62 @@ +//! aarch64 bare-metal bring-up sequence. +//! +//! Driven from the assembly `_start` (boot.rs) via `aarch64_start`. Runs the +//! platform-primitive milestones in dependency order, printing progress over +//! PL011 serial at every step: +//! +//! 1. SERIAL — PL011 UART online (the debug lifeline) +//! 2. MMU — ARMv8 translation tables, MAIR/TCR, SCTLR_EL1.M=1 +//! 3. VECTORS — VBAR_EL1 → 16-entry EL1 vector table, trap frame save/restore +//! 4. GIC — distributor + CPU interface, EOI, timer IRQ routing +//! 5. TIMER — CNTP/CNTV periodic tick → scheduler-tick analogue +//! 6. SVC — EL0 `svc #0` → capture user GPRs → dispatch → ERET +//! 7. EL0 — enter a user process and service its syscalls +//! +//! This is a self-contained proof of the ARM platform layer: it exercises every +//! arch primitive the shared kernel needs, without depending on the Limine boot +//! protocol (which the x86 production path uses). Once these primitives are +//! proven, the shared scheduler/mm/syscall layers can be brought up on top. + +use crate::arch::aarch64::uart; +use crate::arch::aarch64::uart::a64println; + +/// Entry from `aarch64_start`. Never returns. +pub fn run() -> ! { + // ── Milestone 1: SERIAL ──────────────────────────────────────────────── + uart::init(); + a64println!(); + a64println!("========================================"); + a64println!(" OSCortex aarch64 bring-up"); + a64println!("========================================"); + a64println!("[boot] PL011 UART online @ 0x09000000"); + + // Report the Exception Level we landed at and key system registers, so the + // serial log alone tells us the CPU state we're bringing up from. + let current_el: u64; + let sctlr: u64; + let mpidr: u64; + let cntfrq: u64; + unsafe { + core::arch::asm!("mrs {}, CurrentEL", out(reg) current_el, options(nomem, nostack)); + core::arch::asm!("mrs {}, sctlr_el1", out(reg) sctlr, options(nomem, nostack)); + core::arch::asm!("mrs {}, mpidr_el1", out(reg) mpidr, options(nomem, nostack)); + core::arch::asm!("mrs {}, cntfrq_el0", out(reg) cntfrq, options(nomem, nostack)); + } + a64println!("[boot] CurrentEL = EL{}", (current_el >> 2) & 0x3); + uart::puts("[boot] SCTLR_EL1 = "); + uart::puthex_full(sctlr); + uart::puts(" (MMU "); + uart::puts(if sctlr & 1 != 0 { "ON" } else { "OFF" }); + uart::puts(")\n"); + uart::puts("[boot] MPIDR_EL1 = "); + uart::puthex_full(mpidr); + uart::puts("\n"); + uart::puts("[boot] CNTFRQ_EL0 = "); + uart::puthex(cntfrq); + a64println!(" ({} Hz timer)", cntfrq); + + a64println!("[boot] Milestone 1 (SERIAL) complete."); + + // Remaining milestones are wired in subsequent steps. + crate::arch::aarch64::bringup_mmu::run(); +} diff --git a/kernel/src/arch/aarch64/bringup_irq.rs b/kernel/src/arch/aarch64/bringup_irq.rs new file mode 100644 index 0000000..c96c70d --- /dev/null +++ b/kernel/src/arch/aarch64/bringup_irq.rs @@ -0,0 +1,64 @@ +//! aarch64 bring-up milestones 4+5: GIC + generic timer periodic tick. + +use crate::arch::aarch64::uart::a64println; +use crate::arch::aarch64::{gic, timer, vectors}; +use core::sync::atomic::Ordering; + +/// IRQ handler: acknowledge at the GIC, service the timer, EOI. +fn irq_handler(_f: &mut vectors::TrapFrame) { + let intid = gic::acknowledge(); + if intid == timer::TIMER_PPI { + timer::on_tick(); + } + if intid < 1020 { + gic::eoi(intid); + } +} + +pub fn run() -> ! { + a64println!("[boot] Milestone 4 (GIC): init distributor + CPU interface..."); + gic::init_distributor(); + gic::init_cpu_interface(); + a64println!("[boot] GIC online (GICv2 @ GICD 0x08000000 / GICC 0x08010000)."); + + a64println!("[boot] Milestone 5 (TIMER): arming generic-timer @ 100 Hz..."); + vectors::set_irq_handler(irq_handler); + timer::init_periodic(100); + + // Unmask IRQ delivery and let the timer tick. + unsafe { core::arch::asm!("msr daifclr, #0b0010", options(nomem, nostack)) }; + + // Spin until we observe a handful of ticks, then report. + let start = timer::count(); + let mut last_report = 0u64; + loop { + let ticks = timer::TICKS.load(Ordering::Relaxed); + if ticks >= 5 { + break; + } + if ticks != last_report { + last_report = ticks; + } + // Wait for the next interrupt. + unsafe { core::arch::asm!("wfi", options(nomem, nostack)) }; + } + let elapsed = timer::count() - start; + let freq = timer::freq_hz(); + + a64println!( + "[boot] timer ticks = {}, IRQs = {}, elapsed counts = {} (~{} ms @ {} Hz)", + timer::TICKS.load(Ordering::Relaxed), + vectors::IRQ_COUNT.load(Ordering::Relaxed), + elapsed, + if freq != 0 { elapsed * 1000 / freq } else { 0 }, + freq + ); + + if timer::TICKS.load(Ordering::Relaxed) >= 5 { + a64println!("[boot] Milestone 4+5 (GIC + TIMER) complete — periodic IRQ ticking."); + } else { + a64println!("[boot] Milestone 4+5 FAILED — no timer ticks observed."); + } + + crate::arch::aarch64::bringup_user::run(); +} diff --git a/kernel/src/arch/aarch64/bringup_mmu.rs b/kernel/src/arch/aarch64/bringup_mmu.rs new file mode 100644 index 0000000..a95be7b --- /dev/null +++ b/kernel/src/arch/aarch64/bringup_mmu.rs @@ -0,0 +1,44 @@ +//! aarch64 bring-up milestone 2: enable the MMU. + +use crate::arch::aarch64::mmu; +use crate::arch::aarch64::uart; +use crate::arch::aarch64::uart::a64println; + +pub fn run() -> ! { + a64println!("[boot] Milestone 2 (MMU): building translation tables..."); + + // Identity-map device MMIO (low 1 GiB) + 2 GiB of RAM from 0x4000_0000. + unsafe { mmu::enable(2) }; + + // Read SCTLR_EL1 back to confirm M=1. If we get here at all, the MMU is on + // and translating — the very next instruction fetch was a translated VA. + let sctlr: u64; + let ttbr0: u64; + unsafe { + core::arch::asm!("mrs {}, sctlr_el1", out(reg) sctlr, options(nomem, nostack)); + core::arch::asm!("mrs {}, ttbr0_el1", out(reg) ttbr0, options(nomem, nostack)); + } + uart::puts("[boot] SCTLR_EL1 = "); + uart::puthex_full(sctlr); + uart::puts(" (MMU "); + uart::puts(if sctlr & 1 != 0 { "ON" } else { "OFF" }); + uart::puts(")\n"); + uart::puts("[boot] TTBR0_EL1 = "); + uart::puthex_full(ttbr0); + uart::puts("\n"); + + // Prove translation works: a read+write through a RAM VA round-trips. (If + // the map were wrong this access would take a translation fault.) + let probe: u64 = 0x4010_0000; // 1 MiB into RAM, page-aligned, scratch + unsafe { + core::ptr::write_volatile(probe as *mut u64, 0xCAFE_F00D_1234_5678); + let v = core::ptr::read_volatile(probe as *const u64); + uart::puts("[boot] RAM r/w probe @ 0x40100000 = "); + uart::puthex_full(v); + uart::puts(if v == 0xCAFE_F00D_1234_5678 { " OK\n" } else { " FAIL\n" }); + } + + a64println!("[boot] Milestone 2 (MMU) complete — translating."); + + crate::arch::aarch64::bringup_vectors::run(); +} diff --git a/kernel/src/arch/aarch64/bringup_smp.rs b/kernel/src/arch/aarch64/bringup_smp.rs new file mode 100644 index 0000000..167a4f5 --- /dev/null +++ b/kernel/src/arch/aarch64/bringup_smp.rs @@ -0,0 +1,122 @@ +//! aarch64 bring-up milestone 8 (optional): secondary-core wake via PSCI. +//! +//! On QEMU `-M virt` the boot CPU lands at EL1 and the PSCI conduit is HVC. +//! We bring CPU 1 online with PSCI CPU_ON, pointing it at a physical secondary +//! entry stub that sets up its stack, enables FP, installs the shared vector +//! table, and reports in over serial. This proves multi-core wake-up; the full +//! per-CPU scheduler is a larger follow-up (the BSP path already runs userspace). +//! +//! Single-core boot remains the proven path — this milestone only runs when a +//! second CPU is present and PSCI accepts the CPU_ON call. + +use crate::arch::aarch64::uart; +use crate::arch::aarch64::uart::a64println; +use core::arch::global_asm; +use core::sync::atomic::{AtomicU64, Ordering}; + +/// Set by the secondary core once it is running at EL1. +pub static SECONDARY_ONLINE: AtomicU64 = AtomicU64::new(0); +/// The secondary core records its MPIDR here. +pub static SECONDARY_MPIDR: AtomicU64 = AtomicU64::new(0); + +// Secondary boot stack (16 KiB). +global_asm!( + r#" +.section .bss.secondary_stack, "aw", @nobits +.balign 16 +SECONDARY_STACK_BOTTOM: + .skip 16384 +.globl SECONDARY_STACK_TOP +SECONDARY_STACK_TOP: +"# +); + +extern "C" { + static SECONDARY_STACK_TOP: u8; +} + +// Physical secondary entry stub. PSCI starts it with the MMU OFF at EL1, so it +// must run identity-mapped (which our identity map provides) and set up a stack +// before calling into Rust. +global_asm!( + r#" +.section .text +.globl __secondary_entry +__secondary_entry: + // Enable FP/AdvSIMD (Rust codegen needs it; see boot.rs). + mrs x0, cpacr_el1 + orr x0, x0, #(3 << 20) + msr cpacr_el1, x0 + isb + // Stack. + adrp x0, SECONDARY_STACK_TOP + add x0, x0, :lo12:SECONDARY_STACK_TOP + mov sp, x0 + // Into Rust. + bl {secondary_main} +0: wfe + b 0b +"#, + secondary_main = sym secondary_main, +); + +/// Rust entry for the secondary core (runs at EL1, MMU initially off — but the +/// BSP's tables are global; we just need this core's TTBR/SCTLR. For this +/// report-in milestone we run translation-off and only touch device MMIO + a +/// couple of static atomics, both physically addressed.) +#[no_mangle] +extern "C" fn secondary_main() -> ! { + let mpidr: u64; + unsafe { core::arch::asm!("mrs {}, mpidr_el1", out(reg) mpidr, options(nomem, nostack)) }; + SECONDARY_MPIDR.store(mpidr & 0xFF_FFFF, Ordering::Release); + SECONDARY_ONLINE.store(1, Ordering::Release); + // Park; the per-CPU scheduler is a follow-up. + loop { + unsafe { core::arch::asm!("wfe", options(nomem, nostack)) }; + } +} + +extern "C" { + static __secondary_entry: u8; +} + +/// Try to bring CPU 1 online via PSCI CPU_ON. Reports the outcome over serial. +pub fn try_wake_secondary() { + use crate::arch::aarch64::psci; + + let entry = unsafe { core::ptr::addr_of!(__secondary_entry) as u64 }; + // CPU 1's MPIDR on QEMU virt cortex-a72 is affinity 1 (0x1). + let target_mpidr: u64 = 1; + + a64println!("[boot] Milestone 8 (SMP): PSCI CPU_ON cpu1 @ entry..."); + uart::puts("[boot] secondary entry PA = "); + uart::puthex_full(entry); + uart::puts("\n"); + + let rc = psci::cpu_on(target_mpidr, entry, 0); + uart::puts("[boot] PSCI CPU_ON returned "); + uart::puthex(rc as u64); + uart::puts("\n"); + + if rc != 0 { + a64println!("[boot] SMP: PSCI CPU_ON not available in this boot mode (single-core path proven)."); + return; + } + + // Wait briefly for the secondary to report in. + for _ in 0..2_000_000u64 { + if SECONDARY_ONLINE.load(Ordering::Acquire) != 0 { + break; + } + core::hint::spin_loop(); + } + + if SECONDARY_ONLINE.load(Ordering::Acquire) != 0 { + uart::puts("[boot] SMP: CPU 1 ONLINE, MPIDR = "); + uart::puthex_full(SECONDARY_MPIDR.load(Ordering::Acquire)); + uart::puts("\n"); + a64println!("[boot] Milestone 8 (SMP) complete — secondary core woken via PSCI."); + } else { + a64println!("[boot] SMP: secondary did not report in (single-core path proven)."); + } +} diff --git a/kernel/src/arch/aarch64/bringup_user.rs b/kernel/src/arch/aarch64/bringup_user.rs new file mode 100644 index 0000000..f757571 --- /dev/null +++ b/kernel/src/arch/aarch64/bringup_user.rs @@ -0,0 +1,320 @@ +//! aarch64 bring-up milestones 6+7: SVC syscall entry + EL0 user process. +//! +//! This stage proves the kernel can drop to EL0, run a userspace program, and +//! service its `svc #0` syscalls — the headline success bar for the ARM port. +//! +//! We build a tiny self-contained EL0 program (hand-assembled AArch64 machine +//! code), map it + a stack into an EL0-accessible page, install an SVC handler +//! that captures the user GPRs and dispatches the syscall, then `eret` into the +//! program. The program issues several `write` syscalls and an `exit`; the SVC +//! handler prints the writes over serial and stops the machine on exit. +//! +//! Syscall ABI (Linux aarch64 subset): nr in x8, args x0..x5, return in x0. +//! nr 64 = write(fd, buf, len) → buf/len printed to the serial console +//! nr 93 = exit(code) → halts the bring-up with the code +//! nr 172 = getpid() → returns a fixed pid (proves a returning call) + +use crate::arch::aarch64::uart; +use crate::arch::aarch64::uart::a64println; +use crate::arch::aarch64::{enter_user, mmu, vectors}; +use core::sync::atomic::{AtomicBool, AtomicU64, Ordering}; + +/// User VA the program + stack are mapped at (L1 index 16 = 16 GiB, free of the +/// kernel RAM identity map which covers indices 1..2). +const USER_VA_BASE: u64 = 0x4_0000_0000; +/// Offset of the user stack top within the mapped page (page is 4 KiB; code +/// lives at the bottom, stack grows down from near the top). +const USER_STACK_OFF: u64 = 0x0F00; + +/// Physical backing page for the EL0 program (BSS, identity-mapped so the +/// kernel can write code into it). 4 KiB, page-aligned. +#[repr(C, align(4096))] +struct UserPage([u8; 4096]); +static mut USER_PAGE: UserPage = UserPage([0; 4096]); + +/// How many `write` syscalls the EL0 program issued. +static WRITES: AtomicU64 = AtomicU64::new(0); +/// Set true when the program calls exit. +static EXITED: AtomicBool = AtomicBool::new(false); +static EXIT_CODE: AtomicU64 = AtomicU64::new(0); + +/// SVC handler: read the syscall nr/args from the saved user frame, service it, +/// write the return value back into x0. Runs at EL1 with the user frame on the +/// kernel stack; returning lets the vector stub restore + eret to EL0. +fn svc_handler(f: &mut vectors::TrapFrame) { + // Capture the trapping thread's user registers into the per-CPU snapshot, + // exactly as the shared dispatch path expects (user_rip/user_rsp/user_gprs). + crate::arch::aarch64::syscall::capture_from_trap(f); + + // On the first syscall, prove the shared per-CPU snapshot accessors now + // reflect the trapping EL0 thread (user_rip should be just past the svc, + // user_rsp should be the EL0 stack). + static SNAP_CHECKED: AtomicBool = AtomicBool::new(false); + if !SNAP_CHECKED.swap(true, Ordering::Relaxed) { + let rip = crate::arch::aarch64::syscall::user_rip(); + let rsp = crate::arch::aarch64::syscall::user_rsp(); + uart::puts("[svc] snapshot: user_rip="); + uart::puthex_full(rip); + uart::puts(" user_rsp="); + uart::puthex_full(rsp); + uart::puts("\n"); + } + + let nr = f.x[8]; + match nr { + 64 => { + // write(fd=x0, buf=x1, len=x2). buf is a user VA; since the user + // page is also identity-mapped via its physical address we read it + // through the physical alias for safety. + let buf_va = f.x[1]; + let len = f.x[2] as usize; + print_user_buf(buf_va, len); + WRITES.fetch_add(1, Ordering::Relaxed); + f.x[0] = len as u64; // bytes written + } + 172 => { + // getpid() — return a fixed value to prove a value round-trips. + f.x[0] = 0x05; + } + 93 => { + EXIT_CODE.store(f.x[0], Ordering::Release); + EXITED.store(true, Ordering::Release); + // Leave the frame; the post-eret instruction is a spin we never + // re-enter because the bring-up loop below sees EXITED and stops. + f.x[0] = 0; + } + _ => { + uart::puts("[svc] unknown nr="); + uart::puthex(nr); + uart::puts("\n"); + f.x[0] = u64::MAX; // -1 + } + } +} + +/// Print `len` bytes from user VA `buf_va`. The user VA maps to a known offset +/// inside USER_PAGE; translate via that physical alias. +fn print_user_buf(buf_va: u64, len: usize) { + let page_base = unsafe { core::ptr::addr_of!(USER_PAGE) as u64 }; + // user VA = USER_VA_BASE + offset → physical = page_base + offset. + if buf_va < USER_VA_BASE || buf_va >= USER_VA_BASE + 4096 { + return; + } + let off = (buf_va - USER_VA_BASE) as usize; + let end = (off + len).min(4096); + let base = unsafe { core::ptr::addr_of!((*core::ptr::addr_of!(USER_PAGE)).0) as *const u8 }; + for i in off..end { + let b = unsafe { core::ptr::read_volatile(base.add(i)) }; + uart::putb(b); + } + let _ = page_base; +} + +/// Assemble the EL0 test program into the user page and return (entry_off, +/// msg_off, msg_len). The program issues two writes, a getpid, a third write, +/// then exit(7). +fn build_user_program() -> (u64, u64, usize) { + // Message lives at the top half of the page; code at the bottom. + let msg = b"[EL0] hello from userspace via svc #0\n"; + let msg_off = 0x800usize; + let pid_msg = b"[EL0] getpid returned a value, calling exit(7)\n"; + let pid_msg_off = 0x880usize; + + unsafe { + let page = &mut (*core::ptr::addr_of_mut!(USER_PAGE)).0; + page[msg_off..msg_off + msg.len()].copy_from_slice(msg); + page[pid_msg_off..pid_msg_off + pid_msg.len()].copy_from_slice(pid_msg); + } + + // Hand-assembled AArch64 at code offset 0. We compute absolute user VAs for + // the message buffers (USER_VA_BASE + off) and emit instructions to load + // them. We keep it simple: a sequence of MOV-immediate + svc. + let code_off = 0usize; + let mut asm: heapless_vec::Vec = heapless_vec::Vec::new(); + + let msg_va = USER_VA_BASE + msg_off as u64; + let pid_msg_va = USER_VA_BASE + pid_msg_off as u64; + + // write(1, msg_va, msg.len()) + emit_load_imm(&mut asm, 0, 1); // x0 = 1 (fd) + emit_load_imm(&mut asm, 1, msg_va); // x1 = buf + emit_load_imm(&mut asm, 2, msg.len() as u64); // x2 = len + emit_load_imm(&mut asm, 8, 64); // x8 = write + asm.push(SVC0); + + // write again (proves repeated syscalls + that x-regs survive) + emit_load_imm(&mut asm, 0, 1); + emit_load_imm(&mut asm, 1, msg_va); + emit_load_imm(&mut asm, 2, msg.len() as u64); + emit_load_imm(&mut asm, 8, 64); + asm.push(SVC0); + + // getpid() + emit_load_imm(&mut asm, 8, 172); + asm.push(SVC0); + + // write(1, pid_msg_va, pid_msg.len()) + emit_load_imm(&mut asm, 0, 1); + emit_load_imm(&mut asm, 1, pid_msg_va); + emit_load_imm(&mut asm, 2, pid_msg.len() as u64); + emit_load_imm(&mut asm, 8, 64); + asm.push(SVC0); + + // exit(7) + emit_load_imm(&mut asm, 0, 7); + emit_load_imm(&mut asm, 8, 93); + asm.push(SVC0); + + // Safety net: infinite loop if exit ever returns. + asm.push(0x14000000); // b . + + // Copy the assembled words into the page at code_off. + unsafe { + let page = &mut (*core::ptr::addr_of_mut!(USER_PAGE)).0; + for (i, w) in asm.iter().enumerate() { + let b = w.to_le_bytes(); + let o = code_off + i * 4; + page[o..o + 4].copy_from_slice(&b); + } + } + + (code_off as u64, msg_off as u64, msg.len()) +} + +/// `svc #0` encoding. +const SVC0: u32 = 0xD4000001; + +/// Emit a MOVZ/MOVK sequence loading the 64-bit immediate `imm` into x`reg`. +fn emit_load_imm(asm: &mut heapless_vec::Vec, reg: u32, imm: u64) { + // MOVZ Xd, #imm16, LSL #shift : 0xD2800000 | (hw<<21) | (imm16<<5) | Rd + // MOVK Xd, #imm16, LSL #shift : 0xF2800000 | (hw<<21) | (imm16<<5) | Rd + let mut first = true; + for hw in 0..4u32 { + let imm16 = ((imm >> (hw * 16)) & 0xFFFF) as u32; + if imm16 == 0 && !(first && hw == 3) { + // Skip zero halfwords, but always emit at least one (MOVZ #0). + if !first { + continue; + } + } + let base = if first { 0xD280_0000 } else { 0xF280_0000 }; + let insn = base | (hw << 21) | (imm16 << 5) | reg; + asm.push(insn); + first = false; + } + if first { + // imm was all zero → emit MOVZ Xreg, #0. + asm.push(0xD280_0000 | reg); + } +} + +pub fn run() -> ! { + // Optional milestone 8: try to wake a secondary core via PSCI before we + // drop to EL0. Best-effort — single-core is the proven path. + crate::arch::aarch64::bringup_smp::try_wake_secondary(); + + a64println!("[boot] Milestone 6 (SVC) + 7 (EL0): launching a user process..."); + + // Map the user page into EL0 at USER_VA_BASE. + let pa = unsafe { core::ptr::addr_of!(USER_PAGE) as u64 }; + unsafe { mmu::map_user_page(USER_VA_BASE, pa) }; + uart::puts("[boot] mapped EL0 page: VA "); + uart::puthex_full(USER_VA_BASE); + uart::puts(" -> PA "); + uart::puthex_full(pa); + uart::puts("\n"); + + // Assemble the EL0 program + install the SVC handler. + let (entry_off, _msg_off, _msg_len) = build_user_program(); + vectors::set_svc_handler(svc_handler); + + let user_pc = USER_VA_BASE + entry_off; + let user_sp = USER_VA_BASE + USER_STACK_OFF; + uart::puts("[boot] entering EL0: PC "); + uart::puthex_full(user_pc); + uart::puts(" SP "); + uart::puthex_full(user_sp); + uart::puts("\n"); + + // The SVC handler runs on the EL1 stack and returns to EL0 via the vector + // stub's eret. exit() sets EXITED; an EL1 IRQ (timer) periodically re-enters + // the kernel, but to deterministically observe exit we poll a shared flag + // from a watcher that the timer IRQ can't see — so instead we make exit + // *not* return to EL0: it leaves the user PC spinning while we detect the + // flag here. We launch EL0 on a separate path that returns control to this + // loop once EXITED is set by routing exit through a longjmp-free re-entry. + // + // Simplest correct approach: enter EL0; the program runs to exit which sets + // the flag; the post-exit `b .` spins in EL0; the EL1 timer IRQ fires, + // re-enters the kernel, and our IRQ-side check below notices EXITED and + // finalizes. We install that check by wrapping the existing IRQ handler. + install_exit_watch_irq(); + + unsafe { enter_user::enter_el0_at(user_pc, user_sp, 0, 0, 0) } +} + +/// Wrap the timer IRQ so that, once the EL0 program has called exit(), the next +/// timer tick finalizes the bring-up from EL1 (the EL0 program is spinning in +/// `b .`). This lets us observe the syscall results and stop cleanly. +fn install_exit_watch_irq() { + use crate::arch::aarch64::{gic, timer}; + fn watch_irq(_f: &mut vectors::TrapFrame) { + let intid = gic::acknowledge(); + if intid == timer::TIMER_PPI { + timer::on_tick(); + if EXITED.load(Ordering::Acquire) { + gic::eoi(intid); + finalize(); + } + } + if intid < 1020 { + gic::eoi(intid); + } + } + vectors::set_irq_handler(watch_irq); +} + +/// Report the syscall results and stop the machine. +fn finalize() -> ! { + let writes = WRITES.load(Ordering::Relaxed); + let code = EXIT_CODE.load(Ordering::Relaxed); + a64println!(); + a64println!( + "[boot] EL0 process serviced: writes={} exit_code={}", + writes, code + ); + if writes == 3 && code == 7 { + a64println!("[boot] Milestone 6+7 (SVC + EL0) complete — userspace serviced over svc #0."); + a64println!(); + a64println!("========================================"); + a64println!(" OSCortex aarch64: BOOT -> EL0 -> SYSCALLS -> EXIT"); + a64println!(" All bring-up milestones passed."); + a64println!("========================================"); + } else { + a64println!("[boot] Milestone 6+7 FAILED — unexpected syscall results."); + } + // Power off via PSCI SYSTEM_OFF so the test harness exits cleanly. + crate::arch::aarch64::psci::system_off(); +} + +// ── Minimal fixed-capacity Vec for assembling the EL0 program (no alloc dep) ── +mod heapless_vec { + pub struct Vec { + buf: [u32; 256], + len: usize, + } + impl Vec { + pub fn new() -> Self { + Vec { buf: [0; 256], len: 0 } + } + pub fn push(&mut self, v: u32) { + if self.len < self.buf.len() { + self.buf[self.len] = v; + self.len += 1; + } + } + pub fn iter(&self) -> core::slice::Iter<'_, u32> { + self.buf[..self.len].iter() + } + } +} diff --git a/kernel/src/arch/aarch64/bringup_vectors.rs b/kernel/src/arch/aarch64/bringup_vectors.rs new file mode 100644 index 0000000..ae4b6d7 --- /dev/null +++ b/kernel/src/arch/aarch64/bringup_vectors.rs @@ -0,0 +1,63 @@ +//! aarch64 bring-up milestone 3: EL1 exception vectors. + +use crate::arch::aarch64::uart; +use crate::arch::aarch64::uart::a64println; +use crate::arch::aarch64::vectors; +use core::sync::atomic::{AtomicU64, Ordering}; + +/// Test SVC handler: bumps a counter and returns (the vector restores+erets). +static TEST_SVC_HITS: AtomicU64 = AtomicU64::new(0); + +fn test_svc_handler(_f: &mut vectors::TrapFrame) { + TEST_SVC_HITS.fetch_add(1, Ordering::Relaxed); +} + +pub fn run() -> ! { + a64println!("[boot] Milestone 3 (VECTORS): installing VBAR_EL1..."); + vectors::install(); + uart::puts("[boot] VBAR_EL1 = "); + uart::puthex_full(vectors::vbar()); + uart::puts("\n"); + + // Prove the full vector round-trip with a *recoverable* trap: issue + // `svc #0` from EL1. The CPU vectors to Current-EL Synchronous, the asm + // stub saves a TrapFrame, the dispatcher sees EC=SVC64 and runs our test + // handler (which bumps a counter), then the stub restores every register + // and `eret`s back to the instruction after the `svc`. If the save/restore + // were wrong we'd fault here; instead execution continues normally. + vectors::set_svc_handler(test_svc_handler); + + let before = vectors::SYNC_EL1.load(Ordering::Relaxed); + // Mark a register before the trap and check it survives the round-trip. + let sentinel_in: u64 = 0xA5A5_1234_DEAD_BEEF; + let sentinel_out: u64; + unsafe { + core::arch::asm!( + "mov x9, {inv}", + "svc #0", + "mov {outv}, x9", + inv = in(reg) sentinel_in, + outv = out(reg) sentinel_out, + out("x9") _, + options(nomem, nostack), + ); + } + let after = vectors::SYNC_EL1.load(Ordering::Relaxed); + let hits = TEST_SVC_HITS.load(Ordering::Relaxed); + + a64println!( + "[boot] svc#0 round-trip: SYNC_EL1 {} -> {}, handler hits = {}", + before, after, hits + ); + uart::puts("[boot] callee reg preserved across trap: "); + uart::puthex_full(sentinel_out); + uart::puts(if sentinel_out == sentinel_in { " OK\n" } else { " CORRUPT\n" }); + + if after == before + 1 && hits == 1 && sentinel_out == sentinel_in { + a64println!("[boot] Milestone 3 (VECTORS) complete — trap save/dispatch/restore/eret OK."); + } else { + a64println!("[boot] Milestone 3 (VECTORS) FAILED — trap round-trip mismatch."); + } + + crate::arch::aarch64::bringup_irq::run(); +} diff --git a/kernel/src/arch/aarch64/cpu.rs b/kernel/src/arch/aarch64/cpu.rs new file mode 100644 index 0000000..2bd7316 --- /dev/null +++ b/kernel/src/arch/aarch64/cpu.rs @@ -0,0 +1,99 @@ +//! CPU feature detection and setup — aarch64. +//! +//! Scaffolding stubs for the ARM port. Real implementations (FPU/SIMD enable +//! via CPACR_EL1, TLS via TPIDR_EL0, FP/SIMD context save/restore) come later. + +use core::arch::asm; + +/// Assert that this CPU supports all features OSCortex requires. +/// +/// On aarch64 the AArch64 base profile already mandates FP+AdvSIMD, so this is +/// currently a no-op placeholder. +pub fn assert_required_features() {} + +/// Enable FP / AdvSIMD (and later SVE) for the AI inference engine. +/// +/// TODO(arm): clear CPACR_EL1.FPEN trap bits so EL0/EL1 may use FP/SIMD. +pub fn enable_fpu_simd() {} + +/// SYSCALL path init — see syscall module. No-op on aarch64 (SVC is always on). +pub fn enable_syscall() {} + +/// Returns true when running under a hypervisor. +/// +/// TODO(arm): inspect MIDR_EL1 / device-tree. Conservatively returns true since +/// the bring-up target is QEMU `virt`. +pub fn running_under_hypervisor() -> bool { + true +} + +/// True when running on a QEMU-like hypervisor. +/// +/// TODO(arm): real detection. The bring-up target is QEMU `virt`, so assume true. +pub fn is_qemu_like_hypervisor() -> bool { + true +} + +/// Set userspace TLS base for the current CPU context (x86 FS-base analogue). +/// +/// On aarch64 user TLS lives in `TPIDR_EL0`. +pub fn set_fs_base(fs_base: u64) { + unsafe { asm!("msr tpidr_el0, {}", in(reg) fs_base, options(nomem, nostack)) }; +} + +/// Read userspace TLS base for the current CPU context. +pub fn get_fs_base() -> u64 { + let v: u64; + unsafe { asm!("mrs {}, tpidr_el0", out(reg) v, options(nomem, nostack)) }; + v +} + +/// Save the current FP/SIMD (and later SVE) register state to `ptr`. +/// +/// # Safety +/// `ptr` must point to a sufficiently large, suitably aligned buffer. +/// +/// TODO(arm): implement via `st1 {v0-v31}` + FPSR/FPCR. Currently a no-op. +#[inline] +pub unsafe fn save_xstate_to(_ptr: *mut u8) {} + +/// Restore the FP/SIMD (and later SVE) register state from `ptr`. +/// +/// # Safety +/// `ptr` must point to a buffer previously written by [`save_xstate_to`]. +/// +/// TODO(arm): implement via `ld1 {v0-v31}` + FPSR/FPCR. Currently a no-op. +#[inline] +pub unsafe fn restore_xstate_from(_ptr: *const u8) {} + +/// Full memory fence (Data Memory Barrier, system scope). +#[inline(always)] +pub fn memory_fence() { + unsafe { asm!("dmb sy", options(nomem, nostack)) }; +} + +/// Hint the CPU during poll loops. +#[inline(always)] +pub fn spin_pause() { + unsafe { asm!("yield", options(nomem, nostack, preserves_flags)) }; +} + +/// Save DAIF (interrupt mask) state and disable IRQ/FIQ; returns the prior DAIF. +#[inline(always)] +pub fn interrupts_save_and_disable() -> u64 { + let daif: u64; + unsafe { + asm!("mrs {}, daif", out(reg) daif, options(nomem, nostack)); + asm!("msr daifset, #0b0011", options(nomem, nostack)); // mask IRQ+FIQ + } + daif +} + +/// Restore interrupt-enable state from [`interrupts_save_and_disable`]. +#[inline(always)] +pub fn interrupts_restore(daif: u64) { + // DAIF.I is bit 7. If it was clear (IRQs were enabled), re-enable them. + if daif & (1 << 7) == 0 { + unsafe { asm!("msr daifclr, #0b0010", options(nomem, nostack)) }; + } +} diff --git a/kernel/src/arch/aarch64/enter_user.rs b/kernel/src/arch/aarch64/enter_user.rs new file mode 100644 index 0000000..8f09fae --- /dev/null +++ b/kernel/src/arch/aarch64/enter_user.rs @@ -0,0 +1,173 @@ +//! aarch64 user-mode entry transitions (EL1 → EL0 via ERET). +//! +//! Mirrors the x86_64 `enter_user` hooks so the shared `process` layer can +//! transfer into ring-3 (EL0) architecture-neutrally. On aarch64 both the IRET +//! and SYSRET variants collapse to an `ERET` after restoring the GPRs from +//! `regs` and setting ELR_EL1 (user PC), SP_EL0 (user SP) and SPSR_EL1 (EL0t, +//! interrupts unmasked). +//! +//! The shared `EnterUserRegs` uses x86 field names; on aarch64 we map them onto +//! AArch64 registers so a userspace built for the Linux aarch64 ABI sees its +//! arguments in the right places: +//! +//! rip → ELR_EL1 (user PC) rsp → SP_EL0 (user SP) +//! rdi → x0 rsi → x1 rdx → x2 rcx → x3 r8 → x4 r9 → x5 +//! rax → x8 (syscall-nr / return register, matching the Linux ABI) +//! rbx → x19 rbp → x29 r10 → x9 r11 → x10 +//! r12 → x20 r13 → x21 r14 → x22 r15 → x23 +//! +//! The exact mapping only matters for the *first* entry of a fresh process +//! (where the shared layer seeds rip/rsp/args); on a syscall-return re-entry the +//! SVC handler has already written the full user frame, so the values flow +//! straight back. + +/// Full user register state required to enter EL0. Field names mirror the +/// x86_64 backend so shared code stays architecture-neutral. +#[derive(Clone, Copy)] +pub struct EnterUserRegs { + pub rip: u64, + pub rsp: u64, + pub rflags: u64, + pub rax: u64, + pub rbx: u64, + pub rcx: u64, + pub rdx: u64, + pub rsi: u64, + pub rdi: u64, + pub rbp: u64, + pub r8: u64, + pub r9: u64, + pub r10: u64, + pub r11: u64, + pub r12: u64, + pub r13: u64, + pub r14: u64, + pub r15: u64, +} + +/// SPSR_EL1 value for "return to EL0t (SP_EL0), all interrupts unmasked". +/// +/// M[3:0] = 0b0000 → EL0t +/// D,A,I,F = 0 → debug/SError/IRQ/FIQ all unmasked in EL0 +const SPSR_EL0T: u64 = 0; + +/// Build the 34-entry image the asm loads from: x0..x30, SP_EL0, PC, SPSR. +#[inline(always)] +fn build_image(regs: &EnterUserRegs) -> [u64; 34] { + let mut img = [0u64; 34]; + img[0] = regs.rdi; // x0 + img[1] = regs.rsi; // x1 + img[2] = regs.rdx; // x2 + img[3] = regs.rcx; // x3 + img[4] = regs.r8; // x4 + img[5] = regs.r9; // x5 + // x6, x7 unused by the seed path — leave zero. + img[8] = regs.rax; // x8 (syscall nr / return) + img[9] = regs.r10; // x9 + img[10] = regs.r11; // x10 + // x11..x18 zero. + img[19] = regs.rbx; // x19 (callee-saved) + img[20] = regs.r12; // x20 + img[21] = regs.r13; // x21 + img[22] = regs.r14; // x22 + img[23] = regs.r15; // x23 + img[29] = regs.rbp; // x29 (frame pointer) + // x30 (LR): the shared enter path stashes the user link register captured at + // the SVC boundary in the otherwise-unused `rflags` slot (SPSR is a constant + // here). A thread that yields mid-syscall MUST resume with x30 intact or its + // next `ret` branches to whatever was here (was hardwired 0 → crash). On the + // very first entry of a fresh thread user_lr is 0, which is correct (the + // thread-return trampoline address sits at [SP], not in LR). + img[30] = regs.rflags; // x30 (LR) + img[31] = regs.rsp; // SP_EL0 + img[32] = regs.rip; // PC → ELR_EL1 + img[33] = SPSR_EL0T; // SPSR_EL1 + img +} + +/// Restore the full user GPR file from `img`, set SP_EL0 / ELR_EL1 / SPSR_EL1, +/// and `eret` to EL0. Never returns. +/// +/// # Safety +/// TTBR0_EL1 must already point at the target thread's address space and `img` +/// must describe a valid EL0 context (mapped PC/SP). +#[inline(never)] +unsafe fn eret_to_el0(img: &[u64; 34]) -> ! { + core::arch::asm!( + // x18 = base pointer to the image. It is the last GPR we load, so it + // can serve as the base for every prior load without clobbering a value + // we still need. + "mov x18, {base}", + // SP_EL0, ELR_EL1, SPSR_EL1 first (scratch via x16). + "ldr x16, [x18, #(31 * 8)]", // SP_EL0 + "msr sp_el0, x16", + "ldr x16, [x18, #(32 * 8)]", // PC + "msr elr_el1, x16", + "ldr x16, [x18, #(33 * 8)]", // SPSR + "msr spsr_el1, x16", + // Restore x0..x30 from the image. + "ldp x0, x1, [x18, #(0 * 8)]", + "ldp x2, x3, [x18, #(2 * 8)]", + "ldp x4, x5, [x18, #(4 * 8)]", + "ldp x6, x7, [x18, #(6 * 8)]", + "ldp x8, x9, [x18, #(8 * 8)]", + "ldp x10, x11, [x18, #(10 * 8)]", + "ldp x12, x13, [x18, #(12 * 8)]", + "ldp x14, x15, [x18, #(14 * 8)]", + "ldp x19, x20, [x18, #(19 * 8)]", + "ldp x21, x22, [x18, #(21 * 8)]", + "ldp x23, x24, [x18, #(23 * 8)]", + "ldp x25, x26, [x18, #(25 * 8)]", + "ldp x27, x28, [x18, #(27 * 8)]", + "ldr x29, [x18, #(29 * 8)]", + "ldr x30, [x18, #(30 * 8)]", + // x16, x17 last among the temporaries (restore real values). + "ldr x17, [x18, #(17 * 8)]", + "ldr x16, [x18, #(16 * 8)]", + // x18 itself last (base stays valid through every prior load). + "ldr x18, [x18, #(18 * 8)]", + "eret", + base = in(reg) img.as_ptr(), + options(noreturn), + ) +} + +/// ERET entry mirroring the x86_64 IRETQ path (timer-preempted re-entry). +/// +/// # Safety +/// TTBR0_EL1 must already point at the target thread's address space. +#[inline(never)] +pub unsafe fn enter_user_iret(regs: &EnterUserRegs) -> ! { + let img = build_image(regs); + eret_to_el0(&img) +} + +/// ERET entry mirroring the x86_64 SYSRETQ path (syscall-yield re-entry). +/// +/// # Safety +/// TTBR0_EL1 must already point at the target thread's address space. +#[inline(never)] +pub unsafe fn enter_user_sysret(regs: &EnterUserRegs) -> ! { + let img = build_image(regs); + eret_to_el0(&img) +} + +/// Directly enter EL0 at a given PC/SP with seeded x0..x2 (bring-up helper). +/// +/// The minimal path the aarch64 bring-up uses to launch a self-contained EL0 +/// test program without the full shared `process` layer. +/// +/// # Safety +/// `pc`/`sp` must be mapped, executable/writable EL0 addresses under the active +/// TTBR0_EL1. +#[inline(never)] +pub unsafe fn enter_el0_at(pc: u64, sp: u64, x0: u64, x1: u64, x2: u64) -> ! { + let mut img = [0u64; 34]; + img[0] = x0; + img[1] = x1; + img[2] = x2; + img[31] = sp; + img[32] = pc; + img[33] = SPSR_EL0T; + eret_to_el0(&img) +} diff --git a/kernel/src/arch/aarch64/fdt.rs b/kernel/src/arch/aarch64/fdt.rs new file mode 100644 index 0000000..b857a4d --- /dev/null +++ b/kernel/src/arch/aarch64/fdt.rs @@ -0,0 +1,288 @@ +//! Minimal flattened-device-tree (FDT / DTB) reader for the aarch64 port. +//! +//! On `qemu-system-aarch64 -M virt` there is no Limine bootloader: QEMU loads +//! the kernel via `-kernel` and passes a pointer to a Flattened Device Tree blob +//! in `x0` at `_start`. The DTB describes the machine — RAM base/size, and the +//! MMIO layout of the UART, GIC, timer, etc. +//! +//! This is a deliberately small, hand-rolled, `no_std`/no-alloc parser (no +//! external crate). It understands just enough of the DTB v17 structure to: +//! * validate the header magic, +//! * walk the structure block, +//! * extract the `/memory` node's `reg` (RAM base + size), and +//! * read `#address-cells` / `#size-cells` so multi-cell `reg` values decode. +//! +//! Spec reference: the Devicetree Specification (flattened format). All +//! multi-byte integers in a DTB are **big-endian**. + +use core::sync::atomic::{AtomicU64, Ordering}; + +// ── FDT header constants ──────────────────────────────────────────────────── + +const FDT_MAGIC: u32 = 0xd00d_feed; + +const FDT_BEGIN_NODE: u32 = 0x0000_0001; +const FDT_END_NODE: u32 = 0x0000_0002; +const FDT_PROP: u32 = 0x0000_0003; +const FDT_NOP: u32 = 0x0000_0004; +const FDT_END: u32 = 0x0000_0009; + +/// The DTB pointer QEMU passed in x0, captured at `_start`. Zero until set. +static DTB_PTR: AtomicU64 = AtomicU64::new(0); + +/// Record the physical address of the DTB (called from the boot path with the +/// value `_start` saved out of x0). +pub fn set_dtb_ptr(pa: u64) { + DTB_PTR.store(pa, Ordering::Release); +} + +/// The recorded DTB pointer (0 if none was captured). +pub fn dtb_ptr() -> u64 { + DTB_PTR.load(Ordering::Acquire) +} + +// ── Big-endian readers over the raw blob ──────────────────────────────────── + +#[inline] +unsafe fn read_be32(p: *const u8) -> u32 { + u32::from_be_bytes([*p, *p.add(1), *p.add(2), *p.add(3)]) +} + +#[inline] +unsafe fn read_be64(p: *const u8) -> u64 { + let hi = read_be32(p) as u64; + let lo = read_be32(p.add(4)) as u64; + (hi << 32) | lo +} + +// ── Header view ───────────────────────────────────────────────────────────── + +#[derive(Clone, Copy)] +struct FdtHeader { + base: *const u8, + off_dt_struct: u32, + off_dt_strings: u32, + size_dt_struct: u32, +} + +impl FdtHeader { + /// Validate and parse the FDT header at `base`. Returns `None` if the magic + /// is wrong (no/invalid DTB). + unsafe fn parse(base: *const u8) -> Option { + if base.is_null() { + return None; + } + let magic = read_be32(base); + if magic != FDT_MAGIC { + return None; + } + Some(FdtHeader { + base, + // Layout: magic(0) totalsize(4) off_dt_struct(8) off_dt_strings(12) + // off_mem_rsvmap(16) version(20) ... + // size_dt_strings(32) size_dt_struct(36) + off_dt_struct: read_be32(base.add(8)), + off_dt_strings: read_be32(base.add(12)), + size_dt_struct: read_be32(base.add(36)), + }) + } + + #[inline] + unsafe fn struct_start(&self) -> *const u8 { + self.base.add(self.off_dt_struct as usize) + } + + #[inline] + unsafe fn struct_end(&self) -> *const u8 { + self.base + .add(self.off_dt_struct as usize + self.size_dt_struct as usize) + } + + /// Resolve a property name by its offset into the strings block. + unsafe fn string_at(&self, off: u32) -> &'static str { + let p = self.base.add(self.off_dt_strings as usize + off as usize); + let mut len = 0usize; + while *p.add(len) != 0 { + len += 1; + } + let bytes = core::slice::from_raw_parts(p, len); + core::str::from_utf8(bytes).unwrap_or("") + } +} + +/// A single discovered memory region (physical base + length, in bytes). +#[derive(Clone, Copy, Debug)] +pub struct MemRegion { + pub base: u64, + pub size: u64, +} + +/// Result of parsing the device tree for the bits the kernel needs. +#[derive(Clone, Copy)] +pub struct FdtInfo { + /// Discovered RAM regions (from `/memory` nodes). Up to 4 retained. + pub mem: [MemRegion; 4], + pub mem_count: usize, +} + +impl FdtInfo { + const fn empty() -> Self { + FdtInfo { + mem: [MemRegion { base: 0, size: 0 }; 4], + mem_count: 0, + } + } + + /// Total RAM bytes across all discovered regions. + pub fn ram_total(&self) -> u64 { + let mut t = 0u64; + for i in 0..self.mem_count { + t += self.mem[i].size; + } + t + } +} + +/// Walk the DTB structure block and extract the `/memory` node(s). +/// +/// Returns `None` if no valid DTB is present at `dtb_pa`. +/// +/// # Safety +/// `dtb_pa` must point at a DTB blob mapped/identity-accessible at this address. +pub unsafe fn parse(dtb_pa: u64) -> Option { + let hdr = FdtHeader::parse(dtb_pa as *const u8)?; + + let mut info = FdtInfo::empty(); + + // Root cell sizes default to 2/1 per spec; QEMU virt uses 2/2. + let mut root_addr_cells: u32 = 2; + let mut root_size_cells: u32 = 2; + // Cell sizes that apply to the *current* node's children (set when we see + // the property on the parent). We track only one level of nesting context + // since /memory sits directly under the root. + let mut cur_addr_cells: u32 = 2; + let mut cur_size_cells: u32 = 2; + + let mut p = hdr.struct_start(); + let end = hdr.struct_end(); + + // Depth tracking: depth 0 = root. /memory is depth 1. + let mut depth: i32 = -1; + // Whether the node we're currently inside is a `/memory*` node. + let mut in_memory = false; + + while p < end { + let token = read_be32(p); + p = p.add(4); + match token { + t if t == FDT_BEGIN_NODE => { + // Node name is a NUL-terminated string, padded to 4 bytes. + let name_ptr = p; + let mut nlen = 0usize; + while *name_ptr.add(nlen) != 0 { + nlen += 1; + } + let name = core::str::from_utf8(core::slice::from_raw_parts(name_ptr, nlen)) + .unwrap_or(""); + // Advance past name + NUL, aligned up to 4. + p = p.add((nlen + 4) & !3); + + depth += 1; + if depth == 1 { + // Entering a top-level node: inherit the root cell sizes as + // the defaults for this node's `reg`. + cur_addr_cells = root_addr_cells; + cur_size_cells = root_size_cells; + // A memory node is named "memory" or "memory@". + in_memory = name == "memory" || name.starts_with("memory@"); + } + } + t if t == FDT_END_NODE => { + if depth == 1 { + in_memory = false; + } + depth -= 1; + } + t if t == FDT_PROP => { + // prop header: len(4) nameoff(4), then `len` bytes of value, + // padded to 4. + let len = read_be32(p); + let nameoff = read_be32(p.add(4)); + let val = p.add(8); + let pname = hdr.string_at(nameoff); + + if depth == 0 { + // Root-level cell sizes. + if pname == "#address-cells" && len == 4 { + root_addr_cells = read_be32(val); + } else if pname == "#size-cells" && len == 4 { + root_size_cells = read_be32(val); + } + } else if depth == 1 && in_memory && pname == "reg" { + // `reg` = a list of (address, size) pairs, each cell being a + // big-endian u32; address spans `cur_addr_cells` cells, size + // spans `cur_size_cells` cells. + let pair_cells = (cur_addr_cells + cur_size_cells) as usize; + let total_cells = (len as usize) / 4; + let mut c = 0usize; + while c + pair_cells <= total_cells { + let base = read_cells(val.add(c * 4), cur_addr_cells); + let size = read_cells( + val.add((c + cur_addr_cells as usize) * 4), + cur_size_cells, + ); + if size != 0 && info.mem_count < info.mem.len() { + info.mem[info.mem_count] = MemRegion { base, size }; + info.mem_count += 1; + } + c += pair_cells; + } + } + + // Advance past the value (padded to 4). + p = p.add(((len as usize) + 3) & !3); + } + t if t == FDT_NOP => { /* skip */ } + t if t == FDT_END => break, + _ => break, // malformed — stop rather than run off the blob + } + } + + if info.mem_count == 0 { + None + } else { + Some(info) + } +} + +/// Scan a physical address range for the FDT magic on an 8-byte stride and +/// return the first address whose blob parses to a valid `/memory` node. +/// +/// QEMU's `-M virt` ELF-kernel boot does not reliably pass the DTB pointer in x0 +/// and may place the generated blob at a position that depends on the kernel +/// image size. As a robust last resort, we scan a bounded window of RAM for it. +/// +/// # Safety +/// `[start, end)` must be readable (identity-mapped) physical RAM. +pub unsafe fn scan_for_dtb(start: u64, end: u64) -> Option { + let mut p = start & !0x7; + while p < end { + if read_be32(p as *const u8) == FDT_MAGIC { + if parse(p).is_some() { + return Some(p); + } + } + p += 8; + } + None +} + +/// Read a 1- or 2-cell big-endian value (cells beyond 2 are ignored / unsupported). +#[inline] +unsafe fn read_cells(p: *const u8, cells: u32) -> u64 { + match cells { + 1 => read_be32(p) as u64, + 2 => read_be64(p), + _ => read_be64(p), // clamp: treat >2 as 2 (top 64 bits) + } +} diff --git a/kernel/src/arch/aarch64/gdt.rs b/kernel/src/arch/aarch64/gdt.rs new file mode 100644 index 0000000..803f77a --- /dev/null +++ b/kernel/src/arch/aarch64/gdt.rs @@ -0,0 +1,18 @@ +//! Segment-selector compatibility shim — aarch64. +//! +//! AArch64 has no GDT/segmentation. The shared kernel still references the +//! `USER_CS` / `USER_DS` selector constants when building an initial user +//! context (the values are stored into the saved register frame and ignored by +//! the aarch64 exception-return path, which uses SPSR_EL1 instead). We keep the +//! symbols so cross-arch process-setup code compiles; the values are nominal. + +/// User code "selector" placeholder (no segmentation on aarch64). +pub const USER_CS: u16 = 0; +/// User data "selector" placeholder (no segmentation on aarch64). +pub const USER_DS: u16 = 0; + +/// Load the (nonexistent) descriptor table on the BSP. No-op on aarch64. +pub fn init() {} + +/// Per-AP descriptor-table load. No-op on aarch64. +pub fn init_ap(_cpu_idx: u32) {} diff --git a/kernel/src/arch/aarch64/gic.rs b/kernel/src/arch/aarch64/gic.rs new file mode 100644 index 0000000..4c7914f --- /dev/null +++ b/kernel/src/arch/aarch64/gic.rs @@ -0,0 +1,113 @@ +//! GICv2 interrupt controller — QEMU `-M virt` (distributor + CPU interface). +//! +//! `-M virt` defaults to GICv2 with: +//! * GICD (distributor) @ 0x0800_0000 +//! * GICC (CPU interface) @ 0x0801_0000 +//! +//! We enable the distributor + this core's CPU interface, set a permissive +//! priority mask, and provide enable/EOI for individual interrupt IDs. The ARM +//! generic timer's EL1 physical-timer interrupt is PPI 30 on `-M virt`. + +const GICD_BASE: u64 = 0x0800_0000; +const GICC_BASE: u64 = 0x0801_0000; + +// Distributor registers (offsets from GICD_BASE). +const GICD_CTLR: u64 = 0x000; // Distributor control +const GICD_ISENABLER: u64 = 0x100; // Set-enable (1 bit/IRQ, 32/reg) +const GICD_ICENABLER: u64 = 0x180; // Clear-enable +const GICD_IPRIORITYR: u64 = 0x400; // Priority (1 byte/IRQ) +const GICD_ITARGETSR: u64 = 0x800; // CPU targets (1 byte/IRQ, SPIs only) +const GICD_ICFGR: u64 = 0xC00; // Config (2 bits/IRQ: edge/level) + +// CPU interface registers (offsets from GICC_BASE). +const GICC_CTLR: u64 = 0x00; // CPU interface control +const GICC_PMR: u64 = 0x04; // Priority mask +const GICC_BPR: u64 = 0x08; // Binary point +const GICC_IAR: u64 = 0x0C; // Interrupt acknowledge +const GICC_EOIR: u64 = 0x10; // End of interrupt + +#[inline(always)] +unsafe fn gicd_read(off: u64) -> u32 { + core::ptr::read_volatile((GICD_BASE + off) as *const u32) +} +#[inline(always)] +unsafe fn gicd_write(off: u64, v: u32) { + core::ptr::write_volatile((GICD_BASE + off) as *mut u32, v); +} +#[inline(always)] +unsafe fn gicc_read(off: u64) -> u32 { + core::ptr::read_volatile((GICC_BASE + off) as *const u32) +} +#[inline(always)] +unsafe fn gicc_write(off: u64, v: u32) { + core::ptr::write_volatile((GICC_BASE + off) as *mut u32, v); +} + +/// Initialise the distributor (system-wide, call once on the BSP). +pub fn init_distributor() { + unsafe { + // Disable the distributor while configuring. + gicd_write(GICD_CTLR, 0); + // Leave SGIs/PPIs (0..31) at reset; they're per-CPU and routed locally. + // Enable the distributor (Group 0 forwarding). + gicd_write(GICD_CTLR, 1); + } +} + +/// Initialise this core's CPU interface (call on every core). +pub fn init_cpu_interface() { + unsafe { + // Lowest priority threshold = 0xFF lets all priorities through. + gicc_write(GICC_PMR, 0xFF); + // No sub-priority grouping. + gicc_write(GICC_BPR, 0); + // Enable signalling of Group 0 interrupts to this CPU. + gicc_write(GICC_CTLR, 1); + } +} + +/// Enable a specific interrupt ID at the distributor. +pub fn enable_irq(intid: u32) { + let reg = (intid / 32) as u64; + let bit = intid % 32; + unsafe { + gicd_write(GICD_ISENABLER + reg * 4, 1 << bit); + } +} + +/// Disable a specific interrupt ID. +pub fn disable_irq(intid: u32) { + let reg = (intid / 32) as u64; + let bit = intid % 32; + unsafe { + gicd_write(GICD_ICENABLER + reg * 4, 1 << bit); + } +} + +/// Set an interrupt's priority (0 = highest). Lower numbers preempt higher. +pub fn set_priority(intid: u32, prio: u8) { + unsafe { + let addr = GICD_BASE + GICD_IPRIORITYR + intid as u64; + core::ptr::write_volatile(addr as *mut u8, prio); + } +} + +/// Route an SPI (intid >= 32) to CPU 0. PPIs/SGIs are implicitly local. +pub fn route_to_cpu0(intid: u32) { + if intid >= 32 { + unsafe { + let addr = GICD_BASE + GICD_ITARGETSR + intid as u64; + core::ptr::write_volatile(addr as *mut u8, 0x01); + } + } +} + +/// Acknowledge the pending interrupt; returns its INTID (1023 = spurious). +pub fn acknowledge() -> u32 { + unsafe { gicc_read(GICC_IAR) & 0x3FF } +} + +/// Signal end-of-interrupt for `intid`. +pub fn eoi(intid: u32) { + unsafe { gicc_write(GICC_EOIR, intid) }; +} diff --git a/kernel/src/arch/aarch64/idt.rs b/kernel/src/arch/aarch64/idt.rs new file mode 100644 index 0000000..7a63dba --- /dev/null +++ b/kernel/src/arch/aarch64/idt.rs @@ -0,0 +1,19 @@ +//! Exception-vector scaffolding — aarch64 (EL1 vector table / VBAR_EL1). +//! +//! AArch64 has no IDT; synchronous/IRQ/FIQ/SError exceptions are dispatched +//! through a 16-entry, 2 KiB-aligned vector table pointed to by VBAR_EL1. This +//! module names the table-install entry points the way the x86 `idt` module +//! does (`init`/`load`) and re-exports the cross-arch [`InterruptFrame`] saved +//! by the (future) vector handlers. + +pub use super::InterruptFrame; + +/// Install the EL1 exception vector table on the BSP. +/// +/// TODO(arm): build the vector table and write its base to VBAR_EL1. +pub fn init() {} + +/// Re-load the (already-built) vector table on an AP. +/// +/// TODO(arm): write the shared vector-table base to this core's VBAR_EL1. +pub fn load() {} diff --git a/kernel/src/arch/aarch64/interrupts.rs b/kernel/src/arch/aarch64/interrupts.rs new file mode 100644 index 0000000..9fd4d6a --- /dev/null +++ b/kernel/src/arch/aarch64/interrupts.rs @@ -0,0 +1,20 @@ +//! Interrupt helpers — aarch64. +//! +//! x86 has a legacy 8259 PIC to disable; aarch64 has no such thing (the GIC is +//! configured in `apic`/GIC init), so `disable_pic` is a no-op kept for parity +//! with the x86 early-init sequence. + +use core::arch::asm; + +/// No-op on aarch64 — there is no legacy PIC to disable. +/// +/// # Safety +/// Trivially safe; kept `unsafe` to match the x86 signature called from +/// `early_init`. +pub unsafe fn disable_pic() {} + +/// Data Memory Barrier (system scope). +#[inline] +pub fn memory_fence() { + unsafe { asm!("dmb sy", options(nomem, nostack)) }; +} diff --git a/kernel/src/arch/aarch64/memory.rs b/kernel/src/arch/aarch64/memory.rs new file mode 100644 index 0000000..a435877 --- /dev/null +++ b/kernel/src/arch/aarch64/memory.rs @@ -0,0 +1,30 @@ +//! aarch64 memory helpers (translation-table base, etc.) +//! +//! The x86 `read_cr3`/`write_cr3` map to the EL1 translation-table base +//! registers `TTBR0_EL1` (user/low half) and `TTBR1_EL1` (kernel/high half). +//! The shared kernel only needs a single "current address-space root" value; +//! we expose `TTBR0_EL1` under the historical `read_cr3` name. + +use core::arch::asm; + +/// Read the current low-half translation-table base (x86 CR3 analogue). +pub fn read_cr3() -> u64 { + let v: u64; + unsafe { asm!("mrs {}, ttbr0_el1", out(reg) v, options(nomem, nostack)) }; + v +} + +/// Write the low-half translation-table base and synchronise the TLB. +pub fn write_cr3(val: u64) { + unsafe { + asm!( + "msr ttbr0_el1, {}", + "dsb ish", + "tlbi vmalle1is", + "dsb ish", + "isb", + in(reg) val, + options(nostack), + ); + } +} diff --git a/kernel/src/arch/aarch64/mmu.rs b/kernel/src/arch/aarch64/mmu.rs new file mode 100644 index 0000000..9b6d234 --- /dev/null +++ b/kernel/src/arch/aarch64/mmu.rs @@ -0,0 +1,231 @@ +//! ARMv8-A MMU bring-up — translation tables + SCTLR_EL1.M enable. +//! +//! Configuration (the simplest correct setup for QEMU `-M virt`): +//! * 4 KiB granule, 48-bit VA (T0SZ = T1SZ = 16). +//! * TTBR0_EL1 walks the low half (identity map covering devices + RAM). +//! * TTBR1_EL1 walks the kernel high half (mirror of the same identity map +//! at 0xFFFF_0000_0000_0000+ so the kernel can later run higher-half). +//! * MAIR attr0 = Normal Write-Back, attr1 = Device-nGnRE. +//! +//! We use L1 1 GiB block descriptors — the whole address space is covered by a +//! single 512-entry L1 table per TTBR, no L2/L3 needed for the identity map. +//! Region 0 (0x0–0x4000_0000) holds the GIC/UART MMIO and is mapped Device; +//! regions 1..N (RAM from 0x4000_0000) are mapped Normal cacheable. +//! +//! After this runs the MMU is ON and the kernel executes translated (the +//! identity map means VAs equal PAs, so code/stack/UART pointers stay valid). + +use core::sync::atomic::{AtomicBool, Ordering}; + +// ── Descriptor + register bit definitions ─────────────────────────────────── + +/// L1 block descriptor: valid, block (bit[1]=0), AF, plus attr/shareability. +const DESC_VALID: u64 = 1 << 0; +const DESC_BLOCK: u64 = 0 << 1; // 0 = block at L1/L2; 1 = table/page +const DESC_AF: u64 = 1 << 10; // Access Flag (else first access faults) +const DESC_SH_INNER: u64 = 3 << 8; // Inner shareable +const DESC_AP_RW_EL1: u64 = 0 << 6; // EL1 RW, EL0 no access +const DESC_AP_RW_ALL: u64 = 1 << 6; // EL1 RW, EL0 RW + +// AttrIndx[2:0] selects a MAIR attr byte. +const ATTR_IDX_NORMAL: u64 = 0 << 2; // MAIR attr0 +const ATTR_IDX_DEVICE: u64 = 1 << 2; // MAIR attr1 + +/// MAIR_EL1: attr0 = Normal Inner/Outer Write-Back non-transient (0xFF), +/// attr1 = Device-nGnRE (0x04). +const MAIR_VALUE: u64 = 0x04 << 8 | 0xFF; + +/// TCR_EL1 for 4 KiB granule, 39-bit VA on both halves, WB-cacheable walks, +/// inner-shareable. IPS = 40-bit (1 TiB) physical, plenty for QEMU virt. +/// +/// T0SZ/T1SZ = 25 → 39-bit VA (512 GiB). With a 4 KiB granule this makes the +/// translation walk *start at level 1*, so a single 512-entry L1 table with +/// 1 GiB block descriptors covers the whole address space — no L0 needed. +const TCR_T0SZ: u64 = 25; // 64-25 = 39-bit VA (TTBR0) +const TCR_T1SZ: u64 = 25 << 16; // 39-bit VA (TTBR1) +const TCR_IRGN0_WB: u64 = 1 << 8; +const TCR_ORGN0_WB: u64 = 1 << 10; +const TCR_SH0_INNER: u64 = 3 << 12; +const TCR_TG0_4K: u64 = 0 << 14; // 4 KiB granule (TTBR0) +const TCR_IRGN1_WB: u64 = 1 << 24; +const TCR_ORGN1_WB: u64 = 1 << 26; +const TCR_SH1_INNER: u64 = 3 << 28; +const TCR_TG1_4K: u64 = 2 << 30; // 4 KiB granule (TTBR1) +const TCR_IPS_40BIT: u64 = 2 << 32; + +const TCR_VALUE: u64 = TCR_T0SZ + | TCR_T1SZ + | TCR_IRGN0_WB + | TCR_ORGN0_WB + | TCR_SH0_INNER + | TCR_TG0_4K + | TCR_IRGN1_WB + | TCR_ORGN1_WB + | TCR_SH1_INNER + | TCR_TG1_4K + | TCR_IPS_40BIT; + +// SCTLR_EL1 bits. +const SCTLR_M: u64 = 1 << 0; // MMU enable +const SCTLR_C: u64 = 1 << 2; // data cache enable +const SCTLR_I: u64 = 1 << 12; // instruction cache enable + +// ── Translation tables (BSS-resident, page-aligned) ───────────────────────── + +/// A single 512-entry, 4 KiB-aligned translation table. +#[repr(C, align(4096))] +struct Table([u64; 512]); + +/// L1 table for the low half (TTBR0): identity map. +static mut L1_LOW: Table = Table([0; 512]); +/// L1 table for the high half (TTBR1): identity map mirror. +static mut L1_HIGH: Table = Table([0; 512]); + +static MMU_ON: AtomicBool = AtomicBool::new(false); + +/// Build an L1 block descriptor mapping the 1 GiB region at `pa` with the given +/// attribute index and access permissions. +#[inline] +fn block_desc(pa: u64, attr_idx: u64, ap: u64) -> u64 { + (pa & 0x0000_FFFF_C000_0000) // 1 GiB-aligned output address (bits[47:30]) + | DESC_VALID + | DESC_BLOCK + | DESC_AF + | DESC_SH_INNER + | attr_idx + | ap +} + +/// Populate one L1 table with an identity map: index 0 (0x0–0x4000_0000) is +/// Device memory (covers GIC @ 0x0800_0000 and PL011 @ 0x0900_0000), indices +/// 1..=`ram_gibs` are Normal cacheable RAM starting at 0x4000_0000. +unsafe fn fill_identity(table: &mut Table, ram_gibs: u64) { + // Entry 0: low 1 GiB = device MMIO (also covers the flash/boot ROM region). + // Kernel-only (EL1 RW), Device-nGnRE. + table.0[0] = block_desc(0x0000_0000, ATTR_IDX_DEVICE, DESC_AP_RW_EL1); + // RAM region: 0x4000_0000 onward, one 1 GiB block per L1 entry. Kernel-only + // RW+executable (EL0 has no access to the kernel identity map). + let mut g = 1u64; + while g <= ram_gibs && g < 512 { + let pa = g << 30; // g GiB + table.0[g as usize] = block_desc(pa, ATTR_IDX_NORMAL, DESC_AP_RW_EL1); + g += 1; + } +} + +/// Returns the physical address of the active TTBR0 L1 table (for the shared +/// `arch::memory::read_cr3` view). Until per-process tables exist, all address +/// spaces share this identity map. +pub fn kernel_ttbr0() -> u64 { + core::ptr::addr_of!(L1_LOW) as u64 +} + +/// Build the tables, program MAIR/TCR/TTBR, and enable the MMU + caches. +/// +/// `ram_gibs` is how many GiB of RAM (from 0x4000_0000) to identity-map. +/// +/// # Safety +/// Must run once, at EL1, with the MMU OFF and interrupts masked. +pub unsafe fn enable(ram_gibs: u64) { + let l1_low = &mut *core::ptr::addr_of_mut!(L1_LOW); + let l1_high = &mut *core::ptr::addr_of_mut!(L1_HIGH); + fill_identity(l1_low, ram_gibs); + fill_identity(l1_high, ram_gibs); + + let ttbr0 = core::ptr::addr_of!(L1_LOW) as u64; + let ttbr1 = core::ptr::addr_of!(L1_HIGH) as u64; + + core::arch::asm!( + // Program memory attributes + translation control. + "msr mair_el1, {mair}", + "msr tcr_el1, {tcr}", + "msr ttbr0_el1, {ttbr0}", + "msr ttbr1_el1, {ttbr1}", + "isb", + // Invalidate all TLB entries for EL1, then barrier. + "tlbi vmalle1", + "dsb nsh", + "isb", + mair = in(reg) MAIR_VALUE, + tcr = in(reg) TCR_VALUE, + ttbr0 = in(reg) ttbr0, + ttbr1 = in(reg) ttbr1, + options(nostack, preserves_flags), + ); + + // Enable MMU + I/D caches in SCTLR_EL1. + let mut sctlr: u64; + core::arch::asm!("mrs {}, sctlr_el1", out(reg) sctlr, options(nomem, nostack)); + sctlr |= SCTLR_M | SCTLR_C | SCTLR_I; + core::arch::asm!( + "msr sctlr_el1, {}", + "isb", + in(reg) sctlr, + options(nostack, preserves_flags), + ); + + MMU_ON.store(true, Ordering::Release); +} + +/// True once [`enable`] has turned the MMU on. +pub fn is_on() -> bool { + MMU_ON.load(Ordering::Acquire) +} + +// ── EL0 user mapping (bring-up) ────────────────────────────────────────────── + +/// Table descriptor: valid + table type (points at the next-level table). +const DESC_TABLE: u64 = (1 << 0) | (1 << 1); + +/// Page (L3) descriptor type bit (bit[1]=1 means page at L3). +const DESC_PAGE: u64 = (1 << 0) | (1 << 1); + +/// UXN/PXN — execute-never controls (bits[54]=UXN, [53]=PXN). +const DESC_UXN: u64 = 1 << 54; +const DESC_PXN: u64 = 1 << 53; + +/// L2 + L3 tables backing one user region (covers up to 2 MiB of 4 KiB pages +/// under a single L2 entry). One static instance is enough for the bring-up +/// EL0 test process. +static mut USER_L2: Table = Table([0; 512]); +static mut USER_L3: Table = Table([0; 512]); + +/// Map a single 4 KiB physical page `pa` at user VA `va` with EL0 RW + execute +/// (PXN=1 so EL1 can't execute it, UXN=0 so EL0 can). The VA must lie in the +/// 2 MiB window selected by the L1/L2 indices computed here, and `va`'s L1 +/// index must be free in the current TTBR0 L1 table. +/// +/// # Safety +/// Must run at EL1 with the MMU on; mutates the shared TTBR0 L1 plus the static +/// USER_L2/USER_L3 tables. Intended for the single bring-up user region. +pub unsafe fn map_user_page(va: u64, pa: u64) { + let l1_idx = ((va >> 30) & 0x1FF) as usize; + let l2_idx = ((va >> 21) & 0x1FF) as usize; + let l3_idx = ((va >> 12) & 0x1FF) as usize; + + let l1 = &mut *core::ptr::addr_of_mut!(L1_LOW); + let l2 = &mut *core::ptr::addr_of_mut!(USER_L2); + let l3 = &mut *core::ptr::addr_of_mut!(USER_L3); + + // L1[l1_idx] → USER_L2 table. + l1.0[l1_idx] = (core::ptr::addr_of!(USER_L2) as u64 & 0x0000_FFFF_FFFF_F000) | DESC_TABLE; + // L2[l2_idx] → USER_L3 table. + l2.0[l2_idx] = (core::ptr::addr_of!(USER_L3) as u64 & 0x0000_FFFF_FFFF_F000) | DESC_TABLE; + // L3[l3_idx] → the 4 KiB page, EL0 RW + executable, Normal memory. + l3.0[l3_idx] = (pa & 0x0000_FFFF_FFFF_F000) + | DESC_PAGE + | DESC_AF + | DESC_SH_INNER + | ATTR_IDX_NORMAL + | DESC_AP_RW_ALL // EL0 + EL1 RW + | DESC_PXN; // privileged execute-never; UXN=0 → EL0 may execute + + // Make the new mappings visible and flush the affected VA. + core::arch::asm!( + "dsb ishst", + "tlbi vmalle1is", + "dsb ish", + "isb", + options(nostack, preserves_flags), + ); +} diff --git a/kernel/src/arch/aarch64/mod.rs b/kernel/src/arch/aarch64/mod.rs index 384f68e..4e141a0 100644 --- a/kernel/src/arch/aarch64/mod.rs +++ b/kernel/src/arch/aarch64/mod.rs @@ -1,35 +1,207 @@ -//! aarch64 architecture stub. +//! aarch64 architecture backend (scaffold). +//! +//! This is the ARM port's scaffolding layer: every module and public function +//! the shared kernel reaches through `crate::arch::*` exists here with a correct +//! signature, but most bodies are stubs (`unimplemented!()`-free no-ops or sane +//! defaults) pending real implementations. The intent is that the kernel +//! *compiles and links* for `aarch64-unknown-none`; it does NOT yet boot. +//! +//! Mapping from the x86_64 backend to aarch64 primitives: +//! * GDT/segmentation → none (flat); `gdt` keeps nominal selector constants. +//! * IDT → EL1 exception vector table (VBAR_EL1), in `idt`. +//! * Local APIC → GIC CPU interface + ARM generic timer, in `apic`. +//! * SYSCALL/SYSRET → `SVC` trap to the EL1 sync vector, in `syscall`. +//! * CR3 → TTBR0_EL1/TTBR1_EL1, in `memory`. +//! * SMP wake → PSCI `CPU_ON`, reschedule via GIC SGIs, in `smp`. -/// Placeholder interrupt frame — no x86 IDT on aarch64. -/// Used by cortex subsystem for cross-arch compatibility. -/// `ip` mirrors x86_64 InterruptFrame field name for generic kernel code. +pub mod gdt; +pub mod idt; +pub mod interrupts; +pub mod memory; +pub mod apic; +pub mod cpu; +pub mod syscall; +pub mod acpi; +pub mod smp; +pub mod enter_user; + +// ── Bare-metal bring-up (direct `-kernel` boot) ───────────────────────────── +pub mod uart; +pub mod fdt; +pub mod ramfb; +pub mod boot; +pub mod boot_prod; +pub mod mmu; +pub mod vectors; +pub mod gic; +pub mod timer; +pub mod psci; +pub mod bringup; +pub mod bringup_mmu; +pub mod bringup_vectors; +pub mod bringup_irq; +pub mod bringup_user; +pub mod bringup_smp; + +pub use enter_user::{enter_user_iret, enter_user_sysret, EnterUserRegs}; + +use core::arch::asm; + +/// Cross-arch interrupt/exception frame. +/// +/// `ip` mirrors the x86_64 `InterruptFrame` field name so generic kernel code +/// (the cortex observer) stays architecture-neutral. On aarch64 these fields map +/// to the exception-syndrome registers saved by the EL1 vector handler. #[derive(Debug)] #[repr(C)] pub struct InterruptFrame { - pub ip: u64, // Instruction pointer (= ELR_EL1) - pub spsr: u64, // Saved Program Status Register - pub esr: u64, // Exception Syndrome Register - pub far: u64, // Fault Address Register + /// Instruction pointer that faulted (= ELR_EL1). + pub ip: u64, + /// Saved Program Status Register (= SPSR_EL1). + pub spsr: u64, + /// Exception Syndrome Register (= ESR_EL1). + pub esr: u64, + /// Fault Address Register (= FAR_EL1). + pub far: u64, } -/// No-op stub — context switching not yet implemented on aarch64. -pub unsafe extern "C" fn context_switch(_old_sp: *mut u64, _new_sp: u64) {} -/// No-op stub — task entry trampoline not yet implemented on aarch64. -pub unsafe extern "C" fn task_entry() { loop { core::hint::spin_loop() } } - +/// Perform early arch initialisation (BSP path). +/// +/// Mirrors the x86_64 sequence; most steps are scaffold no-ops for now. pub fn early_init() { - crate::logger::early_print("[arch::aarch64] early_init stub (Milestone 4b)\r\n"); + crate::logger::early_print("[ARCH] assert_required_features\r\n"); + cpu::assert_required_features(); + crate::logger::early_print("[ARCH] idt::init (VBAR_EL1)\r\n"); + idt::init(); + crate::logger::early_print("[ARCH] apic::init_bsp (GIC + timer)\r\n"); + apic::init_bsp(); + crate::logger::early_print("[ARCH] cpu::enable_fpu_simd\r\n"); + cpu::enable_fpu_simd(); + crate::logger::early_print("[ARCH] syscall::init (SVC)\r\n"); + syscall::init(); + crate::logger::early_print("[ARCH] aarch64 early_init complete (scaffold)\r\n"); +} + +/// Per-AP init — called once each Application Processor is brought up. +pub fn ap_init(cpu_idx: u32) { + gdt::init_ap(cpu_idx); + idt::load(); + apic::init_ap(); + cpu::enable_fpu_simd(); + syscall::init_ap(cpu_idx); +} + +/// Wake all APs and bring SMP online (single-core scaffold for now). +pub fn smp_init(resp: Option<&'static limine::request::MpResponse>) { + smp::init(resp); +} + +/// Halt the current core until an interrupt (or event) arrives. +#[inline(always)] +pub fn halt() { + unsafe { asm!("wfi", options(nomem, nostack)) }; +} + +/// Unmask interrupts, wait for one (WFI), then mask again. +/// +/// Mirror of the x86_64 `sti; hlt; cli` idle pattern. +#[inline(always)] +pub fn enable_and_halt() { + unsafe { + asm!( + "msr daifclr, #0b0011", + "wfi", + "msr daifset, #0b0011", + options(nomem, nostack), + ) + }; +} + +/// Permanently halt this core (used in panic). +pub fn halt_forever() -> ! { + loop { + unsafe { asm!("msr daifset, #0b0011", "wfi", options(nomem, nostack)) }; + } } -pub fn ap_init() { - crate::logger::early_print("[arch::aarch64] ap_init stub\r\n"); + +/// Enable IRQ/FIQ delivery on this core. +#[inline(always)] +pub fn enable_interrupts() { + unsafe { asm!("msr daifclr, #0b0011", options(nomem, nostack)) }; +} + +/// Disable IRQ/FIQ delivery on this core. +#[inline(always)] +pub fn disable_interrupts() { + unsafe { asm!("msr daifset, #0b0011", options(nomem, nostack)) }; } -pub fn smp_init(_resp: Option<&'static limine::request::MpResponse>) { /* stub */ } -pub fn halt() { loop { core::hint::spin_loop() } } -pub fn halt_forever() -> ! { loop { core::hint::spin_loop() } } -pub fn enable_interrupts() {} -pub fn disable_interrupts() {} + +/// Read a free-running cycle/time counter (x86 `rdtsc` analogue). +/// +/// Uses the virtual count register `CNTVCT_EL0` from the ARM generic timer. +#[inline(always)] pub fn rdtsc() -> u64 { let cnt: u64; - unsafe { core::arch::asm!("mrs {}, cntvct_el0", out(reg) cnt) }; + unsafe { asm!("mrs {}, cntvct_el0", out(reg) cnt, options(nomem, nostack)) }; cnt } + +// ── Context switch ──────────────────────────────────────────────────────────── + +/// Switch from the current task to a new one. +/// +/// Saves the AAPCS64 callee-saved registers (x19–x30, including the frame +/// pointer x29 and link register x30) of the old task onto its kernel stack, +/// stores SP into `*old_sp`, loads `new_sp`, restores the new task's saved +/// registers, and returns into it. +/// +/// # Safety +/// `old_sp` must point to the `kernel_sp` field of the currently running task; +/// `new_sp` must be a kernel stack prepared by `spawn_kernel_task` or a prior +/// call to this function. Must be called with interrupts disabled. +#[unsafe(naked)] +pub unsafe extern "C" fn context_switch(old_sp: *mut u64, new_sp: u64) { + // AAPCS64: x0 = old_sp, x1 = new_sp. + core::arch::naked_asm!( + // Save callee-saved registers of the old task (x19..x30) as pairs. + "stp x29, x30, [sp, #-16]!", + "stp x27, x28, [sp, #-16]!", + "stp x25, x26, [sp, #-16]!", + "stp x23, x24, [sp, #-16]!", + "stp x21, x22, [sp, #-16]!", + "stp x19, x20, [sp, #-16]!", + // Save current SP into *old_sp. + "mov x2, sp", + "str x2, [x0]", + // Load new SP. + "mov sp, x1", + // Restore new task's callee-saved registers. + "ldp x19, x20, [sp], #16", + "ldp x21, x22, [sp], #16", + "ldp x23, x24, [sp], #16", + "ldp x25, x26, [sp], #16", + "ldp x27, x28, [sp], #16", + "ldp x29, x30, [sp], #16", + // Return — branches to the new task's saved link register. + "ret", + ) +} + +/// Trampoline entered by a freshly spawned task on its very first schedule. +/// +/// The task function pointer is the top-of-stack word left by +/// `spawn_kernel_task`. We pop it, enable interrupts, then call it. If it ever +/// returns (it should not for kernel tasks), we park. +#[unsafe(naked)] +pub unsafe extern "C" fn task_entry() { + core::arch::naked_asm!( + "ldr x9, [sp], #16", // task fn pointer pushed by spawn_kernel_task + "msr daifclr, #0b0011", // enable IRQ+FIQ for this task + "blr x9", // run the task function + // Task returned — should not happen for kernel tasks; park. + "0:", + "msr daifset, #0b0011", + "wfi", + "b 0b", + ) +} diff --git a/kernel/src/arch/aarch64/psci.rs b/kernel/src/arch/aarch64/psci.rs new file mode 100644 index 0000000..5889987 --- /dev/null +++ b/kernel/src/arch/aarch64/psci.rs @@ -0,0 +1,66 @@ +//! PSCI (Power State Coordination Interface) — QEMU `-M virt`. +//! +//! QEMU exposes PSCI via the SMC conduit (HVC when virtualization is on). We +//! use it for clean SYSTEM_OFF (test-harness exit) and, later, CPU_ON for SMP. + +/// PSCI function IDs (SMC64 calling convention). +const PSCI_SYSTEM_OFF: u32 = 0x8400_0008; +const PSCI_CPU_ON: u64 = 0xC400_0003; // SMC64 variant + +/// Power the machine off via PSCI SYSTEM_OFF. Never returns. +/// +/// QEMU `-M virt` exposes PSCI over the conduit selected by the firmware/boot +/// EL: HVC when an EL2 is present, SMC when an EL3 (secure firmware) is. When +/// the kernel is `-kernel`-booted straight into EL1 with neither EL2 nor EL3, +/// *neither* conduit is reachable (both trap as Undefined at EL1), so we mask +/// interrupts and park instead — the test harness sees the kernel idle. +pub fn system_off() -> ! { + // Only attempt the SMC/HVC call if we are at EL2+ (where a conduit exists). + let el: u64; + unsafe { core::arch::asm!("mrs {}, CurrentEL", out(reg) el, options(nomem, nostack)) }; + if (el >> 2) & 0x3 >= 2 { + unsafe { + core::arch::asm!( + "mov w0, {fn_id:w}", + "hvc #0", + fn_id = in(reg) PSCI_SYSTEM_OFF, + out("x0") _, out("x1") _, out("x2") _, out("x3") _, + options(nostack), + ); + } + } + // Park with interrupts masked. + unsafe { core::arch::asm!("msr daifset, #0xf", options(nomem, nostack)) }; + loop { + unsafe { core::arch::asm!("wfi", options(nomem, nostack)) }; + } +} + +/// Bring a secondary CPU online at `entry` (physical) with context `ctx`. +/// Returns the PSCI status (0 = success, negative = error). SMP milestone +/// helper. +/// +/// QEMU `-M virt` boots the kernel at EL1 and provides the PSCI conduit over +/// HVC; we issue HVC here. If no conduit is active the call traps as Undefined +/// (caught by the EL1 sync vector) — callers should treat SMP as best-effort. +pub fn cpu_on(target_mpidr: u64, entry: u64, ctx: u64) -> i64 { + let ret: i64; + unsafe { + core::arch::asm!( + "mov x0, {fn_id}", + "mov x1, {mpidr}", + "mov x2, {entry}", + "mov x3, {ctx}", + "hvc #0", + "mov {ret}, x0", + fn_id = in(reg) PSCI_CPU_ON, + mpidr = in(reg) target_mpidr, + entry = in(reg) entry, + ctx = in(reg) ctx, + ret = out(reg) ret, + out("x0") _, out("x1") _, out("x2") _, out("x3") _, + options(nostack), + ); + } + ret +} diff --git a/kernel/src/arch/aarch64/ramfb.rs b/kernel/src/arch/aarch64/ramfb.rs new file mode 100644 index 0000000..00463de --- /dev/null +++ b/kernel/src/arch/aarch64/ramfb.rs @@ -0,0 +1,297 @@ +//! QEMU `ramfb` display via the `fw_cfg` interface — aarch64 `-M virt`. +//! +//! `qemu-system-aarch64 -M virt` provides no Limine framebuffer (Limine is the +//! x86_64 boot protocol). QEMU's `-device ramfb` instead exposes a software +//! framebuffer that the guest configures through **fw_cfg**: the guest writes a +//! big-endian [`RamfbCfg`] describing a guest-RAM buffer (physical address, +//! DRM fourcc pixel format, dimensions, stride) to the `etc/ramfb` fw_cfg file, +//! and QEMU thereafter scans that buffer out to the display every refresh. +//! +//! ## fw_cfg on `-M virt` +//! The MMIO fw_cfg device sits at a fixed base (QEMU virt: `0x0902_0000`). Its +//! register block (big-endian, "DMA" interface) is: +//! +//! * `+0x00` (8 bytes) `data` — legacy data port (unused here). +//! * `+0x08` (2 bytes) `selector` — write the 16-bit file/key selector (BE). +//! * `+0x10` (8 bytes) `dma` — write the **physical** address of a +//! [`FwCfgDmaAccess`] control block (BE) to +//! kick a DMA transfer. +//! +//! To configure ramfb we: +//! 1. Read the fw_cfg file directory (`FW_CFG_FILE_DIR`, selector `0x0019`) +//! and find the `etc/ramfb` entry → its write-only selector key. +//! 2. Allocate a framebuffer in guest RAM (identity-mapped, so VA == PA). +//! 3. DMA-write a big-endian [`RamfbCfg`] into that file selector. +//! +//! After step 3 QEMU latches the configuration and the framebuffer is live; any +//! pixels the kernel writes into the buffer appear on screen at the next refresh. +//! +//! The framebuffer is then published to the shared [`crate::drivers::fb`] console +//! through its arch-neutral `init_raw` hook, so the existing compositor / fb text +//! console (which x86 feeds from Limine) works unchanged on ARM. + +use core::sync::atomic::{AtomicU64, Ordering}; + +/// fw_cfg MMIO base on QEMU `-M virt`. +const FW_CFG_BASE: u64 = 0x0902_0000; + +/// MMIO register offset for the fw_cfg DMA control-block pointer (big-endian). +/// All fw_cfg access here goes through the DMA interface, which folds item +/// selection into the DMA control word, so the legacy selector port is unused. +const FW_CFG_REG_DMA: u64 = 0x10; + +/// Well-known fw_cfg selector for the file directory. +const FW_CFG_FILE_DIR: u16 = 0x0019; + +/// fw_cfg DMA control bits (the `control` field of [`FwCfgDmaAccess`], BE). +const FW_CFG_DMA_CTL_ERROR: u32 = 0x01; +const FW_CFG_DMA_CTL_READ: u32 = 0x02; +const FW_CFG_DMA_CTL_SKIP: u32 = 0x04; +const FW_CFG_DMA_CTL_SELECT: u32 = 0x08; +const FW_CFG_DMA_CTL_WRITE: u32 = 0x10; + +/// DRM fourcc for `XRGB8888` (little-endian: byte0=B, byte1=G, byte2=R, byte3=X) +/// — the format the shared fb console expects (32 bpp, X in the high byte). +const DRM_FORMAT_XRGB8888: u32 = 0x3432_5258; // 'XR24' + +/// Default display geometry. QEMU ramfb is fully software-defined, so any size +/// the guest picks is honoured; 1280×800 matches the x86 default shell surface. +const FB_WIDTH: u32 = 1280; +const FB_HEIGHT: u32 = 800; +const FB_BPP: u32 = 4; // bytes per pixel (XRGB8888) + +/// fw_cfg DMA access control block. All fields big-endian per the fw_cfg spec. +#[repr(C)] +struct FwCfgDmaAccess { + control: u32, // BE + length: u32, // BE + address: u64, // BE (physical address of the data buffer) +} + +/// The `etc/ramfb` configuration record. **All fields big-endian.** +/// +/// `#[repr(C, packed)]` is required: the on-wire record is exactly 28 bytes +/// (8+4+4+4+4+4), but a naturally-aligned `repr(C)` would pad the leading `u64` +/// out to 32, and QEMU rejects an `etc/ramfb` write whose length exceeds the +/// file's declared 28-byte size (the cause of the earlier DMA error bit). +#[repr(C, packed)] +struct RamfbCfg { + addr: u64, // BE — physical address of the framebuffer + fourcc: u32, // BE — DRM fourcc pixel format + flags: u32, // BE — reserved (0) + width: u32, // BE + height: u32, // BE + stride: u32, // BE — bytes per row +} + +/// One fw_cfg file-directory entry (big-endian fields), spec layout. +#[repr(C)] +struct FwCfgFile { + size: u32, // BE — file size in bytes + select: u16, // BE — selector key + reserved: u16, // BE + name: [u8; 56], // NUL-padded ASCII name +} + +/// Published framebuffer physical base once ramfb is configured (0 = not up). +static FB_PHYS: AtomicU64 = AtomicU64::new(0); + +#[inline(always)] +unsafe fn mmio_write_u64(off: u64, v: u64) { + core::ptr::write_volatile((FW_CFG_BASE + off) as *mut u64, v); +} + +/// Kick a fw_cfg DMA transfer described by `dma` (which must be a populated, +/// big-endian [`FwCfgDmaAccess`] in guest RAM) and spin until QEMU clears the +/// busy control bits. Returns `false` if QEMU flagged an error. +unsafe fn fw_cfg_dma(dma: &mut FwCfgDmaAccess) -> bool { + // The DMA address register is big-endian; write the *physical* address of the + // control block (identity map → VA == PA). + let dma_pa = dma as *mut FwCfgDmaAccess as u64; + mmio_write_u64(FW_CFG_REG_DMA, dma_pa.to_be()); + + // QEMU processes the transfer synchronously under TCG, but per spec we must + // poll `control` until all bits except a possible ERROR are cleared. + loop { + let ctl = core::ptr::read_volatile(&dma.control as *const u32); + let ctl = u32::from_be(ctl); + if ctl & FW_CFG_DMA_CTL_ERROR != 0 { + log::warn!("[ramfb] fw_cfg DMA error: control={:#x}", ctl); + return false; + } + // All action bits cleared → transfer complete. + if ctl + & (FW_CFG_DMA_CTL_READ + | FW_CFG_DMA_CTL_WRITE + | FW_CFG_DMA_CTL_SELECT + | FW_CFG_DMA_CTL_SKIP) + == 0 + { + return true; + } + core::hint::spin_loop(); + } +} + +/// Select a fw_cfg item and DMA-read `len` bytes of it into `buf` (physical). +unsafe fn fw_cfg_read(selector: u16, buf: *mut u8, len: u32) -> bool { + let mut dma = FwCfgDmaAccess { + control: ((FW_CFG_DMA_CTL_SELECT | FW_CFG_DMA_CTL_READ) + | ((selector as u32) << 16)) + .to_be(), + length: len.to_be(), + address: (buf as u64).to_be(), + }; + fw_cfg_dma(&mut dma) +} + +/// Select a fw_cfg item by key and DMA-write `len` bytes from `buf` into it. +/// +/// QEMU rejects a combined SELECT+WRITE in one DMA control word (it returns the +/// error bit), so we issue a SELECT-only DMA first to point fw_cfg at the file, +/// then a WRITE-only DMA carrying the payload. +unsafe fn fw_cfg_write(selector: u16, buf: *const u8, len: u32) -> bool { + // 1. SELECT-only: point fw_cfg at the target file (resets its offset to 0). + let mut sel = FwCfgDmaAccess { + control: (FW_CFG_DMA_CTL_SELECT | ((selector as u32) << 16)).to_be(), + length: 0u32.to_be(), + address: 0u64.to_be(), + }; + if !fw_cfg_dma(&mut sel) { + return false; + } + // 2. WRITE-only: stream the payload into the now-selected file. + let mut dma = FwCfgDmaAccess { + control: FW_CFG_DMA_CTL_WRITE.to_be(), + length: len.to_be(), + address: (buf as u64).to_be(), + }; + fw_cfg_dma(&mut dma) +} + +/// Walk the fw_cfg file directory and return the selector key of `etc/ramfb`. +unsafe fn find_ramfb_selector() -> Option { + // The directory begins with a big-endian u32 count, then `count` FwCfgFile + // records. Read the count first. + let mut count_be: u32 = 0; + if !fw_cfg_read( + FW_CFG_FILE_DIR, + &mut count_be as *mut u32 as *mut u8, + 4, + ) { + return None; + } + let count = u32::from_be(count_be); + if count == 0 || count > 1024 { + return None; + } + + // Re-select + re-read so the directory stream restarts at the count, then + // skip the 4-byte count and stream the entries. + let entries_len = count + .saturating_mul(core::mem::size_of::() as u32) + .saturating_add(4); + let buf_pa = crate::mm::frame_allocator::alloc_contiguous_frames( + ((entries_len as usize) + 4095) / 4096, + )?; + let ok = fw_cfg_read(FW_CFG_FILE_DIR, buf_pa as *mut u8, entries_len); + if !ok { + crate::mm::frame_allocator::free_frame(buf_pa); + return None; + } + + let mut found: Option = None; + let base = (buf_pa + 4) as *const FwCfgFile; // skip the leading count + for i in 0..count as usize { + let f = &*base.add(i); + // Match the NUL-terminated name against "etc/ramfb". + let name = &f.name; + let target = b"etc/ramfb"; + let mut matches = true; + for (j, &c) in target.iter().enumerate() { + if name[j] != c { + matches = false; + break; + } + } + if matches && name[target.len()] == 0 { + found = Some(u16::from_be(f.select)); + log::debug!( + "[ramfb] fw_cfg dir: etc/ramfb sel={:#x} size={} (of {} files)", + u16::from_be(f.select), + u32::from_be(f.size), + count + ); + break; + } + } + + crate::mm::frame_allocator::free_frame(buf_pa); + found +} + +/// Bring up the QEMU ramfb display and publish it to the shared fb console. +/// +/// Returns `(phys_addr, width, height, stride_bytes)` on success. Must be called +/// after the frame allocator is online (so we can reserve the framebuffer) and +/// with the low-1 GiB device MMIO identity-mapped (so fw_cfg is reachable). +pub fn init() -> Option<(u64, u32, u32, u32)> { + unsafe { + let selector = match find_ramfb_selector() { + Some(s) => s, + None => { + log::warn!("[ramfb] etc/ramfb not found in fw_cfg directory"); + return None; + } + }; + + let stride = FB_WIDTH * FB_BPP; + let fb_bytes = (stride as usize) * (FB_HEIGHT as usize); + let pages = (fb_bytes + 4095) / 4096; + let fb_phys = crate::mm::frame_allocator::alloc_contiguous_frames(pages)?; + + // Zero the framebuffer (identity-mapped → write directly). + { + let p = fb_phys as *mut u32; + for i in 0..(fb_bytes / 4) { + core::ptr::write_volatile(p.add(i), 0x0000_0000); + } + } + + // Build + DMA-write the big-endian RAMFBCfg into etc/ramfb. + let cfg = RamfbCfg { + addr: fb_phys.to_be(), + fourcc: DRM_FORMAT_XRGB8888.to_be(), + flags: 0u32.to_be(), + width: FB_WIDTH.to_be(), + height: FB_HEIGHT.to_be(), + stride: stride.to_be(), + }; + let ok = fw_cfg_write( + selector, + &cfg as *const RamfbCfg as *const u8, + core::mem::size_of::() as u32, + ); + if !ok { + log::warn!("[ramfb] fw_cfg write of etc/ramfb failed"); + return None; + } + + FB_PHYS.store(fb_phys, Ordering::Release); + log::info!( + "[ramfb] up: {}x{} stride={} @ phys {:#x} (sel={:#x})", + FB_WIDTH, + FB_HEIGHT, + stride, + fb_phys, + selector + ); + + Some((fb_phys, FB_WIDTH, FB_HEIGHT, stride)) + } +} + +/// Physical base of the ramfb framebuffer (0 until [`init`] succeeds). +pub fn fb_phys() -> u64 { + FB_PHYS.load(Ordering::Acquire) +} diff --git a/kernel/src/arch/aarch64/smp.rs b/kernel/src/arch/aarch64/smp.rs new file mode 100644 index 0000000..70ab1be --- /dev/null +++ b/kernel/src/arch/aarch64/smp.rs @@ -0,0 +1,91 @@ +//! SMP — aarch64 multiprocessor scaffolding (PSCI / GICv3 SGI based). +//! +//! This is a single-core-only scaffold: the per-CPU table and `this_cpu()` +//! lookup are real (so shared scheduler/process code compiles and runs on the +//! BSP), but AP wake-up via PSCI `CPU_ON` and reschedule IPIs via GIC SGIs are +//! not yet implemented. + +use core::sync::atomic::{AtomicBool, AtomicU32, Ordering}; +use limine::request::MpResponse; + +/// Maximum CPUs supported. +pub const MAX_CPUS: usize = 64; + +/// Per-CPU mutable state — one entry per logical processor. +/// +/// Field layout mirrors the x86_64 backend so shared code is portable. +#[repr(C)] +pub struct PerCpuData { + pub cpu_id: u32, + /// MPIDR_EL1 affinity of this core (aarch64 analogue of the x86 LAPIC id). + pub mpidr: u64, + pub online: AtomicBool, + pub current_pid: AtomicU32, +} + +const ZERO_CPU: PerCpuData = PerCpuData { + cpu_id: 0, + mpidr: 0, + online: AtomicBool::new(false), + current_pid: AtomicU32::new(0), +}; +static mut PER_CPU_DATA: [PerCpuData; MAX_CPUS] = [ZERO_CPU; MAX_CPUS]; + +/// Total logical CPUs online (including BSP). +pub static CPU_COUNT: AtomicU32 = AtomicU32::new(1); + +/// Read this core's MPIDR_EL1 affinity bits. +#[inline] +fn read_mpidr() -> u64 { + let v: u64; + unsafe { core::arch::asm!("mrs {}, mpidr_el1", out(reg) v, options(nomem, nostack)) }; + v & 0x00FF_FFFF // Aff0..Aff2 +} + +/// Return this CPU's ID (by matching MPIDR_EL1 against the per-CPU table). +pub fn current_cpu_id() -> u32 { + let n = CPU_COUNT.load(Ordering::Relaxed) as usize; + // Single-core fast path: the BSP is always CPU 0. This also makes the timer + // ISR's per-CPU lookup robust against a transiently-inconsistent PER_CPU_DATA + // read (the lookup result indexes PTABLE_LOCK_RECURSION/PER_CPU_DATA, and a + // garbage index would panic the kernel from inside the IRQ handler). + if n <= 1 { + return 0; + } + let mpidr = read_mpidr(); + unsafe { + let table = &*core::ptr::addr_of!(PER_CPU_DATA); + for i in 0..n.min(MAX_CPUS) { + if table[i].mpidr == mpidr { + return i as u32; + } + } + } + 0 +} + +/// Return this CPU's per-CPU entry. +pub fn this_cpu() -> &'static PerCpuData { + let cpu_idx = current_cpu_id() as usize; + unsafe { &(*core::ptr::addr_of!(PER_CPU_DATA))[cpu_idx] } +} + +/// Wake all APs. Currently single-core only. +/// +/// TODO(arm): bring APs online via PSCI `CPU_ON` (SMC/HVC) and have each call +/// `crate::arch::ap_init()` then `crate::sched::ap_start()`. +pub fn init(_smp_resp: Option<&'static MpResponse>) { + unsafe { + let table = &mut *core::ptr::addr_of_mut!(PER_CPU_DATA); + table[0].cpu_id = 0; + table[0].mpidr = read_mpidr(); + table[0].online.store(true, Ordering::Release); + } + CPU_COUNT.store(1, Ordering::Release); + log::info!("[SMP] aarch64 single-core scaffold (AP wake via PSCI not implemented)"); +} + +/// Broadcast a reschedule IPI to all other online CPUs. +/// +/// TODO(arm): send a GIC SGI to the target CPU interface(s). No-op (single core). +pub fn broadcast_resched_ipi() {} diff --git a/kernel/src/arch/aarch64/syscall.rs b/kernel/src/arch/aarch64/syscall.rs new file mode 100644 index 0000000..4b226fc --- /dev/null +++ b/kernel/src/arch/aarch64/syscall.rs @@ -0,0 +1,211 @@ +//! SVC fast-path syscall scaffolding — aarch64. +//! +//! On x86 user syscalls arrive via the `SYSCALL` instruction and a per-CPU GS +//! scratch area captures the user register snapshot at entry. On aarch64 the +//! equivalent is the `SVC` instruction trapping to the EL1 synchronous exception +//! vector, which would stash the user GPRs (x0..x30, SP_EL0, ELR/SPSR) into a +//! per-CPU scratch. +//! +//! This module preserves the public surface the shared kernel calls +//! (`set_active_stack_top`, `user_rsp`/`user_rip`/`user_r9`/`user_rbp`, +//! `user_gprs`, `init`, `init_ap`). The per-CPU snapshot capture itself (the +//! exception-vector entry) is not yet implemented, so the accessors return the +//! last value written into the scratch (zero until the vector is wired up). + +use core::sync::atomic::{AtomicU64, Ordering}; + +const MAX_CPUS: usize = 64; + +/// Per-CPU snapshot of the user register file captured at SVC entry. +/// +/// Field names mirror the x86_64 [`UserGprSnapshot`] so cross-arch process code +/// (e.g. `save_full_user_gprs`) is portable. On aarch64 these map to the AAPCS64 +/// argument/callee-saved registers (rip→ELR_EL1, rsp→SP_EL0, etc.). +#[repr(C)] +struct CpuScratch { + user_rsp: u64, + user_rip: u64, + syscall_stack_top: u64, + rdi: u64, rsi: u64, rdx: u64, + r10: u64, r8: u64, r9: u64, + rbx: u64, rbp: u64, + r12: u64, r13: u64, r14: u64, r15: u64, + // aarch64-only: the user link register (x30) at SVC entry. AArch64 has no + // on-stack return address — a function that yields inside a syscall must + // have its LR restored, or its later `ret` branches to whatever build_image + // leaves in x30 (previously hardwired to 0 → branch-to-0 crash). + lr: u64, +} + +const ZERO_SCRATCH: CpuScratch = CpuScratch { + user_rsp: 0, user_rip: 0, syscall_stack_top: 0, + rdi: 0, rsi: 0, rdx: 0, r10: 0, r8: 0, r9: 0, + rbx: 0, rbp: 0, r12: 0, r13: 0, r14: 0, r15: 0, + lr: 0, +}; + +static mut CPU_SCRATCHES: [CpuScratch; MAX_CPUS] = [ZERO_SCRATCH; MAX_CPUS]; + +/// Active kernel syscall-stack top, per CPU (set by the scheduler before +/// returning to user). Mirrors the x86 `set_active_stack_top` behaviour. +static ACTIVE_STACK_TOP: [AtomicU64; MAX_CPUS] = [const { AtomicU64::new(0) }; MAX_CPUS]; + +/// Snapshot of all general-purpose user registers captured at SVC entry. +/// +/// Field names are kept identical to the x86_64 backend so shared code stays +/// architecture-neutral. +#[derive(Clone, Copy, Default)] +pub struct UserGprSnapshot { + pub rdi: u64, pub rsi: u64, pub rdx: u64, + pub r10: u64, pub r8: u64, pub r9: u64, + pub rbx: u64, pub rbp: u64, + pub r12: u64, pub r13: u64, pub r14: u64, pub r15: u64, + /// aarch64 user link register (x30) at SVC entry. Unused on x86. + pub lr: u64, +} + +#[inline] +fn cpu_idx() -> usize { + crate::arch::aarch64::smp::current_cpu_id() as usize +} + +/// Capture the user register snapshot from an EL0 SVC trap frame into this +/// CPU's scratch, mirroring the x86 syscall-entry GS snapshot. After this, the +/// shared accessors (`user_rip`, `user_rsp`, `user_gprs`, ...) return the +/// trapping thread's state, so the architecture-neutral `dispatch_fast` path can +/// read it. +/// +/// AArch64→x86 field mapping (Linux aarch64 ABI): x0..x5 → rdi/rsi/rdx/r10/r8/r9 +/// (the syscall arg registers), x8 → syscall number (handled by the caller), +/// SP_EL0 → user_rsp, ELR_EL1 → user_rip; callee-saved x19..x23/x29 fill the +/// rbx/r12..r15/rbp slots used by `save_full_user_gprs`. +pub fn capture_from_trap(frame: &super::vectors::TrapFrame) { + let i = cpu_idx(); + unsafe { + let s = &mut (*core::ptr::addr_of_mut!(CPU_SCRATCHES))[i]; + s.user_rsp = frame.sp_el0; + s.user_rip = frame.elr; + s.rdi = frame.x[0]; + s.rsi = frame.x[1]; + s.rdx = frame.x[2]; + s.r10 = frame.x[3]; + s.r8 = frame.x[4]; + s.r9 = frame.x[5]; + // Callee-saved mapping for cross-arch register preservation. + s.rbx = frame.x[19]; + s.r12 = frame.x[20]; + s.r13 = frame.x[21]; + s.r14 = frame.x[22]; + s.r15 = frame.x[23]; + s.rbp = frame.x[29]; + s.lr = frame.x[30]; + } +} + +/// Production SVC (EL0 synchronous) handler — routes a userspace `svc #0` into +/// the shared syscall dispatcher. +/// +/// The Linux aarch64 syscall ABI puts the number in x8 and args in x0..x5; the +/// OSCortex shared dispatcher (`syscall::dispatch_fast`) takes (nr, a0..a4) and +/// returns the result in x0. We capture the trapping thread's user GPRs into the +/// per-CPU snapshot first (so the dispatcher's `user_rip`/`user_rsp`/`user_gprs` +/// accessors reflect the caller), dispatch, then write the return value back into +/// x0 of the saved frame so the vector stub's `eret` resumes EL0 with it. +/// +/// The faulting instruction is `svc #0`; ELR_EL1 already points at the *next* +/// instruction on a synchronous SVC, so no manual advance is needed. +/// +/// If the call terminated the process (exit / exit_group), the EL0 context is no +/// longer valid to resume, so we hand off to the next runnable user process via +/// the shared scheduler instead of returning to the vector stub. +fn production_svc_handler(f: &mut super::vectors::TrapFrame) { + capture_from_trap(f); + + let nr = f.x[8]; + let ret = crate::syscall::dispatch_fast( + nr, f.x[0], f.x[1], f.x[2], f.x[3], f.x[4], + ); + f.x[0] = ret as u64; + + // exit(60) / exit_group(231): the current process is now a zombie; do not + // eret back into it. Pick the next runnable user thread and enter it. + if nr == 60 || nr == 231 { + let cur = crate::process::current_pid(); + match crate::process::next_runnable_pid(cur) { + Some(next) => crate::process::enter_user_by_pid_noreturn(next), + None => { + // No more user work — park this core. The shared idle/cortex loop + // keeps the kernel alive; an exiting init with nothing to run is a + // clean stop for the bring-up. + log::info!("[svc] init exited (code={}) — no runnable user process; parking", ret); + loop { + crate::arch::halt(); + } + } + } + } +} + +/// Initialise the SVC fast path on the BSP. +/// +/// Installs the production SVC handler into the EL1 vector table (VBAR_EL1 was +/// programmed earlier by `vectors::install`), so a userspace `svc #0` reaches +/// the shared syscall dispatcher. +pub fn init() { + super::vectors::set_svc_handler(production_svc_handler); + log::info!("[Syscall] aarch64 SVC entry ready (production dispatch_fast wired)"); +} + +/// Per-AP SVC init. +/// +/// TODO(arm): point this core's VBAR_EL1 at the shared vector table. +pub fn init_ap(_cpu_idx: u32) {} + +/// Set the active kernel syscall-stack top for the current CPU. +pub fn set_active_stack_top(stack_top: u64) { + let i = cpu_idx(); + ACTIVE_STACK_TOP[i].store(stack_top, Ordering::Release); + unsafe { + let s = &mut (*core::ptr::addr_of_mut!(CPU_SCRATCHES))[i]; + s.syscall_stack_top = stack_top; + } +} + +/// User SP_EL0 captured at SVC entry (x86 user RSP analogue). +pub fn user_rsp() -> u64 { + unsafe { (*core::ptr::addr_of!(CPU_SCRATCHES))[cpu_idx()].user_rsp } +} + +/// User ELR_EL1 (return address) captured at SVC entry (x86 user RIP analogue). +pub fn user_rip() -> u64 { + unsafe { (*core::ptr::addr_of!(CPU_SCRATCHES))[cpu_idx()].user_rip } +} + +/// Sixth user argument register captured at SVC entry (x86 R9 analogue). +pub fn user_r9() -> u64 { + unsafe { (*core::ptr::addr_of!(CPU_SCRATCHES))[cpu_idx()].r9 } +} + +/// User frame-pointer register captured at SVC entry (x86 RBP analogue). +pub fn user_rbp() -> u64 { + unsafe { (*core::ptr::addr_of!(CPU_SCRATCHES))[cpu_idx()].rbp } +} + +/// Read the full user GPR snapshot stashed by the SVC entry vector. +pub fn user_gprs() -> UserGprSnapshot { + unsafe { + let s = &(*core::ptr::addr_of!(CPU_SCRATCHES))[cpu_idx()]; + UserGprSnapshot { + rdi: s.rdi, rsi: s.rsi, rdx: s.rdx, + r10: s.r10, r8: s.r8, r9: s.r9, + rbx: s.rbx, rbp: s.rbp, + r12: s.r12, r13: s.r13, r14: s.r14, r15: s.r15, + lr: s.lr, + } + } +} + +/// User link register (x30) captured at SVC entry. AArch64-only. +pub fn user_lr() -> u64 { + unsafe { (*core::ptr::addr_of!(CPU_SCRATCHES))[cpu_idx()].lr } +} diff --git a/kernel/src/arch/aarch64/timer.rs b/kernel/src/arch/aarch64/timer.rs new file mode 100644 index 0000000..a8144d9 --- /dev/null +++ b/kernel/src/arch/aarch64/timer.rs @@ -0,0 +1,74 @@ +//! ARM generic timer — periodic EL1 physical-timer tick (CNTP_*). +//! +//! Drives the same scheduler tick the x86 APIC timer drives. We use the EL1 +//! physical timer (CNTP_CTL_EL0 / CNTP_TVAL_EL0); on QEMU `-M virt` its +//! interrupt is PPI 30. CNTFRQ_EL0 gives the tick frequency (62.5 MHz on virt). + +use core::sync::atomic::{AtomicU64, Ordering}; + +/// EL1 physical-timer PPI on QEMU `-M virt`. +pub const TIMER_PPI: u32 = 30; + +/// Timer frequency in Hz (read from CNTFRQ_EL0 at init). +static FREQ_HZ: AtomicU64 = AtomicU64::new(0); +/// Reload interval in timer ticks (set from the requested Hz). +static INTERVAL_TICKS: AtomicU64 = AtomicU64::new(0); +/// Number of timer interrupts serviced. +pub static TICKS: AtomicU64 = AtomicU64::new(0); + +#[inline] +fn cntfrq() -> u64 { + let v: u64; + unsafe { core::arch::asm!("mrs {}, cntfrq_el0", out(reg) v, options(nomem, nostack)) }; + v +} + +/// Current physical count. +#[inline] +pub fn count() -> u64 { + let v: u64; + unsafe { core::arch::asm!("mrs {}, cntpct_el0", out(reg) v, options(nomem, nostack)) }; + v +} + +/// Program the timer for a periodic tick at `hz`, register it with the GIC, and +/// arm the first interval. Interrupts must still be unmasked by the caller. +pub fn init_periodic(hz: u64) { + let freq = cntfrq(); + FREQ_HZ.store(freq, Ordering::Release); + let interval = if hz == 0 { freq } else { freq / hz }; + INTERVAL_TICKS.store(interval, Ordering::Release); + + // Route + enable the timer PPI in the GIC. + super::gic::set_priority(TIMER_PPI, 0x80); + super::gic::enable_irq(TIMER_PPI); + + arm_next(interval); + + unsafe { + // CNTP_CTL_EL0: ENABLE=1, IMASK=0 (bit0=enable, bit1=imask). + core::arch::asm!("msr cntp_ctl_el0, {}", in(reg) 1u64, options(nomem, nostack)); + core::arch::asm!("isb", options(nomem, nostack)); + } +} + +#[inline] +fn arm_next(interval: u64) { + unsafe { + // TVAL counts down from `interval`; the interrupt fires at zero. + core::arch::asm!("msr cntp_tval_el0, {}", in(reg) interval, options(nomem, nostack)); + } +} + +/// Called from the IRQ handler when the timer PPI fires: bump the tick count +/// and re-arm the next interval. +pub fn on_tick() { + TICKS.fetch_add(1, Ordering::Relaxed); + let interval = INTERVAL_TICKS.load(Ordering::Acquire); + arm_next(interval); +} + +/// Timer frequency in Hz (0 until init). +pub fn freq_hz() -> u64 { + FREQ_HZ.load(Ordering::Acquire) +} diff --git a/kernel/src/arch/aarch64/uart.rs b/kernel/src/arch/aarch64/uart.rs new file mode 100644 index 0000000..e4a78cd --- /dev/null +++ b/kernel/src/arch/aarch64/uart.rs @@ -0,0 +1,169 @@ +//! PL011 UART driver — aarch64 serial console (QEMU `-M virt` @ 0x0900_0000). +//! +//! This is the bring-up printf/debug lifeline. QEMU's `-M virt` machine wires a +//! PrimeCell PL011 UART at physical 0x0900_0000; with `-serial mon:stdio` its +//! TX shows up on the host terminal. We only need polled TX (and optional RX) +//! for bring-up, so this is deliberately minimal. +//! +//! The driver works both before and after the MMU is enabled: pre-MMU the base +//! is a raw physical address; once the MMU is on we map the same physical page +//! at an identity/uncached VA and keep using it (the bring-up identity-maps the +//! device region, so the same address stays valid). + +use core::sync::atomic::{AtomicU64, Ordering}; + +/// PL011 base on QEMU `-M virt`. Overridable once the MMU remaps it. +const PL011_PHYS: u64 = 0x0900_0000; + +/// Active MMIO base for the UART (physical until the MMU is on; then the VA the +/// device page was identity-mapped to — same value with an identity map). +static UART_BASE: AtomicU64 = AtomicU64::new(PL011_PHYS); + +// PL011 register offsets. +const UARTDR: u64 = 0x00; // Data register +const UARTFR: u64 = 0x18; // Flag register +const UARTIBRD: u64 = 0x24; // Integer baud-rate divisor +const UARTFBRD: u64 = 0x28; // Fractional baud-rate divisor +const UARTLCR_H: u64 = 0x2c; // Line control +const UARTCR: u64 = 0x30; // Control +const UARTIMSC: u64 = 0x38; // Interrupt mask set/clear + +// UARTFR bits. +const FR_TXFF: u32 = 1 << 5; // transmit FIFO full +const FR_RXFE: u32 = 1 << 4; // receive FIFO empty + +#[inline(always)] +fn base() -> u64 { + UART_BASE.load(Ordering::Relaxed) +} + +#[inline(always)] +unsafe fn reg_read(off: u64) -> u32 { + core::ptr::read_volatile((base() + off) as *const u32) +} + +#[inline(always)] +unsafe fn reg_write(off: u64, val: u32) { + core::ptr::write_volatile((base() + off) as *mut u32, val); +} + +/// Initialise the PL011 for polled 115200-8N1 TX. Safe to call before the MMU +/// is on. QEMU ignores the exact baud divisor, but real hardware needs it. +pub fn init() { + unsafe { + // Disable the UART while reconfiguring. + reg_write(UARTCR, 0); + // Mask all interrupts (we poll). + reg_write(UARTIMSC, 0); + // 115200 baud assuming a 24 MHz UARTCLK (QEMU virt): divisor = 24e6 / + // (16*115200) = 13.0208 → IBRD=13, FBRD=round(0.0208*64)=1. + reg_write(UARTIBRD, 13); + reg_write(UARTFBRD, 1); + // 8 bits, FIFO enabled, no parity. + reg_write(UARTLCR_H, (1 << 4) | (3 << 5)); + // Enable UART + TX + RX. + reg_write(UARTCR, (1 << 0) | (1 << 8) | (1 << 9)); + } +} + +/// Point the driver at a new base address (used after the MMU remaps the device +/// page to a non-identity VA, if ever). For an identity map this is a no-op. +pub fn set_base(new_base: u64) { + UART_BASE.store(new_base, Ordering::Release); +} + +/// Write one byte, blocking while the TX FIFO is full. +#[inline] +pub fn putb(b: u8) { + unsafe { + while reg_read(UARTFR) & FR_TXFF != 0 { + core::hint::spin_loop(); + } + reg_write(UARTDR, b as u32); + } +} + +/// Write a string, expanding `\n` → `\r\n`. +pub fn puts(s: &str) { + for &b in s.as_bytes() { + if b == b'\n' { + putb(b'\r'); + } + putb(b); + } +} + +/// Write a 64-bit value as `0x`-prefixed, zero-padded hex. +pub fn puthex(mut v: u64) { + const HEX: &[u8; 16] = b"0123456789abcdef"; + putb(b'0'); + putb(b'x'); + let mut started = false; + for i in (0..16).rev() { + let nib = ((v >> (i * 4)) & 0xf) as usize; + if nib != 0 || started || i == 0 { + putb(HEX[nib]); + started = true; + } + } + let _ = &mut v; +} + +/// Write a u64 as `0x`-prefixed, full-width (16 nibble) hex — handy for raw +/// register dumps where leading zeros matter. +pub fn puthex_full(v: u64) { + const HEX: &[u8; 16] = b"0123456789abcdef"; + putb(b'0'); + putb(b'x'); + for i in (0..16).rev() { + putb(HEX[((v >> (i * 4)) & 0xf) as usize]); + } +} + +/// Non-blocking single-byte read. Returns `None` when the RX FIFO is empty. +pub fn try_getb() -> Option { + unsafe { + if reg_read(UARTFR) & FR_RXFE != 0 { + None + } else { + Some((reg_read(UARTDR) & 0xff) as u8) + } + } +} + +/// `core::fmt::Write` adapter so bring-up code can `write!`/`writeln!`. +pub struct Writer; + +impl core::fmt::Write for Writer { + fn write_str(&mut self, s: &str) -> core::fmt::Result { + puts(s); + Ok(()) + } +} + +/// Formatted write to the UART (the bring-up `print!` analogue). +pub fn _fmt(args: core::fmt::Arguments) { + use core::fmt::Write as _; + let _ = Writer.write_fmt(args); +} + +/// `println!`-style helper for the bring-up path. Defined as a plain macro +/// (not `#[macro_export]`) so it stays addressable within the arch module tree +/// without tripping the absolute-path lint on macro-expanded exports. +macro_rules! a64println { + () => { $crate::arch::aarch64::uart::puts("\n") }; + ($($arg:tt)*) => {{ + $crate::arch::aarch64::uart::_fmt(format_args!($($arg)*)); + $crate::arch::aarch64::uart::puts("\n"); + }}; +} +pub(crate) use a64println; + +/// `print!`-style helper for the bring-up path. +macro_rules! a64print { + ($($arg:tt)*) => {{ + $crate::arch::aarch64::uart::_fmt(format_args!($($arg)*)); + }}; +} +#[allow(unused_imports)] +pub(crate) use a64print; diff --git a/kernel/src/arch/aarch64/vectors.rs b/kernel/src/arch/aarch64/vectors.rs new file mode 100644 index 0000000..85f1ba5 --- /dev/null +++ b/kernel/src/arch/aarch64/vectors.rs @@ -0,0 +1,521 @@ +//! EL1 exception vector table (VBAR_EL1) + trap-frame entry/exit. +//! +//! AArch64 dispatches all exceptions through a 16-entry, 2 KiB-aligned vector +//! table. Each entry is 0x80 bytes (32 instructions) and corresponds to one of +//! 4 groups × 4 kinds: +//! +//! group 0: Current EL with SP_EL0 (we never run on SP_EL0 → unused) +//! group 1: Current EL with SP_ELx (kernel-mode exceptions/IRQs) +//! group 2: Lower EL using AArch64 (EL0 syscalls/IRQs — the user path) +//! group 3: Lower EL using AArch32 (unused — we are AArch64-only) +//! +//! kinds: +0x000 Synchronous, +0x080 IRQ, +0x100 FIQ, +0x180 SError +//! +//! Each vector saves the full integer register file (x0–x30) plus SP_EL0, +//! ELR_EL1 and SPSR_EL1 into a [`TrapFrame`] on the current stack, calls the +//! Rust dispatcher with a pointer to it, then restores and `eret`s. The frame +//! layout is shared with `enter_user` so user-entry and exception-return use one +//! ABI. + +use core::arch::global_asm; +use core::sync::atomic::{AtomicU64, Ordering}; + +/// Full integer register context saved on every EL1 exception entry. +/// +/// Layout is fixed and matched byte-for-byte by the asm save/restore in +/// `vectors.S`-equivalent `global_asm!` below. Total size: 36 × 8 = 288 bytes +/// (34 fields + ELR/SPSR), kept 16-byte aligned. +#[repr(C)] +#[derive(Debug, Default, Clone, Copy)] +pub struct TrapFrame { + pub x: [u64; 31], // x0..x30 (x30 = LR) + pub sp_el0: u64, // user stack pointer (SP_EL0) + pub elr: u64, // ELR_EL1 (return address) + pub spsr: u64, // SPSR_EL1 (saved PSTATE) + pub esr: u64, // ESR_EL1 (syndrome — filled by the dispatcher path) +} + +/// Count of each exception kind seen, for bring-up visibility. +pub static SYNC_EL1: AtomicU64 = AtomicU64::new(0); +pub static IRQ_COUNT: AtomicU64 = AtomicU64::new(0); +pub static SYNC_EL0: AtomicU64 = AtomicU64::new(0); + +extern "C" { + /// Vector table base symbol defined in the `global_asm!` below. + static __vectors_el1: u8; +} + +/// Install the vector table into VBAR_EL1 for the current core. +pub fn install() { + let base = unsafe { core::ptr::addr_of!(__vectors_el1) as u64 }; + unsafe { + core::arch::asm!( + "msr vbar_el1, {}", + "isb", + in(reg) base, + options(nostack, preserves_flags), + ); + } +} + +/// Read the installed VBAR_EL1 (for verification). +pub fn vbar() -> u64 { + let v: u64; + unsafe { core::arch::asm!("mrs {}, vbar_el1", out(reg) v, options(nomem, nostack)) }; + v +} + +// ── Vector table + common save/restore (assembly) ─────────────────────────── +// +// The macro `VENTRY` emits one vector slot that jumps to a shared save stub, +// tagged with the exception "kind" so the Rust dispatcher knows the source. +global_asm!( + r#" +.section .text +.balign 2048 +.globl __vectors_el1 +__vectors_el1: + + // ── Group 0: Current EL, SP_EL0 (unused) ──────────────────────────────── + .balign 0x80 + b __vec_curr_sync // +0x000 Sync + .balign 0x80 + b __vec_curr_irq // +0x080 IRQ + .balign 0x80 + b __vec_curr_fiq // +0x100 FIQ + .balign 0x80 + b __vec_curr_serror // +0x180 SError + + // ── Group 1: Current EL, SP_ELx (kernel) ──────────────────────────────── + .balign 0x80 + b __vec_curr_sync // +0x200 Sync + .balign 0x80 + b __vec_curr_irq // +0x280 IRQ + .balign 0x80 + b __vec_curr_fiq // +0x300 FIQ + .balign 0x80 + b __vec_curr_serror // +0x380 SError + + // ── Group 2: Lower EL, AArch64 (EL0 user) ─────────────────────────────── + .balign 0x80 + b __vec_lower_sync // +0x400 Sync (SVC syscalls, user faults) + .balign 0x80 + b __vec_lower_irq // +0x480 IRQ + .balign 0x80 + b __vec_lower_fiq // +0x500 FIQ + .balign 0x80 + b __vec_lower_serror // +0x580 SError + + // ── Group 3: Lower EL, AArch32 (unused) ───────────────────────────────── + .balign 0x80 + b __vec_lower_sync + .balign 0x80 + b __vec_lower_irq + .balign 0x80 + b __vec_lower_fiq + .balign 0x80 + b __vec_lower_serror + +// ── Save macro: push a TrapFrame onto the current stack ───────────────────── +.macro SAVE_FRAME + sub sp, sp, #(36 * 8) // room for x0..x30, sp_el0, elr, spsr, esr + stp x0, x1, [sp, #(0 * 8)] + stp x2, x3, [sp, #(2 * 8)] + stp x4, x5, [sp, #(4 * 8)] + stp x6, x7, [sp, #(6 * 8)] + stp x8, x9, [sp, #(8 * 8)] + stp x10, x11, [sp, #(10 * 8)] + stp x12, x13, [sp, #(12 * 8)] + stp x14, x15, [sp, #(14 * 8)] + stp x16, x17, [sp, #(16 * 8)] + stp x18, x19, [sp, #(18 * 8)] + stp x20, x21, [sp, #(20 * 8)] + stp x22, x23, [sp, #(22 * 8)] + stp x24, x25, [sp, #(24 * 8)] + stp x26, x27, [sp, #(26 * 8)] + stp x28, x29, [sp, #(28 * 8)] + str x30, [sp, #(30 * 8)] + mrs x0, sp_el0 + str x0, [sp, #(31 * 8)] + mrs x0, elr_el1 + mrs x1, spsr_el1 + stp x0, x1, [sp, #(32 * 8)] // elr, spsr + mrs x0, esr_el1 + str x0, [sp, #(34 * 8)] // esr +.endm + +// ── Restore macro: pop a TrapFrame and eret ───────────────────────────────── +.macro RESTORE_FRAME + ldp x0, x1, [sp, #(32 * 8)] // elr, spsr + msr elr_el1, x0 + msr spsr_el1, x1 + ldr x0, [sp, #(31 * 8)] + msr sp_el0, x0 + ldp x0, x1, [sp, #(0 * 8)] + ldp x2, x3, [sp, #(2 * 8)] + ldp x4, x5, [sp, #(4 * 8)] + ldp x6, x7, [sp, #(6 * 8)] + ldp x8, x9, [sp, #(8 * 8)] + ldp x10, x11, [sp, #(10 * 8)] + ldp x12, x13, [sp, #(12 * 8)] + ldp x14, x15, [sp, #(14 * 8)] + ldp x16, x17, [sp, #(16 * 8)] + ldp x18, x19, [sp, #(18 * 8)] + ldp x20, x21, [sp, #(20 * 8)] + ldp x22, x23, [sp, #(22 * 8)] + ldp x24, x25, [sp, #(24 * 8)] + ldp x26, x27, [sp, #(26 * 8)] + ldp x28, x29, [sp, #(28 * 8)] + ldr x30, [sp, #(30 * 8)] + add sp, sp, #(36 * 8) +.endm + +// ── Per-kind entry stubs: save, call dispatcher (x0=frame, x1=kind), restore ─ +__vec_curr_sync: + SAVE_FRAME + mov x0, sp + mov x1, #1 // KIND_CURR_SYNC + bl {dispatch} + RESTORE_FRAME + eret + +__vec_curr_irq: + SAVE_FRAME + mov x0, sp + mov x1, #2 // KIND_CURR_IRQ + bl {dispatch} + RESTORE_FRAME + eret + +__vec_curr_fiq: + SAVE_FRAME + mov x0, sp + mov x1, #3 // KIND_CURR_FIQ + bl {dispatch} + RESTORE_FRAME + eret + +__vec_curr_serror: + SAVE_FRAME + mov x0, sp + mov x1, #4 // KIND_CURR_SERROR + bl {dispatch} + RESTORE_FRAME + eret + +__vec_lower_sync: + SAVE_FRAME + mov x0, sp + mov x1, #5 // KIND_LOWER_SYNC (EL0) + bl {dispatch} + RESTORE_FRAME + eret + +__vec_lower_irq: + SAVE_FRAME + mov x0, sp + mov x1, #6 // KIND_LOWER_IRQ + bl {dispatch} + RESTORE_FRAME + eret + +__vec_lower_fiq: + SAVE_FRAME + mov x0, sp + mov x1, #7 // KIND_LOWER_FIQ + bl {dispatch} + RESTORE_FRAME + eret + +__vec_lower_serror: + SAVE_FRAME + mov x0, sp + mov x1, #8 // KIND_LOWER_SERROR + bl {dispatch} + RESTORE_FRAME + eret +"#, + dispatch = sym dispatch, +); + +// Exception "kind" tags passed from the asm stubs. +pub const KIND_CURR_SYNC: u64 = 1; +pub const KIND_CURR_IRQ: u64 = 2; +pub const KIND_CURR_FIQ: u64 = 3; +pub const KIND_CURR_SERROR: u64 = 4; +pub const KIND_LOWER_SYNC: u64 = 5; +pub const KIND_LOWER_IRQ: u64 = 6; +pub const KIND_LOWER_FIQ: u64 = 7; +pub const KIND_LOWER_SERROR: u64 = 8; + +// ESR_EL1 exception classes we care about. +const EC_SVC64: u64 = 0x15; // SVC instruction from AArch64 +// EC encodings (ARM ARM D17.2.37): +// 0x20 = Instruction Abort from a lower EL +// 0x21 = Instruction Abort taken without a change in EL (same EL) +// 0x24 = Data Abort from a lower EL (EL0 user fault → KIND_LOWER_SYNC) +// 0x25 = Data Abort without a change in EL (EL1 kernel fault → KIND_CURR_SYNC) +// The Flutter engine demand-faults from BOTH: EL0 when its own code touches an +// unmapped Dart heap page (0x24), and EL1 when a syscall handler dereferences a +// not-yet-paged user pointer (0x25). Both must route to the demand pager. +const EC_DABT_LOWER: u64 = 0x24; // Data abort, lower EL (EL0) +const EC_DABT_CURR: u64 = 0x25; // Data abort, same EL (EL1 touching a user VA) +const EC_IABT_LOWER: u64 = 0x20; // Instruction abort, lower EL + +/// Optional override for synchronous-from-EL0 handling, installed by the +/// syscall layer once SVC dispatch is wired (milestone 6). Returns true if it +/// fully handled the exception. +pub static SVC_HANDLER: AtomicU64 = AtomicU64::new(0); +/// Optional override for IRQ handling, installed by the timer/GIC layer +/// (milestone 4/5). +pub static IRQ_HANDLER: AtomicU64 = AtomicU64::new(0); + +type SvcFn = fn(&mut TrapFrame); +type IrqFn = fn(&mut TrapFrame); + +/// Install the SVC (EL0 synchronous) handler. +pub fn set_svc_handler(f: SvcFn) { + SVC_HANDLER.store(f as usize as u64, Ordering::Release); +} + +/// Install the IRQ handler. +pub fn set_irq_handler(f: IrqFn) { + IRQ_HANDLER.store(f as usize as u64, Ordering::Release); +} + +/// The C-ABI exception dispatcher called by every vector stub. +/// +/// `frame` points at the saved [`TrapFrame`] on the trapping stack; `kind` is +/// the exception group/kind tag from the stub. +#[no_mangle] +extern "C" fn dispatch(frame: *mut TrapFrame, kind: u64) { + let f = unsafe { &mut *frame }; + let ec = (f.esr >> 26) & 0x3f; + + match kind { + KIND_LOWER_IRQ | KIND_CURR_IRQ => { + IRQ_COUNT.fetch_add(1, Ordering::Relaxed); + let h = IRQ_HANDLER.load(Ordering::Acquire); + if h != 0 { + let func: IrqFn = unsafe { core::mem::transmute(h as usize) }; + func(f); + } + } + KIND_LOWER_SYNC => { + SYNC_EL0.fetch_add(1, Ordering::Relaxed); + if ec == EC_SVC64 { + let h = SVC_HANDLER.load(Ordering::Acquire); + if h != 0 { + // EAGER user-GPR capture while IRQs are STILL MASKED. Snapshot + // this thread's callee-saved regs (x19–x23/x29) + link + // register (x30) from the trap frame into its PTABLE slot + // now, before we unmask below. Once IRQs are on, a generic- + // timer tick can switch to a sibling thread whose own SVC + // overwrites the shared per-CPU snapshot; gating the later + // yield-time `save_full_user_gprs` on this capture stops that + // stale snapshot from leaking back as our resume context + // (the bug: x30 captured stale → on resume `ret` branches to + // 0 → EL0 UDF right after FlutterEngineRunInitialized). This + // is the aarch64 counterpart of the x86 SYSCALL-entry capture + // (interrupts there stay masked until the handler `sti`s). + crate::arch::aarch64::syscall::capture_from_trap(f); + crate::process::capture_user_gprs_at_entry(crate::process::current_pid()); + // Unmask IRQs for the duration of the syscall, mirroring x86 + // where the SYSCALL entry stub runs handlers with interrupts + // ENABLED. Exception entry masked DAIF.I; without re-enabling + // it the generic-timer scheduler tick cannot fire while a + // syscall runs. The Flutter engine's cooperative bring-up + // makes long, spinning syscalls (epoll_wait bootstrap-spin, + // condvar waits) that hand off between sibling threads WITHIN + // the syscall — if the timer is masked the whole time, a + // thread that spins waiting for a sibling can never be + // preempted and the engine deadlocks with the timer frozen. + // The timer ISR uses try-locks throughout (ISR-safe), so + // taking a tick mid-syscall is safe. Re-mask on the way out + // so the vector restore/eret sees the same DAIF it expects. + unsafe { + core::arch::asm!("msr daifclr, #0b0010", options(nomem, nostack)); + } + let func: SvcFn = unsafe { core::mem::transmute(h as usize) }; + func(f); + unsafe { + core::arch::asm!("msr daifset, #0b0010", options(nomem, nostack)); + } + } else { + report_unhandled(f, kind, ec); + } + } else if ec == EC_DABT_LOWER || ec == EC_IABT_LOWER { + // EL0 data/instruction abort. The Flutter engine relies on + // demand-paged anonymous memory (mmap'd Dart heaps, executable + // AOT regions) and the kernel's VA-mirror window. Route the fault + // through the shared `demand_page` pager exactly like the x86 + // page-fault handler does; if it resolves, `eret` retries the + // faulting instruction. Otherwise surface the fault. + if try_demand_page_el0(f, ec) { + // Resolved — return; the vector restore path will eret and + // re-execute the faulting access against the new mapping. + } else { + report_unhandled(f, kind, ec); + } + } else { + // Other EL0 synchronous fault — surface it. + report_unhandled(f, kind, ec); + } + } + KIND_CURR_SYNC => { + SYNC_EL1.fetch_add(1, Ordering::Relaxed); + if ec == EC_SVC64 { + // Recoverable kernel-side SVC (used by the bring-up self-test + // to prove the vector save/dispatch/restore/eret round-trip). + let h = SVC_HANDLER.load(Ordering::Acquire); + if h != 0 { + let func: SvcFn = unsafe { core::mem::transmute(h as usize) }; + func(f); + } + } else if ec == EC_DABT_CURR { + // Kernel (EL1) touched a demand-paged user VA — e.g. a syscall + // handler dereferencing a not-yet-faulted Dart heap pointer. Run + // the SAME pager as the EL0 path; on success eret re-executes the + // faulting kernel access against the freshly mapped page. + if try_demand_page_el0(f, ec) { + // resolved — fall through to restore + eret + } else { + report_unhandled(f, kind, ec); + } + } else if ec == 0 { + // EC=0 "Unknown reason" at EL1 = an Undefined instruction. This + // is how an unsupported SMC/HVC conduit (e.g. a PSCI probe when + // no conduit is active) surfaces. Treat it as a no-op: report + // once, set x0 to the PSCI NOT_SUPPORTED status, and step over + // the faulting instruction so the probe returns gracefully. + f.x[0] = (-1i64) as u64; // PSCI NOT_SUPPORTED + f.elr = f.elr.wrapping_add(4); + } else { + report_unhandled(f, kind, ec); + } + } + _ => report_unhandled(f, kind, ec), + } +} + +/// Attempt to resolve an EL0 data/instruction abort via the shared demand pager. +/// +/// Reads the faulting address from FAR_EL1 and decodes the ESR fault-status code +/// to distinguish a *translation* fault (page not mapped → demand-page it) from a +/// *permission* fault (page mapped but access not allowed). Returns true if the +/// pager mapped the page (the caller should eret and retry the access). +/// +/// The shared `demand_page(addr, error)` mirrors the x86 page-fault ABI where +/// `error` bit0 = "page present" (a permission, not a translation, fault). We +/// synthesise that bit from the ARM DFSC so the shared pager's +/// `if error & 0x1 != 0 { return false }` early-out behaves identically. +fn try_demand_page_el0(f: &mut TrapFrame, ec: u64) -> bool { + // FAR_EL1 holds the faulting virtual address for aborts. + let far: u64; + unsafe { + core::arch::asm!("mrs {}, far_el1", out(reg) far, options(nomem, nostack)); + } + + // Decode the fault status. For data aborts (EC=0x24) the DFSC is ESR[5:0]; + // for instruction aborts (EC=0x20) the IFSC is likewise ESR[5:0]. + let fsc = f.esr & 0x3f; + // DFSC/IFSC 0b0001xx = translation fault (levels 0..3). 0b0011xx = access + // flag fault, 0b0010xx... ; permission faults are 0b0011xx (0xC..0xF). + let is_translation = matches!(fsc, 0x04 | 0x05 | 0x06 | 0x07); + let is_permission = matches!(fsc, 0x0C | 0x0D | 0x0E | 0x0F); + + // Mirror the x86 error encoding: bit0 = present (permission fault). + let error: u64 = if is_permission { 0x1 } else { 0x0 }; + let _ = (ec, is_translation); + + crate::mm::paging::demand_page(far, error) +} + +/// Print an unhandled-exception report over serial. For bring-up we park after +/// printing so the cause is visible (rather than looping in the vector). +fn report_unhandled(f: &TrapFrame, kind: u64, ec: u64) -> ! { + use crate::arch::aarch64::uart; + uart::puts("\n[EXC] UNHANDLED exception: kind="); + uart::puthex(kind); + uart::puts(" EC="); + uart::puthex(ec); + uart::puts("\n ESR_EL1="); + uart::puthex_full(f.esr); + uart::puts(" ELR="); + uart::puthex_full(f.elr); + uart::puts(" SPSR="); + uart::puthex_full(f.spsr); + uart::puts("\n SP_EL0="); + uart::puthex_full(f.sp_el0); + uart::puts(" x0="); + uart::puthex_full(f.x[0]); + let far: u64; + unsafe { core::arch::asm!("mrs {}, far_el1", out(reg) far, options(nomem, nostack)); } + uart::puts("\n FAR_EL1="); + uart::puthex_full(far); + uart::puts(" x30(LR)="); + uart::puthex_full(f.x[30]); + uart::puts(" x1="); + uart::puthex_full(f.x[1]); + uart::puts(" x16="); + uart::puthex_full(f.x[16]); + uart::puts(" x17="); + uart::puthex_full(f.x[17]); + uart::puts(" pid="); + uart::puthex(crate::process::current_pid() as u64); + uart::puts("\n"); + + // Frame-pointer chain walk: with ELR=0 and x30=0 the immediate caller is lost + // from the registers, but the AArch64 AAPCS frame record ([x29]=saved FP, + // [x29+8]=saved LR) lets us recover the call stack from the EL0 stack. TTBR0 + // is still the faulting thread's during this EL1 handler, so EL0 VAs read + // directly. Print the saved-LR chain so the crash can be symbolised. + uart::puts(" x29(FP)="); + uart::puthex_full(f.x[29]); + uart::puts("\n bt:"); + { + let mut fp = f.x[29]; + let mut depth = 0u32; + // Heuristic validity check for an EL0 user VA (low half, non-tiny). + let plausible = |a: u64| a >= 0x1000 && a < 0x0000_8000_0000_0000; + while depth < 16 && plausible(fp) && (fp & 0x7) == 0 { + let saved_fp = unsafe { core::ptr::read_volatile(fp as *const u64) }; + let saved_lr = unsafe { core::ptr::read_volatile((fp + 8) as *const u64) }; + uart::puts(" "); + uart::puthex_full(saved_lr); + // FP must climb the stack monotonically; stop on any inconsistency. + if saved_fp <= fp { break; } + fp = saved_fp; + depth += 1; + } + } + // Raw stack scan: with x30=0/FP=0 the call chain is lost from registers and + // the frame walk yields nothing, but non-leaf callers still pushed their LR + // onto the EL0 stack. Scan upward from SP_EL0 and print any word that lands + // in the engine .so code window [0x1_4100_0000, 0x1_4300_0000) — those are + // almost certainly saved return addresses. addr2line them against + // libflutter_engine.so (base 0x141000000) to recover where the `ret` to 0 + // originated. + uart::puts(" stkscan:"); + { + let sp = f.sp_el0; + let plausible_sp = sp >= 0x1000 && sp < 0x0000_8000_0000_0000 && (sp & 0x7) == 0; + if plausible_sp { + let mut printed = 0u32; + let mut off = 0u64; + while off < 0x2000 && printed < 24 { + let w = unsafe { core::ptr::read_volatile((sp + off) as *const u64) }; + if w >= 0x1_4100_0000 && w < 0x1_4300_0000 { + uart::puts(" "); + uart::puthex_full(w); + printed += 1; + } + off += 8; + } + } + } + uart::puts("\n"); + loop { + unsafe { core::arch::asm!("wfe", options(nomem, nostack)) }; + } +} diff --git a/kernel/src/arch/mod.rs b/kernel/src/arch/mod.rs index 6ee7cc2..d5645fd 100644 --- a/kernel/src/arch/mod.rs +++ b/kernel/src/arch/mod.rs @@ -8,7 +8,7 @@ cfg_if::cfg_if! { mod x86_64; pub use x86_64::*; } else if #[cfg(target_arch = "aarch64")] { - mod aarch64; + pub mod aarch64; pub use aarch64::*; } else if #[cfg(target_arch = "riscv64")] { mod riscv64; @@ -27,7 +27,22 @@ pub mod mmio; cfg_if::cfg_if! { if #[cfg(target_arch = "x86_64")] { - pub use x86_64::cpu::{interrupts_restore, interrupts_save_and_disable, memory_fence, spin_pause}; + pub use x86_64::cpu::{interrupts_disable, interrupts_restore, interrupts_save_and_disable, memory_fence, spin_pause}; + } else if #[cfg(target_arch = "aarch64")] { + pub fn memory_fence() { + core::sync::atomic::fence(core::sync::atomic::Ordering::SeqCst); + } + pub fn spin_pause() { + core::hint::spin_loop(); + } + /// Mask IRQ/FIQ at the current EL (DAIF.I/F). + pub fn interrupts_disable() { + unsafe { core::arch::asm!("msr daifset, #3", options(nomem, nostack, preserves_flags)); } + } + pub fn interrupts_save_and_disable() -> u64 { + 0 + } + pub fn interrupts_restore(_rflags: u64) {} } else { pub fn memory_fence() { core::sync::atomic::fence(core::sync::atomic::Ordering::SeqCst); @@ -35,6 +50,7 @@ cfg_if::cfg_if! { pub fn spin_pause() { core::hint::spin_loop(); } + pub fn interrupts_disable() {} pub fn interrupts_save_and_disable() -> u64 { 0 } @@ -42,6 +58,30 @@ cfg_if::cfg_if! { } } +/// Read the current frame-pointer register (for best-effort backtraces). +/// +/// On x86_64 this is RBP; on aarch64 it is X29. Returns 0 on architectures +/// where no fixed frame pointer is available. +#[inline(always)] +pub fn read_frame_pointer() -> u64 { + #[cfg(target_arch = "x86_64")] + { + let fp: u64; + unsafe { core::arch::asm!("mov {}, rbp", out(reg) fp, options(nomem, nostack)); } + fp + } + #[cfg(target_arch = "aarch64")] + { + let fp: u64; + unsafe { core::arch::asm!("mov {}, x29", out(reg) fp, options(nomem, nostack)); } + fp + } + #[cfg(not(any(target_arch = "x86_64", target_arch = "aarch64")))] + { + 0 + } +} + /// Trigger an ACPI S5 soft-off (architecture-specific implementation). pub fn acpi_shutdown() -> ! { #[cfg(target_arch = "x86_64")] diff --git a/kernel/src/arch/x86_64/cpu.rs b/kernel/src/arch/x86_64/cpu.rs index 43e3cfc..6061e24 100644 --- a/kernel/src/arch/x86_64/cpu.rs +++ b/kernel/src/arch/x86_64/cpu.rs @@ -257,6 +257,14 @@ pub fn spin_pause() { } } +/// Disable interrupts unconditionally (no save/restore). +#[inline(always)] +pub fn interrupts_disable() { + unsafe { + core::arch::asm!("cli", options(nomem, nostack, preserves_flags)); + } +} + /// Save RFLAGS and disable interrupts; returns RFLAGS for [`interrupts_restore`]. #[inline(always)] pub fn interrupts_save_and_disable() -> u64 { diff --git a/kernel/src/arch/x86_64/enter_user.rs b/kernel/src/arch/x86_64/enter_user.rs new file mode 100644 index 0000000..53322b5 --- /dev/null +++ b/kernel/src/arch/x86_64/enter_user.rs @@ -0,0 +1,167 @@ +//! x86_64 user-mode entry transitions. +//! +//! These are the final, no-return transfers into ring-3. The shared +//! `process` layer prepares the full register state, switches CR3 and +//! delivers pending errno; it then calls one of these hooks to actually +//! enter user mode. Two variants exist: +//! +//! * [`enter_user_iret`] — IRETQ path, used when a thread was preempted +//! by the APIC timer at an arbitrary user instruction. Restores every +//! register (including the exact RCX/R11/RFLAGS) so no architectural +//! state is lost. +//! * [`enter_user_sysret`] — SYSRETQ path, the native user-entry used when +//! a thread yielded via a SYSCALL (e.g. futex_wait). Restores the full +//! user GPR set so callee-saved registers survive the SysV-ABI call +//! boundary of the userspace SYSCALL trampoline. +//! +//! The asm is load-bearing for the render path; it must remain byte-for-byte +//! identical to its prior inline form in `process::mod`. + +use core::arch::asm; + +/// Full user register state required to enter ring-3. +#[derive(Clone, Copy)] +pub struct EnterUserRegs { + pub rip: u64, + pub rsp: u64, + pub rflags: u64, + pub rax: u64, + pub rbx: u64, + pub rcx: u64, + pub rdx: u64, + pub rsi: u64, + pub rdi: u64, + pub rbp: u64, + pub r8: u64, + pub r9: u64, + pub r10: u64, + pub r11: u64, + pub r12: u64, + pub r13: u64, + pub r14: u64, + pub r15: u64, +} + +/// IRETQ entry: restore every register and return via the hardware IRET frame. +/// +/// # Safety +/// CR3 must already point at the target thread's address space, and `regs` +/// must describe a valid ring-3 context (mapped RIP/RSP, sane RFLAGS). +#[inline(never)] +pub unsafe fn enter_user_iret(regs: &EnterUserRegs) -> ! { + // Layout of iret_frame (accessed via r11 as base register): + // [r11 + 0x00] RIP ─┐ + // [r11 + 0x08] CS │ hardware IRET frame pushed onto + // [r11 + 0x10] RFLAGS │ the kernel stack by the `push` sequence + // [r11 + 0x18] RSP ─┘ (ring-3 switch includes RSP+SS) + // [r11 + 0x20] SS + // [r11 + 0x28] RAX .. R15 (all user GPRs, r11 loaded last) + let iret_frame: [u64; 20] = [ + /* 0x00 */ regs.rip, + /* 0x08 */ super::gdt::USER_CS as u64, // 0x2B + /* 0x10 */ regs.rflags | 0x200, // ensure IF=1 + /* 0x18 */ regs.rsp, + /* 0x20 */ super::gdt::USER_DS as u64, // 0x23 + /* 0x28 */ regs.rax, + /* 0x30 */ regs.rbx, + /* 0x38 */ regs.rcx, // real RCX (not SYSCALL convention) + /* 0x40 */ regs.rdx, + /* 0x48 */ regs.rdi, + /* 0x50 */ regs.rsi, + /* 0x58 */ regs.rbp, + /* 0x60 */ regs.r8, + /* 0x68 */ regs.r9, + /* 0x70 */ regs.r10, + /* 0x78 */ regs.r11, // real R11 (not RFLAGS convention) + /* 0x80 */ regs.r12, + /* 0x88 */ regs.r13, + /* 0x90 */ regs.r14, + /* 0x98 */ regs.r15, + ]; + asm!( + // Push the 5-word IRET frame: SS, RSP, RFLAGS, CS, RIP. + // These land on the current kernel stack; IRETQ will consume + // them and switch to the user stack (RSP) automatically. + "push qword ptr [r11 + 0x20]", // SS + "push qword ptr [r11 + 0x18]", // RSP + "push qword ptr [r11 + 0x10]", // RFLAGS + "push qword ptr [r11 + 0x08]", // CS + "push qword ptr [r11 + 0x00]", // RIP + // Restore all user GPRs (r11 last because it is our base ptr). + "mov rax, [r11 + 0x28]", + "mov rbx, [r11 + 0x30]", + "mov rcx, [r11 + 0x38]", + "mov rdx, [r11 + 0x40]", + "mov rdi, [r11 + 0x48]", + "mov rsi, [r11 + 0x50]", + "mov rbp, [r11 + 0x58]", + "mov r8, [r11 + 0x60]", + "mov r9, [r11 + 0x68]", + "mov r10, [r11 + 0x70]", + "mov r12, [r11 + 0x80]", + "mov r13, [r11 + 0x88]", + "mov r14, [r11 + 0x90]", + "mov r15, [r11 + 0x98]", + "mov r11, [r11 + 0x78]", // real R11 — must be last + "iretq", + in("r11") iret_frame.as_ptr(), + options(noreturn), + ) +} + +/// SYSRETQ entry: restore the full user GPR set and return to ring-3. +/// +/// # Safety +/// CR3 must already point at the target thread's address space, and `regs` +/// must describe a valid ring-3 context (mapped RIP/RSP, sane RFLAGS). +#[inline(never)] +pub unsafe fn enter_user_sysret(regs: &EnterUserRegs) -> ! { + // Stage values in a fixed-layout stack array and load them inside the asm + // block via a single pointer. RSP and the syscall-consumed regs (RCX, + // R11) are loaded last so all intermediate reads complete first. + let frame: [u64; 16] = [ + /* 0x00 */ regs.rip, + /* 0x08 */ regs.rflags, + /* 0x10 */ regs.rsp, + /* 0x18 */ regs.rdi, + /* 0x20 */ regs.rsi, + /* 0x28 */ regs.rdx, + /* 0x30 */ regs.rax, + /* 0x38 */ regs.rbx, + /* 0x40 */ regs.rbp, + /* 0x48 */ regs.r8, + /* 0x50 */ regs.r9, + /* 0x58 */ regs.r10, + /* 0x60 */ regs.r12, + /* 0x68 */ regs.r13, + /* 0x70 */ regs.r14, + /* 0x78 */ regs.r15, + ]; + asm!( + // Pin the frame base in r11. r11 is consumed by SYSRET (as user + // RFLAGS) and is the LAST register we load — every other load + // can therefore use [r11 + disp] safely. If we let the compiler + // pick the base register, it can alias the base with the first + // destination (e.g. rax) and have the very first `mov` clobber + // the pointer before any further reads happen. + "mov rax, [r11 + 0x30]", + "mov rdi, [r11 + 0x18]", + "mov rsi, [r11 + 0x20]", + "mov rdx, [r11 + 0x28]", + "mov rbx, [r11 + 0x38]", + "mov rbp, [r11 + 0x40]", + "mov r8, [r11 + 0x48]", + "mov r9, [r11 + 0x50]", + "mov r10, [r11 + 0x58]", + "mov r12, [r11 + 0x60]", + "mov r13, [r11 + 0x68]", + "mov r14, [r11 + 0x70]", + "mov r15, [r11 + 0x78]", + "mov rcx, [r11 + 0x00]", // user RIP → RCX (consumed by sysretq) + "mov rsp, [r11 + 0x10]", // switch to user stack (mapped in user PML4) + "mov r11, [r11 + 0x08]", // user RFL → R11 (consumed by sysretq) — load LAST + "sysretq", + in("r11") frame.as_ptr(), + options(noreturn), + ) +} diff --git a/kernel/src/arch/x86_64/mod.rs b/kernel/src/arch/x86_64/mod.rs index ae8a090..8ff5ecc 100644 --- a/kernel/src/arch/x86_64/mod.rs +++ b/kernel/src/arch/x86_64/mod.rs @@ -13,6 +13,9 @@ pub(crate) mod port_io; pub mod syscall; pub mod acpi; pub mod smp; +pub mod enter_user; + +pub use enter_user::{enter_user_iret, enter_user_sysret, EnterUserRegs}; // Re-export commonly used types so external code uses `crate::arch::InterruptFrame` pub use idt::InterruptFrame; @@ -68,6 +71,15 @@ pub fn halt() { unsafe { asm!("hlt") } } +/// Enable interrupts, halt until one arrives, then disable interrupts again. +/// +/// The `sti; hlt` pair is atomic on x86 (STI defers IF until after the next +/// instruction), so no interrupt can sneak in between enabling and halting. +#[inline(always)] +pub fn enable_and_halt() { + unsafe { asm!("sti; hlt; cli", options(nomem, nostack)) } +} + /// Permanently halt (used in panic). pub fn halt_forever() -> ! { loop { diff --git a/kernel/src/drivers/fb.rs b/kernel/src/drivers/fb.rs index 105fb33..c20f7fd 100644 --- a/kernel/src/drivers/fb.rs +++ b/kernel/src/drivers/fb.rs @@ -208,6 +208,29 @@ pub fn init(fb_resp: &limine::request::FramebufferResponse) { FB_READY.store(true, Ordering::Release); } +/// Initialise the framebuffer console from raw parameters (arch-neutral). +/// +/// The x86 path feeds the framebuffer from the Limine response (`init` above); +/// architectures without a bootloader framebuffer (e.g. aarch64 QEMU ramfb, set +/// up by `arch::aarch64::ramfb`) call this with the buffer they configured. The +/// `addr` must be a directly-writable virtual address (identity-mapped on ARM, +/// so VA == PA) and `pitch_bytes` is the number of bytes per row. Only 32 bpp +/// (XRGB) is supported, matching the rest of this console. +pub fn init_raw(addr: u64, width: u32, height: u32, pitch_bytes: u32) { + if addr == 0 || width == 0 || height == 0 || pitch_bytes < width * 4 { + return; + } + FB_ADDR .store(addr, Ordering::Release); + FB_PITCH_PX.store(pitch_bytes / 4, Ordering::Release); + FB_WIDTH .store(width, Ordering::Release); + FB_HEIGHT .store(height, Ordering::Release); + FB_COLS .store(width / CHAR_W, Ordering::Release); + FB_ROWS .store(height / CHAR_H, Ordering::Release); + + clear(); + FB_READY.store(true, Ordering::Release); +} + /// Write a string to the framebuffer console. /// /// Handles `\n` (advance to next row, scroll if needed) and @@ -530,6 +553,12 @@ pub fn swap_buffers() { // ── Internal helpers ────────────────────────────────────────────────────────── fn write_byte(b: u8) { + // The text console is a best-effort serial mirror. If the FB geometry is + // unpublished or degenerate (e.g. a corrupted FB_ROWS/FB_COLS static), do + // nothing rather than risk an arithmetic-overflow panic in the kernel. + if FB_ROWS.load(Ordering::Relaxed) == 0 || FB_COLS.load(Ordering::Relaxed) == 0 { + return; + } match b { b'\r' => { CURSOR.lock().0 = 0; @@ -539,7 +568,7 @@ fn write_byte(b: u8) { cur.0 = 0; cur.1 += 1; let rows = FB_ROWS.load(Ordering::Relaxed); - if cur.1 >= rows { + if rows > 0 && cur.1 >= rows { drop(cur); scroll_up(); CURSOR.lock().1 = rows - 1; @@ -578,10 +607,19 @@ fn blit_char(ch: u8, col: u32, row: u32) { let height = FB_HEIGHT .load(Ordering::Relaxed); let glyph_idx = ch.wrapping_sub(0x20) as usize; + if glyph_idx >= FONT.len() { return; } let glyph = &FONT[glyph_idx]; - let px0 = col * CHAR_W; - let py0 = row * CHAR_H; + // Bound the cell to the framebuffer. A corrupted/overlarge col or row (e.g. + // before the FB geometry is published, or if the CURSOR static is clobbered) + // must NOT panic the kernel via an arithmetic overflow in `col * CHAR_W` — + // the text console is only a serial mirror and is never worth a panic. + let cols = if width != 0 { width / CHAR_W } else { 0 }; + let rows = if height != 0 { height / CHAR_H } else { 0 }; + if col >= cols || row >= rows { return; } + + let px0 = col.saturating_mul(CHAR_W); + let py0 = row.saturating_mul(CHAR_H); let mut gy = 0; while gy < CHAR_H { @@ -613,6 +651,13 @@ fn scroll_up() { let height = FB_HEIGHT .load(Ordering::Relaxed); let rows = FB_ROWS .load(Ordering::Relaxed); + // Guard against an unpublished/degenerate FB geometry: the serial-mirror + // console must never panic the kernel (subtract-with-overflow on height or + // rows). Bail out cleanly if there is nothing safe to scroll. + if addr == 0 || height < CHAR_H || rows == 0 || pitch_px == 0 { + return; + } + // Number of pixel rows to copy. let copy_rows = height - CHAR_H; let words_per_row = pitch_px as usize; diff --git a/kernel/src/logger.rs b/kernel/src/logger.rs index 02bdf52..60f4924 100644 --- a/kernel/src/logger.rs +++ b/kernel/src/logger.rs @@ -85,6 +85,8 @@ impl SerialPort { /// Initialize UART: 115200 baud, 8-N-1, disable interrupts (x86_64 only). fn init_uart(&mut self) { + #[cfg(target_arch = "aarch64")] + crate::arch::aarch64::uart::init(); #[cfg(target_arch = "x86_64")] unsafe { cpu_out8(self.port + 1, 0x00); // Disable all interrupts @@ -104,9 +106,13 @@ impl SerialPort { while (cpu_in8(self.port + 5) & 0x20) == 0 {} cpu_out8(self.port, byte); } - // On non-x86 arches the serial stub is a no-op until a PL011/UART driver - // is implemented. The framebuffer console still receives all output. - #[cfg(not(target_arch = "x86_64"))] + // aarch64: route bytes to the PL011 UART driver so the shared kernel's + // serial output (early_print + the log framework) reaches the host + // terminal under QEMU `-M virt`. + #[cfg(target_arch = "aarch64")] + crate::arch::aarch64::uart::putb(byte); + // Other non-x86 arches: no serial backend yet; framebuffer console only. + #[cfg(not(any(target_arch = "x86_64", target_arch = "aarch64")))] let _ = byte; } } diff --git a/kernel/src/main.rs b/kernel/src/main.rs index 5630754..cb73e3f 100644 --- a/kernel/src/main.rs +++ b/kernel/src/main.rs @@ -54,22 +54,37 @@ use core::sync::atomic::{AtomicBool, Ordering}; /// Global kernel initialisation lock — prevents double-init on SMP wake-up. pub static KERNEL_INIT_DONE: AtomicBool = AtomicBool::new(false); -// ── Limine boot requests ───────────────────────────────────────────────────── +// ── Limine boot requests (x86_64 boot protocol) ────────────────────────────── +// +// Only the x86_64 production path boots via Limine. The aarch64 path boots via +// `qemu-system-aarch64 -M virt -kernel` (no bootloader) and discovers hardware +// from the device tree — see `arch::aarch64::boot_prod`. The Limine request +// statics are therefore compiled only for x86_64 so the ARM kernel carries no +// dead Limine boot section. +#[cfg(target_arch = "x86_64")] use limine::request::{ FramebufferRequest, HhdmRequest, ExecutableAddressRequest, MemmapRequest, MpRequest, ModulesRequest, }; +#[cfg(target_arch = "x86_64")] use limine::BaseRevision; /// Limine base revision — required for Limine 9+ to recognise this kernel. +#[cfg(target_arch = "x86_64")] #[used] static BASE_REVISION: BaseRevision = BaseRevision::new(); +#[cfg(target_arch = "x86_64")] static FB_REQUEST: FramebufferRequest = FramebufferRequest::new(); +#[cfg(target_arch = "x86_64")] static HHDM_REQUEST: HhdmRequest = HhdmRequest::new(); +#[cfg(target_arch = "x86_64")] static MMAP_REQUEST: MemmapRequest = MemmapRequest::new(); +#[cfg(target_arch = "x86_64")] static KADDR_REQUEST: ExecutableAddressRequest = ExecutableAddressRequest::new(); +#[cfg(target_arch = "x86_64")] static SMP_REQUEST: MpRequest = MpRequest::new(limine::mp::MP_FLAG_X2APIC); +#[cfg(target_arch = "x86_64")] static MODULES_REQUEST: ModulesRequest = ModulesRequest::new(); /// Slice of libflutter_engine.so as provided by Limine, stored once at boot. @@ -92,6 +107,7 @@ pub static FLUTTER_ENGINE_BYTES: spin::Mutex> = /// 7. AI Cortex init ← the magic /// 8. SMP wake-up (other cores join here after KERNEL_INIT_DONE) /// 9. Idle loop (Cortex takes over scheduling from here) +#[cfg(target_arch = "x86_64")] #[no_mangle] pub extern "C" fn kernel_main() -> ! { // ── 0. Early serial — must be first so any crash below is visible ───── @@ -189,6 +205,83 @@ pub extern "C" fn kernel_main() -> ! { log::warn!("[BOOT] No Limine modules response — libflutter_engine.so unavailable"); } + // The architecture-neutral subsystem init + process spawn + idle loop is + // shared with the aarch64 boot path. + shared_init_and_run() +} + +/// aarch64 production kernel entry. +/// +/// Called from `arch::aarch64::boot_prod::aarch64_kernel_boot` after the ARM +/// platform primitives (serial, MMU, EL1 vectors, GIC) are live and the device +/// tree has been parsed into the neutral [`mm::BootMemMap`]. This mirrors the +/// x86 `kernel_main` prologue using arch-provided inputs instead of Limine, then +/// joins the same `shared_init_and_run`. +#[cfg(target_arch = "aarch64")] +pub fn kernel_main_arch(map: mm::BootMemMap, hhdm_offset: u64) -> ! { + logger::early_print("[OSCORTEX] kernel_main_arch reached (aarch64)\r\n"); + + // HHDM offset is 0 on aarch64 (RAM is identity-mapped → VA == PA). + mm::frame_allocator::set_hhdm_offset(hhdm_offset); + + // ── 1. Architecture early init (FPU/SIMD, syscall/SVC readiness) ────── + // The interrupt controller + exception vectors are already live (set up in + // boot_prod), but early_init also enables FP/SIMD and logs syscall setup. + arch::early_init(); + logger::early_print("[OSCORTEX] arch::early_init done (aarch64)\r\n"); + + // ── 2. Memory management from the device-tree region list ──────────── + mm::init_from_regions(&map, hhdm_offset); + + // ── 3. Logger (serial only — no Limine framebuffer on ARM yet) ─────── + logger::init(None); + + log::warn!( + "[MM::FrameAlloc] capacity at boot: total_usable_frames={} ({} MiB), used_at_boot={} ({} MiB)", + mm::frame_allocator::frames_total(), + (mm::frame_allocator::frames_total() * 4096) / (1024 * 1024), + mm::frame_allocator::frames_used(), + (mm::frame_allocator::frames_used() * 4096) / (1024 * 1024) + ); + log::info!("OSCortex kernel {} booting (aarch64)...", env!("CARGO_PKG_VERSION")); + + // ── 3b. Framebuffer via QEMU ramfb (fw_cfg) ────────────────────────── + // `-M virt` has no Limine framebuffer; configure QEMU's `-device ramfb` + // through fw_cfg and publish the buffer to the shared fb console so the + // compositor path works unchanged. Requires the frame allocator (online + // after `mm::init_from_regions`). + match arch::aarch64::ramfb::init() { + Some((addr, w, h, pitch)) => { + // Identity-mapped RAM → the physical framebuffer base is directly + // writable as a virtual address (VA == PA on aarch64). + drivers::fb::init_raw(addr, w, h, pitch); + // Paint a visible boot fill so the display is provably driven even + // before any userspace renders (top band teal, body dark navy). + drivers::fb::fill_rect(0, 0, w, 48, 0x0014_B8A6); + drivers::fb::fill_rect(0, 48, w, h - 48, 0x001A_1A2E); + drivers::fb::write_str("\n OSCortex aarch64 — ramfb framebuffer up\n"); + // Stop mirroring kernel logs into the framebuffer console: under TCG, + // glyph rendering into the 1280x800 buffer (64 volatile writes/glyph) + // is so slow per log line it dwarfs the rest of boot. The serial log + // is the lifeline; the fb stays driven (the boot fill is visible) and + // userspace/compositor renders into it normally. + drivers::fb::disable_fb_logging(); + log::info!("[BOOT] aarch64 framebuffer online: {}x{} via ramfb", w, h); + } + None => { + log::warn!("[BOOT] aarch64 ramfb unavailable (need -device ramfb); serial-only."); + } + } + + shared_init_and_run() +} + +/// Architecture-neutral subsystem init, init-process spawn, and idle loop. +/// +/// Runs after each arch's boot prologue has brought up serial, the memory +/// manager, and the logger. This is the part of `kernel_main` that does not +/// touch any boot protocol directly. +fn shared_init_and_run() -> ! { // ── 4. Platform drivers (input probe) ─────────────────────────────── let qemu_like = arch::cpu::is_qemu_like_hypervisor(); drivers::platform::init_early(qemu_like); @@ -223,22 +316,84 @@ pub extern "C" fn kernel_main() -> ! { // ── 9. Signal APs + bring SMP online ──────────────────────────────── KERNEL_INIT_DONE.store(true, Ordering::Release); + #[cfg(target_arch = "x86_64")] arch::smp_init(SMP_REQUEST.response()); + #[cfg(not(target_arch = "x86_64"))] + arch::smp_init(None); // ── 9b. Spawn init process from initramfs ──────────────────────────── match fs::lookup("/init") { Some(elf_bytes) => { + // The init/host process is the Flutter shell host on BOTH arches now: + // seed rdi = HOST_MODE_SHELL so `_start`/`main_embedder` boots the + // shell host (registers as the engine host, dlopen's the engine, runs + // the shell). When the staged `/init` is the preemption self-test + // (no engine present), HOST_MODE_SHELL is ignored by that program. + let first_rdi: u64 = crate::app_registry::HOST_MODE_SHELL; let bootstrap = process::SpawnBootstrap { - rdi: crate::app_registry::HOST_MODE_SHELL, + rdi: first_rdi, rsi: 0, rdx: 0, parent_pid: 0, }; match process::spawn_with_bootstrap(elf_bytes, "init", bootstrap) { Ok(pid) => { - process::schedule_user_launch(pid); crate::wm::set_focus_pid(pid); log::info!("[INIT] Spawned shell host as PID {}", pid); + + // x86: defer the user launch to a scheduler kernel task; the + // APIC-timer-driven cooperative scheduler switches into it. + #[cfg(target_arch = "x86_64")] + process::schedule_user_launch(pid); + + // aarch64: the generic-timer ISR drives the SAME shared + // cooperative-scheduler hand-off the x86 APIC timer uses. + #[cfg(target_arch = "aarch64")] + { + // Detect whether the staged `/init` is the real Flutter + // shell host (engine .so present) or the bring-up + // preemption self-test (engine absent). The host runs as a + // single foreground EL0 process and spawns its own engine + // threads via SYS_THREAD_CREATE; the self-test needs a + // SECOND compute-loop thread to prove timer preemption. + let engine_present = + fs::lookup("/system/lib/libflutter_engine.so").is_some(); + + if engine_present { + log::info!( + "[INIT] aarch64: Flutter engine present — entering shell host PID {} (single foreground)", + pid + ); + } else { + let bootstrap_b = process::SpawnBootstrap { + rdi: 1, // selects message B in the self-test program + rsi: 0, + rdx: 0, + parent_pid: 0, + }; + match process::spawn_with_bootstrap(elf_bytes, "init-b", bootstrap_b) { + Ok(pid_b) => log::info!( + "[INIT] aarch64: spawned 2nd EL0 thread PID {} (preemption test)", + pid_b + ), + Err(e) => log::warn!("[INIT] aarch64: 2nd thread spawn failed: {}", e), + } + log::info!( + "[INIT] aarch64: entering PID {} at EL0; timer will preempt between threads", + pid + ); + } + // Arm the generic-timer scheduler tick now (kernel fully + // initialised) so preemption begins the moment we drop to EL0. + // The engine NEEDS preemption: pid 1 busy-spins in EL0 after + // creating its UI/raster/IO worker threads (it waits on a + // userspace flag the workers set), and with the nested + // immediate-child-enter disabled on arm those workers only + // get the CPU when a timer tick preempts pid 1. + let _ = engine_present; + arch::aarch64::apic::start_scheduler_tick(); + process::enter_user_by_pid_noreturn(pid); + } } Err(e) => log::warn!("[INIT] Failed to spawn /init: {}", e), } diff --git a/kernel/src/mm/frame_allocator.rs b/kernel/src/mm/frame_allocator.rs index de44f83..26744c7 100644 --- a/kernel/src/mm/frame_allocator.rs +++ b/kernel/src/mm/frame_allocator.rs @@ -3,8 +3,6 @@ //! Uses a static bitmap where each bit represents a 4 KiB physical frame. //! The bitmap itself is placed in the first usable memory region. -use limine::memmap::MEMMAP_USABLE; -use limine::request::MemmapResponse; use spin::Mutex; use core::sync::atomic::{AtomicU64, Ordering}; @@ -117,23 +115,53 @@ impl FrameBitmap { } } -pub fn init(mmap: &MemmapResponse, hhdm_offset: u64) { +/// Initialise the frame bitmap from an arch-neutral usable-region list. +/// +/// Both the x86_64 (Limine) and aarch64 (device-tree) boot paths funnel through +/// here via [`crate::mm::init_from_regions`]. +pub fn init_from_regions(map: &crate::mm::BootMemMap, hhdm_offset: u64) { set_hhdm_offset(hhdm_offset); let mut bm = BITMAP.lock(); bm.hhdm = hhdm_offset; - for entry in mmap.entries() { - if entry.type_ == MEMMAP_USABLE { - let start_frame = (entry.base as usize) / FRAME_SIZE; - let end_frame = ((entry.base + entry.length) as usize) / FRAME_SIZE; - for frame in start_frame.max(MIN_ALLOC_FRAME)..end_frame.min(MAX_FRAMES) { - bm.mark_free(frame); - } + for region in map.regions() { + let start_frame = (region.base as usize) / FRAME_SIZE; + let end_frame = ((region.base + region.len) as usize) / FRAME_SIZE; + for frame in start_frame.max(MIN_ALLOC_FRAME)..end_frame.min(MAX_FRAMES) { + bm.mark_free(frame); } } log::info!("[MM::FrameAlloc] {} MiB usable physical memory", (bm.total * FRAME_SIZE) / (1024 * 1024)); } +/// Reserve a physical address range `[base, base+len)` so the allocator never +/// hands it out (e.g. the loaded kernel image, the DTB, or a device-claimed +/// buffer). Frames are rounded outward to 4 KiB boundaries. Idempotent. +/// +/// Must be called AFTER `init_from_regions` (which marks RAM free) and BEFORE +/// the first allocation, otherwise a free-marked frame inside the range could be +/// handed out — on aarch64 the kernel is loaded into the same RAM region the +/// device tree reports as usable, so without this the heap can overlap the +/// kernel image. +pub fn reserve_range(base: u64, len: u64) { + if len == 0 { + return; + } + let start_frame = (base / FRAME_SIZE as u64) as usize; + let end_frame = (((base + len + FRAME_SIZE as u64 - 1) / FRAME_SIZE as u64) as usize) + .min(MAX_FRAMES); + let mut bm = BITMAP.lock(); + for frame in start_frame..end_frame { + let mask = 1u64 << (frame % 64); + let idx = frame / 64; + // Only flip frames currently free (bit set) so `used`/`total` stay sane. + if (bm.bits[idx] & mask) != 0 { + bm.bits[idx] &= !mask; + bm.used += 1; + } + } +} + /// Allocate a single 4 KiB physical frame. Returns the physical address. pub fn alloc_frame() -> Option { let res = BITMAP.lock().alloc(); diff --git a/kernel/src/mm/mod.rs b/kernel/src/mm/mod.rs index 970d83c..9c0ef18 100644 --- a/kernel/src/mm/mod.rs +++ b/kernel/src/mm/mod.rs @@ -6,14 +6,123 @@ pub mod paging; pub use paging::{map_mmio, map_page, PageFlags}; +#[cfg(target_arch = "x86_64")] use limine::request::{MemmapResponse, ExecutableAddressResponse}; +// ── Architecture-neutral boot memory map ───────────────────────────────────── +// +// The frame allocator needs a list of *usable* physical RAM regions. x86_64 +// gets that from the Limine memory map; aarch64 (QEMU `-M virt`, `-kernel`, +// no bootloader) gets it from the device tree. To keep the shared mm init free +// of any single boot protocol, both arches translate their source into this +// small, fixed-capacity, no-alloc region list and hand it to `frame_allocator`. + +/// One usable physical RAM region: `[base, base+len)`. +#[derive(Clone, Copy, Debug)] +pub struct BootMemRegion { + pub base: u64, + pub len: u64, +} + +/// Maximum distinct usable regions we retain. Limine typically emits well under +/// a dozen usable entries (and QEMU virt one or two), but a fragmented memory map +/// can have more; 128 is generous headroom so no usable RAM is ever dropped. +pub const MAX_BOOT_REGIONS: usize = 128; + +/// A fixed-capacity list of usable physical RAM regions, built by the arch boot +/// path and consumed by `frame_allocator::init_from_regions`. +#[derive(Clone, Copy)] +pub struct BootMemMap { + regions: [BootMemRegion; MAX_BOOT_REGIONS], + count: usize, +} + +impl BootMemMap { + pub const fn new() -> Self { + BootMemMap { + regions: [BootMemRegion { base: 0, len: 0 }; MAX_BOOT_REGIONS], + count: 0, + } + } + + /// Append a usable region (silently dropped if capacity is exceeded or the + /// region is empty). + pub fn push(&mut self, base: u64, len: u64) { + if len == 0 || self.count >= MAX_BOOT_REGIONS { + return; + } + self.regions[self.count] = BootMemRegion { base, len }; + self.count += 1; + } + + /// The usable regions discovered at boot. + pub fn regions(&self) -> &[BootMemRegion] { + &self.regions[..self.count] + } +} + +impl Default for BootMemMap { + fn default() -> Self { + Self::new() + } +} + +// ── x86_64 boot path (Limine) ──────────────────────────────────────────────── + +/// Initialise memory management from the Limine boot info (x86_64 path). +/// +/// Behaviour is byte-equivalent to the prior direct-Limine implementation: it +/// simply translates the Limine memory map into the neutral [`BootMemMap`] and +/// forwards to the shared [`init_from_regions`]. +#[cfg(target_arch = "x86_64")] pub fn init( mmap: &MemmapResponse, hhdm_offset: u64, _kaddr: &ExecutableAddressResponse, ) { - frame_allocator::init(mmap, hhdm_offset); + use limine::memmap::MEMMAP_USABLE; + let mut map = BootMemMap::new(); + for entry in mmap.entries() { + if entry.type_ == MEMMAP_USABLE { + map.push(entry.base, entry.length); + } + } + init_from_regions(&map, hhdm_offset); +} + +// ── Shared init (arch-neutral) ─────────────────────────────────────────────── + +/// Initialise memory management from an arch-provided usable-region list. +/// +/// `hhdm_offset` is the physical→virtual direct-map offset (Limine HHDM on +/// x86_64; 0 on aarch64 where the bring-up identity-maps RAM so VA == PA). +pub fn init_from_regions(map: &BootMemMap, hhdm_offset: u64) { + frame_allocator::init_from_regions(map, hhdm_offset); + + // aarch64: the kernel is loaded by `-kernel` INTO the same RAM region the + // device tree reports as usable (QEMU virt: image at 0x40080000, RAM from + // 0x40000000). The frame allocator just marked all of that free, so without + // reserving the kernel image the very first contiguous allocation (the 64 MiB + // heap) is handed 0x40000000 and its span overwrites live kernel code — a + // latent corruption that surfaces as a hang on the first large heap alloc. + // Reserve [RAM base .. __kernel_end] before the heap is carved out. + // (x86 gets this for free from the Limine memory-map types.) + #[cfg(target_arch = "aarch64")] + { + extern "C" { + static __kernel_end: u8; + } + const RAM_BASE: u64 = 0x4000_0000; + let kernel_end = unsafe { core::ptr::addr_of!(__kernel_end) as u64 }; + frame_allocator::reserve_range(RAM_BASE, kernel_end.saturating_sub(RAM_BASE)); + log::info!( + "[MM] aarch64: reserved kernel image [{:#x}..{:#x}] ({} KiB)", + RAM_BASE, + kernel_end, + (kernel_end - RAM_BASE) / 1024 + ); + } + paging::init(hhdm_offset); heap::init(); log::info!("[MM] Memory management online"); diff --git a/kernel/src/mm/paging.rs b/kernel/src/mm/paging.rs index 583b831..dac7443 100644 --- a/kernel/src/mm/paging.rs +++ b/kernel/src/mm/paging.rs @@ -9,6 +9,92 @@ use crate::mm::frame_allocator; pub(crate) static PAGE_TABLE_LOCK: Mutex<()> = Mutex::new(()); +/// Re-entrancy depth for the page-table critical section on this (single) core. +/// +/// The page-fault / demand-pager path can be entered *while the page-table lock +/// is already held by an outer map/translate on the same core*: on aarch64 the +/// SVC handler now runs with IRQs unmasked (so a syscall can be preempted/woken +/// like on x86), and a fault taken during a page-table operation re-enters +/// `demand_page` → `map_user_page_with_flags`. A plain spin `Mutex` is NOT +/// re-entrant, so the nested acquire spins forever against the held lock with +/// IRQs masked (in the abort handler) — an instant single-core self-deadlock +/// (observed freezing the engine's 3rd worker-thread stack setup). +/// +/// `lock_page_table()` makes the section re-entrant on a single core: the outer +/// acquire takes the real spin lock; nested acquires just bump a depth counter +/// and hand back a guard that releases nothing until depth returns to zero. This +/// is sound because all writers run on one core (the scheduler is cooperative / +/// single-CPU) — there is no second core to race the page-table structure. +static PAGE_TABLE_LOCK_DEPTH: core::sync::atomic::AtomicU32 = + core::sync::atomic::AtomicU32::new(0); + +pub(crate) struct PageTableGuard { + outer: Option>, + /// aarch64 only: DAIF captured by the OUTER acquisition before it masked + /// IRQs. Restored when the outer guard drops. Unused on nested guards (IRQs + /// stay masked for the whole outer section). + #[cfg(target_arch = "aarch64")] + saved_daif: u64, +} + +impl Drop for PageTableGuard { + fn drop(&mut self) { + if self.outer.is_some() { + // Outermost guard: clear depth, release the real lock, THEN re-enable + // IRQs. The lock byte must be released before IRQs come back on, or a + // freshly-unmasked timer tick could switch to a thread that spins on + // the still-held lock with IRQs masked → single-core deadlock. + PAGE_TABLE_LOCK_DEPTH.store(0, core::sync::atomic::Ordering::Release); + let _ = self.outer.take(); // drop the MutexGuard now (IRQs still masked) + #[cfg(target_arch = "aarch64")] + crate::arch::aarch64::cpu::interrupts_restore(self.saved_daif); + } else { + PAGE_TABLE_LOCK_DEPTH.fetch_sub(1, core::sync::atomic::Ordering::Release); + } + } +} + +/// Acquire the page-table critical section, re-entrant on a single core. +/// +/// On aarch64 the outer acquisition also MASKS IRQs for the whole critical +/// section. The reentrant depth counter is only sound if the section cannot be +/// interleaved by another thread; but syscalls run with IRQs unmasked, so a +/// generic-timer tick can preempt a lock holder mid-section and switch to a +/// sibling, which then sees a non-zero depth and proceeds as a bogus "nested" +/// writer — desynchronising the depth counter from the real lock and eventually +/// stranding the lock held while depth reads 0 (a later outer acquire then spins +/// forever with IRQs masked in the demand-abort handler). Masking IRQs around +/// the outer section makes it genuinely uninterruptible, so nesting only ever +/// means true same-stack re-entry (e.g. a demand fault during a page-table walk, +/// which already runs IRQs-masked). x86 keeps its original behaviour. +#[inline] +pub(crate) fn lock_page_table() -> PageTableGuard { + // Mask IRQs BEFORE touching the depth counter so the fetch_add → lock + // sequence cannot be preempted. On a nested call IRQs are already masked + // (the outer masked them); this is then an idempotent no-op. + #[cfg(target_arch = "aarch64")] + let saved_daif = crate::arch::aarch64::cpu::interrupts_save_and_disable(); + let prev = PAGE_TABLE_LOCK_DEPTH.fetch_add(1, core::sync::atomic::Ordering::Acquire); + if prev == 0 { + // First (outer) acquisition on this core: take the real lock. + let g = PAGE_TABLE_LOCK.lock(); + PageTableGuard { + outer: Some(g), + #[cfg(target_arch = "aarch64")] + saved_daif, + } + } else { + // Nested acquisition: the outer guard already holds the lock (and already + // masked IRQs). `saved_daif` here is "already masked" and is discarded on + // drop — only the outer guard restores DAIF. + PageTableGuard { + outer: None, + #[cfg(target_arch = "aarch64")] + saved_daif, + } + } +} + // ─── Page-table entry flags (shared across architectures) ───────────────────── bitflags! { @@ -404,18 +490,335 @@ mod x86_64_impl { // ─── Non-x86_64 stubs ───────────────────────────────────────────────────────── -#[cfg(not(target_arch = "x86_64"))] -mod stub_impl { +// ─── aarch64 implementation ─────────────────────────────────────────────────── +// +// A 4 KiB-granule, 4-level (L0→L1→L2→L3) translation-table walker for the EL0 +// (TTBR0) low half. Per-process address spaces each get a fresh L0 table from +// the frame allocator. The bring-up identity-maps RAM (VA == PA), so a table's +// physical address is also its directly-accessible kernel virtual address +// (HHDM offset is 0 on aarch64); we read/write tables through their PA. +// +// This matches the TCR_EL1 the bring-up programs (T0SZ=25 → 39-bit VA). With a +// 39-bit VA and 4 KiB granule, translation starts at level 1, so the top table +// is an L1 (512 entries × 1 GiB). We treat the per-process root as that L1. +#[cfg(target_arch = "aarch64")] +mod aarch64_impl { use super::PageFlags; + use crate::mm::frame_allocator; + + const PAGE_SIZE: usize = 4096; + /// Output-address mask for a table/page descriptor (bits[47:12]). + const ADDR_MASK: u64 = 0x0000_FFFF_FFFF_F000; + + // Descriptor type/attribute bits (ARMv8-A, stage-1). + const DESC_VALID: u64 = 1 << 0; + const DESC_TABLE: u64 = 1 << 1; // at L1/L2: 1 = table, 0 = block. at L3: must be 1 (page). + const DESC_AF: u64 = 1 << 10; // Access Flag + const DESC_SH_INNER: u64 = 3 << 8; // inner shareable + const DESC_AP_RW_ALL: u64 = 1 << 6; // EL0+EL1 RW (AP[1]=1) + const DESC_AP_RO_ALL: u64 = 3 << 6; // EL0+EL1 RO + const DESC_UXN: u64 = 1 << 54; // unprivileged execute-never + const DESC_PXN: u64 = 1 << 53; // privileged execute-never + const ATTR_IDX_NORMAL: u64 = 0 << 2; // MAIR attr0 = Normal WB (matches mmu.rs) + + /// 39-bit VA index extraction. With start level 1: L1[38:30], L2[29:21], + /// L3[20:12]. + #[inline] + fn idx_l1(va: u64) -> usize { ((va >> 30) & 0x1FF) as usize } + #[inline] + fn idx_l2(va: u64) -> usize { ((va >> 21) & 0x1FF) as usize } + #[inline] + fn idx_l3(va: u64) -> usize { ((va >> 12) & 0x1FF) as usize } + + /// A table is 512 u64 entries at a 4 KiB-aligned physical address. On the + /// identity map, the PA is directly dereferenceable. + #[inline] + unsafe fn entry(table_pa: u64, idx: usize) -> *mut u64 { + (table_pa as *mut u64).add(idx) + } + + /// Allocate and zero a fresh translation table; returns its physical address. + fn alloc_table() -> Option { + let pa = frame_allocator::alloc_frame()?; + unsafe { core::ptr::write_bytes(pa as *mut u8, 0, PAGE_SIZE) }; + Some(pa) + } + + /// Allocate a fresh per-process root table (the L1 for a 39-bit VA space). + /// + /// CRITICAL: on the aarch64 bring-up the kernel itself runs from the low + /// (TTBR0) half — its code, stack, vectors, and the UART/GIC MMIO are in the + /// identity map programmed at boot. A per-process TTBR0 that contained only + /// user pages would unmap the kernel the instant `write_cr3` loaded it, so + /// the very next instruction fetch (and the fault handler) would fault with + /// no way to recover. We therefore seed each new root with a copy of the + /// kernel's identity-map L1 entries (the 1 GiB block descriptors for the + /// device region + RAM). User L1 slots (which the kernel map leaves empty) + /// are then filled by `map_page_in` for the process's own pages. + pub fn alloc_user_pml4() -> Option { + let root = alloc_table()?; + let kernel_l1 = crate::arch::aarch64::mmu::kernel_ttbr0(); + unsafe { + // Copy all 512 L1 entries from the kernel identity map. The kernel + // map uses 1 GiB block descriptors at low indices; user VAs live at + // higher indices (e.g. ~511 for the stack) which are zero in the + // kernel map and get populated per-process. + core::ptr::copy_nonoverlapping( + kernel_l1 as *const u64, + root as *mut u64, + 512, + ); + } + Some(root) + } + + /// Ensure the next-level table referenced by `*slot` exists, allocating it if + /// the descriptor is empty. Returns the child table's physical address. + /// + /// `level_shift` is the VA shift of the *child* block size at this level: + /// 21 when `slot` is an L1 entry (children are 2 MiB L2 blocks), 12 when it is + /// an L2 entry (children are 4 KiB L3 pages). It is used only when this slot + /// currently holds a *block* descriptor and must be split into a table. + /// + /// CRITICAL (user low-VA mapping): each per-process root is seeded with a copy + /// of the kernel identity map, whose low L1 slots are 1 GiB *block* descriptors + /// (device MMIO at slot 0, RAM at slots 1..). The Flutter host loads at + /// USER_ELF_BASE = 0x400000, which lives inside the slot-0 block. Mapping a + /// 4 KiB user page there requires splitting that block into a table WITHOUT + /// losing the original identity mapping (the kernel runs from this same root + /// after the CR3 switch and still needs the UART/GIC MMIO + its own RAM). We + /// therefore expand the block into a full table of next-level blocks that + /// reproduce the original output addresses + attributes, then return it so the + /// caller can overlay the finer user mapping on top. + unsafe fn ensure_table(slot: *mut u64, level_shift: u32) -> Option { + let desc = core::ptr::read_volatile(slot); + if desc & DESC_VALID != 0 { + if desc & DESC_TABLE != 0 { + // Already a table descriptor. + return Some(desc & ADDR_MASK); + } + // VALID but not a table → a block descriptor. Split it. + let child = alloc_table()?; + let block_base = desc & ADDR_MASK; // output PA of the block + // Lower + upper attribute bits to carry into each sub-block. Keep + // everything except the output address and the table/page bit[1]; + // re-add bit[1]=0 (block at L1/L2) below. + let attrs = desc & !ADDR_MASK & !DESC_TABLE; + let child_size: u64 = 1u64 << level_shift; // 2 MiB at L2, 4 KiB at L3 + // At L3 a valid descriptor MUST be a page (bit[1]=1); at L2 a block + // has bit[1]=0. `level_shift==12` ⇒ the children are L3 pages. + let child_is_page = level_shift == 12; + for i in 0..512usize { + let sub_pa = block_base + (i as u64) * child_size; + let mut sub_desc = (sub_pa & ADDR_MASK) | attrs | DESC_VALID; + if child_is_page { sub_desc |= DESC_TABLE; } // page bit at L3 + core::ptr::write_volatile(entry(child, i), sub_desc); + } + core::ptr::write_volatile(slot, (child & ADDR_MASK) | DESC_VALID | DESC_TABLE); + return Some(child); + } + let child = alloc_table()?; + core::ptr::write_volatile(slot, (child & ADDR_MASK) | DESC_VALID | DESC_TABLE); + Some(child) + } + + /// Build an L3 page descriptor for a user page. + fn page_desc(phys: u64, flags: PageFlags) -> u64 { + let mut d = (phys & ADDR_MASK) + | DESC_VALID + | DESC_TABLE // bit[1]=1 → page at L3 + | DESC_AF + | DESC_SH_INNER + | ATTR_IDX_NORMAL; + if flags.contains(PageFlags::WRITABLE) { + d |= DESC_AP_RW_ALL; + } else { + d |= DESC_AP_RO_ALL; + } + // Execution: x86 NO_EXECUTE bit → UXN here. We always set PXN (EL1 must + // never execute user pages). When NO_EXECUTE is set, also set UXN. + d |= DESC_PXN; + if flags.contains(PageFlags::NO_EXECUTE) || !flags.contains(PageFlags::USER) { + d |= DESC_UXN; + } + d + } + + /// Map a 4 KiB page `phys` at user VA `virt` in the address space rooted at + /// `root_pa` (the L1 table), allocating intermediate tables as needed. + /// + /// # Safety + /// `root_pa` must be a valid table from [`alloc_user_pml4`]; `virt`/`phys` + /// must be 4 KiB-aligned. + pub unsafe fn map_page_in( + root_pa: u64, + virt: u64, + phys: u64, + flags: PageFlags, + ) -> Result<(), &'static str> { + let l1 = root_pa; + // L1 entry → L2 table (children are 2 MiB blocks: shift 21). + let l2 = ensure_table(entry(l1, idx_l1(virt)), 21).ok_or("OOM: l2 table")?; + // L2 entry → L3 table (children are 4 KiB pages: shift 12). + let l3 = ensure_table(entry(l2, idx_l2(virt)), 12).ok_or("OOM: l3 table")?; + core::ptr::write_volatile(entry(l3, idx_l3(virt)), page_desc(phys, flags)); + // Publish the new mapping + flush this VA in the EL0 space. + core::arch::asm!( + "dsb ishst", + "tlbi vmalle1is", + "dsb ish", + "isb", + options(nostack, preserves_flags), + ); + Ok(()) + } - pub unsafe fn map_page(_virt: u64, _phys: u64, _flags: PageFlags) { - // Milestone 4b: aarch64/riscv64 page table walker (TTBR0/TTBR1, SATP). + /// Walk to the L3 descriptor for `virt`; returns (l3_slot, descriptor) if the + /// full path to a 4 KiB page is present, else None. + /// + /// CRITICAL: each per-process root is seeded with a copy of the kernel + /// identity map, whose low L1 entries are 1 GiB **block** descriptors (and L2 + /// could be 2 MiB blocks). A block is VALID but is NOT a table pointer — its + /// `ADDR_MASK` bits are an *output PA*, not a child-table PA. The earlier walk + /// dereferenced that output PA as if it were a table, reading garbage that + /// often had DESC_VALID set, so `translate_user_page` reported low user VAs + /// (e.g. the Flutter host's 0x40xxxx .rodata) as "already mapped to identity" + /// — which made the ELF loader REUSE the identity PA instead of allocating a + /// real frame, so the kernel then read 0xFF device/identity memory for those + /// pages. We must therefore treat a block descriptor at L1/L2 as "no 4 KiB + /// leaf here" (return None); the caller allocates a frame and `map_page_in` + /// splits the block into a real table before overlaying the page. + unsafe fn walk(root_pa: u64, virt: u64) -> Option<(*mut u64, u64)> { + let l1d = core::ptr::read_volatile(entry(root_pa, idx_l1(virt))); + if l1d & DESC_VALID == 0 { return None; } + // VALID but not a table ⇒ a 1 GiB block: no L3 leaf for this VA. + if l1d & DESC_TABLE == 0 { return None; } + let l2 = l1d & ADDR_MASK; + let l2d = core::ptr::read_volatile(entry(l2, idx_l2(virt))); + if l2d & DESC_VALID == 0 { return None; } + // VALID but not a table ⇒ a 2 MiB block: no L3 leaf for this VA. + if l2d & DESC_TABLE == 0 { return None; } + let l3 = l2d & ADDR_MASK; + let slot = entry(l3, idx_l3(virt)); + let d = core::ptr::read_volatile(slot); + if d & DESC_VALID == 0 { return None; } + Some((slot, d)) } - pub unsafe fn map_mmio(_phys: u64, _virt: u64, _size: usize) { - // Milestone 4b: identity/MMIO map via arch-specific page tables. + pub fn translate_user_page(root_pa: u64, virt: u64) -> Option { + let page = virt & !0xFFF; + let off = virt & 0xFFF; + unsafe { walk(root_pa, page).map(|(_, d)| (d & ADDR_MASK) | off) } } + pub fn translate_user_page_flags(root_pa: u64, virt: u64) -> Option<(u64, bool, bool)> { + let page = virt & !0xFFF; + unsafe { + walk(root_pa, page).map(|(_, d)| { + let phys = d & ADDR_MASK; + let writable = (d & DESC_AP_RW_ALL) == DESC_AP_RW_ALL; + let exec = (d & DESC_UXN) == 0; + (phys, writable, exec) + }) + } + } + + /// Update the permission bits of an already-mapped user page. + /// + /// # Safety + /// `root_pa` must be a valid allocated table. + pub unsafe fn update_user_page_flags( + root_pa: u64, + virt: u64, + writable: bool, + exec: bool, + ) -> Result<(), &'static str> { + let page = virt & !0xFFF; + let (slot, d) = walk(root_pa, page).ok_or("page not mapped")?; + let mut nd = d & !(DESC_AP_RO_ALL | DESC_UXN); + nd |= if writable { DESC_AP_RW_ALL } else { DESC_AP_RO_ALL }; + if !exec { nd |= DESC_UXN; } + core::ptr::write_volatile(slot, nd); + core::arch::asm!( + "dsb ishst", "tlbi vmalle1is", "dsb ish", "isb", + options(nostack, preserves_flags), + ); + Ok(()) + } + + /// Unmap a single page; returns the physical frame it pointed at. + /// + /// # Safety + /// `root_pa` must be a valid allocated table. + pub unsafe fn unmap_user_page(root_pa: u64, virt: u64) -> Result { + let page = virt & !0xFFF; + let (slot, d) = walk(root_pa, page).ok_or("not present")?; + let phys = d & ADDR_MASK; + core::ptr::write_volatile(slot, 0); + core::arch::asm!( + "dsb ishst", "tlbi vmalle1is", "dsb ish", "isb", + options(nostack, preserves_flags), + ); + Ok(phys) + } + + /// Walk all three levels of a user root table and free every backing frame + /// (leaf pages + intermediate tables + the root). Best-effort. + /// + /// IMPORTANT: each per-process root is seeded (in `alloc_user_pml4`) with a + /// copy of the kernel's identity-map L1 entries, which are **1 GiB block + /// descriptors** (bit[1] == 0), not table pointers. We MUST NOT recurse into + /// or free those — their "address" is kernel RAM, not a process-owned table. + /// Only L1 entries that are *table* descriptors (bit[1] == 1), i.e. the L2 + /// tables created for this process's own high-VA pages, are walked and freed. + pub fn free_user_pml4(root_pa: u64) { + if root_pa == 0 { return; } + unsafe { + for i1 in 0..512usize { + let l1d = core::ptr::read_volatile(entry(root_pa, i1)); + // Skip invalid AND block descriptors (the shared kernel identity + // map). Only table descriptors point at process-owned L2 tables. + if l1d & DESC_VALID == 0 || l1d & DESC_TABLE == 0 { continue; } + let l2 = l1d & ADDR_MASK; + for i2 in 0..512usize { + let l2d = core::ptr::read_volatile(entry(l2, i2)); + if l2d & DESC_VALID == 0 || l2d & DESC_TABLE == 0 { continue; } + let l3 = l2d & ADDR_MASK; + for i3 in 0..512usize { + let l3d = core::ptr::read_volatile(entry(l3, i3)); + if l3d & DESC_VALID != 0 { + frame_allocator::free_frame(l3d & ADDR_MASK); + } + } + frame_allocator::free_frame(l3); + } + frame_allocator::free_frame(l2); + } + frame_allocator::free_frame(root_pa); + } + } + + // Kernel-space single mappings: the bring-up identity-maps devices + RAM, so + // a fresh kernel `map_page`/`map_mmio` is only needed for windows outside the + // boot map. For now these are no-ops (everything the kernel touches is in the + // identity map); a follow-on can extend the kernel L1 if a new window is + // required. + pub unsafe fn map_page(_virt: u64, _phys: u64, _flags: PageFlags) {} + pub unsafe fn map_mmio(_phys: u64, _virt: u64, _size: usize) {} + + pub fn init(_hhdm_offset: u64) { + crate::logger::early_print("[MM::Paging] aarch64 TTBR0 page-table walker online\r\n"); + } +} + +// ─── non-x86, non-aarch64 stub ──────────────────────────────────────────────── +#[cfg(not(any(target_arch = "x86_64", target_arch = "aarch64")))] +mod stub_impl { + use super::PageFlags; + + pub unsafe fn map_page(_virt: u64, _phys: u64, _flags: PageFlags) {} + pub unsafe fn map_mmio(_phys: u64, _virt: u64, _size: usize) {} + pub fn init(_hhdm_offset: u64) { crate::logger::early_print("[MM::Paging] Virtual memory manager online (stub — non-x86)\r\n"); } @@ -427,7 +830,9 @@ mod stub_impl { pub fn init(hhdm_offset: u64) { #[cfg(target_arch = "x86_64")] x86_64_impl::init(hhdm_offset); - #[cfg(not(target_arch = "x86_64"))] + #[cfg(target_arch = "aarch64")] + aarch64_impl::init(hhdm_offset); + #[cfg(not(any(target_arch = "x86_64", target_arch = "aarch64")))] stub_impl::init(hhdm_offset); } @@ -437,10 +842,12 @@ pub fn init(hhdm_offset: u64) { /// * `set_hhdm_offset()` must have run before this. /// * `virt` and `phys` must be 4-KiB aligned. pub unsafe fn map_page(virt: u64, phys: u64, flags: PageFlags) { - let _lock = PAGE_TABLE_LOCK.lock(); + let _lock = lock_page_table(); #[cfg(target_arch = "x86_64")] unsafe { x86_64_impl::map_page(virt, phys, flags) } - #[cfg(not(target_arch = "x86_64"))] + #[cfg(target_arch = "aarch64")] + unsafe { aarch64_impl::map_page(virt, phys, flags) } + #[cfg(not(any(target_arch = "x86_64", target_arch = "aarch64")))] unsafe { stub_impl::map_page(virt, phys, flags) } } @@ -449,10 +856,12 @@ pub unsafe fn map_page(virt: u64, phys: u64, flags: PageFlags) { /// # Safety /// Same preconditions as [`map_page`]. pub unsafe fn map_mmio(phys: u64, virt: u64, size: usize) { - let _lock = PAGE_TABLE_LOCK.lock(); + let _lock = lock_page_table(); #[cfg(target_arch = "x86_64")] unsafe { x86_64_impl::map_mmio(phys, virt, size) } - #[cfg(not(target_arch = "x86_64"))] + #[cfg(target_arch = "aarch64")] + unsafe { aarch64_impl::map_mmio(phys, virt, size) } + #[cfg(not(any(target_arch = "x86_64", target_arch = "aarch64")))] unsafe { stub_impl::map_mmio(phys, virt, size) } } @@ -461,9 +870,7 @@ pub fn demand_page(cr2: u64, error: u64) -> bool { if error & 0x1 != 0 { return false; } // page present — protection fault, not demand page // Read current CR3 to get the active PML4. - let cr3_phys: u64; - unsafe { core::arch::asm!("mov {}, cr3", out(reg) cr3_phys) }; - let cr3_phys = cr3_phys & 0x000f_ffff_ffff_f000; + let cr3_phys = crate::arch::memory::read_cr3() & 0x000f_ffff_ffff_f000; let page_va = cr2 & !0xFFF; @@ -473,6 +880,18 @@ pub fn demand_page(cr2: u64, error: u64) -> bool { unsafe { core::arch::asm!("invlpg [{}]", in(reg) page_va, options(nostack, preserves_flags)); } + #[cfg(target_arch = "aarch64")] + unsafe { + // Invalidate the stale TLB entry for this VA (EL1, inner-shareable), + // then re-execute the faulting access. + core::arch::asm!( + "dsb ishst", + "tlbi vmalle1is", + "dsb ish", + "isb", + options(nostack, preserves_flags), + ); + } return true; } @@ -526,7 +945,9 @@ pub fn demand_page(cr2: u64, error: u64) -> bool { pub fn alloc_user_pml4() -> Option { #[cfg(target_arch = "x86_64")] return x86_64_impl::alloc_user_pml4(); - #[cfg(not(target_arch = "x86_64"))] + #[cfg(target_arch = "aarch64")] + return aarch64_impl::alloc_user_pml4(); + #[cfg(not(any(target_arch = "x86_64", target_arch = "aarch64")))] return None; } @@ -537,7 +958,7 @@ pub fn alloc_user_pml4() -> Option { pub unsafe fn map_user_page(pml4_phys: u64, virt: u64, phys: u64) -> Result<(), &'static str> { - let _lock = PAGE_TABLE_LOCK.lock(); + let _lock = lock_page_table(); #[cfg(target_arch = "x86_64")] return unsafe { x86_64_impl::map_page_in( @@ -545,7 +966,14 @@ pub unsafe fn map_user_page(pml4_phys: u64, virt: u64, phys: u64) PageFlags::PRESENT | PageFlags::WRITABLE | PageFlags::USER | PageFlags::NO_EXECUTE, ) }; - #[cfg(not(target_arch = "x86_64"))] + #[cfg(target_arch = "aarch64")] + return unsafe { + aarch64_impl::map_page_in( + pml4_phys, virt, phys, + PageFlags::PRESENT | PageFlags::WRITABLE | PageFlags::USER | PageFlags::NO_EXECUTE, + ) + }; + #[cfg(not(any(target_arch = "x86_64", target_arch = "aarch64")))] { let _ = (pml4_phys, virt, phys); Ok(()) } } @@ -557,7 +985,7 @@ pub unsafe fn map_user_page_with_flags( pml4_phys: u64, virt: u64, phys: u64, writable: bool, exec: bool, ) -> Result<(), &'static str> { - let _lock = PAGE_TABLE_LOCK.lock(); + let _lock = lock_page_table(); #[cfg(target_arch = "x86_64")] { let mut flags = PageFlags::PRESENT | PageFlags::USER; @@ -565,36 +993,49 @@ pub unsafe fn map_user_page_with_flags( if !exec { flags |= PageFlags::NO_EXECUTE; } return unsafe { x86_64_impl::map_page_in(pml4_phys, virt, phys, flags) }; } - #[cfg(not(target_arch = "x86_64"))] + #[cfg(target_arch = "aarch64")] + { + let mut flags = PageFlags::PRESENT | PageFlags::USER; + if writable { flags |= PageFlags::WRITABLE; } + if !exec { flags |= PageFlags::NO_EXECUTE; } + return unsafe { aarch64_impl::map_page_in(pml4_phys, virt, phys, flags) }; + } + #[cfg(not(any(target_arch = "x86_64", target_arch = "aarch64")))] { let _ = (pml4_phys, virt, phys, writable, exec); Ok(()) } } /// Walk a user PML4 and return the physical frame already mapped at `virt`, /// or `None` if the page is not yet mapped. pub fn translate_user_page(pml4_phys: u64, virt: u64) -> Option { - let _lock = PAGE_TABLE_LOCK.lock(); + let _lock = lock_page_table(); #[cfg(target_arch = "x86_64")] { x86_64_impl::translate_user_page(pml4_phys, virt) } - #[cfg(not(target_arch = "x86_64"))] + #[cfg(target_arch = "aarch64")] + { aarch64_impl::translate_user_page(pml4_phys, virt) } + #[cfg(not(any(target_arch = "x86_64", target_arch = "aarch64")))] { let _ = (pml4_phys, virt); None } } /// Walk a user PML4 and return the physical frame mapped at `virt` along with /// its writable and executable flags. pub fn translate_user_page_flags(pml4_phys: u64, virt: u64) -> Option<(u64, bool, bool)> { - let _lock = PAGE_TABLE_LOCK.lock(); + let _lock = lock_page_table(); #[cfg(target_arch = "x86_64")] { x86_64_impl::translate_user_page_flags(pml4_phys, virt) } - #[cfg(not(target_arch = "x86_64"))] + #[cfg(target_arch = "aarch64")] + { aarch64_impl::translate_user_page_flags(pml4_phys, virt) } + #[cfg(not(any(target_arch = "x86_64", target_arch = "aarch64")))] { let _ = (pml4_phys, virt); None } } /// Free all frames in a user PML4 and the PML4 itself. pub fn free_user_pml4(pml4_phys: u64) { - let _lock = PAGE_TABLE_LOCK.lock(); + let _lock = lock_page_table(); #[cfg(target_arch = "x86_64")] x86_64_impl::free_user_pml4(pml4_phys); - #[cfg(not(target_arch = "x86_64"))] + #[cfg(target_arch = "aarch64")] + aarch64_impl::free_user_pml4(pml4_phys); + #[cfg(not(any(target_arch = "x86_64", target_arch = "aarch64")))] let _ = pml4_phys; } @@ -608,12 +1049,16 @@ pub unsafe fn update_user_page( writable: bool, exec: bool, ) -> Result<(), &'static str> { - let _lock = PAGE_TABLE_LOCK.lock(); + let _lock = lock_page_table(); #[cfg(target_arch = "x86_64")] { unsafe { x86_64_impl::update_user_page_flags(pml4_phys, virt, writable, exec) } } - #[cfg(not(target_arch = "x86_64"))] + #[cfg(target_arch = "aarch64")] + { + unsafe { aarch64_impl::update_user_page_flags(pml4_phys, virt, writable, exec) } + } + #[cfg(not(any(target_arch = "x86_64", target_arch = "aarch64")))] { let _ = (pml4_phys, virt, writable, exec); Ok(()) @@ -625,12 +1070,16 @@ pub unsafe fn update_user_page( /// # Safety /// `pml4_phys` must be a valid allocated PML4. pub unsafe fn unmap_user_page(pml4_phys: u64, virt: u64) -> Result { - let _lock = PAGE_TABLE_LOCK.lock(); + let _lock = lock_page_table(); #[cfg(target_arch = "x86_64")] { unsafe { x86_64_impl::unmap_user_page(pml4_phys, virt) } } - #[cfg(not(target_arch = "x86_64"))] + #[cfg(target_arch = "aarch64")] + { + unsafe { aarch64_impl::unmap_user_page(pml4_phys, virt) } + } + #[cfg(not(any(target_arch = "x86_64", target_arch = "aarch64")))] { let _ = (pml4_phys, virt); Err("unsupported") diff --git a/kernel/src/panic.rs b/kernel/src/panic.rs index 438fa28..d925163 100644 --- a/kernel/src/panic.rs +++ b/kernel/src/panic.rs @@ -44,8 +44,7 @@ fn panic(info: &PanicInfo) -> ! { // Best-effort backtrace: walk the saved RBP chain a few frames and print // each return address. Stops on a bogus pointer. unsafe { - let mut rbp: u64; - core::arch::asm!("mov {}, rbp", out(reg) rbp, options(nomem, nostack)); + let mut rbp: u64 = crate::arch::read_frame_pointer(); crate::logger::early_print("backtrace:\r\n"); for _ in 0..16 { if rbp < 0xffff_8000_0000_0000 || (rbp & 7) != 0 { break; } diff --git a/kernel/src/process/dl.rs b/kernel/src/process/dl.rs index ae05d34..3fa47e8 100644 --- a/kernel/src/process/dl.rs +++ b/kernel/src/process/dl.rs @@ -13,6 +13,17 @@ //! | `R_X86_64_GLOB_DAT`| 6 | `S` | //! | `R_X86_64_JUMP_SLOT`| 7 | `S` | //! +//! ## Supported aarch64 relocation types +//! | Type | Encoding | Formula | +//! |-----------------------|----------|---------| +//! | `R_AARCH64_ABS64` | 257 | `S + A` | +//! | `R_AARCH64_GLOB_DAT` | 1025 | `S + A` | +//! | `R_AARCH64_JUMP_SLOT` | 1026 | `S + A` | +//! | `R_AARCH64_RELATIVE` | 1027 | `B + A` | +//! +//! The loader selects the relocation set from the ELF `e_machine`, so a single +//! `dlopen` path loads both the x86 and the ARM `libflutter_engine.so`. +//! //! Undefined externals (`st_shndx == SHN_UNDEF`) are left at zero. The //! Flutter engine fills all mandatory platform callbacks through the //! `FlutterEngineProcTable` struct passed during `FlutterEngineInitialize`, @@ -36,6 +47,7 @@ const ELFCLASS64: u8 = 2; const ELFDATA2LSB: u8 = 1; const ET_DYN: u16 = 3; const EM_X86_64: u16 = 62; +const EM_AARCH64: u16 = 183; const PT_LOAD: u32 = 1; const PT_DYNAMIC: u32 = 2; @@ -59,6 +71,24 @@ const R_X86_64_GLOB_DAT: u32 = 6; const R_X86_64_JUMP_SLOT: u32 = 7; const R_X86_64_RELATIVE: u32 = 8; +// aarch64 dynamic relocation types (ELF for the Arm 64-bit Architecture). +const R_AARCH64_ABS64: u32 = 257; +const R_AARCH64_GLOB_DAT: u32 = 1025; +const R_AARCH64_JUMP_SLOT: u32 = 1026; +const R_AARCH64_RELATIVE: u32 = 1027; +// TLS relocations. The engine's C++ `thread_local`s use the TLSDESC model. +const R_AARCH64_TLS_DTPMOD64: u32 = 1028; +const R_AARCH64_TLS_DTPREL64: u32 = 1029; +const R_AARCH64_TLS_TPREL64: u32 = 1030; +const R_AARCH64_TLSDESC: u32 = 1031; + +/// AArch64 variant-I TLS: TPIDR_EL0 points at the TCB; the static TLS block of +/// the (single, statically-loaded) module begins after a 16-byte TCB reserve, +/// rounded up to the TLS alignment. The engine's PT_TLS is 8-byte aligned, so +/// the block starts at TP + 16. A TLSDESC/TPREL variable at module offset `A` +/// therefore lives at `TP + TLS_TP_OFFSET + A`. +const TLS_TP_OFFSET: u64 = 16; + const STB_GLOBAL: u8 = 1; const STB_WEAK: u8 = 2; const SHN_UNDEF: u16 = 0; @@ -451,6 +481,7 @@ fn apply_rela_table( let mut n_jmp = 0usize; let mut n_other = 0usize; let mut n_oob = 0usize; + let mut n_tls = 0usize; for i in 0..count { let rela: Elf64Rela = match read_pod(bytes, rela_file + i * ent_sz) { @@ -461,11 +492,68 @@ fn apply_rela_table( let sym_idx = (rela.r_info >> 32) as usize; let target = load_base.wrapping_add(rela.r_offset); + // ── aarch64 TLS relocations (handled specially: they may write a PAIR + // and encode a TP-relative offset, not an absolute VA) ───────────── + // The module's TLS offset of the referenced variable is `st_value+A` + // for a symbol, or just `A` for a local (sym_idx==0) reloc. The + // TP-relative offset is then TLS_TP_OFFSET + that (variant-I layout). + if rtype == R_AARCH64_TLSDESC + || rtype == R_AARCH64_TLS_TPREL64 + || rtype == R_AARCH64_TLS_DTPREL64 + || rtype == R_AARCH64_TLS_DTPMOD64 + { + n_tls += 1; + let module_off = if sym_idx != 0 { + // sym_vaddr = load_base + st_value; recover st_value, add addend. + sym_vaddr(sym_idx) + .wrapping_sub(load_base) + .wrapping_add(rela.r_addend as u64) + } else { + rela.r_addend as u64 + }; + let tp_off = TLS_TP_OFFSET.wrapping_add(module_off); + unsafe { + match rtype { + R_AARCH64_TLSDESC => { + // Descriptor pair: [+0]=static resolver, [+8]=TP offset. + // The resolver trampoline only exists on aarch64 (a TLSDESC + // reloc can only appear in an aarch64 object); on x86 this + // arm is unreachable, so fall back to 0 to keep that build + // green without pulling in the arch-gated symbol. + #[cfg(target_arch = "aarch64")] + let resolver = crate::process::posix_trampolines::TLSDESC_RESOLVER_VA; + #[cfg(not(target_arch = "aarch64"))] + let resolver = 0u64; + write_user_u64(target, resolver); + write_user_u64(target.wrapping_add(8), tp_off); + } + R_AARCH64_TLS_TPREL64 => write_user_u64(target, tp_off), + // Single-module static image: DTPMOD is always module 1, + // DTPREL is the in-block offset (no TCB reserve). + R_AARCH64_TLS_DTPMOD64 => write_user_u64(target, 1), + R_AARCH64_TLS_DTPREL64 => write_user_u64(target, module_off), + _ => {} + } + } + continue; + } + + // Both x86_64 and aarch64 use the same RELA encoding (r_offset/r_info/ + // r_addend) and identical formulas; only the type *numbers* differ. We + // accept both sets so a single loader handles either engine .so. On + // aarch64, GLOB_DAT/JUMP_SLOT add the addend (S + A); on x86 the addend + // is conventionally zero for those, so `S + A` is also correct there. let value = match rtype { + // ── x86_64 ── R_X86_64_RELATIVE => { n_rel += 1; load_base.wrapping_add(rela.r_addend as u64) }, R_X86_64_64 => { n_64 += 1; sym_vaddr(sym_idx).wrapping_add(rela.r_addend as u64) }, R_X86_64_GLOB_DAT => { n_glob += 1; sym_vaddr(sym_idx) }, R_X86_64_JUMP_SLOT => { n_jmp += 1; sym_vaddr(sym_idx) }, + // ── aarch64 ── + R_AARCH64_RELATIVE => { n_rel += 1; load_base.wrapping_add(rela.r_addend as u64) }, + R_AARCH64_ABS64 => { n_64 += 1; sym_vaddr(sym_idx).wrapping_add(rela.r_addend as u64) }, + R_AARCH64_GLOB_DAT => { n_glob += 1; sym_vaddr(sym_idx).wrapping_add(rela.r_addend as u64) }, + R_AARCH64_JUMP_SLOT => { n_jmp += 1; sym_vaddr(sym_idx).wrapping_add(rela.r_addend as u64) }, 0 => continue, _ => { n_other += 1; continue; }, }; @@ -481,8 +569,8 @@ fn apply_rela_table( } log::info!( - "[dl] rela table sz={} count={} applied: REL={} _64={} GLOB={} JMP={} other={} oob={}", - table_sz, count, n_rel, n_64, n_glob, n_jmp, n_other, n_oob + "[dl] rela table sz={} count={} applied: REL={} _64={} GLOB={} JMP={} TLS={} other={} oob={}", + table_sz, count, n_rel, n_64, n_glob, n_jmp, n_tls, n_other, n_oob ); } @@ -616,7 +704,14 @@ pub fn dlopen(pid: u32, pml4_phys: u64, elf_bytes: &[u8], path: &[u8]) -> Result if hdr.e_ident[4] != ELFCLASS64 { return Err("not ELF64"); } if hdr.e_ident[5] != ELFDATA2LSB { return Err("not little-endian"); } if hdr.e_type != ET_DYN { return Err("not a shared object (ET_DYN required)"); } + // Accept only the machine type matching the kernel build: the loaded code + // executes natively in the process, so a foreign-arch .so would fault. + #[cfg(target_arch = "x86_64")] if hdr.e_machine != EM_X86_64 { return Err("not x86_64"); } + #[cfg(target_arch = "aarch64")] + if hdr.e_machine != EM_AARCH64 { return Err("not aarch64"); } + #[cfg(not(any(target_arch = "x86_64", target_arch = "aarch64")))] + { let _ = (EM_X86_64, EM_AARCH64); } // ── Allocate load base ──────────────────────────────────────────────── // Key the bump cursor on pml4_phys, not pid, so threads sharing an @@ -832,9 +927,13 @@ pub fn dlopen(pid: u32, pml4_phys: u64, elf_bytes: &[u8], path: &[u8]) -> Result h }; - // ── Runtime patch: libflutter_engine.so ────────────────────────────── - // Identified by ELF size > 50 MiB (the engine is ~91 MiB; no other - // library we load comes close). + // ── Runtime patch: libflutter_engine.so (x86_64 only) ──────────────── + // These patches rewrite specific *x86 instruction byte sequences* in the + // engine (e.g. ImageReader::GetInstructionsAt sign-extension). They are + // inherently architecture-specific and must NOT run against the aarch64 + // engine (different opcodes, different offsets). Gate the whole block. + // Identified by ELF size > 50 MiB (the x86 engine is ~91 MiB). + #[cfg(target_arch = "x86_64")] if elf_bytes.len() > 50 * 1024 * 1024 { // Helper: write N bytes at a virtual address via HHDM. let mut write_va = |va: u64, data: &[u8]| -> bool { diff --git a/kernel/src/process/elf.rs b/kernel/src/process/elf.rs index 09df765..314f437 100644 --- a/kernel/src/process/elf.rs +++ b/kernel/src/process/elf.rs @@ -2,6 +2,7 @@ //! into a process's user address space. use crate::mm::{frame_allocator, paging}; +use alloc::collections::BTreeMap; // ── ELF64 constants ─────────────────────────────────────────────────────────── @@ -11,7 +12,16 @@ const ELFDATA2LSB: u8 = 1; // little-endian const ET_EXEC: u16 = 2; const ET_DYN: u16 = 3; // PIE const EM_X86_64: u16 = 62; +const EM_AARCH64: u16 = 183; const PT_LOAD: u32 = 1; + +/// The ELF machine type this kernel build can execute natively. +#[cfg(target_arch = "x86_64")] +const EM_NATIVE: u16 = EM_X86_64; +#[cfg(target_arch = "aarch64")] +const EM_NATIVE: u16 = EM_AARCH64; +#[cfg(not(any(target_arch = "x86_64", target_arch = "aarch64")))] +const EM_NATIVE: u16 = EM_X86_64; const PF_X: u32 = 1 << 0; // executable const PF_W: u32 = 1 << 1; // writable @@ -76,7 +86,7 @@ pub fn load(bytes: &[u8], pml4_phys: u64) -> Result { if hdr.e_ident[0..4] != ELF_MAGIC { return Err("bad ELF magic"); } if hdr.e_ident[4] != ELFCLASS64 { return Err("not ELF64"); } if hdr.e_ident[5] != ELFDATA2LSB { return Err("not little-endian"); } - if hdr.e_machine != EM_X86_64 { return Err("not x86_64"); } + if hdr.e_machine != EM_NATIVE { return Err("wrong ELF machine for arch"); } if hdr.e_type != ET_EXEC && hdr.e_type != ET_DYN { return Err("not executable ELF"); } @@ -89,6 +99,19 @@ pub fn load(bytes: &[u8], pml4_phys: u64) -> Result { let ph_num = hdr.e_phnum as usize; let ph_size = hdr.e_phentsize as usize; + // Pages this call has already backed with a freshly-allocated frame, keyed by + // page VA → frame PA. We MUST NOT use the page table to detect "already + // mapped" pages: on aarch64 the per-process root is seeded with the kernel + // identity map (block descriptors), and splitting a block to overlay one user + // page leaves the *other* pages of that block populated with valid identity + // sub-page descriptors. Querying the page table would then report those + // identity pages as already-mapped and the loader would reuse the identity PA + // (device/0xFF memory) instead of allocating a real frame. Tracking only the + // frames WE allocated in this load keeps the genuine code+rodata-overlap reuse + // (two PT_LOADs sharing one 4 KiB page) while always allocating fresh frames + // for identity-block pages. + let mut mapped: BTreeMap = BTreeMap::new(); + // ── Walk PT_LOAD segments ───────────────────────────────────────────── for i in 0..ph_num { let ph_base = ph_off + i * ph_size; @@ -112,16 +135,17 @@ pub fn load(bytes: &[u8], pml4_phys: u64) -> Result { for p in 0..num_pages { let page_virt = page_start + (p * 4096) as u64; - // Check whether this virtual page was already mapped by a previous - // PT_LOAD segment (common when code and rodata share a 4 KiB page). - // If so, reuse the existing physical frame so we don't lose the - // bytes written by the earlier segment, and preserve the more- - // permissive (exec) permission that the earlier segment set. - let existing_phys = crate::mm::paging::translate_user_page(pml4_phys, page_virt); + // Check whether THIS load already backed this virtual page (common + // when code and rodata share a 4 KiB page across two PT_LOADs). If so, + // reuse that frame so we don't lose the earlier segment's bytes and we + // keep the more-permissive (exec) permission. We intentionally consult + // our own `mapped` table, not the page table — see the note above. + let existing_phys = mapped.get(&page_virt).copied(); let (page_phys, is_new_frame) = if let Some(phys) = existing_phys { (phys, false) } else { let phys = frame_allocator::alloc_frame().ok_or("OOM: elf segment")?; + mapped.insert(page_virt, phys); (phys, true) }; diff --git a/kernel/src/process/mod.rs b/kernel/src/process/mod.rs index bfc9da9..9c266fc 100644 --- a/kernel/src/process/mod.rs +++ b/kernel/src/process/mod.rs @@ -44,7 +44,14 @@ pub const MAX_PROCS: usize = 256; pub const USER_ELF_BASE: u64 = 0x0000_0000_0040_0000; // 4 MiB /// User stack top (grows downward; bottom = TOP − STACK_SIZE). +/// +/// x86_64 uses a 47-bit canonical-low address. aarch64's bring-up MMU configures +/// a 39-bit TTBR0 VA window (T0SZ=25 → max user VA 512 GiB), so the ARM stack top +/// is placed just under that boundary instead. +#[cfg(not(target_arch = "aarch64"))] pub const USER_STACK_TOP: u64 = 0x0000_7FFF_FFFF_0000; +#[cfg(target_arch = "aarch64")] +pub const USER_STACK_TOP: u64 = 0x0000_007F_FFF0_0000; // ~512 GiB − 1 MiB (39-bit VA) pub const USER_STACK_SIZE: usize = 4 * 1024 * 1024; // 4 MiB const SYSCALL_STACK_SIZE: usize = 64 * 1024; const XSTATE_SIZE: usize = 4096; @@ -164,6 +171,27 @@ pub struct Process { pub user_stack_size: u64, /// CPU core index that currently executes this process context (None if idle/blocked). pub current_cpu: Option, + /// aarch64 only: full EL0 trap-frame snapshot (x0..x30, SP_EL0, ELR, SPSR, + /// ESR) captured when this thread is timer-preempted at an arbitrary user + /// instruction. The shared `UserRegs` carries only the x86-named subset, so + /// ARM scratch registers (x6/x7, x11–x18, x24–x28, x30) would otherwise be + /// lost across a preemptive switch. The ARM timer ISR saves the live + /// `vectors::TrapFrame` here and restores it on resume, giving full-fidelity + /// preemption while the shared scheduler still owns the switch *decision*. + /// Layout matches `arch::aarch64::vectors::TrapFrame` (36 × u64). + #[cfg(target_arch = "aarch64")] + pub arch_trapframe: [u64; 36], + /// aarch64 only: true once `arch_trapframe` holds a valid timer-preempt + /// snapshot to restore (vs. a fresh/syscall-entered thread). + #[cfg(target_arch = "aarch64")] + pub arch_frame_valid: bool, + /// aarch64 only: the user link register (x30) captured at the last SVC + /// boundary. AArch64 keeps return addresses in LR, not on the stack, so a + /// thread that yields mid-syscall must have x30 restored on resume — without + /// it the cooperative `build_image` re-entry leaves x30=0 and the thread's + /// next `ret` branches to address 0. + #[cfg(target_arch = "aarch64")] + pub user_lr: u64, } impl Process { @@ -196,10 +224,53 @@ impl Process { user_stack_base: 0, user_stack_size: 0, current_cpu: None, + #[cfg(target_arch = "aarch64")] + arch_trapframe: [0u64; 36], + #[cfg(target_arch = "aarch64")] + arch_frame_valid: false, + #[cfg(target_arch = "aarch64")] + user_lr: 0, } } } +/// aarch64: save a full EL0 trap-frame snapshot into `pid`'s arch slot, taken +/// when the generic-timer ISR preempts the thread. Restored verbatim by +/// `arch_take_trapframe` so ARM scratch registers survive the switch. +/// +/// Called from the timer IRQ handler, so it uses `try_lock` and never blocks — +/// blocking on a contended PTABLE_LOCK from inside an ISR would deadlock. Returns +/// false if the lock was busy (the caller then skips the switch this tick). +#[cfg(target_arch = "aarch64")] +pub fn arch_store_trapframe(pid: u32, frame: &[u64; 36]) -> bool { + if let Some(_g) = PTABLE_LOCK.try_lock() { + let p = unsafe { &mut PTABLE[idx_of(pid)] }; + if p.pid == pid { + p.arch_trapframe = *frame; + p.arch_frame_valid = true; + } + true + } else { + false + } +} + +/// aarch64: fetch `pid`'s saved trap-frame snapshot (and whether it is valid). +/// The valid flag is consumed (cleared) so a subsequent syscall-entry re-entry +/// rebuilds the frame from `regs` instead of replaying a stale snapshot. Uses +/// `try_lock` for the same ISR-safety reason as `arch_store_trapframe`. +#[cfg(target_arch = "aarch64")] +pub fn arch_take_trapframe(pid: u32) -> Option<[u64; 36]> { + let _g = PTABLE_LOCK.try_lock()?; + let p = unsafe { &mut PTABLE[idx_of(pid)] }; + if p.pid == pid && p.arch_frame_valid { + p.arch_frame_valid = false; + Some(p.arch_trapframe) + } else { + None + } +} + /// Persist the FS base for `pid` (called on arch_prctl(ARCH_SET_FS) and on /// the FS-bootstrap path) so it is restored on the next user entry. pub fn set_proc_fs_base(pid: u32, fs: u64) { @@ -271,6 +342,18 @@ pub struct PTableLock { holder: AtomicU32, } +/// CPU index used to key PTABLE_LOCK_RECURSION / the holder atomic. Always a +/// valid in-range index. We use the computed `current_cpu_id()` (which on +/// aarch64 short-circuits to 0 for single-core) rather than reading the +/// `PerCpuData.cpu_id` *field*, and hard-clamp to the recursion-array bound so +/// the timer ISR can never index out of bounds and panic the kernel even if a +/// per-CPU struct read is momentarily inconsistent. +#[inline(always)] +fn ptable_cpu_idx() -> u32 { + let id = crate::arch::smp::current_cpu_id(); + if (id as usize) < crate::arch::smp::MAX_CPUS { id } else { 0 } +} + impl PTableLock { pub const fn new() -> Self { Self { @@ -280,7 +363,7 @@ impl PTableLock { } pub fn lock(&self) -> PTableGuard { - let my_cpu = crate::arch::smp::this_cpu().cpu_id; + let my_cpu = ptable_cpu_idx(); if self.holder.load(Ordering::Acquire) == my_cpu { unsafe { PTABLE_LOCK_RECURSION[my_cpu as usize] += 1; @@ -298,7 +381,7 @@ impl PTableLock { } pub fn try_lock(&self) -> Option { - let my_cpu = crate::arch::smp::this_cpu().cpu_id; + let my_cpu = ptable_cpu_idx(); if self.holder.load(Ordering::Acquire) == my_cpu { unsafe { PTABLE_LOCK_RECURSION[my_cpu as usize] += 1; @@ -322,14 +405,14 @@ impl PTableLock { impl Drop for PTableGuard { fn drop(&mut self) { if self.is_outer { - let my_cpu = crate::arch::smp::this_cpu().cpu_id; + let my_cpu = ptable_cpu_idx(); unsafe { PTABLE_LOCK_RECURSION[my_cpu as usize] = 0; PTABLE_LOCK.holder.store(0xFFFF_FFFF, Ordering::Release); PTABLE_LOCK_GUARD = None; } } else { - let my_cpu = crate::arch::smp::this_cpu().cpu_id; + let my_cpu = ptable_cpu_idx(); unsafe { if PTABLE_LOCK_RECURSION[my_cpu as usize] > 0 { PTABLE_LOCK_RECURSION[my_cpu as usize] -= 1; @@ -509,7 +592,11 @@ pub fn spawn_with_bootstrap( } } - // Map POSIX trampoline + sysdata pages so glibc symbols resolve correctly. + // Map POSIX trampoline + sysdata pages so libc/glibc symbols the Flutter + // engine imports resolve to in-process syscall stubs. encode_stub() emits + // native machine code per arch (x86 on x86_64, AArch64 on aarch64), so this + // is required on BOTH arches once a dynamically-linked host (the Flutter + // embedder) runs — the engine's DT_INIT_ARRAY jumps into these stubs. posix_trampolines::map_system_pages(pml4_phys)?; let idx = idx_of(pid); @@ -547,6 +634,11 @@ pub fn spawn_with_bootstrap( p.sig_handlers = [0u64; 32]; p.errno_to_deliver = 0; p.preempted_by_timer = false; + // aarch64: start with no stale timer-preempt frame snapshot. + #[cfg(target_arch = "aarch64")] + { + p.arch_frame_valid = false; + } } log::info!( @@ -679,6 +771,18 @@ pub fn current_pid() -> u32 { crate::arch::smp::this_cpu().current_pid.load(Ordering::Acquire) } +/// Physical address of the current process's top-level page table (the L1 / TTBR0 +/// root on aarch64, the PML4 on x86). Used by diagnostics / kernel-side user-VA +/// translation. Returns 0 if there is no current process. +pub fn current_user_root() -> u64 { + let pid = current_pid(); + if pid == 0 { return 0; } + let idx = idx_of(pid); + let _g = PTABLE_LOCK.lock(); + let p = unsafe { &PTABLE[idx] }; + if p.pid == pid { p.pml4_phys } else { 0 } +} + /// Wait for `pid` to become zombie, then reap it and return exit code. /// /// Returns: @@ -858,6 +962,35 @@ pub fn sibling_pids(current: u32) -> alloc::vec::Vec { out } +/// Compact debug string of pid/state/current_cpu for the first few PIDs, for +/// diagnosing why the timer-preempt scheduler can't find a runnable thread. +pub fn debug_runnable_states() -> alloc::string::String { + use core::fmt::Write; + let mut s = alloc::string::String::new(); + if let Some(_g) = PTABLE_LOCK.try_lock() { + // Print the RAW slot contents for indices 1..=12 (the slot's own .pid + // field, not the loop var) so a slot whose .pid != index reveals a + // reused/cleared entry — the mechanism behind a worker thread that runs + // yet never appears under its expected index. + for idx in 1usize..=12 { + let p = unsafe { &PTABLE[idx] }; + if p.pid == 0 { continue; } + let st = match p.state { + ProcState::Running => 'R', + ProcState::Blocked => 'B', + ProcState::Zombie(_) => 'Z', + ProcState::Dead => 'D', + }; + let cpu = match p.current_cpu { Some(c) => c as i32, None => -1 }; + // slot[idx]=pid:state c cpu + let _ = write!(s, "s{}={}{}{} ", idx, p.pid, st, cpu); + } + } else { + s.push_str("(locked)"); + } + s +} + pub fn next_runnable_pid_locked(current: u32, my_cpu: u32) -> Option { // FOREGROUND-EXCLUSIVE SCHEDULING. When an app is foreground (focus is not the // shell pid 1), run ONLY that app's thread group and leave the backgrounded @@ -1249,13 +1382,7 @@ pub fn enter_user_by_pid_noreturn(pid: u32) -> ! { crate::arch::cpu::set_fs_base(fs_base); // Switch to the user address space. - unsafe { - core::arch::asm!( - "mov cr3, {cr3}", - cr3 = in(reg) pml4_phys, - options(nostack, nomem, preserves_flags), - ); - } + crate::arch::memory::write_cr3(pml4_phys); // Deliver pending errno (e.g. EINTR=4 from sys_epoll_wait_real) NOW, // after the CR3 switch so we write into the resuming thread's own SYSDATA @@ -1270,6 +1397,26 @@ pub fn enter_user_by_pid_noreturn(pid: u32) -> ! { } } + // aarch64: carry the user link register (x30) in the otherwise-unused + // `rflags` slot (build_image maps rflags→x30 on aarch64; SPSR is constant + // there). Without this the cooperative SYSRET resume leaves x30=0 and the + // resuming thread's next `ret` branches to address 0. + #[cfg(target_arch = "aarch64")] + let rflags = { + let _g = PTABLE_LOCK.lock(); + let p = unsafe { &PTABLE[idx_of(pid)] }; + if p.pid == pid { p.user_lr } else { rflags } + }; + + // Assemble the full user register state for the arch entry hook. The + // arch backend owns the actual ring-3 transition asm (IRETQ vs SYSRETQ on + // x86_64); the shared code above owns all the PTABLE/CR3/errno logic. + let enter_regs = crate::arch::EnterUserRegs { + rip, rsp, rflags, + rax, rbx, rcx, rdx, rsi, rdi, rbp, + r8, r9, r10, r11, r12, r13, r14, r15, + }; + if preempted_by_timer { // ── IRET path ──────────────────────────────────────────────────────── // This thread was timer-preempted at an arbitrary user instruction, so @@ -1277,67 +1424,7 @@ pub fn enter_user_by_pid_noreturn(pid: u32) -> ! { // RFLAGS) before returning. SYSRETQ is wrong here because it would // set RIP=RCX and RFLAGS=R11, corrupting those registers and losing // the real FLAGS state (e.g. CF/ZF from a cmp instruction). - // - // Layout of iret_frame (accessed via r11 as base register): - // [r11 + 0x00] RIP ─┐ - // [r11 + 0x08] CS │ hardware IRET frame pushed onto - // [r11 + 0x10] RFLAGS │ the kernel stack by the `push` sequence - // [r11 + 0x18] RSP ─┘ (ring-3 switch includes RSP+SS) - // [r11 + 0x20] SS - // [r11 + 0x28] RAX .. R15 (all user GPRs, r11 loaded last) - let iret_frame: [u64; 20] = [ - /* 0x00 */ rip, - /* 0x08 */ crate::arch::gdt::USER_CS as u64, // 0x2B - /* 0x10 */ rflags | 0x200, // ensure IF=1 - /* 0x18 */ rsp, - /* 0x20 */ crate::arch::gdt::USER_DS as u64, // 0x23 - /* 0x28 */ rax, - /* 0x30 */ rbx, - /* 0x38 */ rcx, // real RCX (not SYSCALL convention) - /* 0x40 */ rdx, - /* 0x48 */ rdi, - /* 0x50 */ rsi, - /* 0x58 */ rbp, - /* 0x60 */ r8, - /* 0x68 */ r9, - /* 0x70 */ r10, - /* 0x78 */ r11, // real R11 (not RFLAGS convention) - /* 0x80 */ r12, - /* 0x88 */ r13, - /* 0x90 */ r14, - /* 0x98 */ r15, - ]; - unsafe { - core::arch::asm!( - // Push the 5-word IRET frame: SS, RSP, RFLAGS, CS, RIP. - // These land on the current kernel stack; IRETQ will consume - // them and switch to the user stack (RSP) automatically. - "push qword ptr [r11 + 0x20]", // SS - "push qword ptr [r11 + 0x18]", // RSP - "push qword ptr [r11 + 0x10]", // RFLAGS - "push qword ptr [r11 + 0x08]", // CS - "push qword ptr [r11 + 0x00]", // RIP - // Restore all user GPRs (r11 last because it is our base ptr). - "mov rax, [r11 + 0x28]", - "mov rbx, [r11 + 0x30]", - "mov rcx, [r11 + 0x38]", - "mov rdx, [r11 + 0x40]", - "mov rdi, [r11 + 0x48]", - "mov rsi, [r11 + 0x50]", - "mov rbp, [r11 + 0x58]", - "mov r8, [r11 + 0x60]", - "mov r9, [r11 + 0x68]", - "mov r10, [r11 + 0x70]", - "mov r12, [r11 + 0x80]", - "mov r13, [r11 + 0x88]", - "mov r14, [r11 + 0x90]", - "mov r15, [r11 + 0x98]", - "mov r11, [r11 + 0x78]", // real R11 — must be last - "iretq", - in("r11") iret_frame.as_ptr(), - options(noreturn), - ) - } + unsafe { crate::arch::enter_user_iret(&enter_regs) } } // ── SYSRET path (syscall-yield context) ────────────────────────────────── @@ -1348,58 +1435,8 @@ pub fn enter_user_by_pid_noreturn(pid: u32) -> ! { // RAX changed to the syscall return value. This is required because the // SysV ABI mandates that callee-saved regs (rbx, rbp, r12–r15) survive // a function call, and the userspace SYSCALL trampoline is such a call. - // - // Stage values in a fixed-layout stack array and load them inside the asm - // block via a single pointer. RSP and the syscall-consumed regs (RCX, - // R11) are loaded last so all intermediate reads complete first. - let frame: [u64; 16] = [ - /* 0x00 */ rip, - /* 0x08 */ rflags, - /* 0x10 */ rsp, - /* 0x18 */ rdi, - /* 0x20 */ rsi, - /* 0x28 */ rdx, - /* 0x30 */ rax, - /* 0x38 */ rbx, - /* 0x40 */ rbp, - /* 0x48 */ r8, - /* 0x50 */ r9, - /* 0x58 */ r10, - /* 0x60 */ r12, - /* 0x68 */ r13, - /* 0x70 */ r14, - /* 0x78 */ r15, - ]; log::trace!("[enter_user] about to SYSRET. rip={:#x} rsp={:#x} rflags={:#x} pml4_phys={:#x}", rip, rsp, rflags, pml4_phys); - unsafe { - core::arch::asm!( - // Pin the frame base in r11. r11 is consumed by SYSRET (as user - // RFLAGS) and is the LAST register we load — every other load - // can therefore use [r11 + disp] safely. If we let the compiler - // pick the base register, it can alias the base with the first - // destination (e.g. rax) and have the very first `mov` clobber - // the pointer before any further reads happen. - "mov rax, [r11 + 0x30]", - "mov rdi, [r11 + 0x18]", - "mov rsi, [r11 + 0x20]", - "mov rdx, [r11 + 0x28]", - "mov rbx, [r11 + 0x38]", - "mov rbp, [r11 + 0x40]", - "mov r8, [r11 + 0x48]", - "mov r9, [r11 + 0x50]", - "mov r10, [r11 + 0x58]", - "mov r12, [r11 + 0x60]", - "mov r13, [r11 + 0x68]", - "mov r14, [r11 + 0x70]", - "mov r15, [r11 + 0x78]", - "mov rcx, [r11 + 0x00]", // user RIP → RCX (consumed by sysretq) - "mov rsp, [r11 + 0x10]", // switch to user stack (mapped in user PML4) - "mov r11, [r11 + 0x08]", // user RFL → R11 (consumed by sysretq) — load LAST - "sysretq", - in("r11") frame.as_ptr(), - options(noreturn), - ) - } + unsafe { crate::arch::enter_user_sysret(&enter_regs) } } pub fn enter_user_by_pid_noreturn_try(pid: u32) -> bool { @@ -1431,6 +1468,12 @@ pub fn enter_user_by_pid_noreturn_try(pid: u32) -> bool { syscall_stack_top, fs_base, errno_to_deliver, preempted_by_timer) = context; crate::process::set_current_pid(pid); + if rip == 0 || rip < 0x1000 { + log::error!( + "[enter-user-ZEROPC] pid={} rip={:#x} rsp={:#x} rax={:#x} fs={:#x} preempted={}", + pid, rip, rsp, rax, fs_base, preempted_by_timer + ); + } if pid == 8 || pid == 9 { static ENTER_USER_89_LOG: AtomicU32 = AtomicU32::new(0); let n = ENTER_USER_89_LOG.fetch_add(1, Ordering::Relaxed); @@ -1452,13 +1495,7 @@ pub fn enter_user_by_pid_noreturn_try(pid: u32) -> bool { crate::arch::cpu::set_fs_base(fs_base); // Switch to the user address space. - unsafe { - core::arch::asm!( - "mov cr3, {cr3}", - cr3 = in(reg) pml4_phys, - options(nostack, nomem, preserves_flags), - ); - } + crate::arch::memory::write_cr3(pml4_phys); if errno_to_deliver != 0 { unsafe { @@ -1469,99 +1506,29 @@ pub fn enter_user_by_pid_noreturn_try(pid: u32) -> bool { } } + // aarch64: carry the user link register (x30) in the otherwise-unused + // `rflags` slot. build_image maps rflags→x30 on aarch64; SPSR is a constant + // there, so rflags is free. (The timer-preempt/IRET path restores x30 from + // arch_trapframe instead, so this only matters for the SYSRET/cooperative + // resume below.) + #[cfg(target_arch = "aarch64")] + let rflags = { + let _g = PTABLE_LOCK.lock(); + let p = unsafe { &PTABLE[idx_of(pid)] }; + if p.pid == pid { p.user_lr } else { rflags } + }; + + let enter_regs = crate::arch::EnterUserRegs { + rip, rsp, rflags, + rax, rbx, rcx, rdx, rsi, rdi, rbp, + r8, r9, r10, r11, r12, r13, r14, r15, + }; + if preempted_by_timer { - let iret_frame: [u64; 20] = [ - /* 0x00 */ rip, - /* 0x08 */ crate::arch::gdt::USER_CS as u64, - /* 0x10 */ rflags | 0x200, - /* 0x18 */ rsp, - /* 0x20 */ crate::arch::gdt::USER_DS as u64, - /* 0x28 */ rax, - /* 0x30 */ rbx, - /* 0x38 */ rcx, - /* 0x40 */ rdx, - /* 0x48 */ rdi, - /* 0x50 */ rsi, - /* 0x58 */ rbp, - /* 0x60 */ r8, - /* 0x68 */ r9, - /* 0x70 */ r10, - /* 0x78 */ r11, - /* 0x80 */ r12, - /* 0x88 */ r13, - /* 0x90 */ r14, - /* 0x98 */ r15, - ]; - unsafe { - core::arch::asm!( - "push qword ptr [r11 + 0x20]", - "push qword ptr [r11 + 0x18]", - "push qword ptr [r11 + 0x10]", - "push qword ptr [r11 + 0x08]", - "push qword ptr [r11 + 0x00]", - "mov rax, [r11 + 0x28]", - "mov rbx, [r11 + 0x30]", - "mov rcx, [r11 + 0x38]", - "mov rdx, [r11 + 0x40]", - "mov rdi, [r11 + 0x48]", - "mov rsi, [r11 + 0x50]", - "mov rbp, [r11 + 0x58]", - "mov r8, [r11 + 0x60]", - "mov r9, [r11 + 0x68]", - "mov r10, [r11 + 0x70]", - "mov r12, [r11 + 0x80]", - "mov r13, [r11 + 0x88]", - "mov r14, [r11 + 0x90]", - "mov r15, [r11 + 0x98]", - "mov r11, [r11 + 0x78]", - "iretq", - in("r11") iret_frame.as_ptr(), - options(noreturn), - ) - } + unsafe { crate::arch::enter_user_iret(&enter_regs) } } - let frame: [u64; 16] = [ - /* 0x00 */ rip, - /* 0x08 */ rflags, - /* 0x10 */ rsp, - /* 0x18 */ rdi, - /* 0x20 */ rsi, - /* 0x28 */ rdx, - /* 0x30 */ rax, - /* 0x38 */ rbx, - /* 0x40 */ rbp, - /* 0x48 */ r8, - /* 0x50 */ r9, - /* 0x58 */ r10, - /* 0x60 */ r12, - /* 0x68 */ r13, - /* 0x70 */ r14, - /* 0x78 */ r15, - ]; - unsafe { - core::arch::asm!( - "mov rax, [r11 + 0x30]", - "mov rdi, [r11 + 0x18]", - "mov rsi, [r11 + 0x20]", - "mov rdx, [r11 + 0x28]", - "mov rbx, [r11 + 0x38]", - "mov rbp, [r11 + 0x40]", - "mov r8, [r11 + 0x48]", - "mov r9, [r11 + 0x50]", - "mov r10, [r11 + 0x58]", - "mov r12, [r11 + 0x60]", - "mov r13, [r11 + 0x68]", - "mov r14, [r11 + 0x70]", - "mov r15, [r11 + 0x78]", - "mov rcx, [r11 + 0x00]", - "mov rsp, [r11 + 0x10]", - "mov r11, [r11 + 0x08]", - "sysretq", - in("r11") frame.as_ptr(), - options(noreturn), - ) - } + unsafe { crate::arch::enter_user_sysret(&enter_regs) } } /// Kernel-task entry point: loads the user PML4 and SYSRETs to ring-3. @@ -1577,7 +1544,7 @@ fn user_launch_task() { // Disable interrupts so the APIC timer cannot preempt between // mark_current_zombie() and sysretq. SYSRET restores RFLAGS from R11 // (0x0202 = IF=1), so the user process runs with interrupts enabled. - unsafe { core::arch::asm!("cli", options(nomem, nostack, preserves_flags)); } + crate::arch::interrupts_disable(); // Mark this kernel task as Zombie before we SYSRET. The SYSRET transfers // execution to ring-3 permanently; this task slot must never be resumed as // a kernel task again. Marking it Zombie prevents the scheduler from @@ -1607,7 +1574,7 @@ fn user_launch_task() { /// /// Returns `None` if there is no other runnable thread (no switch occurs). pub fn timer_preempt_switch(cur_pid: u32, cur_regs: &UserRegs) -> Option<(u32, UserRegs, u64)> { - let my_cpu = crate::arch::smp::this_cpu().cpu_id; + let my_cpu = crate::arch::smp::current_cpu_id(); let fs = crate::arch::cpu::get_fs_base(); let _g = PTABLE_LOCK.lock(); @@ -1666,7 +1633,7 @@ pub fn timer_preempt_switch(cur_pid: u32, cur_regs: &UserRegs) -> Option<(u32, U /// Try-lock variant of `timer_preempt_switch` to avoid ISR deadlocks. pub fn timer_preempt_switch_try(cur_pid: u32, cur_regs: &UserRegs) -> Option<(u32, UserRegs, u64)> { - let my_cpu = crate::arch::smp::this_cpu().cpu_id; + let my_cpu = crate::arch::smp::current_cpu_id(); let fs = crate::arch::cpu::get_fs_base(); if let Some(_g) = PTABLE_LOCK.try_lock() { @@ -1746,8 +1713,34 @@ pub fn save_cooperative_yield_context(pid: u32, rip: u64, rsp: u64) { /// Save the user-space return context (RIP + RSP after syscall) into the /// process's register file so `enter_user_by_pid_noreturn` resumes correctly. +/// Length in bytes of the user syscall-entry instruction. On x86_64 `syscall` +/// is 2 bytes; on aarch64 `svc #0` is 4 bytes. After a syscall the user PC +/// (ELR_EL1 / the saved RIP) points at the *next* instruction, so to make a +/// blocked thread RE-EXECUTE its syscall on resume the kernel rewinds the saved +/// PC by exactly one syscall instruction. +#[cfg(target_arch = "x86_64")] +pub const SYSCALL_INSN_LEN: u64 = 2; +#[cfg(target_arch = "aarch64")] +pub const SYSCALL_INSN_LEN: u64 = 4; + +/// Save a thread's resume context rewound to RE-EXECUTE the syscall it is +/// currently in (used by the blocking syscalls — epoll_wait/poll/wm_event_wait/ +/// futex — which resume by re-entering the kernel and re-checking readiness). +/// Rewinds the saved PC by one syscall instruction (arch-correct length). +/// Using a hardcoded `-2` here mis-rewinds aarch64's 4-byte `svc` into the +/// middle of the instruction → a PC-alignment fault (EC=0x22) on resume. +pub fn save_return_context_reexec(pid: u32, urip: u64, ursp: u64) { + save_return_context(pid, urip.wrapping_sub(SYSCALL_INSN_LEN), ursp); +} + pub fn save_return_context(pid: u32, rip: u64, rsp: u64) { let fs = crate::arch::cpu::get_fs_base(); + // aarch64: capture the user link register (x30) from the SVC-entry snapshot + // BEFORE taking the lock. AArch64 return addresses live in LR, not on the + // stack, so a thread that yields inside this syscall must resume with x30 + // intact or its next `ret` jumps to whatever build_image left there (0). + #[cfg(target_arch = "aarch64")] + let lr = crate::arch::syscall::user_lr(); let _g = PTABLE_LOCK.lock(); let p = unsafe { &mut PTABLE[idx_of(pid)] }; if p.pid == pid { @@ -1757,69 +1750,68 @@ pub fn save_return_context(pid: u32, rip: u64, rsp: u64) { // Saved at a SYSCALL boundary — SYSRETQ is correct for this thread. p.preempted_by_timer = false; p.fs_base = fs; + #[cfg(target_arch = "aarch64")] + { p.user_lr = lr; } } } -/// Snapshot the full user GPR set (as captured at SYSCALL entry) into the -/// process's register file. Must be called from inside a syscall handler -/// before yielding the CPU to another process via -/// `enter_user_by_pid_noreturn`, otherwise the yielding thread will resume -/// with stale rbx/rbp/r12–r15/rdi/rsi/etc. and corrupt its C++ caller's -/// `this` pointer and locals. -/// Per-CPU "already captured the user GPRs for the current syscall" flag. +/// Guards against a since-clobbered per-CPU user-GPR snapshot leaking into a +/// thread's saved context. /// -/// The user GPR snapshot lives in a PER-CPU scratch area (`gs:[..]`), written by -/// the syscall entry stub. It is shared by every thread on that core, so if a -/// thread switch lands mid-handler (the wait loops `sti; hlt` and the timer ISR -/// can switch threads), another thread's syscall entry OVERWRITES it. A later -/// `save_full_user_gprs` would then read the *other* thread's callee-saved regs -/// and store them into our context → on resume `rbx`/`rbp`/`r12-15` are garbage -/// (observed: `MessageLoopOscortex::Run` resuming from `epoll_wait` with -/// `rbx`=a Skia stride → SIGSEGV; the root of the "sporadic" render crashes). +/// The user GPR snapshot lives in a PER-CPU scratch area, written at syscall +/// entry from the trapping thread's frame. It is shared by every thread on that +/// core, so once a syscall hands off the CPU (a cooperative `epoll`/`futex`/ +/// `cond` yield, or — on aarch64 — a generic-timer preempt that lands a +/// sibling's SVC mid-handler), another thread's syscall entry OVERWRITES it. A +/// later `save_full_user_gprs` would then read the *other* thread's callee-saved +/// regs / link register and store them into our context → on resume rbx/rbp/ +/// r12–15 (x19–x23/x29) or x30 are garbage, and the next `ret` branches to a +/// stale address (observed on aarch64: x30=0 → `ret` to 0 → EL0 UDF after +/// `FlutterEngineRunInitialized`; the x86 analogue corrupted `MessageLoop::Run`'s +/// `this`). /// -/// Fix: capture ONCE, eagerly, at syscall entry (`capture_user_gprs_at_entry`, -/// called from `dispatch_fast` while the snapshot is still fresh for THIS -/// thread). This flag makes every subsequent yield-time `save_full_user_gprs` -/// in the same syscall a no-op, so a clobbered per-CPU snapshot can never leak -/// into our saved context. +/// Fix: capture ONCE, eagerly, at syscall entry (`capture_user_gprs_at_entry`), +/// while the snapshot is still fresh for THIS thread (x86: interrupts masked +/// until the handler `sti`s; aarch64: from inside the IRQ-masked SVC window in +/// the vector dispatch). This flag makes every subsequent yield-time +/// `save_full_user_gprs` in the same syscall a no-op, so a clobbered per-CPU +/// snapshot can never leak into our saved context. const GPR_CAP_MAX_CPUS: usize = 64; static GPRS_CAPTURED: [core::sync::atomic::AtomicBool; GPR_CAP_MAX_CPUS] = [const { core::sync::atomic::AtomicBool::new(false) }; GPR_CAP_MAX_CPUS]; #[inline] fn gpr_cap_cpu_idx() -> usize { - (crate::arch::smp::this_cpu().cpu_id as usize).min(GPR_CAP_MAX_CPUS - 1) + #[cfg(target_arch = "aarch64")] + { (crate::arch::aarch64::smp::current_cpu_id() as usize).min(GPR_CAP_MAX_CPUS - 1) } + #[cfg(not(target_arch = "aarch64"))] + { (crate::arch::smp::this_cpu().cpu_id as usize).min(GPR_CAP_MAX_CPUS - 1) } } /// Capture the user GPRs into `pid`'s PTABLE slot at syscall ENTRY, while the /// per-CPU snapshot is guaranteed fresh for this thread. Marks them captured so /// later `save_full_user_gprs` calls this syscall don't re-read the (possibly -/// since-clobbered) shared per-CPU snapshot. +/// since-clobbered) shared per-CPU snapshot. MUST be invoked while the snapshot +/// cannot yet be overwritten by a sibling (IRQs masked / pre-`sti`). pub fn capture_user_gprs_at_entry(pid: u32) { GPRS_CAPTURED[gpr_cap_cpu_idx()].store(false, core::sync::atomic::Ordering::Relaxed); save_full_user_gprs(pid); } +/// Snapshot the full user GPR set (as captured at SYSCALL entry) into the +/// process's register file. Must be called from inside a syscall handler +/// before yielding the CPU to another process via +/// `enter_user_by_pid_noreturn`, otherwise the yielding thread will resume +/// with stale rbx/rbp/r12–r15/rdi/rsi/etc. and corrupt its C++ caller's +/// `this` pointer and locals. pub fn save_full_user_gprs(pid: u32) { // Only the first call per syscall (the eager entry capture) reads the - // per-CPU snapshot; it is fresh then. Later yield-time calls are no-ops. + // per-CPU snapshot; it is fresh then. Later yield-time calls are no-ops so a + // since-clobbered snapshot cannot leak into our saved context. if GPRS_CAPTURED[gpr_cap_cpu_idx()].swap(true, core::sync::atomic::Ordering::Relaxed) { return; } let snap = crate::arch::syscall::user_gprs(); - if pid == 2 { - log::warn!( - "[save_full_user_gprs] pid=2 rdi={:#x} rsi={:#x} rbx={:#x} rbp={:#x} r12={:#x} r13={:#x} r14={:#x} r15={:#x}", - snap.rdi, - snap.rsi, - snap.rbx, - snap.rbp, - snap.r12, - snap.r13, - snap.r14, - snap.r15 - ); - } let _g = PTABLE_LOCK.lock(); let p = unsafe { &mut PTABLE[idx_of(pid)] }; if p.pid != pid { return; } @@ -1835,6 +1827,10 @@ pub fn save_full_user_gprs(pid: u32) { p.regs.r13 = snap.r13; p.regs.r14 = snap.r14; p.regs.r15 = snap.r15; + // aarch64: persist the user link register (x30) so the thread can `ret` + // correctly after resuming from this syscall yield. + #[cfg(target_arch = "aarch64")] + { p.user_lr = snap.lr; } } /// Set the `rax` return value that will be delivered when this process is diff --git a/kernel/src/process/posix_trampolines.rs b/kernel/src/process/posix_trampolines.rs index 17c3a3a..66c72c8 100644 --- a/kernel/src/process/posix_trampolines.rs +++ b/kernel/src/process/posix_trampolines.rs @@ -25,6 +25,20 @@ use crate::mm::{frame_allocator, paging}; /// VA(i) = TRAMPOLINE_PAGES_VA + i * 16. Max 512 stubs. pub const TRAMPOLINE_PAGES_VA: u64 = 0x7FFF_E000; +/// VA of the aarch64 static TLSDESC resolver `_dl_tlsdesc_return`. +/// +/// The Flutter engine's C++ `thread_local` accesses on aarch64 use the TLSDESC +/// relocation model: the compiler emits `ldr x1,[desc#0]; blr x1` where the +/// descriptor's first word is a resolver function and the second word its +/// argument. For a statically-loaded module the resolver simply returns the +/// (TP-relative) offset stored in the descriptor's second word, i.e. +/// `ldr x0, [x0, #8]; ret` +/// `dl::apply_aarch64_tlsdesc` points every R_AARCH64_TLSDESC descriptor's +/// first word at this stub and writes the static TP offset into the second. +/// It lives in the last 16-byte slot of the (executable) trampoline pages. +#[cfg(target_arch = "aarch64")] +pub const TLSDESC_RESOLVER_VA: u64 = TRAMPOLINE_PAGES_VA + 511 * 16; + /// 1-page system data region (0x7FFF_C000 – 0x7FFF_CFFF). pub const SYSDATA_VA: u64 = 0x7FFF_C000; @@ -701,13 +715,71 @@ fn map_trampoline_pages(pml4_phys: u64) -> Result<(), &'static str> { let page1 = unsafe { core::slice::from_raw_parts_mut((phys1 + hhdm) as *mut u8, 4096) }; - page0.fill(0x90); // nop - page1.fill(0x90); // nop + // x86 pads with NOP (0x90) so fall-through reaches the trailing ret. aarch64 + // is fixed-width, so pad with RET (0xD65F03C0, LE) — any stray slot entry + // returns cleanly instead of executing leftover bytes as a bad instruction. + #[cfg(target_arch = "x86_64")] + { page0.fill(0x90); page1.fill(0x90); } + #[cfg(target_arch = "aarch64")] + { + let ret = 0xD65F_03C0u32.to_le_bytes(); + for p in [&mut *page0, &mut *page1] { + for chunk in p.chunks_exact_mut(4) { chunk.copy_from_slice(&ret); } + } + } - for (i, &(_, kind)) in STUBS.iter().enumerate() { + for (i, &(name, kind)) in STUBS.iter().enumerate() { let page = if i < 256 { &mut *page0 } else { &mut *page1 }; let off = (i % 256) * 16; encode_stub(page, off, kind); + // DIAGNOSTIC: confirm the fallback stub encoding (the engine init array + // jumps here for unresolved symbols; a bad encoding faults at EL0). + if name == "__oscortex_fallback_stub" { + let va = TRAMPOLINE_PAGES_VA + (i as u64) * 16; + log::warn!( + "[tramp/dbg] '{}' idx={} va={:#x} bytes={:02x?}", + name, i, va, &page[off..off + 16] + ); + } + } + + // aarch64: emit the static TLSDESC resolver into the last 16-byte slot + // (TLSDESC_RESOLVER_VA). It is `ldr x0,[x0,#8]; ret`: given a TLSDESC + // descriptor pointer in x0, return the TP-relative offset stored at +8. + #[cfg(target_arch = "aarch64")] + { + // Last slot of page1 = byte offset 0xFF0 (slot 511, 511%256=255 -> 255*16). + let off = 255 * 16; + emit32(page1, off, 0xF940_0400); // ldr x0, [x0, #8] + emit32(page1, off + 4, a64::ret()); // ret + // remaining 8 bytes of the slot stay RET-padded (harmless). + } + + // aarch64: we just wrote instructions through the data path (HHDM). Before + // EL0 executes them the data cache must be cleaned to the point of unification + // and the instruction cache invalidated, or stale/garbage may be fetched on + // real hardware (and under HVF). Sweep both freshly-written frames by VA via + // the HHDM mirror (which is what the data writes touched). + #[cfg(target_arch = "aarch64")] + unsafe { + for &base in &[phys0 + hhdm, phys1 + hhdm] { + let mut a = base; + let end = base + 4096; + while a < end { + core::arch::asm!("dc cvau, {0}", in(reg) a, options(nostack, preserves_flags)); + a += 64; // cache line granule (CTR_EL0 min is 64 on cortex-a72/Apple) + } + } + core::arch::asm!("dsb ish", options(nostack, preserves_flags)); + for &base in &[phys0 + hhdm, phys1 + hhdm] { + let mut a = base; + let end = base + 4096; + while a < end { + core::arch::asm!("ic ivau, {0}", in(reg) a, options(nostack, preserves_flags)); + a += 64; + } + } + core::arch::asm!("dsb ish", "isb", options(nostack, preserves_flags)); } // Map as read-execute (not writable) for user. @@ -718,6 +790,161 @@ fn map_trampoline_pages(pml4_phys: u64) -> Result<(), &'static str> { Ok(()) } +/// Write a 32-bit aarch64 instruction (little-endian) into the trampoline page. +#[cfg(target_arch = "aarch64")] +fn emit32(page: &mut [u8], off: usize, insn: u32) { + page[off..off + 4].copy_from_slice(&insn.to_le_bytes()); +} + +// AArch64 instruction encoders for the fixed shapes we need. All registers are +// 64-bit (X) unless noted. `imm16`/`shift` follow the MOVZ/MOVK encoding. +#[cfg(target_arch = "aarch64")] +mod a64 { + /// MOVZ Xd, #imm16, LSL #(shift*16) (zero other bits) + pub fn movz(rd: u32, imm16: u16, shift: u32) -> u32 { + 0xD280_0000 | (shift << 21) | ((imm16 as u32) << 5) | rd + } + /// MOVK Xd, #imm16, LSL #(shift*16) (keep other bits) + pub fn movk(rd: u32, imm16: u16, shift: u32) -> u32 { + 0xF280_0000 | (shift << 21) | ((imm16 as u32) << 5) | rd + } + /// MOVZ Wd, #imm16, LSL #(shift*16) (32-bit dest) + pub fn movz_w(rd: u32, imm16: u16, shift: u32) -> u32 { + 0x5280_0000 | (shift << 21) | ((imm16 as u32) << 5) | rd + } + /// MOVK Wd, #imm16, LSL #(shift*16) + pub fn movk_w(rd: u32, imm16: u16, shift: u32) -> u32 { + 0x7280_0000 | (shift << 21) | ((imm16 as u32) << 5) | rd + } + /// SVC #0 + pub const fn svc0() -> u32 { 0xD400_0001 } + /// RET (uses X30) + pub const fn ret() -> u32 { 0xD65F_03C0 } + /// BRK #0 (trap) + pub const fn brk0() -> u32 { 0xD420_0000 } + /// FMOV Dd, Xn (GPR -> SIMD scalar double) + pub fn fmov_d_x(dd: u32, xn: u32) -> u32 { 0x9E67_0000 | (xn << 5) | dd } + /// FMOV Xd, Dn (SIMD scalar double -> GPR) + pub fn fmov_x_d(xd: u32, dn: u32) -> u32 { 0x9E66_0000 | (dn << 5) | xd } + /// FMOV Dd, XZR (zero a double) — FMOV Dd, #0.0 isn't encodable, use xzr. + pub fn fmov_d_zero(dd: u32) -> u32 { fmov_d_x(dd, 31) } +} + +/// Load a full 64-bit immediate into Xd using up to 4 MOVZ/MOVK instructions. +/// Returns the number of *instructions* written. +#[cfg(target_arch = "aarch64")] +fn emit_mov_imm64(page: &mut [u8], off: usize, rd: u32, val: u64) -> usize { + // First non-trivial half goes via MOVZ, the rest via MOVK. Always emit at + // least one instruction (MOVZ #0) so x0 is defined. + let halves = [ + (val & 0xFFFF) as u16, + ((val >> 16) & 0xFFFF) as u16, + ((val >> 32) & 0xFFFF) as u16, + ((val >> 48) & 0xFFFF) as u16, + ]; + let mut n = 0usize; + let mut first = true; + for (i, &h) in halves.iter().enumerate() { + if h == 0 && !(first && i == 3) { continue; } + let insn = if first { a64::movz(rd, h, i as u32) } else { a64::movk(rd, h, i as u32) }; + emit32(page, off + n * 4, insn); + n += 1; + first = false; + } + if n == 0 { + emit32(page, off, a64::movz(rd, 0, 0)); + n = 1; + } + n +} + +#[cfg(target_arch = "aarch64")] +fn encode_stub(page: &mut [u8], off: usize, kind: StubKind) { + // The kernel's aarch64 SVC path reads the syscall number from x8 and args + // from x0..x5 (AAPCS64 already places the C args there). So a "syscall stub" + // is simply: movz x8,#nr ; svc #0 ; ret. 16-byte slots hold 4 instructions. + match kind { + StubKind::Syscall(nr) => { + // nr < 0x10000 for every entry → a single MOVZ suffices. + emit32(page, off, a64::movz(8, nr as u16, 0)); + emit32(page, off + 4, a64::svc0()); + emit32(page, off + 8, a64::ret()); + emit32(page, off + 12, a64::ret()); // pad (harmless) + } + StubKind::FloatZero => { + emit32(page, off, a64::fmov_d_zero(0)); // d0 = 0.0 + emit32(page, off + 4, a64::ret()); + } + StubKind::RetAddr(va) => { + let n = emit_mov_imm64(page, off, 0, va); // x0 = va + emit32(page, off + n * 4, a64::ret()); + } + StubKind::RetU32(v) => { + // w0 = v (32-bit). Up to two halves. + emit32(page, off, a64::movz_w(0, (v & 0xFFFF) as u16, 0)); + let mut nxt = off + 4; + if (v >> 16) != 0 { + emit32(page, nxt, a64::movk_w(0, ((v >> 16) & 0xFFFF) as u16, 1)); + nxt += 4; + } + emit32(page, nxt, a64::ret()); + } + StubKind::SyscallRetAsArg0(nr) => { + // x0 already holds the caller's return value (the "arg0"); issue the + // syscall and trap if it returns (noreturn). + emit32(page, off, a64::movz(8, nr as u16, 0)); + emit32(page, off + 4, a64::svc0()); + emit32(page, off + 8, a64::brk0()); + } + StubKind::MathSyscall1(nr) => { + // d0 -> x0 ; svc nr ; x0 -> d0 ; ret + emit32(page, off, a64::fmov_x_d(0, 0)); + emit32(page, off + 4, a64::movz(8, nr as u16, 0)); + emit32(page, off + 8, a64::svc0()); + emit32(page, off + 12, a64::fmov_d_x(0, 0)); + emit32(page, off + 16, a64::ret()); + } + StubKind::MathSyscall2(nr) => { + emit32(page, off, a64::fmov_x_d(0, 0)); // x0 = d0 + emit32(page, off + 4, a64::fmov_x_d(1, 1)); // x1 = d1 + emit32(page, off + 8, a64::movz(8, nr as u16, 0)); + emit32(page, off + 12, a64::svc0()); + emit32(page, off + 16, a64::fmov_d_x(0, 0)); // d0 = x0 + emit32(page, off + 20, a64::ret()); + } + StubKind::MathSyscall3(nr) => { + emit32(page, off, a64::fmov_x_d(0, 0)); // x0 = d0 + emit32(page, off + 4, a64::fmov_x_d(1, 1)); // x1 = d1 + emit32(page, off + 8, a64::fmov_x_d(2, 2)); // x2 = d2 + emit32(page, off + 12, a64::movz(8, nr as u16, 0)); + emit32(page, off + 16, a64::svc0()); + emit32(page, off + 20, a64::fmov_d_x(0, 0)); // d0 = x0 + emit32(page, off + 24, a64::ret()); + } + StubKind::Qsort => { + // Route to the kernel qsort syscall (0x3CB) like a normal stub rather + // than inlining x86 machine code. The kernel handler does the sort. + emit32(page, off, a64::movz(8, 0x3CB, 0)); + emit32(page, off + 4, a64::svc0()); + emit32(page, off + 8, a64::ret()); + } + StubKind::Setjmp => { + // Minimal setjmp: store the caller-saved state needed to resume. + // For bring-up we provide a non-unwinding stub: zero x0 and return + // (engine setjmp use is limited; full impl can follow if exercised). + emit32(page, off, a64::movz(0, 0, 0)); // x0 = 0 (first return) + emit32(page, off + 4, a64::ret()); + } + StubKind::Longjmp => { + // Minimal longjmp stub: return the value (x1) — cannot truly unwind + // without a real jmp_buf, but avoids illegal-instruction faults. + emit32(page, off, a64::ret()); + } + StubKind::Padding => {} + } +} + +#[cfg(target_arch = "x86_64")] fn encode_stub(page: &mut [u8], off: usize, kind: StubKind) { match kind { StubKind::Syscall(nr) => { diff --git a/kernel/src/sched/mod.rs b/kernel/src/sched/mod.rs index e04ac6b..ccfb8c7 100644 --- a/kernel/src/sched/mod.rs +++ b/kernel/src/sched/mod.rs @@ -144,6 +144,7 @@ pub fn spawn_kernel_task(name: &'static str, f: fn()) -> u64 { // [r13 = 0] // [r14 = 0] // [r15 = 0] ← kernel_sp points here + #[cfg(not(target_arch = "aarch64"))] let kernel_sp = unsafe { let stack_top = stack_base.add(STACK_SIZE) as *mut u64; let mut sp = stack_top; @@ -158,6 +159,30 @@ pub fn spawn_kernel_task(name: &'static str, f: fn()) -> u64 { sp as u64 }; + // aarch64 uses a different context_switch/task_entry ABI (AAPCS64 callee-saved + // pairs x19..x30, then `ret` to x30; task_entry pops the fn ptr via + // `ldr x9,[sp],#16` then `blr x9`). Build a matching first-run frame. + // + // Layout (low → high address): + // sp+0 x19 sp+8 x20 + // sp+16 x21 sp+24 x22 + // sp+32 x23 sp+40 x24 + // sp+48 x25 sp+56 x26 + // sp+64 x27 sp+72 x28 + // sp+80 x29 sp+88 x30 ← x30 = task_entry (context_switch `ret`s here) + // sp+96 fn ptr ← read by task_entry's `ldr x9,[sp],#16` + #[cfg(target_arch = "aarch64")] + let kernel_sp = unsafe { + let stack_top = stack_base.add(STACK_SIZE) as *mut u64; + // Reserve the fn-ptr slot at the very top, then the 12-word save area. + let fn_slot = stack_top.sub(1); + fn_slot.write(f as u64); + let frame = fn_slot.sub(12); // 12 u64 = x19..x30 + for i in 0..12 { frame.add(i).write(0); } + frame.add(11).write(crate::arch::task_entry as unsafe extern "C" fn() as u64); // x30 + frame as u64 + }; + let task = Task { pid, state: TaskState::Ready, priority: 128, name, kernel_sp, stack_base }; let mut s = SCHED.lock(); diff --git a/kernel/src/syscall/dispatch.rs b/kernel/src/syscall/dispatch.rs index 1487d17..587124b 100644 --- a/kernel/src/syscall/dispatch.rs +++ b/kernel/src/syscall/dispatch.rs @@ -27,11 +27,19 @@ pub extern "C" fn dispatch_fast(number: u64, arg0: u64, arg1: u64, arg2: u64, ar } // Eagerly snapshot this thread's user GPRs NOW, while the per-CPU entry - // snapshot (gs:[..]) is still fresh for us — before any handler runs, yields, - // or opens an interrupt window in which another thread's syscall entry could - // clobber the shared per-CPU snapshot. This is what makes a thread that - // yields inside a syscall (e.g. epoll_wait) resume with its OWN callee-saved - // registers intact, instead of leaking another thread's rbx/rbp/r12-15. + // snapshot is still fresh for us — before any handler runs, yields, or opens + // an interrupt window in which another thread's syscall entry could clobber + // the shared per-CPU snapshot. This is what makes a thread that yields inside + // a syscall (e.g. epoll_wait) resume with its OWN callee-saved registers + // intact, instead of leaking another thread's rbx/rbp/r12-15 (or, on + // aarch64, x30 → `ret` to a stale address). + // + // x86 reaches here with interrupts still masked (the SYSCALL entry stub + // hasn't sti'd yet), so the snapshot is fresh. aarch64 UNMASKS IRQs in the + // vector dispatch BEFORE the handler runs, so an unmasked capture here could + // already be clobbered — the aarch64 eager capture is therefore done earlier, + // inside the IRQ-masked SVC window in `vectors.rs`. Don't redo it here. + #[cfg(not(target_arch = "aarch64"))] { let cur = crate::process::current_pid(); if cur != 0 { @@ -715,7 +723,7 @@ pub extern "C" fn dispatch_fast(number: u64, arg0: u64, arg1: u64, arg2: u64, ar if buf == 0 || len == 0 { return 0; } // Best-effort: cap at 256 MiB to avoid runaway. let n = len.min(256 * 1024 * 1024); - let mut s: u64 = unsafe { core::arch::x86_64::_rdtsc() } ^ 0xA5A5_F00D_DEAD_BEEFu64; + let mut s: u64 = crate::arch::rdtsc() ^ 0xA5A5_F00D_DEAD_BEEFu64; unsafe { let p = buf as *mut u8; let mut i = 0usize; diff --git a/kernel/src/syscall/handlers/engine.rs b/kernel/src/syscall/handlers/engine.rs index 4d80a27..d5e3c53 100644 --- a/kernel/src/syscall/handlers/engine.rs +++ b/kernel/src/syscall/handlers/engine.rs @@ -20,24 +20,35 @@ pub(crate) fn sys_dlopen(path_ptr: u64, path_len: u64, _flags: u64) -> i64 { Some(ctx) => ctx.pml4_phys, None => return -9, // EBADF }; - // Serve libflutter_engine.so from the Limine module if available. - let engine_guard; + // Serve libflutter_engine.so from the Limine module if available (x86 ships + // the 33 MB engine as a Limine boot module to keep it out of the initramfs). + // On aarch64 there is NO Limine — the engine travels INSIDE the initramfs at + // /system/lib/libflutter_engine.so — so when the module is absent we fall + // through to the normal VFS lookup instead of failing. + let engine_guard = if path.contains("libflutter_engine") { + Some(crate::FLUTTER_ENGINE_BYTES.lock()) + } else { + None + }; let elf: &[u8] = if path.contains("libflutter_engine") { - engine_guard = crate::FLUTTER_ENGINE_BYTES.lock(); - if let Some(data) = engine_guard.as_ref() { - log::info!("[dlopen] Serving libflutter_engine.so from Limine module ({} bytes)", data.len()); - data - } else { - log::warn!("[dlopen] libflutter_engine.so requested but not available as Limine module"); - return -2; // ENOENT + // Prefer the Limine module (x86); fall back to the initramfs VFS (aarch64). + match engine_guard.as_ref().and_then(|g| g.as_ref()) { + Some(data) => { + log::info!("[dlopen] Serving libflutter_engine.so from Limine module ({} bytes)", data.len()); + data + } + None => match crate::fs::lookup(path) { + Some(data) => { + log::info!("[dlopen] Serving libflutter_engine.so from initramfs ({} bytes)", data.len()); + data + } + None => { + log::warn!("[dlopen] libflutter_engine.so not in Limine module nor initramfs"); + return -2; // ENOENT + } + }, } } else { - drop({ - // Need to satisfy borrow checker — create a temporary guard that we immediately drop. - // We don't actually use it here. - let _unused = crate::FLUTTER_ENGINE_BYTES.lock(); - _unused - }); match crate::fs::lookup(path) { Some(data) => data, None => return -2, // ENOENT @@ -1359,12 +1370,12 @@ pub(crate) fn sys_sched_yield() -> i64 { } else if cur == 1 { // OnVsync posts work to runner threads — sleep briefly so they can run (SMP or IRQ wake). unsafe { - core::arch::asm!("sti; hlt; cli", options(nomem, nostack)); + { crate::arch::enable_and_halt(); } } } else { // No other runnable process — sleep until the next IRQ (timerfd, vsync…). unsafe { - core::arch::asm!("sti; hlt; cli", options(nomem, nostack)); + { crate::arch::enable_and_halt(); } } } } diff --git a/kernel/src/syscall/handlers/fd.rs b/kernel/src/syscall/handlers/fd.rs index ae9108a..72af8c2 100644 --- a/kernel/src/syscall/handlers/fd.rs +++ b/kernel/src/syscall/handlers/fd.rs @@ -278,6 +278,31 @@ pub(crate) fn sys_write(fd: u64, buf_ptr: u64, len: u64) -> i64 { }; if let Ok(s) = core::str::from_utf8(bytes) { crate::logger::early_print(s); + } else { + // DIAGNOSTIC: a non-UTF8 stdout write would silently vanish. Log its + // length + first bytes so we can see writes that aren't text. + log::warn!( + "[sys_write] fd={} non-utf8 len={} first={:?}", + fd, len, &bytes[..bytes.len().min(8)] + ); + #[cfg(target_arch = "aarch64")] + { + let root = crate::process::current_user_root(); + match crate::mm::paging::translate_user_page(root, buf_ptr) { + Some(pa) => { + let hh = (pa + crate::mm::frame_allocator::hhdm_offset()) as *const u8; + let b0 = unsafe { core::ptr::read_volatile(hh) }; + log::warn!( + "[sys_write/dbg] root={:#x} va={:#x} -> pa={:#x} hhdm_b0={:#x}", + root, buf_ptr, pa, b0 + ); + } + None => log::warn!( + "[sys_write/dbg] root={:#x} va={:#x} -> walk failed", + root, buf_ptr + ), + } + } } len as i64 } @@ -640,7 +665,7 @@ pub(crate) fn sys_exit(code: u64) -> i64 { } // No runnable user process left. loop { - unsafe { core::arch::asm!("sti; hlt; cli", options(nomem, nostack)); } + crate::arch::enable_and_halt(); } } @@ -768,8 +793,5 @@ pub(crate) fn sys_exec_wait(path_ptr: u64, path_len: u64) -> i64 { pub(crate) fn sys_poweroff() -> i64 { log::info!("[syscall] sys_poweroff requested — triggering ACPI S5 shutdown"); - #[cfg(target_arch = "x86_64")] crate::arch::acpi_shutdown(); - #[cfg(not(target_arch = "x86_64"))] - loop { unsafe { core::arch::asm!("hlt", options(nomem, nostack)); } } } diff --git a/kernel/src/syscall/handlers/futex.rs b/kernel/src/syscall/handlers/futex.rs index b8a8ea4..26cd412 100644 --- a/kernel/src/syscall/handlers/futex.rs +++ b/kernel/src/syscall/handlers/futex.rs @@ -486,13 +486,13 @@ pub(crate) fn sys_futex(uaddr: u64, op: u32, val: u32, sys_nr: u64) -> i64 { { #[cfg(target_arch = "x86_64")] unsafe { - core::arch::asm!("sti; hlt; cli", options(nomem, nostack)); + { crate::arch::enable_and_halt(); } } if let Some(next) = crate::process::next_runnable_pid(pid) { if next != pid { let urip = crate::arch::syscall::user_rip(); let ursp = crate::arch::syscall::user_rsp(); - crate::process::save_return_context(pid, urip - 2, ursp); + crate::process::save_return_context_reexec(pid, urip, ursp); crate::process::save_full_user_gprs(pid); crate::process::set_rax(pid, sys_nr); crate::process::save_xstate(pid); @@ -585,13 +585,13 @@ pub(crate) fn sys_futex(uaddr: u64, op: u32, val: u32, sys_nr: u64) -> i64 { { #[cfg(target_arch = "x86_64")] unsafe { - core::arch::asm!("sti; hlt; cli", options(nomem, nostack)); + { crate::arch::enable_and_halt(); } } if let Some(next) = crate::process::next_runnable_pid(pid) { if next != pid { let urip = crate::arch::syscall::user_rip(); let ursp = crate::arch::syscall::user_rsp(); - crate::process::save_return_context(pid, urip - 2, ursp); + crate::process::save_return_context_reexec(pid, urip, ursp); crate::process::save_full_user_gprs(pid); crate::process::set_rax(pid, sys_nr); crate::process::save_xstate(pid); @@ -649,6 +649,8 @@ pub(crate) fn sys_pty_ioctl(fd: u64, cmd: u64, arg: u64) -> i64 { /// writes `*out = tid`, returns `0` on success. pub(crate) fn sys_thread_create(arg0: u64, arg1: u64, arg2: u64, arg3: u64) -> i64 { let parent_pid = crate::process::current_pid(); + log::error!("[thread-create] ENTER pid={} a0={:#x} a1={:#x} a2={:#x} a3={:#x}", + parent_pid, arg0, arg1, arg2, arg3); if parent_pid == 0 { return -1; // EPERM } @@ -682,29 +684,43 @@ pub(crate) fn sys_thread_create(arg0: u64, arg1: u64, arg2: u64, arg3: u64) -> i unsafe { *(arg0 as *mut u64) = fs_base; } (0_i64, Some(tid)) } - Err(errno) => (-errno, None), + Err(errno) => { + log::error!("[thread-create] POSIX spawn FAILED entry={:#x} arg={:#x} stk={:#x} errno={}", + arg2, arg3, stack_size, errno); + (-errno, None) + } }; - log::trace!("[trace] sys_thread_create POSIX out={:#x} attr={:#x} entry={:#x} arg={:#x} stk={:#x} -> {}", + log::warn!("[trace] sys_thread_create POSIX out={:#x} attr={:#x} entry={:#x} arg={:#x} stk={:#x} -> {}", arg0, arg1, arg2, arg3, stack_size, r); if let Some(child_pid) = child_pid_opt { log::warn!("[thread-create] spawned child={} entry={:#x}", child_pid, arg2); - // Give the newborn thread one immediate slice. Without this, the - // creator can monopolize userspace and delay child startup long - // enough to starve wake-source setup (pipe/timer arming). - let parent = crate::process::current_pid(); - if parent != 0 && child_pid != parent { - let my_cpu = crate::arch::smp::this_cpu().cpu_id; - if crate::process::try_claim_cpu_for(child_pid, my_cpu) { - let urip = crate::arch::syscall::user_rip(); - let ursp = crate::arch::syscall::user_rsp(); - crate::process::save_return_context(parent, urip, ursp); - crate::process::save_full_user_gprs(parent); - crate::process::set_rax(parent, 0); - crate::process::save_xstate(parent); - crate::process::enter_user_by_pid_noreturn(child_pid); + // x86 gave the newborn an immediate slice by entering the child + // from inside the creator's pthread_create syscall (never returning; + // delivering r=0 on the creator's later cooperative resume). On + // aarch64 that nested enter-while-in-a-syscall corrupts the creator's + // resume so the *next* fml::Thread's pthread_create returns non-zero + // (thread.cc:80 FML_CHECK abort) — reproducible even with the x30/LR + // and 4-byte-svc-rewind fixes in place. Keep it OFF on aarch64: the + // child becomes runnable here and is picked up by normal cooperative + // scheduling / timer preemption the next time the creator yields. + #[cfg(not(target_arch = "aarch64"))] + { + let parent = crate::process::current_pid(); + if parent != 0 && child_pid != parent { + let my_cpu = crate::arch::smp::this_cpu().cpu_id; + if crate::process::try_claim_cpu_for(child_pid, my_cpu) { + let urip = crate::arch::syscall::user_rip(); + let ursp = crate::arch::syscall::user_rsp(); + crate::process::save_return_context(parent, urip, ursp); + crate::process::save_full_user_gprs(parent); + crate::process::set_rax(parent, 0); + crate::process::save_xstate(parent); + crate::process::enter_user_by_pid_noreturn(child_pid); + } } } + let _ = child_pid; } r } else { @@ -767,7 +783,7 @@ pub(crate) fn sys_thread_join(thread_handle: u64, retval: u64) -> i64 { if retval != 0 { unsafe { *(retval as *mut u64) = 0; } } return 0; } - unsafe { core::arch::asm!("pause", options(nomem, nostack, preserves_flags)); } + crate::arch::spin_pause(); } Err(_) => { if retval != 0 { unsafe { *(retval as *mut u64) = 0; } } diff --git a/kernel/src/syscall/handlers/ipc_display.rs b/kernel/src/syscall/handlers/ipc_display.rs index 202faba..b86c26b 100644 --- a/kernel/src/syscall/handlers/ipc_display.rs +++ b/kernel/src/syscall/handlers/ipc_display.rs @@ -609,7 +609,7 @@ pub(crate) fn sys_wm_event_wait(ev_ptr: u64, ev_len: u64, timeout_ms: u64) -> i6 // Save return context and registers so we can be resumed. let urip = crate::arch::syscall::user_rip(); let ursp = crate::arch::syscall::user_rsp(); - crate::process::save_return_context(cur, urip - 2, ursp); + crate::process::save_return_context_reexec(cur, urip, ursp); crate::process::save_full_user_gprs(cur); crate::process::set_rax(cur, eabi::SYS_WM_EVENT_WAIT); if block_log_n < 32 { @@ -669,7 +669,7 @@ pub(crate) fn sys_wm_event_wait(ev_ptr: u64, ev_len: u64, timeout_ms: u64) -> i6 while crate::process::is_blocked(cur) { #[cfg(target_arch = "x86_64")] unsafe { - core::arch::asm!("sti; hlt; cli", options(nomem, nostack)); + { crate::arch::enable_and_halt(); } } if !crate::process::is_blocked(cur) { diff --git a/kernel/src/syscall/mod.rs b/kernel/src/syscall/mod.rs index 65a413e..1b9bfcf 100644 --- a/kernel/src/syscall/mod.rs +++ b/kernel/src/syscall/mod.rs @@ -109,4 +109,4 @@ pub(crate) use state::{ FS_BOOTSTRAP_LOGGED, WM_WAITER_DEADLINE, WM_WAITER_PID, }; pub(crate) use trace::{POSTEXIT_TRACE_ACTIVE, POSTEXIT_TRACE_COUNT, POSTEXIT_TRACE_LIMIT}; -pub(crate) use wait::{futex_waiter_add, futex_waiter_for, futex_waiter_present, futex_waiter_remove}; +pub(crate) use wait::{futex_waiter_add, futex_waiter_for, futex_waiter_present, futex_waiter_remove, futex_waiter_remove_try}; diff --git a/kernel/src/syscall/poll.rs b/kernel/src/syscall/poll.rs index 9da0a3e..874a6b4 100644 --- a/kernel/src/syscall/poll.rs +++ b/kernel/src/syscall/poll.rs @@ -551,7 +551,11 @@ pub fn check_timerfds_and_wake_try() { exps }; for (pid, mutex, cond) in expired { - super::futex_waiter_remove(cond, pid); + // ISR-safe: never block on FUTEX_WAITERS here. An interrupted syscall + // (e.g. spawn_thread, a futex op) may hold it; a non-try lock would + // self-deadlock this single core with IRQs masked. On contention the + // removal is simply retried on the next timer tick. + super::futex_waiter_remove_try(cond, pid); static COND_EXPIRE_LOG: AtomicU32 = AtomicU32::new(0); let n = COND_EXPIRE_LOG.fetch_add(1, Ordering::Relaxed); if n < 32 { @@ -613,10 +617,7 @@ pub fn monotonic_ns() -> u64 { // 1_700_000_000 * 10^9 + tsc_ns // timerfd_settime with TFD_TIMER_ABSTIME passes this value, so our // "now" must use the same epoch or timers will never fire. - let lo: u32; - let hi: u32; - unsafe { core::arch::asm!("rdtsc", out("eax") lo, out("edx") hi, options(nomem, nostack)); } - let tsc_ns = (((hi as u64) << 32) | lo as u64) / 3; + let tsc_ns = crate::arch::rdtsc() / 3; tsc_ns.saturating_add(1_700_000_000u64 * 1_000_000_000u64) } @@ -1002,7 +1003,7 @@ pub(crate) fn sys_epoll_wait_real(epfd: u64, events_out: u64, maxevents: u64, ti if crate::process::try_claim_cpu_for(target, my_cpu) { let urip = crate::arch::syscall::user_rip(); let ursp = crate::arch::syscall::user_rsp(); - crate::process::save_return_context(cur, urip.wrapping_sub(2), ursp); + crate::process::save_return_context_reexec(cur, urip, ursp); crate::process::save_full_user_gprs(cur); crate::process::set_rax(cur, 0x47B); // re-execute epoll_wait on resume crate::process::save_xstate(cur); @@ -1136,7 +1137,7 @@ pub(crate) fn sys_epoll_wait_real(epfd: u64, events_out: u64, maxevents: u64, ti if crate::process::try_claim_cpu_for(focus, my_cpu) { let urip = crate::arch::syscall::user_rip(); let ursp = crate::arch::syscall::user_rsp(); - crate::process::save_return_context(cur, urip.wrapping_sub(2), ursp); + crate::process::save_return_context_reexec(cur, urip, ursp); crate::process::save_full_user_gprs(cur); crate::process::set_rax(cur, 0x47B); // SYS epoll_wait (re-enter) crate::process::save_xstate(cur); @@ -1294,7 +1295,7 @@ pub(crate) fn sys_epoll_wait_real(epfd: u64, events_out: u64, maxevents: u64, ti // blocking again — exactly like sys_wm_event_wait does. let urip = crate::arch::syscall::user_rip(); let ursp = crate::arch::syscall::user_rsp(); - crate::process::save_return_context(cur, urip.wrapping_sub(2), ursp); + crate::process::save_return_context_reexec(cur, urip, ursp); crate::process::save_full_user_gprs(cur); crate::process::set_rax(cur, 0x47B); // SYS epoll_wait (re-enter) crate::process::save_xstate(cur); @@ -1320,7 +1321,7 @@ pub(crate) fn sys_epoll_wait_real(epfd: u64, events_out: u64, maxevents: u64, ti crate::process::set_state(1, crate::process::ProcState::Running); let urip = crate::arch::syscall::user_rip(); let ursp = crate::arch::syscall::user_rsp(); - crate::process::save_return_context(cur, urip.wrapping_sub(2), ursp); + crate::process::save_return_context_reexec(cur, urip, ursp); crate::process::save_full_user_gprs(cur); crate::process::set_rax(cur, 0x47B); crate::process::save_xstate(cur); @@ -1330,7 +1331,7 @@ pub(crate) fn sys_epoll_wait_real(epfd: u64, events_out: u64, maxevents: u64, ti #[cfg(target_arch = "x86_64")] unsafe { - core::arch::asm!("sti; hlt; cli", options(nomem, nostack)); + { crate::arch::enable_and_halt(); } } } } diff --git a/kernel/src/syscall/posix.rs b/kernel/src/syscall/posix.rs index 51f81c8..36a6f4d 100644 --- a/kernel/src/syscall/posix.rs +++ b/kernel/src/syscall/posix.rs @@ -952,7 +952,7 @@ pub fn sys_pthread_mutex_lock(mutex: u64, sys_nr: u64) -> i64 { if crate::process::try_claim_cpu_for(owner_candidate, my_cpu) { let urip = crate::arch::syscall::user_rip(); let ursp = crate::arch::syscall::user_rsp(); - crate::process::save_return_context(pid, urip - 2, ursp); + crate::process::save_return_context_reexec(pid, urip, ursp); crate::process::save_full_user_gprs(pid); crate::process::set_rax(pid, sys_nr); crate::process::save_xstate(pid); @@ -994,7 +994,7 @@ pub fn sys_pthread_mutex_lock(mutex: u64, sys_nr: u64) -> i64 { let urip = crate::arch::syscall::user_rip(); let ursp = crate::arch::syscall::user_rsp(); // Save context pointing to the syscall instruction itself so we retry on re-entry. - crate::process::save_return_context(pid, urip - 2, ursp); + crate::process::save_return_context_reexec(pid, urip, ursp); crate::process::save_full_user_gprs(pid); crate::process::set_rax(pid, sys_nr); // Restore the original syscall number crate::process::save_xstate(pid); @@ -1020,13 +1020,13 @@ pub fn sys_pthread_mutex_lock(mutex: u64, sys_nr: u64) -> i64 { while atom.load(Ordering::Acquire) != 0 && super::futex_waiter_present(mutex, pid) { #[cfg(target_arch = "x86_64")] unsafe { - core::arch::asm!("sti; hlt; cli", options(nomem, nostack)); + { crate::arch::enable_and_halt(); } } if let Some(next) = mutex_handoff_target() { if next != pid { let urip = crate::arch::syscall::user_rip(); let ursp = crate::arch::syscall::user_rsp(); - crate::process::save_return_context(pid, urip - 2, ursp); + crate::process::save_return_context_reexec(pid, urip, ursp); crate::process::save_full_user_gprs(pid); crate::process::set_rax(pid, sys_nr); crate::process::save_xstate(pid); @@ -1173,7 +1173,7 @@ pub fn sys_pthread_once(once: u64, func: u64, sys_nr: u64) -> i64 { let urip = crate::arch::syscall::user_rip(); let ursp = crate::arch::syscall::user_rsp(); // Save context pointing to the syscall instruction itself so we retry on re-entry. - crate::process::save_return_context(pid, urip - 2, ursp); + crate::process::save_return_context_reexec(pid, urip, ursp); crate::process::save_full_user_gprs(pid); crate::process::set_rax(pid, sys_nr); // sys_pthread_once syscall number crate::process::save_xstate(pid); @@ -1186,13 +1186,13 @@ pub fn sys_pthread_once(once: u64, func: u64, sys_nr: u64) -> i64 { while atom.load(Ordering::Acquire) != 2 && super::futex_waiter_present(once, pid) { #[cfg(target_arch = "x86_64")] unsafe { - core::arch::asm!("sti; hlt; cli", options(nomem, nostack)); + { crate::arch::enable_and_halt(); } } if let Some(next) = crate::process::next_runnable_pid(pid) { if next != pid { let urip = crate::arch::syscall::user_rip(); let ursp = crate::arch::syscall::user_rsp(); - crate::process::save_return_context(pid, urip - 2, ursp); + crate::process::save_return_context_reexec(pid, urip, ursp); crate::process::save_full_user_gprs(pid); crate::process::set_rax(pid, sys_nr); crate::process::save_xstate(pid); @@ -1320,7 +1320,7 @@ pub fn sys_pthread_cond_wait_timeout(cond: u64, mutex: u64, timeout_ns: u64, sys if next != pid { let urip = crate::arch::syscall::user_rip(); let ursp = crate::arch::syscall::user_rsp(); - crate::process::save_return_context(pid, urip - 2, ursp); + crate::process::save_return_context_reexec(pid, urip, ursp); crate::process::save_full_user_gprs(pid); crate::process::set_rax(pid, sys_nr); // Restore the original syscall number crate::process::save_xstate(pid); @@ -1350,13 +1350,13 @@ pub fn sys_pthread_cond_wait_timeout(cond: u64, mutex: u64, timeout_ns: u64, sys } #[cfg(target_arch = "x86_64")] unsafe { - core::arch::asm!("sti; hlt; cli", options(nomem, nostack)); + { crate::arch::enable_and_halt(); } } if let Some(next) = super::cooperative_sched_target(pid) { if next != pid { let urip = crate::arch::syscall::user_rip(); let ursp = crate::arch::syscall::user_rsp(); - crate::process::save_return_context(pid, urip - 2, ursp); + crate::process::save_return_context_reexec(pid, urip, ursp); crate::process::save_full_user_gprs(pid); crate::process::set_rax(pid, sys_nr); crate::process::save_xstate(pid); @@ -1979,9 +1979,7 @@ pub fn sys_sem_wait(sem: u64) -> i64 { { break; } - unsafe { - core::arch::asm!("pause", options(nomem, nostack, preserves_flags)); - } + crate::arch::spin_pause(); } 0 } @@ -2121,9 +2119,7 @@ pub fn sys_abort() -> ! { super::dump_user_backtrace(16); super::sys_exit((-6i64) as u64); loop { - unsafe { - core::arch::asm!("sti; hlt; cli", options(nomem, nostack)); - } + crate::arch::enable_and_halt(); } } @@ -2160,21 +2156,14 @@ pub fn sys_nanosleep(req: u64, _rem: u64) -> i64 { let secs = unsafe { *(req as *const u64) }; // Yield for approximately secs * 1000 iterations (very rough). for _ in 0..(secs * 1000).min(10000) { - unsafe { - core::arch::asm!("pause", options(nomem, nostack, preserves_flags)); - } + crate::arch::spin_pause(); } 0 } /// Fake monotonic time: use TSC / 1000 as nanoseconds. fn read_tsc() -> u64 { - let lo: u32; - let hi: u32; - unsafe { - core::arch::asm!("rdtsc", out("eax") lo, out("edx") hi, options(nomem, nostack)); - } - ((hi as u64) << 32) | lo as u64 + crate::arch::rdtsc() } pub fn sys_gettimeofday(tv: u64, _tz: u64) -> i64 { diff --git a/kernel/src/syscall/wait.rs b/kernel/src/syscall/wait.rs index b84b446..8a8baa3 100644 --- a/kernel/src/syscall/wait.rs +++ b/kernel/src/syscall/wait.rs @@ -19,6 +19,26 @@ pub(crate) fn futex_waiter_remove(addr: u64, pid: u32) { } } +/// ISR-safe variant of [`futex_waiter_remove`]. The generic-timer ISR drives +/// cond-wait timeout expiry (poll::check_timerfds_and_wake_try) and must never +/// block on a lock an interrupted syscall already holds — on a single core that +/// is an instant, IRQ-masked self-deadlock. Returns `false` on contention so the +/// caller can retry on the next tick rather than spin. +pub(crate) fn futex_waiter_remove_try(addr: u64, pid: u32) -> bool { + match FUTEX_WAITERS.try_lock() { + Some(mut table) => { + if let Some(waiters) = table.get_mut(&addr) { + waiters.retain(|&waiter| waiter != pid); + if waiters.is_empty() { + table.remove(&addr); + } + } + true + } + None => false, + } +} + pub(crate) fn futex_waiter_present(addr: u64, pid: u32) -> bool { let table = FUTEX_WAITERS.lock(); table.get(&addr).map_or(false, |waiters| waiters.contains(&pid)) diff --git a/scripts/build-aarch64-shell.sh b/scripts/build-aarch64-shell.sh new file mode 100755 index 0000000..15bfd2a --- /dev/null +++ b/scripts/build-aarch64-shell.sh @@ -0,0 +1,101 @@ +#!/usr/bin/env bash +# Assemble + build the OSCortex aarch64 Flutter shell. +# +# This is the ARM counterpart to scripts/build-iso.sh (which targets x86 + Limine). +# On -M virt there is no Limine; the kernel boots via `-kernel` and everything the +# Flutter shell host needs is delivered through the initramfs (assembled into the +# kernel by kernel/build.rs from the `initramfs/` directory): +# +# /init → the aarch64 Flutter embedder (oscortex-host) +# /system/lib/libflutter_engine.so → the aarch64 Flutter engine (AOT) +# /system/lib/liboscortex_libc.so → aarch64 libc/math helpers +# /system/flutter/libapp.so → the aarch64 AOT shell snapshot +# /system/flutter/icudtl.dat → ICU data +# /system/flutter/flutter_assets/ → fonts + manifests (arch-neutral) +# +# The aarch64 engine .so + gen_snapshot live in the `oscx-engine` Docker image +# (built by the engine port); the AOT shell snapshot (libapp_arm64.so) is produced +# from the app's AOT dill by the version-matched arm64 gen_snapshot. We stage the +# already-built artifacts here. +# +# Usage: scripts/build-aarch64-shell.sh +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$ROOT" + +TARGET=aarch64-unknown-none +EMBEDDER_DIR="$ROOT/tools/flutter-embedder" +CONTAINER=oscx-engine +ARM_OUT=/work/engine/engine/src/out/oscortex_arm64_release + +# Where the x86 build already prepared the arch-neutral shell assets (fonts + +# manifests). We reuse those rather than re-running the Flutter toolchain. +X86_ASSETS_SRC="${X86_ASSETS_SRC:-$ROOT/initramfs/system/flutter/flutter_assets}" + +echo "[arm-shell] 1/6 building the aarch64 embedder host (oscortex-host)..." +( + cd "$EMBEDDER_DIR" + cargo +nightly build --release --target "$TARGET" \ + -Z build-std=core,compiler_builtins \ + -Z build-std-features=compiler-builtins-mem +) +HOST_BIN="$EMBEDDER_DIR/target/$TARGET/release/oscortex-host" +[ -f "$HOST_BIN" ] || { echo "ERROR: $HOST_BIN not built" >&2; exit 1; } + +echo "[arm-shell] 2/6 staging the embedder as /init + /bin/oscortex-host..." +mkdir -p "$ROOT/initramfs/bin" "$ROOT/initramfs/system/lib" "$ROOT/initramfs/system/flutter/flutter_assets" +cp "$HOST_BIN" "$ROOT/initramfs/bin/oscortex-host" +cp "$HOST_BIN" "$ROOT/initramfs/init" + +echo "[arm-shell] 3/6 staging the aarch64 Flutter engine + ICU..." +docker cp "$CONTAINER:$ARM_OUT/libflutter_engine.so" "$ROOT/initramfs/system/lib/libflutter_engine.so" +docker cp "$CONTAINER:$ARM_OUT/icudtl.dat" "$ROOT/initramfs/system/flutter/icudtl.dat" + +echo "[arm-shell] 4/6 staging the aarch64 AOT shell snapshot (libapp.so)..." +# Produced by the arm64 gen_snapshot from the app AOT dill. Prefer the staged +# /work/libapp_arm64.so; fall back to compiling it from /work/app_aot.dill. +if docker exec "$CONTAINER" test -f /work/libapp_arm64.so; then + docker cp "$CONTAINER:/work/libapp_arm64.so" "$ROOT/initramfs/system/flutter/libapp.so" +else + echo "[arm-shell] /work/libapp_arm64.so absent — compiling from /work/app_aot.dill" + docker exec "$CONTAINER" "$ARM_OUT/gen_snapshot" \ + --snapshot_kind=app-aot-elf --elf=/work/libapp_arm64.so --strip /work/app_aot.dill + docker cp "$CONTAINER:/work/libapp_arm64.so" "$ROOT/initramfs/system/flutter/libapp.so" +fi + +echo "[arm-shell] 5/6 building the aarch64 libc helper + staging assets..." +docker run --rm --platform linux/arm64 \ + -v "$ROOT:$ROOT" -w "$ROOT" \ + gcc:12 \ + gcc -shared -fPIC -ffreestanding -nostdlib -O2 \ + -o "$ROOT/initramfs/system/lib/liboscortex_libc.so" \ + "$ROOT/userspace/libc/libc.c" + +# Arch-neutral shell assets (fonts, FontManifest, AssetManifest). Under AOT we do +# NOT ship kernel_blob.bin (the libapp.so snapshot carries the program). If the +# x86 build already prepared them, reuse; otherwise the embedder still boots with +# just the engine-default fonts (text may be blank until fonts land). +if [ -d "$X86_ASSETS_SRC" ] && [ "$X86_ASSETS_SRC" != "$ROOT/initramfs/system/flutter/flutter_assets" ]; then + cp -R "$X86_ASSETS_SRC/." "$ROOT/initramfs/system/flutter/flutter_assets/" +fi +# Remove any JIT-only blobs that may have been copied in (AOT carries the program). +rm -f "$ROOT/initramfs/system/flutter/flutter_assets/kernel_blob.bin" \ + "$ROOT/initramfs/system/flutter/flutter_assets/vm_snapshot_data" \ + "$ROOT/initramfs/system/flutter/flutter_assets/isolate_snapshot_data" + +echo "[arm-shell] 6/6 building the aarch64 kernel (embeds the initramfs)..." +# Force a rebuild of the initramfs (kernel/build.rs reads the initramfs/ dir). +touch "$ROOT/kernel/src/fs/initramfs.rs" 2>/dev/null || true +cargo build --target "$TARGET" -p oscortex-kernel \ + --no-default-features --features arch-aarch64 \ + -Z build-std=core,compiler_builtins,alloc \ + -Z build-std-features=compiler-builtins-mem + +echo "[arm-shell] done. Boot with: scripts/run-aarch64.sh" +echo "[arm-shell] staged initramfs contents:" +ls -la "$ROOT/initramfs/init" \ + "$ROOT/initramfs/system/lib/libflutter_engine.so" \ + "$ROOT/initramfs/system/lib/liboscortex_libc.so" \ + "$ROOT/initramfs/system/flutter/libapp.so" \ + "$ROOT/initramfs/system/flutter/icudtl.dat" 2>/dev/null diff --git a/scripts/run-aarch64.sh b/scripts/run-aarch64.sh new file mode 100755 index 0000000..fa478e9 --- /dev/null +++ b/scripts/run-aarch64.sh @@ -0,0 +1,64 @@ +#!/usr/bin/env bash +# Build + boot the OSCortex aarch64 kernel directly under qemu-system-aarch64. +# +# Direct `-kernel` boot (no Limine — that is the x86_64 boot protocol). The ARM +# `_start` (kernel/src/arch/aarch64/boot.rs) saves the device-tree pointer from +# x0 and routes into the PRODUCTION boot path (boot_prod.rs): PL011 serial → MMU +# → EL1 exception vectors → GIC → device-tree RAM discovery → the SHARED +# `kernel_main`. That runs the real subsystem init (heap, scheduler, VFS, process +# table, Cortex runtime, …) and spawns the `/init` process at EL0, whose syscalls +# are serviced through the shared dispatcher — all over PL011 serial. +# +# The self-contained platform-primitive bring-up (EL1/MMU/vectors/GIC/timer/SVC/ +# EL0 self-test, kernel/src/arch/aarch64/bringup*.rs) is still present and was the +# proving ground for these primitives; the production path above now reuses them. +# +# Usage: +# scripts/run-aarch64.sh # single-core boot +# scripts/run-aarch64.sh smp2 # 2 cores (exercises the PSCI CPU_ON path) +set -euo pipefail + +cd "$(dirname "$0")/.." + +TARGET=aarch64-unknown-none +KERNEL=target/$TARGET/debug/kernel + +echo "[run-aarch64] building kernel for $TARGET ..." +cargo build --target "$TARGET" -p oscortex-kernel \ + --no-default-features --features arch-aarch64 \ + -Z build-std=core,compiler_builtins,alloc \ + -Z build-std-features=compiler-builtins-mem + +SMP=1 +DISPLAY_MODE="${DISPLAY_MODE:-none}" # none | gtk | cocoa | sdl +for arg in "$@"; do + case "$arg" in + smp2) SMP=2 ;; + gtk|cocoa|sdl) DISPLAY_MODE="$arg" ;; + graphical) DISPLAY_MODE="cocoa" ;; # macOS default backend + esac +done + +# `-device ramfb`: QEMU's software framebuffer, configured by the guest through +# fw_cfg (kernel/src/arch/aarch64/ramfb.rs). With DISPLAY_MODE=none the kernel +# still drives the buffer (capture it via the QEMU monitor `screendump`); a real +# display backend (cocoa/gtk/sdl) shows it on screen. +COMMON=( + -M virt -cpu cortex-a72 -smp "$SMP" -m 2G + -kernel "$KERNEL" + -device ramfb +) + +if [[ "$DISPLAY_MODE" == "none" ]]; then + echo "[run-aarch64] booting (smp=$SMP, headless ramfb) — Ctrl-A X to quit ..." + echo "[run-aarch64] monitor: /tmp/qemu-arm-monitor.sock (screendump to capture the fb)" + exec qemu-system-aarch64 "${COMMON[@]}" \ + -display none \ + -serial mon:stdio \ + -monitor unix:/tmp/qemu-arm-monitor.sock,server,nowait +else + echo "[run-aarch64] booting (smp=$SMP, display=$DISPLAY_MODE) — close window or Ctrl-A X to quit ..." + exec qemu-system-aarch64 "${COMMON[@]}" \ + -display "$DISPLAY_MODE" \ + -serial mon:stdio +fi diff --git a/tools/flutter-embedder/.cargo/config.toml b/tools/flutter-embedder/.cargo/config.toml index b5e1c24..9182993 100644 --- a/tools/flutter-embedder/.cargo/config.toml +++ b/tools/flutter-embedder/.cargo/config.toml @@ -13,3 +13,16 @@ rustflags = [ "-C", "code-model=small", ] +[target.aarch64-unknown-none] +# Same rationale as x86_64: the kernel ELF loader maps the embedder at its +# link-time VA but does not run a userspace `ld.so`, so it never populates the +# GOT/PLT. Static relocation avoids GOT-indirect references that would +# dereference an unpopulated (zero) GOT slot at runtime. The aarch64-unknown-none +# spec is already non-PIC, but we set it explicitly to keep the contract obvious +# and match the x86 path. `code-model=small` keeps the binary in the low VA +# window the loader maps at 0x400000. +rustflags = [ + "-C", "relocation-model=static", + "-C", "code-model=small", +] + diff --git a/tools/flutter-embedder/build.rs b/tools/flutter-embedder/build.rs index 07466c6..477802b 100644 --- a/tools/flutter-embedder/build.rs +++ b/tools/flutter-embedder/build.rs @@ -3,9 +3,17 @@ // at USER_ELF_BASE (0x400000) in each process's address space. fn main() { let dir = std::env::var("CARGO_MANIFEST_DIR").unwrap(); - println!("cargo:rustc-link-arg=-T{dir}/user.ld"); + // The linker script is architecture-specific (OUTPUT_FORMAT/OUTPUT_ARCH + // differ), so select it from the build target arch. + let arch = std::env::var("CARGO_CFG_TARGET_ARCH").unwrap_or_default(); + let ld = match arch.as_str() { + "aarch64" => "user_aarch64.ld", + _ => "user.ld", // x86_64 / default + }; + println!("cargo:rustc-link-arg=-T{dir}/{ld}"); println!("cargo:rustc-link-arg=-z"); println!("cargo:rustc-link-arg=noexecstack"); println!("cargo:rerun-if-changed=user.ld"); + println!("cargo:rerun-if-changed=user_aarch64.ld"); println!("cargo:rerun-if-changed=build.rs"); } diff --git a/tools/flutter-embedder/src/main.rs b/tools/flutter-embedder/src/main.rs index e50f1cd..4d828e7 100644 --- a/tools/flutter-embedder/src/main.rs +++ b/tools/flutter-embedder/src/main.rs @@ -27,6 +27,7 @@ use sys::*; /// Bare-metal userspace entry. The kernel sets up RBP=0 and RSP pointing to /// a fresh user stack before jumping here. +#[cfg(target_arch = "x86_64")] #[unsafe(no_mangle)] #[unsafe(naked)] unsafe extern "C" fn _start() -> ! { @@ -73,6 +74,52 @@ unsafe extern "C" fn _start() -> ! { } } +/// aarch64 userspace entry. The kernel enters EL0 with x0=host_mode, x1=app_id, +/// x2=aot_va (the SpawnBootstrap rdi/rsi/rdx mapped to x0/x1/x2 by +/// `enter_user.rs::build_image`) and SP pointing at a fresh 16-byte-aligned user +/// stack. We preserve x0..x2 across the breadcrumb write (callee-saved x19..x21 +/// survive the `svc`), then call `main_embedder(x0, x1, x2)`. +#[cfg(target_arch = "aarch64")] +#[unsafe(no_mangle)] +#[unsafe(naked)] +unsafe extern "C" fn _start() -> ! { + unsafe { + core::arch::naked_asm!( + // Zero the frame pointer (x29) — terminates backtraces cleanly. + "mov x29, #0", + "mov x30, #0", + // AAPCS64 requires SP to be 16-byte aligned. SP cannot be a direct + // operand of AND, so round it down via a scratch GPR (x9). + "mov x9, sp", + "and x9, x9, #-16", + "mov sp, x9", + // Preserve the bootstrap args (x0=host_mode, x1=app_id, x2=aot_va) + // across the breadcrumb write syscall, which clobbers x0..x2. + "mov x19, x0", + "mov x20, x1", + "mov x21, x2", + // Earliest userspace breadcrumb: write one '!' byte to fd 1. + "mov x8, #1", // SYS_WRITE + "mov x0, #1", // fd = 1 + "adr x1, 2f", // buf = label 2 + "mov x2, #1", // len = 1 + "svc #0", + // Restore the bootstrap args for main_embedder(x0, x1, x2). + "mov x0, x19", + "mov x1, x20", + "mov x2, x21", + "bl {main}", + // If main returns, exit(0). + "mov x8, #60", // SYS_EXIT + "mov x0, #0", + "svc #0", + "2:", + ".ascii \"!\"", + main = sym main_embedder, + ); + } +} + // ── Flutter ABI types (repr(C), matches kernel/src/embedder/abi.rs) ────────── /// Kernel-registered proc-table of resolved Flutter engine fn pointers. @@ -2227,27 +2274,16 @@ const ENGINE_LIB_PATH: &[u8] = b"/system/lib/libflutter_engine.so"; const HOST_MODE_SHELL: u64 = 1; const HOST_MODE_APP: u64 = 2; -fn read_host_bootstrap() -> (u64, u64, u64) { - let rdi: u64; - let rsi: u64; - let rdx: u64; - unsafe { - asm!( - "mov {0}, rdi", - "mov {1}, rsi", - "mov {2}, rdx", - out(reg) rdi, - out(reg) rsi, - out(reg) rdx, - ); - } - (rdi, rsi, rdx) -} - // ── Main embedder logic ─────────────────────────────────────────────────────── -extern "C" fn main_embedder() { - let (host_mode, app_id, aot_va) = read_host_bootstrap(); +/// Entry called from `_start` with the kernel's bootstrap registers passed as +/// the first three C arguments (x86: rdi/rsi/rdx, aarch64: x0/x1/x2): +/// `host_mode` (1 = shell, 2 = launched app), `app_id`, `aot_va`. +extern "C" fn main_embedder(host_mode: u64, app_id: u64, aot_va: u64) { + // DIAGNOSTIC (aarch64 bring-up): emit a raw breadcrumb via the syscall + // wrapper so we can confirm main_embedder is entered and `write()` works + // against a .rodata buffer before the heavier init runs. + write(b"[host] main_embedder entered\n"); CURRENT_HOST_MODE.store(host_mode, Ordering::Release); CURRENT_APP_ID.store(app_id, Ordering::Release); write(b"[host] starting\n"); @@ -2474,10 +2510,16 @@ extern "C" fn main_embedder() { // layer (livelock). Strategy: make the heap large enough that NO GC fires during // the first frame — no safepoint, no livelock. (Do NOT use --marker_tasks=0 / // --scavenger_tasks=0: the debug VM asserts num_tasks>0 and SIGABRTs.) - static ARG8: &[u8] = - b"--dart-flags=--old_gen_heap_size=512,--new_gen_semi_max_size=64,--no_background_compilation\0"; + // NOTE: we deliberately do NOT pass a `--dart-flags=...` switch. The engine's + // SettingsFromCommandLine (shell/common/switches.cc) runs every comma-split + // entry through IsAllowedDartVMFlag against a tiny allowlist and FML_LOG(FATAL)s + // on anything else — `--old_gen_heap_size` / `--new_gen_semi_max_size` / + // `--no_background_compilation` are NOT on that list, so passing them via + // --dart-flags aborts the engine (switches.cc:478). The original GC-heap sizing + // was a workaround for an x86-TCG/JIT livelock that the later diagnosis traced + // to font fallback + slow JIT; under AOT on ARM we let the VM use its defaults. #[repr(transparent)] - struct ArgvPtrs([*const u8; 9]); + struct ArgvPtrs([*const u8; 8]); unsafe impl Sync for ArgvPtrs {} static ENGINE_ARGV: ArgvPtrs = ArgvPtrs([ @@ -2489,7 +2531,6 @@ extern "C" fn main_embedder() { ARG5.as_ptr(), ARG6.as_ptr(), ARG7.as_ptr(), - ARG8.as_ptr(), ]); let mut project_args = FlutterProjectArgsRaw { @@ -2611,11 +2652,17 @@ extern "C" fn main_embedder() { } } - // Save main thread RSP so our task runner callback can detect if we are on the platform thread. - let mut rsp: u64; + // Save the platform-thread stack pointer so the task-runner callback can + // detect whether it is running on the platform thread. + let rsp: u64; + #[cfg(target_arch = "x86_64")] unsafe { asm!("mov {}, rsp", out(reg) rsp); } + #[cfg(target_arch = "aarch64")] + unsafe { + asm!("mov {}, sp", out(reg) rsp); + } PLATFORM_THREAD_RSP.store(rsp, Ordering::SeqCst); // Install Custom Task Runners in project args. @@ -3348,10 +3395,14 @@ fn write_hex_u64(label: &[u8], val: u64) { write(&buf); } -/// Read the x86 TSC and return an approximate nanosecond timestamp. -/// Uses a nominal 3 GHz frequency (Phase 33-A keeps the kernel calibrated). -/// Must match sys_clock_gettime(CLOCK_MONOTONIC) exactly by adding the Unix epoch -/// offset used by the kernel (1,700,000,000 seconds). +/// Read a monotonic nanosecond timestamp from the CPU cycle/tick counter. +/// +/// x86: the TSC at a nominal 3 GHz. aarch64: the architected generic timer +/// virtual count (`CNTVCT_EL0`) scaled by its frequency (`CNTFRQ_EL0`, typically +/// 62.5 MHz under QEMU). Both add the same Unix-epoch offset the kernel's +/// `sys_clock_gettime(CLOCK_MONOTONIC)` uses (1,700,000,000 s) so the engine's +/// timeline matches the kernel's. +#[cfg(target_arch = "x86_64")] #[inline(always)] fn rdtsc_ns() -> u64 { let lo: u32; @@ -3369,6 +3420,22 @@ fn rdtsc_ns() -> u64 { tsc_ns.saturating_add(1_700_000_000u64 * 1_000_000_000u64) } +#[cfg(target_arch = "aarch64")] +#[inline(always)] +fn rdtsc_ns() -> u64 { + let cnt: u64; + let frq: u64; + unsafe { + core::arch::asm!("mrs {}, cntvct_el0", out(reg) cnt, options(nomem, nostack)); + core::arch::asm!("mrs {}, cntfrq_el0", out(reg) frq, options(nomem, nostack)); + } + // ns = count * 1e9 / freq. Guard against a zero frequency (shouldn't happen + // on QEMU virt, where CNTFRQ_EL0 is programmed). Use u128 to avoid overflow. + let freq = if frq == 0 { 62_500_000u64 } else { frq }; + let ns = (cnt as u128 * 1_000_000_000u128 / freq as u128) as u64; + ns.saturating_add(1_700_000_000u64 * 1_000_000_000u64) +} + fn write_hex(mut v: u64) { let mut buf = [0u8; 18]; buf[0] = b'0'; diff --git a/tools/flutter-embedder/src/sys.rs b/tools/flutter-embedder/src/sys.rs index e0b03dd..9417c29 100644 --- a/tools/flutter-embedder/src/sys.rs +++ b/tools/flutter-embedder/src/sys.rs @@ -6,6 +6,16 @@ //! rax = syscall number //! rdi = arg0, rsi = arg1, rdx = arg2, r10 = arg3 //! Return value in rax (negative = error). +//! +//! ABI (aarch64 Linux-compatible): +//! x8 = syscall number +//! x0 = arg0, x1 = arg1, x2 = arg2, x3 = arg3 +//! `svc #0`; return value in x0 (negative = error). +//! +//! Both backends expose the same `syscallN(nr, …)` surface so the typed wrappers +//! below are architecture-neutral. The kernel's aarch64 SVC handler +//! (`arch::aarch64::syscall::production_svc_handler`) reads x8 as the number and +//! x0..x4 as the dispatcher args, mirroring the x86 SYSCALL fast path. use core::arch::asm; @@ -128,6 +138,9 @@ pub const WM_EVENT_SIZE: usize = core::mem::size_of::(); // ── Raw syscall helpers ─────────────────────────────────────────────────────── +// ── x86_64 backend (SYSCALL: nr=rax, args rdi/rsi/rdx/r10) ──────────────────── + +#[cfg(target_arch = "x86_64")] #[inline(always)] pub unsafe fn syscall0(nr: u64) -> i64 { let ret: i64; @@ -143,6 +156,7 @@ pub unsafe fn syscall0(nr: u64) -> i64 { ret } +#[cfg(target_arch = "x86_64")] #[inline(always)] pub unsafe fn syscall1(nr: u64, a0: u64) -> i64 { let ret: i64; @@ -159,6 +173,7 @@ pub unsafe fn syscall1(nr: u64, a0: u64) -> i64 { ret } +#[cfg(target_arch = "x86_64")] #[inline(always)] pub unsafe fn syscall2(nr: u64, a0: u64, a1: u64) -> i64 { let ret: i64; @@ -176,6 +191,7 @@ pub unsafe fn syscall2(nr: u64, a0: u64, a1: u64) -> i64 { ret } +#[cfg(target_arch = "x86_64")] #[inline(always)] pub unsafe fn syscall3(nr: u64, a0: u64, a1: u64, a2: u64) -> i64 { let ret: i64; @@ -194,6 +210,7 @@ pub unsafe fn syscall3(nr: u64, a0: u64, a1: u64, a2: u64) -> i64 { ret } +#[cfg(target_arch = "x86_64")] #[inline(always)] pub unsafe fn syscall4(nr: u64, a0: u64, a1: u64, a2: u64, a3: u64) -> i64 { let ret: i64; @@ -213,6 +230,92 @@ pub unsafe fn syscall4(nr: u64, a0: u64, a1: u64, a2: u64, a3: u64) -> i64 { ret } +// ── aarch64 backend (SVC #0: nr=x8, args x0..x5) ────────────────────────────── +// +// The kernel SVC handler reads x8 as the syscall number and x0..x4 as the five +// dispatcher arguments, then writes the return value back into x0. We mark x8 as +// `in` (the kernel does not clobber it) and x0 as `inlateout` since it carries +// arg0 in and the return out. We do NOT use `options(nostack)` blanket here +// because the SVC trap saves/restores the full frame; `nomem` is unsafe to +// assume (the syscall may read/write user buffers passed by pointer), so we omit +// it to keep the compiler from reordering memory ops across the trap. + +#[cfg(target_arch = "aarch64")] +#[inline(always)] +pub unsafe fn syscall0(nr: u64) -> i64 { + let ret: i64; + unsafe { + asm!("svc #0", + in("x8") nr, + lateout("x0") ret, + options(nostack), + ); + } + ret +} + +#[cfg(target_arch = "aarch64")] +#[inline(always)] +pub unsafe fn syscall1(nr: u64, a0: u64) -> i64 { + let ret: i64; + unsafe { + asm!("svc #0", + in("x8") nr, + inlateout("x0") a0 => ret, + options(nostack), + ); + } + ret +} + +#[cfg(target_arch = "aarch64")] +#[inline(always)] +pub unsafe fn syscall2(nr: u64, a0: u64, a1: u64) -> i64 { + let ret: i64; + unsafe { + asm!("svc #0", + in("x8") nr, + inlateout("x0") a0 => ret, + in("x1") a1, + options(nostack), + ); + } + ret +} + +#[cfg(target_arch = "aarch64")] +#[inline(always)] +pub unsafe fn syscall3(nr: u64, a0: u64, a1: u64, a2: u64) -> i64 { + let ret: i64; + unsafe { + asm!("svc #0", + in("x8") nr, + inlateout("x0") a0 => ret, + in("x1") a1, + in("x2") a2, + options(nostack), + ); + } + ret +} + +#[cfg(target_arch = "aarch64")] +#[inline(always)] +pub unsafe fn syscall4(nr: u64, a0: u64, a1: u64, a2: u64, a3: u64) -> i64 { + let ret: i64; + unsafe { + asm!("svc #0", + in("x8") nr, + inlateout("x0") a0 => ret, + in("x1") a1, + in("x2") a2, + in("x3") a3, + options(nostack), + ); + } + ret +} + // ── Typed wrappers ──────────────────────────────────────────────────────────── /// Write `buf` to a kernel-side serial/debug output (fd 1 = stdout). diff --git a/tools/flutter-embedder/user_aarch64.ld b/tools/flutter-embedder/user_aarch64.ld new file mode 100644 index 0000000..c0d9206 --- /dev/null +++ b/tools/flutter-embedder/user_aarch64.ld @@ -0,0 +1,49 @@ +/* tools/flutter-embedder/user_aarch64.ld + * Userspace ELF linker script for OSCortex — aarch64. + * Load address: USER_ELF_BASE = 0x400000 (4 MiB), matching the kernel's + * process loader expectation (shared across architectures). + */ + +OUTPUT_FORMAT("elf64-littleaarch64") +OUTPUT_ARCH(aarch64) +ENTRY(_start) + +SECTIONS { + . = 0x400000; + + .text : { + KEEP(*(.text._start)) + *(.text .text.*) + *(.plt*) + } + + .rodata : { + *(.rodata .rodata.*) + } + + . = ALIGN(4096); + + .data : { + *(.data .data.*) + *(.got*) + } + + .bss : { + __bss_start = .; + *(.bss .bss.*) + *(COMMON) + __bss_end = .; + } + + /DISCARD/ : { + *(.comment*) + *(.note*) + *(.eh_frame*) + *(.gnu.hash) + *(.dynsym) + *(.dynstr) + *(.dynamic) + *(.rel*) + *(.rela*) + } +} diff --git a/userspace/libc/libc.c b/userspace/libc/libc.c index c335cb6..72bbe3e 100644 --- a/userspace/libc/libc.c +++ b/userspace/libc/libc.c @@ -79,6 +79,14 @@ char *strchr(const char *s, int c) { } // --- Mathematical functions --- +// +// On x86_64 the x87 FPU provides fsin/fcos/fsqrt directly. AArch64 has no +// transcendental instructions, so sin/cos use a range-reduced minimax-style +// polynomial and sqrt uses the FSQRT instruction via the compiler builtin. The +// engine only needs these for occasional layout/animation math, so a few ULP of +// error is fine. + +#if defined(__x86_64__) double sin(double x) { double result; @@ -116,6 +124,58 @@ float sqrtf(float x) { return result; } +#else /* aarch64 (and any non-x86 host) */ + +/* Reduce x into [-pi, pi] using a precise pi, then evaluate an 11th-order + * Taylor series — accurate to a few ULP across that range. */ +static double m_reduce(double x) { + const double TWO_PI = 6.283185307179586476925286766559; + const double PI = 3.141592653589793238462643383279; + /* k = round(x / 2pi) */ + double k = x / TWO_PI; + long long ki = (long long)(k >= 0 ? k + 0.5 : k - 0.5); + x -= (double)ki * TWO_PI; + if (x > PI) x -= TWO_PI; + if (x < -PI) x += TWO_PI; + return x; +} + +double sin(double x) { + x = m_reduce(x); + double x2 = x * x; + /* x - x^3/3! + x^5/5! - x^7/7! + x^9/9! - x^11/11! */ + double t = x; + double s = x; + t *= -x2 / (2*3); s += t; + t *= -x2 / (4*5); s += t; + t *= -x2 / (6*7); s += t; + t *= -x2 / (8*9); s += t; + t *= -x2 / (10*11); s += t; + return s; +} + +double cos(double x) { + x = m_reduce(x); + double x2 = x * x; + /* 1 - x^2/2! + x^4/4! - x^6/6! + x^8/8! - x^10/10! */ + double t = 1.0; + double s = 1.0; + t *= -x2 / (1*2); s += t; + t *= -x2 / (3*4); s += t; + t *= -x2 / (5*6); s += t; + t *= -x2 / (7*8); s += t; + t *= -x2 / (9*10); s += t; + return s; +} + +float sinf(float x) { return (float)sin((double)x); } +float cosf(float x) { return (float)cos((double)x); } + +double sqrt(double x) { return __builtin_sqrt(x); } +float sqrtf(float x) { return __builtin_sqrtf(x); } + +#endif + double floor(double x) { long long i = (long long)x; double d = (double)i;