diff --git a/issues/an-ap-loads-the-idt-before-its-control-registers.md b/issues/an-ap-loads-the-idt-before-its-control-registers.md index 3a2807162d4..6e37a6a06a4 100644 --- a/issues/an-ap-loads-the-idt-before-its-control-registers.md +++ b/issues/an-ap-loads-the-idt-before-its-control-registers.md @@ -16,7 +16,7 @@ ordering is stated at the site. trampoline's long-mode half sets `CR4.PAE` and nothing else, then sets the GS base, then `lidt`s the kernel's table — and `CR4.OSFXSR` is not set until `percpu::init_ap` reaches `control_regs::init`, several Rust statements later. -Between them an AP runs `control_regs::init_cr0`, `pat::init` (MSR writes and a +Between them an AP runs `control_regs::init_paging`, `pat::init` (MSR writes and a CR0 round trip) and `mm::paging::load_kernel_flush` (a CR3 reload) with a table loaded whose every stub is `#UD` on entry. Any exception in that window is a triple fault with nothing on any channel. diff --git a/issues/kernel-text-is-writable-and-every-kernel-page-executable.md b/issues/kernel-text-is-writable-and-every-kernel-page-executable.md deleted file mode 100644 index ca320451646..00000000000 --- a/issues/kernel-text-is-writable-and-every-kernel-page-executable.md +++ /dev/null @@ -1,23 +0,0 @@ ---- -status: open -kind: defect -opened: 2026-09-29 ---- - -# Kernel text is writable, and every kernel page is executable - -The kernel maps the whole direct map `PRESENT | WRITE` with no `NX` -(`kernel_leaf`, which `paging::init` maps every 2 MiB with, -`kernel/src/arch/x86_64/paging.rs`), and the kernel image runs from -it, at `PHYS_OFFSET` plus its physical address. Its text can be overwritten, -and its heap, its stacks and every user page's direct-map alias can be -executed at CPL 0. Linux at `Ubuntu-6.8.0-142.142`, under the T14's -`CONFIG_STRICT_KERNEL_RWX=y`, makes text and rodata read-only and everything -else non-executable (`arch/x86/mm/init_64.c:1402-1447`), and under -`CONFIG_DEBUG_WX=y` walks the tables for a writable and executable mapping at -boot (`arch/x86/mm/dump_pagetables.c:434`). - -**Exit**: kernel text is mapped read-only and executable, rodata read-only and -`NX`, and every other kernel mapping `NX`; each boot walks the kernel tables -and panics on a writable executable leaf, and mapping the direct map as it is -today reds that walk. diff --git a/issues/the-kernel-is-at-least-as-secure-as-linux-on-every-machine-toyos-supports.md b/issues/the-kernel-is-at-least-as-secure-as-linux-on-every-machine-toyos-supports.md index ad84b186430..9c42ec14874 100644 --- a/issues/the-kernel-is-at-least-as-secure-as-linux-on-every-machine-toyos-supports.md +++ b/issues/the-kernel-is-at-least-as-secure-as-linux-on-every-machine-toyos-supports.md @@ -37,7 +37,8 @@ work needs. - `issues/every-syscall-runs-at-one-kernel-stack-offset.md` - `issues/kernel-functions-return-with-their-used-registers-intact.md` - `issues/a-threads-kernel-stack-has-no-guard-page.md` -- `issues/kernel-text-is-writable-and-every-kernel-page-executable.md` +- `issues/the-kernels-relro-stays-writable.md` +- `issues/x86-64s-boot-map-is-writable-and-executable.md` - `issues/the-kernel-heap-has-none-of-slubs-hardening.md` - `issues/a-device-without-a-domain-of-its-own-reaches-all-memory.md` - `issues/user-programs-run-without-a-shadow-stack.md` diff --git a/issues/the-kernels-relro-stays-writable.md b/issues/the-kernels-relro-stays-writable.md new file mode 100644 index 00000000000..1c1622e3731 --- /dev/null +++ b/issues/the-kernels-relro-stays-writable.md @@ -0,0 +1,20 @@ +--- +status: open +kind: defect +opened: 2026-10-21 +--- + +# The kernel's RELRO stays writable + +The kernel maps its own image page by page from its `PT_LOAD` flags +(`kernel/src/mm/image.rs`), so its `PT_GNU_RELRO` — `.data.rel.ro`, `.got` and +`.dynamic`, about 70 KiB on x86-64 — is read-write and execute-never for the +machine's life. The loader applies the image's relocations before the kernel +runs, so nothing needs to write it afterwards, and a stray kernel write can +rewrite a vtable or a function pointer there. Linux at the parity tag of +`issues/the-kernel-is-at-least-as-secure-as-linux-on-every-machine-toyos-supports.md` +maps the same data read-only. `toyos-elf` parses no `PT_GNU_RELRO`, which +`issues/an-executables-relro-stays-writable.md` needs too. + +Exit: the kernel's `PT_GNU_RELRO` pages are read-only in its tables from +`mm::init` on, and a boot check or test write shows it. diff --git a/issues/x86-64s-boot-map-is-writable-and-executable.md b/issues/x86-64s-boot-map-is-writable-and-executable.md new file mode 100644 index 00000000000..843fc865f95 --- /dev/null +++ b/issues/x86-64s-boot-map-is-writable-and-executable.md @@ -0,0 +1,23 @@ +--- +status: open +kind: defect +opened: 2026-10-21 +--- + +# x86-64's boot map is writable and executable + +The loader's x86-64 boot map (`lib/bootmap/src/x86_64.rs`) maps the kernel's +text read-only and every other page writable, and every page executable: bit 63 +is execute-disable only under `IA32_EFER.NXE`, and the loader switches to the +map from firmware's state, before the kernel declares `NXE`. The boot CPU runs +the kernel on it until `mm::init`, and each AP from its trampoline, whose page +at 0x8000 the map holds writable, until `smp::ap_entry` loads the kernel's +tables. The boot's audit (`mm::audit_wx`) walks the kernel's tables, not this +map. AArch64's boot map is not writable and executable anywhere: the kernel's +entry declares `SCTLR_EL1.WXN` before it turns translation on under it, and the +loader maps everything but the text execute-never. + +Exit: as on AArch64, the kernel's entry applies its declared `EFER` before it +switches to the boot map; every leaf there but the text's and the trampoline +page's is execute-disable, and those two are read-only; and the plan's host +tests show it. diff --git a/kernel/pure/lib.rs b/kernel/pure/lib.rs index 83953977488..9f13369f627 100644 --- a/kernel/pure/lib.rs +++ b/kernel/pure/lib.rs @@ -23,4 +23,5 @@ pub mod pipe; pub mod pointers; pub mod proclife; pub mod sched; +pub mod wx; pub mod xsave; diff --git a/kernel/pure/wx.rs b/kernel/pure/wx.rs new file mode 100644 index 00000000000..ad94391a84d --- /dev/null +++ b/kernel/pure/wx.rs @@ -0,0 +1,332 @@ +//! The boot's audit of the kernel's own translation tables: no leaf is both +//! writable and executable, and none is executable outside the kernel's text. +//! +//! A leaf is judged by its own bits. A table entry above it can only narrow +//! what the leaf grants, and no narrowing is credited, so the audit can be +//! stricter than the hardware and never looser. + +#![forbid(unsafe_code)] + +use core::ops::Range; + +/// One entry as the audit reads it. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub enum Entry { + /// Translates nothing. + Absent, + /// Names the next table down, at this physical address. + Table(u64), + /// Maps all of its span. + Leaf { writable: bool, executable: bool }, +} + +/// A four-level format of 512 eight-byte entries per table: the root is level +/// 0 and translates bits 47:39, level 3 translates bits 20:12. +pub trait Format { + /// `entry`, found at `level`. + fn read(level: usize, entry: u64) -> Entry; + /// The virtual address whose bits 47:0 are `bits`. + fn address(bits: u64) -> u64; +} + +/// Four-level paging, Intel SDM Vol. 3A §4.5, Tables 4-15 to 4-20, with +/// `IA32_EFER.NXE` set: a supervisor write needs `R/W` (bit 1), and bit 63 +/// (`XD`) forbids a fetch. +pub struct X86_64; + +impl Format for X86_64 { + fn read(level: usize, entry: u64) -> Entry { + const PRESENT: u64 = 1 << 0; + const WRITABLE: u64 = 1 << 1; + /// A PDPTE's 1 GiB page or a PDE's 2 MiB one; reserved in a PML4E. + const PAGE_SIZE: u64 = 1 << 7; + const EXECUTE_DISABLE: u64 = 1 << 63; + /// Bits 51:12, the widest `MAXPHYADDR` §4.1.4 allows. + const ADDRESS: u64 = 0x000F_FFFF_FFFF_F000; + if entry & PRESENT == 0 { + Entry::Absent + } else if level == 3 || (level > 0 && entry & PAGE_SIZE != 0) { + Entry::Leaf { writable: entry & WRITABLE != 0, executable: entry & EXECUTE_DISABLE == 0 } + } else { + Entry::Table(entry & ADDRESS) + } + } + + /// Canonical: bits 63:48 copy bit 47 (§3.3.7.1). + fn address(bits: u64) -> u64 { + (((bits << 16) as i64) >> 16) as u64 + } +} + +/// VMSAv8-64 stage 1, 4 KiB granule, under `TTBR1_EL1` with `T1SZ` = 16 and +/// Direct permissions, Arm ARM DDI 0487 M.d D8.3 (Tables D8-48 and D8-52): +/// bit 0 marks a valid descriptor, and bit 1 a table at levels 0 to 2 and a +/// page at level 3, where clear is reserved and treated as invalid; `AP[2]` +/// (bit 7) makes it read-only and `PXN` (bit 53) forbids an EL1 fetch. +pub struct Aarch64; + +impl Format for Aarch64 { + fn read(level: usize, entry: u64) -> Entry { + const VALID: u64 = 1 << 0; + const TABLE_OR_PAGE: u64 = 1 << 1; + const READ_ONLY: u64 = 1 << 7; + const PXN: u64 = 1 << 53; + /// Bits 47:12 of a 4 KiB granule's next-level table address. + const ADDRESS: u64 = 0x0000_FFFF_FFFF_F000; + match (entry & VALID != 0, entry & TABLE_OR_PAGE != 0, level) { + (false, _, _) | (true, false, 3) => Entry::Absent, + (true, true, 0..=2) => Entry::Table(entry & ADDRESS), + (true, _, _) => Entry::Leaf { writable: entry & READ_ONLY == 0, executable: entry & PXN == 0 }, + } + } + + /// `TTBR1_EL1` translates the top 2^(64 - T1SZ) bytes (D8.2.4). + fn address(bits: u64) -> u64 { + 0xFFFF_0000_0000_0000 | bits + } +} + +/// The leaves one rule caught: how many, and the lowest address among them. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub struct Finding { + pub leaves: u64, + pub first: u64, +} + +/// What the walk of one root found. +#[derive(Clone, Copy, PartialEq, Eq, Debug, Default)] +pub struct Verdict { + /// Every leaf it read. + pub leaves: u64, + pub writable_executable: Option, + /// An executable leaf not wholly inside the text. + pub executable_outside_text: Option, +} + +impl Verdict { + pub fn holds(&self) -> bool { + self.writable_executable.is_none() && self.executable_outside_text.is_none() + } +} + +/// Walk every table under `root`, reading a table at a physical address +/// through `table`, and judge each leaf against `text`, the virtual addresses +/// that may be executable. +pub fn audit<'t, F: Format>(root: &'t [u64; 512], text: Range, table: &impl Fn(u64) -> &'t [u64; 512]) -> Verdict { + let mut verdict = Verdict::default(); + descend::(root, 0, 0, &text, table, &mut verdict); + verdict +} + +fn descend<'t, F: Format>( + entries: &'t [u64; 512], + level: usize, + base: u64, + text: &Range, + table: &impl Fn(u64) -> &'t [u64; 512], + verdict: &mut Verdict, +) { + let shift = 39 - 9 * level as u32; + for (index, &entry) in entries.iter().enumerate() { + let bits = base | (index as u64) << shift; + match F::read(level, entry) { + Entry::Absent => {} + Entry::Table(phys) => descend::(table(phys), level + 1, bits, text, table, verdict), + Entry::Leaf { writable, executable } => { + verdict.leaves += 1; + let start = F::address(bits); + let end = start + (1 << shift); + if executable && writable { + note(&mut verdict.writable_executable, start); + } + if executable && !(text.start <= start && end <= text.end) { + note(&mut verdict.executable_outside_text, start); + } + } + } + } +} + +/// Entries are walked in ascending address order, so the first noted is the lowest. +fn note(finding: &mut Option, at: u64) { + match finding { + Some(found) => found.leaves += 1, + None => *finding = Some(Finding { leaves: 1, first: at }), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use alloc::boxed::Box; + use alloc::vec::Vec; + + /// Tables at physical addresses `0x1000 * (i + 1)`, the root first. + struct Tables(Vec>); + + impl Tables { + fn new(count: usize) -> Self { + Self((0..count).map(|_| Box::new([0; 512])).collect()) + } + + fn phys(index: usize) -> u64 { + 0x1000 * (index as u64 + 1) + } + + fn audit(&self, text: Range) -> Verdict { + let table = |phys: u64| -> &[u64; 512] { &self.0[(phys / 0x1000 - 1) as usize] }; + audit::(&self.0[0], text, &table) + } + } + + mod x86_64 { + use super::*; + + const P: u64 = 1; + const RW: u64 = 1 << 1; + const PS: u64 = 1 << 7; + const XD: u64 = 1 << 63; + const HIGH: u64 = 0xFFFF_8000_0000_0000; + + /// The kernel half's first slot down to one directory and one table of 4 KiB pages: + /// the shape the kernel's direct map takes around its image. + fn direct_map() -> Tables { + let mut t = Tables::new(4); + t.0[0][256] = Tables::phys(1) | P | RW; + t.0[1][0] = Tables::phys(2) | P | RW; + t.0[2][0] = Tables::phys(3) | P | RW; + for i in 1..512 { + t.0[2][i] = (i as u64) << 21 | P | RW | PS | XD; + } + t + } + + #[test] + fn text_read_only_and_executable_and_the_rest_neither_holds() { + let mut t = direct_map(); + t.0[3][0] = P | RW | XD; + t.0[3][1] = 0x1000 | P; + t.0[3][2] = 0x2000 | P | XD; + t.0[3][3] = 0x3000 | P | RW | XD; + let verdict = t.audit::(HIGH + 0x1000..HIGH + 0x2000); + assert_eq!(verdict, Verdict { leaves: 4 + 511, writable_executable: None, executable_outside_text: None }); + assert!(verdict.holds()); + } + + #[test] + fn a_writable_leaf_without_xd_is_caught_at_its_address() { + let mut t = direct_map(); + t.0[3][1] = 0x1000 | P; + t.0[2][7] = 7 << 21 | P | RW | PS; + t.0[2][9] = 9 << 21 | P | RW | PS; + let verdict = t.audit::(HIGH + 0x1000..HIGH + 0x2000); + let both = Some(Finding { leaves: 2, first: HIGH + (7 << 21) }); + assert_eq!(verdict.writable_executable, both); + assert_eq!(verdict.executable_outside_text, both); + assert!(!verdict.holds()); + } + + #[test] + fn a_read_only_leaf_without_xd_outside_the_text_is_caught() { + let mut t = direct_map(); + t.0[3][1] = 0x1000 | P; + t.0[3][2] = 0x2000 | P; + let verdict = t.audit::(HIGH + 0x1000..HIGH + 0x2000); + assert_eq!(verdict.writable_executable, None); + assert_eq!(verdict.executable_outside_text, Some(Finding { leaves: 1, first: HIGH + 0x2000 })); + } + + /// A 2 MiB leaf that holds the text but reaches past it is executable outside it. + #[test] + fn a_large_leaf_straddling_the_text_is_outside_it() { + let mut t = direct_map(); + t.0[2][0] = P | PS; + let verdict = t.audit::(HIGH + 0x1000..HIGH + 0x2000); + assert_eq!(verdict.executable_outside_text, Some(Finding { leaves: 1, first: HIGH })); + assert_eq!(verdict.writable_executable, None); + } + + /// `PS` makes a PDPTE a 1 GiB leaf, and in a PML4E it is not a size at all. + #[test] + fn page_size_ends_the_walk_below_the_root_only() { + let mut t = Tables::new(3); + t.0[0][256] = Tables::phys(1) | P | RW | PS; + t.0[1][3] = 3 << 30 | P | RW | PS; + let verdict = t.audit::(0..0); + assert_eq!(verdict.leaves, 1); + assert_eq!(verdict.writable_executable, Some(Finding { leaves: 1, first: HIGH + (3 << 30) })); + } + + /// The user half's addresses stay canonical, and a not-present entry's other bits are nothing. + #[test] + fn absent_entries_are_skipped_and_addresses_are_canonical() { + let mut t = Tables::new(3); + t.0[0][0] = Tables::phys(1) | P | RW; + t.0[0][255] = Tables::phys(2) | P | RW; + t.0[1][0] = RW | PS; + t.0[2][511] = 511 << 30 | P | RW | PS; + let verdict = t.audit::(0..0); + assert_eq!(verdict.leaves, 1); + assert_eq!(verdict.writable_executable, Some(Finding { leaves: 1, first: 0x0000_7FFF_C000_0000 })); + } + } + + mod aarch64 { + use super::*; + + const VALID: u64 = 1; + const TABLE: u64 = 1 << 1; + const RO: u64 = 1 << 7; + const PXN: u64 = 1 << 53; + const UXN: u64 = 1 << 54; + const HIGH: u64 = 0xFFFF_8000_0000_0000; + + fn direct_map() -> Tables { + let mut t = Tables::new(4); + t.0[0][256] = Tables::phys(1) | VALID | TABLE; + t.0[1][0] = Tables::phys(2) | VALID | TABLE; + t.0[2][0] = Tables::phys(3) | VALID | TABLE; + t.0[2][1] = 1 << 21 | VALID | PXN | UXN; + t + } + + #[test] + fn text_read_only_and_executable_and_the_rest_neither_holds() { + let mut t = direct_map(); + t.0[3][0] = VALID | TABLE | PXN | UXN; + t.0[3][1] = 0x1000 | VALID | TABLE | RO | UXN; + t.0[3][2] = 0x2000 | VALID | TABLE | RO | PXN | UXN; + let verdict = t.audit::(HIGH + 0x1000..HIGH + 0x2000); + assert_eq!(verdict, Verdict { leaves: 4, writable_executable: None, executable_outside_text: None }); + } + + /// `UXN` is EL0's: a block with `PXN` clear is executable at EL1 whatever it says. + #[test] + fn a_writable_block_without_pxn_is_caught() { + let mut t = direct_map(); + t.0[2][1] = 1 << 21 | VALID | UXN; + let verdict = t.audit::(HIGH + 0x1000..HIGH + 0x2000); + assert_eq!(verdict.writable_executable, Some(Finding { leaves: 1, first: HIGH + (1 << 21) })); + } + + /// A level 1 block is a leaf; bit 1 clear at level 3 is no page at all. + #[test] + fn blocks_end_the_walk_and_level_3_needs_bit_1() { + let mut t = direct_map(); + t.0[1][1] = 1 << 30 | VALID; + t.0[3][5] = 0x5000 | VALID; + let verdict = t.audit::(0..0); + assert_eq!(verdict.leaves, 2); + assert_eq!(verdict.writable_executable, Some(Finding { leaves: 1, first: HIGH + (1 << 30) })); + } + + /// Every address under `TTBR1_EL1` has its top sixteen bits set, the root's first slot included. + #[test] + fn addresses_are_ttbr1s() { + let mut t = Tables::new(1); + t.0[0][0] = VALID; + let verdict = t.audit::(0..0); + assert_eq!(verdict.executable_outside_text, Some(Finding { leaves: 1, first: 0xFFFF_0000_0000_0000 })); + } + } +} diff --git a/kernel/smmu/src/table.rs b/kernel/smmu/src/table.rs index f8f9a2cf669..e59081fba06 100644 --- a/kernel/smmu/src/table.rs +++ b/kernel/smmu/src/table.rs @@ -8,7 +8,7 @@ //! takes a message on is the one 4 KiB page at level 3: a block there would //! hand the device every register that shares its 2 MiB. -use toyos_bootmap::aarch64::{block, page, table, MAIR, PXN}; +use toyos_bootmap::aarch64::{block, page, table, MAIR}; use toyos_bootmap::Cache; use toyos_phys::Phys; @@ -39,10 +39,12 @@ const AP_READ_ONLY: u64 = 1 << 7; /// and no invalidation by ASID removes it. const NOT_GLOBAL: u64 = 1 << 11; -/// What a device's leaf adds to the loader's: unprivileged, its context's -/// alone, and executable at no level — the loader's memory is executable at -/// EL1, which no device's access is. -const DEVICES: u64 = AP_UNPRIVILEGED | NOT_GLOBAL | PXN; +/// What a device's leaf adds to the loader's data leaf, which is executable at +/// no level: unprivileged, and its context's alone. +const DEVICES: u64 = AP_UNPRIVILEGED | NOT_GLOBAL; + +/// The loader's leaves that are not the kernel's text. +const DATA: toyos_bootmap::Access = toyos_bootmap::Access::Data; /// What a device may do to memory mapped for it. #[derive(Clone, Copy, PartialEq, Eq, Debug)] @@ -100,9 +102,9 @@ pub const fn plan(at: u64, leaf: Leaf, unit: &Unit) -> Option { Access::Read => AP_READ_ONLY, Access::ReadWrite => 0, }; - (3, 1u64 << 21, memory.get(), block(memory.get(), Cache::Memory) | DEVICES | access) + (3, 1u64 << 21, memory.get(), block(memory.get(), Cache::Memory, DATA) | DEVICES | access) } - Leaf::Doorbell(register) => (4, 1u64 << 12, register.get(), page(register.get(), Cache::Device) | DEVICES), + Leaf::Doorbell(register) => (4, 1u64 << 12, register.get(), page(register.get(), Cache::Device, DATA) | DEVICES), }; if at >> INPUT_BITS != 0 || at & (size - 1) != 0 || !unit.reaches(output, size) { return None; diff --git a/kernel/src/actuator.rs b/kernel/src/actuator.rs index 573c8a13db6..a6c2324be7c 100644 --- a/kernel/src/actuator.rs +++ b/kernel/src/actuator.rs @@ -119,6 +119,11 @@ actuators! { /// Leave every AP holding the CR0/CR4 that INIT left it. no_ap_control_regs = "no-ap-control-regs"; + /// Make one direct-map leaf writable and executable before the boot's + /// audit of the kernel's tables, which must stop the boot on it. Judged by + /// `kernel_wx_leaf_is_refused` and `virt_kernel_wx_leaf_is_refused`. + kernel_wx_leaf = "kernel-wx-leaf"; + /// Hand every CPU the loader's Intel update with its header's revision /// raised by one and its checksum sealed again: the read-back's negative /// control on real Intel, whose payload still says the revision it did. diff --git a/kernel/src/arch/aarch64/control_regs.rs b/kernel/src/arch/aarch64/control_regs.rs index 6a652ab9a64..8c6b24b9494 100644 --- a/kernel/src/arch/aarch64/control_regs.rs +++ b/kernel/src/arch/aarch64/control_regs.rs @@ -21,10 +21,12 @@ pub use toyos_bootmap::aarch64::MAIR; /// stack alignment checked at EL1 and EL0 (`SA`, `SA0`), AArch32 EL0's `IT` /// and `SETEND` disabled (`ITD`, `SED` — RES1 on a CPU with no AArch32 EL0, /// and nothing this kernel runs is AArch32), over the bits Armv8.0 makes -/// RES1 (29, 28, 23, 22, 20, 11). `WXN` stays clear: the direct map the -/// kernel runs from is writable and executable both. Little-endian at both -/// levels; EL0's cache maintenance and `WFI`/`WFE` trap. -pub const SCTLR: u64 = SCTLR_RES1 | 1 << 0 | 1 << 2 | 1 << 3 | 1 << 4 | 1 << 7 | 1 << 8 | 1 << 12; +/// RES1 (29, 28, 23, 22, 20, 11). `WXN` (bit 19, DDI 0601 2023-12): a page +/// writable at EL1 is never executed there, nor one writable at EL0 at EL0, +/// whatever its descriptor says (DDI 0487 M.d D8.4.1.2.4) — the loader's +/// tables, which map the kernel's text read-only, included. Little-endian at +/// both levels; EL0's cache maintenance and `WFI`/`WFE` trap. +pub const SCTLR: u64 = SCTLR_RES1 | 1 << 0 | 1 << 2 | 1 << 3 | 1 << 4 | 1 << 7 | 1 << 8 | 1 << 12 | 1 << 19; const SCTLR_RES1: u64 = 1 << 29 | 1 << 28 | 1 << 23 | 1 << 22 | 1 << 20 | 1 << 11; diff --git a/kernel/src/arch/aarch64/paging.rs b/kernel/src/arch/aarch64/paging.rs index 40af9f5b839..1dc8e60ca50 100644 --- a/kernel/src/arch/aarch64/paging.rs +++ b/kernel/src/arch/aarch64/paging.rs @@ -22,6 +22,9 @@ //! translation dropped on every CPU (`super::tlb`), and only then written //! again (D8.14.1). An entry that was invalid owes nothing, since no TLB holds //! a translation that faulted. +//! +//! **No kernel leaf is both writable and executable**, and only the image's +//! text is executable (`mm::audit_wx`). use alloc::boxed::Box; use alloc::vec::Vec; @@ -102,10 +105,20 @@ fn user_leaf(prot: Prot, cache: CachePolicy) -> u64 { VALID | AF | NOT_GLOBAL | typed(cache) | prot.user_bits() } -/// A kernel leaf: EL1 read-write, EL0 nothing, global, and executable at EL1 -/// only where it is memory — the kernel runs from the direct map. -fn kernel_leaf(cache: CachePolicy) -> u64 { - VALID | AF | typed(cache) | UXN +impl Prot { + /// A kernel leaf's permissions: EL0 has none, and EL1 executes only code. + fn kernel_bits(self) -> u64 { + match self { + Self::Read => AP_READ_ONLY | PXN, + Self::ReadWrite => PXN, + Self::ReadExec => AP_READ_ONLY, + } + } +} + +/// A kernel leaf, bar its address and whether it is a block or a page: global. +fn kernel_leaf(cache: CachePolicy, prot: Prot) -> u64 { + VALID | AF | typed(cache) | prot.kernel_bits() | UXN } /// A leaf with its address and its block-or-page bit taken out: what two @@ -656,21 +669,30 @@ pub const MMIO_GRAIN: u64 = PAGE_4K; /// asked for and free. No invalidation is owed, since only an invalid entry is /// ever written. pub fn map_mmio(phys: u64, size: u64, policy: MmioPolicy) -> crate::mm::Mmio { - high().as_mut().expect("map_mmio: before the kernel's tables exist").map_direct(phys, size, kernel_leaf(policy.cache())); + high() + .as_mut() + .expect("map_mmio: before the kernel's tables exist") + .map_direct(phys, size, kernel_leaf(policy.cache(), Prot::ReadWrite)); let installed = kernel().lock().direct_map_policy(phys).expect("map_mmio: the window was just mapped"); assert!(installed == policy.cache(), "map_mmio: {phys:#x} installed {installed:?}"); crate::log!("mmio: {phys:#x}+{size:#x} {installed:?}"); crate::mm::Mmio::new(DirectMap::from_phys(phys), size) } -/// Build the kernel's tables — every page of memory firmware's map names, -/// and the console UART and the scanout the boot's own records reach — -/// switch `TTBR1_EL1` to them and `TTBR0_EL1` to the kernel's empty user -/// half, and answer where the direct map ends. -pub(crate) fn init(memory_map: &[MemoryMapEntry], scanout: Option<(u64, u64)>) -> toyos_bootmap::DirectMapEnd { +/// Build the kernel's tables — every page of memory firmware's map names, in +/// 4 KiB pages where a 2 MiB one holds more than one of `image`'s +/// protections, and the console +/// UART and the scanout the boot's own records reach — switch `TTBR1_EL1` to +/// them and `TTBR0_EL1` to the kernel's empty user half, and answer where the +/// direct map ends. +pub(crate) fn init( + memory_map: &[MemoryMapEntry], + scanout: Option<(u64, u64)>, + image: &crate::mm::Image, +) -> toyos_bootmap::DirectMapEnd { let extent = direct_map_end(memory_map).unwrap_or_else(|refusal| panic!("paging: firmware's memory map: {refusal}")); let end = extent.get(); - let memory = kernel_leaf(CachePolicy::Normal); + let memory = |phys: u64| kernel_leaf(CachePolicy::Normal, image.prot(phys).unwrap_or(Prot::ReadWrite)); let mut tables = Tables::new(); let (mut blocks, mut pages) = (0u64, 0u64); let mut at = 0; @@ -678,14 +700,15 @@ pub(crate) fn init(memory_map: &[MemoryMapEntry], scanout: Option<(u64, u64)>) - let va = PHYS_OFFSET + at; match coverage(memory_map, at) { Coverage::Nothing => {} - Coverage::Whole => { - tables.directory(va).set(index(va, 2), at | memory); + Coverage::Whole if image.block(at).is_some() => { + tables.directory(va).set(index(va, 2), at | memory(at)); blocks += 1; } - held @ Coverage::Pages(_) => { + held => { let mut table = Table::new(); for j in (0..512).filter(|&j| held.holds(j)) { - table.0[j as usize] = (at + j * PAGE_4K) | memory | TABLE; + let page = at + j * PAGE_4K; + table.0[j as usize] = page | memory(page) | TABLE; pages += 1; } let table = tables.adopt(table); @@ -695,10 +718,10 @@ pub(crate) fn init(memory_map: &[MemoryMapEntry], scanout: Option<(u64, u64)>) - at += PAGE_2M; } if let Some(uart) = super::console_uart::frame() { - tables.map_direct(uart, PAGE_4K, kernel_leaf(CachePolicy::Uncacheable)); + tables.map_direct(uart, PAGE_4K, kernel_leaf(CachePolicy::Uncacheable, Prot::ReadWrite)); } if let Some((scanout, size)) = scanout { - tables.map_direct(scanout, size, kernel_leaf(CachePolicy::WriteCombining)); + tables.map_direct(scanout, size, kernel_leaf(CachePolicy::WriteCombining, Prot::ReadWrite)); } KERNEL_TTBR1.store(tables.root.phys(), core::sync::atomic::Ordering::Release); @@ -716,23 +739,58 @@ pub(crate) fn init(memory_map: &[MemoryMapEntry], scanout: Option<(u64, u64)>) - /// Put this CPU on the kernel's tables — `TTBR1_EL1` the kernel's root, /// `TTBR0_EL1` the kernel space's empty user half — from the loader's or from -/// [`bringup_root`], either of which maps the code, stack and data this runs -/// on exactly as the kernel's root does. The local `TLBI` then drops every -/// translation the old tables left, global ones among them. +/// [`bringup_root`]. +/// +/// The loader's map and the kernel's need not agree on a leaf's size or +/// permissions for the same address: the loader maps 2 MiB blocks where the +/// kernel splits its image into pages, and leaves rodata writable. A change of +/// block size needs break-before-make (Arm ARM DDI 0487 M.d D8.17.1), or the +/// TLB may hold both translations at once and take a TLB conflict abort. So the +/// old `TTBR1_EL1` is replaced by an empty root and every translation dropped +/// with `TLBI VMALLE1` before the kernel's root is installed, while this CPU +/// runs from its image's identity alias under the old `TTBR0_EL1`, where both +/// the loader's map and `bringup_root` map the kernel's text executable. The +/// new `TTBR0_EL1` maps nothing, so switching it only removes translations. +/// Interrupts stay masked across it: the vectors are at `PHYS_OFFSET`. pub(crate) fn join() { let ttbr1 = KERNEL_TTBR1.load(core::sync::atomic::Ordering::Acquire); let ttbr0 = KERNEL_TTBR0.load(core::sync::atomic::Ordering::Acquire); // SAFETY: both roots are published by `init` and live forever, and the - // tables this CPU ran on map everything this function touches as they do. + // kernel space's root is an empty table, so it is the break's `TTBR1_EL1` + // too. Between the two branches nothing is fetched or accessed but this + // block's instructions, through the identity alias the old `TTBR0_EL1` + // maps; the image runs at `PHYS_OFFSET` plus its physical address, so that + // alias is the link address less `PHYS_OFFSET`. unsafe { core::arch::asm!( + "mrs {daif}, daif", + "msr daifset, #0xf", "dsb ishst", + "adr {at}, 1f", + "sub {at}, {at}, {offset}", + "br {at}", + "1:", + "msr ttbr1_el1, {empty}", + "isb", + "tlbi vmalle1", + "dsb nsh", + "isb", "msr ttbr1_el1, {ttbr1}", + "isb", + "adr {at}, 2f", + "add {at}, {at}, {offset}", + "br {at}", + "2:", "msr ttbr0_el1, {ttbr0}", "isb", "tlbi vmalle1", "dsb nsh", "isb", + "msr daif, {daif}", + daif = out(reg) _, + at = out(reg) _, + offset = in(reg) PHYS_OFFSET, + empty = in(reg) ttbr0 & ADDR, ttbr1 = in(reg) ttbr1, ttbr0 = in(reg) ttbr0, options(nostack, preserves_flags), @@ -783,6 +841,41 @@ pub fn bringup_root() -> u64 { /// holds a copy of the kernel's root slots to fall behind. pub(crate) fn seal_kernel_half() {} +/// [`kernel::wx::audit`] of the kernel's tables, `TTBR1_EL1`'s on every CPU; +/// the kernel space's `TTBR0_EL1` half maps nothing. +pub fn audit_wx(text: core::ops::Range) -> kernel::wx::Verdict { + let high = high(); + let tables = high.as_ref().expect("audit_wx: before the kernel's tables exist"); + // SAFETY: every table the kernel's root names is one this file linked, and + // the lock held keeps them in place while they are read. + let table = |phys: u64| unsafe { &Table::at(phys).0 }; + kernel::wx::audit::(&tables.root.0, text, &table) +} + +/// `kernel-wx-leaf`: the direct map's leaf over `phys` made executable at +/// EL1, so [`audit_wx`] has a writable executable leaf to find. Answers its +/// address. Nothing runs from it, so no invalidation is owed. +#[cfg(feature = "boot-actuators")] +pub fn stage_writable_executable(phys: u64) -> u64 { + let va = PHYS_OFFSET + phys; + let mut high = high(); + let tables = high.as_mut().expect("kernel-wx-leaf: before the kernel's tables exist"); + let i = index(va, 2); + let entry = tables.directory(va).0[i]; + assert!(entry & VALID != 0, "kernel-wx-leaf: {phys:#x} is not in the direct map"); + let (table, slot, at) = if entry & TABLE == 0 { + (tables.directory(va), i, va & !(PAGE_2M - 1)) + } else { + // SAFETY: a table this root linked, reached under the lock. + (unsafe { Table::at_mut(entry & ADDR) }, index(va, 3), va & !(PAGE_4K - 1)) + }; + let leaf = table.0[slot]; + assert!(leaf & VALID != 0 && leaf & AP_READ_ONLY == 0, "kernel-wx-leaf: {phys:#x} is not under a writable leaf"); + table.set(slot, leaf & !PXN); + published(); + at +} + /// `PAR_EL1` after the MMU translated `addr` for an EL1 read in the tables /// this CPU runs on now (`AT S1E1R`; Arm ARM K.a, C6.2.x and D24.2.131): /// bit 0 set is a fault, and otherwise bits 63:56 are the memory type. diff --git a/kernel/src/arch/x86_64/boot.rs b/kernel/src/arch/x86_64/boot.rs index 548a3d3c6db..d1b28cc9301 100644 --- a/kernel/src/arch/x86_64/boot.rs +++ b/kernel/src/arch/x86_64/boot.rs @@ -54,10 +54,10 @@ pub fn after_console(args: &KernelArgs, _maps: &[MemoryMapEntry]) { panic!("test-early-fault: x86-64 has no vectors of its own this early"); } // After actuator::init, whose table the `control-regs-bench` probe inside - // this call reads. `pat::init` restored the `CR0` it found, so a firmware - // `CD` — which would make every mapping uncacheable whatever the PAT - // says — ends here. - control_regs::init_cr0(0); + // this call reads, and before `mm::init` loads the kernel's tables. + // `pat::init` restored the `CR0` it found, so a firmware `CD` — which + // would make every mapping uncacheable whatever the PAT says — ends here. + control_regs::init_paging(0); // The first point with every channel a refusal reaches, and before // anything reads what an update changes: `random::key`'s RDRAND and // RDSEED, and `control_regs::init`'s CPUID. diff --git a/kernel/src/arch/x86_64/control_regs.rs b/kernel/src/arch/x86_64/control_regs.rs index 8ab158cad14..436ed9dcffb 100644 --- a/kernel/src/arch/x86_64/control_regs.rs +++ b/kernel/src/arch/x86_64/control_regs.rs @@ -2,14 +2,16 @@ //! `IA32_APIC_BASE` and the performance request hold on every CPU in this //! machine. One declaration, applied by the BSP and every AP and checked on //! each; nothing else may write any of them but [`flush_global`], which puts -//! back the `CR4` it read. Each register is written whole: `CR0` and `EFER` are -//! constants, `CR4` is required bits plus whatever optional bits this CPU +//! back the `CR4` it read, and an AP's trampoline, which enters long mode +//! under [`EFER_BEFORE_LONG_MODE`]. Each register is written whole: `CR0` and +//! `EFER` are constants, `CR4` is required bits plus whatever optional bits this CPU //! offers, `XCR0` and the save form are what `kernel::xsave` makes of this //! CPU's CPUID, `IA32_XSS` is zero wherever it exists (no supervisor component //! is enabled), and `IA32_APIC_BASE` is the local APIC at its architectural address in the one //! mode CPUID offers — x2APIC where it exists, xAPIC where it does not. //! `EFER.NXE` lets bit 63 of a paging entry mean *not executable* -//! ([`Prot`](crate::mm::policy::Prot)). +//! ([`Prot`](crate::mm::policy::Prot)), and `CR0.WP` makes a read-only one +//! read-only at CPL 0: both are set before a CPU loads the kernel's tables. //! //! The performance request is HWP's: `IA32_HWP_INTERRUPT` where it exists, //! `IA32_PM_ENABLE`, `IA32_HWP_REQUEST` as `kernel::cpu::hwp_request` derives @@ -163,6 +165,11 @@ const _: () = assert!(CR4_OPTIONAL & CR4_FORBIDDEN == 0); /// here, never by `arch::syscall::init`, so one register keeps one owner. pub const EFER: u64 = efer::SCE | efer::LME | efer::LMA | efer::NXE; +/// [`EFER`] but `LMA`, which the CPU sets itself as paging turns on in +/// long mode (SDM Vol. 3A, "Initializing IA-32e Mode"): what an AP writes +/// before it does. +pub const EFER_BEFORE_LONG_MODE: u64 = EFER & !efer::LMA; + /// The declaration as the BSP computed it. Zero means not yet declared — also [`pcid_active`]'s correct answer before then. static DECLARED_CR4: AtomicU64 = AtomicU64::new(0); @@ -174,12 +181,24 @@ static DECLARED_XCR0: AtomicU64 = AtomicU64::new(0); /// evidence of the check having run and not a restatement of the CPU count. static CHECKED: AtomicU64 = AtomicU64::new(0); -/// Puts this CPU's `CR0` into [`CR0`]. [`pat::init`](super::pat::init) -/// restores the `CR0` it finds, so a firmware `CD` survives that write and is -/// cleared here: every AP runs this first, and the BSP runs it after, because -/// the BSP's `pat::init` has to precede the panel it would report a refusal on. -pub fn init_cr0(cpu_id: u32) { +/// Puts this CPU's `CR0` into [`CR0`] and its `IA32_EFER` into [`EFER`]: what +/// the kernel's tables need before this CPU loads them. +/// [`pat::init`](super::pat::init) restores the `CR0` it finds, so a firmware +/// `CD` survives that write and is cleared here: every AP runs this first, and +/// the BSP runs it after, because the BSP's `pat::init` has to precede the +/// panel it would report a refusal on. +pub fn init_paging(cpu_id: u32) { let before = bench::sample(); + // `NX` is `CPUID.80000001H:EDX[20]` (SDM Vol. 2A Table 3-8); the extended + // leaf must exist before it means anything. Asked first: setting `NXE` + // without it is `#GP`, and the BSP has no vectors of its own yet. + let (max_ext, _, _, _) = cpu::cpuid(0x8000_0000, 0); + let ext_edx = if max_ext >= 0x8000_0001 { cpu::cpuid(0x8000_0001, 0).3 } else { 0 }; + assert!( + ext_edx & (1 << 20) != 0, + "control_regs: cpu{cpu_id} has no NX bit, so no mapping this kernel writes could \ + be made non-executable and W^X would silently not exist", + ); if !skipped(cpu_id) { let live = cpu::read_cr0(); if live & (cr0::CD | cr0::NW) != 0 { @@ -193,8 +212,12 @@ pub fn init_cr0(cpu_id: u32) { } } // SAFETY: `CR0`'s value is this file's declaration, argued in its own - // doc comment. - unsafe { cpu::write_cr0(CR0) }; + // doc comment; `wrmsr` writes [`EFER`], whose `NXE` this CPU has just + // said it defines and whose `LMA` it, in long mode, already holds. + unsafe { + cpu::write_cr0(CR0); + cpu::wrmsr(efer::MSR, EFER); + } } bench::report(cpu_id, before); } @@ -234,10 +257,11 @@ fn hwp_declared(cpu_id: u32) -> Option { verdict.ok() } -/// Puts this CPU's `CR4`, `EFER`, `XCR0` and `IA32_XSS` into the declaration, -/// checks them against it, and declares the save form and the area they imply. Must run -/// after [`init_cr0`], before `arch::syscall::init`, which needs `SCE` set, -/// and before the first task starts, which loads and saves by `XCR0`. +/// Puts this CPU's `CR4`, `XCR0` and `IA32_XSS` into the declaration, checks +/// them, `CR0` and `EFER` against it, and declares the save form and the area +/// they imply. Must run after [`init_paging`], before `arch::syscall::init`, +/// which needs `SCE` set, and before the first task starts, which loads and +/// saves by `XCR0`. pub fn init(cpu_id: u32) { let declared = declaration(cpu_id); let (xcr0, form) = xsave_declaration(cpu_id); @@ -245,15 +269,12 @@ pub fn init(cpu_id: u32) { if !skipped(cpu_id) { // SAFETY: `write_cr4` faults only on an undefined bit, on clearing `PAE` // in long mode, or on `PCIDE` with a nonzero PCID — `declaration` checked - // the first two for both values and both callers use PCID 0; `wrmsr` - // writes [`EFER`], whose bits `declaration` has just confirmed this CPU - // defines and whose `LMA` is the one this CPU, in long mode, already holds. + // the first two for both values and both callers use PCID 0. unsafe { // Through `PGE` clear: only a change of `PGE` drops the global // translations a firmware that left it set may still hold. cpu::write_cr4(declared & !cr4::PGE); cpu::write_cr4(declared); - cpu::wrmsr(efer::MSR, EFER); // SAFETY: `CR4.OSXSAVE` is set by the write above, and // `xcr0_declaration` took only components this CPU enumerates, // AVX-512's three together and with AVX (SDM Vol. 1 §13.3). @@ -424,20 +445,13 @@ fn declaration(cpu_id: u32) -> u64 { ); let declared = CR4_REQUIRED | (have & CR4_OPTIONAL); - // `SYSCALL` and `NX` are `CPUID.80000001H:EDX` bits 11 and 20 (SDM Vol. - // 2A Table 3-8); the extended leaf must exist before they mean anything. - let (max_ext, _, _, _) = cpu::cpuid(0x8000_0000, 0); - let ext_edx = if max_ext >= 0x8000_0001 { cpu::cpuid(0x8000_0001, 0).3 } else { 0 }; + // `SYSCALL` is `CPUID.80000001H:EDX[11]` (SDM Vol. 2A Table 3-8), a leaf + // `init_paging` found. assert!( - ext_edx & (1 << 11) != 0, + cpu::cpuid(0x8000_0001, 0).3 & (1 << 11) != 0, "control_regs: cpu{cpu_id} has no SYSCALL/SYSRET, which is this kernel's only \ way into and out of Ring 3", ); - assert!( - ext_edx & (1 << 20) != 0, - "control_regs: cpu{cpu_id} has no NX bit, so no mapping this kernel writes could \ - be made non-executable and W^X would silently not exist", - ); // Changing `LA57` with paging on is `#GP`, so this only ever reads it. let live = cpu::read_cr4(); diff --git a/kernel/src/arch/x86_64/paging.rs b/kernel/src/arch/x86_64/paging.rs index 4e569046bcf..1deddb2998d 100644 --- a/kernel/src/arch/x86_64/paging.rs +++ b/kernel/src/arch/x86_64/paging.rs @@ -5,7 +5,9 @@ // `arch::tlb::shootdown` is the caller's job for the rest of the machine. // A leaf is global if and only if it is the same in every root that will ever // be loaded, which is the kernel half and only [`kernel_leaf`] writes it; a -// mapping that differs between roots needs a constructor of its own. +// mapping that differs between roots needs a constructor of its own. No +// kernel leaf is both writable and executable, and only the image's text is +// executable (`mm::audit_wx`). use crate::log; use alloc::boxed::Box; @@ -39,7 +41,8 @@ const PAGE_GLOBAL: u64 = 1 << 8; const PAGE_PAT_2M: u64 = 1 << 12; /// The PAT bit of a 4 KiB PTE, where a PDE has its size bit. const PAGE_PAT_4K: u64 = 1 << 7; -/// Not-executable only because `arch::control_regs` asserts `EFER.NXE` on every CPU. +/// Not-executable only because `arch::control_regs` sets `EFER.NXE` on every +/// CPU before it loads these tables. const PAGE_NX: u64 = 1 << 63; const ADDR_MASK: u64 = 0x000F_FFFF_FFFF_F000; const ADDR_MASK_2M: u64 = 0x000F_FFFF_FFE0_0000; @@ -58,15 +61,29 @@ impl Prot { Self::ReadExec => common, } } + + /// The permission bits a kernel leaf carries. A supervisor write to a + /// read-only one faults because `CR0.WP` is declared (SDM Vol. 3A §4.6.1). + fn kernel_bits(self) -> u64 { + match self { + Self::Read => PAGE_NX, + Self::ReadWrite => PAGE_WRITE | PAGE_NX, + Self::ReadExec => 0, + } + } } -/// Every bit of a 2 MiB kernel-half leaf but its address and size: the one -/// writer of [`PAGE_GLOBAL`]. Writable and executable, because the kernel's -/// image is in the direct map. -fn kernel_leaf(cache: CachePolicy) -> u64 { - PAGE_PRESENT | PAGE_WRITE | PAGE_GLOBAL | cache.pde_bits() +/// Every bit of a kernel-half leaf but its address and a 2 MiB leaf's size +/// bit: the one writer of [`PAGE_GLOBAL`]. Its PAT bit is a 2 MiB leaf's (bit +/// 12; a 4 KiB one's is bit 7), so a 4 KiB leaf takes it only for `Normal`, +/// which sets neither. +fn kernel_leaf(cache: CachePolicy, prot: Prot) -> u64 { + PAGE_PRESENT | PAGE_GLOBAL | prot.kernel_bits() | cache.pde_bits() } +/// Every upper-level entry over the kernel half: its leaves decide. +const KERNEL_TABLE: u64 = PAGE_PRESENT | PAGE_WRITE; + /// Which PAT entry a 2 MiB mapping selects: `Normal` is entry 0 (WB), whose /// type for a range is the MTRR's (SDM Vol. 3A Table 11-7); `Uncacheable` is /// [`pat::UC_ENTRY`](crate::arch::pat::UC_ENTRY), UC under every MTRR type; @@ -712,7 +729,7 @@ impl AddressSpace { let end = (phys + size + PAGE_2M - 1) & !(PAGE_2M - 1); let mut cur = start; while cur < end { - self.map_2m(cur, kernel_leaf(cache)); + self.map_2m(cur, kernel_leaf(cache, Prot::ReadWrite)); cur += PAGE_2M; } crate::mm::Mmio::new(crate::mm::DirectMap::from_phys(phys), size) @@ -778,7 +795,7 @@ impl AddressSpace { } let pt_phys = pt.phys(); self.children.push(pt); - pd.write_pde(pd_idx, virt, pt_phys | PAGE_PRESENT | PAGE_WRITE).discharge(target); + pd.write_pde(pd_idx, virt, pt_phys | KERNEL_TABLE).discharge(target); } // SAFETY: `pd[pd_idx]` names a page table either way, permanently — @@ -794,7 +811,7 @@ impl AddressSpace { fn map_2m(&mut self, phys: u64, flags: u64) { let virt = crate::mm::DirectMap::from_phys(phys).as_ptr::() as u64; let pd_idx = indices(virt).2; - let pd = self.ensure_table(virt, flags); + let pd = self.ensure_table(virt, KERNEL_TABLE); let entry = phys | flags | PAGE_SIZE_BIT; let existing = pd[pd_idx]; assert!( @@ -809,6 +826,22 @@ impl AddressSpace { pd.write_pde(pd_idx, virt, entry).subsumed_by_flush(); } + /// The 2 MiB page of the direct map at `phys`, which holds more than one of + /// the image's protections, in 4 KiB leaves each as the image says. Only + /// `init` calls it, before any TLB holds these tables. + fn map_image_pages(&mut self, phys: u64, image: &crate::mm::Image) { + let virt = crate::mm::DirectMap::from_phys(phys).as_ptr::() as u64; + let mut pt = Box::new(PageTablePage([0; 512])); + for i in 0..512 { + let page = phys + i as u64 * PAGE_4K; + pt.init_entry(i, page | kernel_leaf(CachePolicy::Normal, image.prot(page).unwrap_or(Prot::ReadWrite))); + } + let pt_phys = pt.phys(); + self.children.push(pt); + let pd = self.ensure_table(virt, KERNEL_TABLE); + pd.write_pde(indices(virt).2, virt, pt_phys | KERNEL_TABLE).expect_install("map_image_pages"); + } + /// Masks `flags` to `TABLE_FLAGS`: bit 12 is a leaf's PAT bit but an /// upper entry's address bit, so leaf flags can't move a table 4 KiB. fn ensure_table(&mut self, va: u64, flags: u64) -> &mut PageTablePage { @@ -982,7 +1015,7 @@ pub fn debug_global_leaf(how: u64) -> u64 { unsafe { core::ptr::write_volatile(at.as_mut_ptr::(), at.phys()) }; } let (old_phys, new_phys) = (old.direct_map().phys(), new.direct_map().phys()); - let leaf = kernel_leaf(CachePolicy::Normal) | PAGE_SIZE_BIT; + let leaf = kernel_leaf(CachePolicy::Normal, Prot::ReadWrite) | PAGE_SIZE_BIT; // SAFETY: every read below is made while `SCRATCH` maps one of the two // frames this call owns, or a translation of one of them. let read = || unsafe { core::ptr::read_volatile(SCRATCH as *const u64) }; @@ -991,7 +1024,7 @@ pub fn debug_global_leaf(how: u64) -> u64 { let mut space = kernel().lock(); let _closed = crate::arch::IrqGuard::close(); let target = space.root(); - let pd = space.ensure_table(SCRATCH, leaf); + let pd = space.ensure_table(SCRATCH, KERNEL_TABLE); let idx = indices(SCRATCH).2; if pd[idx] & PAGE_PRESENT != 0 { return SyscallError::AlreadyExists.to_u64(); @@ -1032,11 +1065,16 @@ pub fn debug_global_leaf(how: u64) -> u64 { } } -/// Build kernel page tables: the direct map in the high half, in 2 MiB pages, -/// as far as [`toyos_bootmap::x86_64::direct_map_end`] reaches, and answer -/// that end. The scanout is not mapped for itself: in the low map the MTRRs -/// type it until `panic_console::remap` gives it a type of its own. -pub(crate) fn init(memory_map: &[MemoryMapEntry], _scanout: Option<(u64, u64)>) -> toyos_bootmap::DirectMapEnd { +/// Build kernel page tables: the direct map in the high half, in 2 MiB pages +/// but where one holds more than one of `image`'s protections, as far as +/// [`toyos_bootmap::x86_64::direct_map_end`] reaches, and answer that end. The +/// scanout is not mapped for itself: in the low map the MTRRs type it until +/// `panic_console::remap` gives it a type of its own. +pub(crate) fn init( + memory_map: &[MemoryMapEntry], + _scanout: Option<(u64, u64)>, + image: &crate::mm::Image, +) -> toyos_bootmap::DirectMapEnd { let extent = toyos_bootmap::x86_64::direct_map_end(memory_map) .unwrap_or_else(|refusal| panic!("paging: firmware's memory map: {refusal}")); let end = extent.get(); @@ -1055,7 +1093,10 @@ pub(crate) fn init(memory_map: &[MemoryMapEntry], _scanout: Option<(u64, u64)>) } let mut addr: u64 = 0; while addr < end { - kernel.map_2m(addr, kernel_leaf(CachePolicy::Normal)); + match image.block(addr) { + Some(prot) => kernel.map_2m(addr, kernel_leaf(CachePolicy::Normal, prot)), + None => kernel.map_image_pages(addr, image), + } addr += PAGE_2M; } crate::log!("paging: the direct map covers 0x0..{end:#x}"); @@ -1091,6 +1132,40 @@ pub(crate) fn seal_kernel_half() { } } +/// [`kernel::wx::audit`] of the kernel's tables, whose kernel half every +/// address space shares. +pub fn audit_wx(text: core::ops::Range) -> kernel::wx::Verdict { + let space = kernel().lock(); + // SAFETY: every table the kernel's root names is one this module linked, + // and the lock held keeps them in place while they are read. + let table = |phys: u64| unsafe { &PageTablePage::from_phys(phys).0 }; + kernel::wx::audit::(&space.root.0, text, &table) +} + +/// `kernel-wx-leaf`: the direct map's leaf over `phys` made executable, so +/// [`audit_wx`] has a writable executable leaf to find. Answers its address. +#[cfg(feature = "boot-actuators")] +pub fn stage_writable_executable(phys: u64) -> u64 { + let virt = crate::mm::DirectMap::from_phys(phys).as_ptr::() as u64; + let mut space = kernel().lock(); + let target = space.root(); + let pd = space.ensure_table(virt, KERNEL_TABLE); + let i = indices(virt).2; + let pde = pd[i]; + assert!(pde & PAGE_PRESENT != 0, "kernel-wx-leaf: {phys:#x} is not in the direct map"); + let (table, slot, at) = if pde & PAGE_SIZE_BIT != 0 { + (pd, i, virt & !(PAGE_2M - 1)) + } else { + // SAFETY: a present PDE without the size bit names a table this + // module linked; the kernel's lock gives the exclusivity. + (unsafe { PageTablePage::from_phys_mut(pde & ADDR_MASK) }, ((virt >> 12) & 0x1FF) as usize, virt & !(PAGE_4K - 1)) + }; + let leaf = table[slot]; + assert!(leaf & PAGE_PRESENT != 0 && leaf & PAGE_WRITE != 0, "kernel-wx-leaf: {phys:#x} is not under a writable leaf"); + table.write(slot, at, leaf & !PAGE_NX).discharge(target); + at +} + fn has(entry: u64, flag: u64) -> u8 { if entry & flag != 0 { 1 diff --git a/kernel/src/arch/x86_64/smp.rs b/kernel/src/arch/x86_64/smp.rs index 118206d2a56..5b4b9d72a61 100644 --- a/kernel/src/arch/x86_64/smp.rs +++ b/kernel/src/arch/x86_64/smp.rs @@ -74,9 +74,10 @@ struct TrampolineData { _pad5: [u8; 2], // +0x6E cs_reload_addr: u64, // +0x70 percpu_ptr: u64, // +0x78 + efer: u64, // +0x80 } -const _: () = assert!(size_of::() == 0x80); +const _: () = assert!(size_of::() == 0x88); @@ -107,7 +108,9 @@ macro_rules! asm_label_addr { }}; } -/// Copies the trampoline blob to physical page 0x8000 via the direct map; there is no identity map this early. +/// Copies the trampoline blob to physical page 0x8000 via the direct map, where +/// it is data and never executed; an AP, started only after the copy, fetches +/// it through the boot map's identity view. fn copy_trampoline() { let start = asm_label_addr!(_trampoline_start); let end = asm_label_addr!(_trampoline_end); @@ -167,6 +170,7 @@ fn build_trampoline_data() -> TrampolineData { _pad5: [0; 2], cs_reload_addr, percpu_ptr: 0, // filled per-AP + efer: crate::arch::control_regs::EFER_BEFORE_LONG_MODE, } } @@ -295,7 +299,7 @@ extern "C" fn ap_entry() -> ! { AP_TSC.store(cpu::rdtsc(), Ordering::Release); // Must run before `pat::init`, which restores the CR0 this call sets. - crate::arch::control_regs::init_cr0(percpu::cpu_id()); + crate::arch::control_regs::init_paging(percpu::cpu_id()); // Before `control_regs::init` reads what an update changes. crate::arch::microcode::load(percpu::cpu_id()); @@ -370,10 +374,11 @@ global_asm!( "mov eax, [0x8F00]", "mov cr3, eax", - // Enable Long Mode (IA32_EFER.LME) + // IA32_EFER from TrampolineData.efer (+0x80): long mode, and `NXE` before + // the kernel's tables, where bit 63 is otherwise a reserved bit. "mov ecx, 0xC0000080", - "rdmsr", - "or eax, 0x100", + "mov eax, [0x8F80]", + "mov edx, [0x8F84]", "wrmsr", // Enable Paging (CR0.PG) diff --git a/kernel/src/main.rs b/kernel/src/main.rs index 66250531908..625f448714a 100644 --- a/kernel/src/main.rs +++ b/kernel/src/main.rs @@ -384,11 +384,14 @@ pub(crate) unsafe extern "C" fn kernel_main(loader_args: &mut KernelArgs) -> ! { // what `mm::init` withholds. let [image, elf, black_box, root, map, microcode] = loader; let reserved = [image, elf, black_box, root, map, microcode, arch::boot::reserved()]; + let layout = toyos_elf::Layout::parse(kernel_elf, arch::ELF_MACHINE) + .unwrap_or_else(|e| panic!("boot: the loader's kernel.elf: {e}")); + let image = mm::Image::new(&layout, image); // Before the first hash container, `mm::init`'s address space. hasher::seed(); - mm::init(maps, &reserved); + mm::init(maps, &reserved, &image); drivers::panic_console::remap(); // Before the first table is decoded for its contents: what a machine owner @@ -401,9 +404,7 @@ pub(crate) unsafe extern "C" fn kernel_main(loader_args: &mut KernelArgs) -> ! { let platform = arch::boot::interrupts(kernel_args.rsdp_addr); counters::bring_up(); symbols::set_kernel_base(kernel_args.kernel_memory_addr); - if !kernel_elf.is_empty() { - symbols::load_kernel(kernel_elf, mm::PHYS_OFFSET + kernel_args.kernel_memory_addr); - } + symbols::load_kernel(kernel_elf, mm::PHYS_OFFSET + kernel_args.kernel_memory_addr); arch::boot::clock(kernel_args); trace::enable(); @@ -502,6 +503,18 @@ pub(crate) unsafe extern "C" fn kernel_main(loader_args: &mut KernelArgs) -> ! { // userland runs and asks for a claim. iommu::drivers_settled(); + // Once the boot's own mappings are made: every later kernel leaf is + // written by the same constructors. + #[cfg(feature = "boot-actuators")] + if actuator::kernel_wx_leaf() { + let frame = mm::pmm::alloc_page().expect("kernel-wx-leaf: no frame to stage on"); + let at = mm::paging::stage_writable_executable(frame.direct_map().phys()); + log!("kernel-wx-leaf: the leaf at {at:#x} is writable and executable"); + #[expect(clippy::disallowed_methods, reason = "the audit below ends this boot on the leaf staged over it")] + core::mem::forget(frame); + } + mm::audit_wx(&image); + report_log_destination(); let complete = cpu::counter(); boot_phase!("complete", 0); diff --git a/kernel/src/mm/image.rs b/kernel/src/mm/image.rs new file mode 100644 index 00000000000..a4461767784 --- /dev/null +++ b/kernel/src/mm/image.rs @@ -0,0 +1,82 @@ +//! The kernel's own image where the loader placed it, and what each of its +//! pages may be used for: its `PT_LOAD` segments' flags, page by page. + +use toyos_elf::{Layout, Segment}; + +use super::policy::Prot; +use super::{Region, PAGE_SIZE, PHYS_OFFSET}; + +/// The loader's allocation for the kernel: the image at its own link +/// addresses, then the boot stack. +pub struct Image<'a> { + memory: Region, + /// Where image offset 0 is. + base: u64, + segments: &'a [Segment], + /// The one executable segment's pages, physical. + text: Region, +} + +impl<'a> Image<'a> { + /// The image `layout` describes, loaded into `memory`. + /// + /// Panics where a page could not have a single protection that is not + /// both writable and executable: an executable segment that is writable, + /// two segments that share a page, or other than one executable segment, + /// which is what the loader's boot map maps read-only. + pub fn new(layout: &'a Layout, memory: Region) -> Self { + let segments = layout.segments(); + if let Some((a, b)) = layout.overlapping_load_pages(PAGE_SIZE) { + panic!("kernel image: PT_LOAD segments {a} and {b} share a page"); + } + let mut executable = segments.iter().filter(|segment| segment.flags().executable()); + let (Some(text), None) = (executable.next(), executable.next()) else { + panic!("kernel image: not exactly one executable PT_LOAD segment"); + }; + assert!(!text.writable(), "kernel image: its executable segment is writable"); + let base = memory.start + layout.extent().min(); + let (start, end) = text.page_range(PAGE_SIZE); + let text = Region { start: base + start, end: base + end }; + assert!(memory.start <= text.start && text.end <= memory.end, "kernel image: its text is outside its memory"); + Self { memory, base, segments, text } + } + + /// Whether any byte of `[phys, phys + len)` is the image's. + fn overlaps(&self, phys: u64, len: u64) -> bool { + phys < self.memory.end && self.memory.start < phys + len + } + + /// What the 4 KiB page at `phys` may be used for, or `None` for a page + /// that is not the image's: its segment's flags, and a page no segment + /// holds — the boot stack — is data. + pub fn prot(&self, phys: u64) -> Option { + if !self.overlaps(phys, PAGE_SIZE) { + return None; + } + let holder = phys.checked_sub(self.base).and_then(|offset| { + self.segments.iter().find(|segment| { + let (start, end) = segment.page_range(PAGE_SIZE); + (start..end).contains(&offset) + }) + }); + Some(match holder { + Some(segment) if segment.flags().executable() => Prot::ReadExec, + Some(segment) if !segment.writable() => Prot::Read, + _ => Prot::ReadWrite, + }) + } + + /// What all of the 2 MiB page at `phys` may be used for, where it is one + /// thing: a page not the image's is data, and a page that holds two + /// protections is `None`, to be mapped page by page. + pub fn block(&self, phys: u64) -> Option { + let of = |page: u64| self.prot(page).unwrap_or(Prot::ReadWrite); + let first = of(phys); + (1..super::PAGE_2M / PAGE_SIZE).all(|i| of(phys + i * PAGE_SIZE) == first).then_some(first) + } + + /// The text's pages, as the direct map addresses them. + pub fn text(&self) -> core::ops::Range { + PHYS_OFFSET + self.text.start..PHYS_OFFSET + self.text.end + } +} diff --git a/kernel/src/mm/mod.rs b/kernel/src/mm/mod.rs index 42538cedd83..494ee3c0320 100644 --- a/kernel/src/mm/mod.rs +++ b/kernel/src/mm/mod.rs @@ -7,6 +7,7 @@ pub mod pmm; pub use crate::arch::paging; mod alloc; mod dma; +mod image; mod mmio; pub mod policy; mod region; @@ -22,6 +23,7 @@ pub use alloc::sweep as sweep_heap_bands; #[cfg(feature = "heap-lockspin")] pub use alloc::hold_lock as hold_heap_lock; pub use dma::{Dma, DmaPool, Unaligned}; +pub use image::Image; pub use mmio::Mmio; pub use region::{Allocation, KernelSlice}; pub use unmapped::Unmapped; @@ -191,17 +193,43 @@ pub fn firmware_map() -> &'static [MemoryMapEntry] { /// Call once at boot, in order: pmm (physical pages) → paging (direct map) → /// alloc (heap) → every kernel root slot, before the first user space copies -/// them. `reserved` holds `memory_map`'s own memory. -pub fn init(memory_map: &'static [MemoryMapEntry], reserved: &[Region]) { +/// them. `reserved` holds `memory_map`'s own memory and `image`'s. +pub fn init(memory_map: &'static [MemoryMapEntry], reserved: &[Region], image: &Image) { FIRMWARE_MAP.1.store(memory_map.len(), core::sync::atomic::Ordering::Release); FIRMWARE_MAP.0.store(memory_map.as_ptr().cast_mut(), core::sync::atomic::Ordering::Release); alloc::init_early(); pmm::init(memory_map, reserved); - DIRECT_MAP_END.set(paging::init(memory_map, crate::drivers::panic_console::scanout())); + DIRECT_MAP_END.set(paging::init(memory_map, crate::drivers::panic_console::scanout(), image)); alloc::init(); paging::seal_kernel_half(); } +/// Walk the kernel's tables and stop the boot on a leaf that is writable and +/// executable, or executable outside `image`'s text. +pub fn audit_wx(image: &Image) { + let text = image.text(); + let verdict = paging::audit_wx(text.clone()); + let said = |finding: Option| match finding { + Some(found) => ::alloc::format!("{} (the first at {:#x})", found.leaves, found.first), + None => ::alloc::string::String::from("none"), + }; + assert!( + verdict.holds(), + "paging: of {} kernel leaves, writable and executable: {}; executable outside the text {:#x}..{:#x}: {}", + verdict.leaves, + said(verdict.writable_executable), + text.start, + text.end, + said(verdict.executable_outside_text), + ); + crate::log!( + "paging: {} kernel leaves, none writable and executable, none executable outside the text {:#x}..{:#x}", + verdict.leaves, + text.start, + text.end + ); +} + /// The two memory facts every crash report ends its contexts with: how deep /// any task's kernel stack went, and what the heap sweep last saw. /// Unconditional: it runs on every crash and must build with no sweep present. diff --git a/lib/bootmap/src/aarch64.rs b/lib/bootmap/src/aarch64.rs index 48a46d25ad5..b70ca8b62f8 100644 --- a/lib/bootmap/src/aarch64.rs +++ b/lib/bootmap/src/aarch64.rs @@ -6,7 +6,7 @@ use toyos_abi::boot::MemoryMapEntry; -use crate::{is_read_as_memory, Cache, DirectMapEnd, Encoding, Refusal, DIRECT_MAP_WINDOW, PAGE_2M, PAGE_4K}; +use crate::{is_read_as_memory, Access, Cache, DirectMapEnd, Encoding, Refusal, DIRECT_MAP_WINDOW, PAGE_2M, PAGE_4K}; /// `MAIR_EL1` index 0: Device-nGnRE, registers. pub const ATTR_DEVICE: u64 = 0; @@ -32,6 +32,8 @@ const INNER_SHAREABLE: u64 = 0b11 << 8; const OUTER_SHAREABLE: u64 = 0b10 << 8; /// The access flag, bit 10: set, so the first access does not fault. const AF: u64 = 1 << 10; +/// `AP[2]`, bit 7: read-only. `AP[1]`, bit 6, stays clear: EL0 has no access. +const READ_ONLY: u64 = 1 << 7; /// Privileged execute-never, bit 53. pub const PXN: u64 = 1 << 53; /// Unprivileged execute-never, bit 54. @@ -45,26 +47,32 @@ pub const fn table(phys: u64) -> u64 { phys | TABLE } -/// A 2 MiB block at `phys`: EL1 read-write and EL0 nothing (`AP` = 0b00), and -/// executable at EL1 only where it is memory — the kernel's image is. A -/// device is never executable, since a speculative fetch from one is a read -/// of its registers. -pub const fn block(phys: u64, cache: Cache) -> u64 { - phys | BLOCK | AF | attributes(cache) +/// A 2 MiB block at `phys`: EL0 nothing, and at EL1 the kernel's text +/// read-only and executable and everything else read-write and never +/// executable. A device is not executable even as text, since a speculative +/// fetch from one is a read of its registers. +pub const fn block(phys: u64, cache: Cache, access: Access) -> u64 { + phys | BLOCK | AF | attributes(cache, access) } -const fn attributes(cache: Cache) -> u64 { - match cache { - Cache::Memory => ATTR_NORMAL << 2 | INNER_SHAREABLE | UXN, - Cache::Device => ATTR_DEVICE << 2 | PXN | UXN, - Cache::Scanout => ATTR_NORMAL_NC << 2 | OUTER_SHAREABLE | PXN | UXN, +const fn attributes(cache: Cache, access: Access) -> u64 { + let typed = match cache { + Cache::Memory => ATTR_NORMAL << 2 | INNER_SHAREABLE, + Cache::Device => ATTR_DEVICE << 2, + Cache::Scanout => ATTR_NORMAL_NC << 2 | OUTER_SHAREABLE, Cache::Firmware => panic!("AArch64 has no range registers to type memory: its plan types by the map"), - } + }; + let access = match (access, cache) { + (Access::Text, Cache::Memory) => READ_ONLY, + (Access::Text, _) => READ_ONLY | PXN, + (Access::Data, _) => PXN, + }; + typed | access | UXN } /// A 4 KiB page at `phys`, with [`block`]'s attributes. -pub const fn page(phys: u64, cache: Cache) -> u64 { - phys | PAGE | AF | attributes(cache) +pub const fn page(phys: u64, cache: Cache, access: Access) -> u64 { + phys | PAGE | AF | attributes(cache, access) } /// The three, for [`Plan::write`](crate::Plan::write). diff --git a/lib/bootmap/src/lib.rs b/lib/bootmap/src/lib.rs index 25019bd86a7..689c5ee6c02 100644 --- a/lib/bootmap/src/lib.rs +++ b/lib/bootmap/src/lib.rs @@ -5,7 +5,8 @@ //! there is one mapping in the machine, and everything the kernel touches in //! that window has to be in it — its own image, the boot parameter, and the //! panel it reports a wedge on — and so is the loader's own image, wherever -//! firmware put it: on x86-64 the switch runs from it. +//! firmware put it: on x86-64 the switch runs from it. The kernel's text is +//! read-only in it ([`Access`]). //! //! Both architectures walk the same shape — a root (x86-64's PML4, AArch64's //! L0), a table per 512 GiB (PDPT, L1), a table per GiB (page directory, L2) @@ -19,9 +20,10 @@ //! every directory. The map reaches as far as the high half can hold, and //! refuses by name what lies past it. //! -//! Pure: the scanout, the loader's image and, where the architecture types -//! memory by firmware's map, the write-back ranges in; a [`Plan`] out, which -//! [`Plan::write`] lays into a pool of [`Table`]s the loader allocates. +//! Pure: the scanout, the loader's image, the kernel's text and, where the +//! architecture types memory by firmware's map, the write-back ranges in; a +//! [`Plan`] out, which [`Plan::write`] lays into a pool of [`Table`]s the +//! loader allocates. //! //! What the kernel takes from firmware's map when it builds its own tables is //! here too, because it may never map less than this map did: which types the @@ -148,8 +150,9 @@ pub const PAGE_4K: u64 = 4096; /// Leaves in one table. const PAGES_PER_TABLE: u64 = PAGE_2M / PAGE_4K; -/// The 2 MiB pages a scanout covers only in part: at most its first and its last. -const FINE_TABLES: usize = 2; +/// The 2 MiB pages split into 4 KiB leaves: the first and last a scanout +/// covers only in part, and the same two of the kernel's text. +const FINE_TABLES: usize = 4; /// The pool a builder needs: a root, every second-level table, every /// directory, and a table of 4 KiB leaves per split page. @@ -170,8 +173,8 @@ impl Table { #[derive(Clone, Copy)] pub struct Encoding { pub table: fn(u64) -> u64, - pub block: fn(u64, Cache) -> u64, - pub page: fn(u64, Cache) -> u64, + pub block: fn(u64, Cache, Access) -> u64, + pub page: fn(u64, Cache, Access) -> u64, } /// Why a machine's memory does not fit these tables. @@ -192,6 +195,8 @@ pub enum Refusal { /// A range of firmware's map that begins or ends here, off the 4 KiB page /// UEFI describes memory in. OffPage(u64), + /// The kernel's text is not whole 4 KiB pages inside [`BOOT_MAP_BYTES`]. + Text { base: u64, len: u64 }, } impl fmt::Display for Refusal { @@ -216,6 +221,10 @@ impl fmt::Display for Refusal { "a range ends at {end:#x}, past the {DIRECT_MAP_WINDOW:#x} bytes a direct map can hold" ), Self::OffPage(at) => write!(f, "a range of firmware's map is bounded at {at:#x}, off a {PAGE_4K:#x}-byte page"), + Self::Text { base, len } => write!( + f, + "the kernel's text at {base:#x}+{len:#x} is not whole {PAGE_4K:#x}-byte pages below {BOOT_MAP_BYTES:#x}" + ), } } } @@ -234,6 +243,16 @@ pub enum Cache { Scanout, } +/// What a leaf may be used for. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub enum Access { + /// The kernel's text: read and executed, never written. + Text, + /// Everything else: read and written. Where the architecture's encoding + /// can, never executed. + Data, +} + /// How the pages that are not the scanout are typed. #[derive(Clone, Copy, PartialEq, Eq, Debug)] pub enum Typing<'a> { @@ -280,6 +299,7 @@ pub struct Entry { pub phys: u64, pub slot: Slot, pub cache: Cache, + pub access: Access, } /// Where a machine's memory, its scanout and the loader itself go in the @@ -297,12 +317,14 @@ pub struct Plan<'a> { scanout: Option<(u64, u64)>, /// The loader's image in whole 2 MiB pages: `(first page, bytes)`. loader: (u64, u64), + /// The kernel's text, `(base, end)`. + text: (u64, u64), typing: Typing<'a>, } impl<'a> Plan<'a> { /// Lay out [`BOOT_MAP_BYTES`], `scanout` and `loader`, typed by `typing`, - /// or say why they cannot be. + /// with `text` read-only, or say why they cannot be. /// /// `scanout` is firmware's framebuffer as firmware reports it. Under /// [`Typing::Firmware`] its base must be 2 MiB aligned and its length is @@ -319,7 +341,21 @@ impl<'a> Plan<'a> { /// so its pages are rounded out both ways and typed as the rest of memory. /// /// A byte of either past [`DIRECT_MAP_WINDOW`] is [`Refusal::PastWindow`]. - pub fn new(scanout: Option<(u64, u64)>, loader: (u64, u64), typing: Typing<'a>) -> Result { + /// + /// `text` is the kernel's text, `(base, len)` in whole 4 KiB pages inside + /// the low map, where the loader places the kernel. A 2 MiB page it covers + /// in part is split, so the pages beside it stay writable. + pub fn new( + scanout: Option<(u64, u64)>, + loader: (u64, u64), + text: (u64, u64), + typing: Typing<'a>, + ) -> Result { + let (base, len) = text; + let whole = base.is_multiple_of(PAGE_4K) && len.is_multiple_of(PAGE_4K) && len > 0; + if !whole || base.checked_add(len).is_none_or(|end| end > BOOT_MAP_BYTES) { + return Err(Refusal::Text { base, len }); + } let mut plan = Self { gibs: [0; MAX_DIRECTORIES], directories: 0, @@ -329,6 +365,7 @@ impl<'a> Plan<'a> { fines: 0, scanout: None, loader: whole_pages(loader.0, loader.1)?, + text: (base, base + len), typing, }; let scanout = scanout.map(|(base, len)| plan.scanout_extent(base, len)).transpose()?; @@ -360,6 +397,7 @@ impl<'a> Plan<'a> { if let Some((base, end)) = scanout { plan.split_scanout(base, end); } + plan.split_partial(plan.text.0, plan.text.1); // Every leaf that is not the scanout typed once here, so `entries` // cannot meet a refusal. for page in 0..BOOT_MAP_BYTES / PAGE_2M { @@ -400,9 +438,14 @@ impl<'a> Plan<'a> { /// Split the pages the scanout `base..end` covers in part, and record it. fn split_scanout(&mut self, base: u64, end: u64) { + self.split_partial(base, end); + self.scanout = Some((base, end - base)); + } + + /// Split the pages `base..end` covers in part: at most its first and its last. + fn split_partial(&mut self, base: u64, end: u64) { let first = base / PAGE_2M * PAGE_2M; let last = (end - 1) / PAGE_2M * PAGE_2M; - // At most the first and the last page are covered in part. for page in [first, last] { let whole = base <= page && page + PAGE_2M <= end; if !whole && !self.is_split(page) { @@ -410,7 +453,6 @@ impl<'a> Plan<'a> { self.fines += 1; } } - self.scanout = Some((base, end - base)); } /// Write this map over whatever `pool` holds, its first byte at physical @@ -436,10 +478,10 @@ impl<'a> Plan<'a> { for entry in self.entries() { match entry.slot { Slot::Directory { directory, index } => { - pool[first_directory + directory].0[index] = (encoding.block)(entry.phys, entry.cache) + pool[first_directory + directory].0[index] = (encoding.block)(entry.phys, entry.cache, entry.access) } Slot::Fine { table, index } => { - pool[first_fine + table].0[index] = (encoding.page)(entry.phys, entry.cache) + pool[first_fine + table].0[index] = (encoding.page)(entry.phys, entry.cache, entry.access) } } } @@ -494,6 +536,11 @@ impl<'a> Plan<'a> { self.loader } + /// The kernel's text, `(base, len)`. + pub fn text(&self) -> (u64, u64) { + (self.text.0, self.text.1 - self.text.0) + } + /// The pages the loader's image adds: those past the low map that neither /// the scanout nor a split page already holds. fn loader_pages(&self) -> impl Iterator + '_ { @@ -528,6 +575,7 @@ impl<'a> Plan<'a> { phys, slot: Slot::Fine { table, index: index as usize }, cache: self.cache_of(phys, PAGE_4K), + access: self.access_of(phys, PAGE_4K), } }) }); @@ -552,6 +600,16 @@ impl<'a> Plan<'a> { self.typing.of(phys, size).expect("`new` typed every leaf") } + /// A leaf is the text's only where all of it is, and a page the text + /// covers in part is split, so no leaf is the text's in part. + fn access_of(&self, phys: u64, size: u64) -> Access { + if self.text.0 <= phys && phys + size <= self.text.1 { + Access::Text + } else { + Access::Data + } + } + fn directory_slot(&self, phys: u64) -> (usize, usize) { let directory = self .directories() @@ -563,7 +621,7 @@ impl<'a> Plan<'a> { fn leaf(&self, phys: u64, cache: Cache) -> Entry { let (directory, index) = self.directory_slot(phys); - Entry { phys, slot: Slot::Directory { directory, index }, cache } + Entry { phys, slot: Slot::Directory { directory, index }, cache, access: self.access_of(phys, PAGE_2M) } } /// Name the directory for `gib`, unless it is already named. Infallible: diff --git a/lib/bootmap/src/x86_64.rs b/lib/bootmap/src/x86_64.rs index e8ac240cb75..5de9a5ab0bb 100644 --- a/lib/bootmap/src/x86_64.rs +++ b/lib/bootmap/src/x86_64.rs @@ -5,7 +5,7 @@ use toyos_abi::boot::MemoryMapEntry; -use crate::{is_read_as_memory, Cache, DirectMapEnd, Encoding, Refusal, BOOT_MAP_BYTES, DIRECT_MAP_WINDOW, PAGE_2M}; +use crate::{is_read_as_memory, Access, Cache, DirectMapEnd, Encoding, Refusal, BOOT_MAP_BYTES, DIRECT_MAP_WINDOW, PAGE_2M}; /// One past the kernel direct map's last byte: [`BOOT_MAP_BYTES`], or the end /// of the highest range the kernel reads as memory in whole [`PAGE_2M`] pages, @@ -43,18 +43,27 @@ pub const fn table(phys: u64) -> u64 { /// PWT with the PAT bit clear), uncacheable under every MTRR type and under an /// unprogrammed PAT, which is what the machine has until the kernel's /// `pat::init`. -pub const fn block(phys: u64, cache: Cache) -> u64 { +/// +/// The text is read-only, and every leaf is executable: bit 63 means +/// execute-disable only under `IA32_EFER.NXE` and is reserved otherwise (§4.5), +/// and the loader switches to these tables, and each AP enters them, before +/// the kernel declares `NXE`. +pub const fn block(phys: u64, cache: Cache, access: Access) -> u64 { let typed = match cache { Cache::Firmware | Cache::Memory => 0, Cache::Device | Cache::Scanout => PCD | PWT, }; - phys | PRESENT | WRITABLE | PAGE_SIZE | typed + let writable = match access { + Access::Text => 0, + Access::Data => WRITABLE, + }; + phys | PRESENT | writable | PAGE_SIZE | typed } /// A 4 KiB leaf at `phys` (Table 4-19), typed as [`block`] types a 2 MiB one; /// a 4 KiB entry's PAT bit is bit 7, and it stays clear. -pub const fn page(phys: u64, cache: Cache) -> u64 { - block(phys, cache) & !PAGE_SIZE +pub const fn page(phys: u64, cache: Cache, access: Access) -> u64 { + block(phys, cache, access) & !PAGE_SIZE } /// The three, for [`Plan::write`](crate::Plan::write). diff --git a/lib/bootmap/tests/plan.rs b/lib/bootmap/tests/plan.rs index c55492552b2..4fca49db0ba 100644 --- a/lib/bootmap/tests/plan.rs +++ b/lib/bootmap/tests/plan.rs @@ -3,8 +3,8 @@ //! and a loader firmware put below the map's end or above it. use toyos_bootmap::{ - aarch64, x86_64, Cache, Entry, Plan, Refusal, Slot, Typing, BOOT_MAP_BYTES, DIRECT_MAP_WINDOW, MAX_PAGES, - PAGE_2M, PAGE_4K, ROOT_HIGH_HALF, ROOT_IDENTITY, + aarch64, x86_64, Access, Cache, Entry, Plan, Refusal, Slot, Typing, BOOT_MAP_BYTES, DIRECT_MAP_WINDOW, + MAX_PAGES, PAGE_2M, PAGE_4K, ROOT_HIGH_HALF, ROOT_IDENTITY, }; const GIB: u64 = 1 << 30; @@ -12,6 +12,8 @@ const GIB: u64 = 1 << 30; const PANEL: u64 = 0x7e9000; /// The loader as OVMF loads it on a 2 GiB guest: relocated low, and not on a page. const LOADER: (u64, u64) = (0x7e5f_0000, 0x3_5000); +/// A kernel's text on a 2 MiB page of its own, so it splits nothing. +const TEXT: (u64, u64) = (0x7c00_0000, PAGE_2M); /// Every leaf lands in a table the plan named, at an index inside it, no two /// land in the same place, and no leaf lands where a split page's table is @@ -63,7 +65,7 @@ fn directory(e: &Entry) -> usize { #[test] fn a_machine_with_no_framebuffer_is_the_low_map_and_nothing_else() { - let plan = Plan::new(None, LOADER, Typing::Firmware).expect("the low map alone"); + let plan = Plan::new(None, LOADER, TEXT, Typing::Firmware).expect("the low map alone"); assert_eq!(plan.directories(), [0, 1, 2, 3]); assert_eq!(plan.scanout(), None); assert_eq!(plan.entries().count() as u64, BOOT_MAP_BYTES / PAGE_2M); @@ -75,7 +77,7 @@ fn a_machine_with_no_framebuffer_is_the_low_map_and_nothing_else() { /// retypes pages the low map already holds. #[test] fn a_framebuffer_inside_the_low_map_adds_no_directory() { - let plan = Plan::new(Some((0xc000_0000, PANEL)), LOADER, Typing::Firmware).expect("inside the low map"); + let plan = Plan::new(Some((0xc000_0000, PANEL)), LOADER, TEXT, Typing::Firmware).expect("inside the low map"); assert_eq!(plan.directories(), [0, 1, 2, 3]); // Rounded up to whole pages, and up only. assert_eq!(plan.scanout(), Some((0xc000_0000, 0x80_0000))); @@ -89,7 +91,7 @@ fn a_framebuffer_inside_the_low_map_adds_no_directory() { /// the low map untouched. #[test] fn a_framebuffer_above_the_low_map_adds_its_own_directory() { - let plan = Plan::new(Some((256 * GIB, PANEL)), LOADER, Typing::Firmware).expect("above the low map"); + let plan = Plan::new(Some((256 * GIB, PANEL)), LOADER, TEXT, Typing::Firmware).expect("above the low map"); assert_eq!(plan.directories(), [0, 1, 2, 3, 256]); assert_eq!(plan.scanout(), Some((256 * GIB, 0x80_0000))); let mine: Vec = plan @@ -109,13 +111,13 @@ fn a_framebuffer_above_the_low_map_adds_its_own_directory() { /// past it, not the last GiB of it. #[test] fn a_framebuffer_at_the_boundary_is_outside_the_low_map() { - let plan = Plan::new(Some((BOOT_MAP_BYTES, PANEL)), LOADER, Typing::Firmware).expect("at the boundary"); + let plan = Plan::new(Some((BOOT_MAP_BYTES, PANEL)), LOADER, TEXT, Typing::Firmware).expect("at the boundary"); assert_eq!(plan.directories(), [0, 1, 2, 3, 4]); assert!(plan.entries().filter(|e| e.cache == Cache::Scanout).all(|e| directory(&e) == 4)); is_consistent(&plan); // One page below it is the low map's last page, and adds nothing. - let inside = Plan::new(Some((BOOT_MAP_BYTES - PAGE_2M, PAGE_2M)), LOADER, Typing::Firmware).expect("the last page"); + let inside = Plan::new(Some((BOOT_MAP_BYTES - PAGE_2M, PAGE_2M)), LOADER, TEXT, Typing::Firmware).expect("the last page"); assert_eq!(inside.directories(), [0, 1, 2, 3]); is_consistent(&inside); } @@ -126,7 +128,7 @@ fn a_framebuffer_at_the_boundary_is_outside_the_low_map() { #[test] fn a_framebuffer_that_straddles_the_low_maps_end_is_mapped_from_both_arms() { let base = BOOT_MAP_BYTES - PAGE_2M; - let plan = Plan::new(Some((base, 4 * PAGE_2M)), LOADER, Typing::Firmware).expect("across the end"); + let plan = Plan::new(Some((base, 4 * PAGE_2M)), LOADER, TEXT, Typing::Firmware).expect("across the end"); assert_eq!(plan.directories(), [0, 1, 2, 3, 4]); let mine: Vec<(u64, usize)> = plan .entries() @@ -152,7 +154,7 @@ fn a_framebuffer_that_straddles_the_low_maps_end_is_mapped_from_both_arms() { #[test] fn a_framebuffer_that_straddles_a_gib_claims_both() { let base = 8 * GIB - PAGE_2M; - let plan = Plan::new(Some((base, 4 * PAGE_2M)), LOADER, Typing::Firmware).expect("straddling"); + let plan = Plan::new(Some((base, 4 * PAGE_2M)), LOADER, TEXT, Typing::Firmware).expect("straddling"); assert_eq!(plan.directories(), [0, 1, 2, 3, 7, 8]); assert!(plan.fine_tables().is_empty()); is_consistent(&plan); @@ -160,7 +162,7 @@ fn a_framebuffer_that_straddles_a_gib_claims_both() { // With a loader straddling another, every directory the budget has; the // two split-page tables are a map-typed plan's, and this one splits nothing. let loader = (12 * GIB - 0x1000, 0x3_5000); - let full = Plan::new(Some((base, 4 * PAGE_2M)), loader, Typing::Firmware).expect("both straddling"); + let full = Plan::new(Some((base, 4 * PAGE_2M)), loader, TEXT, Typing::Firmware).expect("both straddling"); assert_eq!(full.directories(), [0, 1, 2, 3, 7, 8, 11, 12]); assert_eq!(full.regions(), [0]); is_consistent(&full); @@ -177,7 +179,7 @@ fn the_high_half_slot_is_the_top_nine_bits_of_phys_offset() { #[test] fn a_base_off_the_page_is_refused_rather_than_rounded_down() { - assert_eq!(Plan::new(Some((0xc000_1000, PANEL)), LOADER, Typing::Firmware), Err(Refusal::Unaligned(0xc000_1000))); + assert_eq!(Plan::new(Some((0xc000_1000, PANEL)), LOADER, TEXT, Typing::Firmware), Err(Refusal::Unaligned(0xc000_1000))); // The refusal names the address, because a machine owner reads it. assert!(Refusal::Unaligned(0xc000_1000).to_string().contains("0xc0001000")); } @@ -186,11 +188,11 @@ fn a_base_off_the_page_is_refused_rather_than_rounded_down() { fn a_range_no_map_can_hold_is_refused_by_name() { // Its own end does not fit an address, either as it is given... let base = u64::MAX - PAGE_2M + 1; - assert_eq!(Plan::new(Some((base, u64::MAX)), LOADER, Typing::Firmware), Err(Refusal::Extent { base, len: u64::MAX })); + assert_eq!(Plan::new(Some((base, u64::MAX)), LOADER, TEXT, Typing::Firmware), Err(Refusal::Extent { base, len: u64::MAX })); // ...or once rounded up to the page it must end on. - assert_eq!(Plan::new(Some((base, 1)), LOADER, Typing::Firmware), Err(Refusal::Extent { base, len: 1 })); + assert_eq!(Plan::new(Some((base, 1)), LOADER, TEXT, Typing::Firmware), Err(Refusal::Extent { base, len: 1 })); // Wider than the directories a plan may name, and told what it would need. - assert_eq!(Plan::new(Some((16 * GIB, 10 * GIB)), LOADER, Typing::Firmware), Err(Refusal::Directories(14))); + assert_eq!(Plan::new(Some((16 * GIB, 10 * GIB)), LOADER, TEXT, Typing::Firmware), Err(Refusal::Directories(14))); assert!(Refusal::Directories(14).to_string().contains("14 page directories")); } @@ -203,11 +205,17 @@ fn uncacheable_is_pat_entry_three_and_plain_memory_is_entry_zero() { const PCD: u64 = 1 << 4; const PAT_2M: u64 = 1 << 12; let at = 0x4020_0000; - assert_eq!(x86_64::block(at, Cache::Firmware), at | PRESENT_WRITABLE_2M); - assert_eq!(x86_64::block(at, Cache::Scanout), at | PRESENT_WRITABLE_2M | PCD | PWT); + assert_eq!(x86_64::block(at, Cache::Firmware, Access::Data), at | PRESENT_WRITABLE_2M); + assert_eq!(x86_64::block(at, Cache::Scanout, Access::Data), at | PRESENT_WRITABLE_2M | PCD | PWT); // The PAT bit is what would select entry 4, and nothing here sets it. for cache in [Cache::Firmware, Cache::Memory, Cache::Device, Cache::Scanout] { - assert_eq!(x86_64::block(at, cache) & PAT_2M, 0, "{cache:?}"); + assert_eq!(x86_64::block(at, cache, Access::Data) & PAT_2M, 0, "{cache:?}"); + } + // The text is the same leaf with R/W (bit 1) clear, and nothing sets XD (bit 63). + assert_eq!(x86_64::block(at, Cache::Firmware, Access::Text), at | 1 | 1 << 7); + assert_eq!(x86_64::page(at, Cache::Firmware, Access::Text), at | 1); + for access in [Access::Text, Access::Data] { + assert_eq!(x86_64::block(at, Cache::Firmware, access) & 1 << 63, 0, "{access:?}"); } assert_eq!(x86_64::table(0x1000), 0x1003); } @@ -217,7 +225,7 @@ fn uncacheable_is_pat_entry_three_and_plain_memory_is_entry_zero() { #[test] fn a_map_typed_plan_is_memory_where_firmware_says_write_back_and_a_device_elsewhere() { let ram = [(0x4000_0000, 0x4000_0000)]; - let plan = Plan::new(None, LOADER, Typing::ByMap(&ram)).expect("virt's map"); + let plan = Plan::new(None, LOADER, TEXT, Typing::ByMap(&ram)).expect("virt's map"); for entry in plan.entries() { let want = if (0x4000_0000..0x8000_0000).contains(&entry.phys) { Cache::Memory } else { Cache::Device }; assert_eq!(entry.cache, want, "{entry:?}"); @@ -229,56 +237,66 @@ fn a_map_typed_plan_is_memory_where_firmware_says_write_back_and_a_device_elsewh #[test] fn a_page_that_is_part_memory_is_refused_by_address() { let ram = [(0x4000_0000, 0x4000_0000 + 0x1000)]; - assert_eq!(Plan::new(None, LOADER, Typing::ByMap(&ram)), Err(Refusal::Mixed(0x8000_0000))); + assert_eq!(Plan::new(None, LOADER, TEXT, Typing::ByMap(&ram)), Err(Refusal::Mixed(0x8000_0000))); assert!(Refusal::Mixed(0x8000_0000).to_string().contains("0x80000000")); // Two adjacent ranges that meet inside a page make it whole. let split = [(0x4000_0000, 0x4010_0000 - 0x4000_0000), (0x4010_0000, 0x3ff0_0000)]; - assert!(Plan::new(None, LOADER, Typing::ByMap(&split)).is_ok()); + assert!(Plan::new(None, LOADER, TEXT, Typing::ByMap(&split)).is_ok()); } /// A scanout inside memory is the scanout's leaves, not a mixed page. #[test] fn a_scanout_inside_memory_is_typed_as_the_scanout() { let ram = [(0x4000_0000, 0x4000_0000)]; - let plan = Plan::new(Some((0x5000_0000, 0x10_0000)), LOADER, Typing::ByMap(&ram)).expect("inside"); + let plan = Plan::new(Some((0x5000_0000, 0x10_0000)), LOADER, TEXT, Typing::ByMap(&ram)).expect("inside"); let scanout: Vec = plan.entries().filter(|e| e.cache == Cache::Scanout).map(|e| e.phys).collect(); assert_eq!(scanout, (0..256).map(|page| 0x5000_0000 + page * PAGE_4K).collect::>()); is_consistent(&plan); // And a scanout over a page that is part memory takes that page whole. let part = [(0x4000_0000, 0x1000)]; - assert!(Plan::new(Some((0x4000_0000, PAGE_2M)), LOADER, Typing::ByMap(&part)).is_ok()); - assert_eq!(Plan::new(None, LOADER, Typing::ByMap(&part)), Err(Refusal::Mixed(0x4000_0000))); + assert!(Plan::new(Some((0x4000_0000, PAGE_2M)), LOADER, TEXT, Typing::ByMap(&part)).is_ok()); + assert_eq!(Plan::new(None, LOADER, TEXT, Typing::ByMap(&part)), Err(Refusal::Mixed(0x4000_0000))); } -/// The AArch64 descriptors, against the Arm ARM's bit positions (K.a, D8.3.1 -/// and D8.3.2): a table is 0b11; a block 0b01 with `AttrIndx` at 4:2, `SH` at -/// 9:8, `AF` at 10, `PXN` at 53 and `UXN` at 54; and `MAIR_EL1`'s bytes are the -/// Device-nGnRE, Normal write-back and Normal non-cacheable encodings of -/// D24.2.110. +/// The AArch64 descriptors, against the Arm ARM's bit positions (DDI 0487 +/// M.d, D8.3, Table D8-52): a table is 0b11; a block 0b01 with `AttrIndx` at +/// 4:2, `AP[2:1]` at 7:6, `SH` at 9:8, `AF` at 10, `PXN` at 53 and `UXN` at 54; +/// and `MAIR_EL1`'s bytes are the Device-nGnRE, Normal write-back and Normal +/// non-cacheable encodings of D24.2.110. #[test] fn the_aarch64_descriptors_carry_the_attribute_index_the_mair_names() { let at = 0x4020_0000; let attr_index = |d: u64| (d >> 2) & 0b111; - let memory = aarch64::block(at, Cache::Memory); - let device = aarch64::block(at, Cache::Device); - let scanout = aarch64::block(at, Cache::Scanout); - for d in [memory, device, scanout] { + let memory = aarch64::block(at, Cache::Memory, Access::Data); + let device = aarch64::block(at, Cache::Device, Access::Data); + let scanout = aarch64::block(at, Cache::Scanout, Access::Data); + let text = aarch64::block(at, Cache::Memory, Access::Text); + for d in [memory, device, scanout, text] { assert_eq!(d & 0b11, 0b01, "a block"); assert_ne!(d & 1 << 10, 0, "the access flag"); - assert_eq!(d & 0b11 << 6, 0, "EL1 read-write, EL0 nothing"); + assert_eq!(d & 1 << 6, 0, "EL0 nothing"); assert_ne!(d & 1 << 54, 0, "never executable at EL0"); assert_eq!(d & 0x0000_FFFF_FFE0_0000, at, "the output address"); } assert_eq!(aarch64::ATTRS[attr_index(memory) as usize], 0xFF); assert_eq!(aarch64::ATTRS[attr_index(device) as usize], 0x04); assert_eq!(aarch64::ATTRS[attr_index(scanout) as usize], 0x44); - assert_eq!(memory & 1 << 53, 0, "the kernel executes from memory"); - assert_ne!(device & 1 << 53, 0, "and never from a device"); + for d in [memory, device, scanout] { + assert_eq!(d & 1 << 7, 0, "EL1 read-write"); + assert_ne!(d & 1 << 53, 0, "never executable at EL1"); + } + assert_ne!(text & 1 << 7, 0, "the text is read-only"); + assert_eq!(text & 1 << 53, 0, "and the kernel executes it"); + assert_ne!(aarch64::block(at, Cache::Device, Access::Text) & 1 << 53, 0, "never a device's"); + assert_eq!(text & !(1 << 7 | 1 << 53), memory & !(1 << 53), "and is memory as the rest is"); assert_eq!((memory >> 8) & 0b11, 0b11, "memory is inner shareable"); assert_eq!(aarch64::MAIR, 0x44_FF_04); // A level-3 page is 0b11 with the same attributes. - assert_eq!(aarch64::page(0x5000_1000, Cache::Scanout), (scanout & !0x0000_FFFF_FFFF_F000 & !0b11) | 0x5000_1000 | 0b11); + assert_eq!( + aarch64::page(0x5000_1000, Cache::Scanout, Access::Data), + (scanout & !0x0000_FFFF_FFFF_F000 & !0b11) | 0x5000_1000 | 0b11 + ); assert_eq!(aarch64::table(0x1000), 0x1003); } @@ -290,7 +308,7 @@ fn the_aarch64_descriptors_carry_the_attribute_index_the_mair_names() { fn a_scanout_carved_out_of_ram_splits_the_pages_it_shares() { let ram = [(0x4000_0000, 0x8000_0000)]; let (base, len) = (0xbc7a_0000, 0x30_0000); - let plan = Plan::new(Some((base, len)), LOADER, Typing::ByMap(&ram)).expect("ramfb"); + let plan = Plan::new(Some((base, len)), LOADER, TEXT, Typing::ByMap(&ram)).expect("ramfb"); assert_eq!(plan.fine_tables(), [0xbc60_0000, 0xbca0_0000]); let scanout: Vec = plan.entries().filter(|e| e.cache == Cache::Scanout).collect(); let bytes: u64 = scanout @@ -309,7 +327,7 @@ fn a_scanout_carved_out_of_ram_splits_the_pages_it_shares() { assert_eq!(at(0xbc80_0000), Cache::Scanout); is_consistent(&plan); // The same framebuffer under range-register typing is refused, as before. - assert_eq!(Plan::new(Some((base, len)), LOADER, Typing::Firmware), Err(Refusal::Unaligned(base))); + assert_eq!(Plan::new(Some((base, len)), LOADER, TEXT, Typing::Firmware), Err(Refusal::Unaligned(base))); } /// A map-typed scanout must still start on a 4 KiB page. @@ -317,7 +335,7 @@ fn a_scanout_carved_out_of_ram_splits_the_pages_it_shares() { fn a_map_typed_scanout_off_a_small_page_is_refused() { let ram = [(0x4000_0000, 0x8000_0000)]; assert_eq!( - Plan::new(Some((0xbc7a_0800, 0x1000)), LOADER, Typing::ByMap(&ram)), + Plan::new(Some((0xbc7a_0800, 0x1000)), LOADER, TEXT, Typing::ByMap(&ram)), Err(Refusal::Unaligned(0xbc7a_0800)) ); } @@ -328,7 +346,7 @@ fn a_map_typed_scanout_off_a_small_page_is_refused() { #[test] fn a_loader_above_the_low_map_adds_its_own_directory_of_plain_memory() { let loader = (0x1_4000_0000, 0x3_4a00); - let plan = Plan::new(None, loader, Typing::Firmware).expect("above the low map"); + let plan = Plan::new(None, loader, TEXT, Typing::Firmware).expect("above the low map"); assert_eq!(plan.directories(), [0, 1, 2, 3, 5]); assert_eq!(plan.loader(), (0x1_4000_0000, PAGE_2M)); let mine: Vec<(u64, usize, Cache)> = plan @@ -343,7 +361,7 @@ fn a_loader_above_the_low_map_adds_its_own_directory_of_plain_memory() { /// A loader inside the low map is already mapped, and adds nothing. #[test] fn a_loader_inside_the_low_map_adds_nothing() { - let plan = Plan::new(None, LOADER, Typing::Firmware).expect("inside"); + let plan = Plan::new(None, LOADER, TEXT, Typing::Firmware).expect("inside"); assert_eq!(plan.directories(), [0, 1, 2, 3]); assert_eq!(plan.entries().count() as u64, BOOT_MAP_BYTES / PAGE_2M); assert_eq!(plan.loader(), (0x7e40_0000, 2 * PAGE_2M)); @@ -354,7 +372,7 @@ fn a_loader_inside_the_low_map_adds_nothing() { #[test] fn a_loader_across_a_page_is_both_pages() { let loader = (6 * GIB + PAGE_2M - 0x1000, 0x3_4a00); - let plan = Plan::new(None, loader, Typing::Firmware).expect("across a page"); + let plan = Plan::new(None, loader, TEXT, Typing::Firmware).expect("across a page"); assert_eq!(plan.loader(), (6 * GIB, 2 * PAGE_2M)); let mine: Vec = plan.entries().filter(|e| e.phys >= BOOT_MAP_BYTES).map(|e| e.phys).collect(); assert_eq!(mine, [6 * GIB, 6 * GIB + PAGE_2M]); @@ -365,7 +383,7 @@ fn a_loader_across_a_page_is_both_pages() { /// scanout holds is not written twice. #[test] fn a_loader_beside_the_scanout_shares_its_directory() { - let plan = Plan::new(Some((256 * GIB, PANEL)), (256 * GIB + 0x7f_0000, 0x3_4a00), Typing::Firmware) + let plan = Plan::new(Some((256 * GIB, PANEL)), (256 * GIB + 0x7f_0000, 0x3_4a00), TEXT, Typing::Firmware) .expect("one GiB for both"); assert_eq!(plan.directories(), [0, 1, 2, 3, 256]); let above: Vec<(u64, Cache)> = @@ -388,14 +406,14 @@ fn a_loader_beside_the_scanout_shares_its_directory() { #[test] fn a_map_typed_loader_is_memory() { let ram = [(0x4000_0000, 0x4000_0000), (5 * GIB, GIB)]; - let plan = Plan::new(None, (5 * GIB + 0x1000, 0x3_4a00), Typing::ByMap(&ram)).expect("in memory"); + let plan = Plan::new(None, (5 * GIB + 0x1000, 0x3_4a00), TEXT, Typing::ByMap(&ram)).expect("in memory"); let above: Vec<(u64, Cache)> = plan.entries().filter(|e| e.phys >= BOOT_MAP_BYTES).map(|e| (e.phys, e.cache)).collect(); assert_eq!(above, [(5 * GIB, Cache::Memory)]); is_consistent(&plan); let part = [(0x4000_0000, 0x4000_0000), (5 * GIB, 0x10_0000)]; assert_eq!( - Plan::new(None, (5 * GIB, 0x3_4a00), Typing::ByMap(&part)), + Plan::new(None, (5 * GIB, 0x3_4a00), TEXT, Typing::ByMap(&part)), Err(Refusal::Mixed(5 * GIB)) ); } @@ -403,16 +421,16 @@ fn a_map_typed_loader_is_memory() { #[test] fn a_loader_no_map_can_hold_is_refused_by_name() { assert_eq!( - Plan::new(None, (0x1_4000_0000, 0), Typing::Firmware), + Plan::new(None, (0x1_4000_0000, 0), TEXT, Typing::Firmware), Err(Refusal::Extent { base: 0x1_4000_0000, len: 0 }) ); // Its directories count against the same budget as the scanout's, and a // GiB both need is counted once. assert_eq!( - Plan::new(Some((16 * GIB, 4 * GIB)), (32 * GIB, 0x3_4a00), Typing::Firmware), + Plan::new(Some((16 * GIB, 4 * GIB)), (32 * GIB, 0x3_4a00), TEXT, Typing::Firmware), Err(Refusal::Directories(9)) ); - let shared = Plan::new(Some((16 * GIB, 4 * GIB)), (17 * GIB, 0x3_4a00), Typing::Firmware) + let shared = Plan::new(Some((16 * GIB, 4 * GIB)), (17 * GIB, 0x3_4a00), TEXT, Typing::Firmware) .expect("the loader inside the scanout's GiBs"); assert_eq!(shared.directories(), [0, 1, 2, 3, 16, 17, 18, 19]); is_consistent(&shared); @@ -424,7 +442,7 @@ fn a_loader_no_map_can_hold_is_refused_by_name() { #[test] fn a_framebuffer_past_512_gib_adds_its_own_second_level_table() { let base = 0xfc_9000_0000; - let plan = Plan::new(Some((base, 8_294_400)), LOADER, Typing::Firmware).expect("at 1010 GiB"); + let plan = Plan::new(Some((base, 8_294_400)), LOADER, TEXT, Typing::Firmware).expect("at 1010 GiB"); assert_eq!(base / GIB, 1010); assert_eq!(plan.directories(), [0, 1, 2, 3, 1010]); assert_eq!(plan.regions(), [0, 1]); @@ -440,7 +458,7 @@ fn a_framebuffer_past_512_gib_adds_its_own_second_level_table() { #[test] fn a_framebuffer_across_512_gib_names_both_second_level_tables() { let base = 512 * GIB - PAGE_2M; - let plan = Plan::new(Some((base, 4 * PAGE_2M)), (520 * GIB, 0x3_4a00), Typing::Firmware) + let plan = Plan::new(Some((base, 4 * PAGE_2M)), (520 * GIB, 0x3_4a00), TEXT, Typing::Firmware) .expect("across 512 GiB"); assert_eq!(plan.directories(), [0, 1, 2, 3, 511, 512, 520]); assert_eq!(plan.regions(), [0, 1]); @@ -450,13 +468,15 @@ fn a_framebuffer_across_512_gib_names_both_second_level_tables() { /// The most the map can be asked for: a scanout and a loader each straddling a /// 512 GiB boundary, four second-level tables past the low map's, the last -/// below the high half's last root slot. +/// below the high half's last root slot, and text that splits two pages. #[test] fn every_second_level_table_the_budget_has_fits_the_pool() { let scanout = (1024 * GIB - PAGE_2M, 4 * PAGE_2M); let loader = (DIRECT_MAP_WINDOW - 512 * GIB - 0x1000, 0x3_4a00); - let plan = Plan::new(Some(scanout), loader, Typing::Firmware).expect("both straddling"); + let text = (0x7c1f_f000, PAGE_2M + 0x2000); + let plan = Plan::new(Some(scanout), loader, text, Typing::Firmware).expect("both straddling"); assert_eq!(plan.regions(), [0, 1, 2, 254, 255]); + // Short of the pool only by the two pages a map-typed scanout splits. assert_eq!(1 + plan.regions().len() + plan.directories().len() + plan.fine_tables().len(), MAX_PAGES - 2); is_consistent(&plan); } @@ -466,14 +486,76 @@ fn every_second_level_table_the_budget_has_fits_the_pool() { #[test] fn a_range_past_the_window_is_refused_by_name() { assert_eq!( - Plan::new(Some((DIRECT_MAP_WINDOW, PANEL)), LOADER, Typing::Firmware), + Plan::new(Some((DIRECT_MAP_WINDOW, PANEL)), LOADER, TEXT, Typing::Firmware), Err(Refusal::PastWindow(DIRECT_MAP_WINDOW + 0x80_0000)) ); assert_eq!( - Plan::new(None, (DIRECT_MAP_WINDOW - 0x1000, 0x3_4a00), Typing::Firmware), + Plan::new(None, (DIRECT_MAP_WINDOW - 0x1000, 0x3_4a00), TEXT, Typing::Firmware), Err(Refusal::PastWindow(DIRECT_MAP_WINDOW + PAGE_2M)) ); - let last = Plan::new(Some((DIRECT_MAP_WINDOW - 0x80_0000, PANEL)), LOADER, Typing::Firmware).expect("the window's last GiB"); + let last = Plan::new(Some((DIRECT_MAP_WINDOW - 0x80_0000, PANEL)), LOADER, TEXT, Typing::Firmware).expect("the window's last GiB"); assert_eq!(last.regions(), [0, 255]); is_consistent(&last); } + +/// The kernel's text as rust-lld lays it out: inside one 2 MiB page, which is +/// split so exactly the text's 4 KiB pages are read-only. +#[test] +fn the_text_inside_one_page_splits_that_page() { + let text = (0x7c04_1000, 0x13_1000); + let plan = Plan::new(None, LOADER, text, Typing::Firmware).expect("the text"); + assert_eq!(plan.text(), text); + assert_eq!(plan.fine_tables(), [0x7c00_0000]); + let mine: Vec = plan.entries().filter(|e| e.access == Access::Text).map(|e| e.phys).collect(); + assert_eq!(mine, (0..0x131).map(|page| text.0 + page * PAGE_4K).collect::>()); + assert!(plan.entries().filter(|e| e.access == Access::Text).all(|e| matches!(e.slot, Slot::Fine { .. }))); + is_consistent(&plan); +} + +/// Text across two 2 MiB boundaries: the page wholly inside it is one +/// read-only leaf, and the two it shares are split. +#[test] +fn text_across_pages_splits_the_two_it_shares() { + let text = (0x7c1f_f000, PAGE_2M + 0x2000); + let plan = Plan::new(None, LOADER, text, Typing::Firmware).expect("the text"); + assert_eq!(plan.fine_tables(), [0x7c00_0000, 0x7c40_0000]); + let mine: Vec<(u64, bool)> = plan + .entries() + .filter(|e| e.access == Access::Text) + .map(|e| (e.phys, matches!(e.slot, Slot::Fine { .. }))) + .collect(); + assert_eq!(mine, [(0x7c20_0000, false), (0x7c1f_f000, true), (0x7c40_0000, true)]); + is_consistent(&plan); +} + +/// A scanout carved out of RAM beside the kernel's text shares its split page. +#[test] +fn text_beside_the_scanout_shares_its_split_page() { + let ram = [(0x4000_0000, 0x8000_0000)]; + let (base, len) = (0xbc7a_0000, 0x30_0000); + let text = (0xbc60_0000, 0x1_0000); + let plan = Plan::new(Some((base, len)), LOADER, text, Typing::ByMap(&ram)).expect("both"); + assert_eq!(plan.fine_tables(), [0xbc60_0000, 0xbca0_0000]); + let at = |phys: u64| plan.entries().find(|e| e.phys == phys).map(|e| (e.cache, e.access)); + assert_eq!(at(text.0), Some((Cache::Memory, Access::Text))); + assert_eq!(at(text.0 + text.1), Some((Cache::Memory, Access::Data))); + assert_eq!(at(base), Some((Cache::Scanout, Access::Data))); + is_consistent(&plan); +} + +/// Text off a 4 KiB page, empty, or reaching past the low map is refused by its extent. +#[test] +fn text_that_is_not_whole_pages_in_the_low_map_is_refused() { + let refused = [ + (0x7c04_1800, 0x1000), + (0x7c04_1000, 0x800), + (0x7c04_1000, 0), + (BOOT_MAP_BYTES - 0x1000, 0x2000), + (!0xfff, 0x1000), + ]; + for (base, len) in refused { + let plan = Plan::new(None, LOADER, (base, len), Typing::Firmware); + assert_eq!(plan, Err(Refusal::Text { base, len }), "{base:#x}+{len:#x}"); + } + assert!(Refusal::Text { base: 0x7c04_1800, len: 0x1000 }.to_string().contains("0x7c041800+0x1000")); +} diff --git a/lib/bootmap/tests/walk.rs b/lib/bootmap/tests/walk.rs index 5fcaafcdf44..e05b4e233c3 100644 --- a/lib/bootmap/tests/walk.rs +++ b/lib/bootmap/tests/walk.rs @@ -3,7 +3,7 @@ //! than from the encoder, so a table named from the wrong slot or the wrong //! second-level table is an address that does not translate. -use toyos_bootmap::{x86_64, Cache, Plan, Table, Typing, BOOT_MAP_BYTES, MAX_PAGES, PAGE_2M, PAGE_4K}; +use toyos_bootmap::{x86_64, Access, Cache, Plan, Table, Typing, BOOT_MAP_BYTES, MAX_PAGES, PAGE_2M, PAGE_4K}; const GIB: u64 = 1 << 30; const PHYS_OFFSET: u64 = 0xFFFF_8000_0000_0000; @@ -14,6 +14,8 @@ const AT: u64 = 0x7f80_0000; const LOADER: (u64, u64) = (0x7e5f_0000, 0x3_5000); /// The AMD laptop's framebuffer, as its GOP reports it. const LAPTOP: (u64, u64) = (0xfc_9000_0000, 8_294_400); +/// A kernel's text on a 2 MiB page of its own, so it splits nothing. +const TEXT: (u64, u64) = (0x7c00_0000, PAGE_2M); /// Bits M-1:12 of an entry that names a page, for the widest M (52) §4.1.4 allows. const ADDRESS: u64 = 0x000F_FFFF_FFFF_F000; @@ -107,7 +109,8 @@ fn expected_pat(cache: Cache) -> u64 { } /// Every leaf the plan names translates, at identity and at `PHYS_OFFSET`, its -/// first byte and its last, to itself, writable and of its kind's type. +/// first byte and its last, to itself, of its kind's type, and writable unless +/// it is the text's. fn every_leaf_translates_in_both_views(plan: &Plan, pool: &[Table; MAX_PAGES], root: u64) { for entry in plan.entries() { let size = match entry.slot { @@ -115,7 +118,7 @@ fn every_leaf_translates_in_both_views(plan: &Plan, pool: &[Table; MAX_PAGES], r toyos_bootmap::Slot::Fine { .. } => PAGE_4K, }; for phys in [entry.phys, entry.phys + size - 1] { - let want = Translation { phys, pat: expected_pat(entry.cache), writable: true }; + let want = Translation { phys, pat: expected_pat(entry.cache), writable: entry.access == Access::Data }; assert_eq!(walk(pool, root, phys).as_ref(), Some(&want), "{entry:?} at identity"); assert_eq!(walk(pool, root, PHYS_OFFSET + phys).as_ref(), Some(&want), "{entry:?} at PHYS_OFFSET"); } @@ -128,7 +131,7 @@ fn every_leaf_translates_in_both_views(plan: &Plan, pool: &[Table; MAX_PAGES], r #[test] fn the_laptops_scanout_at_1010_gib_translates_in_both_views() { let (base, len) = LAPTOP; - let plan = Plan::new(Some(LAPTOP), LOADER, Typing::Firmware).expect("the laptop's plan"); + let plan = Plan::new(Some(LAPTOP), LOADER, TEXT, Typing::Firmware).expect("the laptop's plan"); let (pool, root) = written(&plan); assert_eq!(root, AT, "the root is the pool's first page"); @@ -138,7 +141,7 @@ fn the_laptops_scanout_at_1010_gib_translates_in_both_views() { assert_eq!(walk(&pool, root, phys), scanout(phys), "identity {phys:#x}"); assert_eq!(walk(&pool, root, PHYS_OFFSET + phys), scanout(phys), "PHYS_OFFSET + {phys:#x}"); } - for phys in [0, BOOT_MAP_BYTES - 1, LOADER.0] { + for phys in [0, TEXT.0 - 1, TEXT.0 + TEXT.1, BOOT_MAP_BYTES - 1, LOADER.0] { assert_eq!(walk(&pool, root, phys), memory(phys), "identity {phys:#x}"); assert_eq!(walk(&pool, root, PHYS_OFFSET + phys), memory(phys), "PHYS_OFFSET + {phys:#x}"); } @@ -156,7 +159,7 @@ fn the_laptops_scanout_at_1010_gib_translates_in_both_views() { /// second-level tables: each directory under its own 512 GiB's table. #[test] fn every_second_level_table_is_reached_from_its_own_root_slots() { - let across = Plan::new(Some((512 * GIB - PAGE_2M, 4 * PAGE_2M)), (520 * GIB, 0x3_4a00), Typing::Firmware) + let across = Plan::new(Some((512 * GIB - PAGE_2M, 4 * PAGE_2M)), (520 * GIB, 0x3_4a00), TEXT, Typing::Firmware) .expect("across 512 GiB"); let (pool, root) = written(&across); every_leaf_translates_in_both_views(&across, &pool, root); @@ -164,6 +167,7 @@ fn every_second_level_table_is_reached_from_its_own_root_slots() { let budget = Plan::new( Some((1024 * GIB - PAGE_2M, 4 * PAGE_2M)), (toyos_bootmap::DIRECT_MAP_WINDOW - 512 * GIB - 0x1000, 0x3_4a00), + TEXT, Typing::Firmware, ) .expect("both straddling"); @@ -178,9 +182,29 @@ fn every_second_level_table_is_reached_from_its_own_root_slots() { fn a_split_page_translates_through_its_table_of_small_leaves() { let ram = [(0x4000_0000, 0x8000_0000)]; let (base, len) = (0xbc7a_0000, 0x30_0000); - let plan = Plan::new(Some((base, len)), LOADER, Typing::ByMap(&ram)).expect("ramfb"); + let plan = Plan::new(Some((base, len)), LOADER, TEXT, Typing::ByMap(&ram)).expect("ramfb"); let (pool, root) = written(&plan); assert_eq!(walk(&pool, root, base - 1), Some(Translation { phys: base - 1, pat: 0, writable: true })); assert_eq!(walk(&pool, root, base), Some(Translation { phys: base, pat: 3, writable: true })); every_leaf_translates_in_both_views(&plan, &pool, root); } + +/// The kernel's text read-only in both views, over the pages it shares split, +/// and the bytes on either side of it still writable. +#[test] +fn the_text_is_read_only_and_its_neighbours_are_writable() { + let text = (0x7c04_1000, 0x13_1000); + let plan = Plan::new(None, LOADER, text, Typing::Firmware).expect("the text"); + let (pool, root) = written(&plan); + let at = |phys: u64, writable: bool| Some(Translation { phys, pat: 0, writable }); + for (phys, writable) in [ + (text.0 - 1, true), + (text.0, false), + (text.0 + text.1 - 1, false), + (text.0 + text.1, true), + ] { + assert_eq!(walk(&pool, root, phys), at(phys, writable), "identity {phys:#x}"); + assert_eq!(walk(&pool, root, PHYS_OFFSET + phys), at(phys, writable), "PHYS_OFFSET + {phys:#x}"); + } + every_leaf_translates_in_both_views(&plan, &pool, root); +} diff --git a/loader/src/main.rs b/loader/src/main.rs index 00f31cd7c84..ed4e7a76719 100644 --- a/loader/src/main.rs +++ b/loader/src/main.rs @@ -76,6 +76,8 @@ struct LoadedKernel { pub entry_offset: usize, pub stack_offset: usize, pub stack_size: usize, + /// The text's whole pages, `(offset, len)` in `memory`. + pub text: (usize, usize), } /// `path` on the volume firmware loaded this image from: `LoadedImage`'s @@ -293,6 +295,20 @@ fn load_kernel_elf(kernel_elf_bytes: &[u8]) -> LoadedKernel { process_mem[vstart..vstart + src.len()].copy_from_slice(src); } + // The boot map's read-only text (`toyos_bootmap::Access`): the image's one + // executable segment, which no writable byte shares a page with. + let page = PAGE_SIZE as u64; + let mut executable = layout.segments().iter().filter(|segment| segment.flags().executable()); + let (Some(text), None) = (executable.next(), executable.next()) else { + panic!("kernel.elf: the boot map maps one executable segment, and this image has another number of them"); + }; + assert!(!text.writable(), "kernel.elf: its executable segment is writable"); + if let Some((a, b)) = layout.overlapping_load_pages(page) { + panic!("kernel.elf: PT_LOAD segments {a} and {b} share a page, which cannot have both their protections"); + } + let (text_start, text_end) = text.page_range(page); + let text = ((extent.min() + text_start) as usize, (text_end - text_start) as usize); + let rela_sections = sections.rela_sections().unwrap_or_else(|form| panic!("kernel.elf: {form} is not supported")); @@ -344,6 +360,7 @@ fn load_kernel_elf(kernel_elf_bytes: &[u8]) -> LoadedKernel { entry_offset: at(layout.entry()), stack_offset: placed.stack as usize, stack_size, + text, } } @@ -556,9 +573,13 @@ fn start_kernel(kernel: LoadedKernel, kernel_elf_bytes: vec::Vec, cmdline: v // pages by it. Before the exit, while the map can still be asked for; the // attributes a descriptor carries do not change across it. let write_back = write_back_memory(&system_table); + let kernel_phys = kernel.memory.as_ptr() as u64; + report_reach("Kernel image", kernel_phys, kernel.memory.len() as u64); + let text = (kernel_phys + kernel.text.0 as u64, kernel.text.1 as u64); let planned = Plan::new( gop.as_ref().map(|g| (g.framebuffer, g.framebuffer_size)), loader, + text, arch::typing(&write_back), ); match &planned { @@ -574,11 +595,12 @@ fn start_kernel(kernel: LoadedKernel, kernel_elf_bytes: vec::Vec, cmdline: v } let (at, len) = plan.loader(); println!("Loader image: {:#x}+{:#x}, mapped at identity as {at:#x}+{len:#x}", loader.0, loader.1); + println!("Kernel text: {:#x}+{:#x}, mapped read-only", text.0, text.1); } Err(why) => println!("Boot map: NO MAP HOLDS THIS MACHINE, {why}"), } let plan = planned - .unwrap_or_else(|why| panic!("the boot map cannot hold the scanout and the loader: {why}")); + .unwrap_or_else(|why| panic!("the boot map cannot hold the scanout, the loader and the kernel's text: {why}")); // SAFETY: `pt_mem` is the zeroed allocation of `pt_layout` above, which // nothing else holds, so it is a valid `[Table; MAX_PAGES]` of empty @@ -588,9 +610,6 @@ fn start_kernel(kernel: LoadedKernel, kernel_elf_bytes: vec::Vec, cmdline: v let pml4_phys = plan.write(arch::encoding::ENCODING, pool, pt_mem as u64); println!("Boot map: root {pml4_phys:#x}, {BOOT_MAP_BYTES:#x} bytes at identity and at PHYS_OFFSET"); - let kernel_phys = kernel.memory.as_ptr() as u64; - report_reach("Kernel image", kernel_phys, kernel.memory.len() as u64); - // An empty cmdline's pointer names nothing and is not reported as reachable. match (!cmdline.is_empty()).then_some((cmdline.as_ptr() as u64, cmdline.len() as u64)) { None => println!("Parameter buffer: none"), diff --git a/tests/common/faults.rs b/tests/common/faults.rs index 190a8d0cec2..eab226b15d8 100644 --- a/tests/common/faults.rs +++ b/tests/common/faults.rs @@ -98,3 +98,40 @@ pub fn crash_report_reads_no_kernel_memory(kernel: &Serial) -> Result<(), String None => Ok(()), } } + +/// What `kernel-wx-leaf` says of the leaf it stages (`kernel/src/main.rs`). +const STAGED: &str = "kernel-wx-leaf: the leaf at "; +/// The middle of the boot's audit refusal (`kernel/src/mm/mod.rs`). +const AUDIT_REFUSED: &str = " kernel leaves, writable and executable: "; + +/// The boot's audit of the kernel's tables against one direct-map leaf +/// `kernel-wx-leaf` made writable and executable: the refusal names that leaf +/// under both rules and counts no other, so every leaf the boot made itself +/// passed the same walk. +pub fn kernel_wx_leaf_is_refused(test_config: &Path, profile: qemu::Profile) -> Result<(), String> { + let qemu = QemuInstance::boot_with_options( + test_config, + &[], + &[], + BootOptions { profile, kernel_params: &["kernel-wx-leaf"], ready_marker: AUDIT_REFUSED, ..Default::default() }, + ); + let serial = qemu.boot_log(); + let at = serial + .lines() + .find_map(|line| line.split(STAGED).nth(1)?.split_whitespace().next()) + .ok_or_else(|| format!("no {STAGED:?} line\n{serial}"))?; + let refusal = serial + .lines() + .find(|line| line.contains(AUDIT_REFUSED)) + .map(str::trim_end) + .ok_or_else(|| format!("no audit refusal\n{serial}"))?; + let once = format!(": 1 (the first at {at})"); + let (both, outside) = (format!("writable and executable{once}"), "; executable outside the text "); + match refusal.split_once(outside) { + Some((writable, executable)) if writable.ends_with(&both) && executable.ends_with(&once) => { + eprintln!(" [kernel_wx] {refusal}"); + Ok(()) + } + _ => Err(format!("the audit did not name {at} alone under both rules: {refusal}\n{serial}")), + } +} diff --git a/tests/toyos.rs b/tests/toyos.rs index 3e3ec8bceae..d4e4e7581f9 100644 --- a/tests/toyos.rs +++ b/tests/toyos.rs @@ -290,6 +290,10 @@ const DRIVEN_AND_SHARED: &[&str] = &[ "wall_clock_now", ]; +/// What the boot's audit of the kernel's tables says when every leaf passed +/// (`kernel/src/mm/mod.rs`). +const KERNEL_WX_HOLDS: &str = " kernel leaves, none writable and executable, none executable outside the text "; + /// What `test-early-panic` panics with (`kernel/src/main.rs`): the last line its /// report puts on serial. const EARLY_PANIC_MESSAGE: &str = "test-early-panic: on-screen console check"; @@ -314,6 +318,8 @@ const SCREEN_TESTS: &[(&str, qemu::Profile)] = &[ ("screen_loader_clears", qemu::Profile::Metal), ("virt_early_panic", qemu::Profile::Virt), ("virt_early_fault", qemu::Profile::Virt), + // `kernel_wx_leaf_is_refused` on `virt`: no metal bench is AArch64. + ("virt_kernel_wx_leaf_is_refused", qemu::Profile::Virt), ("virt_el2_drop", qemu::Profile::VirtEl2NoVhe), ("virt_user_mode", qemu::Profile::Virt), ("virt_timer_preempts", qemu::Profile::Virt), @@ -436,6 +442,11 @@ const MACHINE_TESTS: &[&str] = &[ // The nested-NMI report is a raw write to the 16550, which the T14 does not // have. "nested_nmi_is_loud", + // The boot's W^X audit refusing a leaf an actuator staged: the panic ends + // the boot, so it cannot share one, and the T14's `kernel_wx` row reads + // the same audit passing. q35 on `Profile::Metal`, whose 16550 carries + // the panic. + "kernel_wx_leaf_is_refused", // A press of the fixed power button reaches the `acpi` claim's holder as // one SCI, its line masked until the holder serves it: the kernel's SCI // and its mask have no host build, QEMU raises the event on demand, and @@ -867,7 +878,7 @@ const METAL: &[(&str, metal::Metal)] = &[ ), // ---- the boot facts, riding the same tests/images/testcases.toml image ---- // - // **Six names and no extra minute of the machine.** Every needle below is a + // **Seven names and no extra minute of the machine.** Every needle below is a // record the *shipping* kernel writes on any boot it takes, so each rides // whatever image is already going on the stick; the two rows after them are // the two things this suite measures rather than reads, and those cost a @@ -914,6 +925,20 @@ const METAL: &[(&str, metal::Metal)] = &[ "pci_inventory", metal::Metal { arms: TESTCASES, judge: |b| pci_inventory(b[0].kernel().text()) }, ), + ( + // The shipping kernel's W^X audit of its own tables, on the firmware + // and memory map it was built for: the T14 and the AMD bench. Its + // negative control is `kernel_wx_leaf_is_refused`, on QEMU. + "kernel_wx", + metal::Metal { + arms: TESTCASES, + judge: |b| { + let kernel = b[0].kernel(); + eprintln!(" [kernel_wx] {}", kernel.must_say(KERNEL_WX_HOLDS)?); + Ok(()) + }, + }, + ), // ---- one image: tests/images/latencycase.toml armed with the shootdown bench ---- ( "tlb_shootdown_cost", @@ -3616,6 +3641,7 @@ fn run_screen_test(name: &str, profile: qemu::Profile, test_config: &Path) -> Re } Ok(()) } + "virt_kernel_wx_leaf_is_refused" => faults::kernel_wx_leaf_is_refused(test_config, profile), "virt_early_fault" => { // The vectors, judged by the one thing a broken table cannot do: // report. An undefined instruction right after the console step @@ -4005,6 +4031,7 @@ fn run_machine_test(name: &str, test_config: &Path) -> Result<(), String> { "amdvi_firmware_left" => common::iommu::amdvi_firmware_left(test_config), "amdvi_selftest" => common::iommu::amdvi_selftest(test_config), "nested_nmi_is_loud" => faults::nested_nmi_is_loud(test_config), + "kernel_wx_leaf_is_refused" => faults::kernel_wx_leaf_is_refused(test_config, qemu::Profile::Metal), "acpi_press_is_one_sci" => acpi_press_is_one_sci(), "acpi_mediated_access" => acpi_mediated_access(2), "acpi_lock_given_back_on_one_cpu" => acpi_mediated_access(1),