From e50e1e0758c90d8cdf5059f79ae12b7d7d1a2052 Mon Sep 17 00:00:00 2001 From: japabu Date: Wed, 21 Oct 2026 20:45:00 +0200 Subject: [PATCH 1/2] Kernel memory is never writable and executable, and a boot audit holds it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Stage 0 of the native memory design: kernel W^X on x86-64 and AArch64. Before this, the kernel's direct map was one writable, executable 2 MiB leaf per page (x86-64 `kernel_leaf` with no NX, AArch64 with PXN clear), and the image ran from it: its text was writable and its heap, stacks and every user frame's alias executable at CPL 0 / EL1. Now: - Kernel leaves take a `Prot`. The image's 2 MiB pages of the direct map are split into 4 KiB leaves from its PT_LOAD flags (`mm::Image`): text read-only and executable, rodata read-only and execute-never, data and the boot stack read-write and execute-never. Every other kernel leaf, the rest of the direct map, MMIO and the scanout included, is read-write and execute-never. The image still runs from the direct map, so its text has one mapping and no writable alias. - x86-64: `control_regs::init_cr0` becomes `init_paging`, which writes the declared CR0 (WP) and EFER (NXE) before a CPU loads the kernel's tables: the BSP before `mm::init`, an AP first in `ap_entry`. The NX CPUID check moves there, ahead of the write. The AP trampoline now writes the declared EFER less LMA from its data block instead of OR-ing in LME, so an AP enters long mode under the declaration and can walk NX entries even under `no-ap-control-regs`. - AArch64: SCTLR_EL1.WXN is declared. The entry writes it before translation turns on under the loader's tables, so those map the kernel's text read-only too: the boot map plan takes the text range (`toyos_bootmap::Access`), splits the pages it shares, and the AArch64 encoding makes everything else PXN. x86-64's encoding makes the text read-only and cannot use XD there (no NXE yet when the loader switches). - `kernel::wx::audit` walks the kernel's tables once boot mappings are made (after `iommu::drivers_settled`) and panics on any leaf that is writable and executable, or executable outside the text. A leaf is judged by its own bits only, so the audit is never looser than the hardware. It logs its pass with the leaf count. - `kernel-wx-leaf` stages one writable, executable direct-map leaf for the negative control. Bit positions: Intel SDM Vol. 3A §4.5 tables; AArch64 descriptors from Arm DDI 0487 M.d D8.3 (Tables D8-48, D8-52), WXN semantics D8.4.1.2.4, SCTLR_EL1.WXN bit 19 from Arm's system register XML (DDI 0601) 2023-12. Closes `issues/kernel-text-is-writable-and-every-kernel-page-executable.md`. Files what remains: the kernel's RELRO stays writable, and x86-64's boot map is writable and executable until each CPU leaves it. Co-Authored-By: Claude Opus 5.5 --- ...ds-the-idt-before-its-control-registers.md | 2 +- ...itable-and-every-kernel-page-executable.md | 23 -- ...s-linux-on-every-machine-toyos-supports.md | 2 +- issues/the-kernels-relro-stays-writable.md | 20 ++ ...64s-boot-map-is-writable-and-executable.md | 23 ++ kernel/pure/lib.rs | 1 + kernel/pure/wx.rs | 332 ++++++++++++++++++ kernel/smmu/src/table.rs | 16 +- kernel/src/actuator.rs | 5 + kernel/src/arch/aarch64/control_regs.rs | 10 +- kernel/src/arch/aarch64/paging.rs | 90 ++++- kernel/src/arch/x86_64/boot.rs | 8 +- kernel/src/arch/x86_64/control_regs.rs | 70 ++-- kernel/src/arch/x86_64/paging.rs | 104 +++++- kernel/src/arch/x86_64/smp.rs | 17 +- kernel/src/main.rs | 21 +- kernel/src/mm/image.rs | 73 ++++ kernel/src/mm/mod.rs | 34 +- lib/bootmap/src/aarch64.rs | 38 +- lib/bootmap/src/lib.rs | 88 ++++- lib/bootmap/src/x86_64.rs | 19 +- lib/bootmap/tests/plan.rs | 192 +++++++--- lib/bootmap/tests/walk.rs | 38 +- loader/src/main.rs | 27 +- tests/common/faults.rs | 37 ++ tests/toyos.rs | 29 +- 26 files changed, 1102 insertions(+), 217 deletions(-) delete mode 100644 issues/kernel-text-is-writable-and-every-kernel-page-executable.md create mode 100644 issues/the-kernels-relro-stays-writable.md create mode 100644 issues/x86-64s-boot-map-is-writable-and-executable.md create mode 100644 kernel/pure/wx.rs create mode 100644 kernel/src/mm/image.rs diff --git a/issues/an-ap-loads-the-idt-before-its-control-registers.md b/issues/an-ap-loads-the-idt-before-its-control-registers.md index 3a2807162d..6e37a6a06a 100644 --- a/issues/an-ap-loads-the-idt-before-its-control-registers.md +++ b/issues/an-ap-loads-the-idt-before-its-control-registers.md @@ -16,7 +16,7 @@ ordering is stated at the site. trampoline's long-mode half sets `CR4.PAE` and nothing else, then sets the GS base, then `lidt`s the kernel's table — and `CR4.OSFXSR` is not set until `percpu::init_ap` reaches `control_regs::init`, several Rust statements later. -Between them an AP runs `control_regs::init_cr0`, `pat::init` (MSR writes and a +Between them an AP runs `control_regs::init_paging`, `pat::init` (MSR writes and a CR0 round trip) and `mm::paging::load_kernel_flush` (a CR3 reload) with a table loaded whose every stub is `#UD` on entry. Any exception in that window is a triple fault with nothing on any channel. diff --git a/issues/kernel-text-is-writable-and-every-kernel-page-executable.md b/issues/kernel-text-is-writable-and-every-kernel-page-executable.md deleted file mode 100644 index ca32045164..0000000000 --- a/issues/kernel-text-is-writable-and-every-kernel-page-executable.md +++ /dev/null @@ -1,23 +0,0 @@ ---- -status: open -kind: defect -opened: 2026-09-29 ---- - -# Kernel text is writable, and every kernel page is executable - -The kernel maps the whole direct map `PRESENT | WRITE` with no `NX` -(`kernel_leaf`, which `paging::init` maps every 2 MiB with, -`kernel/src/arch/x86_64/paging.rs`), and the kernel image runs from -it, at `PHYS_OFFSET` plus its physical address. Its text can be overwritten, -and its heap, its stacks and every user page's direct-map alias can be -executed at CPL 0. Linux at `Ubuntu-6.8.0-142.142`, under the T14's -`CONFIG_STRICT_KERNEL_RWX=y`, makes text and rodata read-only and everything -else non-executable (`arch/x86/mm/init_64.c:1402-1447`), and under -`CONFIG_DEBUG_WX=y` walks the tables for a writable and executable mapping at -boot (`arch/x86/mm/dump_pagetables.c:434`). - -**Exit**: kernel text is mapped read-only and executable, rodata read-only and -`NX`, and every other kernel mapping `NX`; each boot walks the kernel tables -and panics on a writable executable leaf, and mapping the direct map as it is -today reds that walk. diff --git a/issues/the-kernel-is-at-least-as-secure-as-linux-on-every-machine-toyos-supports.md b/issues/the-kernel-is-at-least-as-secure-as-linux-on-every-machine-toyos-supports.md index ad84b18643..ef13162c9b 100644 --- a/issues/the-kernel-is-at-least-as-secure-as-linux-on-every-machine-toyos-supports.md +++ b/issues/the-kernel-is-at-least-as-secure-as-linux-on-every-machine-toyos-supports.md @@ -37,7 +37,7 @@ work needs. - `issues/every-syscall-runs-at-one-kernel-stack-offset.md` - `issues/kernel-functions-return-with-their-used-registers-intact.md` - `issues/a-threads-kernel-stack-has-no-guard-page.md` -- `issues/kernel-text-is-writable-and-every-kernel-page-executable.md` +- `issues/the-kernels-relro-stays-writable.md` - `issues/the-kernel-heap-has-none-of-slubs-hardening.md` - `issues/a-device-without-a-domain-of-its-own-reaches-all-memory.md` - `issues/user-programs-run-without-a-shadow-stack.md` diff --git a/issues/the-kernels-relro-stays-writable.md b/issues/the-kernels-relro-stays-writable.md new file mode 100644 index 0000000000..1c1622e373 --- /dev/null +++ b/issues/the-kernels-relro-stays-writable.md @@ -0,0 +1,20 @@ +--- +status: open +kind: defect +opened: 2026-10-21 +--- + +# The kernel's RELRO stays writable + +The kernel maps its own image page by page from its `PT_LOAD` flags +(`kernel/src/mm/image.rs`), so its `PT_GNU_RELRO` — `.data.rel.ro`, `.got` and +`.dynamic`, about 70 KiB on x86-64 — is read-write and execute-never for the +machine's life. The loader applies the image's relocations before the kernel +runs, so nothing needs to write it afterwards, and a stray kernel write can +rewrite a vtable or a function pointer there. Linux at the parity tag of +`issues/the-kernel-is-at-least-as-secure-as-linux-on-every-machine-toyos-supports.md` +maps the same data read-only. `toyos-elf` parses no `PT_GNU_RELRO`, which +`issues/an-executables-relro-stays-writable.md` needs too. + +Exit: the kernel's `PT_GNU_RELRO` pages are read-only in its tables from +`mm::init` on, and a boot check or test write shows it. diff --git a/issues/x86-64s-boot-map-is-writable-and-executable.md b/issues/x86-64s-boot-map-is-writable-and-executable.md new file mode 100644 index 0000000000..843fc865f9 --- /dev/null +++ b/issues/x86-64s-boot-map-is-writable-and-executable.md @@ -0,0 +1,23 @@ +--- +status: open +kind: defect +opened: 2026-10-21 +--- + +# x86-64's boot map is writable and executable + +The loader's x86-64 boot map (`lib/bootmap/src/x86_64.rs`) maps the kernel's +text read-only and every other page writable, and every page executable: bit 63 +is execute-disable only under `IA32_EFER.NXE`, and the loader switches to the +map from firmware's state, before the kernel declares `NXE`. The boot CPU runs +the kernel on it until `mm::init`, and each AP from its trampoline, whose page +at 0x8000 the map holds writable, until `smp::ap_entry` loads the kernel's +tables. The boot's audit (`mm::audit_wx`) walks the kernel's tables, not this +map. AArch64's boot map is not writable and executable anywhere: the kernel's +entry declares `SCTLR_EL1.WXN` before it turns translation on under it, and the +loader maps everything but the text execute-never. + +Exit: as on AArch64, the kernel's entry applies its declared `EFER` before it +switches to the boot map; every leaf there but the text's and the trampoline +page's is execute-disable, and those two are read-only; and the plan's host +tests show it. diff --git a/kernel/pure/lib.rs b/kernel/pure/lib.rs index 8395397748..9f13369f62 100644 --- a/kernel/pure/lib.rs +++ b/kernel/pure/lib.rs @@ -23,4 +23,5 @@ pub mod pipe; pub mod pointers; pub mod proclife; pub mod sched; +pub mod wx; pub mod xsave; diff --git a/kernel/pure/wx.rs b/kernel/pure/wx.rs new file mode 100644 index 0000000000..ad94391a84 --- /dev/null +++ b/kernel/pure/wx.rs @@ -0,0 +1,332 @@ +//! The boot's audit of the kernel's own translation tables: no leaf is both +//! writable and executable, and none is executable outside the kernel's text. +//! +//! A leaf is judged by its own bits. A table entry above it can only narrow +//! what the leaf grants, and no narrowing is credited, so the audit can be +//! stricter than the hardware and never looser. + +#![forbid(unsafe_code)] + +use core::ops::Range; + +/// One entry as the audit reads it. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub enum Entry { + /// Translates nothing. + Absent, + /// Names the next table down, at this physical address. + Table(u64), + /// Maps all of its span. + Leaf { writable: bool, executable: bool }, +} + +/// A four-level format of 512 eight-byte entries per table: the root is level +/// 0 and translates bits 47:39, level 3 translates bits 20:12. +pub trait Format { + /// `entry`, found at `level`. + fn read(level: usize, entry: u64) -> Entry; + /// The virtual address whose bits 47:0 are `bits`. + fn address(bits: u64) -> u64; +} + +/// Four-level paging, Intel SDM Vol. 3A §4.5, Tables 4-15 to 4-20, with +/// `IA32_EFER.NXE` set: a supervisor write needs `R/W` (bit 1), and bit 63 +/// (`XD`) forbids a fetch. +pub struct X86_64; + +impl Format for X86_64 { + fn read(level: usize, entry: u64) -> Entry { + const PRESENT: u64 = 1 << 0; + const WRITABLE: u64 = 1 << 1; + /// A PDPTE's 1 GiB page or a PDE's 2 MiB one; reserved in a PML4E. + const PAGE_SIZE: u64 = 1 << 7; + const EXECUTE_DISABLE: u64 = 1 << 63; + /// Bits 51:12, the widest `MAXPHYADDR` §4.1.4 allows. + const ADDRESS: u64 = 0x000F_FFFF_FFFF_F000; + if entry & PRESENT == 0 { + Entry::Absent + } else if level == 3 || (level > 0 && entry & PAGE_SIZE != 0) { + Entry::Leaf { writable: entry & WRITABLE != 0, executable: entry & EXECUTE_DISABLE == 0 } + } else { + Entry::Table(entry & ADDRESS) + } + } + + /// Canonical: bits 63:48 copy bit 47 (§3.3.7.1). + fn address(bits: u64) -> u64 { + (((bits << 16) as i64) >> 16) as u64 + } +} + +/// VMSAv8-64 stage 1, 4 KiB granule, under `TTBR1_EL1` with `T1SZ` = 16 and +/// Direct permissions, Arm ARM DDI 0487 M.d D8.3 (Tables D8-48 and D8-52): +/// bit 0 marks a valid descriptor, and bit 1 a table at levels 0 to 2 and a +/// page at level 3, where clear is reserved and treated as invalid; `AP[2]` +/// (bit 7) makes it read-only and `PXN` (bit 53) forbids an EL1 fetch. +pub struct Aarch64; + +impl Format for Aarch64 { + fn read(level: usize, entry: u64) -> Entry { + const VALID: u64 = 1 << 0; + const TABLE_OR_PAGE: u64 = 1 << 1; + const READ_ONLY: u64 = 1 << 7; + const PXN: u64 = 1 << 53; + /// Bits 47:12 of a 4 KiB granule's next-level table address. + const ADDRESS: u64 = 0x0000_FFFF_FFFF_F000; + match (entry & VALID != 0, entry & TABLE_OR_PAGE != 0, level) { + (false, _, _) | (true, false, 3) => Entry::Absent, + (true, true, 0..=2) => Entry::Table(entry & ADDRESS), + (true, _, _) => Entry::Leaf { writable: entry & READ_ONLY == 0, executable: entry & PXN == 0 }, + } + } + + /// `TTBR1_EL1` translates the top 2^(64 - T1SZ) bytes (D8.2.4). + fn address(bits: u64) -> u64 { + 0xFFFF_0000_0000_0000 | bits + } +} + +/// The leaves one rule caught: how many, and the lowest address among them. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub struct Finding { + pub leaves: u64, + pub first: u64, +} + +/// What the walk of one root found. +#[derive(Clone, Copy, PartialEq, Eq, Debug, Default)] +pub struct Verdict { + /// Every leaf it read. + pub leaves: u64, + pub writable_executable: Option, + /// An executable leaf not wholly inside the text. + pub executable_outside_text: Option, +} + +impl Verdict { + pub fn holds(&self) -> bool { + self.writable_executable.is_none() && self.executable_outside_text.is_none() + } +} + +/// Walk every table under `root`, reading a table at a physical address +/// through `table`, and judge each leaf against `text`, the virtual addresses +/// that may be executable. +pub fn audit<'t, F: Format>(root: &'t [u64; 512], text: Range, table: &impl Fn(u64) -> &'t [u64; 512]) -> Verdict { + let mut verdict = Verdict::default(); + descend::(root, 0, 0, &text, table, &mut verdict); + verdict +} + +fn descend<'t, F: Format>( + entries: &'t [u64; 512], + level: usize, + base: u64, + text: &Range, + table: &impl Fn(u64) -> &'t [u64; 512], + verdict: &mut Verdict, +) { + let shift = 39 - 9 * level as u32; + for (index, &entry) in entries.iter().enumerate() { + let bits = base | (index as u64) << shift; + match F::read(level, entry) { + Entry::Absent => {} + Entry::Table(phys) => descend::(table(phys), level + 1, bits, text, table, verdict), + Entry::Leaf { writable, executable } => { + verdict.leaves += 1; + let start = F::address(bits); + let end = start + (1 << shift); + if executable && writable { + note(&mut verdict.writable_executable, start); + } + if executable && !(text.start <= start && end <= text.end) { + note(&mut verdict.executable_outside_text, start); + } + } + } + } +} + +/// Entries are walked in ascending address order, so the first noted is the lowest. +fn note(finding: &mut Option, at: u64) { + match finding { + Some(found) => found.leaves += 1, + None => *finding = Some(Finding { leaves: 1, first: at }), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use alloc::boxed::Box; + use alloc::vec::Vec; + + /// Tables at physical addresses `0x1000 * (i + 1)`, the root first. + struct Tables(Vec>); + + impl Tables { + fn new(count: usize) -> Self { + Self((0..count).map(|_| Box::new([0; 512])).collect()) + } + + fn phys(index: usize) -> u64 { + 0x1000 * (index as u64 + 1) + } + + fn audit(&self, text: Range) -> Verdict { + let table = |phys: u64| -> &[u64; 512] { &self.0[(phys / 0x1000 - 1) as usize] }; + audit::(&self.0[0], text, &table) + } + } + + mod x86_64 { + use super::*; + + const P: u64 = 1; + const RW: u64 = 1 << 1; + const PS: u64 = 1 << 7; + const XD: u64 = 1 << 63; + const HIGH: u64 = 0xFFFF_8000_0000_0000; + + /// The kernel half's first slot down to one directory and one table of 4 KiB pages: + /// the shape the kernel's direct map takes around its image. + fn direct_map() -> Tables { + let mut t = Tables::new(4); + t.0[0][256] = Tables::phys(1) | P | RW; + t.0[1][0] = Tables::phys(2) | P | RW; + t.0[2][0] = Tables::phys(3) | P | RW; + for i in 1..512 { + t.0[2][i] = (i as u64) << 21 | P | RW | PS | XD; + } + t + } + + #[test] + fn text_read_only_and_executable_and_the_rest_neither_holds() { + let mut t = direct_map(); + t.0[3][0] = P | RW | XD; + t.0[3][1] = 0x1000 | P; + t.0[3][2] = 0x2000 | P | XD; + t.0[3][3] = 0x3000 | P | RW | XD; + let verdict = t.audit::(HIGH + 0x1000..HIGH + 0x2000); + assert_eq!(verdict, Verdict { leaves: 4 + 511, writable_executable: None, executable_outside_text: None }); + assert!(verdict.holds()); + } + + #[test] + fn a_writable_leaf_without_xd_is_caught_at_its_address() { + let mut t = direct_map(); + t.0[3][1] = 0x1000 | P; + t.0[2][7] = 7 << 21 | P | RW | PS; + t.0[2][9] = 9 << 21 | P | RW | PS; + let verdict = t.audit::(HIGH + 0x1000..HIGH + 0x2000); + let both = Some(Finding { leaves: 2, first: HIGH + (7 << 21) }); + assert_eq!(verdict.writable_executable, both); + assert_eq!(verdict.executable_outside_text, both); + assert!(!verdict.holds()); + } + + #[test] + fn a_read_only_leaf_without_xd_outside_the_text_is_caught() { + let mut t = direct_map(); + t.0[3][1] = 0x1000 | P; + t.0[3][2] = 0x2000 | P; + let verdict = t.audit::(HIGH + 0x1000..HIGH + 0x2000); + assert_eq!(verdict.writable_executable, None); + assert_eq!(verdict.executable_outside_text, Some(Finding { leaves: 1, first: HIGH + 0x2000 })); + } + + /// A 2 MiB leaf that holds the text but reaches past it is executable outside it. + #[test] + fn a_large_leaf_straddling_the_text_is_outside_it() { + let mut t = direct_map(); + t.0[2][0] = P | PS; + let verdict = t.audit::(HIGH + 0x1000..HIGH + 0x2000); + assert_eq!(verdict.executable_outside_text, Some(Finding { leaves: 1, first: HIGH })); + assert_eq!(verdict.writable_executable, None); + } + + /// `PS` makes a PDPTE a 1 GiB leaf, and in a PML4E it is not a size at all. + #[test] + fn page_size_ends_the_walk_below_the_root_only() { + let mut t = Tables::new(3); + t.0[0][256] = Tables::phys(1) | P | RW | PS; + t.0[1][3] = 3 << 30 | P | RW | PS; + let verdict = t.audit::(0..0); + assert_eq!(verdict.leaves, 1); + assert_eq!(verdict.writable_executable, Some(Finding { leaves: 1, first: HIGH + (3 << 30) })); + } + + /// The user half's addresses stay canonical, and a not-present entry's other bits are nothing. + #[test] + fn absent_entries_are_skipped_and_addresses_are_canonical() { + let mut t = Tables::new(3); + t.0[0][0] = Tables::phys(1) | P | RW; + t.0[0][255] = Tables::phys(2) | P | RW; + t.0[1][0] = RW | PS; + t.0[2][511] = 511 << 30 | P | RW | PS; + let verdict = t.audit::(0..0); + assert_eq!(verdict.leaves, 1); + assert_eq!(verdict.writable_executable, Some(Finding { leaves: 1, first: 0x0000_7FFF_C000_0000 })); + } + } + + mod aarch64 { + use super::*; + + const VALID: u64 = 1; + const TABLE: u64 = 1 << 1; + const RO: u64 = 1 << 7; + const PXN: u64 = 1 << 53; + const UXN: u64 = 1 << 54; + const HIGH: u64 = 0xFFFF_8000_0000_0000; + + fn direct_map() -> Tables { + let mut t = Tables::new(4); + t.0[0][256] = Tables::phys(1) | VALID | TABLE; + t.0[1][0] = Tables::phys(2) | VALID | TABLE; + t.0[2][0] = Tables::phys(3) | VALID | TABLE; + t.0[2][1] = 1 << 21 | VALID | PXN | UXN; + t + } + + #[test] + fn text_read_only_and_executable_and_the_rest_neither_holds() { + let mut t = direct_map(); + t.0[3][0] = VALID | TABLE | PXN | UXN; + t.0[3][1] = 0x1000 | VALID | TABLE | RO | UXN; + t.0[3][2] = 0x2000 | VALID | TABLE | RO | PXN | UXN; + let verdict = t.audit::(HIGH + 0x1000..HIGH + 0x2000); + assert_eq!(verdict, Verdict { leaves: 4, writable_executable: None, executable_outside_text: None }); + } + + /// `UXN` is EL0's: a block with `PXN` clear is executable at EL1 whatever it says. + #[test] + fn a_writable_block_without_pxn_is_caught() { + let mut t = direct_map(); + t.0[2][1] = 1 << 21 | VALID | UXN; + let verdict = t.audit::(HIGH + 0x1000..HIGH + 0x2000); + assert_eq!(verdict.writable_executable, Some(Finding { leaves: 1, first: HIGH + (1 << 21) })); + } + + /// A level 1 block is a leaf; bit 1 clear at level 3 is no page at all. + #[test] + fn blocks_end_the_walk_and_level_3_needs_bit_1() { + let mut t = direct_map(); + t.0[1][1] = 1 << 30 | VALID; + t.0[3][5] = 0x5000 | VALID; + let verdict = t.audit::(0..0); + assert_eq!(verdict.leaves, 2); + assert_eq!(verdict.writable_executable, Some(Finding { leaves: 1, first: HIGH + (1 << 30) })); + } + + /// Every address under `TTBR1_EL1` has its top sixteen bits set, the root's first slot included. + #[test] + fn addresses_are_ttbr1s() { + let mut t = Tables::new(1); + t.0[0][0] = VALID; + let verdict = t.audit::(0..0); + assert_eq!(verdict.executable_outside_text, Some(Finding { leaves: 1, first: 0xFFFF_0000_0000_0000 })); + } + } +} diff --git a/kernel/smmu/src/table.rs b/kernel/smmu/src/table.rs index f8f9a2cf66..e59081fba0 100644 --- a/kernel/smmu/src/table.rs +++ b/kernel/smmu/src/table.rs @@ -8,7 +8,7 @@ //! takes a message on is the one 4 KiB page at level 3: a block there would //! hand the device every register that shares its 2 MiB. -use toyos_bootmap::aarch64::{block, page, table, MAIR, PXN}; +use toyos_bootmap::aarch64::{block, page, table, MAIR}; use toyos_bootmap::Cache; use toyos_phys::Phys; @@ -39,10 +39,12 @@ const AP_READ_ONLY: u64 = 1 << 7; /// and no invalidation by ASID removes it. const NOT_GLOBAL: u64 = 1 << 11; -/// What a device's leaf adds to the loader's: unprivileged, its context's -/// alone, and executable at no level — the loader's memory is executable at -/// EL1, which no device's access is. -const DEVICES: u64 = AP_UNPRIVILEGED | NOT_GLOBAL | PXN; +/// What a device's leaf adds to the loader's data leaf, which is executable at +/// no level: unprivileged, and its context's alone. +const DEVICES: u64 = AP_UNPRIVILEGED | NOT_GLOBAL; + +/// The loader's leaves that are not the kernel's text. +const DATA: toyos_bootmap::Access = toyos_bootmap::Access::Data; /// What a device may do to memory mapped for it. #[derive(Clone, Copy, PartialEq, Eq, Debug)] @@ -100,9 +102,9 @@ pub const fn plan(at: u64, leaf: Leaf, unit: &Unit) -> Option { Access::Read => AP_READ_ONLY, Access::ReadWrite => 0, }; - (3, 1u64 << 21, memory.get(), block(memory.get(), Cache::Memory) | DEVICES | access) + (3, 1u64 << 21, memory.get(), block(memory.get(), Cache::Memory, DATA) | DEVICES | access) } - Leaf::Doorbell(register) => (4, 1u64 << 12, register.get(), page(register.get(), Cache::Device) | DEVICES), + Leaf::Doorbell(register) => (4, 1u64 << 12, register.get(), page(register.get(), Cache::Device, DATA) | DEVICES), }; if at >> INPUT_BITS != 0 || at & (size - 1) != 0 || !unit.reaches(output, size) { return None; diff --git a/kernel/src/actuator.rs b/kernel/src/actuator.rs index 573c8a13db..a6c2324be7 100644 --- a/kernel/src/actuator.rs +++ b/kernel/src/actuator.rs @@ -119,6 +119,11 @@ actuators! { /// Leave every AP holding the CR0/CR4 that INIT left it. no_ap_control_regs = "no-ap-control-regs"; + /// Make one direct-map leaf writable and executable before the boot's + /// audit of the kernel's tables, which must stop the boot on it. Judged by + /// `kernel_wx_leaf_is_refused` and `virt_kernel_wx_leaf_is_refused`. + kernel_wx_leaf = "kernel-wx-leaf"; + /// Hand every CPU the loader's Intel update with its header's revision /// raised by one and its checksum sealed again: the read-back's negative /// control on real Intel, whose payload still says the revision it did. diff --git a/kernel/src/arch/aarch64/control_regs.rs b/kernel/src/arch/aarch64/control_regs.rs index 6a652ab9a6..8c6b24b949 100644 --- a/kernel/src/arch/aarch64/control_regs.rs +++ b/kernel/src/arch/aarch64/control_regs.rs @@ -21,10 +21,12 @@ pub use toyos_bootmap::aarch64::MAIR; /// stack alignment checked at EL1 and EL0 (`SA`, `SA0`), AArch32 EL0's `IT` /// and `SETEND` disabled (`ITD`, `SED` — RES1 on a CPU with no AArch32 EL0, /// and nothing this kernel runs is AArch32), over the bits Armv8.0 makes -/// RES1 (29, 28, 23, 22, 20, 11). `WXN` stays clear: the direct map the -/// kernel runs from is writable and executable both. Little-endian at both -/// levels; EL0's cache maintenance and `WFI`/`WFE` trap. -pub const SCTLR: u64 = SCTLR_RES1 | 1 << 0 | 1 << 2 | 1 << 3 | 1 << 4 | 1 << 7 | 1 << 8 | 1 << 12; +/// RES1 (29, 28, 23, 22, 20, 11). `WXN` (bit 19, DDI 0601 2023-12): a page +/// writable at EL1 is never executed there, nor one writable at EL0 at EL0, +/// whatever its descriptor says (DDI 0487 M.d D8.4.1.2.4) — the loader's +/// tables, which map the kernel's text read-only, included. Little-endian at +/// both levels; EL0's cache maintenance and `WFI`/`WFE` trap. +pub const SCTLR: u64 = SCTLR_RES1 | 1 << 0 | 1 << 2 | 1 << 3 | 1 << 4 | 1 << 7 | 1 << 8 | 1 << 12 | 1 << 19; const SCTLR_RES1: u64 = 1 << 29 | 1 << 28 | 1 << 23 | 1 << 22 | 1 << 20 | 1 << 11; diff --git a/kernel/src/arch/aarch64/paging.rs b/kernel/src/arch/aarch64/paging.rs index 40af9f5b83..77d04ed215 100644 --- a/kernel/src/arch/aarch64/paging.rs +++ b/kernel/src/arch/aarch64/paging.rs @@ -22,6 +22,9 @@ //! translation dropped on every CPU (`super::tlb`), and only then written //! again (D8.14.1). An entry that was invalid owes nothing, since no TLB holds //! a translation that faulted. +//! +//! **No kernel leaf is both writable and executable**, and only the image's +//! text is executable (`mm::audit_wx`). use alloc::boxed::Box; use alloc::vec::Vec; @@ -102,10 +105,20 @@ fn user_leaf(prot: Prot, cache: CachePolicy) -> u64 { VALID | AF | NOT_GLOBAL | typed(cache) | prot.user_bits() } -/// A kernel leaf: EL1 read-write, EL0 nothing, global, and executable at EL1 -/// only where it is memory — the kernel runs from the direct map. -fn kernel_leaf(cache: CachePolicy) -> u64 { - VALID | AF | typed(cache) | UXN +impl Prot { + /// A kernel leaf's permissions: EL0 has none, and EL1 executes only code. + fn kernel_bits(self) -> u64 { + match self { + Self::Read => AP_READ_ONLY | PXN, + Self::ReadWrite => PXN, + Self::ReadExec => AP_READ_ONLY, + } + } +} + +/// A kernel leaf, bar its address and whether it is a block or a page: global. +fn kernel_leaf(cache: CachePolicy, prot: Prot) -> u64 { + VALID | AF | typed(cache) | prot.kernel_bits() | UXN } /// A leaf with its address and its block-or-page bit taken out: what two @@ -656,21 +669,29 @@ pub const MMIO_GRAIN: u64 = PAGE_4K; /// asked for and free. No invalidation is owed, since only an invalid entry is /// ever written. pub fn map_mmio(phys: u64, size: u64, policy: MmioPolicy) -> crate::mm::Mmio { - high().as_mut().expect("map_mmio: before the kernel's tables exist").map_direct(phys, size, kernel_leaf(policy.cache())); + high() + .as_mut() + .expect("map_mmio: before the kernel's tables exist") + .map_direct(phys, size, kernel_leaf(policy.cache(), Prot::ReadWrite)); let installed = kernel().lock().direct_map_policy(phys).expect("map_mmio: the window was just mapped"); assert!(installed == policy.cache(), "map_mmio: {phys:#x} installed {installed:?}"); crate::log!("mmio: {phys:#x}+{size:#x} {installed:?}"); crate::mm::Mmio::new(DirectMap::from_phys(phys), size) } -/// Build the kernel's tables — every page of memory firmware's map names, -/// and the console UART and the scanout the boot's own records reach — -/// switch `TTBR1_EL1` to them and `TTBR0_EL1` to the kernel's empty user -/// half, and answer where the direct map ends. -pub(crate) fn init(memory_map: &[MemoryMapEntry], scanout: Option<(u64, u64)>) -> toyos_bootmap::DirectMapEnd { +/// Build the kernel's tables — every page of memory firmware's map names, in +/// 4 KiB pages each as `image` says where the image lies, and the console +/// UART and the scanout the boot's own records reach — switch `TTBR1_EL1` to +/// them and `TTBR0_EL1` to the kernel's empty user half, and answer where the +/// direct map ends. +pub(crate) fn init( + memory_map: &[MemoryMapEntry], + scanout: Option<(u64, u64)>, + image: &crate::mm::Image, +) -> toyos_bootmap::DirectMapEnd { let extent = direct_map_end(memory_map).unwrap_or_else(|refusal| panic!("paging: firmware's memory map: {refusal}")); let end = extent.get(); - let memory = kernel_leaf(CachePolicy::Normal); + let memory = |phys: u64| kernel_leaf(CachePolicy::Normal, image.prot(phys).unwrap_or(Prot::ReadWrite)); let mut tables = Tables::new(); let (mut blocks, mut pages) = (0u64, 0u64); let mut at = 0; @@ -678,14 +699,15 @@ pub(crate) fn init(memory_map: &[MemoryMapEntry], scanout: Option<(u64, u64)>) - let va = PHYS_OFFSET + at; match coverage(memory_map, at) { Coverage::Nothing => {} - Coverage::Whole => { - tables.directory(va).set(index(va, 2), at | memory); + Coverage::Whole if !image.overlaps(at, PAGE_2M) => { + tables.directory(va).set(index(va, 2), at | memory(at)); blocks += 1; } - held @ Coverage::Pages(_) => { + held => { let mut table = Table::new(); for j in (0..512).filter(|&j| held.holds(j)) { - table.0[j as usize] = (at + j * PAGE_4K) | memory | TABLE; + let page = at + j * PAGE_4K; + table.0[j as usize] = page | memory(page) | TABLE; pages += 1; } let table = tables.adopt(table); @@ -695,10 +717,10 @@ pub(crate) fn init(memory_map: &[MemoryMapEntry], scanout: Option<(u64, u64)>) - at += PAGE_2M; } if let Some(uart) = super::console_uart::frame() { - tables.map_direct(uart, PAGE_4K, kernel_leaf(CachePolicy::Uncacheable)); + tables.map_direct(uart, PAGE_4K, kernel_leaf(CachePolicy::Uncacheable, Prot::ReadWrite)); } if let Some((scanout, size)) = scanout { - tables.map_direct(scanout, size, kernel_leaf(CachePolicy::WriteCombining)); + tables.map_direct(scanout, size, kernel_leaf(CachePolicy::WriteCombining, Prot::ReadWrite)); } KERNEL_TTBR1.store(tables.root.phys(), core::sync::atomic::Ordering::Release); @@ -783,6 +805,40 @@ pub fn bringup_root() -> u64 { /// holds a copy of the kernel's root slots to fall behind. pub(crate) fn seal_kernel_half() {} +/// [`kernel::wx::audit`] of the kernel's tables, `TTBR1_EL1`'s on every CPU; +/// the kernel space's `TTBR0_EL1` half maps nothing. +pub fn audit_wx(text: core::ops::Range) -> kernel::wx::Verdict { + let high = high(); + let tables = high.as_ref().expect("audit_wx: before the kernel's tables exist"); + // SAFETY: every table the kernel's root names is one this file linked, and + // the lock held keeps them in place while they are read. + let table = |phys: u64| unsafe { &Table::at(phys).0 }; + kernel::wx::audit::(&tables.root.0, text, &table) +} + +/// `kernel-wx-leaf`: the direct map's leaf over `phys` made executable at +/// EL1, so [`audit_wx`] has a writable executable leaf to find. Answers its +/// address. Nothing runs from it, so no invalidation is owed. +#[cfg(feature = "boot-actuators")] +pub fn stage_writable_executable(phys: u64) -> u64 { + let va = PHYS_OFFSET + phys; + let mut high = high(); + let tables = high.as_mut().expect("kernel-wx-leaf: before the kernel's tables exist"); + let i = index(va, 2); + let entry = tables.directory(va).0[i]; + let (table, slot, at) = if entry & (VALID | TABLE) == VALID { + (tables.directory(va), i, va & !(PAGE_2M - 1)) + } else { + // SAFETY: a table this root linked, reached under the lock. + (unsafe { Table::at_mut(entry & ADDR) }, index(va, 3), va & !(PAGE_4K - 1)) + }; + let leaf = table.0[slot]; + assert!(leaf & VALID != 0 && leaf & AP_READ_ONLY == 0, "kernel-wx-leaf: {phys:#x} is not under a writable leaf"); + table.set(slot, leaf & !PXN); + published(); + at +} + /// `PAR_EL1` after the MMU translated `addr` for an EL1 read in the tables /// this CPU runs on now (`AT S1E1R`; Arm ARM K.a, C6.2.x and D24.2.131): /// bit 0 set is a fault, and otherwise bits 63:56 are the memory type. diff --git a/kernel/src/arch/x86_64/boot.rs b/kernel/src/arch/x86_64/boot.rs index 548a3d3c6d..d1b28cc930 100644 --- a/kernel/src/arch/x86_64/boot.rs +++ b/kernel/src/arch/x86_64/boot.rs @@ -54,10 +54,10 @@ pub fn after_console(args: &KernelArgs, _maps: &[MemoryMapEntry]) { panic!("test-early-fault: x86-64 has no vectors of its own this early"); } // After actuator::init, whose table the `control-regs-bench` probe inside - // this call reads. `pat::init` restored the `CR0` it found, so a firmware - // `CD` — which would make every mapping uncacheable whatever the PAT - // says — ends here. - control_regs::init_cr0(0); + // this call reads, and before `mm::init` loads the kernel's tables. + // `pat::init` restored the `CR0` it found, so a firmware `CD` — which + // would make every mapping uncacheable whatever the PAT says — ends here. + control_regs::init_paging(0); // The first point with every channel a refusal reaches, and before // anything reads what an update changes: `random::key`'s RDRAND and // RDSEED, and `control_regs::init`'s CPUID. diff --git a/kernel/src/arch/x86_64/control_regs.rs b/kernel/src/arch/x86_64/control_regs.rs index 8ab158cad1..436ed9dcff 100644 --- a/kernel/src/arch/x86_64/control_regs.rs +++ b/kernel/src/arch/x86_64/control_regs.rs @@ -2,14 +2,16 @@ //! `IA32_APIC_BASE` and the performance request hold on every CPU in this //! machine. One declaration, applied by the BSP and every AP and checked on //! each; nothing else may write any of them but [`flush_global`], which puts -//! back the `CR4` it read. Each register is written whole: `CR0` and `EFER` are -//! constants, `CR4` is required bits plus whatever optional bits this CPU +//! back the `CR4` it read, and an AP's trampoline, which enters long mode +//! under [`EFER_BEFORE_LONG_MODE`]. Each register is written whole: `CR0` and +//! `EFER` are constants, `CR4` is required bits plus whatever optional bits this CPU //! offers, `XCR0` and the save form are what `kernel::xsave` makes of this //! CPU's CPUID, `IA32_XSS` is zero wherever it exists (no supervisor component //! is enabled), and `IA32_APIC_BASE` is the local APIC at its architectural address in the one //! mode CPUID offers — x2APIC where it exists, xAPIC where it does not. //! `EFER.NXE` lets bit 63 of a paging entry mean *not executable* -//! ([`Prot`](crate::mm::policy::Prot)). +//! ([`Prot`](crate::mm::policy::Prot)), and `CR0.WP` makes a read-only one +//! read-only at CPL 0: both are set before a CPU loads the kernel's tables. //! //! The performance request is HWP's: `IA32_HWP_INTERRUPT` where it exists, //! `IA32_PM_ENABLE`, `IA32_HWP_REQUEST` as `kernel::cpu::hwp_request` derives @@ -163,6 +165,11 @@ const _: () = assert!(CR4_OPTIONAL & CR4_FORBIDDEN == 0); /// here, never by `arch::syscall::init`, so one register keeps one owner. pub const EFER: u64 = efer::SCE | efer::LME | efer::LMA | efer::NXE; +/// [`EFER`] but `LMA`, which the CPU sets itself as paging turns on in +/// long mode (SDM Vol. 3A, "Initializing IA-32e Mode"): what an AP writes +/// before it does. +pub const EFER_BEFORE_LONG_MODE: u64 = EFER & !efer::LMA; + /// The declaration as the BSP computed it. Zero means not yet declared — also [`pcid_active`]'s correct answer before then. static DECLARED_CR4: AtomicU64 = AtomicU64::new(0); @@ -174,12 +181,24 @@ static DECLARED_XCR0: AtomicU64 = AtomicU64::new(0); /// evidence of the check having run and not a restatement of the CPU count. static CHECKED: AtomicU64 = AtomicU64::new(0); -/// Puts this CPU's `CR0` into [`CR0`]. [`pat::init`](super::pat::init) -/// restores the `CR0` it finds, so a firmware `CD` survives that write and is -/// cleared here: every AP runs this first, and the BSP runs it after, because -/// the BSP's `pat::init` has to precede the panel it would report a refusal on. -pub fn init_cr0(cpu_id: u32) { +/// Puts this CPU's `CR0` into [`CR0`] and its `IA32_EFER` into [`EFER`]: what +/// the kernel's tables need before this CPU loads them. +/// [`pat::init`](super::pat::init) restores the `CR0` it finds, so a firmware +/// `CD` survives that write and is cleared here: every AP runs this first, and +/// the BSP runs it after, because the BSP's `pat::init` has to precede the +/// panel it would report a refusal on. +pub fn init_paging(cpu_id: u32) { let before = bench::sample(); + // `NX` is `CPUID.80000001H:EDX[20]` (SDM Vol. 2A Table 3-8); the extended + // leaf must exist before it means anything. Asked first: setting `NXE` + // without it is `#GP`, and the BSP has no vectors of its own yet. + let (max_ext, _, _, _) = cpu::cpuid(0x8000_0000, 0); + let ext_edx = if max_ext >= 0x8000_0001 { cpu::cpuid(0x8000_0001, 0).3 } else { 0 }; + assert!( + ext_edx & (1 << 20) != 0, + "control_regs: cpu{cpu_id} has no NX bit, so no mapping this kernel writes could \ + be made non-executable and W^X would silently not exist", + ); if !skipped(cpu_id) { let live = cpu::read_cr0(); if live & (cr0::CD | cr0::NW) != 0 { @@ -193,8 +212,12 @@ pub fn init_cr0(cpu_id: u32) { } } // SAFETY: `CR0`'s value is this file's declaration, argued in its own - // doc comment. - unsafe { cpu::write_cr0(CR0) }; + // doc comment; `wrmsr` writes [`EFER`], whose `NXE` this CPU has just + // said it defines and whose `LMA` it, in long mode, already holds. + unsafe { + cpu::write_cr0(CR0); + cpu::wrmsr(efer::MSR, EFER); + } } bench::report(cpu_id, before); } @@ -234,10 +257,11 @@ fn hwp_declared(cpu_id: u32) -> Option { verdict.ok() } -/// Puts this CPU's `CR4`, `EFER`, `XCR0` and `IA32_XSS` into the declaration, -/// checks them against it, and declares the save form and the area they imply. Must run -/// after [`init_cr0`], before `arch::syscall::init`, which needs `SCE` set, -/// and before the first task starts, which loads and saves by `XCR0`. +/// Puts this CPU's `CR4`, `XCR0` and `IA32_XSS` into the declaration, checks +/// them, `CR0` and `EFER` against it, and declares the save form and the area +/// they imply. Must run after [`init_paging`], before `arch::syscall::init`, +/// which needs `SCE` set, and before the first task starts, which loads and +/// saves by `XCR0`. pub fn init(cpu_id: u32) { let declared = declaration(cpu_id); let (xcr0, form) = xsave_declaration(cpu_id); @@ -245,15 +269,12 @@ pub fn init(cpu_id: u32) { if !skipped(cpu_id) { // SAFETY: `write_cr4` faults only on an undefined bit, on clearing `PAE` // in long mode, or on `PCIDE` with a nonzero PCID — `declaration` checked - // the first two for both values and both callers use PCID 0; `wrmsr` - // writes [`EFER`], whose bits `declaration` has just confirmed this CPU - // defines and whose `LMA` is the one this CPU, in long mode, already holds. + // the first two for both values and both callers use PCID 0. unsafe { // Through `PGE` clear: only a change of `PGE` drops the global // translations a firmware that left it set may still hold. cpu::write_cr4(declared & !cr4::PGE); cpu::write_cr4(declared); - cpu::wrmsr(efer::MSR, EFER); // SAFETY: `CR4.OSXSAVE` is set by the write above, and // `xcr0_declaration` took only components this CPU enumerates, // AVX-512's three together and with AVX (SDM Vol. 1 §13.3). @@ -424,20 +445,13 @@ fn declaration(cpu_id: u32) -> u64 { ); let declared = CR4_REQUIRED | (have & CR4_OPTIONAL); - // `SYSCALL` and `NX` are `CPUID.80000001H:EDX` bits 11 and 20 (SDM Vol. - // 2A Table 3-8); the extended leaf must exist before they mean anything. - let (max_ext, _, _, _) = cpu::cpuid(0x8000_0000, 0); - let ext_edx = if max_ext >= 0x8000_0001 { cpu::cpuid(0x8000_0001, 0).3 } else { 0 }; + // `SYSCALL` is `CPUID.80000001H:EDX[11]` (SDM Vol. 2A Table 3-8), a leaf + // `init_paging` found. assert!( - ext_edx & (1 << 11) != 0, + cpu::cpuid(0x8000_0001, 0).3 & (1 << 11) != 0, "control_regs: cpu{cpu_id} has no SYSCALL/SYSRET, which is this kernel's only \ way into and out of Ring 3", ); - assert!( - ext_edx & (1 << 20) != 0, - "control_regs: cpu{cpu_id} has no NX bit, so no mapping this kernel writes could \ - be made non-executable and W^X would silently not exist", - ); // Changing `LA57` with paging on is `#GP`, so this only ever reads it. let live = cpu::read_cr4(); diff --git a/kernel/src/arch/x86_64/paging.rs b/kernel/src/arch/x86_64/paging.rs index 4e569046bc..cee598f6dc 100644 --- a/kernel/src/arch/x86_64/paging.rs +++ b/kernel/src/arch/x86_64/paging.rs @@ -5,7 +5,9 @@ // `arch::tlb::shootdown` is the caller's job for the rest of the machine. // A leaf is global if and only if it is the same in every root that will ever // be loaded, which is the kernel half and only [`kernel_leaf`] writes it; a -// mapping that differs between roots needs a constructor of its own. +// mapping that differs between roots needs a constructor of its own. No +// kernel leaf is both writable and executable, and only the image's text is +// executable (`mm::audit_wx`). use crate::log; use alloc::boxed::Box; @@ -39,7 +41,8 @@ const PAGE_GLOBAL: u64 = 1 << 8; const PAGE_PAT_2M: u64 = 1 << 12; /// The PAT bit of a 4 KiB PTE, where a PDE has its size bit. const PAGE_PAT_4K: u64 = 1 << 7; -/// Not-executable only because `arch::control_regs` asserts `EFER.NXE` on every CPU. +/// Not-executable only because `arch::control_regs` sets `EFER.NXE` on every +/// CPU before it loads these tables. const PAGE_NX: u64 = 1 << 63; const ADDR_MASK: u64 = 0x000F_FFFF_FFFF_F000; const ADDR_MASK_2M: u64 = 0x000F_FFFF_FFE0_0000; @@ -58,15 +61,33 @@ impl Prot { Self::ReadExec => common, } } + + /// The permission bits a kernel leaf carries. A supervisor write to a + /// read-only one faults because `CR0.WP` is declared (SDM Vol. 3A §4.6.1). + fn kernel_bits(self) -> u64 { + match self { + Self::Read => PAGE_NX, + Self::ReadWrite => PAGE_WRITE | PAGE_NX, + Self::ReadExec => 0, + } + } } /// Every bit of a 2 MiB kernel-half leaf but its address and size: the one -/// writer of [`PAGE_GLOBAL`]. Writable and executable, because the kernel's -/// image is in the direct map. -fn kernel_leaf(cache: CachePolicy) -> u64 { - PAGE_PRESENT | PAGE_WRITE | PAGE_GLOBAL | cache.pde_bits() +/// writer of [`PAGE_GLOBAL`]. +fn kernel_leaf(cache: CachePolicy, prot: Prot) -> u64 { + PAGE_PRESENT | PAGE_GLOBAL | prot.kernel_bits() | cache.pde_bits() } +/// [`kernel_leaf`] for a 4 KiB page of memory: `Normal` sets no PAT bit, the +/// one bit a 2 MiB leaf and a 4 KiB one place apart. +fn kernel_page(prot: Prot) -> u64 { + kernel_leaf(CachePolicy::Normal, prot) +} + +/// Every upper-level entry over the kernel half: its leaves decide. +const KERNEL_TABLE: u64 = PAGE_PRESENT | PAGE_WRITE; + /// Which PAT entry a 2 MiB mapping selects: `Normal` is entry 0 (WB), whose /// type for a range is the MTRR's (SDM Vol. 3A Table 11-7); `Uncacheable` is /// [`pat::UC_ENTRY`](crate::arch::pat::UC_ENTRY), UC under every MTRR type; @@ -712,7 +733,7 @@ impl AddressSpace { let end = (phys + size + PAGE_2M - 1) & !(PAGE_2M - 1); let mut cur = start; while cur < end { - self.map_2m(cur, kernel_leaf(cache)); + self.map_2m(cur, kernel_leaf(cache, Prot::ReadWrite)); cur += PAGE_2M; } crate::mm::Mmio::new(crate::mm::DirectMap::from_phys(phys), size) @@ -778,7 +799,7 @@ impl AddressSpace { } let pt_phys = pt.phys(); self.children.push(pt); - pd.write_pde(pd_idx, virt, pt_phys | PAGE_PRESENT | PAGE_WRITE).discharge(target); + pd.write_pde(pd_idx, virt, pt_phys | KERNEL_TABLE).discharge(target); } // SAFETY: `pd[pd_idx]` names a page table either way, permanently — @@ -794,7 +815,7 @@ impl AddressSpace { fn map_2m(&mut self, phys: u64, flags: u64) { let virt = crate::mm::DirectMap::from_phys(phys).as_ptr::() as u64; let pd_idx = indices(virt).2; - let pd = self.ensure_table(virt, flags); + let pd = self.ensure_table(virt, KERNEL_TABLE); let entry = phys | flags | PAGE_SIZE_BIT; let existing = pd[pd_idx]; assert!( @@ -809,6 +830,22 @@ impl AddressSpace { pd.write_pde(pd_idx, virt, entry).subsumed_by_flush(); } + /// The 2 MiB page of the direct map at `phys`, which the kernel's image + /// shares, in 4 KiB leaves each as the image says. Only `init` calls it, + /// before any TLB holds these tables. + fn map_image_pages(&mut self, phys: u64, image: &crate::mm::Image) { + let virt = crate::mm::DirectMap::from_phys(phys).as_ptr::() as u64; + let mut pt = Box::new(PageTablePage([0; 512])); + for i in 0..512 { + let page = phys + i as u64 * PAGE_4K; + pt.init_entry(i, page | kernel_page(image.prot(page).unwrap_or(Prot::ReadWrite))); + } + let pt_phys = pt.phys(); + self.children.push(pt); + let pd = self.ensure_table(virt, KERNEL_TABLE); + pd.write_pde(indices(virt).2, virt, pt_phys | KERNEL_TABLE).expect_install("map_image_pages"); + } + /// Masks `flags` to `TABLE_FLAGS`: bit 12 is a leaf's PAT bit but an /// upper entry's address bit, so leaf flags can't move a table 4 KiB. fn ensure_table(&mut self, va: u64, flags: u64) -> &mut PageTablePage { @@ -982,7 +1019,7 @@ pub fn debug_global_leaf(how: u64) -> u64 { unsafe { core::ptr::write_volatile(at.as_mut_ptr::(), at.phys()) }; } let (old_phys, new_phys) = (old.direct_map().phys(), new.direct_map().phys()); - let leaf = kernel_leaf(CachePolicy::Normal) | PAGE_SIZE_BIT; + let leaf = kernel_leaf(CachePolicy::Normal, Prot::ReadWrite) | PAGE_SIZE_BIT; // SAFETY: every read below is made while `SCRATCH` maps one of the two // frames this call owns, or a translation of one of them. let read = || unsafe { core::ptr::read_volatile(SCRATCH as *const u64) }; @@ -991,7 +1028,7 @@ pub fn debug_global_leaf(how: u64) -> u64 { let mut space = kernel().lock(); let _closed = crate::arch::IrqGuard::close(); let target = space.root(); - let pd = space.ensure_table(SCRATCH, leaf); + let pd = space.ensure_table(SCRATCH, KERNEL_TABLE); let idx = indices(SCRATCH).2; if pd[idx] & PAGE_PRESENT != 0 { return SyscallError::AlreadyExists.to_u64(); @@ -1032,11 +1069,16 @@ pub fn debug_global_leaf(how: u64) -> u64 { } } -/// Build kernel page tables: the direct map in the high half, in 2 MiB pages, -/// as far as [`toyos_bootmap::x86_64::direct_map_end`] reaches, and answer -/// that end. The scanout is not mapped for itself: in the low map the MTRRs -/// type it until `panic_console::remap` gives it a type of its own. -pub(crate) fn init(memory_map: &[MemoryMapEntry], _scanout: Option<(u64, u64)>) -> toyos_bootmap::DirectMapEnd { +/// Build kernel page tables: the direct map in the high half, in 2 MiB pages +/// of data but where `image` lies, as far as +/// [`toyos_bootmap::x86_64::direct_map_end`] reaches, and answer that end. The +/// scanout is not mapped for itself: in the low map the MTRRs type it until +/// `panic_console::remap` gives it a type of its own. +pub(crate) fn init( + memory_map: &[MemoryMapEntry], + _scanout: Option<(u64, u64)>, + image: &crate::mm::Image, +) -> toyos_bootmap::DirectMapEnd { let extent = toyos_bootmap::x86_64::direct_map_end(memory_map) .unwrap_or_else(|refusal| panic!("paging: firmware's memory map: {refusal}")); let end = extent.get(); @@ -1055,7 +1097,11 @@ pub(crate) fn init(memory_map: &[MemoryMapEntry], _scanout: Option<(u64, u64)>) } let mut addr: u64 = 0; while addr < end { - kernel.map_2m(addr, kernel_leaf(CachePolicy::Normal)); + if image.overlaps(addr, PAGE_2M) { + kernel.map_image_pages(addr, image); + } else { + kernel.map_2m(addr, kernel_leaf(CachePolicy::Normal, Prot::ReadWrite)); + } addr += PAGE_2M; } crate::log!("paging: the direct map covers 0x0..{end:#x}"); @@ -1091,6 +1137,30 @@ pub(crate) fn seal_kernel_half() { } } +/// [`kernel::wx::audit`] of the kernel's tables, whose kernel half every +/// address space shares. +pub fn audit_wx(text: core::ops::Range) -> kernel::wx::Verdict { + let space = kernel().lock(); + // SAFETY: every table the kernel's root names is one this module linked, + // and the lock held keeps them in place while they are read. + let table = |phys: u64| unsafe { &PageTablePage::from_phys(phys).0 }; + kernel::wx::audit::(&space.root.0, text, &table) +} + +/// `kernel-wx-leaf`: the direct map's leaf over `phys` made executable, so +/// [`audit_wx`] has a writable executable leaf to find. Answers its address. +#[cfg(feature = "boot-actuators")] +pub fn stage_writable_executable(phys: u64) -> u64 { + let virt = crate::mm::DirectMap::from_phys(phys).as_ptr::() as u64 & !(PAGE_2M - 1); + let mut space = kernel().lock(); + let target = space.root(); + let pd = space.ensure_table(virt, KERNEL_TABLE); + let pde = pd[indices(virt).2]; + assert!(pde & PAGE_SIZE_BIT != 0 && pde & PAGE_WRITE != 0, "kernel-wx-leaf: {phys:#x} is not under a writable 2 MiB leaf"); + pd.write_pde(indices(virt).2, virt, pde & !PAGE_NX).discharge(target); + virt +} + fn has(entry: u64, flag: u64) -> u8 { if entry & flag != 0 { 1 diff --git a/kernel/src/arch/x86_64/smp.rs b/kernel/src/arch/x86_64/smp.rs index 118206d2a5..5b4b9d72a6 100644 --- a/kernel/src/arch/x86_64/smp.rs +++ b/kernel/src/arch/x86_64/smp.rs @@ -74,9 +74,10 @@ struct TrampolineData { _pad5: [u8; 2], // +0x6E cs_reload_addr: u64, // +0x70 percpu_ptr: u64, // +0x78 + efer: u64, // +0x80 } -const _: () = assert!(size_of::() == 0x80); +const _: () = assert!(size_of::() == 0x88); @@ -107,7 +108,9 @@ macro_rules! asm_label_addr { }}; } -/// Copies the trampoline blob to physical page 0x8000 via the direct map; there is no identity map this early. +/// Copies the trampoline blob to physical page 0x8000 via the direct map, where +/// it is data and never executed; an AP, started only after the copy, fetches +/// it through the boot map's identity view. fn copy_trampoline() { let start = asm_label_addr!(_trampoline_start); let end = asm_label_addr!(_trampoline_end); @@ -167,6 +170,7 @@ fn build_trampoline_data() -> TrampolineData { _pad5: [0; 2], cs_reload_addr, percpu_ptr: 0, // filled per-AP + efer: crate::arch::control_regs::EFER_BEFORE_LONG_MODE, } } @@ -295,7 +299,7 @@ extern "C" fn ap_entry() -> ! { AP_TSC.store(cpu::rdtsc(), Ordering::Release); // Must run before `pat::init`, which restores the CR0 this call sets. - crate::arch::control_regs::init_cr0(percpu::cpu_id()); + crate::arch::control_regs::init_paging(percpu::cpu_id()); // Before `control_regs::init` reads what an update changes. crate::arch::microcode::load(percpu::cpu_id()); @@ -370,10 +374,11 @@ global_asm!( "mov eax, [0x8F00]", "mov cr3, eax", - // Enable Long Mode (IA32_EFER.LME) + // IA32_EFER from TrampolineData.efer (+0x80): long mode, and `NXE` before + // the kernel's tables, where bit 63 is otherwise a reserved bit. "mov ecx, 0xC0000080", - "rdmsr", - "or eax, 0x100", + "mov eax, [0x8F80]", + "mov edx, [0x8F84]", "wrmsr", // Enable Paging (CR0.PG) diff --git a/kernel/src/main.rs b/kernel/src/main.rs index 6625053190..625f448714 100644 --- a/kernel/src/main.rs +++ b/kernel/src/main.rs @@ -384,11 +384,14 @@ pub(crate) unsafe extern "C" fn kernel_main(loader_args: &mut KernelArgs) -> ! { // what `mm::init` withholds. let [image, elf, black_box, root, map, microcode] = loader; let reserved = [image, elf, black_box, root, map, microcode, arch::boot::reserved()]; + let layout = toyos_elf::Layout::parse(kernel_elf, arch::ELF_MACHINE) + .unwrap_or_else(|e| panic!("boot: the loader's kernel.elf: {e}")); + let image = mm::Image::new(&layout, image); // Before the first hash container, `mm::init`'s address space. hasher::seed(); - mm::init(maps, &reserved); + mm::init(maps, &reserved, &image); drivers::panic_console::remap(); // Before the first table is decoded for its contents: what a machine owner @@ -401,9 +404,7 @@ pub(crate) unsafe extern "C" fn kernel_main(loader_args: &mut KernelArgs) -> ! { let platform = arch::boot::interrupts(kernel_args.rsdp_addr); counters::bring_up(); symbols::set_kernel_base(kernel_args.kernel_memory_addr); - if !kernel_elf.is_empty() { - symbols::load_kernel(kernel_elf, mm::PHYS_OFFSET + kernel_args.kernel_memory_addr); - } + symbols::load_kernel(kernel_elf, mm::PHYS_OFFSET + kernel_args.kernel_memory_addr); arch::boot::clock(kernel_args); trace::enable(); @@ -502,6 +503,18 @@ pub(crate) unsafe extern "C" fn kernel_main(loader_args: &mut KernelArgs) -> ! { // userland runs and asks for a claim. iommu::drivers_settled(); + // Once the boot's own mappings are made: every later kernel leaf is + // written by the same constructors. + #[cfg(feature = "boot-actuators")] + if actuator::kernel_wx_leaf() { + let frame = mm::pmm::alloc_page().expect("kernel-wx-leaf: no frame to stage on"); + let at = mm::paging::stage_writable_executable(frame.direct_map().phys()); + log!("kernel-wx-leaf: the leaf at {at:#x} is writable and executable"); + #[expect(clippy::disallowed_methods, reason = "the audit below ends this boot on the leaf staged over it")] + core::mem::forget(frame); + } + mm::audit_wx(&image); + report_log_destination(); let complete = cpu::counter(); boot_phase!("complete", 0); diff --git a/kernel/src/mm/image.rs b/kernel/src/mm/image.rs new file mode 100644 index 0000000000..f730d75cbc --- /dev/null +++ b/kernel/src/mm/image.rs @@ -0,0 +1,73 @@ +//! The kernel's own image where the loader placed it, and what each of its +//! pages may be used for: its `PT_LOAD` segments' flags, page by page. + +use toyos_elf::{Layout, Segment}; + +use super::policy::Prot; +use super::{Region, PAGE_SIZE, PHYS_OFFSET}; + +/// The loader's allocation for the kernel: the image at its own link +/// addresses, then the boot stack. +pub struct Image<'a> { + memory: Region, + /// Where image offset 0 is. + base: u64, + segments: &'a [Segment], + /// The one executable segment's pages, physical. + text: Region, +} + +impl<'a> Image<'a> { + /// The image `layout` describes, loaded into `memory`. + /// + /// Panics where a page could not have a single protection that is not + /// both writable and executable: an executable segment that is writable, + /// two segments that share a page, or other than one executable segment, + /// which is what the loader's boot map maps read-only. + pub fn new(layout: &'a Layout, memory: Region) -> Self { + let segments = layout.segments(); + if let Some((a, b)) = layout.overlapping_load_pages(PAGE_SIZE) { + panic!("kernel image: PT_LOAD segments {a} and {b} share a page"); + } + let mut executable = segments.iter().filter(|segment| segment.flags().executable()); + let (Some(text), None) = (executable.next(), executable.next()) else { + panic!("kernel image: not exactly one executable PT_LOAD segment"); + }; + assert!(!text.writable(), "kernel image: its executable segment is writable"); + let base = memory.start + layout.extent().min(); + let (start, end) = text.page_range(PAGE_SIZE); + let text = Region { start: base + start, end: base + end }; + assert!(memory.start <= text.start && text.end <= memory.end, "kernel image: its text is outside its memory"); + Self { memory, base, segments, text } + } + + /// Whether any byte of `[phys, phys + len)` is the image's. + pub fn overlaps(&self, phys: u64, len: u64) -> bool { + phys < self.memory.end && self.memory.start < phys + len + } + + /// What the 4 KiB page at `phys` may be used for, or `None` for a page + /// that is not the image's: its segment's flags, and a page no segment + /// holds — the boot stack — is data. + pub fn prot(&self, phys: u64) -> Option { + if !self.overlaps(phys, PAGE_SIZE) { + return None; + } + let holder = phys.checked_sub(self.base).and_then(|offset| { + self.segments.iter().find(|segment| { + let (start, end) = segment.page_range(PAGE_SIZE); + (start..end).contains(&offset) + }) + }); + Some(match holder { + Some(segment) if segment.flags().executable() => Prot::ReadExec, + Some(segment) if !segment.writable() => Prot::Read, + _ => Prot::ReadWrite, + }) + } + + /// The text's pages, as the direct map addresses them. + pub fn text(&self) -> core::ops::Range { + PHYS_OFFSET + self.text.start..PHYS_OFFSET + self.text.end + } +} diff --git a/kernel/src/mm/mod.rs b/kernel/src/mm/mod.rs index 42538cedd8..494ee3c032 100644 --- a/kernel/src/mm/mod.rs +++ b/kernel/src/mm/mod.rs @@ -7,6 +7,7 @@ pub mod pmm; pub use crate::arch::paging; mod alloc; mod dma; +mod image; mod mmio; pub mod policy; mod region; @@ -22,6 +23,7 @@ pub use alloc::sweep as sweep_heap_bands; #[cfg(feature = "heap-lockspin")] pub use alloc::hold_lock as hold_heap_lock; pub use dma::{Dma, DmaPool, Unaligned}; +pub use image::Image; pub use mmio::Mmio; pub use region::{Allocation, KernelSlice}; pub use unmapped::Unmapped; @@ -191,17 +193,43 @@ pub fn firmware_map() -> &'static [MemoryMapEntry] { /// Call once at boot, in order: pmm (physical pages) → paging (direct map) → /// alloc (heap) → every kernel root slot, before the first user space copies -/// them. `reserved` holds `memory_map`'s own memory. -pub fn init(memory_map: &'static [MemoryMapEntry], reserved: &[Region]) { +/// them. `reserved` holds `memory_map`'s own memory and `image`'s. +pub fn init(memory_map: &'static [MemoryMapEntry], reserved: &[Region], image: &Image) { FIRMWARE_MAP.1.store(memory_map.len(), core::sync::atomic::Ordering::Release); FIRMWARE_MAP.0.store(memory_map.as_ptr().cast_mut(), core::sync::atomic::Ordering::Release); alloc::init_early(); pmm::init(memory_map, reserved); - DIRECT_MAP_END.set(paging::init(memory_map, crate::drivers::panic_console::scanout())); + DIRECT_MAP_END.set(paging::init(memory_map, crate::drivers::panic_console::scanout(), image)); alloc::init(); paging::seal_kernel_half(); } +/// Walk the kernel's tables and stop the boot on a leaf that is writable and +/// executable, or executable outside `image`'s text. +pub fn audit_wx(image: &Image) { + let text = image.text(); + let verdict = paging::audit_wx(text.clone()); + let said = |finding: Option| match finding { + Some(found) => ::alloc::format!("{} (the first at {:#x})", found.leaves, found.first), + None => ::alloc::string::String::from("none"), + }; + assert!( + verdict.holds(), + "paging: of {} kernel leaves, writable and executable: {}; executable outside the text {:#x}..{:#x}: {}", + verdict.leaves, + said(verdict.writable_executable), + text.start, + text.end, + said(verdict.executable_outside_text), + ); + crate::log!( + "paging: {} kernel leaves, none writable and executable, none executable outside the text {:#x}..{:#x}", + verdict.leaves, + text.start, + text.end + ); +} + /// The two memory facts every crash report ends its contexts with: how deep /// any task's kernel stack went, and what the heap sweep last saw. /// Unconditional: it runs on every crash and must build with no sweep present. diff --git a/lib/bootmap/src/aarch64.rs b/lib/bootmap/src/aarch64.rs index 48a46d25ad..b70ca8b62f 100644 --- a/lib/bootmap/src/aarch64.rs +++ b/lib/bootmap/src/aarch64.rs @@ -6,7 +6,7 @@ use toyos_abi::boot::MemoryMapEntry; -use crate::{is_read_as_memory, Cache, DirectMapEnd, Encoding, Refusal, DIRECT_MAP_WINDOW, PAGE_2M, PAGE_4K}; +use crate::{is_read_as_memory, Access, Cache, DirectMapEnd, Encoding, Refusal, DIRECT_MAP_WINDOW, PAGE_2M, PAGE_4K}; /// `MAIR_EL1` index 0: Device-nGnRE, registers. pub const ATTR_DEVICE: u64 = 0; @@ -32,6 +32,8 @@ const INNER_SHAREABLE: u64 = 0b11 << 8; const OUTER_SHAREABLE: u64 = 0b10 << 8; /// The access flag, bit 10: set, so the first access does not fault. const AF: u64 = 1 << 10; +/// `AP[2]`, bit 7: read-only. `AP[1]`, bit 6, stays clear: EL0 has no access. +const READ_ONLY: u64 = 1 << 7; /// Privileged execute-never, bit 53. pub const PXN: u64 = 1 << 53; /// Unprivileged execute-never, bit 54. @@ -45,26 +47,32 @@ pub const fn table(phys: u64) -> u64 { phys | TABLE } -/// A 2 MiB block at `phys`: EL1 read-write and EL0 nothing (`AP` = 0b00), and -/// executable at EL1 only where it is memory — the kernel's image is. A -/// device is never executable, since a speculative fetch from one is a read -/// of its registers. -pub const fn block(phys: u64, cache: Cache) -> u64 { - phys | BLOCK | AF | attributes(cache) +/// A 2 MiB block at `phys`: EL0 nothing, and at EL1 the kernel's text +/// read-only and executable and everything else read-write and never +/// executable. A device is not executable even as text, since a speculative +/// fetch from one is a read of its registers. +pub const fn block(phys: u64, cache: Cache, access: Access) -> u64 { + phys | BLOCK | AF | attributes(cache, access) } -const fn attributes(cache: Cache) -> u64 { - match cache { - Cache::Memory => ATTR_NORMAL << 2 | INNER_SHAREABLE | UXN, - Cache::Device => ATTR_DEVICE << 2 | PXN | UXN, - Cache::Scanout => ATTR_NORMAL_NC << 2 | OUTER_SHAREABLE | PXN | UXN, +const fn attributes(cache: Cache, access: Access) -> u64 { + let typed = match cache { + Cache::Memory => ATTR_NORMAL << 2 | INNER_SHAREABLE, + Cache::Device => ATTR_DEVICE << 2, + Cache::Scanout => ATTR_NORMAL_NC << 2 | OUTER_SHAREABLE, Cache::Firmware => panic!("AArch64 has no range registers to type memory: its plan types by the map"), - } + }; + let access = match (access, cache) { + (Access::Text, Cache::Memory) => READ_ONLY, + (Access::Text, _) => READ_ONLY | PXN, + (Access::Data, _) => PXN, + }; + typed | access | UXN } /// A 4 KiB page at `phys`, with [`block`]'s attributes. -pub const fn page(phys: u64, cache: Cache) -> u64 { - phys | PAGE | AF | attributes(cache) +pub const fn page(phys: u64, cache: Cache, access: Access) -> u64 { + phys | PAGE | AF | attributes(cache, access) } /// The three, for [`Plan::write`](crate::Plan::write). diff --git a/lib/bootmap/src/lib.rs b/lib/bootmap/src/lib.rs index 25019bd86a..689c5ee6c0 100644 --- a/lib/bootmap/src/lib.rs +++ b/lib/bootmap/src/lib.rs @@ -5,7 +5,8 @@ //! there is one mapping in the machine, and everything the kernel touches in //! that window has to be in it — its own image, the boot parameter, and the //! panel it reports a wedge on — and so is the loader's own image, wherever -//! firmware put it: on x86-64 the switch runs from it. +//! firmware put it: on x86-64 the switch runs from it. The kernel's text is +//! read-only in it ([`Access`]). //! //! Both architectures walk the same shape — a root (x86-64's PML4, AArch64's //! L0), a table per 512 GiB (PDPT, L1), a table per GiB (page directory, L2) @@ -19,9 +20,10 @@ //! every directory. The map reaches as far as the high half can hold, and //! refuses by name what lies past it. //! -//! Pure: the scanout, the loader's image and, where the architecture types -//! memory by firmware's map, the write-back ranges in; a [`Plan`] out, which -//! [`Plan::write`] lays into a pool of [`Table`]s the loader allocates. +//! Pure: the scanout, the loader's image, the kernel's text and, where the +//! architecture types memory by firmware's map, the write-back ranges in; a +//! [`Plan`] out, which [`Plan::write`] lays into a pool of [`Table`]s the +//! loader allocates. //! //! What the kernel takes from firmware's map when it builds its own tables is //! here too, because it may never map less than this map did: which types the @@ -148,8 +150,9 @@ pub const PAGE_4K: u64 = 4096; /// Leaves in one table. const PAGES_PER_TABLE: u64 = PAGE_2M / PAGE_4K; -/// The 2 MiB pages a scanout covers only in part: at most its first and its last. -const FINE_TABLES: usize = 2; +/// The 2 MiB pages split into 4 KiB leaves: the first and last a scanout +/// covers only in part, and the same two of the kernel's text. +const FINE_TABLES: usize = 4; /// The pool a builder needs: a root, every second-level table, every /// directory, and a table of 4 KiB leaves per split page. @@ -170,8 +173,8 @@ impl Table { #[derive(Clone, Copy)] pub struct Encoding { pub table: fn(u64) -> u64, - pub block: fn(u64, Cache) -> u64, - pub page: fn(u64, Cache) -> u64, + pub block: fn(u64, Cache, Access) -> u64, + pub page: fn(u64, Cache, Access) -> u64, } /// Why a machine's memory does not fit these tables. @@ -192,6 +195,8 @@ pub enum Refusal { /// A range of firmware's map that begins or ends here, off the 4 KiB page /// UEFI describes memory in. OffPage(u64), + /// The kernel's text is not whole 4 KiB pages inside [`BOOT_MAP_BYTES`]. + Text { base: u64, len: u64 }, } impl fmt::Display for Refusal { @@ -216,6 +221,10 @@ impl fmt::Display for Refusal { "a range ends at {end:#x}, past the {DIRECT_MAP_WINDOW:#x} bytes a direct map can hold" ), Self::OffPage(at) => write!(f, "a range of firmware's map is bounded at {at:#x}, off a {PAGE_4K:#x}-byte page"), + Self::Text { base, len } => write!( + f, + "the kernel's text at {base:#x}+{len:#x} is not whole {PAGE_4K:#x}-byte pages below {BOOT_MAP_BYTES:#x}" + ), } } } @@ -234,6 +243,16 @@ pub enum Cache { Scanout, } +/// What a leaf may be used for. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub enum Access { + /// The kernel's text: read and executed, never written. + Text, + /// Everything else: read and written. Where the architecture's encoding + /// can, never executed. + Data, +} + /// How the pages that are not the scanout are typed. #[derive(Clone, Copy, PartialEq, Eq, Debug)] pub enum Typing<'a> { @@ -280,6 +299,7 @@ pub struct Entry { pub phys: u64, pub slot: Slot, pub cache: Cache, + pub access: Access, } /// Where a machine's memory, its scanout and the loader itself go in the @@ -297,12 +317,14 @@ pub struct Plan<'a> { scanout: Option<(u64, u64)>, /// The loader's image in whole 2 MiB pages: `(first page, bytes)`. loader: (u64, u64), + /// The kernel's text, `(base, end)`. + text: (u64, u64), typing: Typing<'a>, } impl<'a> Plan<'a> { /// Lay out [`BOOT_MAP_BYTES`], `scanout` and `loader`, typed by `typing`, - /// or say why they cannot be. + /// with `text` read-only, or say why they cannot be. /// /// `scanout` is firmware's framebuffer as firmware reports it. Under /// [`Typing::Firmware`] its base must be 2 MiB aligned and its length is @@ -319,7 +341,21 @@ impl<'a> Plan<'a> { /// so its pages are rounded out both ways and typed as the rest of memory. /// /// A byte of either past [`DIRECT_MAP_WINDOW`] is [`Refusal::PastWindow`]. - pub fn new(scanout: Option<(u64, u64)>, loader: (u64, u64), typing: Typing<'a>) -> Result { + /// + /// `text` is the kernel's text, `(base, len)` in whole 4 KiB pages inside + /// the low map, where the loader places the kernel. A 2 MiB page it covers + /// in part is split, so the pages beside it stay writable. + pub fn new( + scanout: Option<(u64, u64)>, + loader: (u64, u64), + text: (u64, u64), + typing: Typing<'a>, + ) -> Result { + let (base, len) = text; + let whole = base.is_multiple_of(PAGE_4K) && len.is_multiple_of(PAGE_4K) && len > 0; + if !whole || base.checked_add(len).is_none_or(|end| end > BOOT_MAP_BYTES) { + return Err(Refusal::Text { base, len }); + } let mut plan = Self { gibs: [0; MAX_DIRECTORIES], directories: 0, @@ -329,6 +365,7 @@ impl<'a> Plan<'a> { fines: 0, scanout: None, loader: whole_pages(loader.0, loader.1)?, + text: (base, base + len), typing, }; let scanout = scanout.map(|(base, len)| plan.scanout_extent(base, len)).transpose()?; @@ -360,6 +397,7 @@ impl<'a> Plan<'a> { if let Some((base, end)) = scanout { plan.split_scanout(base, end); } + plan.split_partial(plan.text.0, plan.text.1); // Every leaf that is not the scanout typed once here, so `entries` // cannot meet a refusal. for page in 0..BOOT_MAP_BYTES / PAGE_2M { @@ -400,9 +438,14 @@ impl<'a> Plan<'a> { /// Split the pages the scanout `base..end` covers in part, and record it. fn split_scanout(&mut self, base: u64, end: u64) { + self.split_partial(base, end); + self.scanout = Some((base, end - base)); + } + + /// Split the pages `base..end` covers in part: at most its first and its last. + fn split_partial(&mut self, base: u64, end: u64) { let first = base / PAGE_2M * PAGE_2M; let last = (end - 1) / PAGE_2M * PAGE_2M; - // At most the first and the last page are covered in part. for page in [first, last] { let whole = base <= page && page + PAGE_2M <= end; if !whole && !self.is_split(page) { @@ -410,7 +453,6 @@ impl<'a> Plan<'a> { self.fines += 1; } } - self.scanout = Some((base, end - base)); } /// Write this map over whatever `pool` holds, its first byte at physical @@ -436,10 +478,10 @@ impl<'a> Plan<'a> { for entry in self.entries() { match entry.slot { Slot::Directory { directory, index } => { - pool[first_directory + directory].0[index] = (encoding.block)(entry.phys, entry.cache) + pool[first_directory + directory].0[index] = (encoding.block)(entry.phys, entry.cache, entry.access) } Slot::Fine { table, index } => { - pool[first_fine + table].0[index] = (encoding.page)(entry.phys, entry.cache) + pool[first_fine + table].0[index] = (encoding.page)(entry.phys, entry.cache, entry.access) } } } @@ -494,6 +536,11 @@ impl<'a> Plan<'a> { self.loader } + /// The kernel's text, `(base, len)`. + pub fn text(&self) -> (u64, u64) { + (self.text.0, self.text.1 - self.text.0) + } + /// The pages the loader's image adds: those past the low map that neither /// the scanout nor a split page already holds. fn loader_pages(&self) -> impl Iterator + '_ { @@ -528,6 +575,7 @@ impl<'a> Plan<'a> { phys, slot: Slot::Fine { table, index: index as usize }, cache: self.cache_of(phys, PAGE_4K), + access: self.access_of(phys, PAGE_4K), } }) }); @@ -552,6 +600,16 @@ impl<'a> Plan<'a> { self.typing.of(phys, size).expect("`new` typed every leaf") } + /// A leaf is the text's only where all of it is, and a page the text + /// covers in part is split, so no leaf is the text's in part. + fn access_of(&self, phys: u64, size: u64) -> Access { + if self.text.0 <= phys && phys + size <= self.text.1 { + Access::Text + } else { + Access::Data + } + } + fn directory_slot(&self, phys: u64) -> (usize, usize) { let directory = self .directories() @@ -563,7 +621,7 @@ impl<'a> Plan<'a> { fn leaf(&self, phys: u64, cache: Cache) -> Entry { let (directory, index) = self.directory_slot(phys); - Entry { phys, slot: Slot::Directory { directory, index }, cache } + Entry { phys, slot: Slot::Directory { directory, index }, cache, access: self.access_of(phys, PAGE_2M) } } /// Name the directory for `gib`, unless it is already named. Infallible: diff --git a/lib/bootmap/src/x86_64.rs b/lib/bootmap/src/x86_64.rs index e8ac240cb7..5de9a5ab0b 100644 --- a/lib/bootmap/src/x86_64.rs +++ b/lib/bootmap/src/x86_64.rs @@ -5,7 +5,7 @@ use toyos_abi::boot::MemoryMapEntry; -use crate::{is_read_as_memory, Cache, DirectMapEnd, Encoding, Refusal, BOOT_MAP_BYTES, DIRECT_MAP_WINDOW, PAGE_2M}; +use crate::{is_read_as_memory, Access, Cache, DirectMapEnd, Encoding, Refusal, BOOT_MAP_BYTES, DIRECT_MAP_WINDOW, PAGE_2M}; /// One past the kernel direct map's last byte: [`BOOT_MAP_BYTES`], or the end /// of the highest range the kernel reads as memory in whole [`PAGE_2M`] pages, @@ -43,18 +43,27 @@ pub const fn table(phys: u64) -> u64 { /// PWT with the PAT bit clear), uncacheable under every MTRR type and under an /// unprogrammed PAT, which is what the machine has until the kernel's /// `pat::init`. -pub const fn block(phys: u64, cache: Cache) -> u64 { +/// +/// The text is read-only, and every leaf is executable: bit 63 means +/// execute-disable only under `IA32_EFER.NXE` and is reserved otherwise (§4.5), +/// and the loader switches to these tables, and each AP enters them, before +/// the kernel declares `NXE`. +pub const fn block(phys: u64, cache: Cache, access: Access) -> u64 { let typed = match cache { Cache::Firmware | Cache::Memory => 0, Cache::Device | Cache::Scanout => PCD | PWT, }; - phys | PRESENT | WRITABLE | PAGE_SIZE | typed + let writable = match access { + Access::Text => 0, + Access::Data => WRITABLE, + }; + phys | PRESENT | writable | PAGE_SIZE | typed } /// A 4 KiB leaf at `phys` (Table 4-19), typed as [`block`] types a 2 MiB one; /// a 4 KiB entry's PAT bit is bit 7, and it stays clear. -pub const fn page(phys: u64, cache: Cache) -> u64 { - block(phys, cache) & !PAGE_SIZE +pub const fn page(phys: u64, cache: Cache, access: Access) -> u64 { + block(phys, cache, access) & !PAGE_SIZE } /// The three, for [`Plan::write`](crate::Plan::write). diff --git a/lib/bootmap/tests/plan.rs b/lib/bootmap/tests/plan.rs index c55492552b..4fca49db0b 100644 --- a/lib/bootmap/tests/plan.rs +++ b/lib/bootmap/tests/plan.rs @@ -3,8 +3,8 @@ //! and a loader firmware put below the map's end or above it. use toyos_bootmap::{ - aarch64, x86_64, Cache, Entry, Plan, Refusal, Slot, Typing, BOOT_MAP_BYTES, DIRECT_MAP_WINDOW, MAX_PAGES, - PAGE_2M, PAGE_4K, ROOT_HIGH_HALF, ROOT_IDENTITY, + aarch64, x86_64, Access, Cache, Entry, Plan, Refusal, Slot, Typing, BOOT_MAP_BYTES, DIRECT_MAP_WINDOW, + MAX_PAGES, PAGE_2M, PAGE_4K, ROOT_HIGH_HALF, ROOT_IDENTITY, }; const GIB: u64 = 1 << 30; @@ -12,6 +12,8 @@ const GIB: u64 = 1 << 30; const PANEL: u64 = 0x7e9000; /// The loader as OVMF loads it on a 2 GiB guest: relocated low, and not on a page. const LOADER: (u64, u64) = (0x7e5f_0000, 0x3_5000); +/// A kernel's text on a 2 MiB page of its own, so it splits nothing. +const TEXT: (u64, u64) = (0x7c00_0000, PAGE_2M); /// Every leaf lands in a table the plan named, at an index inside it, no two /// land in the same place, and no leaf lands where a split page's table is @@ -63,7 +65,7 @@ fn directory(e: &Entry) -> usize { #[test] fn a_machine_with_no_framebuffer_is_the_low_map_and_nothing_else() { - let plan = Plan::new(None, LOADER, Typing::Firmware).expect("the low map alone"); + let plan = Plan::new(None, LOADER, TEXT, Typing::Firmware).expect("the low map alone"); assert_eq!(plan.directories(), [0, 1, 2, 3]); assert_eq!(plan.scanout(), None); assert_eq!(plan.entries().count() as u64, BOOT_MAP_BYTES / PAGE_2M); @@ -75,7 +77,7 @@ fn a_machine_with_no_framebuffer_is_the_low_map_and_nothing_else() { /// retypes pages the low map already holds. #[test] fn a_framebuffer_inside_the_low_map_adds_no_directory() { - let plan = Plan::new(Some((0xc000_0000, PANEL)), LOADER, Typing::Firmware).expect("inside the low map"); + let plan = Plan::new(Some((0xc000_0000, PANEL)), LOADER, TEXT, Typing::Firmware).expect("inside the low map"); assert_eq!(plan.directories(), [0, 1, 2, 3]); // Rounded up to whole pages, and up only. assert_eq!(plan.scanout(), Some((0xc000_0000, 0x80_0000))); @@ -89,7 +91,7 @@ fn a_framebuffer_inside_the_low_map_adds_no_directory() { /// the low map untouched. #[test] fn a_framebuffer_above_the_low_map_adds_its_own_directory() { - let plan = Plan::new(Some((256 * GIB, PANEL)), LOADER, Typing::Firmware).expect("above the low map"); + let plan = Plan::new(Some((256 * GIB, PANEL)), LOADER, TEXT, Typing::Firmware).expect("above the low map"); assert_eq!(plan.directories(), [0, 1, 2, 3, 256]); assert_eq!(plan.scanout(), Some((256 * GIB, 0x80_0000))); let mine: Vec = plan @@ -109,13 +111,13 @@ fn a_framebuffer_above_the_low_map_adds_its_own_directory() { /// past it, not the last GiB of it. #[test] fn a_framebuffer_at_the_boundary_is_outside_the_low_map() { - let plan = Plan::new(Some((BOOT_MAP_BYTES, PANEL)), LOADER, Typing::Firmware).expect("at the boundary"); + let plan = Plan::new(Some((BOOT_MAP_BYTES, PANEL)), LOADER, TEXT, Typing::Firmware).expect("at the boundary"); assert_eq!(plan.directories(), [0, 1, 2, 3, 4]); assert!(plan.entries().filter(|e| e.cache == Cache::Scanout).all(|e| directory(&e) == 4)); is_consistent(&plan); // One page below it is the low map's last page, and adds nothing. - let inside = Plan::new(Some((BOOT_MAP_BYTES - PAGE_2M, PAGE_2M)), LOADER, Typing::Firmware).expect("the last page"); + let inside = Plan::new(Some((BOOT_MAP_BYTES - PAGE_2M, PAGE_2M)), LOADER, TEXT, Typing::Firmware).expect("the last page"); assert_eq!(inside.directories(), [0, 1, 2, 3]); is_consistent(&inside); } @@ -126,7 +128,7 @@ fn a_framebuffer_at_the_boundary_is_outside_the_low_map() { #[test] fn a_framebuffer_that_straddles_the_low_maps_end_is_mapped_from_both_arms() { let base = BOOT_MAP_BYTES - PAGE_2M; - let plan = Plan::new(Some((base, 4 * PAGE_2M)), LOADER, Typing::Firmware).expect("across the end"); + let plan = Plan::new(Some((base, 4 * PAGE_2M)), LOADER, TEXT, Typing::Firmware).expect("across the end"); assert_eq!(plan.directories(), [0, 1, 2, 3, 4]); let mine: Vec<(u64, usize)> = plan .entries() @@ -152,7 +154,7 @@ fn a_framebuffer_that_straddles_the_low_maps_end_is_mapped_from_both_arms() { #[test] fn a_framebuffer_that_straddles_a_gib_claims_both() { let base = 8 * GIB - PAGE_2M; - let plan = Plan::new(Some((base, 4 * PAGE_2M)), LOADER, Typing::Firmware).expect("straddling"); + let plan = Plan::new(Some((base, 4 * PAGE_2M)), LOADER, TEXT, Typing::Firmware).expect("straddling"); assert_eq!(plan.directories(), [0, 1, 2, 3, 7, 8]); assert!(plan.fine_tables().is_empty()); is_consistent(&plan); @@ -160,7 +162,7 @@ fn a_framebuffer_that_straddles_a_gib_claims_both() { // With a loader straddling another, every directory the budget has; the // two split-page tables are a map-typed plan's, and this one splits nothing. let loader = (12 * GIB - 0x1000, 0x3_5000); - let full = Plan::new(Some((base, 4 * PAGE_2M)), loader, Typing::Firmware).expect("both straddling"); + let full = Plan::new(Some((base, 4 * PAGE_2M)), loader, TEXT, Typing::Firmware).expect("both straddling"); assert_eq!(full.directories(), [0, 1, 2, 3, 7, 8, 11, 12]); assert_eq!(full.regions(), [0]); is_consistent(&full); @@ -177,7 +179,7 @@ fn the_high_half_slot_is_the_top_nine_bits_of_phys_offset() { #[test] fn a_base_off_the_page_is_refused_rather_than_rounded_down() { - assert_eq!(Plan::new(Some((0xc000_1000, PANEL)), LOADER, Typing::Firmware), Err(Refusal::Unaligned(0xc000_1000))); + assert_eq!(Plan::new(Some((0xc000_1000, PANEL)), LOADER, TEXT, Typing::Firmware), Err(Refusal::Unaligned(0xc000_1000))); // The refusal names the address, because a machine owner reads it. assert!(Refusal::Unaligned(0xc000_1000).to_string().contains("0xc0001000")); } @@ -186,11 +188,11 @@ fn a_base_off_the_page_is_refused_rather_than_rounded_down() { fn a_range_no_map_can_hold_is_refused_by_name() { // Its own end does not fit an address, either as it is given... let base = u64::MAX - PAGE_2M + 1; - assert_eq!(Plan::new(Some((base, u64::MAX)), LOADER, Typing::Firmware), Err(Refusal::Extent { base, len: u64::MAX })); + assert_eq!(Plan::new(Some((base, u64::MAX)), LOADER, TEXT, Typing::Firmware), Err(Refusal::Extent { base, len: u64::MAX })); // ...or once rounded up to the page it must end on. - assert_eq!(Plan::new(Some((base, 1)), LOADER, Typing::Firmware), Err(Refusal::Extent { base, len: 1 })); + assert_eq!(Plan::new(Some((base, 1)), LOADER, TEXT, Typing::Firmware), Err(Refusal::Extent { base, len: 1 })); // Wider than the directories a plan may name, and told what it would need. - assert_eq!(Plan::new(Some((16 * GIB, 10 * GIB)), LOADER, Typing::Firmware), Err(Refusal::Directories(14))); + assert_eq!(Plan::new(Some((16 * GIB, 10 * GIB)), LOADER, TEXT, Typing::Firmware), Err(Refusal::Directories(14))); assert!(Refusal::Directories(14).to_string().contains("14 page directories")); } @@ -203,11 +205,17 @@ fn uncacheable_is_pat_entry_three_and_plain_memory_is_entry_zero() { const PCD: u64 = 1 << 4; const PAT_2M: u64 = 1 << 12; let at = 0x4020_0000; - assert_eq!(x86_64::block(at, Cache::Firmware), at | PRESENT_WRITABLE_2M); - assert_eq!(x86_64::block(at, Cache::Scanout), at | PRESENT_WRITABLE_2M | PCD | PWT); + assert_eq!(x86_64::block(at, Cache::Firmware, Access::Data), at | PRESENT_WRITABLE_2M); + assert_eq!(x86_64::block(at, Cache::Scanout, Access::Data), at | PRESENT_WRITABLE_2M | PCD | PWT); // The PAT bit is what would select entry 4, and nothing here sets it. for cache in [Cache::Firmware, Cache::Memory, Cache::Device, Cache::Scanout] { - assert_eq!(x86_64::block(at, cache) & PAT_2M, 0, "{cache:?}"); + assert_eq!(x86_64::block(at, cache, Access::Data) & PAT_2M, 0, "{cache:?}"); + } + // The text is the same leaf with R/W (bit 1) clear, and nothing sets XD (bit 63). + assert_eq!(x86_64::block(at, Cache::Firmware, Access::Text), at | 1 | 1 << 7); + assert_eq!(x86_64::page(at, Cache::Firmware, Access::Text), at | 1); + for access in [Access::Text, Access::Data] { + assert_eq!(x86_64::block(at, Cache::Firmware, access) & 1 << 63, 0, "{access:?}"); } assert_eq!(x86_64::table(0x1000), 0x1003); } @@ -217,7 +225,7 @@ fn uncacheable_is_pat_entry_three_and_plain_memory_is_entry_zero() { #[test] fn a_map_typed_plan_is_memory_where_firmware_says_write_back_and_a_device_elsewhere() { let ram = [(0x4000_0000, 0x4000_0000)]; - let plan = Plan::new(None, LOADER, Typing::ByMap(&ram)).expect("virt's map"); + let plan = Plan::new(None, LOADER, TEXT, Typing::ByMap(&ram)).expect("virt's map"); for entry in plan.entries() { let want = if (0x4000_0000..0x8000_0000).contains(&entry.phys) { Cache::Memory } else { Cache::Device }; assert_eq!(entry.cache, want, "{entry:?}"); @@ -229,56 +237,66 @@ fn a_map_typed_plan_is_memory_where_firmware_says_write_back_and_a_device_elsewh #[test] fn a_page_that_is_part_memory_is_refused_by_address() { let ram = [(0x4000_0000, 0x4000_0000 + 0x1000)]; - assert_eq!(Plan::new(None, LOADER, Typing::ByMap(&ram)), Err(Refusal::Mixed(0x8000_0000))); + assert_eq!(Plan::new(None, LOADER, TEXT, Typing::ByMap(&ram)), Err(Refusal::Mixed(0x8000_0000))); assert!(Refusal::Mixed(0x8000_0000).to_string().contains("0x80000000")); // Two adjacent ranges that meet inside a page make it whole. let split = [(0x4000_0000, 0x4010_0000 - 0x4000_0000), (0x4010_0000, 0x3ff0_0000)]; - assert!(Plan::new(None, LOADER, Typing::ByMap(&split)).is_ok()); + assert!(Plan::new(None, LOADER, TEXT, Typing::ByMap(&split)).is_ok()); } /// A scanout inside memory is the scanout's leaves, not a mixed page. #[test] fn a_scanout_inside_memory_is_typed_as_the_scanout() { let ram = [(0x4000_0000, 0x4000_0000)]; - let plan = Plan::new(Some((0x5000_0000, 0x10_0000)), LOADER, Typing::ByMap(&ram)).expect("inside"); + let plan = Plan::new(Some((0x5000_0000, 0x10_0000)), LOADER, TEXT, Typing::ByMap(&ram)).expect("inside"); let scanout: Vec = plan.entries().filter(|e| e.cache == Cache::Scanout).map(|e| e.phys).collect(); assert_eq!(scanout, (0..256).map(|page| 0x5000_0000 + page * PAGE_4K).collect::>()); is_consistent(&plan); // And a scanout over a page that is part memory takes that page whole. let part = [(0x4000_0000, 0x1000)]; - assert!(Plan::new(Some((0x4000_0000, PAGE_2M)), LOADER, Typing::ByMap(&part)).is_ok()); - assert_eq!(Plan::new(None, LOADER, Typing::ByMap(&part)), Err(Refusal::Mixed(0x4000_0000))); + assert!(Plan::new(Some((0x4000_0000, PAGE_2M)), LOADER, TEXT, Typing::ByMap(&part)).is_ok()); + assert_eq!(Plan::new(None, LOADER, TEXT, Typing::ByMap(&part)), Err(Refusal::Mixed(0x4000_0000))); } -/// The AArch64 descriptors, against the Arm ARM's bit positions (K.a, D8.3.1 -/// and D8.3.2): a table is 0b11; a block 0b01 with `AttrIndx` at 4:2, `SH` at -/// 9:8, `AF` at 10, `PXN` at 53 and `UXN` at 54; and `MAIR_EL1`'s bytes are the -/// Device-nGnRE, Normal write-back and Normal non-cacheable encodings of -/// D24.2.110. +/// The AArch64 descriptors, against the Arm ARM's bit positions (DDI 0487 +/// M.d, D8.3, Table D8-52): a table is 0b11; a block 0b01 with `AttrIndx` at +/// 4:2, `AP[2:1]` at 7:6, `SH` at 9:8, `AF` at 10, `PXN` at 53 and `UXN` at 54; +/// and `MAIR_EL1`'s bytes are the Device-nGnRE, Normal write-back and Normal +/// non-cacheable encodings of D24.2.110. #[test] fn the_aarch64_descriptors_carry_the_attribute_index_the_mair_names() { let at = 0x4020_0000; let attr_index = |d: u64| (d >> 2) & 0b111; - let memory = aarch64::block(at, Cache::Memory); - let device = aarch64::block(at, Cache::Device); - let scanout = aarch64::block(at, Cache::Scanout); - for d in [memory, device, scanout] { + let memory = aarch64::block(at, Cache::Memory, Access::Data); + let device = aarch64::block(at, Cache::Device, Access::Data); + let scanout = aarch64::block(at, Cache::Scanout, Access::Data); + let text = aarch64::block(at, Cache::Memory, Access::Text); + for d in [memory, device, scanout, text] { assert_eq!(d & 0b11, 0b01, "a block"); assert_ne!(d & 1 << 10, 0, "the access flag"); - assert_eq!(d & 0b11 << 6, 0, "EL1 read-write, EL0 nothing"); + assert_eq!(d & 1 << 6, 0, "EL0 nothing"); assert_ne!(d & 1 << 54, 0, "never executable at EL0"); assert_eq!(d & 0x0000_FFFF_FFE0_0000, at, "the output address"); } assert_eq!(aarch64::ATTRS[attr_index(memory) as usize], 0xFF); assert_eq!(aarch64::ATTRS[attr_index(device) as usize], 0x04); assert_eq!(aarch64::ATTRS[attr_index(scanout) as usize], 0x44); - assert_eq!(memory & 1 << 53, 0, "the kernel executes from memory"); - assert_ne!(device & 1 << 53, 0, "and never from a device"); + for d in [memory, device, scanout] { + assert_eq!(d & 1 << 7, 0, "EL1 read-write"); + assert_ne!(d & 1 << 53, 0, "never executable at EL1"); + } + assert_ne!(text & 1 << 7, 0, "the text is read-only"); + assert_eq!(text & 1 << 53, 0, "and the kernel executes it"); + assert_ne!(aarch64::block(at, Cache::Device, Access::Text) & 1 << 53, 0, "never a device's"); + assert_eq!(text & !(1 << 7 | 1 << 53), memory & !(1 << 53), "and is memory as the rest is"); assert_eq!((memory >> 8) & 0b11, 0b11, "memory is inner shareable"); assert_eq!(aarch64::MAIR, 0x44_FF_04); // A level-3 page is 0b11 with the same attributes. - assert_eq!(aarch64::page(0x5000_1000, Cache::Scanout), (scanout & !0x0000_FFFF_FFFF_F000 & !0b11) | 0x5000_1000 | 0b11); + assert_eq!( + aarch64::page(0x5000_1000, Cache::Scanout, Access::Data), + (scanout & !0x0000_FFFF_FFFF_F000 & !0b11) | 0x5000_1000 | 0b11 + ); assert_eq!(aarch64::table(0x1000), 0x1003); } @@ -290,7 +308,7 @@ fn the_aarch64_descriptors_carry_the_attribute_index_the_mair_names() { fn a_scanout_carved_out_of_ram_splits_the_pages_it_shares() { let ram = [(0x4000_0000, 0x8000_0000)]; let (base, len) = (0xbc7a_0000, 0x30_0000); - let plan = Plan::new(Some((base, len)), LOADER, Typing::ByMap(&ram)).expect("ramfb"); + let plan = Plan::new(Some((base, len)), LOADER, TEXT, Typing::ByMap(&ram)).expect("ramfb"); assert_eq!(plan.fine_tables(), [0xbc60_0000, 0xbca0_0000]); let scanout: Vec = plan.entries().filter(|e| e.cache == Cache::Scanout).collect(); let bytes: u64 = scanout @@ -309,7 +327,7 @@ fn a_scanout_carved_out_of_ram_splits_the_pages_it_shares() { assert_eq!(at(0xbc80_0000), Cache::Scanout); is_consistent(&plan); // The same framebuffer under range-register typing is refused, as before. - assert_eq!(Plan::new(Some((base, len)), LOADER, Typing::Firmware), Err(Refusal::Unaligned(base))); + assert_eq!(Plan::new(Some((base, len)), LOADER, TEXT, Typing::Firmware), Err(Refusal::Unaligned(base))); } /// A map-typed scanout must still start on a 4 KiB page. @@ -317,7 +335,7 @@ fn a_scanout_carved_out_of_ram_splits_the_pages_it_shares() { fn a_map_typed_scanout_off_a_small_page_is_refused() { let ram = [(0x4000_0000, 0x8000_0000)]; assert_eq!( - Plan::new(Some((0xbc7a_0800, 0x1000)), LOADER, Typing::ByMap(&ram)), + Plan::new(Some((0xbc7a_0800, 0x1000)), LOADER, TEXT, Typing::ByMap(&ram)), Err(Refusal::Unaligned(0xbc7a_0800)) ); } @@ -328,7 +346,7 @@ fn a_map_typed_scanout_off_a_small_page_is_refused() { #[test] fn a_loader_above_the_low_map_adds_its_own_directory_of_plain_memory() { let loader = (0x1_4000_0000, 0x3_4a00); - let plan = Plan::new(None, loader, Typing::Firmware).expect("above the low map"); + let plan = Plan::new(None, loader, TEXT, Typing::Firmware).expect("above the low map"); assert_eq!(plan.directories(), [0, 1, 2, 3, 5]); assert_eq!(plan.loader(), (0x1_4000_0000, PAGE_2M)); let mine: Vec<(u64, usize, Cache)> = plan @@ -343,7 +361,7 @@ fn a_loader_above_the_low_map_adds_its_own_directory_of_plain_memory() { /// A loader inside the low map is already mapped, and adds nothing. #[test] fn a_loader_inside_the_low_map_adds_nothing() { - let plan = Plan::new(None, LOADER, Typing::Firmware).expect("inside"); + let plan = Plan::new(None, LOADER, TEXT, Typing::Firmware).expect("inside"); assert_eq!(plan.directories(), [0, 1, 2, 3]); assert_eq!(plan.entries().count() as u64, BOOT_MAP_BYTES / PAGE_2M); assert_eq!(plan.loader(), (0x7e40_0000, 2 * PAGE_2M)); @@ -354,7 +372,7 @@ fn a_loader_inside_the_low_map_adds_nothing() { #[test] fn a_loader_across_a_page_is_both_pages() { let loader = (6 * GIB + PAGE_2M - 0x1000, 0x3_4a00); - let plan = Plan::new(None, loader, Typing::Firmware).expect("across a page"); + let plan = Plan::new(None, loader, TEXT, Typing::Firmware).expect("across a page"); assert_eq!(plan.loader(), (6 * GIB, 2 * PAGE_2M)); let mine: Vec = plan.entries().filter(|e| e.phys >= BOOT_MAP_BYTES).map(|e| e.phys).collect(); assert_eq!(mine, [6 * GIB, 6 * GIB + PAGE_2M]); @@ -365,7 +383,7 @@ fn a_loader_across_a_page_is_both_pages() { /// scanout holds is not written twice. #[test] fn a_loader_beside_the_scanout_shares_its_directory() { - let plan = Plan::new(Some((256 * GIB, PANEL)), (256 * GIB + 0x7f_0000, 0x3_4a00), Typing::Firmware) + let plan = Plan::new(Some((256 * GIB, PANEL)), (256 * GIB + 0x7f_0000, 0x3_4a00), TEXT, Typing::Firmware) .expect("one GiB for both"); assert_eq!(plan.directories(), [0, 1, 2, 3, 256]); let above: Vec<(u64, Cache)> = @@ -388,14 +406,14 @@ fn a_loader_beside_the_scanout_shares_its_directory() { #[test] fn a_map_typed_loader_is_memory() { let ram = [(0x4000_0000, 0x4000_0000), (5 * GIB, GIB)]; - let plan = Plan::new(None, (5 * GIB + 0x1000, 0x3_4a00), Typing::ByMap(&ram)).expect("in memory"); + let plan = Plan::new(None, (5 * GIB + 0x1000, 0x3_4a00), TEXT, Typing::ByMap(&ram)).expect("in memory"); let above: Vec<(u64, Cache)> = plan.entries().filter(|e| e.phys >= BOOT_MAP_BYTES).map(|e| (e.phys, e.cache)).collect(); assert_eq!(above, [(5 * GIB, Cache::Memory)]); is_consistent(&plan); let part = [(0x4000_0000, 0x4000_0000), (5 * GIB, 0x10_0000)]; assert_eq!( - Plan::new(None, (5 * GIB, 0x3_4a00), Typing::ByMap(&part)), + Plan::new(None, (5 * GIB, 0x3_4a00), TEXT, Typing::ByMap(&part)), Err(Refusal::Mixed(5 * GIB)) ); } @@ -403,16 +421,16 @@ fn a_map_typed_loader_is_memory() { #[test] fn a_loader_no_map_can_hold_is_refused_by_name() { assert_eq!( - Plan::new(None, (0x1_4000_0000, 0), Typing::Firmware), + Plan::new(None, (0x1_4000_0000, 0), TEXT, Typing::Firmware), Err(Refusal::Extent { base: 0x1_4000_0000, len: 0 }) ); // Its directories count against the same budget as the scanout's, and a // GiB both need is counted once. assert_eq!( - Plan::new(Some((16 * GIB, 4 * GIB)), (32 * GIB, 0x3_4a00), Typing::Firmware), + Plan::new(Some((16 * GIB, 4 * GIB)), (32 * GIB, 0x3_4a00), TEXT, Typing::Firmware), Err(Refusal::Directories(9)) ); - let shared = Plan::new(Some((16 * GIB, 4 * GIB)), (17 * GIB, 0x3_4a00), Typing::Firmware) + let shared = Plan::new(Some((16 * GIB, 4 * GIB)), (17 * GIB, 0x3_4a00), TEXT, Typing::Firmware) .expect("the loader inside the scanout's GiBs"); assert_eq!(shared.directories(), [0, 1, 2, 3, 16, 17, 18, 19]); is_consistent(&shared); @@ -424,7 +442,7 @@ fn a_loader_no_map_can_hold_is_refused_by_name() { #[test] fn a_framebuffer_past_512_gib_adds_its_own_second_level_table() { let base = 0xfc_9000_0000; - let plan = Plan::new(Some((base, 8_294_400)), LOADER, Typing::Firmware).expect("at 1010 GiB"); + let plan = Plan::new(Some((base, 8_294_400)), LOADER, TEXT, Typing::Firmware).expect("at 1010 GiB"); assert_eq!(base / GIB, 1010); assert_eq!(plan.directories(), [0, 1, 2, 3, 1010]); assert_eq!(plan.regions(), [0, 1]); @@ -440,7 +458,7 @@ fn a_framebuffer_past_512_gib_adds_its_own_second_level_table() { #[test] fn a_framebuffer_across_512_gib_names_both_second_level_tables() { let base = 512 * GIB - PAGE_2M; - let plan = Plan::new(Some((base, 4 * PAGE_2M)), (520 * GIB, 0x3_4a00), Typing::Firmware) + let plan = Plan::new(Some((base, 4 * PAGE_2M)), (520 * GIB, 0x3_4a00), TEXT, Typing::Firmware) .expect("across 512 GiB"); assert_eq!(plan.directories(), [0, 1, 2, 3, 511, 512, 520]); assert_eq!(plan.regions(), [0, 1]); @@ -450,13 +468,15 @@ fn a_framebuffer_across_512_gib_names_both_second_level_tables() { /// The most the map can be asked for: a scanout and a loader each straddling a /// 512 GiB boundary, four second-level tables past the low map's, the last -/// below the high half's last root slot. +/// below the high half's last root slot, and text that splits two pages. #[test] fn every_second_level_table_the_budget_has_fits_the_pool() { let scanout = (1024 * GIB - PAGE_2M, 4 * PAGE_2M); let loader = (DIRECT_MAP_WINDOW - 512 * GIB - 0x1000, 0x3_4a00); - let plan = Plan::new(Some(scanout), loader, Typing::Firmware).expect("both straddling"); + let text = (0x7c1f_f000, PAGE_2M + 0x2000); + let plan = Plan::new(Some(scanout), loader, text, Typing::Firmware).expect("both straddling"); assert_eq!(plan.regions(), [0, 1, 2, 254, 255]); + // Short of the pool only by the two pages a map-typed scanout splits. assert_eq!(1 + plan.regions().len() + plan.directories().len() + plan.fine_tables().len(), MAX_PAGES - 2); is_consistent(&plan); } @@ -466,14 +486,76 @@ fn every_second_level_table_the_budget_has_fits_the_pool() { #[test] fn a_range_past_the_window_is_refused_by_name() { assert_eq!( - Plan::new(Some((DIRECT_MAP_WINDOW, PANEL)), LOADER, Typing::Firmware), + Plan::new(Some((DIRECT_MAP_WINDOW, PANEL)), LOADER, TEXT, Typing::Firmware), Err(Refusal::PastWindow(DIRECT_MAP_WINDOW + 0x80_0000)) ); assert_eq!( - Plan::new(None, (DIRECT_MAP_WINDOW - 0x1000, 0x3_4a00), Typing::Firmware), + Plan::new(None, (DIRECT_MAP_WINDOW - 0x1000, 0x3_4a00), TEXT, Typing::Firmware), Err(Refusal::PastWindow(DIRECT_MAP_WINDOW + PAGE_2M)) ); - let last = Plan::new(Some((DIRECT_MAP_WINDOW - 0x80_0000, PANEL)), LOADER, Typing::Firmware).expect("the window's last GiB"); + let last = Plan::new(Some((DIRECT_MAP_WINDOW - 0x80_0000, PANEL)), LOADER, TEXT, Typing::Firmware).expect("the window's last GiB"); assert_eq!(last.regions(), [0, 255]); is_consistent(&last); } + +/// The kernel's text as rust-lld lays it out: inside one 2 MiB page, which is +/// split so exactly the text's 4 KiB pages are read-only. +#[test] +fn the_text_inside_one_page_splits_that_page() { + let text = (0x7c04_1000, 0x13_1000); + let plan = Plan::new(None, LOADER, text, Typing::Firmware).expect("the text"); + assert_eq!(plan.text(), text); + assert_eq!(plan.fine_tables(), [0x7c00_0000]); + let mine: Vec = plan.entries().filter(|e| e.access == Access::Text).map(|e| e.phys).collect(); + assert_eq!(mine, (0..0x131).map(|page| text.0 + page * PAGE_4K).collect::>()); + assert!(plan.entries().filter(|e| e.access == Access::Text).all(|e| matches!(e.slot, Slot::Fine { .. }))); + is_consistent(&plan); +} + +/// Text across two 2 MiB boundaries: the page wholly inside it is one +/// read-only leaf, and the two it shares are split. +#[test] +fn text_across_pages_splits_the_two_it_shares() { + let text = (0x7c1f_f000, PAGE_2M + 0x2000); + let plan = Plan::new(None, LOADER, text, Typing::Firmware).expect("the text"); + assert_eq!(plan.fine_tables(), [0x7c00_0000, 0x7c40_0000]); + let mine: Vec<(u64, bool)> = plan + .entries() + .filter(|e| e.access == Access::Text) + .map(|e| (e.phys, matches!(e.slot, Slot::Fine { .. }))) + .collect(); + assert_eq!(mine, [(0x7c20_0000, false), (0x7c1f_f000, true), (0x7c40_0000, true)]); + is_consistent(&plan); +} + +/// A scanout carved out of RAM beside the kernel's text shares its split page. +#[test] +fn text_beside_the_scanout_shares_its_split_page() { + let ram = [(0x4000_0000, 0x8000_0000)]; + let (base, len) = (0xbc7a_0000, 0x30_0000); + let text = (0xbc60_0000, 0x1_0000); + let plan = Plan::new(Some((base, len)), LOADER, text, Typing::ByMap(&ram)).expect("both"); + assert_eq!(plan.fine_tables(), [0xbc60_0000, 0xbca0_0000]); + let at = |phys: u64| plan.entries().find(|e| e.phys == phys).map(|e| (e.cache, e.access)); + assert_eq!(at(text.0), Some((Cache::Memory, Access::Text))); + assert_eq!(at(text.0 + text.1), Some((Cache::Memory, Access::Data))); + assert_eq!(at(base), Some((Cache::Scanout, Access::Data))); + is_consistent(&plan); +} + +/// Text off a 4 KiB page, empty, or reaching past the low map is refused by its extent. +#[test] +fn text_that_is_not_whole_pages_in_the_low_map_is_refused() { + let refused = [ + (0x7c04_1800, 0x1000), + (0x7c04_1000, 0x800), + (0x7c04_1000, 0), + (BOOT_MAP_BYTES - 0x1000, 0x2000), + (!0xfff, 0x1000), + ]; + for (base, len) in refused { + let plan = Plan::new(None, LOADER, (base, len), Typing::Firmware); + assert_eq!(plan, Err(Refusal::Text { base, len }), "{base:#x}+{len:#x}"); + } + assert!(Refusal::Text { base: 0x7c04_1800, len: 0x1000 }.to_string().contains("0x7c041800+0x1000")); +} diff --git a/lib/bootmap/tests/walk.rs b/lib/bootmap/tests/walk.rs index 5fcaafcdf4..e05b4e233c 100644 --- a/lib/bootmap/tests/walk.rs +++ b/lib/bootmap/tests/walk.rs @@ -3,7 +3,7 @@ //! than from the encoder, so a table named from the wrong slot or the wrong //! second-level table is an address that does not translate. -use toyos_bootmap::{x86_64, Cache, Plan, Table, Typing, BOOT_MAP_BYTES, MAX_PAGES, PAGE_2M, PAGE_4K}; +use toyos_bootmap::{x86_64, Access, Cache, Plan, Table, Typing, BOOT_MAP_BYTES, MAX_PAGES, PAGE_2M, PAGE_4K}; const GIB: u64 = 1 << 30; const PHYS_OFFSET: u64 = 0xFFFF_8000_0000_0000; @@ -14,6 +14,8 @@ const AT: u64 = 0x7f80_0000; const LOADER: (u64, u64) = (0x7e5f_0000, 0x3_5000); /// The AMD laptop's framebuffer, as its GOP reports it. const LAPTOP: (u64, u64) = (0xfc_9000_0000, 8_294_400); +/// A kernel's text on a 2 MiB page of its own, so it splits nothing. +const TEXT: (u64, u64) = (0x7c00_0000, PAGE_2M); /// Bits M-1:12 of an entry that names a page, for the widest M (52) §4.1.4 allows. const ADDRESS: u64 = 0x000F_FFFF_FFFF_F000; @@ -107,7 +109,8 @@ fn expected_pat(cache: Cache) -> u64 { } /// Every leaf the plan names translates, at identity and at `PHYS_OFFSET`, its -/// first byte and its last, to itself, writable and of its kind's type. +/// first byte and its last, to itself, of its kind's type, and writable unless +/// it is the text's. fn every_leaf_translates_in_both_views(plan: &Plan, pool: &[Table; MAX_PAGES], root: u64) { for entry in plan.entries() { let size = match entry.slot { @@ -115,7 +118,7 @@ fn every_leaf_translates_in_both_views(plan: &Plan, pool: &[Table; MAX_PAGES], r toyos_bootmap::Slot::Fine { .. } => PAGE_4K, }; for phys in [entry.phys, entry.phys + size - 1] { - let want = Translation { phys, pat: expected_pat(entry.cache), writable: true }; + let want = Translation { phys, pat: expected_pat(entry.cache), writable: entry.access == Access::Data }; assert_eq!(walk(pool, root, phys).as_ref(), Some(&want), "{entry:?} at identity"); assert_eq!(walk(pool, root, PHYS_OFFSET + phys).as_ref(), Some(&want), "{entry:?} at PHYS_OFFSET"); } @@ -128,7 +131,7 @@ fn every_leaf_translates_in_both_views(plan: &Plan, pool: &[Table; MAX_PAGES], r #[test] fn the_laptops_scanout_at_1010_gib_translates_in_both_views() { let (base, len) = LAPTOP; - let plan = Plan::new(Some(LAPTOP), LOADER, Typing::Firmware).expect("the laptop's plan"); + let plan = Plan::new(Some(LAPTOP), LOADER, TEXT, Typing::Firmware).expect("the laptop's plan"); let (pool, root) = written(&plan); assert_eq!(root, AT, "the root is the pool's first page"); @@ -138,7 +141,7 @@ fn the_laptops_scanout_at_1010_gib_translates_in_both_views() { assert_eq!(walk(&pool, root, phys), scanout(phys), "identity {phys:#x}"); assert_eq!(walk(&pool, root, PHYS_OFFSET + phys), scanout(phys), "PHYS_OFFSET + {phys:#x}"); } - for phys in [0, BOOT_MAP_BYTES - 1, LOADER.0] { + for phys in [0, TEXT.0 - 1, TEXT.0 + TEXT.1, BOOT_MAP_BYTES - 1, LOADER.0] { assert_eq!(walk(&pool, root, phys), memory(phys), "identity {phys:#x}"); assert_eq!(walk(&pool, root, PHYS_OFFSET + phys), memory(phys), "PHYS_OFFSET + {phys:#x}"); } @@ -156,7 +159,7 @@ fn the_laptops_scanout_at_1010_gib_translates_in_both_views() { /// second-level tables: each directory under its own 512 GiB's table. #[test] fn every_second_level_table_is_reached_from_its_own_root_slots() { - let across = Plan::new(Some((512 * GIB - PAGE_2M, 4 * PAGE_2M)), (520 * GIB, 0x3_4a00), Typing::Firmware) + let across = Plan::new(Some((512 * GIB - PAGE_2M, 4 * PAGE_2M)), (520 * GIB, 0x3_4a00), TEXT, Typing::Firmware) .expect("across 512 GiB"); let (pool, root) = written(&across); every_leaf_translates_in_both_views(&across, &pool, root); @@ -164,6 +167,7 @@ fn every_second_level_table_is_reached_from_its_own_root_slots() { let budget = Plan::new( Some((1024 * GIB - PAGE_2M, 4 * PAGE_2M)), (toyos_bootmap::DIRECT_MAP_WINDOW - 512 * GIB - 0x1000, 0x3_4a00), + TEXT, Typing::Firmware, ) .expect("both straddling"); @@ -178,9 +182,29 @@ fn every_second_level_table_is_reached_from_its_own_root_slots() { fn a_split_page_translates_through_its_table_of_small_leaves() { let ram = [(0x4000_0000, 0x8000_0000)]; let (base, len) = (0xbc7a_0000, 0x30_0000); - let plan = Plan::new(Some((base, len)), LOADER, Typing::ByMap(&ram)).expect("ramfb"); + let plan = Plan::new(Some((base, len)), LOADER, TEXT, Typing::ByMap(&ram)).expect("ramfb"); let (pool, root) = written(&plan); assert_eq!(walk(&pool, root, base - 1), Some(Translation { phys: base - 1, pat: 0, writable: true })); assert_eq!(walk(&pool, root, base), Some(Translation { phys: base, pat: 3, writable: true })); every_leaf_translates_in_both_views(&plan, &pool, root); } + +/// The kernel's text read-only in both views, over the pages it shares split, +/// and the bytes on either side of it still writable. +#[test] +fn the_text_is_read_only_and_its_neighbours_are_writable() { + let text = (0x7c04_1000, 0x13_1000); + let plan = Plan::new(None, LOADER, text, Typing::Firmware).expect("the text"); + let (pool, root) = written(&plan); + let at = |phys: u64, writable: bool| Some(Translation { phys, pat: 0, writable }); + for (phys, writable) in [ + (text.0 - 1, true), + (text.0, false), + (text.0 + text.1 - 1, false), + (text.0 + text.1, true), + ] { + assert_eq!(walk(&pool, root, phys), at(phys, writable), "identity {phys:#x}"); + assert_eq!(walk(&pool, root, PHYS_OFFSET + phys), at(phys, writable), "PHYS_OFFSET + {phys:#x}"); + } + every_leaf_translates_in_both_views(&plan, &pool, root); +} diff --git a/loader/src/main.rs b/loader/src/main.rs index 00f31cd7c8..ed4e7a7671 100644 --- a/loader/src/main.rs +++ b/loader/src/main.rs @@ -76,6 +76,8 @@ struct LoadedKernel { pub entry_offset: usize, pub stack_offset: usize, pub stack_size: usize, + /// The text's whole pages, `(offset, len)` in `memory`. + pub text: (usize, usize), } /// `path` on the volume firmware loaded this image from: `LoadedImage`'s @@ -293,6 +295,20 @@ fn load_kernel_elf(kernel_elf_bytes: &[u8]) -> LoadedKernel { process_mem[vstart..vstart + src.len()].copy_from_slice(src); } + // The boot map's read-only text (`toyos_bootmap::Access`): the image's one + // executable segment, which no writable byte shares a page with. + let page = PAGE_SIZE as u64; + let mut executable = layout.segments().iter().filter(|segment| segment.flags().executable()); + let (Some(text), None) = (executable.next(), executable.next()) else { + panic!("kernel.elf: the boot map maps one executable segment, and this image has another number of them"); + }; + assert!(!text.writable(), "kernel.elf: its executable segment is writable"); + if let Some((a, b)) = layout.overlapping_load_pages(page) { + panic!("kernel.elf: PT_LOAD segments {a} and {b} share a page, which cannot have both their protections"); + } + let (text_start, text_end) = text.page_range(page); + let text = ((extent.min() + text_start) as usize, (text_end - text_start) as usize); + let rela_sections = sections.rela_sections().unwrap_or_else(|form| panic!("kernel.elf: {form} is not supported")); @@ -344,6 +360,7 @@ fn load_kernel_elf(kernel_elf_bytes: &[u8]) -> LoadedKernel { entry_offset: at(layout.entry()), stack_offset: placed.stack as usize, stack_size, + text, } } @@ -556,9 +573,13 @@ fn start_kernel(kernel: LoadedKernel, kernel_elf_bytes: vec::Vec, cmdline: v // pages by it. Before the exit, while the map can still be asked for; the // attributes a descriptor carries do not change across it. let write_back = write_back_memory(&system_table); + let kernel_phys = kernel.memory.as_ptr() as u64; + report_reach("Kernel image", kernel_phys, kernel.memory.len() as u64); + let text = (kernel_phys + kernel.text.0 as u64, kernel.text.1 as u64); let planned = Plan::new( gop.as_ref().map(|g| (g.framebuffer, g.framebuffer_size)), loader, + text, arch::typing(&write_back), ); match &planned { @@ -574,11 +595,12 @@ fn start_kernel(kernel: LoadedKernel, kernel_elf_bytes: vec::Vec, cmdline: v } let (at, len) = plan.loader(); println!("Loader image: {:#x}+{:#x}, mapped at identity as {at:#x}+{len:#x}", loader.0, loader.1); + println!("Kernel text: {:#x}+{:#x}, mapped read-only", text.0, text.1); } Err(why) => println!("Boot map: NO MAP HOLDS THIS MACHINE, {why}"), } let plan = planned - .unwrap_or_else(|why| panic!("the boot map cannot hold the scanout and the loader: {why}")); + .unwrap_or_else(|why| panic!("the boot map cannot hold the scanout, the loader and the kernel's text: {why}")); // SAFETY: `pt_mem` is the zeroed allocation of `pt_layout` above, which // nothing else holds, so it is a valid `[Table; MAX_PAGES]` of empty @@ -588,9 +610,6 @@ fn start_kernel(kernel: LoadedKernel, kernel_elf_bytes: vec::Vec, cmdline: v let pml4_phys = plan.write(arch::encoding::ENCODING, pool, pt_mem as u64); println!("Boot map: root {pml4_phys:#x}, {BOOT_MAP_BYTES:#x} bytes at identity and at PHYS_OFFSET"); - let kernel_phys = kernel.memory.as_ptr() as u64; - report_reach("Kernel image", kernel_phys, kernel.memory.len() as u64); - // An empty cmdline's pointer names nothing and is not reported as reachable. match (!cmdline.is_empty()).then_some((cmdline.as_ptr() as u64, cmdline.len() as u64)) { None => println!("Parameter buffer: none"), diff --git a/tests/common/faults.rs b/tests/common/faults.rs index 190a8d0cec..eab226b15d 100644 --- a/tests/common/faults.rs +++ b/tests/common/faults.rs @@ -98,3 +98,40 @@ pub fn crash_report_reads_no_kernel_memory(kernel: &Serial) -> Result<(), String None => Ok(()), } } + +/// What `kernel-wx-leaf` says of the leaf it stages (`kernel/src/main.rs`). +const STAGED: &str = "kernel-wx-leaf: the leaf at "; +/// The middle of the boot's audit refusal (`kernel/src/mm/mod.rs`). +const AUDIT_REFUSED: &str = " kernel leaves, writable and executable: "; + +/// The boot's audit of the kernel's tables against one direct-map leaf +/// `kernel-wx-leaf` made writable and executable: the refusal names that leaf +/// under both rules and counts no other, so every leaf the boot made itself +/// passed the same walk. +pub fn kernel_wx_leaf_is_refused(test_config: &Path, profile: qemu::Profile) -> Result<(), String> { + let qemu = QemuInstance::boot_with_options( + test_config, + &[], + &[], + BootOptions { profile, kernel_params: &["kernel-wx-leaf"], ready_marker: AUDIT_REFUSED, ..Default::default() }, + ); + let serial = qemu.boot_log(); + let at = serial + .lines() + .find_map(|line| line.split(STAGED).nth(1)?.split_whitespace().next()) + .ok_or_else(|| format!("no {STAGED:?} line\n{serial}"))?; + let refusal = serial + .lines() + .find(|line| line.contains(AUDIT_REFUSED)) + .map(str::trim_end) + .ok_or_else(|| format!("no audit refusal\n{serial}"))?; + let once = format!(": 1 (the first at {at})"); + let (both, outside) = (format!("writable and executable{once}"), "; executable outside the text "); + match refusal.split_once(outside) { + Some((writable, executable)) if writable.ends_with(&both) && executable.ends_with(&once) => { + eprintln!(" [kernel_wx] {refusal}"); + Ok(()) + } + _ => Err(format!("the audit did not name {at} alone under both rules: {refusal}\n{serial}")), + } +} diff --git a/tests/toyos.rs b/tests/toyos.rs index 3e3ec8bcea..d4e4e7581f 100644 --- a/tests/toyos.rs +++ b/tests/toyos.rs @@ -290,6 +290,10 @@ const DRIVEN_AND_SHARED: &[&str] = &[ "wall_clock_now", ]; +/// What the boot's audit of the kernel's tables says when every leaf passed +/// (`kernel/src/mm/mod.rs`). +const KERNEL_WX_HOLDS: &str = " kernel leaves, none writable and executable, none executable outside the text "; + /// What `test-early-panic` panics with (`kernel/src/main.rs`): the last line its /// report puts on serial. const EARLY_PANIC_MESSAGE: &str = "test-early-panic: on-screen console check"; @@ -314,6 +318,8 @@ const SCREEN_TESTS: &[(&str, qemu::Profile)] = &[ ("screen_loader_clears", qemu::Profile::Metal), ("virt_early_panic", qemu::Profile::Virt), ("virt_early_fault", qemu::Profile::Virt), + // `kernel_wx_leaf_is_refused` on `virt`: no metal bench is AArch64. + ("virt_kernel_wx_leaf_is_refused", qemu::Profile::Virt), ("virt_el2_drop", qemu::Profile::VirtEl2NoVhe), ("virt_user_mode", qemu::Profile::Virt), ("virt_timer_preempts", qemu::Profile::Virt), @@ -436,6 +442,11 @@ const MACHINE_TESTS: &[&str] = &[ // The nested-NMI report is a raw write to the 16550, which the T14 does not // have. "nested_nmi_is_loud", + // The boot's W^X audit refusing a leaf an actuator staged: the panic ends + // the boot, so it cannot share one, and the T14's `kernel_wx` row reads + // the same audit passing. q35 on `Profile::Metal`, whose 16550 carries + // the panic. + "kernel_wx_leaf_is_refused", // A press of the fixed power button reaches the `acpi` claim's holder as // one SCI, its line masked until the holder serves it: the kernel's SCI // and its mask have no host build, QEMU raises the event on demand, and @@ -867,7 +878,7 @@ const METAL: &[(&str, metal::Metal)] = &[ ), // ---- the boot facts, riding the same tests/images/testcases.toml image ---- // - // **Six names and no extra minute of the machine.** Every needle below is a + // **Seven names and no extra minute of the machine.** Every needle below is a // record the *shipping* kernel writes on any boot it takes, so each rides // whatever image is already going on the stick; the two rows after them are // the two things this suite measures rather than reads, and those cost a @@ -914,6 +925,20 @@ const METAL: &[(&str, metal::Metal)] = &[ "pci_inventory", metal::Metal { arms: TESTCASES, judge: |b| pci_inventory(b[0].kernel().text()) }, ), + ( + // The shipping kernel's W^X audit of its own tables, on the firmware + // and memory map it was built for: the T14 and the AMD bench. Its + // negative control is `kernel_wx_leaf_is_refused`, on QEMU. + "kernel_wx", + metal::Metal { + arms: TESTCASES, + judge: |b| { + let kernel = b[0].kernel(); + eprintln!(" [kernel_wx] {}", kernel.must_say(KERNEL_WX_HOLDS)?); + Ok(()) + }, + }, + ), // ---- one image: tests/images/latencycase.toml armed with the shootdown bench ---- ( "tlb_shootdown_cost", @@ -3616,6 +3641,7 @@ fn run_screen_test(name: &str, profile: qemu::Profile, test_config: &Path) -> Re } Ok(()) } + "virt_kernel_wx_leaf_is_refused" => faults::kernel_wx_leaf_is_refused(test_config, profile), "virt_early_fault" => { // The vectors, judged by the one thing a broken table cannot do: // report. An undefined instruction right after the console step @@ -4005,6 +4031,7 @@ fn run_machine_test(name: &str, test_config: &Path) -> Result<(), String> { "amdvi_firmware_left" => common::iommu::amdvi_firmware_left(test_config), "amdvi_selftest" => common::iommu::amdvi_selftest(test_config), "nested_nmi_is_loud" => faults::nested_nmi_is_loud(test_config), + "kernel_wx_leaf_is_refused" => faults::kernel_wx_leaf_is_refused(test_config, qemu::Profile::Metal), "acpi_press_is_one_sci" => acpi_press_is_one_sci(), "acpi_mediated_access" => acpi_mediated_access(2), "acpi_lock_given_back_on_one_cpu" => acpi_mediated_access(1), From 7f7b4268077cf43385c060bc380b225d91f5673c Mon Sep 17 00:00:00 2001 From: japabu Date: Wed, 21 Oct 2026 21:26:09 +0200 Subject: [PATCH 2/2] AArch64 join breaks before it makes, and only mixed 2 MiB pages are split Review r1 of #984. - AArch64 `paging::join` installed the kernel's TTBR1 root over the loader's while running on it and flushed afterwards. The two maps differ in block size (the loader's 2 MiB blocks against the kernel's pages around its image) and in permissions, and a size change needs break-before-make (Arm ARM DDI 0487 M.d D8.17.1) or the TLB may hold both translations and take a TLB conflict abort. `join` now runs from the image's identity alias under the old TTBR0, swaps TTBR1 to an empty root, invalidates with `TLBI VMALLE1`, installs the kernel's root, returns to the link address and only then switches TTBR0, whose new root maps nothing. Interrupts stay masked across it. QEMU models no TLB conflict, so this rests on the architecture, not on a test. - Both architectures map a 2 MiB page of the direct map as one leaf wherever the image's protections are uniform across it (`mm::Image::block`), and split only pages that hold two. The boot stack and whole pages of data stay blocks: 4092 kernel leaves on q35 instead of 6647, 3184 on `virt` instead of 5739. - x86-64 `kernel_leaf`'s doc covers both leaf sizes and the PAT-bit caveat; the `kernel_page` wrapper is gone. - Both `stage_writable_executable` controls handle a leaf under a split 2 MiB page and refuse an absent one. - The x86-64 boot map issue joins the security track's list. Co-Authored-By: Claude Opus 5.5 --- ...s-linux-on-every-machine-toyos-supports.md | 1 + kernel/src/arch/aarch64/paging.rs | 51 ++++++++++++++++--- kernel/src/arch/x86_64/paging.rs | 49 ++++++++++-------- kernel/src/mm/image.rs | 11 +++- 4 files changed, 82 insertions(+), 30 deletions(-) diff --git a/issues/the-kernel-is-at-least-as-secure-as-linux-on-every-machine-toyos-supports.md b/issues/the-kernel-is-at-least-as-secure-as-linux-on-every-machine-toyos-supports.md index ef13162c9b..9c42ec1487 100644 --- a/issues/the-kernel-is-at-least-as-secure-as-linux-on-every-machine-toyos-supports.md +++ b/issues/the-kernel-is-at-least-as-secure-as-linux-on-every-machine-toyos-supports.md @@ -38,6 +38,7 @@ work needs. - `issues/kernel-functions-return-with-their-used-registers-intact.md` - `issues/a-threads-kernel-stack-has-no-guard-page.md` - `issues/the-kernels-relro-stays-writable.md` +- `issues/x86-64s-boot-map-is-writable-and-executable.md` - `issues/the-kernel-heap-has-none-of-slubs-hardening.md` - `issues/a-device-without-a-domain-of-its-own-reaches-all-memory.md` - `issues/user-programs-run-without-a-shadow-stack.md` diff --git a/kernel/src/arch/aarch64/paging.rs b/kernel/src/arch/aarch64/paging.rs index 77d04ed215..1dc8e60ca5 100644 --- a/kernel/src/arch/aarch64/paging.rs +++ b/kernel/src/arch/aarch64/paging.rs @@ -680,7 +680,8 @@ pub fn map_mmio(phys: u64, size: u64, policy: MmioPolicy) -> crate::mm::Mmio { } /// Build the kernel's tables — every page of memory firmware's map names, in -/// 4 KiB pages each as `image` says where the image lies, and the console +/// 4 KiB pages where a 2 MiB one holds more than one of `image`'s +/// protections, and the console /// UART and the scanout the boot's own records reach — switch `TTBR1_EL1` to /// them and `TTBR0_EL1` to the kernel's empty user half, and answer where the /// direct map ends. @@ -699,7 +700,7 @@ pub(crate) fn init( let va = PHYS_OFFSET + at; match coverage(memory_map, at) { Coverage::Nothing => {} - Coverage::Whole if !image.overlaps(at, PAGE_2M) => { + Coverage::Whole if image.block(at).is_some() => { tables.directory(va).set(index(va, 2), at | memory(at)); blocks += 1; } @@ -738,23 +739,58 @@ pub(crate) fn init( /// Put this CPU on the kernel's tables — `TTBR1_EL1` the kernel's root, /// `TTBR0_EL1` the kernel space's empty user half — from the loader's or from -/// [`bringup_root`], either of which maps the code, stack and data this runs -/// on exactly as the kernel's root does. The local `TLBI` then drops every -/// translation the old tables left, global ones among them. +/// [`bringup_root`]. +/// +/// The loader's map and the kernel's need not agree on a leaf's size or +/// permissions for the same address: the loader maps 2 MiB blocks where the +/// kernel splits its image into pages, and leaves rodata writable. A change of +/// block size needs break-before-make (Arm ARM DDI 0487 M.d D8.17.1), or the +/// TLB may hold both translations at once and take a TLB conflict abort. So the +/// old `TTBR1_EL1` is replaced by an empty root and every translation dropped +/// with `TLBI VMALLE1` before the kernel's root is installed, while this CPU +/// runs from its image's identity alias under the old `TTBR0_EL1`, where both +/// the loader's map and `bringup_root` map the kernel's text executable. The +/// new `TTBR0_EL1` maps nothing, so switching it only removes translations. +/// Interrupts stay masked across it: the vectors are at `PHYS_OFFSET`. pub(crate) fn join() { let ttbr1 = KERNEL_TTBR1.load(core::sync::atomic::Ordering::Acquire); let ttbr0 = KERNEL_TTBR0.load(core::sync::atomic::Ordering::Acquire); // SAFETY: both roots are published by `init` and live forever, and the - // tables this CPU ran on map everything this function touches as they do. + // kernel space's root is an empty table, so it is the break's `TTBR1_EL1` + // too. Between the two branches nothing is fetched or accessed but this + // block's instructions, through the identity alias the old `TTBR0_EL1` + // maps; the image runs at `PHYS_OFFSET` plus its physical address, so that + // alias is the link address less `PHYS_OFFSET`. unsafe { core::arch::asm!( + "mrs {daif}, daif", + "msr daifset, #0xf", "dsb ishst", + "adr {at}, 1f", + "sub {at}, {at}, {offset}", + "br {at}", + "1:", + "msr ttbr1_el1, {empty}", + "isb", + "tlbi vmalle1", + "dsb nsh", + "isb", "msr ttbr1_el1, {ttbr1}", + "isb", + "adr {at}, 2f", + "add {at}, {at}, {offset}", + "br {at}", + "2:", "msr ttbr0_el1, {ttbr0}", "isb", "tlbi vmalle1", "dsb nsh", "isb", + "msr daif, {daif}", + daif = out(reg) _, + at = out(reg) _, + offset = in(reg) PHYS_OFFSET, + empty = in(reg) ttbr0 & ADDR, ttbr1 = in(reg) ttbr1, ttbr0 = in(reg) ttbr0, options(nostack, preserves_flags), @@ -826,7 +862,8 @@ pub fn stage_writable_executable(phys: u64) -> u64 { let tables = high.as_mut().expect("kernel-wx-leaf: before the kernel's tables exist"); let i = index(va, 2); let entry = tables.directory(va).0[i]; - let (table, slot, at) = if entry & (VALID | TABLE) == VALID { + assert!(entry & VALID != 0, "kernel-wx-leaf: {phys:#x} is not in the direct map"); + let (table, slot, at) = if entry & TABLE == 0 { (tables.directory(va), i, va & !(PAGE_2M - 1)) } else { // SAFETY: a table this root linked, reached under the lock. diff --git a/kernel/src/arch/x86_64/paging.rs b/kernel/src/arch/x86_64/paging.rs index cee598f6dc..1deddb2998 100644 --- a/kernel/src/arch/x86_64/paging.rs +++ b/kernel/src/arch/x86_64/paging.rs @@ -73,18 +73,14 @@ impl Prot { } } -/// Every bit of a 2 MiB kernel-half leaf but its address and size: the one -/// writer of [`PAGE_GLOBAL`]. +/// Every bit of a kernel-half leaf but its address and a 2 MiB leaf's size +/// bit: the one writer of [`PAGE_GLOBAL`]. Its PAT bit is a 2 MiB leaf's (bit +/// 12; a 4 KiB one's is bit 7), so a 4 KiB leaf takes it only for `Normal`, +/// which sets neither. fn kernel_leaf(cache: CachePolicy, prot: Prot) -> u64 { PAGE_PRESENT | PAGE_GLOBAL | prot.kernel_bits() | cache.pde_bits() } -/// [`kernel_leaf`] for a 4 KiB page of memory: `Normal` sets no PAT bit, the -/// one bit a 2 MiB leaf and a 4 KiB one place apart. -fn kernel_page(prot: Prot) -> u64 { - kernel_leaf(CachePolicy::Normal, prot) -} - /// Every upper-level entry over the kernel half: its leaves decide. const KERNEL_TABLE: u64 = PAGE_PRESENT | PAGE_WRITE; @@ -830,15 +826,15 @@ impl AddressSpace { pd.write_pde(pd_idx, virt, entry).subsumed_by_flush(); } - /// The 2 MiB page of the direct map at `phys`, which the kernel's image - /// shares, in 4 KiB leaves each as the image says. Only `init` calls it, - /// before any TLB holds these tables. + /// The 2 MiB page of the direct map at `phys`, which holds more than one of + /// the image's protections, in 4 KiB leaves each as the image says. Only + /// `init` calls it, before any TLB holds these tables. fn map_image_pages(&mut self, phys: u64, image: &crate::mm::Image) { let virt = crate::mm::DirectMap::from_phys(phys).as_ptr::() as u64; let mut pt = Box::new(PageTablePage([0; 512])); for i in 0..512 { let page = phys + i as u64 * PAGE_4K; - pt.init_entry(i, page | kernel_page(image.prot(page).unwrap_or(Prot::ReadWrite))); + pt.init_entry(i, page | kernel_leaf(CachePolicy::Normal, image.prot(page).unwrap_or(Prot::ReadWrite))); } let pt_phys = pt.phys(); self.children.push(pt); @@ -1070,7 +1066,7 @@ pub fn debug_global_leaf(how: u64) -> u64 { } /// Build kernel page tables: the direct map in the high half, in 2 MiB pages -/// of data but where `image` lies, as far as +/// but where one holds more than one of `image`'s protections, as far as /// [`toyos_bootmap::x86_64::direct_map_end`] reaches, and answer that end. The /// scanout is not mapped for itself: in the low map the MTRRs type it until /// `panic_console::remap` gives it a type of its own. @@ -1097,10 +1093,9 @@ pub(crate) fn init( } let mut addr: u64 = 0; while addr < end { - if image.overlaps(addr, PAGE_2M) { - kernel.map_image_pages(addr, image); - } else { - kernel.map_2m(addr, kernel_leaf(CachePolicy::Normal, Prot::ReadWrite)); + match image.block(addr) { + Some(prot) => kernel.map_2m(addr, kernel_leaf(CachePolicy::Normal, prot)), + None => kernel.map_image_pages(addr, image), } addr += PAGE_2M; } @@ -1151,14 +1146,24 @@ pub fn audit_wx(text: core::ops::Range) -> kernel::wx::Verdict { /// [`audit_wx`] has a writable executable leaf to find. Answers its address. #[cfg(feature = "boot-actuators")] pub fn stage_writable_executable(phys: u64) -> u64 { - let virt = crate::mm::DirectMap::from_phys(phys).as_ptr::() as u64 & !(PAGE_2M - 1); + let virt = crate::mm::DirectMap::from_phys(phys).as_ptr::() as u64; let mut space = kernel().lock(); let target = space.root(); let pd = space.ensure_table(virt, KERNEL_TABLE); - let pde = pd[indices(virt).2]; - assert!(pde & PAGE_SIZE_BIT != 0 && pde & PAGE_WRITE != 0, "kernel-wx-leaf: {phys:#x} is not under a writable 2 MiB leaf"); - pd.write_pde(indices(virt).2, virt, pde & !PAGE_NX).discharge(target); - virt + let i = indices(virt).2; + let pde = pd[i]; + assert!(pde & PAGE_PRESENT != 0, "kernel-wx-leaf: {phys:#x} is not in the direct map"); + let (table, slot, at) = if pde & PAGE_SIZE_BIT != 0 { + (pd, i, virt & !(PAGE_2M - 1)) + } else { + // SAFETY: a present PDE without the size bit names a table this + // module linked; the kernel's lock gives the exclusivity. + (unsafe { PageTablePage::from_phys_mut(pde & ADDR_MASK) }, ((virt >> 12) & 0x1FF) as usize, virt & !(PAGE_4K - 1)) + }; + let leaf = table[slot]; + assert!(leaf & PAGE_PRESENT != 0 && leaf & PAGE_WRITE != 0, "kernel-wx-leaf: {phys:#x} is not under a writable leaf"); + table.write(slot, at, leaf & !PAGE_NX).discharge(target); + at } fn has(entry: u64, flag: u64) -> u8 { diff --git a/kernel/src/mm/image.rs b/kernel/src/mm/image.rs index f730d75cbc..a446176778 100644 --- a/kernel/src/mm/image.rs +++ b/kernel/src/mm/image.rs @@ -42,7 +42,7 @@ impl<'a> Image<'a> { } /// Whether any byte of `[phys, phys + len)` is the image's. - pub fn overlaps(&self, phys: u64, len: u64) -> bool { + fn overlaps(&self, phys: u64, len: u64) -> bool { phys < self.memory.end && self.memory.start < phys + len } @@ -66,6 +66,15 @@ impl<'a> Image<'a> { }) } + /// What all of the 2 MiB page at `phys` may be used for, where it is one + /// thing: a page not the image's is data, and a page that holds two + /// protections is `None`, to be mapped page by page. + pub fn block(&self, phys: u64) -> Option { + let of = |page: u64| self.prot(page).unwrap_or(Prot::ReadWrite); + let first = of(phys); + (1..super::PAGE_2M / PAGE_SIZE).all(|i| of(phys + i * PAGE_SIZE) == first).then_some(first) + } + /// The text's pages, as the direct map addresses them. pub fn text(&self) -> core::ops::Range { PHYS_OFFSET + self.text.start..PHYS_OFFSET + self.text.end