diff --git a/Cargo.lock b/Cargo.lock index 3886c2eda5..b4ee63ff39 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1823,6 +1823,7 @@ dependencies = [ "hashbrown 0.16.1", "toyos-abi", "toyos-acpi", + "toyos-amdvi", "toyos-blackbox", "toyos-blockhold", "toyos-bootmap", @@ -4065,6 +4066,13 @@ dependencies = [ "toyos-bootmap", ] +[[package]] +name = "toyos-amdvi" +version = "0.1.0" +dependencies = [ + "toyos-phys", +] + [[package]] name = "toyos-aml" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index c7de5b2fd0..79f6d90e89 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -25,6 +25,7 @@ members = [ "build/root", "build/ssh", "kernel", + "kernel/amdvi", "kernel/cpuvuln", "kernel/dma", "kernel/gicv3", diff --git a/build/sourcegate.rs b/build/sourcegate.rs index 23d89cd1f6..87da023f34 100644 --- a/build/sourcegate.rs +++ b/build/sourcegate.rs @@ -102,6 +102,7 @@ const BUS_MASTER_SITES: &[(&str, usize)] = &[ ("kernel/src/drivers/virtio.rs", 1), ("kernel/src/drivers/hda.rs", 1), ("kernel/src/drivers/xhci/wait/boot.rs", 1), + ("kernel/src/arch/x86_64/amdvi/mod.rs", 1), ]; /// The one place in `kernel/` a `!!!` may still appear, and how many times. diff --git a/issues/an-amd-vi-machine-isolates-no-device.md b/issues/an-amd-vi-machine-isolates-no-device.md index 96d655ac89..58c8657787 100644 --- a/issues/an-amd-vi-machine-isolates-no-device.md +++ b/issues/an-amd-vi-machine-isolates-no-device.md @@ -10,30 +10,31 @@ Held by the orchestrator's AMD laptop track: its stage 5, amdvi-translate, build translation, and its stage 6, amdvi-intremap, interrupt remapping. On a machine whose IVRS describes AMD-Vi units, `kernel/src/arch/x86_64/amdvi/` -switches every unit off, so every device's DMA reaches all of memory and every -message-signalled interrupt is delivered as written. `DeviceSpace::create` -answers `Untranslated`, and a function a process would drive is refused its -claim (`iommu::remapping` is `NotRemapped`), so no userland driver runs there. - -What remains is translation, then interrupt remapping: - -- **Translation.** A device table blocking by default, every one of the 65,536 - entries `V=1, TV=1` with `IR=IW=0` — an entry left `V=0` passes its - requester's DMA untranslated — identity domains for kernel drivers, a - command buffer whose `COMPLETION_WAIT` is waited on under a `Tripwire`, and - the event log feeding `iommu::fault`. -- **Interrupt remapping.** Interrupt remapping tables, the I/O APIC's and the - HPET's requester ids from the IVRS's special entries, and `claim_msi`. +translates (stage 5): every requester id's entry blocks unless it names a +domain, enumerated functions sit on an identity domain until a kernel driver +moves one to its own, and the event log feeds `iommu::fault`. No interrupt +is remapped: every entry's interrupt half is invalid, so every +message-signalled interrupt is delivered as written, and a function a +process would drive is refused its claim (`iommu::remapping` is +`NotRemapped`), so no userland driver runs there. + +What remains is interrupt remapping (stage 6): a remapping table per +requester indexed by vector, the I/O APICs' requester ids from the IVRS's +special entries, `claim_msi`, a claim clearing its entry's pass bits and +`SysMgt`, and a claim of a function an IVMD names refused. Constraints a reader would otherwise pay to re-derive: - QEMU 11.1.1's `amd-iommu` translates a function only with `dma-remap=on`, - and only once an `INVALIDATE_DEVTAB_ENTRY` names it; it takes a device table - entry of `Mode` 0 as pass-through whatever `IR` and `IW` say, where the - specification blocks it (`hw/i386/amd_iommu.c` at v11.1.1). -- No AMD machine is in reach of the metal suite. - -**Exit:** a guest boot on `Profile::MetalAmdVi` in which a kernel driver's -DMA goes through a domain of its own, a DMA outside it is recorded as a -fault, and a claimed function's message is delivered through its own -remapping entry. + and only once an `INVALIDATE_DEVTAB_ENTRY` names it + (`issues/qemus-invalidate-iommu-all-leaves-every-function-untranslated.md`); it takes a + device table entry of `Mode` 0 as pass-through whatever `IR` and `IW` say, + where the specification blocks it, so the blocked default is unobservable + there; and its `IO_PAGE_FAULT` record carries no address + (`issues/qemus-amd-iommu-page-fault-record-carries-no-address.md`). +- The AMD bench is in the metal suite; the orchestrator runs every boot. + +**Exit:** a guest boot on `Profile::MetalAmdVi` and a run on the AMD bench +in which a kernel driver's DMA goes through a domain of its own, a DMA +outside it is recorded as a fault, and a claimed function's message is +delivered through its own remapping entry. diff --git a/issues/qemus-amd-iommu-page-fault-record-carries-no-address.md b/issues/qemus-amd-iommu-page-fault-record-carries-no-address.md new file mode 100644 index 0000000000..3ed1fe0ac9 --- /dev/null +++ b/issues/qemus-amd-iommu-page-fault-record-carries-no-address.md @@ -0,0 +1,29 @@ +--- +status: open +kind: tooling +opened: 2026-10-13 +--- + +# QEMU's `amd-iommu` writes an `IO_PAGE_FAULT` record with no address + +AMD 48882 rev 3.11, Figure 56 and Table 57: an `IO_PAGE_FAULT` record +carries the device address the requester tried in its second doubleword, +and in its first `RW` at bit 53 (+04 bit 21), `PR` at 52, `I` at 51 and +`NX` at 49. QEMU 11.1.1's `amd-iommu`, refusing an `iommu-testdev` write at +`0x400000000000` under `amdvi-selftest`, wrote +`0x200a000000000030 0x0000000000000000`: the right code and DeviceID +(`00:06.0`), the address zero, and bits 51 and 49 set, which 48882 reads as +an interrupt with `NX`. Measured by logging the raw record in +`kernel/src/arch/x86_64/amdvi/fault.rs` on PR #909's branch. + +So `amdvi_selftest` (`tests/common/iommu.rs`) holds the record to its +requester and its name, not its address or its access, and the record's +`addr=`/`access=` under QEMU are QEMU's, not the device's. + +Owner: the orchestrator's AMD laptop track +(`issues/an-amd-vi-machine-isolates-no-device.md`); the AMD bench's +`usbd` fault row reads a real unit's record at its aimed address. + +**Exit:** under the QEMU `build/ci/qemu-version` names, the record of +`amdvi-selftest`'s refused write names `0x400000000000` and a write, and +`amdvi_selftest` asserts both. diff --git a/kernel/Cargo.toml b/kernel/Cargo.toml index e0d4e77b46..13c8dc5098 100644 --- a/kernel/Cargo.toml +++ b/kernel/Cargo.toml @@ -479,6 +479,7 @@ toyos-abi = { path = "../sdk/abi" } [target.'cfg(target_os = "none")'.dependencies] bcachefs = { path = "../lib/bcachefs", default-features = false } toyos-acpi = { path = "../lib/acpi" } +toyos-amdvi = { path = "amdvi" } toyos-dma = { path = "dma" } toyos-blackbox = { path = "../lib/blackbox" } toyos-blockhold = { path = "../lib/blockhold" } diff --git a/kernel/amdvi/Cargo.toml b/kernel/amdvi/Cargo.toml new file mode 100644 index 0000000000..8ca426e833 --- /dev/null +++ b/kernel/amdvi/Cargo.toml @@ -0,0 +1,19 @@ +# A member of the host workspace (root `Cargo.toml`), like toyos-smmu: it is +# pure and its tests run on the host. +# +# What lives here is every word an AMD-Vi unit and its driver exchange. The +# unit decides which memory a device reaches, so a field one bit off is an +# isolation hole no boot shows. Each encoding is held on the host to AMD +# 48882's own field positions, and the kernel's `arch::x86_64::amdvi` is left +# the registers and the memory. + +[package] +name = "toyos-amdvi" +description = "AMD-Vi's encodings: device table entries, host page table entries, commands, event records and register fields, pure, forbid(unsafe_code)." +version = "0.1.0" +edition = "2021" +license = "MIT OR Apache-2.0" +publish = false + +[dependencies] +toyos-phys = { path = "../phys" } diff --git a/kernel/amdvi/src/dte.rs b/kernel/amdvi/src/dte.rs new file mode 100644 index 0000000000..611024a858 --- /dev/null +++ b/kernel/amdvi/src/dte.rs @@ -0,0 +1,67 @@ +//! The device table: one 256-bit entry per requester id (§2.2.2, Figure 7, +//! Table 7), indexed by the id. + +use toyos_phys::Phys; + +/// Every requester id a segment has; the table holds an entry for each. +pub const DEVICE_IDS: usize = 1 << 16; +/// Bytes per entry. +pub const ENTRY_BYTES: usize = 32; + +const V: u64 = 1 << 0; +const TV: u64 = 1 << 1; +const MODE_SHIFT: u32 = 9; +const IR: u64 = 1 << 61; +const IW: u64 = 1 << 62; +/// `SysMgt`, bits 105:104: bits 41:40 of the second doubleword. +const SYS_MGT_SHIFT: u32 = 40; + +/// The host table depth every translating entry names: `Mode` 100b, a 48-bit +/// input. +pub const MODE: u8 = 4; + +/// What an IVHD device entry's DTE-setting byte declares for its requesters +/// (§5.2.2, Table 106). Of it, a translation entry carries `SysMgt` alone: +/// the pass bits are the interrupt half's, which nothing here sets valid. +#[derive(Clone, Copy, PartialEq, Eq, Debug, Default)] +pub struct Setting(pub u8); + +impl Setting { + /// Bits 5:4, the `SysMgt` firmware declares the requester needs. + pub const fn sys_mgt(self) -> u64 { + (self.0 as u64 >> 4) & 0b11 + } +} + +/// One entry's translation half, the 128 bits a translation change replaces +/// in one write (§2.2.2.2); the interrupt half above it stays zero, `IV` +/// clear, so the unit forwards a requester's interrupts unremapped (Table 9). +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub struct Dte([u64; 2]); + +impl Dte { + /// `V`, `TV`, `Mode` 000b and `IR`/`IW` clear: every read and write the + /// requester makes is target-aborted (Table 7 bits 11:9, 61, 62). Tagged + /// with `domain`, which Table 8 requires valid wherever `V` is set. + pub const fn blocked(domain: u16, setting: Setting) -> Self { + Self([V | TV, domain as u64 | setting.sys_mgt() << SYS_MGT_SHIFT]) + } + + /// Translating through the four-level table at `root`, tagged `domain`: + /// the requester reaches what that table maps and nothing else. + pub const fn translated(root: Phys<12>, domain: u16, setting: Setting) -> Self { + Self([ + V | TV | (MODE as u64) << MODE_SHIFT | root.get() | IR | IW, + domain as u64 | setting.sys_mgt() << SYS_MGT_SHIFT, + ]) + } + + pub const fn words(self) -> [u64; 2] { + self.0 + } + + /// The domain an entry read back out of the table is tagged with. + pub const fn domain_of(words: [u64; 2]) -> u16 { + words[1] as u16 + } +} diff --git a/kernel/amdvi/src/lib.rs b/kernel/amdvi/src/lib.rs new file mode 100644 index 0000000000..9b22602ca6 --- /dev/null +++ b/kernel/amdvi/src/lib.rs @@ -0,0 +1,18 @@ +//! Every word an AMD-Vi unit and its driver exchange, as AMD 48882 rev 3.11 +//! lays it out: the device table entry ([`dte`]), the host page tables a +//! translating entry names ([`table`]), the commands and event records +//! ([`queue`]), and the register fields this driver writes or reads ([`reg`]). +//! +//! The crate encodes one configuration: a requester is blocked, or translated +//! through a four-level host table that maps 2 MiB pages, read and write, +//! forced coherent. No entry passes DMA untranslated. +//! +//! `no_std`, no allocation, no `unsafe`. + +#![no_std] +#![forbid(unsafe_code)] + +pub mod dte; +pub mod queue; +pub mod reg; +pub mod table; diff --git a/kernel/amdvi/src/queue.rs b/kernel/amdvi/src/queue.rs new file mode 100644 index 0000000000..a85789543d --- /dev/null +++ b/kernel/amdvi/src/queue.rs @@ -0,0 +1,83 @@ +//! The command buffer's commands (§2.4) and the event log's records (§2.5): +//! 128 bits each, a four-bit code in bits 63:60 of the first doubleword. + +use toyos_phys::Phys; + +/// Entries a 4 KiB command buffer or event log holds, its length field's +/// minimum (MMIO 0008h and 0010h, `1000b`). +pub const ENTRIES: usize = 256; + +const CODE_SHIFT: u32 = 60; + +/// A command this driver sends. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub enum Command { + /// `COMPLETION_WAIT` (Figure 44) with `s`: once every earlier command has + /// completed, the unit stores `data` at `store`. + CompletionWait { store: Phys<3>, data: u64 }, + /// `INVALIDATE_DEVTAB_ENTRY` (Figure 45): the unit rereads this id's entry. + InvalidateDevice(u16), + /// `INVALIDATE_IOMMU_PAGES` (Figure 46) over every address of one domain, + /// directories with them: `S` and `PDE` set, address 7_FFFF_FFFF_FFFFh. + InvalidateDomain(u16), + /// `INVALIDATE_IOMMU_ALL` (Figure 51): every cache the unit holds. + InvalidateAll, +} + +impl Command { + pub const fn words(self) -> [u64; 2] { + match self { + Self::CompletionWait { store, data } => [1 << CODE_SHIFT | store.get() | 1, data], + Self::InvalidateDevice(id) => [2 << CODE_SHIFT | id as u64, 0], + Self::InvalidateDomain(domain) => [3 << CODE_SHIFT | (domain as u64) << 32, 0x7_FFFF_FFFF_FFFF << 12 | 0b11], + Self::InvalidateAll => [8 << CODE_SHIFT, 0], + } + } +} + +/// One event record, decoded as far as what it ends. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub enum Event { + /// A record a requester caused, which the unit has already refused or + /// dropped: `IO_PAGE_FAULT` (Figure 56), `IOTLB_INV_TIMEOUT` (Figure 61), + /// `INVALID_DEVICE_REQUEST` (Figure 62), and `INVALID_PPR_REQUEST` with + /// `RX` clear (Figure 64). `write` is the access where the record states + /// one; `address` is the record's, whatever it names. + Requester { code: u8, device: u16, address: u64, write: Option }, + /// Every other record: an entry or command of this driver's the unit + /// refused, or the unit's own hardware failing. + Unit { code: u8 }, +} + +pub const fn event(words: [u64; 2]) -> Event { + let device = words[0] as u16; + let code = (words[0] >> CODE_SHIFT) as u8; + let write = match code { + // `I` [51]: an interrupt carries no read or write; else `RW` [53]. + 0b0010 if words[0] & 1 << 51 != 0 => None, + 0b0010 => Some(words[0] & 1 << 53 != 0), + // `RX` [48] set is a `COMPLETE_PPR_REQUEST` of this driver's failing. + 0b1001 if words[0] & 1 << 48 != 0 => return Event::Unit { code }, + // `WP` [53]: the write permission the peripheral asked for. + 0b1001 => Some(words[0] & 1 << 53 != 0), + 0b0111 | 0b1000 => None, + _ => return Event::Unit { code }, + }; + Event::Requester { code, device, address: words[1], write } +} + +/// The name 48882 gives an event code, for a record's line. +pub const fn name(code: u8) -> &'static str { + match code { + 0b0001 => "ILLEGAL_DEV_TABLE_ENTRY", + 0b0010 => "IO_PAGE_FAULT", + 0b0011 => "DEV_TAB_HARDWARE_ERROR", + 0b0100 => "PAGE_TAB_HARDWARE_ERROR", + 0b0101 => "ILLEGAL_COMMAND_ERROR", + 0b0110 => "COMMAND_HARDWARE_ERROR", + 0b0111 => "IOTLB_INV_TIMEOUT", + 0b1000 => "INVALID_DEVICE_REQUEST", + 0b1001 => "INVALID_PPR_REQUEST", + _ => "reserved", + } +} diff --git a/kernel/amdvi/src/reg.rs b/kernel/amdvi/src/reg.rs new file mode 100644 index 0000000000..34d141cbb4 --- /dev/null +++ b/kernel/amdvi/src/reg.rs @@ -0,0 +1,85 @@ +//! The MMIO registers this driver writes or reads (§3.4), by offset, and the +//! fields of each it uses. + +use toyos_phys::Phys; + +use crate::dte::{DEVICE_IDS, ENTRY_BYTES}; + +pub const DEVICE_TABLE: u64 = 0x0000; +pub const COMMAND_BASE: u64 = 0x0008; +pub const EVENT_BASE: u64 = 0x0010; +pub const CONTROL: u64 = 0x0018; +pub const EXCLUSION_BASE: u64 = 0x0020; +pub const EXCLUSION_LIMIT: u64 = 0x0028; +pub const EXTENDED_FEATURES: u64 = 0x0030; +pub const COMMAND_HEAD: u64 = 0x2000; +pub const COMMAND_TAIL: u64 = 0x2008; +pub const EVENT_HEAD: u64 = 0x2010; +pub const EVENT_TAIL: u64 = 0x2018; +pub const STATUS: u64 = 0x2020; + +/// `CONTROL` (MMIO 0018h). +pub mod control { + pub const IOMMU_EN: u64 = 1 << 0; + pub const EVENT_LOG_EN: u64 = 1 << 2; + pub const EVENT_INT_EN: u64 = 1 << 3; + /// Device table fetches snooped; reset 1. + pub const COHERENT: u64 = 1 << 10; + pub const CMD_BUF_EN: u64 = 1 << 12; + pub const PPR_LOG_EN: u64 = 1 << 13; + pub const PPR_INT_EN: u64 = 1 << 14; + pub const GA_LOG_EN: u64 = 1 << 28; + pub const GA_INT_EN: u64 = 1 << 29; +} + +/// `STATUS` (MMIO 2020h). +pub mod status { + /// Write 1 to clear. + pub const EVENT_OVERFLOW: u32 = 1 << 0; + /// Write 1 to clear. + pub const EVENT_LOG_INT: u32 = 1 << 1; + pub const EVENT_LOG_RUN: u32 = 1 << 3; + pub const CMD_BUF_RUN: u32 = 1 << 4; + pub const PPR_LOG_RUN: u32 = 1 << 7; + pub const GA_LOG_RUN: u32 = 1 << 8; +} + +/// The extended feature register (MMIO 0030h), as far as this driver asks it. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub struct Features(pub u64); + +impl Features { + /// `IASup`: the unit takes `INVALIDATE_IOMMU_ALL`. + pub const fn invalidate_all(self) -> bool { + self.0 & 1 << 6 != 0 + } + + /// `HATS`, the host table levels the unit walks: 4, 5 or 6, or `None` + /// for the reserved encoding. + pub const fn host_levels(self) -> Option { + match (self.0 >> 10) & 0b11 { + 0b11 => None, + n => Some(4 + n as u8), + } + } +} + +/// `DEVICE_TABLE`'s value for a whole table at `base`: `Size` 1FFh, 2 MiB. +pub const fn device_table(base: Phys<12>) -> u64 { + base.get() | ((DEVICE_IDS * ENTRY_BYTES / 4096 - 1) as u64) +} + +/// `COMMAND_BASE`'s or `EVENT_BASE`'s value for a 4 KiB ring at `base`: +/// length 1000b, 256 entries. +pub const fn ring(base: Phys<12>) -> u64 { + base.get() | 0b1000 << 56 +} + +/// The byte offset a head or tail register holds for entry `slot`, and back. +pub const fn offset(slot: usize) -> u64 { + slot as u64 * 16 +} + +pub const fn slot(offset: u64) -> usize { + ((offset >> 4) & 0x7FFF) as usize +} diff --git a/kernel/amdvi/src/table.rs b/kernel/amdvi/src/table.rs new file mode 100644 index 0000000000..f3567c98a1 --- /dev/null +++ b/kernel/amdvi/src/table.rs @@ -0,0 +1,56 @@ +//! Host page tables (§2.2.3): 4 KiB tables of 512 eight-byte entries, a +//! four-level walk over a 48-bit device address (Table 15), ending at level 2 +//! in a translation entry for a 2 MiB page (Figures 9 and 10, Tables 17 and +//! 18). No level is skipped. + +use toyos_phys::Phys; + +const PR: u64 = 1 << 0; +const NEXT_LEVEL_SHIFT: u32 = 9; +const NEXT_LEVEL: u64 = 0b111 << NEXT_LEVEL_SHIFT; +/// Force Coherent: the unit marks the request snooped whatever the device +/// asked, so a device's no-snoop setting cannot read stale memory. +const FC: u64 = 1 << 60; +const IR: u64 = 1 << 61; +const IW: u64 = 1 << 62; + +/// The level the root table is at, `Mode` of the entry naming it. +pub const ROOT_LEVEL: u8 = crate::dte::MODE; +/// The level whose translation entries are 2 MiB pages. +pub const LEAF_LEVEL: u8 = 2; + +/// The index `at` takes in the table at `level` (Table 15: bits +/// `12 + 9 * (level - 1)` up). +pub const fn index(at: u64, level: u8) -> usize { + ((at >> (12 + 9 * (level as u32 - 1))) & 0x1FF) as usize +} + +/// An entry at `level` pointing at the table one level down, granting both +/// read and write, which the walk ANDs down the levels (Table 18). +pub const fn directory(below: Phys<12>, level: u8) -> u64 { + PR | ((level as u64 - 1) << NEXT_LEVEL_SHIFT) | below.get() | IR | IW +} + +/// A level-2 translation entry for the 2 MiB page at `page`, read and write +/// (Table 17, `NextLevel` 000b). +pub const fn leaf(page: Phys<21>) -> u64 { + PR | page.get() | FC | IR | IW +} + +/// What one entry is, by `PR` and `NextLevel`. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub enum Entry { + Absent, + Directory(Phys<12>), + Leaf, +} + +pub const fn entry(word: u64) -> Entry { + if word & PR == 0 { + return Entry::Absent; + } + match (word & NEXT_LEVEL) >> NEXT_LEVEL_SHIFT { + 0 | 0b111 => Entry::Leaf, + _ => Entry::Directory(Phys::of(word)), + } +} diff --git a/kernel/amdvi/tests/encodings.rs b/kernel/amdvi/tests/encodings.rs new file mode 100644 index 0000000000..6641a07fbc --- /dev/null +++ b/kernel/amdvi/tests/encodings.rs @@ -0,0 +1,183 @@ +//! Every encoding against AMD 48882 rev 3.11's own field positions: each +//! expected value is built from `(msb, lsb, value)` triples as the +//! specification's figures and tables give them, and compared whole, so a +//! field in the wrong place and a stray bit are both a difference. + +use toyos_amdvi::dte::{Dte, Setting, DEVICE_IDS, ENTRY_BYTES}; +use toyos_amdvi::queue::{event, name, Command, Event, ENTRIES}; +use toyos_amdvi::reg::{device_table, offset, ring, slot, Features}; +use toyos_amdvi::table::{directory, entry, index, leaf, Entry, LEAF_LEVEL, ROOT_LEVEL}; +use toyos_phys::Phys; + +/// `N` doublewords holding `value` in bits `[msb:lsb]` for every triple, and +/// zero everywhere else. A field may cross a doubleword. +fn layout(fields: &[(u32, u32, u64)]) -> [u64; N] { + let mut words = [0u64; N]; + for &(msb, lsb, value) in fields { + let width = msb - lsb + 1; + assert!(width == 64 || value >> width == 0, "{value:#x} does not fit [{msb}:{lsb}]"); + for bit in 0..width { + if value >> bit & 1 != 0 { + let at = lsb + bit; + assert_eq!(words[(at / 64) as usize] >> (at % 64) & 1, 0, "two fields claim bit {at}"); + words[(at / 64) as usize] |= 1 << (at % 64); + } + } + } + words +} + +fn word(fields: &[(u32, u32, u64)]) -> u64 { + layout::<1>(fields)[0] +} + +const ROOT: u64 = 0x12_3456_7000; + +/// Figure 7: `V` [0], `TV` [1], `Mode` [11:9] 000b, `IR` [61] and `IW` [62] +/// clear, `DomainID` [79:64]; Table 7 bits 11:9 make every access the IR/IW +/// pair's to allow, and both are clear. +#[test] +fn a_blocked_entry_is_valid_untranslated_and_grants_nothing() { + let want = layout::<2>(&[(0, 0, 1), (1, 1, 1), (79, 64, 0)]); + assert_eq!(Dte::blocked(0, Setting(0)).words(), want); +} + +/// Figure 7: the root at [51:12], `Mode` 100b, `IR` and `IW` set, the domain +/// at [79:64]. +#[test] +fn a_translated_entry_names_its_four_level_root_and_domain() { + let want = layout::<2>(&[(0, 0, 1), (1, 1, 1), (11, 9, 0b100), (51, 12, ROOT >> 12), (61, 61, 1), (62, 62, 1), (79, 64, 0xBEEF)]); + let root = Phys::new(ROOT).expect("aligned"); + assert_eq!(Dte::translated(root, 0xBEEF, Setting(0)).words(), want); + assert_eq!(Dte::domain_of(want), 0xBEEF); +} + +/// Table 106 puts `SysMgt` at bits 5:4 of the IVHD byte, Table 7 at DTE bits +/// 105:104; the pass bits around it (7, 6, 3, 2, 1, 0) reach no translation +/// half. +#[test] +fn the_ivhd_byte_gives_an_entry_its_sys_mgt_alone() { + let declared = Setting(0b1101_0111); + let want = layout::<2>(&[(0, 0, 1), (1, 1, 1), (79, 64, 7), (105, 104, 0b01)]); + assert_eq!(Dte::blocked(7, declared).words(), want); + let translated = Dte::translated(Phys::new(ROOT).expect("aligned"), 7, declared).words(); + assert_eq!(translated[1], want[1]); +} + +/// §2.2.2: 65,536 entries of 256 bits, 2 MiB; MMIO 0000h `Size` [8:0] counts +/// 4 KiB units less one, so 1FFh, the base at [51:12]. +#[test] +fn the_device_table_register_names_two_mebibytes() { + assert_eq!(DEVICE_IDS * ENTRY_BYTES, 2 << 20); + assert_eq!(device_table(Phys::new(ROOT).expect("aligned")), word(&[(8, 0, 0x1FF), (51, 12, ROOT >> 12)])); +} + +/// MMIO 0008h and 0010h: the base at [51:12], length [59:56] 1000b for 256 +/// entries; the head and tail registers count 16-byte entries at [18:4]. +#[test] +fn a_ring_register_names_a_four_kibibyte_ring() { + assert_eq!(ring(Phys::new(ROOT).expect("aligned")), word(&[(51, 12, ROOT >> 12), (59, 56, 0b1000)])); + assert_eq!(ENTRIES * 16, 4096); + assert_eq!(offset(255), word(&[(18, 4, 255)])); + assert_eq!(slot(offset(255)), 255); +} + +/// Table 15: level 4 indexes by bits 47:39, level 3 by 38:30, level 2 by +/// 29:21. +#[test] +fn each_level_indexes_by_its_nine_bits() { + let at = word(&[(47, 39, 0x1A5), (38, 30, 0x0F3), (29, 21, 0x15A), (20, 0, 0x1F_FFFF)]); + assert_eq!((index(at, 4), index(at, 3), index(at, 2)), (0x1A5, 0x0F3, 0x15A)); + assert_eq!((ROOT_LEVEL, LEAF_LEVEL), (4, 2)); +} + +/// Figure 10 / Table 18: `PR` [0], `NextLevel` [11:9] the level below, the +/// next table at [51:12], `IR` [61], `IW` [62]. +#[test] +fn a_directory_entry_names_the_level_below() { + let below = Phys::new(ROOT).expect("aligned"); + let want = word(&[(0, 0, 1), (11, 9, 3), (51, 12, ROOT >> 12), (61, 61, 1), (62, 62, 1)]); + assert_eq!(directory(below, 4), want); + assert_eq!(entry(want), Entry::Directory(below)); + assert_eq!(entry(directory(below, 3)), Entry::Directory(below)); +} + +/// Figure 9 / Table 17 at level 2: `PR`, `NextLevel` 000b, the page at +/// [51:21] of a 2 MiB-aligned address, `FC` [60], `IR`, `IW`. +#[test] +fn a_leaf_is_a_two_mebibyte_page_read_write_and_coherent() { + const PAGE: u64 = 0x7_6540_0000; + let want = word(&[(0, 0, 1), (51, 21, PAGE >> 21), (60, 60, 1), (61, 61, 1), (62, 62, 1)]); + assert_eq!(leaf(Phys::new(PAGE).expect("aligned")), want); + assert_eq!(entry(want), Entry::Leaf); + assert_eq!(entry(want & !1), Entry::Absent); + assert_eq!(entry(0), Entry::Absent); +} + +/// Figures 44, 45, 46 and 51: the code at [63:60] of the first doubleword. +#[test] +fn each_command_is_its_code_and_operands() { + const STORE: u64 = 0xA_BCDE_F008; + let wait = Command::CompletionWait { store: Phys::new(STORE).expect("aligned"), data: 0x600D_F00D }; + assert_eq!(wait.words(), [word(&[(0, 0, 1), (51, 3, STORE >> 3), (63, 60, 1)]), 0x600D_F00D]); + assert_eq!(Command::InvalidateDevice(0x0318).words(), [word(&[(15, 0, 0x0318), (63, 60, 2)]), 0]); + // `S` [0] and `PDE` [1] of the second doubleword, address [63:12] all ones + // but bit 63: every address of the domain (§2.4.3's software note). + assert_eq!( + Command::InvalidateDomain(0xBEEF).words(), + [word(&[(47, 32, 0xBEEF), (63, 60, 3)]), word(&[(0, 0, 1), (1, 1, 1), (62, 12, (1 << 51) - 1)])] + ); + assert_eq!(Command::InvalidateAll.words(), [word(&[(63, 60, 8)]), 0]); +} + +/// Figure 56 / Table 57: `DeviceID` [15:0], `I` [51], `RW` [53], the +/// address in the second doubleword. +#[test] +fn a_page_fault_is_its_requesters_with_its_access() { + let read = [word(&[(15, 0, 0x0303), (47, 32, 2), (52, 52, 1), (63, 60, 2)]), 0x4000_0000_0000]; + assert_eq!(event(read), Event::Requester { code: 2, device: 0x0303, address: 0x4000_0000_0000, write: Some(false) }); + let write = [read[0] | word(&[(53, 53, 1)]), read[1]]; + assert!(matches!(event(write), Event::Requester { write: Some(true), .. })); + let message = [word(&[(15, 0, 0x0303), (51, 51, 1), (53, 53, 1), (63, 60, 2)]), 0xFEE0_0000]; + assert!(matches!(event(message), Event::Requester { device: 0x0303, write: None, .. })); +} + +/// Every record a requester can cause is its, never the unit's: Figure 62 +/// (`INVALID_DEVICE_REQUEST`), Figure 61 (`IOTLB_INV_TIMEOUT`), and Figure +/// 64 (`INVALID_PPR_REQUEST`, `RX` [+04 16] clear, `WP` [+04 21]), whose +/// `DeviceID` is the peripheral's (Table 67); with `RX` set it is a command +/// of this driver's (Figure 65). +#[test] +fn a_record_a_requester_causes_is_its_own() { + let request = [word(&[(15, 0, 0x00A2), (63, 60, 0b1000)]), 0xFD_F800_0000]; + assert_eq!(event(request), Event::Requester { code: 0b1000, device: 0x00A2, address: 0xFD_F800_0000, write: None }); + let timeout = [word(&[(15, 0, 0x0318), (63, 60, 0b0111)]), 0x1000]; + assert!(matches!(event(timeout), Event::Requester { code: 0b0111, device: 0x0318, write: None, .. })); + let ppr = [word(&[(15, 0, 0xFF07), (53, 53, 1), (63, 60, 0b1001)]), 0x7000]; + assert_eq!(event(ppr), Event::Requester { code: 0b1001, device: 0xFF07, address: 0x7000, write: Some(true) }); + let completion = [ppr[0] | word(&[(48, 48, 1)]), 0x10]; + assert_eq!(event(completion), Event::Unit { code: 0b1001 }); +} + +/// Table 42: every other code is a table, command or the unit's own. +#[test] +fn every_other_record_is_the_units() { + for code in [0b0000u64, 0b0001, 0b0011, 0b0100, 0b0101, 0b0110, 0b1010, 0b1111] { + assert_eq!(event([code << 60 | 0x0303, 0]), Event::Unit { code: code as u8 }); + } + assert_eq!(name(0b0001), "ILLEGAL_DEV_TABLE_ENTRY"); + assert_eq!(name(0b1001), "INVALID_PPR_REQUEST"); + assert_eq!(name(0b0000), "reserved"); +} + +/// MMIO 0030h: `IASup` [6], `HATS` [11:10] counting from four levels, 11b +/// reserved. The AMD bench's unit reads 0x206d73ef22254ade. +#[test] +fn the_feature_register_says_invalidate_all_and_host_levels() { + let bench = Features(0x206d_73ef_2225_4ade); + assert!(bench.invalidate_all()); + assert_eq!(bench.host_levels(), Some(6)); + assert!(!Features(word(&[(5, 0, 0x3F), (7, 7, 1)])).invalidate_all()); + assert_eq!(Features(0).host_levels(), Some(4)); + assert_eq!(Features(word(&[(11, 10, 0b11)])).host_levels(), None); +} diff --git a/kernel/src/actuator.rs b/kernel/src/actuator.rs index f61d1cd649..db5fbfecb0 100644 --- a/kernel/src/actuator.rs +++ b/kernel/src/actuator.rs @@ -130,6 +130,13 @@ actuators! { /// `iommu_firmware_left` and `amdvi_firmware_left`. iommu_firmware_left = "iommu-firmware-left"; + /// QEMU's two `iommu-testdev` asked three things of the AMD-Vi unit: + /// one on a domain of its own writes where it maps, the other on the + /// identity domain writes to memory, then where that domain maps + /// nothing, and the machine halts on the record. Judged by + /// `amdvi_selftest`. + amdvi_selftest = "amdvi-selftest"; + /// Read the DMAR's flags with `INTR_REMAP` clear, as on a platform whose /// firmware says it does not remap. Judged by `claim_refused_without_remapping`. iommu_no_remap = "iommu-no-remap"; diff --git a/kernel/src/arch/aarch64/smmu/domain.rs b/kernel/src/arch/aarch64/smmu/domain.rs index f40e4c6413..6860760f02 100644 --- a/kernel/src/arch/aarch64/smmu/domain.rs +++ b/kernel/src/arch/aarch64/smmu/domain.rs @@ -1,7 +1,7 @@ //! A device's address space: one context descriptor, tagged with an ASID of //! its own, over stage 1 tables that map 2 MiB blocks //! (`toyos_smmu::table`), and the device addresses it hands out, over the -//! unit's 48-bit input (`crate::iommu::window`). +//! unit's 48-bit input (`crate::iommu::domains`). //! //! **And the ITS's doorbell, the one 4 KiB page at its own address**: the unit //! translates a function's message as it does any write it makes, so a domain @@ -20,10 +20,10 @@ use toyos_smmu::table::{entry, next, plan, Access, Entry, Leaf, INPUT_BITS}; use toyos_smmu::Asid; use super::{window, Live, UNIT}; -use crate::iommu::window::Window; +use crate::iommu::domains; use crate::iommu::{DomainId, IommuError, Iova, StreamId}; use crate::log; -use crate::mm::{PAGE_2M, PAGE_SIZE}; +use crate::mm::PAGE_SIZE; /// One 2 MiB block of memory at `phys`, which a device reads and writes. fn block(phys: u64) -> Leaf { @@ -31,20 +31,16 @@ fn block(phys: u64) -> Leaf { } /// Domain ids start at 1: `DomainId` is never 0. -const FIRST: u16 = 1; +pub(super) const FIRST: u16 = 1; -pub(super) struct Domain { +#[derive(Clone, Copy)] +pub(super) struct Space { asid: Asid, root: Phys<12>, context: Phys<6>, - addresses: Window, } impl Live { - fn domain(&mut self, id: DomainId) -> &mut Domain { - &mut self.domains[usize::from(id.raw() - FIRST)] - } - /// `leaf` at `at`, its tables grown on the way. fn put(&mut self, root: Phys<12>, at: u64, leaf: Leaf) { let path = plan(at, leaf, &self.unit).expect("SMMU: a handed-out address"); @@ -90,16 +86,21 @@ impl Live { } } +/// Why `function` keeps the machine's own addresses rather than a domain of +/// its own: never, since nothing this kernel reads of the IORT says so. +pub fn untranslated(_function: StreamId) -> Option<&'static str> { + None +} + /// A new domain, with `room` bytes of it handed out first: refused, with no id /// spent, where it would have less than that. pub fn create(room: u64) -> Result<(DomainId, Iova), IommuError> { let doorbell = super::super::irqchip::its::doorbell_page(); let mut held = UNIT.lock(); let live = held.as_mut().ok_or(IommuError::NoUnit)?; - let domains = if live.unit.asid(0x100).is_some() { 1 << 16 } else { 1 << 8 }; - let id = u16::try_from(usize::from(FIRST) + live.domains.len()).map_err(|_| IommuError::DomainsExhausted(domains))?; - let asid = live.unit.asid(id).ok_or(IommuError::DomainsExhausted(domains))?; - let (addresses, first) = Window::new(INPUT_BITS as u8, &live.reserved, room)?; + let ceiling = if live.unit.asid(0x100).is_some() { 1 << 16 } else { 1 << 8 }; + let next = live.domains.prepare(INPUT_BITS as u8, ceiling, room)?; + let asid = live.unit.asid(next.id).expect("SMMU: an id below the unit's ASIDs"); let root = Phys::new(live.memory.alloc(PAGE_SIZE)).expect("SMMU: a table below the unit's output size"); let context = Phys::new(live.memory.alloc(64)).expect("SMMU: a descriptor below the unit's output size"); let words = Cd::new(root, asid, &live.unit).expect("SMMU: a table below the unit's output size").words(); @@ -110,39 +111,29 @@ pub fn create(room: u64) -> Result<(DomainId, Iova), IommuError> { } descriptor.write_u64(0, words[0]); if let Some(page) = doorbell { - assert!(page < addresses.floor(), "SMMU: the doorbell at {page:#x} is inside the addresses a domain hands out"); + assert!(page < next.addresses.floor(), "SMMU: the doorbell at {page:#x} is inside the addresses a domain hands out"); live.put(root, page, Leaf::Doorbell(Phys::new(page).expect("SMMU: a 4 KiB page below 2^48"))); } - let domain = Domain { asid, root, context, addresses }; - log!( - "iommu: domain{id} root={:#x} context={:#x} asid={} addresses from {:#x} to {:#x}, the doorbell {}", + let what = alloc::format!( + "root={:#x} context={:#x} asid={} the doorbell {}", root.get(), context.get(), asid.get(), - addresses.floor(), - addresses.ceiling(), match doorbell { - Some(page) => alloc::format!("page at {page:#x}"), - None => "page nowhere: no ITS is armed".into(), + Some(page) => alloc::format!("page at {page:#x},"), + None => "page nowhere: no ITS is armed,".into(), }, ); - live.domains.push(domain); - Ok((DomainId::new(id), first)) + Ok(live.domains.add(next, Space { asid, root, context }, what)) } pub fn map(id: DomainId, phys: u64, bytes: u64) -> Result { - if !phys.is_multiple_of(PAGE_2M) { - return Err(IommuError::Unaligned(phys)); - } let mut held = UNIT.lock(); let live = held.as_mut().expect("a domain exists only on an armed unit"); - let domain = live.domain(id); - let at = domain.addresses.reserve(bytes).ok_or(IommuError::AddressesExhausted(domain.addresses.ceiling()))?; - let root = domain.root; - for offset in (0..bytes).step_by(PAGE_2M as usize) { - live.put(root, at.raw() + offset, block(phys + offset)); + let (space, at) = live.domains.map(id, phys, bytes)?; + for (at, phys) in domains::leaves(at, phys, bytes) { + live.put(space.root, at, block(phys)); } - log!("iommu: domain{} maps {phys:#x}..{:#x} at {:#x}", id.raw(), phys + bytes.next_multiple_of(PAGE_2M), at.raw()); Ok(at) } @@ -150,43 +141,31 @@ pub fn map(id: DomainId, phys: u64, bytes: u64) -> Result { /// and whose mapping was taken back. pub fn map_at(id: DomainId, at: Iova, phys: u64, bytes: u64) -> Result<(), IommuError> { place(id, at, phys, bytes)?; - log!("iommu: domain{} maps {phys:#x}..{:#x} at {:#x} again", id.raw(), phys + bytes.next_multiple_of(PAGE_2M), at.raw()); + domains::placed_again(id, at, phys, bytes); Ok(()) } /// [`map_at`] without its record line, for a mapping a holder makes and takes -/// back as often as it likes. Room never handed out is a kernel bug: [`map`] -/// may yet hand it out. -pub fn place(id: DomainId, at: Iova, phys: u64, bytes: u64) -> Result { - if !phys.is_multiple_of(PAGE_2M) { - return Err(IommuError::Unaligned(phys)); - } +/// back as often as it likes. +pub fn place(id: DomainId, at: Iova, phys: u64, bytes: u64) -> Result<(), IommuError> { let mut held = UNIT.lock(); let live = held.as_mut().expect("a domain exists only on an armed unit"); - let domain = live.domain(id); - assert!(domain.addresses.handed_out(at, bytes), "iommu: domain{} never handed out {:#x}+{bytes:#x}", id.raw(), at.raw()); - let root = domain.root; - for offset in (0..bytes).step_by(PAGE_2M as usize) { - live.put(root, at.raw() + offset, block(phys + offset)); + let space = live.domains.place(id, at, phys, bytes)?; + for (at, phys) in domains::leaves(at, phys, bytes) { + live.put(space.root, at, block(phys)); } - Ok(id.raw()) + Ok(()) } /// Take `bytes` at `at` back: no device reaches them once this returns. pub fn unmap(id: DomainId, at: Iova, bytes: u64) -> Result<(), IommuError> { let mut held = UNIT.lock(); let live = held.as_mut().expect("a domain exists only on an armed unit"); - let (root, asid) = (live.domain(id).root, live.domain(id).asid); - let mut result = Ok(()); - for offset in (0..bytes).step_by(PAGE_2M as usize) { - let here = at.raw() + offset; - if !live.take(root, here) { - result = Err(IommuError::NotMapped(Iova::translated(here))); - break; - } - } + let space = live.domains.space(id); + let result = domains::leaves(at, 0, bytes) + .try_for_each(|(here, _)| if live.take(space.root, here) { Ok(()) } else { Err(IommuError::NotMapped(Iova::translated(here))) }); // Whatever was cleared before a refusal is gone from the unit too. - live.issue(&[Command::InvalidateAsid(asid)]); + live.issue(&[Command::InvalidateAsid(space.asid)]); result } @@ -196,8 +175,7 @@ pub fn attach(function: StreamId, id: DomainId) { let mut held = UNIT.lock(); let live = held.as_mut().expect("a domain exists only on an armed unit"); let stream = live.stream(function); - let context = live.domain(id).context; - let ste = Ste::stage1(context, &live.unit).expect("SMMU: a descriptor below the unit's output size"); + let ste = Ste::stage1(live.domains.space(id).context, &live.unit).expect("SMMU: a descriptor below the unit's output size"); live.write_entry(stream, ste); crate::iommu::fault::attached(function, id.raw()); log!("iommu: {function} moves to domain{}", id.raw()); diff --git a/kernel/src/arch/aarch64/smmu/mod.rs b/kernel/src/arch/aarch64/smmu/mod.rs index dea774b864..17e379b803 100644 --- a/kernel/src/arch/aarch64/smmu/mod.rs +++ b/kernel/src/arch/aarch64/smmu/mod.rs @@ -123,9 +123,7 @@ struct Live { /// StreamID, and the DeviceID the unit's own node maps that stream on to /// at an ITS. routes: Vec<(StreamId, u32, Option)>, - domains: Vec, - /// What no domain's addresses may reach, as `(start, end)`. - reserved: Vec<(u64, u64)>, + domains: crate::iommu::domains::Domains, } impl Live { @@ -401,9 +399,9 @@ fn program( streams, memory, routes, - domains: Vec::new(), - reserved, + domains: crate::iommu::domains::Domains::new(domain::FIRST), }; + live.domains.avoid(reserved); live.issue(&[Command::ForgetAll, Command::InvalidateAll]); regs.write64(reg::EVENTQ_BASE, events_base); diff --git a/kernel/src/arch/aarch64/smmu/selftest.rs b/kernel/src/arch/aarch64/smmu/selftest.rs index 8d34b7774b..8d3db5c07e 100644 --- a/kernel/src/arch/aarch64/smmu/selftest.rs +++ b/kernel/src/arch/aarch64/smmu/selftest.rs @@ -22,33 +22,12 @@ //! recorded nothing. use crate::drivers::pci::PciDevice; +use crate::iommu::testdev::{registers, verdict, write, DEVICE, LANDED, REFUSED, VENDOR}; use crate::iommu::OwnSpace; use crate::log; -use crate::mm::policy::MmioPolicy; -use crate::mm::{Mmio, PAGE_2M}; +use crate::mm::PAGE_2M; use crate::time::{Duration, Tripwire}; -/// `hw/misc/iommu-testdev.h`: Red Hat's vendor and its test device. -const VENDOR: u16 = 0x1b36; -const DEVICE: u16 = 0x0005; - -/// Its BAR0: reading `TRIGGERING` runs an armed DMA; `GVA` is the device -/// address written, `GPA` the physical one read back, `LEN` the bytes, -/// `DBELL` bit 0 arms, and `RESULT` says how it went. -const TRIGGERING: u64 = 0x00; -const GVA_LO: u64 = 0x04; -const GVA_HI: u64 = 0x08; -const LEN: u64 = 0x0c; -const RESULT: u64 = 0x10; -const DBELL: u64 = 0x14; -const GPA_LO: u64 = 0x1c; -const GPA_HI: u64 = 0x20; -const BAR_BYTES: u64 = 0x1000; - -/// `RESULT`: the word landed at `GPA`; the write was refused. -const LANDED: u32 = 0; -const REFUSED: u32 = 0xdead_0002; - /// How long a refused write's events are given to reach this CPU once it /// unmasks. const EVENT_TIMEOUT: Tripwire = Tripwire::absurd( @@ -63,35 +42,6 @@ pub(super) fn stands_unrouted(device: &PciDevice, devices: &[PciDevice]) -> bool device.is_id(VENDOR, DEVICE) && devices.iter().any(|d| d.is_id(VENDOR, DEVICE) && at(d) > at(device)) } -/// `device`'s registers, its memory decode on. No Bus Master Enable: the -/// device writes through the unit's address space whatever its `COMMAND` says. -fn registers(device: &PciDevice) -> Mmio { - let bar = device.memory_bar(0).expect("smmu-selftest: an iommu-testdev's BAR0 is unassigned").address(); - let regs = crate::mm::paging::map_mmio(bar, BAR_BYTES, MmioPolicy::Uncacheable); - device.enable_memory_space(); - regs -} - -/// The device's write of four bytes at `at`, read back at `phys`: its `RESULT`. -fn write(regs: Mmio, at: u64, phys: u64) -> u32 { - regs.write_u32(GVA_LO, at as u32); - regs.write_u32(GVA_HI, (at >> 32) as u32); - regs.write_u32(GPA_LO, phys as u32); - regs.write_u32(GPA_HI, (phys >> 32) as u32); - regs.write_u32(LEN, 4); - regs.write_u32(DBELL, 1); - regs.read_u32(TRIGGERING); - regs.read_u32(RESULT) -} - -fn verdict(result: u32, want: u32) -> &'static str { - match (result == want, want) { - (true, REFUSED) => "refused", - (true, _) => "landed there", - (false, _) => "FAIL", - } -} - pub(super) fn run(devices: &[PciDevice]) { let mut testdevs = devices.iter().filter(|d| d.is_id(VENDOR, DEVICE)); let (Some(stray), Some(device), None) = (testdevs.next(), testdevs.next(), testdevs.next()) else { diff --git a/kernel/src/arch/x86_64/amdvi/domain.rs b/kernel/src/arch/x86_64/amdvi/domain.rs new file mode 100644 index 0000000000..1012393b43 --- /dev/null +++ b/kernel/src/arch/x86_64/amdvi/domain.rs @@ -0,0 +1,211 @@ +//! Address spaces a requester is put in: four-level host tables mapping +//! 2 MiB pages (`toyos_amdvi::table`), tagged with a domain id of their own, +//! over the device addresses `iommu::domains` hands out. +//! +//! Every change is published before it returns: a leaf put in or taken out +//! by `INVALIDATE_IOMMU_PAGES` for the domain, since a unit may keep a +//! translation that faulted (§2.1.3), and a requester moved by +//! `INVALIDATE_DEVTAB_ENTRY` and then the pages of the domain it left +//! (§2.4.2), each behind a completion wait (§2.4.1). +//! +//! A requester an IVMD names stays on the identity domain: firmware declares +//! memory it must keep reaching, which a domain of its own would not map. +//! +//! `DOMAINS` is taken before `UNITS` or `TABLES`, and those two are never held together. + +use alloc::vec::Vec; + +use crate::iommu::domains::{self, Domains}; +use crate::iommu::{DomainId, IommuError, Iova, StreamId}; +use crate::log; +use crate::sync::Lock; +use toyos_acpi::Requesters; +use toyos_amdvi::dte::{Dte, Setting}; +use toyos_amdvi::table::{directory, entry, index, leaf, Entry, LEAF_LEVEL, ROOT_LEVEL}; +use toyos_phys::Phys as Address; + +use super::{memory, submit, Tables, TABLES}; +use toyos_amdvi::queue::Command; + +/// The id every blocked entry is tagged with, which Table 8 requires valid +/// wherever `V` is set; no translating entry uses it. +pub(super) const BLOCKED: u16 = 0; +/// The identity domain's id: every enumerated function until a driver moves it. +pub(super) const IDENTITY: u16 = 1; + +/// What a translating entry names: its domain's root table. +#[derive(Clone, Copy)] +pub struct Space { + root: Address<12>, +} + +struct Amd { + domains: Domains, + /// Firmware's DTE setting for each requester id, which an entry keeps + /// whichever domain it names. + settings: Vec, + /// The requesters an IVMD names. + kept: Vec, + /// The ids a programmed unit serves, as inclusive ranges. + translated: Vec<(u16, u16)>, +} + +static DOMAINS: Lock> = Lock::new(None); + +/// Before any domain is made, once a unit will translate: every one is built +/// clear of `reserved`, `kept`'s requesters stay on the identity domain, and +/// only a requester in `translated` is given one. +pub(super) fn start(reserved: Vec<(u64, u64)>, settings: Vec, kept: Vec, translated: Vec<(u16, u16)>) { + let mut domains = Domains::new(IDENTITY + 1); + domains.avoid(reserved); + *DOMAINS.lock() = Some(Amd { domains, settings, kept, translated }); +} + +/// Why `stream` keeps the machine's own addresses rather than a domain of its +/// own, if it does: no programmed unit serves it, so nothing would translate +/// one; or an IVMD names it, so firmware reserves memory it must keep +/// reaching, which the identity domain maps and a domain of its own would not. +pub fn untranslated(stream: StreamId) -> Option<&'static str> { + let held = DOMAINS.lock(); + let amd = held.as_ref()?; + let id = stream.requester(); + if !amd.translated.iter().any(|&(first, last)| (first..=last).contains(&id)) { + Some("no unit this kernel programmed serves it") + } else if keeps_named(amd, id) { + Some("firmware reserves memory it must keep reaching") + } else { + None + } +} + +/// A new domain, with `room` bytes of it handed out first: refused, with no id +/// spent, where the domain would have less than that. +pub fn create(room: u64) -> Result<(DomainId, Iova), IommuError> { + let mut held = DOMAINS.lock(); + let amd = held.as_mut().ok_or(IommuError::NoUnit)?; + let next = amd.domains.prepare(9 * ROOT_LEVEL + 12, 1 << 16, room)?; + let space = Space { root: TABLES.lock().alloc() }; + Ok(amd.domains.add(next, space, alloc::format!("root={:#x} mode={ROOT_LEVEL}", space.root.get()))) +} + +pub fn map(id: DomainId, phys: u64, bytes: u64) -> Result { + let mut held = DOMAINS.lock(); + let amd = held.as_mut().expect("amdvi: a domain exists only once a unit translates"); + let (space, at) = amd.domains.map(id, phys, bytes)?; + place_leaves(id, space, at, phys, bytes); + Ok(at) +} + +/// Put `bytes` at `phys` at `at`, room this domain handed out before and whose +/// mapping was taken back: a device still aimed there reaches these pages. +pub fn map_at(id: DomainId, at: Iova, phys: u64, bytes: u64) -> Result<(), IommuError> { + place(id, at, phys, bytes)?; + domains::placed_again(id, at, phys, bytes); + Ok(()) +} + +/// [`map_at`] without its record line, for a mapping a holder makes and +/// takes back as often as it likes. +pub fn place(id: DomainId, at: Iova, phys: u64, bytes: u64) -> Result<(), IommuError> { + let held = DOMAINS.lock(); + let amd = held.as_ref().expect("amdvi: a domain exists only once a unit translates"); + let space = amd.domains.place(id, at, phys, bytes)?; + place_leaves(id, space, at, phys, bytes); + Ok(()) +} + +/// Under `DOMAINS`: the leaves, then every unit's cache of the domain. +fn place_leaves(id: DomainId, space: Space, at: Iova, phys: u64, bytes: u64) { + { + let mut tables = TABLES.lock(); + for (at, phys) in domains::leaves(at, phys, bytes) { + put(&mut tables, space.root, at, phys); + } + } + submit(&[Command::InvalidateDomain(id.raw())]); +} + +/// Take `bytes` at `at` back; whatever was cleared before a refusal is gone +/// from every unit too. +pub fn unmap(id: DomainId, at: Iova, bytes: u64) -> Result<(), IommuError> { + let held = DOMAINS.lock(); + let amd = held.as_ref().expect("amdvi: a domain exists only once a unit translates"); + let space = amd.domains.space(id); + let result = domains::leaves(at, 0, bytes) + .try_for_each(|(here, _)| if take(space.root, here) { Ok(()) } else { Err(IommuError::NotMapped(Iova::translated(here))) }); + submit(&[Command::InvalidateDomain(id.raw())]); + result +} + +/// Moves `stream` onto domain `id`, its translation half replaced in one +/// write (§2.2.2.2): translating through it, and only it, once this returns. +pub fn attach(stream: StreamId, id: DomainId) { + let held = DOMAINS.lock(); + let amd = held.as_ref().expect("amdvi: a domain exists only once a unit translates"); + let requester = stream.requester(); + assert!( + amd.translated.iter().any(|&(first, last)| (first..=last).contains(&requester)) && !keeps_named(amd, requester), + "amdvi: {stream} was moved to a domain of its own, which `untranslated` refuses it" + ); + let space = amd.domains.space(id); + let slot = super::entry(requester); + let old = [slot.read_u64(0), slot.read_u64(8)]; + let new = Dte::translated(space.root, id.raw(), amd.settings[usize::from(requester)]).words(); + super::super::iommu_unit::exchange16(slot, old, new); + submit(&[Command::InvalidateDevice(requester), Command::InvalidateDomain(Dte::domain_of(old))]); + crate::iommu::fault::attached(stream, id.raw()); + log!("iommu: {stream} moves to domain{}", id.raw()); +} + +fn keeps_named(amd: &Amd, id: u16) -> bool { + amd.kept.iter().any(|named| match *named { + Requesters::All => true, + Requesters::One(one) => one == id, + Requesters::Range { first, last } => (first..=last).contains(&id), + }) +} + +/// The 8-byte entry `index` of the table at `table`. +fn slot(table: Address<12>, index: usize) -> crate::mm::Mmio { + memory(table.get() + index as u64 * 8, 8) +} + +/// The 2 MiB leaf at `at` under `root` reads and writes `phys`, its tables +/// grown on the way. +pub(super) fn put(tables: &mut Tables, root: Address<12>, at: u64, phys: u64) { + let mut table = root; + for level in (LEAF_LEVEL + 1..=ROOT_LEVEL).rev() { + let here = slot(table, index(at, level)); + table = match entry(here.read_u64(0)) { + Entry::Directory(below) => below, + Entry::Absent => { + let below = tables.alloc(); + here.write_u64(0, directory(below, level)); + below + } + Entry::Leaf => panic!("amdvi: {at:#x}'s level {level} entry is a page, where a table goes"), + }; + } + let here = slot(table, index(at, LEAF_LEVEL)); + // A live leaf here is memory a holder still reaches: the caller takes a + // mapping back before it puts another at its address. + assert!(entry(here.read_u64(0)) == Entry::Absent, "amdvi: {at:#x} was still mapped when a new leaf was written there"); + here.write_u64(0, leaf(Address::new(phys).expect("amdvi: a 2 MiB-aligned page below 2^48"))); +} + +/// Clears the leaf at `at`; `false` where none is mapped. +fn take(root: Address<12>, at: u64) -> bool { + let mut table = root; + for level in (LEAF_LEVEL + 1..=ROOT_LEVEL).rev() { + match entry(slot(table, index(at, level)).read_u64(0)) { + Entry::Directory(below) => table = below, + Entry::Absent | Entry::Leaf => return false, + } + } + let here = slot(table, index(at, LEAF_LEVEL)); + if entry(here.read_u64(0)) != Entry::Leaf { + return false; + } + here.write_u64(0, 0); + true +} diff --git a/kernel/src/arch/x86_64/amdvi/fault.rs b/kernel/src/arch/x86_64/amdvi/fault.rs new file mode 100644 index 0000000000..a1ccfd5f92 --- /dev/null +++ b/kernel/src/arch/x86_64/amdvi/fault.rs @@ -0,0 +1,123 @@ +//! The event log, drained on the unit's own interrupt: bounded by the log's +//! 256 records, allocating nothing and taking no lock; unit state lives in a +//! fixed array of atomics, published before the unit is switched on. What a +//! record ends is `crate::iommu::fault`'s. +//! +//! A record a requester caused is that requester's, whatever it names, and +//! never stops the machine on its own. A record no requester caused — an +//! entry or command of this driver's the unit refused, or the unit's own +//! hardware failing — is a panic: past it nothing can state what the unit +//! lets through. + +use core::sync::atomic::{AtomicU32, AtomicU64, Ordering}; + +use crate::iommu::fault::{self as policy, Access, Fault, Who}; +use crate::log; +use crate::mm::{DirectMap, Mmio, PAGE_SIZE}; +use toyos_amdvi::queue::{event, name, Event, ENTRIES}; +use toyos_amdvi::reg::{self, control, status}; + +use super::{requester, spin_until, REGISTER_WINDOW}; +use crate::arch::x86_64::iommu_unit::MAX_UNITS; + +struct FaultUnit { + /// Physical base of the register window; 0 means this slot is unused. + regs: AtomicU64, + /// Physical base of the event log. + log: AtomicU64, + /// Faults this unit has reported for the life of the boot. + faults: AtomicU32, +} + +static UNITS: [FaultUnit; MAX_UNITS] = + [const { FaultUnit { regs: AtomicU64::new(0), log: AtomicU64::new(0), faults: AtomicU32::new(0) } }; MAX_UNITS]; + +/// Publish unit `index`'s registers and event log to the handler, before its +/// interrupt is armed. +pub(super) fn arm(index: usize, regs: Mmio, log: u64) { + UNITS[index].log.store(log, Ordering::Relaxed); + // Last, with Release: the handler tests this field first. + UNITS[index].regs.store(DirectMap::phys_of(regs.addr() as *const u8), Ordering::Release); +} + +/// `bytes` at `phys`, through the direct map. +fn window(phys: u64, bytes: u64) -> Mmio { + // SAFETY: `phys` is a register window `paging::map_mmio` mapped at boot + // and never unmapped, or an event log `pmm` handed out and the unit + // writes for the machine's life; the direct map covers both. + unsafe { Mmio::over_phys(DirectMap::from_phys(phys), bytes) } +} + +/// A unit raised its event interrupt. +pub fn service() { + let mut kernel_owned = 0usize; + for (index, unit) in UNITS.iter().enumerate() { + let regs = unit.regs.load(Ordering::Acquire); + if regs != 0 { + kernel_owned += drain(index, window(regs, REGISTER_WINDOW), window(unit.log.load(Ordering::Relaxed), PAGE_SIZE)); + } + } + policy::conclude(kernel_owned); + crate::arch::apic::eoi(); +} + +/// Every record from head to tail, of which the answer counts only the ones +/// on an enumerated function this kernel drives. +fn drain(index: usize, regs: Mmio, log: Mmio) -> usize { + // Cleared first, so a record written after the tail is read raises the + // interrupt again (MMIO 2020h `EventLogInt`). + let seen = regs.read_u32(reg::STATUS); + regs.write_u32(reg::STATUS, seen & status::EVENT_LOG_INT); + let tail = reg::slot(regs.read_u64(reg::EVENT_TAIL)); + let mut head = reg::slot(regs.read_u64(reg::EVENT_HEAD)); + let mut halts = 0; + while head != tail { + let at = reg::offset(head); + let words = [log.read_u64(at), log.read_u64(at + 8)]; + let fault = match event(words) { + Event::Requester { code, device, address, write } => Fault { + who: Who::Function(requester(device)), + address, + access: match write { + Some(true) => Access::Write, + Some(false) => Access::Read, + None => Access::Unrecorded, + }, + reason: code, + name: name(code), + }, + Event::Unit { code } => panic!( + "iommu: amdvi unit{index} recorded {} ({code:#x}): {:#018x}:{:#018x}", + name(code), + words[1], + words[0] + ), + }; + halts += usize::from(policy::report(index, &UNITS[index].faults, fault)); + head = (head + 1) % ENTRIES; + } + regs.write_u64(reg::EVENT_HEAD, reg::offset(head)); + if seen & status::EVENT_OVERFLOW != 0 { + restart(index, regs); + } + halts +} + +/// The event log, logging again after it overflowed (§2.5.1): what it +/// dropped is said, never read. +fn restart(index: usize, regs: Mmio) { + log!("iommu: amdvi unit{index} event log overflowed — earlier faults are lost"); + regs.write_u64(reg::CONTROL, regs.read_u64(reg::CONTROL) & !control::EVENT_LOG_EN); + spin_until( + || regs.read_u32(reg::STATUS) & status::EVENT_LOG_RUN == 0, + || alloc::format!("iommu: amdvi unit{index} never stopped its event log: STATUS={:#010x}", regs.read_u32(reg::STATUS)), + ); + regs.write_u32(reg::STATUS, status::EVENT_OVERFLOW); + regs.write_u64(reg::CONTROL, regs.read_u64(reg::CONTROL) | control::EVENT_LOG_EN); +} + +/// Records every unit has reported, for the selftest to wait on. +#[cfg(feature = "boot-actuators")] +pub(super) fn recorded() -> u32 { + UNITS.iter().map(|unit| unit.faults.load(Ordering::Relaxed)).sum() +} diff --git a/kernel/src/arch/x86_64/amdvi/mod.rs b/kernel/src/arch/x86_64/amdvi/mod.rs index f669e23569..5302d931ee 100644 --- a/kernel/src/arch/x86_64/amdvi/mod.rs +++ b/kernel/src/arch/x86_64/amdvi/mod.rs @@ -1,61 +1,140 @@ -//! AMD-Vi, the AMD I/O Virtualization Technology (IOMMU Specification 48882). -//! Every register layout and every AMD name of the unit is at or below this -//! module. +//! AMD-Vi, the AMD I/O Virtualization Technology (AMD 48882 rev 3.11). Every +//! register access and every AMD name of the unit is at or below this module; +//! the words the unit and this driver exchange are `toyos_amdvi`'s. //! //! Finds every unit the IVRS describes, logs what each was handed over with, -//! and switches off whatever firmware left on, each field confirmed before -//! the next, so DMA reaches memory untranslated. No unit is programmed: no -//! domain is given and no interrupt is remapped on an AMD-Vi machine -//! (`issues/an-amd-vi-machine-isolates-no-device.md`). +//! switches off whatever firmware left on, and programs each it can use over +//! one device table every unit reads (§2.2.1): an entry per requester id, +//! blocked unless it names a domain, every enumerated function on the +//! identity domain until a driver moves it to one of its own ([`domain`]), +//! and the event log feeding `iommu::fault` ([`fault`]). A unit it cannot use +//! is logged by register value and left off. No interrupt is remapped: +//! every entry's interrupt half is invalid, so the unit forwards messages +//! as written (Table 9). + +pub mod domain; +pub mod fault; +#[cfg(feature = "boot-actuators")] +pub(super) mod selftest; use alloc::format; use alloc::string::String; +use alloc::vec::Vec; +use core::sync::atomic::{AtomicU64, Ordering}; +use crate::drivers::pci::PciDevice; use crate::iommu::StreamId; use crate::log; -use crate::mm::Mmio; +use crate::mm::pmm::{self, PhysPage}; +use crate::mm::{DirectMap, Mmio, PAGE_2M, PAGE_SIZE}; +use crate::sync::Lock; use crate::time::{Duration, Tripwire}; +use toyos_abi::boot::RootBridgeWindow; +use toyos_amdvi::dte::{Dte, Setting, DEVICE_IDS, ENTRY_BYTES}; +use toyos_amdvi::queue::{Command, ENTRIES}; +use toyos_amdvi::reg::{self, control, status, Features}; +use toyos_phys::Phys as Address; -use super::iommu_unit::{register_window, yn}; -use toyos_acpi::{DeviceEntry, Features, Ivrs, IvrsBlock, IvrsRefused, Phys, Requesters, TableError, Uid, Unit}; +use super::iommu_unit::{register_window, yn, MAX_UNITS}; +use toyos_acpi::{ + DeviceEntry, Features as Image, Ivmd, Ivrs, IvrsBlock, IvrsRefused, Phys, Requesters, TableError, Uid, Unit, +}; /// A unit's register window: everything below the performance counters, /// which nothing here reads. const REGISTER_WINDOW: u64 = 0x4000; -const EXTENDED_FEATURES: u64 = 0x0030; -const CONTROL: u64 = 0x0018; -const STATUS: u64 = 0x2020; - -const IOMMU_EN: u64 = 1 << 0; -const EVENT_LOG_EN: u64 = 1 << 2; -const CMD_BUF_EN: u64 = 1 << 12; - -/// `CONTROL` fields firmware may leave on, in Linux's `iommu_disable` order, -/// each with the `STATUS` bit that says it stopped, or `None` where `CONTROL` -/// reading back is the answer: the logs and their interrupts before the unit, -/// which logs nothing once off. +/// `CONTROL` fields firmware may leave on, each with the `STATUS` bit that +/// says it stopped, or `None` where `CONTROL` reading back is the answer: each +/// log stopped before its interrupt and the unit last, since a log's +/// registers move only once its run bit is clear (§2.4, §2.5.1) and a unit +/// switched off logs nothing. const FIRMWARE_LEFT: [(u64, Option, &str); 8] = [ - (CMD_BUF_EN, Some(1 << 4), "CmdBufEn"), - (1 << 3, None, "EventIntEn"), - (EVENT_LOG_EN, Some(1 << 3), "EventLogEn"), - (1 << 28, Some(1 << 8), "GALogEn"), - (1 << 29, None, "GAIntEn"), - (1 << 13, Some(1 << 7), "PPRLogEn"), - (1 << 14, None, "PPRIntEn"), - (IOMMU_EN, None, "IommuEn"), + (control::CMD_BUF_EN, Some(status::CMD_BUF_RUN), "CmdBufEn"), + (control::EVENT_INT_EN, None, "EventIntEn"), + (control::EVENT_LOG_EN, Some(status::EVENT_LOG_RUN), "EventLogEn"), + (control::GA_LOG_EN, Some(status::GA_LOG_RUN), "GALogEn"), + (control::GA_INT_EN, None, "GAIntEn"), + (control::PPR_LOG_EN, Some(status::PPR_LOG_RUN), "PPRLogEn"), + (control::PPR_INT_EN, None, "PPRIntEn"), + (control::IOMMU_EN, None, "IommuEn"), ]; -/// How long a `CONTROL` write is given to show before the kernel panics: a -/// unit that will not stop is one whose reach nothing can state. +/// IVMD flags (Table 113). +const IVMD_UNITY: u8 = 1 << 0; +const IVMD_EXCLUSION: u8 = 1 << 3; + +/// How long the unit is given to show a `CONTROL` write or finish a command +/// before the kernel panics: a unit half-way through either is one whose +/// reach nothing can state. const COMMAND_TIMEOUT: Tripwire = Tripwire::absurd( Duration::from_secs(1), - "a unit that does not stop within a second of being told to is one whose reach nothing can state", + "a unit that does not act on a register write or a command within a second is one whose reach nothing can state", ); -/// Every unit the IVRS describes, switched off; `false` where firmware -/// published no IVRS. -pub fn init(rsdp_addr: u64) -> bool { +/// The 4 KiB tables every unit reads, carved out of 2 MiB pages and never +/// freed: a unit may still be walking any of them. +struct Tables { + /// The device table's own page. + device_table: Option, + pages: Vec, + /// Bytes taken from the newest page; starts full so the first call allocates one. + used: u64, +} + +impl Tables { + const fn new() -> Self { + Self { device_table: None, pages: Vec::new(), used: PAGE_2M } + } + + /// One zeroed 4 KiB table. + fn alloc(&mut self) -> Address<12> { + if self.used == PAGE_2M { + self.pages.push(pmm::alloc_page().expect("amdvi: no physical memory for a table")); + self.used = 0; + } + let at = self.pages.last().expect("amdvi: the page just pushed").direct_map().phys() + self.used; + self.used += PAGE_SIZE; + Address::new(at).expect("amdvi: a table below 2^48") + } +} + +static TABLES: Lock = Lock::new(Tables::new()); + +/// Every translating unit, and the single owner of each one's command buffer: +/// a buffer has one tail, so a second holder would write behind its back. +static UNITS: Lock> = Lock::new(Vec::new()); + +/// `bytes` at `phys`, through the direct map. +fn memory(phys: u64, bytes: u64) -> Mmio { + // SAFETY: every caller names a table, ring or page this module took from + // `pmm` and never frees, which the direct map covers for the machine's life. + unsafe { Mmio::over_phys(DirectMap::from_phys(phys), bytes) } +} + +/// The physical address of the device table every unit reads, one 2 MiB +/// page of 65,536 entries kept in [`TABLES`]; 0 until [`init`] made it. +static DEVICE_TABLE: AtomicU64 = AtomicU64::new(0); + +/// The entry for requester `id`, its translation half. +fn entry(id: u16) -> Mmio { + let table = DEVICE_TABLE.load(Ordering::Acquire); + assert!(table != 0, "amdvi: an entry written before the device table was made"); + memory(table + u64::from(id) * ENTRY_BYTES as u64, 16) +} + +/// A unit described, switched off and found usable, not yet programmed. +struct Ready { + index: usize, + base: u64, + regs: Mmio, + function: PciDevice, +} + +/// Every unit the IVRS describes, switched off, and each this kernel can use +/// programmed; `false` where firmware published no IVRS. `windows` and every +/// IVMD range are what no domain's addresses reach. +pub fn init(rsdp_addr: u64, devices: &[PciDevice], windows: &[RootBridgeWindow]) -> bool { let ivrs = match toyos_acpi::ivrs(crate::drivers::acpi::direct_phys(), rsdp_addr) { Ok(ivrs) => ivrs, Err(IvrsRefused::Table(TableError::Absent)) => return false, @@ -74,15 +153,38 @@ pub fn init(rsdp_addr: u64) -> bool { info.physical_bits, info.virtual_bits, ); - let (mut units, mut off) = (0usize, 0usize); + let enumerated = |id: u16| devices.iter().any(|d| StreamId::pci(d.bus, d.dev, d.func).requester() == id); + // What firmware declares each requester needs, combined where entries + // overlap: a special entry names in its byte what the range around it + // does not. + let mut settings = alloc::vec![Setting(0); DEVICE_IDS]; + let mut ready: Vec = Vec::new(); + let mut ivmds: Vec = Vec::new(); + // The ids a unit that will be programmed serves: a function outside them + // is behind a unit left off, and no domain of its own would translate. + let mut translated: Vec<(u16, u16)> = Vec::new(); + let mut units = 0usize; for block in ivrs.blocks() { match block { IvrsBlock::Unit(unit) => { + if units == MAX_UNITS { + log!("iommu: more than {MAX_UNITS} AMD-Vi units described; the rest are not inventoried"); + break; + } describe(units, &unit); + let function = describe_function(units, &unit, devices); + let mut usable = Ok(()); + let mut serves: Vec<(u16, u16)> = Vec::new(); for entry in ivrs.devices(&unit) { match entry { - Ok(entry) => log!("iommu: amdvi unit{units} serves {}", served(&ivrs, &entry)), - Err(e) => log!("iommu: amdvi unit{units} device entries refused: {e:?}"), + Ok(entry) => { + log!("iommu: amdvi unit{units} serves {}", served(&ivrs, &entry)); + match declare(&entry, &mut settings, enumerated) { + Ok(ids) => serves.extend(ids), + Err(why) => usable = Err(why), + } + } + Err(e) => usable = Err(format!("its device entries are refused: {e:?}")), } } if let Some(regs) = present(units, &unit) { @@ -91,7 +193,13 @@ pub fn init(rsdp_addr: u64) -> bool { leave_on(regs); } switch_off(units, &unit, regs); - off += 1; + match usable.and_then(|()| usable_unit(&unit, regs, function)) { + Ok(function) => { + translated.extend(serves); + ready.push(Ready { index: units, base: unit.base, regs, function }); + } + Err(why) => log!("iommu: amdvi unit{units} is left off: {why}"), + } } units += 1; } @@ -102,32 +210,262 @@ pub fn init(rsdp_addr: u64) -> bool { requester(unit.device) ), IvrsBlock::Memory(m) => { - let requesters = match m.requesters { - Requesters::All => String::from("every requester"), - Requesters::One(id) => format!("{}", requester(id)), - Requesters::Range { first, last } => format!("{}..={}", requester(first), requester(last)), - }; log!( - "iommu: IVRS IVMD seg={} {requesters} {:#018x}+{:#x} unity={} r={} w={} exclusion={}", + "iommu: IVRS IVMD seg={} {} {:#018x}+{:#x} unity={} r={} w={} exclusion={}", m.segment, + requesters(m.requesters), m.start, m.length, - yn(m.flags & 1 != 0), + yn(m.flags & IVMD_UNITY != 0), yn(m.flags & 2 != 0), yn(m.flags & 4 != 0), - yn(m.flags & 8 != 0), + yn(m.flags & IVMD_EXCLUSION != 0), ); + ivmds.push(m); } IvrsBlock::Other { kind, at, len } => { log!("iommu: IVRS block type {kind:#04x} at +{at}, {len} bytes — not used by this kernel") } } } + if let Some(m) = ivmds.iter().find(|m| m.flags & IVMD_EXCLUSION != 0 || m.flags & IVMD_UNITY == 0) { + log!( + "iommu: AMD-Vi isolates no device this boot: firmware's IVMD at {:#x}+{:#x} is {}, which this kernel \ + does not honour, so every unit is left off", + m.start, + m.length, + // Table 113: `ExclusionRange` overrides the rest; `Unity` clear + // lets the range sit at any device address. + if m.flags & IVMD_EXCLUSION != 0 { "an exclusion range" } else { "a range without Unity" }, + ); + return true; + } + if ready.is_empty() { + log!("iommu: AMD-Vi isolates no device this boot: {units} unit(s) described, none usable"); + return true; + } + + // Before any unit is armed: the handler reaches a faulting function's + // config space through this and cannot take a lock to find it. + crate::iommu::fault::describe(devices); + let top = pmm::top(); + for m in &ivmds { + let end = m.start + m.length; + log!( + "iommu: IVMD {} {:#018x}..{end:#018x} {} the identity domain 0x0..{top:#x}", + requesters(m.requesters), + m.start, + if end <= top { "inside" } else { "OUTSIDE" } + ); + } + let identity = build(devices, &settings); + let reserved = windows.iter().map(|w| (w.base, w.end())).chain(ivmds.iter().map(|m| (m.start, m.start + m.length))); + domain::start(reserved.collect(), settings, ivmds.iter().map(|m| m.requesters).collect(), translated); + log!("iommu: amdvi identity domain covers 0x0..{top:#x}, root={:#x}", identity.get()); + for unit in ready { + program(unit, devices); + } + true +} + +/// The ids `entry` declares a DTE setting for, combined into `settings`, and +/// the ids whose requests reach the unit under them, which it serves; +/// refused where it makes an enumerated function issue its requests under +/// another's id, which would share that id's domain. +fn declare( + entry: &DeviceEntry, + settings: &mut [Setting], + enumerated: impl Fn(u16) -> bool, +) -> Result, String> { + let mut set = |first: u16, last: u16, data: u8| { + for id in first..=last { + settings[usize::from(id)].0 |= data; + } + }; + let (first, last, data) = match *entry { + DeviceEntry::All { data } => (0, u16::MAX, data), + DeviceEntry::Select { id, data } | DeviceEntry::Extended { id, data, .. } | DeviceEntry::Hid { id, data, .. } => { + (id, id, data) + } + DeviceEntry::Range { first, last, data } | DeviceEntry::ExtendedRange { first, last, data, .. } => { + (first, last, data) + } + DeviceEntry::Special { used, data, .. } => (used, used, data), + DeviceEntry::Alias { id: first, used, data } | DeviceEntry::AliasRange { first, used, data, .. } => { + let last = match *entry { + DeviceEntry::AliasRange { last, .. } => last, + _ => first, + }; + if let Some(id) = (first..=last).find(|&id| id != used && enumerated(id)) { + return Err(format!("the IVRS has {} issue its requests as {}", requester(id), requester(used))); + } + (used, used, data) + } + DeviceEntry::Other(_) => return Ok(None), + }; + set(first, last, data); + Ok(Some((first, last))) +} + +/// The unit's function, where this driver can program the unit: segment 0, +/// `INVALIDATE_IOMMU_ALL`, a host table depth it can name, and a function +/// with an MSI capability to report faults on. +fn usable_unit(unit: &Unit, regs: Mmio, function: Option) -> Result { + let features = Features(regs.read_u64(reg::EXTENDED_FEATURES)); + if unit.segment != 0 { + return Err(format!("it serves segment {}, and a requester id here names segment 0", unit.segment)); + } + if !features.invalidate_all() { + return Err(format!("EFR {:#018x} has IASup clear", features.0)); + } + if features.host_levels().is_none() { + return Err(format!("EFR {:#018x} has HATS 11b, a reserved depth", features.0)); + } + let function = function.ok_or_else(|| String::from("its own function is not among the enumerated ones"))?; + function + .capability(toyos_pci::msi::CAP_ID) + .map_err(|_| String::from("its function has no MSI capability to report a fault on"))?; + Ok(function) +} + +/// The device table, every entry blocked but each enumerated function's, +/// which names the identity domain; answers the identity domain's root. +fn build(devices: &[PciDevice], settings: &[Setting]) -> Address<12> { + let mut tables = TABLES.lock(); + let page = pmm::alloc_page().expect("amdvi: no physical memory for the device table"); + let at = page.direct_map().phys(); + // Kept, never freed: every unit reads it for the machine's life. + tables.device_table = Some(page); + DEVICE_TABLE.store(at, Ordering::Release); + for (id, setting) in (0..=u16::MAX).zip(settings) { + let words = Dte::blocked(domain::BLOCKED, *setting).words(); + let slot = entry(id); + slot.write_u64(8, words[1]); + slot.write_u64(0, words[0]); + } + let root = tables.alloc(); + let mut phys = 0; + while phys < pmm::top() { + domain::put(&mut tables, root, phys, phys); + phys += PAGE_2M; + } + for device in devices { + let id = StreamId::pci(device.bus, device.dev, device.func).requester(); + let words = Dte::translated(root, domain::IDENTITY, settings[usize::from(id)]).words(); + let slot = entry(id); + slot.write_u64(8, words[1]); + slot.write_u64(0, words[0]); + } + root +} + +/// A unit past [`program`]: its window, its command buffer, and the word its +/// completion waits store. +struct Live { + index: usize, + regs: Mmio, + commands: Mmio, + tail: usize, + done: Address<12>, + waited: u64, +} + +impl Live { + /// `commands`, then a `COMPLETION_WAIT`, finished: every one has taken + /// effect when this returns (§2.4.1). + fn submit(&mut self, commands: &[Command]) { + for chunk in commands.chunks(ENTRIES - 2) { + for command in chunk { + self.push(*command); + } + self.waited += 1; + let (index, regs, waited) = (self.index, self.regs, self.waited); + self.push(Command::CompletionWait { store: Address::new(self.done.get()).expect("aligned"), data: waited }); + regs.write_u64(reg::COMMAND_TAIL, reg::offset(self.tail)); + let done = memory(self.done.get(), 8); + spin_until( + || done.read_u64(0) == waited, + || { + format!( + "iommu: amdvi unit{index} never finished a completion wait: STATUS={:#010x} head={:#x} tail={:#x}", + regs.read_u32(reg::STATUS), + regs.read_u64(reg::COMMAND_HEAD), + regs.read_u64(reg::COMMAND_TAIL) + ) + }, + ); + } + } + + fn push(&mut self, command: Command) { + let [low, high] = command.words(); + let at = reg::offset(self.tail); + self.commands.write_u64(at + 8, high); + self.commands.write_u64(at, low); + self.tail = (self.tail + 1) % ENTRIES; + } +} + +/// `commands` on every translating unit, finished on each before this returns. +fn submit(commands: &[Command]) { + for unit in UNITS.lock().iter_mut() { + unit.submit(commands); + } +} + +/// Programs the unit in the order §2.3.1 gives — the device table, the +/// command buffer and the event log, then `CONTROL` — with the event +/// interrupt armed and the exclusion range cleared before the unit is on; +/// then empties every cache it holds and names each enumerated +/// function's entry. +fn program(unit: Ready, devices: &[PciDevice]) { + let Ready { index, base, regs, function } = unit; + let (commands, events, done) = { + let mut tables = TABLES.lock(); + (tables.alloc(), tables.alloc(), tables.alloc()) + }; + let table = Address::new(DEVICE_TABLE.load(Ordering::Acquire)).expect("amdvi: a device table below 2^48"); + regs.write_u64(reg::DEVICE_TABLE, reg::device_table(table)); + // Writing a base resets the ring's head and tail (§2.4, §2.5). + regs.write_u64(reg::COMMAND_BASE, reg::ring(commands)); + regs.write_u64(reg::EVENT_BASE, reg::ring(events)); + // `ExEn` and `Allow` clear: no address is passed untranslated (MMIO 0020h). + regs.write_u64(reg::EXCLUSION_BASE, 0); + regs.write_u64(reg::EXCLUSION_LIMIT, 0); + fault::arm(index, regs, events.get()); + function.enable_bus_master(); + let (address, data) = super::apic::msi_message(crate::arch::idt::DMA_FAULT_VECTOR); + assert!(function.arm_unit_msi(address, data), "amdvi: unit{index}'s function lost its MSI capability"); + // `Coherent`, its reset value and the one Table 97 recommends, has the + // unit snoop its device table fetches: nothing here flushes a table. + // Written whole: a field firmware left, `DevTblSegEn` among them, would + // otherwise change how the unit reads this table. `switch_off` logged + // what it was handed over with. + let on = control::COHERENT | control::EVENT_LOG_EN | control::EVENT_INT_EN | control::CMD_BUF_EN | control::IOMMU_EN; + regs.write_u64(reg::CONTROL, on); + let running = status::CMD_BUF_RUN | status::EVENT_LOG_RUN; + spin_until( + || regs.read_u32(reg::STATUS) & running == running, + || format!("iommu: amdvi unit{index} never ran its command buffer and event log: STATUS={:#010x}", regs.read_u32(reg::STATUS)), + ); + let mut live = Live { index, regs, commands: memory(commands.get(), PAGE_SIZE), tail: 0, done, waited: 0 }; + // §2.4.8 needs `INVALIDATE_IOMMU_ALL` alone. QEMU 11.1.1 moves a function + // off pass-through only on an `INVALIDATE_DEVTAB_ENTRY` that names it + // (`issues/qemus-invalidate-iommu-all-leaves-every-function-untranslated.md`), + // so each enumerated function is named too, at one command each. + let reread = devices.iter().map(|d| Command::InvalidateDevice(StreamId::pci(d.bus, d.dev, d.func).requester())); + live.submit(&[Command::InvalidateAll].into_iter().chain(reread).collect::>()); log!( - "iommu: AMD-Vi isolates no device this boot: {units} unit(s) described, {off} switched off, so DMA \ - behind those reaches memory untranslated" + "iommu: amdvi unit{index} @{base:#x} translating control={:#018x} status={:#010x} table={:#x} \ + commands={:#x} events={:#x} msi={address:#010x}/{data:#06x} streams={}", + regs.read_u64(reg::CONTROL), + regs.read_u32(reg::STATUS), + table.get(), + commands.get(), + events.get(), + devices.len(), ); - true + UNITS.lock().push(live); } /// A requester id, printed as `pci::enumerate` prints a function. @@ -136,10 +474,18 @@ fn requester(id: u16) -> StreamId { StreamId::pci(bus, devfn >> 3, devfn & 7) } +fn requesters(named: Requesters) -> String { + match named { + Requesters::All => String::from("every requester"), + Requesters::One(id) => format!("{}", requester(id)), + Requesters::Range { first, last } => format!("{}..={}", requester(first), requester(last)), + } +} + fn describe(index: usize, unit: &Unit) { let image = match unit.features { - Features::Reported(word) => format!("features={word:#010x}"), - Features::Image { attributes, efr, efr2 } => { + Image::Reported(word) => format!("features={word:#010x}"), + Image::Image { attributes, efr, efr2 } => { format!("attributes={attributes:#010x} efr_image={efr:#018x} efr2_image={efr2:#018x}") } }; @@ -155,6 +501,28 @@ fn describe(index: usize, unit: &Unit) { ); } +/// The unit's own PCI function, said: its capability ids, which say whether +/// its event interrupt is an MSI or an MSI-X one (§3.1), and its +/// Miscellaneous Information Register 0 (Capability Offset 10h, §3.2): the +/// address widths and the message number that interrupt is raised with. +fn describe_function(index: usize, unit: &Unit, devices: &[PciDevice]) -> Option { + let at = requester(unit.device); + let Some(function) = devices.iter().find(|d| StreamId::pci(d.bus, d.dev, d.func) == at) else { + log!("iommu: amdvi unit{index} function {at} is not among the enumerated functions"); + return None; + }; + let ids: String = function.capabilities().map(|cap| format!(" {:#04x}", cap.id())).collect(); + let misc = function.read_config_u32(u64::from(unit.capability) + 0x10); + log!( + "iommu: amdvi unit{index} function {at} caps=[{}] misc0={misc:#010x} va={} pa={} msinum={}", + ids.trim_start(), + (misc >> 15) & 0x7F, + (misc >> 8) & 0x7F, + misc & 0x1F, + ); + Some(*function) +} + /// What one device entry names, and its DTE setting byte. fn served(ivrs: &Ivrs, entry: &DeviceEntry) -> String { let id = requester; @@ -201,7 +569,7 @@ fn present(index: usize, unit: &Unit) -> Option { return None; }; // A described unit whose window does not decode: firmware bug, or the unit is powered down. - if regs.read_u32(STATUS) == u32::MAX { + if regs.read_u32(reg::STATUS) == u32::MAX { log!( "iommu: amdvi unit{index} @{:#x}: STATUS reads all ones, the unit is described but not present, \ and whatever firmware left on stays on", @@ -226,115 +594,91 @@ fn spin_until(mut done: impl FnMut() -> bool, stuck: impl Fn() -> String) { /// the next. fn switch_off(index: usize, unit: &Unit, regs: Mmio) { log!( - "iommu: amdvi unit{index} handed over control={:#018x} status={:#010x} efr={:#018x}", - regs.read_u64(CONTROL), - regs.read_u32(STATUS), - regs.read_u64(EXTENDED_FEATURES), + "iommu: amdvi unit{index} handed over control={:#018x} status={:#010x} efr={:#018x} \ + exclusion={:#018x}..{:#018x}", + regs.read_u64(reg::CONTROL), + regs.read_u32(reg::STATUS), + regs.read_u64(reg::EXTENDED_FEATURES), + regs.read_u64(reg::EXCLUSION_BASE), + regs.read_u64(reg::EXCLUSION_LIMIT), ); for (bit, run, what) in FIRMWARE_LEFT { - if regs.read_u64(CONTROL) & bit == 0 { + if regs.read_u64(reg::CONTROL) & bit == 0 { continue; } log!("iommu: amdvi unit{index} was handed over with {what} on; it goes off"); - regs.write_u64(CONTROL, regs.read_u64(CONTROL) & !bit); + regs.write_u64(reg::CONTROL, regs.read_u64(reg::CONTROL) & !bit); let stopped = || match run { - Some(run) => regs.read_u32(STATUS) & run == 0, - None => regs.read_u64(CONTROL) & bit == 0, + Some(run) => regs.read_u32(reg::STATUS) & run == 0, + None => regs.read_u64(reg::CONTROL) & bit == 0, }; spin_until(stopped, || { format!( "iommu: amdvi unit{index} never stopped {what}: CONTROL={:#018x} STATUS={:#010x}", - regs.read_u64(CONTROL), - regs.read_u32(STATUS) + regs.read_u64(reg::CONTROL), + regs.read_u32(reg::STATUS) ) }); } log!( "iommu: amdvi unit{index} @{:#x} switched off control={:#018x} status={:#010x}", unit.base, - regs.read_u64(CONTROL), - regs.read_u32(STATUS) + regs.read_u64(reg::CONTROL), + regs.read_u32(reg::STATUS) ); } /// [`crate::actuator::iommu_firmware_left`]: the unit left on as firmware /// protecting memory before the operating system runs may leave it — every -/// requester's device table entry translating through a table that maps -/// nothing, so every DMA is blocked, with its command buffer and event log -/// running. One `INVALIDATE_IOMMU_ALL` empties every cache the unit holds, -/// so it reads every entry as written, and a completion wait says it did. +/// requester's entry translating through a table that maps nothing, so every +/// DMA is blocked, with its command buffer and event log running. One +/// `INVALIDATE_IOMMU_ALL` empties every cache the unit holds, so it reads +/// every entry as written, and a completion wait says it did. /// /// The tables are leaked, as firmware's are its own: the unit reads them /// until it is switched off, and freed they could read back as an entry that /// passes DMA untranslated. #[cfg(feature = "boot-actuators")] fn leave_on(regs: Mmio) { - use crate::mm::{pmm, DirectMap}; - - const DEVICE_TABLE: u64 = 0x0000; - const COMMAND_BASE: u64 = 0x0008; - const EVENT_BASE: u64 = 0x0010; - const COMMAND_HEAD: u64 = 0x2000; - const COMMAND_TAIL: u64 = 0x2008; - const EVENT_HEAD: u64 = 0x2010; - const EVENT_TAIL: u64 = 0x2018; - /// 2^8 entries of 16 bytes, 4 KiB, in bits 59:56 of either base. - const LOG_LEN: u64 = 8 << 56; - const DEVICE_IDS: u64 = 1 << 16; - const ENTRY: u64 = 32; - /// `V`, `TV`, and a four-level table: `Mode` 100b at 11:9; `IR` and `IW` - /// clear. - const BLOCKED: u64 = 1 | (1 << 1) | (4 << 9); - /// `IASup`, set where the unit takes `INVALIDATE_IOMMU_ALL`. - const IA_SUP: u64 = 1 << 6; - /// The opcode alone: every other bit of the command is reserved. - const INVALIDATE_IOMMU_ALL: u64 = 8 << 60; - /// The opcode, and `S`: store the second word at the address. - const COMPLETION_WAIT: u64 = (1 << 60) | 1; - const DONE: u64 = 0x600d_f00d; - - let efr = regs.read_u64(EXTENDED_FEATURES); + let features = Features(regs.read_u64(reg::EXTENDED_FEATURES)); assert!( - efr & IA_SUP != 0, - "amdvi: the unit's EFR {efr:#018x} has IASup clear, so it takes no INVALIDATE_IOMMU_ALL and the actuator \ - cannot make it read the entries it leaves" + features.invalidate_all(), + "amdvi: the unit's EFR {:#018x} has IASup clear, so it takes no INVALIDATE_IOMMU_ALL and the actuator \ + cannot make it read the entries it leaves", + features.0 ); - - let window = |phys: u64, bytes: u64| { - // SAFETY: `phys` is inside a page `pmm::alloc_page` handed out and - // this function leaks, which the direct map covers for the machine's life. - unsafe { Mmio::over_phys(DirectMap::from_phys(phys), bytes) } - }; let table = pmm::alloc_page().expect("amdvi: no memory for the device table the actuator leaves on"); let rest = pmm::alloc_page().expect("amdvi: no memory for the logs the actuator leaves on"); - let table_at = table.direct_map().phys(); - let at = rest.direct_map().phys(); - let (commands_at, events_at, root_at, done_at) = (at, at + 0x1000, at + 0x2000, at + 0x3000); + let (table_at, at) = (table.direct_map().phys(), rest.direct_map().phys()); #[expect(clippy::disallowed_methods, reason = "the unit reads them until it is switched off, as firmware's own tables")] core::mem::forget((table, rest)); + let address = |at: u64| Address::<12>::new(at).expect("amdvi: a page below 2^48"); + let (commands, events, root, done) = (address(at), address(at + 0x1000), address(at + 0x2000), address(at + 0x3000)); - let entries = window(table_at, DEVICE_IDS * ENTRY); - for id in 0..DEVICE_IDS { - entries.write_u64(id * ENTRY, BLOCKED | root_at); - entries.write_u64(id * ENTRY + 8, 1); + let entries = memory(table_at, (DEVICE_IDS * ENTRY_BYTES) as u64); + let blocking = Dte::translated(root, 0, Setting(0)).words(); + for id in 0..DEVICE_IDS as u64 { + entries.write_u64(id * ENTRY_BYTES as u64 + 8, blocking[1]); + entries.write_u64(id * ENTRY_BYTES as u64, blocking[0]); } - regs.write_u64(DEVICE_TABLE, table_at | (DEVICE_IDS * ENTRY / 0x1000 - 1)); - regs.write_u64(COMMAND_BASE, commands_at | LOG_LEN); - regs.write_u64(EVENT_BASE, events_at | LOG_LEN); - for at in [COMMAND_HEAD, COMMAND_TAIL, EVENT_HEAD, EVENT_TAIL] { - regs.write_u64(at, 0); - } - regs.write_u64(CONTROL, regs.read_u64(CONTROL) | CMD_BUF_EN | EVENT_LOG_EN | IOMMU_EN); + regs.write_u64(reg::DEVICE_TABLE, reg::device_table(address(table_at))); + regs.write_u64(reg::COMMAND_BASE, reg::ring(commands)); + regs.write_u64(reg::EVENT_BASE, reg::ring(events)); + let on = control::CMD_BUF_EN | control::EVENT_LOG_EN | control::IOMMU_EN; + regs.write_u64(reg::CONTROL, regs.read_u64(reg::CONTROL) | on); - let commands = [INVALIDATE_IOMMU_ALL, 0, COMPLETION_WAIT | done_at, DONE]; - let ring = window(commands_at, commands.len() as u64 * 8); - for (word, value) in (0..).zip(commands) { - ring.write_u64(word * 8, value); + const DONE: u64 = 0x600d_f00d; + let wait = Command::CompletionWait { store: Address::new(done.get()).expect("aligned"), data: DONE }; + let ring = memory(commands.get(), PAGE_SIZE); + for (slot, command) in [Command::InvalidateAll, wait].into_iter().enumerate() { + let [low, high] = command.words(); + ring.write_u64(reg::offset(slot) + 8, high); + ring.write_u64(reg::offset(slot), low); } - regs.write_u64(COMMAND_TAIL, commands.len() as u64 * 8); - let done = window(done_at, 8); + regs.write_u64(reg::COMMAND_TAIL, reg::offset(2)); + let stored = memory(done.get(), 8); spin_until( - || done.read_u64(0) == DONE, - || format!("amdvi: the actuator's completion wait never landed: STATUS={:#010x}", regs.read_u32(STATUS)), + || stored.read_u64(0) == DONE, + || format!("amdvi: the actuator's completion wait never landed: STATUS={:#010x}", regs.read_u32(reg::STATUS)), ); } diff --git a/kernel/src/arch/x86_64/amdvi/selftest.rs b/kernel/src/arch/x86_64/amdvi/selftest.rs new file mode 100644 index 0000000000..96fe0d7b87 --- /dev/null +++ b/kernel/src/arch/x86_64/amdvi/selftest.rs @@ -0,0 +1,80 @@ +//! `amdvi-selftest`: QEMU's two `iommu-testdev` asked three things of the +//! programmed unit, the first enumerated on a domain of its own and the +//! second left on the identity domain: +//! +//! 1. the first writes at an address its domain maps, and lands at the +//! memory mapped there and nowhere else; +//! 2. the second writes at a physical address of memory, which the identity +//! domain maps to itself, and lands there; +//! 3. the second writes at an address the identity domain does not map, and +//! is refused and recorded, and the machine halts: the function is one +//! this kernel drives. +//! +//! The record is what says the unit translated: a write the unit passed +//! untranslated to 2^46 reaches no memory on a guest of 40 physical bits, +//! and the device answers it refused all the same. The write of 3 is made with this CPU's interrupts masked, so its verdict +//! is on the console before its event is taken. Each step says what it saw +//! on its own line; the harness judges them, and the record by the handler's. + +use crate::drivers::pci::PciDevice; +use crate::iommu::testdev::{registers, verdict, write, DEVICE, LANDED, REFUSED, VENDOR}; +use crate::iommu::{OwnSpace, StreamId}; +use crate::log; +use crate::mm::PAGE_2M; +use crate::time::{Duration, Tripwire}; + +/// How long the refused write's event is given to reach this CPU once it +/// unmasks. +const EVENT_TIMEOUT: Tripwire = Tripwire::absurd( + Duration::from_secs(1), + "the unit writes the record and raises its interrupt while the refused write is still being made", +); + +pub fn run(devices: &[PciDevice]) { + let mut testdevs = devices.iter().filter(|d| d.is_id(VENDOR, DEVICE)); + let (Some(moved), Some(kept), None) = (testdevs.next(), testdevs.next(), testdevs.next()) else { + panic!("amdvi-selftest: this machine has not two iommu-testdevs (1b36:0005)"); + }; + let (moved_regs, kept_regs) = (registers(moved), registers(kept)); + let page = crate::mm::pmm::alloc_page().expect("amdvi-selftest: no page to aim at"); + let phys = page.direct_map().phys(); + let (on_own, on_identity) = + (StreamId::pci(moved.bus, moved.dev, moved.func), StreamId::pci(kept.bus, kept.dev, kept.func)); + + let (space, at) = OwnSpace::create(PAGE_2M).expect("amdvi-selftest: a domain on the programmed unit"); + space.place(at, phys, PAGE_2M).expect("amdvi-selftest: the domain's own first leaf"); + space.attach(moved.bus, moved.dev, moved.func); + let landed = write(moved_regs, at + 0x40, phys + 0x40); + log!( + "amdvi-selftest: {on_own}'s write at {:#x}, mapped to {:#x}, answered {landed:#x}: {}", + at + 0x40, + phys + 0x40, + verdict(landed, LANDED) + ); + + let identity = write(kept_regs, phys + 0x80, phys + 0x80); + log!( + "amdvi-selftest: {on_identity}'s write at {:#x}, on the identity domain, answered {identity:#x}: {}", + phys + 0x80, + verdict(identity, LANDED) + ); + + let before = super::fault::recorded(); + let masked = crate::arch::IrqGuard::close(); + let refused = write(kept_regs, at, phys); + log!( + "amdvi-selftest: {on_identity}'s write at {at:#x}, which the identity domain does not map, answered \ + {refused:#x}: {}", + verdict(refused, REFUSED) + ); + drop(masked); + let deadline = crate::clock::nanos_since_boot() + EVENT_TIMEOUT.nanos(); + while super::fault::recorded() == before { + assert!( + crate::clock::nanos_since_boot() < deadline, + "amdvi-selftest: FAIL: no record of {on_identity}'s refused write was read {EVENT_TIMEOUT} after it" + ); + core::hint::spin_loop(); + } + panic!("amdvi-selftest: FAIL: the handler read the record of a function this kernel drives and the machine went on"); +} diff --git a/kernel/src/arch/x86_64/iommu_unit.rs b/kernel/src/arch/x86_64/iommu_unit.rs index 987ba7edee..951afe6f5f 100644 --- a/kernel/src/arch/x86_64/iommu_unit.rs +++ b/kernel/src/arch/x86_64/iommu_unit.rs @@ -1,10 +1,9 @@ //! The x86-64 IOMMU: VT-d where firmware publishes a DMAR, AMD-Vi where it -//! publishes an IVRS. -//! -//! Only VT-d gives a domain or remaps an interrupt. AMD-Vi programs no unit, -//! so on its machines VT-d holds none: every [`domain`] is refused `NoUnit` -//! and no [`interrupt`] is armed -//! (`issues/an-amd-vi-machine-isolates-no-device.md`). +//! publishes an IVRS, chosen once at [`init`] and asked through [`domain`] +//! and [`fault`]. Only VT-d remaps an interrupt +//! (`issues/an-amd-vi-machine-isolates-no-device.md`, stage 6). + +use core::sync::atomic::{AtomicU8, Ordering}; use crate::drivers::pci::PciDevice; use crate::log; @@ -12,7 +11,26 @@ use crate::mm::policy::MmioPolicy; use crate::mm::Mmio; use toyos_abi::boot::RootBridgeWindow; -pub use super::vtd::{domain, fault, interrupt}; +pub use super::vtd::interrupt; + +/// Which backend [`init`] found: none until it has, and then fixed. +#[derive(Clone, Copy, PartialEq, Eq)] +#[repr(u8)] +enum Backend { + None, + Vtd, + AmdVi, +} + +static BACKEND: AtomicU8 = AtomicU8::new(Backend::None as u8); + +fn backend() -> Backend { + match BACKEND.load(Ordering::Acquire) { + 1 => Backend::Vtd, + 2 => Backend::AmdVi, + _ => Backend::None, + } +} /// x86-64's 52-bit physical-address ceiling: a register base at or above this /// is not an address at all, and would otherwise wrap `DirectMap::as_ptr`'s @@ -21,7 +39,20 @@ const MAX_PHYS: u64 = 1 << 52; /// The units of whichever table firmware published. pub fn init(rsdp_addr: u64, devices: &[PciDevice], windows: &[RootBridgeWindow]) { - if super::vtd::init(rsdp_addr, devices, windows) || super::amdvi::init(rsdp_addr) { + let found = if super::vtd::init(rsdp_addr, devices, windows) { + Backend::Vtd + } else if super::amdvi::init(rsdp_addr, devices, windows) { + Backend::AmdVi + } else { + Backend::None + }; + BACKEND.store(found as u8, Ordering::Release); + // Once the backend is chosen: the selftest makes a domain through it. + #[cfg(feature = "boot-actuators")] + if found == Backend::AmdVi && crate::actuator::amdvi_selftest() { + super::amdvi::selftest::run(devices); + } + if found != Backend::None { return; } // ACPI cannot tell "no IOMMU silicon" from "the IOMMU disabled in @@ -32,6 +63,9 @@ pub fn init(rsdp_addr: u64, devices: &[PciDevice], windows: &[RootBridgeWindow]) ); } +/// Ceiling on units inventoried; a machine past it is told, not silently truncated. +pub(super) const MAX_UNITS: usize = 16; + /// Firmware's register base, mapped only if aligned to its `len`-byte window /// and within the physical range — never clamped to fit, since a base in /// usable RAM would decode as plausible registers until a write lands in @@ -43,6 +77,42 @@ pub(super) fn register_window(base: u64, len: u64) -> Option { Some(crate::mm::paging::map_mmio(base, len, MmioPolicy::Uncacheable)) } +/// Swaps the present 16-byte entry `old` at `entry` for `new` in one +/// `lock cmpxchg16b`: a unit may fetch it at any moment and must read the +/// one or the other, never a half of each. +pub(super) fn exchange16(entry: Mmio, old: [u64; 2], new: [u64; 2]) { + let entry = entry.subregion(0, 16); + assert!(entry.addr().is_multiple_of(16), "iommu: a 16-byte entry at {:#x} is not aligned to it", entry.addr()); + let (found_lo, found_hi): (u64, u64); + // SAFETY: `entry` is 16 bytes of a table the caller's unit reads, 16-byte + // aligned as asserted; `rbx`, which LLVM reserves, holds `new`'s low half + // only between the two moves. Every operand names its register: LLVM may + // give `rbx` to a `reg` operand, and the `xchg` would then swap the + // address out of it. + unsafe { + core::arch::asm!( + "xchg r8, rbx", + "lock cmpxchg16b xmmword ptr [rsi]", + "mov rbx, r8", + in("rsi") entry.addr(), + inout("r8") new[0] => _, + in("rcx") new[1], + inout("rax") old[0] => found_lo, + inout("rdx") old[1] => found_hi, + options(nostack), + ); + } + // `found` is `old` exactly when the swap happened; nothing else writes an + // entry under its backend's lock, so anything else is a kernel bug. + assert!( + [found_lo, found_hi] == old, + "iommu: the entry at {:#x} held {found_hi:#x}:{found_lo:#x}, not the {:#x}:{:#x} read before it was replaced", + entry.addr(), + old[1], + old[0] + ); +} + /// One character per boolean; `n` is printed rather than omitted, since an /// absent field would look like a forgotten one. pub(super) fn yn(v: bool) -> char { @@ -52,3 +122,85 @@ pub(super) fn yn(v: bool) -> char { 'n' } } + +/// A device's address spaces, on whichever backend translates. +pub mod domain { + use super::{backend, Backend}; + use crate::arch::x86_64::{amdvi, vtd}; + use crate::iommu::{DomainId, IommuError, Iova, StreamId}; + + pub fn create(room: u64) -> Result<(DomainId, Iova), IommuError> { + match backend() { + Backend::Vtd => vtd::domain::create(room), + Backend::AmdVi => amdvi::domain::create(room), + Backend::None => Err(IommuError::NoUnit), + } + } + + /// Why `stream` keeps the machine's own addresses rather than a domain of + /// its own, if it does: AMD-Vi's reasons. VT-d answers `None` for every + /// function, an RMRR's included, which it moves off the identity domain + /// like any other. + pub fn untranslated(stream: StreamId) -> Option<&'static str> { + match backend() { + Backend::AmdVi => amdvi::domain::untranslated(stream), + Backend::Vtd | Backend::None => None, + } + } + + /// The backend that minted a `DomainId`; there is one, or none was minted. + fn minted() -> Backend { + let found = backend(); + assert!(found != Backend::None, "iommu: a domain used on a machine where no unit translates"); + found + } + + pub fn map(id: DomainId, phys: u64, bytes: u64) -> Result { + match minted() { + Backend::AmdVi => amdvi::domain::map(id, phys, bytes), + _ => vtd::domain::map(id, phys, bytes), + } + } + + pub fn map_at(id: DomainId, at: Iova, phys: u64, bytes: u64) -> Result<(), IommuError> { + match minted() { + Backend::AmdVi => amdvi::domain::map_at(id, at, phys, bytes), + _ => vtd::domain::map_at(id, at, phys, bytes), + } + } + + pub fn place(id: DomainId, at: Iova, phys: u64, bytes: u64) -> Result<(), IommuError> { + match minted() { + Backend::AmdVi => amdvi::domain::place(id, at, phys, bytes), + _ => vtd::domain::place(id, at, phys, bytes), + } + } + + pub fn unmap(id: DomainId, at: Iova, bytes: u64) -> Result<(), IommuError> { + match minted() { + Backend::AmdVi => amdvi::domain::unmap(id, at, bytes), + _ => vtd::domain::unmap(id, at, bytes), + } + } + + pub fn attach(stream: StreamId, id: DomainId) { + match minted() { + Backend::AmdVi => amdvi::domain::attach(stream, id), + _ => vtd::domain::attach(stream, id), + } + } +} + +/// The fault interrupt, on whichever backend armed it. +pub mod fault { + use super::{backend, Backend}; + use crate::arch::x86_64::{amdvi, vtd}; + + pub fn service() { + match backend() { + Backend::Vtd => vtd::fault::service(), + Backend::AmdVi => amdvi::fault::service(), + Backend::None => panic!("iommu: a fault interrupt on a machine where no unit is armed"), + } + } +} diff --git a/kernel/src/arch/x86_64/vtd/domain.rs b/kernel/src/arch/x86_64/vtd/domain.rs index 35283ea9c3..2d2e04ce2e 100644 --- a/kernel/src/arch/x86_64/vtd/domain.rs +++ b/kernel/src/arch/x86_64/vtd/domain.rs @@ -18,14 +18,13 @@ use crate::log; use alloc::vec::Vec; +use crate::iommu::domains::{self, Domains}; use crate::iommu::{AddressWidth, DomainId, IommuError, Iova, StreamId}; use crate::sync::Lock; -use super::table::{self, Domain}; +use super::table::{self, Space}; use super::{TABLES, UNITS}; -const FIRST: u16 = table::KERNEL_DOMAIN + 1; - /// What every enabled unit agreed on, which is what a domain can be built to. enum Agreement { None, @@ -36,27 +35,22 @@ enum Agreement { Split, } -struct Domains { +struct Vtd { agreement: Agreement, - /// By id minus [`FIRST`]; never shrinks, since a released id would name a - /// domain some unit may still have cached. - live: Vec, - /// What no domain's addresses may reach: the root bridges' windows and - /// the regions firmware reserved, as `(start, end)`. - reserved: Vec<(u64, u64)>, + domains: Domains, } -static DOMAINS: Lock = - Lock::new(Domains { agreement: Agreement::None, live: Vec::new(), reserved: Vec::new() }); +static DOMAINS: Lock = + Lock::new(Vtd { agreement: Agreement::None, domains: Domains::new(table::KERNEL_DOMAIN + 1) }); /// Before any domain is made: every one is built clear of these. pub fn avoid(reserved: Vec<(u64, u64)>) { - DOMAINS.lock().reserved = reserved; + DOMAINS.lock().domains.avoid(reserved); } pub fn unit_agrees(width: AddressWidth, ceiling: u32, mgaw: u8) { - let mut domains = DOMAINS.lock(); - domains.agreement = match domains.agreement { + let mut held = DOMAINS.lock(); + held.agreement = match held.agreement { Agreement::None => Agreement::One(width, ceiling, mgaw), Agreement::One(seen, cap, seen_mgaw) if seen == width => { Agreement::One(width, cap.min(ceiling), seen_mgaw.min(mgaw)) @@ -68,113 +62,76 @@ pub fn unit_agrees(width: AddressWidth, ceiling: u32, mgaw: u8) { /// A new domain, with `room` bytes of it handed out first: refused, with no id /// spent, where the domain would have less than that. pub fn create(room: u64) -> Result<(DomainId, Iova), IommuError> { - let mut domains = DOMAINS.lock(); - let (width, ceiling, mgaw) = match domains.agreement { + let mut held = DOMAINS.lock(); + let (width, ceiling, mgaw) = match held.agreement { Agreement::None => return Err(IommuError::NoUnit), Agreement::Split => return Err(IommuError::WidthsDisagree), Agreement::One(width, ceiling, mgaw) => (width, ceiling, mgaw), }; - let id = FIRST + domains.live.len() as u16; - if u32::from(id) >= ceiling { - return Err(IommuError::DomainsExhausted(ceiling)); - } - let (domain, first) = Domain::new(&mut TABLES.lock(), id, width, mgaw, &domains.reserved, room)?; - log!( - "iommu: domain{id} root={:#x} aw={} mgaw={} addresses from {:#x} to {:#x}", - domain.root().phys(), - width.bits(), - mgaw, - domain.window().floor(), - domain.window().ceiling() - ); - domains.live.push(domain); - Ok((DomainId::new(id), first)) + let next = held.domains.prepare(table::translatable_bits(width, mgaw), ceiling, room)?; + let space = Space { root: TABLES.lock().alloc(), width }; + let what = alloc::format!("root={:#x} aw={} mgaw={mgaw}", space.root.phys(), width.bits()); + Ok(held.domains.add(next, space, what)) } pub fn map(id: DomainId, phys: u64, bytes: u64) -> Result { - if !phys.is_multiple_of(crate::mm::PAGE_2M) { - return Err(IommuError::Unaligned(phys)); - } - let mut domains = DOMAINS.lock(); - let domain = domains.at(id); - let at = domain.window_mut().reserve(bytes).ok_or(IommuError::AddressesExhausted(domain.window().ceiling()))?; - let (did, domain) = (domain.id(), *domain); - let mut units = UNITS.lock(); - table::map(&mut TABLES.lock(), &domain, at, phys, bytes); - for unit in units.iter_mut() { - unit.invalidate_domain(did); - } - log!( - "iommu: domain{did} maps {:#x}..{:#x} at {:#x}", - phys, - phys + bytes.next_multiple_of(crate::mm::PAGE_2M), - at.raw(), - ); + let mut held = DOMAINS.lock(); + let (space, at) = held.domains.map(id, phys, bytes)?; + put(id, space, at, phys, bytes); Ok(at) } /// Put `bytes` at `phys` at `at`, room this domain handed out before and whose /// mapping was taken back: a device still aimed there reaches these pages. -/// Room it never handed out is a kernel bug, since [`map`] may yet hand it out. pub fn map_at(id: DomainId, at: Iova, phys: u64, bytes: u64) -> Result<(), IommuError> { - let did = place(id, at, phys, bytes)?; - log!( - "iommu: domain{did} maps {:#x}..{:#x} at {:#x} again", - phys, - phys + bytes.next_multiple_of(crate::mm::PAGE_2M), - at.raw(), - ); + place(id, at, phys, bytes)?; + domains::placed_again(id, at, phys, bytes); Ok(()) } /// [`map_at`] without its record line, for a mapping a holder makes and /// takes back as often as it likes: a line each would let one process flood /// the record ring. -pub fn place(id: DomainId, at: Iova, phys: u64, bytes: u64) -> Result { - if !phys.is_multiple_of(crate::mm::PAGE_2M) { - return Err(IommuError::Unaligned(phys)); - } - let mut domains = DOMAINS.lock(); - let domain = *domains.at(id); - assert!( - domain.window().handed_out(at, bytes), - "iommu: domain{} never handed out {:#x}+{bytes:#x}", - domain.id(), - at.raw() - ); +pub fn place(id: DomainId, at: Iova, phys: u64, bytes: u64) -> Result<(), IommuError> { + let held = DOMAINS.lock(); + let space = held.domains.place(id, at, phys, bytes)?; + put(id, space, at, phys, bytes); + Ok(()) +} + +/// Under `DOMAINS`: the leaves, then every unit's cache of the domain. +fn put(id: DomainId, space: Space, at: Iova, phys: u64, bytes: u64) { let mut units = UNITS.lock(); - table::map(&mut TABLES.lock(), &domain, at, phys, bytes); + let mut tables = TABLES.lock(); + for (at, phys) in domains::leaves(at, phys, bytes) { + table::put(&mut tables, space, at, phys); + } for unit in units.iter_mut() { - unit.invalidate_domain(domain.id()); + unit.invalidate_domain(id.raw()); } - Ok(domain.id()) } +/// Take `bytes` at `at` back; whatever was cleared before a refusal is gone +/// from every unit too. pub fn unmap(id: DomainId, at: Iova, bytes: u64) -> Result<(), IommuError> { - let mut domains = DOMAINS.lock(); - let domain = *domains.at(id); + let held = DOMAINS.lock(); + let space = held.domains.space(id); let mut units = UNITS.lock(); - table::unmap(&domain, at, bytes)?; + let result = domains::leaves(at, 0, bytes) + .try_for_each(|(here, _)| if table::take(space, here) { Ok(()) } else { Err(IommuError::NotMapped(Iova::translated(here))) }); for unit in units.iter_mut() { - unit.invalidate_domain(domain.id()); + unit.invalidate_domain(id.raw()); } - Ok(()) + result } pub fn attach(stream: StreamId, id: DomainId) { - let mut domains = DOMAINS.lock(); - let domain = *domains.at(id); + let held = DOMAINS.lock(); + let space = held.domains.space(id); let mut units = UNITS.lock(); for unit in units.iter_mut() { - unit.attach(stream, &domain); - } - crate::iommu::fault::attached(stream, domain.id()); - log!("iommu: {stream} moves to domain{}", domain.id()); -} - -impl Domains { - /// A `DomainId` only [`create`] mints, so the index is always in range. - fn at(&mut self, id: DomainId) -> &mut Domain { - &mut self.live[usize::from(id.raw() - FIRST)] + unit.attach(stream, id.raw(), space); } + crate::iommu::fault::attached(stream, id.raw()); + log!("iommu: {stream} moves to domain{}", id.raw()); } diff --git a/kernel/src/arch/x86_64/vtd/mod.rs b/kernel/src/arch/x86_64/vtd/mod.rs index 39ddf8f659..e2f131c081 100644 --- a/kernel/src/arch/x86_64/vtd/mod.rs +++ b/kernel/src/arch/x86_64/vtd/mod.rs @@ -26,7 +26,7 @@ use crate::mm::Mmio; use crate::sync::Lock; use crate::time::{Duration, Tripwire}; -use super::iommu_unit::yn; +use super::iommu_unit::{yn, MAX_UNITS}; use dmar::{Dmar, Malformed, Scope, Scopes, Structure}; use queue::Queue; use table::{Table, Tables}; @@ -78,8 +78,6 @@ const COMMAND_TIMEOUT: Tripwire = Tripwire::absurd( "a unit half-way through being enabled is a unit whose reach nothing can state", ); -/// Ceiling on units inventoried; a machine past it is told, not silently truncated. -pub(super) const MAX_UNITS: usize = 16; /// Remapping tables outlive `init`: the unit keeps reading them while it is on. static TABLES: Lock = Lock::new(Tables::new()); @@ -106,8 +104,8 @@ impl Live { /// Moves `stream` onto `domain` in this unit's tables, and out of /// whatever this unit still holds for it under its old one. - pub(super) fn attach(&mut self, stream: StreamId, domain: &table::Domain) { - let displaced = table::bind(self.root, stream, domain); + pub(super) fn attach(&mut self, stream: StreamId, id: u16, space: table::Space) { + let displaced = table::bind(self.root, stream, id, space); self.queue.invalidate_context(self.regs, displaced); } } diff --git a/kernel/src/arch/x86_64/vtd/table.rs b/kernel/src/arch/x86_64/vtd/table.rs index 6b44d87cd7..cce6d6bada 100644 --- a/kernel/src/arch/x86_64/vtd/table.rs +++ b/kernel/src/arch/x86_64/vtd/table.rs @@ -7,8 +7,8 @@ use alloc::vec::Vec; -use crate::iommu::window::{self, Window}; -use crate::iommu::{AddressWidth, IommuError, Iova, StreamId}; +use crate::iommu::window; +use crate::iommu::{AddressWidth, Iova, StreamId}; use crate::mm::pmm::{self, PhysPage}; use crate::mm::{DirectMap, Mmio, PAGE_2M}; @@ -123,38 +123,7 @@ impl Table { /// the one or the other, never a half of each (§6.2.2.1). fn replace_pair(self, index: usize, old: (u64, u64), new: (u64, u64)) { let entry = self.window().subregion(index as u64 * 16, 16); - let (found_lo, found_hi): (u64, u64); - // SAFETY: `entry` is 16 bytes of a table `Tables::alloc` never frees, - // 16-byte aligned since a table is 4 KiB-aligned; `rbx`, which LLVM - // reserves, holds `new`'s low half only between the two moves. - // Every operand names its register: LLVM may give `rbx` to a `reg` - // operand, and the `xchg` would then swap the address out of it. - unsafe { - core::arch::asm!( - "xchg r8, rbx", - "lock cmpxchg16b xmmword ptr [rsi]", - "mov rbx, r8", - in("rsi") entry.addr(), - inout("r8") new.0 => _, - in("rcx") new.1, - inout("rax") old.0 => found_lo, - inout("rdx") old.1 => found_hi, - options(nostack), - ); - } - let found = (found_lo, found_hi); - // `found` is `old` exactly when the swap happened; nothing else - // writes an entry under `UNITS`, so anything else is a kernel bug. - assert!( - found == old, - "iommu: entry {index} of table {:#x} held {:#x}:{:#x}, not the {:#x}:{:#x} read before \ - it was replaced", - self.phys, - found.1, - found.0, - old.1, - old.0 - ); + super::super::iommu_unit::exchange16(entry, [old.0, old.1], [new.0, new.1]); flush(entry.addr() as usize); } @@ -219,59 +188,25 @@ pub fn identity_domain(tables: &mut Tables, width: AddressWidth, top: u64) -> (T (root, frames) } -/// One domain's second-level tables, its id, and its addresses. +/// One domain's second-level tables, at the depth they were built for. #[derive(Clone, Copy)] -pub struct Domain { - root: Table, - id: u16, - width: AddressWidth, - window: Window, +pub struct Space { + pub root: Table, + pub width: AddressWidth, } -impl Domain { - /// The bits of device address a unit will translate: the lesser of the page - /// tables' depth and what the hardware accepts at all. - /// - /// **`SAGAW` and `MGAW` are two different limits and the smaller binds.** - /// VT-d Rev. 4.0 D51397-015, 11.4.2 Capability Register: `SAGAW` 12:8 is the - /// set of page-table depths a unit supports, `MGAW` 21:16 (encoded one less - /// than it is) is the maximum DMA virtual addressability it has. A unit - /// reporting a 48-bit `SAGAW` and a 39-bit `MGAW` walks four levels and - /// still faults `address-beyond-mgaw` on anything from `1 << 39` up, so a - /// window placed by table depth alone is unreachable on it. - const fn translatable_bits(width: AddressWidth, mgaw: u8) -> u8 { - if mgaw < width.bits() { mgaw } else { width.bits() } - } - - /// A domain with `room` bytes of its [`Window`] handed out at the address - /// answered, or the reason it has not. - pub fn new( - tables: &mut Tables, - id: u16, - width: AddressWidth, - mgaw: u8, - reserved: &[(u64, u64)], - room: u64, - ) -> Result<(Self, Iova), IommuError> { - let (window, first) = Window::new(Self::translatable_bits(width, mgaw), reserved, room)?; - Ok((Self { root: tables.alloc(), id, width, window }, first)) - } - - pub fn root(&self) -> Table { - self.root - } - - pub fn id(&self) -> u16 { - self.id - } - - pub const fn window(&self) -> &Window { - &self.window - } - - pub fn window_mut(&mut self) -> &mut Window { - &mut self.window - } +/// The bits of device address a unit will translate: the lesser of the page +/// tables' depth and what the hardware accepts at all. +/// +/// **`SAGAW` and `MGAW` are two different limits and the smaller binds.** +/// VT-d Rev. 4.0 D51397-015, 11.4.2 Capability Register: `SAGAW` 12:8 is the +/// set of page-table depths a unit supports, `MGAW` 21:16 (encoded one less +/// than it is) is the maximum DMA virtual addressability it has. A unit +/// reporting a 48-bit `SAGAW` and a 39-bit `MGAW` walks four levels and +/// still faults `address-beyond-mgaw` on anything from `1 << 39` up, so a +/// window placed by table depth alone is unreachable on it. +pub const fn translatable_bits(width: AddressWidth, mgaw: u8) -> u8 { + if mgaw < width.bits() { mgaw } else { width.bits() } } /// A unit whose `MGAW` is narrower than its `SAGAW` gets its window under the @@ -281,49 +216,32 @@ impl Domain { /// `intel-iommu` derives both fields from one `aw-bits` property, so its model /// cannot report a `SAGAW` wider than its `MGAW` at all. const _: () = { - assert!(Domain::translatable_bits(AddressWidth::Bits48, 39) == 39); + assert!(translatable_bits(AddressWidth::Bits48, 39) == 39); assert!(window::first_address(39) < 1 << 39); // What a window placed by the table depth alone would be, against the // ceiling such a unit reports. assert!(window::first_address(48) >= 1 << 39); // A unit whose two limits agree is unchanged by any of this. - assert!(Domain::translatable_bits(AddressWidth::Bits48, 48) == 48); - assert!(Domain::translatable_bits(AddressWidth::Bits39, 39) == 39); + assert!(translatable_bits(AddressWidth::Bits48, 48) == 48); + assert!(translatable_bits(AddressWidth::Bits39, 39) == 39); // And tables shallower than `MGAW` bind it the other way round. - assert!(Domain::translatable_bits(AddressWidth::Bits39, 48) == 39); + assert!(translatable_bits(AddressWidth::Bits39, 48) == 39); }; -pub fn map(tables: &mut Tables, domain: &Domain, at: Iova, phys: u64, bytes: u64) { - let levels = levels(domain.width); - let mut offset = 0u64; - while offset < bytes { - map_2m( - tables, - domain.root, - levels, - Iova::translated(at.raw() + offset), - phys + offset, - LEAF_PERM, - ); - offset += PAGE_2M; - } +/// The 2 MiB leaf at `at` in `space` reads and writes `phys`. +pub fn put(tables: &mut Tables, space: Space, at: u64, phys: u64) { + map_2m(tables, space.root, levels(space.width), Iova::translated(at), phys, LEAF_PERM); } -/// Clears the leaves covering `bytes` at `at`; the caller invalidates before the pages behind them are reused. -pub fn unmap(domain: &Domain, at: Iova, bytes: u64) -> Result<(), IommuError> { - let levels = levels(domain.width); - let mut offset = 0u64; - while offset < bytes { - let here = Iova::translated(at.raw() + offset); - let (table, index) = - leaf_of(domain.root, levels, here).ok_or(IommuError::NotMapped(here))?; - if table.read(index) & (SL_READ | SL_WRITE) == 0 { - return Err(IommuError::NotMapped(here)); - } - table.write(index, 0); - offset += PAGE_2M; +/// Clears the leaf at `at`; `false` where none is mapped. The caller +/// invalidates before the page behind it is reused. +pub fn take(space: Space, at: u64) -> bool { + let Some((table, index)) = leaf_of(space.root, levels(space.width), Iova::translated(at)) else { return false }; + if table.read(index) & (SL_READ | SL_WRITE) == 0 { + return false; } - Ok(()) + table.write(index, 0); + true } /// Walks to the page-directory holding `at`'s leaf, growing the tables on the way. @@ -430,7 +348,7 @@ pub fn bind_identity( /// Moves `stream`'s present context entry in one unit's root table onto a /// domain of its own, and answers what that unit may still hold for the old one. -pub fn bind(root: Table, stream: StreamId, domain: &Domain) -> Displaced { +pub fn bind(root: Table, stream: StreamId, id: u16, space: Space) -> Displaced { let entry = root.read(stream.bus() as usize * 2); let context = Table { phys: entry & ADDR_MASK }; let index = stream.devfn() as usize; @@ -441,7 +359,7 @@ pub fn bind(root: Table, stream: StreamId, domain: &Domain) -> Displaced { "iommu: {stream} has no context entry to move — it was not enumerated when its unit \ was programmed" ); - let (new, displaced) = rebind(old, stream.requester(), domain.root, domain.id, domain.width); + let (new, displaced) = rebind(old, stream.requester(), space.root, id, space.width); context.replace_pair(index, old, new); displaced } diff --git a/kernel/src/drivers/hda.rs b/kernel/src/drivers/hda.rs index 4b97e8c956..c0abd8b65d 100644 --- a/kernel/src/drivers/hda.rs +++ b/kernel/src/drivers/hda.rs @@ -612,7 +612,8 @@ pub fn init(devices: &[PciDevice]) { return; } - let space = crate::iommu::DeviceSpace::create(); + let candidates: Vec = list.iter().map(|listed| listed.pci).collect(); + let space = crate::iommu::DeviceSpace::create(&candidates); let bdl = super::DmaPool::alloc_in(PERIODS * 16, space).leak(); let pcm = super::DmaPool::alloc_in(PERIODS * PERIOD_BYTES, space).leak(); // Unaligned: the descriptor layout is the HDA specification's (§3.6.2), not a Rust one. diff --git a/kernel/src/drivers/pci.rs b/kernel/src/drivers/pci.rs index c0b035dff0..eb9ac0881c 100644 --- a/kernel/src/drivers/pci.rs +++ b/kernel/src/drivers/pci.rs @@ -382,6 +382,18 @@ impl PciDevice { true } + /// Point an IOMMU unit's own function's MSI at `address`/`data` and + /// enable it: the one message no unit remaps (AMD 48882 Capability Offset + /// 10h), so it is armed here rather than through [`Self::message`]. + /// `false` where the function has no MSI capability. + pub(crate) fn arm_unit_msi(&self, address: u32, data: u32) -> bool { + let Ok(cap) = self.capability(msi::CAP_ID) else { + return false; + }; + self.write_msi(cap, address, data); + true + } + fn write_msi(&self, cap: Capability<'_>, address: u32, data: u32) { let control = cap.read_u16(msi::MESSAGE_CONTROL); let msi = msi::Msi::decode(control); diff --git a/kernel/src/drivers/virtio_console.rs b/kernel/src/drivers/virtio_console.rs index c31975a2e5..0ebf3c8a63 100644 --- a/kernel/src/drivers/virtio_console.rs +++ b/kernel/src/drivers/virtio_console.rs @@ -233,7 +233,7 @@ pub fn init(devices: &[PciDevice]) -> bool { log!("virtio-console: found at PCI {:02x}:{:02x}.{}", pci_dev.bus, pci_dev.dev, pci_dev.func); // An address space of this device's own, holding one pool and nothing else. - let space = crate::iommu::DeviceSpace::create(); + let space = crate::iommu::DeviceSpace::create(&[pci_dev]); // Leaked, not held in a `static`: the console is never unbound. let dma = DmaPool::alloc_in(DMA_SIZE, space).leak(); // Before the device is told an address, and after the only mapping it gets. diff --git a/kernel/src/drivers/virtio_gpu.rs b/kernel/src/drivers/virtio_gpu.rs index fa8ade0c68..7ae304658d 100644 --- a/kernel/src/drivers/virtio_gpu.rs +++ b/kernel/src/drivers/virtio_gpu.rs @@ -534,7 +534,7 @@ fn fb_size_bytes(width: u32, height: u32) -> Option { pub fn init(devices: &[PciDevice]) -> Option<(Box, GpuInfo)> { let pci_dev = *devices.iter().find(|d| d.is_id(VIRTIO_VENDOR, VIRTIO_GPU_DEVICE))?; log!("VirtIO GPU: found at PCI {:02x}:{:02x}.{}", pci_dev.bus, pci_dev.dev, pci_dev.func); - let space = DeviceSpace::create(); + let space = DeviceSpace::create(&[pci_dev]); // Leaked rather than held in a `static`: the display is never unbound. let dma = DmaPool::alloc_in(DMA_SIZE, space).leak(); // Before the device is told any address, and after the command pool's mapping. diff --git a/kernel/src/drivers/xhci/wait/boot.rs b/kernel/src/drivers/xhci/wait/boot.rs index 909e1f0f26..9c24061703 100644 --- a/kernel/src/drivers/xhci/wait/boot.rs +++ b/kernel/src/drivers/xhci/wait/boot.rs @@ -336,7 +336,7 @@ fn init_one(pci_dev: &PciDevice) -> Option { // An address space of this controller's own, holding one pool and nothing // else; attached before the first address is written to a register. - let space = crate::iommu::DeviceSpace::create(); + let space = crate::iommu::DeviceSpace::create(&[*pci_dev]); // After the reset: a controller refused above never allocates, and // `DmaPool` frees the pool — and takes it back out of the domain — on every // refusal below when it drops. diff --git a/kernel/src/iommu/domains.rs b/kernel/src/iommu/domains.rs new file mode 100644 index 0000000000..da58854b08 --- /dev/null +++ b/kernel/src/iommu/domains.rs @@ -0,0 +1,113 @@ +//! The domains a unit translates for, whatever the backend: their ids, each +//! one's device addresses, and which 2 MiB leaves a map or a place covers. A +//! backend owns the tables these name and the invalidation that publishes +//! them, and holds this under the lock it writes those under. + +use alloc::vec::Vec; + +use crate::iommu::window::Window; +use crate::iommu::{DomainId, IommuError, Iova}; +use crate::mm::PAGE_2M; + +/// Every domain one backend made, by id from `first` up, each with the +/// backend's own name for its tables. +pub struct Domains { + first: u16, + /// By id minus `first`; never shrinks, since a released id would name a + /// domain some unit may still have cached. + live: Vec<(S, Window)>, + /// What no domain's addresses may reach: the root bridges' windows and + /// the regions firmware reserved, as `(start, end)`. + reserved: Vec<(u64, u64)>, +} + +/// A domain [`Domains::prepare`] admitted, not yet made. +pub struct Next { + pub id: u16, + pub addresses: Window, + first: Iova, +} + +impl Domains { + pub const fn new(first: u16) -> Self { + Self { first, live: Vec::new(), reserved: Vec::new() } + } + + /// Before any domain is made: every one is built clear of these. + pub fn avoid(&mut self, reserved: Vec<(u64, u64)>) { + self.reserved = reserved; + } + + /// The next domain's id and its window over `translatable` bits, with + /// `room` bytes handed out first; refused, with no id spent, where ids + /// reach `ceiling` or the window would hold less. + pub fn prepare(&self, translatable: u8, ceiling: u32, room: u64) -> Result { + let id = u16::try_from(usize::from(self.first) + self.live.len()) + .ok() + .filter(|id| u32::from(*id) < ceiling) + .ok_or(IommuError::DomainsExhausted(ceiling))?; + let (addresses, first) = Window::new(translatable, &self.reserved, room)?; + Ok(Next { id, addresses, first }) + } + + /// `next`, made over `space`, which `what` describes in its record. + pub fn add(&mut self, next: Next, space: S, what: impl core::fmt::Display) -> (DomainId, Iova) { + log!( + "iommu: domain{} {what} addresses from {:#x} to {:#x}", + next.id, + next.addresses.floor(), + next.addresses.ceiling() + ); + self.live.push((space, next.addresses)); + (DomainId::new(next.id), next.first) + } + + pub fn space(&self, id: DomainId) -> S { + self.live[usize::from(id.raw() - self.first)].0 + } + + /// Room for `bytes` at `phys`, never handed out before: the domain's + /// tables, and where the leaves go. + pub fn map(&mut self, id: DomainId, phys: u64, bytes: u64) -> Result<(S, Iova), IommuError> { + aligned(phys)?; + // A `DomainId` only `add` mints, so the index is always in range. + let (space, window) = &mut self.live[usize::from(id.raw() - self.first)]; + let at = window.reserve(bytes).ok_or(IommuError::AddressesExhausted(window.ceiling()))?; + log!("iommu: domain{} maps {phys:#x}..{:#x} at {:#x}", id.raw(), phys + bytes.next_multiple_of(PAGE_2M), at.raw()); + Ok((*space, at)) + } + + /// The domain's tables, for `bytes` at `phys` placed at `at`, room this + /// domain already handed out. Room it never handed out is a kernel bug, + /// since [`Self::map`] may yet hand it out. + pub fn place(&self, id: DomainId, at: Iova, phys: u64, bytes: u64) -> Result { + aligned(phys)?; + let (space, window) = &self.live[usize::from(id.raw() - self.first)]; + assert!(window.handed_out(at, bytes), "iommu: domain{} never handed out {:#x}+{bytes:#x}", id.raw(), at.raw()); + Ok(*space) + } +} + +fn aligned(phys: u64) -> Result<(), IommuError> { + if phys.is_multiple_of(PAGE_2M) { + Ok(()) + } else { + Err(IommuError::Unaligned(phys)) + } +} + +/// Each 2 MiB leaf `bytes` at `at` covers, as `(device address, physical +/// address)` of `phys` onwards. +pub fn leaves(at: Iova, phys: u64, bytes: u64) -> impl Iterator { + (0..bytes).step_by(PAGE_2M as usize).map(move |offset| (at.raw() + offset, phys + offset)) +} + +/// The record a mapping put back where a device may still be aimed leaves. +pub fn placed_again(id: DomainId, at: Iova, phys: u64, bytes: u64) { + log!( + "iommu: domain{} maps {phys:#x}..{:#x} at {:#x} again", + id.raw(), + phys + bytes.next_multiple_of(PAGE_2M), + at.raw() + ); +} diff --git a/kernel/src/iommu/fault.rs b/kernel/src/iommu/fault.rs index f550438637..516b46c678 100644 --- a/kernel/src/iommu/fault.rs +++ b/kernel/src/iommu/fault.rs @@ -139,8 +139,8 @@ fn find(who: Who) -> Option<&'static Function> { /// Whom a record names. #[derive(Clone, Copy)] pub enum Who { - /// A requester id: VT-d's source-id, or the function the IORT routes an - /// SMMU StreamID from. + /// A requester id: VT-d's source-id, AMD-Vi's DeviceID, or the function + /// the IORT routes an SMMU StreamID from. Function(StreamId), /// An SMMU StreamID no enumerated function is routed from. // The SMMUv3 is AArch64's alone, so x86-64 builds a variant it never makes. @@ -172,9 +172,7 @@ impl core::fmt::Display for Who { pub enum Access { Read, Write, - /// An SMMU event that carries no transaction. - // The SMMUv3 is AArch64's alone, so x86-64 builds a variant it never makes. - #[allow(dead_code)] + /// A record that carries no transaction. Unrecorded, } diff --git a/kernel/src/iommu/mod.rs b/kernel/src/iommu/mod.rs index f6afe2a5ec..1844305a9b 100644 --- a/kernel/src/iommu/mod.rs +++ b/kernel/src/iommu/mod.rs @@ -12,7 +12,10 @@ use crate::arch::iommu_unit as unit; use crate::arch::message_unit as messages; +pub(crate) mod domains; pub(crate) mod fault; +#[cfg(feature = "boot-actuators")] +pub(crate) mod testdev; pub(crate) mod window; /// The address width a device's translations cover. @@ -166,9 +169,18 @@ pub enum DeviceSpace { } impl DeviceSpace { - /// One of a device's own, or the machine's own with the reason. For a - /// driver **in this kernel**, whose addresses are the kernel's either way. - pub fn create() -> Self { + /// One for whichever of `functions` is attached to it, or the machine's + /// own with the reason: the machine's own where any of them keeps it, so + /// the one attached reaches what its device addresses name. For a driver + /// **in this kernel**, whose addresses are the kernel's either way. + pub fn create(functions: &[crate::drivers::pci::PciDevice]) -> Self { + for function in functions { + let stream = StreamId::pci(function.bus, function.dev, function.func); + if let Some(why) = unit::domain::untranslated(stream) { + log!("iommu: {stream} keeps the machine's addresses: {why}"); + return Self::Untranslated; + } + } match unit::domain::create(0) { Ok((id, _)) => Self::Own(id), Err(why) => { @@ -238,7 +250,7 @@ impl OwnSpace { /// and write no record of it: for a mapping its holder makes and takes back /// as often as it likes. pub fn place(self, at: u64, phys: u64, bytes: u64) -> Result<(), IommuError> { - unit::domain::place(self.0, Iova::translated(at), phys, bytes).map(|_| ()) + unit::domain::place(self.0, Iova::translated(at), phys, bytes) } /// [`DeviceSpace::unmap`]. diff --git a/kernel/src/iommu/testdev.rs b/kernel/src/iommu/testdev.rs new file mode 100644 index 0000000000..97dfe3e934 --- /dev/null +++ b/kernel/src/iommu/testdev.rs @@ -0,0 +1,58 @@ +//! QEMU's `iommu-testdev` (1b36:0005), whose one register read makes a DMA +//! write of a known word at a device address and reads it back at a physical +//! one: the only DMA a guest of this suite can aim at an address of its +//! choosing, for each backend's selftest. + +use crate::drivers::pci::PciDevice; +use crate::mm::policy::MmioPolicy; +use crate::mm::Mmio; + +/// `hw/misc/iommu-testdev.h`: Red Hat's vendor and its test device. +pub const VENDOR: u16 = 0x1b36; +pub const DEVICE: u16 = 0x0005; + +/// Its BAR0: reading `TRIGGERING` runs an armed DMA; `GVA` is the device +/// address written, `GPA` the physical one read back, `LEN` the bytes, +/// `DBELL` bit 0 arms, and `RESULT` says how it went. +const TRIGGERING: u64 = 0x00; +const GVA_LO: u64 = 0x04; +const GVA_HI: u64 = 0x08; +const LEN: u64 = 0x0c; +const RESULT: u64 = 0x10; +const DBELL: u64 = 0x14; +const GPA_LO: u64 = 0x1c; +const GPA_HI: u64 = 0x20; +const BAR_BYTES: u64 = 0x1000; + +/// `RESULT`: the word landed at `GPA`; the write was refused. +pub const LANDED: u32 = 0; +pub const REFUSED: u32 = 0xdead_0002; + +/// `device`'s registers, its memory decode on. No Bus Master Enable: the +/// device writes through the unit's address space whatever its `COMMAND` says. +pub fn registers(device: &PciDevice) -> Mmio { + let bar = device.memory_bar(0).expect("iommu selftest: an iommu-testdev's BAR0 is unassigned").address(); + let regs = crate::mm::paging::map_mmio(bar, BAR_BYTES, MmioPolicy::Uncacheable); + device.enable_memory_space(); + regs +} + +/// The device's write of four bytes at `at`, read back at `phys`: its `RESULT`. +pub fn write(regs: Mmio, at: u64, phys: u64) -> u32 { + regs.write_u32(GVA_LO, at as u32); + regs.write_u32(GVA_HI, (at >> 32) as u32); + regs.write_u32(GPA_LO, phys as u32); + regs.write_u32(GPA_HI, (phys >> 32) as u32); + regs.write_u32(LEN, 4); + regs.write_u32(DBELL, 1); + regs.read_u32(TRIGGERING); + regs.read_u32(RESULT) +} + +pub fn verdict(result: u32, want: u32) -> &'static str { + match (result == want, want) { + (true, REFUSED) => "refused", + (true, _) => "landed there", + (false, _) => "FAIL", + } +} diff --git a/tests/common/iommu.rs b/tests/common/iommu.rs index eb9625969d..64062b25c4 100644 --- a/tests/common/iommu.rs +++ b/tests/common/iommu.rs @@ -367,9 +367,11 @@ const AMDVI_SUPERSEDED: &str = "iommu: IVRS IVHD 0x10 at +48 describes 00:02.0 a /// The xHCI the boot stick is on, among the functions the unit serves. const AMDVI_SERVES_XHCI: &str = "iommu: amdvi unit0 serves 00:03.0 data=0x00"; const AMDVI_OFF: &str = "iommu: amdvi unit0 @0xfed80000 switched off control=0x0000000000000000"; -const AMDVI_NO_ISOLATION: &str = "iommu: AMD-Vi isolates no device this boot: 1 unit(s) described, 1 switched off"; -/// A kernel driver's domain refused for want of a unit that translates. -const NO_DOMAIN: &str = "iommu: no domain of its own for a device: no unit on this machine translates"; +/// The unit programmed and on. +const AMDVI_TRANSLATING: &str = "iommu: amdvi unit0 @0xfed80000 translating control="; +/// The boot stick's controller on the first domain a driver made, the one +/// past the identity domain. +const XHCI_MOVED: &str = "iommu: 00:03.0 moves to domain2"; /// The USB gate: the boot stick bound through the controller behind the unit, /// which only DMA the unit lets through can do. const STICK_BOUND: &str = "usb-storage: 1 device(s)"; @@ -381,12 +383,13 @@ pub(crate) fn amdvi_left_on(unit: &str) -> [String; 3] { .map(|field| format!("iommu: amdvi {unit} was handed over with {field} on; it goes off")) } -/// **A unit an AMD machine's firmware left on is switched off, and the boot -/// stick's DMA reaches memory.** QEMU's `amd-iommu` is the only AMD-Vi unit +/// **A unit an AMD machine's firmware left on is switched off and programmed, +/// and the boot stick's DMA reaches memory through its controller's own +/// domain.** QEMU's `amd-iommu` is the only AMD-Vi unit /// the merge queue runs: the T14 is an Intel machine, and the hand-over's /// register sequence and the DMA it frees are neither of them a host test's. /// Two boots: the shipped kernel finds a unit nothing left on and switches -/// nothing; the test kernel's `iommu-firmware-left` leaves the unit blocking +/// nothing off; the test kernel's `iommu-firmware-left` leaves the unit blocking /// every device's DMA with its command buffer and event log running, and /// each goes off by its own write before the stick is driven. With the /// switch-off reverted that boot reds on the fields' lines and on `control` @@ -414,8 +417,9 @@ pub fn amdvi_firmware_left(test_config: &Path) -> Result<(), String> { log.must_say(AMDVI_SUPERSEDED)?; log.must_say(AMDVI_SERVES_XHCI)?; log.must_say(AMDVI_OFF)?; - log.must_say(AMDVI_NO_ISOLATION)?; - log.must_say(NO_DOMAIN)?; + log.must_say(AMDVI_TRANSLATING)?; + log.must_say(XHCI_MOVED)?; + log.must_not_say("isolates no device")?; log.must_not_say("no DMAR or IVRS table")?; let handed = log.must_say("iommu: amdvi unit0 handed over control=")?; if left { @@ -436,3 +440,75 @@ pub fn amdvi_firmware_left(test_config: &Path) -> Result<(), String> { } Ok(()) } + +/// **A translating AMD-Vi unit refuses what no domain maps, and the handler +/// reads its record.** `amdvi-selftest` (`kernel/src/arch/x86_64/amdvi/selftest.rs`) +/// on QEMU's `amd-iommu` with two `iommu-testdev` behind it: the first, on a +/// domain of its own, lands a write where that domain maps it; the second, +/// on the identity domain, lands one in memory and is refused one where the +/// identity domain maps nothing, recorded as the kernel's, and the machine +/// halts. The record, not the device's answer, says the unit translated: a +/// write passed through to 2^46 reaches nothing on this 40-bit guest and the +/// device answers it refused either way. QEMU 11.1.1 passes a function +/// through until an `INVALIDATE_DEVTAB_ENTRY` names it, so with the +/// per-function invalidations at boot removed no record is read and this +/// reds. A kernel-owned record halts, so no metal row reaches the event log +/// of a unit that translates for the kernel alone. +pub fn amdvi_selftest(test_config: &Path) -> Result<(), String> { + let halted = |l: &str| l.contains("panic: rebooting") || l.contains("panic: holding this panel"); + let options = BootOptions { + profile: Profile::MetalAmdViTestdev, + kernel_params: &["amdvi-selftest"], + ready_marker: "iommu: IVRS rev=", + ..Default::default() + }; + let argv = qemu::profile_argv(&options); + if argv.iter().filter(|a| a.as_str() == "iommu-testdev").count() != 2 { + return Err(format!("the machine has not two iommu-testdevs: {argv:?}")); + } + let mut qemu = QemuInstance::boot_with_options(test_config, &[], &[], options); + let rest = qemu.drain_until(std::time::Duration::from_secs(30), halted); + let serial = format!("{}\n{rest}", qemu.boot_log()); + let lines: Vec<&str> = serial.lines().collect(); + let at = |want: &str| lines.iter().position(|l| l.contains(want)); + if let Some(failed) = at("amdvi-selftest: FAIL") { + return Err(format!("{}\nserial:\n{serial}", lines[failed])); + } + let stop = lines + .iter() + .position(|l| halted(l)) + .ok_or_else(|| format!("the machine never reached the halt path's line past the stop\nserial:\n{serial}"))?; + at(AMDVI_TRANSLATING).ok_or_else(|| format!("the unit was never programmed\nserial:\n{serial}"))?; + let mut said = Vec::new(); + for (step, verdict) in + [("mapped to", "landed there"), ("on the identity domain", "landed there"), ("which the identity domain does not map", "refused")] + { + let found = at(step).map(|i| lines[i]).ok_or_else(|| format!("the selftest never said {step:?}\nserial:\n{serial}"))?; + eprintln!(" [amdvi] {found}"); + if !found.ends_with(verdict) { + return Err(format!("{found}\nserial:\n{serial}")); + } + said.push(found); + } + // `'s write at
, which the identity domain …`. The + // record is held to its function alone: QEMU 11.1.1 writes it with no + // address and its flags off 48882's positions + // (`issues/qemus-amd-iommu-page-fault-record-carries-no-address.md`). + let function = said[2] + .split_once("amdvi-selftest: ") + .and_then(|(_, rest)| rest.split_once("'s write at ")) + .map(|(function, _)| function) + .ok_or_else(|| format!("{:?} names no function", said[2]))?; + let record = format!("iommu: DMA FAULT owner=kernel unit0 stream={function} "); + let faults: Vec = (0..lines.len()).filter(|&i| lines[i].contains("iommu: DMA FAULT owner=")).collect(); + match faults.as_slice() { + [one] if lines[*one].contains(&record) && lines[*one].ends_with("IO_PAGE_FAULT") && *one < stop => { + eprintln!(" [amdvi] {}", lines[*one]); + } + _ => return Err(format!("not one record {record:?} … IO_PAGE_FAULT before the stop\nserial:\n{serial}")), + } + if let Some(death) = lines.iter().find(|l| super::serial::died(l).is_some() && !l.contains("owner=kernel")) { + return Err(format!("the machine died of something other than the record: {death}\nserial:\n{serial}")); + } + Ok(()) +} diff --git a/tests/common/qemu.rs b/tests/common/qemu.rs index 138eb8d648..4c020f44dd 100644 --- a/tests/common/qemu.rs +++ b/tests/common/qemu.rs @@ -707,6 +707,10 @@ pub enum Profile { /// stick's controller behind an AMD-Vi unit, which the kernel finds by the /// IVRS and switches off. MetalAmdVi, + /// [`Profile::MetalAmdVi`] with two of QEMU's `iommu-testdev` behind the + /// unit: the only machine on which a translating AMD-Vi unit's refusal + /// can be made and read. + MetalAmdViTestdev, /// QEMU `virt` on AArch64 (GICv3, AAVMF): a GOP from `ramfb`, the boot /// stick on an xHCI, the PL011, a virtio-rng for firmware's /// `EFI_RNG_PROTOCOL`, and nothing else — no NIC, NVMe or IOMMU. The @@ -768,7 +772,8 @@ impl Profile { | Self::HeadlessUsbSpare | Self::Metal | Self::MetalNoX2apic - | Self::MetalAmdVi => Arch::X86_64, + | Self::MetalAmdVi + | Self::MetalAmdViTestdev => Arch::X86_64, } } @@ -914,8 +919,10 @@ struct Shape { /// profile because absence is a shape and because the unit's own /// capabilities are what the kernel reads at boot. iommu: Option, - /// `virt`'s SMMUv3, which is a machine property rather than a device. - smmu: Smmu, + /// What a test aims a DMA or a message from; on `virt` with its SMMUv3, + /// a machine property rather than a device, and on a q35 behind + /// [`Shape::iommu`]. + probes: Probes, /// A virtio-rng, which is firmware's alone: edk2's driver puts /// `EFI_RNG_PROTOCOL` behind it for the loader's seed, and the kernel /// drives no such device. `virt` has it because an HVF guest's CPU has no @@ -926,19 +933,22 @@ struct Shape { virtio_gpu: bool, } -/// Whether `virt` has its SMMUv3. With it come two of QEMU's `iommu-testdev`, -/// whose writes are the only DMA a guest of this suite can aim at an address -/// of its choosing: a unit no function writes through is a unit no test reads. +/// The devices a machine has for a test to aim a DMA or a message from, and +/// on `virt` the SMMUv3 that comes with them: a unit no function writes +/// through is a unit no test reads. #[derive(Clone, Copy, PartialEq, Eq)] -enum Smmu { +enum Probes { Absent, - WithTestdev, - /// With the ITS behind it ([`Arch::irqchip`]), one `edu`, whose message is - /// the only one a guest of this suite can raise on demand, and an - /// `e1000e` behind a root port of a `pxb-pcie` with `bypass_iommu=on`, - /// which QEMU's IORT routes straight to the ITS: a plain root port's - /// buses it routes through the unit. - WithIts, + /// Two of QEMU's `iommu-testdev`, the only DMA a guest of this suite can + /// aim at an address of its choosing, behind the machine's unit. + Testdevs, + /// On `virt`, its SMMUv3 with the ITS behind it + /// ([`Arch::irqchip`]), one `edu`, whose message is the only one a guest + /// of this suite can raise on demand, and an `e1000e` behind a root port + /// of a `pxb-pcie` with `bypass_iommu=on`, which QEMU's IORT routes + /// straight to the ITS: a plain root port's buses it routes through the + /// unit. + Its, } /// Where a machine's image and its DATA are. A size is stated because a @@ -985,8 +995,8 @@ impl Profile { match self { Self::VirtEl2 | Self::VirtEl2NoVhe | Self::VirtTcg | Self::VirtLowEcam => Self::Virt.shape(), Self::VirtNoRng => Shape { rng: false, ..Self::Virt.shape() }, - Self::VirtSmmu => Shape { smmu: Smmu::WithTestdev, ..Self::Virt.shape() }, - Self::VirtIts => Shape { smmu: Smmu::WithIts, ..Self::Virt.shape() }, + Self::VirtSmmu => Shape { probes: Probes::Testdevs, ..Self::Virt.shape() }, + Self::VirtIts => Shape { probes: Probes::Its, ..Self::Virt.shape() }, Self::Virt => Shape { vga: "std", panel: None, @@ -997,7 +1007,7 @@ impl Profile { blockdevs: &[], storage: Storage::Stick { nvme_bytes: 0 }, iommu: None, - smmu: Smmu::Absent, + probes: Probes::Absent, rng: true, virtio_gpu: false, }, @@ -1011,7 +1021,7 @@ impl Profile { blockdevs: &[], storage: Storage::Disk { data_bytes: NVME_SMALL }, iommu: Some(Unit::Vtd(IOMMU_DEFAULT)), - smmu: Smmu::Absent, + probes: Probes::Absent, rng: false, virtio_gpu: false, }, @@ -1029,12 +1039,13 @@ impl Profile { blockdevs: &[], storage: Storage::Stick { nvme_bytes: NVME_SMALL }, iommu: Some(Unit::Vtd(IOMMU_DEFAULT)), - smmu: Smmu::Absent, + probes: Probes::Absent, rng: false, virtio_gpu: false, }, Self::MetalNoX2apic => Self::Metal.shape(), Self::MetalAmdVi => Shape { iommu: Some(Unit::AmdVi), ..Self::Metal.shape() }, + Self::MetalAmdViTestdev => Shape { probes: Probes::Testdevs, ..Self::MetalAmdVi.shape() }, Self::HeadlessNoIommu => Shape { iommu: None, ..Self::Headless.shape() }, Self::HeadlessNoUsb => Shape { xhci: &[], usb: &[], ..Self::Headless.shape() }, Self::HeadlessVirtioGpu => Shape { virtio_gpu: true, ..Self::Headless.shape() }, @@ -2127,15 +2138,18 @@ fn qemu_command( // AMD-Vi is staged with no interrupt remapping, and `virt_smmu`'s unit // needs no message of a function's. - let remapped = match (&shape.iommu, shape.smmu) { + let remapped = match (&shape.iommu, shape.probes) { (Some(Unit::Vtd(unit)), _) => unit.intremap, - (_, Smmu::WithIts) => true, + (_, Probes::Its) => true, _ => false, }; let mut machine = vec![arch.machine()]; machine.extend(arch.irqchip(accel, remapped)); match arch { - Arch::X86_64 => assert!(shape.smmu == Smmu::Absent, "a q35 has no SMMUv3"), + Arch::X86_64 => assert!( + shape.probes == Probes::Absent || (shape.probes == Probes::Testdevs && shape.iommu.is_some()), + "a q35 has no SMMUv3, and an iommu-testdev on one writes through no unit" + ), Arch::Aarch64 => { // A unit a profile declares is VT-d or AMD-Vi, which `virt` has none of. assert!(shape.iommu.is_none(), "`virt` has no VT-d or AMD-Vi"); @@ -2144,7 +2158,7 @@ fn qemu_command( Profile::VirtLowEcam => Some("highmem-ecam=off"), _ => None, }); - if shape.smmu != Smmu::Absent { + if shape.probes != Probes::Absent { machine.push("iommu=smmuv3"); } } @@ -2283,13 +2297,14 @@ fn qemu_command( if shape.virtio_gpu { qemu.arg("-device").arg(format!("virtio-gpu-pci{platform}")); } - if shape.smmu == Smmu::WithTestdev { - // Two, the first enumerated below the second: the kernel's selftest - // routes nothing for the first, whose entry the table holds. + if shape.probes == Probes::Testdevs { + // Two, the first enumerated below the second: the SMMU's selftest + // routes nothing for the first, whose entry the table holds; the + // AMD-Vi's moves the first to a domain and leaves the second. qemu.arg("-device").arg("iommu-testdev"); qemu.arg("-device").arg("iommu-testdev"); } - if shape.smmu == Smmu::WithIts { + if shape.probes == Probes::Its { qemu.arg("-device").arg("edu"); qemu.arg("-device").arg("pxb-pcie,id=bypass,bus_nr=8,bypass_iommu=on"); qemu.arg("-device").arg("pcie-root-port,id=bypassport,bus=bypass,chassis=1"); diff --git a/tests/toyos.rs b/tests/toyos.rs index 5fd8c8bad0..899df34969 100644 --- a/tests/toyos.rs +++ b/tests/toyos.rs @@ -316,11 +316,15 @@ const MACHINE_TESTS: &[&str] = &[ // behind its emulated VT-d unit and without one: no shipped machine has a // virtio function, so only a QEMU machine can be asked. "iommu_virtio_platform", - // An AMD-Vi unit left blocking every device, switched off before the boot - // stick is driven behind it: the T14 is an Intel machine, no AMD one is - // in reach, and the register sequence and the DMA it frees are no host - // test's. + // An AMD-Vi unit left blocking every device, switched off and programmed + // before the boot stick is driven behind it: the register sequence and the + // DMA it lets through are no host test's, and the AMD bench is staged by + // hand rather than by the merge queue. "amdvi_firmware_left", + // A translating AMD-Vi unit's refusal, recorded and read: a kernel-owned + // record halts the machine, so no metal row reaches it, and only QEMU's + // `iommu-testdev` aims a DMA where no domain maps. + "amdvi_selftest", // The nested-NMI report is a raw write to the 16550, which the T14 does not // have. "nested_nmi_is_loud", @@ -979,6 +983,13 @@ const METAL: &[(&str, metal::Metal)] = &[ "domain_ends_below_the_host_bridges", metal::Metal { arms: TESTCASES, judge: |b| claims::clear_of_host_bridges(&b[0].kernel()) }, ), + ( + // An AMD-Vi unit programmed, and the boot stick's controller read + // through a domain of its own: the register sequence, the device + // table and the host tables on real silicon. The AMD bench's row. + "amdvi_translates_the_stick_controller", + metal::Metal { arms: TESTCASES, judge: |b| amdvi_translates(&b[0].kernel()) }, + ), ( // A machine whose units do not remap: the claim is refused before // anything on the function changes, and the boot goes on. @@ -3619,6 +3630,7 @@ fn run_machine_test(name: &str, test_config: &Path) -> Result<(), String> { match name { "iommu_virtio_platform" => common::iommu::iommu_virtio_platform(test_config), "amdvi_firmware_left" => common::iommu::amdvi_firmware_left(test_config), + "amdvi_selftest" => common::iommu::amdvi_selftest(test_config), "nested_nmi_is_loud" => faults::nested_nmi_is_loud(test_config), "acpi_press_is_one_sci" => acpi_press_is_one_sci(), "acpi_mediated_access" => acpi_mediated_access(2), @@ -3762,6 +3774,36 @@ fn usbd_drives_the_spare(test_config: &Path) -> Result<(), String> { Ok(()) } +/// Every AMD-Vi unit translating, every xHCI controller the kernel drives +/// moved to a domain of its own, the boot stick bound through one, and no +/// DMA refused. +fn amdvi_translates(kernel: &serial::Serial) -> Result<(), String> { + let text = kernel.text(); + let units: Vec<&str> = + text.lines().filter(|l| l.contains("iommu: amdvi unit") && l.contains(" translating control=")).collect(); + if units.is_empty() { + return Err(format!("no AMD-Vi unit is translating:\n{text}")); + } + kernel.must_not_say("isolates no device")?; + kernel.must_not_say("iommu: DMA FAULT")?; + let controllers: Vec<&str> = text + .lines() + .filter_map(|l| l.split_once("xHCI: found at PCI ")?.1.split_whitespace().next()) + .collect(); + if controllers.is_empty() { + return Err(format!("the kernel drove no xHCI controller:\n{text}")); + } + for at in &controllers { + kernel.must_say(&format!("iommu: {at} moves to domain"))?; + } + kernel.must_say("usb-storage: 1 device(s)")?; + for unit in units { + eprintln!(" [amdvi] {}", unit.trim()); + } + eprintln!(" [amdvi] {} controller(s) on domains of their own, the stick bound", controllers.len()); + Ok(()) +} + /// The T14's Type-C controller left to a claim and driven by usbd: the /// kernel left it, the claim was handed over with its MSI in remappable /// format and its remapping entry written, the message reached the slot, and