diff --git a/Cargo.lock b/Cargo.lock index f6c30dc67bf..ea7c7022adb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4037,6 +4037,7 @@ dependencies = [ "toyos-logstream", "toyos-manifest", "toyos-osrelease", + "toyos-pci-claim", "toyos-quiesce", "toyos-supervisor", "toyos-swap", diff --git a/build/ci.rs b/build/ci.rs index abf46e1279a..93d905f31ea 100644 --- a/build/ci.rs +++ b/build/ci.rs @@ -633,6 +633,12 @@ pub(crate) const CONTROLS: &[Control] = &[ Fails("pcid::tests::two_live_address_spaces_never_share_a_pcid"), Passes("pcid::oracle::tests::the_counting_allocator_produces_a_cross_space_read"), ]), + // A grant past its holder's limit is issued, truncated, as another grant. + red(KERNEL, "iova-ignores-limit", None, &[ + Fails("iova::tests::a_function_that_drops_its_upper_bits_reaches_only_its_own_grant"), + Fails("iova::tests::every_grant_is_the_lowest_allowed_run_below_its_limit"), + Fails("iova::tests::the_amd_benchs_layout"), + ]), red(KERNEL, "mutate-spawn-skips-the-insert-recheck", None, &[ Fails("proclife::interleave::tests::a_published_exit_leaves_no_unretired_thread"), Fails("proclife::interleave::tests::a_kill_racing_a_spawn_leaves_no_unretired_thread"), diff --git a/issues/a-function-that-truncates-its-dma-addresses-faults-in-its-domain.md b/issues/a-function-that-truncates-its-dma-addresses-faults-in-its-domain.md deleted file mode 100644 index 77797af15b9..00000000000 --- a/issues/a-function-that-truncates-its-dma-addresses-faults-in-its-domain.md +++ /dev/null @@ -1,33 +0,0 @@ ---- -status: owner -kind: question -opened: 2026-10-20 ---- - -# A function that truncates its DMA addresses faults in its domain - -Every domain's addresses start a quarter of the way up the units' translated -width (`kernel/src/iommu/window.rs`), which on the AMD bench is -`0x400000000000`. The bench's HDMI audio function `03:00.1` (`1002:1637`), -reachable since P1 of `issues/there-is-no-wifi.md` hands BARs over in place, -does not reach that high: given its CORB and RIRB at `0x400010000000` it -read and wrote `0x10000000`, `0x10000400` and `0x10000410`, the same offsets -with the bits above its real width dropped. It advertises 64-bit addressing -(GCAP `0x4001`, 64OK, HDA 1.0a §3.3.2), the `hda` driver writes both halves -of each ring base, and `03:00.6` (`1022:15e3`) at the same domain base works -in the same boot. The unit refuses every one of those accesses and the claim -is refused with it, so nothing is corrupted; the function plays nothing, and -`amdvi_translates_the_stick_controller`, which refuses any DMA fault, is red -on the AMD bench for this reason alone. - -Evidence: the AMD `testcases` boot at `33adc6b47`, -https://github.com/ToyOSOrg/evidence/blob/b0148546c1c8b77c0fca941fdee729c35349ad1a/p1/bench/round6/amd/testcases/kernel.log -(lines 337 to 345). - -Owner: the domain layout, which #916 settled. The orchestrator put the -choice to the owner, recommending that each domain start at the lowest gap -above RAM that clears every declared window rather than a quarter of the way -up, with no per-device quirk. - -Exit: the owner's choice implemented, and `amdvi_translates_the_stick_controller` -green on the AMD bench with `03:00.1` claimed. diff --git a/issues/a-narrow-functions-device-addresses-are-spent-once-per-boot.md b/issues/a-narrow-functions-device-addresses-are-spent-once-per-boot.md new file mode 100644 index 00000000000..fd1b0b01d89 --- /dev/null +++ b/issues/a-narrow-functions-device-addresses-are-spent-once-per-boot.md @@ -0,0 +1,20 @@ +--- +status: open +kind: finding +opened: 2026-10-21 +--- + +# A narrow function's device addresses are spent once per boot + +A domain hands each device address out once, for the boot +(`kernel/pure/iova.rs`), and a function keeps its domain across claims. So +the addresses below a function's width are a per-boot budget. On the AMD +bench a 32-bit function has 1,704 leaves of 2 MiB below 4 GiB, the lend +window's 128 among them. A claim may hold up to 16 leaves of grants, and +retired grant addresses are never reused. Past roughly a hundred claims of +one narrow function in a boot, its next grant is refused (`ResourceExhausted`). +That refusal is clean, but nothing else announces the limit. + +Exit: either the budget is shown unreachable for every narrow function a +shipped driver claims, or an address whose function's reset has completed is +handed out again, which the residue rules would have to allow. diff --git a/issues/a-virtio-function-that-offers-no-access-platform-is-handed-over-as-translated.md b/issues/a-virtio-function-that-offers-no-access-platform-is-handed-over-as-translated.md deleted file mode 100644 index b2713094f02..00000000000 --- a/issues/a-virtio-function-that-offers-no-access-platform-is-handed-over-as-translated.md +++ /dev/null @@ -1,54 +0,0 @@ ---- -status: open -kind: defect -opened: 2026-10-12 ---- - -# A virtio function that offers no `ACCESS_PLATFORM` is handed over as translated - -Virtio 1.2 §6, `VIRTIO_F_ACCESS_PLATFORM`: "If this feature bit is set to 0, -then the device has same access to memory addresses supplied to it as the -driver has. In particular, the device will always use physical addresses -matching addresses used by the driver ... When clear, this overrides any -platform-specific description of whether device access is limited or -translated in any way, e.g. whether an IOMMU may be present." And §6.1: "If -VIRTIO_F_ACCESS_PLATFORM is not offered, then a driver MUST pass only physical -addresses to the device." - -`pcidev` hands a process a function once the unit the firmware's tables put -it behind has given it a domain (`kernel/src/pcidev/mod.rs`), and never asks -whether a virtio function offers the bit; the holder's driver accepts it where -offered and goes on without it (`Offer::accept`, `lib/virtio/src/pci.rs`). A -function that does not offer it is not behind the unit at all: QEMU keeps it -on the machine's own address space whatever the `DMAR` says -(`tests/common/iommu.rs`). So an honest -holder's descriptors, which carry domain addresses above all memory -(`kernel/src/iommu/window.rs`), reach nothing, and a hostile holder's, which -carry physical addresses, reach all of memory: the claim's isolation is void -for exactly the functions it is promised for. - -The launcher and the harness hide it. Each passes `iommu_platform=on` to every -virtio function wherever a unit exists (`build/qemu.rs`), and -`iommu_virtio_platform` reds a profile that does not; QEMU's own default is -off. A transitional virtio function cannot offer the bit in QEMU at all, and -VirtualBox's virtio core offers it nowhere -(`issues/the-virtio-drivers-match-only-a-non-transitional-device-id.md`). - -The check is not the holder's, for the holder is who the domain protects the -machine from, nor the kernel's, which the owner's target keeps free of virtio -(`issues/every-driver-is-still-in-the-kernel.md`; `kernel/src/pcidev` and -`kernel/src/iommu` name none today). By the orchestrator's ruling (#896, -issuecomment-6131213771, ruling 1) it is the supervisor's, which mints the -claim and is trusted: it reads the function's offered features before it moves -the claim in, and on a machine whose functions are translated refuses a virtio -function that does not offer the bit. How it reads the word is the stage's -design. - -**Owner**: whoever holds `issues/toyos-runs-in-any-open-source-hypervisor.md`. - -**Exit**: a guest boot under the harness's VT-d with netstack's virtio-net -created without `iommu_platform=on` reads the supervisor's line refusing -netstack that function for its missing `VIRTIO_F_ACCESS_PLATFORM`, and -`netstack: no NIC on this machine`; the same boot with the check deleted -endows netstack the function; and `git grep -i virtio kernel/src/pcidev -kernel/src/iommu` still finds nothing. diff --git a/issues/i219-msi-passes-only-when-the-network-sends-it-a-frame.md b/issues/i219-msi-passes-only-when-the-network-sends-it-a-frame.md new file mode 100644 index 00000000000..aa00f754ff7 --- /dev/null +++ b/issues/i219-msi-passes-only-when-the-network-sends-it-a-frame.md @@ -0,0 +1,26 @@ +--- +status: open +kind: defect +opened: 2026-10-21 +--- + +# `i219_msi` passes only when the network sends it a frame + +`i219_messages_on_its_msi_vector` fails unless a frame arrives within +`LISTEN_NANOS` after the job's own DHCP discover +(`tests/guest/rust-tests/src/bin/i219_msi.rs`, "no frame arrived on a message +under traffic"). The frame has to come from a peer on the T14's wire: the +row's receive claim is about whatever traffic the network happens to carry, +and the job controls none of it. Its passing runs counted one or two frames. + +On 2026-10-21 the T14's wire was quiet. #974 at `94968aca2` and main +`b580e0078` (with the same test-only output line, `bf581bdeb`) were booted +back to back, and each failed alike. The MAC's own counters read "saw 0 +frame(s), took 0 good, missed 0, 0 with CRC errors", while 65 frames were +sent and written back and the link came up after 2.7 s: +https://github.com/ToyOSOrg/evidence/tree/e240b6db1696f7e5c805f1146578d0451e88673a/t14-i219 + +Exit: the job makes the traffic it receives, by MAC or PHY loopback or a +frame addressed to itself, so its receive claim no longer depends on the +network sending anything. Its verdict is then the same on a quiet wire and a +busy one. diff --git a/issues/the-hdmi-audio-function-1002-1637-is-driven-at-no-address-width.md b/issues/the-hdmi-audio-function-1002-1637-is-driven-at-no-address-width.md new file mode 100644 index 00000000000..aae8b03da77 --- /dev/null +++ b/issues/the-hdmi-audio-function-1002-1637-is-driven-at-no-address-width.md @@ -0,0 +1,30 @@ +--- +status: open +kind: defect +opened: 2026-10-21 +--- + +# The HDMI audio function 1002:1637 is driven at no address width + +The AMD bench's HDMI audio function `03:00.1` (`1002:1637`) sets GCAP's 64OK +(`0x4001`, HDA 1.0a §3.3.2) and keeps all 32 bits of its upper base registers. +But it issues only bits 0 to 39 of an address, in both its command engine and +its stream engine. Bit 40 and every bit above it are dropped, and bit 39 is +issued as programmed. + +So its 64OK is contradicted, and `hda` refuses the function by name +(`address_bits` in `system/hda/driver/src/controller.rs`) rather than state a +width for it. The 40 bits are this machine's measurement, not a property of +every `1002:1637`: neither a vendor document nor an AMD engineer's code has +been read for it. + +Evidence: the probe boot at `c8acfbc07` on the AMD bench (LENOVO 82A2, BIOS +DMCN38WW), the link and stream ladders and the bit patterns: +https://github.com/ToyOSOrg/evidence/tree/30bf1f4f70c2ce4771f0e7393fcc9a64d1c08cd1/probe +(`amd-round1/kernel.log` lines 396–501; the probe itself is +`c8acfbc07-probe.patch`). + +Exit: a width for `1002:1637` that an independent source supports. That is +AMD's documentation, or AMD's own code read clean-room with the fact's +provenance in evidence. `hda` then drives the function at that width, and the +AMD bench's `testcases` boot walks `03:00.1`'s codec with no `DMA FAULT`. diff --git a/issues/the-t14-answers-only-through-a-usb-stick.md b/issues/the-t14-answers-only-through-a-usb-stick.md index bf95c91252b..0dc82cfd65d 100644 --- a/issues/the-t14-answers-only-through-a-usb-stick.md +++ b/issues/the-t14-answers-only-through-a-usb-stick.md @@ -40,9 +40,9 @@ Constraints a reader would otherwise pay to re-derive: - **QEMU's `virtio-net-pci-non-transitional` on `q35` advertises no PCIe function-level reset** — measured, not assumed: `pcidev`'s refusal on that ground reddened every netd registration at once. So a re-claim is made safe by - ordering instead: bus mastering starts on the claim's **first grant**, never at - hand-over, so a function still holding its last holder's queue addresses can - act on none of them. `release` asks for a reset where the function advertises + ordering instead: bus mastering starts when the claim's holder **starts** it + (`device_dma_start`), never at hand-over, so a function still holding its + last holder's queue addresses can act on none of them before it is stopped. `release` asks for a reset where the function advertises one; the I219 does, so on the T14 both hold. - **The machine served its own log, and the Mac found it by name.** `logd` served the boot from its first line on a TCP port, and netd answers multicast diff --git a/issues/the-t14-hung-after-rebooting-with-its-i219-faulted.md b/issues/the-t14-hung-after-rebooting-with-its-i219-faulted.md index 3a45eb7c519..2deccf2f194 100644 --- a/issues/the-t14-hung-after-rebooting-with-its-i219-faulted.md +++ b/issues/the-t14-hung-after-rebooting-with-its-i219-faulted.md @@ -32,6 +32,6 @@ swapped-in netd's first grant started it mastering while it still ran the old netd's rings, its write faulted at the unit, and the fault handler cleared its bus mastering with receive still enabled. It stayed that way to the reset. The swap's fix (the kernel resets a released function, netd stops the part before -its first grant) removes that state; the next swap run says whether the hang +it starts it mastering) removes that state; the next swap run says whether the hang goes with it. If it does not, the reboot path is owed a quiesce of every function a process holds, as it already does for its own xHCI controllers. diff --git a/issues/the-virtio-drivers-match-only-a-non-transitional-device-id.md b/issues/the-virtio-drivers-match-only-a-non-transitional-device-id.md index 8358f95c154..9e580acc8f4 100644 --- a/issues/the-virtio-drivers-match-only-a-non-transitional-device-id.md +++ b/issues/the-virtio-drivers-match-only-a-non-transitional-device-id.md @@ -35,9 +35,8 @@ already whole. A transitional device offers no `VIRTIO_F_ACCESS_PLATFORM` on either: QEMU refuses `iommu_platform=on` on a transitional device, and VirtualBox's virtio -devices offer the bit nowhere. So behind a unit such a NIC is one no -claim may take -(`issues/a-virtio-function-that-offers-no-access-platform-is-handed-over-as-translated.md`); +devices offer the bit nowhere. So behind a unit such a NIC is one the +supervisor refuses every holder (`translated` in `system/supervisor/src/main.rs`); the matches reach a card only on the path `issues/a-machine-without-an-iommu-refuses-every-claim.md` rules for a machine with no unit, and the kernel's console on every machine. diff --git a/issues/toyos-runs-in-any-open-source-hypervisor.md b/issues/toyos-runs-in-any-open-source-hypervisor.md index 214fbcb5a52..e0b979772ea 100644 --- a/issues/toyos-runs-in-any-open-source-hypervisor.md +++ b/issues/toyos-runs-in-any-open-source-hypervisor.md @@ -30,7 +30,6 @@ signature. What they do depend on, each filed: - `issues/diskserver-claims-qemus-nvme-by-its-pci-id.md` - `issues/the-virtio-drivers-match-only-a-non-transitional-device-id.md` -- `issues/a-virtio-function-that-offers-no-access-platform-is-handed-over-as-translated.md` - `issues/the-x86-clock-calibrates-against-an-hpet-or-the-boot-panics.md` - `issues/a-hardware-reduced-x86-machine-has-no-power-off-and-no-power-button.md` - `issues/no-claim-gets-an-address-space-under-virtio-iommu.md` diff --git a/kernel/Cargo.toml b/kernel/Cargo.toml index 464de72d9fa..36c68e0cb79 100644 --- a/kernel/Cargo.toml +++ b/kernel/Cargo.toml @@ -124,6 +124,10 @@ protocol-port = [] # counter that wrapped at 4095 and reissued a live tag, and the isolation tests # red under it. Never a default and reached by no kernel build. counting-allocator = [] +# `pure/iova.rs`'s negative control: a grant is handed out whatever the +# holder's address limit, and the truncation test reds under it. Never a +# default and reached by no kernel build. +iova-ignores-limit = [] # The process lifecycle's controls (`pure/proclife/`), declared for the # library's own tests and never enabled by any kernel build: each reverts one # decision to the form this kernel really had, so the host model that refuses diff --git a/kernel/pci/src/af.rs b/kernel/pci/src/af.rs index a261418ec54..9f077eccf33 100644 --- a/kernel/pci/src/af.rs +++ b/kernel/pci/src/af.rs @@ -35,6 +35,12 @@ pub const fn resets(af_capabilities: u8) -> bool { af_capabilities & (FLR_CAPABLE | TP_CAPABLE) == FLR_CAPABLE | TP_CAPABLE } +/// Whether the function implements AF Status' Transactions Pending, from AF +/// Capabilities. +pub const fn reports_pending(af_capabilities: u8) -> bool { + af_capabilities & TP_CAPABLE != 0 +} + /// Whether the function still has transactions outstanding. pub const fn pending(af_status: u8) -> bool { af_status & TRANSACTIONS_PENDING != 0 @@ -53,6 +59,15 @@ mod tests { } } + #[test] + fn transactions_pending_is_reported_by_bit_zero_alone() { + assert!(reports_pending(0b01)); + assert!(reports_pending(0xff)); + for without in [0b00, 0b10, 0xfe] { + assert!(!reports_pending(without), "{without:#x}"); + } + } + #[test] fn pending_is_bit_zero_of_status() { assert!(pending(1)); diff --git a/kernel/pci/src/express.rs b/kernel/pci/src/express.rs index 519e8d8360a..6723952a4df 100644 --- a/kernel/pci/src/express.rs +++ b/kernel/pci/src/express.rs @@ -9,7 +9,8 @@ //! attached. The only reset that means the same thing on every function is the //! one the function itself implements — and QEMU's virtio functions implement //! none, so `pcidev` does not rest on this: what makes a re-claim safe there is -//! that bus mastering starts on the first grant. This is the belt, taken where +//! that bus mastering starts only when its next holder starts it, once that +//! holder has stopped it. This is the belt, taken where //! a function offers it. /// This capability's id in a function's capability list. @@ -19,6 +20,7 @@ pub const CAP_ID: u8 = 0x10; pub const CAPABILITIES: u64 = 0x02; pub const DEVICE_CAPABILITIES: u64 = 0x04; pub const DEVICE_CONTROL: u64 = 0x08; +pub const DEVICE_STATUS: u64 = 0x0A; /// Device Control 2 (§7.5.3.16), which only a version 2 structure carries. pub const DEVICE_CONTROL_2: u64 = 0x28; @@ -32,6 +34,22 @@ const FLR_CAPABLE: u32 = 1 << 28; /// and self-clearing: it always reads zero. const INITIATE_FLR: u16 = 1 << 15; +/// Device Status bit 5, "Transactions Pending" (§7.5.3.5): a Non-Posted +/// Request the function issued has not yet been completed or timed out. +const TRANSACTIONS_PENDING: u16 = 1 << 5; + +/// How long a function that stopped mastering is given for its last +/// Non-Posted Requests to complete or time out: twice the default range's top, +/// 50 ms (§2.8, Device Control 2 §7.5.3.16). One whose request outlasts this +/// is refused. +pub const QUIESCE_NANOS: u64 = 100_000_000; + +/// Whether the function has a Non-Posted Request outstanding, from its Device +/// Status. +pub const fn pending(device_status: u16) -> bool { + device_status & TRANSACTIONS_PENDING != 0 +} + /// §6.6.2: after initiating the reset, software waits at least 100 ms before /// it may access the function again. pub const SETTLE_NANOS: u64 = 100_000_000; @@ -67,6 +85,18 @@ pub const fn restored(device_control: u16) -> u16 { mod tests { use super::*; + /// Bit 5 and nothing else: its neighbours are AUX Power Detected (4) and + /// Emergency Power Reduction Detected (6). + #[test] + fn only_bit_five_says_a_transaction_is_pending() { + assert!(pending(1 << 5)); + assert!(!pending(0)); + assert!(!pending(!(1u16 << 5))); + for bit in (0..16).filter(|bit| *bit != 5) { + assert!(!pending(1 << bit), "bit {bit}"); + } + } + /// Bit 28 and nothing else. The neighbours are Role-Based Error Reporting /// (bit 15) and the Captured Slot Power values (25:18), so an off-by-one /// here reads a power limit as a reset capability. diff --git a/kernel/pure/iova.rs b/kernel/pure/iova.rs new file mode 100644 index 00000000000..11efb2557f2 --- /dev/null +++ b/kernel/pure/iova.rs @@ -0,0 +1,271 @@ +//! A domain's device addresses: which a unit may hand out, in what order, and +//! never at or above what the function they are handed to can issue. +//! +//! The allowed addresses are the unit's translated range less what the +//! backend reserves — the first leaf, the root bridges' windows, firmware's +//! reservations, the architecture's special ranges — each widened to whole +//! leaves. They are handed out lowest first and each exactly once, so under +//! hand-out-once no other order keeps a function of any width in range +//! longer. +//! +//! **Every grant ends at or below its holder's [`Limit`].** A function that +//! drops its upper address bits issues an address above its limit as one +//! below it, which can be another grant of the same domain: so refusing is +//! the only answer once nothing below the limit is left. + +#![forbid(unsafe_code)] + +use alloc::vec::Vec; + +/// The one leaf size a domain maps. +pub const LEAF: u64 = 2 << 20; + +/// One past the highest device address a function issues, as its driver states +/// it in bits. +#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Debug)] +pub struct Limit(u64); + +impl Limit { + /// The narrowest a driver may state: 32 bits. + pub const NARROWEST: Self = Self(1 << 32); + /// 64 bits. + pub const WIDEST: Self = Self(u64::MAX); + + /// `bits` of address, from 32 to 64; `None` outside that. + pub const fn bits(bits: u8) -> Option { + match bits { + 32..=63 => Some(Self(1 << bits)), + // Not representable one past the end; no unit translates the top leaf. + 64 => Some(Self::WIDEST), + _ => None, + } + } + + pub const fn end(self) -> u64 { + self.0 + } +} + +/// One domain's allowed addresses and how far up they are handed out. +pub struct Window { + /// Ascending, disjoint, whole leaves, none empty. + allowed: Vec<(u64, u64)>, + /// The first address not yet handed out. + next: u64, +} + +impl Window { + /// `[LEAF, 1 << translatable)` less every `(start, end)` of `reserved`, in + /// any order, each widened out to whole leaves. + /// + /// # Panics + /// If `translatable` is past 63: no unit translates 64 bits. + pub fn new(translatable: u8, reserved: &[(u64, u64)]) -> Self { + assert!(translatable < 64, "iova: a {translatable}-bit unit"); + let mut allowed = alloc::vec![(LEAF, 1u64 << translatable)]; + for &(start, end) in reserved.iter().filter(|(start, end)| end > start) { + let start = start & !(LEAF - 1); + let end = end.checked_next_multiple_of(LEAF).unwrap_or(!(LEAF - 1)); + allowed = allowed + .into_iter() + .flat_map(|(from, to)| [(from, to.min(start)), (from.max(end), to)]) + .filter(|(from, to)| to > from) + .collect(); + } + Self { allowed, next: 0 } + } + + pub fn allowed(&self) -> &[(u64, u64)] { + &self.allowed + } + + /// The lowest run of `bytes`, rounded up to whole leaves, never handed out + /// and ending at or below `limit`; `None` where none is left. A refusal + /// hands nothing out, so a holder with a wider limit is still served. + pub fn reserve(&mut self, bytes: u64, limit: Limit) -> Option { + let span = bytes.checked_next_multiple_of(LEAF).filter(|span| *span != 0)?; + for &(start, end) in &self.allowed { + let at = start.max(self.next); + let stop = at.checked_add(span)?; + if stop > end { + continue; + } + if stop > limit.end() && !cfg!(feature = "iova-ignores-limit") { + return None; + } + self.next = stop; + return Some(at); + } + None + } + + /// Whether `bytes` at `at` is room [`Self::reserve`] already handed out: + /// the only room a mapping may be placed in by address. `at` must be a + /// leaf boundary and the run inside one allowed range. + pub fn handed_out(&self, at: u64, bytes: u64) -> bool { + let Some(span) = bytes.checked_next_multiple_of(LEAF).filter(|span| *span != 0) else { return false }; + let Some(stop) = at.checked_add(span) else { return false }; + at.is_multiple_of(LEAF) && stop <= self.next && self.allowed.iter().any(|&(start, end)| start <= at && stop <= end) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use alloc::vec; + use std::collections::{BTreeMap, BTreeSet}; + + const GIB: u64 = 1 << 30; + + fn bits(bits: u8) -> Limit { + Limit::bits(bits).unwrap() + } + + /// Every grant until `window` refuses one of `bytes` under `limit`. + fn drain(window: &mut Window, bytes: u64, limit: Limit) -> Vec<(u64, u64)> { + core::iter::from_fn(|| window.reserve(bytes, limit).map(|at| (at, at + bytes.next_multiple_of(LEAF)))) + .take(1 << 14) + .collect() + } + + #[test] + fn a_limit_is_32_to_64_bits() { + assert_eq!(Limit::bits(31), None); + assert_eq!(Limit::bits(65), None); + assert_eq!(Limit::bits(32), Some(Limit::NARROWEST)); + assert_eq!(bits(40).end(), 1 << 40); + assert_eq!(bits(64).end(), u64::MAX); + } + + #[test] + fn addresses_go_lowest_first_from_the_first_leaf() { + let mut window = Window::new(48, &[]); + assert_eq!(window.reserve(1, Limit::NARROWEST), Some(LEAF)); + assert_eq!(window.reserve(LEAF + 1, Limit::NARROWEST), Some(2 * LEAF)); + assert_eq!(window.reserve(LEAF, Limit::NARROWEST), Some(4 * LEAF)); + assert_eq!(window.reserve(0, Limit::NARROWEST), None); + } + + #[test] + fn a_reserved_range_is_skipped_by_whole_leaves() { + // Unaligned at both ends, and a run too long for the gap before it. + let reserved = [(3 * LEAF + 1, 5 * LEAF - 1)]; + let mut window = Window::new(48, &reserved); + assert_eq!(window.allowed()[..2], [(LEAF, 3 * LEAF), (5 * LEAF, 1 << 48)]); + assert_eq!(window.reserve(LEAF, Limit::NARROWEST), Some(LEAF)); + assert_eq!(window.reserve(2 * LEAF, Limit::NARROWEST), Some(5 * LEAF)); + // The leaf the long run could not use is never handed out after it. + assert_eq!(window.reserve(LEAF, Limit::NARROWEST), Some(7 * LEAF)); + } + + /// The defect a limit exists for: a 40-bit function told an address at or + /// above `1 << 40` issues it with those bits dropped, and every grant the + /// window hands out under its limit must stay distinct under that + /// truncation, to the last one before the refusal. + #[test] + fn a_function_that_drops_its_upper_bits_reaches_only_its_own_grant() { + const WIDTH: u8 = 40; + // A few leaves below the limit, the rest of the unit above it. + let mut window = Window::new(44, &[(8 * LEAF, (1 << WIDTH) - 4 * LEAF)]); + let granted = drain(&mut window, LEAF, bits(WIDTH)); + let mut issued = BTreeMap::new(); + for &(at, end) in &granted { + let truncated = at & ((1 << WIDTH) - 1); + if let Some(other) = issued.insert(truncated, at) { + panic!("the grant at {at:#x} is issued as {truncated:#x}, which is the grant at {other:#x}"); + } + assert!(end <= 1 << WIDTH, "the grant {at:#x}..{end:#x} ends above the function's limit"); + } + assert_eq!(granted.len(), 7 + 4, "every leaf below the limit, and none above"); + // The room above stays for a holder that states the wider width. + assert_eq!(window.reserve(LEAF, bits(44)), Some(1 << WIDTH)); + } + + /// Against a straightforward model, over windows and limits a fixed seed + /// varies: what `reserve` hands out is the lowest allowed run at or above + /// the cursor, never reserved, never past its limit, and `None` exactly + /// where no such run is left. + #[test] + fn every_grant_is_the_lowest_allowed_run_below_its_limit() { + let mut seed = 0x9E37_79B9_7F4A_7C15u64; + let mut draw = |n: u64| { + seed ^= seed << 13; + seed ^= seed >> 7; + seed ^= seed << 17; + seed % n + }; + for _ in 0..200 { + let reserved: Vec<(u64, u64)> = (0..draw(6)) + .map(|_| { + let start = draw(4096) * LEAF + draw(LEAF); + (start, start + draw(512) * LEAF + draw(LEAF)) + }) + .collect(); + let mut window = Window::new(33, &reserved); + let leaves: BTreeSet = + (1..(1u64 << 33) / LEAF).map(|leaf| leaf * LEAF).filter(|at| window.allowed().iter().any(|&(s, e)| s <= *at && at + LEAF <= e)).collect(); + let mut next = 0; + for _ in 0..60 { + let span = (1 + draw(64)) * LEAF; + let limit = if draw(2) == 0 { Limit::NARROWEST } else { bits(33) }; + let fits = |at: &u64| *at >= next && (0..span / LEAF).all(|i| leaves.contains(&(at + i * LEAF))); + let lowest = leaves.iter().copied().find(fits).filter(|at| at + span <= limit.end()); + let got = window.reserve(span, limit); + assert_eq!(got, lowest, "reserve({span:#x}) under {limit:?} with {reserved:x?}"); + if let Some(at) = got { + assert!(reserved.iter().all(|&(s, e)| at + span <= s || e <= at), "{at:#x} inside {reserved:x?}"); + assert!(window.handed_out(at, span)); + next = at + span; + } + } + } + } + + #[test] + fn only_what_was_handed_out_whole_is_room() { + let mut window = Window::new(48, &[(4 * LEAF, 5 * LEAF)]); + let first = window.reserve(3 * LEAF, Limit::NARROWEST).unwrap(); + let after_the_hole = window.reserve(LEAF, Limit::NARROWEST).unwrap(); + assert_eq!((first, after_the_hole), (LEAF, 5 * LEAF)); + assert!(window.handed_out(first, 3 * LEAF)); + assert!(window.handed_out(2 * LEAF, 1)); + assert!(window.handed_out(after_the_hole, LEAF)); + // Across the reserved leaf, off a boundary, past the cursor, empty, and + // past the end of the address space: none of it. + assert!(!window.handed_out(3 * LEAF, 2 * LEAF)); + assert!(!window.handed_out(LEAF + 1, LEAF)); + assert!(!window.handed_out(6 * LEAF, LEAF)); + assert!(!window.handed_out(LEAF, 0)); + assert!(!window.handed_out(!(LEAF - 1), LEAF)); + } + + /// The AMD bench as its firmware declares it (`pcidev: firmware declared + /// root bridge memory`), with the x86 interrupt range and AMD-Vi's special + /// range reserved: the lend window first, then what a 32-bit and a 40-bit + /// function each get before they are refused. + #[test] + fn the_amd_benchs_layout() { + let reserved = vec![ + (0xd000_0000, 0xd070_0000), + (0xfc_9000_0000, 0xfc_a020_0000), + (0xd070_0000, 0xf800_0000), + (0xfc00_0000, 0xfdc0_0000), + (0xff00_0000, 0xffea_b000), + (0x4_5020_0000, 0xfc_9000_0000), + (0xfc_a020_0000, 0xfc_b000_0000), + (0xfee0_0000, 0xfef0_0000), + (0xfd_0000_0000, 0x100_0000_0000), + ]; + let mut window = Window::new(48, &reserved); + assert_eq!(window.reserve(256 << 20, Limit::NARROWEST), Some(LEAF)); + let narrow = drain(&mut window, LEAF, Limit::NARROWEST).len(); + let wide = drain(&mut window, LEAF, bits(40)).len(); + // 1663 leaves below the first window, 32 over ECAM, 9 below the + // interrupt range, less the lend window's 128. + assert_eq!(narrow, 1663 + 32 + 9 - 128); + // Then 6785 up to the 64-bit window and 640 between it and the special range. + assert_eq!(wide, 6785 + 640); + assert!(narrow as u64 * LEAF > 3 * GIB); + assert_eq!(window.reserve(LEAF, bits(41)), Some(1 << 40)); + } +} diff --git a/kernel/pure/lib.rs b/kernel/pure/lib.rs index eb8066d4c33..41098a89349 100644 --- a/kernel/pure/lib.rs +++ b/kernel/pure/lib.rs @@ -13,6 +13,7 @@ pub mod bootwrite; pub mod clock; pub mod cpu; pub mod dirnodes; +pub mod iova; pub mod microcode; pub mod mtrr; pub mod owners; diff --git a/kernel/src/arch/aarch64/cpu.rs b/kernel/src/arch/aarch64/cpu.rs index 0dd00716154..37738467f76 100644 --- a/kernel/src/arch/aarch64/cpu.rs +++ b/kernel/src/arch/aarch64/cpu.rs @@ -2,26 +2,6 @@ use core::arch::asm; -/// How many bits wide a physical address this CPU issues is: -/// `ID_AA64MMFR0_EL1.PARange` (Arm ARM DDI 0487): 0b0111, 56 bits, is -/// FEAT_D128's, and the encodings past it are reserved. -pub fn physical_address_bits() -> u8 { - let mmfr0: u64; - // SAFETY: reads an EL1 ID register, which EL1 may always read. - unsafe { asm!("mrs {}, id_aa64mmfr0_el1", out(reg) mmfr0, options(nomem, nostack, preserves_flags)) }; - match mmfr0 & 0xF { - 0 => 32, - 1 => 36, - 2 => 40, - 3 => 42, - 4 => 44, - 5 => 48, - 6 => 52, - 7 => 56, - reserved => panic!("ID_AA64MMFR0_EL1.PARange holds the reserved value {reserved:#x}"), - } -} - /// The CPU's free-running counter: the generic timer's virtual count, /// `CNTVCT_EL0`. The `ISB` keeps the read from being taken early, ahead of /// the code it times (Arm ARM K.a, D12.2.2). diff --git a/kernel/src/arch/aarch64/smmu/domain.rs b/kernel/src/arch/aarch64/smmu/domain.rs index 710c7127b05..ae58e56feff 100644 --- a/kernel/src/arch/aarch64/smmu/domain.rs +++ b/kernel/src/arch/aarch64/smmu/domain.rs @@ -21,7 +21,7 @@ use toyos_smmu::Asid; use super::{window, Live, UNIT}; use crate::iommu::domains; -use crate::iommu::{DomainId, IommuError, Iova, StreamId}; +use crate::iommu::{DomainId, IommuError, Iova, Limit, StreamId}; use crate::log; use crate::mm::PAGE_SIZE; @@ -116,7 +116,10 @@ pub fn create(room: u64) -> Result<(DomainId, Iova), IommuError> { } descriptor.write_u64(0, words[0]); if let Some(page) = doorbell { - assert!(page < next.addresses.floor(), "SMMU: the doorbell at {page:#x} is inside the addresses a domain hands out"); + assert!( + !next.addresses.allowed().iter().any(|&(start, end)| start <= page && page < end), + "SMMU: the doorbell at {page:#x} is inside the addresses a domain hands out" + ); live.put(root, page, Leaf::Doorbell(Phys::new(page).expect("SMMU: a 4 KiB page below 2^48"))); } let what = alloc::format!( @@ -132,10 +135,10 @@ pub fn create(room: u64) -> Result<(DomainId, Iova), IommuError> { Ok(live.domains.add(next, Space { asid, root, context }, what)) } -pub fn map(id: DomainId, phys: u64, bytes: u64) -> Result { +pub fn map(id: DomainId, phys: u64, bytes: u64, limit: Limit) -> Result { let mut held = UNIT.lock(); let live = held.as_mut().expect("a domain exists only on an armed unit"); - let (space, at) = live.domains.map(id, phys, bytes)?; + let (space, at) = live.domains.map(id, phys, bytes, limit)?; for (at, phys) in domains::leaves(at, phys, bytes) { live.put(space.root, at, block(phys)); } diff --git a/kernel/src/arch/aarch64/smmu/mod.rs b/kernel/src/arch/aarch64/smmu/mod.rs index 56862339320..5a783a99758 100644 --- a/kernel/src/arch/aarch64/smmu/mod.rs +++ b/kernel/src/arch/aarch64/smmu/mod.rs @@ -271,7 +271,10 @@ pub fn init(rsdp_addr: u64, devices: &[PciDevice], windows: Option<&[toyos_abi:: Err(why) => log!("IOMMU: {function}'s IORT route is refused: {why:?}"), } } - let reserved = windows.map(|w| w.iter().map(|w| (w.base, w.end())).collect()); + // The ITS's doorbell page is mapped at its own address in every domain + // (`domain::create`), so no grant may be handed out over it. + let doorbell = super::irqchip::its::doorbell_page().map(|page| (page, page + crate::mm::PAGE_SIZE)); + let reserved = windows.map(|w| w.iter().map(|w| (w.base, w.end())).chain(doorbell).collect()); let Some(live) = program(regs, smmu.base, smmu.coherent_override, smmu.event, routes, reserved) else { return; }; diff --git a/kernel/src/arch/x86_64/amdvi/domain.rs b/kernel/src/arch/x86_64/amdvi/domain.rs index 7d1934032b1..5350c88a29e 100644 --- a/kernel/src/arch/x86_64/amdvi/domain.rs +++ b/kernel/src/arch/x86_64/amdvi/domain.rs @@ -16,7 +16,7 @@ use alloc::vec::Vec; use crate::iommu::domains::{self, Domains}; -use crate::iommu::{DomainId, IommuError, Iova, StreamId}; +use crate::iommu::{DomainId, IommuError, Iova, Limit, StreamId}; use crate::log; use crate::sync::Lock; use toyos_acpi::Requesters; @@ -121,10 +121,10 @@ pub fn create(room: u64) -> Result<(DomainId, Iova), IommuError> { Ok(amd.domains.add(next, space, alloc::format!("root={:#x} mode={ROOT_LEVEL}", space.root.get()))) } -pub fn map(id: DomainId, phys: u64, bytes: u64) -> Result { +pub fn map(id: DomainId, phys: u64, bytes: u64, limit: Limit) -> Result { let mut held = DOMAINS.lock(); let amd = held.as_mut().expect("amdvi: a domain exists only once a unit translates"); - let (space, at) = amd.domains.map(id, phys, bytes)?; + let (space, at) = amd.domains.map(id, phys, bytes, limit)?; place_leaves(id, space, at, phys, bytes); Ok(at) } diff --git a/kernel/src/arch/x86_64/amdvi/mod.rs b/kernel/src/arch/x86_64/amdvi/mod.rs index a9a96b6db53..5bf3a6b33d5 100644 --- a/kernel/src/arch/x86_64/amdvi/mod.rs +++ b/kernel/src/arch/x86_64/amdvi/mod.rs @@ -46,6 +46,11 @@ use toyos_acpi::{ /// which nothing here reads. const REGISTER_WINDOW: u64 = 0x4000; +/// What an untranslated request in it means is fixed by the platform — +/// interrupts, system management, I/O and configuration — and not by a page +/// table (48882 Rev. 3.09 §2.1.2, Table 3), so no domain's addresses enter it. +const SPECIAL_RANGE: (u64, u64) = (0xFD_0000_0000, 0x100_0000_0000); + /// `CONTROL` fields firmware may leave on, each with the `STATUS` bit that /// says it stopped, or `None` where `CONTROL` reading back is the answer: each /// log stopped before its interrupt and the unit last, since a log's @@ -314,7 +319,11 @@ pub fn init(rsdp_addr: u64, devices: &[PciDevice], windows: Option<&[RootBridgeW let identity = build(devices, &settings, remap.is_some()); // Nothing where no declaration is held, as `vtd` treats its RMRRs. let reserved = windows.map(|w| { - w.iter().map(|w| (w.base, w.end())).chain(ivmds.iter().map(|m| (m.start, m.start + m.length))).collect() + w.iter() + .map(|w| (w.base, w.end())) + .chain(ivmds.iter().map(|m| (m.start, m.start + m.length))) + .chain([super::iommu_unit::INTERRUPT_RANGE, SPECIAL_RANGE]) + .collect() }); let mut ids: Vec = platform.iter().map(|&(id, _)| id).collect(); ids.sort_unstable(); diff --git a/kernel/src/arch/x86_64/amdvi/selftest.rs b/kernel/src/arch/x86_64/amdvi/selftest.rs index 96fe0d7b873..4754c86daa9 100644 --- a/kernel/src/arch/x86_64/amdvi/selftest.rs +++ b/kernel/src/arch/x86_64/amdvi/selftest.rs @@ -61,9 +61,12 @@ pub fn run(devices: &[PciDevice]) { let before = super::fault::recorded(); let masked = crate::arch::IrqGuard::close(); - let refused = write(kept_regs, at, phys); + // Past the memory the identity domain maps; a domain's own addresses are + // handed out lowest first, so they are no such address. + let unmapped = crate::mm::pmm::top().next_multiple_of(PAGE_2M); + let refused = write(kept_regs, unmapped, phys); log!( - "amdvi-selftest: {on_identity}'s write at {at:#x}, which the identity domain does not map, answered \ + "amdvi-selftest: {on_identity}'s write at {unmapped:#x}, which the identity domain does not map, answered \ {refused:#x}: {}", verdict(refused, REFUSED) ); diff --git a/kernel/src/arch/x86_64/cpu.rs b/kernel/src/arch/x86_64/cpu.rs index 4181efb11d3..1ed9d6a96b1 100644 --- a/kernel/src/arch/x86_64/cpu.rs +++ b/kernel/src/arch/x86_64/cpu.rs @@ -159,15 +159,6 @@ pub fn leaf_d(subleaf: u32) -> (u32, u32, u32, u32) { if cpuid(0, 0).0 >= 0xD { cpuid(0xD, subleaf) } else { (0, 0, 0, 0) } } -/// How many bits wide a physical address this CPU issues is: CPUID -/// 8000_0008H `EAX[7:0]` (Intel SDM Vol. 2A, CPUID; AMD APM Vol. 3, -/// Fn8000_0008_EAX `PhysAddrSize`), a leaf every x86-64 CPU of this -/// project's range implements. -pub fn physical_address_bits() -> u8 { - assert!(cpuid(0x8000_0000, 0).0 >= 0x8000_0008, "CPUID does not reach leaf 8000_0008H"); - cpuid(0x8000_0008, 0).0 as u8 -} - /// CPUID with both index registers, as `(eax, ebx, ecx, edx)`. /// /// `core`'s intrinsic and not an `asm!` of our own: `rbx` is CPUID's output diff --git a/kernel/src/arch/x86_64/iommu_unit.rs b/kernel/src/arch/x86_64/iommu_unit.rs index 291a2a0d883..9114a2d7cc0 100644 --- a/kernel/src/arch/x86_64/iommu_unit.rs +++ b/kernel/src/arch/x86_64/iommu_unit.rs @@ -128,11 +128,16 @@ pub(super) fn yn(v: bool) -> char { } } +/// The interrupt address range, which no domain's addresses may enter: a +/// request there is a message, not a DMA (VT-d Rev. 5.0 §3.15; AMD 48882 Rev. +/// 3.09 §2.2.9 for requests without a PASID). +pub const INTERRUPT_RANGE: (u64, u64) = (0xFEE0_0000, 0xFEF0_0000); + /// A device's address spaces, on whichever backend translates. pub mod domain { use super::{backend, Backend}; use crate::arch::x86_64::{amdvi, vtd}; - use crate::iommu::{DomainId, IommuError, Iova, StreamId}; + use crate::iommu::{DomainId, IommuError, Iova, Limit, StreamId}; pub fn create(room: u64) -> Result<(DomainId, Iova), IommuError> { match backend() { @@ -170,10 +175,10 @@ pub mod domain { found } - pub fn map(id: DomainId, phys: u64, bytes: u64) -> Result { + pub fn map(id: DomainId, phys: u64, bytes: u64, limit: Limit) -> Result { match minted() { - Backend::AmdVi => amdvi::domain::map(id, phys, bytes), - _ => vtd::domain::map(id, phys, bytes), + Backend::AmdVi => amdvi::domain::map(id, phys, bytes, limit), + _ => vtd::domain::map(id, phys, bytes, limit), } } diff --git a/kernel/src/arch/x86_64/vtd/domain.rs b/kernel/src/arch/x86_64/vtd/domain.rs index d31fe481c9a..7b92994f969 100644 --- a/kernel/src/arch/x86_64/vtd/domain.rs +++ b/kernel/src/arch/x86_64/vtd/domain.rs @@ -19,7 +19,7 @@ use crate::log; use alloc::vec::Vec; use crate::iommu::domains::{self, Domains}; -use crate::iommu::{AddressWidth, DomainId, IommuError, Iova, StreamId}; +use crate::iommu::{AddressWidth, DomainId, IommuError, Iova, Limit, StreamId}; use crate::sync::Lock; use super::table::{self, Space}; @@ -74,9 +74,9 @@ pub fn create(room: u64) -> Result<(DomainId, Iova), IommuError> { Ok(held.domains.add(next, space, what)) } -pub fn map(id: DomainId, phys: u64, bytes: u64) -> Result { +pub fn map(id: DomainId, phys: u64, bytes: u64, limit: Limit) -> Result { let mut held = DOMAINS.lock(); - let (space, at) = held.domains.map(id, phys, bytes)?; + let (space, at) = held.domains.map(id, phys, bytes, limit)?; put(id, space, at, phys, bytes); Ok(at) } diff --git a/kernel/src/arch/x86_64/vtd/mod.rs b/kernel/src/arch/x86_64/vtd/mod.rs index 4fa6ccdfde9..a6bbb126c8c 100644 --- a/kernel/src/arch/x86_64/vtd/mod.rs +++ b/kernel/src/arch/x86_64/vtd/mod.rs @@ -153,7 +153,8 @@ pub fn init(rsdp_addr: u64, devices: &[PciDevice], windows: Option<&[RootBridgeW let mut ready: Vec<(Unit, Plan)> = Vec::new(); // What no domain's addresses may reach, RMRRs added below as the walk // meets them; nothing where no declaration is held. - let mut reserved: Option> = windows.map(|w| w.iter().map(|w| (w.base, w.end())).collect()); + let mut reserved: Option> = windows + .map(|w| w.iter().map(|w| (w.base, w.end())).chain([super::iommu_unit::INTERRUPT_RANGE]).collect()); for structure in dmar.structures() { match structure { Ok(Structure::Drhd(drhd)) => { @@ -180,8 +181,8 @@ pub fn init(rsdp_addr: u64, devices: &[PciDevice], windows: Option<&[RootBridgeW // at or above `top` is one firmware requires stay mapped and this // kernel does not map — a device in its scope would fault on the // access firmware reserved the range for. The other half, that it - // is mapped *nowhere else*, is structural: a driver domain's window - // begins above `top` or `Domain::new` refuses it. + // is mapped *nowhere else*, is structural: every domain's addresses + // are built clear of it (`domain::avoid`). Ok(Structure::Rmrr(rmrr)) => { let held = rmrr.limit() < identity_top; // Spelled whole in each arm rather than with the verdict as a diff --git a/kernel/src/arch/x86_64/vtd/table.rs b/kernel/src/arch/x86_64/vtd/table.rs index cce6d6badad..115414e579f 100644 --- a/kernel/src/arch/x86_64/vtd/table.rs +++ b/kernel/src/arch/x86_64/vtd/table.rs @@ -7,7 +7,6 @@ use alloc::vec::Vec; -use crate::iommu::window; use crate::iommu::{AddressWidth, Iova, StreamId}; use crate::mm::pmm::{self, PhysPage}; use crate::mm::{DirectMap, Mmio, PAGE_2M}; @@ -209,18 +208,14 @@ pub const fn translatable_bits(width: AddressWidth, mgaw: u8) -> u8 { if mgaw < width.bits() { mgaw } else { width.bits() } } -/// A unit whose `MGAW` is narrower than its `SAGAW` gets its window under the -/// smaller of the two. +/// A unit whose `MGAW` is narrower than its `SAGAW` gets its addresses under +/// the smaller of the two. /// /// Asserted here rather than in a guest because no guest holds the shape: QEMU's /// `intel-iommu` derives both fields from one `aw-bits` property, so its model /// cannot report a `SAGAW` wider than its `MGAW` at all. const _: () = { assert!(translatable_bits(AddressWidth::Bits48, 39) == 39); - assert!(window::first_address(39) < 1 << 39); - // What a window placed by the table depth alone would be, against the - // ceiling such a unit reports. - assert!(window::first_address(48) >= 1 << 39); // A unit whose two limits agree is unchanged by any of this. assert!(translatable_bits(AddressWidth::Bits48, 48) == 48); assert!(translatable_bits(AddressWidth::Bits39, 39) == 39); diff --git a/kernel/src/drivers/virtio.rs b/kernel/src/drivers/virtio.rs index b4f3b82e9f8..386b8456357 100644 --- a/kernel/src/drivers/virtio.rs +++ b/kernel/src/drivers/virtio.rs @@ -575,7 +575,9 @@ pub fn cap_selftest() { // guess but inside the BAR's real size has a case to be accepted by. const WINDOW: u64 = 0x8000; // One buffer for both the config space and the BAR window, so a permitted subregion stays inside it. - let pool = DmaPool::alloc_in(WINDOW as usize, crate::iommu::DeviceSpace::Untranslated); + // No device is told it: it stands in for config space. + let pool = DmaPool::alloc_in(WINDOW as usize, crate::iommu::DeviceSpace::Untranslated, crate::iommu::Limit::WIDEST) + .expect("pci-cap-selftest: a 64-bit untranslated pool is never refused"); let phys = pool.view().host_phys(); // SAFETY: the pool owns this page until it drops at end of function, and no Mmio built over it escapes. let cfg = unsafe { Mmio::over_phys(DirectMap::from_phys(phys), WINDOW) }; diff --git a/kernel/src/drivers/virtio_console.rs b/kernel/src/drivers/virtio_console.rs index 0ebf3c8a63e..29e603f96c3 100644 --- a/kernel/src/drivers/virtio_console.rs +++ b/kernel/src/drivers/virtio_console.rs @@ -14,7 +14,7 @@ use core::sync::atomic::{AtomicBool, Ordering}; use super::pci::PciDevice; use super::serial::BackendGuard; -use super::virtio::{wait_used, BufDir, DescSlot, Offered, Virtqueue, VirtioDevice, VIRTIO_F_VERSION_1}; +use super::virtio::{wait_used, BufDir, DescSlot, Offered, Virtqueue, VirtioDevice, VIRTIO_F_ACCESS_PLATFORM, VIRTIO_F_VERSION_1}; use super::DmaPool; use crate::log; use crate::mm::{Dma, Unaligned}; @@ -232,10 +232,29 @@ pub fn init(devices: &[PciDevice]) -> bool { }; log!("virtio-console: found at PCI {:02x}:{:02x}.{}", pci_dev.bus, pci_dev.dev, pci_dev.func); - // An address space of this device's own, holding one pool and nothing else. - let space = crate::iommu::DeviceSpace::create(&[pci_dev]); + // An address space of this device's own, holding one pool and nothing + // else — unless it does not offer ACCESS_PLATFORM: then it uses physical + // addresses whatever translates the bus, and a driver MUST give it physical + // ones (virtio 1.2 §6.1); a domain's addresses would be other memory's. + let space = if offered.features & VIRTIO_F_ACCESS_PLATFORM != 0 { + crate::iommu::DeviceSpace::create(&[pci_dev]) + } else { + log!( + "virtio-console: PCI {:02x}:{:02x}.{} offers no ACCESS_PLATFORM, so it is told physical addresses", + pci_dev.bus, pci_dev.dev, pci_dev.func + ); + crate::iommu::DeviceSpace::Untranslated + }; // Leaked, not held in a `static`: the console is never unbound. - let dma = DmaPool::alloc_in(DMA_SIZE, space).leak(); + // Virtio 1.x carries every buffer address as 64 bits (§2.7.5). + let dma = match DmaPool::alloc_in(DMA_SIZE, space, crate::iommu::Limit::WIDEST) { + Ok(pool) => pool.leak(), + Err(why) => { + log!("virtio-console: PCI {:02x}:{:02x}.{} {why} — device refused", pci_dev.bus, pci_dev.dev, pci_dev.func); + offered.decline(); + return false; + } + }; // Before the device is told an address, and after the only mapping it gets. space.attach(pci_dev.bus, pci_dev.dev, pci_dev.func); diff --git a/kernel/src/drivers/xhci/wait/boot.rs b/kernel/src/drivers/xhci/wait/boot.rs index 423babdc859..5b74a7daba7 100644 --- a/kernel/src/drivers/xhci/wait/boot.rs +++ b/kernel/src/drivers/xhci/wait/boot.rs @@ -340,7 +340,15 @@ fn init_one(pci_dev: &PciDevice) -> Option { // After the reset: a controller refused above never allocates, and // `DmaPool` frees the pool — and takes it back out of the domain — on every // refusal below when it drops. - let pool = DmaPool::alloc_in(layout.pool_size, space); + // xHCI 1.2 §5.3.6: HCCPARAMS1's AC64 says whether it issues 64-bit addresses. + let limit = if hccparams1 & 1 != 0 { crate::iommu::Limit::WIDEST } else { crate::iommu::Limit::NARROWEST }; + let pool = match DmaPool::alloc_in(layout.pool_size, space, limit) { + Ok(pool) => pool, + Err(why) => { + refuse(format_args!("its pool cannot be placed where it reaches: {why}")); + return None; + } + }; space.attach(pci_dev.bus, pci_dev.dev, pci_dev.func); // MaxSlotsEn caps what this driver can track; the controller then refuses diff --git a/kernel/src/iommu/domains.rs b/kernel/src/iommu/domains.rs index fef333fc4f2..9c44421b999 100644 --- a/kernel/src/iommu/domains.rs +++ b/kernel/src/iommu/domains.rs @@ -1,11 +1,12 @@ //! The domains a unit translates for, whatever the backend: their ids, each -//! one's device addresses, and which 2 MiB leaves a map or a place covers. A -//! backend owns the tables these name and the invalidation that publishes -//! them, and holds this under the lock it writes those under. +//! one's device addresses (`kernel::iova`), and which 2 MiB leaves a map or a +//! place covers. A backend owns the tables these name and the invalidation +//! that publishes them, and holds this under the lock it writes those under. use alloc::vec::Vec; -use crate::iommu::window::Window; +use kernel::iova::{Limit, Window}; + use crate::iommu::{DomainId, IommuError, Iova}; use crate::mm::PAGE_2M; @@ -16,9 +17,10 @@ pub struct Domains { /// By id minus `first`; never shrinks, since a released id would name a /// domain some unit may still have cached. live: Vec<(S, Window)>, - /// What no domain's addresses may reach: the root bridges' declared - /// windows and the regions firmware reserved, as `(start, end)`; `None` - /// where no declaration is held ([`Window::new`]). + /// What no domain's addresses may reach, as `(start, end)`: the root + /// bridges' declared windows, the regions firmware reserved, and the + /// backend's own special ranges; `None` where no declaration of the + /// windows is held, and then there is no domain. reserved: Option>, } @@ -39,25 +41,32 @@ impl Domains { self.reserved = reserved; } - /// The next domain's id and its window over `translatable` bits, with - /// `room` bytes handed out first; refused, with no id spent, where ids - /// reach `ceiling` or the window would hold less. + /// The next domain's id and its addresses over `translatable` bits, with + /// `room` bytes below 4 GiB handed out first, which any holder can reach; + /// refused, with no id spent, where ids reach `ceiling` or the room is not + /// there. pub fn prepare(&self, translatable: u8, ceiling: u32, room: u64) -> Result { let id = u16::try_from(usize::from(self.first) + self.live.len()) .ok() .filter(|id| u32::from(*id) < ceiling) .ok_or(IommuError::DomainsExhausted(ceiling))?; - let (addresses, first) = Window::new(translatable, self.reserved.as_deref(), room)?; - Ok(Next { id, addresses, first }) + let mut addresses = Window::new(translatable, self.reserved.as_deref().ok_or(IommuError::Undeclared)?); + let first = match room { + 0 => 0, + room => addresses.reserve(room, Limit::NARROWEST).ok_or(IommuError::NoRoom { room })?, + }; + Ok(Next { id, addresses, first: Iova::translated(first) }) } /// `next`, made over `space`, which `what` describes in its record. pub fn add(&mut self, next: Next, space: S, what: impl core::fmt::Display) -> (DomainId, Iova) { + let allowed = next.addresses.allowed(); log!( - "iommu: domain{} {what} addresses from {:#x} to {:#x}", + "iommu: domain{} {what} addresses in {} ranges from {:#x} to {:#x}", next.id, - next.addresses.floor(), - next.addresses.ceiling() + allowed.len(), + allowed.first().map_or(0, |range| range.0), + allowed.last().map_or(0, |range| range.1), ); self.live.push((space, next.addresses)); (DomainId::new(next.id), next.first) @@ -67,15 +76,15 @@ impl Domains { self.live[usize::from(id.raw() - self.first)].0 } - /// Room for `bytes` at `phys`, never handed out before: the domain's - /// tables, and where the leaves go. - pub fn map(&mut self, id: DomainId, phys: u64, bytes: u64) -> Result<(S, Iova), IommuError> { + /// Room for `bytes` at `phys`, never handed out before and ending at or + /// below `limit`: the domain's tables, and where the leaves go. + pub fn map(&mut self, id: DomainId, phys: u64, bytes: u64, limit: Limit) -> Result<(S, Iova), IommuError> { aligned(phys)?; // A `DomainId` only `add` mints, so the index is always in range. let (space, window) = &mut self.live[usize::from(id.raw() - self.first)]; - let at = window.reserve(bytes).ok_or(IommuError::AddressesExhausted(window.ceiling()))?; - log!("iommu: domain{} maps {phys:#x}..{:#x} at {:#x}", id.raw(), phys + bytes.next_multiple_of(PAGE_2M), at.raw()); - Ok((*space, at)) + let at = window.reserve(bytes, limit).ok_or(IommuError::AddressesExhausted(limit))?; + log!("iommu: domain{} maps {phys:#x}..{:#x} at {at:#x}", id.raw(), phys + bytes.next_multiple_of(PAGE_2M)); + Ok((*space, Iova::translated(at))) } /// The domain's tables, for `bytes` at `phys` placed at `at`, room this @@ -84,7 +93,7 @@ impl Domains { pub fn place(&self, id: DomainId, at: Iova, phys: u64, bytes: u64) -> Result { aligned(phys)?; let (space, window) = &self.live[usize::from(id.raw() - self.first)]; - assert!(window.handed_out(at, bytes), "iommu: domain{} never handed out {:#x}+{bytes:#x}", id.raw(), at.raw()); + assert!(window.handed_out(at.raw(), bytes), "iommu: domain{} never handed out {:#x}+{bytes:#x}", id.raw(), at.raw()); Ok(*space) } } diff --git a/kernel/src/iommu/mod.rs b/kernel/src/iommu/mod.rs index 1a4993a2764..8821b14cae5 100644 --- a/kernel/src/iommu/mod.rs +++ b/kernel/src/iommu/mod.rs @@ -1,6 +1,6 @@ //! The unit that decides what a device may reach. //! -//! Inventories the machine's IOMMU units, turns translation on, and hands a driver an address space of its own to put its DMA in; an unusable unit is logged and left off rather than halting boot. Names above the backend stay backend-neutral, and what every backend decides alike is declared here once: a domain's addresses ([`window`]) and what a fault record ends ([`fault`]). +//! Inventories the machine's IOMMU units, turns translation on, and hands a driver an address space of its own to put its DMA in; an unusable unit is logged and left off rather than halting boot. Names above the backend stay backend-neutral, and what every backend decides alike is declared here once: a domain's addresses (`kernel::iova`, over what each backend reserves) and what a fault record ends ([`fault`]). //! //! The refusal is deliberately not yet built for a driver in this kernel: landing it before any userspace driver exists would cost every machine and protect nothing. A function a *process* drives is the other case and is refused ([`OwnSpace`], [`remapping`]), because a descriptor it writes a physical address into is an arbitrary read and write over all of memory, and a message nothing translates for it alone raises any claim's interrupt; its message raises its own interrupt, translated for it alone ([`Remapped`]). //! @@ -11,12 +11,12 @@ use crate::arch::iommu_unit as unit; use crate::arch::message_unit as messages; +pub use kernel::iova::Limit; pub(crate) mod domains; pub(crate) mod fault; #[cfg(feature = "boot-actuators")] pub(crate) mod testdev; -pub(crate) mod window; /// The address width a device's translations cover. /// @@ -116,18 +116,13 @@ pub enum IommuError { /// The units disagree on the depth a domain's tables would be built at. WidthsDisagree, DomainsExhausted(u32), - /// Every address up to this ceiling is handed out. - AddressesExhausted(u64), - /// What this machine's units translate does not reach above its memory, so - /// a device window has nowhere to sit that a stale descriptor would miss. - WindowBelowMemory { translatable: u8, floor: u64, top: u64 }, - /// The units translate no further than the CPU's physical addresses, so a - /// domain has to end under the root bridges' windows, and no declaration - /// of them is held. - Undeclared { translatable: u8, cpu: u8 }, - /// From the floor to the first root-bridge window or reserved region above - /// it is less than a new domain was asked to hand out. - NoRoom { floor: u64, ceiling: u64, room: u64 }, + /// Every allowed address below this limit is handed out. + AddressesExhausted(Limit), + /// No declaration of the root bridges' windows is held, so no address is + /// known to be clear of them. + Undeclared, + /// Less than a new domain was asked to hand out is allowed below 4 GiB. + NoRoom { room: u64 }, /// Not a whole number of the 2 MiB leaves this kernel writes. Unaligned(u64), NotMapped(Iova), @@ -143,25 +138,17 @@ impl core::fmt::Display for IommuError { Self::DomainsExhausted(ceiling) => { write!(f, "every one of this machine's {ceiling} domains is taken") } - Self::AddressesExhausted(ceiling) => { - write!(f, "a domain's device addresses up to {ceiling:#x} are all handed out") + Self::AddressesExhausted(limit) => { + write!(f, "a domain's device addresses below {:#x} are all handed out", limit.end()) } - Self::WindowBelowMemory { translatable, floor, top } => write!( + Self::Undeclared => write!( f, - "this machine's units translate {translatable} bits, whose device window would \ - start at {floor:#x}, at or below the {top:#x} its memory reaches" - ), - Self::Undeclared { translatable, cpu } => write!( - f, - "this machine's units translate {translatable} bits and its CPUs issue {cpu}, so a \ - domain has to end under the root bridges' windows, and no declaration of them \ - is held" - ), - Self::NoRoom { floor, ceiling, room } => write!( - f, - "a domain's addresses from {floor:#x} end at {ceiling:#x}, where a root bridge's \ - window or a reserved region begins, short of the {room:#x} bytes asked of it" + "no declaration of the root bridges' windows is held, so no device address is known \ + to be clear of them" ), + Self::NoRoom { room } => { + write!(f, "less than the {room:#x} bytes a new domain hands out first is allowed below 4 GiB") + } Self::Unaligned(at) => write!(f, "{at:#x} is not a 2 MiB boundary"), Self::NotMapped(at) => write!(f, "{:#x} is not mapped in this domain", at.raw()), } @@ -200,17 +187,19 @@ impl DeviceSpace { } } - /// Put `bytes` at `phys` in this space; returns what to program the device - /// with. + /// Put `bytes` at `phys` in this space, ending at or below `limit`, the + /// device's own address width; returns what to program the device with. /// /// Read and write both, always: nothing here can give a permission set a /// second value. The only leaf is 2 MiB, coarser than any split a driver's /// pools offer, and QEMU drops an access its cached translation denies /// rather than recording a fault — unexpressible and unobservable both. - pub fn map(self, phys: u64, bytes: u64) -> Result { + pub fn map(self, phys: u64, bytes: u64, limit: Limit) -> Result { match self { + // The pages are the address, so nothing chooses one the device reaches. + Self::Untranslated if phys.saturating_add(bytes) > limit.end() => Err(IommuError::AddressesExhausted(limit)), Self::Untranslated => Ok(phys), - Self::Own(id) => unit::domain::map(id, phys, bytes).map(Iova::raw), + Self::Own(id) => unit::domain::map(id, phys, bytes, limit).map(Iova::raw), } } @@ -238,16 +227,16 @@ impl DeviceSpace { pub struct OwnSpace(DomainId); impl OwnSpace { - /// One with `room` bytes of it handed out at the address answered, or the - /// reason there is none: a machine with no unit and a machine out of - /// domains are both refusals. + /// One with `room` bytes of it below 4 GiB handed out at the address + /// answered, or the reason there is none: a machine with no unit and a + /// machine out of domains are both refusals. pub fn create(room: u64) -> Result<(Self, u64), IommuError> { unit::domain::create(room).map(|(id, at)| (Self(id), at.raw())) } /// [`DeviceSpace::map`]. - pub fn map(self, phys: u64, bytes: u64) -> Result { - unit::domain::map(self.0, phys, bytes).map(Iova::raw) + pub fn map(self, phys: u64, bytes: u64, limit: Limit) -> Result { + unit::domain::map(self.0, phys, bytes, limit).map(Iova::raw) } /// Put `bytes` at `phys` at `at` again, where a device may still be aimed @@ -294,7 +283,6 @@ pub fn init( devices: &[crate::drivers::pci::PciDevice], windows: Option<&toyos_pci::aperture::Declaration>>, ) { - log!("iommu: the CPUs issue {}-bit physical addresses", crate::arch::cpu::physical_address_bits()); unit::init(rsdp_addr, devices, windows.map(|declared| declared.windows())); } diff --git a/kernel/src/iommu/window.rs b/kernel/src/iommu/window.rs deleted file mode 100644 index 4392aeec9e5..00000000000 --- a/kernel/src/iommu/window.rs +++ /dev/null @@ -1,188 +0,0 @@ -//! A domain's device addresses — where they start, where they end and which -//! of them are handed out — for every backend. -//! -//! They lie above all memory, so a descriptor still carrying one names -//! nothing a domain maps and faults rather than landing on a page, and clear -//! of every root-bridge window: a bridge may route a window peer-to-peer -//! before the unit sees the request (PCIe Base §2.4). Where the unit -//! translates past every address the CPU issues, they lie above every such -//! address, and nothing needs declaring. Where it translates no more bits -//! than the CPU's physical addresses, whatever the backend, they start a -//! quarter of the way up and end under the first declared window or region -//! firmware reserved for a device (an RMRR or IVMD) over that start, and with -//! no declaration accepted there is no domain. An address is handed out once, -//! unmapped or not: a device holding a stale one would reach whatever took -//! its place. - -use crate::iommu::{IommuError, Iova}; -use crate::mm::PAGE_2M; - -/// One domain's device addresses. -#[derive(Clone, Copy)] -pub struct Window { - floor: u64, - ceiling: u64, - /// The first not yet handed out. - next: u64, -} - -/// A quarter of the way up `translatable` bits of device address. -pub const fn first_address(translatable: u8) -> u64 { - 1 << (translatable - 2) -} - -/// Where a window over `translatable` bits starting at `floor` ends: under the -/// first of `reserved` that reaches above `floor`; at or below `floor` where -/// one of them covers it. -const fn ceiling(translatable: u8, floor: u64, reserved: &[(u64, u64)]) -> u64 { - let mut ceiling = 1u64 << translatable; - let mut i = 0; - while i < reserved.len() { - let (start, end) = reserved[i]; - if end > floor && start < ceiling { - ceiling = start; - } - i += 1; - } - ceiling -} - -/// Where a domain over `translatable` bits begins and ends on a CPU issuing -/// `cpu`-bit physical addresses, `reserved` the declared root-bridge windows -/// and firmware's reserved regions, or `None` where no declaration is held. -const fn bounds(translatable: u8, cpu: u8, reserved: Option<&[(u64, u64)]>) -> Result<(u64, u64), IommuError> { - if translatable > cpu { - return Ok((1 << cpu, 1 << translatable)); - } - let floor = first_address(translatable); - match reserved { - Some(reserved) => Ok((floor, ceiling(translatable, floor, reserved))), - None => Err(IommuError::Undeclared { translatable, cpu }), - } -} - -impl Window { - /// A new domain's window over `translatable` bits ([`bounds`]), with - /// `room` bytes of it handed out first; refused where it would start at or - /// below memory or hold less than that, and at least one leaf. - pub fn new(translatable: u8, reserved: Option<&[(u64, u64)]>, room: u64) -> Result<(Self, Iova), IommuError> { - let (floor, ceiling) = bounds(translatable, crate::arch::cpu::physical_address_bits(), reserved)?; - let top = crate::mm::pmm::top(); - if floor <= top { - return Err(IommuError::WindowBelowMemory { translatable, floor, top }); - } - let mut window = Self::starting_at(floor, ceiling); - match window.reserve(room) { - Some(first) if ceiling.saturating_sub(floor) >= PAGE_2M => Ok((window, first)), - _ => Err(IommuError::NoRoom { floor, ceiling, room }), - } - } - - const fn starting_at(floor: u64, ceiling: u64) -> Self { - Self { floor, ceiling, next: floor } - } - - pub const fn floor(&self) -> u64 { - self.floor - } - - pub const fn ceiling(&self) -> u64 { - self.ceiling - } - - /// `bytes`, rounded up to whole 2 MiB leaves, of addresses never handed out. - pub const fn reserve(&mut self, bytes: u64) -> Option { - let Some(span) = bytes.checked_next_multiple_of(PAGE_2M) else { return None }; - let Some(end) = self.next.checked_add(span) else { return None }; - if end > self.ceiling { - return None; - } - let at = Iova::translated(self.next); - self.next = end; - Some(at) - } - - /// Whether `bytes` at `at` is room [`Self::reserve`] already handed out: - /// the only room a mapping may be placed in by address. - /// - /// `at` must itself be a leaf `reserve` could have returned, not merely - /// inside a handed-out span: a leaf's index floors to the enclosing leaf, - /// so an unaligned `at` this let through would place a mapping short of, - /// or overlapping, where the caller named. - pub const fn handed_out(&self, at: Iova, bytes: u64) -> bool { - if !at.raw().is_multiple_of(PAGE_2M) || at.raw() < self.floor { - return false; - } - let Some(span) = bytes.checked_next_multiple_of(PAGE_2M) else { return false }; - match at.raw().checked_add(span) { - Some(end) => end <= self.next, - None => false, - } - } -} - -/// [`Window`] and [`ceiling`] at their boundaries, at compile time: the -/// kernel binary has no test harness. -const _: () = { - const FLOOR: u64 = first_address(48); - let mut one = Window::starting_at(FLOOR, FLOOR + 2 * PAGE_2M); - assert!(matches!(one.reserve(1), Some(at) if at.raw() == FLOOR)); - // Exactly what one `reserve` handed out. - assert!(one.handed_out(Iova::translated(FLOOR), PAGE_2M)); - // Short of the floor, past what has been reserved, rounded up past it, - // off a leaf boundary, and past the end of the address space: nothing. - assert!(!one.handed_out(Iova::translated(FLOOR - PAGE_2M), PAGE_2M)); - assert!(!one.handed_out(Iova::translated(FLOOR + PAGE_2M), PAGE_2M)); - assert!(!one.handed_out(Iova::translated(FLOOR), PAGE_2M + 1)); - assert!(!one.handed_out(Iova::translated(FLOOR + 1), 0)); - assert!(!one.handed_out(Iova::translated(u64::MAX - PAGE_2M + 1), PAGE_2M)); - assert!(!one.handed_out(Iova::translated(FLOOR), u64::MAX)); - // Past the ceiling, and past the end of the address space, nothing. - assert!(one.reserve(PAGE_2M + 1).is_none()); - assert!(one.reserve(u64::MAX).is_none()); - assert!(matches!(one.reserve(PAGE_2M), Some(at) if at.raw() == FLOOR + PAGE_2M)); - // Two leaves handed out; the second is room in its own right, and a - // mid-span address off a leaf boundary is not. - assert!(one.handed_out(Iova::translated(FLOOR + PAGE_2M), PAGE_2M)); - assert!(!one.handed_out(Iova::translated(FLOOR + 1), PAGE_2M)); - assert!(one.reserve(1).is_none()); -}; - -/// [`ceiling`] over the windows the T14's firmware declares, unsorted as it -/// declares them, and over `virt`'s: a 39-bit unit's window ends where the -/// first root-bridge window above its floor begins, one reaching over the -/// floor leaves it nothing, and windows all below the floor leave the whole -/// input. -const _: () = { - const FLOOR: u64 = first_address(39); - const T14: [(u64, u64); 7] = [ - (0xA200_0000, 0xBD00_0000), - (0x40_0000_0000, 0x60_3DC0_0000), - (0xA080_0000, 0xA200_0000), - (0xBD00_0000, 0xC000_0000), - (0xFF00_0000, 0xFFB8_0000), - (0xFFD3_A070, 0x1_0000_0000), - (0x60_3DC0_0000, 0x80_0000_0000), - ]; - assert!(FLOOR == 0x20_0000_0000); - assert!(ceiling(39, FLOOR, &T14) == 0x40_0000_0000); - assert!(ceiling(39, FLOOR, &[]) == 1 << 39); - assert!(ceiling(39, FLOOR, &[(0x10_0000_0000, FLOOR + 1)]) < FLOOR); - const VIRT: [(u64, u64); 2] = [(0x1000_0000, 0x3f00_0000), (0x80_0000_0000, 0x100_0000_0000)]; - assert!(ceiling(48, first_address(48), &VIRT) == 1 << 48); -}; - -/// [`bounds`] on each kind of machine: a unit wider than the CPU's addresses -/// places above them, declared or not; a unit no wider places under its -/// first declared window, and with nothing declared nowhere. -const _: () = { - const T14: [(u64, u64); 1] = [(0x40_0000_0000, 0x60_3DC0_0000)]; - assert!(matches!(bounds(48, 40, None), Ok((0x100_0000_0000, 0x1_0000_0000_0000)))); - assert!(matches!(bounds(48, 40, Some(&T14)), Ok((0x100_0000_0000, 0x1_0000_0000_0000)))); - assert!(matches!(bounds(39, 39, Some(&T14)), Ok((0x20_0000_0000, 0x40_0000_0000)))); - assert!(matches!(bounds(39, 39, None), Err(IommuError::Undeclared { translatable: 39, cpu: 39 }))); - assert!(matches!(bounds(39, 46, None), Err(IommuError::Undeclared { .. }))); - // The AMD bench: 48-bit host tables on a CPU of 48, all its windows below - // the quarter floor, so the whole of the upper quarter. - assert!(matches!(bounds(48, 48, Some(&[(0x4_5020_0000, 0xfc_9000_0000)])), Ok((0x4000_0000_0000, 0x1_0000_0000_0000)))); -}; diff --git a/kernel/src/mm/dma.rs b/kernel/src/mm/dma.rs index 0081a1370ff..66547e1ee22 100644 --- a/kernel/src/mm/dma.rs +++ b/kernel/src/mm/dma.rs @@ -8,7 +8,7 @@ use alloc::vec::Vec; use core::marker::PhantomData; use core::ptr::{copy_nonoverlapping, read_unaligned, read_volatile, write_bytes, write_volatile}; -use crate::iommu::DeviceSpace; +use crate::iommu::{DeviceSpace, IommuError, Limit}; use super::pmm::PhysPage; use super::DirectMap; @@ -208,19 +208,23 @@ pub struct DmaPool { impl DmaPool { /// Take enough contiguous 2 MiB pages to cover `size` bytes, mapped into - /// `space` and reachable by nothing outside it. - pub fn alloc_in(size: usize, space: DeviceSpace) -> Self { + /// `space` below `limit`, the device's address width, and reachable by + /// nothing outside it; refused where none below it is left — for an + /// untranslated device, where the pages lie above it. + pub fn alloc_in(size: usize, space: DeviceSpace, limit: Limit) -> Result { let pages_2m = size.div_ceil(super::PAGE_2M as usize); let pages = super::pmm::alloc_contiguous(pages_2m) .expect("DmaPool: out of physical memory"); let base = pages[0].direct_map(); let size = pages_2m * super::PAGE_2M as usize; - // A domain this kernel cannot map into is a kernel bug, not a device's - // answer: the addresses are the allocator's own and the pages are 2 MiB. - let device = space - .map(base.phys(), size as u64) - .unwrap_or_else(|why| panic!("DmaPool: {why}")); - Self { pages, base, size, space, device } + // A device's width is its answer; any other refusal is a kernel bug: + // the addresses are the allocator's own and the pages are 2 MiB. + let device = match space.map(base.phys(), size as u64, limit) { + Ok(device) => device, + Err(why @ IommuError::AddressesExhausted(_)) => return Err(why), + Err(why) => panic!("DmaPool: {why}"), + }; + Ok(Self { pages, base, size, space, device }) } /// The whole pool as a view, borrowed so it cannot outlive `self`. diff --git a/kernel/src/pcidev/mod.rs b/kernel/src/pcidev/mod.rs index 4fe181b4685..fda9fd42685 100644 --- a/kernel/src/pcidev/mod.rs +++ b/kernel/src/pcidev/mod.rs @@ -51,17 +51,25 @@ //! interrupts this machine does not remap**, because its message would be //! compatibility format, which raises any vector on any CPU. //! -//! **A function masters the bus only once it has memory it may reach.** What -//! comes back from a process still holds the device addresses of a domain that -//! no longer maps them, so bus mastering is not started at hand-over: it starts -//! on the claim's first grant, after that grant is in the function's domain. +//! **Every grant ends below the address limit its holder stated** +//! ([`dma_limit`]), and no grant or lend is made before one is: a function +//! that drops its upper address bits would issue a grant above its limit as +//! another grant's address. +//! +//! **A function masters the bus only when its holder starts it** +//! ([`dma_start`]), never at hand-over: what comes back from a process, or +//! from firmware, may still hold addresses and run queues. Its holder has the +//! registers first, with mastering off, to stop and reset the function by its +//! own means and lay out its grants; then it starts it. The kernel takes that +//! as no proof of anything: whatever the function then does reaches only its +//! domain. //! //! **A function no mechanism resets is still aimed at what its last holder //! granted.** Nothing but that function's own driver can stop its queues, and //! no register write retracts a transfer it had taken in before its holder -//! died: the first grant that starts it mastering again lets that transfer out. +//! died: the start that lets it master again lets that transfer out. //! So [`release`] takes those grants back like any other — mastering is off, so -//! the function reaches nothing until its next claim's first grant — and keeps +//! the function reaches nothing until its next holder starts it — and keeps //! only their device addresses, as the function's [`Function::residue`], for //! the rest of the boot if it is never claimed again. A residue address is //! never lent ([`Bound::lend_room`]) and never a fresh grant's, so the next @@ -121,7 +129,7 @@ use toyos_pci::{af, aperture, bar, express, pages, pm}; use crate::device::{Claim, ClaimError}; use crate::drivers::pci::{arm_claimed_msi, arm_claimed_msix, mask_claimed_msix, NoCapability, NoEntry, PciDevice}; -use crate::iommu::{IommuError, OwnSpace, Refused, Remapped}; +use crate::iommu::{IommuError, Limit, OwnSpace, Refused, Remapped}; use crate::mm::policy::CachePolicy; use crate::mm::{align_2m, DirectMap, PAGE_2M}; use crate::object::shm::{Region, SharedMemObject}; @@ -280,9 +288,13 @@ struct Bound { /// to nothing and counted against nothing, and where a grant of a range's /// size is placed. residue: Vec, - /// Whether this function may issue a transaction yet. False until its first - /// grant is in its domain, so a function carrying a previous holder's queue - /// addresses can act on none of them. + /// What the holder stated the function issues ([`dma_limit`]); no grant + /// or lend is made before it is. + limit: Option, + /// Whether this function may issue a transaction yet: false until its + /// holder starts it ([`dma_start`]), so a function carrying a previous + /// holder's or firmware's queue addresses acts on none of them before its + /// holder has stopped it. mastering: bool, } @@ -302,9 +314,7 @@ impl Bound { .find(|at| !taken(*at)) } - /// Let the function master the bus, once: called after a grant is in its - /// domain and never before, so the first thing it may reach exists before - /// it may reach anything. + /// Let the function master the bus, once. fn start_mastering(&mut self) { if !self.mastering { self.pci.start_bus_mastering(); @@ -646,6 +656,9 @@ enum Refusal { BarUnrouted(u8), /// This BAR's pages are not its own ([`pages::of_bar`]). BarPages(u8, pages::Refused), + /// With mastering stopped, its Transactions Pending stayed set past + /// [`express::QUIESCE_NANOS`]. + StillTransacting, } impl core::fmt::Display for Refusal { @@ -712,6 +725,12 @@ impl core::fmt::Display for Refusal { read of it would come back" ), Self::BarPages(i, why) => write!(f, "BAR {i}: {why}"), + Self::StillTransacting => write!( + f, + "with its mastering stopped, a request it had issued was still outstanding after {} ms, \ + and it would land in a domain that maps none of it", + express::QUIESCE_NANOS / 1_000_000 + ), } } } @@ -805,11 +824,8 @@ pub fn claim(arrival: Arrival) -> Result<(PciFunctionInfo, Claim), ClaimError> { } /// Choose the BARs its holder is given, give the function an address space, -/// arm the interrupt, and only then let it master the bus. -/// -/// The order is the whole security argument: a function that could master the -/// bus before its domain existed would be reaching physical memory with -/// whatever addresses its registers still held. +/// arm the interrupt, stop its mastering and let what it had in flight finish, +/// then attach the domain. Its holder starts it ([`dma_start`]). fn bring_up(pci: PciDevice, id: PciId, number: usize, windows: &[RootBridgeWindow]) -> Result { let remapping = crate::iommu::remapping().map_err(Refusal::NotRemapped)?; crate::iommu::isolates(&pci).map_err(Refusal::NotIsolated)?; @@ -844,6 +860,7 @@ fn bring_up(pci: PciDevice, id: PciId, number: usize, windows: &[RootBridgeWindo Err(NoEntry::NoTable(NoCapability::Absent)) if arm_claimed_msi(&message) => Armed::Msi(message), Err(NoEntry::NoTable(NoCapability::Absent)) => return Err(Refusal::NoInterrupt), }; + quiesce(&pci)?; space.attach(pci.bus, pci.dev, pci.func); Ok(Bound { pci, @@ -855,6 +872,7 @@ fn bring_up(pci: PciDevice, id: PciId, number: usize, windows: &[RootBridgeWindo bar_bytes, grants: Vec::new(), residue: Vec::new(), + limit: None, mastering: false, }) } @@ -1122,6 +1140,41 @@ fn settle_after_reset(pci: &PciDevice) { kept.restore(pci); } +/// Stop a function mastering and let everything it already issued finish +/// before it moves onto a domain that maps none of those addresses. Firmware +/// may have left it running queues at physical addresses, and what lands after +/// the attach would fault against its holder. +/// +/// Clearing Bus Master Enable stops new requests. A config read's completion +/// passes none of the function's earlier posted writes (PCIe Base §2.4.1), but +/// Non-Posted Requests, its DMA reads, are only known finished once Transactions +/// Pending clears: Device Status' (§7.5.3.5), or on a conventional function +/// the Advanced Features capability's, under the same bound as a hang ceiling, +/// conventional PCI having no Completion Timeout. A function with neither has +/// no pending signal: its posted writes are flushed, and a read it still had in +/// flight may fault in the new domain, against its holder; isolation holds, +/// since that domain maps only the function's own. +fn quiesce(pci: &PciDevice) -> Result<(), Refusal> { + pci.disable_bus_master(); + pci.drain_messages(); + let pending: Box bool + '_> = match (pci.capability(express::CAP_ID), pci.capability(af::CAP_ID)) { + (Err(NoCapability::Truncated), _) | (_, Err(NoCapability::Truncated)) => return Err(Refusal::CapsTruncated), + (Ok(cap), _) => Box::new(move || express::pending(cap.read_u16(express::DEVICE_STATUS))), + (Err(NoCapability::Absent), Ok(cap)) if af::reports_pending(cap.read_u8(af::AF_CAPABILITIES)) => { + Box::new(move || af::pending(cap.read_u8(af::AF_STATUS))) + } + (Err(NoCapability::Absent), _) => return Ok(()), + }; + let deadline = crate::clock::nanos_since_boot() + express::QUIESCE_NANOS; + while pending() { + if crate::clock::nanos_since_boot() >= deadline { + return Err(Refusal::StillTransacting); + } + core::hint::spin_loop(); + } + Ok(()) +} + fn wait_until(at: u64) { while crate::clock::nanos_since_boot() < at { core::hint::spin_loop(); @@ -1233,7 +1286,7 @@ fn tear_down(function: &Function, mut bound: Bound) { } // Nothing reset it, so whatever it had taken in is still aimed at these // addresses; mastering is off, so it reaches none of them until the - // next claim's first grant. + // next holder starts it. None => { for grant in grants.iter() { if let Err(why) = bound.space.unmap(grant.at, grant.bytes) { @@ -1307,6 +1360,7 @@ pub fn dma_alloc( ) -> Result<(Arc, u64, u64), SyscallError> { let function = binding.function(); with_bound(binding, |bound| { + let limit = bound.limit.ok_or(SyscallError::InvalidArgument)?; if bytes == 0 || bytes > MAX_GRANT_BYTES { return Err(SyscallError::InvalidArgument); } @@ -1325,7 +1379,7 @@ pub fn dma_alloc( // The pages are the allocator's own and 2 MiB, so a domain that cannot // take them is a kernel bug rather than the device's answer. let refused = |why| -> ! { panic!("pcidev: {function} could not map a grant: {why}") }; - let (at, residual) = match bound.residue.iter().position(|aimed| aimed.bytes == span) { + let (at, residual) = match bound.residue.iter().position(|aimed| aimed.bytes == span && aimed.at + span <= limit.end()) { // The function may still be aimed here, so this holder is the one // whatever it had in flight lands on. Some(index) => { @@ -1335,7 +1389,7 @@ pub fn dma_alloc( } // A domain its claims have spent is the machine's limit and not // a kernel bug: the holder is refused, and the pages go back. - None => match bound.space.map(phys, span) { + None => match bound.space.map(phys, span, limit) { Ok(at) => (at, false), Err(IommuError::AddressesExhausted(_)) => return Err(SyscallError::ResourceExhausted), Err(why) => refused(why), @@ -1343,13 +1397,37 @@ pub fn dma_alloc( }; let origin = Origin::Allocated { residual }; bound.grants.push(Grant { memory: Arc::clone(&memory), at, bytes: span, origin }); - // After the mapping and never before: the first thing this function may - // reach has to exist before it may reach anything. - bound.start_mastering(); Ok((memory, at, span)) }) } +/// The function issues device addresses of `bits` bits, as its holder states +/// it: once per claim and before any grant or lend, which are all placed +/// below it. `InvalidArgument` outside 32 to 64 bits, `AlreadyExists` for a +/// second statement. +pub fn dma_limit(binding: &Binding, bits: u64) -> Result<(), SyscallError> { + let limit = u8::try_from(bits).ok().and_then(Limit::bits).ok_or(SyscallError::InvalidArgument)?; + with_bound(binding, |bound| { + if bound.limit.is_some() { + return Err(SyscallError::AlreadyExists); + } + // `OwnSpace::create` hands the lend window out below 4 GiB, which + // every limit reaches. + assert!(bound.lend + LEND_WINDOW <= limit.end(), "pcidev: a lend window at {:#x} above 4 GiB", bound.lend); + bound.limit = Some(limit); + Ok(()) + }) +} + +/// Let the function master the bus, as its holder asks once it has stopped +/// and laid out what the function may reach; nothing on a second call. +pub fn dma_start(binding: &Binding) -> Result<(), SyscallError> { + with_bound(binding, |bound| { + bound.start_mastering(); + Ok(()) + }) +} + /// Take a grant back, for a caller that could not be given a handle to it. /// /// **A partial success is not left behind.** Without this the grant would stay @@ -1394,6 +1472,7 @@ pub fn dma_undo(binding: &Binding, memory: &Arc) { pub fn dma_map(binding: &Binding, region: &Arc) -> Result<(u64, u64), SyscallError> { let function = binding.function(); with_bound(binding, |bound| { + bound.limit.ok_or(SyscallError::InvalidArgument)?; let (phys, span) = region.ram().ok_or(SyscallError::InvalidArgument)?; // One region, one address: a second mapping of it would be two grants // naming one set of pages, and taking either back would leave the @@ -1408,7 +1487,6 @@ pub fn dma_map(binding: &Binding, region: &Arc) -> Result<(u64, panic!("pcidev: {function} could not lend a region at {at:#x}: {why}"); } bound.grants.push(Grant { memory: Arc::clone(region), at, bytes: span, origin: Origin::Mapped }); - bound.start_mastering(); Ok((at, span)) }) } diff --git a/kernel/src/syscall/device.rs b/kernel/src/syscall/device.rs index 30149fc3521..99b144cab9f 100644 --- a/kernel/src/syscall/device.rs +++ b/kernel/src/syscall/device.rs @@ -304,6 +304,30 @@ pub(super) fn sys_device_dma_unmap(handle: RawHandle, device_addr: u64) -> u64 { } } +/// State the address width the claimed function issues. +pub(super) fn sys_device_dma_limit(handle: RawHandle, bits: u64) -> u64 { + let claim = match pci_claim(handle) { + Ok(claim) => claim, + Err(e) => return e.refuse(), + }; + match on_function(&claim, |function| crate::pcidev::dma_limit(function, bits)) { + Ok(()) => 0, + Err(e) => e.to_u64(), + } +} + +/// Let the claimed function master the bus. +pub(super) fn sys_device_dma_start(handle: RawHandle) -> u64 { + let claim = match pci_claim(handle) { + Ok(claim) => claim, + Err(e) => return e.refuse(), + }; + match on_function(&claim, crate::pcidev::dma_start) { + Ok(()) => 0, + Err(e) => e.to_u64(), + } +} + /// The grant as the bytes that cross the boundary; its `const _` in `toyos-abi` /// proves the layout has no gap, so nothing of this kernel's stack goes with it. fn grant_bytes(grant: &toyos_abi::pci::DmaGrant) -> &[u8] { diff --git a/kernel/src/syscall/dispatch.rs b/kernel/src/syscall/dispatch.rs index 62d69217455..5174514df7d 100644 --- a/kernel/src/syscall/dispatch.rs +++ b/kernel/src/syscall/dispatch.rs @@ -23,7 +23,7 @@ use super::HANDLE_LEN; use super::debug::{canary, debug_heap_alloc, ring0_timer_in_syscall, FATAL_HALT_NONCE, LOCK_ACROSS_SWITCH}; use super::device::{ sys_acpi, sys_device_bar_map, sys_device_claim, sys_device_dma_alloc, sys_device_dma_map, - sys_device_dma_unmap, + sys_device_dma_limit, sys_device_dma_start, sys_device_dma_unmap, sys_device_reg_read, sys_partition_transfer, Transfer, }; use super::fs::{ @@ -390,6 +390,8 @@ pub(crate) fn syscall_dispatch(num: u64, a1: u64, a2: u64, a3: u64, a4: u64) -> sys_device_dma_map(&ctx, RawHandle(a1 as u32), RawHandle(a2 as u32), out) } SYS_DEVICE_DMA_UNMAP => sys_device_dma_unmap(RawHandle(a1 as u32), a2), + SYS_DEVICE_DMA_LIMIT => sys_device_dma_limit(RawHandle(a1 as u32), a2), + SYS_DEVICE_DMA_START => sys_device_dma_start(RawHandle(a1 as u32)), SYS_SYMLINK => { let target = match ctx.user_str(UserAddr::new(a1), a2) { Ok(s) => s, Err(e) => return e.to_u64() }; let link = match ctx.user_str(UserAddr::new(a3), a4) { Ok(s) => s, Err(e) => return e.to_u64() }; diff --git a/lib/pci-claim/src/lib.rs b/lib/pci-claim/src/lib.rs index af3eb5a7c9c..90c28781c69 100644 --- a/lib/pci-claim/src/lib.rs +++ b/lib/pci-claim/src/lib.rs @@ -28,6 +28,26 @@ pub fn describe(dev: &PciDev) -> Result { dev.describe().map_err(KernelRefused::on("the claim's description")) } +/// The claim's function's vendor and device id, config space's first dword. +pub fn id(dev: &PciDev) -> Result<(u16, u16), KernelRefused> { + let word = dev + .config_read(0, toyos_abi::syscall::RegWidth::U32) + .map_err(KernelRefused::on("the vendor and device id"))?; + Ok((word as u16, (word >> 16) as u16)) +} + +/// State the address width the claim's function issues (`PciDev::dma_limit`), +/// before its first grant or lend. +pub fn limit(dev: &PciDev, bits: u8) -> Result<(), KernelRefused> { + dev.dma_limit(bits).map_err(KernelRefused::on("the address limit")) +} + +/// Let the claim's function master the bus (`PciDev::dma_start`), once its +/// driver has stopped and reset it. +pub fn start(dev: &PciDev) -> Result<(), KernelRefused> { + dev.dma_start().map_err(KernelRefused::on("the bus-master start")) +} + /// `region`, which a peer lent, put into the claim's function's domain /// (`PciDev::dma_map`). pub fn lend(dev: &PciDev, region: RawHandle) -> Result { diff --git a/lib/pci-claim/src/virtio.rs b/lib/pci-claim/src/virtio.rs index 75b1461f308..ea7b63250f8 100644 --- a/lib/pci-claim/src/virtio.rs +++ b/lib/pci-claim/src/virtio.rs @@ -1,6 +1,7 @@ //! A virtio function's claim, brought to the features its driver and device //! agreed: the capability walk, the register window, and §3.1.1 up to -//! `FEATURES_OK`, in the order `toyos-virtio`'s types fix. +//! `FEATURES_OK`, in the order `toyos-virtio`'s types fix; then its 64-bit +//! address width stated and the function started. //! //! What is the device type's own — its queues, their vectors, its //! configuration — begins at the [`Setup`] this answers. @@ -8,7 +9,9 @@ use toyos::shm::SharedMemory; use toyos::PciDev; use toyos_abi::syscall::RegWidth; -use toyos_virtio::pci::{vendor_caps, ConfigSpace, Layout, Offer, Setup, WalkRefusal, VIRTIO_F_ACCESS_PLATFORM}; +use toyos_virtio::pci::{ + offered_features, vendor_caps, ConfigSpace, Layout, Offer, Setup, WalkRefusal, VIRTIO_F_ACCESS_PLATFORM, +}; use crate::{Bar, KernelRefused, Why}; @@ -94,6 +97,17 @@ impl std::fmt::Display for Features { } } +/// Whether the claim's virtio function offers `VIRTIO_F_ACCESS_PLATFORM`, +/// read without resetting it. One that does not uses physical addresses +/// whatever translates the bus (virtio 1.2 §6.1): no domain confines it. +/// +/// Not the claim's description, which is its holder's to read once. +pub fn offers_access_platform(dev: &PciDev) -> Result { + let layout = Layout::of(&vendor_caps(&ClaimConfig(dev)).map_err(Refusal::Walk)?)?; + let (window, _mapping) = Bar::map(dev, u32::from(layout.bar()), layout.common_end())?; + Ok(offered_features(&window, &layout)? & VIRTIO_F_ACCESS_PLATFORM != 0) +} + /// Walk the claim's capabilities, map the BAR they name, and negotiate /// `wanted` beside what `toyos-virtio` accepts wherever it is offered. /// @@ -112,6 +126,10 @@ pub fn negotiate(dev: &PciDev, wanted: u64) -> Result { let offer = Offer::acknowledge(window, &layout)?; let offered = offer.features(); let setup = offer.accept(wanted)?; + // Every address a virtqueue carries is 64 bits (virtio 1.2 §2.7.5), and + // the device just reset (§3.1.1) uses none before `DRIVER_OK`. + crate::limit(dev, 64)?; + crate::start(dev)?; let features = Features { at: (info.bus, info.dev, info.func), offered, negotiated: setup.features() }; Ok(Negotiated { setup, mapping, features }) } diff --git a/lib/virtio/src/pci.rs b/lib/virtio/src/pci.rs index 2abff7bdd51..0cb974c7d25 100644 --- a/lib/virtio/src/pci.rs +++ b/lib/virtio/src/pci.rs @@ -301,6 +301,12 @@ impl Layout { pub fn bar(&self) -> u8 { self.common.bar } + + /// How much of [`Self::bar`] a window needs for [`offered_features`]: up + /// to the end of the common structure. + pub fn common_end(&self) -> u64 { + u64::from(self.common.offset) + u64::from(self.common.length) + } } #[derive(Clone, Copy)] @@ -367,6 +373,18 @@ impl Wires { } } +/// Every feature bit the device behind `layout` offers, read with nothing +/// written but the feature select (§4.1.4.3): whoever hands a function over +/// asks it before its driver resets it. +pub fn offered_features(regs: &R, layout: &Layout) -> Result { + let common = Region::of(&layout.common, "COMMON_CFG", regs.bytes(), common::BYTES, 4)?; + regs.write32(common.at + common::DEVICE_FEATURE_SELECT, 0); + let low = regs.read32(common.at + common::DEVICE_FEATURE); + regs.write32(common.at + common::DEVICE_FEATURE_SELECT, 1); + let high = regs.read32(common.at + common::DEVICE_FEATURE); + Ok((high as u64) << 32 | low as u64) +} + /// A device reset and acknowledged, and the features it offers. pub struct Offer { wires: Wires, diff --git a/lib/virtio/src/tests.rs b/lib/virtio/src/tests.rs index aa4fa463116..c19d0c0881c 100644 --- a/lib/virtio/src/tests.rs +++ b/lib/virtio/src/tests.rs @@ -6,6 +6,7 @@ use std::vec::Vec; use crate::pci::{ + offered_features, status, Layout, Live, Offer, Refusal, Setup, Source, VendorCap, WalkRefusal, VIRTIO_F_ACCESS_PLATFORM, VIRTIO_F_VERSION_1, }; @@ -1080,3 +1081,19 @@ fn a_part_off_its_alignment_is_the_drivers_mistake() { let machine = Machine::new(); let _ = Virtqueue::new(machine, 0, SIZE, Parts { desc: 0, avail: 128, used: 154 }); } + +/// The offer is read without the device being reset or told anything: its +/// status is untouched, and a device that withholds `ACCESS_PLATFORM` is +/// seen to. +#[test] +fn the_offer_is_read_without_touching_the_device() { + let machine = Machine::new(); + assert_eq!( + offered_features(&machine, &layout(&machine)), + Ok(VIRTIO_F_VERSION_1 | VIRTIO_F_ACCESS_PLATFORM | FEATURE_OFFERED) + ); + assert_eq!(machine.statuses(), Vec::::new()); + machine.state(|s| s.offered &= !VIRTIO_F_ACCESS_PLATFORM); + let offered = offered_features(&machine, &layout(&machine)).expect("read"); + assert_eq!(offered & VIRTIO_F_ACCESS_PLATFORM, 0); +} diff --git a/sdk/abi/src/pci.rs b/sdk/abi/src/pci.rs index cfe406d5a83..c8eb921c79b 100644 --- a/sdk/abi/src/pci.rs +++ b/sdk/abi/src/pci.rs @@ -8,12 +8,15 @@ //! claim handle. Nothing here is a physical address, and nothing here is //! specific to what the function *is*. //! -//! **The function masters the bus from its first grant and not from the -//! hand-over**, so a driver takes its window and its first grant before the -//! device can do anything at all, an interrupt included. +//! **The function masters the bus from [`syscall::device_dma_start`] and not +//! from the hand-over**, so a driver stops and resets it through its window, +//! states its address width ([`syscall::device_dma_limit`]) and takes its +//! grants before the device can do anything at all, an interrupt included. //! //! [`syscall::device_dma_alloc`]: crate::syscall::device_dma_alloc //! [`syscall::device_dma_map`]: crate::syscall::device_dma_map +//! [`syscall::device_dma_start`]: crate::syscall::device_dma_start +//! [`syscall::device_dma_limit`]: crate::syscall::device_dma_limit /// The six BAR slots a Type 0 PCI header has (PCI 3.0 §6.1), and the bound /// every index in this module is checked against. diff --git a/sdk/abi/src/syscall.rs b/sdk/abi/src/syscall.rs index ed43c04c981..c272cdc8840 100644 --- a/sdk/abi/src/syscall.rs +++ b/sdk/abi/src/syscall.rs @@ -289,6 +289,11 @@ pub const SYS_HANDLE_DUP_SLOT: u64 = 129; /// A fresh null handle: every read answers end of file and every write takes /// all it is given. See [`null`]. pub const SYS_NULL: u64 = 130; +/// State the address width a claimed PCI function issues. See +/// [`device_dma_limit`]. +pub const SYS_DEVICE_DMA_LIMIT: u64 = 131; +/// Let a claimed PCI function master the bus. See [`device_dma_start`]. +pub const SYS_DEVICE_DMA_START: u64 = 132; /// Bins in the per-process syscall profile — one for every number this ABI /// issues, and one at the end for every number it does not. @@ -296,13 +301,13 @@ pub const SYS_NULL: u64 = 130; /// **The profile's parts sum to its total, and that is the whole requirement.** /// A bin array narrower than the ABI reaches drops calls out of the line while /// the total goes on counting them. -pub const SYSCALL_PROFILE_BINS: usize = 132; +pub const SYSCALL_PROFILE_BINS: usize = 134; /// Where a number this ABI does not issue is counted. Merging is a degradation /// a reader can see in the line; dropping is one nobody can. pub const SYSCALL_PROFILE_OTHER: usize = SYSCALL_PROFILE_BINS - 1; -const _: () = assert!(SYS_NULL < SYSCALL_PROFILE_OTHER as u64); +const _: () = assert!(SYS_DEVICE_DMA_START < SYSCALL_PROFILE_OTHER as u64); pub const WNOHANG: u64 = 1; @@ -2311,8 +2316,8 @@ pub fn pipe_map(handle: RawHandle) -> Result<*mut u8, SyscallError> { /// Every call answers an object of its own over the same window, so a BAR /// whose handle was closed can be asked for again. /// -/// A window alone drives nothing: the function masters the bus from its first -/// [`device_dma_alloc`] and not before. +/// A window alone drives nothing: the function masters the bus from +/// [`device_dma_start`] and not before. pub fn device_bar_map(claim: RawHandle, bar: u32) -> Result { check(syscall(SYS_DEVICE_BAR_MAP, claim.0 as u64, bar as u64, 0, 0)) .map(|v| RawHandle(v as u32)) @@ -2323,15 +2328,9 @@ pub fn device_bar_map(claim: RawHandle, bar: u32) -> Result Result { let mut grant = DmaGrant { shm: HANDLE_INVALID, _pad: 0, device_addr: 0, bytes: 0 }; @@ -2345,6 +2344,30 @@ pub fn device_dma_alloc(claim: RawHandle, bytes: u64) -> Result Result<(), SyscallError> { + check_unit(syscall(SYS_DEVICE_DMA_LIMIT, claim.0 as u64, bits as u64, 0, 0)) +} + +/// Let the function `claim` holds master the bus; nothing on a second call. +/// +/// **The function issues no bus transaction at all, an MSI-X message +/// included, until this is called.** A claim is handed over with Bus Master +/// Enable clear, because a previous holder or firmware may have left it +/// running queues at addresses its domain no longer maps; its driver stops and +/// resets it through its registers first, lays out its grants, and then starts +/// it. What it does from then on reaches only what its domain maps. +pub fn device_dma_start(claim: RawHandle) -> Result<(), SyscallError> { + check_unit(syscall(SYS_DEVICE_DMA_START, claim.0 as u64, 0, 0, 0)) +} + /// Put the region `shm` names into the address space of the function `claim` /// holds, and answer where the function reaches it: the region's whole length, /// read and write, and nothing on either side of it. @@ -2357,8 +2380,8 @@ pub fn device_dma_alloc(claim: RawHandle, bytes: u64) -> Result Result { let mut mapping = DmaMapping { device_addr: 0, bytes: 0 }; check_unit(syscall( diff --git a/sdk/toyos/src/device.rs b/sdk/toyos/src/device.rs index ad80f34fd1a..cef28b4156f 100644 --- a/sdk/toyos/src/device.rs +++ b/sdk/toyos/src/device.rs @@ -145,7 +145,26 @@ impl PciDev { crate::shm::SharedMemory::adopt(handle, bytes as usize) } - /// `bytes` of memory this function can reach and nothing else can. + /// Give up ownership, for a claim about to be endowed. + pub fn into_raw(self) -> RawHandle { + self.0.into_raw() + } + + /// State that this function issues `bits`-bit device addresses, before any + /// grant or lend: every one is placed below `1 << bits`. + pub fn dma_limit(&self, bits: u8) -> Result<(), SyscallError> { + syscall::device_dma_limit(self.0.as_handle(), bits) + } + + /// Let this function master the bus, once its driver has stopped it and + /// laid out what it may reach. Until then it issues nothing, an interrupt + /// included. + pub fn dma_start(&self) -> Result<(), SyscallError> { + syscall::device_dma_start(self.0.as_handle()) + } + + /// `bytes` of memory this function can reach and nothing else can, below + /// the width [`Self::dma_limit`] stated. pub fn dma_alloc(&self, bytes: u64) -> Result { let grant = syscall::device_dma_alloc(self.0.as_handle(), bytes)?; let memory = crate::shm::SharedMemory::adopt(grant.shm, grant.bytes as usize)?; diff --git a/system/hda/driver/src/controller.rs b/system/hda/driver/src/controller.rs index da72c90ba44..5529a472133 100644 --- a/system/hda/driver/src/controller.rs +++ b/system/hda/driver/src/controller.rs @@ -155,6 +155,44 @@ impl Gcap { } } +/// Functions whose GCAP 64OK a measurement contradicts, with where it is +/// recorded: no width they are known to reach, so none is driven. +const CONTRADICTED: &[(u16, u16, &str)] = &[( + 0x1002, + 0x1637, + "issues/the-hdmi-audio-function-1002-1637-is-driven-at-no-address-width.md", +)]; + +/// The device-address width a controller is driven at: 64 bits where GCAP +/// says 64OK (§3.3.2), 32 where it does not, and none where its 64OK is +/// contradicted. +pub fn address_bits(vendor: u16, device: u16, gcap: Gcap) -> Result { + match CONTRADICTED.iter().find(|(v, d, _)| (*v, *d) == (vendor, device)) { + Some(&(_, _, record)) => Err(Contradicted { vendor, device, record }), + None if gcap.wide() => Ok(64), + None => Ok(32), + } +} + +/// A function [`address_bits`] knows no width for. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub struct Contradicted { + pub vendor: u16, + pub device: u16, + pub record: &'static str, +} + +impl core::fmt::Display for Contradicted { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + write!( + f, + "{:04x}:{:04x} sets 64OK and is measured dropping address bits, so no width it reaches is \ + known ({})", + self.vendor, self.device, self.record + ) + } +} + /// Which ring a refusal is about. #[derive(Clone, Copy, PartialEq, Eq, Debug)] pub enum Ring { diff --git a/system/hda/driver/src/controller/tests.rs b/system/hda/driver/src/controller/tests.rs index ba7511a0859..c15b96e7773 100644 --- a/system/hda/driver/src/controller/tests.rs +++ b/system/hda/driver/src/controller/tests.rs @@ -684,3 +684,14 @@ fn a_stream_that_never_stops_is_refused() { stream.start().unwrap(); assert_eq!(stream.stop(), Err(StreamFault::NeverStopped)); } + +/// 64OK decides the width, except for a function a measurement contradicts, +/// which is driven at none. +#[test] +fn the_address_width_is_64ok_unless_it_is_contradicted() { + assert_eq!(address_bits(0x8086, 0xa0c8, Gcap(LAPTOP_GCAP)), Ok(64)); + assert_eq!(address_bits(0x8086, 0xa0c8, Gcap(LAPTOP_GCAP & !1)), Ok(32)); + assert_eq!(address_bits(0x1022, 0x15e3, Gcap(0x4401)), Ok(64)); + let refused = address_bits(0x1002, 0x1637, Gcap(0x4001)).unwrap_err(); + assert_eq!((refused.vendor, refused.device), (0x1002, 0x1637)); +} diff --git a/system/hda/src/main.rs b/system/hda/src/main.rs index 4abba76e46c..173d0d2ec78 100644 --- a/system/hda/src/main.rs +++ b/system/hda/src/main.rs @@ -58,6 +58,8 @@ enum Refusal { Kernel(KernelRefused), NoRegisters, Controller(controller::Refusal), + /// No width this function reaches is known. + Width(controller::Contradicted), Link(LinkFault), NoOutput(PathError), /// The converter behind the chosen pin offers none of soundserver's rates @@ -108,6 +110,7 @@ impl core::fmt::Display for Refusal { Self::Kernel(why) => write!(f, "{why}"), Self::NoRegisters => write!(f, "the claim gives its register BAR no window"), Self::Controller(why) => write!(f, "the controller refused: {why}"), + Self::Width(why) => write!(f, "{why}"), Self::Link(why) => write!(f, "the codec link refused: {why}"), Self::NoOutput(why) => { write!(f, "its codecs reach no output: ")?; @@ -208,11 +211,18 @@ impl Driver { return Err(Refusal::NoRegisters); } let (bar, bar_mapping) = Bar::map(&dev, REGISTERS, bar_bytes)?; + // With bus mastering still off: the reset stops every DMA engine + // before CRST (§3.3.7), so nothing firmware or a previous holder left + // running reaches the grants laid out below. let reset = controller::reset(&bar, || std::thread::sleep(Duration::from_micros(CODEC_DETECT_MICROS)))?; say!("{at}: gcap={:#06x} statests={:#06x}", reset.gcap.0, reset.statests); + let (vendor, device) = toyos_pci_claim::id(&dev)?; + let bits = controller::address_bits(vendor, device, reset.gcap).map_err(Refusal::Width)?; + toyos_pci_claim::limit(&dev, bits)?; let (grant, region) = Grant::alloc(&dev, GRANT_BYTES as u64)?; grant.window().zero(); + toyos_pci_claim::start(&dev)?; let mut link = Link::open(bar, grant, &reset)?; let codecs = walk(at, &mut link, reset.statests); if let Some(fault) = link.refused() { diff --git a/system/netstack/i219/src/lib.rs b/system/netstack/i219/src/lib.rs index dbd41d35c39..113b366195b 100644 --- a/system/netstack/i219/src/lib.rs +++ b/system/netstack/i219/src/lib.rs @@ -607,11 +607,11 @@ pub(crate) enum Whole { WithThePhy { after: Option }, } -/// Stop what a previous holder left the part doing, before this driver's first -/// grant lets it master the bus again: every interrupt masked, receive and -/// transmit off. +/// Stop what a previous holder left the part doing, before this driver starts +/// it mastering the bus again: every interrupt masked, receive and transmit +/// off. /// -/// **The function keeps its rings across a release**, and the grant is the +/// **The function keeps its rings across a release**, and the start is the /// instant it may use them — an enabled receive unit would write the next /// frame into the previous holder's descriptors. The kernel resets a released /// function where it advertises a reset; this is the driver not trusting that diff --git a/system/netstack/src/i219.rs b/system/netstack/src/i219.rs index 299b1fce47b..5a2548ceaca 100644 --- a/system/netstack/src/i219.rs +++ b/system/netstack/src/i219.rs @@ -132,11 +132,12 @@ fn registers(dev: &PciDev) -> Result<(Bar, SharedMemory), Opening> { /// Everything the claim is asked for before the part is reached, in the order /// it is asked: the register window, the part stopped, then one grant — as the -/// driver reaches its descriptors — with the two mappings they point into. +/// driver reaches its descriptors — with the two mappings they point into, and +/// the function started. fn granted(dev: &PciDev) -> Result<(Bar, Grant, SharedMemory, DmaRegion), Opening> { let (bar, mapped) = registers(dev)?; - // Before the grant: the grant is what starts the function mastering, and a - // part a previous holder left receiving would write into its descriptors. + // Before the start: a part a previous holder left receiving would write + // into its descriptors. // Said first: whether the kernel's release reset the part is read here, in // the receive and transmit enables it handed over. crate::say!( @@ -145,7 +146,10 @@ fn granted(dev: &PciDev) -> Result<(Bar, Grant, SharedMemory, DmaRegion), Openin bar.read32(toyos_i219::regs::TCTL) ); toyos_i219::quiesce(&bar); + // Every buffer address a descriptor carries is 64 bits (82574 §7.1.3, §7.2.10.1). + toyos_pci_claim::limit(dev, 64).map_err(Opening::Kernel)?; let (grant, region) = Grant::alloc(dev, toyos_i219::GRANT_BYTES).map_err(Opening::Kernel)?; + toyos_pci_claim::start(dev).map_err(Opening::Kernel)?; Ok((bar, grant, mapped, region)) } diff --git a/system/nvme/src/nvme.rs b/system/nvme/src/nvme.rs index 8a02edbfb1b..21a642cc591 100644 --- a/system/nvme/src/nvme.rs +++ b/system/nvme/src/nvme.rs @@ -239,6 +239,8 @@ impl Controller { if mqes < IO_DEPTH as u32 { return Err(Refusal::Geometry(format!("the controller's queues hold {mqes} entries"))); } + // Every address NVMe carries — queue bases and PRP entries — is 64 bits. + dev.dma_limit(64).map_err(|e| Refusal::Kernel("the address limit", e))?; let grant = dev.dma_alloc(GRANT_BYTES).map_err(|e| Refusal::Kernel("a DMA grant", e))?; // SAFETY: the grant is at least `GRANT_BYTES` and lives as long as // `grant`, which this struct holds. @@ -306,6 +308,8 @@ impl Controller { self.regs.write::(REG_CC, cc & !1); } self.settle(false, "CSTS.RDY clearing")?; + // Disabled, so it runs no queue a previous holder or firmware left. + self.dev.dma_start().map_err(|e| Refusal::Kernel("the bus-master start", e))?; self.dma.sub(OFF_ADMIN_SQ, PAGE).zero(); self.dma.sub(OFF_ADMIN_CQ, PAGE).zero(); self.admin = Queue::new(0, ADMIN_DEPTH, OFF_ADMIN_SQ, OFF_ADMIN_CQ); diff --git a/system/supervisor/Cargo.toml b/system/supervisor/Cargo.toml index b0cb8ddc01a..7d92ca048fa 100644 --- a/system/supervisor/Cargo.toml +++ b/system/supervisor/Cargo.toml @@ -25,6 +25,9 @@ toyos-gpt = { path = "../../lib/gpt" } toyos-blockring = { path = "../../lib/blockring" } # The slot table, read through a session where the block service serves it. diskserver = { path = "../diskserver" } +# Whether a virtio function offers `VIRTIO_F_ACCESS_PLATFORM`, read through +# its claim before the claim moves to its holder. +toyos-pci-claim = { path = "../../lib/pci-claim" } # How long a stop is prepared before the kernel is asked for it. toyos-quiesce = { path = "../../lib/quiesce" } diff --git a/system/supervisor/lifecycle/src/claims.rs b/system/supervisor/lifecycle/src/claims.rs index 47dc08b86e1..1e96d6c268c 100644 --- a/system/supervisor/lifecycle/src/claims.rs +++ b/system/supervisor/lifecycle/src/claims.rs @@ -40,6 +40,17 @@ pub enum Missing { Unresolved(Unresolved), /// The kernel refused the claim. Refused(SyscallError), + /// A virtio function that offers no `VIRTIO_F_ACCESS_PLATFORM` uses + /// physical addresses whatever translates the bus (virtio 1.2 §6.1), so + /// no domain confines what it reaches; or one whose offer could not be + /// read, which is the same unknown. + Untranslated(Option), +} + +/// Virtio 1.2 §4.1.2: a virtio function's vendor and device ids, transitional +/// and modern. +pub fn is_virtio(pci: &Pci) -> bool { + pci.vendor == 0x1af4 && (0x1000..=0x107f).contains(&pci.device) } /// Whether a start goes on without `entry`'s claim. A matcher naming several @@ -73,6 +84,17 @@ mod tests { PciAddr { segment: 0, bus: 0, dev, func: 0 } } + #[test] + fn a_virtio_function_is_known_by_its_ids_and_an_untranslated_one_is_gone_without() { + let virtio = |device| Pci { vendor: 0x1af4, device, ..edu(1, 3, Driven::Free) }; + assert!(is_virtio(&virtio(0x1000)) && is_virtio(&virtio(0x1041)) && is_virtio(&virtio(0x107f))); + assert!(!is_virtio(&virtio(0x1080)) && !is_virtio(&virtio(0x0fff))); + assert!(!is_virtio(&edu(1, 3, Driven::Free))); + let refused = Missing::Untranslated(None); + assert!(goes_on_without("pci:02:00:00", &refused, &[])); + assert!(!goes_on_without("pci:02:00:00", &refused, &["pci:02:00:00".to_string()])); + } + #[test] fn a_matcher_names_the_one_function_the_kernel_does_not_drive() { assert_eq!(function(&EDU, &[edu(3, 3, Driven::Kernel), edu(4, 4, Driven::Free)]), Ok(Arrival(4))); diff --git a/system/supervisor/src/main.rs b/system/supervisor/src/main.rs index 161cfa657f4..8db2b5c66d2 100644 --- a/system/supervisor/src/main.rs +++ b/system/supervisor/src/main.rs @@ -976,6 +976,7 @@ impl<'a> Supervisor<'a> { } let name = entry.to_string(); let claim = self.syscap.claim_function::(function.arrival).map_err(|e| refused(&name, e))?; + let claim = translated(claim, &function).map_err(|why| missing(&name, &why))?; self.make_home(row); let mut extra = Extra { claims: vec![(format!("{DEV_PREFIX}{name}"), claim)], ..Extra::default() }; // A disk driver is who this start says it is: its `disk` connector @@ -2514,6 +2515,32 @@ fn missing(name: &str, why: &Missing) -> String { format!("{name} names {}, and its row has one process to drive one", functions(at)) } Missing::Refused(e) => refused(name, *e), + Missing::Untranslated(None) => format!( + "{name} is a virtio function that offers no VIRTIO_F_ACCESS_PLATFORM, so no unit translates its DMA \ + (virtio 1.2 §6.1): refused" + ), + Missing::Untranslated(Some(why)) => format!( + "{name} is a virtio function whose offer was not read ({why}), so whether a unit translates its DMA is \ + unknown: refused" + ), + } +} + +/// `claim` on `function`, refused where `function` is a virtio function whose +/// DMA no unit confines: a holder is never handed one. +fn translated(claim: toyos::Device, function: &Pci) -> Result { + if !claims::is_virtio(function) { + return Ok(claim); + } + // SAFETY: the claim is this process's own, just minted, and moves in whole. + let dev = unsafe { ::from_handle(claim.into_raw()) }; + let offered = toyos_pci_claim::virtio::offers_access_platform(&dev); + // SAFETY: the same handle, moving back out whole. + let claim = unsafe { ::from_handle(dev.into_raw()) }; + match offered { + Ok(true) => Ok(claim), + Ok(false) => Err(Missing::Untranslated(None)), + Err(why) => Err(Missing::Untranslated(Some(why.to_string()))), } } @@ -2675,17 +2702,24 @@ fn start<'a>( // The build refuses to render an entry that names no device, so a // failure here is an image whose halves disagree. let entry = Entry::parse(name).unwrap_or_else(|| panic!("supervisor: `{name}` names no device")); - let request = match entry { - Entry::Fixed(request) => Ok(request), + let (request, function) = match entry { + Entry::Fixed(request) => (Ok(request), None), Entry::Pci(matcher) => { let records = match &mut functions { Some(records) => records, None => functions.insert(pci_records(syscap).map_err(StartError::Refused)?), }; - claims::function(&matcher, records).map(DeviceRequest::Function).map_err(Missing::Unresolved) + let found = claims::function(&matcher, records); + let function = found.as_ref().ok().and_then(|at| records.iter().find(|pci| pci.arrival == *at)).copied(); + (found.map(DeviceRequest::Function).map_err(Missing::Unresolved), function) } }; - let minted = request.and_then(|request| claim(syscap, request).map_err(Missing::Refused)); + let minted = request.and_then(|request| claim(syscap, request).map_err(Missing::Refused)).and_then(|minted| { + match &function { + Some(function) => translated(minted, function), + None => Ok(minted), + } + }); // The label is the manifest's own spelling, which is exactly what the // claimant looks the claim up by: one string, written once. let why = match minted { diff --git a/system/usbd/src/hc.rs b/system/usbd/src/hc.rs index 9276019ec16..92dd30c9fa8 100644 --- a/system/usbd/src/hc.rs +++ b/system/usbd/src/hc.rs @@ -295,8 +295,9 @@ impl Controller { settle("it held HCRST", || regs.read::(op + OP_USBCMD) & USBCMD_HCRST == 0)?; settle("it stayed Controller Not Ready", || regs.read::(op + OP_USBSTS) & USBSTS_CNR == 0)?; - // After the reset, so a controller refused above never had a grant: - // the grant is what starts the function mastering the bus. + // After the reset, so a controller refused above never had a grant. + // AC64 is required above, so its addresses are 64 bits (§5.3.6). + dev.dma_limit(64).map_err(|e| Refusal::Kernel("the address limit", e))?; let dev_blocks = max_slots.min(MAX_DEVICES); let bytes = (OFF_SCRATCH + scratch * PAGE + dev_blocks * DEV_STRIDE).next_multiple_of(GRANT_UNIT); let region = dev.dma_alloc(bytes as u64).map_err(|e| Refusal::Kernel("a DMA grant", e))?; @@ -305,6 +306,10 @@ impl Controller { let grant = unsafe { Window::new(region.memory.as_ptr(), bytes) }; grant.zero(); let base = region.device_addr; + // Halted and reset with mastering off: started before it is told any + // address, which a controller may read the moment it is written + // (QEMU's reads the segment table on the ERSTBA write). + dev.dma_start().map_err(|e| Refusal::Kernel("the bus-master start", e))?; println!( "usbd: PCI {:02x}:{:02x}.{} reset: {max_ports} ports, {max_slots} slots ({dev_blocks} given room), \ {context_size}-byte contexts, {scratch} scratchpad buffers, a {} KiB grant", diff --git a/tests/checks.rs b/tests/checks.rs index 1a90ef16471..cd9597982a1 100644 --- a/tests/checks.rs +++ b/tests/checks.rs @@ -952,8 +952,8 @@ mod checks { } #[test] - fn metal_domains_end_below_the_host_bridges() { - claims_checks::domains_end_below_the_windows(); + fn metal_domains_map_clear_of_the_host_bridges() { + claims_checks::domains_map_clear_of_the_windows(); } #[test] diff --git a/tests/checks/claims.rs b/tests/checks/claims.rs index 55261db24b9..83dd23cc6c5 100644 --- a/tests/checks/claims.rs +++ b/tests/checks/claims.rs @@ -11,6 +11,8 @@ use super::*; const RECORDED: &str = r"[2026-09-29 11:54:39 0.065 cpu0 kernel] PCI 00:1f.6 [0200] vendor=8086 device=15fc prog_if=00 bars=[bar0=0xbcf00000] [2026-09-29 11:54:39 0.149 cpu0 kernel] pcidev: firmware declared root bridge memory: mem 0xa2000000..0xbd000000, mem 0x4000000000..0x603dc00000, mem 0xa0800000..0xa2000000, mem 0xbd000000..0xc0000000, mem 0xff000000..0xffb80000, mem 0xffd3a070..0x100000000, mem 0x603dc00000..0x8000000000 [2026-09-29 11:54:39 0.250 cpu0 kernel] iommu: domain2 root=0xa31000 aw=48 mgaw=39 addresses from 0x2000000000 to 0x8000000000 +[2026-09-29 11:54:39 0.251 cpu0 kernel] iommu: domain2 maps 0x1c00000..0x1e00000 at 0x3fffe00000 +[2026-09-29 11:54:39 0.252 cpu0 kernel] iommu: domain2 maps 0x1e00000..0x2200000 at 0x4000000000 [2026-09-29 12:10:34 1.191 cpu0 kernel] iommu: irte5 source=00:1f.6 p=1 sid=0x00fe svt=1 sq=0 vector=0x28 apic=0x0 dst=0x0 trigger=edge [2026-09-29 12:10:34 1.192 cpu0 kernel] pcidev: PCI 00:1f.6 [8086:15fc] handed over on slot 0, vector 0x28 [2026-09-29 12:10:53 20.233 cpu1 kernel] pcidev: PCI 00:1f.6 [8086:15fc] released from slot 0; reset by nothing (Express: no capability; AF: no capability; PM: No_Soft_Reset set), so where it may still be aimed is kept for its next claim @@ -70,10 +72,14 @@ pub fn one_entry_per_slot() { assert!(why.contains(r#"left (entry, p) [("0", "1"), ("0", "1")]"#), "{why}"); } -/// The recorded domain reaches into `0x4000000000..`; capped there, it passes. -pub fn domains_end_below_the_windows() { - let why = claims::clear_of_host_bridges(&log(RECORDED)).expect_err("the recorded domain"); +/// A mapping reaching into `0x4000000000..` reds; the same mapping placed +/// below the window, and one placed again there, pass. +pub fn domains_map_clear_of_the_windows() { + let why = claims::clear_of_host_bridges(&log(RECORDED)).expect_err("the mapping inside a window"); assert!(why.contains("inside the root-bridge window 0x4000000000..0x603dc00000"), "{why}"); - let capped = RECORDED.replace("to 0x8000000000", "to 0x4000000000"); - assert_eq!(claims::clear_of_host_bridges(&log(&capped)), Ok(())); + let clear = RECORDED.replace("at 0x4000000000", "at 0x200000 again"); + assert_eq!(claims::clear_of_host_bridges(&log(&clear)), Ok(())); + // One that starts below the window and ends inside it. + let straddles = RECORDED.replace("at 0x4000000000", "at 0x3fffe00000"); + assert!(claims::clear_of_host_bridges(&log(&straddles)).is_err()); } diff --git a/tests/common/claims.rs b/tests/common/claims.rs index ac34c25f77b..c4f83fd457c 100644 --- a/tests/common/claims.rs +++ b/tests/common/claims.rs @@ -125,41 +125,42 @@ pub fn reuses_its_entry(log: &Serial) -> Result<(), String> { Ok(()) } -/// Every domain's addresses lie outside every root-bridge window firmware -/// declared: a bridge may route a request in one of those before the unit -/// sees it. +/// Every address a domain mapped lies outside every root-bridge window +/// firmware declared: a bridge may route a request in one of those before the +/// unit sees it. pub fn clear_of_host_bridges(log: &Serial) -> Result<(), String> { const DECLARED: &str = "pcidev: firmware declared root bridge memory: "; let declared = records(log) .find_map(|m| m.strip_prefix(DECLARED)) .ok_or_else(|| format!("no `{DECLARED}` record:\n{}", log.text()))?; + let hex = |t: &str| u64::from_str_radix(t.trim().strip_prefix("0x")?, 16).ok(); let range = |text: &str| -> Option<(u64, u64)> { let (from, to) = text.split_once("..")?; - let hex = |t: &str| u64::from_str_radix(t.trim().strip_prefix("0x")?, 16).ok(); Some((hex(from)?, hex(to)?)) }; let windows = declared .split(", ") .map(|w| w.strip_prefix("mem ").and_then(range).ok_or_else(|| format!("{w:?} in {declared:?}"))) .collect::, _>>()?; - let mut domains = 0; - for record in records(log).filter(|m| m.starts_with("iommu: domain")) { - let Some((_, span)) = record.split_once(" addresses from ") else { continue }; - let (floor, ceiling) = span - .split_once(" to ") - .and_then(|(from, to)| range(&format!("{from}..{to}"))) - .ok_or_else(|| format!("an unreadable domain record: {record:?}"))?; - if let Some((base, end)) = windows.iter().find(|(base, end)| *end > floor && *base < ceiling) { - return Err(format!( - "{record:?} hands out addresses inside the root-bridge window {base:#x}..{end:#x}" - )); + let mut mapped = 0; + // `iommu: domainN maps .. at [ again]`. + for record in records(log).filter(|m| m.starts_with("iommu: domain") && m.contains(" maps ")) { + let parsed = record.split_once(" maps ").and_then(|(_, rest)| { + let (phys, at) = rest.split_once(" at ")?; + let (from, to) = range(phys)?; + let at = hex(at.trim_end_matches(" again"))?; + Some((at, at + (to - from))) + }); + let (at, end) = parsed.ok_or_else(|| format!("an unreadable mapping record: {record:?}"))?; + if let Some((base, top)) = windows.iter().find(|(base, top)| *top > at && *base < end) { + return Err(format!("{record:?} maps device addresses inside the root-bridge window {base:#x}..{top:#x}")); } - domains += 1; + mapped += 1; } - if domains == 0 { - return Err(format!("no domain was made on this boot:\n{}", log.text())); + if mapped == 0 { + return Err(format!("no domain mapped anything on this boot:\n{}", log.text())); } - eprintln!(" [claims] {domains} domain(s), none reaching a root-bridge window"); + eprintln!(" [claims] {mapped} mapping(s), none reaching a root-bridge window"); Ok(()) } diff --git a/tests/common/iommu.rs b/tests/common/iommu.rs index e138f1acc16..d3902bd7cc0 100644 --- a/tests/common/iommu.rs +++ b/tests/common/iommu.rs @@ -468,3 +468,45 @@ pub fn amdvi_selftest(test_config: &Path) -> Result<(), String> { } Ok(()) } + +/// What the supervisor says refusing netcase's NIC to netstack, on a machine +/// whose virtio NIC was created without `iommu_platform=on`. +pub const UNTRANSLATED_NIC: &str = "supervisor: netstack: pci:02:00:00/1af4:1041 is a virtio function that offers \ + no VIRTIO_F_ACCESS_PLATFORM, so no unit translates its DMA (virtio 1.2 §6.1): refused"; + +/// A virtio NIC that bypasses the unit ([`Profile::HeadlessNicUntranslated`]) +/// is never handed to netstack: the supervisor reads its offer through the +/// claim, refuses it by name, and netstack runs with no NIC. Only a QEMU +/// machine can make one: every bench function is behind its unit. +pub fn a_virtio_function_without_access_platform_is_refused() -> Result<(), String> { + let options = BootOptions { profile: Profile::HeadlessNicUntranslated, ..Default::default() }; + let mut qemu = QemuInstance::boot_with_options(&netcase(), &[], &[], options); + let gone = "netstack: no NIC on this machine"; + let mut text = qemu.boot_log().to_string(); + qemu::await_guest(&mut qemu, &mut text, "the supervisor's refusal and netstack without a NIC", |t| { + t.contains(UNTRANSLATED_NIC) && t.contains(gone) + }) + .map_err(|e| format!("{e}\n{text}"))?; + let log = Serial::named("boot console", text); + log.must_be_clean()?; + log.must_not_say("netstack: VirtIO: PCI")?; + eprintln!(" [iommu] {UNTRANSLATED_NIC}"); + Ok(()) +} + +/// The kernel's virtio console created without `iommu_platform=on` +/// ([`Profile::HeadlessConsoleUntranslated`]) bypasses the unit, so the kernel +/// tells it physical addresses and the boot reaches its ready marker over it. +/// Given a domain's addresses instead, it reads and writes whatever memory +/// those are, and the boot's console is lost. q35 only: no bench has a virtio +/// function. +pub fn the_kernel_tells_a_bypassing_console_physical_addresses() -> Result<(), String> { + let options = BootOptions { profile: Profile::HeadlessConsoleUntranslated, ..Default::default() }; + let qemu = QemuInstance::boot_with_options(&netcase(), &[], &[], options); + let log = Serial::named("boot console", qemu.boot_log().to_string()); + log.must_be_clean()?; + log.must_say("offers no ACCESS_PLATFORM, so it is told physical addresses")?; + log.must_say("Boot: complete")?; + eprintln!(" [iommu] the console bypassing the unit was told physical addresses and carried the boot"); + Ok(()) +} diff --git a/tests/common/qemu.rs b/tests/common/qemu.rs index 173afd16c0a..cd962af2884 100644 --- a/tests/common/qemu.rs +++ b/tests/common/qemu.rs @@ -694,6 +694,14 @@ pub enum Profile { /// a function a claim may take whose BAR has to land inside the port's /// windows, as every function behind a bridge on a laptop does. HeadlessNicBehindPort, + /// [`Profile::Headless`] with its virtio NIC created without + /// `iommu_platform=on`, so behind the unit on paper and bypassing it in + /// fact: the function a claim must never be handed. + HeadlessNicUntranslated, + /// [`Profile::Headless`] with the kernel's virtio console created without + /// `iommu_platform=on`: a device the kernel itself must tell physical + /// addresses. + HeadlessConsoleUntranslated, /// [`Profile::Headless`] with two of QEMU's `edu`, which the kernel drives /// neither of: two identical functions, each a `drives` row's to start a /// process on. @@ -787,6 +795,8 @@ impl Profile { | Self::HeadlessVirtioGpu | Self::HeadlessUsbSpare | Self::HeadlessNicBehindPort + | Self::HeadlessNicUntranslated + | Self::HeadlessConsoleUntranslated | Self::HeadlessTwoEdu | Self::Metal | Self::MetalTwoHda @@ -910,6 +920,8 @@ enum Nic { Virtio, /// The virtio NIC on the secondary bus of a root port of its own. BehindPort, + /// The virtio NIC without `iommu_platform=on`, whatever unit the machine has. + Untranslated, } /// Everything a profile decides about the machine, in one table. A new @@ -929,6 +941,9 @@ struct Shape { /// channel on a virtio-serial function of its own. virtio: Virtio, nic: Nic, + /// The virtio console created without `iommu_platform=on` on a machine + /// with a unit, so its DMA bypasses the unit. + console_bypasses: bool, /// The `-device` argument for each xHCI controller, port and slot counts /// included. A list because a machine can have more than one and the T14 /// does — its keyboard is on the second. @@ -1045,6 +1060,7 @@ impl Profile { rng: true, virtio_gpu: false, hda: false, + console_bypasses: false, }, Self::Headless => Shape { vga: "none", @@ -1060,6 +1076,7 @@ impl Profile { rng: false, virtio_gpu: false, hda: false, + console_bypasses: false, }, Self::Metal => Shape { vga: "std", @@ -1079,6 +1096,7 @@ impl Profile { rng: false, virtio_gpu: false, hda: false, + console_bypasses: false, }, Self::MetalNoX2apic => Self::Metal.shape(), Self::MetalTwoHda => Shape { hda: true, ..Self::Metal.shape() }, @@ -1094,6 +1112,8 @@ impl Profile { Self::HeadlessNoUsb => Shape { xhci: &[], usb: &[], ..Self::Headless.shape() }, Self::HeadlessVirtioGpu => Shape { virtio_gpu: true, ..Self::Headless.shape() }, Self::HeadlessNicBehindPort => Shape { nic: Nic::BehindPort, ..Self::Headless.shape() }, + Self::HeadlessNicUntranslated => Shape { nic: Nic::Untranslated, ..Self::Headless.shape() }, + Self::HeadlessConsoleUntranslated => Shape { console_bypasses: true, ..Self::Headless.shape() }, Self::HeadlessTwoEdu => Shape { probes: Probes::Edus, ..Self::Headless.shape() }, Self::HeadlessUsbSpare => Shape { xhci: &[XHCI_DEFAULT, XHCI_SPARE], @@ -2403,6 +2423,9 @@ fn qemu_command( "virtio-net-pci-non-transitional,netdev=net0{platform}" )); } + Nic::Untranslated => { + qemu.arg("-netdev").arg("user,id=net0").arg("-device").arg("virtio-net-pci-non-transitional,netdev=net0"); + } Nic::BehindPort => { qemu.arg("-device").arg("pcie-root-port,id=nicport,chassis=9"); qemu.arg("-netdev").arg("user,id=net0").arg("-device").arg(format!( @@ -2434,7 +2457,8 @@ fn qemu_command( .arg("stdio,id=cs0,signal=off") .arg("-device") .arg(format!( - "virtio-serial-pci-non-transitional,id=virtio-serial0,max_ports=1{platform}" + "virtio-serial-pci-non-transitional,id=virtio-serial0,max_ports=1{}", + if shape.console_bypasses { "" } else { platform } )) .arg("-device") .arg("virtconsole,chardev=cs0,id=console0"); diff --git a/tests/guest/rust-tests/src/bin/claim_lpi.rs b/tests/guest/rust-tests/src/bin/claim_lpi.rs index 7e878f34a61..3fbc73d0ba0 100644 --- a/tests/guest/rust-tests/src/bin/claim_lpi.rs +++ b/tests/guest/rust-tests/src/bin/claim_lpi.rs @@ -44,7 +44,9 @@ fn main() { let regs = unsafe { Window::new(bar.as_ptr(), info.bar_bytes[0] as usize) }; let id = regs.read::(IDENTIFICATION); assert_eq!(id & 0xff, EDU_ID, "claim_lpi: BAR 0 answers {id:#010x}, which is not edu's identification"); - let _grant = dev.dma_alloc(4096).expect("claim_lpi: a grant"); + // A message is a write only a started function masters; edu's own DMA + // engine is not used, so it needs no grant. + dev.dma_start().expect("claim_lpi: the start"); assert_eq!(dev.irq().map(|r| r.count), Err(SyscallError::WouldBlock), "claim_lpi: an interrupt before any was raised"); let poller = Poller::new(1); diff --git a/tests/guest/rust-tests/src/bin/i219_msi.rs b/tests/guest/rust-tests/src/bin/i219_msi.rs index b3cab4e94f9..dcc63fa047c 100644 --- a/tests/guest/rust-tests/src/bin/i219_msi.rs +++ b/tests/guest/rust-tests/src/bin/i219_msi.rs @@ -266,11 +266,13 @@ fn main() { // A second mapping of the same BAR, for what is read beside the driver. let (regs, _held) = Bar::map(&dev, index, bytes).unwrap_or_else(|why| fail(&format!("{why}"))); - // The grant is what lets the function master the bus, and a message is a + // The start is what lets the function master the bus, and a message is a // write it masters. let (driven, _mapping) = Bar::map(&dev, index, bytes).unwrap_or_else(|why| fail(&format!("{why}"))); toyos_i219::quiesce(&driven); + toyos_pci_claim::limit(&dev, 64).unwrap_or_else(|why| fail(&format!("{why}"))); let (grant, _region) = Grant::alloc(&dev, toyos_i219::GRANT_BYTES).unwrap_or_else(|why| fail(&format!("{why}"))); + toyos_pci_claim::start(&dev).unwrap_or_else(|why| fail(&format!("{why}"))); let frames = grant.window(); let claim = Claim(Rc::clone(&dev)); @@ -367,6 +369,13 @@ fn main() { counters.stranded, wire.sent, )); + // The MAC's own counts of the wire: a network that sent nothing reads as + // nothing seen, and a frame the MAC took but no descriptor got as more + // taken than received. + say(&format!( + "the wire: the MAC saw {} frame(s), took {} good, missed {}, {} with CRC errors", + wire.seen, wire.received, wire.missed, wire.crc_errors + )); drop(driver); diff --git a/tests/guest/rust-tests/src/usbd_spare.rs b/tests/guest/rust-tests/src/usbd_spare.rs index 5ddf4a70c6f..574d711c8b5 100644 --- a/tests/guest/rust-tests/src/usbd_spare.rs +++ b/tests/guest/rust-tests/src/usbd_spare.rs @@ -199,8 +199,11 @@ pub fn fault(cap: &SysCap) { let bar = dev.map_bar(0, info.bar_bytes[0]).expect("usbd_spare: BAR 0"); // SAFETY: the mapping is the BAR's length and lives as long as `bar`. let regs = unsafe { Window::new(bar.as_ptr(), info.bar_bytes[0] as usize) }; - // One page granted, which is what starts the function mastering the bus. + // One page granted, and the function started: mastering is what its + // aimed fetch needs. + dev.dma_limit(64).expect("usbd_spare: the address limit"); let grant = dev.dma_alloc(4096).expect("usbd_spare: a grant"); + dev.dma_start().expect("usbd_spare: the start"); let op = usize::from(regs.read::(0)); let db = (regs.read::(0x14) & !3) as usize; let settle = |what: &str, done: &dyn Fn() -> bool| { diff --git a/tests/toyos.rs b/tests/toyos.rs index d465394a797..9dccfd05ac5 100644 --- a/tests/toyos.rs +++ b/tests/toyos.rs @@ -414,6 +414,14 @@ const MACHINE_TESTS: &[&str] = &[ // behind its emulated VT-d unit and without one: no shipped machine has a // virtio function, so only a QEMU machine can be asked. "iommu_virtio_platform", + // A virtio NIC created without `iommu_platform=on` on q35 with VT-d, so + // its DMA bypasses the unit: the supervisor refuses it to netstack by + // name. Only QEMU can create such a function; q35 only. + "a_virtio_function_without_access_platform_is_refused", + // The kernel's own virtio console created without `iommu_platform=on` + // on q35 with VT-d: the kernel tells it physical addresses, and the boot + // runs over it. q35 only. + "the_kernel_tells_a_bypassing_console_physical_addresses", // An AMD-Vi unit left blocking every device, switched off and programmed // before the boot stick is driven behind it: the register sequence and the // DMA it lets through are no host test's, and the AMD bench is staged by @@ -1155,9 +1163,9 @@ const METAL: &[(&str, metal::Metal)] = &[ }, ), ( - // Every domain's addresses end below the first root-bridge window - // above where they start. - "domain_ends_below_the_host_bridges", + // Every device address a domain maps is outside every root-bridge + // window firmware declared. + "domains_map_clear_of_the_host_bridges", metal::Metal { arms: TESTCASES, judge: |b| claims::clear_of_host_bridges(&b[0].kernel()) }, ), ( @@ -3923,6 +3931,12 @@ fn scanout_wc(console: &str) -> Result<(), String> { fn run_machine_test(name: &str, test_config: &Path) -> Result<(), String> { match name { "iommu_virtio_platform" => common::iommu::iommu_virtio_platform(), + "a_virtio_function_without_access_platform_is_refused" => { + common::iommu::a_virtio_function_without_access_platform_is_refused() + } + "the_kernel_tells_a_bypassing_console_physical_addresses" => { + common::iommu::the_kernel_tells_a_bypassing_console_physical_addresses() + } "amdvi_firmware_left" => common::iommu::amdvi_firmware_left(test_config), "amdvi_selftest" => common::iommu::amdvi_selftest(test_config), "nested_nmi_is_loud" => faults::nested_nmi_is_loud(test_config),