From 80edb7fe1b9ec023637146622bb4650f31ad1361 Mon Sep 17 00:00:00 2001 From: japabu Date: Wed, 28 Oct 2026 19:11:04 +0100 Subject: [PATCH 01/11] clock: the TSC keeps CPUID 15H's rate or one the HPET refines, and every CPU holds it to the HPET The x86-64 clock took the TSC's rate from one 50 ms count against the HPET at boot and never looked again. On the AMD bench that count read 41 to 66 ppm low (HPET read after the first TSC read and before the last, so its interval brackets the TSC's), the T14 kept its own 285 ppm-off measurement beside the 2,419,200,000 Hz CPUID 15H states, and nothing checked that the TSC was invariant, that the CPUs agreed or that the rate held. The rate. Where CPUID 15H enumerates its crystal, the clock keeps that rate (SDM Vol. 2A, CPUID 15H). Leaf 16H is no longer read as a rate: the SDM confines its data to display. Elsewhere the boot counts the TSC against the HPET over 50 ms in six samples, each the narrowest of three reads bracketed by RDTSCP;LFENCE, and derives the rate's bound from the brackets and the HPET's tick; a sample off the line between first and last (a step inside the window) refuses the calibration, which is retaken up to three times. The check. Each CPU samples the HPET once a second from its scheduler pass and holds it, at the kept rate, against the previous sample (a step) and the first (a drift), allowing the readings' brackets, one HPET tick and the kept rate's bound plus the HPET specification's 0.05%. Reports at 1, 2, 4, ... s log the rate the HPET measured with its bound; where the kept rate was measured, a better measurement replaces it. A step, a drift or a write that IA32_TSC_ADJUST shows panics by name. A CPU that does not state an invariant TSC is refused, except under a hypervisor (TCG cannot state one), where the HPET alone holds it, and says so. Every AP's TSC is bounded against the BSP's by 256 round trips of a cache line as it starts, replacing the 11 ms bracket; an offset the trips prove panics. The clock page carries a line (counter, stamp, ns per tick in 32.32) behind a two-copy latch instead of four constants, so the kernel can retune the rate from where the clock stands without a step, and an NMI or a process reads it without waiting on the writer. The kernel reads its own clock off the same page. The latch has a loom model and a negative control. Closes issues/the-kernels-50-ms-tsc-calibration-misses-the-tscs-rate.md, issues/the-system-clock-takes-the-tsc-on-one-reading-and-never-checks-it-again.md and issues/ap-tsc-trail-is-assumed-and-never-checked.md, on the metal rows `timer_calibration`, `smp_roster_and_tsc_trail` and `clock_holds_over_minutes`. Co-Authored-By: Claude Opus 5.5 --- build/arch.rs | 2 +- build/ci.rs | 4 + ...-tsc-trail-is-assumed-and-never-checked.md | 77 --- ...ms-tsc-calibration-misses-the-tscs-rate.md | 52 -- ...e-on-an-x86-cpu-that-states-no-tsc-rate.md | 21 +- ...n-one-reading-and-never-checks-it-again.md | 53 -- ...ates-against-an-hpet-or-the-boot-panics.md | 15 +- kernel/loom/Cargo.toml | 10 + kernel/loom/src/lib.rs | 6 + kernel/loom/tests/clock_latch.rs | 62 +++ kernel/pure/clock.rs | 53 -- kernel/pure/clock/mod.rs | 44 ++ kernel/pure/clock/reference.rs | 497 ++++++++++++++++++ kernel/src/actuator.rs | 6 + kernel/src/arch/aarch64/boot.rs | 2 +- kernel/src/arch/aarch64/clocksource.rs | 6 + kernel/src/arch/aarch64/mod.rs | 1 + kernel/src/arch/x86_64/boot.rs | 18 +- kernel/src/arch/x86_64/clocksource.rs | 402 ++++++++++++++ kernel/src/arch/x86_64/cpu.rs | 53 +- kernel/src/arch/x86_64/hpet.rs | 97 ---- kernel/src/arch/x86_64/idle.rs | 3 +- kernel/src/arch/x86_64/mod.rs | 4 +- kernel/src/arch/x86_64/smp.rs | 64 +-- kernel/src/clock.rs | 161 +++--- kernel/src/sched/driver.rs | 1 + lib/trace/src/lib.rs | 18 +- lib/tsc/Cargo.toml | 2 +- lib/tsc/src/lib.rs | 40 +- loader/src/arch/x86_64.rs | 3 +- sdk/abi/Cargo.toml | 7 + sdk/abi/src/arch/aarch64.rs | 2 +- sdk/abi/src/arch/x86_64.rs | 2 +- sdk/abi/src/clock.rs | 270 +++++++--- sdk/abi/src/clock/latch.rs | 74 +++ tests/common/faults.rs | 34 ++ tests/guest/osbench/src/around.rs | 4 +- tests/guest/osbench/src/sys/toyos.rs | 2 +- .../src/bin/abuse_readonly_copyout.rs | 14 +- tests/toyos.rs | 190 ++++--- 40 files changed, 1676 insertions(+), 700 deletions(-) delete mode 100644 issues/ap-tsc-trail-is-assumed-and-never-checked.md delete mode 100644 issues/the-kernels-50-ms-tsc-calibration-misses-the-tscs-rate.md delete mode 100644 issues/the-system-clock-takes-the-tsc-on-one-reading-and-never-checks-it-again.md create mode 100644 kernel/loom/tests/clock_latch.rs delete mode 100644 kernel/pure/clock.rs create mode 100644 kernel/pure/clock/mod.rs create mode 100644 kernel/pure/clock/reference.rs create mode 100644 kernel/src/arch/aarch64/clocksource.rs create mode 100644 kernel/src/arch/x86_64/clocksource.rs delete mode 100644 kernel/src/arch/x86_64/hpet.rs create mode 100644 sdk/abi/src/clock/latch.rs diff --git a/build/arch.rs b/build/arch.rs index 7f56ca52222..9dd4681c042 100644 --- a/build/arch.rs +++ b/build/arch.rs @@ -164,7 +164,7 @@ impl Arch { pub const fn cpu(self, accel: Accel) -> &'static str { match (self, accel.is_hardware()) { (Arch::X86_64, true) => "host,+rdrand,+smap,+fsgsbase,+x2apic,+smep,+ssse3,+sse4.1,+sse4.2,+xsave,+xsaveopt,+avx,+avx2", - (Arch::X86_64, false) => "qemu64,+rdrand,+smap,+fsgsbase,+x2apic,+smep,+ssse3,+sse4.1,+sse4.2,+xsave,+xsaveopt,+avx,+avx2", + (Arch::X86_64, false) => "qemu64,+rdrand,+rdtscp,+smap,+fsgsbase,+x2apic,+smep,+ssse3,+sse4.1,+sse4.2,+xsave,+xsaveopt,+avx,+avx2", (Arch::Aarch64, true) => "host", (Arch::Aarch64, false) => "max", } diff --git a/build/ci.rs b/build/ci.rs index d28728a8b26..4b09debd254 100644 --- a/build/ci.rs +++ b/build/ci.rs @@ -534,6 +534,10 @@ pub(crate) const CONTROLS: &[Control] = &[ red(KERNEL_LOOM, "seqlock-writer-fence-off", Some("panic_console_publish"), &[ Fails("a_snapshot_is_one_publication_whole"), ]), + red(KERNEL_LOOM, "clock-latch-fence-off", Some("clock_latch"), &[ + Fails("a_read_is_one_line_whole"), + Fails("a_later_read_is_never_an_older_line"), + ]), red(KERNEL_LOOM, "serial-try-lock-then-some", Some("serial_lock"), &[ Fails("a_lost_try_lock_leaves_the_lock_held"), Fails("two_writers_never_overlap"), diff --git a/issues/ap-tsc-trail-is-assumed-and-never-checked.md b/issues/ap-tsc-trail-is-assumed-and-never-checked.md deleted file mode 100644 index 6623469fb78..00000000000 --- a/issues/ap-tsc-trail-is-assumed-and-never-checked.md +++ /dev/null @@ -1,77 +0,0 @@ ---- -status: open -kind: tooling -opened: 2026-08-14 ---- - -# Every timestamp in the kernel assumes an AP's TSC does not trail the BSP's, and nothing measures it - -`clock::nanos_since_boot` subtracts one global `TSC_BOOT` — stored by the BSP -during `clock::init` — from whatever `rdtsc` this CPU answers. A CPU whose TSC -reads below that value has no honest answer to give. - -The assumption has been stated out loud, and `kernel/src/clock.rs` carries it: - -> **Cross-CPU ordering rests on the TSC being invariant and firmware-synchronised.** -> ToyOS targets 2020+ x86-64, where it is. If it is not, two records from two -> CPUs may be merged out of order by the skew; nothing breaks and the reader -> cannot tell. - -That is an accurate description of a *small* skew. It is not a description of a -CPU whose TSC starts behind the BSP's by more than the boot took, which is the -case the subtraction cannot represent at all. - -## What changed, and what did not - -Until 2026-08-14 the subtraction was `rdtsc() - TSC_BOOT`, and with -overflow-checks on — which every guest build has — a trailing TSC **panicked the -kernel**. `log::emit` then began reading the clock inside its publication -bracket, where such a panic would fire with `IF` clear and a reservation taken -and uncommitted, and the panic handler's own `log!` would reenter the same -shard. The subtraction is `saturating_sub` now, so the failure is a timestamp of -zero instead of a fault. - -**That makes it survivable, not correct.** A CPU stuck behind stamps every -record it writes with zero for as long as it trails, and those records read as -the oldest thing the machine has: `log/read.rs`'s merge sorts them last, and its -per-shard descent — which relies on `at_ns` descending with the sequence number -— stops at the first of them. Every other consumer of `nanos_since_boot` on that CPU is -wrong by the same amount: deadlines, the scheduler's accounting, `at_ns` in -`/log`. - -## What is not known - -Nothing in this tree has ever measured TSC agreement across CPUs. Both halves -are open: - -- **Does it happen on the T14?** `smp` brings APs up and no code compares their - TSC to the BSP's, before or after. One `rdtsc` per CPU logged at the end of - `init_ap` would answer it, and the metal session is where that has to run — - QEMU's TSCs are synthesised from one host clock and will agree whatever the - firmware does. -- **What should happen if it does?** The choices are a per-CPU `TSC_BOOT` - captured at AP bring-up (which fixes the arithmetic and leaves cross-CPU - comparison meaningless), `IA32_TSC_ADJUST` written to align them (which is - what firmware is supposed to have done), or accepting the skew and saying so - where records are merged. Not this branch's call. - -**2026-08-25, promoted to `defect`.** "Nothing would tell us" is the defect: -`kernel/src/clock.rs` cites this file for the assumption its whole arithmetic -rests on, and the instrument that would confirm or refute it does not exist — -one `rdtsc` per CPU logged at the end of `init_ap` is the entire measurement. -QEMU synthesises every guest TSC from one host clock and cannot answer, so this -belongs on the metal session's table beside the control-register delta -(`issues/a-metal-session-runs-a-pre-flash-gate-first.md`). Owed by -whoever prepares that session: build the log line first, then read it. - -**2026-10-28, part measured.** `smp` now logs each AP's `rdtsc` inside a -bracket of the BSP's, about 19.8 M cycles (11 ms) wide on the AMD bench, so -it answers "trails by more than the boot took" and no smaller offset. Under -the AMD bench's Linux, CPU 0 and each of the other 15 took turns reading -their TSCs through idle gaps up to 1 s (`tsc-probe warp`, PR #1079's -evidence). Every pair's worst step was positive, 54 to 342 cycles, which -bounds each CPU's offset from CPU 0 by 342 cycles, some 190 ns. That is that -firmware's synchronisation under Linux, not a ToyOS measurement: the log -line this file owes is still a bracket, not an offset. -`issues/the-system-clock-takes-the-tsc-on-one-reading-and-never-checks-it-again.md` -holds the clock's other unchecked assumptions. diff --git a/issues/the-kernels-50-ms-tsc-calibration-misses-the-tscs-rate.md b/issues/the-kernels-50-ms-tsc-calibration-misses-the-tscs-rate.md deleted file mode 100644 index a18cc3dfb0e..00000000000 --- a/issues/the-kernels-50-ms-tsc-calibration-misses-the-tscs-rate.md +++ /dev/null @@ -1,52 +0,0 @@ ---- -status: open -kind: defect -opened: 2026-10-28 ---- - -# The kernel's 50 ms TSC calibration misses the TSC's rate - -The kernel measures the TSC against the HPET once, over 50 ms -(`kernel/src/arch/x86_64/hpet.rs`), and keeps time at that rate for the -whole boot. On the AMD bench (Ryzen 7 4800U) that rate is consistently low. -The TSC counted 1796.6248 MHz against the same HPET over 10 minutes and -against the ACPI PM timer over 5, the same in every idle and busy phase -(`tsc-drift` under the bench's Linux, 2026-10-28, PR #1079's evidence). All -but one of the kernel's calibrations in the session's AMD boot logs read -1796.506 to 1796.551 MHz, 41 to 66 ppm low, and 1796.519 to 1796.532 MHz in -most; the one other reads 1796.597 MHz, 15 ppm low. A rate read low is a -period kept long (`hpet.rs` stores HPET femtoseconds per TSC cycle), so every -interval reads long: at 58 ppm a ToyOS clock on that machine gains about 5 s -a day. - -The error is a bias, not noise: every reading falls on one side. It is not -the period's resolution either: one step of the femtoseconds the kernel -stores is 3,227 Hz there, 1.8 ppm, and truncation biases the other way. An -inference, not measured: the HPET is read before the first TSC read and after -the last, so the HPET interval brackets the TSC interval by up to two -uncached MMIO reads. About 1.5 µs over 50 ms is about 30 ppm low, the sign -and size seen. - -The T14 states its TSC's rate (CPUID 15H, 2,419,200,000 Hz), and the kernel -logs its own measurement beside it. Over 5 minutes against the HPET under -its Linux, the TSC counted 2419.2002 MHz, the stated rate within 0.1 ppm, -worst residual 0.2 µs (`tsc-drift`, 2026-10-28). The session's T14 boot logs -hold two clusters, 285 ppm under and 59 ppm over, in 345 and 250 log files -(some of them copies of one boot), and a few readings between and beyond -them (404 to 440 and 759 to 766 ppm under). Those are the kernel's error: the -285 ppm cluster gains about 25 s a day and the 59 ppm one loses about 5. The -bracketing above explains neither, which needs 14 µs and the other sign. -That the two clusters are two kernel revisions is an inference no one has -checked. On that machine a stated rate is there to be used, and right, and -the kernel keeps its measurement instead. - -**Exit**: on the T14 and the AMD bench, the rate the kernel keeps is within a -bound the fix derives and logs. That bound is set by its window, its reads' -cost and the reference's own accuracy, against the rate measured over minutes -against the same reference: `tsc-drift` on the AMD bench, and CPUID 15H or -`tsc-drift` on the T14. - -Whether the TSC may keep time at all, and whether it still does, is -`issues/the-system-clock-takes-the-tsc-on-one-reading-and-never-checks-it-again.md`. - -**Owner**: the kernel's x86-64 clock (`kernel/src/arch/x86_64/hpet.rs`). diff --git a/issues/the-loaders-lines-carry-no-time-on-an-x86-cpu-that-states-no-tsc-rate.md b/issues/the-loaders-lines-carry-no-time-on-an-x86-cpu-that-states-no-tsc-rate.md index 8162e2ca67d..3d0d9ed830a 100644 --- a/issues/the-loaders-lines-carry-no-time-on-an-x86-cpu-that-states-no-tsc-rate.md +++ b/issues/the-loaders-lines-carry-no-time-on-an-x86-cpu-that-states-no-tsc-rate.md @@ -8,12 +8,13 @@ opened: 2026-10-04 Every line of the log opens with its time since the CPU's counter last started (`toyos_abi::log::Head`). The loader converts the counter at the rate CPUID -leaves 15H/16H state (`loader/src/stamp.rs`, `toyos_tsc::stated_hz`), and -the kernel does the same for its records until it has measured the TSC against -the HPET (`kernel/src/clock.rs`'s `stamp`). A CPU that states neither leaf gets +leaf 15H states (`loader/src/stamp.rs`, `toyos_tsc::stated_hz`), and the +kernel does the same for its records until its clock starts +(`kernel/src/clock.rs`'s `stamp`). Leaf 16H's base frequency is no rate: the +SDM confines its data to display. A CPU that states no rate in 15H gets `--.---` (`toyos_abi::log::UNTIMED`) in place of the time on every loader line, one `Loader clock: this CPU states no counter rate` line, and `--.---` on every -kernel record before `clock: HPET at …` — `boot:`, `ACPI:`, the interrupt +kernel record before the clock starts — `boot:`, `ACPI:`, the interrupt controller's and the per-CPU area's among them. The records after it carry their time from the same zero. @@ -22,8 +23,9 @@ Every AMD part is one. InstLatx64's recorded CPUID dumps largest standard leaf, as `00000010` on Zen 2 (`0870F10_K17_Matisse_CPUID`), Zen 3+ (`0A40F41_K19_Rembrandt_01_CPUID`), Zen 4 (`0A60F12_K19_Raphael_01_CPUID`) and Zen 5 (`0B00F21_K20_Turin_01_CPUID`, `0B20F40_K20_StrixPoint_06_CPUID`, -`0B40F40_K20_GraniteRidge_02_CPUID`): neither leaf exists there. QEMU's -`qemu64` states neither either, and under nightly's `kvm / suite`'s +`0B40F40_K20_GraniteRidge_02_CPUID`): neither leaf exists there, and an Intel +part whose leaf 15H leaves its crystal's hertz zero states none either. QEMU's +`qemu64` has no leaf 15H, and under nightly's `kvm / suite`'s `-cpu host` whether a line is timed is the runner's CPU's to decide. A second rate source is the fix, and the choice of one is open: the @@ -35,7 +37,6 @@ Until then no guest test asserts a time on the screen: `screen_loader_clears` reads `--.---` under `qemu64` and whatever the runner's CPU states under `-cpu host`. The fix is where that test starts asserting a time in it. -Exit: a loader line and a kernel record written before `clock: HPET at …` carry -their time on an x86-64 CPU that states neither leaf 15H's ratio nor leaf 16H's -base frequency, and `screen_loader_clears` asserts that line's time on the -screen. +Exit: a loader line and a kernel record written before the clock starts carry +their time on an x86-64 CPU whose leaf 15H states no rate, and +`screen_loader_clears` asserts that line's time on the screen. diff --git a/issues/the-system-clock-takes-the-tsc-on-one-reading-and-never-checks-it-again.md b/issues/the-system-clock-takes-the-tsc-on-one-reading-and-never-checks-it-again.md deleted file mode 100644 index ee21047bf49..00000000000 --- a/issues/the-system-clock-takes-the-tsc-on-one-reading-and-never-checks-it-again.md +++ /dev/null @@ -1,53 +0,0 @@ ---- -status: open -kind: defect -opened: 2026-10-28 ---- - -# The system clock takes the TSC on one reading and never checks it again - -x86-64's system clock is the TSC. The boot measures it against the HPET -once, over 50 ms (`kernel/src/arch/x86_64/hpet.rs`), and `clock::set_counter` -publishes that one period for the machine's life. Three things the clock -then relies on go unchecked: - -- **That the TSC may keep time at all.** CPUID 80000007H:EDX[8], the - invariant TSC (Intel's SDM "Invariant TSC"; AMD's PPRs - `CPUID_Fn80000007_EDX[TscInvariant]`), is read only to admit deep idle - states (`kernel/src/arch/x86_64/idle.rs::facts`). The clock starts on a - TSC without it the same way. -- **That every CPU's TSC agrees.** Each AP's reading at start is checked to - fall inside a bracket of the BSP's, about 19.8 M cycles wide on the AMD - bench, 11 ms (the `SMP:` lines), and nothing compares the CPUs again - (`issues/ap-tsc-trail-is-assumed-and-never-checked.md`). -- **That the rate stays the one measured.** No later reading is set against - the HPET or another reference, so a step or a change of rate after boot - goes into every timestamp, timeout and deadline unannounced. - -On the AMD bench (Ryzen 7 4800U) the one calibration is also 41 to 66 ppm -off (`issues/the-kernels-50-ms-tsc-calibration-misses-the-tscs-rate.md`). -That CPU states the invariant bit. Under its Linux, it counts one rate in -every phase over 10 minutes, and over 15 minutes across 16 CPUs and idle -gaps up to 1 s no CPU's TSC read behind another's and no step exceeded a -read's 1.2 µs. But in 5 of 30 boots' kernel logs that Linux's watchdog saw -the TSC some 1.7 ms ahead of the HPET over half a second, 2.6 s into boot in -the one boot whose dmesg was kept. Every time it was the same size, and it -came with a calibration that was right. The cause is unknown (`tsc-probe` -and the boots' figures, PR #1079's evidence). ToyOS's one calibration also -runs early in boot. A step inside its 50 ms would be a 3% error in every -timestamp after it, and a clock that never checks would not see it. - -**Exit**: the kernel keeps time on the TSC only where every CPU states it -invariant. It bounds the CPUs' offsets at start to a figure it logs and -derives, and checks the rate and offsets against a reference while it runs. -When they leave their bound it fails clearly, or falls back to a counter -that holds and says so in the log. The runtime bound is derived from the -reference's and the reads' own error, so the check does not mistake its -own calibration for the TSC's: 12 of the 17 times the AMD's Linux watchdog -fired, it was Linux's own calibration that was wrong. Where a vendor -documents a cheaper detector, the check prefers it, such as `IA32_TSC_ADJUST` -for a write to the TSC (Intel's SDM, Vol. 3). Each part is shown on the T14 -and the AMD bench. - -**Owner**: the kernel's x86-64 clock (`kernel/src/arch/x86_64/hpet.rs`, -`kernel/src/clock.rs`). diff --git a/issues/the-x86-clock-calibrates-against-an-hpet-or-the-boot-panics.md b/issues/the-x86-clock-calibrates-against-an-hpet-or-the-boot-panics.md index 85da7001334..9d7c5f32e21 100644 --- a/issues/the-x86-clock-calibrates-against-an-hpet-or-the-boot-panics.md +++ b/issues/the-x86-clock-calibrates-against-an-hpet-or-the-boot-panics.md @@ -7,11 +7,11 @@ opened: 2026-10-12 # The x86 clock calibrates against an HPET, or the boot panics `arch::boot::clock` (`kernel/src/arch/x86_64/boot.rs`) takes the HPET's base -from its ACPI table with `.expect("ACPI: HPET not found")`, and every rate the -kernel keeps descends from that one measurement: the TSC's -(`kernel/src/arch/x86_64/hpet.rs`) and the local APIC timer's, calibrated -against the TSC (`apic::init_timer`). CPUID 15H/16H is read after it, as a -cross-check only. A machine whose firmware publishes no `HPET` table stops +from its ACPI table with `.expect("ACPI: HPET not found")`, and the clock +needs it whatever the CPU states: it measures the TSC's rate against the HPET +where CPUID 15H states none, and holds the TSC to it while it runs, a stated +rate too (`kernel/src/arch/x86_64/clocksource.rs`); the local APIC timer is +calibrated against the TSC (`apic::init_timer`). A machine whose firmware publishes no `HPET` table stops there, before a timer ticks, and before PCI is enumerated, so a console that is a virtio function never carries the panic. @@ -44,5 +44,6 @@ a third-party copy of ACPI 6.6 numbers them; 6.5 itself is unread. **Owner**: whoever holds `issues/toyos-runs-in-any-open-source-hypervisor.md`. **Exit**: a q35 boot with `-machine hpet=off` reaches `Boot: complete`, its -`TSC:` line naming the source its rate came from; and a machine that states no -source is refused by a line that names each one it looked for. +`clock:` line naming the source its rate came from and the reference that +holds it; and a machine that states no source is refused by a line that names +each one it looked for. diff --git a/kernel/loom/Cargo.toml b/kernel/loom/Cargo.toml index d2ba72efec6..3a8a7c31f1c 100644 --- a/kernel/loom/Cargo.toml +++ b/kernel/loom/Cargo.toml @@ -197,6 +197,16 @@ tlb-tags-enter-relaxed = [] # Never on by default and never reachable from a kernel build, which declares # the same name only so `cfg` checking knows it. seqlock-writer-fence-off = [] +# The negative control for the clock page's latch. It drops the fence between +# the writer's move of readers to the other copy and its rewrite of the one +# they left (`sdk/abi/src/clock/latch.rs`), and `clock_latch.rs` must red: +# +# cargo test --manifest-path kernel/loom/Cargo.toml --features clock-latch-fence-off \ +# --test clock_latch +# +# Never on by default; `toyos-abi` declares the same name only so `cfg` +# checking knows it. +clock-latch-fence-off = [] # The negative control for the console backend's lock. It builds # `serial_lock.rs`'s `try_lock` with `then_some`, whose argument is built on a diff --git a/kernel/loom/src/lib.rs b/kernel/loom/src/lib.rs index b2e8411c536..85483bbc364 100644 --- a/kernel/loom/src/lib.rs +++ b/kernel/loom/src/lib.rs @@ -407,6 +407,12 @@ pub mod capture_access; #[path = "../../../sdk/toyos/src/log/ring.rs"] pub mod log_ring; +/// The clock page's latch, which the kernel replaces the clock's line through +/// while every process and the kernel's own NMIs read it: not the kernel's +/// file but the ABI's, driven by `tests/clock_latch.rs`. +#[path = "../../../sdk/abi/src/clock/latch.rs"] +pub mod clock_latch; + /// The seqlock the panic console's descriptor and each CPU's counters are /// published through, driven by `tests/panic_console_publish.rs` and /// `tests/counters_round.rs`. diff --git a/kernel/loom/tests/clock_latch.rs b/kernel/loom/tests/clock_latch.rs new file mode 100644 index 00000000000..e21c3ed5895 --- /dev/null +++ b/kernel/loom/tests/clock_latch.rs @@ -0,0 +1,62 @@ +//! Loom: a reader of the clock page's latch gets one line whole, and never an +//! older one after a newer. +//! +//! The line is three words a reader turns into a time; half of one +//! publication and half of the next is a clock that steps. One writer +//! replaces it (the kernel's retune) while a reader on another CPU reads it +//! with no lock, and a reader never waits for the writer. The negative control +//! is the `clock-latch-fence-off` feature, which must red this file. +#![cfg(feature = "loom")] + +use kernel_loom::clock_latch::Latch; +use loom::sync::Arc; +use loom::thread; + +const WORDS: usize = 3; + +/// Interleavings past this many preemptions add nothing a two-party protocol +/// of this length can do, and without a bound the two publications' twelve +/// stores do not finish exploring. +const PREEMPTIONS: usize = 3; + +fn model(f: impl Fn() + Sync + Send + 'static) { + let mut builder = loom::model::Builder::new(); + builder.preemption_bound = Some(PREEMPTIONS); + builder.check(f); +} + +/// Two publications, so the reader can meet either copy being rewritten. +#[test] +fn a_read_is_one_line_whole() { + model(|| { + let latch = Arc::new(Latch::::new()); + latch.publish([1; WORDS]); + let writer = { + let latch = latch.clone(); + thread::spawn(move || { + latch.publish([2; WORDS]); + latch.publish([3; WORDS]); + }) + }; + let line = latch.read(); + assert!(line == [1; WORDS] || line == [2; WORDS] || line == [3; WORDS], "a read kept two lines' words: {line:?}"); + writer.join().unwrap(); + assert_eq!(latch.read(), [3; WORDS]); + }); +} + +#[test] +fn a_later_read_is_never_an_older_line() { + model(|| { + let latch = Arc::new(Latch::::new()); + latch.publish([1; WORDS]); + let writer = { + let latch = latch.clone(); + thread::spawn(move || latch.publish([2; WORDS])) + }; + let first = latch.read(); + let second = latch.read(); + assert!(second[0] >= first[0], "a later read went back to an older line: {first:?} then {second:?}"); + writer.join().unwrap(); + }); +} diff --git a/kernel/pure/clock.rs b/kernel/pure/clock.rs deleted file mode 100644 index 38dc0e5b9bd..00000000000 --- a/kernel/pure/clock.rs +++ /dev/null @@ -1,53 +0,0 @@ -//! What the clock computes from counter ticks: their span in nanoseconds -//! ([`ticks_to_nanos`]), and the Unix second at boot that a wall-clock reading -//! taken at some counter value fixes ([`anchor`]). - -#![forbid(unsafe_code)] - -const NANOS_PER_SEC: u64 = 1_000_000_000; - -pub use toyos_abi::clock::ticks_to_nanos; - -/// Unix seconds at the counter's `boot`, from a reading of `secs` true when the -/// counter read `at`, which is after boot where the kernel read the clock and -/// before it where the loader did. Only the whole seconds between the two move -/// the answer. -pub const fn anchor(secs: u64, at: u64, boot: u64, period_fs: u64) -> u64 { - if at >= boot { - secs.saturating_sub(ticks_to_nanos(at - boot, period_fs) / NANOS_PER_SEC) - } else { - secs.saturating_add(ticks_to_nanos(boot - at, period_fs) / NANOS_PER_SEC) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - /// A one-nanosecond tick, so a tick count reads as nanoseconds. - const NS: u64 = 1_000_000; - const SECS: u64 = 1_993_799_665; - const BOOT: u64 = 50_000_000_000; - - #[test] - fn a_reading_after_boot_is_carried_back_to_it() { - assert_eq!(anchor(SECS, BOOT + 3_700_000_000, BOOT, NS), SECS - 3); - } - - #[test] - fn a_reading_before_boot_is_carried_forward_to_it() { - assert_eq!(anchor(SECS, BOOT - 2_500_000_000, BOOT, NS), SECS + 2); - } - - #[test] - fn a_reading_at_boot_is_boot() { - assert_eq!(anchor(SECS, BOOT, BOOT, NS), SECS); - } - - #[test] - fn a_tick_is_counted_at_its_period() { - // A 24 MHz counter: 41_666_666 fs a tick, so 24e6 ticks fall just short of a second. - assert_eq!(ticks_to_nanos(24_000_000, 41_666_666), 999_999_984); - assert_eq!(anchor(SECS, BOOT - 48_000_001, BOOT, 41_666_666), SECS + 2); - } -} diff --git a/kernel/pure/clock/mod.rs b/kernel/pure/clock/mod.rs new file mode 100644 index 00000000000..f3d4fd2b66a --- /dev/null +++ b/kernel/pure/clock/mod.rs @@ -0,0 +1,44 @@ +//! What the clock computes: the Unix second at boot that a wall-clock reading +//! fixes ([`anchor`]), and the counter held to a reference timer +//! ([`reference`]). + +#![forbid(unsafe_code)] + +pub mod reference; + +const NANOS_PER_SEC: u64 = 1_000_000_000; + +/// Unix seconds at boot, from a reading of `secs` true when the clock read +/// `at`, which is after boot where the kernel read the wall clock and before +/// it where the loader did; both are stamps, nanoseconds since the counter's +/// zero. Only the whole seconds between the two move the answer. +pub const fn anchor(secs: u64, at: u64, boot: u64) -> u64 { + if at >= boot { + secs.saturating_sub((at - boot) / NANOS_PER_SEC) + } else { + secs.saturating_add((boot - at) / NANOS_PER_SEC) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + const SECS: u64 = 1_993_799_665; + const BOOT: u64 = 50_000_000_000; + + #[test] + fn a_reading_after_boot_is_carried_back_to_it() { + assert_eq!(anchor(SECS, BOOT + 3_700_000_000, BOOT), SECS - 3); + } + + #[test] + fn a_reading_before_boot_is_carried_forward_to_it() { + assert_eq!(anchor(SECS, BOOT - 2_500_000_000, BOOT), SECS + 2); + } + + #[test] + fn a_reading_at_boot_is_boot() { + assert_eq!(anchor(SECS, BOOT, BOOT), SECS); + } +} diff --git a/kernel/pure/clock/reference.rs b/kernel/pure/clock/reference.rs new file mode 100644 index 00000000000..be86d0754d3 --- /dev/null +++ b/kernel/pure/clock/reference.rs @@ -0,0 +1,497 @@ +//! A free-running counter held to a reference timer: what a reading of the +//! reference between two of the counter's says, the rate two such readings +//! measure and how far it can be off, and whether the counter still keeps the +//! rate a clock was given. +//! +//! Every bound here is derived from the readings themselves: the counter +//! readings either side of a reference reading bracket the moment it latched, +//! and the reference's tick bounds where in that tick it was. Nothing is a +//! statistic, so a bound that fails is a counter, a reference or a read that +//! did not do what it says. + +/// Parts per billion in one. +pub const PPB: u64 = 1_000_000_000; + +const FS_PER_NS: u128 = 1_000_000; + +/// A reference timer: its tick, and how far its rate may wander from that tick +/// by its own specification, which every check that compares rates over time +/// allows on top of what the readings bound. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Reference { + pub period_fs: u64, + pub tolerance_ppb: u64, +} + +/// One reading of the reference, latched at some moment the counter read +/// between `before` and `after`. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Sample { + pub before: u64, + pub after: u64, + pub reference: u64, +} + +impl Sample { + /// The counter ticks the reading's latch could have been anywhere in. + pub const fn width(&self) -> u64 { + self.after - self.before + } + + /// The bracket's midpoint, doubled so it stays whole. + const fn mid2(&self) -> u128 { + self.before as u128 + self.after as u128 + } +} + +/// A rate two samples measure: nanoseconds per counter tick in the clock's +/// fixed point (`toyos_abi::clock::Line::mult`), at most `bound_ppb` off, over +/// `span_ns` of the reference. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Rate { + pub mult: u64, + pub bound_ppb: u64, + pub span_ns: u64, +} + +/// Why two samples measure no rate. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Unmeasurable { + /// The later sample's counter or reference reads no later than the + /// earlier's. + NotLater, + /// The brackets are a quarter or more of the span between them, which + /// bounds nothing worth keeping. + TooWide, +} + +/// The rate from `a` to `b`, which is later. +/// +/// The counter moved `Δc` between the latches, known to the brackets' half +/// widths `(wa + wb) / 2`, while the reference moved `Δn` ticks, known to one: +/// so the relative error is `u = (wa + wb) / (2 Δc)` plus `r = 1 / Δn`, each +/// taken against the measured span, and with the second-order terms of both +/// the ratio and those spans `(u + r)·(1 + 6·max(u, r))`, which holds while +/// both are under a quarter. `mult` is rounded down, which is one unit of it +/// more. +pub fn rate(reference: Reference, a: Sample, b: Sample) -> Result { + if b.mid2() <= a.mid2() || b.reference <= a.reference { + return Err(Unmeasurable::NotLater); + } + let counter2 = b.mid2() - a.mid2(); + let ticks = (b.reference - a.reference) as u128; + let span_fs = ticks * reference.period_fs as u128; + let widths = a.width() as u128 + b.width() as u128; + if widths * 4 >= counter2 || ticks < 4 { + return Err(Unmeasurable::TooWide); + } + let u = (widths * PPB as u128).div_ceil(counter2); + let r = (PPB as u128).div_ceil(ticks); + let first = u + r; + // ns per tick, times 2^32: `(span_fs / 10⁶) / (counter2 / 2) · 2^32`. + let mult = (span_fs << 33) / (FS_PER_NS * counter2); + let bound = first + (6 * first * u.max(r)).div_ceil(PPB as u128) + (PPB as u128).div_ceil(mult.max(1)); + Ok(Rate { mult: mult as u64, bound_ppb: bound as u64, span_ns: (span_fs / FS_PER_NS) as u64 }) +} + +/// How the counter and the reference agree over the span from one sample to +/// a later one, at a given rate. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Agreement { + /// The span the reference counted, in nanoseconds. + pub reference_ns: u64, + /// The counter's span at the rate, less the reference's. + pub apart_ns: i64, + /// What the readings and the rate's tolerance allow `apart_ns` to be. + pub allowed_ns: u64, +} + +impl Agreement { + pub const fn holds(&self) -> bool { + self.apart_ns.unsigned_abs() <= self.allowed_ns + } +} + +/// The counter's span from `a` to `b` at `mult`, against the reference's. +/// +/// Allowed: the brackets' half widths at `mult`, one reference tick, and +/// `tolerance_ppb` of the span, which is what the rate itself may be off by; +/// two nanoseconds more cover the rounding of each conversion. A `b` whose +/// reference reads behind `a`'s is a span of zero, so a reference that went +/// backwards is a disagreement. +pub fn agree(reference: Reference, a: Sample, b: Sample, mult: u64, tolerance_ppb: u64) -> Agreement { + let nanos = |ticks2: u128| (ticks2 * mult as u128) >> 33; + let counter_ns = if b.mid2() >= a.mid2() { nanos(b.mid2() - a.mid2()) as i128 } else { -(nanos(a.mid2() - b.mid2()) as i128) }; + let reference_ns = (b.reference.saturating_sub(a.reference) as u128 * reference.period_fs as u128 / FS_PER_NS) as u64; + let readings = nanos(a.width() as u128 + b.width() as u128) as u64; + let tick = (reference.period_fs as u128).div_ceil(FS_PER_NS) as u64; + let rated = (reference_ns as u128 * tolerance_ppb as u128).div_ceil(PPB as u128) as u64; + Agreement { + reference_ns, + apart_ns: (counter_ns - reference_ns as i128).clamp(i64::MIN as i128, i64::MAX as i128) as i64, + allowed_ns: readings + tick + rated + 2, + } +} + +/// Where a calibration's samples left the straight line between its first +/// and last. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Calibration { + Unmeasurable(Unmeasurable), + /// The sample at `index` sits off the line by more than the readings + /// allow: something moved the counter or the reference inside the window. + Disturbed { index: usize, agreement: Agreement }, +} + +/// The rate over `samples`, first to last, once every sample between is shown +/// to sit on that line within what the readings allow: a step inside the +/// window, which would skew the rate, is refused rather than averaged in. +pub fn calibrate(reference: Reference, samples: &[Sample]) -> Result { + let (Some(&first), Some(&last)) = (samples.first(), samples.last()) else { + return Err(Calibration::Unmeasurable(Unmeasurable::NotLater)); + }; + let fit = rate(reference, first, last).map_err(Calibration::Unmeasurable)?; + for (index, &sample) in samples.iter().enumerate().skip(1) { + let agreement = agree(reference, first, sample, fit.mult, fit.bound_ppb); + if !agreement.holds() { + return Err(Calibration::Disturbed { index, agreement }); + } + } + Ok(fit) +} + +/// The rate a clock keeps, and whether it was measured, so the watch may +/// replace it with a better measurement, or stated by the hardware. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Kept { + pub mult: u64, + pub bound_ppb: u64, + pub measured: bool, +} + +/// The first span [`Watch`] reports at, and the factor between reports. +const FIRST_REPORT_NS: u64 = 1_000_000_000; +const REPORT_FACTOR: u64 = 2; + +/// A counter's agreement with the reference over the life of a clock kept on +/// it: each sample is held against the one before it, which catches a step, +/// and against the first, which catches a rate that has moved. +#[derive(Clone, Copy, Debug)] +pub struct Watch { + reference: Reference, + anchor: Sample, + last: Sample, + kept: Kept, + next_report_ns: u64, +} + +/// What a sample showed, where it held. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Report { + /// The rate measured since the first sample. + pub measured: Rate, + /// The rate the clock keeps from here: a better measurement where the kept + /// one was measured, and otherwise the one it kept. + pub kept: Kept, + /// Whether `kept` replaced the rate the clock was keeping. + pub retuned: bool, +} + +/// What a sample showed, where it did not hold. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Failure { + /// Against the sample before it: the counter stepped. + Step { from: Sample, to: Sample, agreement: Agreement }, + /// Against the first: the counter's rate is not the one the clock keeps. + Drift { from: Sample, to: Sample, agreement: Agreement }, +} + +impl Watch { + pub const fn new(reference: Reference, anchor: Sample, kept: Kept) -> Self { + Self { reference, anchor, last: anchor, kept, next_report_ns: FIRST_REPORT_NS } + } + + pub const fn kept(&self) -> Kept { + self.kept + } + + /// The sample taken last, every one before it having held. + pub const fn last(&self) -> Sample { + self.last + } + + /// Hold `sample`, taken after every sample before it, to the kept rate. + /// `Some` at each report: the first a second after the anchor, then at + /// twice the span of the one before. + pub fn take(&mut self, sample: Sample) -> Result, Failure> { + let tolerance = self.kept.bound_ppb + self.reference.tolerance_ppb; + let step = agree(self.reference, self.last, sample, self.kept.mult, tolerance); + if !step.holds() { + return Err(Failure::Step { from: self.last, to: sample, agreement: step }); + } + let drift = agree(self.reference, self.anchor, sample, self.kept.mult, tolerance); + if !drift.holds() { + return Err(Failure::Drift { from: self.anchor, to: sample, agreement: drift }); + } + self.last = sample; + if drift.reference_ns < self.next_report_ns { + return Ok(None); + } + while self.next_report_ns <= drift.reference_ns { + self.next_report_ns = self.next_report_ns.saturating_mul(REPORT_FACTOR); + } + let Ok(measured) = rate(self.reference, self.anchor, sample) else { + return Ok(None); + }; + let retuned = self.kept.measured && measured.bound_ppb < self.kept.bound_ppb; + if retuned { + self.kept = Kept { mult: measured.mult, bound_ppb: measured.bound_ppb, measured: true }; + } + Ok(Some(Report { measured, kept: self.kept, retuned })) + } +} + +/// How far one CPU's counter can be from another's, from round trips of a +/// cache line between them: each side writes the counter it read just before +/// its store, and the other reads its own just after it sees the store. +/// +/// With `δ` the second counter less the first at one instant, a trip out +/// reads `δ` plus the trip's latency, never less, and a trip back reads `-δ` +/// plus its own; so the least of each bounds `δ` to `[-least_back, +/// least_out]`. An interval that excludes zero is an offset proven. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Offset { + pub least_out: i64, + pub least_back: i64, +} + +impl Offset { + /// `[lo, hi]` for `δ`, or the offset the trips prove, in ticks: negative + /// for a second counter behind the first. + pub const fn bound(&self) -> Result<(i64, i64), i64> { + if self.least_out < 0 { + Err(self.least_out) + } else if self.least_back < 0 { + Err(-self.least_back) + } else { + Ok((-self.least_back, self.least_out)) + } + } +} + +/// The 64-bit reading of a reference whose counter is 32 bits wide: the value +/// with `low` as its low half nearest `predicted`, the earlier 64-bit reading +/// plus the ticks the counter says have passed since. Right while that +/// prediction is within 2^31 ticks. +pub const fn extend(predicted: u64, low: u32) -> u64 { + let off = (low as u64).wrapping_sub(predicted) as u32 as i32; + predicted.wrapping_add_signed(off as i64) +} + +#[cfg(test)] +mod tests { + use super::*; + use alloc::vec::Vec; + + /// The AMD bench's HPET: 14.318 MHz, and the HPET specification's 0.05%. + const HPET: Reference = Reference { period_fs: 69_841_279, tolerance_ppb: 500_000 }; + /// A counter at 1796.6248 MHz. + const HZ: u128 = 1_796_624_800; + /// What a read of the reference costs in counter ticks: 1.2 µs. + const READ: u64 = 2_156; + + /// The sample a perfect counter at `hz` takes when the reference reads + /// `n`, the latch landing `at` ticks into a bracket `READ` wide, the + /// counter `step` ticks off from where it should be. + fn sample_at(hz: u128, n: u64, at: u64, step: i64) -> Sample { + let true_counter = (n as u128 * HPET.period_fs as u128 * hz / 1_000_000_000_000_000) as u64; + let latch = true_counter.wrapping_add_signed(step); + Sample { before: latch - at, after: latch - at + READ, reference: n } + } + + fn sample(n: u64) -> Sample { + sample_at(HZ, n, READ / 2, 0) + } + + fn hz_of(mult: u64) -> u128 { + (1_000_000_000u128 << 32) / mult as u128 + } + + fn ppb_apart(mult: u64, hz: u128) -> u128 { + hz_of(mult).abs_diff(hz) * PPB as u128 / hz + } + + const MS_50: u64 = 715_909; + const SECOND: u64 = 14_318_180; + + #[test] + fn a_rate_is_within_its_bound_wherever_the_latches_land() { + for (at_a, at_b) in [(0, READ), (READ, 0), (READ / 2, READ / 2), (0, 0)] { + let rate = rate(HPET, sample_at(HZ, 1_000, at_a, 0), sample_at(HZ, 1_000 + MS_50, at_b, 0)).unwrap(); + assert!(ppb_apart(rate.mult, HZ) <= rate.bound_ppb as u128, "{rate:?} at {at_a}, {at_b}"); + // Two 1.2 µs reads over 50 ms: 24 ppm, and the reference's tick 1.4. + assert!((24_000..27_000).contains(&rate.bound_ppb), "{rate:?}"); + } + } + + /// The same reads over ten minutes bound the rate to 6 ppb. + #[test] + fn a_long_span_bounds_the_rate_tightly() { + let rate = rate(HPET, sample_at(HZ, 0, 0, 0), sample_at(HZ, 600 * SECOND, READ, 0)).unwrap(); + assert!(ppb_apart(rate.mult, HZ) <= rate.bound_ppb as u128); + assert!(rate.bound_ppb <= 6, "{rate:?}"); + assert_eq!(rate.span_ns / 1_000_000_000, 600); + } + + #[test] + fn samples_out_of_order_or_too_wide_measure_nothing() { + assert_eq!(rate(HPET, sample(10_000), sample(1_000)), Err(Unmeasurable::NotLater)); + assert_eq!(rate(HPET, sample(1_000), sample(1_000)), Err(Unmeasurable::NotLater)); + let near = sample(100_000); + let wide = Sample { before: near.before - 1_000_000, after: near.after + 1_000_000, ..near }; + assert_eq!(rate(HPET, sample(99_000), wide), Err(Unmeasurable::TooWide)); + } + + fn window(steps: &[(usize, i64)]) -> Vec { + (0..=5u64) + .map(|i| { + let step = steps.iter().filter(|&&(at, _)| i as usize >= at).map(|&(_, by)| by).sum(); + sample_at(HZ, 1_000 + i * MS_50 / 5, (i * 431) % READ, step) + }) + .collect() + } + + #[test] + fn a_still_window_calibrates() { + let rate = calibrate(HPET, &window(&[])).unwrap(); + assert!(ppb_apart(rate.mult, HZ) <= rate.bound_ppb as u128); + } + + /// The 1.7 ms the AMD bench's TSC once read ahead of its HPET, and a step + /// of 5 µs, a tenth of a 50 ms window's 100 ppm, either way, wherever in the + /// window it lands: each is refused, not averaged into the rate. + #[test] + fn a_step_inside_the_window_is_refused() { + for step in [3_054_262i64, 8_983, -8_983] { + for at in 1..=5 { + match calibrate(HPET, &window(&[(at, step)])) { + Err(Calibration::Disturbed { agreement, .. }) => assert!(!agreement.holds()), + other => panic!("a {step}-tick step at sample {at} calibrated as {other:?}"), + } + } + } + } + + fn watch(measured: bool) -> Watch { + let first = calibrate(HPET, &window(&[])).unwrap(); + Watch::new(HPET, sample(1_000), Kept { mult: first.mult, bound_ppb: first.bound_ppb, measured }) + } + + /// A second's samples, each across a reference read of its own. + fn run(watch: &mut Watch, hz: u128, seconds: core::ops::Range, step: i64) -> Result, Failure> { + let mut reports = Vec::new(); + for s in seconds { + if let Some(report) = watch.take(sample_at(hz, 1_000 + s * SECOND, (s * 977) % READ, step))? { + reports.push(report); + } + } + Ok(reports) + } + + /// A measured rate is replaced at each report by a better one, and the + /// one an hour keeps is within its bound of the counter's, which is under + /// 2 ppb. + #[test] + fn a_measured_rate_is_refined() { + let mut watch = watch(true); + let reports = run(&mut watch, HZ, 1..3_601, 0).unwrap(); + let spans: Vec = reports.iter().map(|r| r.measured.span_ns / 1_000_000_000).collect(); + assert_eq!(spans, [1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048]); + assert!(reports.iter().all(|r| r.retuned)); + let kept = watch.kept(); + assert!(ppb_apart(kept.mult, HZ) <= kept.bound_ppb as u128); + assert!(kept.bound_ppb < 10, "{kept:?}"); + } + + /// A stated rate is reported against and never replaced. + #[test] + fn a_stated_rate_is_kept() { + let mut watch = watch(false); + let before = watch.kept(); + let reports = run(&mut watch, HZ, 1..100, 0).unwrap(); + assert!(!reports.is_empty() && reports.iter().all(|r| !r.retuned && r.kept == before)); + } + + /// A 1.7 ms step at runtime fails the sample after it, as a step. + #[test] + fn a_step_at_runtime_fails() { + let mut watch = watch(true); + run(&mut watch, HZ, 1..30, 0).unwrap(); + match run(&mut watch, HZ, 30..31, 3_054_262) { + Err(Failure::Step { agreement, .. }) => assert!(agreement.apart_ns > 1_600_000), + other => panic!("{other:?}"), + } + } + + /// A counter whose rate moves by 0.1% fails within seconds. One that moves + /// by 0.04%, inside the reference's own tolerance, is measured and kept. + #[test] + fn a_rate_that_moves_past_the_reference_tolerance_fails() { + let mut watch = watch(true); + run(&mut watch, HZ, 1..10, 0).unwrap(); + let faster = HZ + HZ / 1_000; + // The rate's change shows against the anchor as the seconds pass. + let at = (1_000 + 10 * SECOND) as u128 * HPET.period_fs as u128; + let offset = (at * HZ / 1_000_000_000_000_000) as i64 - (at * faster / 1_000_000_000_000_000) as i64; + let failed = run(&mut watch, faster, 10..20, offset); + assert!(matches!(failed, Err(Failure::Drift { .. }) | Err(Failure::Step { .. })), "{failed:?}"); + + let mut tolerant = watch_fresh(); + let slightly = HZ + HZ * 4 / 10_000; + assert!(run(&mut tolerant, slightly, 1..20, 0).is_ok()); + } + + fn watch_fresh() -> Watch { + Watch::new(HPET, sample(1_000), Kept { mult: (1_000_000_000u128 << 32).div_ceil(HZ) as u64, bound_ppb: 10, measured: true }) + } + + /// Two CPUs' samples 100 µs apart, each read slowed to 20 µs by an SMI and + /// latched at opposite ends of its bracket: a still counter, which the + /// readings' own widths must allow, since the reference's tolerance over + /// that span is 50 ns. + #[test] + fn close_samples_hold_across_wide_reads() { + let mut watch = watch(true); + run(&mut watch, HZ, 1..3, 0).unwrap(); + let wide = 35_932; + let n = watch.last().reference + 1_432; + let latch = |n: u64| sample_at(HZ, n, 0, 0).before; + let early = Sample { before: latch(n), after: latch(n) + wide, reference: n }; + let late = Sample { before: latch(n + 1_432) - wide, after: latch(n + 1_432), reference: n + 1_432 }; + assert_eq!(watch.take(early).map(|_| ()), Ok(())); + assert_eq!(watch.take(late).map(|_| ()), Ok(())); + } + + /// A reference that reads backwards is a disagreement, never a span. + #[test] + fn a_reference_that_goes_back_fails() { + let mut watch = watch(true); + run(&mut watch, HZ, 1..5, 0).unwrap(); + let back = Sample { reference: watch.last().reference - 10, ..sample(1_000 + 6 * SECOND) }; + assert!(watch.take(back).is_err()); + } + + #[test] + fn round_trips_bound_an_offset_or_prove_one() { + assert_eq!(Offset { least_out: 120, least_back: 80 }.bound(), Ok((-80, 120))); + assert_eq!(Offset { least_out: -30, least_back: 200 }.bound(), Err(-30)); + assert_eq!(Offset { least_out: 200, least_back: -45 }.bound(), Err(45)); + } + + #[test] + fn a_32_bit_reference_is_extended_across_its_wrap() { + assert_eq!(extend(0x1_ffff_fff0, 0x0000_0010), 0x2_0000_0010); + assert_eq!(extend(0x2_0000_0010, 0xffff_fff0), 0x1_ffff_fff0); + assert_eq!(extend(0x5_1234_5678, 0x1234_5679), 0x5_1234_5679); + } +} diff --git a/kernel/src/actuator.rs b/kernel/src/actuator.rs index c4c79a0b14c..c9a7de616c8 100644 --- a/kernel/src/actuator.rs +++ b/kernel/src/actuator.rs @@ -124,6 +124,12 @@ actuators! { /// `kernel_wx_leaf_is_refused` and `virt_kernel_wx_leaf_is_refused`. kernel_wx_leaf = "kernel-wx-leaf"; + /// Read one CPU's TSC 10 ms ahead once the clock has held for a second, + /// as a step `IA32_TSC_ADJUST` would not show, so the HPET alone can catch + /// it. The negative control on the clock's runtime check, + /// `a_stepped_tsc_panics`. + tsc_step = "tsc-step"; + /// Hand every CPU the loader's Intel update with its header's revision /// raised by one and its checksum sealed again: the read-back's negative /// control on real Intel, whose payload still says the revision it did. diff --git a/kernel/src/arch/aarch64/boot.rs b/kernel/src/arch/aarch64/boot.rs index 473d3fda692..0c6a778de4a 100644 --- a/kernel/src/arch/aarch64/boot.rs +++ b/kernel/src/arch/aarch64/boot.rs @@ -349,7 +349,7 @@ pub fn interrupts(rsdp_addr: u64) -> Platform { /// what the counter counts at; and the wall clock the loader handed over. pub fn clock(args: &KernelArgs) { let hz = super::cpu::stated_counter_hz().expect("clock: CNTFRQ_EL0 states no rate for the generic timer"); - crate::clock::set_counter(super::cpu::counter(), 1_000_000_000_000_000 / hz); + crate::clock::set_counter(super::cpu::counter(), toyos_abi::clock::Line::mult_of_hz(hz)); log!("clock: the generic timer counts at {hz} Hz"); crate::clock::init_wall(super::rtc::read(args)); } diff --git a/kernel/src/arch/aarch64/clocksource.rs b/kernel/src/arch/aarch64/clocksource.rs new file mode 100644 index 00000000000..e5f955788ae --- /dev/null +++ b/kernel/src/arch/aarch64/clocksource.rs @@ -0,0 +1,6 @@ +//! The generic timer is one system counter every CPU reads, at the rate +//! `CNTFRQ_EL0` states, which the boot keeps (`boot::clock`): there is no +//! second timer here to hold it to. + +/// No reference to sample against. +pub fn check(_now: u64) {} diff --git a/kernel/src/arch/aarch64/mod.rs b/kernel/src/arch/aarch64/mod.rs index e17e5ba4d07..f45fa171378 100644 --- a/kernel/src/arch/aarch64/mod.rs +++ b/kernel/src/arch/aarch64/mod.rs @@ -15,6 +15,7 @@ pub mod acpi_mode; pub mod barrier; pub mod boot; pub mod cache; +pub mod clocksource; pub mod console_uart; pub mod control_regs; pub mod counters; diff --git a/kernel/src/arch/x86_64/boot.rs b/kernel/src/arch/x86_64/boot.rs index fcfebe0485f..3cbf302c0bb 100644 --- a/kernel/src/arch/x86_64/boot.rs +++ b/kernel/src/arch/x86_64/boot.rs @@ -107,12 +107,10 @@ pub fn interrupts(rsdp_addr: u64) -> Platform { Platform { madt, facs } } -/// The clock: the TSC, calibrated against the HPET, and the CMOS wall clock. +/// The clock: the TSC, held to the HPET, and the CMOS wall clock. pub fn clock(args: &KernelArgs) { - // HPET clock — enables profiling for everything from here on - let hpet_base = acpi::find_hpet_base(args.rsdp_addr) - .expect("ACPI: HPET not found"); - super::hpet::calibrate_counter(hpet_base); + let hpet_base = acpi::find_hpet_base(args.rsdp_addr).expect("ACPI: HPET not found"); + super::clocksource::start(hpet_base); let century_reg = match acpi::rtc_century_register(args.rsdp_addr) { Ok(reg) => reg, Err(e) => { @@ -124,16 +122,14 @@ pub fn clock(args: &KernelArgs) { } /// Where the TSC counts from: `IA32_TSC_ADJUST`, where CPUID says the CPU has -/// it. Every write to the TSC since reset is added to it (Intel SDM Vol. 3B, -/// "Time-Stamp Counter Adjustment"), so zero is a counter firmware never wrote -/// and the TSC is time since power-on. +/// it, which is zero on a counter firmware never wrote, so the TSC is time +/// since power-on. pub fn report_counter_origin() { - const IA32_TSC_ADJUST: u32 = 0x3b; - if super::cpu::leaf_7().1 & (1 << 1) == 0 { + if !super::cpu::has_tsc_adjust() { log!("boot: IA32_TSC_ADJUST not on this CPU"); return; } - log!("boot: IA32_TSC_ADJUST {}", super::cpu::rdmsr(IA32_TSC_ADJUST) as i64); + log!("boot: IA32_TSC_ADJUST {}", super::cpu::rdmsr(super::cpu::IA32_TSC_ADJUST) as i64); } /// The per-CPU timer, once the clock converts its bound. diff --git a/kernel/src/arch/x86_64/clocksource.rs b/kernel/src/arch/x86_64/clocksource.rs new file mode 100644 index 00000000000..db9c4d9f05d --- /dev/null +++ b/kernel/src/arch/x86_64/clocksource.rs @@ -0,0 +1,402 @@ +//! x86-64's clock: the TSC, at the rate CPUID leaf 15H states where the part +//! states one and at one measured against the HPET where it does not, held to +//! the HPET by every CPU while it runs (`kernel::clock::reference`). +//! +//! The clock counts the TSC only where every CPU states it invariant, or under +//! a hypervisor, whose TSC is its own to synthesise and whose CPUID need not +//! say so; there the HPET is all that holds it. Every CPU's TSC is bounded +//! against the BSP's as it starts ([`measure_offset`]), and a write to it is +//! caught where `IA32_TSC_ADJUST` is there to show one. Each failure panics: +//! no other counter here can take the clock's place, because every process +//! reads the TSC itself. + +use core::sync::atomic::{AtomicBool, AtomicU64, Ordering}; + +use kernel::clock::reference::{self, Calibration, Failure, Kept, Offset, Reference, Sample, Watch}; +use toyos_abi::clock::Line; + +use super::cpu; +use crate::log; +use crate::mm::Mmio; +use crate::mm::policy::MmioPolicy; +use crate::sync::Lock; +use crate::time::{Delay, Duration, AP_START}; + +const HPET_CAP: u64 = 0x000; +const HPET_CFG: u64 = 0x010; +const HPET_COUNTER: u64 = 0x0F0; +/// `GCAP_ID` bit 13, `COUNT_SIZE_CAP`: the main counter is 64 bits wide +/// (IA-PC HPET Specification 1.0a, the General Capabilities and ID Register). +const COUNT_SIZE_CAP: u64 = 1 << 13; +/// The main counter's tick is accurate to 0.05% over any interval of 1 ms or +/// more (IA-PC HPET Specification 1.0a): the most the TSC may seem to +/// drift against it without having moved. +const HPET_TOLERANCE_PPB: u64 = 500_000; + +/// Reads of the HPET per sample, of which the narrowest bracket is kept: an +/// SMI or a host's preemption inside one read widens that read alone. +const READS: usize = 3; + +const CALIBRATION: Delay = Delay::to_measure( + Duration::from_millis(50), + "the TSC counted against the HPET where no CPUID leaf states its rate; the runtime check refines it", +); +/// Samples across the calibration, so a step inside it is refused. +const CALIBRATION_SAMPLES: u64 = 6; +/// Calibrations the boot tries before it refuses the machine. +const CALIBRATION_TRIES: u32 = 3; + +/// What each CPU waits between its samples: about the half millisecond the +/// HPET's tolerance allows a second, so a step of the 1.7 ms seen on the AMD +/// bench fails the sample after it. Three reads of the HPET, some 4 µs, once +/// a second on each CPU. +const CHECK_EVERY_NS: u64 = 1_000_000_000; + +/// Round trips of a cache line between the BSP and each AP as it starts. +const ROUNDS: u32 = 256; + +struct Hpet { + mmio: Mmio, + wide: bool, + reference: Reference, +} + +impl Hpet { + /// The main counter, as 64 bits: a 32-bit counter's reading is taken as + /// the one nearest `near` (`reference::extend`). + fn read(&self, near: u64) -> u64 { + if self.wide { + self.mmio.read_u64(HPET_COUNTER) + } else { + reference::extend(near, self.mmio.read_u32(HPET_COUNTER)) + } + } + + /// The main counter as it reads, a 32-bit one's from zero. + fn raw(&self) -> u64 { + if self.wide { self.mmio.read_u64(HPET_COUNTER) } else { u64::from(self.mmio.read_u32(HPET_COUNTER)) } + } + + /// The narrowest bracket of [`READS`] readings near `near`. + fn sample(&self, near: u64) -> Sample { + let mut best: Option = None; + for _ in 0..READS { + let before = cpu::rdtsc_ordered(); + let reference = self.read(near); + let after = cpu::rdtsc_ordered(); + let sample = Sample { before, after, reference }; + if best.is_none_or(|best| sample.width() < best.width()) { + best = Some(sample); + } + } + best.expect("READS is not zero") + } + + /// Where the counter should read now, from `last` and the rate the clock + /// keeps: what a 32-bit counter's wraps are counted against. + fn near(&self, last: Sample, mult: u64) -> u64 { + let nanos = Line { counter: 0, stamp: 0, mult }.nanos(cpu::counter().saturating_sub(last.before)); + last.reference + (u128::from(nanos) * 1_000_000 / u128::from(self.reference.period_fs)) as u64 + } +} + +/// What the runtime check holds, once the boot has started the clock. +struct Held { + hpet: Hpet, + watch: Watch, +} + +static HELD: Lock> = Lock::new(None); + +/// Whether CPUID states `IA32_TSC_ADJUST`, read once on the BSP. +static TSC_ADJUST: AtomicBool = AtomicBool::new(false); + +/// Each CPU's part of the check: when it next samples, and the +/// `IA32_TSC_ADJUST` it started with. +pub struct Local { + due_ns: AtomicU64, + adjust: AtomicU64, + /// What `tsc-step` adds to this CPU's readings. + #[cfg(feature = "boot-actuators")] + staged: AtomicU64, +} + +impl Local { + pub const fn new() -> Self { + Self { + due_ns: AtomicU64::new(0), + adjust: AtomicU64::new(0), + #[cfg(feature = "boot-actuators")] + staged: AtomicU64::new(0), + } + } +} + +/// Start the clock on the TSC, at its stated rate or one measured against the +/// HPET at `hpet_base`, and hand the HPET to the runtime check. +pub fn start(hpet_base: u64) { + admit("cpu0"); + TSC_ADJUST.store(cpu::has_tsc_adjust(), Ordering::Relaxed); + note_adjust(); + + let mmio = crate::mm::paging::map_mmio(hpet_base, 0x1000, MmioPolicy::Uncacheable); + let cap = mmio.read_u64(HPET_CAP); + let period_fs = cap >> 32; + assert!(period_fs > 0, "HPET: invalid counter period"); + let cfg = mmio.read_u64(HPET_CFG); + mmio.write_u64(HPET_CFG, cfg | 1); + let hpet = Hpet { mmio, wide: cap & COUNT_SIZE_CAP != 0, reference: Reference { period_fs, tolerance_ppb: HPET_TOLERANCE_PPB } }; + log!( + "clock: HPET at {hpet_base:#x} enabled, period={period_fs}fs, {} bits, counter reads {}", + if hpet.wide { 64 } else { 32 }, + hpet.raw(), + ); + + let (anchor, kept) = match cpu::stated_counter_hz() { + Some(hz) => { + let anchor = hpet.sample(hpet.raw()); + log!( + "clock: the TSC counts at {hz} Hz, as CPUID 15H states, held to the HPET from a reading {} cycles wide", + anchor.width() + ); + (anchor, Kept { mult: Line::mult_of_hz(hz), bound_ppb: 1, measured: false }) + } + None => calibrate(&hpet), + }; + crate::clock::set_counter(anchor.before, kept.mult); + *HELD.lock() = Some(Held { watch: Watch::new(hpet.reference, anchor, kept), hpet }); +} + +/// The TSC's rate against the HPET over [`CALIBRATION`], first sample to last, +/// refused where a sample between leaves the line, and the first sample. +fn calibrate(hpet: &Hpet) -> (Sample, Kept) { + let span_ticks = CALIBRATION.nanos() * 1_000_000 / hpet.reference.period_fs; + for attempt in 1..=CALIBRATION_TRIES { + // Every sample is within the window of the first, far under the 2^31 + // ticks a 32-bit counter's reading is taken near. + let first = hpet.sample(hpet.raw()); + let mut samples = [first; CALIBRATION_SAMPLES as usize]; + for (i, slot) in samples.iter_mut().enumerate().skip(1) { + let target = first.reference + span_ticks * i as u64 / (CALIBRATION_SAMPLES - 1); + wait_for(hpet, target, first); + *slot = hpet.sample(first.reference); + } + match reference::calibrate(hpet.reference, &samples) { + Ok(rate) => { + let line = Line { counter: 0, stamp: 0, mult: rate.mult }; + log!( + "clock: the TSC counts at {} mHz against the HPET over {} ms, within {} ppb; CPUID 15H states no rate, so the HPET goes on refining it", + line.millihertz(), + rate.span_ns / 1_000_000, + rate.bound_ppb, + ); + return (first, Kept { mult: rate.mult, bound_ppb: rate.bound_ppb, measured: true }); + } + Err(Calibration::Unmeasurable(why)) => { + log!("clock: calibration {attempt} measured nothing ({why:?}): {samples:?}") + } + Err(Calibration::Disturbed { index, agreement }) => log!( + "clock: calibration {attempt} refused: sample {index} sits {} ns off the line, {} allowed: {samples:?}", + agreement.apart_ns, + agreement.allowed_ns, + ), + } + } + panic!("clock: {CALIBRATION_TRIES} calibrations of the TSC against the HPET were each disturbed, so this machine keeps no time"); +} + +/// Spin until the HPET reads `target`. A main counter that does not advance +/// would spin here forever, and this is the boot's last wait before it has a +/// clock: the only unit available to bound it is the TSC's own, so the budget +/// is the calibration converted at a frequency no x86-64 part reaches, twice. +fn wait_for(hpet: &Hpet, target: u64, first: Sample) { + const TSC_CEILING_HZ: u64 = 10_000_000_000; + let budget = 2 * CALIBRATION.nanos() * (TSC_CEILING_HZ / 1_000_000_000); + while hpet.read(first.reference) < target { + assert!( + cpu::counter().wrapping_sub(first.after) <= budget, + "clock: the HPET main counter did not reach {target} in {budget} TSC cycles (it read {} at the \ + first sample and reads {}), so this machine offers no clock to calibrate against", + first.reference, + hpet.read(first.reference), + ); + } +} + +/// Refuse a CPU whose TSC this clock cannot keep time on. +fn admit(cpu: &str) { + assert!(cpu::has_rdtscp(), "clock: {cpu} has no RDTSCP, which every read the clock bounds is ordered by"); + if cpu::invariant_tsc() { + return; + } + assert!( + cpu::under_hypervisor(), + "clock: {cpu} does not state its TSC invariant (CPUID.80000007H:EDX[8]), so its rate may change with its \ + power state and no clock can be kept on it" + ); + log!("clock: {cpu}'s hypervisor does not state its TSC invariant; the HPET alone holds it"); +} + +fn note_adjust() { + if TSC_ADJUST.load(Ordering::Relaxed) { + crate::cpu_local::here().arch.clock.adjust.store(cpu::rdmsr(cpu::IA32_TSC_ADJUST), Ordering::Relaxed); + } +} + +/// This AP's part of its start, before it echoes: the same refusal as the BSP's. +pub fn start_ap(cpu_id: u32) { + admit(&alloc::format!("cpu{cpu_id}")); + note_adjust(); +} + +/// The round trips' line: whose turn it is, odd the AP's and even the BSP's, +/// and the counter its last writer read just before it passed the turn on. A +/// turn and not the counter's own value says whose word it is, because a +/// hypervisor's counter can read the same twice. +#[repr(align(64))] +struct Trip { + turn: AtomicU64, + counter: AtomicU64, +} + +static TRIP: Trip = Trip { turn: AtomicU64::new(0), counter: AtomicU64::new(0) }; +/// The AP's least trip out, once it has made every one. +static LEAST_OUT: AtomicU64 = AtomicU64::new(0); +static OUT_DONE: AtomicBool = AtomicBool::new(false); + +/// Pass the turn on as `turn`, with the counter read just before. +fn pass(turn: u64) { + TRIP.counter.store(cpu::rdtsc_ordered(), Ordering::Relaxed); + TRIP.turn.store(turn, Ordering::Release); +} + +/// Wait for `turn`, and answer the counter its writer read and the trip: +/// this CPU's counter now, less that one. +fn arrive(turn: u64) -> Option { + if !crate::clock::settles(AP_START.nanos(), || TRIP.turn.load(Ordering::Acquire) == turn) { + return None; + } + let theirs = TRIP.counter.load(Ordering::Relaxed); + Some(cpu::rdtsc_ordered().wrapping_sub(theirs) as i64) +} + +/// The BSP's half of [`ROUNDS`] round trips with the AP just started: how far +/// `cpu_id`'s TSC can be from this one's (`reference::Offset`). Panics on an +/// offset the trips prove, and on an AP that stops answering. +pub fn measure_offset(cpu_id: u32) { + let mut least_back = i64::MAX; + for round in 0..u64::from(ROUNDS) { + pass(2 * round + 1); + let back = arrive(2 * round + 2); + let back = back.unwrap_or_else(|| panic!("clock: cpu{cpu_id} stopped answering the round trips that bound its TSC at round {round}")); + least_back = least_back.min(back); + } + let done = crate::clock::settles(AP_START.nanos(), || OUT_DONE.load(Ordering::Acquire)); + assert!(done, "clock: cpu{cpu_id} never reported its round trips"); + OUT_DONE.store(false, Ordering::Relaxed); + TRIP.turn.store(0, Ordering::Relaxed); + let offset = Offset { least_out: LEAST_OUT.load(Ordering::Relaxed) as i64, least_back }; + match offset.bound() { + Ok((lo, hi)) => log!("clock: cpu{cpu_id}'s TSC is within {lo}..={hi} cycles of cpu0's, over {ROUNDS} round trips"), + Err(proven) => panic!( + "clock: cpu{cpu_id}'s TSC is {} cpu0's by at least {} cycles, which {ROUNDS} round trips prove ({offset:?})", + if proven < 0 { "behind" } else { "ahead of" }, + proven.unsigned_abs(), + ), + } +} + +/// The AP's half of [`measure_offset`], after its echo. +pub fn answer_offset() { + let mut least_out = i64::MAX; + for round in 0..u64::from(ROUNDS) { + let out = arrive(2 * round + 1); + let out = out.unwrap_or_else(|| { + panic!("clock: cpu{} waited for a round trip the BSP never sent, round {round}", super::percpu::cpu_id()) + }); + least_out = least_out.min(out); + pass(2 * round + 2); + } + LEAST_OUT.store(least_out as u64, Ordering::Relaxed); + OUT_DONE.store(true, Ordering::Release); +} + +/// This CPU's sample against the HPET, at most once a [`CHECK_EVERY_NS`]: +/// called from every scheduler pass, where a CPU not due pays one load. +pub fn check(now_ns: u64) { + let local = &crate::cpu_local::here().arch.clock; + if now_ns < local.due_ns.load(Ordering::Relaxed) { + return; + } + // Another CPU sampling now: this one samples on its next pass. + let Some(mut held) = HELD.try_lock() else { + return; + }; + let Some(held) = held.as_mut() else { + return; + }; + local.due_ns.store(now_ns + CHECK_EVERY_NS, Ordering::Relaxed); + let cpu_id = super::percpu::cpu_id(); + if TSC_ADJUST.load(Ordering::Relaxed) { + let started = local.adjust.load(Ordering::Relaxed); + let adjust = cpu::rdmsr(cpu::IA32_TSC_ADJUST); + assert!( + adjust == started, + "clock: cpu{cpu_id}'s TSC was written: IA32_TSC_ADJUST reads {} and read {} when it started", + adjust as i64, + started as i64, + ); + } + let sample = held.hpet.sample(held.hpet.near(held.watch.last(), held.watch.kept().mult)); + #[cfg(feature = "boot-actuators")] + let sample = staged(local, sample, false); + match held.watch.take(sample) { + Ok(None) => {} + Ok(Some(report)) => { + #[cfg(feature = "boot-actuators")] + staged(local, sample, true); + let kept = Line { counter: 0, stamp: 0, mult: report.kept.mult }; + if report.retuned { + crate::clock::retune(report.kept.mult); + } + log!( + "clock: over {} s the TSC counted {} mHz against the HPET, within {} ppb; the clock {} {} mHz, within {} ppb", + report.measured.span_ns / 1_000_000_000, + Line { counter: 0, stamp: 0, mult: report.measured.mult }.millihertz(), + report.measured.bound_ppb, + if report.retuned { "now keeps" } else if report.kept.measured { "keeps" } else { "keeps CPUID 15H's" }, + kept.millihertz(), + report.kept.bound_ppb, + ); + } + Err(Failure::Step { from, to, agreement }) => panic!( + "clock: cpu{cpu_id}'s TSC stepped: {} ns apart from the HPET over {} ns since the last sample, {} allowed \ + ({from:?} to {to:?})", + agreement.apart_ns, + agreement.reference_ns, + agreement.allowed_ns, + ), + Err(Failure::Drift { from, to, agreement }) => panic!( + "clock: cpu{cpu_id}'s TSC drifted from the rate the clock keeps: {} ns apart from the HPET over {} ns, {} \ + allowed ({from:?} to {to:?})", + agreement.apart_ns, + agreement.reference_ns, + agreement.allowed_ns, + ), + } +} + +/// `tsc-step`: from the first report on, the first CPU to report reads its +/// TSC 10 ms ahead of where it is. A hypervisor's TSC may ignore a write +/// (QEMU's TCG does), so the step is staged in the readings the check takes. +#[cfg(feature = "boot-actuators")] +fn staged(local: &Local, sample: Sample, reported: bool) -> Sample { + static STEPPED: AtomicBool = AtomicBool::new(false); + if crate::actuator::tsc_step() && reported && !STEPPED.swap(true, Ordering::Relaxed) { + let ahead = crate::clock::counter_ticks(10_000_000); + local.staged.store(ahead, Ordering::Relaxed); + log!("tsc-step: cpu{}'s TSC reads {ahead} cycles ahead from here", super::percpu::cpu_id()); + } + let ahead = local.staged.load(Ordering::Relaxed); + Sample { before: sample.before + ahead, after: sample.after + ahead, ..sample } +} diff --git a/kernel/src/arch/x86_64/cpu.rs b/kernel/src/arch/x86_64/cpu.rs index e7088b74331..f0f3d6d94be 100644 --- a/kernel/src/arch/x86_64/cpu.rs +++ b/kernel/src/arch/x86_64/cpu.rs @@ -50,6 +50,46 @@ pub fn rdtsc() -> u64 { (hi as u64) << 32 | lo as u64 } +/// The TSC read once every earlier instruction has executed and every earlier +/// load is globally visible, and before any later instruction starts: +/// `RDTSCP; LFENCE` (SDM Vol. 2B, RDTSCP). The clock's boot refuses a CPU +/// without RDTSCP ([`has_rdtscp`]). +#[inline] +pub fn rdtsc_ordered() -> u64 { + let lo: u32; + let hi: u32; + // SAFETY: reads the time-stamp counter and `IA32_TSC_AUX` into the declared outputs; unprivileged since CR4.TSD is clear. + unsafe { + asm!("rdtscp", "lfence", out("eax") lo, out("edx") hi, out("ecx") _, options(nostack, preserves_flags)); + } + (hi as u64) << 32 | lo as u64 +} + +/// `CPUID.80000001H:EDX[27]`. +pub fn has_rdtscp() -> bool { + cpuid(0x8000_0000, 0).0 >= 0x8000_0001 && cpuid(0x8000_0001, 0).3 & 1 << 27 != 0 +} + +/// `CPUID.80000007H:EDX[8]`, the invariant TSC: one rate in every P-, C- and +/// T-state (SDM Vol. 3B, "Invariant TSC"; AMD's PPRs, `TscInvariant`). +pub fn invariant_tsc() -> bool { + cpuid(0x8000_0000, 0).0 >= 0x8000_0007 && cpuid(0x8000_0007, 0).3 & 1 << 8 != 0 +} + +/// `CPUID.01H:ECX[31]`, which a hypervisor sets for its guest (APM Vol. 3, +/// 24594 rev 3.36, §E.3.2). +pub fn under_hypervisor() -> bool { + cpuid(1, 0).2 & 1 << 31 != 0 +} + +/// `CPUID.07H:EBX[1]`: `IA32_TSC_ADJUST`, to which every write to the TSC +/// since reset is added (SDM Vol. 3B, "Time-Stamp Counter Adjustment"). +pub fn has_tsc_adjust() -> bool { + leaf_7().1 & 1 << 1 != 0 +} + +pub const IA32_TSC_ADJUST: u32 = 0x3b; + /// `CPUID.01H:ECX[30]`; without it `RDRAND` is `#UD`, so [`rdrand`] may not be /// reached at all. #[inline] @@ -457,17 +497,10 @@ pub fn frame_pointer() -> u64 { rbp } -/// The TSC's frequency in hertz as CPUID *states* it, for the one caller that -/// may run before the clock is calibrated — the panic path, which has to bound a wait on a -/// machine that never reached the HPET. Nothing calibrates against it and no -/// third source is guessed at: a CPU that states neither leaf answers `None` -/// and its caller says so rather than inventing a rate. +/// The TSC's frequency in hertz as CPUID leaf 15H states it, or `None` on a +/// part that states none (`toyos_tsc::stated_hz`). pub fn stated_counter_hz() -> Option { - let max_leaf = cpuid(0, 0).0; - toyos_tsc::stated_hz( - (max_leaf >= 0x15).then(|| cpuid(0x15, 0)), - (max_leaf >= 0x16).then(|| cpuid(0x16, 0)), - ) + toyos_tsc::stated_hz((cpuid(0, 0).0 >= 0x15).then(|| cpuid(0x15, 0))) } diff --git a/kernel/src/arch/x86_64/hpet.rs b/kernel/src/arch/x86_64/hpet.rs deleted file mode 100644 index 76e8a704fef..00000000000 --- a/kernel/src/arch/x86_64/hpet.rs +++ /dev/null @@ -1,97 +0,0 @@ -//! The TSC, calibrated against the HPET: x86-64's free-running counter has no -//! stated rate every part can be trusted for, so the boot measures it. - -use crate::log; -use crate::mm::policy::MmioPolicy; -use crate::time::{Delay, Duration}; - -use super::cpu; - -const HPET_CAP: u64 = 0x000; -const HPET_CFG: u64 = 0x010; -const HPET_COUNTER: u64 = 0x0F0; - -/// Measure the TSC against the HPET at `hpet_base` and start the clock on it. -pub fn calibrate_counter(hpet_base: u64) { - let hpet = crate::mm::paging::map_mmio(hpet_base, 0x1000, MmioPolicy::Uncacheable); - - let cap = hpet.read_u64(HPET_CAP); - let hpet_period_fs = cap >> 32; - assert!(hpet_period_fs > 0, "HPET: invalid counter period"); - - let cfg = hpet.read_u64(HPET_CFG); - hpet.write_u64(HPET_CFG, cfg | 1); - - const CALIBRATION: Delay = Delay::to_measure( - Duration::from_millis(50), - "TSC ticks counted against the HPET; longer is a better ratio and boot time is what it costs", - ); - let calibration_ns = CALIBRATION.nanos(); - let calibration_hpet_ticks = calibration_ns * 1_000_000 / hpet_period_fs; - - let hpet_start = hpet.read_u64(HPET_COUNTER); - let tsc_start = cpu::counter(); - let hpet_target = hpet_start + calibration_hpet_ticks; - log!( - "clock: HPET at {:#x} enabled, period={}fs, counter reads {}, calibrating over {} ticks", - hpet_base, - hpet_period_fs, - hpet_start, - calibration_hpet_ticks, - ); - - // A main counter that does not advance would spin here forever, and this is - // the boot's last wait before it has a clock: the only unit available to - // bound it is the TSC's own, so the budget is the calibration converted at - // a frequency no x86-64 part reaches, which makes it an over-estimate of - // the cycles the calibration can legitimately take on any machine. - const TSC_CEILING_HZ: u64 = 10_000_000_000; - // Times two, so a machine merely slower than the ceiling is not refused for it. - let stall_budget_cycles = 2 * calibration_ns * (TSC_CEILING_HZ / 1_000_000_000); - while hpet.read_u64(HPET_COUNTER) < hpet_target { - assert!( - cpu::counter().wrapping_sub(tsc_start) <= stall_budget_cycles, - "clock: the HPET main counter at {:#x} did not reach {} in {} TSC cycles (it started \ - at {} and reads {}), so this machine offers no clock to calibrate against", - hpet_base, - hpet_target, - stall_budget_cycles, - hpet_start, - hpet.read_u64(HPET_COUNTER), - ); - } - let tsc_end = cpu::counter(); - let hpet_end = hpet.read_u64(HPET_COUNTER); - - let hpet_elapsed_fs = (hpet_end - hpet_start) as u128 * hpet_period_fs as u128; - let tsc_delta = tsc_end - tsc_start; - let tsc_period_fs = (hpet_elapsed_fs / tsc_delta as u128) as u64; - - crate::clock::set_counter(tsc_start, tsc_period_fs); - - let tsc_freq_mhz = 1_000_000_000_000_000u64 / tsc_period_fs / 1_000_000; - log!("TSC: {}MHz (period={}fs, calibrated over {}ms)", tsc_freq_mhz, tsc_period_fs, calibration_ns / 1_000_000); - - // **The one cross-source check this machine offers.** Everything else the - // kernel times is derived from the measurement just taken, so it could only - // agree with itself; CPUID 15H/16H is the part's own statement of the same - // frequency, arrived at by neither the HPET nor this counting loop, and the - // parts-per-million between the two is what a metal profile can hold a - // ceiling against. - let measured_hz = 1_000_000_000_000_000u64 / tsc_period_fs; - match cpu::stated_counter_hz() { - Some(stated) => { - let apart = measured_hz.abs_diff(stated); - log!( - "clock: TSC measured {measured_hz}Hz against the HPET, CPUID states {stated}Hz, \ - {}ppm apart", - apart * 1_000_000 / stated, - ); - } - None => log!( - "clock: TSC measured {measured_hz}Hz against the HPET; CPUID leaves 15H and 16H \ - stating no frequency, so nothing independent confirms it" - ), - } -} - diff --git a/kernel/src/arch/x86_64/idle.rs b/kernel/src/arch/x86_64/idle.rs index 67dc25b9e43..d644ec26407 100644 --- a/kernel/src/arch/x86_64/idle.rs +++ b/kernel/src/arch/x86_64/idle.rs @@ -30,13 +30,12 @@ pub fn facts() -> CpuIdle { } else { (0, 0) }; - let invariant_tsc = cpu::cpuid(0x8000_0000, 0).0 >= 0x8000_0007 && cpu::cpuid(0x8000_0007, 0).3 & 1 << 8 != 0; CpuIdle { monitor: cpuid_1_ecx & 1 << 3 != 0, mwait_extensions: cpuid_5_ecx & 1 != 0, mwait_substates: cpuid_5_edx, always_running_timer: cpu::leaf_6().0 & 1 << 2 != 0, - invariant_tsc, + invariant_tsc: cpu::invariant_tsc(), } } diff --git a/kernel/src/arch/x86_64/mod.rs b/kernel/src/arch/x86_64/mod.rs index e6d5e826f45..6c744b9c6fd 100644 --- a/kernel/src/arch/x86_64/mod.rs +++ b/kernel/src/arch/x86_64/mod.rs @@ -15,6 +15,7 @@ pub mod apic; pub mod barrier; pub mod boot; pub mod cache; +pub mod clocksource; pub mod control_regs; pub mod counters; pub mod console_uart; @@ -22,7 +23,6 @@ pub mod cpu; pub mod entropy; pub mod entry; pub mod fpu; -pub mod hpet; pub mod hw; pub mod idle; pub mod idt; @@ -123,6 +123,7 @@ pub unsafe fn percpu_fetch_add( /// This architecture's part of a CPU's `cpu_local` record. pub struct CpuLocal { pub counters: counters::Local, + pub clock: clocksource::Local, pub idle_line: idle::Line, #[cfg(feature = "switch-witness")] pub hw: hw::Local, @@ -132,6 +133,7 @@ impl CpuLocal { pub fn new() -> Self { Self { counters: counters::Local::new(), + clock: clocksource::Local::new(), idle_line: idle::Line::new(), #[cfg(feature = "switch-witness")] hw: hw::Local::new(), diff --git a/kernel/src/arch/x86_64/smp.rs b/kernel/src/arch/x86_64/smp.rs index 5b4b9d72a61..79a8f27f9c4 100644 --- a/kernel/src/arch/x86_64/smp.rs +++ b/kernel/src/arch/x86_64/smp.rs @@ -1,6 +1,5 @@ use core::arch::global_asm; use core::mem::size_of; -use core::sync::atomic::{AtomicU64, Ordering}; use crate::arch::cpu; use crate::arch::{apic, percpu, syscall}; @@ -18,18 +17,8 @@ const TRAMPOLINE_PAGE: u64 = 0x8000; const TRAMPOLINE_VECTOR: u8 = 0x08; const DATA_OFFSET: usize = 0xF00; -/// The `rdtsc` the AP being started read on its first instruction in Rust. -/// -/// **The whole of the TSC-trail measurement.** `clock::nanos_since_boot` -/// subtracts one BSP-sampled `COUNTER_AT_BOOT` whatever CPU asks, so a CPU whose TSC -/// reads below the BSP's has no honest answer to give; the reading is bracketed -/// by two BSP samples taken either side of this AP's whole bring-up, which is -/// serialised, so a synchronised TSC lands inside the bracket by construction -/// and a skewed one is outside it by at least the skew. -static AP_TSC: AtomicU64 = AtomicU64::new(0); - /// The started AP's [`cpu::topology_keys`], written before its echo and taken -/// after it, like [`AP_TSC`]: bring-up is serialised, so one slot serves. +/// after it: bring-up is serialised, so one slot serves. static AP_TOPOLOGY: Lock>> = Lock::new(None); // Field offsets are hardcoded in the global_asm! trampoline below; the static assertion at the bottom checks the match. @@ -194,7 +183,6 @@ pub fn boot_aps(madt: &MadtInfo, boot_cr3: u64) { data.cr3 = boot_cr3; let target = crate::DirectMap::from_phys(TRAMPOLINE_PAGE + DATA_OFFSET as u64).as_mut_ptr::(); - let mut bracketed = 0u32; for &ap_id in &madt.apic_ids { if ap_id == bsp_id { continue; } @@ -208,11 +196,6 @@ pub fn boot_aps(madt: &MadtInfo, boot_cr3: u64) { // SAFETY: `target` is reserved physical memory written only here; unaligned because `TrampolineData` is `repr(C, packed)`; this AP has not been sent its SIPI yet, and the loop reaches a second write only after the previous AP committed — which happens in `ap_entry` past every trampoline read — so no CPU is reading it; a failed AP breaks the loop below instead of reaching another write. unsafe { core::ptr::write_unaligned(target, data); } - AP_TSC.store(0, Ordering::Release); - // The bracket's lower edge: taken after the previous AP committed and - // before this one is sent anything, so nothing this AP does precedes it. - let bracket_lo = cpu::rdtsc(); - // Both delays are spent, not waited on: nothing is polled across either. const AFTER_INIT: Delay = Delay::from_spec(Duration::from_millis(10), "SDM §8.4.4.1, after the INIT IPI"); @@ -234,10 +217,7 @@ pub fn boot_aps(madt: &MadtInfo, boot_cr3: u64) { let deadline = clock::nanos_since_boot() + AP_START.nanos(); // Commit only on this attempt's own token, so `0..cpu_count()` stays dense. if ROSTER.await_echo(attempt, || clock::nanos_since_boot() >= deadline) { - let bracket_hi = cpu::rdtsc(); - if tsc_inside(attempt.id(), bracket_lo, bracket_hi) { - bracketed += 1; - } + super::clocksource::measure_offset(attempt.id()); ROSTER.commit(attempt, ap_id); topology.push(AP_TOPOLOGY.lock().take().expect("an AP reads its topology before it echoes")); log!("SMP: AP cpu{} lapic={} online", attempt.id(), ap_id); @@ -249,9 +229,8 @@ pub fn boot_aps(madt: &MadtInfo, boot_cr3: u64) { } let online = smp::cpu_count(); log!( - "SMP: {online} of {} MADT cpus online, {bracketed} of {} APs inside the BSP's TSC bracket", + "SMP: {online} of {} MADT cpus online", madt.apic_ids.len(), - online.saturating_sub(1), ); crate::arch::control_regs::report(online); smp::set_topology( @@ -263,41 +242,7 @@ pub fn boot_aps(madt: &MadtInfo, boot_cr3: u64) { ); } -/// Whether the AP just started read a TSC between the two the BSP took either -/// side of its bring-up, and the record of where it read. -/// -/// Cycles, not nanoseconds: the AP's own conversion is what is under test, and -/// the reading is a raw `rdtsc` taken before this CPU has run `clock`'s -/// arithmetic at all. -fn tsc_inside(cpu_id: u32, lo: u64, hi: u64) -> bool { - let sample = AP_TSC.load(Ordering::Acquire); - let span = hi.wrapping_sub(lo); - if sample >= lo && sample <= hi { - log!("SMP: cpu{cpu_id} tsc={sample} inside the BSP's {lo}..{hi} ({span} cycles wide)"); - return true; - } - // Spelled whole in each arm rather than with the verb as a field: the host - // profile holds these two words against this file's source, and a literal - // assembled at run time is one no reader of the source can find. - if sample < lo { - log!( - "SMP: cpu{cpu_id} tsc={sample} trails the BSP's {lo}..{hi} by {} cycles", - lo - sample - ); - } else { - log!( - "SMP: cpu{cpu_id} tsc={sample} leads the BSP's {lo}..{hi} by {} cycles", - sample - hi - ); - } - false -} - extern "C" fn ap_entry() -> ! { - // First, so the reading is this CPU's TSC and not a measure of what runs - // after it; `rdtsc` needs nothing this AP has not got out of the trampoline. - AP_TSC.store(cpu::rdtsc(), Ordering::Release); - // Must run before `pat::init`, which restores the CR0 this call sets. crate::arch::control_regs::init_paging(percpu::cpu_id()); // Before `control_regs::init` reads what an update changes. @@ -314,12 +259,15 @@ extern "C" fn ap_entry() -> ! { syscall::init(); // Calibration is a one-time BSP measurement; nothing left for an AP to do here. apic::init_ap(); + super::clocksource::start_ap(percpu::cpu_id()); // Before the echo, so every CPU the roster counts has said its line. crate::arch::mtrr::compare(percpu::cpu_id()); *AP_TOPOLOGY.lock() = Some(cpu::topology_keys()); // Echo this attempt's token, so the BSP counts this AP for its own attempt. ROSTER.echo(percpu::ap_token()); + // The BSP's next step once it sees the echo. + super::clocksource::answer_offset(); process::ap_idle(); } diff --git a/kernel/src/clock.rs b/kernel/src/clock.rs index 409c11d8012..1ef51999f9c 100644 --- a/kernel/src/clock.rs +++ b/kernel/src/clock.rs @@ -1,49 +1,93 @@ //! The machine's clocks: monotonic since boot, read off the CPU's free-running -//! counter at the period the architecture's boot gives [`set_counter`] (on -//! x86-64, measured against the HPET); wall-clock, the architecture's one -//! reading of the RTC — on x86-64 a CMOS read, which can block for up to a -//! second, and on AArch64 firmware's, which the loader hands over — anchored -//! in [`init_wall`] and answered after as that reading plus -//! [`nanos_since_boot`]; and the log's stamp ([`stamp`]), the same clock -//! counted from the counter's zero, which is where every line of the log — -//! the loader's, the kernel's and every program's — counts from. - -use core::sync::atomic::{AtomicBool, AtomicU64, Ordering::{Acquire, Relaxed, Release}}; +//! counter along the line the architecture's boot starts in [`set_counter`] +//! and may retune once it measures the counter's rate better ([`retune`]); +//! wall-clock, the architecture's one reading of the RTC — on x86-64 a CMOS +//! read, which can block for up to a second, and on AArch64 firmware's, which +//! the loader hands over — anchored in [`init_wall`] and answered after as +//! that reading plus [`nanos_since_boot`]; and the log's stamp ([`stamp`]), the +//! same clock counted from the counter's zero, which is where every line of +//! the log — the loader's, the kernel's and every program's — counts from. +//! +//! The line lives on the clock page every process maps +//! (`toyos_abi::clock`), and the kernel reads it there too, so the two clocks +//! are one. + +use core::sync::atomic::{AtomicBool, AtomicPtr, AtomicU64, Ordering::{Acquire, Relaxed, Release}}; + +use alloc::sync::Arc; +use toyos_abi::clock::{Clock, ClockPage, Line}; use crate::arch::cpu; use crate::log::LogStamp; +use crate::object::shm::SharedMemObject; +use crate::sync::Lock; use crate::time::Instant; -use kernel::clock::{anchor, ticks_to_nanos}; +use kernel::clock::anchor; use toyos_wallclock::Civil; -static COUNTER_AT_BOOT: AtomicU64 = AtomicU64::new(0); -static COUNTER_PERIOD_FS: AtomicU64 = AtomicU64::new(0); -/// The counter's zero to `COUNTER_AT_BOOT`, in nanoseconds. -static STAMP_AT_BOOT: AtomicU64 = AtomicU64::new(0); -/// One counter tick at the rate the CPU states, in femtoseconds; zero where it -/// states none, or before [`state`]. -static STATED_FS: AtomicU64 = AtomicU64::new(0); +/// The [`Line::mult`] of the rate the CPU states; zero where it states none, +/// or before [`state`]. +static STATED_MULT: AtomicU64 = AtomicU64::new(0); + +/// The clock page through the direct map, null before [`set_counter`]. +static LINES: AtomicPtr = AtomicPtr::new(core::ptr::null_mut()); + +/// The clock page as an object, for every address space to map. +static PAGE: AtomicPtr> = AtomicPtr::new(core::ptr::null_mut()); + +/// The page's one writer at a time. +static RETUNING: Lock<()> = Lock::new(()); /// Take the rate the CPU states, the loader's, for the stamps the log carries /// before [`set_counter`]. The kernel's entry calls this before its first record. pub fn state() { - let stated = cpu::stated_counter_hz().map_or(0, |hz| 1_000_000_000_000_000 / hz); - STATED_FS.store(stated, Relaxed); + let stated = cpu::stated_counter_hz().map_or(0, Line::mult_of_hz); + STATED_MULT.store(stated, Relaxed); } -/// Start the clock on the counter: its reading at boot and its measured period. -/// The architecture's boot calls this once, after it has the period. +/// Start the clock on the counter: its reading at boot and its rate, as a +/// [`Line::mult`]. The architecture's boot calls this once, after it has the +/// rate, and before any address space is built. /// /// **The stamps go on from where the stated rate left them**: a record before /// this read the counter at that rate, so the stamp at `boot` is that reading, /// and no record after this one is stamped earlier than one before it. -pub fn set_counter(boot: u64, period_fs: u64) { - let stated = STATED_FS.load(Relaxed); - let at_boot = ticks_to_nanos(boot, if stated != 0 { stated } else { period_fs }); - STAMP_AT_BOOT.store(at_boot, Relaxed); - COUNTER_AT_BOOT.store(boot, Relaxed); - COUNTER_PERIOD_FS.store(period_fs, Relaxed); - publish_page(boot, period_fs, at_boot); +pub fn set_counter(boot: u64, mult: u64) { + let stated = STATED_MULT.load(Relaxed); + let at_boot = Line { counter: 0, stamp: 0, mult: if stated != 0 { stated } else { mult } }.nanos(boot); + let page = SharedMemObject::create(crate::mm::charge::boot(), crate::mm::PAGE_SIZE).expect("clock: no frame for the clock page"); + let lines = crate::DirectMap::from_phys(page.page(0)).as_mut_ptr::(); + // SAFETY: the page is this object's own, which no address space maps yet + // and none unmaps after: held below for the machine's life. Every field of + // `ClockPage` is an atomic, so a shared reference reads and writes it as + // atomics while processes read it too. + let lines = unsafe { &*lines }; + lines.init(at_boot, Line { counter: boot, stamp: at_boot, mult }); + PAGE.store(alloc::boxed::Box::into_raw(alloc::boxed::Box::new(page)), Release); + LINES.store(lines as *const ClockPage as *mut ClockPage, Release); +} + +/// Go on at `mult` from now: the line from here starts where the clock +/// stands, so it bends and never steps. The line now in force. +pub fn retune(mult: u64) -> Line { + let page = lines().expect("clock: retuned before it started"); + let _one = RETUNING.lock(); + let now = cpu::counter(); + let at = page.read().expect("clock: the page `set_counter` laid out carries no clock").stamp_of(now); + let line = Line { counter: now, stamp: at, mult }; + page.publish(line); + line +} + +fn lines() -> Option<&'static ClockPage> { + let page = LINES.load(Acquire); + // SAFETY: `set_counter` stored a pointer into a page it holds for the + // machine's life, and the pointer is never cleared. + (!page.is_null()).then(|| unsafe { &*page }) +} + +fn clock() -> Option { + lines().and_then(ClockPage::read) } /// Now as a log line's time: nanoseconds since the counter's zero — power-on, @@ -52,30 +96,11 @@ pub fn set_counter(boot: u64, period_fs: u64) { /// started: no rate reads the counter yet. Never panics, as /// [`nanos_since_boot`]. pub fn stamp() -> Option { - if calibrated() { - return Some(LogStamp::since_zero(STAMP_AT_BOOT.load(Relaxed).saturating_add(nanos_since_boot()))); + if let Some(clock) = clock() { + return Some(LogStamp::since_zero(clock.stamp_of(cpu::counter()))); } - let stated = STATED_FS.load(Relaxed); - (stated != 0).then(|| LogStamp::since_zero(ticks_to_nanos(cpu::counter(), stated))) -} - -/// The clock page, published once by [`set_counter`]: the object every address -/// space maps read-only at [`toyos_abi::clock::CLOCK_PAGE`]. -static PAGE: core::sync::atomic::AtomicPtr> = - core::sync::atomic::AtomicPtr::new(core::ptr::null_mut()); - -/// Lay the calibration out on a page of its own, for every process to read -/// the clock this module reads without asking it. Laid out before any address -/// space can map it, and never written again. -fn publish_page(counter_at_boot: u64, period_fs: u64, stamp_at_boot: u64) { - use toyos_abi::clock::{ClockPage, CLOCK_MAGIC}; - let page = crate::object::shm::SharedMemObject::create(crate::mm::charge::boot(), crate::mm::PAGE_SIZE).expect("clock: no frame for the clock page"); - let value = ClockPage { magic: CLOCK_MAGIC, counter_at_boot, period_fs, stamp_at_boot }; - // SAFETY: `ClockPage` is four `u64`s, `repr(C)`: every byte is a field's. - let bytes = unsafe { core::slice::from_raw_parts((&value as *const ClockPage).cast::(), core::mem::size_of::()) }; - page.write(0, bytes); - // Held for the machine's life: every process maps it. - PAGE.store(alloc::boxed::Box::into_raw(alloc::boxed::Box::new(page)), Release); + let stated = STATED_MULT.load(Relaxed); + (stated != 0).then(|| LogStamp::since_zero(Line { counter: 0, stamp: 0, mult: stated }.nanos(cpu::counter()))) } /// Map the clock page into a fresh address space, read-only, at the address @@ -83,25 +108,24 @@ fn publish_page(counter_at_boot: u64, period_fs: u64, stamp_at_boot: u64) { pub fn map_page(space: &mut crate::mm::space::AddressSpace) { let page = PAGE.load(Acquire); assert!(!page.is_null(), "clock: an address space was built before the clock page"); - // SAFETY: `publish_page` stored a box it never frees, and the pointer is + // SAFETY: `set_counter` stored a box it never frees, and the pointer is // never cleared. - let page = alloc::sync::Arc::clone(unsafe { &*page }); + let page = Arc::clone(unsafe { &*page }); let at = crate::mm::space::At::Fixed(crate::UserAddr::new(toyos_abi::clock::CLOCK_PAGE)); let len = page.size(); space.map(page, 0, len, kernel::vm::Prot::Read, at).expect("clock: the clock page's address is a fresh space's"); } -/// Whether [`nanos_since_boot`] measures anything yet; false before [`init`]. +/// Whether [`nanos_since_boot`] measures anything yet; false before [`set_counter`]. pub fn calibrated() -> bool { - COUNTER_PERIOD_FS.load(Relaxed) != 0 + !LINES.load(Relaxed).is_null() } - /// Nanoseconds since boot; lock-free, no MMIO, and never panics — `log::emit` /// reads it from inside a bracket where panicking would reenter the log. -/// Saturating, not wrapping: a trailing CPU reads as oldest, not lying newest after a 584-year wrap. +/// Zero before [`set_counter`], and a CPU whose counter trails boot reads boot. pub fn nanos_since_boot() -> u64 { - ticks_to_nanos(cpu::counter().saturating_sub(COUNTER_AT_BOOT.load(Relaxed)), COUNTER_PERIOD_FS.load(Relaxed)) + clock().map_or(0, |clock| clock.nanos_since_boot(cpu::counter())) } /// The same reading as an [`Instant`], the type arithmetic on it is allowed in. @@ -118,29 +142,25 @@ pub fn counter_deadline(nanos: u64) -> u64 { /// `nanos` as a count of [`cpu::counter`] ticks: a span converted once and then compared /// against counter differences, which is what a sampler that may not divide -/// needs. Before [`init`] the period is unknown and this is zero, so a bound +/// needs. Before [`set_counter`] the rate is unknown and this is zero, so a bound /// derived from it is one its arm has to refuse. pub fn counter_ticks(nanos: u64) -> u64 { - let period_fs = COUNTER_PERIOD_FS.load(Relaxed); - if period_fs == 0 { - return 0; - } - ((nanos as u128 * 1_000_000) / period_fs as u128) as u64 + clock().map_or(0, |clock| clock.line.ticks(nanos)) } /// The span a count of [`counter_ticks`] stands for, for a caller that measured -/// before there was a period to measure with and converts once, afterwards. -/// Zero while the period is unknown, so a span taken on a machine that never +/// before there was a rate to measure with and converts once, afterwards. +/// Zero while the rate is unknown, so a span taken on a machine that never /// calibrated reads as no time rather than as an invented one. pub fn nanos_of_ticks(ticks: u64) -> u64 { - ticks_to_nanos(ticks, COUNTER_PERIOD_FS.load(Relaxed)) + clock().map_or(0, |clock| clock.line.nanos(ticks)) } /// Polls `ready` until it holds or `nanos` pass; `false` is the deadline. /// Reads the counter, not [`nanos_since_boot`], because that clock's out-of-line divide /// would land an NMI sample's PC in `compiler_builtins` rather than in the wait. /// -/// **Before [`init`] there is no period to measure a span with, so this asks +/// **Before [`set_counter`] there is no rate to measure a span with, so this asks /// `ready` once and answers it.** A wait nothing can bound is the one thing an /// unattended machine may not enter: a device denied its recovery window is a /// `false` its caller reports, and a spin nothing ends reports nothing at all. @@ -175,7 +195,8 @@ pub fn init_wall(reading: Result<(Civil, u64), impl core::fmt::Display>) { } }; - BOOT_SECS.store(anchor(civil.to_unix_secs(), at, COUNTER_AT_BOOT.load(Relaxed), COUNTER_PERIOD_FS.load(Relaxed)), Relaxed); + let clock = clock().expect("clock: the wall clock anchored before the counter started"); + BOOT_SECS.store(anchor(civil.to_unix_secs(), clock.stamp_of(at), clock.stamp_at_boot), Relaxed); WALL_KNOWN.store(true, Release); log!("clock: the RTC reads {civil} UTC"); } diff --git a/kernel/src/sched/driver.rs b/kernel/src/sched/driver.rs index 18a482ce350..3f0364fd58c 100644 --- a/kernel/src/sched/driver.rs +++ b/kernel/src/sched/driver.rs @@ -519,6 +519,7 @@ pub fn pass(dispose: Dispose) { crate::object::drain_zero_handles(); let now = HW.now(); crate::arch::watchdog::feed(now.0); + crate::arch::clocksource::check(now.0); #[cfg(feature = "heap-sweep")] maybe_sweep(now); #[cfg(feature = "pass-spin")] diff --git a/lib/trace/src/lib.rs b/lib/trace/src/lib.rs index 15e3abb1afb..66699e27360 100644 --- a/lib/trace/src/lib.rs +++ b/lib/trace/src/lib.rs @@ -4,9 +4,9 @@ //! A record crossed the syscall boundary or came off a file, so it is input: //! [`Entry::decode`] refuses a kind the ABI does not name, and a kind that //! names its own task but carries no thread, each by name. A stamp is the -//! writing CPU's counter, which the machine's `ClockPage` reads as a log -//! line's time, from the counter's zero on the kernel's own formula, so a -//! diary and a log line order against each other. +//! writing CPU's counter, which the machine's `Clock` reads as a log line's +//! time, from the counter's zero on the kernel's own formula, so a diary and a +//! log line order against each other. //! //! Pure: `core` only, no `unsafe`, no I/O. @@ -15,7 +15,7 @@ use core::fmt; -use toyos_abi::clock::ClockPage; +use toyos_abi::clock::Clock; use toyos_abi::trace::{Kind, TraceRecord, NO_THREAD}; /// A thread, by its process's id and its own, each as the writing CPU held @@ -129,7 +129,7 @@ impl Entry { /// The line this entry prints as, its stamp read on `clock`: /// `[12.000345678 cpu3] wake 7/0`. - pub fn line(&self, clock: ClockPage) -> Line<'_> { + pub fn line(&self, clock: Clock) -> Line<'_> { Line { entry: self, clock } } } @@ -137,7 +137,7 @@ impl Entry { /// What [`Entry::line`] renders. pub struct Line<'a> { entry: &'a Entry, - clock: ClockPage, + clock: Clock, } impl fmt::Display for Line<'_> { @@ -188,10 +188,8 @@ mod tests { /// A 1 GHz counter that read 1000 at boot, two seconds after its zero: /// one tick a nanosecond. - const CLOCK: ClockPage = ClockPage { - magic: toyos_abi::clock::CLOCK_MAGIC, - counter_at_boot: 1_000, - period_fs: 1_000_000, + const CLOCK: Clock = Clock { + line: toyos_abi::clock::Line { counter: 1_000, stamp: 2_000_000_000, mult: 1 << toyos_abi::clock::MULT_SHIFT }, stamp_at_boot: 2_000_000_000, }; diff --git a/lib/tsc/Cargo.toml b/lib/tsc/Cargo.toml index 23094e14759..ee0c03fcbed 100644 --- a/lib/tsc/Cargo.toml +++ b/lib/tsc/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "toyos-tsc" -description = "The rate x86-64's time-stamp counter counts at, as CPUID leaves 15H and 16H state it, pure." +description = "The rate x86-64's time-stamp counter counts at, as CPUID leaf 15H states it, pure." version = "0.1.0" edition = "2024" license = "MIT OR Apache-2.0" diff --git a/lib/tsc/src/lib.rs b/lib/tsc/src/lib.rs index e597f923dad..94b3b26f373 100644 --- a/lib/tsc/src/lib.rs +++ b/lib/tsc/src/lib.rs @@ -8,32 +8,24 @@ pub type Leaf = (u32, u32, u32, u32); /// SDM Vol. 2A, CPUID leaf 15H: EAX is the denominator and EBX the numerator of -/// the core crystal's ratio to the TSC, ECX the crystal's hertz — any of the -/// three reading zero means the leaf states nothing. Leaf 16H's EAX is the -/// processor base frequency in MHz, which an invariant TSC counts at. `None` -/// for a leaf the CPU does not implement, and as the answer where neither -/// states a rate: nothing here guesses one. -pub const fn stated_hz(leaf15: Option, leaf16: Option) -> Option { - if let Some((denominator, numerator, crystal_hz, _)) = leaf15 - && denominator != 0 - && numerator != 0 - && crystal_hz != 0 - { - return Some(crystal_hz as u64 * numerator as u64 / denominator as u64); +/// the TSC's ratio to the core crystal, ECX the crystal's hertz, and the TSC +/// counts at their product; any of the three reading zero states no rate. +/// `None` for a CPU without the leaf. Leaf 16H's base frequency is no rate: +/// the SDM confines its data to display. +pub const fn stated_hz(leaf15: Option) -> Option { + match leaf15 { + Some((denominator, numerator, crystal_hz, _)) if denominator != 0 && numerator != 0 && crystal_hz != 0 => { + Some(crystal_hz as u64 * numerator as u64 / denominator as u64) + } + _ => None, } - if let Some((base_mhz, _, _, _)) = leaf16 - && base_mhz != 0 - { - return Some(base_mhz as u64 * 1_000_000); - } - None } const _: () = { - // The ratio, then the fall-through to the base frequency when the crystal - // is not enumerated, then the CPU that states neither. - assert!(matches!(stated_hz(Some((2, 4, 25_000_000, 0)), None), Some(50_000_000))); - assert!(matches!(stated_hz(Some((0, 0, 0, 0)), Some((2_400, 0, 0, 0))), Some(2_400_000_000))); - assert!(stated_hz(None, Some((0, 0, 0, 0))).is_none()); - assert!(stated_hz(None, None).is_none()); + // A 38.4 MHz crystal at 63/1. + assert!(matches!(stated_hz(Some((1, 63, 38_400_000, 0))), Some(2_419_200_000))); + assert!(matches!(stated_hz(Some((2, 4, 25_000_000, 0))), Some(50_000_000))); + // A crystal the leaf does not enumerate states nothing. + assert!(stated_hz(Some((2, 176, 0, 0))).is_none()); + assert!(stated_hz(None).is_none()); }; diff --git a/loader/src/arch/x86_64.rs b/loader/src/arch/x86_64.rs index 4903d66aa7a..fe23de90e84 100644 --- a/loader/src/arch/x86_64.rs +++ b/loader/src/arch/x86_64.rs @@ -29,8 +29,7 @@ pub fn counter_hz() -> Option { let r = __cpuid(at); (r.eax, r.ebx, r.ecx, r.edx) }; - let max = leaf(0).0; - toyos_tsc::stated_hz((max >= 0x15).then(|| leaf(0x15)), (max >= 0x16).then(|| leaf(0x16))) + toyos_tsc::stated_hz((leaf(0).0 >= 0x15).then(|| leaf(0x15))) } /// This CPU as its vendor's microcode names it, or why it loads none. diff --git a/sdk/abi/Cargo.toml b/sdk/abi/Cargo.toml index 8eda9709e01..bcc434f233e 100644 --- a/sdk/abi/Cargo.toml +++ b/sdk/abi/Cargo.toml @@ -8,6 +8,13 @@ repository = "https://github.com/ToyOSOrg/ToyOS" [features] rustc-dep-of-std = ["core"] +# Declared, never enabled here: `kernel-loom` compiles `src/clock/latch.rs` +# into its own crate with `loom` on, so its atomics resolve to loom's, and with +# `clock-latch-fence-off`, its negative control, which drops the writer's +# fence and must red `kernel/loom/tests/clock_latch.rs`. Declared only so `cfg` +# checking knows the names. +loom = [] +clock-latch-fence-off = [] [dependencies] core = { version = "1.0.0", optional = true, package = "rustc-std-workspace-core" } diff --git a/sdk/abi/src/arch/aarch64.rs b/sdk/abi/src/arch/aarch64.rs index 4fc56f7835c..2f2c6fe4dca 100644 --- a/sdk/abi/src/arch/aarch64.rs +++ b/sdk/abi/src/arch/aarch64.rs @@ -17,7 +17,7 @@ pub fn current_tid() -> Tid { Tid(unsafe { core::ptr::read_volatile((tp as usize + TCB_TID) as *const u32) }) } -/// The counter the clock page's two words describe: the generic timer's +/// The counter the clock page's line describes: the generic timer's /// virtual count, which EL0 may read, read after every earlier instruction, as /// the x86-64 one is. #[inline] diff --git a/sdk/abi/src/arch/x86_64.rs b/sdk/abi/src/arch/x86_64.rs index aae795211be..6d5e15e15ad 100644 --- a/sdk/abi/src/arch/x86_64.rs +++ b/sdk/abi/src/arch/x86_64.rs @@ -24,7 +24,7 @@ pub fn current_tid() -> Tid { Tid(tid) } -/// The counter the clock page's two words describe: the time-stamp counter, +/// The counter the clock page's line describes: the time-stamp counter, /// read after every earlier load has completed, so a stamp taken after reading /// another writer's record cannot be older than that record's. #[inline] diff --git a/sdk/abi/src/clock.rs b/sdk/abi/src/clock.rs index 396aaa632c4..48c764e3cce 100644 --- a/sdk/abi/src/clock.rs +++ b/sdk/abi/src/clock.rs @@ -1,16 +1,23 @@ //! The monotonic clock a process reads without a syscall. //! -//! The kernel calibrates the counter once at boot and never again, so the -//! numbers that turn a counter reading into nanoseconds since boot are -//! constants for the machine's life. It writes them into one page before the -//! first process exists and maps that page read-only at [`CLOCK_PAGE`] in every -//! address space; [`nanos_since_boot`] is the kernel's own arithmetic on the -//! same words, and [`stamp_ns`] its log's, so a stamp taken here and one taken -//! by the kernel's log order against each other. +//! The clock is a straight [`Line`] through the counter: from a counter +//! reading, the nanoseconds since the counter's zero, and the rate after it. +//! The kernel lays the line out on one page before the first process exists, +//! maps that page read-only at [`CLOCK_PAGE`] in every address space, and +//! replaces the line when it learns the counter's rate better, starting the +//! new one where the old one stands, so the clock never steps. [`page`] reads +//! the line in force through the page's [`latch`], which the kernel's own +//! clock reads too, so a stamp taken here and one taken by the kernel's log +//! order against each other. //! -//! **No sequence word, because nothing is ever rewritten.** A clock page that a -//! kernel adjusted would need one; this one is laid out before it is mapped and -//! is immutable after, which is the whole of its concurrency argument. +//! **Every word of the page is atomic.** The kernel writes it while processes +//! read it, so it is reached as atomics and never as plain memory. + +mod latch; + +use core::sync::atomic::{AtomicU64, Ordering}; + +pub use latch::Latch; /// Where every address space finds the page: below the lowest address an /// allocation or a placed `mmap` may take, so nothing a process maps can land @@ -19,49 +26,125 @@ pub const CLOCK_PAGE: u64 = 0x1_0000_0000; /// What the page opens with, so a reader that finds anything else refuses /// rather than computes a time out of it. -pub const CLOCK_MAGIC: u64 = 0x544F_594F_5343_4C4B; // "TOYOSCLK" +pub const CLOCK_MAGIC: u64 = 0x544F_594F_434C_4B32; // "TOYOCLK2" -/// The page's layout. Written once by the kernel before it is mapped anywhere. -#[repr(C)] +/// The binary point of [`Line::mult`]: nanoseconds per tick in 32.32 fixed +/// point, which keeps a 5 GHz counter's rate to a part in 10⁹ and a 1 Hz +/// counter's tick representable. +pub const MULT_SHIFT: u32 = 32; + +/// One straight piece of the clock. #[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub struct ClockPage { - pub magic: u64, - /// The counter's reading at nanosecond zero. - pub counter_at_boot: u64, - /// One counter tick, in femtoseconds. - pub period_fs: u64, - /// The counter's zero to `counter_at_boot`, in nanoseconds: what turns a - /// reading of this clock into a log line's time ([`stamp_ns`]). +pub struct Line { + /// The counter reading the line starts at. + pub counter: u64, + /// The clock at `counter`: nanoseconds since the counter's zero. + pub stamp: u64, + /// Nanoseconds per counter tick, times 2^[`MULT_SHIFT`]. + pub mult: u64, +} + +impl Line { + /// The [`Line::mult`] of a counter at `hz`, rounded to nearest. + pub const fn mult_of_hz(hz: u64) -> u64 { + let ns = 1_000_000_000u128 << MULT_SHIFT; + ((ns + hz as u128 / 2) / hz as u128) as u64 + } + + /// The rate [`Line::mult`] stands for, in millihertz. + pub const fn millihertz(&self) -> u64 { + ((1_000_000_000_000u128 << MULT_SHIFT) / self.mult as u128) as u64 + } + + /// `ticks` of the counter, in nanoseconds; saturating, which no count a + /// machine reaches in a century touches. + pub const fn nanos(&self, ticks: u64) -> u64 { + let nanos = (ticks as u128 * self.mult as u128) >> MULT_SHIFT; + if nanos > u64::MAX as u128 { + u64::MAX + } else { + nanos as u64 + } + } + + /// The counter ticks `nanos` takes, rounded down. + pub const fn ticks(&self, nanos: u64) -> u64 { + (((nanos as u128) << MULT_SHIFT) / self.mult as u128) as u64 + } + + /// The clock at `counter`. A reading before the line's start is carried + /// back along it, which is where a record stamped before the kernel last + /// replaced the line lands, and a CPU whose counter trails reads behind; + /// nothing reads before the counter's zero. + pub const fn stamp_of(&self, counter: u64) -> u64 { + if counter >= self.counter { + self.stamp.saturating_add(self.nanos(counter - self.counter)) + } else { + self.stamp.saturating_sub(self.nanos(self.counter - counter)) + } + } + + const fn words(&self) -> [u64; 3] { + [self.counter, self.stamp, self.mult] + } + + const fn of_words(words: [u64; 3]) -> Self { + Self { counter: words[0], stamp: words[1], mult: words[2] } + } +} + +/// The clock as one reading of the page found it. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Clock { + /// The line in force. + pub line: Line, + /// The clock at boot: what [`Clock::nanos_since_boot`] counts from. pub stamp_at_boot: u64, } -impl ClockPage { +impl Clock { /// A counter reading as a log line's time: nanoseconds since the counter's - /// zero, on the kernel's formula. + /// zero. pub const fn stamp_of(&self, counter: u64) -> u64 { - self.stamp_at_boot.saturating_add(nanos_between(self.counter_at_boot, self.period_fs, counter)) + self.line.stamp_of(counter) + } + + /// A counter reading as nanoseconds since boot; a counter that reads + /// before boot reads as boot. + pub const fn nanos_since_boot(&self, counter: u64) -> u64 { + self.stamp_of(counter).saturating_sub(self.stamp_at_boot) } } -/// Nanoseconds between `counter_at_boot` and `now`, on the kernel's formula. -/// -/// Saturating below boot: a counter read on a CPU that trails reads as the -/// oldest instant, never as a wrap 584 years ahead. -pub const fn nanos_between(counter_at_boot: u64, period_fs: u64, now: u64) -> u64 { - ticks_to_nanos(now.saturating_sub(counter_at_boot), period_fs) +/// The page's layout. +#[repr(C)] +pub struct ClockPage { + magic: AtomicU64, + stamp_at_boot: AtomicU64, + line: Latch<3>, } -/// `ticks` of a counter whose tick is `period_fs` femtoseconds, in nanoseconds: -/// `⌊ticks · period_fs / 10⁶⌋` in its low 64 bits, the one formula the kernel -/// and every clock-page reader share. -/// -/// Split at 10⁶ ticks, `⌊(q·10⁶ + r)·p / 10⁶⌋ = q·p + ⌊r·p / 10⁶⌋`, so it -/// takes two 64-bit divisions by a constant, which compile to multiplications, -/// in place of a 128-bit division, which is a library call. Exact for any tick -/// period under 18 ms (`10⁶ · period_fs < 2⁶⁴`). -pub const fn ticks_to_nanos(ticks: u64, period_fs: u64) -> u64 { - const FS_PER_NS: u64 = 1_000_000; - (ticks / FS_PER_NS).wrapping_mul(period_fs).wrapping_add(ticks % FS_PER_NS * period_fs / FS_PER_NS) +impl ClockPage { + /// Lay the page out: the kernel's, once, before any address space maps it. + pub fn init(&self, stamp_at_boot: u64, line: Line) { + self.stamp_at_boot.store(stamp_at_boot, Ordering::Relaxed); + self.line.publish(line.words()); + self.magic.store(CLOCK_MAGIC, Ordering::Release); + } + + /// Replace the line: the kernel's, one writer at a time. + pub fn publish(&self, line: Line) { + self.line.publish(line.words()); + } + + /// The clock now, or `None` on a page that carries none. + pub fn read(&self) -> Option { + if self.magic.load(Ordering::Acquire) != CLOCK_MAGIC { + return None; + } + let line = Line::of_words(self.line.read()); + Some(Clock { line, stamp_at_boot: self.stamp_at_boot.load(Ordering::Relaxed) }) + } } /// Nanoseconds since boot, off this process's clock page. @@ -69,9 +152,8 @@ pub const fn ticks_to_nanos(ticks: u64, period_fs: u64) -> u64 { /// # Panics /// As [`page`]. pub fn nanos_since_boot() -> u64 { - let page = page(); - let now = crate::arch::counter(); - nanos_between(page.counter_at_boot, page.period_fs, now) + let clock = page(); + clock.nanos_since_boot(crate::arch::counter()) } /// Now as a log line's time: nanoseconds since the counter's zero, the kernel @@ -84,61 +166,83 @@ pub fn stamp_ns() -> u64 { page().stamp_of(crate::arch::counter()) } -/// This process's clock page. +/// The clock, off this process's page. /// /// # Panics /// On a page that does not carry [`CLOCK_MAGIC`]: every address space the /// kernel builds maps one, so its absence is a kernel that broke the ABI. -pub fn page() -> ClockPage { +pub fn page() -> Clock { // SAFETY: the kernel maps the page read-only at `CLOCK_PAGE` in every - // address space before the first instruction runs, and never writes it - // again, so a volatile copy of the whole value is a read of constants. - let page = unsafe { core::ptr::read_volatile(CLOCK_PAGE as *const ClockPage) }; - assert!(page.magic == CLOCK_MAGIC, "the clock page at {CLOCK_PAGE:#x} carries no clock"); - page + // address space before the first instruction runs and never unmaps it; + // every field is an atomic, so a shared reference reads it as atomics. + let page = unsafe { &*(CLOCK_PAGE as *const ClockPage) }; + page.read().unwrap_or_else(|| panic!("the clock page at {CLOCK_PAGE:#x} carries no clock")) } #[cfg(test)] mod tests { use super::*; - /// The arithmetic the kernel's `clock::nanos_since_boot` does, so the two - /// clocks agree on every counter value: a 1 GHz counter is one tick per - /// nanosecond, and a counter behind boot reads zero. + /// A 1 GHz counter: one tick a nanosecond. + const GHZ: u64 = 1 << MULT_SHIFT; + + #[test] + fn a_line_counts_from_its_start() { + let line = Line { counter: 100, stamp: 5_000, mult: GHZ }; + assert_eq!(line.stamp_of(1_100), 6_000); + assert_eq!(line.stamp_of(100), 5_000); + // Carried back before the start, and never before the counter's zero. + assert_eq!(line.stamp_of(40), 4_940); + assert_eq!(Line { counter: 10_000, stamp: 5, mult: GHZ }.stamp_of(0), 0); + // Saturating at a counter no machine reaches. + assert_eq!(line.stamp_of(u64::MAX), u64::MAX); + } + #[test] - fn the_formula_is_the_kernels() { - assert_eq!(nanos_between(100, 1_000_000, 1_100), 1_000); - assert_eq!(nanos_between(100, 1_000_000, 50), 0); - // 2.5 GHz: 400 fs a tick. - assert_eq!(nanos_between(0, 400_000, 2_500_000_000), 1_000_000_000); - // No overflow at a counter the machine will not reach in a century. - assert_eq!(nanos_between(0, 1_000_000, u64::MAX), u64::MAX); - } - - /// The split formula is the 128-bit product's quotient for every period a - /// counter has, at every count: the ends, the split's own boundary, and a - /// spread between. + fn boot_reads_zero_and_before_boot_reads_boot() { + let clock = Clock { line: Line { counter: 1_000, stamp: 2_000_000_000, mult: GHZ }, stamp_at_boot: 2_000_000_000 }; + assert_eq!(clock.nanos_since_boot(1_000), 0); + assert_eq!(clock.nanos_since_boot(1_500), 500); + assert_eq!(clock.nanos_since_boot(999), 0); + } + + /// The rate round-trips through `mult` to a part in 10⁹ for every counter + /// a machine has, the generic timer's 24 MHz to a 5 GHz TSC, and a second + /// of ticks is a second to the nanosecond. #[test] - fn the_split_is_the_wide_quotient() { - let wide = |ticks: u64, fs: u64| ((ticks as u128 * fs as u128) / 1_000_000) as u64; - let mut x = 0x2545_f491_4f6c_dd1du64; - let mut next = || { - x = x.wrapping_mul(6_364_136_223_846_793_005).wrapping_add(1_442_695_040_888_963_407); - x - }; - let mut periods = std::vec![1, 999_999, 1_000_000, 1_000_001, 41_666_666, 413_393, 18_000_000_000_000]; - periods.extend((0..64).map(|_| 1 + next() % 100_000_000)); - let mut ticks = std::vec![0, 1, 999_999, 1_000_000, 1_000_001, u64::MAX - 1, u64::MAX]; - ticks.extend((0..256).map(|_| next())); - for &fs in &periods { - for &t in &ticks { - assert_eq!(ticks_to_nanos(t, fs), wide(t, fs), "{t} ticks of {fs} fs"); - } + fn mult_keeps_the_rate() { + for hz in [24_000_000, 1_796_624_812, 2_419_200_000, 5_000_000_000] { + let line = Line { counter: 0, stamp: 0, mult: Line::mult_of_hz(hz) }; + let millihertz = hz * 1_000; + let error_ppb = line.millihertz().abs_diff(millihertz) as u128 * 1_000_000_000 / millihertz as u128; + assert!(error_ppb <= 1, "{hz} Hz came back as {} mHz", line.millihertz()); + assert!(line.nanos(hz).abs_diff(1_000_000_000) <= 1, "{hz} Hz: a second is {} ns", line.nanos(hz)); + assert!(line.ticks(1_000_000_000).abs_diff(hz) <= 1 + hz / 1_000_000_000); } } + /// Half a unit of `mult` at 2.4192 GHz is 0.28 ppb, so a day of a stated + /// rate's ticks is a day to 25 µs; a period kept in whole femtoseconds read + /// it 1.9 ppm short, 164 ms. + #[test] + fn a_day_at_a_stated_rate_is_a_day() { + let hz = 2_419_200_000u64; + let line = Line { counter: 0, stamp: 0, mult: Line::mult_of_hz(hz) }; + let day = 86_400u64; + assert!(line.nanos(hz * day).abs_diff(day * 1_000_000_000) <= 25_000); + } + #[test] - fn the_page_is_four_words() { - assert_eq!(core::mem::size_of::(), 32); + fn the_page_publishes_whole_lines() { + let page = ClockPage { magic: AtomicU64::new(0), stamp_at_boot: AtomicU64::new(0), line: Latch::new() }; + assert_eq!(page.read(), None); + let first = Line { counter: 7, stamp: 9, mult: GHZ }; + page.init(9, first); + assert_eq!(page.read(), Some(Clock { line: first, stamp_at_boot: 9 })); + let second = Line { counter: 70, stamp: 72, mult: GHZ / 2 }; + page.publish(second); + assert_eq!(page.read().map(|c| c.line), Some(second)); + page.publish(first); + assert_eq!(page.read().map(|c| c.line), Some(first)); } } diff --git a/sdk/abi/src/clock/latch.rs b/sdk/abi/src/clock/latch.rs new file mode 100644 index 00000000000..da034f75736 --- /dev/null +++ b/sdk/abi/src/clock/latch.rs @@ -0,0 +1,74 @@ +//! A value one writer replaces while any reader, an NMI on the writer's own +//! CPU among them, reads it with no lock and no wait for the writer: two +//! copies, and a sequence whose parity names the one to read. +//! +//! The writer moves readers to the second copy, rewrites the first, moves them +//! back and rewrites the second, so a reader always has a copy nobody is +//! writing; it retries only when the sequence moved while it read. Every +//! payload word is atomic, so a torn read is words thrown away, not a data +//! race. No `crate::` references: `kernel-loom` compiles this file under +//! `feature = "loom"`. + +#[cfg(not(feature = "loom"))] +use core::sync::atomic::{fence, AtomicU64, Ordering}; + +#[cfg(feature = "loom")] +use loom::sync::atomic::{fence, AtomicU64, Ordering}; + +#[repr(C)] +pub struct Latch { + seq: AtomicU64, + copies: [[AtomicU64; N]; 2], +} + +impl Latch { + /// The kernel lays a latch out in place on its page, so only a test builds one. + #[cfg(all(test, not(feature = "loom")))] + pub(super) const fn new() -> Self { + Self { seq: AtomicU64::new(0), copies: [const { [const { AtomicU64::new(0) }; N] }; 2] } + } + + #[allow(clippy::new_without_default)] + #[cfg(feature = "loom")] + pub fn new() -> Self { + Self { seq: AtomicU64::new(0), copies: core::array::from_fn(|_| core::array::from_fn(|_| AtomicU64::new(0))) } + } + + /// Replace the value. One writer at a time: the caller's own exclusion is + /// what makes the load-then-store of `seq` sound. + pub fn publish(&self, words: [u64; N]) { + let seq = self.seq.load(Ordering::Relaxed); + // `Release`, so a reader sent to the other copy sees what the last + // publication left in it. + self.seq.store(seq.wrapping_add(1), Ordering::Release); + // Orders the move before the rewrite: a reader that sees a new word + // sees the sequence moved, and throws the read away. + #[cfg(not(feature = "clock-latch-fence-off"))] + fence(Ordering::Release); + Self::store(&self.copies[(seq & 1) as usize], words); + self.seq.store(seq.wrapping_add(2), Ordering::Release); + fence(Ordering::Release); + Self::store(&self.copies[(seq.wrapping_add(1) & 1) as usize], words); + } + + /// The value one publication left, whole. + pub fn read(&self) -> [u64; N] { + loop { + let seq = self.seq.load(Ordering::Acquire); + let copy = &self.copies[(seq & 1) as usize]; + let words = core::array::from_fn(|i| copy[i].load(Ordering::Relaxed)); + // Orders the word loads before the recheck: a word a later + // publication wrote is one whose move the recheck sees. + fence(Ordering::Acquire); + if self.seq.load(Ordering::Relaxed) == seq { + return words; + } + } + } + + fn store(copy: &[AtomicU64; N], words: [u64; N]) { + for (slot, word) in copy.iter().zip(words) { + slot.store(word, Ordering::Relaxed); + } + } +} diff --git a/tests/common/faults.rs b/tests/common/faults.rs index eab226b15d8..c35c2120c0c 100644 --- a/tests/common/faults.rs +++ b/tests/common/faults.rs @@ -135,3 +135,37 @@ pub fn kernel_wx_leaf_is_refused(test_config: &Path, profile: qemu::Profile) -> _ => Err(format!("the audit did not name {at} alone under both rules: {refusal}\n{serial}")), } } + +/// What `tsc-step` says of the step it stages, and the middle of the clock +/// check's panic at it (`kernel/src/arch/x86_64/clocksource.rs`). +const TSC_STEP_STAGED: &str = "tsc-step: cpu"; +const TSC_STEPPED: &str = "'s TSC stepped: "; + +/// A TSC `tsc-step` reads 10 ms ahead on one CPU: the clock's runtime check +/// panics on that CPU, after the step, having read the HPET 9 ms or more apart +/// from the clock. +pub fn a_stepped_tsc_panics(test_config: &Path) -> Result<(), String> { + let qemu = QemuInstance::boot_with_options( + test_config, + &[], + &[], + BootOptions { profile: qemu::Profile::Metal, kernel_params: &["tsc-step"], ready_marker: TSC_STEPPED, ..Default::default() }, + ); + let serial = qemu.boot_log(); + let lines: Vec<&str> = serial.lines().collect(); + let staged = lines.iter().position(|l| l.contains(TSC_STEP_STAGED)).ok_or_else(|| format!("no {TSC_STEP_STAGED:?} line\n{serial}"))?; + let panicked = lines.iter().position(|l| l.contains(TSC_STEPPED)).ok_or_else(|| format!("no {TSC_STEPPED:?} panic\n{serial}"))?; + let cpu = |line: &str, head: &str| line.split(head).nth(1).and_then(|r| r.split(|c: char| !c.is_ascii_digit()).next()).map(str::to_string); + let stepped_cpu = cpu(lines[staged], TSC_STEP_STAGED); + let panicked_cpu = cpu(lines[panicked], "clock: cpu"); + let apart: Option = lines[panicked].split(TSC_STEPPED).nth(1).and_then(|r| r.split(" ns apart").next()).and_then(|n| n.parse().ok()); + if staged > panicked || stepped_cpu != panicked_cpu || !apart.is_some_and(|ns| ns >= 9_000_000) { + return Err(format!( + "the step and the panic do not match: {:?} then {:?}\n{serial}", + lines[staged].trim(), + lines[panicked].trim() + )); + } + eprintln!(" [tsc_step] {}", lines[panicked].trim()); + Ok(()) +} diff --git a/tests/guest/osbench/src/around.rs b/tests/guest/osbench/src/around.rs index 6f3ecbb3413..7b523493793 100644 --- a/tests/guest/osbench/src/around.rs +++ b/tests/guest/osbench/src/around.rs @@ -82,8 +82,8 @@ pub fn read(d: &crate::sys::diag::Diag) -> Result { #[cfg(target_os = "toyos")] pub fn cpus(bench: &str, before: &Reading, after: &Reading) { use osbench_rules::window; - let page = toyos_abi::clock::page(); - let to_ns = |ticks: u64| toyos_abi::clock::ticks_to_nanos(ticks, page.period_fs); + let line = toyos_abi::clock::page().line; + let to_ns = |ticks: u64| line.nanos(ticks); let span = window::Span { row_ns: u64::try_from(after.called.duration_since(before.called).as_nanos()).expect("a row under 584 years") + after.cost_ns, diff --git a/tests/guest/osbench/src/sys/toyos.rs b/tests/guest/osbench/src/sys/toyos.rs index c15d6a9a55c..37bad22d5f8 100644 --- a/tests/guest/osbench/src/sys/toyos.rs +++ b/tests/guest/osbench/src/sys/toyos.rs @@ -321,7 +321,7 @@ pub mod diag { }; for raw in &buf[..n] { if let Ok(entry) = Entry::decode(raw) { - let since_boot = page.stamp_of(entry.stamp).saturating_sub(page.stamp_at_boot); + let since_boot = page.nanos_since_boot(entry.stamp); out.push((since_boot, entry)); } } diff --git a/tests/guest/rust-tests/src/bin/abuse_readonly_copyout.rs b/tests/guest/rust-tests/src/bin/abuse_readonly_copyout.rs index 2fdedb1230e..2d1b291393f 100644 --- a/tests/guest/rust-tests/src/bin/abuse_readonly_copyout.rs +++ b/tests/guest/rust-tests/src/bin/abuse_readonly_copyout.rs @@ -21,7 +21,7 @@ mod image; use std::os::toyos::io::AsRawHandle; use std::process::Command; -use toyos_abi::clock::{ClockPage, CLOCK_MAGIC, CLOCK_PAGE}; +use toyos_abi::clock::{ClockPage, CLOCK_PAGE}; use toyos_abi::syscall::{self, OpenFlags, VmAt, VmProt, ProcessStats, SeekFrom, SyscallError}; use toyos_abi::RawHandle; @@ -140,8 +140,10 @@ fn second_reader() -> bool { fn main() { if std::env::args().nth(1).as_deref() == Some(CHILD_ARG) { - let page = unsafe { core::ptr::read_volatile(CLOCK_PAGE as *const ClockPage) }; - assert_eq!(page.magic, CLOCK_MAGIC, "a second process found the clock page rewritten"); + // SAFETY: the kernel maps the page read-only here in every address + // space, and every field of it is an atomic. + let page = unsafe { &*(CLOCK_PAGE as *const ClockPage) }; + assert!(page.read().is_some(), "a second process found the clock page rewritten"); std::process::exit(0); } @@ -172,7 +174,11 @@ fn main() { } // Last, and said by its bit alone: its harm reaches every stamping reader. - if refused("the clock page", CLOCK_PAGE, stats).is_err() || !second_reader() { + // Half a page in, past the clock's own words, which the kernel rewrites + // when it retunes: the same frame, and bytes nothing moves. + const CLOCK_SPARE: u64 = CLOCK_PAGE + PAGE_4K / 2; + const _: () = assert!(core::mem::size_of::() <= (PAGE_4K / 2) as usize && LEN <= (PAGE_4K / 2) as usize); + if refused("the clock page", CLOCK_SPARE, stats).is_err() || !second_reader() { wrote |= CLOCK_BIT; } if wrote != 0 { diff --git a/tests/toyos.rs b/tests/toyos.rs index 4dc4052e248..72744ddcb0d 100644 --- a/tests/toyos.rs +++ b/tests/toyos.rs @@ -520,6 +520,12 @@ const MACHINE_TESTS: &[&str] = &[ // the same audit passing. q35 on `Profile::Metal`, whose 16550 carries // the panic. "kernel_wx_leaf_is_refused", + // The clock's runtime check failing loud on a TSC an actuator stepped + // where `IA32_TSC_ADJUST` would not show it: the panic ends the boot, and + // no bench's TSC steps on demand. q35 on `Profile::Metal`, whose 16550 + // carries the panic; the order of the step and the panic, and the size + // the check read, are what it holds. + "a_stepped_tsc_panics", // A press of the fixed power button reaches the `acpi` claim's holder as // one SCI, its line masked until the holder serves it: the kernel's SCI // and its mask have no host build, QEMU raises the event on demand, and @@ -1113,7 +1119,7 @@ const METAL: &[(&str, metal::Metal)] = &[ arms: TESTCASES, judge: |b| { timer_calibration(b[0].kernel().text())?; - tsc_agrees_with_cpuid(b[0].kernel().text()) + tsc_rate_at_boot(b[0].kernel().text()) }, }, ), @@ -1200,6 +1206,13 @@ const METAL: &[(&str, metal::Metal)] = &[ "osbench_energy", metal::Metal { arms: OSBENCH_ENERGY, judge: |b| osbench_energy_measured(b[0]) }, ), + ( + // The clock's boot rate against the HPET's over the minutes the suite + // runs, through its idle, busy and hot phases: the T14 and the AMD + // bench, riding the `osbench` boot. + "clock_holds_over_minutes", + metal::Metal { arms: BENCHCASE, judge: |b| clock_holds_over_minutes(b[0]) }, + ), ( // `osbench lan` against the `osbench-peer` the other bench's Linux runs, // compared with the same program under this bench's Linux on the same @@ -4667,6 +4680,7 @@ fn run_machine_test(name: &str, test_config: &Path) -> Result<(), String> { "amdvi_selftest" => common::iommu::amdvi_selftest(test_config), "nested_nmi_is_loud" => faults::nested_nmi_is_loud(test_config), "kernel_wx_leaf_is_refused" => faults::kernel_wx_leaf_is_refused(test_config, qemu::Profile::Metal), + "a_stepped_tsc_panics" => faults::a_stepped_tsc_panics(test_config), "acpi_press_is_one_sci" => acpi_press_is_one_sci(), "osbench_capability_groups_exit_0" => osbench_capability_groups_exit_0(), "acpi_mediated_access" => acpi_mediated_access(2), @@ -7426,19 +7440,13 @@ fn number_between(log: &str, head: &str, tail: &str) -> Result { } /// Every CPU the firmware named is scheduling, every one of them holds the -/// control-register declaration, and none of their timestamp counters trails -/// the BSP's. +/// control-register declaration, and every AP's timestamp counter is bounded +/// against the BSP's. /// -/// **The third is the one nothing in this tree ever asked.** -/// `clock::nanos_since_boot` subtracts a single BSP-sampled origin whatever CPU -/// reads it, so a CPU whose TSC starts behind that origin saturates to zero and -/// stamps every record it writes as the oldest thing the machine has -///. The kernel -/// brackets each AP's first `rdtsc` between two of the BSP's, taken either side -/// of a bring-up that is serialised — so "inside" is what a synchronised -/// counter gives and nothing else does. **QEMU cannot refute it**: every guest -/// TSC is synthesised from one host clock, which is exactly why the assertion -/// is here to be run on metal. +/// The kernel bounds each AP's offset with round trips of a cache line as it +/// starts and panics on one they prove, so the row reads each AP's bound and +/// prints the widest. **QEMU cannot refute it**: every guest TSC is +/// synthesised from one host clock, which is why it runs on metal. fn smp_roster_and_tsc_trail(log: &str, cpus: u32) -> Result<(), String> { let online = number_between(log, "SMP: ", " of ")?; let named = number_between(log, " of ", " MADT cpus online")?; @@ -7447,33 +7455,28 @@ fn smp_roster_and_tsc_trail(log: &str, cpus: u32) -> Result<(), String> { "the machine was given {cpus} CPUs, its MADT named {named} and {online} came up" )); } - let bracketed = number_between(log, "MADT cpus online, ", " of ")?; + let bounds: Vec<(i64, i64)> = log.lines().filter_map(tsc_offset_bound).collect(); let aps = u64::from(cpus) - 1; - if bracketed != aps { - let outside: Vec<&str> = log - .lines() - .filter(|l| l.contains(" the BSP's ") && !l.contains("inside")) - .collect(); - return Err(format!( - "{bracketed} of {aps} APs read a TSC inside the BSP's bracket; the rest: {outside:#?}" - )); + if bounds.len() as u64 != aps { + return Err(format!("{} of {aps} APs' TSCs were bounded against the BSP's", bounds.len())); } let checked = number_between(log, "control_regs: ", " of ")?; if checked != u64::from(cpus) { return Err(format!("{checked} of {cpus} CPUs were checked against the declaration")); } - let widest = log - .lines() - .filter_map(|l| l.split(" (").nth(1)?.split(" cycles wide)").next()?.parse::().ok()) - .max() - .unwrap_or_default(); - eprintln!( - " [smp] {online} CPUs online, {bracketed} AP TSCs inside a BSP bracket at most \ - {widest} cycles wide" - ); + let widest = bounds.iter().map(|&(lo, hi)| lo.unsigned_abs().max(hi.unsigned_abs())).max().unwrap_or_default(); + eprintln!(" [smp] {online} CPUs online, every AP's TSC within {widest} cycles of the BSP's"); Ok(()) } +/// `lo..=hi` off `clock: cpuN's TSC is within lo..=hi cycles of cpu0's`. +fn tsc_offset_bound(line: &str) -> Option<(i64, i64)> { + let (_, rest) = line.split_once("'s TSC is within ")?; + let (bound, _) = rest.split_once(" cycles of cpu0's")?; + let (lo, hi) = bound.split_once("..=")?; + Some((lo.parse().ok()?, hi.parse().ok()?)) +} + /// The local APIC mode a bench's CPUs offer, by `CPUID.01H:ECX[21]` as each /// bench's scout read it on every CPU (`kernel/sim/tests/scout/`): the T14's /// `0x77fafbbf` sets it and the AMD bench's `0x76d8320b` does not, so the @@ -7608,55 +7611,106 @@ fn acpi_table_inventory(log: &str) -> Result<(), String> { Ok(()) } -/// The LAPIC timer and the TSC each calibrated to a frequency. +/// The LAPIC timer calibrated to a frequency. fn timer_calibration(log: &str) -> Result<(), String> { let lapic_hz = number_between(log, "ticks/10ms, so ", "Hz")?; if lapic_hz == 0 { return Err("the LAPIC timer calibrated to no frequency at all".to_string()); } - let measured = number_between(log, "clock: TSC measured ", "Hz against the HPET")?; - if measured == 0 { - return Err("the TSC calibrated to no frequency at all".to_string()); + eprintln!(" [timer] LAPIC {lapic_hz}Hz"); + Ok(()) +} + +/// The rate the boot started the clock at: CPUID 15H's, or one the HPET +/// measured with its bound, which on the bench is a rate the clock can keep at +/// all. Its bound is held against minutes by [`clock_holds_over_minutes`]. +fn tsc_rate_at_boot(log: &str) -> Result<(), String> { + let boot = boot_rate(log)?; + if boot.millihertz == 0 { + return Err("the TSC was started at no rate at all".to_string()); } - eprintln!(" [timer] TSC {measured}Hz measured; LAPIC {lapic_hz}Hz"); + eprintln!(" [timer] TSC {boot:?}"); Ok(()) } -/// The TSC the whole machine is timed by, against the frequency the part itself -/// states. -/// -/// **The one cross-source check a boot has.** Everything else the kernel times -/// is derived from the HPET calibration, so it can only agree with itself; -/// CPUID leaf 15H's crystal ratio and leaf 16H's base frequency are the CPU's -/// own statement, arrived at by neither the HPET nor the counting loop. A part -/// that states neither is a fact about the part, not a failure, so the ppm -/// bound is asserted only where a statement exists. Judged on metal only: the -/// calibration is a span of a clock, and a guest's clock runs while its host -/// has the vCPU. -fn tsc_agrees_with_cpuid(log: &str) -> Result<(), String> { - /// One percent, which is the widest two timebases can differ and still be - /// counting the same second. A refusal and not a measurement: it catches a - /// machine whose HPET and CPUID have stopped agreeing at all, and nothing - /// narrower is true of every part this kernel may boot on. - const CEILING_PPM: u64 = 10_000; - - let measured = number_between(log, "clock: TSC measured ", "Hz against the HPET")?; - let Ok(stated) = number_between(log, "CPUID states ", "Hz,") else { - let why = log - .lines() - .find(|l| l.contains("CPUID leaves 15H and 16H")) - .ok_or("neither a stated frequency nor the record saying there is none")?; - eprintln!(" [timer] TSC {measured}Hz — {}", why.trim()); - return Ok(()); - }; - let ppm = number_between(log, "Hz, ", "ppm apart")?; - if ppm > CEILING_PPM { +/// The rate the clock started at, in millihertz, and its bound in ppb: zero +/// for CPUID 15H's, which is not measured. +#[derive(Debug)] +struct BootRate { + millihertz: u64, + bound_ppb: u64, + stated: bool, +} + +fn boot_rate(log: &str) -> Result { + if let Ok(hz) = number_between(log, "clock: the TSC counts at ", " Hz, as CPUID 15H states") { + return Ok(BootRate { millihertz: hz * 1_000, bound_ppb: 0, stated: true }); + } + let millihertz = number_between(log, "clock: the TSC counts at ", " mHz against the HPET over ")?; + let bound_ppb = number_between(log, " ms, within ", " ppb; CPUID 15H states no rate")?; + Ok(BootRate { millihertz, bound_ppb, stated: false }) +} + +/// One of the clock's reports: the TSC's rate against the HPET over a span +/// since boot and its bound, and the rate the clock keeps after it. +#[derive(Debug)] +struct ClockReport { + span_s: u64, + measured_millihertz: u64, + measured_ppb: u64, + kept_millihertz: u64, +} + +fn clock_report(line: &str) -> Option { + let (_, rest) = line.split_once("clock: over ")?; + let (span, rest) = rest.split_once(" s the TSC counted ")?; + let (measured, rest) = rest.split_once(" mHz against the HPET, within ")?; + let (measured_ppb, rest) = rest.split_once(" ppb; the clock ")?; + let kept = rest.split(' ').find_map(|word| word.parse().ok())?; + Some(ClockReport { + span_s: span.parse().ok()?, + measured_millihertz: measured.parse().ok()?, + measured_ppb: measured_ppb.parse().ok()?, + kept_millihertz: kept, + }) +} + +/// Parts per billion `a` is from `b`. +fn ppb_apart(a: u64, b: u64) -> u64 { + (u128::from(a.abs_diff(b)) * 1_000_000_000 / u128::from(b)) as u64 +} + +/// **The clock over minutes.** The kernel holds every CPU's TSC to the HPET +/// once a second and panics on a step or a drift, so a boot that ran its list +/// to the end held; what this row reads is whether the rate the boot started at +/// was within the bound it derived, against the rate the HPET measured over +/// the longest span the boot reached, which must be two minutes or more. A +/// stated rate is CPUID 15H's, and against it the HPET's own 0.05% is all a +/// document allows (IA-PC HPET Specification 1.0a), so its distance is printed. +fn clock_holds_over_minutes(boot: &metal::Readback) -> Result<(), String> { + let kernel = boot.kernel(); + let text = kernel.text(); + let started = boot_rate(text)?; + let longest = text + .lines() + .filter_map(clock_report) + .max_by_key(|report| report.span_s) + .ok_or("the clock never reported its rate against the HPET")?; + if longest.span_s < 120 { + return Err(format!("the clock's longest report spans {} s, under two minutes: {longest:?}", longest.span_s)); + } + let apart = ppb_apart(started.millihertz, longest.measured_millihertz); + eprintln!( + " [clock] started at {started:?}; over {} s the HPET measured {} mHz within {} ppb, {apart} ppb from the start; keeps {} mHz", + longest.span_s, longest.measured_millihertz, longest.measured_ppb, longest.kept_millihertz, + ); + let allowed = if started.stated { 500_000 } else { started.bound_ppb } + longest.measured_ppb; + if apart > allowed { return Err(format!( - "the TSC measures {measured}Hz against the HPET and CPUID states {stated}Hz — \ - {ppm}ppm apart, over the {CEILING_PPM}ppm this bound allows" + "the clock started {apart} ppb from the rate the HPET measured over {} s, past the {allowed} ppb its bounds allow", + longest.span_s )); } - eprintln!(" [timer] TSC {measured}Hz measured, {stated}Hz stated, {ppm}ppm apart"); Ok(()) } From d1f10ef8923a7388196188624a0239ef5cef6fe7 Mon Sep 17 00:00:00 2001 From: japabu Date: Wed, 28 Oct 2026 22:45:57 +0100 Subject: [PATCH 02/11] clock: the HPET's drift is the spec's two figures, and a runtime sample is one read MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit IA-PC HPET 1.0a §2.2's minimum implementation allows the main counter ±0.05% over any interval of 1 ms or more and ±0.2% over any of 100 µs or less. The check held every span to the first; samples on two CPUs land tens of µs apart, where the spec allows four times as much. Spans under 1 ms now take the 0.2%; the table says nothing between 100 µs and 1 ms. The T14's boot at 378995d9d bracketed one HPET read at 35,690 TSC cycles, 14.8 µs, so three reads per runtime sample were 44 µs in a scheduler pass once a second on each CPU. Every bound allows a read's own bracket, so the narrowest of three buys a sample precision and nothing the check needs: a runtime sample is one read. The boot's anchor and calibration keep three. The same bracket, 14.3 µs over 50 ms, is the T14's old 285 ppm calibration cluster. Co-Authored-By: Claude Opus 5.5 --- kernel/pure/clock/reference.rs | 32 ++++++++++++++----- kernel/src/arch/x86_64/clocksource.rs | 44 ++++++++++++++++----------- tests/toyos.rs | 3 +- 3 files changed, 53 insertions(+), 26 deletions(-) diff --git a/kernel/pure/clock/reference.rs b/kernel/pure/clock/reference.rs index be86d0754d3..54fb66ad6c3 100644 --- a/kernel/pure/clock/reference.rs +++ b/kernel/pure/clock/reference.rs @@ -16,11 +16,27 @@ const FS_PER_NS: u128 = 1_000_000; /// A reference timer: its tick, and how far its rate may wander from that tick /// by its own specification, which every check that compares rates over time -/// allows on top of what the readings bound. +/// allows on top of what the readings bound: `tolerance_ppb` over a span of +/// `settled_ns` or more, and `short_tolerance_ppb` over a shorter one. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct Reference { pub period_fs: u64, pub tolerance_ppb: u64, + pub settled_ns: u64, + pub short_tolerance_ppb: u64, +} + +impl Reference { + /// The span from `a` to `b` the reference counted, zero where it read + /// backwards. + pub const fn span_ns(&self, a: Sample, b: Sample) -> u64 { + (b.reference.saturating_sub(a.reference) as u128 * self.period_fs as u128 / FS_PER_NS) as u64 + } + + /// What the reference's own rate may be off by over `span_ns`. + pub const fn tolerance_over(&self, span_ns: u64) -> u64 { + if span_ns >= self.settled_ns { self.tolerance_ppb } else { self.short_tolerance_ppb } + } } /// One reading of the reference, latched at some moment the counter read @@ -122,7 +138,7 @@ impl Agreement { pub fn agree(reference: Reference, a: Sample, b: Sample, mult: u64, tolerance_ppb: u64) -> Agreement { let nanos = |ticks2: u128| (ticks2 * mult as u128) >> 33; let counter_ns = if b.mid2() >= a.mid2() { nanos(b.mid2() - a.mid2()) as i128 } else { -(nanos(a.mid2() - b.mid2()) as i128) }; - let reference_ns = (b.reference.saturating_sub(a.reference) as u128 * reference.period_fs as u128 / FS_PER_NS) as u64; + let reference_ns = reference.span_ns(a, b); let readings = nanos(a.width() as u128 + b.width() as u128) as u64; let tick = (reference.period_fs as u128).div_ceil(FS_PER_NS) as u64; let rated = (reference_ns as u128 * tolerance_ppb as u128).div_ceil(PPB as u128) as u64; @@ -224,12 +240,12 @@ impl Watch { /// `Some` at each report: the first a second after the anchor, then at /// twice the span of the one before. pub fn take(&mut self, sample: Sample) -> Result, Failure> { - let tolerance = self.kept.bound_ppb + self.reference.tolerance_ppb; - let step = agree(self.reference, self.last, sample, self.kept.mult, tolerance); + let tolerance = |from: Sample| self.kept.bound_ppb + self.reference.tolerance_over(self.reference.span_ns(from, sample)); + let step = agree(self.reference, self.last, sample, self.kept.mult, tolerance(self.last)); if !step.holds() { return Err(Failure::Step { from: self.last, to: sample, agreement: step }); } - let drift = agree(self.reference, self.anchor, sample, self.kept.mult, tolerance); + let drift = agree(self.reference, self.anchor, sample, self.kept.mult, tolerance(self.anchor)); if !drift.holds() { return Err(Failure::Drift { from: self.anchor, to: sample, agreement: drift }); } @@ -293,8 +309,10 @@ mod tests { use super::*; use alloc::vec::Vec; - /// The AMD bench's HPET: 14.318 MHz, and the HPET specification's 0.05%. - const HPET: Reference = Reference { period_fs: 69_841_279, tolerance_ppb: 500_000 }; + /// The AMD bench's HPET: 14.318 MHz, and the HPET specification's 0.05% + /// from 1 ms and 0.2% under it. + const HPET: Reference = + Reference { period_fs: 69_841_279, tolerance_ppb: 500_000, settled_ns: 1_000_000, short_tolerance_ppb: 2_000_000 }; /// A counter at 1796.6248 MHz. const HZ: u128 = 1_796_624_800; /// What a read of the reference costs in counter ticks: 1.2 µs. diff --git a/kernel/src/arch/x86_64/clocksource.rs b/kernel/src/arch/x86_64/clocksource.rs index db9c4d9f05d..4984b01e7b9 100644 --- a/kernel/src/arch/x86_64/clocksource.rs +++ b/kernel/src/arch/x86_64/clocksource.rs @@ -26,16 +26,21 @@ const HPET_CAP: u64 = 0x000; const HPET_CFG: u64 = 0x010; const HPET_COUNTER: u64 = 0x0F0; /// `GCAP_ID` bit 13, `COUNT_SIZE_CAP`: the main counter is 64 bits wide -/// (IA-PC HPET Specification 1.0a, the General Capabilities and ID Register). +/// (IA-PC HPET Specification 1.0a, §2.3.4). const COUNT_SIZE_CAP: u64 = 1 << 13; -/// The main counter's tick is accurate to 0.05% over any interval of 1 ms or -/// more (IA-PC HPET Specification 1.0a): the most the TSC may seem to -/// drift against it without having moved. -const HPET_TOLERANCE_PPB: u64 = 500_000; +/// The HPET as a reference: its tick, and the main counter's clock frequency +/// drift the specification's minimum implementation allows, ±0.05% over any +/// interval of 1 ms or more and ±0.2% over any of 100 µs or less (IA-PC HPET +/// Specification 1.0a, §2.2), which is what the TSC may seem to drift against +/// it without having moved. The 0.2% is taken for every span under 1 ms, which +/// the table leaves unstated between 100 µs and 1 ms. +const fn hpet_reference(period_fs: u64) -> Reference { + Reference { period_fs, tolerance_ppb: 500_000, settled_ns: 1_000_000, short_tolerance_ppb: 2_000_000 } +} -/// Reads of the HPET per sample, of which the narrowest bracket is kept: an +/// Reads of the HPET per boot sample, of which the narrowest bracket is kept: an /// SMI or a host's preemption inside one read widens that read alone. -const READS: usize = 3; +const BOOT_READS: usize = 3; const CALIBRATION: Delay = Delay::to_measure( Duration::from_millis(50), @@ -48,8 +53,11 @@ const CALIBRATION_TRIES: u32 = 3; /// What each CPU waits between its samples: about the half millisecond the /// HPET's tolerance allows a second, so a step of the 1.7 ms seen on the AMD -/// bench fails the sample after it. Three reads of the HPET, some 4 µs, once -/// a second on each CPU. +/// bench fails the sample after it. One read of the HPET, once a second on +/// each CPU, inside its scheduler pass: 14.8 µs on the T14, whose HPET read +/// brackets 35,690 TSC cycles. One and not the boot's narrowest of three: +/// every bound here allows the read's own bracket, so a read an SMI widened +/// costs that sample precision and never a false failure. const CHECK_EVERY_NS: u64 = 1_000_000_000; /// Round trips of a cache line between the BSP and each AP as it starts. @@ -77,10 +85,10 @@ impl Hpet { if self.wide { self.mmio.read_u64(HPET_COUNTER) } else { u64::from(self.mmio.read_u32(HPET_COUNTER)) } } - /// The narrowest bracket of [`READS`] readings near `near`. - fn sample(&self, near: u64) -> Sample { + /// The narrowest bracket of `reads` readings near `near`. + fn sample(&self, near: u64, reads: usize) -> Sample { let mut best: Option = None; - for _ in 0..READS { + for _ in 0..reads { let before = cpu::rdtsc_ordered(); let reference = self.read(near); let after = cpu::rdtsc_ordered(); @@ -89,7 +97,7 @@ impl Hpet { best = Some(sample); } } - best.expect("READS is not zero") + best.expect("a sample takes a read") } /// Where the counter should read now, from `last` and the rate the clock @@ -145,7 +153,7 @@ pub fn start(hpet_base: u64) { assert!(period_fs > 0, "HPET: invalid counter period"); let cfg = mmio.read_u64(HPET_CFG); mmio.write_u64(HPET_CFG, cfg | 1); - let hpet = Hpet { mmio, wide: cap & COUNT_SIZE_CAP != 0, reference: Reference { period_fs, tolerance_ppb: HPET_TOLERANCE_PPB } }; + let hpet = Hpet { mmio, wide: cap & COUNT_SIZE_CAP != 0, reference: hpet_reference(period_fs) }; log!( "clock: HPET at {hpet_base:#x} enabled, period={period_fs}fs, {} bits, counter reads {}", if hpet.wide { 64 } else { 32 }, @@ -154,7 +162,7 @@ pub fn start(hpet_base: u64) { let (anchor, kept) = match cpu::stated_counter_hz() { Some(hz) => { - let anchor = hpet.sample(hpet.raw()); + let anchor = hpet.sample(hpet.raw(), BOOT_READS); log!( "clock: the TSC counts at {hz} Hz, as CPUID 15H states, held to the HPET from a reading {} cycles wide", anchor.width() @@ -174,12 +182,12 @@ fn calibrate(hpet: &Hpet) -> (Sample, Kept) { for attempt in 1..=CALIBRATION_TRIES { // Every sample is within the window of the first, far under the 2^31 // ticks a 32-bit counter's reading is taken near. - let first = hpet.sample(hpet.raw()); + let first = hpet.sample(hpet.raw(), BOOT_READS); let mut samples = [first; CALIBRATION_SAMPLES as usize]; for (i, slot) in samples.iter_mut().enumerate().skip(1) { let target = first.reference + span_ticks * i as u64 / (CALIBRATION_SAMPLES - 1); wait_for(hpet, target, first); - *slot = hpet.sample(first.reference); + *slot = hpet.sample(first.reference, BOOT_READS); } match reference::calibrate(hpet.reference, &samples) { Ok(rate) => { @@ -347,7 +355,7 @@ pub fn check(now_ns: u64) { started as i64, ); } - let sample = held.hpet.sample(held.hpet.near(held.watch.last(), held.watch.kept().mult)); + let sample = held.hpet.sample(held.hpet.near(held.watch.last(), held.watch.kept().mult), 1); #[cfg(feature = "boot-actuators")] let sample = staged(local, sample, false); match held.watch.take(sample) { diff --git a/tests/toyos.rs b/tests/toyos.rs index 72744ddcb0d..53244fecb78 100644 --- a/tests/toyos.rs +++ b/tests/toyos.rs @@ -7686,7 +7686,8 @@ fn ppb_apart(a: u64, b: u64) -> u64 { /// was within the bound it derived, against the rate the HPET measured over /// the longest span the boot reached, which must be two minutes or more. A /// stated rate is CPUID 15H's, and against it the HPET's own 0.05% is all a -/// document allows (IA-PC HPET Specification 1.0a), so its distance is printed. +/// document allows (IA-PC HPET Specification 1.0a, §2.2), so its distance is +/// printed. fn clock_holds_over_minutes(boot: &metal::Readback) -> Result<(), String> { let kernel = boot.kernel(); let text = kernel.text(); From 7d5b0114a5ce8cae39a769e44f3e41879ba7c2c2 Mon Sep 17 00:00:00 2001 From: japabu Date: Wed, 28 Oct 2026 23:56:19 +0100 Subject: [PATCH 03/11] clock: one CPU samples the HPET, each CPU checks itself, and a span the package may have slept in is left out Review r1 of #1104. One CPU's pass a second takes the HPET sample for the whole machine's rate, drift and steps; the others no longer read the HPET on their wake path. Each CPU a second still checks its IA32_TSC_ADJUST, and now that its TSC reads no earlier than the latest any CPU's check read before it loaded it: a reading behind it proves this CPU behind by the difference, the round trips' argument one way, for a load, an ordered read and a fetch_max. The HPET may stop in a deep package state (on the T14, PC10, which the idle work now reaches); SDM Vol. 4 documents MSR_PKG_C10_RESIDENCY (632H) for neither the T14's 06_8CH nor the AMD, so no residency is read. A package state deeper than a core's own cannot hold while that core is out of every declared idle state, so a span is held only where some CPU's deep-idle counts (now counted at entry too) did not move and it was not in one at the sample. Every other span is left out: the watch keeps held segments, a rate's bound counts a tick and two brackets a segment, and the step and drift checks run over held spans only. The panics say the TSC and the HPET disagree, not which moved. The stated-rate report no longer prints the mult's rounding as a bound. tsc-step stages a whole-machine step in the sampler's readings. Accelerated QEMU guests get +invtsc and take the metal path. Co-Authored-By: Claude Opus 5.5 --- build/arch.rs | 2 +- kernel/loom/tests/clock_latch.rs | 5 +- kernel/pure/clock/reference.rs | 200 ++++++++++++++++++++------ kernel/src/actuator.rs | 6 +- kernel/src/arch/x86_64/clocksource.rs | 178 +++++++++++++++-------- kernel/src/idle.rs | 20 ++- sdk/abi/src/trace.rs | 2 +- tests/common/faults.rs | 22 ++- 8 files changed, 305 insertions(+), 130 deletions(-) diff --git a/build/arch.rs b/build/arch.rs index 9dd4681c042..c029da5319e 100644 --- a/build/arch.rs +++ b/build/arch.rs @@ -163,7 +163,7 @@ impl Arch { /// TCG guest that withholds one is a feature this tree stops exercising. pub const fn cpu(self, accel: Accel) -> &'static str { match (self, accel.is_hardware()) { - (Arch::X86_64, true) => "host,+rdrand,+smap,+fsgsbase,+x2apic,+smep,+ssse3,+sse4.1,+sse4.2,+xsave,+xsaveopt,+avx,+avx2", + (Arch::X86_64, true) => "host,+invtsc,+rdrand,+smap,+fsgsbase,+x2apic,+smep,+ssse3,+sse4.1,+sse4.2,+xsave,+xsaveopt,+avx,+avx2", (Arch::X86_64, false) => "qemu64,+rdrand,+rdtscp,+smap,+fsgsbase,+x2apic,+smep,+ssse3,+sse4.1,+sse4.2,+xsave,+xsaveopt,+avx,+avx2", (Arch::Aarch64, true) => "host", (Arch::Aarch64, false) => "max", diff --git a/kernel/loom/tests/clock_latch.rs b/kernel/loom/tests/clock_latch.rs index e21c3ed5895..79888f3680f 100644 --- a/kernel/loom/tests/clock_latch.rs +++ b/kernel/loom/tests/clock_latch.rs @@ -14,9 +14,8 @@ use loom::thread; const WORDS: usize = 3; -/// Interleavings past this many preemptions add nothing a two-party protocol -/// of this length can do, and without a bound the two publications' twelve -/// stores do not finish exploring. +/// A bound so the two publications' twelve stores finish exploring; under it +/// the `clock-latch-fence-off` control still reaches a torn read. const PREEMPTIONS: usize = 3; fn model(f: impl Fn() + Sync + Send + 'static) { diff --git a/kernel/pure/clock/reference.rs b/kernel/pure/clock/reference.rs index 54fb66ad6c3..4631e66d7fb 100644 --- a/kernel/pure/clock/reference.rs +++ b/kernel/pure/clock/reference.rs @@ -81,28 +81,80 @@ pub enum Unmeasurable { TooWide, } +/// The counter and the reference over one or more segments, each from one +/// sample to a later one: what a rate is measured over, and what a held span +/// is checked over. Segments add, so a span can leave out the stretches the +/// reference cannot be held across. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Span { + /// The counter's span between the brackets' midpoints, doubled. + counter2: i128, + /// The reference's ticks. + ticks: i128, + /// The brackets' widths at every segment's ends. + widths: u128, + /// Segments, each of which knows its reference span to one tick. + segments: u64, + /// The shortest segment's reference span, for the reference's tolerance. + shortest_ns: u64, +} + +impl Span { + pub const EMPTY: Self = Self { counter2: 0, ticks: 0, widths: 0, segments: 0, shortest_ns: u64::MAX }; + + pub fn between(reference: Reference, a: Sample, b: Sample) -> Self { + Self { + counter2: b.mid2() as i128 - a.mid2() as i128, + ticks: b.reference as i128 - a.reference as i128, + widths: a.width() as u128 + b.width() as u128, + segments: 1, + shortest_ns: reference.span_ns(a, b), + } + } + + pub fn then(self, more: Self) -> Self { + Self { + counter2: self.counter2 + more.counter2, + ticks: self.ticks + more.ticks, + widths: self.widths + more.widths, + segments: self.segments + more.segments, + shortest_ns: self.shortest_ns.min(more.shortest_ns), + } + } + + /// The reference's count over the span, in nanoseconds; zero for one + /// that read backwards. + pub fn reference_ns(&self, reference: Reference) -> u64 { + (self.ticks.max(0) as u128 * reference.period_fs as u128 / FS_PER_NS) as u64 + } +} + /// The rate from `a` to `b`, which is later. +pub fn rate(reference: Reference, a: Sample, b: Sample) -> Result { + rate_over(reference, Span::between(reference, a, b)) +} + +/// The rate over `span`. /// /// The counter moved `Δc` between the latches, known to the brackets' half -/// widths `(wa + wb) / 2`, while the reference moved `Δn` ticks, known to one: -/// so the relative error is `u = (wa + wb) / (2 Δc)` plus `r = 1 / Δn`, each +/// widths, while the reference moved `Δn` ticks, known to one a segment: so +/// the relative error is `u = Σw / (2 Δc)` plus `r = segments / Δn`, each /// taken against the measured span, and with the second-order terms of both /// the ratio and those spans `(u + r)·(1 + 6·max(u, r))`, which holds while /// both are under a quarter. `mult` is rounded down, which is one unit of it /// more. -pub fn rate(reference: Reference, a: Sample, b: Sample) -> Result { - if b.mid2() <= a.mid2() || b.reference <= a.reference { +pub fn rate_over(reference: Reference, span: Span) -> Result { + if span.counter2 <= 0 || span.ticks <= 0 { return Err(Unmeasurable::NotLater); } - let counter2 = b.mid2() - a.mid2(); - let ticks = (b.reference - a.reference) as u128; + let counter2 = span.counter2 as u128; + let ticks = span.ticks as u128; let span_fs = ticks * reference.period_fs as u128; - let widths = a.width() as u128 + b.width() as u128; - if widths * 4 >= counter2 || ticks < 4 { + if span.widths * 4 >= counter2 || ticks < 4 * span.segments as u128 { return Err(Unmeasurable::TooWide); } - let u = (widths * PPB as u128).div_ceil(counter2); - let r = (PPB as u128).div_ceil(ticks); + let u = (span.widths * PPB as u128).div_ceil(counter2); + let r = (span.segments as u128 * PPB as u128).div_ceil(ticks); let first = u + r; // ns per tick, times 2^32: `(span_fs / 10⁶) / (counter2 / 2) · 2^32`. let mult = (span_fs << 33) / (FS_PER_NS * counter2); @@ -110,8 +162,7 @@ pub fn rate(reference: Reference, a: Sample, b: Sample) -> Result Agreement { + agree_over(reference, Span::between(reference, a, b), mult, tolerance_ppb) +} + +/// The counter's span over `span` at `mult`, against the reference's. +/// +/// Allowed: the brackets' half widths at `mult`, one reference tick a +/// segment, and `tolerance_ppb` of the span, which is what the rate itself +/// may be off by; two nanoseconds a segment more cover the rounding of each +/// conversion. A segment whose reference reads behind its start counts as a +/// span of zero, so a reference that went backwards is a disagreement. +pub fn agree_over(reference: Reference, span: Span, mult: u64, tolerance_ppb: u64) -> Agreement { let nanos = |ticks2: u128| (ticks2 * mult as u128) >> 33; - let counter_ns = if b.mid2() >= a.mid2() { nanos(b.mid2() - a.mid2()) as i128 } else { -(nanos(a.mid2() - b.mid2()) as i128) }; - let reference_ns = reference.span_ns(a, b); - let readings = nanos(a.width() as u128 + b.width() as u128) as u64; + let counter_ns = if span.counter2 >= 0 { + nanos(span.counter2 as u128) as i128 + } else { + -(nanos(span.counter2.unsigned_abs()) as i128) + }; + let reference_ns = span.reference_ns(reference); + let readings = nanos(span.widths) as u64; let tick = (reference.period_fs as u128).div_ceil(FS_PER_NS) as u64; let rated = (reference_ns as u128 * tolerance_ppb as u128).div_ceil(PPB as u128) as u64; Agreement { reference_ns, apart_ns: (counter_ns - reference_ns as i128).clamp(i64::MIN as i128, i64::MAX as i128) as i64, - allowed_ns: readings + tick + rated + 2, + allowed_ns: readings + span.segments * (tick + 2) + rated, } } @@ -161,7 +221,8 @@ pub enum Calibration { /// The rate over `samples`, first to last, once every sample between is shown /// to sit on that line within what the readings allow: a step inside the -/// window, which would skew the rate, is refused rather than averaged in. +/// window larger than the readings resolve, which would skew the rate, is +/// refused rather than averaged in. pub fn calibrate(reference: Reference, samples: &[Sample]) -> Result { let (Some(&first), Some(&last)) = (samples.first(), samples.last()) else { return Err(Calibration::Unmeasurable(Unmeasurable::NotLater)); @@ -191,40 +252,50 @@ const REPORT_FACTOR: u64 = 2; /// A counter's agreement with the reference over the life of a clock kept on /// it: each sample is held against the one before it, which catches a step, -/// and against the first, which catches a rate that has moved. +/// and against every held span since the first, which catches a rate that +/// has moved. A span the reference cannot be held across is left out whole. #[derive(Clone, Copy, Debug)] pub struct Watch { reference: Reference, - anchor: Sample, + /// The held segments before the one now open. + closed: Span, + /// Where the open segment starts. + opened: Sample, last: Sample, kept: Kept, next_report_ns: u64, + /// Spans left out. + unheld: u64, } /// What a sample showed, where it held. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct Report { - /// The rate measured since the first sample. + /// The rate measured over every held span. pub measured: Rate, /// The rate the clock keeps from here: a better measurement where the kept /// one was measured, and otherwise the one it kept. pub kept: Kept, /// Whether `kept` replaced the rate the clock was keeping. pub retuned: bool, + /// The spans left out so far. + pub unheld: u64, } -/// What a sample showed, where it did not hold. +/// What a sample showed, where it did not hold: the counter and the reference +/// disagree, and the samples cannot say which of the two moved. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum Failure { - /// Against the sample before it: the counter stepped. + /// Against the sample before it: one of them stepped. Step { from: Sample, to: Sample, agreement: Agreement }, - /// Against the first: the counter's rate is not the one the clock keeps. - Drift { from: Sample, to: Sample, agreement: Agreement }, + /// Over every held span: they count at rates further apart than the kept + /// rate's bound and the reference's tolerance. + Drift { since: Sample, to: Sample, agreement: Agreement }, } impl Watch { pub const fn new(reference: Reference, anchor: Sample, kept: Kept) -> Self { - Self { reference, anchor, last: anchor, kept, next_report_ns: FIRST_REPORT_NS } + Self { reference, closed: Span::EMPTY, opened: anchor, last: anchor, kept, next_report_ns: FIRST_REPORT_NS, unheld: 0 } } pub const fn kept(&self) -> Kept { @@ -236,18 +307,31 @@ impl Watch { self.last } - /// Hold `sample`, taken after every sample before it, to the kept rate. - /// `Some` at each report: the first a second after the anchor, then at - /// twice the span of the one before. - pub fn take(&mut self, sample: Sample) -> Result, Failure> { - let tolerance = |from: Sample| self.kept.bound_ppb + self.reference.tolerance_over(self.reference.span_ns(from, sample)); - let step = agree(self.reference, self.last, sample, self.kept.mult, tolerance(self.last)); + /// Take `sample`, taken after every sample before it. `held` says whether + /// the reference can be held to the counter across the span from the last + /// sample; one that cannot is left out, and checks nothing. `Some` at each + /// report: the first at a second of held spans, then at twice the span of + /// the one before. + pub fn take(&mut self, sample: Sample, held: bool) -> Result, Failure> { + if !held { + if self.last != self.opened { + self.closed = self.closed.then(Span::between(self.reference, self.opened, self.last)); + } + self.opened = sample; + self.last = sample; + self.unheld += 1; + return Ok(None); + } + let tolerance = |span: &Span| self.kept.bound_ppb + self.reference.tolerance_over(span.shortest_ns); + let since_last = Span::between(self.reference, self.last, sample); + let step = agree_over(self.reference, since_last, self.kept.mult, tolerance(&since_last)); if !step.holds() { return Err(Failure::Step { from: self.last, to: sample, agreement: step }); } - let drift = agree(self.reference, self.anchor, sample, self.kept.mult, tolerance(self.anchor)); + let whole = self.closed.then(Span::between(self.reference, self.opened, sample)); + let drift = agree_over(self.reference, whole, self.kept.mult, tolerance(&whole)); if !drift.holds() { - return Err(Failure::Drift { from: self.anchor, to: sample, agreement: drift }); + return Err(Failure::Drift { since: self.opened, to: sample, agreement: drift }); } self.last = sample; if drift.reference_ns < self.next_report_ns { @@ -256,14 +340,14 @@ impl Watch { while self.next_report_ns <= drift.reference_ns { self.next_report_ns = self.next_report_ns.saturating_mul(REPORT_FACTOR); } - let Ok(measured) = rate(self.reference, self.anchor, sample) else { + let Ok(measured) = rate_over(self.reference, whole) else { return Ok(None); }; let retuned = self.kept.measured && measured.bound_ppb < self.kept.bound_ppb; if retuned { self.kept = Kept { mult: measured.mult, bound_ppb: measured.bound_ppb, measured: true }; } - Ok(Some(Report { measured, kept: self.kept, retuned })) + Ok(Some(Report { measured, kept: self.kept, retuned, unheld: self.unheld })) } } @@ -409,7 +493,7 @@ mod tests { fn run(watch: &mut Watch, hz: u128, seconds: core::ops::Range, step: i64) -> Result, Failure> { let mut reports = Vec::new(); for s in seconds { - if let Some(report) = watch.take(sample_at(hz, 1_000 + s * SECOND, (s * 977) % READ, step))? { + if let Some(report) = watch.take(sample_at(hz, 1_000 + s * SECOND, (s * 977) % READ, step), true)? { reports.push(report); } } @@ -486,8 +570,34 @@ mod tests { let latch = |n: u64| sample_at(HZ, n, 0, 0).before; let early = Sample { before: latch(n), after: latch(n) + wide, reference: n }; let late = Sample { before: latch(n + 1_432) - wide, after: latch(n + 1_432), reference: n + 1_432 }; - assert_eq!(watch.take(early).map(|_| ()), Ok(())); - assert_eq!(watch.take(late).map(|_| ()), Ok(())); + assert_eq!(watch.take(early, true).map(|_| ()), Ok(())); + assert_eq!(watch.take(late, true).map(|_| ()), Ok(())); + } + + /// A reference that stood still for 30 s while the counter ran, as an HPET + /// may in a package state that stops it: the span is left out, so it is no + /// step, and the rate over the held spans either side is the counter's. + #[test] + fn an_unheld_span_is_left_out() { + let mut watch = watch(true); + run(&mut watch, HZ, 1..20, 0).unwrap(); + // The reference resumes where it stopped; the counter is 30 s on. + let stood = (30 * SECOND as u128 * HPET.period_fs as u128 * HZ / 1_000_000_000_000_000) as i64; + let resumed = sample_at(HZ, 1_000 + 20 * SECOND, 0, stood); + assert!(matches!(watch.take(resumed, true), Err(Failure::Step { .. })), "the stood reference reads as a step when held"); + let mut watch = watch_after(20); + assert_eq!(watch.take(resumed, false), Ok(None)); + let reports = run(&mut watch, HZ, 21..300, stood).unwrap(); + let last = reports.last().expect("held spans past the gap report"); + assert_eq!(last.unheld, 1); + assert!(ppb_apart(last.measured.mult, HZ) <= last.measured.bound_ppb as u128, "{last:?}"); + assert!(last.measured.span_ns / 1_000_000_000 >= 256); + } + + fn watch_after(seconds: u64) -> Watch { + let mut watch = watch(true); + run(&mut watch, HZ, 1..seconds, 0).unwrap(); + watch } /// A reference that reads backwards is a disagreement, never a span. @@ -496,7 +606,7 @@ mod tests { let mut watch = watch(true); run(&mut watch, HZ, 1..5, 0).unwrap(); let back = Sample { reference: watch.last().reference - 10, ..sample(1_000 + 6 * SECOND) }; - assert!(watch.take(back).is_err()); + assert!(watch.take(back, true).is_err()); } #[test] diff --git a/kernel/src/actuator.rs b/kernel/src/actuator.rs index c9a7de616c8..9a279f545ee 100644 --- a/kernel/src/actuator.rs +++ b/kernel/src/actuator.rs @@ -124,9 +124,9 @@ actuators! { /// `kernel_wx_leaf_is_refused` and `virt_kernel_wx_leaf_is_refused`. kernel_wx_leaf = "kernel-wx-leaf"; - /// Read one CPU's TSC 10 ms ahead once the clock has held for a second, - /// as a step `IA32_TSC_ADJUST` would not show, so the HPET alone can catch - /// it. The negative control on the clock's runtime check, + /// Read the TSC 10 ms ahead in the HPET's samples once the clock has + /// held for a second, as a step `IA32_TSC_ADJUST` would not show, so the + /// HPET alone can catch it. The negative control on the clock's runtime check, /// `a_stepped_tsc_panics`. tsc_step = "tsc-step"; diff --git a/kernel/src/arch/x86_64/clocksource.rs b/kernel/src/arch/x86_64/clocksource.rs index 4984b01e7b9..c68a175b5dd 100644 --- a/kernel/src/arch/x86_64/clocksource.rs +++ b/kernel/src/arch/x86_64/clocksource.rs @@ -1,17 +1,21 @@ //! x86-64's clock: the TSC, at the rate CPUID leaf 15H states where the part -//! states one and at one measured against the HPET where it does not, held to -//! the HPET by every CPU while it runs (`kernel::clock::reference`). +//! states one and at one measured against the HPET where it does not, and held +//! to the HPET while it runs (`kernel::clock::reference`). //! //! The clock counts the TSC only where every CPU states it invariant, or under //! a hypervisor, whose TSC is its own to synthesise and whose CPUID need not -//! say so; there the HPET is all that holds it. Every CPU's TSC is bounded -//! against the BSP's as it starts ([`measure_offset`]), and a write to it is -//! caught where `IA32_TSC_ADJUST` is there to show one. Each failure panics: -//! no other counter here can take the clock's place, because every process -//! reads the TSC itself. +//! say so; there the HPET is all that holds it. Every AP's TSC is bounded +//! against the BSP's as it starts ([`measure_offset`]). While it runs, one CPU +//! a second samples the HPET for the whole machine's rate and steps, over the +//! spans in which some CPU stayed awake; and every CPU a second checks its +//! `IA32_TSC_ADJUST` and that its TSC is not behind another's ([`check`]). +//! Each failure panics: no other counter here can take the clock's place, +//! because every process reads the TSC itself. use core::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use alloc::vec::Vec; + use kernel::clock::reference::{self, Calibration, Failure, Kept, Offset, Reference, Sample, Watch}; use toyos_abi::clock::Line; @@ -51,13 +55,9 @@ const CALIBRATION_SAMPLES: u64 = 6; /// Calibrations the boot tries before it refuses the machine. const CALIBRATION_TRIES: u32 = 3; -/// What each CPU waits between its samples: about the half millisecond the -/// HPET's tolerance allows a second, so a step of the 1.7 ms seen on the AMD -/// bench fails the sample after it. One read of the HPET, once a second on -/// each CPU, inside its scheduler pass: 14.8 µs on the T14, whose HPET read -/// brackets 35,690 TSC cycles. One and not the boot's narrowest of three: -/// every bound here allows the read's own bracket, so a read an SMI widened -/// costs that sample precision and never a false failure. +/// Once a second: the step allowance at this gap is about 0.5 ms, under the +/// 1.7 ms step the AMD bench was seen to take. One HPET read a sample: every +/// bound allows its bracket. const CHECK_EVERY_NS: u64 = 1_000_000_000; /// Round trips of a cache line between the BSP and each AP as it starts. @@ -108,10 +108,32 @@ impl Hpet { } } -/// What the runtime check holds, once the boot has started the clock. +/// What the HPET's sampler holds, once the boot has started the clock. struct Held { hpet: Hpet, watch: Watch, + /// Each CPU's [`crate::idle::Local::deep_periods`] at the last sample. + idle: Vec<(u64, u64)>, +} + +impl Held { + /// Whether some CPU stayed out of every declared idle state from the last + /// sample to now: no package state deeper than its own can have held in + /// between, so the HPET counted throughout. Where every CPU spent part of + /// the span in one, the package may have reached a state that stops the + /// HPET, and the span is not held. + fn some_cpu_stayed_awake(&mut self) -> bool { + let cpus = crate::smp::cpu_count() as usize; + self.idle.resize(cpus, (0, 0)); + let mut awake = false; + for (cpu, was) in self.idle.iter_mut().enumerate() { + let local = crate::cpu_local::get(cpu as u32).expect("a CPU the roster counts has its record"); + let now = local.idle.deep_periods(); + awake |= now == *was && now.0 == now.1; + *was = now; + } + awake + } } static HELD: Lock> = Lock::new(None); @@ -119,24 +141,25 @@ static HELD: Lock> = Lock::new(None); /// Whether CPUID states `IA32_TSC_ADJUST`, read once on the BSP. static TSC_ADJUST: AtomicBool = AtomicBool::new(false); -/// Each CPU's part of the check: when it next samples, and the +/// When the HPET is next sampled, by whichever CPU's pass gets there first. +static NEXT_SAMPLE_NS: AtomicU64 = AtomicU64::new(0); + +/// The latest TSC any CPU's check read, on a line of its own. +#[repr(align(64))] +struct Latest(AtomicU64); + +static LATEST: Latest = Latest(AtomicU64::new(0)); + +/// Each CPU's part of the check: when it next checks itself, and the /// `IA32_TSC_ADJUST` it started with. pub struct Local { due_ns: AtomicU64, adjust: AtomicU64, - /// What `tsc-step` adds to this CPU's readings. - #[cfg(feature = "boot-actuators")] - staged: AtomicU64, } impl Local { pub const fn new() -> Self { - Self { - due_ns: AtomicU64::new(0), - adjust: AtomicU64::new(0), - #[cfg(feature = "boot-actuators")] - staged: AtomicU64::new(0), - } + Self { due_ns: AtomicU64::new(0), adjust: AtomicU64::new(0) } } } @@ -172,7 +195,7 @@ pub fn start(hpet_base: u64) { None => calibrate(&hpet), }; crate::clock::set_counter(anchor.before, kept.mult); - *HELD.lock() = Some(Held { watch: Watch::new(hpet.reference, anchor, kept), hpet }); + *HELD.lock() = Some(Held { watch: Watch::new(hpet.reference, anchor, kept), hpet, idle: Vec::new() }); } /// The TSC's rate against the HPET over [`CALIBRATION`], first sample to last, @@ -329,20 +352,19 @@ pub fn answer_offset() { OUT_DONE.store(true, Ordering::Release); } -/// This CPU's sample against the HPET, at most once a [`CHECK_EVERY_NS`]: -/// called from every scheduler pass, where a CPU not due pays one load. +/// This CPU's checks, at most once a [`CHECK_EVERY_NS`], and the machine's +/// HPET sample on the first CPU due: called from every scheduler pass, where a +/// CPU not due pays one load. +/// +/// A CPU checks that its `IA32_TSC_ADJUST` is the one it started with, and +/// that its TSC reads no earlier than the latest any CPU's check read before +/// this one loaded it: a reading behind it is this CPU's TSC proven behind by +/// the difference (`reference::Offset`'s argument, one way). pub fn check(now_ns: u64) { let local = &crate::cpu_local::here().arch.clock; if now_ns < local.due_ns.load(Ordering::Relaxed) { return; } - // Another CPU sampling now: this one samples on its next pass. - let Some(mut held) = HELD.try_lock() else { - return; - }; - let Some(held) = held.as_mut() else { - return; - }; local.due_ns.store(now_ns + CHECK_EVERY_NS, Ordering::Relaxed); let cpu_id = super::percpu::cpu_id(); if TSC_ADJUST.load(Ordering::Relaxed) { @@ -355,38 +377,72 @@ pub fn check(now_ns: u64) { started as i64, ); } + let latest = LATEST.0.load(Ordering::Acquire); + let mine = cpu::rdtsc_ordered(); + assert!( + mine >= latest, + "clock: cpu{cpu_id}'s TSC read {mine}, {} cycles behind the {latest} another CPU's check read before it", + latest - mine, + ); + LATEST.0.fetch_max(mine, Ordering::AcqRel); + + let due = NEXT_SAMPLE_NS.load(Ordering::Relaxed); + if now_ns < due || NEXT_SAMPLE_NS.compare_exchange(due, now_ns + CHECK_EVERY_NS, Ordering::Relaxed, Ordering::Relaxed).is_err() { + return; + } + let Some(mut held) = HELD.try_lock() else { + return; + }; + if let Some(held) = held.as_mut() { + sample(held, cpu_id); + } +} + +/// One sample of the HPET, held to the TSC where the span since the last +/// can be. +fn sample(held: &mut Held, cpu_id: u32) { + let awake = held.some_cpu_stayed_awake(); let sample = held.hpet.sample(held.hpet.near(held.watch.last(), held.watch.kept().mult), 1); #[cfg(feature = "boot-actuators")] - let sample = staged(local, sample, false); - match held.watch.take(sample) { + let sample = staged(sample, false); + match held.watch.take(sample, awake) { Ok(None) => {} Ok(Some(report)) => { #[cfg(feature = "boot-actuators")] - staged(local, sample, true); + staged(sample, true); let kept = Line { counter: 0, stamp: 0, mult: report.kept.mult }; if report.retuned { crate::clock::retune(report.kept.mult); } - log!( - "clock: over {} s the TSC counted {} mHz against the HPET, within {} ppb; the clock {} {} mHz, within {} ppb", - report.measured.span_ns / 1_000_000_000, - Line { counter: 0, stamp: 0, mult: report.measured.mult }.millihertz(), - report.measured.bound_ppb, - if report.retuned { "now keeps" } else if report.kept.measured { "keeps" } else { "keeps CPUID 15H's" }, - kept.millihertz(), - report.kept.bound_ppb, - ); + let measured = Line { counter: 0, stamp: 0, mult: report.measured.mult }.millihertz(); + let (span_s, bound) = (report.measured.span_ns / 1_000_000_000, report.measured.bound_ppb); + let unheld = report.unheld; + match (report.kept.measured, report.retuned) { + (false, _) => log!( + "clock: over {span_s} s the TSC counted {measured} mHz against the HPET, within {bound} ppb; \ + the clock keeps CPUID 15H's {} mHz; {unheld} spans unheld", + kept.millihertz(), + ), + (true, retuned) => log!( + "clock: over {span_s} s the TSC counted {measured} mHz against the HPET, within {bound} ppb; \ + the clock {} {} mHz, within {} ppb; {unheld} spans unheld", + if retuned { "now keeps" } else { "keeps" }, + kept.millihertz(), + report.kept.bound_ppb, + ), + } } Err(Failure::Step { from, to, agreement }) => panic!( - "clock: cpu{cpu_id}'s TSC stepped: {} ns apart from the HPET over {} ns since the last sample, {} allowed \ - ({from:?} to {to:?})", + "clock: the TSC and the HPET disagree by {} ns over the {} ns the HPET counted since the last sample, {} \ + allowed: one of them stepped (cpu{cpu_id}, {from:?} to {to:?})", agreement.apart_ns, agreement.reference_ns, agreement.allowed_ns, ), - Err(Failure::Drift { from, to, agreement }) => panic!( - "clock: cpu{cpu_id}'s TSC drifted from the rate the clock keeps: {} ns apart from the HPET over {} ns, {} \ - allowed ({from:?} to {to:?})", + Err(Failure::Drift { since, to, agreement }) => panic!( + "clock: the TSC and the HPET disagree by {} ns over the {} ns the HPET counted in held spans, {} allowed: \ + their rates are further apart than the kept rate's bound and the HPET's tolerance (cpu{cpu_id}, the \ + open span from {since:?} to {to:?})", agreement.apart_ns, agreement.reference_ns, agreement.allowed_ns, @@ -394,17 +450,17 @@ pub fn check(now_ns: u64) { } } -/// `tsc-step`: from the first report on, the first CPU to report reads its -/// TSC 10 ms ahead of where it is. A hypervisor's TSC may ignore a write -/// (QEMU's TCG does), so the step is staged in the readings the check takes. +/// `tsc-step`: from the first report on, every sample reads the TSC 10 ms +/// ahead of where it is. A hypervisor's TSC may ignore a write (QEMU's TCG +/// does), so the step is staged in the readings the sampler takes. #[cfg(feature = "boot-actuators")] -fn staged(local: &Local, sample: Sample, reported: bool) -> Sample { - static STEPPED: AtomicBool = AtomicBool::new(false); - if crate::actuator::tsc_step() && reported && !STEPPED.swap(true, Ordering::Relaxed) { +fn staged(sample: Sample, reported: bool) -> Sample { + static AHEAD: AtomicU64 = AtomicU64::new(0); + if crate::actuator::tsc_step() && reported && AHEAD.load(Ordering::Relaxed) == 0 { let ahead = crate::clock::counter_ticks(10_000_000); - local.staged.store(ahead, Ordering::Relaxed); - log!("tsc-step: cpu{}'s TSC reads {ahead} cycles ahead from here", super::percpu::cpu_id()); + AHEAD.store(ahead, Ordering::Relaxed); + log!("tsc-step: the TSC reads {ahead} cycles ahead from here"); } - let ahead = local.staged.load(Ordering::Relaxed); + let ahead = AHEAD.load(Ordering::Relaxed); Sample { before: sample.before + ahead, after: sample.after + ahead, ..sample } } diff --git a/kernel/src/idle.rs b/kernel/src/idle.rs index 25ac31c5418..f5e5e9811f3 100644 --- a/kernel/src/idle.rs +++ b/kernel/src/idle.rs @@ -41,8 +41,10 @@ pub struct Local { /// declaration it learned it under. history: AtomicU64, learned_under: AtomicU64, - /// Idle periods spent in a declared state. + /// Idle periods spent in a declared state, counted as each ends, and as + /// each starts. deep: AtomicU64, + entered: AtomicU64, } impl Local { @@ -53,6 +55,7 @@ impl Local { history: AtomicU64::new(0), learned_under: AtomicU64::new(0), deep: AtomicU64::new(0), + entered: AtomicU64::new(0), } } @@ -60,6 +63,13 @@ impl Local { pub fn deep(&self) -> u64 { self.deep.load(Ordering::Relaxed) } + + /// The idle periods in a declared state this CPU has started and ended, + /// in that order: the CPU is in one now where the two differ. + pub fn deep_periods(&self) -> (u64, u64) { + let entered = self.entered.load(Ordering::Acquire); + (entered, self.deep.load(Ordering::Acquire)) + } } /// Read what this CPU can enter: called by each CPU as it first enters its @@ -175,14 +185,18 @@ pub fn wait(plan: &idle::Plan) { crate::sched::driver::publish_idle_exit(0); } match (state, &d) { - (Some(state), Some(d)) => crate::arch::idle::enter(d.entries[state]), + (Some(state), Some(d)) => { + // This CPU's alone to write, so a store and not a locked add. + local.entered.store(local.entered.load(Ordering::Relaxed) + 1, Ordering::Release); + crate::arch::idle::enter(d.entries[state]); + } _ => crate::arch::hw::halt(), } // Awake: no hold has a reason to wake this CPU until it waits again. crate::sched::driver::publish_idle_exit(0); let woke = crate::clock::nanos_since_boot(); if state.is_some() { - local.deep.fetch_add(1, Ordering::Relaxed); + local.deep.fetch_add(1, Ordering::Release); } let Some(d) = d else { return }; let costs = &d.costs[..d.len]; diff --git a/sdk/abi/src/trace.rs b/sdk/abi/src/trace.rs index 5268dae645f..713d2b26030 100644 --- a/sdk/abi/src/trace.rs +++ b/sdk/abi/src/trace.rs @@ -3,7 +3,7 @@ //! always. //! //! A record's [`TraceRecord::stamp`] is the CPU's free-running counter (TSC, -//! `CNTVCT_EL0`) in its own ticks, which [`crate::clock::ClockPage`] turns into +//! `CNTVCT_EL0`) in its own ticks, which a [`crate::clock::Clock`] turns into //! nanoseconds since boot. A read takes a cursor laid out as the log's //! ([`crate::log::CursorHead`]), a position per CPU the firmware names, and //! answers records oldest first, merged across CPUs by stamp, with the diff --git a/tests/common/faults.rs b/tests/common/faults.rs index c35c2120c0c..384392afc0b 100644 --- a/tests/common/faults.rs +++ b/tests/common/faults.rs @@ -136,30 +136,26 @@ pub fn kernel_wx_leaf_is_refused(test_config: &Path, profile: qemu::Profile) -> } } -/// What `tsc-step` says of the step it stages, and the middle of the clock +/// What `tsc-step` says of the step it stages, and the head of the clock /// check's panic at it (`kernel/src/arch/x86_64/clocksource.rs`). -const TSC_STEP_STAGED: &str = "tsc-step: cpu"; -const TSC_STEPPED: &str = "'s TSC stepped: "; +const TSC_STEP_STAGED: &str = "tsc-step: the TSC reads "; +const TSC_DISAGREES: &str = "clock: the TSC and the HPET disagree by "; -/// A TSC `tsc-step` reads 10 ms ahead on one CPU: the clock's runtime check -/// panics on that CPU, after the step, having read the HPET 9 ms or more apart -/// from the clock. +/// A TSC `tsc-step` reads 10 ms ahead: the clock's runtime check panics after +/// the step, having read it 9 ms or more apart from the HPET, as a step. pub fn a_stepped_tsc_panics(test_config: &Path) -> Result<(), String> { let qemu = QemuInstance::boot_with_options( test_config, &[], &[], - BootOptions { profile: qemu::Profile::Metal, kernel_params: &["tsc-step"], ready_marker: TSC_STEPPED, ..Default::default() }, + BootOptions { profile: qemu::Profile::Metal, kernel_params: &["tsc-step"], ready_marker: TSC_DISAGREES, ..Default::default() }, ); let serial = qemu.boot_log(); let lines: Vec<&str> = serial.lines().collect(); let staged = lines.iter().position(|l| l.contains(TSC_STEP_STAGED)).ok_or_else(|| format!("no {TSC_STEP_STAGED:?} line\n{serial}"))?; - let panicked = lines.iter().position(|l| l.contains(TSC_STEPPED)).ok_or_else(|| format!("no {TSC_STEPPED:?} panic\n{serial}"))?; - let cpu = |line: &str, head: &str| line.split(head).nth(1).and_then(|r| r.split(|c: char| !c.is_ascii_digit()).next()).map(str::to_string); - let stepped_cpu = cpu(lines[staged], TSC_STEP_STAGED); - let panicked_cpu = cpu(lines[panicked], "clock: cpu"); - let apart: Option = lines[panicked].split(TSC_STEPPED).nth(1).and_then(|r| r.split(" ns apart").next()).and_then(|n| n.parse().ok()); - if staged > panicked || stepped_cpu != panicked_cpu || !apart.is_some_and(|ns| ns >= 9_000_000) { + let panicked = lines.iter().position(|l| l.contains(TSC_DISAGREES)).ok_or_else(|| format!("no {TSC_DISAGREES:?} panic\n{serial}"))?; + let apart: Option = lines[panicked].split(TSC_DISAGREES).nth(1).and_then(|r| r.split(" ns over").next()).and_then(|n| n.parse().ok()); + if staged > panicked || !lines[panicked].contains("since the last sample") || !apart.is_some_and(|ns| ns >= 9_000_000) { return Err(format!( "the step and the panic do not match: {:?} then {:?}\n{serial}", lines[staged].trim(), From 0fc019ac8cf90e9bef2c1838a7c2ab5684736081 Mon Sep 17 00:00:00 2001 From: japabu Date: Thu, 29 Oct 2026 00:30:41 +0100 Subject: [PATCH 04/11] clock: the ACPI PM timer tells which of the TSC and the HPET moved MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The AMD bench's boot at 68afed54f panicked at 17.585 s on the TSC reading 1,723,545 ns ahead of the HPET over 680 ms, the event Linux's watchdog saw in 5 of 30 boots. Two counters cannot say which of them moved. At every HPET sample the sampler now reads the ACPI PM timer too (the FADT's PM_TMR_BLK and its X_ twin, 24 or 32 bits by TMR_VAL_EXT, declared read-only to the kernel), and where the TSC and the HPET disagree across a span, holds the TSC to the PM timer over the same span: - the PM timer agrees with the TSC: the HPET moved, and the span is left out with all three counters' spans on the record; - it disagrees by what the HPET does: two of the chipset's counters agree against the TSC, which panics, saying so; the two are both the FCH's (the PCH's on Intel) and may share a clock, so a stall common to both reads as the TSC having moved; - neither: all three are on the record and the span is left out. Each report also counts the left-out spans across which the HPET, and the PM timer, read apart from the TSC, which is what holds the idle guard to evidence. A PM timer that is absent or does not count within 100 µs is said, and a step then panics as before. hpet-stall stages an HPET 10 ms behind beside tsc-step's TSC ahead; a_stalled_hpet_is_left_out and a_stepped_tsc_panics hold the two outcomes. Co-Authored-By: Claude Opus 5.5 --- kernel/pure/clock/reference.rs | 96 ++++++++++-- kernel/src/actuator.rs | 6 + kernel/src/arch/x86_64/boot.rs | 2 +- kernel/src/arch/x86_64/clocksource.rs | 216 ++++++++++++++++++++++---- kernel/src/drivers/acpi.rs | 7 + lib/acpi/src/fadt.rs | 37 +++++ lib/acpi/src/lib.rs | 4 +- lib/acpi/tests/fixtures.rs | 6 +- tests/common/faults.rs | 33 ++++ tests/toyos.rs | 6 + 10 files changed, 368 insertions(+), 45 deletions(-) diff --git a/kernel/pure/clock/reference.rs b/kernel/pure/clock/reference.rs index 4631e66d7fb..1bd387e373d 100644 --- a/kernel/pure/clock/reference.rs +++ b/kernel/pure/clock/reference.rs @@ -379,13 +379,44 @@ impl Offset { } } -/// The 64-bit reading of a reference whose counter is 32 bits wide: the value -/// with `low` as its low half nearest `predicted`, the earlier 64-bit reading +/// The 64-bit reading of a reference whose counter is `bits` wide: the value +/// with `low` as its low bits nearest `predicted`, the earlier 64-bit reading /// plus the ticks the counter says have passed since. Right while that -/// prediction is within 2^31 ticks. -pub const fn extend(predicted: u64, low: u32) -> u64 { - let off = (low as u64).wrapping_sub(predicted) as u32 as i32; - predicted.wrapping_add_signed(off as i64) +/// prediction is within half the counter's range. +pub const fn extend(predicted: u64, low: u64, bits: u32) -> u64 { + let range = 1u64 << bits; + let off = low.wrapping_sub(predicted) & (range - 1); + if off >= range / 2 { + predicted.wrapping_add(off).wrapping_sub(range) + } else { + predicted.wrapping_add(off) + } +} + +/// What a second reference says of a step the counter and the first +/// reference disagree on, over the same span. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Arbitration { + /// The second reference agrees with the counter: the first moved. + FirstMoved, + /// The second reference disagrees with the counter by what the first + /// does, within both agreements' allowances: the counter moved, or both + /// references did together. + CounterMoved, + /// The two references disagree with the counter and with each other. + AllDisagree, +} + +/// `first` and `second`: the counter against each reference over one span, +/// the first's an agreement that failed. +pub const fn arbitrate(first: Agreement, second: Agreement) -> Arbitration { + if second.holds() { + Arbitration::FirstMoved + } else if first.apart_ns.abs_diff(second.apart_ns) <= first.allowed_ns + second.allowed_ns { + Arbitration::CounterMoved + } else { + Arbitration::AllDisagree + } } #[cfg(test)] @@ -616,10 +647,57 @@ mod tests { assert_eq!(Offset { least_out: 200, least_back: -45 }.bound(), Err(45)); } + /// The PM timer, at 3.579545 MHz, across the same spans as the HPET. + const PM: Reference = + Reference { period_fs: 279_365_114, tolerance_ppb: 500_000, settled_ns: 1_000_000, short_tolerance_ppb: 2_000_000 }; + + /// The PM timer's sample a perfect counter takes at `ns`, its step `step`. + fn pm_at(ns: u64, step: i64) -> Sample { + let n = (ns as u128 * 1_000_000 / PM.period_fs as u128) as u64; + let counter = (n as u128 * PM.period_fs as u128 * HZ / 1_000_000_000_000_000) as u64; + let latch = counter.wrapping_add_signed(step); + Sample { before: latch - READ / 2, after: latch - READ / 2 + READ, reference: n } + } + + fn ns_of(n: u64) -> u64 { + (n as u128 * HPET.period_fs as u128 / 1_000_000) as u64 + } + + /// The AMD's 1.7 ms over 680 ms: as the TSC's own step, both references + /// read it alike; as the HPET's stall, the PM timer reads the TSC right; + /// as a step of each by a different size, nothing agrees. + #[test] + fn a_second_reference_tells_which_moved() { + let mult = ((1_000_000_000u128 << 32) / HZ) as u64; + let tolerance = 500_000; + let (n0, n1) = (100 * SECOND, 100 * SECOND + 9_739_369); + let step = 3_096_525i64; + let hpet = |n, step| sample_at(HZ, n, READ / 2, step); + let tsc_moved = arbitrate( + agree(HPET, hpet(n0, 0), hpet(n1, step), mult, tolerance), + agree(PM, pm_at(ns_of(n0), 0), pm_at(ns_of(n1), step), mult, tolerance), + ); + assert_eq!(tsc_moved, Arbitration::CounterMoved); + let stalled = n1 - 24_675; + let hpet_moved = arbitrate( + agree(HPET, hpet(n0, 0), Sample { reference: stalled, ..hpet(n1, 0) }, mult, tolerance), + agree(PM, pm_at(ns_of(n0), 0), pm_at(ns_of(n1), 0), mult, tolerance), + ); + assert_eq!(hpet_moved, Arbitration::FirstMoved); + let neither = arbitrate( + agree(HPET, hpet(n0, 0), hpet(n1, step), mult, tolerance), + agree(PM, pm_at(ns_of(n0), 0), pm_at(ns_of(n1), step / 3), mult, tolerance), + ); + assert_eq!(neither, Arbitration::AllDisagree); + } + #[test] fn a_32_bit_reference_is_extended_across_its_wrap() { - assert_eq!(extend(0x1_ffff_fff0, 0x0000_0010), 0x2_0000_0010); - assert_eq!(extend(0x2_0000_0010, 0xffff_fff0), 0x1_ffff_fff0); - assert_eq!(extend(0x5_1234_5678, 0x1234_5679), 0x5_1234_5679); + assert_eq!(extend(0x1_ffff_fff0, 0x0000_0010, 32), 0x2_0000_0010); + assert_eq!(extend(0x2_0000_0010, 0xffff_fff0, 32), 0x1_ffff_fff0); + assert_eq!(extend(0x5_1234_5678, 0x1234_5679, 32), 0x5_1234_5679); + // The ACPI PM timer's 24 bits, across their wrap and back. + assert_eq!(extend(0x3_ff_fff0, 0x00_0010, 24), 0x4_00_0010); + assert_eq!(extend(0x4_00_0010, 0xff_fff0, 24), 0x3_ff_fff0); } } diff --git a/kernel/src/actuator.rs b/kernel/src/actuator.rs index 9a279f545ee..d28157ef16c 100644 --- a/kernel/src/actuator.rs +++ b/kernel/src/actuator.rs @@ -130,6 +130,12 @@ actuators! { /// `a_stepped_tsc_panics`. tsc_step = "tsc-step"; + /// Read the HPET 10 ms behind in its samples once the clock has held for + /// a second, as an HPET that stood still while the TSC and the ACPI PM + /// timer counted: the span is left out and the boot goes on. Judged by + /// `a_stalled_hpet_is_left_out`. + hpet_stall = "hpet-stall"; + /// Hand every CPU the loader's Intel update with its header's revision /// raised by one and its checksum sealed again: the read-back's negative /// control on real Intel, whose payload still says the revision it did. diff --git a/kernel/src/arch/x86_64/boot.rs b/kernel/src/arch/x86_64/boot.rs index 3cbf302c0bb..d9e7f17476a 100644 --- a/kernel/src/arch/x86_64/boot.rs +++ b/kernel/src/arch/x86_64/boot.rs @@ -110,7 +110,7 @@ pub fn interrupts(rsdp_addr: u64) -> Platform { /// The clock: the TSC, held to the HPET, and the CMOS wall clock. pub fn clock(args: &KernelArgs) { let hpet_base = acpi::find_hpet_base(args.rsdp_addr).expect("ACPI: HPET not found"); - super::clocksource::start(hpet_base); + super::clocksource::start(hpet_base, acpi::pm_timer(args.rsdp_addr)); let century_reg = match acpi::rtc_century_register(args.rsdp_addr) { Ok(reg) => reg, Err(e) => { diff --git a/kernel/src/arch/x86_64/clocksource.rs b/kernel/src/arch/x86_64/clocksource.rs index c68a175b5dd..a88124c6945 100644 --- a/kernel/src/arch/x86_64/clocksource.rs +++ b/kernel/src/arch/x86_64/clocksource.rs @@ -16,10 +16,12 @@ use core::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use alloc::vec::Vec; -use kernel::clock::reference::{self, Calibration, Failure, Kept, Offset, Reference, Sample, Watch}; +use kernel::clock::reference::{self, Agreement, Arbitration, Calibration, Failure, Kept, Offset, Reference, Sample, Watch}; use toyos_abi::clock::Line; use super::cpu; +use super::pio::{self, Declared}; +use toyos_userbound::{Mediated, Ports}; use crate::log; use crate::mm::Mmio; use crate::mm::policy::MmioPolicy; @@ -76,7 +78,7 @@ impl Hpet { if self.wide { self.mmio.read_u64(HPET_COUNTER) } else { - reference::extend(near, self.mmio.read_u32(HPET_COUNTER)) + reference::extend(near, u64::from(self.mmio.read_u32(HPET_COUNTER)), 32) } } @@ -108,10 +110,70 @@ impl Hpet { } } +/// The ACPI PM timer, the second reference: which of the TSC and the HPET +/// moved, where the two disagree. **It and the HPET are both the chipset's +/// (the FCH's on AMD, the PCH's on Intel)**, and may share its clock, so a +/// stall common to both reads as the TSC having moved. +struct PmTimer { + port: Declared, + bits: u32, + reference: Reference, + last: Sample, +} + +/// ACPI 6.5 §4.8.3.3: 3.579545 MHz. ACPI states no tolerance for it; the +/// HPET's is taken, the same chipset's clock. +const PM_TIMER_PERIOD_FS: u64 = 279_365_114; + +impl PmTimer { + fn new(named: toyos_acpi::PmTimer) -> Result { + let ports = Ports::new(named.port, 4).ok_or("its block runs past the port space")?; + let port = pio::declare("the ACPI PM timer", ports, Mediated::ReadOnly).map_err(|e| alloc::format!("{e:?}"))?; + let mut timer = Self { + port, + bits: u32::from(named.bits), + reference: hpet_reference(PM_TIMER_PERIOD_FS), + last: Sample { before: 0, after: 0, reference: 0 }, + }; + let first = timer.read_raw(); + // A tick is 279 ns: a counter that does not move within 100 µs counts nothing. + let until = cpu::counter() + crate::clock::counter_ticks(100_000); + while timer.read_raw() == first { + if cpu::counter() > until { + return Err(alloc::format!("port {:#x} reads {first:#x} and does not count", named.port)); + } + } + timer.last = timer.sample(timer.read_raw()); + Ok(timer) + } + + fn read_raw(&self) -> u64 { + u64::from(cpu::inl(self.port.port(0))) & ((1u64 << self.bits) - 1) + } + + fn sample(&self, near: u64) -> Sample { + let before = cpu::rdtsc_ordered(); + let low = self.read_raw(); + let after = cpu::rdtsc_ordered(); + Sample { before, after, reference: reference::extend(near, low, self.bits) } + } + + /// The next sample, its wraps counted from the last by the kept rate. + fn next(&self, mult: u64) -> Sample { + let nanos = Line { counter: 0, stamp: 0, mult }.nanos(cpu::counter().saturating_sub(self.last.before)); + self.sample(self.last.reference + (u128::from(nanos) * 1_000_000 / u128::from(PM_TIMER_PERIOD_FS)) as u64) + } +} + /// What the HPET's sampler holds, once the boot has started the clock. struct Held { hpet: Hpet, watch: Watch, + pm: Option, + /// Spans left out, and of them those across which the HPET, and the PM + /// timer, disagreed with the TSC: what the left-out spans would have read. + unheld_hpet_apart: u64, + unheld_pm_apart: u64, /// Each CPU's [`crate::idle::Local::deep_periods`] at the last sample. idle: Vec<(u64, u64)>, } @@ -165,7 +227,7 @@ impl Local { /// Start the clock on the TSC, at its stated rate or one measured against the /// HPET at `hpet_base`, and hand the HPET to the runtime check. -pub fn start(hpet_base: u64) { +pub fn start(hpet_base: u64, pm_timer: Result) { admit("cpu0"); TSC_ADJUST.store(cpu::has_tsc_adjust(), Ordering::Relaxed); note_adjust(); @@ -195,7 +257,19 @@ pub fn start(hpet_base: u64) { None => calibrate(&hpet), }; crate::clock::set_counter(anchor.before, kept.mult); - *HELD.lock() = Some(Held { watch: Watch::new(hpet.reference, anchor, kept), hpet, idle: Vec::new() }); + let pm = pm_timer.and_then(PmTimer::new); + match &pm { + Ok(pm) => log!("clock: the ACPI PM timer at {:#x}, {} bits, tells the TSC from the HPET where they disagree", pm.port.port(0).number(), pm.bits), + Err(why) => log!("clock: no ACPI PM timer to tell the TSC from the HPET ({why}); a step between them panics"), + } + *HELD.lock() = Some(Held { + watch: Watch::new(hpet.reference, anchor, kept), + hpet, + pm: pm.ok(), + unheld_hpet_apart: 0, + unheld_pm_apart: 0, + idle: Vec::new(), + }); } /// The TSC's rate against the HPET over [`CALIBRATION`], first sample to last, @@ -398,47 +472,90 @@ pub fn check(now_ns: u64) { } } -/// One sample of the HPET, held to the TSC where the span since the last -/// can be. +/// One sample of the HPET, and of the PM timer beside it, held to the TSC +/// where the span since the last can be. fn sample(held: &mut Held, cpu_id: u32) { let awake = held.some_cpu_stayed_awake(); - let sample = held.hpet.sample(held.hpet.near(held.watch.last(), held.watch.kept().mult), 1); + let kept = held.watch.kept(); + let last = held.watch.last(); + let hpet = held.hpet.sample(held.hpet.near(last, kept.mult), 1); + let pm = held.pm.as_ref().map(|pm| (pm.last, pm.next(kept.mult))); #[cfg(feature = "boot-actuators")] - let sample = staged(sample, false); - match held.watch.take(sample, awake) { + let (hpet, pm) = staged(hpet, pm, false, held.hpet.reference.period_fs); + let against = |reference: Reference, from: Sample, to: Sample| { + let span = reference::Span::between(reference, from, to); + let tolerance = kept.bound_ppb + reference.tolerance_over(reference.span_ns(from, to)); + reference::agree_over(reference, span, kept.mult, tolerance) + }; + let pm_reference = held.pm.as_ref().map(|pm| pm.reference); + let pm_agreement = pm.zip(pm_reference).map(|((from, to), reference)| against(reference, from, to)); + if let (Some(timer), Some((_, to))) = (held.pm.as_mut(), pm) { + timer.last = to; + } + if !awake { + held.unheld_hpet_apart += u64::from(!against(held.hpet.reference, last, hpet).holds()); + held.unheld_pm_apart += u64::from(pm_agreement.is_some_and(|a| !a.holds())); + let _ = held.watch.take(hpet, false); + return; + } + match held.watch.take(hpet, true) { Ok(None) => {} Ok(Some(report)) => { #[cfg(feature = "boot-actuators")] - staged(sample, true); + staged(hpet, pm, true, held.hpet.reference.period_fs); let kept = Line { counter: 0, stamp: 0, mult: report.kept.mult }; if report.retuned { crate::clock::retune(report.kept.mult); } let measured = Line { counter: 0, stamp: 0, mult: report.measured.mult }.millihertz(); let (span_s, bound) = (report.measured.span_ns / 1_000_000_000, report.measured.bound_ppb); - let unheld = report.unheld; + let unheld = alloc::format!( + "{} spans unheld, across which the HPET read apart from the TSC in {} and the PM timer in {}", + report.unheld, held.unheld_hpet_apart, held.unheld_pm_apart, + ); match (report.kept.measured, report.retuned) { (false, _) => log!( "clock: over {span_s} s the TSC counted {measured} mHz against the HPET, within {bound} ppb; \ - the clock keeps CPUID 15H's {} mHz; {unheld} spans unheld", + the clock keeps CPUID 15H's {} mHz; {unheld}", kept.millihertz(), ), (true, retuned) => log!( "clock: over {span_s} s the TSC counted {measured} mHz against the HPET, within {bound} ppb; \ - the clock {} {} mHz, within {} ppb; {unheld} spans unheld", + the clock {} {} mHz, within {} ppb; {unheld}", if retuned { "now keeps" } else { "keeps" }, kept.millihertz(), report.kept.bound_ppb, ), } } - Err(Failure::Step { from, to, agreement }) => panic!( - "clock: the TSC and the HPET disagree by {} ns over the {} ns the HPET counted since the last sample, {} \ - allowed: one of them stepped (cpu{cpu_id}, {from:?} to {to:?})", - agreement.apart_ns, - agreement.reference_ns, - agreement.allowed_ns, - ), + Err(Failure::Step { from, to, agreement }) => { + let spans = spans(agreement, pm_agreement, from, to, pm); + match pm_agreement.map(|second| reference::arbitrate(agreement, second)) { + Some(Arbitration::FirstMoved) => { + log!("clock: the HPET moved and the TSC did not, so this span is left out: {spans}"); + let _ = held.watch.take(hpet, false); + } + Some(Arbitration::AllDisagree) => { + log!("clock: the TSC, the HPET and the ACPI PM timer all disagree, so this span is left out: {spans}"); + let _ = held.watch.take(hpet, false); + } + Some(Arbitration::CounterMoved) => panic!( + "clock: the TSC and the HPET disagree by {} ns over the {} ns the HPET counted since the last \ + sample, {} allowed, and the ACPI PM timer agrees with the HPET: two of the chipset's counters \ + agree against the TSC, which moved, or both stalled together (cpu{cpu_id}; {spans})", + agreement.apart_ns, + agreement.reference_ns, + agreement.allowed_ns, + ), + None => panic!( + "clock: the TSC and the HPET disagree by {} ns over the {} ns the HPET counted since the last \ + sample, {} allowed, and no PM timer says which moved (cpu{cpu_id}; {spans})", + agreement.apart_ns, + agreement.reference_ns, + agreement.allowed_ns, + ), + } + } Err(Failure::Drift { since, to, agreement }) => panic!( "clock: the TSC and the HPET disagree by {} ns over the {} ns the HPET counted in held spans, {} allowed: \ their rates are further apart than the kept rate's bound and the HPET's tolerance (cpu{cpu_id}, the \ @@ -450,17 +567,54 @@ fn sample(held: &mut Held, cpu_id: u32) { } } -/// `tsc-step`: from the first report on, every sample reads the TSC 10 ms -/// ahead of where it is. A hypervisor's TSC may ignore a write (QEMU's TCG -/// does), so the step is staged in the readings the sampler takes. +/// All three counters' spans, for a disagreement's record. +fn spans( + hpet: Agreement, + pm: Option, + from: Sample, + to: Sample, + pm_samples: Option<(Sample, Sample)>, +) -> alloc::string::String { + let pm = match (pm, pm_samples) { + (Some(a), Some((pm_from, pm_to))) => alloc::format!( + "the PM timer counted {} ns and reads {} ns apart, {} allowed ({pm_from:?} to {pm_to:?})", + a.reference_ns, a.apart_ns, a.allowed_ns, + ), + _ => "no PM timer".into(), + }; + alloc::format!( + "the TSC counted {} ticks, the HPET {} ns and reads {} ns apart, {} allowed ({from:?} to {to:?}); {pm}", + to.before.wrapping_sub(from.before), + hpet.reference_ns, + hpet.apart_ns, + hpet.allowed_ns, + ) +} + +/// `tsc-step` and `hpet-stall`: from the first report on, every sample reads +/// either the TSC 10 ms ahead of where it is, in both references' readings, +/// or the HPET 10 ms behind, as a TSC that stepped and an HPET that stood +/// still. A hypervisor's TSC may ignore a write (QEMU's TCG does), so each is +/// staged in the readings the sampler takes. #[cfg(feature = "boot-actuators")] -fn staged(sample: Sample, reported: bool) -> Sample { - static AHEAD: AtomicU64 = AtomicU64::new(0); - if crate::actuator::tsc_step() && reported && AHEAD.load(Ordering::Relaxed) == 0 { - let ahead = crate::clock::counter_ticks(10_000_000); - AHEAD.store(ahead, Ordering::Relaxed); - log!("tsc-step: the TSC reads {ahead} cycles ahead from here"); +fn staged(hpet: Sample, pm: Option<(Sample, Sample)>, reported: bool, period_fs: u64) -> (Sample, Option<(Sample, Sample)>) { + static TSC_AHEAD: AtomicU64 = AtomicU64::new(0); + static HPET_BEHIND: AtomicU64 = AtomicU64::new(0); + if reported && TSC_AHEAD.load(Ordering::Relaxed) == 0 && HPET_BEHIND.load(Ordering::Relaxed) == 0 { + if crate::actuator::tsc_step() { + let ahead = crate::clock::counter_ticks(10_000_000); + TSC_AHEAD.store(ahead, Ordering::Relaxed); + log!("tsc-step: the TSC reads {ahead} cycles ahead from here"); + } + if crate::actuator::hpet_stall() { + let behind = 10_000_000_000_000 / period_fs; + HPET_BEHIND.store(behind, Ordering::Relaxed); + log!("hpet-stall: the HPET reads {behind} ticks behind from here"); + } } - let ahead = AHEAD.load(Ordering::Relaxed); - Sample { before: sample.before + ahead, after: sample.after + ahead, ..sample } + let ahead = TSC_AHEAD.load(Ordering::Relaxed); + let forward = |s: Sample| Sample { before: s.before + ahead, after: s.after + ahead, ..s }; + let behind = HPET_BEHIND.load(Ordering::Relaxed); + let hpet = Sample { reference: hpet.reference - behind, ..forward(hpet) }; + (hpet, pm.map(|(from, to)| (from, forward(to)))) } diff --git a/kernel/src/drivers/acpi.rs b/kernel/src/drivers/acpi.rs index 54ccdb4454e..e4c89b5761d 100644 --- a/kernel/src/drivers/acpi.rs +++ b/kernel/src/drivers/acpi.rs @@ -201,6 +201,13 @@ pub fn find_hpet_base(rsdp_addr: u64) -> Option { Some(base) } +/// The ACPI PM timer the FADT names, or why there is none. +pub fn pm_timer(rsdp_addr: u64) -> Result { + let fadt = toyos_acpi::find_table(direct_phys(), rsdp_addr, b"FACP", toyos_acpi::FADT_FOR_FIXED_HARDWARE) + .map_err(|e| alloc::format!("the FADT is unusable ({e:?})"))?; + toyos_acpi::pm_timer(&fadt).map_err(|e| alloc::format!("{e:?}")) +} + /// How many processors the MADT enables, of either architecture's kind: its /// Local APIC and x2APIC entries, or its GICC entries. What every per-CPU /// table is sized by, before either architecture's own walk of the MADT. diff --git a/lib/acpi/src/fadt.rs b/lib/acpi/src/fadt.rs index 049dba9b28e..b2c0b112236 100644 --- a/lib/acpi/src/fadt.rs +++ b/lib/acpi/src/fadt.rs @@ -167,10 +167,12 @@ const FADT_PSTATE_CNT: usize = 55; const FADT_PM1A_EVT_BLK: usize = 56; const FADT_PM1B_EVT_BLK: usize = 60; const FADT_PM1B_CNT_BLK: usize = 68; +const FADT_PM_TMR_BLK: usize = 76; const FADT_GPE0_BLK: usize = 80; const FADT_GPE1_BLK: usize = 84; const FADT_PM1_EVT_LEN: usize = 88; const FADT_PM1_CNT_LEN: usize = 89; +const FADT_PM_TMR_LEN: usize = 91; const FADT_GPE0_BLK_LEN: usize = 92; const FADT_GPE1_BLK_LEN: usize = 93; const FADT_CST_CNT: usize = 95; @@ -178,6 +180,7 @@ const FADT_X_PM1A_EVT_BLK: usize = 148; const FADT_X_PM1B_EVT_BLK: usize = 160; const FADT_X_PM1A_CNT_BLK: usize = 172; const FADT_X_PM1B_CNT_BLK: usize = 184; +const FADT_X_PM_TMR_BLK: usize = 208; const FADT_X_GPE0_BLK: usize = 220; const FADT_X_GPE1_BLK: usize = 232; /// The end of `X_GPE1_BLK`, the last field [`fixed_hardware`] reads where the @@ -187,6 +190,9 @@ const FADT_X_END: usize = FADT_X_GPE1_BLK + 12; /// Table 5.10's `PWR_BUTTON` (bit 4): set, the power button is a control /// method device, which only AML serves; and `HW_REDUCED_ACPI` (bit 20). const PWR_BUTTON: u32 = 1 << 4; +/// Table 5.10's `TMR_VAL_EXT` (bit 8): set, the PM timer's value is 32 bits +/// wide, and clear, 24. +const TMR_VAL_EXT: u32 = 1 << 8; const HW_REDUCED_ACPI: u32 = 1 << 20; /// The bytes [`fixed_hardware`] needs at least: through the flags. @@ -286,6 +292,7 @@ pub enum Field { Pm1aEvent, Pm1aControl, Gpe0, + PmTimer, } /// Why the FADT's fixed hardware is none this kernel serves, one reason each. @@ -346,6 +353,36 @@ fn block(field: Field, address: u64, len: u8) -> Result(fadt: &Table

) -> Result { + let short = FixedRefused::Short { len: fadt.len() }; + let flags = fadt.u32_at(FADT_FLAGS).ok_or(short)?; + if flags & HW_REDUCED_ACPI != 0 { + return Err(FixedRefused::HardwareReduced); + } + // §5.2.9: `PM_TMR_LEN` is 4 where the timer is supported. + let len = fadt.byte(FADT_PM_TMR_LEN).ok_or(short)?; + let at = address(fadt, Field::PmTimer, FADT_PM_TMR_BLK, FADT_X_PM_TMR_BLK)?; + if at == 0 { + return Err(FixedRefused::Absent { field: Field::PmTimer }); + } + if len != 4 { + return Err(FixedRefused::Length { field: Field::PmTimer, len }); + } + let block = block(Field::PmTimer, at, len)?; + Ok(PmTimer { port: block.port, bits: if flags & TMR_VAL_EXT != 0 { 32 } else { 24 } }) +} + /// The PM1a control block, checked against its `X_` twin where the table /// holds one; its length the `PM1_CNT_LEN` byte's. pub fn pm1a_control(fadt: &Table

) -> Result { diff --git a/lib/acpi/src/lib.rs b/lib/acpi/src/lib.rs index 71ce28fa26b..8b9f82f51a2 100644 --- a/lib/acpi/src/lib.rs +++ b/lib/acpi/src/lib.rs @@ -32,9 +32,9 @@ use toyos_bootmap::DirectMapEnd; pub use ecdt::{ecdt, Ec, EcRefused, EcRegister, ECDT_NEEDED}; pub use facs::{acquire, facs, release, Facs, FacsRefused, FACS_GLOBAL_LOCK, OWNED, PENDING}; pub use fadt::{ - century_of, dsdt_address, fixed_hardware, pm1a_control, psci, reset_register, + century_of, dsdt_address, fixed_hardware, pm1a_control, pm_timer, psci, reset_register, rtc_century, - Century, Field, FixedHardware, FixedRefused, LegacyMode, PowerButton, Psci, Reset, SmiCmd, CMOS_RAM, + Century, Field, FixedHardware, FixedRefused, LegacyMode, PmTimer, PowerButton, Psci, Reset, SmiCmd, CMOS_RAM, FADT_FOR_FIXED_HARDWARE, FADT_PM1A_CNT_BLK, }; use fadt::FADT_DSDT; diff --git a/lib/acpi/tests/fixtures.rs b/lib/acpi/tests/fixtures.rs index 89399770910..a46b2d1ed60 100644 --- a/lib/acpi/tests/fixtures.rs +++ b/lib/acpi/tests/fixtures.rs @@ -8,7 +8,7 @@ use toyos_abi::boot::RootBridgeWindow; use toyos_abi::acpi::Block; use toyos_acpi::{ century_of, definition_blocks, dsdt_address, ecam_allocations, find_table, fixed_hardware, hpet_base, - isa_line, madt_entries, memory_windows, pm1a_control, psci, reset_register, rtc_century, sci_line, + isa_line, madt_entries, memory_windows, pm1a_control, pm_timer, psci, reset_register, rtc_century, sci_line, PmTimer, Century, FixedHardware, IoApicEntry, Line, MadtEntry, Polarity, PowerButton, Psci, Reset, SmiCmd, SourceOverride, TableError, Trigger, FADT_FOR_FIXED_HARDWARE, FADT_PM1A_CNT_BLK, MADT_ENTRIES, @@ -208,7 +208,8 @@ fn a_second_firmwares_bytes_decode_to_that_machines_own_windows() { /// q35's ICH9 power-management block at the base QEMU's firmware put it, /// 0x600, as the FADT names it: the blocks the `acpi_press_is_one_sci` guest test -/// takes its SCI through, and the 32-bit fields and their `X_` twins agreeing. +/// takes its SCI through, and the 32-bit fields and their `X_` twins agreeing; +/// and its 24-bit PM timer at 0x608, which the clock arbitrates with. #[test] fn the_q35_fadt_names_the_fixed_hardware_its_sci_is_served_through() { let fadt = find_table(machine(), RSDP, b"FACP", FADT_FOR_FIXED_HARDWARE).expect("FADT"); @@ -223,6 +224,7 @@ fn the_q35_fadt_names_the_fixed_hardware_its_sci_is_served_through() { }) ); assert_eq!(pm1a_control(&fadt), Ok(Block { port: 0x604, len: 2 })); + assert_eq!(pm_timer(&fadt), Ok(PmTimer { port: 0x608, bits: 24 })); } /// `ioapic: iso bus:irq->gsi [... 0:9->9 level/high ...]`: q35 names its SCI's diff --git a/tests/common/faults.rs b/tests/common/faults.rs index 384392afc0b..233612cbf3f 100644 --- a/tests/common/faults.rs +++ b/tests/common/faults.rs @@ -165,3 +165,36 @@ pub fn a_stepped_tsc_panics(test_config: &Path) -> Result<(), String> { eprintln!(" [tsc_step] {}", lines[panicked].trim()); Ok(()) } + +/// What `hpet-stall` says of the stall it stages, the sampler's record of +/// leaving the span out, and a report after it. +const HPET_STALL_STAGED: &str = "hpet-stall: the HPET reads "; +const HPET_MOVED: &str = "clock: the HPET moved and the TSC did not, so this span is left out: "; +const CLOCK_REPORT: &str = "clock: over "; + +/// An HPET `hpet-stall` reads 10 ms behind while the TSC and the ACPI PM timer +/// count on: the sampler says the HPET moved, leaves the span out, and the +/// clock goes on reporting rather than panicking. +pub fn a_stalled_hpet_is_left_out(test_config: &Path) -> Result<(), String> { + let mut qemu = QemuInstance::boot_with_options( + test_config, + &[], + &[], + BootOptions { profile: qemu::Profile::Metal, kernel_params: &["hpet-stall"], ready_marker: HPET_MOVED, ..Default::default() }, + ); + let staged_then_moved = { + let serial = qemu.boot_log(); + let staged = serial.find(HPET_STALL_STAGED); + let moved = serial.find(HPET_MOVED); + match (staged, moved) { + (Some(staged), Some(moved)) if staged < moved => Ok(serial[moved..].lines().next().unwrap_or_default().to_string()), + _ => Err(format!("no {HPET_STALL_STAGED:?} before {HPET_MOVED:?}\n{serial}")), + } + }?; + let after = qemu.drain_until(std::time::Duration::from_secs(60), |l| l.contains(CLOCK_REPORT) || l.contains("PANIC")); + if after.contains("PANIC") || !after.contains(CLOCK_REPORT) { + return Err(format!("the clock did not report again after it left the span out\n{after}")); + } + eprintln!(" [hpet_stall] {}", staged_then_moved.trim()); + Ok(()) +} diff --git a/tests/toyos.rs b/tests/toyos.rs index 53244fecb78..37e0b0ae3a3 100644 --- a/tests/toyos.rs +++ b/tests/toyos.rs @@ -526,6 +526,11 @@ const MACHINE_TESTS: &[&str] = &[ // carries the panic; the order of the step and the panic, and the size // the check read, are what it holds. "a_stepped_tsc_panics", + // Its counterpart: the HPET an actuator stalled while the TSC and the PM + // timer count on is left out, and the boot goes on. q35 on + // `Profile::Metal`, whose PM timer is at 0x608; no bench's HPET stalls on + // demand. + "a_stalled_hpet_is_left_out", // A press of the fixed power button reaches the `acpi` claim's holder as // one SCI, its line masked until the holder serves it: the kernel's SCI // and its mask have no host build, QEMU raises the event on demand, and @@ -4681,6 +4686,7 @@ fn run_machine_test(name: &str, test_config: &Path) -> Result<(), String> { "nested_nmi_is_loud" => faults::nested_nmi_is_loud(test_config), "kernel_wx_leaf_is_refused" => faults::kernel_wx_leaf_is_refused(test_config, qemu::Profile::Metal), "a_stepped_tsc_panics" => faults::a_stepped_tsc_panics(test_config), + "a_stalled_hpet_is_left_out" => faults::a_stalled_hpet_is_left_out(test_config), "acpi_press_is_one_sci" => acpi_press_is_one_sci(), "osbench_capability_groups_exit_0" => osbench_capability_groups_exit_0(), "acpi_mediated_access" => acpi_mediated_access(2), From 471e0d811b37e77d348856c220084faa64b8032f Mon Sep 17 00:00:00 2001 From: japabu Date: Thu, 29 Oct 2026 00:39:29 +0100 Subject: [PATCH 05/11] clock: the PM timer test's literals in even groups Co-Authored-By: Claude Opus 5.5 --- kernel/pure/clock/reference.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kernel/pure/clock/reference.rs b/kernel/pure/clock/reference.rs index 1bd387e373d..e7ec9035b1a 100644 --- a/kernel/pure/clock/reference.rs +++ b/kernel/pure/clock/reference.rs @@ -697,7 +697,7 @@ mod tests { assert_eq!(extend(0x2_0000_0010, 0xffff_fff0, 32), 0x1_ffff_fff0); assert_eq!(extend(0x5_1234_5678, 0x1234_5679, 32), 0x5_1234_5679); // The ACPI PM timer's 24 bits, across their wrap and back. - assert_eq!(extend(0x3_ff_fff0, 0x00_0010, 24), 0x4_00_0010); - assert_eq!(extend(0x4_00_0010, 0xff_fff0, 24), 0x3_ff_fff0); + assert_eq!(extend(0x03ff_fff0, 0x0010, 24), 0x0400_0010); + assert_eq!(extend(0x0400_0010, 0x00ff_fff0, 24), 0x03ff_fff0); } } From 1a65ddc422586fe25ba47adda88a8649a474712f Mon Sep 17 00:00:00 2001 From: japabu Date: Sun, 1 Nov 2026 00:18:53 +0100 Subject: [PATCH 06/11] clock: the RTC tells a TSC step from a chipset stall; the first is slewed away, a second demotes the TSC On the AMD bench, the TSC steps forward about 1.7 ms once in every boot. The HPET, the ACPI PM timer and the RTC all agree on it. The RTC runs off its own 32.768 kHz crystal; the HPET and the PM timer run off the FCH's 14.318 MHz clock. The step lands inside an early spell in which every core sits in port 0x415 (issues/the-amd-benchs-tsc-steps-1-7-ms-forward-once-a-boot.md). Until now the idle guard left such spans out unseen. The guard goes; every span is now arbitrated: - **The TSC and the HPET agree:** the span is held, as before. - **The PM timer agrees with the TSC:** the HPET moved, and the span is left out. - **The PM timer agrees with the HPET, or there is no PM timer:** the RTC decides. Its update-ended interrupt (IRQ8, vector 0x26, counted as `rtc` in the census) times the TSC against each edge. An edge reads late by its interrupt's latency and never early, so the least-latency of three edges on each side of the span gives the RTC's reading. Taken within a quarter of the step: - **near the step:** the TSC stepped; - **near zero:** the chipset's counters moved together, and the span is left out; - **neither, or no edge yet:** taken as a TSC step, logged "unconfirmed by the RTC". - **A TSC step:** - **the boot's first, forward:** logged loudly with its size and slewed away at no more than 500 ppm (RFC 5905's MAXFREQ, the frequency tolerance NTP disciplines a clock within). 1.7 ms takes 3.4 s, and monotonic time never steps. - **a second, or one backward:** demotes the TSC. **Demotion.** A per-CPU check finding `IA32_TSC_ADJUST` changed, or a CPU's TSC behind another's, now demotes the TSC instead of panicking. The demoted TSC leaves for the HPET for the rest of the boot: - the clock page is marked demoted; - `toyos_abi::clock::nanos_since_boot` and `stamp_ns` take the new `SYS_CLOCK_NOW`, which counts by the HPET from the later of the clock now and the last held sample plus the HPET since; - the kernel's deadlines and log stamps stay on the TSC. **ABI.** The page gains its demoted word (TOYOCLK3). **Test.** `a_stepped_tsc_is_slewed_then_demoted` replaces `a_stepped_tsc_panics`. The `tsc-step` actuator now stages a second step once the first is slewed away, and shifts the RTC's edges with the TSC. Co-Authored-By: Claude Opus 5.5 --- ...hs-tsc-steps-1-7-ms-forward-once-a-boot.md | 49 ++ kernel/pure/clock/reference.rs | 249 ++++++++- kernel/src/actuator.rs | 8 +- kernel/src/arch/aarch64/clocksource.rs | 6 + kernel/src/arch/x86_64/boot.rs | 3 + kernel/src/arch/x86_64/clocksource.rs | 495 +++++++++++++----- kernel/src/arch/x86_64/idt/mod.rs | 5 + kernel/src/arch/x86_64/idt/rtc.rs | 12 + kernel/src/arch/x86_64/rtc.rs | 62 +++ kernel/src/clock.rs | 27 +- kernel/src/idle.rs | 20 +- kernel/src/irq_census.rs | 7 +- kernel/src/syscall/dispatch.rs | 1 + sdk/abi/src/clock.rs | 55 +- sdk/abi/src/syscall.rs | 13 +- tests/checks.rs | 4 +- tests/common/faults.rs | 47 +- tests/common/irqcensus.rs | 4 +- tests/toyos.rs | 14 +- 19 files changed, 855 insertions(+), 226 deletions(-) create mode 100644 issues/the-amd-benchs-tsc-steps-1-7-ms-forward-once-a-boot.md create mode 100644 kernel/src/arch/x86_64/idt/rtc.rs diff --git a/issues/the-amd-benchs-tsc-steps-1-7-ms-forward-once-a-boot.md b/issues/the-amd-benchs-tsc-steps-1-7-ms-forward-once-a-boot.md new file mode 100644 index 00000000000..2eb4478d119 --- /dev/null +++ b/issues/the-amd-benchs-tsc-steps-1-7-ms-forward-once-a-boot.md @@ -0,0 +1,49 @@ +--- +status: open +kind: defect +opened: 2026-10-31 +--- + +# The AMD bench's TSC steps 1.7 ms forward once a boot + +On the AMD bench (Lenovo 82A2, BIOS DMCN38WW; Ryzen 7 4800U, CPUID +signature 0x860f01, Family 17h Model 60h), the TSC jumps 1.69–1.73 ms ahead +once in every boot. The HPET, the ACPI PM timer and the RTC agree on it +within 37 µs. The RTC runs off its own 32.768 kHz crystal, and the other two +off the FCH's 14.318 MHz clock. Linux on the same machine sees the event in +5 of 30 boots. + +**Owner:** the clock (`kernel/src/arch/x86_64/clocksource.rs`) and +acpiserver, which hands over the idle state the step lands in. + +**Where it lands.** Inside one spell in which every CPU sits in port 0x415 +(s1, the deepest state acpiserver hands over), early in the boot. One CPU +reads the HPET as each spell ends: +- in `benchcase`, spells 0 to 2 read no step (the third lasted 1.66 ms), and + the TSC is 1.70 ms ahead from the end of spell 3; +- in `testcases`, the boot's first such spell already ends 1.72 ms ahead. + +The stepping spell's TSC length includes the jump. No later spell steps. +`benchcase` reaches its first spells at about 17 s, when `osbench_sched`'s +all-core load ends; `testcases` reaches them at about 5 s, right after the +hand-over. The trigger is the package's deep idle with every core in 0x415. +Whether the TSC's jump across it is the platform's or the state's entry is +not known. + +**What the clock does meanwhile.** A forward step that the HPET and the PM +timer agree on is slewed away at no more than 500 ppm. This happens whether +the RTC confirms the step or cannot tell. The 1.7 ms takes 3.4 s, and +monotonic time never steps. A second step, or a backward one, demotes the TSC: +processes then read now through the kernel, which counts by the HPET. + +Two things stay on the TSC after a demotion: the kernel's own deadlines and +its log stamps. A later step therefore moves them apart from what processes +read. + +**Not read.** AMD's *Processor Programming Reference for Family 17h Model 60h* +(55922), which would cover the TSC in deep C-states and the TSC's relation to +the FCH's clocks. amd.com refuses the build's user agent, so it has not been +fetched. + +**Exit:** the cause found and removed, or documented by AMD, so that no AMD +boot takes a step. diff --git a/kernel/pure/clock/reference.rs b/kernel/pure/clock/reference.rs index e7ec9035b1a..e536269ce89 100644 --- a/kernel/pure/clock/reference.rs +++ b/kernel/pure/clock/reference.rs @@ -9,6 +9,8 @@ //! statistic, so a bound that fails is a counter, a reference or a read that //! did not do what it says. +use toyos_abi::clock::MULT_SHIFT; + /// Parts per billion in one. pub const PPB: u64 = 1_000_000_000; @@ -30,7 +32,12 @@ impl Reference { /// The span from `a` to `b` the reference counted, zero where it read /// backwards. pub const fn span_ns(&self, a: Sample, b: Sample) -> u64 { - (b.reference.saturating_sub(a.reference) as u128 * self.period_fs as u128 / FS_PER_NS) as u64 + self.ns_of(b.reference.saturating_sub(a.reference)) + } + + /// `ticks` of the reference, in nanoseconds. + pub const fn ns_of(&self, ticks: u64) -> u64 { + (ticks as u128 * self.period_fs as u128 / FS_PER_NS) as u64 } /// What the reference's own rate may be off by over `span_ns`. @@ -264,8 +271,6 @@ pub struct Watch { last: Sample, kept: Kept, next_report_ns: u64, - /// Spans left out. - unheld: u64, } /// What a sample showed, where it held. @@ -278,8 +283,6 @@ pub struct Report { pub kept: Kept, /// Whether `kept` replaced the rate the clock was keeping. pub retuned: bool, - /// The spans left out so far. - pub unheld: u64, } /// What a sample showed, where it did not hold: the counter and the reference @@ -295,7 +298,7 @@ pub enum Failure { impl Watch { pub const fn new(reference: Reference, anchor: Sample, kept: Kept) -> Self { - Self { reference, closed: Span::EMPTY, opened: anchor, last: anchor, kept, next_report_ns: FIRST_REPORT_NS, unheld: 0 } + Self { reference, closed: Span::EMPTY, opened: anchor, last: anchor, kept, next_report_ns: FIRST_REPORT_NS } } pub const fn kept(&self) -> Kept { @@ -319,7 +322,6 @@ impl Watch { } self.opened = sample; self.last = sample; - self.unheld += 1; return Ok(None); } let tolerance = |span: &Span| self.kept.bound_ppb + self.reference.tolerance_over(span.shortest_ns); @@ -347,7 +349,7 @@ impl Watch { if retuned { self.kept = Kept { mult: measured.mult, bound_ppb: measured.bound_ppb, measured: true }; } - Ok(Some(Report { measured, kept: self.kept, retuned, unheld: self.unheld })) + Ok(Some(Report { measured, kept: self.kept, retuned })) } } @@ -419,6 +421,153 @@ pub const fn arbitrate(first: Agreement, second: Agreement) -> Arbitration { } } +/// The RTC's edges the sampler has read, the latest [`EDGES_KEPT`]: each +/// edge's count and the counter as its interrupt was taken, which is the edge +/// plus the interrupt's latency, never earlier. +#[derive(Clone, Copy, Debug)] +pub struct Edges { + kept: [(u64, u64); EDGES_KEPT], + len: usize, +} + +pub const EDGES_KEPT: usize = 16; +/// The edges either side of a span its least-latency edge is taken from. +pub const EDGES_EACH_SIDE: usize = 3; + +impl Edges { + pub const fn new() -> Self { + Self { kept: [(0, 0); EDGES_KEPT], len: 0 } + } + + /// Keep the edge `seen` as the counter read `at`, where it is later than + /// the latest kept. + pub fn push(&mut self, seen: u64, at: u64) { + if self.len > 0 && self.kept[self.len - 1].0 >= seen { + return; + } + if self.len == EDGES_KEPT { + self.kept.copy_within(1.., 0); + self.len -= 1; + } + self.kept[self.len] = (seen, at); + self.len += 1; + } + + /// Whether [`EDGES_EACH_SIDE`] edges have come after the counter read `to`. + pub fn complete_after(&self, to: u64) -> bool { + self.kept[..self.len].iter().filter(|&&(_, at)| at >= to).count() >= EDGES_EACH_SIDE + } + + /// The counter's span, at `mult`, less the RTC's seconds, from before + /// `from` to after `to`: from the least-latency of the [`EDGES_EACH_SIDE`] + /// edges before `from` to that of those after `to`. An edge reads late by + /// its interrupt's latency and never early, so the least on each side is + /// the nearest its edge. `None` without an edge either side. + pub fn apart_across(&self, from: u64, to: u64, mult: u64) -> Option { + let kept = &self.kept[..self.len]; + let before = kept.iter().rev().filter(|&&(_, at)| at <= from).take(EDGES_EACH_SIDE); + let after = kept.iter().filter(|&&(_, at)| at >= to).take(EDGES_EACH_SIDE); + let &(seen_ref, at_ref) = kept.iter().rev().find(|&&(_, at)| at <= from)?; + let offset = |&(seen, at): &(u64, u64)| { + let ns = ((at as i128 - at_ref as i128) * mult as i128) >> MULT_SHIFT; + ns - (seen as i128 - seen_ref as i128) * 1_000_000_000 + }; + let least_before = before.map(offset).min()?; + let least_after = after.map(offset).min()?; + Some((least_after - least_before) as i64) + } +} + +impl Default for Edges { + fn default() -> Self { + Self::new() + } +} + +/// Which of the counter's step and no step at all a third reference, on +/// another oscillator, reads nearest: within a quarter of the step of +/// either, or of neither. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Third { + /// It reads the step: the counter moved. + Counter, + /// It reads no step: the two references moved together. + References, + /// It reads neither, or has no edge either side of the span yet. + Unsure, +} + +pub fn third_says(step_ns: i64, apart_ns: Option) -> Third { + let Some(apart) = apart_ns else { return Third::Unsure }; + let zone = step_ns.unsigned_abs() / 4; + if apart.abs_diff(step_ns) <= zone { + Third::Counter + } else if apart.unsigned_abs() <= zone { + Third::References + } else { + Third::Unsure + } +} + +/// What the clock does about a step of its counter: the boot's first, forward, +/// is slewed away; a second, or one back, leaves the counter for the +/// reference. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Response { + Slew, + Demote, +} + +pub const fn respond(step_ns: i64, slewed_before: u32) -> Response { + if step_ns > 0 && slewed_before == 0 { Response::Slew } else { Response::Demote } +} + +/// The most a slew moves the clock's rate: 500 ppm, the frequency tolerance +/// NTP disciplines a clock within (RFC 5905, Appendix A.1.1, `MAXFREQ`), so +/// a reader that tolerates an NTP-disciplined clock tolerates this one. A +/// step of `d` is slewed away in `2000 d`: 1.7 ms in 3.4 s. +pub const SLEW_MOST_PPB: i64 = 500_000; +/// A debt this small is paid. +const SLEW_SETTLED_NS: u64 = 1_000; + +/// How far the clock stands ahead of the reference's time, which a slew pays +/// off by running the clock slower than the kept rate. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Slew { + debt_ns: i64, +} + +impl Slew { + pub const fn new(debt_ns: i64) -> Self { + Self { debt_ns } + } + + pub const fn debt_ns(&self) -> i64 { + self.debt_ns + } + + pub const fn settled(&self) -> bool { + self.debt_ns.unsigned_abs() <= SLEW_SETTLED_NS + } + + /// `ticks` the clock ran at `ran` where the reference's time runs at + /// `kept`: what it fell behind is paid off the debt. + pub fn ran(&mut self, ticks: u64, kept: u64, ran: u64) { + let line = |mult| ((ticks as u128 * mult as u128) >> MULT_SHIFT) as i128; + self.debt_ns -= (line(kept) - line(ran)) as i64; + } + + /// The rate to run at for the next `period_ns`: the debt paid over that + /// period, at most [`SLEW_MOST_PPB`] from `kept`, and `kept` once settled. + pub fn mult(&self, kept: u64, period_ns: u64) -> u64 { + if self.settled() { + return kept; + } + let ppb = (self.debt_ns as i128 * 1_000_000_000 / period_ns as i128).clamp(-(SLEW_MOST_PPB as i128), SLEW_MOST_PPB as i128); + (kept as i128 - kept as i128 * ppb / 1_000_000_000) as u64 + } +} + #[cfg(test)] mod tests { use super::*; @@ -605,11 +754,11 @@ mod tests { assert_eq!(watch.take(late, true).map(|_| ()), Ok(())); } - /// A reference that stood still for 30 s while the counter ran, as an HPET - /// may in a package state that stops it: the span is left out, so it is no - /// step, and the rate over the held spans either side is the counter's. + /// A reference that stood still for 30 s while the counter ran: held, the + /// span reads as a step; left out, it is none, and the rate over the held + /// spans either side is the counter's. #[test] - fn an_unheld_span_is_left_out() { + fn a_span_left_out_checks_nothing() { let mut watch = watch(true); run(&mut watch, HZ, 1..20, 0).unwrap(); // The reference resumes where it stopped; the counter is 30 s on. @@ -620,7 +769,6 @@ mod tests { assert_eq!(watch.take(resumed, false), Ok(None)); let reports = run(&mut watch, HZ, 21..300, stood).unwrap(); let last = reports.last().expect("held spans past the gap report"); - assert_eq!(last.unheld, 1); assert!(ppb_apart(last.measured.mult, HZ) <= last.measured.bound_ppb as u128, "{last:?}"); assert!(last.measured.span_ns / 1_000_000_000 >= 256); } @@ -700,4 +848,79 @@ mod tests { assert_eq!(extend(0x03ff_fff0, 0x0010, 24), 0x0400_0010); assert_eq!(extend(0x0400_0010, 0x00ff_fff0, 24), 0x03ff_fff0); } + + /// A 1 GHz counter, so ticks are nanoseconds. + const NS: u64 = 1 << MULT_SHIFT; + + /// Edges a second apart on the RTC, each read as late as its latency; + /// the counter steps by `step` at `step_at`. + fn edges(latency_ns: &[u64], step_at: u64, step: u64) -> Edges { + let mut edges = Edges::new(); + for (n, latency) in latency_ns.iter().enumerate() { + let edge = 10_000_000_000 + n as u64 * 1_000_000_000; + edges.push(n as u64 + 1, edge + latency + if edge >= step_at { step } else { 0 }); + } + edges + } + + /// The least-latency edge either side reads the step, however late the + /// edge nearest it was taken; and reads none where there was none. + #[test] + fn the_rtc_reads_a_step_through_its_interrupts_latency() { + let latency = [40_000, 700_000, 3_000, 650_000, 520_000, 9_000, 700_000, 2_000]; + let step = 1_700_000; + let stepped = edges(&latency, 14_500_000_000, step); + let (from, to) = (13_900_000_000, 14_900_000_000); + assert!(stepped.complete_after(to + step)); + let apart = stepped.apart_across(from, to + step, NS).expect("edges either side"); + assert!(apart.abs_diff(step as i64) <= 10_000, "read {apart} ns for a {step} ns step"); + assert_eq!(third_says(step as i64, Some(apart)), Third::Counter); + + let still = edges(&latency, u64::MAX, 0); + let apart = still.apart_across(from, to, NS).expect("edges either side"); + assert!(apart.unsigned_abs() <= 10_000, "read {apart} ns where nothing stepped"); + assert_eq!(third_says(step as i64, Some(apart)), Third::References); + } + + #[test] + fn the_rtc_is_unsure_without_an_edge_either_side_or_between_the_two() { + let few = edges(&[1_000, 1_000], u64::MAX, 0); + assert_eq!(few.apart_across(9_000_000_000, 10_500_000_000, NS), None); + assert!(!few.complete_after(10_500_000_000)); + assert_eq!(third_says(1_700_000, None), Third::Unsure); + assert_eq!(third_says(1_700_000, Some(850_000)), Third::Unsure); + assert_eq!(third_says(-1_700_000, Some(-1_600_000)), Third::Counter); + } + + #[test] + fn only_a_first_forward_step_is_slewed() { + assert_eq!(respond(1_700_000, 0), Response::Slew); + assert_eq!(respond(1_700_000, 1), Response::Demote); + assert_eq!(respond(-1_700_000, 0), Response::Demote); + } + + /// Samples land 0.8 to 1.4 s apart: the slew never moves the rate past + /// 500 ppm, and the debt is paid within the time that rate takes and two + /// periods more. + #[test] + fn a_slew_pays_its_debt_within_its_bound() { + let kept = toyos_abi::clock::Line::mult_of_hz(1_796_624_780); + let ticks_per_ns = 1.796_624_78; + let debt = 1_700_000i64; + let mut slew = Slew::new(debt); + let gaps = [1_000_000_000u64, 1_400_000_000, 800_000_000, 1_200_000_000, 1_000_000_000, 1_300_000_000]; + let mut elapsed = 0u64; + for gap in gaps.iter().cycle().take(32) { + if slew.settled() { + break; + } + let ran = slew.mult(kept, 1_000_000_000); + let ppb = (kept as i128 - ran as i128) * 1_000_000_000 / kept as i128; + assert!(ppb.unsigned_abs() <= SLEW_MOST_PPB as u128, "{ppb} ppb"); + slew.ran((*gap as f64 * ticks_per_ns) as u64, kept, ran); + elapsed += gap; + } + assert!(slew.settled(), "{} ns left", slew.debt_ns()); + assert!(elapsed <= 2_000 * debt as u64 + 2 * 1_400_000_000, "took {elapsed} ns"); + } } diff --git a/kernel/src/actuator.rs b/kernel/src/actuator.rs index d28157ef16c..6d1ad9a691f 100644 --- a/kernel/src/actuator.rs +++ b/kernel/src/actuator.rs @@ -124,10 +124,10 @@ actuators! { /// `kernel_wx_leaf_is_refused` and `virt_kernel_wx_leaf_is_refused`. kernel_wx_leaf = "kernel-wx-leaf"; - /// Read the TSC 10 ms ahead in the HPET's samples once the clock has - /// held for a second, as a step `IA32_TSC_ADJUST` would not show, so the - /// HPET alone can catch it. The negative control on the clock's runtime check, - /// `a_stepped_tsc_panics`. + /// Read the TSC 10 ms ahead in the clock's samples once it has held for + /// a second, and 10 ms further once that step is slewed away, as steps + /// `IA32_TSC_ADJUST` would not show: the first is slewed, the second + /// demotes the TSC. Judged by `a_stepped_tsc_is_slewed_then_demoted`. tsc_step = "tsc-step"; /// Read the HPET 10 ms behind in its samples once the clock has held for diff --git a/kernel/src/arch/aarch64/clocksource.rs b/kernel/src/arch/aarch64/clocksource.rs index e5f955788ae..0681f563e31 100644 --- a/kernel/src/arch/aarch64/clocksource.rs +++ b/kernel/src/arch/aarch64/clocksource.rs @@ -4,3 +4,9 @@ /// No reference to sample against. pub fn check(_now: u64) {} + +/// Now as a log line's time, for a process: the generic timer's clock, which +/// nothing here demotes. +pub fn stamp_now() -> u64 { + crate::clock::stamp_at(super::cpu::counter()) +} diff --git a/kernel/src/arch/x86_64/boot.rs b/kernel/src/arch/x86_64/boot.rs index d9e7f17476a..6de6ca6c2ee 100644 --- a/kernel/src/arch/x86_64/boot.rs +++ b/kernel/src/arch/x86_64/boot.rs @@ -141,6 +141,9 @@ pub fn timer() { pub fn platform_devices(platform: &Platform, rsdp_addr: u64) { super::pio::fill_i8042_row(); super::acpi_mode::init(rsdp_addr, platform.facs); + if let Err(why) = super::rtc::arm_update_edges() { + log!("clock: no RTC edges time the TSC: {why}"); + } } /// Every other CPU, running. diff --git a/kernel/src/arch/x86_64/clocksource.rs b/kernel/src/arch/x86_64/clocksource.rs index a88124c6945..8d3207f8b59 100644 --- a/kernel/src/arch/x86_64/clocksource.rs +++ b/kernel/src/arch/x86_64/clocksource.rs @@ -4,19 +4,25 @@ //! //! The clock counts the TSC only where every CPU states it invariant, or under //! a hypervisor, whose TSC is its own to synthesise and whose CPUID need not -//! say so; there the HPET is all that holds it. Every AP's TSC is bounded -//! against the BSP's as it starts ([`measure_offset`]). While it runs, one CPU -//! a second samples the HPET for the whole machine's rate and steps, over the -//! spans in which some CPU stayed awake; and every CPU a second checks its +//! say so. Every AP's TSC is bounded against the BSP's as it starts +//! ([`measure_offset`]). While it runs, one CPU a second samples the HPET, the +//! ACPI PM timer and the RTC's latest edge, and every CPU a second checks its //! `IA32_TSC_ADJUST` and that its TSC is not behind another's ([`check`]). -//! Each failure panics: no other counter here can take the clock's place, -//! because every process reads the TSC itself. +//! +//! **Where the TSC and the HPET disagree, the other two say which moved.** The +//! PM timer agreeing with the TSC is the HPET moving, and the span is left out. +//! The PM timer agreeing with the HPET leaves the chipset's 14.318 MHz clock +//! or the TSC; the RTC's edges, off its own 32.768 kHz crystal, choose: with +//! the TSC, the span is left out; with the chipset's counters, or saying +//! neither, the TSC stepped. The boot's first forward step is slewed away; +//! a second, one back, or a check that finds a CPU's TSC written or behind +//! demotes the TSC ([`demote`]), and every process's reading of now asks the +//! kernel, which counts by the HPET. A rate that drifts past both bounds +//! panics. use core::sync::atomic::{AtomicBool, AtomicU64, Ordering}; -use alloc::vec::Vec; - -use kernel::clock::reference::{self, Agreement, Arbitration, Calibration, Failure, Kept, Offset, Reference, Sample, Watch}; +use kernel::clock::reference::{self, Agreement, Arbitration, Calibration, Edges, Failure, Kept, Offset, Reference, Response, Sample, Slew, Third, Watch}; use toyos_abi::clock::Line; use super::cpu; @@ -65,6 +71,7 @@ const CHECK_EVERY_NS: u64 = 1_000_000_000; /// Round trips of a cache line between the BSP and each AP as it starts. const ROUNDS: u32 = 256; +#[derive(Clone, Copy)] struct Hpet { mmio: Mmio, wide: bool, @@ -112,8 +119,8 @@ impl Hpet { /// The ACPI PM timer, the second reference: which of the TSC and the HPET /// moved, where the two disagree. **It and the HPET are both the chipset's -/// (the FCH's on AMD, the PCH's on Intel)**, and may share its clock, so a -/// stall common to both reads as the TSC having moved. +/// (the FCH's on AMD, the PCH's on Intel)**, and may share its clock, so the +/// RTC's edges judge a stall common to both. struct PmTimer { port: Declared, bits: u32, @@ -170,36 +177,64 @@ struct Held { hpet: Hpet, watch: Watch, pm: Option, - /// Spans left out, and of them those across which the HPET, and the PM - /// timer, disagreed with the TSC: what the left-out spans would have read. - unheld_hpet_apart: u64, - unheld_pm_apart: u64, - /// Each CPU's [`crate::idle::Local::deep_periods`] at the last sample. - idle: Vec<(u64, u64)>, -} - -impl Held { - /// Whether some CPU stayed out of every declared idle state from the last - /// sample to now: no package state deeper than its own can have held in - /// between, so the HPET counted throughout. Where every CPU spent part of - /// the span in one, the package may have reached a state that stops the - /// HPET, and the span is not held. - fn some_cpu_stayed_awake(&mut self) -> bool { - let cpus = crate::smp::cpu_count() as usize; - self.idle.resize(cpus, (0, 0)); - let mut awake = false; - for (cpu, was) in self.idle.iter_mut().enumerate() { - let local = crate::cpu_local::get(cpu as u32).expect("a CPU the roster counts has its record"); - let now = local.idle.deep_periods(); - awake |= now == *was && now.0 == now.1; - *was = now; - } - awake - } + /// The RTC's edges, the third reference, on its own crystal. + edges: Edges, + /// A step the HPET and the PM timer read the TSC take, waiting for the + /// RTC's edges after it. + pending: Option, + /// The steps this boot slewed away, and the slew under way. + slewed: u32, + slew: Option, + /// Spans left out: one reference moved, or the RTC said both did. + left_out: u64, } +/// A step the chipset's counters agree the TSC took over `from..to`. +#[derive(Clone, Copy)] +struct Pending { + step_ns: i64, + from: Sample, + to: Sample, + /// When the sampler gives up waiting for the RTC's edges after it. + until_ns: u64, +} + +/// A slew, and the line it runs on since the counter read `since`: at `ran` +/// where the reference's time runs at `kept`. +#[derive(Clone, Copy)] +struct Slewing { + slew: Slew, + since: u64, + ran: u64, + kept: u64, +} + +/// How long a step waits for the RTC's edges after it before it is taken +/// unconfirmed: [`reference::EDGES_EACH_SIDE`] edges a second apart, and two +/// more for their interrupts. +const RTC_WAIT_NS: u64 = (reference::EDGES_EACH_SIDE as u64 + 2) * 1_000_000_000; + static HELD: Lock> = Lock::new(None); +/// The last sample every check held, and the clock then: where a demoted +/// clock counts on from by the HPET. Published by the sampler. +static LAST_HELD: crate::seqlock::Published<3> = crate::seqlock::Published::new(); + +/// The HPET and the clock where the TSC was demoted, and the latest reading +/// the kernel has answered since, which none after reads before. +struct Demoted { + hpet: Hpet, + at_hpet: u64, + at_stamp: u64, + tsc: u64, + mult: u64, +} + +static DEMOTED: Lock> = Lock::new(None); +static IS_DEMOTED: AtomicBool = AtomicBool::new(false); +static DEMOTED_LATEST: AtomicU64 = AtomicU64::new(0); +static DEMOTED_ANSWERED: AtomicBool = AtomicBool::new(false); + /// Whether CPUID states `IA32_TSC_ADJUST`, read once on the BSP. static TSC_ADJUST: AtomicBool = AtomicBool::new(false); @@ -260,15 +295,19 @@ pub fn start(hpet_base: u64, pm_timer: Result log!("clock: the ACPI PM timer at {:#x}, {} bits, tells the TSC from the HPET where they disagree", pm.port.port(0).number(), pm.bits), - Err(why) => log!("clock: no ACPI PM timer to tell the TSC from the HPET ({why}); a step between them panics"), + Err(why) => log!("clock: no ACPI PM timer to tell the TSC from the HPET ({why}); the RTC alone tells them apart"), } + LAST_HELD.publish([anchor.before, anchor.reference, crate::clock::stamp_at(anchor.before)]); + *HPET_COPY.lock() = Some(hpet); *HELD.lock() = Some(Held { watch: Watch::new(hpet.reference, anchor, kept), hpet, pm: pm.ok(), - unheld_hpet_apart: 0, - unheld_pm_apart: 0, - idle: Vec::new(), + edges: Edges::new(), + pending: None, + slewed: 0, + slew: None, + left_out: 0, }); } @@ -436,7 +475,7 @@ pub fn answer_offset() { /// the difference (`reference::Offset`'s argument, one way). pub fn check(now_ns: u64) { let local = &crate::cpu_local::here().arch.clock; - if now_ns < local.due_ns.load(Ordering::Relaxed) { + if now_ns < local.due_ns.load(Ordering::Relaxed) || IS_DEMOTED.load(Ordering::Relaxed) { return; } local.due_ns.store(now_ns + CHECK_EVERY_NS, Ordering::Relaxed); @@ -444,20 +483,22 @@ pub fn check(now_ns: u64) { if TSC_ADJUST.load(Ordering::Relaxed) { let started = local.adjust.load(Ordering::Relaxed); let adjust = cpu::rdmsr(cpu::IA32_TSC_ADJUST); - assert!( - adjust == started, - "clock: cpu{cpu_id}'s TSC was written: IA32_TSC_ADJUST reads {} and read {} when it started", - adjust as i64, - started as i64, - ); + if adjust != started { + return demote(format_args!( + "cpu{cpu_id}'s TSC was written: IA32_TSC_ADJUST reads {} and read {} when it started", + adjust as i64, + started as i64, + )); + } } let latest = LATEST.0.load(Ordering::Acquire); let mine = cpu::rdtsc_ordered(); - assert!( - mine >= latest, - "clock: cpu{cpu_id}'s TSC read {mine}, {} cycles behind the {latest} another CPU's check read before it", - latest - mine, - ); + if mine < latest { + return demote(format_args!( + "cpu{cpu_id}'s TSC read {mine}, {} cycles behind the {latest} another CPU's check read before it", + latest - mine, + )); + } LATEST.0.fetch_max(mine, Ordering::AcqRel); let due = NEXT_SAMPLE_NS.load(Ordering::Relaxed); @@ -468,103 +509,230 @@ pub fn check(now_ns: u64) { return; }; if let Some(held) = held.as_mut() { - sample(held, cpu_id); + sample(held, now_ns); } } -/// One sample of the HPET, and of the PM timer beside it, held to the TSC -/// where the span since the last can be. -fn sample(held: &mut Held, cpu_id: u32) { - let awake = held.some_cpu_stayed_awake(); +/// Leave the TSC for the HPET for the rest of the boot: every process's +/// reading of now asks the kernel ([`stamp_now`]), which counts on by the +/// HPET from the later of the clock now and the last held sample's clock and +/// the HPET's count since, so no reading after reads before one already +/// answered. The kernel's own deadlines stay on the TSC. +fn demote(why: core::fmt::Arguments) { + let mut demoted = DEMOTED.lock(); + if demoted.is_some() { + return; + } + // A publication in progress ends within its few stores. + let [held_tsc, held_hpet, held_stamp] = loop { + if let Some(words) = LAST_HELD.snapshot() { + break words; + } + core::hint::spin_loop(); + }; + let hpet = HPET_COPY.lock().expect("the clock started before any check"); + let mult = crate::clock::mult(); + let tsc = cpu::counter(); + let near = held_hpet + (u128::from(Line { counter: 0, stamp: 0, mult }.nanos(tsc.saturating_sub(held_tsc))) * 1_000_000 / u128::from(hpet.reference.period_fs)) as u64; + let at_hpet = hpet.read(near); + let by_hpet = held_stamp + hpet.reference.ns_of(at_hpet.saturating_sub(held_hpet)); + let at_stamp = by_hpet.max(crate::clock::stamp_at(tsc)); + DEMOTED_LATEST.store(at_stamp, Ordering::Relaxed); + *demoted = Some(Demoted { hpet, at_hpet, at_stamp, tsc, mult }); + IS_DEMOTED.store(true, Ordering::Release); + crate::clock::demote(); + log!( + "clock: DEMOTED: {why}, so the TSC no longer keeps the clock: every process's reading of now asks the kernel, \ + which counts by the HPET from {at_stamp} ns, {} ns from the TSC's clock; the kernel's deadlines stay on the TSC", + at_stamp as i64 - crate::clock::stamp_at(tsc) as i64, + ); +} + +/// The HPET, for [`demote`] on whichever CPU's check finds the TSC moved. +static HPET_COPY: Lock> = Lock::new(None); + +/// Now as a log line's time, for a process: the TSC's clock, or the HPET's +/// count on from where the TSC was demoted. +pub fn stamp_now() -> u64 { + if !IS_DEMOTED.load(Ordering::Acquire) { + return crate::clock::stamp_at(cpu::counter()); + } + let stamp = { + let demoted = DEMOTED.lock(); + let d = demoted.as_ref().expect("demoted before the flag"); + let elapsed = Line { counter: 0, stamp: 0, mult: d.mult }.nanos(cpu::counter().saturating_sub(d.tsc)); + let near = d.at_hpet + (u128::from(elapsed) * 1_000_000 / u128::from(d.hpet.reference.period_fs)) as u64; + d.at_stamp + d.hpet.reference.ns_of(d.hpet.read(near).saturating_sub(d.at_hpet)) + }; + let answered = DEMOTED_LATEST.fetch_max(stamp, Ordering::AcqRel).max(stamp); + if !DEMOTED_ANSWERED.swap(true, Ordering::Relaxed) { + log!("clock: a process's reading of now asked the kernel since the TSC was demoted, and read {answered} ns"); + } + answered +} + +/// One sample of the HPET, of the PM timer beside it and of the RTC's +/// latest edge: each span the TSC and the HPET agree over is held; where +/// they do not, the PM timer and then the RTC say which moved. +fn sample(held: &mut Held, now_ns: u64) { let kept = held.watch.kept(); let last = held.watch.last(); let hpet = held.hpet.sample(held.hpet.near(last, kept.mult), 1); let pm = held.pm.as_ref().map(|pm| (pm.last, pm.next(kept.mult))); + let edge = super::rtc::last_edge(); #[cfg(feature = "boot-actuators")] - let (hpet, pm) = staged(hpet, pm, false, held.hpet.reference.period_fs); - let against = |reference: Reference, from: Sample, to: Sample| { - let span = reference::Span::between(reference, from, to); - let tolerance = kept.bound_ppb + reference.tolerance_over(reference.span_ns(from, to)); - reference::agree_over(reference, span, kept.mult, tolerance) - }; - let pm_reference = held.pm.as_ref().map(|pm| pm.reference); - let pm_agreement = pm.zip(pm_reference).map(|((from, to), reference)| against(reference, from, to)); + let (hpet, pm, edge) = staged(hpet, pm, edge); + if let Some((seen, at)) = edge { + held.edges.push(seen, at); + } + let pm_agreement = held.pm.as_ref().zip(pm).map(|(timer, (from, to))| { + let span = reference::Span::between(timer.reference, from, to); + let tolerance = kept.bound_ppb + timer.reference.tolerance_over(timer.reference.span_ns(from, to)); + reference::agree_over(timer.reference, span, kept.mult, tolerance) + }); if let (Some(timer), Some((_, to))) = (held.pm.as_mut(), pm) { timer.last = to; } - if !awake { - held.unheld_hpet_apart += u64::from(!against(held.hpet.reference, last, hpet).holds()); - held.unheld_pm_apart += u64::from(pm_agreement.is_some_and(|a| !a.holds())); + if IS_DEMOTED.load(Ordering::Acquire) { let _ = held.watch.take(hpet, false); return; } match held.watch.take(hpet, true) { - Ok(None) => {} - Ok(Some(report)) => { - #[cfg(feature = "boot-actuators")] - staged(hpet, pm, true, held.hpet.reference.period_fs); - let kept = Line { counter: 0, stamp: 0, mult: report.kept.mult }; - if report.retuned { - crate::clock::retune(report.kept.mult); - } - let measured = Line { counter: 0, stamp: 0, mult: report.measured.mult }.millihertz(); - let (span_s, bound) = (report.measured.span_ns / 1_000_000_000, report.measured.bound_ppb); - let unheld = alloc::format!( - "{} spans unheld, across which the HPET read apart from the TSC in {} and the PM timer in {}", - report.unheld, held.unheld_hpet_apart, held.unheld_pm_apart, - ); - match (report.kept.measured, report.retuned) { - (false, _) => log!( - "clock: over {span_s} s the TSC counted {measured} mHz against the HPET, within {bound} ppb; \ - the clock keeps CPUID 15H's {} mHz; {unheld}", - kept.millihertz(), - ), - (true, retuned) => log!( - "clock: over {span_s} s the TSC counted {measured} mHz against the HPET, within {bound} ppb; \ - the clock {} {} mHz, within {} ppb; {unheld}", - if retuned { "now keeps" } else { "keeps" }, - kept.millihertz(), - report.kept.bound_ppb, - ), + Ok(report) => { + LAST_HELD.publish([hpet.before, hpet.reference, crate::clock::stamp_at(hpet.before)]); + if let Some(report) = report { + log_report(held, report); + #[cfg(feature = "boot-actuators")] + stage_first(held.hpet.reference.period_fs); } } Err(Failure::Step { from, to, agreement }) => { let spans = spans(agreement, pm_agreement, from, to, pm); + let _ = held.watch.take(hpet, false); + held.left_out += 1; match pm_agreement.map(|second| reference::arbitrate(agreement, second)) { - Some(Arbitration::FirstMoved) => { - log!("clock: the HPET moved and the TSC did not, so this span is left out: {spans}"); - let _ = held.watch.take(hpet, false); - } + Some(Arbitration::FirstMoved) => log!("clock: the HPET moved and the TSC did not, so this span is left out: {spans}"), Some(Arbitration::AllDisagree) => { - log!("clock: the TSC, the HPET and the ACPI PM timer all disagree, so this span is left out: {spans}"); - let _ = held.watch.take(hpet, false); + log!("clock: the TSC, the HPET and the ACPI PM timer all disagree, so this span is left out: {spans}") + } + Some(Arbitration::CounterMoved) | None => { + if let Some(earlier) = held.pending.take() { + settle(held, earlier); + } + log!( + "clock: the TSC reads {} ns from the chipset's counters over the last span; the RTC's edges \ + after it will say which moved: {spans}", + agreement.apart_ns, + ); + held.pending = Some(Pending { step_ns: agreement.apart_ns, from, to, until_ns: now_ns + RTC_WAIT_NS }); } - Some(Arbitration::CounterMoved) => panic!( - "clock: the TSC and the HPET disagree by {} ns over the {} ns the HPET counted since the last \ - sample, {} allowed, and the ACPI PM timer agrees with the HPET: two of the chipset's counters \ - agree against the TSC, which moved, or both stalled together (cpu{cpu_id}; {spans})", - agreement.apart_ns, - agreement.reference_ns, - agreement.allowed_ns, - ), - None => panic!( - "clock: the TSC and the HPET disagree by {} ns over the {} ns the HPET counted since the last \ - sample, {} allowed, and no PM timer says which moved (cpu{cpu_id}; {spans})", - agreement.apart_ns, - agreement.reference_ns, - agreement.allowed_ns, - ), } } Err(Failure::Drift { since, to, agreement }) => panic!( "clock: the TSC and the HPET disagree by {} ns over the {} ns the HPET counted in held spans, {} allowed: \ - their rates are further apart than the kept rate's bound and the HPET's tolerance (cpu{cpu_id}, the \ - open span from {since:?} to {to:?})", + their rates are further apart than the kept rate's bound and the HPET's tolerance (the open span from \ + {since:?} to {to:?})", agreement.apart_ns, agreement.reference_ns, agreement.allowed_ns, ), } + if let Some(pending) = held.pending.filter(|p| held.edges.complete_after(p.to.after) || now_ns >= p.until_ns) { + held.pending = None; + settle(held, pending); + } + if !IS_DEMOTED.load(Ordering::Acquire) { + slew(held); + } +} + +/// What the RTC says of a pending step, and what the clock does about it. +fn settle(held: &mut Held, pending: Pending) { + let kept = held.watch.kept(); + let apart = held.edges.apart_across(pending.from.before, pending.to.after, kept.mult); + let rtc = match apart { + Some(apart) => alloc::format!("the RTC's edges read {apart} ns"), + None => alloc::string::String::from("no RTC edge either side"), + }; + let step = pending.step_ns; + let confirmed = match reference::third_says(step, apart) { + Third::References => { + return log!( + "clock: the chipset's counters moved {step} ns from the TSC and {rtc}, with the TSC, so the span \ + was left out" + ); + } + Third::Counter => "confirmed by the RTC", + Third::Unsure => "unconfirmed by the RTC", + }; + match reference::respond(step, held.slewed) { + Response::Slew => { + held.slewed += 1; + held.slew = Some(Slewing { slew: Slew::new(step), since: cpu::counter(), ran: crate::clock::mult(), kept: kept.mult }); + log!( + "clock: TSC STEP: the TSC stepped {step} ns forward against the HPET and the PM timer, {confirmed} \ + ({rtc}); the clock slews it away at most {} ppm, in about {} ms", + reference::SLEW_MOST_PPB / 1_000, + u128::from(step.unsigned_abs()) * u128::from(reference::PPB) / reference::SLEW_MOST_PPB as u128 / 1_000_000, + ); + } + Response::Demote => { + demote(format_args!( + "the TSC stepped {step} ns against the HPET and the PM timer, {confirmed} ({rtc}), after {} step(s) slewed", + held.slewed, + )); + } + } +} + +/// The slew under way, paid to now and set for the next second; or, with +/// none, the kept rate where the watch retuned it. +fn slew(held: &mut Held) { + let kept = held.watch.kept().mult; + let Some(mut slewing) = held.slew else { + if crate::clock::mult() != kept { + crate::clock::retune(cpu::counter(), kept); + } + return; + }; + let now = cpu::counter(); + slewing.slew.ran(now - slewing.since, slewing.kept, slewing.ran); + let ran = slewing.slew.mult(kept, CHECK_EVERY_NS); + crate::clock::retune(now, ran); + if slewing.slew.settled() { + held.slew = None; + log!("clock: the TSC's step is slewed away, {} ns left", slewing.slew.debt_ns()); + #[cfg(feature = "boot-actuators")] + stage_again(); + } else { + held.slew = Some(Slewing { since: now, ran, kept, ..slewing }); + } +} + +fn log_report(held: &Held, report: reference::Report) { + let kept = Line { counter: 0, stamp: 0, mult: report.kept.mult }; + let measured = Line { counter: 0, stamp: 0, mult: report.measured.mult }.millihertz(); + let (span_s, bound) = (report.measured.span_ns / 1_000_000_000, report.measured.bound_ppb); + let left_out = alloc::format!( + "{} spans left out; {} TSC step(s) slewed{}", + held.left_out, + held.slewed, + held.slew.map_or_else(alloc::string::String::new, |s| alloc::format!(", {} ns still to slew", s.slew.debt_ns())), + ); + match (report.kept.measured, report.retuned) { + (false, _) => log!( + "clock: over {span_s} s the TSC counted {measured} mHz against the HPET, within {bound} ppb; \ + the clock keeps CPUID 15H's {} mHz; {left_out}", + kept.millihertz(), + ), + (true, retuned) => log!( + "clock: over {span_s} s the TSC counted {measured} mHz against the HPET, within {bound} ppb; \ + the clock {} {} mHz, within {} ppb; {left_out}", + if retuned { "now keeps" } else { "keeps" }, + kept.millihertz(), + report.kept.bound_ppb, + ), + } } /// All three counters' spans, for a disagreement's record. @@ -591,30 +759,71 @@ fn spans( ) } -/// `tsc-step` and `hpet-stall`: from the first report on, every sample reads -/// either the TSC 10 ms ahead of where it is, in both references' readings, -/// or the HPET 10 ms behind, as a TSC that stepped and an HPET that stood -/// still. A hypervisor's TSC may ignore a write (QEMU's TCG does), so each is -/// staged in the readings the sampler takes. +/// The PM timer's span, and the RTC's latest edge, as one sample reads them. #[cfg(feature = "boot-actuators")] -fn staged(hpet: Sample, pm: Option<(Sample, Sample)>, reported: bool, period_fs: u64) -> (Sample, Option<(Sample, Sample)>) { - static TSC_AHEAD: AtomicU64 = AtomicU64::new(0); - static HPET_BEHIND: AtomicU64 = AtomicU64::new(0); - if reported && TSC_AHEAD.load(Ordering::Relaxed) == 0 && HPET_BEHIND.load(Ordering::Relaxed) == 0 { - if crate::actuator::tsc_step() { - let ahead = crate::clock::counter_ticks(10_000_000); - TSC_AHEAD.store(ahead, Ordering::Relaxed); - log!("tsc-step: the TSC reads {ahead} cycles ahead from here"); - } - if crate::actuator::hpet_stall() { - let behind = 10_000_000_000_000 / period_fs; - HPET_BEHIND.store(behind, Ordering::Relaxed); - log!("hpet-stall: the HPET reads {behind} ticks behind from here"); - } +type PmSpan = Option<(Sample, Sample)>; +#[cfg(feature = "boot-actuators")] +type Edge = Option<(u64, u64)>; + +/// `tsc-step` and `hpet-stall`: from the first report on, every reading reads +/// either the TSC 10 ms ahead of where it is, the HPET's, the PM timer's and +/// the RTC's edges alike, or the HPET 10 ms behind, as a TSC that stepped and +/// an HPET that stood still; and the TSC 10 ms further once its first step is +/// slewed away. A hypervisor's TSC may ignore a write (QEMU's TCG does), so +/// each is staged in the readings the sampler takes, from the counter it +/// took effect at. +#[cfg(feature = "boot-actuators")] +mod stage { + use core::sync::atomic::{AtomicU64, Ordering}; + + pub static STEPS_AT: [AtomicU64; 2] = [AtomicU64::new(u64::MAX), AtomicU64::new(u64::MAX)]; + pub static STALLED_AT: AtomicU64 = AtomicU64::new(u64::MAX); + pub static STEP_TICKS: AtomicU64 = AtomicU64::new(0); + pub static STALL_TICKS: AtomicU64 = AtomicU64::new(0); + + /// What a counter read at `at` reads, staged. + pub fn ahead(at: u64) -> u64 { + let steps = STEPS_AT.iter().filter(|s| s.load(Ordering::Relaxed) <= at).count() as u64; + at + steps * STEP_TICKS.load(Ordering::Relaxed) + } +} + +#[cfg(feature = "boot-actuators")] +fn staged(hpet: Sample, pm: PmSpan, edge: Edge) -> (Sample, PmSpan, Edge) { + use stage::*; + let forward = |s: Sample| Sample { before: ahead(s.before), after: ahead(s.after), ..s }; + let stalled = if hpet.before >= STALLED_AT.load(Ordering::Relaxed) { STALL_TICKS.load(Ordering::Relaxed) } else { 0 }; + let hpet = Sample { reference: hpet.reference - stalled, ..forward(hpet) }; + (hpet, pm.map(|(from, to)| (from, forward(to))), edge.map(|(seen, at)| (seen, ahead(at)))) +} + +/// `tsc-step`'s and `hpet-stall`'s first, at the clock's first report. +#[cfg(feature = "boot-actuators")] +fn stage_first(period_fs: u64) { + use stage::*; + let now = cpu::counter(); + if STEPS_AT[0].load(Ordering::Relaxed) != u64::MAX || STALLED_AT.load(Ordering::Relaxed) != u64::MAX { + return; + } + if crate::actuator::tsc_step() { + let ahead = crate::clock::counter_ticks(10_000_000); + STEP_TICKS.store(ahead, Ordering::Relaxed); + STEPS_AT[0].store(now, Ordering::Relaxed); + log!("tsc-step: the TSC reads {ahead} cycles ahead from here"); + } + if crate::actuator::hpet_stall() { + let behind = 10_000_000_000_000 / period_fs; + STALL_TICKS.store(behind, Ordering::Relaxed); + STALLED_AT.store(now, Ordering::Relaxed); + log!("hpet-stall: the HPET reads {behind} ticks behind from here"); + } +} + +/// `tsc-step`'s second step, once the first is slewed away. +#[cfg(feature = "boot-actuators")] +fn stage_again() { + if crate::actuator::tsc_step() && stage::STEPS_AT[1].load(Ordering::Relaxed) == u64::MAX { + stage::STEPS_AT[1].store(cpu::counter(), Ordering::Relaxed); + log!("tsc-step: the TSC reads {} cycles further ahead from here", stage::STEP_TICKS.load(Ordering::Relaxed)); } - let ahead = TSC_AHEAD.load(Ordering::Relaxed); - let forward = |s: Sample| Sample { before: s.before + ahead, after: s.after + ahead, ..s }; - let behind = HPET_BEHIND.load(Ordering::Relaxed); - let hpet = Sample { reference: hpet.reference - behind, ..forward(hpet) }; - (hpet, pm.map(|(from, to)| (from, forward(to)))) } diff --git a/kernel/src/arch/x86_64/idt/mod.rs b/kernel/src/arch/x86_64/idt/mod.rs index 59557caaa9b..ecaf2105628 100644 --- a/kernel/src/arch/x86_64/idt/mod.rs +++ b/kernel/src/arch/x86_64/idt/mod.rs @@ -1,6 +1,7 @@ pub(crate) mod exceptions; mod device_irq; mod dma_fault; +mod rtc; mod isa; mod kick; mod nmi; @@ -36,6 +37,9 @@ pub const ISA_VECTORS: [u8; crate::isa::MAX_ROWS] = [Vector::Isa0 as u8, Vector: /// The vector an IOMMU writes into its own `FEDATA`. pub const DMA_FAULT_VECTOR: u8 = Vector::DmaFault as u8; +/// The vector the RTC's IRQ8 is routed to. +pub const RTC_VECTOR: u8 = Vector::Rtc as u8; + const PF_PRESENT: u64 = 1 << 0; const PF_WRITE: u64 = 1 << 1; const PF_INSTRUCTION_FETCH: u64 = 1 << 4; @@ -242,6 +246,7 @@ idt_vectors! { Nmi = 0x02, nmi::nmi_entry, ist 2; Timer = 0x20, timer::timer_entry; DmaFault = 0x25, dma_fault::dma_fault_entry; + Rtc = 0x26, rtc::rtc_entry; Isa0 = 0x2C, isa::isa0_entry; Isa1 = 0x2D, isa::isa1_entry; Kick = 0xFC, kick::kick_entry; diff --git a/kernel/src/arch/x86_64/idt/rtc.rs b/kernel/src/arch/x86_64/idt/rtc.rs new file mode 100644 index 00000000000..a74244af2df --- /dev/null +++ b/kernel/src/arch/x86_64/idt/rtc.rs @@ -0,0 +1,12 @@ +use super::device_irq::device_irq_entry; + +extern "sysv64" fn rtc_handler() { + crate::arch::percpu::irq_took!(Rtc); + crate::arch::rtc::update_ended(); + crate::arch::apic::eoi(); +} + +device_irq_entry! { + /// The RTC's IRQ8 entry (see `device_irq_entry` for the asm contract). + pub(super) fn rtc_entry => rtc_handler +} diff --git a/kernel/src/arch/x86_64/rtc.rs b/kernel/src/arch/x86_64/rtc.rs index 169e01cd52c..5feb9256606 100644 --- a/kernel/src/arch/x86_64/rtc.rs +++ b/kernel/src/arch/x86_64/rtc.rs @@ -7,6 +7,7 @@ use core::fmt; use crate::arch::cpu; use crate::clock; +use crate::log; use crate::time::{Bound, Duration}; use toyos_wallclock::Civil; @@ -209,3 +210,64 @@ fn port_read(reg: u8) -> u8 { fn cmos_read(reg: u8) -> u8 { port_read(reg) } + +const STATUS_C: u8 = 0x0C; +/// Register B bit 4: raise IRQ8 as each update cycle ends. +const UPDATE_ENDED_INTERRUPT: u8 = 1 << 4; +/// Register C bit 4: an update cycle has ended since register C was last read. +const UPDATE_ENDED: u8 = 1 << 4; +/// ISA IRQ 8, which the RTC raises until register C is read (MC146818). +const RTC_IRQ: u8 = 8; + +/// The latest update-ended edge: how many the handler has seen, and the TSC +/// as it took the last. Published by IRQ8's handler on the one CPU it is routed to. +static EDGE: crate::seqlock::Published<2> = crate::seqlock::Published::new(); + +/// Raise IRQ8 at the end of each update cycle, once a second off the RTC's own +/// 32.768 kHz crystal, and take the TSC as each arrives: a reference outside +/// the chipset's 14.318 MHz domain, which the HPET and the ACPI PM timer share. +/// Called after the boot's [`read`], the only other CMOS access, so the index +/// port is the handler's alone from here. +pub fn arm_update_edges() -> Result<(), alloc::string::String> { + let line = super::ioapic::gsi_for_isa_irq(RTC_IRQ).ok_or("no I/O APIC carries IRQ8")?; + super::ioapic::route(line.gsi, super::idt::RTC_VECTOR, super::apic::MSG_DEST, line.trigger, line.polarity) + .map_err(|why| alloc::format!("{why:?}"))?; + let b = cmos_read(STATUS_B); + port_write(STATUS_B, b | UPDATE_ENDED_INTERRUPT); + let _ = cmos_read(STATUS_C); + super::ioapic::set_masked(line.gsi, false).map_err(|why| alloc::format!("{why:?}"))?; + // An update that ended while the pin was masked holds IRQ8 high with no + // edge the pin saw; reading C again lowers it, so the next update's edge + // arrives. Should the handler run first, it reads the same register. + let _ = cmos_read(STATUS_C); + log!( + "clock: the RTC's update-ended interrupt on gsi {} {} times the TSC against its 32.768 kHz crystal", + line.gsi.0, + super::ioapic::describe(line.trigger, line.polarity), + ); + Ok(()) +} + +/// IRQ8's handler: the TSC first, then register C, whose read lets the RTC +/// raise the next. +pub fn update_ended() { + let at = cpu::rdtsc_ordered(); + if cmos_read(STATUS_C) & UPDATE_ENDED != 0 { + let seen = EDGE.snapshot().map_or(0, |[seen, _]| seen); + EDGE.publish([seen + 1, at]); + } +} + +/// The edges seen and the TSC at the last, or `None` before the first. +pub fn last_edge() -> Option<(u64, u64)> { + EDGE.snapshot().filter(|&[seen, _]| seen > 0).map(|[seen, at]| (seen, at)) +} + +fn port_write(reg: u8, value: u8) { + // SAFETY: as `port_read`'s: `reg` is this module's constant, below 0x80, + // and the byte only enables the update-ended interrupt. + unsafe { + cpu::outb(CMOS_ADDR, reg); + cpu::outb(CMOS_DATA, value); + } +} diff --git a/kernel/src/clock.rs b/kernel/src/clock.rs index 1ef51999f9c..27a2d1c76f6 100644 --- a/kernel/src/clock.rs +++ b/kernel/src/clock.rs @@ -67,18 +67,33 @@ pub fn set_counter(boot: u64, mult: u64) { LINES.store(lines as *const ClockPage as *mut ClockPage, Release); } -/// Go on at `mult` from now: the line from here starts where the clock -/// stands, so it bends and never steps. The line now in force. -pub fn retune(mult: u64) -> Line { +/// Go on at `mult` from the counter reading `at`: the line from there starts +/// where the clock stands, so it bends and never steps. The line now in force. +pub fn retune(at: u64, mult: u64) -> Line { let page = lines().expect("clock: retuned before it started"); let _one = RETUNING.lock(); - let now = cpu::counter(); - let at = page.read().expect("clock: the page `set_counter` laid out carries no clock").stamp_of(now); - let line = Line { counter: now, stamp: at, mult }; + let stamp = page.read().expect("clock: the page `set_counter` laid out carries no clock").stamp_of(at); + let line = Line { counter: at, stamp, mult }; page.publish(line); line } +/// The rate the line in force runs at. +pub fn mult() -> u64 { + clock().expect("clock: read before it started").line.mult +} + +/// The clock at the counter reading `at`, as a log line's time. +pub fn stamp_at(at: u64) -> u64 { + clock().expect("clock: read before it started").stamp_of(at) +} + +/// Send every process's reading of now to the kernel from here on +/// (`toyos_abi::clock::ClockPage::demoted`). +pub fn demote() { + lines().expect("clock: demoted before it started").demote(); +} + fn lines() -> Option<&'static ClockPage> { let page = LINES.load(Acquire); // SAFETY: `set_counter` stored a pointer into a page it holds for the diff --git a/kernel/src/idle.rs b/kernel/src/idle.rs index f5e5e9811f3..25ac31c5418 100644 --- a/kernel/src/idle.rs +++ b/kernel/src/idle.rs @@ -41,10 +41,8 @@ pub struct Local { /// declaration it learned it under. history: AtomicU64, learned_under: AtomicU64, - /// Idle periods spent in a declared state, counted as each ends, and as - /// each starts. + /// Idle periods spent in a declared state. deep: AtomicU64, - entered: AtomicU64, } impl Local { @@ -55,7 +53,6 @@ impl Local { history: AtomicU64::new(0), learned_under: AtomicU64::new(0), deep: AtomicU64::new(0), - entered: AtomicU64::new(0), } } @@ -63,13 +60,6 @@ impl Local { pub fn deep(&self) -> u64 { self.deep.load(Ordering::Relaxed) } - - /// The idle periods in a declared state this CPU has started and ended, - /// in that order: the CPU is in one now where the two differ. - pub fn deep_periods(&self) -> (u64, u64) { - let entered = self.entered.load(Ordering::Acquire); - (entered, self.deep.load(Ordering::Acquire)) - } } /// Read what this CPU can enter: called by each CPU as it first enters its @@ -185,18 +175,14 @@ pub fn wait(plan: &idle::Plan) { crate::sched::driver::publish_idle_exit(0); } match (state, &d) { - (Some(state), Some(d)) => { - // This CPU's alone to write, so a store and not a locked add. - local.entered.store(local.entered.load(Ordering::Relaxed) + 1, Ordering::Release); - crate::arch::idle::enter(d.entries[state]); - } + (Some(state), Some(d)) => crate::arch::idle::enter(d.entries[state]), _ => crate::arch::hw::halt(), } // Awake: no hold has a reason to wake this CPU until it waits again. crate::sched::driver::publish_idle_exit(0); let woke = crate::clock::nanos_since_boot(); if state.is_some() { - local.deep.fetch_add(1, Ordering::Release); + local.deep.fetch_add(1, Ordering::Relaxed); } let Some(d) = d else { return }; let costs = &d.costs[..d.len]; diff --git a/kernel/src/irq_census.rs b/kernel/src/irq_census.rs index df075624e26..9bae6a8e99b 100644 --- a/kernel/src/irq_census.rs +++ b/kernel/src/irq_census.rs @@ -25,6 +25,9 @@ pub enum Source { UserDev, /// Vector 0x25, the remapping unit's fault event. DmaFault, + /// Vector 0x26, x86-64's RTC update-ended interrupt, once a second. + #[cfg_attr(not(target_arch = "x86_64"), allow(dead_code))] + Rtc, /// Vector 0xFE, the TLB shootdown IPI. Tlb, /// Vector 0x02, which `crate::hardlockup` samples a CPU with. @@ -38,11 +41,11 @@ pub enum Source { } impl Source { - pub const COUNT: usize = 8; + pub const COUNT: usize = 9; /// Order `tests/toyos.rs`'s `irq_census_conservation` parses back; must match variant order. pub const NAMES: [&'static str; Self::COUNT] = [ - "timer", "kick", "userdev", "dmafault", "tlb", "nmi", "spurious", "unclaimed", + "timer", "kick", "userdev", "dmafault", "rtc", "tlb", "nmi", "spurious", "unclaimed", ]; } diff --git a/kernel/src/syscall/dispatch.rs b/kernel/src/syscall/dispatch.rs index fd583a8a451..56a9517bf07 100644 --- a/kernel/src/syscall/dispatch.rs +++ b/kernel/src/syscall/dispatch.rs @@ -630,6 +630,7 @@ pub(crate) fn syscall_dispatch(num: u64, a1: u64, a2: u64, a3: u64, a4: u64) -> SYS_EVENT_SIGNAL => sys_event_signal(RawHandle(a1 as u32)), SYS_EVENT_TAKE => sys_event_take(RawHandle(a1 as u32)), SYS_HANDLE_WAIT => sys_handle_wait(RawHandle(a1 as u32), a2, a3), + SYS_CLOCK_NOW => crate::arch::clocksource::stamp_now(), SYS_ACCOUNT_INFO => { if a3 < core::mem::size_of::() as u64 { return SyscallError::InvalidArgument.to_u64(); } let Some(addr) = UserAddr::checked(a2) else { return bad_addr }; diff --git a/sdk/abi/src/clock.rs b/sdk/abi/src/clock.rs index 48c764e3cce..47c02b1ddeb 100644 --- a/sdk/abi/src/clock.rs +++ b/sdk/abi/src/clock.rs @@ -1,4 +1,5 @@ -//! The monotonic clock a process reads without a syscall. +//! The monotonic clock a process reads without a syscall, until the kernel +//! demotes the counter it reads. //! //! The clock is a straight [`Line`] through the counter: from a counter //! reading, the nanoseconds since the counter's zero, and the rate after it. @@ -10,6 +11,12 @@ //! clock reads too, so a stamp taken here and one taken by the kernel's log //! order against each other. //! +//! **A demoted page sends every reading to the kernel.** Where the counter +//! steps again after a slewed step, or steps back, the kernel marks the page +//! demoted ([`ClockPage::demoted`]); [`nanos_since_boot`] and [`stamp_ns`] +//! then ask [`crate::syscall::clock_now`], which reads the reference instead. +//! The line stays, for a recorded counter reading. +//! //! **Every word of the page is atomic.** The kernel writes it while processes //! read it, so it is reached as atomics and never as plain memory. @@ -26,7 +33,7 @@ pub const CLOCK_PAGE: u64 = 0x1_0000_0000; /// What the page opens with, so a reader that finds anything else refuses /// rather than computes a time out of it. -pub const CLOCK_MAGIC: u64 = 0x544F_594F_434C_4B32; // "TOYOCLK2" +pub const CLOCK_MAGIC: u64 = 0x544F_594F_434C_4B33; // "TOYOCLK3" /// The binary point of [`Line::mult`]: nanoseconds per tick in 32.32 fixed /// point, which keeps a 5 GHz counter's rate to a part in 10⁹ and a 1 Hz @@ -121,6 +128,7 @@ impl Clock { pub struct ClockPage { magic: AtomicU64, stamp_at_boot: AtomicU64, + demoted: AtomicU64, line: Latch<3>, } @@ -137,6 +145,16 @@ impl ClockPage { self.line.publish(line.words()); } + /// Send every reading to the kernel from now on: the kernel's, once. + pub fn demote(&self) { + self.demoted.store(1, Ordering::Release); + } + + /// Whether a reading of now asks the kernel rather than the line. + pub fn demoted(&self) -> bool { + self.demoted.load(Ordering::Acquire) != 0 + } + /// The clock now, or `None` on a page that carries none. pub fn read(&self) -> Option { if self.magic.load(Ordering::Acquire) != CLOCK_MAGIC { @@ -147,35 +165,51 @@ impl ClockPage { } } -/// Nanoseconds since boot, off this process's clock page. +/// Nanoseconds since boot, off this process's clock page, or the kernel's +/// clock where the page is demoted. /// /// # Panics /// As [`page`]. pub fn nanos_since_boot() -> u64 { - let clock = page(); + let page = mapped(); + let clock = read(page); + if page.demoted() { + return crate::syscall::clock_now().saturating_sub(clock.stamp_at_boot); + } clock.nanos_since_boot(crate::arch::counter()) } /// Now as a log line's time: nanoseconds since the counter's zero, the kernel /// log's own `clock::stamp`, which every line of the log carries -/// (`log::LogRecord::at_ns`). +/// (`log::LogRecord::at_ns`); the kernel's clock where the page is demoted. /// /// # Panics /// As [`page`]. pub fn stamp_ns() -> u64 { - page().stamp_of(crate::arch::counter()) + let page = mapped(); + if page.demoted() { + return crate::syscall::clock_now(); + } + read(page).stamp_of(crate::arch::counter()) } -/// The clock, off this process's page. +/// The clock, off this process's page: its line, for a counter reading. /// /// # Panics /// On a page that does not carry [`CLOCK_MAGIC`]: every address space the /// kernel builds maps one, so its absence is a kernel that broke the ABI. pub fn page() -> Clock { + read(mapped()) +} + +fn mapped() -> &'static ClockPage { // SAFETY: the kernel maps the page read-only at `CLOCK_PAGE` in every // address space before the first instruction runs and never unmaps it; // every field is an atomic, so a shared reference reads it as atomics. - let page = unsafe { &*(CLOCK_PAGE as *const ClockPage) }; + unsafe { &*(CLOCK_PAGE as *const ClockPage) } +} + +fn read(page: &ClockPage) -> Clock { page.read().unwrap_or_else(|| panic!("the clock page at {CLOCK_PAGE:#x} carries no clock")) } @@ -234,7 +268,7 @@ mod tests { #[test] fn the_page_publishes_whole_lines() { - let page = ClockPage { magic: AtomicU64::new(0), stamp_at_boot: AtomicU64::new(0), line: Latch::new() }; + let page = ClockPage { magic: AtomicU64::new(0), stamp_at_boot: AtomicU64::new(0), demoted: AtomicU64::new(0), line: Latch::new() }; assert_eq!(page.read(), None); let first = Line { counter: 7, stamp: 9, mult: GHZ }; page.init(9, first); @@ -244,5 +278,8 @@ mod tests { assert_eq!(page.read().map(|c| c.line), Some(second)); page.publish(first); assert_eq!(page.read().map(|c| c.line), Some(first)); + assert!(!page.demoted()); + page.demote(); + assert!(page.demoted()); } } diff --git a/sdk/abi/src/syscall.rs b/sdk/abi/src/syscall.rs index 4c95cc77890..95834edfa15 100644 --- a/sdk/abi/src/syscall.rs +++ b/sdk/abi/src/syscall.rs @@ -348,6 +348,8 @@ pub const SYS_EVENT_SIGNAL: u64 = 149; pub const SYS_EVENT_TAKE: u64 = 150; /// Park on one handle's one condition, with a timeout. See [`handle_wait`]. pub const SYS_HANDLE_WAIT: u64 = 151; +/// The clock now, read by the kernel. See [`clock_now`]. +pub const SYS_CLOCK_NOW: u64 = 152; /// Bins in the per-process syscall profile — one for every number this ABI /// issues, and one at the end for every number it does not. @@ -355,13 +357,13 @@ pub const SYS_HANDLE_WAIT: u64 = 151; /// **The profile's parts sum to its total, and that is the whole requirement.** /// A bin array narrower than the ABI reaches drops calls out of the line while /// the total goes on counting them. -pub const SYSCALL_PROFILE_BINS: usize = 153; +pub const SYSCALL_PROFILE_BINS: usize = 154; /// Where a number this ABI does not issue is counted. Merging is a degradation /// a reader can see in the line; dropping is one nobody can. pub const SYSCALL_PROFILE_OTHER: usize = SYSCALL_PROFILE_BINS - 1; -const _: () = assert!(SYS_HANDLE_WAIT < SYSCALL_PROFILE_OTHER as u64); +const _: () = assert!(SYS_CLOCK_NOW < SYSCALL_PROFILE_OTHER as u64); pub const WNOHANG: u64 = 1; /// Every bit `SYS_PROCESS_WAIT`'s flag word defines; a wait carrying another @@ -1416,6 +1418,13 @@ pub fn reboot(syscap: RawHandle) -> SyscallError { } } +/// The clock now as a log line's time, nanoseconds since the counter's zero, +/// read by the kernel: what [`crate::clock::stamp_ns`] answers where the page +/// is demoted. Needs no handle; it fails for nobody. +pub fn clock_now() -> u64 { + syscall(SYS_CLOCK_NOW, 0, 0, 0, 0) +} + /// Which stop [`machine_stops`] asks about. #[derive(Clone, Copy, Debug, PartialEq, Eq)] #[repr(u64)] diff --git a/tests/checks.rs b/tests/checks.rs index b513b75f0bf..ba2968973e5 100644 --- a/tests/checks.rs +++ b/tests/checks.rs @@ -585,7 +585,7 @@ mod checks { let userdev = if cpu == 0 { 40 } else { 0 }; format!( "[ {at} cpu{on} kernel] irq: cpu{cpu} timer=100 kick={kick} userdev={userdev} \ - dmafault=0 tlb={tlb} nmi=0 spurious=0 unclaimed=0\n" + dmafault=0 rtc=0 tlb={tlb} nmi=0 spurious=0 unclaimed=0\n" ) }; let issued = |at: &str, on: u32, total: u64| { @@ -1104,7 +1104,7 @@ mod checks { assert!(judge(&[&readback("jobcase", &done(stopped), jobcase)]).is_err()); // What the seal itself writes of the machine, above the ring's tail. - let census = "| irq: cpu0 timer=9 kick=2 userdev=1729 dmafault=0 tlb=0 \ + let census = "| irq: cpu0 timer=9 kick=2 userdev=1729 dmafault=0 rtc=0 tlb=0 \ nmi=0 spurious=0 unclaimed=0\n\ | tlb: shootdowns=4 wait=12us max=3us dlopen=0 pcid=0 mmio=0 unmap=4 pipe=0 staged=0 \ bench=0\n\ diff --git a/tests/common/faults.rs b/tests/common/faults.rs index 233612cbf3f..871b6190e02 100644 --- a/tests/common/faults.rs +++ b/tests/common/faults.rs @@ -136,33 +136,42 @@ pub fn kernel_wx_leaf_is_refused(test_config: &Path, profile: qemu::Profile) -> } } -/// What `tsc-step` says of the step it stages, and the head of the clock -/// check's panic at it (`kernel/src/arch/x86_64/clocksource.rs`). +/// What `tsc-step` says of each step it stages, and the clock's records of +/// slewing the first away and demoting the TSC at the second +/// (`kernel/src/arch/x86_64/clocksource.rs`). const TSC_STEP_STAGED: &str = "tsc-step: the TSC reads "; -const TSC_DISAGREES: &str = "clock: the TSC and the HPET disagree by "; +const TSC_STEP_SLEWED: &str = "clock: TSC STEP: the TSC stepped "; +const TSC_SLEWED_AWAY: &str = "clock: the TSC's step is slewed away"; +const TSC_STEP_AGAIN: &str = "cycles further ahead from here"; +const TSC_DEMOTED: &str = "clock: DEMOTED: "; +const DEMOTED_READS: &str = "clock: a process's reading of now asked the kernel since the TSC was demoted"; -/// A TSC `tsc-step` reads 10 ms ahead: the clock's runtime check panics after -/// the step, having read it 9 ms or more apart from the HPET, as a step. -pub fn a_stepped_tsc_panics(test_config: &Path) -> Result<(), String> { - let qemu = QemuInstance::boot_with_options( +/// A TSC `tsc-step` reads 10 ms ahead, and 10 ms further once the clock has +/// slewed the first step away: the first is slewed, the second demotes the +/// TSC, and processes' readings of now reach the kernel after it. +pub fn a_stepped_tsc_is_slewed_then_demoted(test_config: &Path) -> Result<(), String> { + let mut qemu = QemuInstance::boot_with_options( test_config, &[], &[], - BootOptions { profile: qemu::Profile::Metal, kernel_params: &["tsc-step"], ready_marker: TSC_DISAGREES, ..Default::default() }, + BootOptions { profile: qemu::Profile::Metal, kernel_params: &["tsc-step"], ready_marker: TSC_STEP_SLEWED, ..Default::default() }, ); - let serial = qemu.boot_log(); + let mut serial = qemu.boot_log().to_string(); + // A 10 ms step at 500 ppm is 20 s of slewing; the rest is the RTC's edges either side of each step. + serial.push_str(&qemu.drain_until(std::time::Duration::from_secs(120), |l| l.contains(DEMOTED_READS) || l.contains("PANIC"))); let lines: Vec<&str> = serial.lines().collect(); - let staged = lines.iter().position(|l| l.contains(TSC_STEP_STAGED)).ok_or_else(|| format!("no {TSC_STEP_STAGED:?} line\n{serial}"))?; - let panicked = lines.iter().position(|l| l.contains(TSC_DISAGREES)).ok_or_else(|| format!("no {TSC_DISAGREES:?} panic\n{serial}"))?; - let apart: Option = lines[panicked].split(TSC_DISAGREES).nth(1).and_then(|r| r.split(" ns over").next()).and_then(|n| n.parse().ok()); - if staged > panicked || !lines[panicked].contains("since the last sample") || !apart.is_some_and(|ns| ns >= 9_000_000) { - return Err(format!( - "the step and the panic do not match: {:?} then {:?}\n{serial}", - lines[staged].trim(), - lines[panicked].trim() - )); + let at = |marker: &str| lines.iter().position(|l| l.contains(marker)).ok_or_else(|| format!("no {marker:?} line\n{serial}")); + let order = [at(TSC_STEP_STAGED)?, at(TSC_STEP_SLEWED)?, at(TSC_SLEWED_AWAY)?, at(TSC_STEP_AGAIN)?, at(TSC_DEMOTED)?, at(DEMOTED_READS)?]; + if !order.is_sorted() || serial.contains("PANIC") { + return Err(format!("the steps, the slew and the demotion are out of order: {order:?}\n{serial}")); + } + let stepped: Option = lines[order[1]].split(TSC_STEP_SLEWED).nth(1).and_then(|r| r.split(" ns").next()).and_then(|n| n.parse().ok()); + if !stepped.is_some_and(|ns| (9_000_000..=11_000_000).contains(&ns)) { + return Err(format!("the slewed step is not the staged 10 ms: {}\n{serial}", lines[order[1]].trim())); + } + for i in [1, 4, 5] { + eprintln!(" [tsc_step] {}", lines[order[i]].trim()); } - eprintln!(" [tsc_step] {}", lines[panicked].trim()); Ok(()) } diff --git a/tests/common/irqcensus.rs b/tests/common/irqcensus.rs index de38f7db579..42154a7119d 100644 --- a/tests/common/irqcensus.rs +++ b/tests/common/irqcensus.rs @@ -15,14 +15,14 @@ use std::collections::BTreeMap; /// and [`Census::parse`] refuses a line whose fields are not exactly these, so /// a source added on one side and not the other is a red rather than a silently /// dropped column. -pub const SOURCES: [&str; 8] = ["timer", "kick", "userdev", "dmafault", "tlb", "nmi", "spurious", "unclaimed"]; +pub const SOURCES: [&str; 9] = ["timer", "kick", "userdev", "dmafault", "rtc", "tlb", "nmi", "spurious", "unclaimed"]; /// The sources whose delivery CPU is chosen by the interrupt controller rather /// than by the CPU that took the work — every device vector, in other words. /// `MSG_ADDR` names physical destination 0 and the one I/O APIC pin this kernel /// routes goes to the BSP, so today every one of these is cpu0's alone. The day /// that stops being true is the day the track's change lands. -pub const DEVICE_SOURCES: [&str; 2] = ["userdev", "dmafault"]; +pub const DEVICE_SOURCES: [&str; 3] = ["userdev", "dmafault", "rtc"]; /// One CPU's counters out of one `irq:` line. #[derive(Clone, Debug, PartialEq, Eq)] diff --git a/tests/toyos.rs b/tests/toyos.rs index 37e0b0ae3a3..7c9365074cf 100644 --- a/tests/toyos.rs +++ b/tests/toyos.rs @@ -520,12 +520,12 @@ const MACHINE_TESTS: &[&str] = &[ // the same audit passing. q35 on `Profile::Metal`, whose 16550 carries // the panic. "kernel_wx_leaf_is_refused", - // The clock's runtime check failing loud on a TSC an actuator stepped - // where `IA32_TSC_ADJUST` would not show it: the panic ends the boot, and - // no bench's TSC steps on demand. q35 on `Profile::Metal`, whose 16550 - // carries the panic; the order of the step and the panic, and the size - // the check read, are what it holds. - "a_stepped_tsc_panics", + // The clock's response to TSC steps an actuator stages where + // `IA32_TSC_ADJUST` would not show them: the first slewed away, the + // second demoting the TSC, after which processes' readings of now reach + // the kernel. q35 on `Profile::Metal`; the order, the step's size and a + // demoted reading are what it holds, and no bench's TSC steps on demand. + "a_stepped_tsc_is_slewed_then_demoted", // Its counterpart: the HPET an actuator stalled while the TSC and the PM // timer count on is left out, and the boot goes on. q35 on // `Profile::Metal`, whose PM timer is at 0x608; no bench's HPET stalls on @@ -4685,7 +4685,7 @@ fn run_machine_test(name: &str, test_config: &Path) -> Result<(), String> { "amdvi_selftest" => common::iommu::amdvi_selftest(test_config), "nested_nmi_is_loud" => faults::nested_nmi_is_loud(test_config), "kernel_wx_leaf_is_refused" => faults::kernel_wx_leaf_is_refused(test_config, qemu::Profile::Metal), - "a_stepped_tsc_panics" => faults::a_stepped_tsc_panics(test_config), + "a_stepped_tsc_is_slewed_then_demoted" => faults::a_stepped_tsc_is_slewed_then_demoted(test_config), "a_stalled_hpet_is_left_out" => faults::a_stalled_hpet_is_left_out(test_config), "acpi_press_is_one_sci" => acpi_press_is_one_sci(), "osbench_capability_groups_exit_0" => osbench_capability_groups_exit_0(), From fec49988cf83b8a855118bab60d38cf3a0dfb008 Mon Sep 17 00:00:00 2001 From: japabu Date: Sun, 1 Nov 2026 03:05:29 +0100 Subject: [PATCH 07/11] review r2: a reading takes the counter inside the line's section, a demotion hands over monotonically, and the sampler goes on after it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes review r2's required findings (R1–R7) and four of its optional ones. - **R1, torn reads.** `Latch::read_with` runs the counter read inside the read section: the words, the counter, an Acquire fence, then the recheck. A publication that lands while the reader is preempted sends it round again. - The page readers (`ClockPage::read_now`, `nanos_since_boot`, `stamp_ns`) and the kernel's own (`clock::stamp`, `nanos_since_boot`) read this way. - The ABI's counter is fenced after its read as well as before (`lfence`/`isb`), and the kernel's is `counter_ordered`. - The loom model adds a counter the writer advances once its line is published. A reading of it always has the new line. A `should_panic` companion shows the old order, counter read after the section, does extrapolate a superseded line. - **R2, demotion handover.** A page reader checks the page's demoted word after reading the counter. `demote` holds its lock throughout: it marks the page, fences, reads the counter, and raises `DEMOTED_LATEST` to the line at that reading. Every value a reader that found the page unmarked returned is then at or below every later syscall answer. - `stamp_now` reads the counter before the mark, as a page reader does. - **Demoted time.** It counts by the HPET from the last held sample's clock, the HPET's own clock, and the floor holds it until the HPET catches up. A forward step no longer leaves processes ahead for good. - **R3, after a demotion.** - `demote` retunes the line to the kept rate, so no slew stays in it. - The per-CPU checks and the sampler go on. A step after the demotion is logged loudly and not slewed, and so is a check failure. - The dead demoted branch in `sample` is gone. - **R4, the metal rows.** `clock_kept_its_counter` prints every TSC STEP, left-out and waiting-on-the-RTC line. It fails `smp_roster_and_tsc_trail`, `timer_calibration` and `clock_holds_over_minutes` on a demotion, and a host test holds it. The "panics on a step" doc is fixed. - **R5, records.** - The new issue `a-demoted-tsc-keeps-the-kernels-clock-apart-from-processes` records the backward-step sequence. - The AMD issue no longer says monotonic time never steps: readers see the forward step, and only the correction never steps back. - **R6, comments.** The `Demoted` docs, the ABI header (the kernel demotes the page when it stops trusting the counter) and `reference.rs`'s header (the RTC's least latency and drift are its one estimate). - **R7, the census.** The census's sources come from a macro the architecture hands its own variants to (`arch::census_sources!`), so generic code names no RTC and carries no `target_arch`. Optional findings: - **Taken:** - `Slew::mult` pays over two periods, so a sample up to a period late cannot overpay. A later one overpays, and the next sample pays it back (host case). The tail now halves each period: 1.7 ms takes about 12 s. - `apart_across` takes out the RTC crystal's drift, measured from edges at least 4 s before the span. A host case shows a 60 ppm crystal reads a stall as none, and reads a step as one; with the correction disabled it fails. - The demoted HPET is extended from its last answer. - The "behind" line names both causes. - **Left:** a publish-once demoted cell, and arbitrating drift and AP-start offsets, which still panic. Co-Authored-By: Claude Opus 5.5 --- ...-the-kernels-clock-apart-from-processes.md | 31 +++++ ...hs-tsc-steps-1-7-ms-forward-once-a-boot.md | 16 ++- kernel/loom/tests/clock_latch.rs | 36 ++++++ kernel/pure/clock/reference.rs | 90 +++++++++++--- kernel/src/arch/aarch64/cpu.rs | 9 ++ kernel/src/arch/aarch64/mod.rs | 9 ++ kernel/src/arch/x86_64/clocksource.rs | 116 +++++++++++------- kernel/src/arch/x86_64/cpu.rs | 6 + kernel/src/arch/x86_64/mod.rs | 12 ++ kernel/src/clock.rs | 13 +- kernel/src/irq_census.rs | 80 ++++++------ sdk/abi/src/arch/aarch64.rs | 8 +- sdk/abi/src/arch/x86_64.rs | 10 +- sdk/abi/src/clock.rs | 49 ++++++-- sdk/abi/src/clock/latch.rs | 17 ++- tests/checks.rs | 9 ++ tests/toyos.rs | 26 +++- 17 files changed, 398 insertions(+), 139 deletions(-) create mode 100644 issues/a-demoted-tsc-keeps-the-kernels-clock-apart-from-processes.md diff --git a/issues/a-demoted-tsc-keeps-the-kernels-clock-apart-from-processes.md b/issues/a-demoted-tsc-keeps-the-kernels-clock-apart-from-processes.md new file mode 100644 index 00000000000..b1cc1d515af --- /dev/null +++ b/issues/a-demoted-tsc-keeps-the-kernels-clock-apart-from-processes.md @@ -0,0 +1,31 @@ +--- +status: open +kind: defect +opened: 2026-11-01 +--- + +# A demoted TSC keeps the kernel's clock apart from processes' + +When the clock demotes the TSC (`kernel/src/arch/x86_64/clocksource.rs`, +`demote`), processes' readings of now move to the HPET through +`SYS_CLOCK_NOW`. The kernel's own clock, its deadlines and its log stamps stay +on the TSC at the kept rate. Every later TSC step therefore moves the two +clocks apart, and a log line a process stamps no longer orders against one +the kernel stamps. + +A backward step plays out like this: +1. Every reading, the kernel's and processes', goes back by the step. +2. The sampler, or a CPU's check, finds it within a second and demotes the + TSC. +3. Processes' time jumps forward to the HPET's count from the last held + sample, which is the true time, and stays monotonic from there. +4. The kernel's clock stays behind by the step. + +After a forward second step, processes' time holds at the TSC's clock until +the HPET's count reaches it, then follows the HPET. The kernel's clock stays +ahead by the step. + +**Owner:** the clock. + +**Exit:** after a demotion, the kernel's clock follows the reference too, so +the kernel and processes read one clock again. diff --git a/issues/the-amd-benchs-tsc-steps-1-7-ms-forward-once-a-boot.md b/issues/the-amd-benchs-tsc-steps-1-7-ms-forward-once-a-boot.md index 2eb4478d119..6c297e7722e 100644 --- a/issues/the-amd-benchs-tsc-steps-1-7-ms-forward-once-a-boot.md +++ b/issues/the-amd-benchs-tsc-steps-1-7-ms-forward-once-a-boot.md @@ -30,15 +30,13 @@ hand-over. The trigger is the package's deep idle with every core in 0x415. Whether the TSC's jump across it is the platform's or the state's entry is not known. -**What the clock does meanwhile.** A forward step that the HPET and the PM -timer agree on is slewed away at no more than 500 ppm. This happens whether -the RTC confirms the step or cannot tell. The 1.7 ms takes 3.4 s, and -monotonic time never steps. A second step, or a backward one, demotes the TSC: -processes then read now through the kernel, which counts by the HPET. - -Two things stay on the TSC after a demotion: the kernel's own deadlines and -its log stamps. A later step therefore moves them apart from what processes -read. +**What the clock does meanwhile.** Readers see the forward step: the clock +reads up to 1.7 ms ahead from the step until the sampler finds it, within a +second, and through the RTC's few seconds after. A forward step the HPET and +the PM timer agree on is then slewed away at no more than 500 ppm. This +happens whether the RTC confirms the step or cannot tell, and the correction +never steps back. A second step, or a backward one, demotes the TSC +(`issues/a-demoted-tsc-keeps-the-kernels-clock-apart-from-processes.md`). **Not read.** AMD's *Processor Programming Reference for Family 17h Model 60h* (55922), which would cover the TSC in deep C-states and the TSC's relation to diff --git a/kernel/loom/tests/clock_latch.rs b/kernel/loom/tests/clock_latch.rs index 79888f3680f..f7ab6c47074 100644 --- a/kernel/loom/tests/clock_latch.rs +++ b/kernel/loom/tests/clock_latch.rs @@ -59,3 +59,39 @@ fn a_later_read_is_never_an_older_line() { writer.join().unwrap(); }); } + +/// The counter a reader turns a line into a time with, modelled as memory the +/// writer advances only once the line that starts there is published. +fn superseded(read: impl Fn(&Latch, &loom::sync::atomic::AtomicU64) -> ([u64; WORDS], u64) + Sync + Send + 'static) { + use loom::sync::atomic::{AtomicU64, Ordering}; + model(move || { + let latch = Arc::new(Latch::::new()); + let counter = Arc::new(AtomicU64::new(0)); + latch.publish([1; WORDS]); + let writer = { + let (latch, counter) = (latch.clone(), counter.clone()); + thread::spawn(move || { + latch.publish([2; WORDS]); + counter.store(2, Ordering::Release); + }) + }; + let (line, now) = read(&latch, &counter); + assert!(now < 2 || line == [2; WORDS], "a reading at {now} extrapolated the superseded line {line:?}"); + writer.join().unwrap(); + }); +} + +/// A counter read inside the section, after the line's words, is read again +/// with the line when a publication lands between them. +#[test] +fn a_counter_read_inside_the_section_never_extrapolates_a_superseded_line() { + superseded(|latch, counter| latch.read_with(|| counter.load(loom::sync::atomic::Ordering::Acquire))); +} + +/// What the clock read before: the counter after the section, which a reader +/// preempted between the two pairs with a line replaced meanwhile. +#[test] +#[should_panic(expected = "extrapolated the superseded line")] +fn a_counter_read_after_the_section_can_extrapolate_a_superseded_line() { + superseded(|latch, counter| (latch.read(), counter.load(loom::sync::atomic::Ordering::Acquire))); +} diff --git a/kernel/pure/clock/reference.rs b/kernel/pure/clock/reference.rs index e536269ce89..e532612c63e 100644 --- a/kernel/pure/clock/reference.rs +++ b/kernel/pure/clock/reference.rs @@ -3,11 +3,13 @@ //! measure and how far it can be off, and whether the counter still keeps the //! rate a clock was given. //! -//! Every bound here is derived from the readings themselves: the counter -//! readings either side of a reference reading bracket the moment it latched, -//! and the reference's tick bounds where in that tick it was. Nothing is a -//! statistic, so a bound that fails is a counter, a reference or a read that -//! did not do what it says. +//! Every bound on a rate and a step is derived from the readings themselves: +//! the counter readings either side of a reference reading bracket the moment +//! it latched, and the reference's tick bounds where in that tick it was, so a +//! bound that fails is a counter, a reference or a read that did not do what +//! it says. The one estimate is a third reference read by its interrupts +//! ([`Edges`]): their least latency, and its crystal's drift carried across a +//! span, which only choose between a step and none ([`third_says`]). use toyos_abi::clock::MULT_SHIFT; @@ -462,22 +464,32 @@ impl Edges { /// `from` to after `to`: from the least-latency of the [`EDGES_EACH_SIDE`] /// edges before `from` to that of those after `to`. An edge reads late by /// its interrupt's latency and never early, so the least on each side is - /// the nearest its edge. `None` without an edge either side. + /// the nearest its edge. Less the RTC crystal's own drift from the + /// counter, where edges at least [`DRIFT_BASE_NS`] before give it: the + /// least-latency of the oldest kept to that before `from`, carried on + /// across the span. `None` without an edge either side. pub fn apart_across(&self, from: u64, to: u64, mult: u64) -> Option { let kept = &self.kept[..self.len]; - let before = kept.iter().rev().filter(|&&(_, at)| at <= from).take(EDGES_EACH_SIDE); - let after = kept.iter().filter(|&&(_, at)| at >= to).take(EDGES_EACH_SIDE); let &(seen_ref, at_ref) = kept.iter().rev().find(|&&(_, at)| at <= from)?; let offset = |&(seen, at): &(u64, u64)| { let ns = ((at as i128 - at_ref as i128) * mult as i128) >> MULT_SHIFT; - ns - (seen as i128 - seen_ref as i128) * 1_000_000_000 + (ns - (seen as i128 - seen_ref as i128) * 1_000_000_000, at) }; - let least_before = before.map(offset).min()?; - let least_after = after.map(offset).min()?; - Some((least_after - least_before) as i64) + let least = |edges: &mut dyn Iterator| edges.map(offset).min_by_key(|&(ns, _)| ns); + let before = least(&mut kept.iter().rev().filter(|&&(_, at)| at <= from).take(EDGES_EACH_SIDE))?; + let after = least(&mut kept.iter().filter(|&&(_, at)| at >= to).take(EDGES_EACH_SIDE))?; + let base_ticks = ((DRIFT_BASE_NS as u128) << MULT_SHIFT) / mult as u128; + let older = least(&mut kept.iter().filter(|&&(_, at)| (at as u128) + base_ticks <= before.1 as u128).take(EDGES_EACH_SIDE)); + let drift = older.map_or(0, |older| (before.0 - older.0) * (after.1 as i128 - before.1 as i128) / (before.1 as i128 - older.1 as i128)); + Some((after.0 - before.0 - drift) as i64) } } +/// How far back edges must reach for [`Edges::apart_across`] to measure the +/// RTC crystal's drift from the counter: over this, a few microseconds of +/// latency are a part per million. +pub const DRIFT_BASE_NS: u64 = 4_000_000_000; + impl Default for Edges { fn default() -> Self { Self::new() @@ -525,7 +537,8 @@ pub const fn respond(step_ns: i64, slewed_before: u32) -> Response { /// The most a slew moves the clock's rate: 500 ppm, the frequency tolerance /// NTP disciplines a clock within (RFC 5905, Appendix A.1.1, `MAXFREQ`), so /// a reader that tolerates an NTP-disciplined clock tolerates this one. A -/// step of `d` is slewed away in `2000 d`: 1.7 ms in 3.4 s. +/// step of `d` is slewed at this rate for `2000 d` less the last millisecond +/// or so, which halves each period: 1.7 ms in about 12 s. pub const SLEW_MOST_PPB: i64 = 500_000; /// A debt this small is paid. const SLEW_SETTLED_NS: u64 = 1_000; @@ -557,13 +570,15 @@ impl Slew { self.debt_ns -= (line(kept) - line(ran)) as i64; } - /// The rate to run at for the next `period_ns`: the debt paid over that - /// period, at most [`SLEW_MOST_PPB`] from `kept`, and `kept` once settled. + /// The rate to run at until the next sample, due in `period_ns`: the debt + /// paid over two periods, at most [`SLEW_MOST_PPB`] from `kept`, and `kept` + /// once settled. A sample up to a period late pays no more than the debt; + /// a later one pays more, which the next pays back. pub fn mult(&self, kept: u64, period_ns: u64) -> u64 { if self.settled() { return kept; } - let ppb = (self.debt_ns as i128 * 1_000_000_000 / period_ns as i128).clamp(-(SLEW_MOST_PPB as i128), SLEW_MOST_PPB as i128); + let ppb = (self.debt_ns as i128 * 1_000_000_000 / (2 * period_ns as i128)).clamp(-(SLEW_MOST_PPB as i128), SLEW_MOST_PPB as i128); (kept as i128 - kept as i128 * ppb / 1_000_000_000) as u64 } } @@ -910,7 +925,7 @@ mod tests { let mut slew = Slew::new(debt); let gaps = [1_000_000_000u64, 1_400_000_000, 800_000_000, 1_200_000_000, 1_000_000_000, 1_300_000_000]; let mut elapsed = 0u64; - for gap in gaps.iter().cycle().take(32) { + for gap in gaps.iter().cycle().take(64) { if slew.settled() { break; } @@ -921,6 +936,45 @@ mod tests { elapsed += gap; } assert!(slew.settled(), "{} ns left", slew.debt_ns()); - assert!(elapsed <= 2_000 * debt as u64 + 2 * 1_400_000_000, "took {elapsed} ns"); + // At the most rate down to two periods' worth, then halving at worst + // to six tenths a period, from a millisecond to the settled microsecond. + assert!(elapsed <= 2_000 * debt as u64 + 14 * 1_400_000_000, "took {elapsed} ns"); + } + + /// A sample a period late pays the debt and no more; one ten periods late + /// pays more, and the next runs the clock faster to pay it back. + #[test] + fn a_late_sample_does_not_overpay_and_a_later_one_is_paid_back() { + let kept = toyos_abi::clock::Line::mult_of_hz(1_000_000_000); + let period = 1_000_000_000u64; + let mut late = Slew::new(300_000); + late.ran(2 * period, kept, late.mult(kept, period)); + assert!(late.debt_ns() >= 0 && late.settled(), "{} ns left", late.debt_ns()); + let mut later = Slew::new(300_000); + later.ran(10 * period, kept, later.mult(kept, period)); + assert!(later.debt_ns() < 0); + assert!(later.mult(kept, period) > kept, "an overpaid slew runs the clock faster"); + } + + /// An RTC crystal 60 ppm off the counter drifts 420 µs over the edges + /// either side of a span, about a quarter of a 1.7 ms step: corrected by + /// its own earlier edges, it reads the step and reads a stall as none. + #[test] + fn the_rtcs_crystal_drift_is_taken_out() { + let drifting = |step: u64| { + let mut edges = Edges::new(); + for n in 0..EDGES_KEPT as u64 { + let edge = 10_000_000_000 + n * 1_000_060_000; + let latency = [3_000, 650_000, 20_000][n as usize % 3]; + edges.push(n + 1, edge + latency + if n >= 11 { step } else { 0 }); + } + edges + }; + let (from, to) = (20_900_000_000, 21_900_000_000); + let apart = drifting(1_700_000).apart_across(from, to + 1_700_000, NS).expect("edges either side"); + assert_eq!(third_says(1_700_000, Some(apart)), Third::Counter, "read {apart} ns"); + let apart = drifting(0).apart_across(from, to, NS).expect("edges either side"); + assert!(apart.unsigned_abs() <= 30_000, "a stall read {apart} ns through the drift"); + assert_eq!(third_says(1_700_000, Some(apart)), Third::References); } } diff --git a/kernel/src/arch/aarch64/cpu.rs b/kernel/src/arch/aarch64/cpu.rs index 37738467f76..41217f27ed6 100644 --- a/kernel/src/arch/aarch64/cpu.rs +++ b/kernel/src/arch/aarch64/cpu.rs @@ -13,6 +13,15 @@ pub fn counter() -> u64 { count } +/// [`counter`] ordered after every earlier instruction and before every later +/// one: the read the clock takes inside its line's read section. +pub fn counter_ordered() -> u64 { + let count: u64; + // SAFETY: as `counter`'s; the second `ISB` touches nothing either. + unsafe { asm!("isb", "mrs {}, cntvct_el0", "isb", out(reg) count, options(nomem, nostack, preserves_flags)) }; + count +} + /// The counter's frequency as firmware states it in `CNTFRQ_EL0`, which the /// Arm ARM makes firmware's duty to program. Zero states nothing. pub fn stated_counter_hz() -> Option { diff --git a/kernel/src/arch/aarch64/mod.rs b/kernel/src/arch/aarch64/mod.rs index f45fa171378..cf8fed2ad31 100644 --- a/kernel/src/arch/aarch64/mod.rs +++ b/kernel/src/arch/aarch64/mod.rs @@ -11,6 +11,15 @@ macro_rules! owed { }; } +/// The interrupt sources only this architecture counts: none past the +/// census's own (`irq_census`). +macro_rules! census_sources { + ($list:ident) => { + $list! {} + }; +} +pub(crate) use census_sources; + pub mod acpi_mode; pub mod barrier; pub mod boot; diff --git a/kernel/src/arch/x86_64/clocksource.rs b/kernel/src/arch/x86_64/clocksource.rs index 8d3207f8b59..4d2b92eca5c 100644 --- a/kernel/src/arch/x86_64/clocksource.rs +++ b/kernel/src/arch/x86_64/clocksource.rs @@ -112,8 +112,14 @@ impl Hpet { /// Where the counter should read now, from `last` and the rate the clock /// keeps: what a 32-bit counter's wraps are counted against. fn near(&self, last: Sample, mult: u64) -> u64 { - let nanos = Line { counter: 0, stamp: 0, mult }.nanos(cpu::counter().saturating_sub(last.before)); - last.reference + (u128::from(nanos) * 1_000_000 / u128::from(self.reference.period_fs)) as u64 + self.ahead_of(last.reference, last.before, cpu::counter(), mult) + } + + /// Where the counter should read at the TSC's `now`, from its reading + /// `reading` at the TSC's `then`. + fn ahead_of(&self, reading: u64, then: u64, now: u64, mult: u64) -> u64 { + let nanos = Line { counter: 0, stamp: 0, mult }.nanos(now.saturating_sub(then)); + reading + (u128::from(nanos) * 1_000_000 / u128::from(self.reference.period_fs)) as u64 } } @@ -220,20 +226,25 @@ static HELD: Lock> = Lock::new(None); /// clock counts on from by the HPET. Published by the sampler. static LAST_HELD: crate::seqlock::Published<3> = crate::seqlock::Published::new(); -/// The HPET and the clock where the TSC was demoted, and the latest reading -/// the kernel has answered since, which none after reads before. +/// Where the TSC was demoted: the HPET, its reading then and the clock by it, +/// and its last reading since, which the next extends from. struct Demoted { hpet: Hpet, at_hpet: u64, at_stamp: u64, - tsc: u64, + last_tsc: u64, + last_hpet: u64, mult: u64, } static DEMOTED: Lock> = Lock::new(None); static IS_DEMOTED: AtomicBool = AtomicBool::new(false); +/// The latest reading the kernel has answered a process since the demotion, +/// which no later answer reads before. static DEMOTED_LATEST: AtomicU64 = AtomicU64::new(0); static DEMOTED_ANSWERED: AtomicBool = AtomicBool::new(false); +/// The rate the watch keeps, for a demotion on any CPU to retune the line to. +static KEPT_MULT: AtomicU64 = AtomicU64::new(0); /// Whether CPUID states `IA32_TSC_ADJUST`, read once on the BSP. static TSC_ADJUST: AtomicBool = AtomicBool::new(false); @@ -299,6 +310,7 @@ pub fn start(hpet_base: u64, pm_timer: Result> = Lock::new(None); -/// Now as a log line's time, for a process: the TSC's clock, or the HPET's -/// count on from where the TSC was demoted. +/// Now as a log line's time, for a process: the TSC's clock, or, once +/// demoted, the HPET's count on from where the TSC was demoted. The counter is +/// read before the mark, as a page reader reads it. pub fn stamp_now() -> u64 { - if !IS_DEMOTED.load(Ordering::Acquire) { - return crate::clock::stamp_at(cpu::counter()); + let on_line = crate::clock::stamp_at(cpu::counter_ordered()); + if !IS_DEMOTED.load(Ordering::SeqCst) { + return on_line; } let stamp = { - let demoted = DEMOTED.lock(); - let d = demoted.as_ref().expect("demoted before the flag"); - let elapsed = Line { counter: 0, stamp: 0, mult: d.mult }.nanos(cpu::counter().saturating_sub(d.tsc)); - let near = d.at_hpet + (u128::from(elapsed) * 1_000_000 / u128::from(d.hpet.reference.period_fs)) as u64; - d.at_stamp + d.hpet.reference.ns_of(d.hpet.read(near).saturating_sub(d.at_hpet)) + let mut demoted = DEMOTED.lock(); + let d = demoted.as_mut().expect("demoted before the mark"); + let now = cpu::counter_ordered(); + let reading = d.hpet.read(d.hpet.ahead_of(d.last_hpet, d.last_tsc, now, d.mult)); + (d.last_tsc, d.last_hpet) = (now, reading); + d.at_stamp + d.hpet.reference.ns_of(reading.saturating_sub(d.at_hpet)) }; let answered = DEMOTED_LATEST.fetch_max(stamp, Ordering::AcqRel).max(stamp); if !DEMOTED_ANSWERED.swap(true, Ordering::Relaxed) { @@ -593,10 +618,6 @@ fn sample(held: &mut Held, now_ns: u64) { if let (Some(timer), Some((_, to))) = (held.pm.as_mut(), pm) { timer.last = to; } - if IS_DEMOTED.load(Ordering::Acquire) { - let _ = held.watch.take(hpet, false); - return; - } match held.watch.take(hpet, true) { Ok(report) => { LAST_HELD.publish([hpet.before, hpet.reference, crate::clock::stamp_at(hpet.before)]); @@ -641,9 +662,8 @@ fn sample(held: &mut Held, now_ns: u64) { held.pending = None; settle(held, pending); } - if !IS_DEMOTED.load(Ordering::Acquire) { - slew(held); - } + KEPT_MULT.store(held.watch.kept().mult, Ordering::Relaxed); + slew(held); } /// What the RTC says of a pending step, and what the clock does about it. @@ -665,13 +685,16 @@ fn settle(held: &mut Held, pending: Pending) { Third::Counter => "confirmed by the RTC", Third::Unsure => "unconfirmed by the RTC", }; + if IS_DEMOTED.load(Ordering::Acquire) { + return log!("clock: TSC STEP: the demoted TSC stepped {step} ns against the HPET and the PM timer, {confirmed} ({rtc}); not slewed"); + } match reference::respond(step, held.slewed) { Response::Slew => { held.slewed += 1; held.slew = Some(Slewing { slew: Slew::new(step), since: cpu::counter(), ran: crate::clock::mult(), kept: kept.mult }); log!( "clock: TSC STEP: the TSC stepped {step} ns forward against the HPET and the PM timer, {confirmed} \ - ({rtc}); the clock slews it away at most {} ppm, in about {} ms", + ({rtc}); the clock slews it away at most {} ppm, most of it in about {} ms", reference::SLEW_MOST_PPB / 1_000, u128::from(step.unsigned_abs()) * u128::from(reference::PPB) / reference::SLEW_MOST_PPB as u128 / 1_000_000, ); @@ -689,6 +712,9 @@ fn settle(held: &mut Held, pending: Pending) { /// none, the kept rate where the watch retuned it. fn slew(held: &mut Held) { let kept = held.watch.kept().mult; + if IS_DEMOTED.load(Ordering::Acquire) { + held.slew = None; + } let Some(mut slewing) = held.slew else { if crate::clock::mult() != kept { crate::clock::retune(cpu::counter(), kept); diff --git a/kernel/src/arch/x86_64/cpu.rs b/kernel/src/arch/x86_64/cpu.rs index f0f3d6d94be..0a43ec0c83a 100644 --- a/kernel/src/arch/x86_64/cpu.rs +++ b/kernel/src/arch/x86_64/cpu.rs @@ -488,6 +488,12 @@ pub fn counter() -> u64 { rdtsc() } +/// [`counter`] ordered after every earlier instruction and before every later +/// one: the read the clock takes inside its line's read section. +pub fn counter_ordered() -> u64 { + rdtsc_ordered() +} + /// This function's caller's frame pointer: `rbp`, which `-Cforce-frame-pointers=yes` makes one. #[inline(always)] pub fn frame_pointer() -> u64 { diff --git a/kernel/src/arch/x86_64/mod.rs b/kernel/src/arch/x86_64/mod.rs index 6c744b9c6fd..b670b86b46c 100644 --- a/kernel/src/arch/x86_64/mod.rs +++ b/kernel/src/arch/x86_64/mod.rs @@ -9,6 +9,18 @@ //! AMD-Vi. Generic code reaches each through the concept it serves //! (`watchdog`, `iommu_unit`, …), never by its name. +/// The interrupt sources only this architecture counts, handed to the +/// census's list (`irq_census`). +macro_rules! census_sources { + ($list:ident) => { + $list! { + /// Vector 0x26, the RTC's update-ended interrupt, once a second. + Rtc => "rtc", + } + }; +} +pub(crate) use census_sources; + pub mod acpi_mode; pub mod amdvi; pub mod apic; diff --git a/kernel/src/clock.rs b/kernel/src/clock.rs index 27a2d1c76f6..c1265d690cb 100644 --- a/kernel/src/clock.rs +++ b/kernel/src/clock.rs @@ -105,14 +105,21 @@ fn clock() -> Option { lines().and_then(ClockPage::read) } +/// The clock and the counter read inside its line's read section, so a line +/// replaced while this CPU was interrupted is read again. The kernel's own +/// clock stays on the counter whether or not the page is demoted. +fn now_on_line() -> Option<(Clock, u64)> { + lines().and_then(|page| page.read_now(cpu::counter_ordered)).map(|(clock, now, _)| (clock, now)) +} + /// Now as a log line's time: nanoseconds since the counter's zero — power-on, /// or the reset since — at the clock's rate, and before [`set_counter`] at the /// rate the CPU states. `None` where it states none and the clock has not /// started: no rate reads the counter yet. Never panics, as /// [`nanos_since_boot`]. pub fn stamp() -> Option { - if let Some(clock) = clock() { - return Some(LogStamp::since_zero(clock.stamp_of(cpu::counter()))); + if let Some((clock, now)) = now_on_line() { + return Some(LogStamp::since_zero(clock.stamp_of(now))); } let stated = STATED_MULT.load(Relaxed); (stated != 0).then(|| LogStamp::since_zero(Line { counter: 0, stamp: 0, mult: stated }.nanos(cpu::counter()))) @@ -140,7 +147,7 @@ pub fn calibrated() -> bool { /// reads it from inside a bracket where panicking would reenter the log. /// Zero before [`set_counter`], and a CPU whose counter trails boot reads boot. pub fn nanos_since_boot() -> u64 { - clock().map_or(0, |clock| clock.nanos_since_boot(cpu::counter())) + now_on_line().map_or(0, |(clock, now)| clock.nanos_since_boot(now)) } /// The same reading as an [`Instant`], the type arithmetic on it is allowed in. diff --git a/kernel/src/irq_census.rs b/kernel/src/irq_census.rs index 9bae6a8e99b..cf268b77784 100644 --- a/kernel/src/irq_census.rs +++ b/kernel/src/irq_census.rs @@ -8,47 +8,51 @@ use core::sync::atomic::{AtomicU64, Ordering}; use crate::arch::percpu; -/// One variant per interrupt source this kernel counts; order matches `arch::idt`'s vector table. -/// Adding a device vector means adding a variant here and one `irq_took!` at its handler. -/// A variant nothing counts does not compile: `irq_took!` is the only reference, so `-D dead-code` refuses an unconstructed one. -#[derive(Clone, Copy, PartialEq, Eq, Debug)] -#[repr(usize)] -pub enum Source { - /// Vector 0x20, this CPU's LAPIC one-shot: an expiry and nothing else. - Timer, - /// Vector 0xFC, `irqchip::kick_cpu`'s IPI. - Kick, - /// Vectors 0x28-0x2C: the MSI-X of a PCI function a process drives, and - /// the lines of an ISA function one does. One count for all five: which - /// claim an interrupt belonged to is the claim's own record, and the census - /// is about this machine's interrupt routing. - UserDev, - /// Vector 0x25, the remapping unit's fault event. - DmaFault, - /// Vector 0x26, x86-64's RTC update-ended interrupt, once a second. - #[cfg_attr(not(target_arch = "x86_64"), allow(dead_code))] - Rtc, - /// Vector 0xFE, the TLB shootdown IPI. - Tlb, - /// Vector 0x02, which `crate::hardlockup` samples a CPU with. - Nmi, - /// Vector 0xFF, the local APIC's spurious vector. - /// A non-zero count on a machine that staged nothing is an interrupt-routing defect. - Spurious, - /// Every vector no `idt_vectors!` row claims — `arch::trap::unclaimed`. - /// A non-zero count a boot staged nothing for is a routing defect the gate kept off `#DF`. - Unclaimed, -} - -impl Source { - pub const COUNT: usize = 9; +/// The census's sources: these, and between the device vectors and the IPIs, +/// those the architecture names (`arch::census_sources!`). +macro_rules! sources { + ($($(#[$meta:meta])* $arch:ident => $name:literal,)*) => { + /// One variant per interrupt source this kernel counts; order matches `arch::idt`'s vector table. + /// Adding a device vector means adding a variant here, or the architecture's, and one `irq_took!` at its handler. + /// A variant nothing counts does not compile: `irq_took!` is the only reference, so `-D dead-code` refuses an unconstructed one. + #[derive(Clone, Copy, PartialEq, Eq, Debug)] + #[repr(usize)] + pub enum Source { + /// Vector 0x20, this CPU's LAPIC one-shot: an expiry and nothing else. + Timer, + /// Vector 0xFC, `irqchip::kick_cpu`'s IPI. + Kick, + /// Vectors 0x28-0x2C: the MSI-X of a PCI function a process drives, and + /// the lines of an ISA function one does. One count for all five: which + /// claim an interrupt belonged to is the claim's own record, and the census + /// is about this machine's interrupt routing. + UserDev, + /// Vector 0x25, the remapping unit's fault event. + DmaFault, + $($(#[$meta])* $arch,)* + /// Vector 0xFE, the TLB shootdown IPI. + Tlb, + /// Vector 0x02, which `crate::hardlockup` samples a CPU with. + Nmi, + /// Vector 0xFF, the local APIC's spurious vector. + /// A non-zero count on a machine that staged nothing is an interrupt-routing defect. + Spurious, + /// Every vector no `idt_vectors!` row claims — `arch::trap::unclaimed`. + /// A non-zero count a boot staged nothing for is a routing defect the gate kept off `#DF`. + Unclaimed, + } - /// Order `tests/toyos.rs`'s `irq_census_conservation` parses back; must match variant order. - pub const NAMES: [&'static str; Self::COUNT] = [ - "timer", "kick", "userdev", "dmafault", "rtc", "tlb", "nmi", "spurious", "unclaimed", - ]; + impl Source { + /// Order `tests/toyos.rs`'s `irq_census_conservation` parses back; must match variant order. + pub const NAMES: &'static [&'static str] = + &["timer", "kick", "userdev", "dmafault", $($name,)* "tlb", "nmi", "spurious", "unclaimed"]; + pub const COUNT: usize = Self::NAMES.len(); + } + }; } +crate::arch::census_sources!(sources); + /// Every source's count summed, bar the NMI's. fn sum_bar_nmi(counts: &[u64; Source::COUNT]) -> u64 { counts.iter().sum::() - counts[Source::Nmi as usize] diff --git a/sdk/abi/src/arch/aarch64.rs b/sdk/abi/src/arch/aarch64.rs index 2f2c6fe4dca..1ac6ffb64b7 100644 --- a/sdk/abi/src/arch/aarch64.rs +++ b/sdk/abi/src/arch/aarch64.rs @@ -18,14 +18,14 @@ pub fn current_tid() -> Tid { } /// The counter the clock page's line describes: the generic timer's -/// virtual count, which EL0 may read, read after every earlier instruction, as -/// the x86-64 one is. +/// virtual count, which EL0 may read, read after every earlier instruction and +/// before any later one, as the x86-64 one is. #[inline] pub fn counter() -> u64 { let now: u64; - // SAFETY: an `isb` and a read of `CNTVCT_EL0` into a register; neither + // SAFETY: two `isb`s and a read of `CNTVCT_EL0` into a register; none // touches memory. - unsafe { core::arch::asm!("isb", "mrs {now}, cntvct_el0", now = out(reg) now, options(nomem, nostack)) }; + unsafe { core::arch::asm!("isb", "mrs {now}, cntvct_el0", "isb", now = out(reg) now, options(nomem, nostack)) }; now } diff --git a/sdk/abi/src/arch/x86_64.rs b/sdk/abi/src/arch/x86_64.rs index 6d5e15e15ad..39e7fcfbef1 100644 --- a/sdk/abi/src/arch/x86_64.rs +++ b/sdk/abi/src/arch/x86_64.rs @@ -26,7 +26,11 @@ pub fn current_tid() -> Tid { /// The counter the clock page's line describes: the time-stamp counter, /// read after every earlier load has completed, so a stamp taken after reading -/// another writer's record cannot be older than that record's. +/// another writer's record cannot be older than that record's, and before any +/// later instruction starts, so a load after it cannot have been answered +/// before it (SDM Vol. 2B, RDTSC: "If software requires RDTSC to be executed +/// prior to execution of any subsequent instruction (including any memory +/// accesses), it can execute the sequence LFENCE immediately after RDTSC"). #[inline] pub fn counter() -> u64 { // SAFETY: `lfence` has no operands; `rdtsc` is unprivileged while CR4.TSD @@ -34,7 +38,9 @@ pub fn counter() -> u64 { // every CPU. unsafe { core::arch::x86_64::_mm_lfence(); - core::arch::x86_64::_rdtsc() + let now = core::arch::x86_64::_rdtsc(); + core::arch::x86_64::_mm_lfence(); + now } } diff --git a/sdk/abi/src/clock.rs b/sdk/abi/src/clock.rs index 47c02b1ddeb..850ea9b8d93 100644 --- a/sdk/abi/src/clock.rs +++ b/sdk/abi/src/clock.rs @@ -9,13 +9,17 @@ //! new one where the old one stands, so the clock never steps. [`page`] reads //! the line in force through the page's [`latch`], which the kernel's own //! clock reads too, so a stamp taken here and one taken by the kernel's log -//! order against each other. +//! order against each other until the kernel demotes the page. //! -//! **A demoted page sends every reading to the kernel.** Where the counter -//! steps again after a slewed step, or steps back, the kernel marks the page -//! demoted ([`ClockPage::demoted`]); [`nanos_since_boot`] and [`stamp_ns`] -//! then ask [`crate::syscall::clock_now`], which reads the reference instead. -//! The line stays, for a recorded counter reading. +//! **A demoted page sends every reading to the kernel.** The kernel demotes +//! the page when it stops trusting the counter ([`ClockPage::demoted`]); +//! [`nanos_since_boot`] and [`stamp_ns`] then ask +//! [`crate::syscall::clock_now`], which reads the reference instead. The line +//! stays, for a recorded counter reading. +//! +//! **A reading takes the counter inside the latch's read section**, and the +//! page's demoted word after the counter, so a line replaced or a demotion +//! made while the reader was preempted sends it round again or to the kernel. //! //! **Every word of the page is atomic.** The kernel writes it while processes //! read it, so it is reached as atomics and never as plain memory. @@ -155,6 +159,20 @@ impl ClockPage { self.demoted.load(Ordering::Acquire) != 0 } + /// The clock, a counter reading `counter` took inside the line's read + /// section, and whether the page was demoted after it; `None` on a page + /// that carries none. + pub fn read_now(&self, counter: impl Fn() -> u64) -> Option<(Clock, u64, bool)> { + if self.magic.load(Ordering::Acquire) != CLOCK_MAGIC { + return None; + } + let (words, (now, demoted)) = self.line.read_with(|| { + let now = counter(); + (now, self.demoted()) + }); + Some((Clock { line: Line::of_words(words), stamp_at_boot: self.stamp_at_boot.load(Ordering::Relaxed) }, now, demoted)) + } + /// The clock now, or `None` on a page that carries none. pub fn read(&self) -> Option { if self.magic.load(Ordering::Acquire) != CLOCK_MAGIC { @@ -171,12 +189,11 @@ impl ClockPage { /// # Panics /// As [`page`]. pub fn nanos_since_boot() -> u64 { - let page = mapped(); - let clock = read(page); - if page.demoted() { + let (clock, now, demoted) = read_now(); + if demoted { return crate::syscall::clock_now().saturating_sub(clock.stamp_at_boot); } - clock.nanos_since_boot(crate::arch::counter()) + clock.nanos_since_boot(now) } /// Now as a log line's time: nanoseconds since the counter's zero, the kernel @@ -186,11 +203,17 @@ pub fn nanos_since_boot() -> u64 { /// # Panics /// As [`page`]. pub fn stamp_ns() -> u64 { - let page = mapped(); - if page.demoted() { + let (clock, now, demoted) = read_now(); + if demoted { return crate::syscall::clock_now(); } - read(page).stamp_of(crate::arch::counter()) + clock.stamp_of(now) +} + +fn read_now() -> (Clock, u64, bool) { + mapped() + .read_now(crate::arch::counter) + .unwrap_or_else(|| panic!("the clock page at {CLOCK_PAGE:#x} carries no clock")) } /// The clock, off this process's page: its line, for a counter reading. diff --git a/sdk/abi/src/clock/latch.rs b/sdk/abi/src/clock/latch.rs index da034f75736..4ffd043946a 100644 --- a/sdk/abi/src/clock/latch.rs +++ b/sdk/abi/src/clock/latch.rs @@ -53,15 +53,26 @@ impl Latch { /// The value one publication left, whole. pub fn read(&self) -> [u64; N] { + self.read_with(|| ()).0 + } + + /// The value one publication left, whole, and what `inside` read while + /// that publication was in force: a publication that lands between the + /// value's loads and the recheck after `inside` sends the read round + /// again. `inside` orders whatever it reads that is not memory, such as a + /// counter, after the loads before it and before the recheck after it. + pub fn read_with(&self, mut inside: impl FnMut() -> T) -> ([u64; N], T) { loop { let seq = self.seq.load(Ordering::Acquire); let copy = &self.copies[(seq & 1) as usize]; let words = core::array::from_fn(|i| copy[i].load(Ordering::Relaxed)); - // Orders the word loads before the recheck: a word a later - // publication wrote is one whose move the recheck sees. + let read = inside(); + // Orders the word loads, and `inside`'s, before the recheck: a + // word a later publication wrote is one whose move the recheck + // sees. fence(Ordering::Acquire); if self.seq.load(Ordering::Relaxed) == seq { - return words; + return (words, read); } } } diff --git a/tests/checks.rs b/tests/checks.rs index ba2968973e5..389441b4b8d 100644 --- a/tests/checks.rs +++ b/tests/checks.rs @@ -575,6 +575,15 @@ mod checks { Ok(()) } + /// A slewed step passes the clock's rows and a demotion fails them. + #[test] + fn a_demoted_clock_fails_its_rows() { + let slewed = "[ 20.778 cpu12 kernel] clock: TSC STEP: the TSC stepped 1695132 ns forward against the HPET\n"; + assert_eq!(clock_kept_its_counter(slewed), Ok(())); + let demoted = format!("{slewed}[ 33.252 cpu1 kernel] clock: DEMOTED: the TSC stepped 10000110 ns\n"); + assert!(clock_kept_its_counter(&demoted).is_err()); + } + /// [`irq_census`] over two censuses, X's from cpu2 and Y's from cpu3, whose /// lines are in no read order: a CPU's census is the largest count each /// source reaches on any of its lines, and the largest issuer total bounds diff --git a/tests/toyos.rs b/tests/toyos.rs index 7c9365074cf..b6944f93a7f 100644 --- a/tests/toyos.rs +++ b/tests/toyos.rs @@ -7454,6 +7454,7 @@ fn number_between(log: &str, head: &str, tail: &str) -> Result { /// prints the widest. **QEMU cannot refute it**: every guest TSC is /// synthesised from one host clock, which is why it runs on metal. fn smp_roster_and_tsc_trail(log: &str, cpus: u32) -> Result<(), String> { + clock_kept_its_counter(log)?; let online = number_between(log, "SMP: ", " of ")?; let named = number_between(log, " of ", " MADT cpus online")?; if online != u64::from(cpus) || named != u64::from(cpus) { @@ -7631,6 +7632,7 @@ fn timer_calibration(log: &str) -> Result<(), String> { /// measured with its bound, which on the bench is a rate the clock can keep at /// all. Its bound is held against minutes by [`clock_holds_over_minutes`]. fn tsc_rate_at_boot(log: &str) -> Result<(), String> { + clock_kept_its_counter(log)?; let boot = boot_rate(log)?; if boot.millihertz == 0 { return Err("the TSC was started at no rate at all".to_string()); @@ -7681,15 +7683,30 @@ fn clock_report(line: &str) -> Option { }) } +/// Whether the boot's clock kept the TSC: every step it slewed, every span it +/// left out and every step it waited on the RTC for is printed, and a +/// demotion fails the row, since a demoted boot's processes read the HPET. +fn clock_kept_its_counter(log: &str) -> Result<(), String> { + const SAID: [&str; 3] = ["clock: TSC STEP: ", "left out", "the RTC's edges after it will say which moved"]; + for line in log.lines().filter(|line| SAID.iter().any(|said| line.contains(said))) { + eprintln!(" [clock] {}", line.trim()); + } + match log.lines().find(|line| line.contains("clock: DEMOTED")) { + Some(demoted) => Err(format!("the clock demoted the TSC: {}", demoted.trim())), + None => Ok(()), + } +} + /// Parts per billion `a` is from `b`. fn ppb_apart(a: u64, b: u64) -> u64 { (u128::from(a.abs_diff(b)) * 1_000_000_000 / u128::from(b)) as u64 } -/// **The clock over minutes.** The kernel holds every CPU's TSC to the HPET -/// once a second and panics on a step or a drift, so a boot that ran its list -/// to the end held; what this row reads is whether the rate the boot started at -/// was within the bound it derived, against the rate the HPET measured over +/// **The clock over minutes.** The kernel holds the TSC to the HPET once a +/// second, panics on a drift and demotes the TSC on a step it does not slew, so +/// a boot that ran its list to the end without a demotion held; what this row +/// reads is whether the rate the boot started at was within the bound it +/// derived, against the rate the HPET measured over /// the longest span the boot reached, which must be two minutes or more. A /// stated rate is CPUID 15H's, and against it the HPET's own 0.05% is all a /// document allows (IA-PC HPET Specification 1.0a, §2.2), so its distance is @@ -7697,6 +7714,7 @@ fn ppb_apart(a: u64, b: u64) -> u64 { fn clock_holds_over_minutes(boot: &metal::Readback) -> Result<(), String> { let kernel = boot.kernel(); let text = kernel.text(); + clock_kept_its_counter(text)?; let started = boot_rate(text)?; let longest = text .lines() From d739b07434beb0b5369a7c8912a9924d21dc5d9a Mon Sep 17 00:00:00 2001 From: japabu Date: Sun, 1 Nov 2026 03:56:11 +0100 Subject: [PATCH 08/11] tests: the clock rows print a span's left-out record, not every report's count of them `clock_kept_its_counter` matched "left out", which every report line carries ("0 spans left out"), so each clock row printed every report. It now matches the records the sampler writes when it leaves a span out ("so this span is left out", "so the span was left out"), beside the TSC STEP lines and the lines that wait on the RTC. Seen judging the T14's r8 readbacks. Co-Authored-By: Claude Opus 5.5 --- tests/toyos.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/toyos.rs b/tests/toyos.rs index b6944f93a7f..a8fc9aaca0d 100644 --- a/tests/toyos.rs +++ b/tests/toyos.rs @@ -7687,7 +7687,7 @@ fn clock_report(line: &str) -> Option { /// left out and every step it waited on the RTC for is printed, and a /// demotion fails the row, since a demoted boot's processes read the HPET. fn clock_kept_its_counter(log: &str) -> Result<(), String> { - const SAID: [&str; 3] = ["clock: TSC STEP: ", "left out", "the RTC's edges after it will say which moved"]; + const SAID: [&str; 4] = ["clock: TSC STEP: ", "so this span is left out", "so the span was left out", "the RTC's edges after it will say which moved"]; for line in log.lines().filter(|line| SAID.iter().any(|said| line.contains(said))) { eprintln!(" [clock] {}", line.trim()); } From f5a102ed965fd413854828ae97c11ceefb0b183b Mon Sep 17 00:00:00 2001 From: japabu Date: Sun, 1 Nov 2026 11:06:11 +0100 Subject: [PATCH 09/11] review r3: a demotion hands over at every CPU's clock, a drift base never reaches across a step, and an AP's proven offset demotes Fixes review r3's findings. - **Cross-CPU handover.** `demote` floored processes' time at the demoting CPU's clock after the mark. A reader on a CPU whose TSC leads, the very case a demotion is for, could have read further ahead, and a later `SYS_CLOCK_NOW` then read behind it. Now, once the page is marked, a round (`HANDOVER`, a `shootdown::Shootdown`) asks every online CPU to floor `DEMOTED_LATEST` at its own clock. Each CPU answers from its kick handler, or itself while it waits. A page reader that found the page unmarked read its counter before its own CPU's floor, so no answer reads before it, however far that TSC leads. - `SYS_CLOCK_NOW` waits for the round to end (`HANDED_OVER`), serving its own CPU while it waits. Every wait panics past `time::DEAF_CPU`. - The first `demote` claims `IS_DEMOTED` by a swap. A later one only logs and never waits, so no CPU waits on the round while holding what another needs. - Before the APs are released, no other CPU has run a process, and only the demoting CPU floors. - What is left: a reader that used a line the sampler replaced before the mark is ahead of its CPU's floor by that retune's bend, its change of rate over the CPU's lead. Every retune already bends that much. - **AArch64 `nomem`.** The ABI's `counter` and the kernel's `counter_ordered` were `nomem`, which let the compiler move the latch's loads across the counter read. Both drop it. - **The drift base across a step.** `Edges::apart_across` measured the RTC crystal's drift from edges at least 4 s before a span. Where an earlier TSC step lay between them, it read the step as drift and took it out of the next step's reading. `settle` now forgets the edges before a step it takes (`Edges::forget_before`). A host case shows a 1.7 ms step after a 4 ms one reads as itself only with the edges forgotten. - **An AP's proven offset demotes.** It panicked at AP start, while the runtime check demotes on the same fact. It now demotes. - **Comments.** `kernel/src/clock.rs`'s header no longer says the two clocks are one after a demotion. The demoted-offset case is added to `issues/a-demoted-tsc-keeps-the-kernels-clock-apart-from-processes.md`. Co-Authored-By: Claude Opus 5.5 --- ...-the-kernels-clock-apart-from-processes.md | 5 + kernel/pure/clock/reference.rs | 38 ++++++ kernel/src/arch/aarch64/cpu.rs | 5 +- kernel/src/arch/x86_64/clocksource.rs | 120 ++++++++++++++---- kernel/src/arch/x86_64/idt/kick.rs | 1 + kernel/src/arch/x86_64/mod.rs | 1 + kernel/src/clock.rs | 2 +- sdk/abi/src/arch/aarch64.rs | 5 +- 8 files changed, 144 insertions(+), 33 deletions(-) diff --git a/issues/a-demoted-tsc-keeps-the-kernels-clock-apart-from-processes.md b/issues/a-demoted-tsc-keeps-the-kernels-clock-apart-from-processes.md index b1cc1d515af..44128b7e3ae 100644 --- a/issues/a-demoted-tsc-keeps-the-kernels-clock-apart-from-processes.md +++ b/issues/a-demoted-tsc-keeps-the-kernels-clock-apart-from-processes.md @@ -25,6 +25,11 @@ After a forward second step, processes' time holds at the TSC's clock until the HPET's count reaches it, then follows the HPET. The kernel's clock stays ahead by the step. +Where one CPU's TSC is apart from the others' (written, found behind, or an +offset an AP's start proves), processes' time holds at the furthest-ahead +CPU's clock at the demotion until the HPET's count reaches it. The kernel's +clock on each CPU stays apart by that CPU's offset. + **Owner:** the clock. **Exit:** after a demotion, the kernel's clock follows the reference too, so diff --git a/kernel/pure/clock/reference.rs b/kernel/pure/clock/reference.rs index e532612c63e..30dadc2e6d3 100644 --- a/kernel/pure/clock/reference.rs +++ b/kernel/pure/clock/reference.rs @@ -455,6 +455,16 @@ impl Edges { self.len += 1; } + /// Drop the edges taken before the counter read `at`, where the counter + /// stepped: an edge from before a step and one from after it measure the + /// step, not the RTC crystal's drift, so neither a drift base nor a side + /// of a later span may reach across it. + pub fn forget_before(&mut self, at: u64) { + let kept = self.kept[..self.len].iter().position(|&(_, taken)| taken >= at).unwrap_or(self.len); + self.kept.copy_within(kept..self.len, 0); + self.len -= kept; + } + /// Whether [`EDGES_EACH_SIDE`] edges have come after the counter read `to`. pub fn complete_after(&self, to: u64) -> bool { self.kept[..self.len].iter().filter(|&&(_, at)| at >= to).count() >= EDGES_EACH_SIDE @@ -977,4 +987,32 @@ mod tests { assert!(apart.unsigned_abs() <= 30_000, "a stall read {apart} ns through the drift"); assert_eq!(third_says(1_700_000, Some(apart)), Third::References); } + + /// A 1.7 ms step after a 4 ms one reads as itself once the edges from + /// before the first are forgotten; with them, the drift base reaches + /// across the first step and takes a quarter of it for the crystal's drift. + #[test] + fn a_drift_base_never_reaches_across_a_step() { + let (first, second) = (4_000_000u64, 1_700_000u64); + let stepped = || { + let mut edges = Edges::new(); + for n in 0..EDGES_KEPT as u64 { + let edge = 10_000_000_000 + n * 1_000_000_000; + let latency = [3_000, 650_000, 20_000][n as usize % 3]; + let steps = if n >= 4 { first } else { 0 } + if n >= 13 { second } else { 0 }; + edges.push(n + 1, edge + latency + steps); + } + edges + }; + let first_taken_at = 14_000_000_000 + first; + let (from, to) = (22_500_000_000 + first, 22_900_000_000 + first + second); + + let mut forgotten = stepped(); + forgotten.forget_before(first_taken_at); + let apart = forgotten.apart_across(from, to, NS).expect("edges either side"); + assert_eq!(third_says(second as i64, Some(apart)), Third::Counter, "read {apart} ns"); + + let apart = stepped().apart_across(from, to, NS).expect("edges either side"); + assert_ne!(third_says(second as i64, Some(apart)), Third::Counter, "read {apart} ns across the first step"); + } } diff --git a/kernel/src/arch/aarch64/cpu.rs b/kernel/src/arch/aarch64/cpu.rs index 41217f27ed6..e79989281c3 100644 --- a/kernel/src/arch/aarch64/cpu.rs +++ b/kernel/src/arch/aarch64/cpu.rs @@ -17,8 +17,9 @@ pub fn counter() -> u64 { /// one: the read the clock takes inside its line's read section. pub fn counter_ordered() -> u64 { let count: u64; - // SAFETY: as `counter`'s; the second `ISB` touches nothing either. - unsafe { asm!("isb", "mrs {}, cntvct_el0", "isb", out(reg) count, options(nomem, nostack, preserves_flags)) }; + // SAFETY: as `counter`'s; the second `ISB` touches nothing either. Not + // `nomem`, which would let the compiler move the latch's loads across it. + unsafe { asm!("isb", "mrs {}, cntvct_el0", "isb", out(reg) count, options(nostack, preserves_flags)) }; count } diff --git a/kernel/src/arch/x86_64/clocksource.rs b/kernel/src/arch/x86_64/clocksource.rs index 4d2b92eca5c..5ac0f853b30 100644 --- a/kernel/src/arch/x86_64/clocksource.rs +++ b/kernel/src/arch/x86_64/clocksource.rs @@ -15,10 +15,10 @@ //! or the TSC; the RTC's edges, off its own 32.768 kHz crystal, choose: with //! the TSC, the span is left out; with the chipset's counters, or saying //! neither, the TSC stepped. The boot's first forward step is slewed away; -//! a second, one back, or a check that finds a CPU's TSC written or behind -//! demotes the TSC ([`demote`]), and every process's reading of now asks the -//! kernel, which counts by the HPET. A rate that drifts past both bounds -//! panics. +//! a second, one back, a check that finds a CPU's TSC written or behind, or +//! an AP's start that proves its offset demotes the TSC ([`demote`]), and +//! every process's reading of now asks the kernel, which counts by the HPET. +//! A rate that drifts past both bounds panics. use core::sync::atomic::{AtomicBool, AtomicU64, Ordering}; @@ -31,6 +31,7 @@ use toyos_userbound::{Mediated, Ports}; use crate::log; use crate::mm::Mmio; use crate::mm::policy::MmioPolicy; +use crate::shootdown::{Reach, Shootdown}; use crate::sync::Lock; use crate::time::{Delay, Duration, AP_START}; @@ -238,10 +239,16 @@ struct Demoted { } static DEMOTED: Lock> = Lock::new(None); +/// Taken by the first [`demote`]; every later one only logs. static IS_DEMOTED: AtomicBool = AtomicBool::new(false); -/// The latest reading the kernel has answered a process since the demotion, -/// which no later answer reads before. +/// What no answer since the demotion reads before: every online CPU's clock +/// once it saw the page marked ([`HANDOVER`]), and every answer given. static DEMOTED_LATEST: AtomicU64 = AtomicU64::new(0); +/// The demotion's one round: each online CPU, once the page is marked, floors +/// [`DEMOTED_LATEST`] at its own clock ([`floor_here`]). +static HANDOVER: Shootdown = Shootdown::new(); +/// Every online CPU has answered [`HANDOVER`]: no process is answered before. +static HANDED_OVER: AtomicBool = AtomicBool::new(false); static DEMOTED_ANSWERED: AtomicBool = AtomicBool::new(false); /// The rate the watch keeps, for a demotion on any CPU to retune the line to. static KEPT_MULT: AtomicU64 = AtomicU64::new(0); @@ -271,6 +278,11 @@ impl Local { } } +/// The handover's answer for each of the `cpus` CPUs the firmware names. +pub fn size(cpus: usize) { + HANDOVER.size(cpus); +} + /// Start the clock on the TSC, at its stated rate or one measured against the /// HPET at `hpet_base`, and hand the HPET to the runtime check. pub fn start(hpet_base: u64, pm_timer: Result) { @@ -437,8 +449,8 @@ fn arrive(turn: u64) -> Option { } /// The BSP's half of [`ROUNDS`] round trips with the AP just started: how far -/// `cpu_id`'s TSC can be from this one's (`reference::Offset`). Panics on an -/// offset the trips prove, and on an AP that stops answering. +/// `cpu_id`'s TSC can be from this one's (`reference::Offset`). Demotes the TSC +/// on an offset the trips prove, and panics on an AP that stops answering. pub fn measure_offset(cpu_id: u32) { let mut least_back = i64::MAX; for round in 0..u64::from(ROUNDS) { @@ -454,11 +466,11 @@ pub fn measure_offset(cpu_id: u32) { let offset = Offset { least_out: LEAST_OUT.load(Ordering::Relaxed) as i64, least_back }; match offset.bound() { Ok((lo, hi)) => log!("clock: cpu{cpu_id}'s TSC is within {lo}..={hi} cycles of cpu0's, over {ROUNDS} round trips"), - Err(proven) => panic!( - "clock: cpu{cpu_id}'s TSC is {} cpu0's by at least {} cycles, which {ROUNDS} round trips prove ({offset:?})", + Err(proven) => demote(format_args!( + "cpu{cpu_id}'s TSC is {} cpu0's by at least {} cycles, which {ROUNDS} round trips prove ({offset:?})", if proven < 0 { "behind" } else { "ahead of" }, proven.unsigned_abs(), - ), + )), } } @@ -530,15 +542,12 @@ pub fn check(now_ns: u64) { /// Leave the TSC for the HPET for the rest of the boot, or, after that, log /// what was found. Every process's reading of now then asks the kernel -/// ([`stamp_now`]), which counts by the HPET from the last held sample's clock; -/// and none answers before the clock at the first counter read after the page -/// was marked, which is after every counter a page reader that found it -/// unmarked read. The line goes back to the kept rate, unslewed, and the -/// kernel's own clock and deadlines stay on it. +/// ([`stamp_now`]), which counts by the HPET from the last held sample's clock, +/// and answers none before every online CPU's clock once it saw the page +/// marked ([`hand_over`]). The line goes back to the kept rate, unslewed, and +/// the kernel's own clock and deadlines stay on it. fn demote(why: core::fmt::Arguments) { - let mut demoted = DEMOTED.lock(); - if demoted.is_some() { - drop(demoted); + if IS_DEMOTED.swap(true, Ordering::SeqCst) { return log!("clock: DEMOTED TSC: {why}"); } // A publication in progress ends within its few stores. @@ -553,37 +562,91 @@ fn demote(why: core::fmt::Arguments) { let first = cpu::counter_ordered(); let at_hpet = hpet.read(hpet.ahead_of(held_hpet, held_tsc, first, mult)); let at_stamp = held_stamp + hpet.reference.ns_of(at_hpet.saturating_sub(held_hpet)); - *demoted = Some(Demoted { hpet, at_hpet, at_stamp, last_tsc: first, last_hpet: at_hpet, mult }); - IS_DEMOTED.store(true, Ordering::SeqCst); + *DEMOTED.lock() = Some(Demoted { hpet, at_hpet, at_stamp, last_tsc: first, last_hpet: at_hpet, mult }); crate::clock::demote(); core::sync::atomic::fence(Ordering::SeqCst); + hand_over(); let after = cpu::counter_ordered(); let on_line = crate::clock::stamp_at(after); - DEMOTED_LATEST.fetch_max(on_line, Ordering::AcqRel); crate::clock::retune(after, mult); - drop(demoted); + HANDED_OVER.store(true, Ordering::Release); log!( "clock: DEMOTED: {why}, so the TSC no longer keeps the clock: every process's reading of now asks the kernel, \ - which counts by the HPET from {at_stamp} ns, {} ns from the TSC's clock and held at that clock till it \ - catches up; the kernel's own clock and deadlines stay on the TSC, at the kept rate", + which counts by the HPET from {at_stamp} ns, {} ns from the TSC's clock and held at every CPU's clock at \ + the handover till it catches up; the kernel's own clock and deadlines stay on the TSC, at the kept rate", at_stamp as i64 - on_line as i64, ); } +/// The handover's round, on the page marked: every online CPU floors +/// [`DEMOTED_LATEST`] at its own clock, this one first. A page reader that +/// found the page unmarked read its counter before its CPU's floor, on that +/// CPU's TSC, so no answer reads before it however far that TSC leads; one +/// that read a line replaced since is ahead of the floor only by the bend +/// every retune makes, the change of rate over that CPU's lead. Before the +/// APs are released no other CPU has run a process, and none is asked. +/// +/// Never logs, as `tlb::wait_for`: a CPU blocked on the console's lock with +/// interrupts closed could not answer. +fn hand_over() { + let generation = HANDOVER.issue(Reach::Narrow); + serve_here(); + if !crate::smp::answering() { + return; + } + super::apic::kick_all_but_self(); + for cpu in 0..crate::smp::cpu_count() as usize { + awaiting("answer to the handover from every CPU", || { + serve_here(); + HANDOVER.served(cpu, generation) + }); + } +} + +/// This CPU's answer to the handover, if it owes one: the kick handler's, and +/// every waiter's on its own CPU. +pub fn serve_here() { + let _closed = super::IrqGuard::close(); + HANDOVER.serve_if_owed(super::percpu::cpu_id() as usize, |_| floor_here()); +} + +/// Floor every answer after the demotion at this CPU's clock now: after the +/// mark, so after the counter of every page reader on this CPU that found +/// the page unmarked. +fn floor_here() { + DEMOTED_LATEST.fetch_max(crate::clock::stamp_at(cpu::counter_ordered()), Ordering::AcqRel); +} + +/// Spin until `done`, which serves this CPU's handover itself, and panic past +/// `time::DEAF_CPU`: a CPU that takes no kick in five seconds is wedged. +fn awaiting(what: &str, mut done: impl FnMut() -> bool) { + let deaf = crate::time::DEAF_CPU.nanos(); + let deadline = crate::clock::nanos_since_boot().saturating_add(deaf); + while !done() { + assert!(crate::clock::nanos_since_boot() < deadline, "clock: no {what} in {deaf} ns since the TSC was demoted"); + core::hint::spin_loop(); + } +} + /// The HPET, for [`demote`] on whichever CPU's check finds the TSC moved. static HPET_COPY: Lock> = Lock::new(None); /// Now as a log line's time, for a process: the TSC's clock, or, once -/// demoted, the HPET's count on from where the TSC was demoted. The counter is -/// read before the mark, as a page reader reads it. +/// demoted, the HPET's count on from where the TSC was demoted, never before +/// the handover's floor. The counter is read before the mark, as a page reader +/// reads it. pub fn stamp_now() -> u64 { let on_line = crate::clock::stamp_at(cpu::counter_ordered()); if !IS_DEMOTED.load(Ordering::SeqCst) { return on_line; } + awaiting("end to the handover", || { + serve_here(); + HANDED_OVER.load(Ordering::Acquire) + }); let stamp = { let mut demoted = DEMOTED.lock(); - let d = demoted.as_mut().expect("demoted before the mark"); + let d = demoted.as_mut().expect("handed over before the demotion was recorded"); let now = cpu::counter_ordered(); let reading = d.hpet.read(d.hpet.ahead_of(d.last_hpet, d.last_tsc, now, d.mult)); (d.last_tsc, d.last_hpet) = (now, reading); @@ -685,6 +748,7 @@ fn settle(held: &mut Held, pending: Pending) { Third::Counter => "confirmed by the RTC", Third::Unsure => "unconfirmed by the RTC", }; + held.edges.forget_before(pending.to.after); if IS_DEMOTED.load(Ordering::Acquire) { return log!("clock: TSC STEP: the demoted TSC stepped {step} ns against the HPET and the PM timer, {confirmed} ({rtc}); not slewed"); } diff --git a/kernel/src/arch/x86_64/idt/kick.rs b/kernel/src/arch/x86_64/idt/kick.rs index 41390867613..605276fbe3a 100644 --- a/kernel/src/arch/x86_64/idt/kick.rs +++ b/kernel/src/arch/x86_64/idt/kick.rs @@ -4,6 +4,7 @@ extern "sysv64" fn kick_handler() { crate::arch::percpu::irq_took!(Kick); crate::counters::serve_here(); crate::arch::smi_cmd::serve_here(); + crate::arch::clocksource::serve_here(); if crate::sched::driver::pass_owed() { crate::preempt::set_need_resched(); } diff --git a/kernel/src/arch/x86_64/mod.rs b/kernel/src/arch/x86_64/mod.rs index b670b86b46c..4fccc30d21d 100644 --- a/kernel/src/arch/x86_64/mod.rs +++ b/kernel/src/arch/x86_64/mod.rs @@ -158,4 +158,5 @@ impl CpuLocal { pub fn size(cpus: usize) { tlb::size(cpus); smi_cmd::size(cpus); + clocksource::size(cpus); } diff --git a/kernel/src/clock.rs b/kernel/src/clock.rs index c1265d690cb..54227729f5a 100644 --- a/kernel/src/clock.rs +++ b/kernel/src/clock.rs @@ -10,7 +10,7 @@ //! //! The line lives on the clock page every process maps //! (`toyos_abi::clock`), and the kernel reads it there too, so the two clocks -//! are one. +//! are one until the architecture demotes the page ([`demote`]). use core::sync::atomic::{AtomicBool, AtomicPtr, AtomicU64, Ordering::{Acquire, Relaxed, Release}}; diff --git a/sdk/abi/src/arch/aarch64.rs b/sdk/abi/src/arch/aarch64.rs index 1ac6ffb64b7..d20b8327fe2 100644 --- a/sdk/abi/src/arch/aarch64.rs +++ b/sdk/abi/src/arch/aarch64.rs @@ -24,8 +24,9 @@ pub fn current_tid() -> Tid { pub fn counter() -> u64 { let now: u64; // SAFETY: two `isb`s and a read of `CNTVCT_EL0` into a register; none - // touches memory. - unsafe { core::arch::asm!("isb", "mrs {now}, cntvct_el0", "isb", now = out(reg) now, options(nomem, nostack)) }; + // touches memory. Not `nomem`, which would let the compiler move memory + // accesses, the clock page's loads among them, across the read. + unsafe { core::arch::asm!("isb", "mrs {now}, cntvct_el0", "isb", now = out(reg) now, options(nostack, preserves_flags)) }; now } From 0650c9d0882db71912f1b6b1b8d83476d923f143 Mon Sep 17 00:00:00 2001 From: japabu Date: Sun, 1 Nov 2026 12:52:38 +0100 Subject: [PATCH 10/11] review r4: the handover's wait answers every round, a demotion before the release floors alone, and floors take the replaced line too MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes review r4's B1–B3 and takes its three optional findings. - **B1, a deadlock.** The demoter's wait served only the handover. With interrupts closed, as a scheduler pass can be, it answered neither a TLB shootdown nor an `SMI_CMD` write. So a CPU waiting on one of those, also with interrupts closed, and the demoter waited on each other until `DEAF_CPU`. `awaiting` now calls `tlb::poll` every spin, as every spinner with `IF` clear must. `tlb::poll` serves the handover too, so a `Lock` spinner with `IF` clear answers it as well. - **B2, a round before the release.** `hand_over` issued a generation even when no AP answered yet, so every AP owed it. Each then floored at its own later clock, which jumped processes' time forward and held it for an offset nothing had read. Before the release, the demoting CPU now floors alone and issues nothing. The docs of `hand_over` and `HANDED_OVER`, and the issue, say so. - **B3, the residual.** A reader that read the line a retune then replaced could be ahead of its CPU's floor by the rate change times its lead. The window included the sampler's own retune to the kept rate during the round. - `clock::demote` now holds every retune under `RETUNING` until `clock::handed_over`, which also makes the demotion's own retune. So `slew()`'s redundant retune no longer lands during the round. - `clock::retune` keeps the line it replaces (`REPLACED`). - `clock::floor_at` floors along the later of the line in force and that one. A reader of either is covered exactly. - A reader of a line two or more retunes old is covered while its CPU's lead stays under the time between those retunes over their change of rate: about 2000 s at a slew's 500 ppm. That remainder, its bound and its exit are in `issues/a-demoted-tsc-keeps-the-kernels-clock-apart-from-processes.md`. - **Optional:** - S1: `serve_here` checks `owes` before it takes the `IrqGuard`, so a kick with no round in flight pays a per-CPU read and two loads. - S2: a finding after the demotion is logged once, not every second. - S3: `awaiting` checks its deadline every 1024 spins. Co-Authored-By: Claude Opus 5.5 --- ...-the-kernels-clock-apart-from-processes.md | 20 ++++-- kernel/src/arch/x86_64/clocksource.rs | 69 +++++++++++-------- kernel/src/arch/x86_64/tlb.rs | 4 +- kernel/src/clock.rs | 51 +++++++++++--- 4 files changed, 102 insertions(+), 42 deletions(-) diff --git a/issues/a-demoted-tsc-keeps-the-kernels-clock-apart-from-processes.md b/issues/a-demoted-tsc-keeps-the-kernels-clock-apart-from-processes.md index 44128b7e3ae..a3a014dde3c 100644 --- a/issues/a-demoted-tsc-keeps-the-kernels-clock-apart-from-processes.md +++ b/issues/a-demoted-tsc-keeps-the-kernels-clock-apart-from-processes.md @@ -25,12 +25,22 @@ After a forward second step, processes' time holds at the TSC's clock until the HPET's count reaches it, then follows the HPET. The kernel's clock stays ahead by the step. -Where one CPU's TSC is apart from the others' (written, found behind, or an -offset an AP's start proves), processes' time holds at the furthest-ahead -CPU's clock at the demotion until the HPET's count reaches it. The kernel's -clock on each CPU stays apart by that CPU's offset. +Where one CPU's TSC is apart from the others' (written or found behind), +processes' time holds at the furthest-ahead CPU's clock at the handover until +the HPET's count reaches it. An offset an AP's start proves demotes before any +process runs, so processes' time starts on the HPET's count. Either way the +kernel's clock on each CPU stays apart by that CPU's offset. + +The handover floors each CPU along the line in force and the one it replaced. +A process that read the page along an older line, at least two retunes before +the demotion, is above its CPU's floor only if that CPU leads the sampling +CPU by more than the time between those retunes over their change of rate. +At a slew's 500 ppm and samples a second apart, that is a lead past about +2000 s. Its answer then reads behind that process's last reading by at most +the change of rate times the lead. **Owner:** the clock. **Exit:** after a demotion, the kernel's clock follows the reference too, so -the kernel and processes read one clock again. +the kernel and processes read one clock again, and the handover floors along +every line a process could have read. diff --git a/kernel/src/arch/x86_64/clocksource.rs b/kernel/src/arch/x86_64/clocksource.rs index 5ac0f853b30..de7f3fba498 100644 --- a/kernel/src/arch/x86_64/clocksource.rs +++ b/kernel/src/arch/x86_64/clocksource.rs @@ -247,8 +247,11 @@ static DEMOTED_LATEST: AtomicU64 = AtomicU64::new(0); /// The demotion's one round: each online CPU, once the page is marked, floors /// [`DEMOTED_LATEST`] at its own clock ([`floor_here`]). static HANDOVER: Shootdown = Shootdown::new(); -/// Every online CPU has answered [`HANDOVER`]: no process is answered before. +/// Every CPU a process may have read the page on has floored +/// [`DEMOTED_LATEST`]: no process is answered before. static HANDED_OVER: AtomicBool = AtomicBool::new(false); +/// A finding after the demotion has been logged; later ones are not. +static FOUND_SINCE: AtomicBool = AtomicBool::new(false); static DEMOTED_ANSWERED: AtomicBool = AtomicBool::new(false); /// The rate the watch keeps, for a demotion on any CPU to retune the line to. static KEPT_MULT: AtomicU64 = AtomicU64::new(0); @@ -548,7 +551,10 @@ pub fn check(now_ns: u64) { /// the kernel's own clock and deadlines stay on it. fn demote(why: core::fmt::Arguments) { if IS_DEMOTED.swap(true, Ordering::SeqCst) { - return log!("clock: DEMOTED TSC: {why}"); + if !FOUND_SINCE.swap(true, Ordering::Relaxed) { + log!("clock: DEMOTED TSC: {why}; later findings are not logged"); + } + return; } // A publication in progress ends within its few stores. let [held_tsc, held_hpet, held_stamp] = loop { @@ -568,7 +574,7 @@ fn demote(why: core::fmt::Arguments) { hand_over(); let after = cpu::counter_ordered(); let on_line = crate::clock::stamp_at(after); - crate::clock::retune(after, mult); + crate::clock::handed_over(after, mult); HANDED_OVER.store(true, Ordering::Release); log!( "clock: DEMOTED: {why}, so the TSC no longer keeps the clock: every process's reading of now asks the kernel, \ @@ -581,50 +587,60 @@ fn demote(why: core::fmt::Arguments) { /// The handover's round, on the page marked: every online CPU floors /// [`DEMOTED_LATEST`] at its own clock, this one first. A page reader that /// found the page unmarked read its counter before its CPU's floor, on that -/// CPU's TSC, so no answer reads before it however far that TSC leads; one -/// that read a line replaced since is ahead of the floor only by the bend -/// every retune makes, the change of rate over that CPU's lead. Before the -/// APs are released no other CPU has run a process, and none is asked. +/// CPU's TSC, so no answer reads before it however far that TSC leads. +/// Before the APs are released no process has run on them, so this CPU +/// floors alone and none is asked. /// /// Never logs, as `tlb::wait_for`: a CPU blocked on the console's lock with /// interrupts closed could not answer. fn hand_over() { - let generation = HANDOVER.issue(Reach::Narrow); - serve_here(); if !crate::smp::answering() { - return; + return floor_here(); } + let generation = HANDOVER.issue(Reach::Narrow); + serve_here(); super::apic::kick_all_but_self(); for cpu in 0..crate::smp::cpu_count() as usize { - awaiting("answer to the handover from every CPU", || { - serve_here(); - HANDOVER.served(cpu, generation) - }); + awaiting("answer to the handover from every CPU", || HANDOVER.served(cpu, generation)); } } -/// This CPU's answer to the handover, if it owes one: the kick handler's, and -/// every waiter's on its own CPU. +/// This CPU's answer to the handover, if it owes one: the kick handler's and +/// `tlb::poll`'s, so every spinner with interrupts closed answers it. pub fn serve_here() { + // The hint before the mask: with no round in flight a kick pays two loads. + if !HANDOVER.owes(super::percpu::cpu_id() as usize) { + return; + } let _closed = super::IrqGuard::close(); HANDOVER.serve_if_owed(super::percpu::cpu_id() as usize, |_| floor_here()); } -/// Floor every answer after the demotion at this CPU's clock now: after the -/// mark, so after the counter of every page reader on this CPU that found -/// the page unmarked. +/// Floor every answer after the demotion at this CPU's clock now, along the +/// line in force and the one it replaced (`clock::floor_at`): after the mark, +/// so after the counter of every page reader on this CPU that found the page +/// unmarked, whichever of the two it read. fn floor_here() { - DEMOTED_LATEST.fetch_max(crate::clock::stamp_at(cpu::counter_ordered()), Ordering::AcqRel); + DEMOTED_LATEST.fetch_max(crate::clock::floor_at(cpu::counter_ordered()), Ordering::AcqRel); } -/// Spin until `done`, which serves this CPU's handover itself, and panic past -/// `time::DEAF_CPU`: a CPU that takes no kick in five seconds is wedged. -fn awaiting(what: &str, mut done: impl FnMut() -> bool) { +/// Spin until `done`, answering every round this CPU owes meanwhile +/// (`tlb::poll`: the TLB's, `SMI_CMD`'s and the handover's), as every spinner +/// with interrupts closed must; and panic past `time::DEAF_CPU`: a CPU that +/// takes no kick in five seconds is wedged. +fn awaiting(what: &str, done: impl Fn() -> bool) { + const SPINS_PER_DEADLINE_CHECK: u32 = 1024; let deaf = crate::time::DEAF_CPU.nanos(); let deadline = crate::clock::nanos_since_boot().saturating_add(deaf); + let mut spins = 0u32; while !done() { - assert!(crate::clock::nanos_since_boot() < deadline, "clock: no {what} in {deaf} ns since the TSC was demoted"); + super::tlb::poll(); core::hint::spin_loop(); + spins += 1; + if spins == SPINS_PER_DEADLINE_CHECK { + spins = 0; + assert!(crate::clock::nanos_since_boot() < deadline, "clock: no {what} in {deaf} ns since the TSC was demoted"); + } } } @@ -640,10 +656,7 @@ pub fn stamp_now() -> u64 { if !IS_DEMOTED.load(Ordering::SeqCst) { return on_line; } - awaiting("end to the handover", || { - serve_here(); - HANDED_OVER.load(Ordering::Acquire) - }); + awaiting("end to the handover", || HANDED_OVER.load(Ordering::Acquire)); let stamp = { let mut demoted = DEMOTED.lock(); let d = demoted.as_mut().expect("handed over before the demotion was recorded"); diff --git a/kernel/src/arch/x86_64/tlb.rs b/kernel/src/arch/x86_64/tlb.rs index 1c94e8c6fae..a001fcdef23 100644 --- a/kernel/src/arch/x86_64/tlb.rs +++ b/kernel/src/arch/x86_64/tlb.rs @@ -277,13 +277,15 @@ pub fn serve_ipi() { /// Answers a pending shootdown without taking a lock or allocating, so it is /// safe from inside `Lock::lock`'s spin; and, on the boot processor, a write -/// to `SMI_CMD` asked of it, which a spinner with `IF` clear owes the same way. +/// to `SMI_CMD` asked of it, and a demoted clock's handover, which a spinner +/// with `IF` clear owes the same way. #[inline] pub fn poll() { super::smi_cmd::serve_here(); if !smp::answering() { return; } + super::clocksource::serve_here(); let cpu = percpu::cpu_id() as usize; SHOOTDOWN.serve_if_owed(cpu, answer(Answerer::Asked)); } diff --git a/kernel/src/clock.rs b/kernel/src/clock.rs index 54227729f5a..620c2b0915a 100644 --- a/kernel/src/clock.rs +++ b/kernel/src/clock.rs @@ -38,6 +38,14 @@ static PAGE: AtomicPtr> = AtomicPtr::new(core::ptr::null_mu /// The page's one writer at a time. static RETUNING: Lock<()> = Lock::new(()); +/// The line the one in force replaced, for [`floor_at`]; zeros before the +/// first retune, which no floor exceeds. Published under [`RETUNING`]. +static REPLACED: crate::seqlock::Published<3> = crate::seqlock::Published::new(); + +/// From [`demote`] to [`handed_over`] no retune publishes: the line in force +/// and the one it replaced are what every page reader read. +static HELD: AtomicBool = AtomicBool::new(false); + /// Take the rate the CPU states, the loader's, for the stamps the log carries /// before [`set_counter`]. The kernel's entry calls this before its first record. pub fn state() { @@ -68,14 +76,21 @@ pub fn set_counter(boot: u64, mult: u64) { } /// Go on at `mult` from the counter reading `at`: the line from there starts -/// where the clock stands, so it bends and never steps. The line now in force. -pub fn retune(at: u64, mult: u64) -> Line { - let page = lines().expect("clock: retuned before it started"); +/// where the clock stands, so it bends and never steps. Nothing while a +/// demotion is handed over ([`demote`]); the caller retunes again later. +pub fn retune(at: u64, mult: u64) { let _one = RETUNING.lock(); - let stamp = page.read().expect("clock: the page `set_counter` laid out carries no clock").stamp_of(at); - let line = Line { counter: at, stamp, mult }; - page.publish(line); - line + if !HELD.load(Relaxed) { + publish(at, mult); + } +} + +/// [`retune`]'s publication, under [`RETUNING`]. +fn publish(at: u64, mult: u64) { + let page = lines().expect("clock: retuned before it started"); + let old = page.read().expect("clock: the page `set_counter` laid out carries no clock").line; + REPLACED.publish([old.counter, old.stamp, old.mult]); + page.publish(Line { counter: at, stamp: old.stamp_of(at), mult }); } /// The rate the line in force runs at. @@ -89,11 +104,31 @@ pub fn stamp_at(at: u64) -> u64 { } /// Send every process's reading of now to the kernel from here on -/// (`toyos_abi::clock::ClockPage::demoted`). +/// (`toyos_abi::clock::ClockPage::demoted`), and hold every retune until +/// [`handed_over`]: a page reader that found the page unmarked read the line +/// in force or an older one, and [`floor_at`] reads the latest two. pub fn demote() { + let _one = RETUNING.lock(); + HELD.store(true, Relaxed); lines().expect("clock: demoted before it started").demote(); } +/// The clock at the counter reading `at` by the line in force and by the one +/// it replaced, whichever is later: no earlier than a page reader's reading +/// of a counter before `at` along either. Lock-free, for a kick handler. +pub fn floor_at(at: u64) -> u64 { + let now = stamp_at(at); + let [counter, stamp, mult] = REPLACED.snapshot().expect("clock: the replaced line is published only under `RETUNING`, which a hold keeps"); + now.max(Line { counter, stamp, mult }.stamp_of(at)) +} + +/// End [`demote`]'s hold, going on at `mult` from the counter reading `at`. +pub fn handed_over(at: u64, mult: u64) { + let _one = RETUNING.lock(); + HELD.store(false, Relaxed); + publish(at, mult); +} + fn lines() -> Option<&'static ClockPage> { let page = LINES.load(Acquire); // SAFETY: `set_counter` stored a pointer into a page it holds for the From 6cc3e1c5ab11d11e84125824b5d3833657c101c2 Mon Sep 17 00:00:00 2001 From: japabu Date: Sun, 1 Nov 2026 13:07:47 +0100 Subject: [PATCH 11/11] review r5: the handover's comments and record say what it covers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes review r5's R1–R3, which are all text, and takes its S1 and S2. - **R1.** The `HELD` and `hand_over` docs claimed every page reader was covered. They now say only the line in force and the one it replaced are, and point to the issue for the rest. - **R2.** The issue's "about 2000 s" assumed retunes a second apart. A CPU leading the previous sampler by λ samples about 1 s − λ later, so coverage needs a lead under about a second. The bound stands: the rate change times the lead, 0.5 ms per second of lead at 500 ppm. - **R3.** `serve_here`'s doc said every spinner with interrupts closed answers. Only the kick handler and `tlb::poll`'s spinners do, and the round ends because the demoter answers every round itself. The "two loads" count is gone. - **S1.** The issue says processes' time may hold up to the last retune's change of rate times the time since it past the furthest-ahead clock. - **S2.** `awaiting` checks `done` before it reads the clock for a deadline. Co-Authored-By: Claude Opus 5.5 --- ...-the-kernels-clock-apart-from-processes.md | 28 +++++++++++-------- kernel/src/arch/x86_64/clocksource.rs | 15 ++++++++-- kernel/src/clock.rs | 4 +-- 3 files changed, 30 insertions(+), 17 deletions(-) diff --git a/issues/a-demoted-tsc-keeps-the-kernels-clock-apart-from-processes.md b/issues/a-demoted-tsc-keeps-the-kernels-clock-apart-from-processes.md index a3a014dde3c..524bb1bd73c 100644 --- a/issues/a-demoted-tsc-keeps-the-kernels-clock-apart-from-processes.md +++ b/issues/a-demoted-tsc-keeps-the-kernels-clock-apart-from-processes.md @@ -26,18 +26,22 @@ the HPET's count reaches it, then follows the HPET. The kernel's clock stays ahead by the step. Where one CPU's TSC is apart from the others' (written or found behind), -processes' time holds at the furthest-ahead CPU's clock at the handover until -the HPET's count reaches it. An offset an AP's start proves demotes before any -process runs, so processes' time starts on the HPET's count. Either way the -kernel's clock on each CPU stays apart by that CPU's offset. - -The handover floors each CPU along the line in force and the one it replaced. -A process that read the page along an older line, at least two retunes before -the demotion, is above its CPU's floor only if that CPU leads the sampling -CPU by more than the time between those retunes over their change of rate. -At a slew's 500 ppm and samples a second apart, that is a lead past about -2000 s. Its answer then reads behind that process's last reading by at most -the change of rate times the lead. +processes' time holds at the furthest-ahead CPU's clock at the handover, or +up to the last retune's change of rate times the time since it past that, +until the HPET's count reaches it. An offset an AP's start proves demotes +before any process runs, so processes' time starts on the HPET's count. +Either way the kernel's clock on each CPU stays apart by that CPU's offset. + +The handover floors each CPU along the line in force and the one it replaced, +which covers a process that read either. A process that read the page along +an older line, at least two retunes before the demotion, can read behind its +last reading. Retunes come a second apart by the sampling CPU's clock, so a +CPU that leads the previous sampler by λ samples about 1 s − λ after it. A +reading on a leading CPU is then covered only while the lead stays under +about a second, where consecutive samples come from CPUs that far apart. +Past that, the answer reads behind the process's last reading by at most the +change of rate times the lead: 0.5 ms per second of lead at a slew's +500 ppm. **Owner:** the clock. diff --git a/kernel/src/arch/x86_64/clocksource.rs b/kernel/src/arch/x86_64/clocksource.rs index de7f3fba498..5f70ba809e0 100644 --- a/kernel/src/arch/x86_64/clocksource.rs +++ b/kernel/src/arch/x86_64/clocksource.rs @@ -587,7 +587,11 @@ fn demote(why: core::fmt::Arguments) { /// The handover's round, on the page marked: every online CPU floors /// [`DEMOTED_LATEST`] at its own clock, this one first. A page reader that /// found the page unmarked read its counter before its CPU's floor, on that -/// CPU's TSC, so no answer reads before it however far that TSC leads. +/// CPU's TSC, so no answer reads before its reading along the line in force +/// or the one it replaced, however far that TSC leads. A reading along an +/// older line is the remainder +/// `issues/a-demoted-tsc-keeps-the-kernels-clock-apart-from-processes.md` +/// records. /// Before the APs are released no process has run on them, so this CPU /// floors alone and none is asked. /// @@ -606,9 +610,11 @@ fn hand_over() { } /// This CPU's answer to the handover, if it owes one: the kick handler's and -/// `tlb::poll`'s, so every spinner with interrupts closed answers it. +/// `tlb::poll`'s, so a `Lock` spinner answers it. Not every spinner with +/// interrupts closed does; the round ends because the demoter answers every +/// round it is asked meanwhile ([`awaiting`]). pub fn serve_here() { - // The hint before the mask: with no round in flight a kick pays two loads. + // The hint, not the mask, with no round in flight. if !HANDOVER.owes(super::percpu::cpu_id() as usize) { return; } @@ -630,6 +636,9 @@ fn floor_here() { /// takes no kick in five seconds is wedged. fn awaiting(what: &str, done: impl Fn() -> bool) { const SPINS_PER_DEADLINE_CHECK: u32 = 1024; + if done() { + return; + } let deaf = crate::time::DEAF_CPU.nanos(); let deadline = crate::clock::nanos_since_boot().saturating_add(deaf); let mut spins = 0u32; diff --git a/kernel/src/clock.rs b/kernel/src/clock.rs index 620c2b0915a..68829bad25a 100644 --- a/kernel/src/clock.rs +++ b/kernel/src/clock.rs @@ -42,8 +42,8 @@ static RETUNING: Lock<()> = Lock::new(()); /// first retune, which no floor exceeds. Published under [`RETUNING`]. static REPLACED: crate::seqlock::Published<3> = crate::seqlock::Published::new(); -/// From [`demote`] to [`handed_over`] no retune publishes: the line in force -/// and the one it replaced are what every page reader read. +/// From [`demote`] to [`handed_over`] no retune publishes, so the two lines +/// [`floor_at`] reads stay unchanged through the handover. static HELD: AtomicBool = AtomicBool::new(false); /// Take the rate the CPU states, the loader's, for the stamps the log carries