diff --git a/build/arch.rs b/build/arch.rs index 7f56ca52222..c029da5319e 100644 --- a/build/arch.rs +++ b/build/arch.rs @@ -163,8 +163,8 @@ impl Arch { /// TCG guest that withholds one is a feature this tree stops exercising. pub const fn cpu(self, accel: Accel) -> &'static str { match (self, accel.is_hardware()) { - (Arch::X86_64, true) => "host,+rdrand,+smap,+fsgsbase,+x2apic,+smep,+ssse3,+sse4.1,+sse4.2,+xsave,+xsaveopt,+avx,+avx2", - (Arch::X86_64, false) => "qemu64,+rdrand,+smap,+fsgsbase,+x2apic,+smep,+ssse3,+sse4.1,+sse4.2,+xsave,+xsaveopt,+avx,+avx2", + (Arch::X86_64, true) => "host,+invtsc,+rdrand,+smap,+fsgsbase,+x2apic,+smep,+ssse3,+sse4.1,+sse4.2,+xsave,+xsaveopt,+avx,+avx2", + (Arch::X86_64, false) => "qemu64,+rdrand,+rdtscp,+smap,+fsgsbase,+x2apic,+smep,+ssse3,+sse4.1,+sse4.2,+xsave,+xsaveopt,+avx,+avx2", (Arch::Aarch64, true) => "host", (Arch::Aarch64, false) => "max", } diff --git a/build/ci.rs b/build/ci.rs index d28728a8b26..4b09debd254 100644 --- a/build/ci.rs +++ b/build/ci.rs @@ -534,6 +534,10 @@ pub(crate) const CONTROLS: &[Control] = &[ red(KERNEL_LOOM, "seqlock-writer-fence-off", Some("panic_console_publish"), &[ Fails("a_snapshot_is_one_publication_whole"), ]), + red(KERNEL_LOOM, "clock-latch-fence-off", Some("clock_latch"), &[ + Fails("a_read_is_one_line_whole"), + Fails("a_later_read_is_never_an_older_line"), + ]), red(KERNEL_LOOM, "serial-try-lock-then-some", Some("serial_lock"), &[ Fails("a_lost_try_lock_leaves_the_lock_held"), Fails("two_writers_never_overlap"), diff --git a/issues/a-demoted-tsc-keeps-the-kernels-clock-apart-from-processes.md b/issues/a-demoted-tsc-keeps-the-kernels-clock-apart-from-processes.md new file mode 100644 index 00000000000..524bb1bd73c --- /dev/null +++ b/issues/a-demoted-tsc-keeps-the-kernels-clock-apart-from-processes.md @@ -0,0 +1,50 @@ +--- +status: open +kind: defect +opened: 2026-11-01 +--- + +# A demoted TSC keeps the kernel's clock apart from processes' + +When the clock demotes the TSC (`kernel/src/arch/x86_64/clocksource.rs`, +`demote`), processes' readings of now move to the HPET through +`SYS_CLOCK_NOW`. The kernel's own clock, its deadlines and its log stamps stay +on the TSC at the kept rate. Every later TSC step therefore moves the two +clocks apart, and a log line a process stamps no longer orders against one +the kernel stamps. + +A backward step plays out like this: +1. Every reading, the kernel's and processes', goes back by the step. +2. The sampler, or a CPU's check, finds it within a second and demotes the + TSC. +3. Processes' time jumps forward to the HPET's count from the last held + sample, which is the true time, and stays monotonic from there. +4. The kernel's clock stays behind by the step. + +After a forward second step, processes' time holds at the TSC's clock until +the HPET's count reaches it, then follows the HPET. The kernel's clock stays +ahead by the step. + +Where one CPU's TSC is apart from the others' (written or found behind), +processes' time holds at the furthest-ahead CPU's clock at the handover, or +up to the last retune's change of rate times the time since it past that, +until the HPET's count reaches it. An offset an AP's start proves demotes +before any process runs, so processes' time starts on the HPET's count. +Either way the kernel's clock on each CPU stays apart by that CPU's offset. + +The handover floors each CPU along the line in force and the one it replaced, +which covers a process that read either. A process that read the page along +an older line, at least two retunes before the demotion, can read behind its +last reading. Retunes come a second apart by the sampling CPU's clock, so a +CPU that leads the previous sampler by λ samples about 1 s − λ after it. A +reading on a leading CPU is then covered only while the lead stays under +about a second, where consecutive samples come from CPUs that far apart. +Past that, the answer reads behind the process's last reading by at most the +change of rate times the lead: 0.5 ms per second of lead at a slew's +500 ppm. + +**Owner:** the clock. + +**Exit:** after a demotion, the kernel's clock follows the reference too, so +the kernel and processes read one clock again, and the handover floors along +every line a process could have read. diff --git a/issues/ap-tsc-trail-is-assumed-and-never-checked.md b/issues/ap-tsc-trail-is-assumed-and-never-checked.md deleted file mode 100644 index 6623469fb78..00000000000 --- a/issues/ap-tsc-trail-is-assumed-and-never-checked.md +++ /dev/null @@ -1,77 +0,0 @@ ---- -status: open -kind: tooling -opened: 2026-08-14 ---- - -# Every timestamp in the kernel assumes an AP's TSC does not trail the BSP's, and nothing measures it - -`clock::nanos_since_boot` subtracts one global `TSC_BOOT` — stored by the BSP -during `clock::init` — from whatever `rdtsc` this CPU answers. A CPU whose TSC -reads below that value has no honest answer to give. - -The assumption has been stated out loud, and `kernel/src/clock.rs` carries it: - -> **Cross-CPU ordering rests on the TSC being invariant and firmware-synchronised.** -> ToyOS targets 2020+ x86-64, where it is. If it is not, two records from two -> CPUs may be merged out of order by the skew; nothing breaks and the reader -> cannot tell. - -That is an accurate description of a *small* skew. It is not a description of a -CPU whose TSC starts behind the BSP's by more than the boot took, which is the -case the subtraction cannot represent at all. - -## What changed, and what did not - -Until 2026-08-14 the subtraction was `rdtsc() - TSC_BOOT`, and with -overflow-checks on — which every guest build has — a trailing TSC **panicked the -kernel**. `log::emit` then began reading the clock inside its publication -bracket, where such a panic would fire with `IF` clear and a reservation taken -and uncommitted, and the panic handler's own `log!` would reenter the same -shard. The subtraction is `saturating_sub` now, so the failure is a timestamp of -zero instead of a fault. - -**That makes it survivable, not correct.** A CPU stuck behind stamps every -record it writes with zero for as long as it trails, and those records read as -the oldest thing the machine has: `log/read.rs`'s merge sorts them last, and its -per-shard descent — which relies on `at_ns` descending with the sequence number -— stops at the first of them. Every other consumer of `nanos_since_boot` on that CPU is -wrong by the same amount: deadlines, the scheduler's accounting, `at_ns` in -`/log`. - -## What is not known - -Nothing in this tree has ever measured TSC agreement across CPUs. Both halves -are open: - -- **Does it happen on the T14?** `smp` brings APs up and no code compares their - TSC to the BSP's, before or after. One `rdtsc` per CPU logged at the end of - `init_ap` would answer it, and the metal session is where that has to run — - QEMU's TSCs are synthesised from one host clock and will agree whatever the - firmware does. -- **What should happen if it does?** The choices are a per-CPU `TSC_BOOT` - captured at AP bring-up (which fixes the arithmetic and leaves cross-CPU - comparison meaningless), `IA32_TSC_ADJUST` written to align them (which is - what firmware is supposed to have done), or accepting the skew and saying so - where records are merged. Not this branch's call. - -**2026-08-25, promoted to `defect`.** "Nothing would tell us" is the defect: -`kernel/src/clock.rs` cites this file for the assumption its whole arithmetic -rests on, and the instrument that would confirm or refute it does not exist — -one `rdtsc` per CPU logged at the end of `init_ap` is the entire measurement. -QEMU synthesises every guest TSC from one host clock and cannot answer, so this -belongs on the metal session's table beside the control-register delta -(`issues/a-metal-session-runs-a-pre-flash-gate-first.md`). Owed by -whoever prepares that session: build the log line first, then read it. - -**2026-10-28, part measured.** `smp` now logs each AP's `rdtsc` inside a -bracket of the BSP's, about 19.8 M cycles (11 ms) wide on the AMD bench, so -it answers "trails by more than the boot took" and no smaller offset. Under -the AMD bench's Linux, CPU 0 and each of the other 15 took turns reading -their TSCs through idle gaps up to 1 s (`tsc-probe warp`, PR #1079's -evidence). Every pair's worst step was positive, 54 to 342 cycles, which -bounds each CPU's offset from CPU 0 by 342 cycles, some 190 ns. That is that -firmware's synchronisation under Linux, not a ToyOS measurement: the log -line this file owes is still a bracket, not an offset. -`issues/the-system-clock-takes-the-tsc-on-one-reading-and-never-checks-it-again.md` -holds the clock's other unchecked assumptions. diff --git a/issues/the-amd-benchs-tsc-steps-1-7-ms-forward-once-a-boot.md b/issues/the-amd-benchs-tsc-steps-1-7-ms-forward-once-a-boot.md new file mode 100644 index 00000000000..6c297e7722e --- /dev/null +++ b/issues/the-amd-benchs-tsc-steps-1-7-ms-forward-once-a-boot.md @@ -0,0 +1,47 @@ +--- +status: open +kind: defect +opened: 2026-10-31 +--- + +# The AMD bench's TSC steps 1.7 ms forward once a boot + +On the AMD bench (Lenovo 82A2, BIOS DMCN38WW; Ryzen 7 4800U, CPUID +signature 0x860f01, Family 17h Model 60h), the TSC jumps 1.69–1.73 ms ahead +once in every boot. The HPET, the ACPI PM timer and the RTC agree on it +within 37 µs. The RTC runs off its own 32.768 kHz crystal, and the other two +off the FCH's 14.318 MHz clock. Linux on the same machine sees the event in +5 of 30 boots. + +**Owner:** the clock (`kernel/src/arch/x86_64/clocksource.rs`) and +acpiserver, which hands over the idle state the step lands in. + +**Where it lands.** Inside one spell in which every CPU sits in port 0x415 +(s1, the deepest state acpiserver hands over), early in the boot. One CPU +reads the HPET as each spell ends: +- in `benchcase`, spells 0 to 2 read no step (the third lasted 1.66 ms), and + the TSC is 1.70 ms ahead from the end of spell 3; +- in `testcases`, the boot's first such spell already ends 1.72 ms ahead. + +The stepping spell's TSC length includes the jump. No later spell steps. +`benchcase` reaches its first spells at about 17 s, when `osbench_sched`'s +all-core load ends; `testcases` reaches them at about 5 s, right after the +hand-over. The trigger is the package's deep idle with every core in 0x415. +Whether the TSC's jump across it is the platform's or the state's entry is +not known. + +**What the clock does meanwhile.** Readers see the forward step: the clock +reads up to 1.7 ms ahead from the step until the sampler finds it, within a +second, and through the RTC's few seconds after. A forward step the HPET and +the PM timer agree on is then slewed away at no more than 500 ppm. This +happens whether the RTC confirms the step or cannot tell, and the correction +never steps back. A second step, or a backward one, demotes the TSC +(`issues/a-demoted-tsc-keeps-the-kernels-clock-apart-from-processes.md`). + +**Not read.** AMD's *Processor Programming Reference for Family 17h Model 60h* +(55922), which would cover the TSC in deep C-states and the TSC's relation to +the FCH's clocks. amd.com refuses the build's user agent, so it has not been +fetched. + +**Exit:** the cause found and removed, or documented by AMD, so that no AMD +boot takes a step. diff --git a/issues/the-kernels-50-ms-tsc-calibration-misses-the-tscs-rate.md b/issues/the-kernels-50-ms-tsc-calibration-misses-the-tscs-rate.md deleted file mode 100644 index a18cc3dfb0e..00000000000 --- a/issues/the-kernels-50-ms-tsc-calibration-misses-the-tscs-rate.md +++ /dev/null @@ -1,52 +0,0 @@ ---- -status: open -kind: defect -opened: 2026-10-28 ---- - -# The kernel's 50 ms TSC calibration misses the TSC's rate - -The kernel measures the TSC against the HPET once, over 50 ms -(`kernel/src/arch/x86_64/hpet.rs`), and keeps time at that rate for the -whole boot. On the AMD bench (Ryzen 7 4800U) that rate is consistently low. -The TSC counted 1796.6248 MHz against the same HPET over 10 minutes and -against the ACPI PM timer over 5, the same in every idle and busy phase -(`tsc-drift` under the bench's Linux, 2026-10-28, PR #1079's evidence). All -but one of the kernel's calibrations in the session's AMD boot logs read -1796.506 to 1796.551 MHz, 41 to 66 ppm low, and 1796.519 to 1796.532 MHz in -most; the one other reads 1796.597 MHz, 15 ppm low. A rate read low is a -period kept long (`hpet.rs` stores HPET femtoseconds per TSC cycle), so every -interval reads long: at 58 ppm a ToyOS clock on that machine gains about 5 s -a day. - -The error is a bias, not noise: every reading falls on one side. It is not -the period's resolution either: one step of the femtoseconds the kernel -stores is 3,227 Hz there, 1.8 ppm, and truncation biases the other way. An -inference, not measured: the HPET is read before the first TSC read and after -the last, so the HPET interval brackets the TSC interval by up to two -uncached MMIO reads. About 1.5 µs over 50 ms is about 30 ppm low, the sign -and size seen. - -The T14 states its TSC's rate (CPUID 15H, 2,419,200,000 Hz), and the kernel -logs its own measurement beside it. Over 5 minutes against the HPET under -its Linux, the TSC counted 2419.2002 MHz, the stated rate within 0.1 ppm, -worst residual 0.2 µs (`tsc-drift`, 2026-10-28). The session's T14 boot logs -hold two clusters, 285 ppm under and 59 ppm over, in 345 and 250 log files -(some of them copies of one boot), and a few readings between and beyond -them (404 to 440 and 759 to 766 ppm under). Those are the kernel's error: the -285 ppm cluster gains about 25 s a day and the 59 ppm one loses about 5. The -bracketing above explains neither, which needs 14 µs and the other sign. -That the two clusters are two kernel revisions is an inference no one has -checked. On that machine a stated rate is there to be used, and right, and -the kernel keeps its measurement instead. - -**Exit**: on the T14 and the AMD bench, the rate the kernel keeps is within a -bound the fix derives and logs. That bound is set by its window, its reads' -cost and the reference's own accuracy, against the rate measured over minutes -against the same reference: `tsc-drift` on the AMD bench, and CPUID 15H or -`tsc-drift` on the T14. - -Whether the TSC may keep time at all, and whether it still does, is -`issues/the-system-clock-takes-the-tsc-on-one-reading-and-never-checks-it-again.md`. - -**Owner**: the kernel's x86-64 clock (`kernel/src/arch/x86_64/hpet.rs`). diff --git a/issues/the-loaders-lines-carry-no-time-on-an-x86-cpu-that-states-no-tsc-rate.md b/issues/the-loaders-lines-carry-no-time-on-an-x86-cpu-that-states-no-tsc-rate.md index 8162e2ca67d..3d0d9ed830a 100644 --- a/issues/the-loaders-lines-carry-no-time-on-an-x86-cpu-that-states-no-tsc-rate.md +++ b/issues/the-loaders-lines-carry-no-time-on-an-x86-cpu-that-states-no-tsc-rate.md @@ -8,12 +8,13 @@ opened: 2026-10-04 Every line of the log opens with its time since the CPU's counter last started (`toyos_abi::log::Head`). The loader converts the counter at the rate CPUID -leaves 15H/16H state (`loader/src/stamp.rs`, `toyos_tsc::stated_hz`), and -the kernel does the same for its records until it has measured the TSC against -the HPET (`kernel/src/clock.rs`'s `stamp`). A CPU that states neither leaf gets +leaf 15H states (`loader/src/stamp.rs`, `toyos_tsc::stated_hz`), and the +kernel does the same for its records until its clock starts +(`kernel/src/clock.rs`'s `stamp`). Leaf 16H's base frequency is no rate: the +SDM confines its data to display. A CPU that states no rate in 15H gets `--.---` (`toyos_abi::log::UNTIMED`) in place of the time on every loader line, one `Loader clock: this CPU states no counter rate` line, and `--.---` on every -kernel record before `clock: HPET at …` — `boot:`, `ACPI:`, the interrupt +kernel record before the clock starts — `boot:`, `ACPI:`, the interrupt controller's and the per-CPU area's among them. The records after it carry their time from the same zero. @@ -22,8 +23,9 @@ Every AMD part is one. InstLatx64's recorded CPUID dumps largest standard leaf, as `00000010` on Zen 2 (`0870F10_K17_Matisse_CPUID`), Zen 3+ (`0A40F41_K19_Rembrandt_01_CPUID`), Zen 4 (`0A60F12_K19_Raphael_01_CPUID`) and Zen 5 (`0B00F21_K20_Turin_01_CPUID`, `0B20F40_K20_StrixPoint_06_CPUID`, -`0B40F40_K20_GraniteRidge_02_CPUID`): neither leaf exists there. QEMU's -`qemu64` states neither either, and under nightly's `kvm / suite`'s +`0B40F40_K20_GraniteRidge_02_CPUID`): neither leaf exists there, and an Intel +part whose leaf 15H leaves its crystal's hertz zero states none either. QEMU's +`qemu64` has no leaf 15H, and under nightly's `kvm / suite`'s `-cpu host` whether a line is timed is the runner's CPU's to decide. A second rate source is the fix, and the choice of one is open: the @@ -35,7 +37,6 @@ Until then no guest test asserts a time on the screen: `screen_loader_clears` reads `--.---` under `qemu64` and whatever the runner's CPU states under `-cpu host`. The fix is where that test starts asserting a time in it. -Exit: a loader line and a kernel record written before `clock: HPET at …` carry -their time on an x86-64 CPU that states neither leaf 15H's ratio nor leaf 16H's -base frequency, and `screen_loader_clears` asserts that line's time on the -screen. +Exit: a loader line and a kernel record written before the clock starts carry +their time on an x86-64 CPU whose leaf 15H states no rate, and +`screen_loader_clears` asserts that line's time on the screen. diff --git a/issues/the-system-clock-takes-the-tsc-on-one-reading-and-never-checks-it-again.md b/issues/the-system-clock-takes-the-tsc-on-one-reading-and-never-checks-it-again.md deleted file mode 100644 index ee21047bf49..00000000000 --- a/issues/the-system-clock-takes-the-tsc-on-one-reading-and-never-checks-it-again.md +++ /dev/null @@ -1,53 +0,0 @@ ---- -status: open -kind: defect -opened: 2026-10-28 ---- - -# The system clock takes the TSC on one reading and never checks it again - -x86-64's system clock is the TSC. The boot measures it against the HPET -once, over 50 ms (`kernel/src/arch/x86_64/hpet.rs`), and `clock::set_counter` -publishes that one period for the machine's life. Three things the clock -then relies on go unchecked: - -- **That the TSC may keep time at all.** CPUID 80000007H:EDX[8], the - invariant TSC (Intel's SDM "Invariant TSC"; AMD's PPRs - `CPUID_Fn80000007_EDX[TscInvariant]`), is read only to admit deep idle - states (`kernel/src/arch/x86_64/idle.rs::facts`). The clock starts on a - TSC without it the same way. -- **That every CPU's TSC agrees.** Each AP's reading at start is checked to - fall inside a bracket of the BSP's, about 19.8 M cycles wide on the AMD - bench, 11 ms (the `SMP:` lines), and nothing compares the CPUs again - (`issues/ap-tsc-trail-is-assumed-and-never-checked.md`). -- **That the rate stays the one measured.** No later reading is set against - the HPET or another reference, so a step or a change of rate after boot - goes into every timestamp, timeout and deadline unannounced. - -On the AMD bench (Ryzen 7 4800U) the one calibration is also 41 to 66 ppm -off (`issues/the-kernels-50-ms-tsc-calibration-misses-the-tscs-rate.md`). -That CPU states the invariant bit. Under its Linux, it counts one rate in -every phase over 10 minutes, and over 15 minutes across 16 CPUs and idle -gaps up to 1 s no CPU's TSC read behind another's and no step exceeded a -read's 1.2 µs. But in 5 of 30 boots' kernel logs that Linux's watchdog saw -the TSC some 1.7 ms ahead of the HPET over half a second, 2.6 s into boot in -the one boot whose dmesg was kept. Every time it was the same size, and it -came with a calibration that was right. The cause is unknown (`tsc-probe` -and the boots' figures, PR #1079's evidence). ToyOS's one calibration also -runs early in boot. A step inside its 50 ms would be a 3% error in every -timestamp after it, and a clock that never checks would not see it. - -**Exit**: the kernel keeps time on the TSC only where every CPU states it -invariant. It bounds the CPUs' offsets at start to a figure it logs and -derives, and checks the rate and offsets against a reference while it runs. -When they leave their bound it fails clearly, or falls back to a counter -that holds and says so in the log. The runtime bound is derived from the -reference's and the reads' own error, so the check does not mistake its -own calibration for the TSC's: 12 of the 17 times the AMD's Linux watchdog -fired, it was Linux's own calibration that was wrong. Where a vendor -documents a cheaper detector, the check prefers it, such as `IA32_TSC_ADJUST` -for a write to the TSC (Intel's SDM, Vol. 3). Each part is shown on the T14 -and the AMD bench. - -**Owner**: the kernel's x86-64 clock (`kernel/src/arch/x86_64/hpet.rs`, -`kernel/src/clock.rs`). diff --git a/issues/the-x86-clock-calibrates-against-an-hpet-or-the-boot-panics.md b/issues/the-x86-clock-calibrates-against-an-hpet-or-the-boot-panics.md index 85da7001334..9d7c5f32e21 100644 --- a/issues/the-x86-clock-calibrates-against-an-hpet-or-the-boot-panics.md +++ b/issues/the-x86-clock-calibrates-against-an-hpet-or-the-boot-panics.md @@ -7,11 +7,11 @@ opened: 2026-10-12 # The x86 clock calibrates against an HPET, or the boot panics `arch::boot::clock` (`kernel/src/arch/x86_64/boot.rs`) takes the HPET's base -from its ACPI table with `.expect("ACPI: HPET not found")`, and every rate the -kernel keeps descends from that one measurement: the TSC's -(`kernel/src/arch/x86_64/hpet.rs`) and the local APIC timer's, calibrated -against the TSC (`apic::init_timer`). CPUID 15H/16H is read after it, as a -cross-check only. A machine whose firmware publishes no `HPET` table stops +from its ACPI table with `.expect("ACPI: HPET not found")`, and the clock +needs it whatever the CPU states: it measures the TSC's rate against the HPET +where CPUID 15H states none, and holds the TSC to it while it runs, a stated +rate too (`kernel/src/arch/x86_64/clocksource.rs`); the local APIC timer is +calibrated against the TSC (`apic::init_timer`). A machine whose firmware publishes no `HPET` table stops there, before a timer ticks, and before PCI is enumerated, so a console that is a virtio function never carries the panic. @@ -44,5 +44,6 @@ a third-party copy of ACPI 6.6 numbers them; 6.5 itself is unread. **Owner**: whoever holds `issues/toyos-runs-in-any-open-source-hypervisor.md`. **Exit**: a q35 boot with `-machine hpet=off` reaches `Boot: complete`, its -`TSC:` line naming the source its rate came from; and a machine that states no -source is refused by a line that names each one it looked for. +`clock:` line naming the source its rate came from and the reference that +holds it; and a machine that states no source is refused by a line that names +each one it looked for. diff --git a/kernel/loom/Cargo.toml b/kernel/loom/Cargo.toml index d2ba72efec6..3a8a7c31f1c 100644 --- a/kernel/loom/Cargo.toml +++ b/kernel/loom/Cargo.toml @@ -197,6 +197,16 @@ tlb-tags-enter-relaxed = [] # Never on by default and never reachable from a kernel build, which declares # the same name only so `cfg` checking knows it. seqlock-writer-fence-off = [] +# The negative control for the clock page's latch. It drops the fence between +# the writer's move of readers to the other copy and its rewrite of the one +# they left (`sdk/abi/src/clock/latch.rs`), and `clock_latch.rs` must red: +# +# cargo test --manifest-path kernel/loom/Cargo.toml --features clock-latch-fence-off \ +# --test clock_latch +# +# Never on by default; `toyos-abi` declares the same name only so `cfg` +# checking knows it. +clock-latch-fence-off = [] # The negative control for the console backend's lock. It builds # `serial_lock.rs`'s `try_lock` with `then_some`, whose argument is built on a diff --git a/kernel/loom/src/lib.rs b/kernel/loom/src/lib.rs index b2e8411c536..85483bbc364 100644 --- a/kernel/loom/src/lib.rs +++ b/kernel/loom/src/lib.rs @@ -407,6 +407,12 @@ pub mod capture_access; #[path = "../../../sdk/toyos/src/log/ring.rs"] pub mod log_ring; +/// The clock page's latch, which the kernel replaces the clock's line through +/// while every process and the kernel's own NMIs read it: not the kernel's +/// file but the ABI's, driven by `tests/clock_latch.rs`. +#[path = "../../../sdk/abi/src/clock/latch.rs"] +pub mod clock_latch; + /// The seqlock the panic console's descriptor and each CPU's counters are /// published through, driven by `tests/panic_console_publish.rs` and /// `tests/counters_round.rs`. diff --git a/kernel/loom/tests/clock_latch.rs b/kernel/loom/tests/clock_latch.rs new file mode 100644 index 00000000000..f7ab6c47074 --- /dev/null +++ b/kernel/loom/tests/clock_latch.rs @@ -0,0 +1,97 @@ +//! Loom: a reader of the clock page's latch gets one line whole, and never an +//! older one after a newer. +//! +//! The line is three words a reader turns into a time; half of one +//! publication and half of the next is a clock that steps. One writer +//! replaces it (the kernel's retune) while a reader on another CPU reads it +//! with no lock, and a reader never waits for the writer. The negative control +//! is the `clock-latch-fence-off` feature, which must red this file. +#![cfg(feature = "loom")] + +use kernel_loom::clock_latch::Latch; +use loom::sync::Arc; +use loom::thread; + +const WORDS: usize = 3; + +/// A bound so the two publications' twelve stores finish exploring; under it +/// the `clock-latch-fence-off` control still reaches a torn read. +const PREEMPTIONS: usize = 3; + +fn model(f: impl Fn() + Sync + Send + 'static) { + let mut builder = loom::model::Builder::new(); + builder.preemption_bound = Some(PREEMPTIONS); + builder.check(f); +} + +/// Two publications, so the reader can meet either copy being rewritten. +#[test] +fn a_read_is_one_line_whole() { + model(|| { + let latch = Arc::new(Latch::::new()); + latch.publish([1; WORDS]); + let writer = { + let latch = latch.clone(); + thread::spawn(move || { + latch.publish([2; WORDS]); + latch.publish([3; WORDS]); + }) + }; + let line = latch.read(); + assert!(line == [1; WORDS] || line == [2; WORDS] || line == [3; WORDS], "a read kept two lines' words: {line:?}"); + writer.join().unwrap(); + assert_eq!(latch.read(), [3; WORDS]); + }); +} + +#[test] +fn a_later_read_is_never_an_older_line() { + model(|| { + let latch = Arc::new(Latch::::new()); + latch.publish([1; WORDS]); + let writer = { + let latch = latch.clone(); + thread::spawn(move || latch.publish([2; WORDS])) + }; + let first = latch.read(); + let second = latch.read(); + assert!(second[0] >= first[0], "a later read went back to an older line: {first:?} then {second:?}"); + writer.join().unwrap(); + }); +} + +/// The counter a reader turns a line into a time with, modelled as memory the +/// writer advances only once the line that starts there is published. +fn superseded(read: impl Fn(&Latch, &loom::sync::atomic::AtomicU64) -> ([u64; WORDS], u64) + Sync + Send + 'static) { + use loom::sync::atomic::{AtomicU64, Ordering}; + model(move || { + let latch = Arc::new(Latch::::new()); + let counter = Arc::new(AtomicU64::new(0)); + latch.publish([1; WORDS]); + let writer = { + let (latch, counter) = (latch.clone(), counter.clone()); + thread::spawn(move || { + latch.publish([2; WORDS]); + counter.store(2, Ordering::Release); + }) + }; + let (line, now) = read(&latch, &counter); + assert!(now < 2 || line == [2; WORDS], "a reading at {now} extrapolated the superseded line {line:?}"); + writer.join().unwrap(); + }); +} + +/// A counter read inside the section, after the line's words, is read again +/// with the line when a publication lands between them. +#[test] +fn a_counter_read_inside_the_section_never_extrapolates_a_superseded_line() { + superseded(|latch, counter| latch.read_with(|| counter.load(loom::sync::atomic::Ordering::Acquire))); +} + +/// What the clock read before: the counter after the section, which a reader +/// preempted between the two pairs with a line replaced meanwhile. +#[test] +#[should_panic(expected = "extrapolated the superseded line")] +fn a_counter_read_after_the_section_can_extrapolate_a_superseded_line() { + superseded(|latch, counter| (latch.read(), counter.load(loom::sync::atomic::Ordering::Acquire))); +} diff --git a/kernel/pure/clock.rs b/kernel/pure/clock.rs deleted file mode 100644 index 38dc0e5b9bd..00000000000 --- a/kernel/pure/clock.rs +++ /dev/null @@ -1,53 +0,0 @@ -//! What the clock computes from counter ticks: their span in nanoseconds -//! ([`ticks_to_nanos`]), and the Unix second at boot that a wall-clock reading -//! taken at some counter value fixes ([`anchor`]). - -#![forbid(unsafe_code)] - -const NANOS_PER_SEC: u64 = 1_000_000_000; - -pub use toyos_abi::clock::ticks_to_nanos; - -/// Unix seconds at the counter's `boot`, from a reading of `secs` true when the -/// counter read `at`, which is after boot where the kernel read the clock and -/// before it where the loader did. Only the whole seconds between the two move -/// the answer. -pub const fn anchor(secs: u64, at: u64, boot: u64, period_fs: u64) -> u64 { - if at >= boot { - secs.saturating_sub(ticks_to_nanos(at - boot, period_fs) / NANOS_PER_SEC) - } else { - secs.saturating_add(ticks_to_nanos(boot - at, period_fs) / NANOS_PER_SEC) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - /// A one-nanosecond tick, so a tick count reads as nanoseconds. - const NS: u64 = 1_000_000; - const SECS: u64 = 1_993_799_665; - const BOOT: u64 = 50_000_000_000; - - #[test] - fn a_reading_after_boot_is_carried_back_to_it() { - assert_eq!(anchor(SECS, BOOT + 3_700_000_000, BOOT, NS), SECS - 3); - } - - #[test] - fn a_reading_before_boot_is_carried_forward_to_it() { - assert_eq!(anchor(SECS, BOOT - 2_500_000_000, BOOT, NS), SECS + 2); - } - - #[test] - fn a_reading_at_boot_is_boot() { - assert_eq!(anchor(SECS, BOOT, BOOT, NS), SECS); - } - - #[test] - fn a_tick_is_counted_at_its_period() { - // A 24 MHz counter: 41_666_666 fs a tick, so 24e6 ticks fall just short of a second. - assert_eq!(ticks_to_nanos(24_000_000, 41_666_666), 999_999_984); - assert_eq!(anchor(SECS, BOOT - 48_000_001, BOOT, 41_666_666), SECS + 2); - } -} diff --git a/kernel/pure/clock/mod.rs b/kernel/pure/clock/mod.rs new file mode 100644 index 00000000000..f3d4fd2b66a --- /dev/null +++ b/kernel/pure/clock/mod.rs @@ -0,0 +1,44 @@ +//! What the clock computes: the Unix second at boot that a wall-clock reading +//! fixes ([`anchor`]), and the counter held to a reference timer +//! ([`reference`]). + +#![forbid(unsafe_code)] + +pub mod reference; + +const NANOS_PER_SEC: u64 = 1_000_000_000; + +/// Unix seconds at boot, from a reading of `secs` true when the clock read +/// `at`, which is after boot where the kernel read the wall clock and before +/// it where the loader did; both are stamps, nanoseconds since the counter's +/// zero. Only the whole seconds between the two move the answer. +pub const fn anchor(secs: u64, at: u64, boot: u64) -> u64 { + if at >= boot { + secs.saturating_sub((at - boot) / NANOS_PER_SEC) + } else { + secs.saturating_add((boot - at) / NANOS_PER_SEC) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + const SECS: u64 = 1_993_799_665; + const BOOT: u64 = 50_000_000_000; + + #[test] + fn a_reading_after_boot_is_carried_back_to_it() { + assert_eq!(anchor(SECS, BOOT + 3_700_000_000, BOOT), SECS - 3); + } + + #[test] + fn a_reading_before_boot_is_carried_forward_to_it() { + assert_eq!(anchor(SECS, BOOT - 2_500_000_000, BOOT), SECS + 2); + } + + #[test] + fn a_reading_at_boot_is_boot() { + assert_eq!(anchor(SECS, BOOT, BOOT), SECS); + } +} diff --git a/kernel/pure/clock/reference.rs b/kernel/pure/clock/reference.rs new file mode 100644 index 00000000000..30dadc2e6d3 --- /dev/null +++ b/kernel/pure/clock/reference.rs @@ -0,0 +1,1018 @@ +//! A free-running counter held to a reference timer: what a reading of the +//! reference between two of the counter's says, the rate two such readings +//! measure and how far it can be off, and whether the counter still keeps the +//! rate a clock was given. +//! +//! Every bound on a rate and a step is derived from the readings themselves: +//! the counter readings either side of a reference reading bracket the moment +//! it latched, and the reference's tick bounds where in that tick it was, so a +//! bound that fails is a counter, a reference or a read that did not do what +//! it says. The one estimate is a third reference read by its interrupts +//! ([`Edges`]): their least latency, and its crystal's drift carried across a +//! span, which only choose between a step and none ([`third_says`]). + +use toyos_abi::clock::MULT_SHIFT; + +/// Parts per billion in one. +pub const PPB: u64 = 1_000_000_000; + +const FS_PER_NS: u128 = 1_000_000; + +/// A reference timer: its tick, and how far its rate may wander from that tick +/// by its own specification, which every check that compares rates over time +/// allows on top of what the readings bound: `tolerance_ppb` over a span of +/// `settled_ns` or more, and `short_tolerance_ppb` over a shorter one. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Reference { + pub period_fs: u64, + pub tolerance_ppb: u64, + pub settled_ns: u64, + pub short_tolerance_ppb: u64, +} + +impl Reference { + /// The span from `a` to `b` the reference counted, zero where it read + /// backwards. + pub const fn span_ns(&self, a: Sample, b: Sample) -> u64 { + self.ns_of(b.reference.saturating_sub(a.reference)) + } + + /// `ticks` of the reference, in nanoseconds. + pub const fn ns_of(&self, ticks: u64) -> u64 { + (ticks as u128 * self.period_fs as u128 / FS_PER_NS) as u64 + } + + /// What the reference's own rate may be off by over `span_ns`. + pub const fn tolerance_over(&self, span_ns: u64) -> u64 { + if span_ns >= self.settled_ns { self.tolerance_ppb } else { self.short_tolerance_ppb } + } +} + +/// One reading of the reference, latched at some moment the counter read +/// between `before` and `after`. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Sample { + pub before: u64, + pub after: u64, + pub reference: u64, +} + +impl Sample { + /// The counter ticks the reading's latch could have been anywhere in. + pub const fn width(&self) -> u64 { + self.after - self.before + } + + /// The bracket's midpoint, doubled so it stays whole. + const fn mid2(&self) -> u128 { + self.before as u128 + self.after as u128 + } +} + +/// A rate two samples measure: nanoseconds per counter tick in the clock's +/// fixed point (`toyos_abi::clock::Line::mult`), at most `bound_ppb` off, over +/// `span_ns` of the reference. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Rate { + pub mult: u64, + pub bound_ppb: u64, + pub span_ns: u64, +} + +/// Why two samples measure no rate. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Unmeasurable { + /// The later sample's counter or reference reads no later than the + /// earlier's. + NotLater, + /// The brackets are a quarter or more of the span between them, which + /// bounds nothing worth keeping. + TooWide, +} + +/// The counter and the reference over one or more segments, each from one +/// sample to a later one: what a rate is measured over, and what a held span +/// is checked over. Segments add, so a span can leave out the stretches the +/// reference cannot be held across. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Span { + /// The counter's span between the brackets' midpoints, doubled. + counter2: i128, + /// The reference's ticks. + ticks: i128, + /// The brackets' widths at every segment's ends. + widths: u128, + /// Segments, each of which knows its reference span to one tick. + segments: u64, + /// The shortest segment's reference span, for the reference's tolerance. + shortest_ns: u64, +} + +impl Span { + pub const EMPTY: Self = Self { counter2: 0, ticks: 0, widths: 0, segments: 0, shortest_ns: u64::MAX }; + + pub fn between(reference: Reference, a: Sample, b: Sample) -> Self { + Self { + counter2: b.mid2() as i128 - a.mid2() as i128, + ticks: b.reference as i128 - a.reference as i128, + widths: a.width() as u128 + b.width() as u128, + segments: 1, + shortest_ns: reference.span_ns(a, b), + } + } + + pub fn then(self, more: Self) -> Self { + Self { + counter2: self.counter2 + more.counter2, + ticks: self.ticks + more.ticks, + widths: self.widths + more.widths, + segments: self.segments + more.segments, + shortest_ns: self.shortest_ns.min(more.shortest_ns), + } + } + + /// The reference's count over the span, in nanoseconds; zero for one + /// that read backwards. + pub fn reference_ns(&self, reference: Reference) -> u64 { + (self.ticks.max(0) as u128 * reference.period_fs as u128 / FS_PER_NS) as u64 + } +} + +/// The rate from `a` to `b`, which is later. +pub fn rate(reference: Reference, a: Sample, b: Sample) -> Result { + rate_over(reference, Span::between(reference, a, b)) +} + +/// The rate over `span`. +/// +/// The counter moved `Δc` between the latches, known to the brackets' half +/// widths, while the reference moved `Δn` ticks, known to one a segment: so +/// the relative error is `u = Σw / (2 Δc)` plus `r = segments / Δn`, each +/// taken against the measured span, and with the second-order terms of both +/// the ratio and those spans `(u + r)·(1 + 6·max(u, r))`, which holds while +/// both are under a quarter. `mult` is rounded down, which is one unit of it +/// more. +pub fn rate_over(reference: Reference, span: Span) -> Result { + if span.counter2 <= 0 || span.ticks <= 0 { + return Err(Unmeasurable::NotLater); + } + let counter2 = span.counter2 as u128; + let ticks = span.ticks as u128; + let span_fs = ticks * reference.period_fs as u128; + if span.widths * 4 >= counter2 || ticks < 4 * span.segments as u128 { + return Err(Unmeasurable::TooWide); + } + let u = (span.widths * PPB as u128).div_ceil(counter2); + let r = (span.segments as u128 * PPB as u128).div_ceil(ticks); + let first = u + r; + // ns per tick, times 2^32: `(span_fs / 10⁶) / (counter2 / 2) · 2^32`. + let mult = (span_fs << 33) / (FS_PER_NS * counter2); + let bound = first + (6 * first * u.max(r)).div_ceil(PPB as u128) + (PPB as u128).div_ceil(mult.max(1)); + Ok(Rate { mult: mult as u64, bound_ppb: bound as u64, span_ns: (span_fs / FS_PER_NS) as u64 }) +} + +/// How the counter and the reference agree over a span, at a given rate. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Agreement { + /// The span the reference counted, in nanoseconds. + pub reference_ns: u64, + /// The counter's span at the rate, less the reference's. + pub apart_ns: i64, + /// What the readings and the rate's tolerance allow `apart_ns` to be. + pub allowed_ns: u64, +} + +impl Agreement { + pub const fn holds(&self) -> bool { + self.apart_ns.unsigned_abs() <= self.allowed_ns + } +} + +/// The counter's span from `a` to `b` at `mult`, against the reference's. +pub fn agree(reference: Reference, a: Sample, b: Sample, mult: u64, tolerance_ppb: u64) -> Agreement { + agree_over(reference, Span::between(reference, a, b), mult, tolerance_ppb) +} + +/// The counter's span over `span` at `mult`, against the reference's. +/// +/// Allowed: the brackets' half widths at `mult`, one reference tick a +/// segment, and `tolerance_ppb` of the span, which is what the rate itself +/// may be off by; two nanoseconds a segment more cover the rounding of each +/// conversion. A segment whose reference reads behind its start counts as a +/// span of zero, so a reference that went backwards is a disagreement. +pub fn agree_over(reference: Reference, span: Span, mult: u64, tolerance_ppb: u64) -> Agreement { + let nanos = |ticks2: u128| (ticks2 * mult as u128) >> 33; + let counter_ns = if span.counter2 >= 0 { + nanos(span.counter2 as u128) as i128 + } else { + -(nanos(span.counter2.unsigned_abs()) as i128) + }; + let reference_ns = span.reference_ns(reference); + let readings = nanos(span.widths) as u64; + let tick = (reference.period_fs as u128).div_ceil(FS_PER_NS) as u64; + let rated = (reference_ns as u128 * tolerance_ppb as u128).div_ceil(PPB as u128) as u64; + Agreement { + reference_ns, + apart_ns: (counter_ns - reference_ns as i128).clamp(i64::MIN as i128, i64::MAX as i128) as i64, + allowed_ns: readings + span.segments * (tick + 2) + rated, + } +} + +/// Where a calibration's samples left the straight line between its first +/// and last. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Calibration { + Unmeasurable(Unmeasurable), + /// The sample at `index` sits off the line by more than the readings + /// allow: something moved the counter or the reference inside the window. + Disturbed { index: usize, agreement: Agreement }, +} + +/// The rate over `samples`, first to last, once every sample between is shown +/// to sit on that line within what the readings allow: a step inside the +/// window larger than the readings resolve, which would skew the rate, is +/// refused rather than averaged in. +pub fn calibrate(reference: Reference, samples: &[Sample]) -> Result { + let (Some(&first), Some(&last)) = (samples.first(), samples.last()) else { + return Err(Calibration::Unmeasurable(Unmeasurable::NotLater)); + }; + let fit = rate(reference, first, last).map_err(Calibration::Unmeasurable)?; + for (index, &sample) in samples.iter().enumerate().skip(1) { + let agreement = agree(reference, first, sample, fit.mult, fit.bound_ppb); + if !agreement.holds() { + return Err(Calibration::Disturbed { index, agreement }); + } + } + Ok(fit) +} + +/// The rate a clock keeps, and whether it was measured, so the watch may +/// replace it with a better measurement, or stated by the hardware. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Kept { + pub mult: u64, + pub bound_ppb: u64, + pub measured: bool, +} + +/// The first span [`Watch`] reports at, and the factor between reports. +const FIRST_REPORT_NS: u64 = 1_000_000_000; +const REPORT_FACTOR: u64 = 2; + +/// A counter's agreement with the reference over the life of a clock kept on +/// it: each sample is held against the one before it, which catches a step, +/// and against every held span since the first, which catches a rate that +/// has moved. A span the reference cannot be held across is left out whole. +#[derive(Clone, Copy, Debug)] +pub struct Watch { + reference: Reference, + /// The held segments before the one now open. + closed: Span, + /// Where the open segment starts. + opened: Sample, + last: Sample, + kept: Kept, + next_report_ns: u64, +} + +/// What a sample showed, where it held. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Report { + /// The rate measured over every held span. + pub measured: Rate, + /// The rate the clock keeps from here: a better measurement where the kept + /// one was measured, and otherwise the one it kept. + pub kept: Kept, + /// Whether `kept` replaced the rate the clock was keeping. + pub retuned: bool, +} + +/// What a sample showed, where it did not hold: the counter and the reference +/// disagree, and the samples cannot say which of the two moved. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Failure { + /// Against the sample before it: one of them stepped. + Step { from: Sample, to: Sample, agreement: Agreement }, + /// Over every held span: they count at rates further apart than the kept + /// rate's bound and the reference's tolerance. + Drift { since: Sample, to: Sample, agreement: Agreement }, +} + +impl Watch { + pub const fn new(reference: Reference, anchor: Sample, kept: Kept) -> Self { + Self { reference, closed: Span::EMPTY, opened: anchor, last: anchor, kept, next_report_ns: FIRST_REPORT_NS } + } + + pub const fn kept(&self) -> Kept { + self.kept + } + + /// The sample taken last, every one before it having held. + pub const fn last(&self) -> Sample { + self.last + } + + /// Take `sample`, taken after every sample before it. `held` says whether + /// the reference can be held to the counter across the span from the last + /// sample; one that cannot is left out, and checks nothing. `Some` at each + /// report: the first at a second of held spans, then at twice the span of + /// the one before. + pub fn take(&mut self, sample: Sample, held: bool) -> Result, Failure> { + if !held { + if self.last != self.opened { + self.closed = self.closed.then(Span::between(self.reference, self.opened, self.last)); + } + self.opened = sample; + self.last = sample; + return Ok(None); + } + let tolerance = |span: &Span| self.kept.bound_ppb + self.reference.tolerance_over(span.shortest_ns); + let since_last = Span::between(self.reference, self.last, sample); + let step = agree_over(self.reference, since_last, self.kept.mult, tolerance(&since_last)); + if !step.holds() { + return Err(Failure::Step { from: self.last, to: sample, agreement: step }); + } + let whole = self.closed.then(Span::between(self.reference, self.opened, sample)); + let drift = agree_over(self.reference, whole, self.kept.mult, tolerance(&whole)); + if !drift.holds() { + return Err(Failure::Drift { since: self.opened, to: sample, agreement: drift }); + } + self.last = sample; + if drift.reference_ns < self.next_report_ns { + return Ok(None); + } + while self.next_report_ns <= drift.reference_ns { + self.next_report_ns = self.next_report_ns.saturating_mul(REPORT_FACTOR); + } + let Ok(measured) = rate_over(self.reference, whole) else { + return Ok(None); + }; + let retuned = self.kept.measured && measured.bound_ppb < self.kept.bound_ppb; + if retuned { + self.kept = Kept { mult: measured.mult, bound_ppb: measured.bound_ppb, measured: true }; + } + Ok(Some(Report { measured, kept: self.kept, retuned })) + } +} + +/// How far one CPU's counter can be from another's, from round trips of a +/// cache line between them: each side writes the counter it read just before +/// its store, and the other reads its own just after it sees the store. +/// +/// With `δ` the second counter less the first at one instant, a trip out +/// reads `δ` plus the trip's latency, never less, and a trip back reads `-δ` +/// plus its own; so the least of each bounds `δ` to `[-least_back, +/// least_out]`. An interval that excludes zero is an offset proven. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Offset { + pub least_out: i64, + pub least_back: i64, +} + +impl Offset { + /// `[lo, hi]` for `δ`, or the offset the trips prove, in ticks: negative + /// for a second counter behind the first. + pub const fn bound(&self) -> Result<(i64, i64), i64> { + if self.least_out < 0 { + Err(self.least_out) + } else if self.least_back < 0 { + Err(-self.least_back) + } else { + Ok((-self.least_back, self.least_out)) + } + } +} + +/// The 64-bit reading of a reference whose counter is `bits` wide: the value +/// with `low` as its low bits nearest `predicted`, the earlier 64-bit reading +/// plus the ticks the counter says have passed since. Right while that +/// prediction is within half the counter's range. +pub const fn extend(predicted: u64, low: u64, bits: u32) -> u64 { + let range = 1u64 << bits; + let off = low.wrapping_sub(predicted) & (range - 1); + if off >= range / 2 { + predicted.wrapping_add(off).wrapping_sub(range) + } else { + predicted.wrapping_add(off) + } +} + +/// What a second reference says of a step the counter and the first +/// reference disagree on, over the same span. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Arbitration { + /// The second reference agrees with the counter: the first moved. + FirstMoved, + /// The second reference disagrees with the counter by what the first + /// does, within both agreements' allowances: the counter moved, or both + /// references did together. + CounterMoved, + /// The two references disagree with the counter and with each other. + AllDisagree, +} + +/// `first` and `second`: the counter against each reference over one span, +/// the first's an agreement that failed. +pub const fn arbitrate(first: Agreement, second: Agreement) -> Arbitration { + if second.holds() { + Arbitration::FirstMoved + } else if first.apart_ns.abs_diff(second.apart_ns) <= first.allowed_ns + second.allowed_ns { + Arbitration::CounterMoved + } else { + Arbitration::AllDisagree + } +} + +/// The RTC's edges the sampler has read, the latest [`EDGES_KEPT`]: each +/// edge's count and the counter as its interrupt was taken, which is the edge +/// plus the interrupt's latency, never earlier. +#[derive(Clone, Copy, Debug)] +pub struct Edges { + kept: [(u64, u64); EDGES_KEPT], + len: usize, +} + +pub const EDGES_KEPT: usize = 16; +/// The edges either side of a span its least-latency edge is taken from. +pub const EDGES_EACH_SIDE: usize = 3; + +impl Edges { + pub const fn new() -> Self { + Self { kept: [(0, 0); EDGES_KEPT], len: 0 } + } + + /// Keep the edge `seen` as the counter read `at`, where it is later than + /// the latest kept. + pub fn push(&mut self, seen: u64, at: u64) { + if self.len > 0 && self.kept[self.len - 1].0 >= seen { + return; + } + if self.len == EDGES_KEPT { + self.kept.copy_within(1.., 0); + self.len -= 1; + } + self.kept[self.len] = (seen, at); + self.len += 1; + } + + /// Drop the edges taken before the counter read `at`, where the counter + /// stepped: an edge from before a step and one from after it measure the + /// step, not the RTC crystal's drift, so neither a drift base nor a side + /// of a later span may reach across it. + pub fn forget_before(&mut self, at: u64) { + let kept = self.kept[..self.len].iter().position(|&(_, taken)| taken >= at).unwrap_or(self.len); + self.kept.copy_within(kept..self.len, 0); + self.len -= kept; + } + + /// Whether [`EDGES_EACH_SIDE`] edges have come after the counter read `to`. + pub fn complete_after(&self, to: u64) -> bool { + self.kept[..self.len].iter().filter(|&&(_, at)| at >= to).count() >= EDGES_EACH_SIDE + } + + /// The counter's span, at `mult`, less the RTC's seconds, from before + /// `from` to after `to`: from the least-latency of the [`EDGES_EACH_SIDE`] + /// edges before `from` to that of those after `to`. An edge reads late by + /// its interrupt's latency and never early, so the least on each side is + /// the nearest its edge. Less the RTC crystal's own drift from the + /// counter, where edges at least [`DRIFT_BASE_NS`] before give it: the + /// least-latency of the oldest kept to that before `from`, carried on + /// across the span. `None` without an edge either side. + pub fn apart_across(&self, from: u64, to: u64, mult: u64) -> Option { + let kept = &self.kept[..self.len]; + let &(seen_ref, at_ref) = kept.iter().rev().find(|&&(_, at)| at <= from)?; + let offset = |&(seen, at): &(u64, u64)| { + let ns = ((at as i128 - at_ref as i128) * mult as i128) >> MULT_SHIFT; + (ns - (seen as i128 - seen_ref as i128) * 1_000_000_000, at) + }; + let least = |edges: &mut dyn Iterator| edges.map(offset).min_by_key(|&(ns, _)| ns); + let before = least(&mut kept.iter().rev().filter(|&&(_, at)| at <= from).take(EDGES_EACH_SIDE))?; + let after = least(&mut kept.iter().filter(|&&(_, at)| at >= to).take(EDGES_EACH_SIDE))?; + let base_ticks = ((DRIFT_BASE_NS as u128) << MULT_SHIFT) / mult as u128; + let older = least(&mut kept.iter().filter(|&&(_, at)| (at as u128) + base_ticks <= before.1 as u128).take(EDGES_EACH_SIDE)); + let drift = older.map_or(0, |older| (before.0 - older.0) * (after.1 as i128 - before.1 as i128) / (before.1 as i128 - older.1 as i128)); + Some((after.0 - before.0 - drift) as i64) + } +} + +/// How far back edges must reach for [`Edges::apart_across`] to measure the +/// RTC crystal's drift from the counter: over this, a few microseconds of +/// latency are a part per million. +pub const DRIFT_BASE_NS: u64 = 4_000_000_000; + +impl Default for Edges { + fn default() -> Self { + Self::new() + } +} + +/// Which of the counter's step and no step at all a third reference, on +/// another oscillator, reads nearest: within a quarter of the step of +/// either, or of neither. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Third { + /// It reads the step: the counter moved. + Counter, + /// It reads no step: the two references moved together. + References, + /// It reads neither, or has no edge either side of the span yet. + Unsure, +} + +pub fn third_says(step_ns: i64, apart_ns: Option) -> Third { + let Some(apart) = apart_ns else { return Third::Unsure }; + let zone = step_ns.unsigned_abs() / 4; + if apart.abs_diff(step_ns) <= zone { + Third::Counter + } else if apart.unsigned_abs() <= zone { + Third::References + } else { + Third::Unsure + } +} + +/// What the clock does about a step of its counter: the boot's first, forward, +/// is slewed away; a second, or one back, leaves the counter for the +/// reference. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Response { + Slew, + Demote, +} + +pub const fn respond(step_ns: i64, slewed_before: u32) -> Response { + if step_ns > 0 && slewed_before == 0 { Response::Slew } else { Response::Demote } +} + +/// The most a slew moves the clock's rate: 500 ppm, the frequency tolerance +/// NTP disciplines a clock within (RFC 5905, Appendix A.1.1, `MAXFREQ`), so +/// a reader that tolerates an NTP-disciplined clock tolerates this one. A +/// step of `d` is slewed at this rate for `2000 d` less the last millisecond +/// or so, which halves each period: 1.7 ms in about 12 s. +pub const SLEW_MOST_PPB: i64 = 500_000; +/// A debt this small is paid. +const SLEW_SETTLED_NS: u64 = 1_000; + +/// How far the clock stands ahead of the reference's time, which a slew pays +/// off by running the clock slower than the kept rate. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Slew { + debt_ns: i64, +} + +impl Slew { + pub const fn new(debt_ns: i64) -> Self { + Self { debt_ns } + } + + pub const fn debt_ns(&self) -> i64 { + self.debt_ns + } + + pub const fn settled(&self) -> bool { + self.debt_ns.unsigned_abs() <= SLEW_SETTLED_NS + } + + /// `ticks` the clock ran at `ran` where the reference's time runs at + /// `kept`: what it fell behind is paid off the debt. + pub fn ran(&mut self, ticks: u64, kept: u64, ran: u64) { + let line = |mult| ((ticks as u128 * mult as u128) >> MULT_SHIFT) as i128; + self.debt_ns -= (line(kept) - line(ran)) as i64; + } + + /// The rate to run at until the next sample, due in `period_ns`: the debt + /// paid over two periods, at most [`SLEW_MOST_PPB`] from `kept`, and `kept` + /// once settled. A sample up to a period late pays no more than the debt; + /// a later one pays more, which the next pays back. + pub fn mult(&self, kept: u64, period_ns: u64) -> u64 { + if self.settled() { + return kept; + } + let ppb = (self.debt_ns as i128 * 1_000_000_000 / (2 * period_ns as i128)).clamp(-(SLEW_MOST_PPB as i128), SLEW_MOST_PPB as i128); + (kept as i128 - kept as i128 * ppb / 1_000_000_000) as u64 + } +} + +#[cfg(test)] +mod tests { + use super::*; + use alloc::vec::Vec; + + /// The AMD bench's HPET: 14.318 MHz, and the HPET specification's 0.05% + /// from 1 ms and 0.2% under it. + const HPET: Reference = + Reference { period_fs: 69_841_279, tolerance_ppb: 500_000, settled_ns: 1_000_000, short_tolerance_ppb: 2_000_000 }; + /// A counter at 1796.6248 MHz. + const HZ: u128 = 1_796_624_800; + /// What a read of the reference costs in counter ticks: 1.2 µs. + const READ: u64 = 2_156; + + /// The sample a perfect counter at `hz` takes when the reference reads + /// `n`, the latch landing `at` ticks into a bracket `READ` wide, the + /// counter `step` ticks off from where it should be. + fn sample_at(hz: u128, n: u64, at: u64, step: i64) -> Sample { + let true_counter = (n as u128 * HPET.period_fs as u128 * hz / 1_000_000_000_000_000) as u64; + let latch = true_counter.wrapping_add_signed(step); + Sample { before: latch - at, after: latch - at + READ, reference: n } + } + + fn sample(n: u64) -> Sample { + sample_at(HZ, n, READ / 2, 0) + } + + fn hz_of(mult: u64) -> u128 { + (1_000_000_000u128 << 32) / mult as u128 + } + + fn ppb_apart(mult: u64, hz: u128) -> u128 { + hz_of(mult).abs_diff(hz) * PPB as u128 / hz + } + + const MS_50: u64 = 715_909; + const SECOND: u64 = 14_318_180; + + #[test] + fn a_rate_is_within_its_bound_wherever_the_latches_land() { + for (at_a, at_b) in [(0, READ), (READ, 0), (READ / 2, READ / 2), (0, 0)] { + let rate = rate(HPET, sample_at(HZ, 1_000, at_a, 0), sample_at(HZ, 1_000 + MS_50, at_b, 0)).unwrap(); + assert!(ppb_apart(rate.mult, HZ) <= rate.bound_ppb as u128, "{rate:?} at {at_a}, {at_b}"); + // Two 1.2 µs reads over 50 ms: 24 ppm, and the reference's tick 1.4. + assert!((24_000..27_000).contains(&rate.bound_ppb), "{rate:?}"); + } + } + + /// The same reads over ten minutes bound the rate to 6 ppb. + #[test] + fn a_long_span_bounds_the_rate_tightly() { + let rate = rate(HPET, sample_at(HZ, 0, 0, 0), sample_at(HZ, 600 * SECOND, READ, 0)).unwrap(); + assert!(ppb_apart(rate.mult, HZ) <= rate.bound_ppb as u128); + assert!(rate.bound_ppb <= 6, "{rate:?}"); + assert_eq!(rate.span_ns / 1_000_000_000, 600); + } + + #[test] + fn samples_out_of_order_or_too_wide_measure_nothing() { + assert_eq!(rate(HPET, sample(10_000), sample(1_000)), Err(Unmeasurable::NotLater)); + assert_eq!(rate(HPET, sample(1_000), sample(1_000)), Err(Unmeasurable::NotLater)); + let near = sample(100_000); + let wide = Sample { before: near.before - 1_000_000, after: near.after + 1_000_000, ..near }; + assert_eq!(rate(HPET, sample(99_000), wide), Err(Unmeasurable::TooWide)); + } + + fn window(steps: &[(usize, i64)]) -> Vec { + (0..=5u64) + .map(|i| { + let step = steps.iter().filter(|&&(at, _)| i as usize >= at).map(|&(_, by)| by).sum(); + sample_at(HZ, 1_000 + i * MS_50 / 5, (i * 431) % READ, step) + }) + .collect() + } + + #[test] + fn a_still_window_calibrates() { + let rate = calibrate(HPET, &window(&[])).unwrap(); + assert!(ppb_apart(rate.mult, HZ) <= rate.bound_ppb as u128); + } + + /// The 1.7 ms the AMD bench's TSC once read ahead of its HPET, and a step + /// of 5 µs, a tenth of a 50 ms window's 100 ppm, either way, wherever in the + /// window it lands: each is refused, not averaged into the rate. + #[test] + fn a_step_inside_the_window_is_refused() { + for step in [3_054_262i64, 8_983, -8_983] { + for at in 1..=5 { + match calibrate(HPET, &window(&[(at, step)])) { + Err(Calibration::Disturbed { agreement, .. }) => assert!(!agreement.holds()), + other => panic!("a {step}-tick step at sample {at} calibrated as {other:?}"), + } + } + } + } + + fn watch(measured: bool) -> Watch { + let first = calibrate(HPET, &window(&[])).unwrap(); + Watch::new(HPET, sample(1_000), Kept { mult: first.mult, bound_ppb: first.bound_ppb, measured }) + } + + /// A second's samples, each across a reference read of its own. + fn run(watch: &mut Watch, hz: u128, seconds: core::ops::Range, step: i64) -> Result, Failure> { + let mut reports = Vec::new(); + for s in seconds { + if let Some(report) = watch.take(sample_at(hz, 1_000 + s * SECOND, (s * 977) % READ, step), true)? { + reports.push(report); + } + } + Ok(reports) + } + + /// A measured rate is replaced at each report by a better one, and the + /// one an hour keeps is within its bound of the counter's, which is under + /// 2 ppb. + #[test] + fn a_measured_rate_is_refined() { + let mut watch = watch(true); + let reports = run(&mut watch, HZ, 1..3_601, 0).unwrap(); + let spans: Vec = reports.iter().map(|r| r.measured.span_ns / 1_000_000_000).collect(); + assert_eq!(spans, [1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048]); + assert!(reports.iter().all(|r| r.retuned)); + let kept = watch.kept(); + assert!(ppb_apart(kept.mult, HZ) <= kept.bound_ppb as u128); + assert!(kept.bound_ppb < 10, "{kept:?}"); + } + + /// A stated rate is reported against and never replaced. + #[test] + fn a_stated_rate_is_kept() { + let mut watch = watch(false); + let before = watch.kept(); + let reports = run(&mut watch, HZ, 1..100, 0).unwrap(); + assert!(!reports.is_empty() && reports.iter().all(|r| !r.retuned && r.kept == before)); + } + + /// A 1.7 ms step at runtime fails the sample after it, as a step. + #[test] + fn a_step_at_runtime_fails() { + let mut watch = watch(true); + run(&mut watch, HZ, 1..30, 0).unwrap(); + match run(&mut watch, HZ, 30..31, 3_054_262) { + Err(Failure::Step { agreement, .. }) => assert!(agreement.apart_ns > 1_600_000), + other => panic!("{other:?}"), + } + } + + /// A counter whose rate moves by 0.1% fails within seconds. One that moves + /// by 0.04%, inside the reference's own tolerance, is measured and kept. + #[test] + fn a_rate_that_moves_past_the_reference_tolerance_fails() { + let mut watch = watch(true); + run(&mut watch, HZ, 1..10, 0).unwrap(); + let faster = HZ + HZ / 1_000; + // The rate's change shows against the anchor as the seconds pass. + let at = (1_000 + 10 * SECOND) as u128 * HPET.period_fs as u128; + let offset = (at * HZ / 1_000_000_000_000_000) as i64 - (at * faster / 1_000_000_000_000_000) as i64; + let failed = run(&mut watch, faster, 10..20, offset); + assert!(matches!(failed, Err(Failure::Drift { .. }) | Err(Failure::Step { .. })), "{failed:?}"); + + let mut tolerant = watch_fresh(); + let slightly = HZ + HZ * 4 / 10_000; + assert!(run(&mut tolerant, slightly, 1..20, 0).is_ok()); + } + + fn watch_fresh() -> Watch { + Watch::new(HPET, sample(1_000), Kept { mult: (1_000_000_000u128 << 32).div_ceil(HZ) as u64, bound_ppb: 10, measured: true }) + } + + /// Two CPUs' samples 100 µs apart, each read slowed to 20 µs by an SMI and + /// latched at opposite ends of its bracket: a still counter, which the + /// readings' own widths must allow, since the reference's tolerance over + /// that span is 50 ns. + #[test] + fn close_samples_hold_across_wide_reads() { + let mut watch = watch(true); + run(&mut watch, HZ, 1..3, 0).unwrap(); + let wide = 35_932; + let n = watch.last().reference + 1_432; + let latch = |n: u64| sample_at(HZ, n, 0, 0).before; + let early = Sample { before: latch(n), after: latch(n) + wide, reference: n }; + let late = Sample { before: latch(n + 1_432) - wide, after: latch(n + 1_432), reference: n + 1_432 }; + assert_eq!(watch.take(early, true).map(|_| ()), Ok(())); + assert_eq!(watch.take(late, true).map(|_| ()), Ok(())); + } + + /// A reference that stood still for 30 s while the counter ran: held, the + /// span reads as a step; left out, it is none, and the rate over the held + /// spans either side is the counter's. + #[test] + fn a_span_left_out_checks_nothing() { + let mut watch = watch(true); + run(&mut watch, HZ, 1..20, 0).unwrap(); + // The reference resumes where it stopped; the counter is 30 s on. + let stood = (30 * SECOND as u128 * HPET.period_fs as u128 * HZ / 1_000_000_000_000_000) as i64; + let resumed = sample_at(HZ, 1_000 + 20 * SECOND, 0, stood); + assert!(matches!(watch.take(resumed, true), Err(Failure::Step { .. })), "the stood reference reads as a step when held"); + let mut watch = watch_after(20); + assert_eq!(watch.take(resumed, false), Ok(None)); + let reports = run(&mut watch, HZ, 21..300, stood).unwrap(); + let last = reports.last().expect("held spans past the gap report"); + assert!(ppb_apart(last.measured.mult, HZ) <= last.measured.bound_ppb as u128, "{last:?}"); + assert!(last.measured.span_ns / 1_000_000_000 >= 256); + } + + fn watch_after(seconds: u64) -> Watch { + let mut watch = watch(true); + run(&mut watch, HZ, 1..seconds, 0).unwrap(); + watch + } + + /// A reference that reads backwards is a disagreement, never a span. + #[test] + fn a_reference_that_goes_back_fails() { + let mut watch = watch(true); + run(&mut watch, HZ, 1..5, 0).unwrap(); + let back = Sample { reference: watch.last().reference - 10, ..sample(1_000 + 6 * SECOND) }; + assert!(watch.take(back, true).is_err()); + } + + #[test] + fn round_trips_bound_an_offset_or_prove_one() { + assert_eq!(Offset { least_out: 120, least_back: 80 }.bound(), Ok((-80, 120))); + assert_eq!(Offset { least_out: -30, least_back: 200 }.bound(), Err(-30)); + assert_eq!(Offset { least_out: 200, least_back: -45 }.bound(), Err(45)); + } + + /// The PM timer, at 3.579545 MHz, across the same spans as the HPET. + const PM: Reference = + Reference { period_fs: 279_365_114, tolerance_ppb: 500_000, settled_ns: 1_000_000, short_tolerance_ppb: 2_000_000 }; + + /// The PM timer's sample a perfect counter takes at `ns`, its step `step`. + fn pm_at(ns: u64, step: i64) -> Sample { + let n = (ns as u128 * 1_000_000 / PM.period_fs as u128) as u64; + let counter = (n as u128 * PM.period_fs as u128 * HZ / 1_000_000_000_000_000) as u64; + let latch = counter.wrapping_add_signed(step); + Sample { before: latch - READ / 2, after: latch - READ / 2 + READ, reference: n } + } + + fn ns_of(n: u64) -> u64 { + (n as u128 * HPET.period_fs as u128 / 1_000_000) as u64 + } + + /// The AMD's 1.7 ms over 680 ms: as the TSC's own step, both references + /// read it alike; as the HPET's stall, the PM timer reads the TSC right; + /// as a step of each by a different size, nothing agrees. + #[test] + fn a_second_reference_tells_which_moved() { + let mult = ((1_000_000_000u128 << 32) / HZ) as u64; + let tolerance = 500_000; + let (n0, n1) = (100 * SECOND, 100 * SECOND + 9_739_369); + let step = 3_096_525i64; + let hpet = |n, step| sample_at(HZ, n, READ / 2, step); + let tsc_moved = arbitrate( + agree(HPET, hpet(n0, 0), hpet(n1, step), mult, tolerance), + agree(PM, pm_at(ns_of(n0), 0), pm_at(ns_of(n1), step), mult, tolerance), + ); + assert_eq!(tsc_moved, Arbitration::CounterMoved); + let stalled = n1 - 24_675; + let hpet_moved = arbitrate( + agree(HPET, hpet(n0, 0), Sample { reference: stalled, ..hpet(n1, 0) }, mult, tolerance), + agree(PM, pm_at(ns_of(n0), 0), pm_at(ns_of(n1), 0), mult, tolerance), + ); + assert_eq!(hpet_moved, Arbitration::FirstMoved); + let neither = arbitrate( + agree(HPET, hpet(n0, 0), hpet(n1, step), mult, tolerance), + agree(PM, pm_at(ns_of(n0), 0), pm_at(ns_of(n1), step / 3), mult, tolerance), + ); + assert_eq!(neither, Arbitration::AllDisagree); + } + + #[test] + fn a_32_bit_reference_is_extended_across_its_wrap() { + assert_eq!(extend(0x1_ffff_fff0, 0x0000_0010, 32), 0x2_0000_0010); + assert_eq!(extend(0x2_0000_0010, 0xffff_fff0, 32), 0x1_ffff_fff0); + assert_eq!(extend(0x5_1234_5678, 0x1234_5679, 32), 0x5_1234_5679); + // The ACPI PM timer's 24 bits, across their wrap and back. + assert_eq!(extend(0x03ff_fff0, 0x0010, 24), 0x0400_0010); + assert_eq!(extend(0x0400_0010, 0x00ff_fff0, 24), 0x03ff_fff0); + } + + /// A 1 GHz counter, so ticks are nanoseconds. + const NS: u64 = 1 << MULT_SHIFT; + + /// Edges a second apart on the RTC, each read as late as its latency; + /// the counter steps by `step` at `step_at`. + fn edges(latency_ns: &[u64], step_at: u64, step: u64) -> Edges { + let mut edges = Edges::new(); + for (n, latency) in latency_ns.iter().enumerate() { + let edge = 10_000_000_000 + n as u64 * 1_000_000_000; + edges.push(n as u64 + 1, edge + latency + if edge >= step_at { step } else { 0 }); + } + edges + } + + /// The least-latency edge either side reads the step, however late the + /// edge nearest it was taken; and reads none where there was none. + #[test] + fn the_rtc_reads_a_step_through_its_interrupts_latency() { + let latency = [40_000, 700_000, 3_000, 650_000, 520_000, 9_000, 700_000, 2_000]; + let step = 1_700_000; + let stepped = edges(&latency, 14_500_000_000, step); + let (from, to) = (13_900_000_000, 14_900_000_000); + assert!(stepped.complete_after(to + step)); + let apart = stepped.apart_across(from, to + step, NS).expect("edges either side"); + assert!(apart.abs_diff(step as i64) <= 10_000, "read {apart} ns for a {step} ns step"); + assert_eq!(third_says(step as i64, Some(apart)), Third::Counter); + + let still = edges(&latency, u64::MAX, 0); + let apart = still.apart_across(from, to, NS).expect("edges either side"); + assert!(apart.unsigned_abs() <= 10_000, "read {apart} ns where nothing stepped"); + assert_eq!(third_says(step as i64, Some(apart)), Third::References); + } + + #[test] + fn the_rtc_is_unsure_without_an_edge_either_side_or_between_the_two() { + let few = edges(&[1_000, 1_000], u64::MAX, 0); + assert_eq!(few.apart_across(9_000_000_000, 10_500_000_000, NS), None); + assert!(!few.complete_after(10_500_000_000)); + assert_eq!(third_says(1_700_000, None), Third::Unsure); + assert_eq!(third_says(1_700_000, Some(850_000)), Third::Unsure); + assert_eq!(third_says(-1_700_000, Some(-1_600_000)), Third::Counter); + } + + #[test] + fn only_a_first_forward_step_is_slewed() { + assert_eq!(respond(1_700_000, 0), Response::Slew); + assert_eq!(respond(1_700_000, 1), Response::Demote); + assert_eq!(respond(-1_700_000, 0), Response::Demote); + } + + /// Samples land 0.8 to 1.4 s apart: the slew never moves the rate past + /// 500 ppm, and the debt is paid within the time that rate takes and two + /// periods more. + #[test] + fn a_slew_pays_its_debt_within_its_bound() { + let kept = toyos_abi::clock::Line::mult_of_hz(1_796_624_780); + let ticks_per_ns = 1.796_624_78; + let debt = 1_700_000i64; + let mut slew = Slew::new(debt); + let gaps = [1_000_000_000u64, 1_400_000_000, 800_000_000, 1_200_000_000, 1_000_000_000, 1_300_000_000]; + let mut elapsed = 0u64; + for gap in gaps.iter().cycle().take(64) { + if slew.settled() { + break; + } + let ran = slew.mult(kept, 1_000_000_000); + let ppb = (kept as i128 - ran as i128) * 1_000_000_000 / kept as i128; + assert!(ppb.unsigned_abs() <= SLEW_MOST_PPB as u128, "{ppb} ppb"); + slew.ran((*gap as f64 * ticks_per_ns) as u64, kept, ran); + elapsed += gap; + } + assert!(slew.settled(), "{} ns left", slew.debt_ns()); + // At the most rate down to two periods' worth, then halving at worst + // to six tenths a period, from a millisecond to the settled microsecond. + assert!(elapsed <= 2_000 * debt as u64 + 14 * 1_400_000_000, "took {elapsed} ns"); + } + + /// A sample a period late pays the debt and no more; one ten periods late + /// pays more, and the next runs the clock faster to pay it back. + #[test] + fn a_late_sample_does_not_overpay_and_a_later_one_is_paid_back() { + let kept = toyos_abi::clock::Line::mult_of_hz(1_000_000_000); + let period = 1_000_000_000u64; + let mut late = Slew::new(300_000); + late.ran(2 * period, kept, late.mult(kept, period)); + assert!(late.debt_ns() >= 0 && late.settled(), "{} ns left", late.debt_ns()); + let mut later = Slew::new(300_000); + later.ran(10 * period, kept, later.mult(kept, period)); + assert!(later.debt_ns() < 0); + assert!(later.mult(kept, period) > kept, "an overpaid slew runs the clock faster"); + } + + /// An RTC crystal 60 ppm off the counter drifts 420 µs over the edges + /// either side of a span, about a quarter of a 1.7 ms step: corrected by + /// its own earlier edges, it reads the step and reads a stall as none. + #[test] + fn the_rtcs_crystal_drift_is_taken_out() { + let drifting = |step: u64| { + let mut edges = Edges::new(); + for n in 0..EDGES_KEPT as u64 { + let edge = 10_000_000_000 + n * 1_000_060_000; + let latency = [3_000, 650_000, 20_000][n as usize % 3]; + edges.push(n + 1, edge + latency + if n >= 11 { step } else { 0 }); + } + edges + }; + let (from, to) = (20_900_000_000, 21_900_000_000); + let apart = drifting(1_700_000).apart_across(from, to + 1_700_000, NS).expect("edges either side"); + assert_eq!(third_says(1_700_000, Some(apart)), Third::Counter, "read {apart} ns"); + let apart = drifting(0).apart_across(from, to, NS).expect("edges either side"); + assert!(apart.unsigned_abs() <= 30_000, "a stall read {apart} ns through the drift"); + assert_eq!(third_says(1_700_000, Some(apart)), Third::References); + } + + /// A 1.7 ms step after a 4 ms one reads as itself once the edges from + /// before the first are forgotten; with them, the drift base reaches + /// across the first step and takes a quarter of it for the crystal's drift. + #[test] + fn a_drift_base_never_reaches_across_a_step() { + let (first, second) = (4_000_000u64, 1_700_000u64); + let stepped = || { + let mut edges = Edges::new(); + for n in 0..EDGES_KEPT as u64 { + let edge = 10_000_000_000 + n * 1_000_000_000; + let latency = [3_000, 650_000, 20_000][n as usize % 3]; + let steps = if n >= 4 { first } else { 0 } + if n >= 13 { second } else { 0 }; + edges.push(n + 1, edge + latency + steps); + } + edges + }; + let first_taken_at = 14_000_000_000 + first; + let (from, to) = (22_500_000_000 + first, 22_900_000_000 + first + second); + + let mut forgotten = stepped(); + forgotten.forget_before(first_taken_at); + let apart = forgotten.apart_across(from, to, NS).expect("edges either side"); + assert_eq!(third_says(second as i64, Some(apart)), Third::Counter, "read {apart} ns"); + + let apart = stepped().apart_across(from, to, NS).expect("edges either side"); + assert_ne!(third_says(second as i64, Some(apart)), Third::Counter, "read {apart} ns across the first step"); + } +} diff --git a/kernel/src/actuator.rs b/kernel/src/actuator.rs index c4c79a0b14c..6d1ad9a691f 100644 --- a/kernel/src/actuator.rs +++ b/kernel/src/actuator.rs @@ -124,6 +124,18 @@ actuators! { /// `kernel_wx_leaf_is_refused` and `virt_kernel_wx_leaf_is_refused`. kernel_wx_leaf = "kernel-wx-leaf"; + /// Read the TSC 10 ms ahead in the clock's samples once it has held for + /// a second, and 10 ms further once that step is slewed away, as steps + /// `IA32_TSC_ADJUST` would not show: the first is slewed, the second + /// demotes the TSC. Judged by `a_stepped_tsc_is_slewed_then_demoted`. + tsc_step = "tsc-step"; + + /// Read the HPET 10 ms behind in its samples once the clock has held for + /// a second, as an HPET that stood still while the TSC and the ACPI PM + /// timer counted: the span is left out and the boot goes on. Judged by + /// `a_stalled_hpet_is_left_out`. + hpet_stall = "hpet-stall"; + /// Hand every CPU the loader's Intel update with its header's revision /// raised by one and its checksum sealed again: the read-back's negative /// control on real Intel, whose payload still says the revision it did. diff --git a/kernel/src/arch/aarch64/boot.rs b/kernel/src/arch/aarch64/boot.rs index 473d3fda692..0c6a778de4a 100644 --- a/kernel/src/arch/aarch64/boot.rs +++ b/kernel/src/arch/aarch64/boot.rs @@ -349,7 +349,7 @@ pub fn interrupts(rsdp_addr: u64) -> Platform { /// what the counter counts at; and the wall clock the loader handed over. pub fn clock(args: &KernelArgs) { let hz = super::cpu::stated_counter_hz().expect("clock: CNTFRQ_EL0 states no rate for the generic timer"); - crate::clock::set_counter(super::cpu::counter(), 1_000_000_000_000_000 / hz); + crate::clock::set_counter(super::cpu::counter(), toyos_abi::clock::Line::mult_of_hz(hz)); log!("clock: the generic timer counts at {hz} Hz"); crate::clock::init_wall(super::rtc::read(args)); } diff --git a/kernel/src/arch/aarch64/clocksource.rs b/kernel/src/arch/aarch64/clocksource.rs new file mode 100644 index 00000000000..0681f563e31 --- /dev/null +++ b/kernel/src/arch/aarch64/clocksource.rs @@ -0,0 +1,12 @@ +//! The generic timer is one system counter every CPU reads, at the rate +//! `CNTFRQ_EL0` states, which the boot keeps (`boot::clock`): there is no +//! second timer here to hold it to. + +/// No reference to sample against. +pub fn check(_now: u64) {} + +/// Now as a log line's time, for a process: the generic timer's clock, which +/// nothing here demotes. +pub fn stamp_now() -> u64 { + crate::clock::stamp_at(super::cpu::counter()) +} diff --git a/kernel/src/arch/aarch64/cpu.rs b/kernel/src/arch/aarch64/cpu.rs index 37738467f76..e79989281c3 100644 --- a/kernel/src/arch/aarch64/cpu.rs +++ b/kernel/src/arch/aarch64/cpu.rs @@ -13,6 +13,16 @@ pub fn counter() -> u64 { count } +/// [`counter`] ordered after every earlier instruction and before every later +/// one: the read the clock takes inside its line's read section. +pub fn counter_ordered() -> u64 { + let count: u64; + // SAFETY: as `counter`'s; the second `ISB` touches nothing either. Not + // `nomem`, which would let the compiler move the latch's loads across it. + unsafe { asm!("isb", "mrs {}, cntvct_el0", "isb", out(reg) count, options(nostack, preserves_flags)) }; + count +} + /// The counter's frequency as firmware states it in `CNTFRQ_EL0`, which the /// Arm ARM makes firmware's duty to program. Zero states nothing. pub fn stated_counter_hz() -> Option { diff --git a/kernel/src/arch/aarch64/mod.rs b/kernel/src/arch/aarch64/mod.rs index e17e5ba4d07..cf8fed2ad31 100644 --- a/kernel/src/arch/aarch64/mod.rs +++ b/kernel/src/arch/aarch64/mod.rs @@ -11,10 +11,20 @@ macro_rules! owed { }; } +/// The interrupt sources only this architecture counts: none past the +/// census's own (`irq_census`). +macro_rules! census_sources { + ($list:ident) => { + $list! {} + }; +} +pub(crate) use census_sources; + pub mod acpi_mode; pub mod barrier; pub mod boot; pub mod cache; +pub mod clocksource; pub mod console_uart; pub mod control_regs; pub mod counters; diff --git a/kernel/src/arch/x86_64/boot.rs b/kernel/src/arch/x86_64/boot.rs index fcfebe0485f..6de6ca6c2ee 100644 --- a/kernel/src/arch/x86_64/boot.rs +++ b/kernel/src/arch/x86_64/boot.rs @@ -107,12 +107,10 @@ pub fn interrupts(rsdp_addr: u64) -> Platform { Platform { madt, facs } } -/// The clock: the TSC, calibrated against the HPET, and the CMOS wall clock. +/// The clock: the TSC, held to the HPET, and the CMOS wall clock. pub fn clock(args: &KernelArgs) { - // HPET clock — enables profiling for everything from here on - let hpet_base = acpi::find_hpet_base(args.rsdp_addr) - .expect("ACPI: HPET not found"); - super::hpet::calibrate_counter(hpet_base); + let hpet_base = acpi::find_hpet_base(args.rsdp_addr).expect("ACPI: HPET not found"); + super::clocksource::start(hpet_base, acpi::pm_timer(args.rsdp_addr)); let century_reg = match acpi::rtc_century_register(args.rsdp_addr) { Ok(reg) => reg, Err(e) => { @@ -124,16 +122,14 @@ pub fn clock(args: &KernelArgs) { } /// Where the TSC counts from: `IA32_TSC_ADJUST`, where CPUID says the CPU has -/// it. Every write to the TSC since reset is added to it (Intel SDM Vol. 3B, -/// "Time-Stamp Counter Adjustment"), so zero is a counter firmware never wrote -/// and the TSC is time since power-on. +/// it, which is zero on a counter firmware never wrote, so the TSC is time +/// since power-on. pub fn report_counter_origin() { - const IA32_TSC_ADJUST: u32 = 0x3b; - if super::cpu::leaf_7().1 & (1 << 1) == 0 { + if !super::cpu::has_tsc_adjust() { log!("boot: IA32_TSC_ADJUST not on this CPU"); return; } - log!("boot: IA32_TSC_ADJUST {}", super::cpu::rdmsr(IA32_TSC_ADJUST) as i64); + log!("boot: IA32_TSC_ADJUST {}", super::cpu::rdmsr(super::cpu::IA32_TSC_ADJUST) as i64); } /// The per-CPU timer, once the clock converts its bound. @@ -145,6 +141,9 @@ pub fn timer() { pub fn platform_devices(platform: &Platform, rsdp_addr: u64) { super::pio::fill_i8042_row(); super::acpi_mode::init(rsdp_addr, platform.facs); + if let Err(why) = super::rtc::arm_update_edges() { + log!("clock: no RTC edges time the TSC: {why}"); + } } /// Every other CPU, running. diff --git a/kernel/src/arch/x86_64/clocksource.rs b/kernel/src/arch/x86_64/clocksource.rs new file mode 100644 index 00000000000..5f70ba809e0 --- /dev/null +++ b/kernel/src/arch/x86_64/clocksource.rs @@ -0,0 +1,941 @@ +//! x86-64's clock: the TSC, at the rate CPUID leaf 15H states where the part +//! states one and at one measured against the HPET where it does not, and held +//! to the HPET while it runs (`kernel::clock::reference`). +//! +//! The clock counts the TSC only where every CPU states it invariant, or under +//! a hypervisor, whose TSC is its own to synthesise and whose CPUID need not +//! say so. Every AP's TSC is bounded against the BSP's as it starts +//! ([`measure_offset`]). While it runs, one CPU a second samples the HPET, the +//! ACPI PM timer and the RTC's latest edge, and every CPU a second checks its +//! `IA32_TSC_ADJUST` and that its TSC is not behind another's ([`check`]). +//! +//! **Where the TSC and the HPET disagree, the other two say which moved.** The +//! PM timer agreeing with the TSC is the HPET moving, and the span is left out. +//! The PM timer agreeing with the HPET leaves the chipset's 14.318 MHz clock +//! or the TSC; the RTC's edges, off its own 32.768 kHz crystal, choose: with +//! the TSC, the span is left out; with the chipset's counters, or saying +//! neither, the TSC stepped. The boot's first forward step is slewed away; +//! a second, one back, a check that finds a CPU's TSC written or behind, or +//! an AP's start that proves its offset demotes the TSC ([`demote`]), and +//! every process's reading of now asks the kernel, which counts by the HPET. +//! A rate that drifts past both bounds panics. + +use core::sync::atomic::{AtomicBool, AtomicU64, Ordering}; + +use kernel::clock::reference::{self, Agreement, Arbitration, Calibration, Edges, Failure, Kept, Offset, Reference, Response, Sample, Slew, Third, Watch}; +use toyos_abi::clock::Line; + +use super::cpu; +use super::pio::{self, Declared}; +use toyos_userbound::{Mediated, Ports}; +use crate::log; +use crate::mm::Mmio; +use crate::mm::policy::MmioPolicy; +use crate::shootdown::{Reach, Shootdown}; +use crate::sync::Lock; +use crate::time::{Delay, Duration, AP_START}; + +const HPET_CAP: u64 = 0x000; +const HPET_CFG: u64 = 0x010; +const HPET_COUNTER: u64 = 0x0F0; +/// `GCAP_ID` bit 13, `COUNT_SIZE_CAP`: the main counter is 64 bits wide +/// (IA-PC HPET Specification 1.0a, §2.3.4). +const COUNT_SIZE_CAP: u64 = 1 << 13; +/// The HPET as a reference: its tick, and the main counter's clock frequency +/// drift the specification's minimum implementation allows, ±0.05% over any +/// interval of 1 ms or more and ±0.2% over any of 100 µs or less (IA-PC HPET +/// Specification 1.0a, §2.2), which is what the TSC may seem to drift against +/// it without having moved. The 0.2% is taken for every span under 1 ms, which +/// the table leaves unstated between 100 µs and 1 ms. +const fn hpet_reference(period_fs: u64) -> Reference { + Reference { period_fs, tolerance_ppb: 500_000, settled_ns: 1_000_000, short_tolerance_ppb: 2_000_000 } +} + +/// Reads of the HPET per boot sample, of which the narrowest bracket is kept: an +/// SMI or a host's preemption inside one read widens that read alone. +const BOOT_READS: usize = 3; + +const CALIBRATION: Delay = Delay::to_measure( + Duration::from_millis(50), + "the TSC counted against the HPET where no CPUID leaf states its rate; the runtime check refines it", +); +/// Samples across the calibration, so a step inside it is refused. +const CALIBRATION_SAMPLES: u64 = 6; +/// Calibrations the boot tries before it refuses the machine. +const CALIBRATION_TRIES: u32 = 3; + +/// Once a second: the step allowance at this gap is about 0.5 ms, under the +/// 1.7 ms step the AMD bench was seen to take. One HPET read a sample: every +/// bound allows its bracket. +const CHECK_EVERY_NS: u64 = 1_000_000_000; + +/// Round trips of a cache line between the BSP and each AP as it starts. +const ROUNDS: u32 = 256; + +#[derive(Clone, Copy)] +struct Hpet { + mmio: Mmio, + wide: bool, + reference: Reference, +} + +impl Hpet { + /// The main counter, as 64 bits: a 32-bit counter's reading is taken as + /// the one nearest `near` (`reference::extend`). + fn read(&self, near: u64) -> u64 { + if self.wide { + self.mmio.read_u64(HPET_COUNTER) + } else { + reference::extend(near, u64::from(self.mmio.read_u32(HPET_COUNTER)), 32) + } + } + + /// The main counter as it reads, a 32-bit one's from zero. + fn raw(&self) -> u64 { + if self.wide { self.mmio.read_u64(HPET_COUNTER) } else { u64::from(self.mmio.read_u32(HPET_COUNTER)) } + } + + /// The narrowest bracket of `reads` readings near `near`. + fn sample(&self, near: u64, reads: usize) -> Sample { + let mut best: Option = None; + for _ in 0..reads { + let before = cpu::rdtsc_ordered(); + let reference = self.read(near); + let after = cpu::rdtsc_ordered(); + let sample = Sample { before, after, reference }; + if best.is_none_or(|best| sample.width() < best.width()) { + best = Some(sample); + } + } + best.expect("a sample takes a read") + } + + /// Where the counter should read now, from `last` and the rate the clock + /// keeps: what a 32-bit counter's wraps are counted against. + fn near(&self, last: Sample, mult: u64) -> u64 { + self.ahead_of(last.reference, last.before, cpu::counter(), mult) + } + + /// Where the counter should read at the TSC's `now`, from its reading + /// `reading` at the TSC's `then`. + fn ahead_of(&self, reading: u64, then: u64, now: u64, mult: u64) -> u64 { + let nanos = Line { counter: 0, stamp: 0, mult }.nanos(now.saturating_sub(then)); + reading + (u128::from(nanos) * 1_000_000 / u128::from(self.reference.period_fs)) as u64 + } +} + +/// The ACPI PM timer, the second reference: which of the TSC and the HPET +/// moved, where the two disagree. **It and the HPET are both the chipset's +/// (the FCH's on AMD, the PCH's on Intel)**, and may share its clock, so the +/// RTC's edges judge a stall common to both. +struct PmTimer { + port: Declared, + bits: u32, + reference: Reference, + last: Sample, +} + +/// ACPI 6.5 §4.8.3.3: 3.579545 MHz. ACPI states no tolerance for it; the +/// HPET's is taken, the same chipset's clock. +const PM_TIMER_PERIOD_FS: u64 = 279_365_114; + +impl PmTimer { + fn new(named: toyos_acpi::PmTimer) -> Result { + let ports = Ports::new(named.port, 4).ok_or("its block runs past the port space")?; + let port = pio::declare("the ACPI PM timer", ports, Mediated::ReadOnly).map_err(|e| alloc::format!("{e:?}"))?; + let mut timer = Self { + port, + bits: u32::from(named.bits), + reference: hpet_reference(PM_TIMER_PERIOD_FS), + last: Sample { before: 0, after: 0, reference: 0 }, + }; + let first = timer.read_raw(); + // A tick is 279 ns: a counter that does not move within 100 µs counts nothing. + let until = cpu::counter() + crate::clock::counter_ticks(100_000); + while timer.read_raw() == first { + if cpu::counter() > until { + return Err(alloc::format!("port {:#x} reads {first:#x} and does not count", named.port)); + } + } + timer.last = timer.sample(timer.read_raw()); + Ok(timer) + } + + fn read_raw(&self) -> u64 { + u64::from(cpu::inl(self.port.port(0))) & ((1u64 << self.bits) - 1) + } + + fn sample(&self, near: u64) -> Sample { + let before = cpu::rdtsc_ordered(); + let low = self.read_raw(); + let after = cpu::rdtsc_ordered(); + Sample { before, after, reference: reference::extend(near, low, self.bits) } + } + + /// The next sample, its wraps counted from the last by the kept rate. + fn next(&self, mult: u64) -> Sample { + let nanos = Line { counter: 0, stamp: 0, mult }.nanos(cpu::counter().saturating_sub(self.last.before)); + self.sample(self.last.reference + (u128::from(nanos) * 1_000_000 / u128::from(PM_TIMER_PERIOD_FS)) as u64) + } +} + +/// What the HPET's sampler holds, once the boot has started the clock. +struct Held { + hpet: Hpet, + watch: Watch, + pm: Option, + /// The RTC's edges, the third reference, on its own crystal. + edges: Edges, + /// A step the HPET and the PM timer read the TSC take, waiting for the + /// RTC's edges after it. + pending: Option, + /// The steps this boot slewed away, and the slew under way. + slewed: u32, + slew: Option, + /// Spans left out: one reference moved, or the RTC said both did. + left_out: u64, +} + +/// A step the chipset's counters agree the TSC took over `from..to`. +#[derive(Clone, Copy)] +struct Pending { + step_ns: i64, + from: Sample, + to: Sample, + /// When the sampler gives up waiting for the RTC's edges after it. + until_ns: u64, +} + +/// A slew, and the line it runs on since the counter read `since`: at `ran` +/// where the reference's time runs at `kept`. +#[derive(Clone, Copy)] +struct Slewing { + slew: Slew, + since: u64, + ran: u64, + kept: u64, +} + +/// How long a step waits for the RTC's edges after it before it is taken +/// unconfirmed: [`reference::EDGES_EACH_SIDE`] edges a second apart, and two +/// more for their interrupts. +const RTC_WAIT_NS: u64 = (reference::EDGES_EACH_SIDE as u64 + 2) * 1_000_000_000; + +static HELD: Lock> = Lock::new(None); + +/// The last sample every check held, and the clock then: where a demoted +/// clock counts on from by the HPET. Published by the sampler. +static LAST_HELD: crate::seqlock::Published<3> = crate::seqlock::Published::new(); + +/// Where the TSC was demoted: the HPET, its reading then and the clock by it, +/// and its last reading since, which the next extends from. +struct Demoted { + hpet: Hpet, + at_hpet: u64, + at_stamp: u64, + last_tsc: u64, + last_hpet: u64, + mult: u64, +} + +static DEMOTED: Lock> = Lock::new(None); +/// Taken by the first [`demote`]; every later one only logs. +static IS_DEMOTED: AtomicBool = AtomicBool::new(false); +/// What no answer since the demotion reads before: every online CPU's clock +/// once it saw the page marked ([`HANDOVER`]), and every answer given. +static DEMOTED_LATEST: AtomicU64 = AtomicU64::new(0); +/// The demotion's one round: each online CPU, once the page is marked, floors +/// [`DEMOTED_LATEST`] at its own clock ([`floor_here`]). +static HANDOVER: Shootdown = Shootdown::new(); +/// Every CPU a process may have read the page on has floored +/// [`DEMOTED_LATEST`]: no process is answered before. +static HANDED_OVER: AtomicBool = AtomicBool::new(false); +/// A finding after the demotion has been logged; later ones are not. +static FOUND_SINCE: AtomicBool = AtomicBool::new(false); +static DEMOTED_ANSWERED: AtomicBool = AtomicBool::new(false); +/// The rate the watch keeps, for a demotion on any CPU to retune the line to. +static KEPT_MULT: AtomicU64 = AtomicU64::new(0); + +/// Whether CPUID states `IA32_TSC_ADJUST`, read once on the BSP. +static TSC_ADJUST: AtomicBool = AtomicBool::new(false); + +/// When the HPET is next sampled, by whichever CPU's pass gets there first. +static NEXT_SAMPLE_NS: AtomicU64 = AtomicU64::new(0); + +/// The latest TSC any CPU's check read, on a line of its own. +#[repr(align(64))] +struct Latest(AtomicU64); + +static LATEST: Latest = Latest(AtomicU64::new(0)); + +/// Each CPU's part of the check: when it next checks itself, and the +/// `IA32_TSC_ADJUST` it started with. +pub struct Local { + due_ns: AtomicU64, + adjust: AtomicU64, +} + +impl Local { + pub const fn new() -> Self { + Self { due_ns: AtomicU64::new(0), adjust: AtomicU64::new(0) } + } +} + +/// The handover's answer for each of the `cpus` CPUs the firmware names. +pub fn size(cpus: usize) { + HANDOVER.size(cpus); +} + +/// Start the clock on the TSC, at its stated rate or one measured against the +/// HPET at `hpet_base`, and hand the HPET to the runtime check. +pub fn start(hpet_base: u64, pm_timer: Result) { + admit("cpu0"); + TSC_ADJUST.store(cpu::has_tsc_adjust(), Ordering::Relaxed); + note_adjust(); + + let mmio = crate::mm::paging::map_mmio(hpet_base, 0x1000, MmioPolicy::Uncacheable); + let cap = mmio.read_u64(HPET_CAP); + let period_fs = cap >> 32; + assert!(period_fs > 0, "HPET: invalid counter period"); + let cfg = mmio.read_u64(HPET_CFG); + mmio.write_u64(HPET_CFG, cfg | 1); + let hpet = Hpet { mmio, wide: cap & COUNT_SIZE_CAP != 0, reference: hpet_reference(period_fs) }; + log!( + "clock: HPET at {hpet_base:#x} enabled, period={period_fs}fs, {} bits, counter reads {}", + if hpet.wide { 64 } else { 32 }, + hpet.raw(), + ); + + let (anchor, kept) = match cpu::stated_counter_hz() { + Some(hz) => { + let anchor = hpet.sample(hpet.raw(), BOOT_READS); + log!( + "clock: the TSC counts at {hz} Hz, as CPUID 15H states, held to the HPET from a reading {} cycles wide", + anchor.width() + ); + (anchor, Kept { mult: Line::mult_of_hz(hz), bound_ppb: 1, measured: false }) + } + None => calibrate(&hpet), + }; + crate::clock::set_counter(anchor.before, kept.mult); + let pm = pm_timer.and_then(PmTimer::new); + match &pm { + Ok(pm) => log!("clock: the ACPI PM timer at {:#x}, {} bits, tells the TSC from the HPET where they disagree", pm.port.port(0).number(), pm.bits), + Err(why) => log!("clock: no ACPI PM timer to tell the TSC from the HPET ({why}); the RTC alone tells them apart"), + } + LAST_HELD.publish([anchor.before, anchor.reference, crate::clock::stamp_at(anchor.before)]); + *HPET_COPY.lock() = Some(hpet); + KEPT_MULT.store(kept.mult, Ordering::Relaxed); + *HELD.lock() = Some(Held { + watch: Watch::new(hpet.reference, anchor, kept), + hpet, + pm: pm.ok(), + edges: Edges::new(), + pending: None, + slewed: 0, + slew: None, + left_out: 0, + }); +} + +/// The TSC's rate against the HPET over [`CALIBRATION`], first sample to last, +/// refused where a sample between leaves the line, and the first sample. +fn calibrate(hpet: &Hpet) -> (Sample, Kept) { + let span_ticks = CALIBRATION.nanos() * 1_000_000 / hpet.reference.period_fs; + for attempt in 1..=CALIBRATION_TRIES { + // Every sample is within the window of the first, far under the 2^31 + // ticks a 32-bit counter's reading is taken near. + let first = hpet.sample(hpet.raw(), BOOT_READS); + let mut samples = [first; CALIBRATION_SAMPLES as usize]; + for (i, slot) in samples.iter_mut().enumerate().skip(1) { + let target = first.reference + span_ticks * i as u64 / (CALIBRATION_SAMPLES - 1); + wait_for(hpet, target, first); + *slot = hpet.sample(first.reference, BOOT_READS); + } + match reference::calibrate(hpet.reference, &samples) { + Ok(rate) => { + let line = Line { counter: 0, stamp: 0, mult: rate.mult }; + log!( + "clock: the TSC counts at {} mHz against the HPET over {} ms, within {} ppb; CPUID 15H states no rate, so the HPET goes on refining it", + line.millihertz(), + rate.span_ns / 1_000_000, + rate.bound_ppb, + ); + return (first, Kept { mult: rate.mult, bound_ppb: rate.bound_ppb, measured: true }); + } + Err(Calibration::Unmeasurable(why)) => { + log!("clock: calibration {attempt} measured nothing ({why:?}): {samples:?}") + } + Err(Calibration::Disturbed { index, agreement }) => log!( + "clock: calibration {attempt} refused: sample {index} sits {} ns off the line, {} allowed: {samples:?}", + agreement.apart_ns, + agreement.allowed_ns, + ), + } + } + panic!("clock: {CALIBRATION_TRIES} calibrations of the TSC against the HPET were each disturbed, so this machine keeps no time"); +} + +/// Spin until the HPET reads `target`. A main counter that does not advance +/// would spin here forever, and this is the boot's last wait before it has a +/// clock: the only unit available to bound it is the TSC's own, so the budget +/// is the calibration converted at a frequency no x86-64 part reaches, twice. +fn wait_for(hpet: &Hpet, target: u64, first: Sample) { + const TSC_CEILING_HZ: u64 = 10_000_000_000; + let budget = 2 * CALIBRATION.nanos() * (TSC_CEILING_HZ / 1_000_000_000); + while hpet.read(first.reference) < target { + assert!( + cpu::counter().wrapping_sub(first.after) <= budget, + "clock: the HPET main counter did not reach {target} in {budget} TSC cycles (it read {} at the \ + first sample and reads {}), so this machine offers no clock to calibrate against", + first.reference, + hpet.read(first.reference), + ); + } +} + +/// Refuse a CPU whose TSC this clock cannot keep time on. +fn admit(cpu: &str) { + assert!(cpu::has_rdtscp(), "clock: {cpu} has no RDTSCP, which every read the clock bounds is ordered by"); + if cpu::invariant_tsc() { + return; + } + assert!( + cpu::under_hypervisor(), + "clock: {cpu} does not state its TSC invariant (CPUID.80000007H:EDX[8]), so its rate may change with its \ + power state and no clock can be kept on it" + ); + log!("clock: {cpu}'s hypervisor does not state its TSC invariant; the HPET alone holds it"); +} + +fn note_adjust() { + if TSC_ADJUST.load(Ordering::Relaxed) { + crate::cpu_local::here().arch.clock.adjust.store(cpu::rdmsr(cpu::IA32_TSC_ADJUST), Ordering::Relaxed); + } +} + +/// This AP's part of its start, before it echoes: the same refusal as the BSP's. +pub fn start_ap(cpu_id: u32) { + admit(&alloc::format!("cpu{cpu_id}")); + note_adjust(); +} + +/// The round trips' line: whose turn it is, odd the AP's and even the BSP's, +/// and the counter its last writer read just before it passed the turn on. A +/// turn and not the counter's own value says whose word it is, because a +/// hypervisor's counter can read the same twice. +#[repr(align(64))] +struct Trip { + turn: AtomicU64, + counter: AtomicU64, +} + +static TRIP: Trip = Trip { turn: AtomicU64::new(0), counter: AtomicU64::new(0) }; +/// The AP's least trip out, once it has made every one. +static LEAST_OUT: AtomicU64 = AtomicU64::new(0); +static OUT_DONE: AtomicBool = AtomicBool::new(false); + +/// Pass the turn on as `turn`, with the counter read just before. +fn pass(turn: u64) { + TRIP.counter.store(cpu::rdtsc_ordered(), Ordering::Relaxed); + TRIP.turn.store(turn, Ordering::Release); +} + +/// Wait for `turn`, and answer the counter its writer read and the trip: +/// this CPU's counter now, less that one. +fn arrive(turn: u64) -> Option { + if !crate::clock::settles(AP_START.nanos(), || TRIP.turn.load(Ordering::Acquire) == turn) { + return None; + } + let theirs = TRIP.counter.load(Ordering::Relaxed); + Some(cpu::rdtsc_ordered().wrapping_sub(theirs) as i64) +} + +/// The BSP's half of [`ROUNDS`] round trips with the AP just started: how far +/// `cpu_id`'s TSC can be from this one's (`reference::Offset`). Demotes the TSC +/// on an offset the trips prove, and panics on an AP that stops answering. +pub fn measure_offset(cpu_id: u32) { + let mut least_back = i64::MAX; + for round in 0..u64::from(ROUNDS) { + pass(2 * round + 1); + let back = arrive(2 * round + 2); + let back = back.unwrap_or_else(|| panic!("clock: cpu{cpu_id} stopped answering the round trips that bound its TSC at round {round}")); + least_back = least_back.min(back); + } + let done = crate::clock::settles(AP_START.nanos(), || OUT_DONE.load(Ordering::Acquire)); + assert!(done, "clock: cpu{cpu_id} never reported its round trips"); + OUT_DONE.store(false, Ordering::Relaxed); + TRIP.turn.store(0, Ordering::Relaxed); + let offset = Offset { least_out: LEAST_OUT.load(Ordering::Relaxed) as i64, least_back }; + match offset.bound() { + Ok((lo, hi)) => log!("clock: cpu{cpu_id}'s TSC is within {lo}..={hi} cycles of cpu0's, over {ROUNDS} round trips"), + Err(proven) => demote(format_args!( + "cpu{cpu_id}'s TSC is {} cpu0's by at least {} cycles, which {ROUNDS} round trips prove ({offset:?})", + if proven < 0 { "behind" } else { "ahead of" }, + proven.unsigned_abs(), + )), + } +} + +/// The AP's half of [`measure_offset`], after its echo. +pub fn answer_offset() { + let mut least_out = i64::MAX; + for round in 0..u64::from(ROUNDS) { + let out = arrive(2 * round + 1); + let out = out.unwrap_or_else(|| { + panic!("clock: cpu{} waited for a round trip the BSP never sent, round {round}", super::percpu::cpu_id()) + }); + least_out = least_out.min(out); + pass(2 * round + 2); + } + LEAST_OUT.store(least_out as u64, Ordering::Relaxed); + OUT_DONE.store(true, Ordering::Release); +} + +/// This CPU's checks, at most once a [`CHECK_EVERY_NS`], and the machine's +/// HPET sample on the first CPU due: called from every scheduler pass, where a +/// CPU not due pays one load. +/// +/// A CPU checks that its `IA32_TSC_ADJUST` is the one it started with, and +/// that its TSC reads no earlier than the latest any CPU's check read before +/// this one loaded it: a reading behind it is this CPU's TSC proven behind by +/// the difference (`reference::Offset`'s argument, one way). +pub fn check(now_ns: u64) { + let local = &crate::cpu_local::here().arch.clock; + if now_ns < local.due_ns.load(Ordering::Relaxed) { + return; + } + local.due_ns.store(now_ns + CHECK_EVERY_NS, Ordering::Relaxed); + let cpu_id = super::percpu::cpu_id(); + if TSC_ADJUST.load(Ordering::Relaxed) { + let started = local.adjust.load(Ordering::Relaxed); + let adjust = cpu::rdmsr(cpu::IA32_TSC_ADJUST); + if adjust != started { + local.adjust.store(adjust, Ordering::Relaxed); + demote(format_args!( + "cpu{cpu_id}'s TSC was written: IA32_TSC_ADJUST reads {} and read {} before", + adjust as i64, + started as i64, + )); + } + } + let latest = LATEST.0.load(Ordering::Acquire); + let mine = cpu::rdtsc_ordered(); + if mine < latest { + demote(format_args!( + "cpu{cpu_id}'s TSC read {mine}, {} cycles behind the latest another CPU's check read: this CPU's TSC \ + trails another's, or every CPU's stepped back", + latest - mine, + )); + } else { + LATEST.0.fetch_max(mine, Ordering::AcqRel); + } + + let due = NEXT_SAMPLE_NS.load(Ordering::Relaxed); + if now_ns < due || NEXT_SAMPLE_NS.compare_exchange(due, now_ns + CHECK_EVERY_NS, Ordering::Relaxed, Ordering::Relaxed).is_err() { + return; + } + let Some(mut held) = HELD.try_lock() else { + return; + }; + if let Some(held) = held.as_mut() { + sample(held, now_ns); + } +} + +/// Leave the TSC for the HPET for the rest of the boot, or, after that, log +/// what was found. Every process's reading of now then asks the kernel +/// ([`stamp_now`]), which counts by the HPET from the last held sample's clock, +/// and answers none before every online CPU's clock once it saw the page +/// marked ([`hand_over`]). The line goes back to the kept rate, unslewed, and +/// the kernel's own clock and deadlines stay on it. +fn demote(why: core::fmt::Arguments) { + if IS_DEMOTED.swap(true, Ordering::SeqCst) { + if !FOUND_SINCE.swap(true, Ordering::Relaxed) { + log!("clock: DEMOTED TSC: {why}; later findings are not logged"); + } + return; + } + // A publication in progress ends within its few stores. + let [held_tsc, held_hpet, held_stamp] = loop { + if let Some(words) = LAST_HELD.snapshot() { + break words; + } + core::hint::spin_loop(); + }; + let hpet = HPET_COPY.lock().expect("the clock started before any check"); + let mult = KEPT_MULT.load(Ordering::Relaxed); + let first = cpu::counter_ordered(); + let at_hpet = hpet.read(hpet.ahead_of(held_hpet, held_tsc, first, mult)); + let at_stamp = held_stamp + hpet.reference.ns_of(at_hpet.saturating_sub(held_hpet)); + *DEMOTED.lock() = Some(Demoted { hpet, at_hpet, at_stamp, last_tsc: first, last_hpet: at_hpet, mult }); + crate::clock::demote(); + core::sync::atomic::fence(Ordering::SeqCst); + hand_over(); + let after = cpu::counter_ordered(); + let on_line = crate::clock::stamp_at(after); + crate::clock::handed_over(after, mult); + HANDED_OVER.store(true, Ordering::Release); + log!( + "clock: DEMOTED: {why}, so the TSC no longer keeps the clock: every process's reading of now asks the kernel, \ + which counts by the HPET from {at_stamp} ns, {} ns from the TSC's clock and held at every CPU's clock at \ + the handover till it catches up; the kernel's own clock and deadlines stay on the TSC, at the kept rate", + at_stamp as i64 - on_line as i64, + ); +} + +/// The handover's round, on the page marked: every online CPU floors +/// [`DEMOTED_LATEST`] at its own clock, this one first. A page reader that +/// found the page unmarked read its counter before its CPU's floor, on that +/// CPU's TSC, so no answer reads before its reading along the line in force +/// or the one it replaced, however far that TSC leads. A reading along an +/// older line is the remainder +/// `issues/a-demoted-tsc-keeps-the-kernels-clock-apart-from-processes.md` +/// records. +/// Before the APs are released no process has run on them, so this CPU +/// floors alone and none is asked. +/// +/// Never logs, as `tlb::wait_for`: a CPU blocked on the console's lock with +/// interrupts closed could not answer. +fn hand_over() { + if !crate::smp::answering() { + return floor_here(); + } + let generation = HANDOVER.issue(Reach::Narrow); + serve_here(); + super::apic::kick_all_but_self(); + for cpu in 0..crate::smp::cpu_count() as usize { + awaiting("answer to the handover from every CPU", || HANDOVER.served(cpu, generation)); + } +} + +/// This CPU's answer to the handover, if it owes one: the kick handler's and +/// `tlb::poll`'s, so a `Lock` spinner answers it. Not every spinner with +/// interrupts closed does; the round ends because the demoter answers every +/// round it is asked meanwhile ([`awaiting`]). +pub fn serve_here() { + // The hint, not the mask, with no round in flight. + if !HANDOVER.owes(super::percpu::cpu_id() as usize) { + return; + } + let _closed = super::IrqGuard::close(); + HANDOVER.serve_if_owed(super::percpu::cpu_id() as usize, |_| floor_here()); +} + +/// Floor every answer after the demotion at this CPU's clock now, along the +/// line in force and the one it replaced (`clock::floor_at`): after the mark, +/// so after the counter of every page reader on this CPU that found the page +/// unmarked, whichever of the two it read. +fn floor_here() { + DEMOTED_LATEST.fetch_max(crate::clock::floor_at(cpu::counter_ordered()), Ordering::AcqRel); +} + +/// Spin until `done`, answering every round this CPU owes meanwhile +/// (`tlb::poll`: the TLB's, `SMI_CMD`'s and the handover's), as every spinner +/// with interrupts closed must; and panic past `time::DEAF_CPU`: a CPU that +/// takes no kick in five seconds is wedged. +fn awaiting(what: &str, done: impl Fn() -> bool) { + const SPINS_PER_DEADLINE_CHECK: u32 = 1024; + if done() { + return; + } + let deaf = crate::time::DEAF_CPU.nanos(); + let deadline = crate::clock::nanos_since_boot().saturating_add(deaf); + let mut spins = 0u32; + while !done() { + super::tlb::poll(); + core::hint::spin_loop(); + spins += 1; + if spins == SPINS_PER_DEADLINE_CHECK { + spins = 0; + assert!(crate::clock::nanos_since_boot() < deadline, "clock: no {what} in {deaf} ns since the TSC was demoted"); + } + } +} + +/// The HPET, for [`demote`] on whichever CPU's check finds the TSC moved. +static HPET_COPY: Lock> = Lock::new(None); + +/// Now as a log line's time, for a process: the TSC's clock, or, once +/// demoted, the HPET's count on from where the TSC was demoted, never before +/// the handover's floor. The counter is read before the mark, as a page reader +/// reads it. +pub fn stamp_now() -> u64 { + let on_line = crate::clock::stamp_at(cpu::counter_ordered()); + if !IS_DEMOTED.load(Ordering::SeqCst) { + return on_line; + } + awaiting("end to the handover", || HANDED_OVER.load(Ordering::Acquire)); + let stamp = { + let mut demoted = DEMOTED.lock(); + let d = demoted.as_mut().expect("handed over before the demotion was recorded"); + let now = cpu::counter_ordered(); + let reading = d.hpet.read(d.hpet.ahead_of(d.last_hpet, d.last_tsc, now, d.mult)); + (d.last_tsc, d.last_hpet) = (now, reading); + d.at_stamp + d.hpet.reference.ns_of(reading.saturating_sub(d.at_hpet)) + }; + let answered = DEMOTED_LATEST.fetch_max(stamp, Ordering::AcqRel).max(stamp); + if !DEMOTED_ANSWERED.swap(true, Ordering::Relaxed) { + log!("clock: a process's reading of now asked the kernel since the TSC was demoted, and read {answered} ns"); + } + answered +} + +/// One sample of the HPET, of the PM timer beside it and of the RTC's +/// latest edge: each span the TSC and the HPET agree over is held; where +/// they do not, the PM timer and then the RTC say which moved. +fn sample(held: &mut Held, now_ns: u64) { + let kept = held.watch.kept(); + let last = held.watch.last(); + let hpet = held.hpet.sample(held.hpet.near(last, kept.mult), 1); + let pm = held.pm.as_ref().map(|pm| (pm.last, pm.next(kept.mult))); + let edge = super::rtc::last_edge(); + #[cfg(feature = "boot-actuators")] + let (hpet, pm, edge) = staged(hpet, pm, edge); + if let Some((seen, at)) = edge { + held.edges.push(seen, at); + } + let pm_agreement = held.pm.as_ref().zip(pm).map(|(timer, (from, to))| { + let span = reference::Span::between(timer.reference, from, to); + let tolerance = kept.bound_ppb + timer.reference.tolerance_over(timer.reference.span_ns(from, to)); + reference::agree_over(timer.reference, span, kept.mult, tolerance) + }); + if let (Some(timer), Some((_, to))) = (held.pm.as_mut(), pm) { + timer.last = to; + } + match held.watch.take(hpet, true) { + Ok(report) => { + LAST_HELD.publish([hpet.before, hpet.reference, crate::clock::stamp_at(hpet.before)]); + if let Some(report) = report { + log_report(held, report); + #[cfg(feature = "boot-actuators")] + stage_first(held.hpet.reference.period_fs); + } + } + Err(Failure::Step { from, to, agreement }) => { + let spans = spans(agreement, pm_agreement, from, to, pm); + let _ = held.watch.take(hpet, false); + held.left_out += 1; + match pm_agreement.map(|second| reference::arbitrate(agreement, second)) { + Some(Arbitration::FirstMoved) => log!("clock: the HPET moved and the TSC did not, so this span is left out: {spans}"), + Some(Arbitration::AllDisagree) => { + log!("clock: the TSC, the HPET and the ACPI PM timer all disagree, so this span is left out: {spans}") + } + Some(Arbitration::CounterMoved) | None => { + if let Some(earlier) = held.pending.take() { + settle(held, earlier); + } + log!( + "clock: the TSC reads {} ns from the chipset's counters over the last span; the RTC's edges \ + after it will say which moved: {spans}", + agreement.apart_ns, + ); + held.pending = Some(Pending { step_ns: agreement.apart_ns, from, to, until_ns: now_ns + RTC_WAIT_NS }); + } + } + } + Err(Failure::Drift { since, to, agreement }) => panic!( + "clock: the TSC and the HPET disagree by {} ns over the {} ns the HPET counted in held spans, {} allowed: \ + their rates are further apart than the kept rate's bound and the HPET's tolerance (the open span from \ + {since:?} to {to:?})", + agreement.apart_ns, + agreement.reference_ns, + agreement.allowed_ns, + ), + } + if let Some(pending) = held.pending.filter(|p| held.edges.complete_after(p.to.after) || now_ns >= p.until_ns) { + held.pending = None; + settle(held, pending); + } + KEPT_MULT.store(held.watch.kept().mult, Ordering::Relaxed); + slew(held); +} + +/// What the RTC says of a pending step, and what the clock does about it. +fn settle(held: &mut Held, pending: Pending) { + let kept = held.watch.kept(); + let apart = held.edges.apart_across(pending.from.before, pending.to.after, kept.mult); + let rtc = match apart { + Some(apart) => alloc::format!("the RTC's edges read {apart} ns"), + None => alloc::string::String::from("no RTC edge either side"), + }; + let step = pending.step_ns; + let confirmed = match reference::third_says(step, apart) { + Third::References => { + return log!( + "clock: the chipset's counters moved {step} ns from the TSC and {rtc}, with the TSC, so the span \ + was left out" + ); + } + Third::Counter => "confirmed by the RTC", + Third::Unsure => "unconfirmed by the RTC", + }; + held.edges.forget_before(pending.to.after); + if IS_DEMOTED.load(Ordering::Acquire) { + return log!("clock: TSC STEP: the demoted TSC stepped {step} ns against the HPET and the PM timer, {confirmed} ({rtc}); not slewed"); + } + match reference::respond(step, held.slewed) { + Response::Slew => { + held.slewed += 1; + held.slew = Some(Slewing { slew: Slew::new(step), since: cpu::counter(), ran: crate::clock::mult(), kept: kept.mult }); + log!( + "clock: TSC STEP: the TSC stepped {step} ns forward against the HPET and the PM timer, {confirmed} \ + ({rtc}); the clock slews it away at most {} ppm, most of it in about {} ms", + reference::SLEW_MOST_PPB / 1_000, + u128::from(step.unsigned_abs()) * u128::from(reference::PPB) / reference::SLEW_MOST_PPB as u128 / 1_000_000, + ); + } + Response::Demote => { + demote(format_args!( + "the TSC stepped {step} ns against the HPET and the PM timer, {confirmed} ({rtc}), after {} step(s) slewed", + held.slewed, + )); + } + } +} + +/// The slew under way, paid to now and set for the next second; or, with +/// none, the kept rate where the watch retuned it. +fn slew(held: &mut Held) { + let kept = held.watch.kept().mult; + if IS_DEMOTED.load(Ordering::Acquire) { + held.slew = None; + } + let Some(mut slewing) = held.slew else { + if crate::clock::mult() != kept { + crate::clock::retune(cpu::counter(), kept); + } + return; + }; + let now = cpu::counter(); + slewing.slew.ran(now - slewing.since, slewing.kept, slewing.ran); + let ran = slewing.slew.mult(kept, CHECK_EVERY_NS); + crate::clock::retune(now, ran); + if slewing.slew.settled() { + held.slew = None; + log!("clock: the TSC's step is slewed away, {} ns left", slewing.slew.debt_ns()); + #[cfg(feature = "boot-actuators")] + stage_again(); + } else { + held.slew = Some(Slewing { since: now, ran, kept, ..slewing }); + } +} + +fn log_report(held: &Held, report: reference::Report) { + let kept = Line { counter: 0, stamp: 0, mult: report.kept.mult }; + let measured = Line { counter: 0, stamp: 0, mult: report.measured.mult }.millihertz(); + let (span_s, bound) = (report.measured.span_ns / 1_000_000_000, report.measured.bound_ppb); + let left_out = alloc::format!( + "{} spans left out; {} TSC step(s) slewed{}", + held.left_out, + held.slewed, + held.slew.map_or_else(alloc::string::String::new, |s| alloc::format!(", {} ns still to slew", s.slew.debt_ns())), + ); + match (report.kept.measured, report.retuned) { + (false, _) => log!( + "clock: over {span_s} s the TSC counted {measured} mHz against the HPET, within {bound} ppb; \ + the clock keeps CPUID 15H's {} mHz; {left_out}", + kept.millihertz(), + ), + (true, retuned) => log!( + "clock: over {span_s} s the TSC counted {measured} mHz against the HPET, within {bound} ppb; \ + the clock {} {} mHz, within {} ppb; {left_out}", + if retuned { "now keeps" } else { "keeps" }, + kept.millihertz(), + report.kept.bound_ppb, + ), + } +} + +/// All three counters' spans, for a disagreement's record. +fn spans( + hpet: Agreement, + pm: Option, + from: Sample, + to: Sample, + pm_samples: Option<(Sample, Sample)>, +) -> alloc::string::String { + let pm = match (pm, pm_samples) { + (Some(a), Some((pm_from, pm_to))) => alloc::format!( + "the PM timer counted {} ns and reads {} ns apart, {} allowed ({pm_from:?} to {pm_to:?})", + a.reference_ns, a.apart_ns, a.allowed_ns, + ), + _ => "no PM timer".into(), + }; + alloc::format!( + "the TSC counted {} ticks, the HPET {} ns and reads {} ns apart, {} allowed ({from:?} to {to:?}); {pm}", + to.before.wrapping_sub(from.before), + hpet.reference_ns, + hpet.apart_ns, + hpet.allowed_ns, + ) +} + +/// The PM timer's span, and the RTC's latest edge, as one sample reads them. +#[cfg(feature = "boot-actuators")] +type PmSpan = Option<(Sample, Sample)>; +#[cfg(feature = "boot-actuators")] +type Edge = Option<(u64, u64)>; + +/// `tsc-step` and `hpet-stall`: from the first report on, every reading reads +/// either the TSC 10 ms ahead of where it is, the HPET's, the PM timer's and +/// the RTC's edges alike, or the HPET 10 ms behind, as a TSC that stepped and +/// an HPET that stood still; and the TSC 10 ms further once its first step is +/// slewed away. A hypervisor's TSC may ignore a write (QEMU's TCG does), so +/// each is staged in the readings the sampler takes, from the counter it +/// took effect at. +#[cfg(feature = "boot-actuators")] +mod stage { + use core::sync::atomic::{AtomicU64, Ordering}; + + pub static STEPS_AT: [AtomicU64; 2] = [AtomicU64::new(u64::MAX), AtomicU64::new(u64::MAX)]; + pub static STALLED_AT: AtomicU64 = AtomicU64::new(u64::MAX); + pub static STEP_TICKS: AtomicU64 = AtomicU64::new(0); + pub static STALL_TICKS: AtomicU64 = AtomicU64::new(0); + + /// What a counter read at `at` reads, staged. + pub fn ahead(at: u64) -> u64 { + let steps = STEPS_AT.iter().filter(|s| s.load(Ordering::Relaxed) <= at).count() as u64; + at + steps * STEP_TICKS.load(Ordering::Relaxed) + } +} + +#[cfg(feature = "boot-actuators")] +fn staged(hpet: Sample, pm: PmSpan, edge: Edge) -> (Sample, PmSpan, Edge) { + use stage::*; + let forward = |s: Sample| Sample { before: ahead(s.before), after: ahead(s.after), ..s }; + let stalled = if hpet.before >= STALLED_AT.load(Ordering::Relaxed) { STALL_TICKS.load(Ordering::Relaxed) } else { 0 }; + let hpet = Sample { reference: hpet.reference - stalled, ..forward(hpet) }; + (hpet, pm.map(|(from, to)| (from, forward(to))), edge.map(|(seen, at)| (seen, ahead(at)))) +} + +/// `tsc-step`'s and `hpet-stall`'s first, at the clock's first report. +#[cfg(feature = "boot-actuators")] +fn stage_first(period_fs: u64) { + use stage::*; + let now = cpu::counter(); + if STEPS_AT[0].load(Ordering::Relaxed) != u64::MAX || STALLED_AT.load(Ordering::Relaxed) != u64::MAX { + return; + } + if crate::actuator::tsc_step() { + let ahead = crate::clock::counter_ticks(10_000_000); + STEP_TICKS.store(ahead, Ordering::Relaxed); + STEPS_AT[0].store(now, Ordering::Relaxed); + log!("tsc-step: the TSC reads {ahead} cycles ahead from here"); + } + if crate::actuator::hpet_stall() { + let behind = 10_000_000_000_000 / period_fs; + STALL_TICKS.store(behind, Ordering::Relaxed); + STALLED_AT.store(now, Ordering::Relaxed); + log!("hpet-stall: the HPET reads {behind} ticks behind from here"); + } +} + +/// `tsc-step`'s second step, once the first is slewed away. +#[cfg(feature = "boot-actuators")] +fn stage_again() { + if crate::actuator::tsc_step() && stage::STEPS_AT[1].load(Ordering::Relaxed) == u64::MAX { + stage::STEPS_AT[1].store(cpu::counter(), Ordering::Relaxed); + log!("tsc-step: the TSC reads {} cycles further ahead from here", stage::STEP_TICKS.load(Ordering::Relaxed)); + } +} diff --git a/kernel/src/arch/x86_64/cpu.rs b/kernel/src/arch/x86_64/cpu.rs index e7088b74331..0a43ec0c83a 100644 --- a/kernel/src/arch/x86_64/cpu.rs +++ b/kernel/src/arch/x86_64/cpu.rs @@ -50,6 +50,46 @@ pub fn rdtsc() -> u64 { (hi as u64) << 32 | lo as u64 } +/// The TSC read once every earlier instruction has executed and every earlier +/// load is globally visible, and before any later instruction starts: +/// `RDTSCP; LFENCE` (SDM Vol. 2B, RDTSCP). The clock's boot refuses a CPU +/// without RDTSCP ([`has_rdtscp`]). +#[inline] +pub fn rdtsc_ordered() -> u64 { + let lo: u32; + let hi: u32; + // SAFETY: reads the time-stamp counter and `IA32_TSC_AUX` into the declared outputs; unprivileged since CR4.TSD is clear. + unsafe { + asm!("rdtscp", "lfence", out("eax") lo, out("edx") hi, out("ecx") _, options(nostack, preserves_flags)); + } + (hi as u64) << 32 | lo as u64 +} + +/// `CPUID.80000001H:EDX[27]`. +pub fn has_rdtscp() -> bool { + cpuid(0x8000_0000, 0).0 >= 0x8000_0001 && cpuid(0x8000_0001, 0).3 & 1 << 27 != 0 +} + +/// `CPUID.80000007H:EDX[8]`, the invariant TSC: one rate in every P-, C- and +/// T-state (SDM Vol. 3B, "Invariant TSC"; AMD's PPRs, `TscInvariant`). +pub fn invariant_tsc() -> bool { + cpuid(0x8000_0000, 0).0 >= 0x8000_0007 && cpuid(0x8000_0007, 0).3 & 1 << 8 != 0 +} + +/// `CPUID.01H:ECX[31]`, which a hypervisor sets for its guest (APM Vol. 3, +/// 24594 rev 3.36, §E.3.2). +pub fn under_hypervisor() -> bool { + cpuid(1, 0).2 & 1 << 31 != 0 +} + +/// `CPUID.07H:EBX[1]`: `IA32_TSC_ADJUST`, to which every write to the TSC +/// since reset is added (SDM Vol. 3B, "Time-Stamp Counter Adjustment"). +pub fn has_tsc_adjust() -> bool { + leaf_7().1 & 1 << 1 != 0 +} + +pub const IA32_TSC_ADJUST: u32 = 0x3b; + /// `CPUID.01H:ECX[30]`; without it `RDRAND` is `#UD`, so [`rdrand`] may not be /// reached at all. #[inline] @@ -448,6 +488,12 @@ pub fn counter() -> u64 { rdtsc() } +/// [`counter`] ordered after every earlier instruction and before every later +/// one: the read the clock takes inside its line's read section. +pub fn counter_ordered() -> u64 { + rdtsc_ordered() +} + /// This function's caller's frame pointer: `rbp`, which `-Cforce-frame-pointers=yes` makes one. #[inline(always)] pub fn frame_pointer() -> u64 { @@ -457,17 +503,10 @@ pub fn frame_pointer() -> u64 { rbp } -/// The TSC's frequency in hertz as CPUID *states* it, for the one caller that -/// may run before the clock is calibrated — the panic path, which has to bound a wait on a -/// machine that never reached the HPET. Nothing calibrates against it and no -/// third source is guessed at: a CPU that states neither leaf answers `None` -/// and its caller says so rather than inventing a rate. +/// The TSC's frequency in hertz as CPUID leaf 15H states it, or `None` on a +/// part that states none (`toyos_tsc::stated_hz`). pub fn stated_counter_hz() -> Option { - let max_leaf = cpuid(0, 0).0; - toyos_tsc::stated_hz( - (max_leaf >= 0x15).then(|| cpuid(0x15, 0)), - (max_leaf >= 0x16).then(|| cpuid(0x16, 0)), - ) + toyos_tsc::stated_hz((cpuid(0, 0).0 >= 0x15).then(|| cpuid(0x15, 0))) } diff --git a/kernel/src/arch/x86_64/hpet.rs b/kernel/src/arch/x86_64/hpet.rs deleted file mode 100644 index 76e8a704fef..00000000000 --- a/kernel/src/arch/x86_64/hpet.rs +++ /dev/null @@ -1,97 +0,0 @@ -//! The TSC, calibrated against the HPET: x86-64's free-running counter has no -//! stated rate every part can be trusted for, so the boot measures it. - -use crate::log; -use crate::mm::policy::MmioPolicy; -use crate::time::{Delay, Duration}; - -use super::cpu; - -const HPET_CAP: u64 = 0x000; -const HPET_CFG: u64 = 0x010; -const HPET_COUNTER: u64 = 0x0F0; - -/// Measure the TSC against the HPET at `hpet_base` and start the clock on it. -pub fn calibrate_counter(hpet_base: u64) { - let hpet = crate::mm::paging::map_mmio(hpet_base, 0x1000, MmioPolicy::Uncacheable); - - let cap = hpet.read_u64(HPET_CAP); - let hpet_period_fs = cap >> 32; - assert!(hpet_period_fs > 0, "HPET: invalid counter period"); - - let cfg = hpet.read_u64(HPET_CFG); - hpet.write_u64(HPET_CFG, cfg | 1); - - const CALIBRATION: Delay = Delay::to_measure( - Duration::from_millis(50), - "TSC ticks counted against the HPET; longer is a better ratio and boot time is what it costs", - ); - let calibration_ns = CALIBRATION.nanos(); - let calibration_hpet_ticks = calibration_ns * 1_000_000 / hpet_period_fs; - - let hpet_start = hpet.read_u64(HPET_COUNTER); - let tsc_start = cpu::counter(); - let hpet_target = hpet_start + calibration_hpet_ticks; - log!( - "clock: HPET at {:#x} enabled, period={}fs, counter reads {}, calibrating over {} ticks", - hpet_base, - hpet_period_fs, - hpet_start, - calibration_hpet_ticks, - ); - - // A main counter that does not advance would spin here forever, and this is - // the boot's last wait before it has a clock: the only unit available to - // bound it is the TSC's own, so the budget is the calibration converted at - // a frequency no x86-64 part reaches, which makes it an over-estimate of - // the cycles the calibration can legitimately take on any machine. - const TSC_CEILING_HZ: u64 = 10_000_000_000; - // Times two, so a machine merely slower than the ceiling is not refused for it. - let stall_budget_cycles = 2 * calibration_ns * (TSC_CEILING_HZ / 1_000_000_000); - while hpet.read_u64(HPET_COUNTER) < hpet_target { - assert!( - cpu::counter().wrapping_sub(tsc_start) <= stall_budget_cycles, - "clock: the HPET main counter at {:#x} did not reach {} in {} TSC cycles (it started \ - at {} and reads {}), so this machine offers no clock to calibrate against", - hpet_base, - hpet_target, - stall_budget_cycles, - hpet_start, - hpet.read_u64(HPET_COUNTER), - ); - } - let tsc_end = cpu::counter(); - let hpet_end = hpet.read_u64(HPET_COUNTER); - - let hpet_elapsed_fs = (hpet_end - hpet_start) as u128 * hpet_period_fs as u128; - let tsc_delta = tsc_end - tsc_start; - let tsc_period_fs = (hpet_elapsed_fs / tsc_delta as u128) as u64; - - crate::clock::set_counter(tsc_start, tsc_period_fs); - - let tsc_freq_mhz = 1_000_000_000_000_000u64 / tsc_period_fs / 1_000_000; - log!("TSC: {}MHz (period={}fs, calibrated over {}ms)", tsc_freq_mhz, tsc_period_fs, calibration_ns / 1_000_000); - - // **The one cross-source check this machine offers.** Everything else the - // kernel times is derived from the measurement just taken, so it could only - // agree with itself; CPUID 15H/16H is the part's own statement of the same - // frequency, arrived at by neither the HPET nor this counting loop, and the - // parts-per-million between the two is what a metal profile can hold a - // ceiling against. - let measured_hz = 1_000_000_000_000_000u64 / tsc_period_fs; - match cpu::stated_counter_hz() { - Some(stated) => { - let apart = measured_hz.abs_diff(stated); - log!( - "clock: TSC measured {measured_hz}Hz against the HPET, CPUID states {stated}Hz, \ - {}ppm apart", - apart * 1_000_000 / stated, - ); - } - None => log!( - "clock: TSC measured {measured_hz}Hz against the HPET; CPUID leaves 15H and 16H \ - stating no frequency, so nothing independent confirms it" - ), - } -} - diff --git a/kernel/src/arch/x86_64/idle.rs b/kernel/src/arch/x86_64/idle.rs index 67dc25b9e43..d644ec26407 100644 --- a/kernel/src/arch/x86_64/idle.rs +++ b/kernel/src/arch/x86_64/idle.rs @@ -30,13 +30,12 @@ pub fn facts() -> CpuIdle { } else { (0, 0) }; - let invariant_tsc = cpu::cpuid(0x8000_0000, 0).0 >= 0x8000_0007 && cpu::cpuid(0x8000_0007, 0).3 & 1 << 8 != 0; CpuIdle { monitor: cpuid_1_ecx & 1 << 3 != 0, mwait_extensions: cpuid_5_ecx & 1 != 0, mwait_substates: cpuid_5_edx, always_running_timer: cpu::leaf_6().0 & 1 << 2 != 0, - invariant_tsc, + invariant_tsc: cpu::invariant_tsc(), } } diff --git a/kernel/src/arch/x86_64/idt/kick.rs b/kernel/src/arch/x86_64/idt/kick.rs index 41390867613..605276fbe3a 100644 --- a/kernel/src/arch/x86_64/idt/kick.rs +++ b/kernel/src/arch/x86_64/idt/kick.rs @@ -4,6 +4,7 @@ extern "sysv64" fn kick_handler() { crate::arch::percpu::irq_took!(Kick); crate::counters::serve_here(); crate::arch::smi_cmd::serve_here(); + crate::arch::clocksource::serve_here(); if crate::sched::driver::pass_owed() { crate::preempt::set_need_resched(); } diff --git a/kernel/src/arch/x86_64/idt/mod.rs b/kernel/src/arch/x86_64/idt/mod.rs index 59557caaa9b..ecaf2105628 100644 --- a/kernel/src/arch/x86_64/idt/mod.rs +++ b/kernel/src/arch/x86_64/idt/mod.rs @@ -1,6 +1,7 @@ pub(crate) mod exceptions; mod device_irq; mod dma_fault; +mod rtc; mod isa; mod kick; mod nmi; @@ -36,6 +37,9 @@ pub const ISA_VECTORS: [u8; crate::isa::MAX_ROWS] = [Vector::Isa0 as u8, Vector: /// The vector an IOMMU writes into its own `FEDATA`. pub const DMA_FAULT_VECTOR: u8 = Vector::DmaFault as u8; +/// The vector the RTC's IRQ8 is routed to. +pub const RTC_VECTOR: u8 = Vector::Rtc as u8; + const PF_PRESENT: u64 = 1 << 0; const PF_WRITE: u64 = 1 << 1; const PF_INSTRUCTION_FETCH: u64 = 1 << 4; @@ -242,6 +246,7 @@ idt_vectors! { Nmi = 0x02, nmi::nmi_entry, ist 2; Timer = 0x20, timer::timer_entry; DmaFault = 0x25, dma_fault::dma_fault_entry; + Rtc = 0x26, rtc::rtc_entry; Isa0 = 0x2C, isa::isa0_entry; Isa1 = 0x2D, isa::isa1_entry; Kick = 0xFC, kick::kick_entry; diff --git a/kernel/src/arch/x86_64/idt/rtc.rs b/kernel/src/arch/x86_64/idt/rtc.rs new file mode 100644 index 00000000000..a74244af2df --- /dev/null +++ b/kernel/src/arch/x86_64/idt/rtc.rs @@ -0,0 +1,12 @@ +use super::device_irq::device_irq_entry; + +extern "sysv64" fn rtc_handler() { + crate::arch::percpu::irq_took!(Rtc); + crate::arch::rtc::update_ended(); + crate::arch::apic::eoi(); +} + +device_irq_entry! { + /// The RTC's IRQ8 entry (see `device_irq_entry` for the asm contract). + pub(super) fn rtc_entry => rtc_handler +} diff --git a/kernel/src/arch/x86_64/mod.rs b/kernel/src/arch/x86_64/mod.rs index e6d5e826f45..4fccc30d21d 100644 --- a/kernel/src/arch/x86_64/mod.rs +++ b/kernel/src/arch/x86_64/mod.rs @@ -9,12 +9,25 @@ //! AMD-Vi. Generic code reaches each through the concept it serves //! (`watchdog`, `iommu_unit`, …), never by its name. +/// The interrupt sources only this architecture counts, handed to the +/// census's list (`irq_census`). +macro_rules! census_sources { + ($list:ident) => { + $list! { + /// Vector 0x26, the RTC's update-ended interrupt, once a second. + Rtc => "rtc", + } + }; +} +pub(crate) use census_sources; + pub mod acpi_mode; pub mod amdvi; pub mod apic; pub mod barrier; pub mod boot; pub mod cache; +pub mod clocksource; pub mod control_regs; pub mod counters; pub mod console_uart; @@ -22,7 +35,6 @@ pub mod cpu; pub mod entropy; pub mod entry; pub mod fpu; -pub mod hpet; pub mod hw; pub mod idle; pub mod idt; @@ -123,6 +135,7 @@ pub unsafe fn percpu_fetch_add( /// This architecture's part of a CPU's `cpu_local` record. pub struct CpuLocal { pub counters: counters::Local, + pub clock: clocksource::Local, pub idle_line: idle::Line, #[cfg(feature = "switch-witness")] pub hw: hw::Local, @@ -132,6 +145,7 @@ impl CpuLocal { pub fn new() -> Self { Self { counters: counters::Local::new(), + clock: clocksource::Local::new(), idle_line: idle::Line::new(), #[cfg(feature = "switch-witness")] hw: hw::Local::new(), @@ -144,4 +158,5 @@ impl CpuLocal { pub fn size(cpus: usize) { tlb::size(cpus); smi_cmd::size(cpus); + clocksource::size(cpus); } diff --git a/kernel/src/arch/x86_64/rtc.rs b/kernel/src/arch/x86_64/rtc.rs index 169e01cd52c..5feb9256606 100644 --- a/kernel/src/arch/x86_64/rtc.rs +++ b/kernel/src/arch/x86_64/rtc.rs @@ -7,6 +7,7 @@ use core::fmt; use crate::arch::cpu; use crate::clock; +use crate::log; use crate::time::{Bound, Duration}; use toyos_wallclock::Civil; @@ -209,3 +210,64 @@ fn port_read(reg: u8) -> u8 { fn cmos_read(reg: u8) -> u8 { port_read(reg) } + +const STATUS_C: u8 = 0x0C; +/// Register B bit 4: raise IRQ8 as each update cycle ends. +const UPDATE_ENDED_INTERRUPT: u8 = 1 << 4; +/// Register C bit 4: an update cycle has ended since register C was last read. +const UPDATE_ENDED: u8 = 1 << 4; +/// ISA IRQ 8, which the RTC raises until register C is read (MC146818). +const RTC_IRQ: u8 = 8; + +/// The latest update-ended edge: how many the handler has seen, and the TSC +/// as it took the last. Published by IRQ8's handler on the one CPU it is routed to. +static EDGE: crate::seqlock::Published<2> = crate::seqlock::Published::new(); + +/// Raise IRQ8 at the end of each update cycle, once a second off the RTC's own +/// 32.768 kHz crystal, and take the TSC as each arrives: a reference outside +/// the chipset's 14.318 MHz domain, which the HPET and the ACPI PM timer share. +/// Called after the boot's [`read`], the only other CMOS access, so the index +/// port is the handler's alone from here. +pub fn arm_update_edges() -> Result<(), alloc::string::String> { + let line = super::ioapic::gsi_for_isa_irq(RTC_IRQ).ok_or("no I/O APIC carries IRQ8")?; + super::ioapic::route(line.gsi, super::idt::RTC_VECTOR, super::apic::MSG_DEST, line.trigger, line.polarity) + .map_err(|why| alloc::format!("{why:?}"))?; + let b = cmos_read(STATUS_B); + port_write(STATUS_B, b | UPDATE_ENDED_INTERRUPT); + let _ = cmos_read(STATUS_C); + super::ioapic::set_masked(line.gsi, false).map_err(|why| alloc::format!("{why:?}"))?; + // An update that ended while the pin was masked holds IRQ8 high with no + // edge the pin saw; reading C again lowers it, so the next update's edge + // arrives. Should the handler run first, it reads the same register. + let _ = cmos_read(STATUS_C); + log!( + "clock: the RTC's update-ended interrupt on gsi {} {} times the TSC against its 32.768 kHz crystal", + line.gsi.0, + super::ioapic::describe(line.trigger, line.polarity), + ); + Ok(()) +} + +/// IRQ8's handler: the TSC first, then register C, whose read lets the RTC +/// raise the next. +pub fn update_ended() { + let at = cpu::rdtsc_ordered(); + if cmos_read(STATUS_C) & UPDATE_ENDED != 0 { + let seen = EDGE.snapshot().map_or(0, |[seen, _]| seen); + EDGE.publish([seen + 1, at]); + } +} + +/// The edges seen and the TSC at the last, or `None` before the first. +pub fn last_edge() -> Option<(u64, u64)> { + EDGE.snapshot().filter(|&[seen, _]| seen > 0).map(|[seen, at]| (seen, at)) +} + +fn port_write(reg: u8, value: u8) { + // SAFETY: as `port_read`'s: `reg` is this module's constant, below 0x80, + // and the byte only enables the update-ended interrupt. + unsafe { + cpu::outb(CMOS_ADDR, reg); + cpu::outb(CMOS_DATA, value); + } +} diff --git a/kernel/src/arch/x86_64/smp.rs b/kernel/src/arch/x86_64/smp.rs index 5b4b9d72a61..79a8f27f9c4 100644 --- a/kernel/src/arch/x86_64/smp.rs +++ b/kernel/src/arch/x86_64/smp.rs @@ -1,6 +1,5 @@ use core::arch::global_asm; use core::mem::size_of; -use core::sync::atomic::{AtomicU64, Ordering}; use crate::arch::cpu; use crate::arch::{apic, percpu, syscall}; @@ -18,18 +17,8 @@ const TRAMPOLINE_PAGE: u64 = 0x8000; const TRAMPOLINE_VECTOR: u8 = 0x08; const DATA_OFFSET: usize = 0xF00; -/// The `rdtsc` the AP being started read on its first instruction in Rust. -/// -/// **The whole of the TSC-trail measurement.** `clock::nanos_since_boot` -/// subtracts one BSP-sampled `COUNTER_AT_BOOT` whatever CPU asks, so a CPU whose TSC -/// reads below the BSP's has no honest answer to give; the reading is bracketed -/// by two BSP samples taken either side of this AP's whole bring-up, which is -/// serialised, so a synchronised TSC lands inside the bracket by construction -/// and a skewed one is outside it by at least the skew. -static AP_TSC: AtomicU64 = AtomicU64::new(0); - /// The started AP's [`cpu::topology_keys`], written before its echo and taken -/// after it, like [`AP_TSC`]: bring-up is serialised, so one slot serves. +/// after it: bring-up is serialised, so one slot serves. static AP_TOPOLOGY: Lock>> = Lock::new(None); // Field offsets are hardcoded in the global_asm! trampoline below; the static assertion at the bottom checks the match. @@ -194,7 +183,6 @@ pub fn boot_aps(madt: &MadtInfo, boot_cr3: u64) { data.cr3 = boot_cr3; let target = crate::DirectMap::from_phys(TRAMPOLINE_PAGE + DATA_OFFSET as u64).as_mut_ptr::(); - let mut bracketed = 0u32; for &ap_id in &madt.apic_ids { if ap_id == bsp_id { continue; } @@ -208,11 +196,6 @@ pub fn boot_aps(madt: &MadtInfo, boot_cr3: u64) { // SAFETY: `target` is reserved physical memory written only here; unaligned because `TrampolineData` is `repr(C, packed)`; this AP has not been sent its SIPI yet, and the loop reaches a second write only after the previous AP committed — which happens in `ap_entry` past every trampoline read — so no CPU is reading it; a failed AP breaks the loop below instead of reaching another write. unsafe { core::ptr::write_unaligned(target, data); } - AP_TSC.store(0, Ordering::Release); - // The bracket's lower edge: taken after the previous AP committed and - // before this one is sent anything, so nothing this AP does precedes it. - let bracket_lo = cpu::rdtsc(); - // Both delays are spent, not waited on: nothing is polled across either. const AFTER_INIT: Delay = Delay::from_spec(Duration::from_millis(10), "SDM §8.4.4.1, after the INIT IPI"); @@ -234,10 +217,7 @@ pub fn boot_aps(madt: &MadtInfo, boot_cr3: u64) { let deadline = clock::nanos_since_boot() + AP_START.nanos(); // Commit only on this attempt's own token, so `0..cpu_count()` stays dense. if ROSTER.await_echo(attempt, || clock::nanos_since_boot() >= deadline) { - let bracket_hi = cpu::rdtsc(); - if tsc_inside(attempt.id(), bracket_lo, bracket_hi) { - bracketed += 1; - } + super::clocksource::measure_offset(attempt.id()); ROSTER.commit(attempt, ap_id); topology.push(AP_TOPOLOGY.lock().take().expect("an AP reads its topology before it echoes")); log!("SMP: AP cpu{} lapic={} online", attempt.id(), ap_id); @@ -249,9 +229,8 @@ pub fn boot_aps(madt: &MadtInfo, boot_cr3: u64) { } let online = smp::cpu_count(); log!( - "SMP: {online} of {} MADT cpus online, {bracketed} of {} APs inside the BSP's TSC bracket", + "SMP: {online} of {} MADT cpus online", madt.apic_ids.len(), - online.saturating_sub(1), ); crate::arch::control_regs::report(online); smp::set_topology( @@ -263,41 +242,7 @@ pub fn boot_aps(madt: &MadtInfo, boot_cr3: u64) { ); } -/// Whether the AP just started read a TSC between the two the BSP took either -/// side of its bring-up, and the record of where it read. -/// -/// Cycles, not nanoseconds: the AP's own conversion is what is under test, and -/// the reading is a raw `rdtsc` taken before this CPU has run `clock`'s -/// arithmetic at all. -fn tsc_inside(cpu_id: u32, lo: u64, hi: u64) -> bool { - let sample = AP_TSC.load(Ordering::Acquire); - let span = hi.wrapping_sub(lo); - if sample >= lo && sample <= hi { - log!("SMP: cpu{cpu_id} tsc={sample} inside the BSP's {lo}..{hi} ({span} cycles wide)"); - return true; - } - // Spelled whole in each arm rather than with the verb as a field: the host - // profile holds these two words against this file's source, and a literal - // assembled at run time is one no reader of the source can find. - if sample < lo { - log!( - "SMP: cpu{cpu_id} tsc={sample} trails the BSP's {lo}..{hi} by {} cycles", - lo - sample - ); - } else { - log!( - "SMP: cpu{cpu_id} tsc={sample} leads the BSP's {lo}..{hi} by {} cycles", - sample - hi - ); - } - false -} - extern "C" fn ap_entry() -> ! { - // First, so the reading is this CPU's TSC and not a measure of what runs - // after it; `rdtsc` needs nothing this AP has not got out of the trampoline. - AP_TSC.store(cpu::rdtsc(), Ordering::Release); - // Must run before `pat::init`, which restores the CR0 this call sets. crate::arch::control_regs::init_paging(percpu::cpu_id()); // Before `control_regs::init` reads what an update changes. @@ -314,12 +259,15 @@ extern "C" fn ap_entry() -> ! { syscall::init(); // Calibration is a one-time BSP measurement; nothing left for an AP to do here. apic::init_ap(); + super::clocksource::start_ap(percpu::cpu_id()); // Before the echo, so every CPU the roster counts has said its line. crate::arch::mtrr::compare(percpu::cpu_id()); *AP_TOPOLOGY.lock() = Some(cpu::topology_keys()); // Echo this attempt's token, so the BSP counts this AP for its own attempt. ROSTER.echo(percpu::ap_token()); + // The BSP's next step once it sees the echo. + super::clocksource::answer_offset(); process::ap_idle(); } diff --git a/kernel/src/arch/x86_64/tlb.rs b/kernel/src/arch/x86_64/tlb.rs index 1c94e8c6fae..a001fcdef23 100644 --- a/kernel/src/arch/x86_64/tlb.rs +++ b/kernel/src/arch/x86_64/tlb.rs @@ -277,13 +277,15 @@ pub fn serve_ipi() { /// Answers a pending shootdown without taking a lock or allocating, so it is /// safe from inside `Lock::lock`'s spin; and, on the boot processor, a write -/// to `SMI_CMD` asked of it, which a spinner with `IF` clear owes the same way. +/// to `SMI_CMD` asked of it, and a demoted clock's handover, which a spinner +/// with `IF` clear owes the same way. #[inline] pub fn poll() { super::smi_cmd::serve_here(); if !smp::answering() { return; } + super::clocksource::serve_here(); let cpu = percpu::cpu_id() as usize; SHOOTDOWN.serve_if_owed(cpu, answer(Answerer::Asked)); } diff --git a/kernel/src/clock.rs b/kernel/src/clock.rs index 409c11d8012..68829bad25a 100644 --- a/kernel/src/clock.rs +++ b/kernel/src/clock.rs @@ -1,49 +1,150 @@ //! The machine's clocks: monotonic since boot, read off the CPU's free-running -//! counter at the period the architecture's boot gives [`set_counter`] (on -//! x86-64, measured against the HPET); wall-clock, the architecture's one -//! reading of the RTC — on x86-64 a CMOS read, which can block for up to a -//! second, and on AArch64 firmware's, which the loader hands over — anchored -//! in [`init_wall`] and answered after as that reading plus -//! [`nanos_since_boot`]; and the log's stamp ([`stamp`]), the same clock -//! counted from the counter's zero, which is where every line of the log — -//! the loader's, the kernel's and every program's — counts from. +//! counter along the line the architecture's boot starts in [`set_counter`] +//! and may retune once it measures the counter's rate better ([`retune`]); +//! wall-clock, the architecture's one reading of the RTC — on x86-64 a CMOS +//! read, which can block for up to a second, and on AArch64 firmware's, which +//! the loader hands over — anchored in [`init_wall`] and answered after as +//! that reading plus [`nanos_since_boot`]; and the log's stamp ([`stamp`]), the +//! same clock counted from the counter's zero, which is where every line of +//! the log — the loader's, the kernel's and every program's — counts from. +//! +//! The line lives on the clock page every process maps +//! (`toyos_abi::clock`), and the kernel reads it there too, so the two clocks +//! are one until the architecture demotes the page ([`demote`]). -use core::sync::atomic::{AtomicBool, AtomicU64, Ordering::{Acquire, Relaxed, Release}}; +use core::sync::atomic::{AtomicBool, AtomicPtr, AtomicU64, Ordering::{Acquire, Relaxed, Release}}; + +use alloc::sync::Arc; +use toyos_abi::clock::{Clock, ClockPage, Line}; use crate::arch::cpu; use crate::log::LogStamp; +use crate::object::shm::SharedMemObject; +use crate::sync::Lock; use crate::time::Instant; -use kernel::clock::{anchor, ticks_to_nanos}; +use kernel::clock::anchor; use toyos_wallclock::Civil; -static COUNTER_AT_BOOT: AtomicU64 = AtomicU64::new(0); -static COUNTER_PERIOD_FS: AtomicU64 = AtomicU64::new(0); -/// The counter's zero to `COUNTER_AT_BOOT`, in nanoseconds. -static STAMP_AT_BOOT: AtomicU64 = AtomicU64::new(0); -/// One counter tick at the rate the CPU states, in femtoseconds; zero where it -/// states none, or before [`state`]. -static STATED_FS: AtomicU64 = AtomicU64::new(0); +/// The [`Line::mult`] of the rate the CPU states; zero where it states none, +/// or before [`state`]. +static STATED_MULT: AtomicU64 = AtomicU64::new(0); + +/// The clock page through the direct map, null before [`set_counter`]. +static LINES: AtomicPtr = AtomicPtr::new(core::ptr::null_mut()); + +/// The clock page as an object, for every address space to map. +static PAGE: AtomicPtr> = AtomicPtr::new(core::ptr::null_mut()); + +/// The page's one writer at a time. +static RETUNING: Lock<()> = Lock::new(()); + +/// The line the one in force replaced, for [`floor_at`]; zeros before the +/// first retune, which no floor exceeds. Published under [`RETUNING`]. +static REPLACED: crate::seqlock::Published<3> = crate::seqlock::Published::new(); + +/// From [`demote`] to [`handed_over`] no retune publishes, so the two lines +/// [`floor_at`] reads stay unchanged through the handover. +static HELD: AtomicBool = AtomicBool::new(false); /// Take the rate the CPU states, the loader's, for the stamps the log carries /// before [`set_counter`]. The kernel's entry calls this before its first record. pub fn state() { - let stated = cpu::stated_counter_hz().map_or(0, |hz| 1_000_000_000_000_000 / hz); - STATED_FS.store(stated, Relaxed); + let stated = cpu::stated_counter_hz().map_or(0, Line::mult_of_hz); + STATED_MULT.store(stated, Relaxed); } -/// Start the clock on the counter: its reading at boot and its measured period. -/// The architecture's boot calls this once, after it has the period. +/// Start the clock on the counter: its reading at boot and its rate, as a +/// [`Line::mult`]. The architecture's boot calls this once, after it has the +/// rate, and before any address space is built. /// /// **The stamps go on from where the stated rate left them**: a record before /// this read the counter at that rate, so the stamp at `boot` is that reading, /// and no record after this one is stamped earlier than one before it. -pub fn set_counter(boot: u64, period_fs: u64) { - let stated = STATED_FS.load(Relaxed); - let at_boot = ticks_to_nanos(boot, if stated != 0 { stated } else { period_fs }); - STAMP_AT_BOOT.store(at_boot, Relaxed); - COUNTER_AT_BOOT.store(boot, Relaxed); - COUNTER_PERIOD_FS.store(period_fs, Relaxed); - publish_page(boot, period_fs, at_boot); +pub fn set_counter(boot: u64, mult: u64) { + let stated = STATED_MULT.load(Relaxed); + let at_boot = Line { counter: 0, stamp: 0, mult: if stated != 0 { stated } else { mult } }.nanos(boot); + let page = SharedMemObject::create(crate::mm::charge::boot(), crate::mm::PAGE_SIZE).expect("clock: no frame for the clock page"); + let lines = crate::DirectMap::from_phys(page.page(0)).as_mut_ptr::(); + // SAFETY: the page is this object's own, which no address space maps yet + // and none unmaps after: held below for the machine's life. Every field of + // `ClockPage` is an atomic, so a shared reference reads and writes it as + // atomics while processes read it too. + let lines = unsafe { &*lines }; + lines.init(at_boot, Line { counter: boot, stamp: at_boot, mult }); + PAGE.store(alloc::boxed::Box::into_raw(alloc::boxed::Box::new(page)), Release); + LINES.store(lines as *const ClockPage as *mut ClockPage, Release); +} + +/// Go on at `mult` from the counter reading `at`: the line from there starts +/// where the clock stands, so it bends and never steps. Nothing while a +/// demotion is handed over ([`demote`]); the caller retunes again later. +pub fn retune(at: u64, mult: u64) { + let _one = RETUNING.lock(); + if !HELD.load(Relaxed) { + publish(at, mult); + } +} + +/// [`retune`]'s publication, under [`RETUNING`]. +fn publish(at: u64, mult: u64) { + let page = lines().expect("clock: retuned before it started"); + let old = page.read().expect("clock: the page `set_counter` laid out carries no clock").line; + REPLACED.publish([old.counter, old.stamp, old.mult]); + page.publish(Line { counter: at, stamp: old.stamp_of(at), mult }); +} + +/// The rate the line in force runs at. +pub fn mult() -> u64 { + clock().expect("clock: read before it started").line.mult +} + +/// The clock at the counter reading `at`, as a log line's time. +pub fn stamp_at(at: u64) -> u64 { + clock().expect("clock: read before it started").stamp_of(at) +} + +/// Send every process's reading of now to the kernel from here on +/// (`toyos_abi::clock::ClockPage::demoted`), and hold every retune until +/// [`handed_over`]: a page reader that found the page unmarked read the line +/// in force or an older one, and [`floor_at`] reads the latest two. +pub fn demote() { + let _one = RETUNING.lock(); + HELD.store(true, Relaxed); + lines().expect("clock: demoted before it started").demote(); +} + +/// The clock at the counter reading `at` by the line in force and by the one +/// it replaced, whichever is later: no earlier than a page reader's reading +/// of a counter before `at` along either. Lock-free, for a kick handler. +pub fn floor_at(at: u64) -> u64 { + let now = stamp_at(at); + let [counter, stamp, mult] = REPLACED.snapshot().expect("clock: the replaced line is published only under `RETUNING`, which a hold keeps"); + now.max(Line { counter, stamp, mult }.stamp_of(at)) +} + +/// End [`demote`]'s hold, going on at `mult` from the counter reading `at`. +pub fn handed_over(at: u64, mult: u64) { + let _one = RETUNING.lock(); + HELD.store(false, Relaxed); + publish(at, mult); +} + +fn lines() -> Option<&'static ClockPage> { + let page = LINES.load(Acquire); + // SAFETY: `set_counter` stored a pointer into a page it holds for the + // machine's life, and the pointer is never cleared. + (!page.is_null()).then(|| unsafe { &*page }) +} + +fn clock() -> Option { + lines().and_then(ClockPage::read) +} + +/// The clock and the counter read inside its line's read section, so a line +/// replaced while this CPU was interrupted is read again. The kernel's own +/// clock stays on the counter whether or not the page is demoted. +fn now_on_line() -> Option<(Clock, u64)> { + lines().and_then(|page| page.read_now(cpu::counter_ordered)).map(|(clock, now, _)| (clock, now)) } /// Now as a log line's time: nanoseconds since the counter's zero — power-on, @@ -52,30 +153,11 @@ pub fn set_counter(boot: u64, period_fs: u64) { /// started: no rate reads the counter yet. Never panics, as /// [`nanos_since_boot`]. pub fn stamp() -> Option { - if calibrated() { - return Some(LogStamp::since_zero(STAMP_AT_BOOT.load(Relaxed).saturating_add(nanos_since_boot()))); + if let Some((clock, now)) = now_on_line() { + return Some(LogStamp::since_zero(clock.stamp_of(now))); } - let stated = STATED_FS.load(Relaxed); - (stated != 0).then(|| LogStamp::since_zero(ticks_to_nanos(cpu::counter(), stated))) -} - -/// The clock page, published once by [`set_counter`]: the object every address -/// space maps read-only at [`toyos_abi::clock::CLOCK_PAGE`]. -static PAGE: core::sync::atomic::AtomicPtr> = - core::sync::atomic::AtomicPtr::new(core::ptr::null_mut()); - -/// Lay the calibration out on a page of its own, for every process to read -/// the clock this module reads without asking it. Laid out before any address -/// space can map it, and never written again. -fn publish_page(counter_at_boot: u64, period_fs: u64, stamp_at_boot: u64) { - use toyos_abi::clock::{ClockPage, CLOCK_MAGIC}; - let page = crate::object::shm::SharedMemObject::create(crate::mm::charge::boot(), crate::mm::PAGE_SIZE).expect("clock: no frame for the clock page"); - let value = ClockPage { magic: CLOCK_MAGIC, counter_at_boot, period_fs, stamp_at_boot }; - // SAFETY: `ClockPage` is four `u64`s, `repr(C)`: every byte is a field's. - let bytes = unsafe { core::slice::from_raw_parts((&value as *const ClockPage).cast::(), core::mem::size_of::()) }; - page.write(0, bytes); - // Held for the machine's life: every process maps it. - PAGE.store(alloc::boxed::Box::into_raw(alloc::boxed::Box::new(page)), Release); + let stated = STATED_MULT.load(Relaxed); + (stated != 0).then(|| LogStamp::since_zero(Line { counter: 0, stamp: 0, mult: stated }.nanos(cpu::counter()))) } /// Map the clock page into a fresh address space, read-only, at the address @@ -83,25 +165,24 @@ fn publish_page(counter_at_boot: u64, period_fs: u64, stamp_at_boot: u64) { pub fn map_page(space: &mut crate::mm::space::AddressSpace) { let page = PAGE.load(Acquire); assert!(!page.is_null(), "clock: an address space was built before the clock page"); - // SAFETY: `publish_page` stored a box it never frees, and the pointer is + // SAFETY: `set_counter` stored a box it never frees, and the pointer is // never cleared. - let page = alloc::sync::Arc::clone(unsafe { &*page }); + let page = Arc::clone(unsafe { &*page }); let at = crate::mm::space::At::Fixed(crate::UserAddr::new(toyos_abi::clock::CLOCK_PAGE)); let len = page.size(); space.map(page, 0, len, kernel::vm::Prot::Read, at).expect("clock: the clock page's address is a fresh space's"); } -/// Whether [`nanos_since_boot`] measures anything yet; false before [`init`]. +/// Whether [`nanos_since_boot`] measures anything yet; false before [`set_counter`]. pub fn calibrated() -> bool { - COUNTER_PERIOD_FS.load(Relaxed) != 0 + !LINES.load(Relaxed).is_null() } - /// Nanoseconds since boot; lock-free, no MMIO, and never panics — `log::emit` /// reads it from inside a bracket where panicking would reenter the log. -/// Saturating, not wrapping: a trailing CPU reads as oldest, not lying newest after a 584-year wrap. +/// Zero before [`set_counter`], and a CPU whose counter trails boot reads boot. pub fn nanos_since_boot() -> u64 { - ticks_to_nanos(cpu::counter().saturating_sub(COUNTER_AT_BOOT.load(Relaxed)), COUNTER_PERIOD_FS.load(Relaxed)) + now_on_line().map_or(0, |(clock, now)| clock.nanos_since_boot(now)) } /// The same reading as an [`Instant`], the type arithmetic on it is allowed in. @@ -118,29 +199,25 @@ pub fn counter_deadline(nanos: u64) -> u64 { /// `nanos` as a count of [`cpu::counter`] ticks: a span converted once and then compared /// against counter differences, which is what a sampler that may not divide -/// needs. Before [`init`] the period is unknown and this is zero, so a bound +/// needs. Before [`set_counter`] the rate is unknown and this is zero, so a bound /// derived from it is one its arm has to refuse. pub fn counter_ticks(nanos: u64) -> u64 { - let period_fs = COUNTER_PERIOD_FS.load(Relaxed); - if period_fs == 0 { - return 0; - } - ((nanos as u128 * 1_000_000) / period_fs as u128) as u64 + clock().map_or(0, |clock| clock.line.ticks(nanos)) } /// The span a count of [`counter_ticks`] stands for, for a caller that measured -/// before there was a period to measure with and converts once, afterwards. -/// Zero while the period is unknown, so a span taken on a machine that never +/// before there was a rate to measure with and converts once, afterwards. +/// Zero while the rate is unknown, so a span taken on a machine that never /// calibrated reads as no time rather than as an invented one. pub fn nanos_of_ticks(ticks: u64) -> u64 { - ticks_to_nanos(ticks, COUNTER_PERIOD_FS.load(Relaxed)) + clock().map_or(0, |clock| clock.line.nanos(ticks)) } /// Polls `ready` until it holds or `nanos` pass; `false` is the deadline. /// Reads the counter, not [`nanos_since_boot`], because that clock's out-of-line divide /// would land an NMI sample's PC in `compiler_builtins` rather than in the wait. /// -/// **Before [`init`] there is no period to measure a span with, so this asks +/// **Before [`set_counter`] there is no rate to measure a span with, so this asks /// `ready` once and answers it.** A wait nothing can bound is the one thing an /// unattended machine may not enter: a device denied its recovery window is a /// `false` its caller reports, and a spin nothing ends reports nothing at all. @@ -175,7 +252,8 @@ pub fn init_wall(reading: Result<(Civil, u64), impl core::fmt::Display>) { } }; - BOOT_SECS.store(anchor(civil.to_unix_secs(), at, COUNTER_AT_BOOT.load(Relaxed), COUNTER_PERIOD_FS.load(Relaxed)), Relaxed); + let clock = clock().expect("clock: the wall clock anchored before the counter started"); + BOOT_SECS.store(anchor(civil.to_unix_secs(), clock.stamp_of(at), clock.stamp_at_boot), Relaxed); WALL_KNOWN.store(true, Release); log!("clock: the RTC reads {civil} UTC"); } diff --git a/kernel/src/drivers/acpi.rs b/kernel/src/drivers/acpi.rs index 54ccdb4454e..e4c89b5761d 100644 --- a/kernel/src/drivers/acpi.rs +++ b/kernel/src/drivers/acpi.rs @@ -201,6 +201,13 @@ pub fn find_hpet_base(rsdp_addr: u64) -> Option { Some(base) } +/// The ACPI PM timer the FADT names, or why there is none. +pub fn pm_timer(rsdp_addr: u64) -> Result { + let fadt = toyos_acpi::find_table(direct_phys(), rsdp_addr, b"FACP", toyos_acpi::FADT_FOR_FIXED_HARDWARE) + .map_err(|e| alloc::format!("the FADT is unusable ({e:?})"))?; + toyos_acpi::pm_timer(&fadt).map_err(|e| alloc::format!("{e:?}")) +} + /// How many processors the MADT enables, of either architecture's kind: its /// Local APIC and x2APIC entries, or its GICC entries. What every per-CPU /// table is sized by, before either architecture's own walk of the MADT. diff --git a/kernel/src/irq_census.rs b/kernel/src/irq_census.rs index df075624e26..cf268b77784 100644 --- a/kernel/src/irq_census.rs +++ b/kernel/src/irq_census.rs @@ -8,44 +8,51 @@ use core::sync::atomic::{AtomicU64, Ordering}; use crate::arch::percpu; -/// One variant per interrupt source this kernel counts; order matches `arch::idt`'s vector table. -/// Adding a device vector means adding a variant here and one `irq_took!` at its handler. -/// A variant nothing counts does not compile: `irq_took!` is the only reference, so `-D dead-code` refuses an unconstructed one. -#[derive(Clone, Copy, PartialEq, Eq, Debug)] -#[repr(usize)] -pub enum Source { - /// Vector 0x20, this CPU's LAPIC one-shot: an expiry and nothing else. - Timer, - /// Vector 0xFC, `irqchip::kick_cpu`'s IPI. - Kick, - /// Vectors 0x28-0x2C: the MSI-X of a PCI function a process drives, and - /// the lines of an ISA function one does. One count for all five: which - /// claim an interrupt belonged to is the claim's own record, and the census - /// is about this machine's interrupt routing. - UserDev, - /// Vector 0x25, the remapping unit's fault event. - DmaFault, - /// Vector 0xFE, the TLB shootdown IPI. - Tlb, - /// Vector 0x02, which `crate::hardlockup` samples a CPU with. - Nmi, - /// Vector 0xFF, the local APIC's spurious vector. - /// A non-zero count on a machine that staged nothing is an interrupt-routing defect. - Spurious, - /// Every vector no `idt_vectors!` row claims — `arch::trap::unclaimed`. - /// A non-zero count a boot staged nothing for is a routing defect the gate kept off `#DF`. - Unclaimed, -} - -impl Source { - pub const COUNT: usize = 8; +/// The census's sources: these, and between the device vectors and the IPIs, +/// those the architecture names (`arch::census_sources!`). +macro_rules! sources { + ($($(#[$meta:meta])* $arch:ident => $name:literal,)*) => { + /// One variant per interrupt source this kernel counts; order matches `arch::idt`'s vector table. + /// Adding a device vector means adding a variant here, or the architecture's, and one `irq_took!` at its handler. + /// A variant nothing counts does not compile: `irq_took!` is the only reference, so `-D dead-code` refuses an unconstructed one. + #[derive(Clone, Copy, PartialEq, Eq, Debug)] + #[repr(usize)] + pub enum Source { + /// Vector 0x20, this CPU's LAPIC one-shot: an expiry and nothing else. + Timer, + /// Vector 0xFC, `irqchip::kick_cpu`'s IPI. + Kick, + /// Vectors 0x28-0x2C: the MSI-X of a PCI function a process drives, and + /// the lines of an ISA function one does. One count for all five: which + /// claim an interrupt belonged to is the claim's own record, and the census + /// is about this machine's interrupt routing. + UserDev, + /// Vector 0x25, the remapping unit's fault event. + DmaFault, + $($(#[$meta])* $arch,)* + /// Vector 0xFE, the TLB shootdown IPI. + Tlb, + /// Vector 0x02, which `crate::hardlockup` samples a CPU with. + Nmi, + /// Vector 0xFF, the local APIC's spurious vector. + /// A non-zero count on a machine that staged nothing is an interrupt-routing defect. + Spurious, + /// Every vector no `idt_vectors!` row claims — `arch::trap::unclaimed`. + /// A non-zero count a boot staged nothing for is a routing defect the gate kept off `#DF`. + Unclaimed, + } - /// Order `tests/toyos.rs`'s `irq_census_conservation` parses back; must match variant order. - pub const NAMES: [&'static str; Self::COUNT] = [ - "timer", "kick", "userdev", "dmafault", "tlb", "nmi", "spurious", "unclaimed", - ]; + impl Source { + /// Order `tests/toyos.rs`'s `irq_census_conservation` parses back; must match variant order. + pub const NAMES: &'static [&'static str] = + &["timer", "kick", "userdev", "dmafault", $($name,)* "tlb", "nmi", "spurious", "unclaimed"]; + pub const COUNT: usize = Self::NAMES.len(); + } + }; } +crate::arch::census_sources!(sources); + /// Every source's count summed, bar the NMI's. fn sum_bar_nmi(counts: &[u64; Source::COUNT]) -> u64 { counts.iter().sum::() - counts[Source::Nmi as usize] diff --git a/kernel/src/sched/driver.rs b/kernel/src/sched/driver.rs index 18a482ce350..3f0364fd58c 100644 --- a/kernel/src/sched/driver.rs +++ b/kernel/src/sched/driver.rs @@ -519,6 +519,7 @@ pub fn pass(dispose: Dispose) { crate::object::drain_zero_handles(); let now = HW.now(); crate::arch::watchdog::feed(now.0); + crate::arch::clocksource::check(now.0); #[cfg(feature = "heap-sweep")] maybe_sweep(now); #[cfg(feature = "pass-spin")] diff --git a/kernel/src/syscall/dispatch.rs b/kernel/src/syscall/dispatch.rs index fd583a8a451..56a9517bf07 100644 --- a/kernel/src/syscall/dispatch.rs +++ b/kernel/src/syscall/dispatch.rs @@ -630,6 +630,7 @@ pub(crate) fn syscall_dispatch(num: u64, a1: u64, a2: u64, a3: u64, a4: u64) -> SYS_EVENT_SIGNAL => sys_event_signal(RawHandle(a1 as u32)), SYS_EVENT_TAKE => sys_event_take(RawHandle(a1 as u32)), SYS_HANDLE_WAIT => sys_handle_wait(RawHandle(a1 as u32), a2, a3), + SYS_CLOCK_NOW => crate::arch::clocksource::stamp_now(), SYS_ACCOUNT_INFO => { if a3 < core::mem::size_of::() as u64 { return SyscallError::InvalidArgument.to_u64(); } let Some(addr) = UserAddr::checked(a2) else { return bad_addr }; diff --git a/lib/acpi/src/fadt.rs b/lib/acpi/src/fadt.rs index 049dba9b28e..b2c0b112236 100644 --- a/lib/acpi/src/fadt.rs +++ b/lib/acpi/src/fadt.rs @@ -167,10 +167,12 @@ const FADT_PSTATE_CNT: usize = 55; const FADT_PM1A_EVT_BLK: usize = 56; const FADT_PM1B_EVT_BLK: usize = 60; const FADT_PM1B_CNT_BLK: usize = 68; +const FADT_PM_TMR_BLK: usize = 76; const FADT_GPE0_BLK: usize = 80; const FADT_GPE1_BLK: usize = 84; const FADT_PM1_EVT_LEN: usize = 88; const FADT_PM1_CNT_LEN: usize = 89; +const FADT_PM_TMR_LEN: usize = 91; const FADT_GPE0_BLK_LEN: usize = 92; const FADT_GPE1_BLK_LEN: usize = 93; const FADT_CST_CNT: usize = 95; @@ -178,6 +180,7 @@ const FADT_X_PM1A_EVT_BLK: usize = 148; const FADT_X_PM1B_EVT_BLK: usize = 160; const FADT_X_PM1A_CNT_BLK: usize = 172; const FADT_X_PM1B_CNT_BLK: usize = 184; +const FADT_X_PM_TMR_BLK: usize = 208; const FADT_X_GPE0_BLK: usize = 220; const FADT_X_GPE1_BLK: usize = 232; /// The end of `X_GPE1_BLK`, the last field [`fixed_hardware`] reads where the @@ -187,6 +190,9 @@ const FADT_X_END: usize = FADT_X_GPE1_BLK + 12; /// Table 5.10's `PWR_BUTTON` (bit 4): set, the power button is a control /// method device, which only AML serves; and `HW_REDUCED_ACPI` (bit 20). const PWR_BUTTON: u32 = 1 << 4; +/// Table 5.10's `TMR_VAL_EXT` (bit 8): set, the PM timer's value is 32 bits +/// wide, and clear, 24. +const TMR_VAL_EXT: u32 = 1 << 8; const HW_REDUCED_ACPI: u32 = 1 << 20; /// The bytes [`fixed_hardware`] needs at least: through the flags. @@ -286,6 +292,7 @@ pub enum Field { Pm1aEvent, Pm1aControl, Gpe0, + PmTimer, } /// Why the FADT's fixed hardware is none this kernel serves, one reason each. @@ -346,6 +353,36 @@ fn block(field: Field, address: u64, len: u8) -> Result(fadt: &Table

) -> Result { + let short = FixedRefused::Short { len: fadt.len() }; + let flags = fadt.u32_at(FADT_FLAGS).ok_or(short)?; + if flags & HW_REDUCED_ACPI != 0 { + return Err(FixedRefused::HardwareReduced); + } + // §5.2.9: `PM_TMR_LEN` is 4 where the timer is supported. + let len = fadt.byte(FADT_PM_TMR_LEN).ok_or(short)?; + let at = address(fadt, Field::PmTimer, FADT_PM_TMR_BLK, FADT_X_PM_TMR_BLK)?; + if at == 0 { + return Err(FixedRefused::Absent { field: Field::PmTimer }); + } + if len != 4 { + return Err(FixedRefused::Length { field: Field::PmTimer, len }); + } + let block = block(Field::PmTimer, at, len)?; + Ok(PmTimer { port: block.port, bits: if flags & TMR_VAL_EXT != 0 { 32 } else { 24 } }) +} + /// The PM1a control block, checked against its `X_` twin where the table /// holds one; its length the `PM1_CNT_LEN` byte's. pub fn pm1a_control(fadt: &Table

) -> Result { diff --git a/lib/acpi/src/lib.rs b/lib/acpi/src/lib.rs index 71ce28fa26b..8b9f82f51a2 100644 --- a/lib/acpi/src/lib.rs +++ b/lib/acpi/src/lib.rs @@ -32,9 +32,9 @@ use toyos_bootmap::DirectMapEnd; pub use ecdt::{ecdt, Ec, EcRefused, EcRegister, ECDT_NEEDED}; pub use facs::{acquire, facs, release, Facs, FacsRefused, FACS_GLOBAL_LOCK, OWNED, PENDING}; pub use fadt::{ - century_of, dsdt_address, fixed_hardware, pm1a_control, psci, reset_register, + century_of, dsdt_address, fixed_hardware, pm1a_control, pm_timer, psci, reset_register, rtc_century, - Century, Field, FixedHardware, FixedRefused, LegacyMode, PowerButton, Psci, Reset, SmiCmd, CMOS_RAM, + Century, Field, FixedHardware, FixedRefused, LegacyMode, PmTimer, PowerButton, Psci, Reset, SmiCmd, CMOS_RAM, FADT_FOR_FIXED_HARDWARE, FADT_PM1A_CNT_BLK, }; use fadt::FADT_DSDT; diff --git a/lib/acpi/tests/fixtures.rs b/lib/acpi/tests/fixtures.rs index 89399770910..a46b2d1ed60 100644 --- a/lib/acpi/tests/fixtures.rs +++ b/lib/acpi/tests/fixtures.rs @@ -8,7 +8,7 @@ use toyos_abi::boot::RootBridgeWindow; use toyos_abi::acpi::Block; use toyos_acpi::{ century_of, definition_blocks, dsdt_address, ecam_allocations, find_table, fixed_hardware, hpet_base, - isa_line, madt_entries, memory_windows, pm1a_control, psci, reset_register, rtc_century, sci_line, + isa_line, madt_entries, memory_windows, pm1a_control, pm_timer, psci, reset_register, rtc_century, sci_line, PmTimer, Century, FixedHardware, IoApicEntry, Line, MadtEntry, Polarity, PowerButton, Psci, Reset, SmiCmd, SourceOverride, TableError, Trigger, FADT_FOR_FIXED_HARDWARE, FADT_PM1A_CNT_BLK, MADT_ENTRIES, @@ -208,7 +208,8 @@ fn a_second_firmwares_bytes_decode_to_that_machines_own_windows() { /// q35's ICH9 power-management block at the base QEMU's firmware put it, /// 0x600, as the FADT names it: the blocks the `acpi_press_is_one_sci` guest test -/// takes its SCI through, and the 32-bit fields and their `X_` twins agreeing. +/// takes its SCI through, and the 32-bit fields and their `X_` twins agreeing; +/// and its 24-bit PM timer at 0x608, which the clock arbitrates with. #[test] fn the_q35_fadt_names_the_fixed_hardware_its_sci_is_served_through() { let fadt = find_table(machine(), RSDP, b"FACP", FADT_FOR_FIXED_HARDWARE).expect("FADT"); @@ -223,6 +224,7 @@ fn the_q35_fadt_names_the_fixed_hardware_its_sci_is_served_through() { }) ); assert_eq!(pm1a_control(&fadt), Ok(Block { port: 0x604, len: 2 })); + assert_eq!(pm_timer(&fadt), Ok(PmTimer { port: 0x608, bits: 24 })); } /// `ioapic: iso bus:irq->gsi [... 0:9->9 level/high ...]`: q35 names its SCI's diff --git a/lib/trace/src/lib.rs b/lib/trace/src/lib.rs index 15e3abb1afb..66699e27360 100644 --- a/lib/trace/src/lib.rs +++ b/lib/trace/src/lib.rs @@ -4,9 +4,9 @@ //! A record crossed the syscall boundary or came off a file, so it is input: //! [`Entry::decode`] refuses a kind the ABI does not name, and a kind that //! names its own task but carries no thread, each by name. A stamp is the -//! writing CPU's counter, which the machine's `ClockPage` reads as a log -//! line's time, from the counter's zero on the kernel's own formula, so a -//! diary and a log line order against each other. +//! writing CPU's counter, which the machine's `Clock` reads as a log line's +//! time, from the counter's zero on the kernel's own formula, so a diary and a +//! log line order against each other. //! //! Pure: `core` only, no `unsafe`, no I/O. @@ -15,7 +15,7 @@ use core::fmt; -use toyos_abi::clock::ClockPage; +use toyos_abi::clock::Clock; use toyos_abi::trace::{Kind, TraceRecord, NO_THREAD}; /// A thread, by its process's id and its own, each as the writing CPU held @@ -129,7 +129,7 @@ impl Entry { /// The line this entry prints as, its stamp read on `clock`: /// `[12.000345678 cpu3] wake 7/0`. - pub fn line(&self, clock: ClockPage) -> Line<'_> { + pub fn line(&self, clock: Clock) -> Line<'_> { Line { entry: self, clock } } } @@ -137,7 +137,7 @@ impl Entry { /// What [`Entry::line`] renders. pub struct Line<'a> { entry: &'a Entry, - clock: ClockPage, + clock: Clock, } impl fmt::Display for Line<'_> { @@ -188,10 +188,8 @@ mod tests { /// A 1 GHz counter that read 1000 at boot, two seconds after its zero: /// one tick a nanosecond. - const CLOCK: ClockPage = ClockPage { - magic: toyos_abi::clock::CLOCK_MAGIC, - counter_at_boot: 1_000, - period_fs: 1_000_000, + const CLOCK: Clock = Clock { + line: toyos_abi::clock::Line { counter: 1_000, stamp: 2_000_000_000, mult: 1 << toyos_abi::clock::MULT_SHIFT }, stamp_at_boot: 2_000_000_000, }; diff --git a/lib/tsc/Cargo.toml b/lib/tsc/Cargo.toml index 23094e14759..ee0c03fcbed 100644 --- a/lib/tsc/Cargo.toml +++ b/lib/tsc/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "toyos-tsc" -description = "The rate x86-64's time-stamp counter counts at, as CPUID leaves 15H and 16H state it, pure." +description = "The rate x86-64's time-stamp counter counts at, as CPUID leaf 15H states it, pure." version = "0.1.0" edition = "2024" license = "MIT OR Apache-2.0" diff --git a/lib/tsc/src/lib.rs b/lib/tsc/src/lib.rs index e597f923dad..94b3b26f373 100644 --- a/lib/tsc/src/lib.rs +++ b/lib/tsc/src/lib.rs @@ -8,32 +8,24 @@ pub type Leaf = (u32, u32, u32, u32); /// SDM Vol. 2A, CPUID leaf 15H: EAX is the denominator and EBX the numerator of -/// the core crystal's ratio to the TSC, ECX the crystal's hertz — any of the -/// three reading zero means the leaf states nothing. Leaf 16H's EAX is the -/// processor base frequency in MHz, which an invariant TSC counts at. `None` -/// for a leaf the CPU does not implement, and as the answer where neither -/// states a rate: nothing here guesses one. -pub const fn stated_hz(leaf15: Option, leaf16: Option) -> Option { - if let Some((denominator, numerator, crystal_hz, _)) = leaf15 - && denominator != 0 - && numerator != 0 - && crystal_hz != 0 - { - return Some(crystal_hz as u64 * numerator as u64 / denominator as u64); +/// the TSC's ratio to the core crystal, ECX the crystal's hertz, and the TSC +/// counts at their product; any of the three reading zero states no rate. +/// `None` for a CPU without the leaf. Leaf 16H's base frequency is no rate: +/// the SDM confines its data to display. +pub const fn stated_hz(leaf15: Option) -> Option { + match leaf15 { + Some((denominator, numerator, crystal_hz, _)) if denominator != 0 && numerator != 0 && crystal_hz != 0 => { + Some(crystal_hz as u64 * numerator as u64 / denominator as u64) + } + _ => None, } - if let Some((base_mhz, _, _, _)) = leaf16 - && base_mhz != 0 - { - return Some(base_mhz as u64 * 1_000_000); - } - None } const _: () = { - // The ratio, then the fall-through to the base frequency when the crystal - // is not enumerated, then the CPU that states neither. - assert!(matches!(stated_hz(Some((2, 4, 25_000_000, 0)), None), Some(50_000_000))); - assert!(matches!(stated_hz(Some((0, 0, 0, 0)), Some((2_400, 0, 0, 0))), Some(2_400_000_000))); - assert!(stated_hz(None, Some((0, 0, 0, 0))).is_none()); - assert!(stated_hz(None, None).is_none()); + // A 38.4 MHz crystal at 63/1. + assert!(matches!(stated_hz(Some((1, 63, 38_400_000, 0))), Some(2_419_200_000))); + assert!(matches!(stated_hz(Some((2, 4, 25_000_000, 0))), Some(50_000_000))); + // A crystal the leaf does not enumerate states nothing. + assert!(stated_hz(Some((2, 176, 0, 0))).is_none()); + assert!(stated_hz(None).is_none()); }; diff --git a/loader/src/arch/x86_64.rs b/loader/src/arch/x86_64.rs index 4903d66aa7a..fe23de90e84 100644 --- a/loader/src/arch/x86_64.rs +++ b/loader/src/arch/x86_64.rs @@ -29,8 +29,7 @@ pub fn counter_hz() -> Option { let r = __cpuid(at); (r.eax, r.ebx, r.ecx, r.edx) }; - let max = leaf(0).0; - toyos_tsc::stated_hz((max >= 0x15).then(|| leaf(0x15)), (max >= 0x16).then(|| leaf(0x16))) + toyos_tsc::stated_hz((leaf(0).0 >= 0x15).then(|| leaf(0x15))) } /// This CPU as its vendor's microcode names it, or why it loads none. diff --git a/sdk/abi/Cargo.toml b/sdk/abi/Cargo.toml index 8eda9709e01..bcc434f233e 100644 --- a/sdk/abi/Cargo.toml +++ b/sdk/abi/Cargo.toml @@ -8,6 +8,13 @@ repository = "https://github.com/ToyOSOrg/ToyOS" [features] rustc-dep-of-std = ["core"] +# Declared, never enabled here: `kernel-loom` compiles `src/clock/latch.rs` +# into its own crate with `loom` on, so its atomics resolve to loom's, and with +# `clock-latch-fence-off`, its negative control, which drops the writer's +# fence and must red `kernel/loom/tests/clock_latch.rs`. Declared only so `cfg` +# checking knows the names. +loom = [] +clock-latch-fence-off = [] [dependencies] core = { version = "1.0.0", optional = true, package = "rustc-std-workspace-core" } diff --git a/sdk/abi/src/arch/aarch64.rs b/sdk/abi/src/arch/aarch64.rs index 4fc56f7835c..d20b8327fe2 100644 --- a/sdk/abi/src/arch/aarch64.rs +++ b/sdk/abi/src/arch/aarch64.rs @@ -17,15 +17,16 @@ pub fn current_tid() -> Tid { Tid(unsafe { core::ptr::read_volatile((tp as usize + TCB_TID) as *const u32) }) } -/// The counter the clock page's two words describe: the generic timer's -/// virtual count, which EL0 may read, read after every earlier instruction, as -/// the x86-64 one is. +/// The counter the clock page's line describes: the generic timer's +/// virtual count, which EL0 may read, read after every earlier instruction and +/// before any later one, as the x86-64 one is. #[inline] pub fn counter() -> u64 { let now: u64; - // SAFETY: an `isb` and a read of `CNTVCT_EL0` into a register; neither - // touches memory. - unsafe { core::arch::asm!("isb", "mrs {now}, cntvct_el0", now = out(reg) now, options(nomem, nostack)) }; + // SAFETY: two `isb`s and a read of `CNTVCT_EL0` into a register; none + // touches memory. Not `nomem`, which would let the compiler move memory + // accesses, the clock page's loads among them, across the read. + unsafe { core::arch::asm!("isb", "mrs {now}, cntvct_el0", "isb", now = out(reg) now, options(nostack, preserves_flags)) }; now } diff --git a/sdk/abi/src/arch/x86_64.rs b/sdk/abi/src/arch/x86_64.rs index aae795211be..39e7fcfbef1 100644 --- a/sdk/abi/src/arch/x86_64.rs +++ b/sdk/abi/src/arch/x86_64.rs @@ -24,9 +24,13 @@ pub fn current_tid() -> Tid { Tid(tid) } -/// The counter the clock page's two words describe: the time-stamp counter, +/// The counter the clock page's line describes: the time-stamp counter, /// read after every earlier load has completed, so a stamp taken after reading -/// another writer's record cannot be older than that record's. +/// another writer's record cannot be older than that record's, and before any +/// later instruction starts, so a load after it cannot have been answered +/// before it (SDM Vol. 2B, RDTSC: "If software requires RDTSC to be executed +/// prior to execution of any subsequent instruction (including any memory +/// accesses), it can execute the sequence LFENCE immediately after RDTSC"). #[inline] pub fn counter() -> u64 { // SAFETY: `lfence` has no operands; `rdtsc` is unprivileged while CR4.TSD @@ -34,7 +38,9 @@ pub fn counter() -> u64 { // every CPU. unsafe { core::arch::x86_64::_mm_lfence(); - core::arch::x86_64::_rdtsc() + let now = core::arch::x86_64::_rdtsc(); + core::arch::x86_64::_mm_lfence(); + now } } diff --git a/sdk/abi/src/clock.rs b/sdk/abi/src/clock.rs index 396aaa632c4..850ea9b8d93 100644 --- a/sdk/abi/src/clock.rs +++ b/sdk/abi/src/clock.rs @@ -1,16 +1,34 @@ -//! The monotonic clock a process reads without a syscall. +//! The monotonic clock a process reads without a syscall, until the kernel +//! demotes the counter it reads. //! -//! The kernel calibrates the counter once at boot and never again, so the -//! numbers that turn a counter reading into nanoseconds since boot are -//! constants for the machine's life. It writes them into one page before the -//! first process exists and maps that page read-only at [`CLOCK_PAGE`] in every -//! address space; [`nanos_since_boot`] is the kernel's own arithmetic on the -//! same words, and [`stamp_ns`] its log's, so a stamp taken here and one taken -//! by the kernel's log order against each other. +//! The clock is a straight [`Line`] through the counter: from a counter +//! reading, the nanoseconds since the counter's zero, and the rate after it. +//! The kernel lays the line out on one page before the first process exists, +//! maps that page read-only at [`CLOCK_PAGE`] in every address space, and +//! replaces the line when it learns the counter's rate better, starting the +//! new one where the old one stands, so the clock never steps. [`page`] reads +//! the line in force through the page's [`latch`], which the kernel's own +//! clock reads too, so a stamp taken here and one taken by the kernel's log +//! order against each other until the kernel demotes the page. //! -//! **No sequence word, because nothing is ever rewritten.** A clock page that a -//! kernel adjusted would need one; this one is laid out before it is mapped and -//! is immutable after, which is the whole of its concurrency argument. +//! **A demoted page sends every reading to the kernel.** The kernel demotes +//! the page when it stops trusting the counter ([`ClockPage::demoted`]); +//! [`nanos_since_boot`] and [`stamp_ns`] then ask +//! [`crate::syscall::clock_now`], which reads the reference instead. The line +//! stays, for a recorded counter reading. +//! +//! **A reading takes the counter inside the latch's read section**, and the +//! page's demoted word after the counter, so a line replaced or a demotion +//! made while the reader was preempted sends it round again or to the kernel. +//! +//! **Every word of the page is atomic.** The kernel writes it while processes +//! read it, so it is reached as atomics and never as plain memory. + +mod latch; + +use core::sync::atomic::{AtomicU64, Ordering}; + +pub use latch::Latch; /// Where every address space finds the page: below the lowest address an /// allocation or a placed `mmap` may take, so nothing a process maps can land @@ -19,126 +37,272 @@ pub const CLOCK_PAGE: u64 = 0x1_0000_0000; /// What the page opens with, so a reader that finds anything else refuses /// rather than computes a time out of it. -pub const CLOCK_MAGIC: u64 = 0x544F_594F_5343_4C4B; // "TOYOSCLK" +pub const CLOCK_MAGIC: u64 = 0x544F_594F_434C_4B33; // "TOYOCLK3" -/// The page's layout. Written once by the kernel before it is mapped anywhere. -#[repr(C)] +/// The binary point of [`Line::mult`]: nanoseconds per tick in 32.32 fixed +/// point, which keeps a 5 GHz counter's rate to a part in 10⁹ and a 1 Hz +/// counter's tick representable. +pub const MULT_SHIFT: u32 = 32; + +/// One straight piece of the clock. #[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub struct ClockPage { - pub magic: u64, - /// The counter's reading at nanosecond zero. - pub counter_at_boot: u64, - /// One counter tick, in femtoseconds. - pub period_fs: u64, - /// The counter's zero to `counter_at_boot`, in nanoseconds: what turns a - /// reading of this clock into a log line's time ([`stamp_ns`]). +pub struct Line { + /// The counter reading the line starts at. + pub counter: u64, + /// The clock at `counter`: nanoseconds since the counter's zero. + pub stamp: u64, + /// Nanoseconds per counter tick, times 2^[`MULT_SHIFT`]. + pub mult: u64, +} + +impl Line { + /// The [`Line::mult`] of a counter at `hz`, rounded to nearest. + pub const fn mult_of_hz(hz: u64) -> u64 { + let ns = 1_000_000_000u128 << MULT_SHIFT; + ((ns + hz as u128 / 2) / hz as u128) as u64 + } + + /// The rate [`Line::mult`] stands for, in millihertz. + pub const fn millihertz(&self) -> u64 { + ((1_000_000_000_000u128 << MULT_SHIFT) / self.mult as u128) as u64 + } + + /// `ticks` of the counter, in nanoseconds; saturating, which no count a + /// machine reaches in a century touches. + pub const fn nanos(&self, ticks: u64) -> u64 { + let nanos = (ticks as u128 * self.mult as u128) >> MULT_SHIFT; + if nanos > u64::MAX as u128 { + u64::MAX + } else { + nanos as u64 + } + } + + /// The counter ticks `nanos` takes, rounded down. + pub const fn ticks(&self, nanos: u64) -> u64 { + (((nanos as u128) << MULT_SHIFT) / self.mult as u128) as u64 + } + + /// The clock at `counter`. A reading before the line's start is carried + /// back along it, which is where a record stamped before the kernel last + /// replaced the line lands, and a CPU whose counter trails reads behind; + /// nothing reads before the counter's zero. + pub const fn stamp_of(&self, counter: u64) -> u64 { + if counter >= self.counter { + self.stamp.saturating_add(self.nanos(counter - self.counter)) + } else { + self.stamp.saturating_sub(self.nanos(self.counter - counter)) + } + } + + const fn words(&self) -> [u64; 3] { + [self.counter, self.stamp, self.mult] + } + + const fn of_words(words: [u64; 3]) -> Self { + Self { counter: words[0], stamp: words[1], mult: words[2] } + } +} + +/// The clock as one reading of the page found it. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Clock { + /// The line in force. + pub line: Line, + /// The clock at boot: what [`Clock::nanos_since_boot`] counts from. pub stamp_at_boot: u64, } -impl ClockPage { +impl Clock { /// A counter reading as a log line's time: nanoseconds since the counter's - /// zero, on the kernel's formula. + /// zero. pub const fn stamp_of(&self, counter: u64) -> u64 { - self.stamp_at_boot.saturating_add(nanos_between(self.counter_at_boot, self.period_fs, counter)) + self.line.stamp_of(counter) + } + + /// A counter reading as nanoseconds since boot; a counter that reads + /// before boot reads as boot. + pub const fn nanos_since_boot(&self, counter: u64) -> u64 { + self.stamp_of(counter).saturating_sub(self.stamp_at_boot) } } -/// Nanoseconds between `counter_at_boot` and `now`, on the kernel's formula. -/// -/// Saturating below boot: a counter read on a CPU that trails reads as the -/// oldest instant, never as a wrap 584 years ahead. -pub const fn nanos_between(counter_at_boot: u64, period_fs: u64, now: u64) -> u64 { - ticks_to_nanos(now.saturating_sub(counter_at_boot), period_fs) +/// The page's layout. +#[repr(C)] +pub struct ClockPage { + magic: AtomicU64, + stamp_at_boot: AtomicU64, + demoted: AtomicU64, + line: Latch<3>, } -/// `ticks` of a counter whose tick is `period_fs` femtoseconds, in nanoseconds: -/// `⌊ticks · period_fs / 10⁶⌋` in its low 64 bits, the one formula the kernel -/// and every clock-page reader share. -/// -/// Split at 10⁶ ticks, `⌊(q·10⁶ + r)·p / 10⁶⌋ = q·p + ⌊r·p / 10⁶⌋`, so it -/// takes two 64-bit divisions by a constant, which compile to multiplications, -/// in place of a 128-bit division, which is a library call. Exact for any tick -/// period under 18 ms (`10⁶ · period_fs < 2⁶⁴`). -pub const fn ticks_to_nanos(ticks: u64, period_fs: u64) -> u64 { - const FS_PER_NS: u64 = 1_000_000; - (ticks / FS_PER_NS).wrapping_mul(period_fs).wrapping_add(ticks % FS_PER_NS * period_fs / FS_PER_NS) +impl ClockPage { + /// Lay the page out: the kernel's, once, before any address space maps it. + pub fn init(&self, stamp_at_boot: u64, line: Line) { + self.stamp_at_boot.store(stamp_at_boot, Ordering::Relaxed); + self.line.publish(line.words()); + self.magic.store(CLOCK_MAGIC, Ordering::Release); + } + + /// Replace the line: the kernel's, one writer at a time. + pub fn publish(&self, line: Line) { + self.line.publish(line.words()); + } + + /// Send every reading to the kernel from now on: the kernel's, once. + pub fn demote(&self) { + self.demoted.store(1, Ordering::Release); + } + + /// Whether a reading of now asks the kernel rather than the line. + pub fn demoted(&self) -> bool { + self.demoted.load(Ordering::Acquire) != 0 + } + + /// The clock, a counter reading `counter` took inside the line's read + /// section, and whether the page was demoted after it; `None` on a page + /// that carries none. + pub fn read_now(&self, counter: impl Fn() -> u64) -> Option<(Clock, u64, bool)> { + if self.magic.load(Ordering::Acquire) != CLOCK_MAGIC { + return None; + } + let (words, (now, demoted)) = self.line.read_with(|| { + let now = counter(); + (now, self.demoted()) + }); + Some((Clock { line: Line::of_words(words), stamp_at_boot: self.stamp_at_boot.load(Ordering::Relaxed) }, now, demoted)) + } + + /// The clock now, or `None` on a page that carries none. + pub fn read(&self) -> Option { + if self.magic.load(Ordering::Acquire) != CLOCK_MAGIC { + return None; + } + let line = Line::of_words(self.line.read()); + Some(Clock { line, stamp_at_boot: self.stamp_at_boot.load(Ordering::Relaxed) }) + } } -/// Nanoseconds since boot, off this process's clock page. +/// Nanoseconds since boot, off this process's clock page, or the kernel's +/// clock where the page is demoted. /// /// # Panics /// As [`page`]. pub fn nanos_since_boot() -> u64 { - let page = page(); - let now = crate::arch::counter(); - nanos_between(page.counter_at_boot, page.period_fs, now) + let (clock, now, demoted) = read_now(); + if demoted { + return crate::syscall::clock_now().saturating_sub(clock.stamp_at_boot); + } + clock.nanos_since_boot(now) } /// Now as a log line's time: nanoseconds since the counter's zero, the kernel /// log's own `clock::stamp`, which every line of the log carries -/// (`log::LogRecord::at_ns`). +/// (`log::LogRecord::at_ns`); the kernel's clock where the page is demoted. /// /// # Panics /// As [`page`]. pub fn stamp_ns() -> u64 { - page().stamp_of(crate::arch::counter()) + let (clock, now, demoted) = read_now(); + if demoted { + return crate::syscall::clock_now(); + } + clock.stamp_of(now) +} + +fn read_now() -> (Clock, u64, bool) { + mapped() + .read_now(crate::arch::counter) + .unwrap_or_else(|| panic!("the clock page at {CLOCK_PAGE:#x} carries no clock")) } -/// This process's clock page. +/// The clock, off this process's page: its line, for a counter reading. /// /// # Panics /// On a page that does not carry [`CLOCK_MAGIC`]: every address space the /// kernel builds maps one, so its absence is a kernel that broke the ABI. -pub fn page() -> ClockPage { +pub fn page() -> Clock { + read(mapped()) +} + +fn mapped() -> &'static ClockPage { // SAFETY: the kernel maps the page read-only at `CLOCK_PAGE` in every - // address space before the first instruction runs, and never writes it - // again, so a volatile copy of the whole value is a read of constants. - let page = unsafe { core::ptr::read_volatile(CLOCK_PAGE as *const ClockPage) }; - assert!(page.magic == CLOCK_MAGIC, "the clock page at {CLOCK_PAGE:#x} carries no clock"); - page + // address space before the first instruction runs and never unmaps it; + // every field is an atomic, so a shared reference reads it as atomics. + unsafe { &*(CLOCK_PAGE as *const ClockPage) } +} + +fn read(page: &ClockPage) -> Clock { + page.read().unwrap_or_else(|| panic!("the clock page at {CLOCK_PAGE:#x} carries no clock")) } #[cfg(test)] mod tests { use super::*; - /// The arithmetic the kernel's `clock::nanos_since_boot` does, so the two - /// clocks agree on every counter value: a 1 GHz counter is one tick per - /// nanosecond, and a counter behind boot reads zero. + /// A 1 GHz counter: one tick a nanosecond. + const GHZ: u64 = 1 << MULT_SHIFT; + #[test] - fn the_formula_is_the_kernels() { - assert_eq!(nanos_between(100, 1_000_000, 1_100), 1_000); - assert_eq!(nanos_between(100, 1_000_000, 50), 0); - // 2.5 GHz: 400 fs a tick. - assert_eq!(nanos_between(0, 400_000, 2_500_000_000), 1_000_000_000); - // No overflow at a counter the machine will not reach in a century. - assert_eq!(nanos_between(0, 1_000_000, u64::MAX), u64::MAX); - } - - /// The split formula is the 128-bit product's quotient for every period a - /// counter has, at every count: the ends, the split's own boundary, and a - /// spread between. + fn a_line_counts_from_its_start() { + let line = Line { counter: 100, stamp: 5_000, mult: GHZ }; + assert_eq!(line.stamp_of(1_100), 6_000); + assert_eq!(line.stamp_of(100), 5_000); + // Carried back before the start, and never before the counter's zero. + assert_eq!(line.stamp_of(40), 4_940); + assert_eq!(Line { counter: 10_000, stamp: 5, mult: GHZ }.stamp_of(0), 0); + // Saturating at a counter no machine reaches. + assert_eq!(line.stamp_of(u64::MAX), u64::MAX); + } + + #[test] + fn boot_reads_zero_and_before_boot_reads_boot() { + let clock = Clock { line: Line { counter: 1_000, stamp: 2_000_000_000, mult: GHZ }, stamp_at_boot: 2_000_000_000 }; + assert_eq!(clock.nanos_since_boot(1_000), 0); + assert_eq!(clock.nanos_since_boot(1_500), 500); + assert_eq!(clock.nanos_since_boot(999), 0); + } + + /// The rate round-trips through `mult` to a part in 10⁹ for every counter + /// a machine has, the generic timer's 24 MHz to a 5 GHz TSC, and a second + /// of ticks is a second to the nanosecond. #[test] - fn the_split_is_the_wide_quotient() { - let wide = |ticks: u64, fs: u64| ((ticks as u128 * fs as u128) / 1_000_000) as u64; - let mut x = 0x2545_f491_4f6c_dd1du64; - let mut next = || { - x = x.wrapping_mul(6_364_136_223_846_793_005).wrapping_add(1_442_695_040_888_963_407); - x - }; - let mut periods = std::vec![1, 999_999, 1_000_000, 1_000_001, 41_666_666, 413_393, 18_000_000_000_000]; - periods.extend((0..64).map(|_| 1 + next() % 100_000_000)); - let mut ticks = std::vec![0, 1, 999_999, 1_000_000, 1_000_001, u64::MAX - 1, u64::MAX]; - ticks.extend((0..256).map(|_| next())); - for &fs in &periods { - for &t in &ticks { - assert_eq!(ticks_to_nanos(t, fs), wide(t, fs), "{t} ticks of {fs} fs"); - } + fn mult_keeps_the_rate() { + for hz in [24_000_000, 1_796_624_812, 2_419_200_000, 5_000_000_000] { + let line = Line { counter: 0, stamp: 0, mult: Line::mult_of_hz(hz) }; + let millihertz = hz * 1_000; + let error_ppb = line.millihertz().abs_diff(millihertz) as u128 * 1_000_000_000 / millihertz as u128; + assert!(error_ppb <= 1, "{hz} Hz came back as {} mHz", line.millihertz()); + assert!(line.nanos(hz).abs_diff(1_000_000_000) <= 1, "{hz} Hz: a second is {} ns", line.nanos(hz)); + assert!(line.ticks(1_000_000_000).abs_diff(hz) <= 1 + hz / 1_000_000_000); } } + /// Half a unit of `mult` at 2.4192 GHz is 0.28 ppb, so a day of a stated + /// rate's ticks is a day to 25 µs; a period kept in whole femtoseconds read + /// it 1.9 ppm short, 164 ms. + #[test] + fn a_day_at_a_stated_rate_is_a_day() { + let hz = 2_419_200_000u64; + let line = Line { counter: 0, stamp: 0, mult: Line::mult_of_hz(hz) }; + let day = 86_400u64; + assert!(line.nanos(hz * day).abs_diff(day * 1_000_000_000) <= 25_000); + } + #[test] - fn the_page_is_four_words() { - assert_eq!(core::mem::size_of::(), 32); + fn the_page_publishes_whole_lines() { + let page = ClockPage { magic: AtomicU64::new(0), stamp_at_boot: AtomicU64::new(0), demoted: AtomicU64::new(0), line: Latch::new() }; + assert_eq!(page.read(), None); + let first = Line { counter: 7, stamp: 9, mult: GHZ }; + page.init(9, first); + assert_eq!(page.read(), Some(Clock { line: first, stamp_at_boot: 9 })); + let second = Line { counter: 70, stamp: 72, mult: GHZ / 2 }; + page.publish(second); + assert_eq!(page.read().map(|c| c.line), Some(second)); + page.publish(first); + assert_eq!(page.read().map(|c| c.line), Some(first)); + assert!(!page.demoted()); + page.demote(); + assert!(page.demoted()); } } diff --git a/sdk/abi/src/clock/latch.rs b/sdk/abi/src/clock/latch.rs new file mode 100644 index 00000000000..4ffd043946a --- /dev/null +++ b/sdk/abi/src/clock/latch.rs @@ -0,0 +1,85 @@ +//! A value one writer replaces while any reader, an NMI on the writer's own +//! CPU among them, reads it with no lock and no wait for the writer: two +//! copies, and a sequence whose parity names the one to read. +//! +//! The writer moves readers to the second copy, rewrites the first, moves them +//! back and rewrites the second, so a reader always has a copy nobody is +//! writing; it retries only when the sequence moved while it read. Every +//! payload word is atomic, so a torn read is words thrown away, not a data +//! race. No `crate::` references: `kernel-loom` compiles this file under +//! `feature = "loom"`. + +#[cfg(not(feature = "loom"))] +use core::sync::atomic::{fence, AtomicU64, Ordering}; + +#[cfg(feature = "loom")] +use loom::sync::atomic::{fence, AtomicU64, Ordering}; + +#[repr(C)] +pub struct Latch { + seq: AtomicU64, + copies: [[AtomicU64; N]; 2], +} + +impl Latch { + /// The kernel lays a latch out in place on its page, so only a test builds one. + #[cfg(all(test, not(feature = "loom")))] + pub(super) const fn new() -> Self { + Self { seq: AtomicU64::new(0), copies: [const { [const { AtomicU64::new(0) }; N] }; 2] } + } + + #[allow(clippy::new_without_default)] + #[cfg(feature = "loom")] + pub fn new() -> Self { + Self { seq: AtomicU64::new(0), copies: core::array::from_fn(|_| core::array::from_fn(|_| AtomicU64::new(0))) } + } + + /// Replace the value. One writer at a time: the caller's own exclusion is + /// what makes the load-then-store of `seq` sound. + pub fn publish(&self, words: [u64; N]) { + let seq = self.seq.load(Ordering::Relaxed); + // `Release`, so a reader sent to the other copy sees what the last + // publication left in it. + self.seq.store(seq.wrapping_add(1), Ordering::Release); + // Orders the move before the rewrite: a reader that sees a new word + // sees the sequence moved, and throws the read away. + #[cfg(not(feature = "clock-latch-fence-off"))] + fence(Ordering::Release); + Self::store(&self.copies[(seq & 1) as usize], words); + self.seq.store(seq.wrapping_add(2), Ordering::Release); + fence(Ordering::Release); + Self::store(&self.copies[(seq.wrapping_add(1) & 1) as usize], words); + } + + /// The value one publication left, whole. + pub fn read(&self) -> [u64; N] { + self.read_with(|| ()).0 + } + + /// The value one publication left, whole, and what `inside` read while + /// that publication was in force: a publication that lands between the + /// value's loads and the recheck after `inside` sends the read round + /// again. `inside` orders whatever it reads that is not memory, such as a + /// counter, after the loads before it and before the recheck after it. + pub fn read_with(&self, mut inside: impl FnMut() -> T) -> ([u64; N], T) { + loop { + let seq = self.seq.load(Ordering::Acquire); + let copy = &self.copies[(seq & 1) as usize]; + let words = core::array::from_fn(|i| copy[i].load(Ordering::Relaxed)); + let read = inside(); + // Orders the word loads, and `inside`'s, before the recheck: a + // word a later publication wrote is one whose move the recheck + // sees. + fence(Ordering::Acquire); + if self.seq.load(Ordering::Relaxed) == seq { + return (words, read); + } + } + } + + fn store(copy: &[AtomicU64; N], words: [u64; N]) { + for (slot, word) in copy.iter().zip(words) { + slot.store(word, Ordering::Relaxed); + } + } +} diff --git a/sdk/abi/src/syscall.rs b/sdk/abi/src/syscall.rs index 4c95cc77890..95834edfa15 100644 --- a/sdk/abi/src/syscall.rs +++ b/sdk/abi/src/syscall.rs @@ -348,6 +348,8 @@ pub const SYS_EVENT_SIGNAL: u64 = 149; pub const SYS_EVENT_TAKE: u64 = 150; /// Park on one handle's one condition, with a timeout. See [`handle_wait`]. pub const SYS_HANDLE_WAIT: u64 = 151; +/// The clock now, read by the kernel. See [`clock_now`]. +pub const SYS_CLOCK_NOW: u64 = 152; /// Bins in the per-process syscall profile — one for every number this ABI /// issues, and one at the end for every number it does not. @@ -355,13 +357,13 @@ pub const SYS_HANDLE_WAIT: u64 = 151; /// **The profile's parts sum to its total, and that is the whole requirement.** /// A bin array narrower than the ABI reaches drops calls out of the line while /// the total goes on counting them. -pub const SYSCALL_PROFILE_BINS: usize = 153; +pub const SYSCALL_PROFILE_BINS: usize = 154; /// Where a number this ABI does not issue is counted. Merging is a degradation /// a reader can see in the line; dropping is one nobody can. pub const SYSCALL_PROFILE_OTHER: usize = SYSCALL_PROFILE_BINS - 1; -const _: () = assert!(SYS_HANDLE_WAIT < SYSCALL_PROFILE_OTHER as u64); +const _: () = assert!(SYS_CLOCK_NOW < SYSCALL_PROFILE_OTHER as u64); pub const WNOHANG: u64 = 1; /// Every bit `SYS_PROCESS_WAIT`'s flag word defines; a wait carrying another @@ -1416,6 +1418,13 @@ pub fn reboot(syscap: RawHandle) -> SyscallError { } } +/// The clock now as a log line's time, nanoseconds since the counter's zero, +/// read by the kernel: what [`crate::clock::stamp_ns`] answers where the page +/// is demoted. Needs no handle; it fails for nobody. +pub fn clock_now() -> u64 { + syscall(SYS_CLOCK_NOW, 0, 0, 0, 0) +} + /// Which stop [`machine_stops`] asks about. #[derive(Clone, Copy, Debug, PartialEq, Eq)] #[repr(u64)] diff --git a/sdk/abi/src/trace.rs b/sdk/abi/src/trace.rs index 5268dae645f..713d2b26030 100644 --- a/sdk/abi/src/trace.rs +++ b/sdk/abi/src/trace.rs @@ -3,7 +3,7 @@ //! always. //! //! A record's [`TraceRecord::stamp`] is the CPU's free-running counter (TSC, -//! `CNTVCT_EL0`) in its own ticks, which [`crate::clock::ClockPage`] turns into +//! `CNTVCT_EL0`) in its own ticks, which a [`crate::clock::Clock`] turns into //! nanoseconds since boot. A read takes a cursor laid out as the log's //! ([`crate::log::CursorHead`]), a position per CPU the firmware names, and //! answers records oldest first, merged across CPUs by stamp, with the diff --git a/tests/checks.rs b/tests/checks.rs index b513b75f0bf..389441b4b8d 100644 --- a/tests/checks.rs +++ b/tests/checks.rs @@ -575,6 +575,15 @@ mod checks { Ok(()) } + /// A slewed step passes the clock's rows and a demotion fails them. + #[test] + fn a_demoted_clock_fails_its_rows() { + let slewed = "[ 20.778 cpu12 kernel] clock: TSC STEP: the TSC stepped 1695132 ns forward against the HPET\n"; + assert_eq!(clock_kept_its_counter(slewed), Ok(())); + let demoted = format!("{slewed}[ 33.252 cpu1 kernel] clock: DEMOTED: the TSC stepped 10000110 ns\n"); + assert!(clock_kept_its_counter(&demoted).is_err()); + } + /// [`irq_census`] over two censuses, X's from cpu2 and Y's from cpu3, whose /// lines are in no read order: a CPU's census is the largest count each /// source reaches on any of its lines, and the largest issuer total bounds @@ -585,7 +594,7 @@ mod checks { let userdev = if cpu == 0 { 40 } else { 0 }; format!( "[ {at} cpu{on} kernel] irq: cpu{cpu} timer=100 kick={kick} userdev={userdev} \ - dmafault=0 tlb={tlb} nmi=0 spurious=0 unclaimed=0\n" + dmafault=0 rtc=0 tlb={tlb} nmi=0 spurious=0 unclaimed=0\n" ) }; let issued = |at: &str, on: u32, total: u64| { @@ -1104,7 +1113,7 @@ mod checks { assert!(judge(&[&readback("jobcase", &done(stopped), jobcase)]).is_err()); // What the seal itself writes of the machine, above the ring's tail. - let census = "| irq: cpu0 timer=9 kick=2 userdev=1729 dmafault=0 tlb=0 \ + let census = "| irq: cpu0 timer=9 kick=2 userdev=1729 dmafault=0 rtc=0 tlb=0 \ nmi=0 spurious=0 unclaimed=0\n\ | tlb: shootdowns=4 wait=12us max=3us dlopen=0 pcid=0 mmio=0 unmap=4 pipe=0 staged=0 \ bench=0\n\ diff --git a/tests/common/faults.rs b/tests/common/faults.rs index eab226b15d8..871b6190e02 100644 --- a/tests/common/faults.rs +++ b/tests/common/faults.rs @@ -135,3 +135,75 @@ pub fn kernel_wx_leaf_is_refused(test_config: &Path, profile: qemu::Profile) -> _ => Err(format!("the audit did not name {at} alone under both rules: {refusal}\n{serial}")), } } + +/// What `tsc-step` says of each step it stages, and the clock's records of +/// slewing the first away and demoting the TSC at the second +/// (`kernel/src/arch/x86_64/clocksource.rs`). +const TSC_STEP_STAGED: &str = "tsc-step: the TSC reads "; +const TSC_STEP_SLEWED: &str = "clock: TSC STEP: the TSC stepped "; +const TSC_SLEWED_AWAY: &str = "clock: the TSC's step is slewed away"; +const TSC_STEP_AGAIN: &str = "cycles further ahead from here"; +const TSC_DEMOTED: &str = "clock: DEMOTED: "; +const DEMOTED_READS: &str = "clock: a process's reading of now asked the kernel since the TSC was demoted"; + +/// A TSC `tsc-step` reads 10 ms ahead, and 10 ms further once the clock has +/// slewed the first step away: the first is slewed, the second demotes the +/// TSC, and processes' readings of now reach the kernel after it. +pub fn a_stepped_tsc_is_slewed_then_demoted(test_config: &Path) -> Result<(), String> { + let mut qemu = QemuInstance::boot_with_options( + test_config, + &[], + &[], + BootOptions { profile: qemu::Profile::Metal, kernel_params: &["tsc-step"], ready_marker: TSC_STEP_SLEWED, ..Default::default() }, + ); + let mut serial = qemu.boot_log().to_string(); + // A 10 ms step at 500 ppm is 20 s of slewing; the rest is the RTC's edges either side of each step. + serial.push_str(&qemu.drain_until(std::time::Duration::from_secs(120), |l| l.contains(DEMOTED_READS) || l.contains("PANIC"))); + let lines: Vec<&str> = serial.lines().collect(); + let at = |marker: &str| lines.iter().position(|l| l.contains(marker)).ok_or_else(|| format!("no {marker:?} line\n{serial}")); + let order = [at(TSC_STEP_STAGED)?, at(TSC_STEP_SLEWED)?, at(TSC_SLEWED_AWAY)?, at(TSC_STEP_AGAIN)?, at(TSC_DEMOTED)?, at(DEMOTED_READS)?]; + if !order.is_sorted() || serial.contains("PANIC") { + return Err(format!("the steps, the slew and the demotion are out of order: {order:?}\n{serial}")); + } + let stepped: Option = lines[order[1]].split(TSC_STEP_SLEWED).nth(1).and_then(|r| r.split(" ns").next()).and_then(|n| n.parse().ok()); + if !stepped.is_some_and(|ns| (9_000_000..=11_000_000).contains(&ns)) { + return Err(format!("the slewed step is not the staged 10 ms: {}\n{serial}", lines[order[1]].trim())); + } + for i in [1, 4, 5] { + eprintln!(" [tsc_step] {}", lines[order[i]].trim()); + } + Ok(()) +} + +/// What `hpet-stall` says of the stall it stages, the sampler's record of +/// leaving the span out, and a report after it. +const HPET_STALL_STAGED: &str = "hpet-stall: the HPET reads "; +const HPET_MOVED: &str = "clock: the HPET moved and the TSC did not, so this span is left out: "; +const CLOCK_REPORT: &str = "clock: over "; + +/// An HPET `hpet-stall` reads 10 ms behind while the TSC and the ACPI PM timer +/// count on: the sampler says the HPET moved, leaves the span out, and the +/// clock goes on reporting rather than panicking. +pub fn a_stalled_hpet_is_left_out(test_config: &Path) -> Result<(), String> { + let mut qemu = QemuInstance::boot_with_options( + test_config, + &[], + &[], + BootOptions { profile: qemu::Profile::Metal, kernel_params: &["hpet-stall"], ready_marker: HPET_MOVED, ..Default::default() }, + ); + let staged_then_moved = { + let serial = qemu.boot_log(); + let staged = serial.find(HPET_STALL_STAGED); + let moved = serial.find(HPET_MOVED); + match (staged, moved) { + (Some(staged), Some(moved)) if staged < moved => Ok(serial[moved..].lines().next().unwrap_or_default().to_string()), + _ => Err(format!("no {HPET_STALL_STAGED:?} before {HPET_MOVED:?}\n{serial}")), + } + }?; + let after = qemu.drain_until(std::time::Duration::from_secs(60), |l| l.contains(CLOCK_REPORT) || l.contains("PANIC")); + if after.contains("PANIC") || !after.contains(CLOCK_REPORT) { + return Err(format!("the clock did not report again after it left the span out\n{after}")); + } + eprintln!(" [hpet_stall] {}", staged_then_moved.trim()); + Ok(()) +} diff --git a/tests/common/irqcensus.rs b/tests/common/irqcensus.rs index de38f7db579..42154a7119d 100644 --- a/tests/common/irqcensus.rs +++ b/tests/common/irqcensus.rs @@ -15,14 +15,14 @@ use std::collections::BTreeMap; /// and [`Census::parse`] refuses a line whose fields are not exactly these, so /// a source added on one side and not the other is a red rather than a silently /// dropped column. -pub const SOURCES: [&str; 8] = ["timer", "kick", "userdev", "dmafault", "tlb", "nmi", "spurious", "unclaimed"]; +pub const SOURCES: [&str; 9] = ["timer", "kick", "userdev", "dmafault", "rtc", "tlb", "nmi", "spurious", "unclaimed"]; /// The sources whose delivery CPU is chosen by the interrupt controller rather /// than by the CPU that took the work — every device vector, in other words. /// `MSG_ADDR` names physical destination 0 and the one I/O APIC pin this kernel /// routes goes to the BSP, so today every one of these is cpu0's alone. The day /// that stops being true is the day the track's change lands. -pub const DEVICE_SOURCES: [&str; 2] = ["userdev", "dmafault"]; +pub const DEVICE_SOURCES: [&str; 3] = ["userdev", "dmafault", "rtc"]; /// One CPU's counters out of one `irq:` line. #[derive(Clone, Debug, PartialEq, Eq)] diff --git a/tests/guest/osbench/src/around.rs b/tests/guest/osbench/src/around.rs index 6f3ecbb3413..7b523493793 100644 --- a/tests/guest/osbench/src/around.rs +++ b/tests/guest/osbench/src/around.rs @@ -82,8 +82,8 @@ pub fn read(d: &crate::sys::diag::Diag) -> Result { #[cfg(target_os = "toyos")] pub fn cpus(bench: &str, before: &Reading, after: &Reading) { use osbench_rules::window; - let page = toyos_abi::clock::page(); - let to_ns = |ticks: u64| toyos_abi::clock::ticks_to_nanos(ticks, page.period_fs); + let line = toyos_abi::clock::page().line; + let to_ns = |ticks: u64| line.nanos(ticks); let span = window::Span { row_ns: u64::try_from(after.called.duration_since(before.called).as_nanos()).expect("a row under 584 years") + after.cost_ns, diff --git a/tests/guest/osbench/src/sys/toyos.rs b/tests/guest/osbench/src/sys/toyos.rs index c15d6a9a55c..37bad22d5f8 100644 --- a/tests/guest/osbench/src/sys/toyos.rs +++ b/tests/guest/osbench/src/sys/toyos.rs @@ -321,7 +321,7 @@ pub mod diag { }; for raw in &buf[..n] { if let Ok(entry) = Entry::decode(raw) { - let since_boot = page.stamp_of(entry.stamp).saturating_sub(page.stamp_at_boot); + let since_boot = page.nanos_since_boot(entry.stamp); out.push((since_boot, entry)); } } diff --git a/tests/guest/rust-tests/src/bin/abuse_readonly_copyout.rs b/tests/guest/rust-tests/src/bin/abuse_readonly_copyout.rs index 2fdedb1230e..2d1b291393f 100644 --- a/tests/guest/rust-tests/src/bin/abuse_readonly_copyout.rs +++ b/tests/guest/rust-tests/src/bin/abuse_readonly_copyout.rs @@ -21,7 +21,7 @@ mod image; use std::os::toyos::io::AsRawHandle; use std::process::Command; -use toyos_abi::clock::{ClockPage, CLOCK_MAGIC, CLOCK_PAGE}; +use toyos_abi::clock::{ClockPage, CLOCK_PAGE}; use toyos_abi::syscall::{self, OpenFlags, VmAt, VmProt, ProcessStats, SeekFrom, SyscallError}; use toyos_abi::RawHandle; @@ -140,8 +140,10 @@ fn second_reader() -> bool { fn main() { if std::env::args().nth(1).as_deref() == Some(CHILD_ARG) { - let page = unsafe { core::ptr::read_volatile(CLOCK_PAGE as *const ClockPage) }; - assert_eq!(page.magic, CLOCK_MAGIC, "a second process found the clock page rewritten"); + // SAFETY: the kernel maps the page read-only here in every address + // space, and every field of it is an atomic. + let page = unsafe { &*(CLOCK_PAGE as *const ClockPage) }; + assert!(page.read().is_some(), "a second process found the clock page rewritten"); std::process::exit(0); } @@ -172,7 +174,11 @@ fn main() { } // Last, and said by its bit alone: its harm reaches every stamping reader. - if refused("the clock page", CLOCK_PAGE, stats).is_err() || !second_reader() { + // Half a page in, past the clock's own words, which the kernel rewrites + // when it retunes: the same frame, and bytes nothing moves. + const CLOCK_SPARE: u64 = CLOCK_PAGE + PAGE_4K / 2; + const _: () = assert!(core::mem::size_of::() <= (PAGE_4K / 2) as usize && LEN <= (PAGE_4K / 2) as usize); + if refused("the clock page", CLOCK_SPARE, stats).is_err() || !second_reader() { wrote |= CLOCK_BIT; } if wrote != 0 { diff --git a/tests/toyos.rs b/tests/toyos.rs index 4dc4052e248..a8fc9aaca0d 100644 --- a/tests/toyos.rs +++ b/tests/toyos.rs @@ -520,6 +520,17 @@ const MACHINE_TESTS: &[&str] = &[ // the same audit passing. q35 on `Profile::Metal`, whose 16550 carries // the panic. "kernel_wx_leaf_is_refused", + // The clock's response to TSC steps an actuator stages where + // `IA32_TSC_ADJUST` would not show them: the first slewed away, the + // second demoting the TSC, after which processes' readings of now reach + // the kernel. q35 on `Profile::Metal`; the order, the step's size and a + // demoted reading are what it holds, and no bench's TSC steps on demand. + "a_stepped_tsc_is_slewed_then_demoted", + // Its counterpart: the HPET an actuator stalled while the TSC and the PM + // timer count on is left out, and the boot goes on. q35 on + // `Profile::Metal`, whose PM timer is at 0x608; no bench's HPET stalls on + // demand. + "a_stalled_hpet_is_left_out", // A press of the fixed power button reaches the `acpi` claim's holder as // one SCI, its line masked until the holder serves it: the kernel's SCI // and its mask have no host build, QEMU raises the event on demand, and @@ -1113,7 +1124,7 @@ const METAL: &[(&str, metal::Metal)] = &[ arms: TESTCASES, judge: |b| { timer_calibration(b[0].kernel().text())?; - tsc_agrees_with_cpuid(b[0].kernel().text()) + tsc_rate_at_boot(b[0].kernel().text()) }, }, ), @@ -1200,6 +1211,13 @@ const METAL: &[(&str, metal::Metal)] = &[ "osbench_energy", metal::Metal { arms: OSBENCH_ENERGY, judge: |b| osbench_energy_measured(b[0]) }, ), + ( + // The clock's boot rate against the HPET's over the minutes the suite + // runs, through its idle, busy and hot phases: the T14 and the AMD + // bench, riding the `osbench` boot. + "clock_holds_over_minutes", + metal::Metal { arms: BENCHCASE, judge: |b| clock_holds_over_minutes(b[0]) }, + ), ( // `osbench lan` against the `osbench-peer` the other bench's Linux runs, // compared with the same program under this bench's Linux on the same @@ -4667,6 +4685,8 @@ fn run_machine_test(name: &str, test_config: &Path) -> Result<(), String> { "amdvi_selftest" => common::iommu::amdvi_selftest(test_config), "nested_nmi_is_loud" => faults::nested_nmi_is_loud(test_config), "kernel_wx_leaf_is_refused" => faults::kernel_wx_leaf_is_refused(test_config, qemu::Profile::Metal), + "a_stepped_tsc_is_slewed_then_demoted" => faults::a_stepped_tsc_is_slewed_then_demoted(test_config), + "a_stalled_hpet_is_left_out" => faults::a_stalled_hpet_is_left_out(test_config), "acpi_press_is_one_sci" => acpi_press_is_one_sci(), "osbench_capability_groups_exit_0" => osbench_capability_groups_exit_0(), "acpi_mediated_access" => acpi_mediated_access(2), @@ -7426,20 +7446,15 @@ fn number_between(log: &str, head: &str, tail: &str) -> Result { } /// Every CPU the firmware named is scheduling, every one of them holds the -/// control-register declaration, and none of their timestamp counters trails -/// the BSP's. +/// control-register declaration, and every AP's timestamp counter is bounded +/// against the BSP's. /// -/// **The third is the one nothing in this tree ever asked.** -/// `clock::nanos_since_boot` subtracts a single BSP-sampled origin whatever CPU -/// reads it, so a CPU whose TSC starts behind that origin saturates to zero and -/// stamps every record it writes as the oldest thing the machine has -///. The kernel -/// brackets each AP's first `rdtsc` between two of the BSP's, taken either side -/// of a bring-up that is serialised — so "inside" is what a synchronised -/// counter gives and nothing else does. **QEMU cannot refute it**: every guest -/// TSC is synthesised from one host clock, which is exactly why the assertion -/// is here to be run on metal. +/// The kernel bounds each AP's offset with round trips of a cache line as it +/// starts and panics on one they prove, so the row reads each AP's bound and +/// prints the widest. **QEMU cannot refute it**: every guest TSC is +/// synthesised from one host clock, which is why it runs on metal. fn smp_roster_and_tsc_trail(log: &str, cpus: u32) -> Result<(), String> { + clock_kept_its_counter(log)?; let online = number_between(log, "SMP: ", " of ")?; let named = number_between(log, " of ", " MADT cpus online")?; if online != u64::from(cpus) || named != u64::from(cpus) { @@ -7447,33 +7462,28 @@ fn smp_roster_and_tsc_trail(log: &str, cpus: u32) -> Result<(), String> { "the machine was given {cpus} CPUs, its MADT named {named} and {online} came up" )); } - let bracketed = number_between(log, "MADT cpus online, ", " of ")?; + let bounds: Vec<(i64, i64)> = log.lines().filter_map(tsc_offset_bound).collect(); let aps = u64::from(cpus) - 1; - if bracketed != aps { - let outside: Vec<&str> = log - .lines() - .filter(|l| l.contains(" the BSP's ") && !l.contains("inside")) - .collect(); - return Err(format!( - "{bracketed} of {aps} APs read a TSC inside the BSP's bracket; the rest: {outside:#?}" - )); + if bounds.len() as u64 != aps { + return Err(format!("{} of {aps} APs' TSCs were bounded against the BSP's", bounds.len())); } let checked = number_between(log, "control_regs: ", " of ")?; if checked != u64::from(cpus) { return Err(format!("{checked} of {cpus} CPUs were checked against the declaration")); } - let widest = log - .lines() - .filter_map(|l| l.split(" (").nth(1)?.split(" cycles wide)").next()?.parse::().ok()) - .max() - .unwrap_or_default(); - eprintln!( - " [smp] {online} CPUs online, {bracketed} AP TSCs inside a BSP bracket at most \ - {widest} cycles wide" - ); + let widest = bounds.iter().map(|&(lo, hi)| lo.unsigned_abs().max(hi.unsigned_abs())).max().unwrap_or_default(); + eprintln!(" [smp] {online} CPUs online, every AP's TSC within {widest} cycles of the BSP's"); Ok(()) } +/// `lo..=hi` off `clock: cpuN's TSC is within lo..=hi cycles of cpu0's`. +fn tsc_offset_bound(line: &str) -> Option<(i64, i64)> { + let (_, rest) = line.split_once("'s TSC is within ")?; + let (bound, _) = rest.split_once(" cycles of cpu0's")?; + let (lo, hi) = bound.split_once("..=")?; + Some((lo.parse().ok()?, hi.parse().ok()?)) +} + /// The local APIC mode a bench's CPUs offer, by `CPUID.01H:ECX[21]` as each /// bench's scout read it on every CPU (`kernel/sim/tests/scout/`): the T14's /// `0x77fafbbf` sets it and the AMD bench's `0x76d8320b` does not, so the @@ -7608,55 +7618,124 @@ fn acpi_table_inventory(log: &str) -> Result<(), String> { Ok(()) } -/// The LAPIC timer and the TSC each calibrated to a frequency. +/// The LAPIC timer calibrated to a frequency. fn timer_calibration(log: &str) -> Result<(), String> { let lapic_hz = number_between(log, "ticks/10ms, so ", "Hz")?; if lapic_hz == 0 { return Err("the LAPIC timer calibrated to no frequency at all".to_string()); } - let measured = number_between(log, "clock: TSC measured ", "Hz against the HPET")?; - if measured == 0 { - return Err("the TSC calibrated to no frequency at all".to_string()); + eprintln!(" [timer] LAPIC {lapic_hz}Hz"); + Ok(()) +} + +/// The rate the boot started the clock at: CPUID 15H's, or one the HPET +/// measured with its bound, which on the bench is a rate the clock can keep at +/// all. Its bound is held against minutes by [`clock_holds_over_minutes`]. +fn tsc_rate_at_boot(log: &str) -> Result<(), String> { + clock_kept_its_counter(log)?; + let boot = boot_rate(log)?; + if boot.millihertz == 0 { + return Err("the TSC was started at no rate at all".to_string()); } - eprintln!(" [timer] TSC {measured}Hz measured; LAPIC {lapic_hz}Hz"); + eprintln!(" [timer] TSC {boot:?}"); Ok(()) } -/// The TSC the whole machine is timed by, against the frequency the part itself -/// states. -/// -/// **The one cross-source check a boot has.** Everything else the kernel times -/// is derived from the HPET calibration, so it can only agree with itself; -/// CPUID leaf 15H's crystal ratio and leaf 16H's base frequency are the CPU's -/// own statement, arrived at by neither the HPET nor the counting loop. A part -/// that states neither is a fact about the part, not a failure, so the ppm -/// bound is asserted only where a statement exists. Judged on metal only: the -/// calibration is a span of a clock, and a guest's clock runs while its host -/// has the vCPU. -fn tsc_agrees_with_cpuid(log: &str) -> Result<(), String> { - /// One percent, which is the widest two timebases can differ and still be - /// counting the same second. A refusal and not a measurement: it catches a - /// machine whose HPET and CPUID have stopped agreeing at all, and nothing - /// narrower is true of every part this kernel may boot on. - const CEILING_PPM: u64 = 10_000; - - let measured = number_between(log, "clock: TSC measured ", "Hz against the HPET")?; - let Ok(stated) = number_between(log, "CPUID states ", "Hz,") else { - let why = log - .lines() - .find(|l| l.contains("CPUID leaves 15H and 16H")) - .ok_or("neither a stated frequency nor the record saying there is none")?; - eprintln!(" [timer] TSC {measured}Hz — {}", why.trim()); - return Ok(()); - }; - let ppm = number_between(log, "Hz, ", "ppm apart")?; - if ppm > CEILING_PPM { +/// The rate the clock started at, in millihertz, and its bound in ppb: zero +/// for CPUID 15H's, which is not measured. +#[derive(Debug)] +struct BootRate { + millihertz: u64, + bound_ppb: u64, + stated: bool, +} + +fn boot_rate(log: &str) -> Result { + if let Ok(hz) = number_between(log, "clock: the TSC counts at ", " Hz, as CPUID 15H states") { + return Ok(BootRate { millihertz: hz * 1_000, bound_ppb: 0, stated: true }); + } + let millihertz = number_between(log, "clock: the TSC counts at ", " mHz against the HPET over ")?; + let bound_ppb = number_between(log, " ms, within ", " ppb; CPUID 15H states no rate")?; + Ok(BootRate { millihertz, bound_ppb, stated: false }) +} + +/// One of the clock's reports: the TSC's rate against the HPET over a span +/// since boot and its bound, and the rate the clock keeps after it. +#[derive(Debug)] +struct ClockReport { + span_s: u64, + measured_millihertz: u64, + measured_ppb: u64, + kept_millihertz: u64, +} + +fn clock_report(line: &str) -> Option { + let (_, rest) = line.split_once("clock: over ")?; + let (span, rest) = rest.split_once(" s the TSC counted ")?; + let (measured, rest) = rest.split_once(" mHz against the HPET, within ")?; + let (measured_ppb, rest) = rest.split_once(" ppb; the clock ")?; + let kept = rest.split(' ').find_map(|word| word.parse().ok())?; + Some(ClockReport { + span_s: span.parse().ok()?, + measured_millihertz: measured.parse().ok()?, + measured_ppb: measured_ppb.parse().ok()?, + kept_millihertz: kept, + }) +} + +/// Whether the boot's clock kept the TSC: every step it slewed, every span it +/// left out and every step it waited on the RTC for is printed, and a +/// demotion fails the row, since a demoted boot's processes read the HPET. +fn clock_kept_its_counter(log: &str) -> Result<(), String> { + const SAID: [&str; 4] = ["clock: TSC STEP: ", "so this span is left out", "so the span was left out", "the RTC's edges after it will say which moved"]; + for line in log.lines().filter(|line| SAID.iter().any(|said| line.contains(said))) { + eprintln!(" [clock] {}", line.trim()); + } + match log.lines().find(|line| line.contains("clock: DEMOTED")) { + Some(demoted) => Err(format!("the clock demoted the TSC: {}", demoted.trim())), + None => Ok(()), + } +} + +/// Parts per billion `a` is from `b`. +fn ppb_apart(a: u64, b: u64) -> u64 { + (u128::from(a.abs_diff(b)) * 1_000_000_000 / u128::from(b)) as u64 +} + +/// **The clock over minutes.** The kernel holds the TSC to the HPET once a +/// second, panics on a drift and demotes the TSC on a step it does not slew, so +/// a boot that ran its list to the end without a demotion held; what this row +/// reads is whether the rate the boot started at was within the bound it +/// derived, against the rate the HPET measured over +/// the longest span the boot reached, which must be two minutes or more. A +/// stated rate is CPUID 15H's, and against it the HPET's own 0.05% is all a +/// document allows (IA-PC HPET Specification 1.0a, §2.2), so its distance is +/// printed. +fn clock_holds_over_minutes(boot: &metal::Readback) -> Result<(), String> { + let kernel = boot.kernel(); + let text = kernel.text(); + clock_kept_its_counter(text)?; + let started = boot_rate(text)?; + let longest = text + .lines() + .filter_map(clock_report) + .max_by_key(|report| report.span_s) + .ok_or("the clock never reported its rate against the HPET")?; + if longest.span_s < 120 { + return Err(format!("the clock's longest report spans {} s, under two minutes: {longest:?}", longest.span_s)); + } + let apart = ppb_apart(started.millihertz, longest.measured_millihertz); + eprintln!( + " [clock] started at {started:?}; over {} s the HPET measured {} mHz within {} ppb, {apart} ppb from the start; keeps {} mHz", + longest.span_s, longest.measured_millihertz, longest.measured_ppb, longest.kept_millihertz, + ); + let allowed = if started.stated { 500_000 } else { started.bound_ppb } + longest.measured_ppb; + if apart > allowed { return Err(format!( - "the TSC measures {measured}Hz against the HPET and CPUID states {stated}Hz — \ - {ppm}ppm apart, over the {CEILING_PPM}ppm this bound allows" + "the clock started {apart} ppb from the rate the HPET measured over {} s, past the {allowed} ppb its bounds allow", + longest.span_s )); } - eprintln!(" [timer] TSC {measured}Hz measured, {stated}Hz stated, {ppm}ppm apart"); Ok(()) }