diff --git a/.cargo/config.toml b/.cargo/config.toml index 42cdffba96f..798bc0d8739 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -42,3 +42,11 @@ rustflags = ["-Dwarnings"] [target.aarch64-unknown-toyos] rustflags = ["-Dwarnings"] + +# `tests/guest/osbench`'s Linux build, static so a bench runs it without a +# toolchain, linked by the toolchain's own LLD so no C toolchain is needed, and +# with the frame pointers the ToyOS targets keep, so both builds run the same +# code. +[target.x86_64-unknown-linux-musl] +linker = "rust-lld" +rustflags = ["-Dwarnings", "-Cforce-frame-pointers=yes"] diff --git a/Cargo.lock b/Cargo.lock index 0dfa500604b..420896dcaa6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2747,6 +2747,15 @@ dependencies = [ "libredox", ] +[[package]] +name = "osbench" +version = "0.1.0" +dependencies = [ + "libc", + "toyos", + "toyos-abi", +] + [[package]] name = "p256" version = "0.13.2" diff --git a/Cargo.toml b/Cargo.toml index ba693aaadc5..62bf7b025bb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -143,6 +143,7 @@ members = [ "system/vdagent/bridge", "tests/guest/kernelprobe", "tests/guest/metalprobe", + "tests/guest/osbench", "tests/guest/pciholder", "tests/guest/test-runner", "tests/metal", diff --git a/build/build.rs b/build/build.rs index d8934a1d0e0..c8395e68a9d 100644 --- a/build/build.rs +++ b/build/build.rs @@ -3111,6 +3111,7 @@ mod tests { "images/diag.toml", "images/console.toml", "tests/images/acpicase.toml", + "tests/images/benchcase.toml", "tests/images/clipboardcase.toml", "tests/images/drivescase.toml", "tests/images/hostedclangcase.toml", diff --git a/build/ci.rs b/build/ci.rs index c782b5296c0..fb953e7f35e 100644 --- a/build/ci.rs +++ b/build/ci.rs @@ -802,9 +802,9 @@ fn run_control(root: &Path, control: &Control) -> Result { /// end, is a test that fills the host's disk one run at a time. /// /// Clippy needs none of the ToyOS toolchain — the kernel and the bootloader -/// lint against every architecture's bare targets ([`crate::clippy::BARE_TARGETS`]), -/// which any rustup installs, and userland carries no clippy shape -/// (`build/clippy.rs`). +/// lint against every architecture's bare targets, and osbench's Linux half +/// against its Linux one ([`crate::clippy::TARGETS`]), which any rustup +/// installs; no other userland carries a clippy shape (`build/clippy.rs`). /// /// In a job that carries the cache ([`cicache::carried`]) the restored entry is /// read before any step. A developer's tree keeps the dates its edits gave it. @@ -870,10 +870,10 @@ fn host(root: &Path, scope: Result<(Scope, String), String>) -> Vec { }), licences(), ]); - steps.push(step("clippy and the bare targets", || { + steps.push(step("clippy and its targets", || { for args in [ vec!["component", "add", "clippy"], - [&["target", "add"][..], &crate::clippy::BARE_TARGETS].concat(), + [&["target", "add"][..], &crate::clippy::TARGETS].concat(), ] { let status = Command::new("rustup").args(&args).status().map_err(|e| e.to_string())?; if !status.success() { diff --git a/build/clippy.rs b/build/clippy.rs index 5d58443fb91..485c02cad92 100644 --- a/build/clippy.rs +++ b/build/clippy.rs @@ -215,6 +215,11 @@ const SHAPES: &[Shape] = &[ before: &["-p", "toyos", "--lib", "--features", "fd-table"], after: &["$ADOPTED", "-D", "warnings"], }, + // osbench's Linux half, which no image builds: a check, so nothing links. + Shape { + before: &["-p", "osbench", "--target", OSBENCH_LINUX], + after: &["$ADOPTED", "-D", "warnings"], + }, // Its own target directory: cargo keeps one check of a unit, so a unit // linted under two sets of lints is checked again by each, every run, and // so is every crate depending on it. @@ -224,10 +229,14 @@ const SHAPES: &[Shape] = &[ }, ]; -/// The bare targets the kernel and bootloader shapes lint against, which -/// `rustup target add` installs: every architecture's. -pub const BARE_TARGETS: [&str; 4] = - [Arch::X86_64.kernel(), Arch::X86_64.loader(), Arch::Aarch64.kernel(), Arch::Aarch64.loader()]; +/// The Linux target `tests/guest/osbench` is built for beside ToyOS. +const OSBENCH_LINUX: &str = "x86_64-unknown-linux-musl"; + +/// The targets the shapes lint against, which `rustup target add` installs: +/// every architecture's bare targets for the kernel and bootloader, and +/// osbench's Linux one. +pub const TARGETS: [&str; 5] = + [Arch::X86_64.kernel(), Arch::X86_64.loader(), Arch::Aarch64.kernel(), Arch::Aarch64.loader(), OSBENCH_LINUX]; impl Shape { /// The command as a reader writes it, its tokens unexpanded. diff --git a/issues/osbench-builds-for-no-aarch64-toyos.md b/issues/osbench-builds-for-no-aarch64-toyos.md new file mode 100644 index 00000000000..e43a224907e --- /dev/null +++ b/issues/osbench-builds-for-no-aarch64-toyos.md @@ -0,0 +1,19 @@ +--- +status: open +kind: defect +opened: 2026-10-22 +--- + +# osbench builds for no AArch64 ToyOS + +`tests/guest/osbench` names the CPU a thread runs on, and the cores CPUs +share, to stage and check its placement benchmarks. Linux answers both itself +(`sched_getcpu`, sysfs topology). ToyOS has no call for either, so osbench +reads them with x86-64's CPUID (`src/arch/x86_64.rs`). AArch64 offers no EL0 +equivalent: `MPIDR_EL1` is not readable at EL0. So `src/arch/mod.rs` refuses +`aarch64-unknown-toyos` with a `compile_error!`, and no AArch64 machine can be +compared. + +**Exit**: osbench builds and runs its placement benchmarks on an AArch64 ToyOS +guest, either from a ToyOS call that names a thread's CPU and its core, or from +an EL0-readable register the kernel sets per CPU. diff --git a/issues/osbench-times-cpus-leaving-idle-and-not-their-settled-clock.md b/issues/osbench-times-cpus-leaving-idle-and-not-their-settled-clock.md new file mode 100644 index 00000000000..4c3aab75076 --- /dev/null +++ b/issues/osbench-times-cpus-leaving-idle-and-not-their-settled-clock.md @@ -0,0 +1,26 @@ +--- +status: open +kind: tooling +opened: 2026-10-21 +--- + +# osbench times CPUs leaving idle, not their settled clock + +`tests/guest/osbench` runs each benchmark straight after the last one, and +its clock gauge, `cpu.chunk`, is a 2 ms single-threaded burst at the start +of each group. So both read a CPU on its way out of idle, and how fast it +gets there is each system's idle and frequency policy rather than the +workload. + +Measured on the T14 (osbench at 951c7d1, Linux with intel_pstate powersave +and EPP balance_performance, ToyOS with HWP at EPP 0x80, idling in `hlt`): the +same chunk took 10.5 µs on Linux in every group, and 7.9 µs, then 3.3 µs, +across ToyOS's groups. Under all-CPU load the two run one clock: 2.389 M and +2.371 M chunks/s machine-wide. Dividing a result by its group's `cpu.chunk` +is therefore no normalisation: the memory bandwidths, equal raw, come out a +quarter apart. + +**Exit**: each benchmark runs after a steady-state warm-up of the CPUs it +uses and prints the chunk time measured beside it, and a T14 run on each +system shows `cpu.chunk` agreeing with the all-CPU rate per CPU within the +run-to-run spread. diff --git a/tests/guest/osbench/Cargo.toml b/tests/guest/osbench/Cargo.toml new file mode 100644 index 00000000000..b44ddf9d588 --- /dev/null +++ b/tests/guest/osbench/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "osbench" +description = "Scheduler and system benchmarks run unchanged on ToyOS and on Linux, so the two can be compared on one machine: one result line per measured distribution." +version = "0.1.0" +edition = "2021" +license = "MIT OR Apache-2.0" + +[target.'cfg(target_os = "toyos")'.dependencies] +toyos-abi = { path = "../../../sdk/abi" } +toyos = { path = "../../../sdk/toyos" } + +[target.'cfg(target_os = "linux")'.dependencies] +libc = "0.2" + +# The upstream compiler that builds the Linux side knows no ToyOS target. +[lints.rust] +unexpected_cfgs = { level = "warn", check-cfg = ['cfg(target_os, values("toyos"))'] } diff --git a/tests/guest/osbench/src/arch/mod.rs b/tests/guest/osbench/src/arch/mod.rs new file mode 100644 index 00000000000..a0a58bef77a --- /dev/null +++ b/tests/guest/osbench/src/arch/mod.rs @@ -0,0 +1,13 @@ +//! What ToyOS's half of `sys` names CPUs by: the architecture's own reading of +//! the CPU a thread runs on and of the cores CPUs share. + +#[cfg(target_arch = "x86_64")] +mod x86_64; +#[cfg(target_arch = "x86_64")] +pub use x86_64::{core_of, cpu_model, current_cpu, threads_per_core}; + +#[cfg(not(target_arch = "x86_64"))] +compile_error!( + "osbench names a ToyOS CPU by its x86-64 APIC ID, and has no AArch64 equivalent: \ + issues/osbench-builds-for-no-aarch64-toyos.md" +); diff --git a/tests/guest/osbench/src/arch/x86_64.rs b/tests/guest/osbench/src/arch/x86_64.rs new file mode 100644 index 00000000000..2e456f485e1 --- /dev/null +++ b/tests/guest/osbench/src/arch/x86_64.rs @@ -0,0 +1,64 @@ +//! CPU identity and topology as x86-64's CPUID reads them, which answers the +//! same under any OS. + +use core::arch::x86_64::__cpuid_count; +use std::sync::OnceLock; + +struct Topology { + /// CPUID leaf 0BH enumerates topology (SDM Vol. 2A: a subleaf 0 whose + /// EBX[15:0] is 0 enumerates nothing). + x2apic: bool, + /// Right shift from an APIC ID to its core's ID. + smt_shift: u32, +} + +fn topology() -> &'static Topology { + static TOPOLOGY: OnceLock = OnceLock::new(); + TOPOLOGY.get_or_init(|| { + let max = __cpuid_count(0, 0).eax; + let leaf_b = __cpuid_count(0xb, 0); + if max >= 0xb && leaf_b.ebx & 0xffff != 0 { + // EAX[4:0] at the SMT level: the shift to the next level's ID. + return Topology { x2apic: true, smt_shift: leaf_b.eax & 0x1f }; + } + // AMD without leaf 0BH: CPUID 8000_001EH EBX[15:8] is threads per + // core minus one (APM Vol. 3), where 8000_0001H ECX[22] says the + // topology extensions exist. + let max_ext = __cpuid_count(0x8000_0000, 0).eax; + let extensions = max_ext >= 0x8000_001e && __cpuid_count(0x8000_0001, 0).ecx & (1 << 22) != 0; + let threads = if extensions { ((__cpuid_count(0x8000_001e, 0).ebx >> 8) & 0xff) + 1 } else { 1 }; + Topology { x2apic: false, smt_shift: threads.next_power_of_two().trailing_zeros() } + }) +} + +/// The APIC ID of the CPU that ran this: leaf 0BH's EDX, or leaf 01H's +/// EBX[31:24] where leaf 0BH enumerates nothing. Stale as soon as the +/// scheduler moves the thread. +pub fn current_cpu() -> u32 { + if topology().x2apic { + __cpuid_count(0xb, 0).edx + } else { + __cpuid_count(1, 0).ebx >> 24 + } +} + +/// The core an APIC ID belongs to: two CPUs of one core are SMT siblings. +pub fn core_of(apic: u32) -> u32 { + apic >> topology().smt_shift +} + +pub fn threads_per_core() -> u32 { + 1 << topology().smt_shift +} + +/// The processor brand string, CPUID 8000_0002H to 8000_0004H. +pub fn cpu_model() -> String { + let mut bytes = Vec::with_capacity(48); + for leaf in 0x8000_0002u32..=0x8000_0004 { + let r = __cpuid_count(leaf, 0); + for reg in [r.eax, r.ebx, r.ecx, r.edx] { + bytes.extend_from_slice(®.to_le_bytes()); + } + } + String::from_utf8_lossy(&bytes).trim_matches(char::from(0)).trim().to_string() +} diff --git a/tests/guest/osbench/src/fair.rs b/tests/guest/osbench/src/fair.rs new file mode 100644 index 00000000000..cf3af89326f --- /dev/null +++ b/tests/guest/osbench/src/fair.rs @@ -0,0 +1,374 @@ +//! Three jobs contending for every CPU, and the share of the machine's work +//! each one got. +//! +//! - `build`: one process, one thread. +//! - `threaded`: one process, one thread per CPU. +//! - `helpers`: a parent keeping one helper process per CPU alive, each +//! working [`HELPER_BURST`] and exiting, as a parallel build's compiler +//! invocations do. +//! +//! Every worker runs the same compute chunk and counts the chunks it finished +//! in each window, on the clock every process shares. A job's share of a +//! window is its chunks over all three jobs' chunks: on CPUs of one kind that +//! is its share of CPU time, and `cpus_milli_p50` is that share of the +//! machine's CPUs. +//! +//! Each system runs it once with all three workloads in this program's own +//! group, and once with a group per workload. On ToyOS the group is the Job +//! object its threads share CPU time under: `onejob` leaves the three in this +//! program's, and `perjob` starts each workload in a Job of its own +//! (`SysCap::new_job`, which needs `Rights::JOBS`). On Linux `flat` leaves +//! them in this process's session and cgroup, where the fair unit is the +//! thread; `session` gives each a session of its own, whose autogroup is the +//! unit where autogroup applies; and `scope` puts each in a transient +//! `systemd-run --user --scope` with equal `CPUWeight`, whose cgroup is the +//! unit where the user's manager has the `cpu` controller. A `units` line says +//! which unit applied: `3xcgroup` is a cgroup per workload, `1xcgroup` one +//! cgroup for all three. + +use std::process::{Command, Stdio}; +use std::thread; +use std::time::Duration; + +use crate::report::{condition, refuse, result}; +use crate::sys::monotonic_ns; + +/// The warm-up between the jobs' start and the first window, in which every +/// worker starts and the shares settle from where each job began. +const SETTLE: Duration = Duration::from_secs(2); +const WINDOW: Duration = Duration::from_secs(2); +const WINDOWS: usize = 5; +const HELPER_BURST: Duration = Duration::from_millis(100); + +const JOBS: [&str; 3] = ["build", "threaded", "helpers"]; + +/// When the windows are, on [`monotonic_ns`]'s clock. +#[derive(Clone, Copy)] +struct Windows { + origin: u64, + width: u64, + count: usize, +} + +impl Windows { + fn end(self) -> u64 { + self.origin + self.width * self.count as u64 + } + + fn args(self) -> [String; 3] { + [self.origin.to_string(), self.width.to_string(), self.count.to_string()] + } + + fn parse(args: &[String]) -> Windows { + let num = |i: usize| -> u64 { args.get(i).and_then(|a| a.parse().ok()).expect("the coordinator's windows") }; + Windows { origin: num(0), width: num(1), count: num(2) as usize } + } + + /// Chunks until `until` or the last window's end, counted by the window + /// each one finished in. + fn work(self, until: u64) -> Vec { + let stop = until.min(self.end()); + let mut counts = vec![0; self.count]; + loop { + std::hint::black_box(crate::chunk()); + let now = monotonic_ns(); + if now >= stop { + return counts; + } + if let Some(since) = now.checked_sub(self.origin) { + counts[(since / self.width) as usize] += 1; + } + } + } +} + +fn add(into: &mut [u64], from: &[u64]) { + for (a, b) in into.iter_mut().zip(from) { + *a += b; + } +} + +fn counts_line(counts: &[u64]) -> String { + counts.iter().map(u64::to_string).collect::>().join(",") +} + +fn parse_counts(text: &str, count: usize) -> Option> { + let counts: Vec = text.split(',').map(|c| c.parse().ok()).collect::>()?; + (counts.len() == count).then_some(counts) +} + +/// `osbench fair-job `: one job, which prints +/// `counts= helpers= group=` and exits. +pub fn job(args: &[String]) { + let kind = args.first().map(String::as_str).expect("a job kind"); + let windows = Windows::parse(&args[1..]); + let cpus = crate::cpus(); + let (counts, helpers) = match kind { + "build" => (windows.work(u64::MAX), 0), + "threaded" => { + let workers: Vec<_> = (0..cpus).map(|_| thread::spawn(move || windows.work(u64::MAX))).collect(); + let mut counts = vec![0; windows.count]; + for worker in workers { + add(&mut counts, &worker.join().expect("a worker panicked")); + } + (counts, 0) + } + "helpers" => helpers(windows, cpus), + other => panic!("no job {other:?}"), + }; + #[cfg(target_os = "linux")] + let group = format!(" group={}", group_key()); + #[cfg(target_os = "toyos")] + let group = String::new(); + println!("counts={} helpers={helpers}{group}", counts_line(&counts)); +} + +/// One helper process per CPU in flight until the windows end. +fn helpers(windows: Windows, cpus: usize) -> (Vec, u64) { + let exe = std::env::current_exe().expect("a path to this program"); + let slots: Vec<_> = (0..cpus) + .map(|_| { + let exe = exe.clone(); + thread::spawn(move || { + let mut counts = vec![0; windows.count]; + let mut spawned = 0; + while monotonic_ns() < windows.end() { + let out = Command::new(&exe) + .arg("fair-helper") + .args(windows.args()) + .arg(HELPER_BURST.as_nanos().to_string()) + .stdin(Stdio::null()) + .stderr(Stdio::inherit()) + .output() + .expect("a helper spawns"); + assert!(out.status.success(), "a helper ended {}", out.status); + let text = String::from_utf8_lossy(&out.stdout); + add(&mut counts, &parse_counts(text.trim(), windows.count).expect("a helper's counts")); + spawned += 1; + } + (counts, spawned) + }) + }) + .collect(); + let mut counts = vec![0; windows.count]; + let mut spawned = 0; + for slot in slots { + let (c, s) = slot.join().expect("a helper slot panicked"); + add(&mut counts, &c); + spawned += s; + } + (counts, spawned) +} + +/// `osbench fair-helper `. +pub fn helper(args: &[String]) { + let windows = Windows::parse(args); + let burst: u64 = args.get(3).and_then(|a| a.parse().ok()).expect("a burst"); + println!("{}", counts_line(&windows.work(monotonic_ns() + burst))); +} + +/// The unit Linux's fair scheduler gave this process a share as: the cgroup +/// whose `cpu` controller governs it, or where none does, its autogroup if +/// autogroup is on. +#[cfg(target_os = "linux")] +fn group_key() -> String { + let cgroup = crate::sys::cpu_cgroup(); + if cgroup != "/" { + format!("cgroup:{cgroup}") + } else if crate::sys::autogroup_enabled() { + format!("autogroup:{}", crate::sys::autogroup()) + } else { + "none".into() + } +} + +/// What the jobs' `group=` keys say applied: `3xcgroup` is a cgroup per job. +/// Compared and never printed whole, since a cgroup path names the user. +#[cfg(target_os = "linux")] +fn units(_: Grouping, keys: &[String]) -> String { + let units: std::collections::BTreeSet<&str> = keys.iter().map(String::as_str).collect(); + let kinds: std::collections::BTreeSet<&str> = units.iter().map(|key| key.split(':').next().unwrap_or(key)).collect(); + format!("{}x{}", units.len(), kinds.into_iter().collect::>().join("+")) +} + +/// A ToyOS process cannot read which job it joined, so the grouping that +/// started them says it. +#[cfg(target_os = "toyos")] +fn units(grouping: Grouping, _: &[String]) -> String { + match grouping { + Grouping::Plain => "1xjob", + Grouping::Job => "3xjob", + } + .into() +} + +/// End workloads already started, so none runs into the next measurement. +fn stop(children: impl IntoIterator) { + for (_, mut child) in children { + let _ = child.kill(); + let _ = child.wait(); + } +} + +/// How the coordinator starts each job. +#[derive(Clone, Copy)] +pub enum Grouping { + /// Every job in this program's group: ToyOS's `onejob`, Linux's `flat`. + Plain, + /// Each workload in a ToyOS Job of its own. + #[cfg(target_os = "toyos")] + Job, + #[cfg(target_os = "linux")] + Session, + #[cfg(target_os = "linux")] + Scope, +} + +impl Grouping { + fn name(self) -> &'static str { + match self { + #[cfg(target_os = "toyos")] + Grouping::Plain => "onejob", + #[cfg(target_os = "toyos")] + Grouping::Job => "perjob", + #[cfg(target_os = "linux")] + Grouping::Plain => "flat", + #[cfg(target_os = "linux")] + Grouping::Session => "session", + #[cfg(target_os = "linux")] + Grouping::Scope => "scope", + } + } + + fn command(self, exe: &std::path::Path) -> Result { + match self { + Grouping::Plain => Ok(Command::new(exe)), + #[cfg(target_os = "toyos")] + Grouping::Job => { + use std::os::toyos::process::CommandExt; + use toyos::syscap::SysCap; + static CAP: std::sync::OnceLock> = std::sync::OnceLock::new(); + let cap = CAP + .get_or_init(|| toyos::endow::Endowments::get().take(toyos::endow::SYSCAP_LABEL)) + .as_ref() + .ok_or("no capability was endowed, so no job can be made")?; + let job = cap.new_job().map_err(|e| format!("a new job was refused: {e:?}"))?; + let mut command = Command::new(exe); + command.endow(toyos::job::JOB_LABEL, job.into_raw().0); + Ok(command) + } + #[cfg(target_os = "linux")] + Grouping::Session => { + use std::os::unix::process::CommandExt; + let mut command = Command::new(exe); + // SAFETY: setsid is async-signal-safe, and a fresh child is + // no process group leader, so it cannot fail. + unsafe { + command.pre_exec(|| { + libc::setsid(); + Ok(()) + }) + }; + Ok(command) + } + #[cfg(target_os = "linux")] + Grouping::Scope => { + let mut command = Command::new("systemd-run"); + command.args(["--user", "--scope", "--quiet", "--collect", "-p", "CPUWeight=100", "--"]).arg(exe); + Ok(command) + } + } + } +} + +/// Run the three jobs once under `grouping` and print each one's share. +pub fn contend(grouping: Grouping) { + let bench = format!("fair.{}", grouping.name()); + let exe = match std::env::current_exe() { + Ok(exe) => exe, + Err(e) => return refuse(&bench, format!("no path to this program: {e}")), + }; + let windows = + Windows { origin: monotonic_ns() + SETTLE.as_nanos() as u64, width: WINDOW.as_nanos() as u64, count: WINDOWS }; + let mut running = Vec::new(); + for kind in JOBS { + let mut command = match grouping.command(&exe) { + Ok(command) => command, + Err(why) => { + stop(running); + return refuse(&bench, format!("job {kind}: {why}")); + } + }; + let child = command + .arg("fair-job") + .arg(kind) + .args(windows.args()) + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn(); + match child { + Ok(child) => running.push((kind, child)), + Err(e) => { + stop(running); + return refuse(&bench, format!("job {kind} would not start: {e}")); + } + } + } + let mut jobs = Vec::new(); + let mut rest = running.into_iter(); + while let Some((kind, child)) = rest.next() { + let out = match child.wait_with_output() { + Ok(out) => out, + Err(e) => { + stop(rest); + return refuse(&bench, format!("job {kind}: {e}")); + } + }; + let text = String::from_utf8_lossy(&out.stdout); + let field = |key: &str| text.split_whitespace().find_map(|w| w.strip_prefix(key)).map(str::to_string); + let counts = field("counts=").and_then(|c| parse_counts(&c, WINDOWS)); + match (out.status.success(), counts) { + (true, Some(counts)) => { + jobs.push((kind, counts, field("helpers=").unwrap_or_default(), field("group=").unwrap_or_default())) + } + _ => { + stop(rest); + let said = String::from_utf8_lossy(&out.stderr); + return refuse( + &bench, + format!("job {kind} ended {}: {}", out.status, said.lines().next().unwrap_or("")), + ); + } + } + } + let totals: Vec = (0..WINDOWS).map(|w| jobs.iter().map(|(_, c, _, _)| c[w]).sum()).collect(); + if totals.contains(&0) { + return refuse(&bench, "a window in which no job finished a chunk"); + } + let keys: Vec = jobs.iter().map(|(_, _, _, key)| key.clone()).collect(); + condition(&format!("{bench}.units"), units(grouping, &keys)); + let cpus = crate::cpus() as u64; + for (kind, counts, helpers, _) in &jobs { + let mut permille: Vec = counts.iter().zip(&totals).map(|(c, t)| c * 1000 / t).collect(); + permille.sort_unstable(); + let cpus_milli = permille[permille.len() / 2] * cpus; + let mut extra = vec![("job", kind.to_string()), ("cpus_milli_p50", cpus_milli.to_string())]; + if *kind == "helpers" { + extra.push(("helpers_spawned", helpers.clone())); + } + result(&bench, "permille", &mut permille, &extra); + } + let mut rate: Vec = totals.iter().map(|t| t * 1_000_000_000 / windows.width).collect(); + result(&format!("{bench}.total"), "chunks/s", &mut rate, &[]); +} + +#[cfg(target_os = "linux")] +pub fn groupings() -> [Grouping; 3] { + [Grouping::Plain, Grouping::Session, Grouping::Scope] +} + +#[cfg(target_os = "toyos")] +pub fn groupings() -> [Grouping; 2] { + [Grouping::Plain, Grouping::Job] +} diff --git a/tests/guest/osbench/src/main.rs b/tests/guest/osbench/src/main.rs new file mode 100644 index 00000000000..8b67bd0be21 --- /dev/null +++ b/tests/guest/osbench/src/main.rs @@ -0,0 +1,152 @@ +//! `osbench`: the same scheduler and system workloads, built from one source +//! for ToyOS and for Linux, so one machine's two systems can be compared. +//! +//! `osbench ` runs a group: `system`, `sched`, `fair` or `all`. A +//! ToyOS job list passes no arguments, so there the group is the name the +//! program was started under, `osbench_`. Every other command is one +//! this program gives its own children. +//! +//! Every result is one line of `key=value` words (`report`): a distribution +//! with its sample count, minimum, p50, p90, p99, p99.9, maximum and mean, and +//! the conditions it was taken under. Each benchmark's own comment states its +//! warm-up and repetitions. Benchmarks report and never judge: a group exits +//! non-zero only when one of them could not measure. +//! +//! File I/O is not measured: ToyOS's storage on the benches is a USB stick and +//! Linux's an NVMe disk, so a comparison would be of the devices. +//! +//! The Linux build is static (`x86_64-unknown-linux-musl`, linked by the +//! toolchain's own LLD per `.cargo/config.toml`), so a bench runs it without a +//! toolchain: +//! +//! ```text +//! rustup target add --toolchain nightly-2026-07-22 x86_64-unknown-linux-musl +//! cargo +nightly-2026-07-22 build -p osbench --target x86_64-unknown-linux-musl --profile toyos +//! ``` +//! +//! That nightly is the build's stage-0 compiler (`build/sysroot.rs`'s +//! `STAGE0_CARGO`, which this follows), the upstream one nearest ToyOS's own. +//! With the `toyos` profile every guest binary is built with, osbench's own +//! code is generated alike for both; std's platform layer and the C library +//! beneath it (musl's on Linux) are each system's own. + +#[cfg(target_os = "toyos")] +mod arch; +mod fair; +mod pingpong; +mod place; +mod report; +mod sys; +mod system; +mod wake; + +use std::time::Instant; + +/// Iterations of the compute chunk every CPU-bound worker here runs: a few +/// microseconds on either bench. +pub const CHUNK_ITERATIONS: u64 = 25_000; + +/// One unit of CPU-bound work: a multiply-add chain, the same instructions +/// on both systems. +pub fn chunk() -> u64 { + let mut x = std::hint::black_box(0x2545_f491_4f6c_dd1d_u64); + for _ in 0..CHUNK_ITERATIONS { + x = x.wrapping_mul(6_364_136_223_846_793_005).wrapping_add(1_442_695_040_888_963_407); + } + std::hint::black_box(x) +} + +pub fn ns_since(t: Instant) -> u64 { + u64::try_from(t.elapsed().as_nanos()).expect("a span under 584 years") +} + +pub fn cpus() -> usize { + std::thread::available_parallelism().expect("the number of CPUs this process may use").get() +} + +fn conditions() { + report::condition("cpu_model", sys::cpu_model()); + report::condition("cpus", cpus()); + report::condition("threads_per_core", sys::threads_per_core()); + sys::conditions(); +} + +fn system() { + system::chunk_rate(); + system::null_syscall(); + system::thread_spawn(); + system::process_spawn(); + system::pipe_throughput(64); + system::pipe_throughput(64 * 1024); + system::anon_memory(); + system::map_unmap_small(); + system::bandwidth(); +} + +fn sched() { + system::chunk_rate(); + wake::periodic("wake.periodic.idle"); + wake::periodic_loaded("wake.periodic.loaded"); + #[cfg(target_os = "linux")] + { + // An ordinary Linux thread's sleep is padded by its timer slack, 50 µs + // by default; these show the wake without it. ToyOS has none. + let slack = sys::timer_slack_ns(); + sys::set_timer_slack_ns(1); + wake::periodic("wake.periodic.idle.slack1ns"); + wake::periodic_loaded("wake.periodic.loaded.slack1ns"); + sys::set_timer_slack_ns(slack); + } + wake::onto_busy(); + wake::storm(); + for same in [true, false] { + pingpong::pipe(same); + pingpong::futex(same); + } +} + +fn fair() { + system::chunk_rate(); + for grouping in fair::groupings() { + fair::contend(grouping); + } +} + +fn group(name: &str, run: fn()) { + report::group(name, "start"); + conditions(); + let t = Instant::now(); + run(); + report::group(name, &format!("done secs={}", t.elapsed().as_secs())); +} + +fn main() { + let args: Vec = std::env::args().collect(); + let invoked = args.first().and_then(|a| a.rsplit('/').next()).unwrap_or_default(); + let command = match args.get(1) { + Some(command) => command.as_str(), + None => invoked.strip_prefix("osbench_").unwrap_or("all"), + }; + let rest = args.get(2..).unwrap_or_default(); + match command { + "system" => group("system", system), + "sched" => group("sched", sched), + "fair" => group("fair", fair), + "all" => { + group("system", system); + group("sched", sched); + group("fair", fair); + } + "nop" => return, + "spin" => return wake::spin(), + "fair-job" => return fair::job(rest), + "fair-helper" => return fair::helper(rest), + other => { + eprintln!("osbench: no command {other:?}; the groups are system, sched, fair and all"); + std::process::exit(2); + } + } + if report::any_refused() { + std::process::exit(1); + } +} diff --git a/tests/guest/osbench/src/pingpong.rs b/tests/guest/osbench/src/pingpong.rs new file mode 100644 index 00000000000..683e26fde01 --- /dev/null +++ b/tests/guest/osbench/src/pingpong.rs @@ -0,0 +1,199 @@ +//! A round trip between two threads, on one CPU and across two cores, through +//! a pipe and through a futex. + +use std::io::{Read, Write}; +use std::sync::atomic::{AtomicU32, Ordering}; +use std::sync::{mpsc, Arc}; +use std::time::Instant; + +use crate::place; +use crate::report::{refuse, result}; +use crate::{ns_since, sys, wake}; + +const WARMUP: u32 = 500; +const ROUNDS: u32 = 10_000; +/// Every this many rounds both sides read their CPU, outside the time kept, +/// and that round is not kept. +const CHECK_EVERY: u32 = 256; + +/// One side of the exchange: `ping` hands the turn over and waits for it +/// back, `pong` waits for it and hands it back. +trait Channel: Send + 'static { + fn ping(&mut self, round: u32) -> std::io::Result<()>; + fn pong(&mut self, round: u32) -> std::io::Result<()>; +} + +struct PipeEnd { + reader: std::io::PipeReader, + writer: std::io::PipeWriter, +} + +impl Channel for PipeEnd { + fn ping(&mut self, _: u32) -> std::io::Result<()> { + self.writer.write_all(&[1])?; + self.reader.read_exact(&mut [0]) + } + + fn pong(&mut self, _: u32) -> std::io::Result<()> { + self.reader.read_exact(&mut [0])?; + self.writer.write_all(&[1]) + } +} + +/// Two words: the round the pinger has handed over and the round the ponger +/// has answered. Each side sleeps on the other's word. +#[derive(Default)] +struct Words { + ping: AtomicU32, + pong: AtomicU32, +} + +#[derive(Clone)] +struct FutexEnd(Arc); + +fn wait_for(word: &AtomicU32, round: u32) { + loop { + let now = word.load(Ordering::Acquire); + if now == round { + return; + } + sys::futex_wait(word, now); + } +} + +impl Channel for FutexEnd { + fn ping(&mut self, round: u32) -> std::io::Result<()> { + self.0.ping.store(round, Ordering::Release); + sys::futex_wake(&self.0.ping, 1); + wait_for(&self.0.pong, round); + Ok(()) + } + + fn pong(&mut self, round: u32) -> std::io::Result<()> { + wait_for(&self.0.ping, round); + self.0.pong.store(round, Ordering::Release); + sys::futex_wake(&self.0.pong, 1); + Ok(()) + } +} + +#[derive(Clone, Copy)] +enum Where { + SameCpu, + CrossCore, +} + +pub fn pipe(same: bool) { + let pair = || -> std::io::Result<(PipeEnd, PipeEnd)> { + let (from_ping, to_pong) = std::io::pipe()?; + let (from_pong, to_ping) = std::io::pipe()?; + Ok((PipeEnd { reader: from_pong, writer: to_pong }, PipeEnd { reader: from_ping, writer: to_ping })) + }; + let at = if same { Where::SameCpu } else { Where::CrossCore }; + match pair() { + Ok((a, b)) => run("pipe", at, a, b), + Err(e) => refuse(&bench("pipe", at), e), + } +} + +pub fn futex(same: bool) { + let words = FutexEnd(Arc::default()); + run("futex", if same { Where::SameCpu } else { Where::CrossCore }, words.clone(), words); +} + +fn bench(kind: &str, at: Where) -> String { + let at = match at { + Where::SameCpu => "same_cpu", + Where::CrossCore => "cross_core", + }; + format!("pingpong.{kind}.{at}") +} + +/// Round-trip time as the pinger sees it, ns. Linux pins both sides; ToyOS +/// stages them and `placed_held` counts the checks that found them where they +/// were put. Neither side is put on the interrupts' CPU. +fn run(kind: &str, at: Where, mut pinger: impl Channel, mut ponger: impl Channel) { + let bench = bench(kind, at); + let (go_ping, ping_go) = mpsc::channel::(); + let (go_pong, pong_go) = mpsc::channel::(); + let (ping_said, ping_heard) = mpsc::channel(); + let (pong_said, pong_heard) = mpsc::channel(); + let ping_side = place::spawn_on( + |on| on != wake::IRQ_CPU, + move || { + if ping_go.recv() != Ok(true) { + return; + } + let mut kept = Vec::with_capacity(ROUNDS as usize); + let mut seen = Vec::new(); + for round in 1..=WARMUP + ROUNDS { + let t = Instant::now(); + if let Err(e) = pinger.ping(round) { + let _ = ping_said.send(Err(e.to_string())); + return; + } + let took = ns_since(t); + if round % CHECK_EVERY == 0 { + seen.push(sys::current_cpu()); + } else if round > WARMUP { + kept.push(took); + } + } + let _ = ping_said.send(Ok((kept, seen))); + }, + ); + let ping_side = match ping_side { + Ok(placed) => placed, + Err(why) => return refuse(&bench, format!("the pinger was not placed: {why}")), + }; + let ping_cpu = ping_side.cpu; + let wanted = move |on: u32| { + on != wake::IRQ_CPU + && match at { + Where::SameCpu => on == ping_cpu, + Where::CrossCore => sys::core_of(on) != sys::core_of(ping_cpu), + } + }; + let pong_side = place::spawn_on(wanted, move || { + if pong_go.recv() != Ok(true) { + return; + } + let mut seen = Vec::new(); + for round in 1..=WARMUP + ROUNDS { + if ponger.pong(round).is_err() { + break; + } + if round % CHECK_EVERY == 0 { + seen.push(sys::current_cpu()); + } + } + let _ = pong_said.send(seen); + }); + let go = pong_side.is_ok(); + // A refused send is a side that already ended, which its join reports. + let _ = (go_ping.send(go), go_pong.send(go)); + ping_side.thread.join().expect("the pinger panicked"); + let pong_side = match pong_side { + Ok(placed) => placed, + Err(why) => return refuse(&bench, format!("the ponger was not placed: {why}")), + }; + pong_side.thread.join().expect("the ponger panicked"); + let (mut kept, ping_seen) = match ping_heard.recv() { + Ok(Ok(measured)) => measured, + Ok(Err(why)) => return refuse(&bench, why), + Err(_) => return refuse(&bench, "the pinger said nothing"), + }; + let pong_seen = pong_heard.recv().unwrap_or_default(); + let checks = ping_seen.len().min(pong_seen.len()); + let held = ping_seen.iter().zip(&pong_seen).filter(|(&ping, &pong)| ping == ping_cpu && wanted(pong)).count(); + result( + &bench, + "ns", + &mut kept, + &[ + ("ping_cpu", ping_cpu.to_string()), + ("pong_cpu", pong_side.cpu.to_string()), + ("placed_held", format!("{held}/{checks}")), + ], + ); +} diff --git a/tests/guest/osbench/src/place.rs b/tests/guest/osbench/src/place.rs new file mode 100644 index 00000000000..6afba565f5e --- /dev/null +++ b/tests/guest/osbench/src/place.rs @@ -0,0 +1,88 @@ +//! Starting a thread on a chosen CPU. +//! +//! **Linux pins it; ToyOS has no affinity, so it is staged.** On ToyOS a new +//! thread runs where spawn placement puts it, so candidates are started until +//! one first runs on an accepted CPU and the rest end; what keeps it there is +//! the scheduler, not a mask, and every user of a placed thread reads where its +//! threads actually ran. + +use std::thread::JoinHandle; + +use crate::sys; + +/// A thread that began its body on `cpu`. +pub struct Placed { + pub thread: JoinHandle<()>, + pub cpu: u32, + #[cfg(target_os = "linux")] + tid: libc::pid_t, +} + +impl Placed { + /// Let the thread run on any CPU from now on. On ToyOS it always could. + pub fn release(&self) { + #[cfg(target_os = "linux")] + sys::unpin(self.tid); + } +} + +/// Start `body` on the first CPU `wanted` accepts. +#[cfg(target_os = "linux")] +pub fn spawn_on(wanted: impl Fn(u32) -> bool, body: impl FnOnce() + Send + 'static) -> Result { + use std::sync::mpsc; + let target = sys::allowed_cpus() + .into_iter() + .map(|cpu| cpu as u32) + .find(|&cpu| wanted(cpu)) + .ok_or("no CPU this process may run on is wanted")?; + let (said, heard) = mpsc::channel(); + let thread = std::thread::spawn(move || { + sys::pin_self(target as usize); + let on = sys::current_cpu(); + said.send((on, sys::tid())).expect("the placer waits"); + if on == target { + body(); + } + }); + let (on, tid) = heard.recv().map_err(|_| "a placed thread ended before it said where it ran")?; + if on != target { + thread.join().map_err(|_| "a misplaced thread panicked")?; + return Err(format!("pinned to CPU {target} and ran on {on}")); + } + Ok(Placed { thread, cpu: on, tid }) +} + +/// Start `body` on the first CPU `wanted` accepts that a new thread lands +/// on, or say none did within [`STAGE`]. +#[cfg(target_os = "toyos")] +pub fn spawn_on(wanted: impl Fn(u32) -> bool, body: impl FnOnce() + Send + 'static) -> Result { + use std::sync::mpsc; + let deadline = std::time::Instant::now() + STAGE; + let mut body = Some(body); + let mut tried = 0; + while std::time::Instant::now() < deadline { + tried += 1; + let (said, heard) = mpsc::channel(); + let (verdict, judged) = mpsc::channel::>>(); + let thread = std::thread::spawn(move || { + said.send(sys::current_cpu()).expect("the placer waits"); + if let Ok(Some(body)) = judged.recv() { + body(); + } + }); + let on = heard.recv().map_err(|_| "a candidate ended before it said where it ran")?; + if wanted(on) { + let body = body.take().expect("a body is kept once"); + verdict.send(Some(Box::new(body))).map_err(|_| "a kept candidate ended")?; + return Ok(Placed { thread, cpu: on }); + } + verdict.send(None).map_err(|_| "a candidate ended before its verdict")?; + thread.join().map_err(|_| "a candidate let go panicked")?; + } + Err(format!("{tried} threads started in {STAGE:?} and none first ran where it was wanted")) +} + +/// How long one placement may take: spawn placement rotates over the CPUs, +/// so a machine of `n` usually needs about `n` candidates for any one CPU. +#[cfg(target_os = "toyos")] +const STAGE: std::time::Duration = std::time::Duration::from_secs(5); diff --git a/tests/guest/osbench/src/report.rs b/tests/guest/osbench/src/report.rs new file mode 100644 index 00000000000..455dab0c6a7 --- /dev/null +++ b/tests/guest/osbench/src/report.rs @@ -0,0 +1,75 @@ +//! The one output format: every line starts `osbench v=1 os=` and is +//! otherwise `key=value` words, so a run's stdout is its data. + +use std::sync::atomic::{AtomicBool, Ordering}; + +static REFUSED: AtomicBool = AtomicBool::new(false); + +pub const OS: &str = if cfg!(target_os = "toyos") { "toyos" } else { "linux" }; + +/// A value as one word: whitespace becomes `_`. +pub fn word(value: impl std::fmt::Display) -> String { + value.to_string().split_whitespace().collect::>().join("_") +} + +fn emit(body: &str) { + println!("osbench v=1 os={OS} {body}"); +} + +/// A fact about the machine or the run that its numbers depend on. +pub fn condition(key: &str, value: impl std::fmt::Display) { + emit(&format!("cond {key}={}", word(value))); +} + +/// A benchmark that did not measure: its line says why, and the process exits +/// non-zero once its group has run. +pub fn refuse(bench: &str, why: impl std::fmt::Display) { + REFUSED.store(true, Ordering::Relaxed); + emit(&format!("bench={bench} refused={}", word(why))); +} + +/// A benchmark that ran and kept no sample, with why: a result, which does +/// not fail its group, where the reason is the system's and not a fault. +pub fn unmeasured(bench: &str, why: &str, extra: &[(&str, String)]) { + let mut line = format!("bench={bench} unmeasured={}", word(why)); + for (key, value) in extra { + line.push_str(&format!(" {key}={}", word(value))); + } + emit(&line); +} + +pub fn any_refused() -> bool { + REFUSED.load(Ordering::Relaxed) +} + +/// One measured distribution, by nearest rank. A percentile the sample count +/// cannot resolve prints `-`: p99 needs 100 samples and p99.9 a thousand. +pub fn result(bench: &str, unit: &str, samples: &mut [u64], extra: &[(&str, String)]) { + if samples.is_empty() { + refuse(bench, "no samples"); + return; + } + samples.sort_unstable(); + let n = samples.len(); + let rank = |permille: usize| samples[(permille * n).div_ceil(1000) - 1]; + let resolved = |permille: usize, needs: usize| if n >= needs { rank(permille).to_string() } else { "-".into() }; + let mean = samples.iter().map(|&s| u128::from(s)).sum::() / n as u128; + let mut line = format!( + "bench={bench} unit={unit} n={n} min={} p50={} p90={} p99={} p999={} max={} mean={mean}", + samples[0], + rank(500), + resolved(900, 10), + resolved(990, 100), + resolved(999, 1000), + samples[n - 1], + ); + for (key, value) in extra { + line.push_str(&format!(" {key}={}", word(value))); + } + emit(&line); +} + +/// A group's first and last line: the judge reads the last as "it ran". +pub fn group(name: &str, phase: &str) { + emit(&format!("group={name} {phase}")); +} diff --git a/tests/guest/osbench/src/sys/linux.rs b/tests/guest/osbench/src/sys/linux.rs new file mode 100644 index 00000000000..f20957bd0a3 --- /dev/null +++ b/tests/guest/osbench/src/sys/linux.rs @@ -0,0 +1,303 @@ +use std::fs; +use std::sync::atomic::AtomicU32; +use std::sync::OnceLock; +use std::time::Duration; + +use crate::report::condition; + +pub fn null_syscall() { + // SAFETY: getpid takes no arguments and cannot fail. Raw, so no libc + // cache answers in its place. + unsafe { libc::syscall(libc::SYS_getpid) }; +} + +/// `CLOCK_MONOTONIC`, which `Instant` reads and every process shares. +pub fn monotonic_ns() -> u64 { + let mut ts = libc::timespec { tv_sec: 0, tv_nsec: 0 }; + // SAFETY: `ts` is a valid out-pointer. + assert_eq!(unsafe { libc::clock_gettime(libc::CLOCK_MONOTONIC, &mut ts) }, 0); + ts.tv_sec as u64 * 1_000_000_000 + ts.tv_nsec as u64 +} + +/// Block while `word` holds `expected`. +pub fn futex_wait(word: &AtomicU32, expected: u32) { + // SAFETY: `word` is a live, aligned u32; a null timeout waits unbounded. + unsafe { + libc::syscall( + libc::SYS_futex, + word.as_ptr(), + libc::FUTEX_WAIT | libc::FUTEX_PRIVATE_FLAG, + expected, + std::ptr::null::(), + ) + }; +} + +/// Wake up to `count` waiters on `word`; answers how many woke. +pub fn futex_wake(word: &AtomicU32, count: u32) -> u32 { + let count = i32::try_from(count).unwrap_or(i32::MAX); + // SAFETY: `word` is a live, aligned u32. + let woken = + unsafe { libc::syscall(libc::SYS_futex, word.as_ptr(), libc::FUTEX_WAKE | libc::FUTEX_PRIVATE_FLAG, count) }; + u32::try_from(woken).expect("FUTEX_WAKE on a valid word answers a count") +} + +pub fn map_anon(len: usize) -> *mut u8 { + // SAFETY: a fresh anonymous mapping aliases nothing. + let ptr = unsafe { + libc::mmap( + std::ptr::null_mut(), + len, + libc::PROT_READ | libc::PROT_WRITE, + libc::MAP_PRIVATE | libc::MAP_ANONYMOUS, + -1, + 0, + ) + }; + assert_ne!(ptr, libc::MAP_FAILED, "mmap of {len} bytes: {}", std::io::Error::last_os_error()); + ptr.cast() +} + +/// # Safety +/// `ptr` and `len` are one whole mapping from [`map_anon`], and nothing +/// reaches it afterwards. +pub unsafe fn unmap(ptr: *mut u8, len: usize) { + // SAFETY: as the caller promises. + assert_eq!(unsafe { libc::munmap(ptr.cast(), len) }, 0); +} + +/// An anonymous mapping with transparent huge pages asked for, which +/// ToyOS's anonymous mappings always are; [`huge_backing`] says what the +/// kernel gave. +pub fn map_anon_huge(len: usize) -> *mut u8 { + let ptr = map_anon(len); + // SAFETY: an advice changes no contents. + unsafe { libc::madvise(ptr.cast(), len, libc::MADV_HUGEPAGE) }; + ptr +} + +/// This process's anonymous memory in huge pages, from +/// `/proc/self/smaps_rollup`. +pub fn huge_backing() -> String { + let kib = fs::read_to_string("/proc/self/smaps_rollup").ok().and_then(|rollup| { + let line = rollup.lines().find(|l| l.starts_with("AnonHugePages:"))?; + line.split_whitespace().nth(1)?.parse::().ok() + }); + kib.map_or_else(|| "unknown".into(), |kib| format!("{kib}KiB")) +} + +/// The CPU that ran this, by Linux's numbering. Stale as soon as the +/// scheduler moves the thread. +pub fn current_cpu() -> u32 { + // SAFETY: sched_getcpu takes nothing. + let cpu = unsafe { libc::sched_getcpu() }; + u32::try_from(cpu).unwrap_or_else(|_| panic!("sched_getcpu: {}", std::io::Error::last_os_error())) +} + +/// The core CPU `cpu` belongs to, as sysfs gives it: two CPUs of one core +/// are SMT siblings. +pub fn core_of(cpu: u32) -> u32 { + static CORES: OnceLock> = OnceLock::new(); + let cores = CORES.get_or_init(|| { + let max = allowed_cpus().into_iter().max().unwrap_or(0); + (0..=max) + .map(|cpu| { + let topology = |file: &str| -> u32 { + read(&format!("/sys/devices/system/cpu/cpu{cpu}/topology/{file}")) + .and_then(|v| v.parse().ok()) + .unwrap_or(u32::MAX) + }; + topology("physical_package_id") << 16 | topology("core_id") + }) + .collect() + }); + cores[cpu as usize] +} + +pub fn threads_per_core() -> u32 { + let cpus = allowed_cpus(); + let first = core_of(cpus[0] as u32); + cpus.iter().filter(|&&cpu| core_of(cpu as u32) == first).count() as u32 +} + +pub fn cpu_model() -> String { + let cpuinfo = fs::read_to_string("/proc/cpuinfo").unwrap_or_default(); + let model = cpuinfo + .lines() + .find_map(|l| l.strip_prefix("model name").and_then(|v| v.split_once(':')).map(|(_, v)| v.trim().to_string())); + model.unwrap_or_else(|| "unknown".into()) +} + +/// The CPUs this process may run on, by Linux's numbering: its main +/// thread's mask, which nothing here pins. +pub fn allowed_cpus() -> Vec { + // SAFETY: a zeroed cpu_set_t is an empty set. + let mut set: libc::cpu_set_t = unsafe { std::mem::zeroed() }; + // SAFETY: getpid cannot fail; `set` is a valid out-pointer of the size given. + let got = unsafe { libc::sched_getaffinity(libc::getpid(), std::mem::size_of::(), &mut set) }; + assert_eq!(got, 0, "sched_getaffinity: {}", std::io::Error::last_os_error()); + // SAFETY: CPU_ISSET reads the set within its size. + (0..libc::CPU_SETSIZE as usize).filter(|&cpu| unsafe { libc::CPU_ISSET(cpu, &set) }).collect() +} + +fn set_affinity(tid: libc::pid_t, cpus: &[usize]) { + // SAFETY: a zeroed cpu_set_t is an empty set. + let mut set: libc::cpu_set_t = unsafe { std::mem::zeroed() }; + for &cpu in cpus { + // SAFETY: `cpu` came from `allowed_cpus`, inside the set's size. + unsafe { libc::CPU_SET(cpu, &mut set) }; + } + // SAFETY: `set` is valid for its size. + let done = unsafe { libc::sched_setaffinity(tid, std::mem::size_of::(), &set) }; + assert_eq!(done, 0, "sched_setaffinity({tid}, {cpus:?}): {}", std::io::Error::last_os_error()); +} + +pub fn tid() -> libc::pid_t { + // SAFETY: gettid takes no arguments and cannot fail. + unsafe { libc::syscall(libc::SYS_gettid) as libc::pid_t } +} + +/// Keep the calling thread on Linux CPU `cpu`. +pub fn pin_self(cpu: usize) { + set_affinity(0, &[cpu]); +} + +/// Let thread `tid` run anywhere this process may. +pub fn unpin(tid: libc::pid_t) { + set_affinity(tid, &allowed_cpus()); +} + +/// The calling thread's timer slack, which pads each of its sleeps by up to +/// this much. +pub fn timer_slack_ns() -> u64 { + // SAFETY: PR_GET_TIMERSLACK answers the slack as its return value. + let slack = unsafe { libc::prctl(libc::PR_GET_TIMERSLACK, 0, 0, 0, 0) }; + u64::try_from(slack).expect("PR_GET_TIMERSLACK cannot fail") +} + +pub fn set_timer_slack_ns(ns: u64) { + // SAFETY: PR_SET_TIMERSLACK takes the slack by value. + let done = unsafe { libc::prctl(libc::PR_SET_TIMERSLACK, ns as libc::c_ulong, 0, 0, 0) }; + assert_eq!(done, 0, "PR_SET_TIMERSLACK: {}", std::io::Error::last_os_error()); +} + +fn read(path: &str) -> Option { + fs::read_to_string(path).ok().map(|s| s.trim().to_string()) +} + +/// Every distinct value one per-CPU sysfs file holds, with how many CPUs hold +/// it: `powersave*8`. +fn per_cpu(file: &str) -> String { + let mut seen: Vec<(String, usize)> = Vec::new(); + for cpu in allowed_cpus() { + let value = read(&format!("/sys/devices/system/cpu/cpu{cpu}/{file}")).unwrap_or_else(|| "absent".into()); + match seen.iter_mut().find(|(v, _)| *v == value) { + Some((_, n)) => *n += 1, + None => seen.push((value, 1)), + } + } + seen.iter().map(|(v, n)| format!("{v}*{n}")).collect::>().join(",") +} + +/// The selected word of a sysfs choice file: `always [madvise] never`. +fn selected(path: &str) -> String { + read(path) + .map(|s| { + s.split_whitespace() + .find(|w| w.starts_with('[')) + .map_or(s.clone(), |w| w.trim_matches(['[', ']']).to_string()) + }) + .unwrap_or_else(|| "absent".into()) +} + +/// The whole machine's busy fraction over `span`, in permille, off +/// `/proc/stat`'s first line: everything but idle and iowait is busy. +fn busy_permille(span: Duration) -> Option { + let sample = || -> Option<(u64, u64)> { + let stat = fs::read_to_string("/proc/stat").ok()?; + let fields: Vec = stat.lines().next()?.split_whitespace().skip(1).filter_map(|f| f.parse().ok()).collect(); + let idle = fields.get(3)? + fields.get(4)?; + Some((fields.iter().sum(), idle)) + }; + let (total0, idle0) = sample()?; + std::thread::sleep(span); + let (total1, idle1) = sample()?; + let total = total1.checked_sub(total0)?; + (total > 0).then(|| (total - (idle1 - idle0)) * 1000 / total) +} + +/// What a Linux number depends on, read and never changed. +pub fn conditions() { + condition("kernel", read("/proc/sys/kernel/osrelease").unwrap_or_default()); + let distro = fs::read_to_string("/etc/os-release").ok().and_then(|text| { + text.lines().find_map(|l| l.strip_prefix("PRETTY_NAME=")).map(|v| v.trim_matches('"').to_string()) + }); + condition("distro", distro.unwrap_or_else(|| "unknown".into())); + condition("cpus_online", read("/sys/devices/system/cpu/online").unwrap_or_default()); + condition("smt_active", read("/sys/devices/system/cpu/smt/active").unwrap_or_else(|| "absent".into())); + condition("smt_control", read("/sys/devices/system/cpu/smt/control").unwrap_or_else(|| "absent".into())); + condition("cpufreq_driver", per_cpu("cpufreq/scaling_driver")); + condition("governor", per_cpu("cpufreq/scaling_governor")); + condition("epp", per_cpu("cpufreq/energy_performance_preference")); + condition("max_khz", per_cpu("cpufreq/scaling_max_freq")); + condition( + "intel_no_turbo", + read("/sys/devices/system/cpu/intel_pstate/no_turbo").unwrap_or_else(|| "absent".into()), + ); + condition("boost", read("/sys/devices/system/cpu/cpufreq/boost").unwrap_or_else(|| "absent".into())); + condition("amd_pstate", read("/sys/devices/system/cpu/amd_pstate/status").unwrap_or_else(|| "absent".into())); + for supply in fs::read_dir("/sys/class/power_supply").into_iter().flatten().flatten() { + let at = supply.path(); + let get = |f: &str| read(&at.join(f).to_string_lossy()); + match get("type").as_deref() { + Some("Mains") => condition("ac_online", get("online").unwrap_or_default()), + Some("Battery") => { + condition("battery_status", get("status").unwrap_or_default()); + condition("battery_capacity", get("capacity").unwrap_or_default()); + } + _ => {} + } + } + condition("thp", selected("/sys/kernel/mm/transparent_hugepage/enabled")); + condition("autogroup", read("/proc/sys/kernel/sched_autogroup_enabled").unwrap_or_else(|| "absent".into())); + condition("timer_slack_ns", read("/proc/self/timerslack_ns").unwrap_or_default()); + for entry in fs::read_dir("/sys/devices/system/cpu/vulnerabilities").into_iter().flatten().flatten() { + let name = entry.file_name().to_string_lossy().into_owned(); + condition(&format!("vuln_{name}"), read(&entry.path().to_string_lossy()).unwrap_or_default()); + } + condition( + "loadavg", + read("/proc/loadavg").map(|l| l.split_whitespace().take(3).collect::>().join(",")).unwrap_or_default(), + ); + condition("busy_permille_1s", busy_permille(Duration::from_secs(1)).map_or("unknown".into(), |p| p.to_string())); +} + +/// The cgroup whose `cpu` controller governs this process: the deepest one +/// on its path whose parent enables `cpu` for its children, or `/` when +/// none does. +pub fn cpu_cgroup() -> String { + let own = fs::read_to_string("/proc/self/cgroup") + .ok() + .and_then(|text| text.lines().find_map(|l| l.strip_prefix("0::").map(str::to_string))) + .unwrap_or_default(); + let mut governing = String::from("/"); + let mut at = String::new(); + for part in own.split('/').filter(|p| !p.is_empty()) { + let parent = format!("/sys/fs/cgroup{at}/cgroup.subtree_control"); + at.push('/'); + at.push_str(part); + if read(&parent).is_some_and(|c| c.split_whitespace().any(|w| w == "cpu")) { + governing.clone_from(&at); + } + } + governing +} + +pub fn autogroup_enabled() -> bool { + read("/proc/sys/kernel/sched_autogroup_enabled").as_deref() == Some("1") +} + +/// This process's autogroup, as `/proc/self/autogroup` names it. +pub fn autogroup() -> String { + read("/proc/self/autogroup").and_then(|s| s.split_whitespace().next().map(str::to_string)).unwrap_or_default() +} diff --git a/tests/guest/osbench/src/sys/mod.rs b/tests/guest/osbench/src/sys/mod.rs new file mode 100644 index 00000000000..0851bbfd84e --- /dev/null +++ b/tests/guest/osbench/src/sys/mod.rs @@ -0,0 +1,46 @@ +//! What std does not offer portably, as each OS offers it: the cheapest +//! syscall, a futex, a clock every process shares, anonymous mappings and the +//! machine's conditions. Both modules export the same names; Linux adds +//! affinity, which ToyOS has none of, and the readers of its cgroup and +//! autogroup. + +#[cfg(target_os = "linux")] +#[path = "linux.rs"] +mod imp; + +#[cfg(target_os = "toyos")] +#[path = "toyos.rs"] +mod imp; + +pub use imp::*; + +/// A whole anonymous read-write mapping, unmapped on drop. +pub struct Mapping { + ptr: *mut u8, + len: usize, +} + +impl Mapping { + /// A mapping backed by huge pages wherever the system gives them. + pub fn huge(len: usize) -> Mapping { + Mapping { ptr: map_anon_huge(len), len } + } + + pub fn ptr(&self) -> *mut u8 { + self.ptr + } + + /// The mapping as words. Anonymous memory reads as zero, so every word is + /// initialised. + pub fn words(&mut self) -> &mut [u64] { + // SAFETY: `ptr` is page-aligned, `len` bytes long and owned by `self`. + unsafe { std::slice::from_raw_parts_mut(self.ptr.cast::(), self.len / 8) } + } +} + +impl Drop for Mapping { + fn drop(&mut self) { + // SAFETY: nothing borrows the mapping past `self`. + unsafe { unmap(self.ptr, self.len) } + } +} diff --git a/tests/guest/osbench/src/sys/toyos.rs b/tests/guest/osbench/src/sys/toyos.rs new file mode 100644 index 00000000000..44df6eef0ee --- /dev/null +++ b/tests/guest/osbench/src/sys/toyos.rs @@ -0,0 +1,61 @@ +use std::sync::atomic::AtomicU32; + +use toyos_abi::syscall::{self, MmapFlags, MmapProt}; + +/// ToyOS has no call that says which CPU runs a thread, so CPUs are named as +/// the architecture's own instructions name them. +pub use crate::arch::{core_of, cpu_model, current_cpu, threads_per_core}; + +pub fn null_syscall() { + std::hint::black_box(syscall::getpid()); +} + +/// Nanoseconds since boot off the clock page, which `Instant` reads and +/// every process shares. +pub fn monotonic_ns() -> u64 { + toyos_abi::clock::nanos_since_boot() +} + +/// Block while `word` holds `expected`. +pub fn futex_wait(word: &AtomicU32, expected: u32) { + // SAFETY: `word` is a live, aligned u32. + unsafe { syscall::futex_wait(word.as_ptr(), expected, None) }; +} + +/// Wake up to `count` waiters on `word`; answers how many woke. +pub fn futex_wake(word: &AtomicU32, count: u32) -> u32 { + // SAFETY: `word` is a live, aligned u32. + let woken = unsafe { syscall::futex_wake(word.as_ptr(), count) }; + u32::try_from(woken).expect("a wake count fits the count asked for") +} + +/// ToyOS commits, zeroes and maps an anonymous range whole, in 2 MiB pages, +/// before `mmap` answers. +pub fn map_anon(len: usize) -> *mut u8 { + // SAFETY: a fresh anonymous mapping aliases nothing. + let ptr = + unsafe { syscall::mmap(std::ptr::null_mut(), len, MmapProt::READ | MmapProt::WRITE, MmapFlags::ANONYMOUS) }; + assert!(!ptr.is_null(), "mmap of {len} bytes was refused"); + ptr +} + +/// Every ToyOS anonymous mapping is in 2 MiB pages. +pub fn map_anon_huge(len: usize) -> *mut u8 { + map_anon(len) +} + +pub fn huge_backing() -> String { + "all".into() +} + +/// # Safety +/// `ptr` and `len` are one whole mapping from [`map_anon`], and nothing +/// reaches it afterwards. +pub unsafe fn unmap(ptr: *mut u8, len: usize) { + // SAFETY: as the caller promises. + unsafe { syscall::munmap(ptr, len) }.expect("munmap of a whole mapping"); +} + +/// Nothing a program can read: the kernel's frequency request and the +/// machine's power source are the boot log's and the operator's to record. +pub fn conditions() {} diff --git a/tests/guest/osbench/src/system.rs b/tests/guest/osbench/src/system.rs new file mode 100644 index 00000000000..6ffb67e8bac --- /dev/null +++ b/tests/guest/osbench/src/system.rs @@ -0,0 +1,238 @@ +//! The primitives every program pays for: a syscall, a thread, a process, a +//! pipe, anonymous memory and the memory bus. + +use std::io::{Read, Write}; +use std::process::{Command, Stdio}; +use std::thread; +use std::time::{Duration, Instant}; + +use crate::report::{refuse, result}; +use crate::sys::{self, Mapping}; +use crate::{chunk, ns_since}; + +const MIB: usize = 1 << 20; + +/// What one chunk of the fairness work costs one thread, 200 after one: the +/// same instructions on both systems, so a difference between them is the +/// clock that thread ran at. It is a 2 ms single-threaded burst, so it reads +/// the clock a CPU leaving idle has, not the clock it settles at. +pub fn chunk_rate() { + chunk(); + let mut ns: Vec = (0..200) + .map(|_| { + let t = Instant::now(); + chunk(); + ns_since(t) + }) + .collect(); + result("cpu.chunk", "ns", &mut ns, &[]); +} + +/// The cheapest syscall each kernel has, getpid on both: 15 rounds of +/// 100,000 calls after one unread round, picoseconds per call. +pub fn null_syscall() { + const CALLS: u64 = 100_000; + let round = || { + let t = Instant::now(); + for _ in 0..CALLS { + sys::null_syscall(); + } + ns_since(t) * 1000 / CALLS + }; + round(); + let mut picos: Vec = (0..15).map(|_| round()).collect(); + result("syscall.null", "ps", &mut picos, &[("calls_per_sample", CALLS.to_string())]); +} + +/// `thread::spawn` of an empty closure and its `join`: 1,000 after 50. +pub fn thread_spawn() { + let once = || { + let t = Instant::now(); + thread::spawn(|| {}).join().expect("an empty thread"); + ns_since(t) + }; + for _ in 0..50 { + once(); + } + let mut ns: Vec = (0..1000).map(|_| once()).collect(); + result("thread.spawn_join", "ns", &mut ns, &[]); +} + +/// This program spawned with null stdio as `osbench nop`, which exits at +/// once, and waited for: 100 after 10. +pub fn process_spawn() { + let exe = match std::env::current_exe() { + Ok(exe) => exe, + Err(e) => return refuse("process.spawn_wait", format!("no path to this program: {e}")), + }; + let once = || -> Result { + let t = Instant::now(); + let status = Command::new(&exe) + .arg("nop") + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .status() + .map_err(|e| format!("spawn {}: {e}", exe.display()))?; + let ns = ns_since(t); + if !status.success() { + return Err(format!("nop ended {status}")); + } + Ok(ns) + }; + let run = || -> Result, String> { + for _ in 0..10 { + once()?; + } + (0..100).map(|_| once()).collect() + }; + match run() { + Ok(mut ns) => result("process.spawn_wait", "ns", &mut ns, &[]), + Err(why) => refuse("process.spawn_wait", why), + } +} + +/// One thread writing `size`-byte writes into a pipe for 100 ms and another +/// reading it in 64 KiB reads: MB/s as the reader saw it from its first byte +/// to end-of-file. One unread round, then seven. +pub fn pipe_throughput(size: usize) { + let bench = format!("pipe.throughput.{size}"); + let round = || -> std::io::Result { + let (mut reader, mut writer) = std::io::pipe()?; + let writing = thread::spawn(move || -> std::io::Result<()> { + let chunk = vec![0x5au8; size]; + let start = Instant::now(); + while start.elapsed() < Duration::from_millis(100) { + for _ in 0..64 { + writer.write_all(&chunk)?; + } + } + Ok(()) + }); + let mut buf = vec![0u8; 64 * 1024]; + // The clock starts at the first byte; what it carried is not counted. + if reader.read(&mut buf)? == 0 { + return Err(std::io::Error::other("the writer ended before writing")); + } + let (mut bytes, start) = (0u64, Instant::now()); + loop { + match reader.read(&mut buf)? { + 0 => break, + n => bytes += n as u64, + } + } + let ns = ns_since(start).max(1); + writing.join().expect("the writer panicked")?; + Ok(bytes * 1000 / ns) + }; + let run = || -> std::io::Result> { + round()?; + (0..7).map(|_| round()).collect() + }; + match run() { + Ok(mut mbps) => result(&bench, "MB/s", &mut mbps, &[("write_bytes", size.to_string())]), + Err(e) => refuse(&bench, e), + } +} + +/// Anonymous memory made usable and given back, ns per MiB of a 32 MiB +/// mapping: the `mmap`, a write to every 4 KiB page, and the `munmap`, each +/// timed alone and summed. Where the cost lands differs by design — Linux faults +/// each page on first touch, ToyOS commits and zeroes the range in `mmap` — +/// so only `total` compares like for like. One unread round, then ten. +pub fn anon_memory() { + const LEN: usize = 32 * MIB; + const MIBS: u64 = (LEN / MIB) as u64; + let round = || { + let t = Instant::now(); + let ptr = sys::map_anon(LEN); + let map = ns_since(t); + let t = Instant::now(); + for page in 0..LEN / 4096 { + // SAFETY: inside the mapping just made. + unsafe { ptr.add(page * 4096).write_volatile(1) }; + } + let touch = ns_since(t); + let t = Instant::now(); + // SAFETY: the whole mapping, unused from here. + unsafe { sys::unmap(ptr, LEN) }; + let unmap = ns_since(t); + [map, touch, unmap] + }; + round(); + let rounds: Vec<[u64; 3]> = (0..10).map(|_| round()).collect(); + let per_mib = |pick: &dyn Fn(&[u64; 3]) -> u64| -> Vec { rounds.iter().map(|r| pick(r) / MIBS).collect() }; + let extra = [("mapping_bytes", LEN.to_string())]; + result("mem.anon.map", "ns/MiB", &mut per_mib(&|r| r[0]), &extra); + result("mem.anon.touch", "ns/MiB", &mut per_mib(&|r| r[1]), &extra); + result("mem.anon.unmap", "ns/MiB", &mut per_mib(&|r| r[2]), &extra); + result("mem.anon.total", "ns/MiB", &mut per_mib(&|r| r.iter().sum()), &extra); +} + +/// A 4 KiB anonymous `mmap` and its `munmap`, never touched: 2,000 after 50. +pub fn map_unmap_small() { + let once = || { + let t = Instant::now(); + let ptr = sys::map_anon(4096); + // SAFETY: the whole mapping, never used. + unsafe { sys::unmap(ptr, 4096) }; + ns_since(t) + }; + for _ in 0..50 { + once(); + } + let mut ns: Vec = (0..2000).map(|_| once()).collect(); + result("mem.map_unmap_4k", "ns", &mut ns, &[]); +} + +/// One thread's bandwidth over two 64 MiB buffers, far past either bench's +/// last-level cache, in MB/s of buffer covered: `read` sums it, `write` +/// stores a pattern, `copy` stores the source xor a constant — a loop the +/// compiler cannot turn into a `memcpy` call, so both systems run the same +/// instructions — and `memcpy` is `copy_from_slice`, each system's own. +/// Linux is asked for huge pages, which every ToyOS anonymous mapping is; the +/// line says how much Linux gave. One unread round, then seven, per kind. +pub fn bandwidth() { + const LEN: usize = 64 * MIB; + let mut src = Mapping::huge(LEN); + let mut dst = Mapping::huge(LEN); + for (i, w) in src.words().iter_mut().enumerate() { + *w = i as u64; + } + dst.words().fill(1); + let huge = [("huge_pages", sys::huge_backing())]; + + let kind = |name: &str, f: &mut dyn FnMut()| { + let mut round = || { + let t = Instant::now(); + f(); + (LEN as u64) * 1000 / ns_since(t).max(1) + }; + round(); + let mut mbps: Vec = (0..7).map(|_| round()).collect(); + result(&format!("mem.bw.{name}"), "MB/s", &mut mbps, &huge); + }; + kind("read", &mut || { + let sum = src.words().iter().fold(0u64, |a, &w| a.wrapping_add(w)); + std::hint::black_box(sum); + }); + let mut seed = 0u64; + kind("write", &mut || { + seed += 1; + for (i, w) in dst.words().iter_mut().enumerate() { + *w = seed ^ i as u64; + } + std::hint::black_box(dst.ptr()); + }); + kind("copy", &mut || { + let (s, d) = (src.words(), dst.words()); + for (d, s) in d.iter_mut().zip(s.iter()) { + *d = *s ^ 0x9e37_79b9_7f4a_7c15; + } + std::hint::black_box(dst.ptr()); + }); + kind("memcpy", &mut || { + dst.words().copy_from_slice(src.words()); + std::hint::black_box(dst.ptr()); + }); +} diff --git a/tests/guest/osbench/src/wake.rs b/tests/guest/osbench/src/wake.rs new file mode 100644 index 00000000000..f3592877b46 --- /dev/null +++ b/tests/guest/osbench/src/wake.rs @@ -0,0 +1,377 @@ +//! How long a woken thread waits to run: on a timer, onto a busy CPU, and in +//! a storm. + +use std::process::{Child, Command, Stdio}; +use std::sync::atomic::{AtomicBool, AtomicU32, AtomicU8, Ordering}; +use std::io::{BufRead, BufReader}; +use std::sync::{Arc, Barrier}; +use std::thread; +use std::time::{Duration, Instant}; + +use crate::place::{self, Placed}; +use crate::report::{refuse, result}; +use crate::{ns_since, sys}; + +/// What a spin-wait on another thread's progress may take before the run is +/// refused: every wait here is microseconds on a working machine. +const STUCK: Duration = Duration::from_secs(5); + +fn spin_until(what: &str, done: impl Fn() -> bool) -> Result<(), String> { + let start = Instant::now(); + while !done() { + if start.elapsed() > STUCK { + return Err(format!("{what} did not happen within {STUCK:?}")); + } + std::hint::spin_loop(); + } + Ok(()) +} + +/// A cyclictest-style periodic wake: sleep to absolute targets every 500 µs +/// and record how late each wake ran, ns, 5,000 after 100. A late wake +/// shortens the next sleep rather than moving every later target. Ordinary +/// priority on both systems, as an unprivileged program gets it. +pub fn periodic(bench: &str) { + const PERIOD: Duration = Duration::from_micros(500); + const WARMUP: u32 = 100; + const SAMPLES: u32 = 5_000; + let start = Instant::now(); + let mut late = Vec::with_capacity(SAMPLES as usize); + for i in 1..=WARMUP + SAMPLES { + let target = start + PERIOD * i; + if let Some(left) = target.checked_duration_since(Instant::now()) { + thread::sleep(left); + } + if i > WARMUP { + late.push(ns_since(target)); + } + } + result(bench, "ns", &mut late, &[("period_ns", PERIOD.as_nanos().to_string())]); +} + +/// The periodic wake with every CPU kept busy by another process's +/// spinners, one per CPU. +pub fn periodic_loaded(bench: &str) { + let mut load = match Spinners::start() { + Ok(load) => load, + Err(why) => return refuse(bench, why), + }; + periodic(bench); + if let Err(why) = load.stop() { + refuse(bench, why); + } +} + +/// `osbench spin`: one spinning thread per CPU, which says `spinning` once +/// all of them have started, until it is killed or [`SPIN_BOUND`] has +/// passed, so a coordinator that died leaves no load behind. +pub fn spin() { + let end = Instant::now() + SPIN_BOUND; + let started = Arc::new(Barrier::new(crate::cpus() + 1)); + for _ in 0..crate::cpus() { + let started = started.clone(); + thread::spawn(move || { + started.wait(); + while Instant::now() < end { + std::hint::black_box(crate::chunk()); + } + }); + } + started.wait(); + println!("spinning"); + thread::sleep(end.saturating_duration_since(Instant::now())); +} + +/// Ten times what one loaded periodic run takes. +const SPIN_BOUND: Duration = Duration::from_secs(30); + +struct Spinners(Child); + +impl Spinners { + fn start() -> Result { + let exe = std::env::current_exe().map_err(|e| format!("no path to this program: {e}"))?; + let mut child = Command::new(exe) + .arg("spin") + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .spawn() + .map_err(|e| format!("the spinners would not start: {e}"))?; + let stdout = child.stdout.take().expect("stdout was asked for piped"); + let (said, heard) = std::sync::mpsc::channel(); + thread::spawn(move || { + let mut line = String::new(); + let _ = said.send(BufReader::new(stdout).read_line(&mut line).map(|_| line)); + }); + let mut spinners = Spinners(child); + match heard.recv_timeout(STUCK) { + Ok(Ok(line)) if line.trim() == "spinning" => Ok(spinners), + outcome => { + let _ = spinners.stop(); + Err(format!("the spinners never said they were spinning: {outcome:?}")) + } + } + } + + fn stop(&mut self) -> Result<(), String> { + self.0.kill().map_err(|e| format!("the spinners would not die: {e}"))?; + self.0.wait().map_err(|e| format!("the spinners were not reaped: {e}"))?; + Ok(()) + } +} + +const PARKED: u8 = 0; +const RUN: u8 = 1; +const DONE: u8 = 2; +const STOP: u8 = 3; + +/// A partner: where it parked, where it ran when woken, and how far it has got. +struct Seat { + parked_on: AtomicU32, + ran_on: AtomicU32, + phase: AtomicU8, +} + +fn park_until(seat: &Seat, until: &[u8]) -> u8 { + loop { + let phase = seat.phase.load(Ordering::Acquire); + if until.contains(&phase) { + return phase; + } + thread::park(); + } +} + +/// The CPU device interrupts reach on ToyOS (`kernel/src/arch/x86_64/apic.rs`), +/// left out of the staging on both systems so an interrupt's pass never +/// drains a wake early. +pub const IRQ_CPU: u32 = 0; +const ROUNDS: usize = 200; + +/// A wake onto a CPU a spinner holds, beside idle CPUs: each round, from a +/// waker spinning on a CPU of its own, wakes a fresh partner that parked on +/// the busy one and times it to running, ns. Linux pins the waker, each +/// partner until just before its wake, and the spinner. ToyOS has no +/// affinity and stages them, so a round is skipped and counted whenever its +/// partner parked elsewhere, the spinner left the busy CPU, or the waker is +/// on the busy CPU or the interrupts'. Either kernel may run the wake where it +/// is or move it to an idle CPU, and `ran_on_busy` counts how often it stayed. +/// A run every round of which was skipped is reported, not refused: on ToyOS +/// that is the placement this benchmark could not hold. +pub fn onto_busy() { + const BENCH: &str = "wake.onto_busy"; + if crate::cpus() < 3 { + return refuse(BENCH, "the waker, the busy CPU and the interrupts' CPU need three"); + } + let outcome = Arc::new(std::sync::Mutex::new(None)); + let waker = { + let outcome = outcome.clone(); + place::spawn_on(|on| on != IRQ_CPU, move || *outcome.lock().unwrap() = Some(rounds_onto_busy())) + }; + let waker = match waker { + Ok(waker) => waker, + Err(why) => return refuse(BENCH, format!("the waker was not placed: {why}")), + }; + waker.thread.join().expect("the waker panicked"); + let measured = outcome.lock().unwrap().take().expect("the waker said nothing"); + match measured { + Ok((ns, extra)) if ns.is_empty() => { + crate::report::unmeasured(BENCH, "every round was skipped, as the counts say", &extra) + } + Ok((mut ns, extra)) => result(BENCH, "ns", &mut ns, &extra), + Err(why) => refuse(BENCH, why), + } +} + +type Measured = Result<(Vec, Vec<(&'static str, String)>), String>; + +fn rounds_onto_busy() -> Measured { + let waker_on = sys::current_cpu(); + let mut partners: Vec<(Placed, Arc)> = Vec::with_capacity(ROUNDS); + let mut busy = None; + let end = |partners: Vec<(Placed, Arc)>| { + for (partner, seat) in partners { + seat.phase.store(STOP, Ordering::Release); + partner.thread.thread().unpark(); + partner.thread.join().expect("a partner panicked"); + } + }; + while partners.len() < ROUNDS { + let seat = Arc::new(Seat { + parked_on: AtomicU32::new(u32::MAX), + ran_on: AtomicU32::new(u32::MAX), + phase: AtomicU8::new(PARKED), + }); + let wanted = |on: u32| on != waker_on && on != IRQ_CPU && busy.is_none_or(|b| b == on); + let body = { + let seat = seat.clone(); + move || { + seat.parked_on.store(sys::current_cpu(), Ordering::Release); + if park_until(&seat, &[RUN, STOP]) == RUN { + seat.ran_on.store(sys::current_cpu(), Ordering::Relaxed); + seat.phase.store(DONE, Ordering::Release); + park_until(&seat, &[STOP]); + } + } + }; + match place::spawn_on(wanted, body) { + Ok(placed) => { + busy = Some(placed.cpu); + partners.push((placed, seat)); + } + Err(why) => { + let staged = partners.len(); + end(partners); + return Err(format!("partner {staged} was not placed: {why}")); + } + } + } + let busy = busy.expect("ROUNDS is not zero"); + + let stop = Arc::new(AtomicBool::new(false)); + let at = Arc::new(AtomicU32::new(u32::MAX)); + let spinner = { + let (stop, at) = (stop.clone(), at.clone()); + place::spawn_on( + |on| on == busy, + move || { + while !stop.load(Ordering::Relaxed) { + at.store(sys::current_cpu(), Ordering::Relaxed); + std::hint::black_box(crate::chunk()); + } + }, + ) + }; + let spinner = match spinner { + Ok(spinner) => spinner, + Err(why) => { + end(partners); + return Err(format!("no spinner on CPU {busy}: {why}")); + } + }; + let spinning = spin_until("the spinner's first word", || at.load(Ordering::Relaxed) != u32::MAX); + + let mut ns = Vec::with_capacity(ROUNDS); + let (mut ran_on_busy, mut spinner_away, mut parked_elsewhere, mut waker_crowded) = (0usize, 0usize, 0usize, 0usize); + let mut failed = spinning.err(); + for (partner, seat) in partners.iter().filter(|_| failed.is_none()) { + if let Err(why) = spin_until("a partner's park", || seat.parked_on.load(Ordering::Acquire) != u32::MAX) { + failed = Some(why); + break; + } + if seat.parked_on.load(Ordering::Acquire) != busy { + parked_elsewhere += 1; + continue; + } + if at.load(Ordering::Relaxed) != busy { + spinner_away += 1; + continue; + } + let waker_at = sys::current_cpu(); + if waker_at == busy || waker_at == IRQ_CPU { + waker_crowded += 1; + continue; + } + partner.release(); + let sent = Instant::now(); + seat.phase.store(RUN, Ordering::Release); + partner.thread.thread().unpark(); + if let Err(why) = spin_until("a woken partner's run", || seat.phase.load(Ordering::Acquire) == DONE) { + failed = Some(why); + break; + } + ns.push(ns_since(sent)); + ran_on_busy += usize::from(seat.ran_on.load(Ordering::Relaxed) == busy); + } + stop.store(true, Ordering::Relaxed); + spinner.thread.join().expect("the spinner panicked"); + end(partners); + if let Some(why) = failed { + return Err(why); + } + Ok(( + ns, + vec![ + ("busy_cpu", busy.to_string()), + ("waker_cpu", waker_on.to_string()), + ("ran_on_busy", ran_on_busy.to_string()), + ("skipped_spinner_away", spinner_away.to_string()), + ("skipped_parked_elsewhere", parked_elsewhere.to_string()), + ("skipped_waker_on_busy_or_irq", waker_crowded.to_string()), + ], + )) +} + +/// The storm's generation: each storm advances it, which is what ends a +/// waiter's wait on both systems (a ToyOS futex wait returns only once the +/// word has changed), and each woken waiter counts itself and waits on the +/// new value. +static WORD: AtomicU32 = AtomicU32::new(0); +static ARRIVED: AtomicU32 = AtomicU32::new(0); +static FINISHED: AtomicBool = AtomicBool::new(false); + +/// A futex wake of `width` parked waiters: what the call costs the waker +/// (`call`) and how long until every woken waiter has run (`drain`), ns, 15 +/// rounds per width. A round counts only if the wake found exactly `width` +/// parked, which is the proof they all were. +pub fn storm() { + const WIDTHS: [u32; 4] = [1, 4, 16, 64]; + const ROUNDS: usize = 15; + const ATTEMPTS: usize = 400; + let mut waiters = Vec::new(); + for width in WIDTHS { + while waiters.len() < width as usize { + waiters.push(thread::spawn(|| { + let mut seen = WORD.load(Ordering::Acquire); + while !FINISHED.load(Ordering::Acquire) { + sys::futex_wait(&WORD, seen); + let now = WORD.load(Ordering::Acquire); + if now != seen { + seen = now; + ARRIVED.fetch_add(1, Ordering::AcqRel); + } + } + })); + } + let (mut call, mut drain) = (Vec::new(), Vec::new()); + let mut failed = None; + for _ in 0..ATTEMPTS { + if call.len() == ROUNDS { + break; + } + // Time for every waiter woken last round to park again. + thread::sleep(Duration::from_millis(2)); + let before = ARRIVED.load(Ordering::Acquire); + let t = Instant::now(); + WORD.fetch_add(1, Ordering::AcqRel); + let woken = sys::futex_wake(&WORD, u32::MAX); + let called = ns_since(t); + if let Err(why) = + spin_until("a storm's drain", || ARRIVED.load(Ordering::Acquire).wrapping_sub(before) >= woken) + { + failed = Some(why); + break; + } + if woken == width { + call.push(called); + drain.push(ns_since(t)); + } + } + let bench = |what: &str| format!("wake.storm.{what}.w{width}"); + match failed { + Some(why) => refuse(&bench("call"), why), + None if call.len() < ROUNDS => { + refuse(&bench("call"), format!("{} of {ATTEMPTS} wakes found all {width} parked", call.len())) + } + None => { + result(&bench("call"), "ns", &mut call, &[]); + result(&bench("drain"), "ns", &mut drain, &[]); + } + } + } + FINISHED.store(true, Ordering::Release); + WORD.fetch_add(1, Ordering::AcqRel); + while sys::futex_wake(&WORD, u32::MAX) > 0 {} + for waiter in waiters { + waiter.join().expect("a storm waiter panicked"); + } +} diff --git a/tests/images/benchcase.toml b/tests/images/benchcase.toml new file mode 100644 index 00000000000..3fcc3b67062 --- /dev/null +++ b/tests/images/benchcase.toml @@ -0,0 +1,61 @@ +# The boot `osbench` measures ToyOS on, for a comparison with the same program +# on the bench's own Linux (`tests/guest/osbench`). +# +# Only what a job list needs to run and to reach the log on the boot stick: no +# compositor, sound, ACPI server or network, so little but the measurement asks +# for a CPU. +# +# The runner's capability carries `jobs`, for the fairness run that starts each +# workload as a job of its own, and `dup`, to hand it down. +[boot] +start = ["logkeeper", "diskserver", "fileserver", "test-runner"] + +[programs.logkeeper] +service = true +syscap = ["logread"] + +[programs.test-runner] +receives = ["power"] +args = ["osbench_system", "osbench_sched", "osbench_fair", "reboot"] +syscap = ["jobs", "dup"] + +[programs.osbench] + +[programs.toybox] + +[symlinks] +"bin/osbench_system" = "/system/bin/osbench" +"bin/osbench_sched" = "/system/bin/osbench" +"bin/osbench_fair" = "/system/bin/osbench" +"bin/reboot" = "/system/bin/toybox" + +# The block service: every disk a driver offers on `disk`, listed and opened +# for the clients `block` grants, each session handed to its disk's driver. +# Started again when it ends; its drivers connect again and offer again. +[programs.diskserver] +service = true +restart = true +serves = ["block", "disk"] + +# The NVMe driver: one process per such controller, reading its own table and +# offering its disk to the block service. QEMU's controller alone, so a metal +# boot leaves the bench's own NVMe disk to its firmware. +[programs.nvme] +drives = ["pci:01:08:02/1b36:0010"] +receives = ["disk"] + +# The USB driver, one process per boot controller: QEMU's `nec-usb-xhci`, the +# T14's 00:14.0 and the AMD bench's 03:00.3. QEMU's spare, the T14's Type-C +# controller and the AMD bench's 03:00.4 stay free for the tests that claim +# them. +[programs.usbd] +drives = ["pci:0c:03:30/1033:0194", "pci:0c:03:30/8086:a0ed", "pci:0c:03:30/1022:1639@03:00.3"] +receives = ["disk"] + +# The file servers: DATA, the log and the boot volume, one process +# each, serving every program the directories of its role. Started again when +# one ends, on the same ports. +[programs.fileserver] +restart = true +roles = ["data", "log", "boot"] +receives = ["block"] diff --git a/tests/toyos.rs b/tests/toyos.rs index eca5af95013..ba233f7f3f2 100644 --- a/tests/toyos.rs +++ b/tests/toyos.rs @@ -987,6 +987,14 @@ const METAL: &[(&str, metal::Metal)] = &[ "job_fairness", metal::Metal { arms: TESTCASES, judge: |b| job_fairness(b[0]) }, ), + // ---- one image: tests/images/benchcase.toml, `osbench`'s ToyOS half ---- + ( + // The numbers compared with the same program on the bench's own Linux. + // Evidence and not a verdict: they are read off the log, and the row + // holds only that every group ran to its end. The T14 and the AMD bench. + "osbench", + metal::Metal { arms: BENCHCASE, judge: |b| osbench_ran(b[0]) }, + ), // ---- tests/images/jobcase.toml, each image armed to end its boot its own way ---- ( // Its own boot, and it must not share one: it is the only arm in this @@ -1411,6 +1419,22 @@ const TESTCASES: &[metal::Arm] = &[metal::once( ], )]; +/// `osbench`'s groups, each a job named by its symlink. +const OSBENCH_GROUPS: [&str; 3] = ["osbench_system", "osbench_sched", "osbench_fair"]; + +const BENCHCASE: &[metal::Arm] = &[metal::once("benchcase", "tests/images/benchcase.toml", &[], &OSBENCH_GROUPS)]; + +/// Every group exited 0, which `osbench` does only once each of its +/// benchmarks measured, and said it was done. +fn osbench_ran(boot: &metal::Readback) -> Result<(), String> { + for job in OSBENCH_GROUPS { + boot.job_passed(job)?; + let group = job.strip_prefix("osbench_").expect("a group's job"); + boot.log().must_say(&format!("group={group} done"))?; + } + Ok(()) +} + /// The same boot, its rows ended on the hold, which waits for the ACPI /// server's power sources, the last line of its start. The members run /// behind it: the hold gives up at a time since boot, which a list of