From 07c14b086c1f8594773274ead0398e780fd73f50 Mon Sep 17 00:00:00 2001 From: japabu Date: Wed, 21 Oct 2026 18:16:37 +0200 Subject: [PATCH 1/4] sched: EEVDF picks over a deadline tree augmented with each subtree's least virtual time The queue kept its entities in a vector and scanned it for every pick, eligibility query, charge and lag read. Every operation is now O(log n): an AVL tree ordered by (virtual deadline, issue order) whose nodes carry their subtree's least virtual time answers the pick (the first eligible entity in deadline order) and `overtaken_in` (the least virtual time among entities due before the running one) by one descent, and a key-to-slot map replaces the linear lookups. Semantics are unchanged, tie-breaks included. `end_request` ends an entity's request where its service stands, which a voluntary yield gives up. Checks: - `the_tree_answers_as_a_scan_of_every_entity_does`: 20,000 random joins, leaves, rejoins, charges, reweights and slice changes, each followed by the tree's shape and augmentation checked whole and its pick and `overtaken_in` compared against a scan. - The stage-1 model (`job_sched`) prints the same reports to the byte with the tree as with the vector. Co-Authored-By: Claude Opus 5.5 --- kernel/pure/sched/eevdf.rs | 427 +++++++++++++++++++++++++++++++++---- 1 file changed, 384 insertions(+), 43 deletions(-) diff --git a/kernel/pure/sched/eevdf.rs b/kernel/pure/sched/eevdf.rs index 2d897cba08..99d37125d3 100644 --- a/kernel/pure/sched/eevdf.rs +++ b/kernel/pure/sched/eevdf.rs @@ -16,7 +16,12 @@ //! so charging never loses service. The only rounding is the queue's mean //! (`V`), which is below one virtual unit: a lag read or a join is exact to //! within `w · 2^-VSHIFT` ns, under `2^-8` ns at [`MAX_WEIGHT`]. +//! +//! Entities sit in an AVL tree ordered by deadline, each node holding its +//! subtree's least `v`, so a pick, an eligibility query and every update are +//! O(log n) in the entities queued. +use alloc::collections::BTreeMap; use alloc::vec::Vec; /// Fine service units per nanosecond, as a shift. @@ -50,6 +55,8 @@ impl Lag { } } +const NIL: u32 = u32::MAX; + struct Entity { key: K, weight: u64, @@ -63,6 +70,13 @@ struct Entity { slice: u64, /// Issue order of the current request: earlier wins a deadline tie. seq: u64, + left: u32, + right: u32, + height: u8, + /// The least `v` in this entity's subtree. + min_v: i128, + /// Where it is in [`Eevdf::order`]. + pos: u32, } /// What [`Eevdf::leave`] hands back: everything needed to rejoin elsewhere. @@ -74,7 +88,15 @@ pub struct Left { } pub struct Eevdf { - entities: Vec>, + /// Every entity queued, and the slots of those that left, which `free` + /// lists for reuse. + nodes: Vec>, + free: Vec, + slots: BTreeMap, + /// The queued slots in join order, a leave moving the last into its place: + /// the order [`Self::keys`] answers in. + order: Vec, + root: u32, total: i128, /// Origin of `sum`, kept within one virtual unit of `V`. base: i128, @@ -96,25 +118,35 @@ fn fine(ns: u64) -> i128 { (ns as i128) << VSHIFT } -impl Eevdf { +impl Eevdf { pub fn new() -> Self { - Self { entities: Vec::new(), total: 0, base: 0, sum: 0, seq: 0 } + Self { + nodes: Vec::new(), + free: Vec::new(), + slots: BTreeMap::new(), + order: Vec::new(), + root: NIL, + total: 0, + base: 0, + sum: 0, + seq: 0, + } } pub fn len(&self) -> usize { - self.entities.len() + self.order.len() } pub fn is_empty(&self) -> bool { - self.entities.is_empty() + self.order.is_empty() } pub fn contains(&self, key: K) -> bool { - self.entities.iter().any(|e| e.key == key) + self.slots.contains_key(&key) } pub fn keys(&self) -> impl Iterator + '_ { - self.entities.iter().map(|e| e.key) + self.order.iter().map(|&i| self.nodes[i as usize].key) } pub fn total_weight(&self) -> u128 { @@ -125,14 +157,168 @@ impl Eevdf { self.get(key).weight } - fn index(&self, key: K) -> usize { - self.entities.iter().position(|e| e.key == key).expect("eevdf: entity is not queued") + fn index(&self, key: K) -> u32 { + *self.slots.get(&key).expect("eevdf: entity is not queued") } fn get(&self, key: K) -> &Entity { - &self.entities[self.index(key)] + &self.nodes[self.index(key) as usize] + } + + fn node(&self, i: u32) -> &Entity { + &self.nodes[i as usize] + } + + fn node_mut(&mut self, i: u32) -> &mut Entity { + &mut self.nodes[i as usize] + } + + /// Tree order: deadline, then issue order. + fn order_of(&self, i: u32) -> (i128, u64) { + let e = self.node(i); + (e.vd, e.seq) + } + + fn eligible(&self, v: i128) -> bool { + mul(self.total, v - self.base) <= self.sum + } + + // ---- The tree ------------------------------------------------------------ + + fn height(&self, i: u32) -> u8 { + if i == NIL { 0 } else { self.node(i).height } + } + + fn update(&mut self, i: u32) { + let (l, r) = (self.node(i).left, self.node(i).right); + let mut min_v = self.node(i).v; + if l != NIL { + min_v = min_v.min(self.node(l).min_v); + } + if r != NIL { + min_v = min_v.min(self.node(r).min_v); + } + let height = 1 + self.height(l).max(self.height(r)); + let e = self.node_mut(i); + e.min_v = min_v; + e.height = height; + } + + fn rotate_right(&mut self, i: u32) -> u32 { + let l = self.node(i).left; + self.node_mut(i).left = self.node(l).right; + self.node_mut(l).right = i; + self.update(i); + self.update(l); + l + } + + fn rotate_left(&mut self, i: u32) -> u32 { + let r = self.node(i).right; + self.node_mut(i).right = self.node(r).left; + self.node_mut(r).left = i; + self.update(i); + self.update(r); + r + } + + fn balance(&mut self, i: u32) -> u32 { + self.update(i); + let (l, r) = (self.node(i).left, self.node(i).right); + let skew = i16::from(self.height(l)) - i16::from(self.height(r)); + if skew > 1 { + if self.height(self.node(l).left) < self.height(self.node(l).right) { + let l = self.rotate_left(l); + self.node_mut(i).left = l; + } + return self.rotate_right(i); + } + if skew < -1 { + if self.height(self.node(r).right) < self.height(self.node(r).left) { + let r = self.rotate_right(r); + self.node_mut(i).right = r; + } + return self.rotate_left(i); + } + i + } + + /// Insert the detached node `n` under `t`; answers the subtree's new root. + fn insert(&mut self, t: u32, n: u32) -> u32 { + if t == NIL { + let e = self.node_mut(n); + (e.left, e.right, e.height, e.min_v) = (NIL, NIL, 1, e.v); + return n; + } + if self.order_of(n) < self.order_of(t) { + let l = self.insert(self.node(t).left, n); + self.node_mut(t).left = l; + } else { + let r = self.insert(self.node(t).right, n); + self.node_mut(t).right = r; + } + self.balance(t) + } + + /// Detach the node at tree order `at` from under `t`. + fn remove(&mut self, t: u32, at: (i128, u64)) -> u32 { + assert!(t != NIL, "eevdf: a queued entity is missing from the tree"); + let here = self.order_of(t); + if at < here { + let l = self.remove(self.node(t).left, at); + self.node_mut(t).left = l; + } else if at > here { + let r = self.remove(self.node(t).right, at); + self.node_mut(t).right = r; + } else { + let (l, r) = (self.node(t).left, self.node(t).right); + if l == NIL { + return r; + } + if r == NIL { + return l; + } + let (next, rest) = self.remove_first(r); + let e = self.node_mut(next); + (e.left, e.right) = (l, rest); + return self.balance(next); + } + self.balance(t) } + /// Detach the first node under `t`: answers it and the subtree's new root. + fn remove_first(&mut self, t: u32) -> (u32, u32) { + let l = self.node(t).left; + if l == NIL { + return (t, self.node(t).right); + } + let (first, l) = self.remove_first(l); + self.node_mut(t).left = l; + (first, self.balance(t)) + } + + /// Recompute the least `v` on the path to the node at `at`, whose `v` + /// changed and whose place did not. + fn refresh(&mut self, t: u32, at: (i128, u64)) { + let here = self.order_of(t); + if at < here { + self.refresh(self.node(t).left, at); + } else if at > here { + self.refresh(self.node(t).right, at); + } + self.update(t); + } + + /// Give node `i` a fresh request, moving it to its new place in the tree. + fn reissue(&mut self, i: u32, request: i128) { + let at = self.order_of(i); + self.root = self.remove(self.root, at); + self.issue(i, request); + self.root = self.insert(self.root, i); + } + + // ---- The queue ----------------------------------------------------------- + /// Move `base` to `⌊V⌋` so the sums stay small. fn rebase(&mut self) { if self.total > 0 { @@ -142,11 +328,13 @@ impl Eevdf { } } - fn issue(&mut self, i: usize, request: i128) { + /// Set a detached node's request. + fn issue(&mut self, i: u32, request: i128) { self.seq += 1; - let e = &mut self.entities[i]; + let seq = self.seq; + let e = self.node_mut(i); e.vd = add(e.v, request.div_euclid(e.weight as i128).max(1)); - e.seq = self.seq; + e.seq = seq; } /// Enter with `lag` and a fresh request of `slice` ns. Returns the lag the @@ -169,9 +357,34 @@ impl Eevdf { let num = mul(w, self.sum - l) - mul(l, self.total); num.div_euclid(mul(self.total, w)) }; - self.entities.push(Entity { key, weight, v: add(self.base, v_rel), rem: 0, vd: 0, slice, seq: 0 }); - let i = self.entities.len() - 1; + let entity = Entity { + key, + weight, + v: add(self.base, v_rel), + rem: 0, + vd: 0, + slice, + seq: 0, + left: NIL, + right: NIL, + height: 1, + min_v: 0, + pos: self.order.len() as u32, + }; + let i = match self.free.pop() { + Some(i) => { + self.nodes[i as usize] = entity; + i + } + None => { + self.nodes.push(entity); + (self.nodes.len() - 1) as u32 + } + }; + self.slots.insert(key, i); + self.order.push(i); self.issue(i, request); + self.root = self.insert(self.root, i); self.sum = add(self.sum, mul(w, v_rel)); self.total += w; self.lag(key) @@ -181,15 +394,24 @@ impl Eevdf { pub fn leave(&mut self, key: K) -> Left { self.rebase(); let lag = self.lag(key); - let i = self.index(key); - let e = self.entities.swap_remove(i); + let i = self.slots.remove(&key).expect("eevdf: entity is not queued"); + let at = self.order_of(i); + self.root = self.remove(self.root, at); + let pos = self.node(i).pos as usize; + self.order.swap_remove(pos); + if let Some(&moved) = self.order.get(pos) { + self.node_mut(moved).pos = pos as u32; + } + self.free.push(i); + let e = self.node(i); let w = e.weight as i128; - self.sum -= mul(w, e.v - self.base); + let (v, vd) = (e.v, e.vd); + self.sum -= mul(w, v - self.base); self.total -= w; if self.total == 0 { self.sum = 0; } - Left { lag, request_left: mul(e.vd - e.v, w) } + Left { lag, request_left: mul(vd - v, w) } } /// Enter carrying what [`Self::leave`] returned elsewhere: the lag and the @@ -214,28 +436,38 @@ impl Eevdf { /// once by the next, whose deadline counts from where service now stands. pub fn charge(&mut self, key: K, ns: u64) { let i = self.index(key); - let e = &mut self.entities[i]; + let e = self.node_mut(i); let total = fine(ns) + e.rem as i128; let w = e.weight as i128; let dv = total / w; e.rem = (total % w) as u64; e.v = add(e.v, dv); + let (v, vd, slice) = (e.v, e.vd, e.slice); self.sum = add(self.sum, mul(dv, w)); - if e.v >= e.vd { - let slice = e.slice; - self.issue(i, fine(slice)); + if v >= vd { + self.reissue(i, fine(slice)); + } else { + self.refresh(self.root, self.order_of(i)); } } /// The eligible entity with the earliest deadline. pub fn pick(&self) -> Option { - let best = self - .entities - .iter() - .filter(|e| mul(self.total, e.v - self.base) <= self.sum) - .min_by_key(|e| (e.vd, e.seq)); - assert!(best.is_some() || self.is_empty(), "eevdf: a non-empty queue with nothing eligible"); - best.map(|e| e.key) + let mut t = self.root; + while t != NIL { + let e = self.node(t); + if e.left != NIL && self.eligible(self.node(e.left).min_v) { + t = e.left; + } else if self.eligible(e.v) { + return Some(e.key); + } else if e.right != NIL && self.eligible(self.node(e.right).min_v) { + t = e.right; + } else { + break; + } + } + assert!(self.is_empty(), "eevdf: a non-empty queue with nothing eligible"); + None } /// Service, in whole ns of `running` being served, until a queued entity @@ -243,15 +475,26 @@ impl Eevdf { /// pick: `V` advances by `1 / total` per unit of service, and an entity /// with lag `l < 0` becomes eligible once it has advanced `−l / w`. pub fn overtaken_in(&self, running: K) -> Option { - let r = self.get(running); - self.entities - .iter() - .filter(|e| (e.vd, e.seq) < (r.vd, r.seq)) - .map(|e| { - let behind = mul(self.total, e.v - self.base) - self.sum; - ((behind.max(0) + (1 << VSHIFT) - 1) >> VSHIFT) as u64 - }) - .min() + let before = self.order_of(self.index(running)); + let mut least: Option = None; + let mut t = self.root; + while t != NIL { + let e = self.node(t); + if self.order_of(t) < before { + let here = match e.left { + NIL => e.v, + l => e.v.min(self.node(l).min_v), + }; + least = Some(least.map_or(here, |m| m.min(here))); + t = e.right; + } else { + t = e.left; + } + } + least.map(|v| { + let behind = mul(self.total, v - self.base) - self.sum; + ((behind.max(0) + (1 << VSHIFT) - 1) >> VSHIFT) as u64 + }) } /// Service, in whole ns rounded up, until `key`'s current request is done. @@ -262,6 +505,14 @@ impl Eevdf { ns.max(1) as u64 } + /// End `key`'s current request where its service stands, and issue the + /// next: what a voluntary yield gives up. + pub fn end_request(&mut self, key: K) { + let i = self.index(key); + let slice = self.node(i).slice; + self.reissue(i, fine(slice)); + } + /// Change weight, keeping lag and the service left in the current request. pub fn reweight(&mut self, key: K, weight: u64) { let slice = self.get(key).slice; @@ -276,11 +527,11 @@ impl Eevdf { /// new size if that ends sooner. pub fn set_slice(&mut self, key: K, slice: u64) { let i = self.index(key); - let e = &mut self.entities[i]; + let e = self.node_mut(i); e.slice = slice; let request = mul(e.vd - e.v, e.weight as i128); if fine(slice) < request { - self.issue(i, fine(slice)); + self.reissue(i, fine(slice)); } } @@ -291,11 +542,11 @@ impl Eevdf { /// `Σ lag`, which is zero up to the mean's rounding: the conservation the /// whole construction rests on, for tests. pub fn lag_sum(&self) -> Lag { - Lag(self.entities.iter().map(|e| self.lag(e.key).0).sum()) + Lag(self.keys().map(|k| self.lag(k).0).sum()) } } -impl Default for Eevdf { +impl Default for Eevdf { fn default() -> Self { Self::new() } @@ -478,6 +729,96 @@ mod tests { assert_eq!(q.weight(0), 7 * UNIT); } + /// The tree's shape and augmentation, and its answers against a scan of + /// every entity, after each of a long random run of joins, leaves, + /// charges, reweights and slice changes. + #[test] + fn the_tree_answers_as_a_scan_of_every_entity_does() { + fn scan_pick(q: &Eevdf) -> Option { + q.order + .iter() + .map(|&i| q.node(i)) + .filter(|e| q.eligible(e.v)) + .min_by_key(|e| (e.vd, e.seq)) + .map(|e| e.key) + } + fn scan_overtaken(q: &Eevdf, running: u32) -> Option { + let r = q.get(running); + q.order + .iter() + .map(|&i| q.node(i)) + .filter(|e| (e.vd, e.seq) < (r.vd, r.seq)) + .map(|e| { + let behind = mul(q.total, e.v - q.base) - q.sum; + ((behind.max(0) + (1 << VSHIFT) - 1) >> VSHIFT) as u64 + }) + .min() + } + /// Height, least `v` and the ordered keys of the subtree at `t`. + fn walk(q: &Eevdf, t: u32, out: &mut Vec<(i128, u64)>) -> (u8, i128) { + if t == NIL { + return (0, i128::MAX); + } + let e = q.node(t); + let (hl, ml) = walk(q, e.left, out); + out.push((e.vd, e.seq)); + let (hr, mr) = walk(q, e.right, out); + assert!(hl.abs_diff(hr) <= 1, "an unbalanced node"); + assert_eq!(e.height, 1 + hl.max(hr)); + assert_eq!(e.min_v, e.v.min(ml).min(mr)); + (e.height, e.min_v) + } + let mut rng = Lcg(23); + let mut q: Eevdf = Eevdf::new(); + let mut running: Option = None; + for step in 0..20_000 { + match rng.below(10) { + 0 | 1 if q.len() < 64 => { + let k = (0..).find(|k| !q.contains(*k)).unwrap(); + let lag = rng.below(8 * MS) as i64 - 4 * MS as i64; + q.join(k, 1 + rng.below(1 << 24), MS / 4 + rng.below(4 * MS), Lag::from_ns(lag)); + } + 2 if !q.is_empty() => { + let k = q.keys().nth(rng.below(q.len() as u64) as usize).unwrap(); + let left = q.leave(k); + if running == Some(k) { + running = None; + } + if rng.below(2) == 0 { + q.rejoin(k, 1 + rng.below(1 << 24), MS / 4 + rng.below(4 * MS), left); + } + } + 3 if !q.is_empty() => { + let k = q.keys().nth(rng.below(q.len() as u64) as usize).unwrap(); + q.reweight(k, 1 + rng.below(1 << 24)); + } + 4 if !q.is_empty() => { + let k = q.keys().nth(rng.below(q.len() as u64) as usize).unwrap(); + q.set_slice(k, MS / 4 + rng.below(4 * MS)); + } + _ => { + if running.is_none() { + running = q.pick(); + } + if let Some(k) = running { + q.charge(k, 1 + rng.below(q.request_left_ns(k))); + if rng.below(3) == 0 { + running = None; + } + } + } + } + let mut ordered = Vec::new(); + walk(&q, q.root, &mut ordered); + assert!(ordered.windows(2).all(|w| w[0] < w[1]), "step {step}: out of deadline order"); + assert_eq!(ordered.len(), q.len()); + assert_eq!(q.pick(), scan_pick(&q), "step {step}: pick"); + if let Some(k) = running { + assert_eq!(q.overtaken_in(k), scan_overtaken(&q, k), "step {step}: overtaken_in"); + } + } + } + #[test] fn charging_loses_no_service_to_rounding() { // A weight that divides nothing: 10^6 one-nanosecond charges must add From 539cc6333c6f6d49e78609cf6dd24b1088a4b3e3 Mon Sep 17 00:00:00 2001 From: japabu Date: Wed, 21 Oct 2026 18:16:38 +0200 Subject: [PATCH 2/4] sched: a job owns its threads' CPU time, and each CPU runs EEVDF over the job groups it holds MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Stage 3 of the scheduler plan (owner decision J1): the per-PID fair shares, the global vruntime frontier and the run queue's frozen insertion keys go together, replaced by jobs and two-level per-CPU EEVDF. Jobs. A job is the owner a process's threads and its children share CPU time under (`kernel/pure/sched/job.rs`, kernel object `Job`). A spawn puts its child in its own job unless it endows another under `JOB_LABEL`; making a new job takes a system capability's `JOBS` right (`SYS_JOB_CREATE`), so names and pids grant nothing. `SYS_JOB_OWN` answers a handle to the caller's job, which a launch carries so the supervisor starts the program in its caller's job. The supervisor makes every service a job of its own, and the compositor every program it starts (its row now holds `jobs`). A job lives while a thread of it runs or a handle to it is held. There is no hierarchy above it. Scheduling. Each CPU's fair band is EEVDF over the job groups it holds and EEVDF over each group's threads (`kernel/pure/sched/queue.rs`). A group weighs its job's weight times its share of the job's runnable threads, refreshed at each decision, so a helper buys no entitlement. Lag goes with whatever leaves: a thread's to its next queue, a dissolving group's with its last thread, clamped to `LAG_BOUND_NS`. A CPU decides again at a request's end, at an earlier-due group's eligibility, and at a wake due first — the last two once the running thread has run `MIN_SLICE_NS`. Every change to a CPU's queues happens in its own pass and every pass arms the timer last, so a remote steal re-arms it as a wake does (the J1 compositor fix). Busy CPUs ask the heaviest CPU for the thread whose move narrows their summed group weight, once per `BALANCE_NS`; idle steals and the push stay. A yield ends the yielder's request at both levels; a running fair thread lent real time leaves the fair band at the lend, its real-time turn starting then. Parameters: `SLICE_NS` 4 ms, `MIN_SLICE_NS` 250 us, `LAG_BOUND_NS` 4 ms, `BALANCE_NS` one slice — the stage-1 model's, under which it met the J1 criteria; `QUANTUM_NS` (10 ms) stays the real-time band's turn. Deleted: `fair.rs` (`FairShare`, `Frontier`, `ShareState`), `SYS_SCHED_INFO` and `SchedInfo`, `PROTECTED_RUN_NS`, `FairOrder`, the PID share table and its removal hook, sched_stress's vruntime-lag check and abuse_page_straddle's `SchedInfo` case (its `ProcessStats` case covers the same copy-out). Simulator. I5 is now equal jobs on a machine holding them alike, against `cpus × 2 × (slice + lag bound + kernel section + 2 chunks)`; I13 is runnable- thread progress, the service a ready thread's CPU owed it while it waited, against the model's `3 × slice + 2 × lag bound` plus two levels of decision lateness; I6 checks each job's runnable count against the bands. Gates: `fair_job_per_thread` and `fair_double_charge` red on I5 alone, `fair_lowest_key_within_group` on I13 alone. `fairness_storm` from 1 to 32 CPUs, 200 seeds each: I5 worst 4/20, 10/40, 14/60, 18/80, 24/120, 26/160, 38/240, 42/320 and 52/480 ms at 1, 2, 3, 4, 6, 8, 12, 16 and 24; I13 worst under 9.3 ms of 24 at every width. The policy suite is restated for the new bounds: a single-threaded rival finishes one slice late at every swarm size from 2 to 64, where it used to fall N+1 quanta behind. `old_commit_before_pass` is caught in 415 of 500 schedules rather than all: the explorer may leave a CPU un-stepped while it holds a task, so a consumer's request can end before it took a step and no block finds the queue empty. Two corpus traces were re-shrunk for the new decisions. Step budgets for the wide workloads follow the measured pass counts. Closed, their mechanism gone with the frozen keys: per-process-fair-split-is-the-policy, granularity-bound-crossed-at-four-widths, i13s-margin-at-32-cpus-is-down-to-two-milliseconds, and interactive-mix-crosses-the-fairness-bound-at-two-cpus (no crossing in 2,000 seeds of either stream with its allowance removed; I5 opens no window there now, and its sleeper's waits are I13's, at worst 8.7 ms of 24). fairness-is-per-thread-so-thread-count-wins is answered by jobs, and the hierarchy it proposed the owner declined. Co-Authored-By: Claude Opus 5.5 --- build/kernelkeys.rs | 5 - images/system.toml | 4 +- ...spinners-quantum-when-every-cpu-is-busy.md | 20 +- ...clippy-stage-two-is-lints-one-at-a-time.md | 2 +- ...ness-is-per-thread-so-thread-count-wins.md | 31 - ...ranularity-bound-crossed-at-four-widths.md | 46 -- .../guest-tests-a-loaded-host-can-turn-red.md | 5 +- ...-at-32-cpus-is-down-to-two-milliseconds.md | 55 -- ...-crosses-the-fairness-bound-at-two-cpus.md | 45 -- ...-a-wake-beside-seven-spinners-to-100-us.md | 8 +- ...pmm-state-shares-lines-every-cpu-writes.md | 5 +- .../per-process-fair-split-is-the-policy.md | 125 ---- ...are-measurements-or-make-several-claims.md | 2 +- ...-capability-end-state-is-twelve-answers.md | 2 +- ...the-scheduler-and-the-clocks-never-talk.md | 44 +- kernel/loom/tests/loom_retire.rs | 23 +- kernel/pure/sched/cpu.rs | 559 ++++++++++-------- kernel/pure/sched/fair.rs | 369 ------------ kernel/pure/sched/invariants.rs | 12 +- kernel/pure/sched/job.rs | 62 ++ kernel/pure/sched/job_sched/checkpoint.rs | 31 + kernel/pure/sched/mod.rs | 4 +- kernel/pure/sched/msg.rs | 9 +- kernel/pure/sched/queue.rs | 418 ++++++++----- kernel/pure/sched/task.rs | 53 +- .../old_migrate_kept_the_corpse_i14.trace | 7 +- kernel/sim/corpus/old_steal_port_i8.trace | 3 +- kernel/sim/src/explore.rs | 46 +- kernel/sim/src/invariants.rs | 508 ++++------------ kernel/sim/src/main.rs | 53 +- kernel/sim/src/payload.rs | 3 - kernel/sim/src/scenarios.rs | 355 +++-------- kernel/sim/src/sweep.rs | 75 ++- kernel/sim/src/vm.rs | 222 +++---- kernel/sim/src/workload.rs | 85 ++- kernel/sim/tests/deadline_claim_race.rs | 17 +- kernel/sim/tests/policy.rs | 491 +++++++-------- kernel/sim/tests/scenarios.rs | 294 +++------ kernel/sim/tests/topology.rs | 2 +- kernel/src/arch/aarch64/irqchip.rs | 6 +- kernel/src/arch/x86_64/apic.rs | 2 +- kernel/src/arch/x86_64/idt/spurious.rs | 2 +- kernel/src/arch/x86_64/idt/unclaimed.rs | 2 +- kernel/src/deadline.rs | 2 +- kernel/src/hw.rs | 2 +- kernel/src/loader/mod.rs | 13 +- kernel/src/loader/start.rs | 23 +- kernel/src/object/job.rs | 40 ++ kernel/src/object/mod.rs | 3 + kernel/src/object/ops.rs | 28 +- kernel/src/process.rs | 17 +- kernel/src/quiesce.rs | 2 +- kernel/src/sched/driver.rs | 18 +- kernel/src/sched/kthread.rs | 10 +- kernel/src/sched/payload.rs | 3 - kernel/src/scheduler.rs | 53 +- kernel/src/syscall/dispatch.rs | 10 +- kernel/src/syscall/machine.rs | 12 +- kernel/src/syscall/proc.rs | 24 + lib/manifest/src/lib.rs | 4 + sdk/abi/src/handle.rs | 17 +- sdk/abi/src/syscall.rs | 65 +- sdk/toyos/src/endow.rs | 2 + sdk/toyos/src/job.rs | 38 ++ sdk/toyos/src/launch.rs | 59 +- sdk/toyos/src/lib.rs | 1 + sdk/toyos/src/spawn.rs | 58 +- sdk/toyos/src/syscap.rs | 8 + system/compositor/src/session.rs | 22 +- system/supervisor/src/main.rs | 32 +- tests/checks/metal.rs | 2 +- tests/common/metal.rs | 2 +- .../rust-tests/src/bin/abuse_page_straddle.rs | 24 +- .../guest/rust-tests/src/bin/job_fairness.rs | 131 ++++ .../guest/rust-tests/src/bin/sched_stress.rs | 74 --- tests/images/testcases.toml | 3 +- tests/toyos.rs | 21 + 77 files changed, 2033 insertions(+), 2902 deletions(-) delete mode 100644 issues/fairness-is-per-thread-so-thread-count-wins.md delete mode 100644 issues/granularity-bound-crossed-at-four-widths.md delete mode 100644 issues/i13s-margin-at-32-cpus-is-down-to-two-milliseconds.md delete mode 100644 issues/interactive-mix-crosses-the-fairness-bound-at-two-cpus.md delete mode 100644 issues/per-process-fair-split-is-the-policy.md delete mode 100644 kernel/pure/sched/fair.rs create mode 100644 kernel/pure/sched/job.rs create mode 100644 kernel/src/object/job.rs create mode 100644 sdk/toyos/src/job.rs create mode 100644 tests/guest/rust-tests/src/bin/job_fairness.rs diff --git a/build/kernelkeys.rs b/build/kernelkeys.rs index 6016dead8c..71358a029a 100644 --- a/build/kernelkeys.rs +++ b/build/kernelkeys.rs @@ -43,11 +43,6 @@ pub struct Declared { /// Every hashed container in `kernel/src`, with the origin of its keys. pub const DECLARED: &[Declared] = &[ - Declared { - file: "kernel/src/scheduler.rs", - ty: "HashMap>", - keys: "`process::Pid`, minted by the process table", - }, Declared { file: "kernel/src/process.rs", ty: "HashMap", diff --git a/images/system.toml b/images/system.toml index 3184d0f5eb..9b192900b0 100644 --- a/images/system.toml +++ b/images/system.toml @@ -79,13 +79,15 @@ receives = ["power"] # `screen`. `volume` is soundserver's master level and mute, which the volume # keys set and no other row receives. The display is a virtio-gpu function the # compositor drives, where the machine has one, and the firmware's framebuffer -# otherwise. +# otherwise. `jobs` makes each program it starts a job of its own, with its own +# share of the CPU. [programs.compositor] service = true serves = ["compositor", "prompt", "clipboard", "input", "display"] receives = ["soundserver", "filepicker", "volume"] devices = ["pci:03:80:00/1af4:1050", "framebuffer"] starts = ["terminal", "files", "/apps"] +syscap = ["jobs"] login = true # The i8042, driven from userland: its ports and its two lines are this diff --git a/issues/a-woken-thread-waits-out-a-sibling-spinners-quantum-when-every-cpu-is-busy.md b/issues/a-woken-thread-waits-out-a-sibling-spinners-quantum-when-every-cpu-is-busy.md index 1ba859cd7c..10f17f4675 100644 --- a/issues/a-woken-thread-waits-out-a-sibling-spinners-quantum-when-every-cpu-is-busy.md +++ b/issues/a-woken-thread-waits-out-a-sibling-spinners-quantum-when-every-cpu-is-busy.md @@ -7,8 +7,8 @@ opened: 2026-10-12 # A woken thread waits out a sibling spinner's quantum when every CPU is busy With every CPU running a spinner of the woken thread's own process, a wake that -lands where it cannot be forwarded and cuts nothing waits for the pick, and the -pick serves the fair band's frozen insertion-time keys: a queued sibling whose +lands where it cannot be forwarded and cuts nothing waited for the pick, and the +pick served the fair band's frozen insertion-time keys: a queued sibling whose key went stale when it was queued sits below the woken thread's, and two threads of one process tie at the running one's. With the busy-CPU wake change (PR #874) the PR's metal scout found phase 2's pair together on one CPU on both @@ -32,9 +32,10 @@ there is not in the logs. One boot per tree each: The logs: [first judgement](https://github.com/ToyOSOrg/evidence/tree/cfb4aa7c18348380fcb873e99809db20cf18af22/metal/judgement), [second](https://github.com/ToyOSOrg/evidence/tree/d63b878f86d09c3aefcad27fd30eb3485fa81e5b/r2/metal/judgement). -Owned by stage S4 of the scheduler design (#872), per-thread EEVDF, which -deletes the frozen insertion keys: the orchestrator's application of the -owner's ruling (#885) on PR #874 files it there. +The frozen insertion keys and the protected run are gone: each CPU runs EEVDF +over its job groups and their threads (`kernel/pure/sched/queue.rs`), and a +wake due first cuts the running request at its minimum run, `MIN_SLICE_NS` +(250 µs). What remains is the measurement. **Before #874**, the scout behind it (`wt/toyos-scout-sys` at `fbb958b17`, https://github.com/ToyOSOrg/ToyOS/pull/866#issuecomment-6119438569) timed a @@ -44,10 +45,9 @@ max 6.69 ms; on the AMD bench p50 ≤2.62 ms, p99 ≤7.86 ms, max 7.55 ms. With core idle, such a wake is the fair band's pick to decide. In EEVDF (Stoica et al., "A Proportional Share Resource Allocation Algorithm for Real-Time, Time-Shared Systems", RTSS 1996) a task woken inside a quantum is picked no -sooner than the next quantum's start, and a wake's quantum here is the -protected run, `PROTECTED_RUN_NS` (1 ms). +sooner than the next quantum's start. -**Exit**: S4's re-run of K1–K4 on both benches, which reads phases 2, 3 and 5 +**Exit**: a re-run of K1–K4 on both benches, which reads phases 2, 3 and 5 against the base numbers above; and a metal row on the T14 and the AMD bench -with eight CPU-bound threads on eight CPUs that holds a woken task to run within -one protected run and reports its p99. +with eight CPU-bound threads on eight CPUs that holds a woken task due first +to run within one minimum run and reports its p99. diff --git a/issues/clippy-stage-two-is-lints-one-at-a-time.md b/issues/clippy-stage-two-is-lints-one-at-a-time.md index 9692447e5b..f79e41606e 100644 --- a/issues/clippy-stage-two-is-lints-one-at-a-time.md +++ b/issues/clippy-stage-two-is-lints-one-at-a-time.md @@ -128,7 +128,7 @@ before adopting: | lint | count | what was measured, and what it does not see | |---|---:|---| -| `significant_drop_in_scrutinee` | 6, 1 a defect | Silent in the gate: it reports only a guard marked `#[clippy::has_significant_drop]`, and no kernel guard is. With `sync::LockGuard` alone marked, the kernel under `x86_64-unknown-none` with default features reported 6 sites, and 5 once `sys_chdir` stopped holding the VFS guard into `process_data` — the one defect. The 5 left (`arch/x86_64/idt/mod.rs`'s `for` over `IDT.lock()`, `loader/mod.rs`'s `/system/lib` fallback, `user_ptr::translate_user`, `sys_readdir`, `sys_dlopen`) take no second lock under the guard, and adopting the lint rewrites each. Unmeasured: AArch64, the actuator and instrument shapes, and the guards the measurement did not mark (`SleepGuard`, `block`'s `Locked`, the serial guard). It cannot join `ADOPTED`, which `build/clippy.rs` splices into the host workspace's shapes too, where std's `MutexGuard` is already marked: it would be a per-shape `-W`, as `undocumented_unsafe_blocks` is. **It sees one spelling, not the order**: an `if let Some(x) = L.lock().as_mut()` whose value borrows from the guard is in the tree and in neither report (`file_cache::touch`, `scheduler::remove_vruntime`, `inbox`'s `answer`), and so is any inversion through a named guard, the common shape. | +| `significant_drop_in_scrutinee` | 6, 1 a defect | Silent in the gate: it reports only a guard marked `#[clippy::has_significant_drop]`, and no kernel guard is. With `sync::LockGuard` alone marked, the kernel under `x86_64-unknown-none` with default features reported 6 sites, and 5 once `sys_chdir` stopped holding the VFS guard into `process_data` — the one defect. The 5 left (`arch/x86_64/idt/mod.rs`'s `for` over `IDT.lock()`, `loader/mod.rs`'s `/system/lib` fallback, `user_ptr::translate_user`, `sys_readdir`, `sys_dlopen`) take no second lock under the guard, and adopting the lint rewrites each. Unmeasured: AArch64, the actuator and instrument shapes, and the guards the measurement did not mark (`SleepGuard`, `block`'s `Locked`, the serial guard). It cannot join `ADOPTED`, which `build/clippy.rs` splices into the host workspace's shapes too, where std's `MutexGuard` is already marked: it would be a per-shape `-W`, as `undocumented_unsafe_blocks` is. **It sees one spelling, not the order**: an `if let Some(x) = L.lock().as_mut()` whose value borrows from the guard is in the tree and in neither report (`file_cache::touch`, `inbox`'s `answer`), and so is any inversion through a named guard, the common shape. | ## `undocumented_unsafe_blocks`, per area diff --git a/issues/fairness-is-per-thread-so-thread-count-wins.md b/issues/fairness-is-per-thread-so-thread-count-wins.md deleted file mode 100644 index acd38f5570..0000000000 --- a/issues/fairness-is-per-thread-so-thread-count-wins.md +++ /dev/null @@ -1,31 +0,0 @@ ---- -status: open -kind: track -opened: 2026-08-16 ---- - -# Fairness is per-thread, so thread count wins - -Queued track from the 2026-08-16 owner conversation on scheduler direction. - -**The premise this was opened on has been answered one level up; the remaining -work is the rest of the hierarchy.** Fair scheduling no longer shares the CPU -between *threads*. All threads of one process share a vruntime, so a second -runnable thread bumps a refcount and buys no second slice -(`kernel/pure/sched/fair.rs:95-99`), and the state it advances is one `KShare` -per `Pid` (`kernel/src/scheduler.rs:249`, minted at `:257-268`). A process with -100 threads no longer out-schedules one with 8 by showing up more. - -What is unbuilt is everything *above* the process: shares are one flat level -per `Pid` with no grouping over them. Production schedulers make shares -hierarchical: groups first, processes and their threads within their group. - -ToyOS has a better grouping handle than Unix's cgroups bolt-on ever was: the -capability domain. A process's share can hang off what its parent endowed it -with, in the same `system.toml` that endows everything else — hierarchical -reservations, since reservations nest naturally. The reservation model is the -substrate; this issue is the hierarchy on top of it. - -Blocked on: the reservations implementation. Design question to settle at pick-up: -whether the hierarchy's nodes are processes, endowment parents, or named domains -in `system.toml`, and what the default share of an unconfigured process is. diff --git a/issues/granularity-bound-crossed-at-four-widths.md b/issues/granularity-bound-crossed-at-four-widths.md deleted file mode 100644 index 1015c60217..0000000000 --- a/issues/granularity-bound-crossed-at-four-widths.md +++ /dev/null @@ -1,46 +0,0 @@ ---- -status: open -kind: defect -opened: 2026-08-01 ---- - -# The scheduler crosses its own derived granularity bound at four of ten widths - -Distinct from `per-process-fair-split-is-the-policy`, and deliberately not merged -with it. That one says fairness degrades as the machine widens. **This one says -the shipped scheduler exceeds a limit its own design implies** — a different and -sharper statement. - -The bound is derived from granularities the policy itself picked: -`lag_spread + (ΣT_i + 1) × (QUANTUM + max KernelSection + 2 × RUN_CHUNK)` -(`kernel/sim/src/invariants.rs`). - -**Re-measured 2026-08-24 at `739af0c2`**, the whole sweep, 500 seeds a width: - -``` -cargo run --release -p toyos-sched-sim -- measure fairness_storm: 500 -``` - -(The package is `kernel-sim` now.) - -| CPUs | 1 | 2 | 3 | 4 | 6 | 8 | 12 | 16 | 24 | 32 | -|---|---|---|---|---|---|---|---|---|---|---| -| I5 worst spread (ms) | 30 | 84 | 125 | 198 | 324 | 418 | 634 | 720 | 1056 | 1386 | -| crossing (ms) | — | — | — | **112** | **324** | **418** | **634** | — | — | — | - -Still four widths and still the same four. The crossing row is -`Outcome::fair_over_bound`: the widest spread seen in a window where `spread` -exceeded the **derived** bound, which is not the same quantity as the amount by -which it exceeded it, and not the same quantity as the worst spread either — -at 4 CPUs the worst window is 198 ms and the worst *crossing* window is 112 ms. -The entry read "crossed ... by 116, 324, 418 and 634 ms" and that was wrong in -kind as well as in one figure; the 4-CPU number is 112 ms today. - -**The gate handles this honestly rather than hiding it**, which is the part worth -preserving. It reds on `max(derived, recorded allowance)` -(`invariants.rs`), so a sampled scenario is gated on not regressing — but -`fair_over_bound` records every crossing of the *derived* bound regardless, and -the sweep prints `N ns PAST THE DERIVED BOUND on the recorded allowance` -(`past_the_bound`, `sweep.rs`). **The allowance cannot quietly become the standard**, which -is the failure mode of every temporary baseline and the reason most of them end -up permanent. diff --git a/issues/guest-tests-a-loaded-host-can-turn-red.md b/issues/guest-tests-a-loaded-host-can-turn-red.md index 64bde05b32..a06cdecfb8 100644 --- a/issues/guest-tests-a-loaded-host-can-turn-red.md +++ b/issues/guest-tests-a-loaded-host-can-turn-red.md @@ -16,9 +16,8 @@ measurement owed. The one observed red of this shape, `virtio_sound_counts`, is halted or reset within 2 s. - `kill_ends_every_wait` (`kill_ends_every_wait.rs`, `HELD` 1 s): a kill that takes more than 1 s of starved wall time to land reads as "the child outlived" it. A timing claim. -- `sched_stress` (`sched_stress.rs`): the post-wake vruntime lag held to ±50 ms after a 10 ms - sleep, where vCPU time the host takes is charged as the thread's runtime. Its connect-storm pace - is `issues/timing-verdicts-ruled-off-qemu-have-no-metal-arm.md`'s. +- `sched_stress` (`sched_stress.rs`): its connect-storm pace is + `issues/timing-verdicts-ruled-off-qemu-have-no-metal-arm.md`'s. - `abuse_thread_table` (`abuse_thread_table.rs`, `RUN_BOUND` 10 s): each thread runs within 10 s. - `netstack_socket_churn` (`netstack_socket_churn.rs`, `LET_GO` 20 s), `netstack_streams` and `netstack_streams_e1000e` (`netstack_streams.rs`, `WOKEN` 60 s, and the harness's 30 s diff --git a/issues/i13s-margin-at-32-cpus-is-down-to-two-milliseconds.md b/issues/i13s-margin-at-32-cpus-is-down-to-two-milliseconds.md deleted file mode 100644 index 864c338eae..0000000000 --- a/issues/i13s-margin-at-32-cpus-is-down-to-two-milliseconds.md +++ /dev/null @@ -1,55 +0,0 @@ ---- -status: open -kind: defect -opened: 2026-08-24 ---- - -# I13's margin at 32 CPUs has closed from 10 ms to 2 ms since it was recorded - -Found while re-running the fairness sweep for -`issues/granularity-bound-crossed-at-four-widths.md`, not looked for. - -`per-process-fair-split-is-the-policy` records the per-thread split as the half -that does *not* degrade with width, and prices its own confidence on that: -entry criterion 3 says "**The margin at 32 CPUs is 1.2× and trending up** — -10 ms at one CPU to 50 ms at 32 against a 60 ms bound — with nothing measured -above 32, while the scheduler's own staged target is 1–128." - -The same command on this tree at `739af0c2`, 500 seeds a width: - -``` -cargo run --release -p toyos-sched-sim -- measure fairness_storm: 500 -``` - -| CPUs | 1 | 2 | 3 | 4 | 6 | 8 | 12 | 16 | 24 | 32 | -|---|---|---|---|---|---|---|---|---|---|---| -| I13 worst, recorded | 10 | 30 | 28 | 28 | 31 | 32 | 35 | 37 | 42 | 50 | -| I13 worst, 2026-08-24 | 10 | 28 | 28 | 28 | 31 | 32 | 35 | 38 | 43 | **58** | - -Every width but the last moves by at most 2 ms, and eight of ten move by at most -1 ms. The widest moves by 8, from 50 to 58 against the 60 ms bound — 1.20× margin -to 1.03×. The run is `clean`: `invariant_i13_is_measured_and_holds` still passes, -and I5's whole row reproduces the recorded numbers exactly at nine of ten widths, -which is what says the two runs are the same measurement rather than two -configurations. - -**Why this is worth a file rather than a shrug.** The number is the entry -criterion for the per-share-FIFO redesign, and that criterion is about *whether -the answer can be trusted at width*, not about whether the gate is green. At -1.03× the next width up is where the bound is crossed, and 64 and 128 are inside -the scheduler's staged target and have never been measured. It is also possible -this is seed noise at the width where the sweep does the least work per seed -(I13's reach there is 37%, the lowest of the ten) — that is the first thing to -establish, by re-running 32 at a larger seed count and by measuring 64. - -Not measured: whether the move is a change in the tree or in the sample. Nobody -has bisected it, and this entry does not claim a regression. - -**2026-08-25, promoted to `defect`.** The number is the stated entry criterion -for the per-share-FIFO redesign, and 1.03× against a 60 ms bound is not a margin -anyone may enter on — that makes it work owed rather than something noticed. -Two runs answer it and both are cheap: `fairness_storm:32` at a seed count well -above 500, which decides whether 58 is the tree or the sample, and -`fairness_storm:64`, which is inside the scheduler's staged 1–128 target and has -never been run. Owed by whoever picks up the per-share-FIFO work, before -starting it rather than after. diff --git a/issues/interactive-mix-crosses-the-fairness-bound-at-two-cpus.md b/issues/interactive-mix-crosses-the-fairness-bound-at-two-cpus.md deleted file mode 100644 index 60c2f4ecec..0000000000 --- a/issues/interactive-mix-crosses-the-fairness-bound-at-two-cpus.md +++ /dev/null @@ -1,45 +0,0 @@ ---- -status: open -kind: defect -opened: 2026-10-12 ---- - -# `interactive_mix` at two CPUs and sixteen hogs crosses invariant I5's derived bound - -`kernel/sim`'s invariant I5 bounds the service spread between two equal shares -over one contention window by -`lag_spread + (ΣT + 1) × (QUANTUM + max KernelSection + 2 × RUN_CHUNK)` -(`kernel/sim/src/invariants.rs`, `check_fairness`). `interactive_mix(2, 16)` — -one sleeper thread against a sixteen-thread hog process on two CPUs — crosses -it on `main` as well as with the busy-CPU wake change (PR #874). - -Measured with I5's ceiling lifted (`with_fair_allowance(u64::MAX)`), seeds -0..2000 alternating the seeded and PCT streams as `sim/tests/policy.rs` does, -by a scratch test ([at `6e323ccf0`](https://github.com/ToyOSOrg/evidence/blob/b036b48d71bb81456b770307ccbd9532d80eb175/sim-investigation/scratch-tests/zz_scratch3.rs), -[its output](https://github.com/ToyOSOrg/evidence/blob/b036b48d71bb81456b770307ccbd9532d80eb175/sim-investigation/i5-2000.txt); -[again at `17c40459d`](https://github.com/ToyOSOrg/evidence/blob/d6d8e852b6e7e249e45d6218c252583ee624b03c/r3/sim/zz_scratch.rs), [its output](https://github.com/ToyOSOrg/evidence/blob/d6d8e852b6e7e249e45d6218c252583ee624b03c/r3/sim/scratch-at-head.log)): - -| tree | runs crossing | worst spread | bound in force there | -|---|---|---|---| -| `main` at `177e72a9e` | 90 / 2000 | 250 ms | 216 ms | -| PR #874 at `6e323ccf0` | 196 / 2000 | 254 ms | 217 ms | -| PR #874 at `17c40459d` | 197 / 2000 | 254 ms | 217 ms | - -The policy suite's forty seeds miss every crossing on `main`, and two of them -(24 and 36) cross with the change, which is why `scenarios::interactive_mix` -now carries a recorded allowance at that shape (`INTERACTIVE_I5_SAMPLE`, the -worst spread plus a quarter, as `FAIRNESS_SAMPLE` does). `Outcome::fair_over_bound` -still reports every crossing of the derived bound. - -What a crossing run shows (`interactive_mix(2, 16)`, seed 24, from a step -timeline): the sleeper's first wake waits behind the eight hogs on its CPU, -whose keys are still the spawn's zero, and each later quantum end on that CPU -hands it to a hog whose key went stale when it was queued, while the sleeper -counts as runnable throughout. The spread is the fair band's insertion-time -keys, the same staleness the bound's `ΣT + 1` term carries, past that term. - -Owned by stage S4 of the scheduler design (#872), per-thread EEVDF, which -deletes the frozen insertion keys and I5's staleness term with them. - -**Exit**: `interactive_mix(2, 16)` meets I5's derived bound over 2,000 seeds, -and `INTERACTIVE_I5_SAMPLE` is deleted. diff --git a/issues/no-row-holds-a-wake-beside-seven-spinners-to-100-us.md b/issues/no-row-holds-a-wake-beside-seven-spinners-to-100-us.md index 69e3a34419..2b4a6558ca 100644 --- a/issues/no-row-holds-a-wake-beside-seven-spinners-to-100-us.md +++ b/issues/no-row-holds-a-wake-beside-seven-spinners-to-100-us.md @@ -11,10 +11,10 @@ could take it. PR #874 (`347e3313d`) changed that. An ordinary wake's doorbell now kicks a busy target on its edge (`Doorbell::ring`, `kernel/pure/sched/mailbox.rs`), and at the drain `place` (`kernel/pure/sched/cpu.rs`) forwards a fair task that would queue behind a -running fair task to an idle CPU. A task that stays with a strictly lower key -cuts the running task's quantum to end `PROTECTED_RUN_NS` (1 ms) after its -dispatch, and one queued task counts as surplus to a steal -(`PUSH_THRESHOLD` = 1). +running fair task to an idle CPU. A task that stays and is due first cuts the +running request at its minimum run, `MIN_SLICE_NS` (250 µs) after its dispatch +(`kernel/pure/sched/queue.rs`), and one queued task counts as surplus to a +steal (`PUSH_THRESHOLD` = 1). **What #874 measured.** The metal row `wake_onto_busy` (`tests/toyos.rs`) times a wake onto a CPU that one spinner holds, beside CPUs with nothing to diff --git a/issues/per-cpu-scheduler-and-pmm-state-shares-lines-every-cpu-writes.md b/issues/per-cpu-scheduler-and-pmm-state-shares-lines-every-cpu-writes.md index e59926896a..687ccac00b 100644 --- a/issues/per-cpu-scheduler-and-pmm-state-shares-lines-every-cpu-writes.md +++ b/issues/per-cpu-scheduler-and-pmm-state-shares-lines-every-cpu-writes.md @@ -20,8 +20,9 @@ every other CPU writes too: so two CPUs' handles share a line. Each pass stores its own `load`, `surplus` and `last_pass` (`cpu.rs:1669-1672`), and every wake does a read-modify-write on its target's doorbell (`kernel/pure/sched/mailbox.rs:475`). -- **`Frontier`.** One global `AtomicU64`, `fetch_max`ed on every pick - (`kernel/pure/sched/fair.rs:44-46`, called at `cpu.rs:1827`). +- **A job's runnable count.** One `AtomicU32` per job + (`kernel/pure/sched/job.rs`), changed by each CPU its threads enter or + leave a fair band on, and read at each of those CPUs' decisions. **Measured on the T14** (`wt/toyos-scout-clock` at `fa6a2a829`, the `internet_download` row's twelve download windows over two boots; diff --git a/issues/per-process-fair-split-is-the-policy.md b/issues/per-process-fair-split-is-the-policy.md deleted file mode 100644 index 77524dee58..0000000000 --- a/issues/per-process-fair-split-is-the-policy.md +++ /dev/null @@ -1,125 +0,0 @@ ---- -status: none -kind: rejected -opened: 2026-08-01 ---- - -# The scheduler's *per-process* fair split degrades as the machine widens — settled: it is the policy - -Worst service spread against the derived bound, in ms, from -`measure fairness_storm: 500`: - -| CPUs | 1 | 2 | 3 | 4 | 6 | 8 | 12 | 16 | 24 | 32 | -|---|---|---|---|---|---|---|---|---|---|---| -| worst | 30 | 84 | 125 | **198** | **324** | **418** | **634** | 720 | 1056 | 1386 | -| bound | 60 | 108 | 156 | 204 | 300 | 396 | 588 | 780 | 1164 | 1548 | - -**Per-process only, and that is a real bound on the defect.** The *per-thread* split does -not degrade with width: measured 10 ms at 1 CPU to 50 ms at 32, against a 60 ms derived -bound — inside its bound at every width — over the same runs where I5 went 30 → 1386. So -threads of a process are shared out fairly among themselves at any machine size; it is the -split *between processes* that widens. The fix has a smaller target than "fairness degrades" -implies. - -**Both questions the earlier filing left open are now measured, not argued.** - -**Offset, not drift.** Holding the seed count and scaling the storm's per-thread -work: one CPU stays at 30 ms at every window length, while eight go 362 → 602 → -548 ms as the window doubles twice. It saturates rather than accumulating. - -**Policy, not model.** Everything deciding who runs next is the shipped core — -`RunQueue`'s insertion-time keys, `FairShare`'s one vruntime pot per process, -`CpuSched::pick`, `answer_steal_requests`' surplus rule. The simulator mocks -time, timer, IPI, halt and switch: the parts that decide *when*, not *who*. - -**The mechanism, which is why this is a design consequence and not an -implementation bug.** Every running thread of a process charges one pot, so the -pot advances at the process's *aggregate* rate while each queued thread's key -stays frozen at its insertion. One dispatch of staleness therefore buys more -wall-clock service the more of that process runs at once. That is why it scales -with width, and why careful coding cannot close it — the fix is a policy change. - -**Caveat, and it is load-bearing.** These are worst-of-N over adversarially -chosen interleavings, seeded and PCT — not the split hardware would show on an -average schedule. **The mechanism and the scaling are the policy's; the magnitude -is a worst case.** Do not quote these numbers as expected behaviour. - -**Connected to the queue's tie-break, and that is why this is hard.** Threads of -a process sharing one vruntime is *why* the insertion sequence exists -(`queue.rs:18-22`). The degradation here and sibling starvation are two faces of -one decision: **per-process accounting with per-thread queueing.** Anything that -fixes one has to answer for the other. - -**But only the per-process face degrades, and that is now measured.** Simulator -invariant I13 measures service per *thread* inside a share over the same -contention windows, narrowed to intervals where every CPU carries the same -number of each member's runnable threads (otherwise the number is placement, not -ordering). From `measure fairness_storm:`, against a derived bound of -60 ms at every width — `(rivals + 1) × (QUANTUM + max KernelSection + -2 × RUN_CHUNK)`, five dispatches of one run queue's fair band, with **no lag -term** because a share holds one vruntime and one lag for all its threads: - -| CPUs | 1 | 2 | 3 | 4 | 6 | 8 | 12 | 16 | 24 | 32 | -|---|---|---|---|---|---|---|---|---|---|---| -| I13 worst | 10 | 30 | 28 | 28 | 31 | 32 | 35 | 37 | 42 | 50 | -| I5 worst | 30 | 102 | 125 | 198 | 324 | 418 | 634 | 612 | 1046 | 1386 | - -Flat where the per-process split runs away. **And the tie-break is not what -keeps it flat** — the pot is charged for every nanosecond any thread of the -share runs, so a re-inserted thread already carries a key strictly above every -sibling queued before it and the band serves them in insertion order whatever -the tie-break is. `(vruntime, TaskKey)` ported literally -(`scenarios::fair_identity_tiebreak`) is invisible to I13, which is why the -negative gate had to be the stronger `fair_identity_within_share`. **The -consequence for the fix**: a redesign replacing per-thread queue keys with an -ordered map of shares each holding a FIFO of its ready threads takes the -ordering job *away* from the pot and hands it to that FIFO, so this face stops -being benign the moment the fix lands. I13 is the gate that says so; it is green -today and its own gate is red on the broken shape, on I13 alone — I5 reports a -perfectly even split while two of three sibling threads never run. - -**Entry criteria for the per-share-FIFO redesign.** I5 and I13 together are -close to sufficient and are not sufficient. Three gaps, all prerequisites rather -than follow-ups, and the first is *the* one — the other two are conditions on -trusting the answer, this one is a hole where the answer would be. - -1. **The redesign's most novel path has no coverage in the workload class that - exercises it.** Where a woken thread lands in its share's order falls out of - the pot today; after the redesign it is decided by the FIFO push, which *is* - the new code. Nothing measures it. A block drops a thread from I13's measured - set, so I13's reach inverts exactly against the workloads that would exercise - it — 96–99% on the fairness storms, where nothing blocks, against - `double_drop_exit_race` 37%, `rt_wake_latency` 29%, `fork_storm` 9%, - `futex_storm` 5% and `audio_pipeline` **0%**. **I13 would stay green straight - through a redesign that got the wake path's ordering wrong**, and it is the - check that nominally guards fairness. A wake-heavy workload with windows long - enough to measure does not exist and has to be built first. -2. **I13's reach is a silent casualty of the change it guards.** Its window - closes when a member's threads stop being evenly spread over the CPUs, and - the redesign must reimplement `pop_surplus`, which feeds - `answer_steal_requests` and can therefore change placement — so a redesign - that disturbs placement makes I13 measure *less* rather than fail, with the - sweep still printing `clean`. Instrumented rather than left as vigilance: - `SweepResult::thread_coverage_pct` publishes the fraction of executed time - I13 had a comparison open for, `invariant_i13_is_measured_and_holds` gates on - it against 96% / 69% / 99%, and forcing the balance condition false takes it - to 0% and reds the test. **A/B that number across the redesign; a collapse is - as loud as a violation.** This is the gate that goes quiet — the change under - test narrowing the gate's own coverage rather than violating it — and this - project's history has more than one of those in it. -3. **The margin at 32 CPUs is 1.2× and trending up** — 10 ms at one CPU to - 50 ms at 32 against a 60 ms bound — with nothing measured above 32, while - the scheduler's own staged target is 1–128. Measure 64 and 128 first, or a - red at high width cannot be attributed to the redesign rather than to the - width. - Compounded by the reach falling with width for an unrelated reason — 55% at - four CPUs, 45% at eight, because threads exit at slightly different moments - and unbalance a wide machine sooner. **At the widths that target reaches, I13 - certifies less than half the run**, which is a limit on the invariant and not - a defect in it. - -Found only because I5 measures *service* — nanoseconds actually delivered — rather -than checking vruntime bookkeeping against itself, which would have been true by -construction. The dead-gate lesson from the other side: the first question about a -gate is not whether it passes, but whether it measures the quantity you care -about. diff --git a/issues/suite-rows-that-are-measurements-or-make-several-claims.md b/issues/suite-rows-that-are-measurements-or-make-several-claims.md index 44c3ae1be5..ee514e49ef 100644 --- a/issues/suite-rows-that-are-measurements-or-make-several-claims.md +++ b/issues/suite-rows-that-are-measurements-or-make-several-claims.md @@ -31,7 +31,7 @@ Guest binaries in `tests/guest/rust-tests/src/bin/` that make several claims: - `allocator_stress.rs`: twelve checks. - `std_process.rs`: eight std process behaviours. - `std_sync.rs`: mutex, condvar, park and rwlock. -- `sched_stress.rs`: io_uring acceptor isolation, the post-wake vruntime lag, and a connect storm. +- `sched_stress.rs`: io_uring acceptor isolation and a connect storm. - `std_fs.rs`: listing, a self-read, NotFound, and file types. Owner: the orchestrator, which dispatches the reshape. diff --git a/issues/the-capability-end-state-is-twelve-answers.md b/issues/the-capability-end-state-is-twelve-answers.md index 34e61f7f9c..2891267dfb 100644 --- a/issues/the-capability-end-state-is-twelve-answers.md +++ b/issues/the-capability-end-state-is-twelve-answers.md @@ -145,7 +145,7 @@ table** can still reach: - **its own address space** — `SYS_MMAP`, `SYS_MUNMAP`, `SYS_TLS_ALLOC_BLOCK`, `SYS_STACK_INFO`; - **its own record** — `SYS_GETPID`, `SYS_GET_ENV`, `SYS_ENDOWMENTS`, - `SYS_QUERY_MODULES`, `SYS_SCHED_INFO`, `SYS_GETCWD`, `SYS_CHDIR`; + `SYS_QUERY_MODULES`, `SYS_GETCWD`, `SYS_CHDIR`; - **machine facts** — `SYS_CLOCK_EPOCH`, `SYS_CPU_COUNT`, `SYS_RANDOM`, and `SYS_SYSINFO`'s header (total and used memory, the CPU count, the live-thread count, the uptime and the two CPU-time accumulators) but not the roster after diff --git a/issues/the-scheduler-and-the-clocks-never-talk.md b/issues/the-scheduler-and-the-clocks-never-talk.md index 6b16275632..37158f3b5b 100644 --- a/issues/the-scheduler-and-the-clocks-never-talk.md +++ b/issues/the-scheduler-and-the-clocks-never-talk.md @@ -71,23 +71,27 @@ scales with crowding. These are model figures for the scenarios that ran, not hardware latency guarantees. Stage 3 validates the kernel implementation on the supported machines. -Stage 3 owes these decision points. A CPU decides again when its running -request ends, and when a queued group whose request is due before the running -group's becomes eligible, once the running thread has run `min_slice`. -Without the second, a job whose request is due first but carries a little -debt waits out every longer request ahead of it. In the model's desktop mix -beside a build, the compositor then finished a frame 9.4 ms after its release -and missed it; with the second, it misses none on the T14 and AMD shapes, -finishing within 1.7 ms. The kernel must re-arm each CPU's preemption timer -for the earliest such instant on every change to that CPU's queues, not only -at its own decisions: an enqueue, a dequeue (a last thread blocking, or -another CPU stealing or migrating a thread away) and a decision. A group in -debt that leaves shifts `V` and makes the others eligible sooner, and a remote -dequeue is no decision on the CPU that loses it; armed only there, the timer -fires late. The query is O(log g) for g groups: the deadline-ordered group -tree, augmented with each subtree's least virtual time, gives the earliest -eligibility among groups due before the running one. The model's cost is -226 to 293 switches per CPU per second in that mix, against 216 to 259 -without. Within a group the group's own request, as short as its shortest -thread's, bounds the wait; deciding there as well cost 361 to 417 switches -and improved no latency. +Stage 3 is in the kernel. A job is a kernel object (`kernel/pure/sched/job.rs`, +`kernel/src/object/job.rs`) a process's threads and its children share unless a +spawn endows another under `JOB_LABEL`, and a launch carries its caller's; only a system capability's `JOBS` right +makes one, which the supervisor makes each service with and the compositor +each program it starts. Each CPU +runs the two EEVDF levels (`kernel/pure/sched/queue.rs`) over a deadline-ordered +tree augmented with each subtree's least virtual time, and decides again at a +request's end, at an earlier-due group's eligibility, and at a wake due first — +the last two once the running thread has run `MIN_SLICE_NS`. Every change to a +CPU's queues is made in that CPU's own pass, and every pass ends by arming the +timer for the earliest of these, so a remote steal re-arms it as a wake does. +Without the eligibility decision a job whose request is due first but carries a +little debt waits out every longer request ahead of it: in the model's desktop +mix beside a build the compositor then missed frames, finishing one 9.4 ms after +its release; with it, none on the T14 and AMD shapes. Busy CPUs balance summed +group weight, one request per CPU per `BALANCE_NS`. A voluntary yield ends the +yielder's request at both levels. + +Stage 3 leaves three things. Explicit latency requests have no syscall: the +queue takes one request size for every thread until a first caller (the +compositor's or the audio worker's) needs a shorter one. `sshserver` starts its +sessions as its own helpers, since it has no ToyOS dependency to make a job +with. std and libc start a child in a new job only by `CommandExt::endow` under +`JOB_LABEL`. diff --git a/kernel/loom/tests/loom_retire.rs b/kernel/loom/tests/loom_retire.rs index b80d512d58..263d494477 100644 --- a/kernel/loom/tests/loom_retire.rs +++ b/kernel/loom/tests/loom_retire.rs @@ -15,11 +15,11 @@ use std::sync::atomic::{AtomicBool, Ordering as StdOrdering}; use loom::sync::Arc; use kernel_loom::sched::cpu::{Balance, CpuHandle, CpuHandles, CpuSched, Env, RunToken, SchedPass}; use kernel_loom::sched::topology::Topology; -use kernel_loom::sched::fair::{FairShare, Frontier, ShareState}; use kernel_loom::sched::hw::{CpuId, Hw, Kicker, Machine, Nanos, TraceEvent}; +use kernel_loom::sched::job::{Job, JobKey}; use kernel_loom::sched::mailbox::{mailbox, MailboxConsumer, Urgency}; use kernel_loom::sched_model::{ - model, IrqGuard, Kicks, LoomLock, Msg, PreemptModel, RemoteGuard, CPU0, CPU1, + model, IrqGuard, Kicks, Msg, PreemptModel, RemoteGuard, CPU0, CPU1, }; use kernel_loom::sched::retire; use kernel_loom::sched::task::{ @@ -333,13 +333,12 @@ fn a_retire_and_a_wake_never_both_claim_a_parked_task() { // The sixth model runs a whole `CpuSched`, so it needs the three things a pass // is built against: a payload, a hardware surface, and the shared half of a CPU. -/// The smallest payload a `CpuSched` can be built over: no address space, no -/// saved context, and loom's mutex as the per-process share cell. +/// The smallest payload a `CpuSched` can be built over: no address space and +/// no saved context. struct Payload; impl SchedPayload for Payload { type Ctx = (); - type ShareLock = LoomLock; fn name(&self) -> u64 { 0 @@ -387,12 +386,11 @@ impl Hw for Silent { struct Owner { cpus: CpuHandles, hw: Silent, - frontier: Frontier, } -/// The instant every pass in the model runs at. Constant on purpose: a quantum -/// that never expires keeps `preempt_if_due` out of the interleavings, which -/// have nothing to do with it. +/// The instant every pass in the model runs at. Constant on purpose: a request +/// that never ends keeps `preempt_if_due` out of the interleavings, which have +/// nothing to do with it. const NOW: Nanos = Nanos(1_000); /// The task the retire and the wake race for. const KEY: TaskKey = TaskKey(1); @@ -438,7 +436,6 @@ fn the_retire_arm_never_loses_a_parked_task_to_a_racing_wake() { let owner = Arc::new(Owner { cpus: CpuHandles::new(vec![CpuHandle::new(CPU0, tx)], Topology::flat(1)), hw: Silent, - frontier: Frontier::new(), }); let mut cpu = CpuSched::::new(CPU0, rx, ()); // A pass runs from the IRQ-exit path, which cannot be preempted, so @@ -450,7 +447,6 @@ fn the_retire_arm_never_loses_a_parked_task_to_a_racing_wake() { let env = Env { hw: &owner.hw, cpus: &owner.cpus, - frontier: &owner.frontier, preempt: &guard, balance: Balance::None, }; @@ -459,12 +455,9 @@ fn the_retire_arm_never_loses_a_parked_task_to_a_racing_wake() { }; // Spawn placement is a message, never a reach into the queue. let spawn = |key: TaskKey| { - let share = Arc::new(FairShare::new(LoomLock::new(ShareState::NonRunnable { - lag: 0, - }))); let task = TaskBuilder { key, - share, + job: Arc::new(Job::new(JobKey(key.0))), ctx: (), ext: Payload, rt: RtState::default(), diff --git a/kernel/pure/sched/cpu.rs b/kernel/pure/sched/cpu.rs index b23fccef67..c0273b80a5 100644 --- a/kernel/pure/sched/cpu.rs +++ b/kernel/pure/sched/cpu.rs @@ -23,18 +23,17 @@ use alloc::collections::{BTreeMap, VecDeque}; use alloc::vec::Vec; use core::marker::PhantomData; -use crate::sched::fair::{Frontier, QUANTUM_NS}; use crate::sched::hw::{CpuId, Hw, Nanos, TraceEvent, TraceKind}; use crate::sched::mailbox::{ Doorbell, Kick, MailboxConsumer, MailboxNode, MailboxProducer, PostSlot, PreemptGuard, Quiesced, SchedMsg, SleepArm, Urgency, }; use crate::sched::msg::Msg; -use crate::sched::queue::RunQueue; +use crate::sched::queue::{RunQueue, MIN_SLICE_NS}; use crate::sched::sync::{fence, Arc, AtomicU32, AtomicU64, Ordering}; use crate::sched::task::{ BlockedTask, Claim, DeadTask, ReadyTask, RunningTask, SchedPayload, TaskKey, TaskShared, - TaskState, TransitTask, WaitClass, WakeCause, WakeReason, + TaskState, TransitTask, WaitClass, WakeCause, WakeReason, QUANTUM_NS, }; use crate::sched::timer::{TimerApplied, TimerPlan}; use crate::sched::topology::{Level, Topology}; @@ -237,9 +236,8 @@ pub struct CpuSched { /// value in exactly one container, and a stopped task still owns a kernel /// stack and an address space that only a reset will release. /// - /// Counted into its share exactly as [`Self::dying`] is, because the state - /// word of everything in here reads `Ready`; excluded from the published - /// load, because nothing queues behind work that will never run. + /// In no fair band, as [`Self::dying`]'s tasks are not; excluded from the + /// published load, because nothing queues behind work that will never run. stopped: VecDeque>, /// The task that exited on this CPU, freed by the NEXT pass — a pass /// cannot free the stack it is running on. @@ -252,15 +250,26 @@ pub struct CpuSched { /// Thieves that asked this pass; answered in `finish` from surplus, after /// the pick, so answering can never hand away the task we were about to /// run. - steal_requests: Vec, - quantum_end: Nanos, - /// When the running task was dispatched: where [`PROTECTED_RUN_NS`] is - /// counted from. Its own field because a cut moves - /// [`Self::quantum_end`], which then no longer says when the quantum began. + /// Each asker, with the summed group weight it asked at: zero for an + /// idle one. + steal_requests: Vec<(CpuId, u64)>, + /// When this CPU next decides for its running task, absent other events: + /// a real-time or dying task's turn end as its dispatch set it, and a fair + /// thread's [`RunQueue::decision`] as each pass recomputes it. + decide_at: Nanos, + /// A queued fair thread the band would run in place of the running one, + /// which takes the CPU at this instant: [`MIN_SLICE_NS`] after the running + /// thread's dispatch, or at once if it has run that long. + cut: Option, + /// When the running task began its current run here: where + /// [`MIN_SLICE_NS`] is counted from. dispatched_at: Nanos, + /// When this CPU next looks for a busier CPU to take a thread from while + /// it runs fair work ([`SchedPass::balance_busy`]). + next_balance: Nanos, /// The running task is a corpse that was dispatched **ahead of a ready /// real-time task** because it had aged, and holds the CPU until - /// [`Self::quantum_end`] — one [`DYING_CHUNK_NS`] — against that band. + /// [`Self::decide_at`] — one [`DYING_CHUNK_NS`] — against that band. /// /// Without this the grant is worth nothing: `preempt_if_due`'s RT arm fires /// at the *next pass*, not at the chunk boundary, so any interrupt landing @@ -332,8 +341,10 @@ impl CpuSched { mailbox, steal_probe: MailboxNode::new(), steal_requests: Vec::new(), - quantum_end: Nanos::ZERO, + decide_at: Nanos::ZERO, + cut: None, dispatched_at: Nanos::ZERO, + next_balance: Nanos::ZERO, aged_grant: false, idle_probes_spent: 0, cursor: 0, @@ -398,8 +409,9 @@ impl CpuSched { self.armed } - pub fn quantum_end(&self) -> Nanos { - self.quantum_end + /// When this CPU next decides for its running task, absent other events. + pub fn decide_at(&self) -> Nanos { + self.decide_at } /// Is this CPU on its idle context? Only then may it halt. @@ -438,19 +450,38 @@ impl CpuSched { /// Lend the running task an RT window: the path for a client /// that was *not* blocked when its producer signalled, and so takes the /// boost at its own consume point instead of through a wake cause. - pub fn boost_current(&mut self, until: Nanos) { + pub fn boost_current(&mut self, now: Nanos, until: Nanos) { if let Some(current) = self.running.as_mut() { current.boost(until); } + self.leave_fair_for_rt(now); } /// `SYS_RT_ENTER` on the running task — permanent RT, as opposed to the /// bounded window a waker lends. The privilege gate lives at the syscall /// layer. - pub fn set_current_rt(&mut self, permanent: bool) { + pub fn set_current_rt(&mut self, now: Nanos, permanent: bool) { if let Some(current) = self.running.as_mut() { current.set_permanent_rt(permanent); } + self.leave_fair_for_rt(now); + } + + /// A running fair thread that has just become real-time leaves the fair + /// band charged up to `now`, and its real-time turn starts now: one lend + /// buys at most one quantum of running at the borrowed priority. + fn leave_fair_for_rt(&mut self, now: Nanos) { + let Some(current) = self.running.as_mut() else { + return; + }; + if !current.is_rt() || !self.rq.running_is_fair() { + return; + } + let ns = current.charge(now); + self.rq.charge_running(ns); + self.rq.leave_running(current.key(), current.standing_mut()); + self.decide_at = now.after(QUANTUM_NS); + self.cut = None; } } @@ -524,7 +555,7 @@ impl CpuSched { &[ (core::mem::offset_of!(Self, id), "id"), (core::mem::offset_of!(Self, running), "running"), - (core::mem::offset_of!(Self, rq), "rq (the ready band: rt deque, fair map, insert_seq)"), + (core::mem::offset_of!(Self, rq), "rq (the ready bands: rt deque, job groups)"), (core::mem::offset_of!(Self, parked), "parked (the park map)"), (core::mem::offset_of!(Self, dying), "dying"), (core::mem::offset_of!(Self, stopped), "stopped"), @@ -532,8 +563,10 @@ impl CpuSched { (core::mem::offset_of!(Self, mailbox), "mailbox"), (core::mem::offset_of!(Self, steal_probe), "steal_probe (excluded: a sibling writes it)"), (core::mem::offset_of!(Self, steal_requests), "steal_requests"), - (core::mem::offset_of!(Self, quantum_end), "quantum_end"), + (core::mem::offset_of!(Self, decide_at), "decide_at"), + (core::mem::offset_of!(Self, cut), "cut"), (core::mem::offset_of!(Self, dispatched_at), "dispatched_at"), + (core::mem::offset_of!(Self, next_balance), "next_balance"), (core::mem::offset_of!(Self, aged_grant), "aged_grant"), (core::mem::offset_of!(Self, idle_probes_spent), "idle_probes_spent"), (core::mem::offset_of!(Self, cursor), "cursor"), @@ -563,12 +596,11 @@ impl CpuSched { /// pre-cutover steal ported as it was, and a port that quietly acquired a /// later fix is a negative gate that no longer reproduces what it names. pub fn steal_ready(&mut self) -> Option> { - self.rq.pop_surplus(None) + self.rq.pop_surplus(None, 0) } pub fn install_stolen(&mut self, task: ReadyTask) { - let vruntime = task.share().runnable_vruntime().unwrap_or(0); - self.rq.insert(vruntime, task); + self.rq.insert(task); } /// Clear the borrowed window at a park only `if now >= until`, so a lend @@ -604,11 +636,14 @@ impl CpuSched { self.rt_outranks_every_corpse = outranks; } - /// Order the fair band by something other than its insertion sequence. - /// Invariant I13 must catch what that does to a share's threads; - /// `scenarios::sibling_storm`'s two gates are what prove it does. - pub fn set_fair_order(&mut self, order: crate::sched::queue::FairOrder) { - self.rq.set_order(order); + /// See [`RunQueue::set_lowest_key_within_group`]. + pub fn set_lowest_key_within_group(&mut self, on: bool) { + self.rq.set_lowest_key_within_group(on); + } + + /// See [`RunQueue::set_double_charge`]. + pub fn set_double_charge(&mut self, job: Option) { + self.rq.set_double_charge(job); } } @@ -763,7 +798,6 @@ pub fn balance_fence() { pub struct Env<'e, H: Hw, P: PreemptGuard> { pub hw: &'e H, pub cpus: &'e CpuHandles>, - pub frontier: &'e Frontier, /// The pass runs preempt-disabled, which is also what its own mailbox /// pushes need (N3). pub preempt: &'e P, @@ -824,24 +858,11 @@ impl CpuSched { self.release(dead, env); } - /// Put a ready task in this CPU's queue, counting it into its share, and - /// answer the key it was queued at. - fn enqueue, P: PreemptGuard>( - &mut self, - task: ReadyTask, - env: Env<'_, H, P>, - ) -> u64 { - let vruntime = task.share().enter_runnable(env.frontier); - self.rq.insert(vruntime, task); - vruntime - } - - /// Transfer a ready task to `dst` as an unconsumed `Adopt`. The caller has - /// already settled the share refcount: a task taken out of this queue must - /// leave it, a task that never entered must not. + /// Transfer a ready task to `dst` as an unconsumed `Adopt`. The task is in + /// no band here: one taken out of the run queue has left it. /// /// **A killed task is kept here instead**, and dispatched on this CPU — - /// see [`CpuSched::begin_dying`]. The retire that carries + /// see [`CpuSched::keep_dying`]. The retire that carries /// `Urgency::Preempt` is consumed and dropped by a destination that gets it /// ahead of the adopt (`handle_retire`'s home-is-me arm), so handing a /// corpse on trades an unwind that could start here for a wait on another @@ -888,7 +909,7 @@ impl CpuSched { // invariant I14: handing a corpse on trades an unwind that could // start here for a wait on another CPU's pick. So it is kept here // and dispatched, not reaped. - self.begin_dying(task, env, now); + self.keep_dying(task, now); return; } let key = task.key(); @@ -938,14 +959,11 @@ impl CpuSched { /// behind RT or fair behind fair, in which case an idle sibling gets it: it /// displaces nobody there. /// - /// A fair task that stays, with a key strictly below the running fair - /// task's, is owed the CPU ahead of it: the running task's quantum is cut - /// to end [`PROTECTED_RUN_NS`] after its dispatch, or now if it has run - /// that long, and the pick serves the lowest key. **Strictly**, because two - /// threads of one process share a pot and so a key: an equal key says - /// nothing about which of them is owed the CPU, and preempting on it would - /// let a thread that sleeps briefly between long spins take nearly all of - /// a sibling's time. + /// A fair task that stays and is what the fair band would now run in place + /// of the running fair thread takes the CPU [`MIN_SLICE_NS`] after that + /// thread's dispatch, or now if it has run that long: the floor is what + /// bounds the switching a stream of short wakes can cost the thread they + /// preempt. fn place, P: PreemptGuard>( &mut self, task: ReadyTask, @@ -959,7 +977,7 @@ impl CpuSched { // it takes no further instruction. Forwarding it to a sibling or // dispatching it to unwind would each undo that. if task.shared().stop_pending() { - self.begin_stopped(task, env); + self.keep_stopped(task); return; } // **The forward is decided first, and the kill check stays inside @@ -968,21 +986,14 @@ impl CpuSched { // takes — which is the path `old_migrate_kept_the_corpse` stages, and // a negative gate that has become unreachable is a gate that has been // weakened. - // The key the running fair task holds, where this one is fair too. A - // running corpse is not fair-band work: the dying list's bound times - // its unwind, not the band's order. - let behind_fair = self - .running - .as_ref() - .filter(|r| !task.is_rt() && !r.is_rt() && !r.shared().kill_pending()) - .map(|running| { - running - .share() - .runnable_vruntime() - .expect("a running task's share is runnable") - }); + // A fair wake behind a running fair thread. A running corpse is not + // fair-band work: the dying list's bound times its unwind, not the + // band's order. + let behind_fair = !task.is_rt() + && self.rq.running_is_fair() + && self.running.as_ref().is_some_and(|r| !r.shared().kill_pending()); let behind_rt = task.is_rt() && self.running.as_ref().is_some_and(|r| r.is_rt()); - if behind_fair.is_some() || behind_rt { + if behind_fair || behind_rt { if let Some(dst) = self.forward_target(env) { self.hand_off(task, dst, env, now); return; @@ -991,58 +1002,29 @@ impl CpuSched { if task.shared().kill_pending() { // A dying task is not queued behind work: it is dispatched next, // so its unwind starts inside this pass's own pick. - self.begin_dying(task, env, now); + self.keep_dying(task, now); return; } - let queued = self.enqueue(task, env); - if behind_fair.is_some_and(|held| queued < held) { - let protected = self.dispatched_at.after(PROTECTED_RUN_NS); - self.quantum_end = self.quantum_end.min(now.max(protected)); + let key = task.key(); + self.rq.insert(task); + if behind_fair && self.rq.would_preempt_for(key) { + let at = now.max(self.dispatched_at.after(MIN_SLICE_NS)); + self.cut = Some(self.cut.map_or(at, |cut| cut.min(at))); } } - /// Put a killed task where the pick takes it first, counting it into its - /// share exactly as [`Self::enqueue`] would. + /// Put a killed task where the pick takes it first. It is in no band: an + /// unwind is not competing for its job's share of the CPU. /// - /// The refcount is the reason this is not a bare `push`: `Ready` and - /// `Running` both count as runnable, so a dying task that skipped - /// `enter_runnable` would desynchronise the per-share count the sim walks - /// in `check_share_refcounts`. - fn begin_dying, P: PreemptGuard>( - &mut self, - task: ReadyTask, - env: Env<'_, H, P>, - now: Nanos, - ) { - let _vruntime = task.share().enter_runnable(env.frontier); - self.keep_dying(task, now); - } - - /// The same, for a task that is *already* counted — one taken out of the - /// run queue, which never left it. - /// - /// Both routes end a borrowed RT window: [`ReadyTask::end_lend`] carries + /// Every route ends a borrowed RT window: [`ReadyTask::end_lend`] carries /// the argument, and this is the one place that has to remember it. fn keep_dying(&mut self, mut task: ReadyTask, now: Nanos) { task.end_lend(); self.dying.push_back(Corpse { since: now, task }); } - /// Band a task that reached this CPU already stopped, counting it into its - /// share exactly as [`Self::begin_dying`] does and for the same reason: the - /// word reads `Ready`, so a task that skipped `enter_runnable` would - /// desynchronise the per-share count. - fn begin_stopped, P: PreemptGuard>( - &mut self, - task: ReadyTask, - env: Env<'_, H, P>, - ) { - let _vruntime = task.share().enter_runnable(env.frontier); - self.keep_stopped(task); - } - - /// The same, for a task that is already counted — the running task at its - /// own safe point, which never left the runnable set. + /// Band a stopped task: one that reached this CPU stopped, or the running + /// task at its own safe point. Like a dying task it is in no fair band. fn keep_stopped(&mut self, mut task: ReadyTask) { task.end_lend(); self.stopped.push_back(task); @@ -1117,8 +1099,7 @@ impl CpuSched { return; } if let Some(ready) = self.rq.remove(key) { - // Out of the fair queue and into the dying list. No refcount - // movement: it was runnable in the queue and it is runnable here. + // Out of its band and into the dying list. self.trace(env, now, TraceKind::Wake { task: key, name: ready.ext().name() }); self.keep_dying(ready, now); return; @@ -1174,12 +1155,21 @@ impl CpuSched { match msg { Msg::Wake { key, cause } => self.handle_wake(key, cause, env, now), Msg::Adopt { task } => self.handle_adopt(task, env, now), - Msg::StealRequest { thief } => self.steal_requests.push(thief), + Msg::StealRequest { thief, load } => self.steal_requests.push((thief, load)), Msg::Retire { shared } => self.handle_retire(&shared, env, now), } } } + /// Bring [`Self::decide_at`] up to date for a running fair thread, whose + /// decision moves with every change to this CPU's queues, and answer it. + fn settle_decision(&mut self, now: Nanos) -> Nanos { + if let Some(at) = self.rq.decision(now, self.dispatched_at) { + self.decide_at = self.cut.map_or(at, |cut| cut.min(at)); + } + self.decide_at + } + /// The earliest deadline this CPU owes, and the only thing `apply_timer` /// arms from. /// @@ -1316,17 +1306,9 @@ pub const DYING_AGE_NS: u64 = QUANTUM_NS; /// of added worst-case jitter once per 10 ms is the trade this picks. pub const DYING_CHUNK_NS: u64 = QUANTUM_NS / 10; -/// How long a fair task keeps the CPU after its dispatch against a fair wake -/// that outranks it: [`CpuSched::place`]'s cut ends a quantum no sooner. -/// -/// **It caps what wakers can charge the task they preempt.** A preemption -/// costs the preempted task two switches and its caches, and a process that -/// wakes briefly and often stays behind a compute process in virtual time for -/// as long as it keeps doing so — without a floor every one of its wakes would -/// preempt. With one, a CPU takes at most one fair wake preemption per this -/// long whatever the wakers do, and an owed wake on a saturated machine waits -/// at most this rather than a quantum. A policy number, like [`QUANTUM_NS`]. -pub const PROTECTED_RUN_NS: u64 = QUANTUM_NS / 10; +/// How often a CPU running fair work looks for a busier one to take a thread +/// from: once per request at most, on a pass it takes anyway. +pub const BALANCE_NS: u64 = crate::sched::queue::SLICE_NS; /// Power-of-two buckets a pass-cost histogram keeps. The top one saturates at /// 2^30 ns ≈ 1.07 s, which is longer than any pass a machine survives. @@ -1536,10 +1518,7 @@ impl<'c, 'e, H: Hw, P: PreemptGuard> SchedPass<'c, 'e, H, P, Undisposed> { if let Some(current) = cpu.running.as_mut() { let ns = current.charge(now); if ns > 0 { - current - .share() - .charge(ns) - .expect("charging a share with no runnable threads"); + cpu.rq.charge_running(ns); } } cpu.drain(env, now); @@ -1566,19 +1545,16 @@ impl<'c, 'e, H: Hw, P: PreemptGuard> SchedPass<'c, 'e, H, P, Undisposed> { self.dispose() } - /// Voluntary yield, or a quantum the driver already decided to end. + /// Voluntary yield: a fair thread gives up the rest of its request. pub fn dispose_yield(self) -> SchedPass<'c, 'e, H, P, Disposed> { if let Some(current) = self.cpu.running.take() { - let task = current.preempt(self.cpu.id, self.now); + let mut task = current.preempt(self.cpu.id, self.now); if task.shared().kill_pending() { + self.cpu.rq.leave_running(task.key(), task.standing_mut()); self.cpu.keep_dying(task, self.now); return self.dispose(); } - let vruntime = task - .share() - .runnable_vruntime() - .expect("a yielding task's share must be runnable"); - self.cpu.rq.insert(vruntime, task); + self.cpu.rq.put_back(task, true); } self.dispose() } @@ -1593,7 +1569,7 @@ impl<'c, 'e, H: Hw, P: PreemptGuard> SchedPass<'c, 'e, H, P, Undisposed> { ticket: CommittedTicket>, deadline: Option, ) -> SchedPass<'c, 'e, H, P, Disposed> { - let current = self + let mut current = self .cpu .running .take() @@ -1601,6 +1577,7 @@ impl<'c, 'e, H: Hw, P: PreemptGuard> SchedPass<'c, 'e, H, P, Undisposed> { let key = current.key(); let name = current.ext().name(); let class = ticket.class(); + self.cpu.rq.leave_running(key, current.standing_mut()); let task = current.park( &ticket, self.cpu.id, @@ -1608,7 +1585,6 @@ impl<'c, 'e, H: Hw, P: PreemptGuard> SchedPass<'c, 'e, H, P, Undisposed> { #[cfg(feature = "protocol-port")] self.cpu.park_keeps_lapsed_lend, ); - task.share().leave_runnable(self.env.frontier); self.cpu.parked.insert( key, ParkedEntry { @@ -1636,7 +1612,8 @@ impl<'c, 'e, H: Hw, P: PreemptGuard> SchedPass<'c, 'e, H, P, Undisposed> { .running .take() .expect("dispose_stop without a running task"); - let task = current.preempt(self.cpu.id, self.now); + let mut task = current.preempt(self.cpu.id, self.now); + self.cpu.rq.leave_running(task.key(), task.standing_mut()); task.shared().mark_stop(); self.cpu.keep_stopped(task); self.dispose() @@ -1645,15 +1622,15 @@ impl<'c, 'e, H: Hw, P: PreemptGuard> SchedPass<'c, 'e, H, P, Undisposed> { /// The current task exits. Its record survives as the zombie until the /// next pass, which runs on another stack. pub fn dispose_exit(self) -> SchedPass<'c, 'e, H, P, Disposed> { - let current = self + let mut current = self .cpu .running .take() .expect("dispose_exit without a running task"); let key = current.key(); let name = current.ext().name(); + self.cpu.rq.leave_running(key, current.standing_mut()); let dead = current.die(self.cpu.id, self.now); - dead.share().leave_runnable(self.env.frontier); self.cpu.dispose_dead(dead, self.env); self.cpu .trace(self.env, self.now, TraceKind::Retire { task: key, name }); @@ -1699,6 +1676,7 @@ impl SchedPass<'_, '_, H, P, Disposed> { self.preempt_if_due(); self.pick(); self.answer_steal_requests(); + self.balance_busy(); // **Two numbers, because two consumers ask two questions** — see // [`CpuHandle::load`] and [`CpuHandle::surplus`]. Spawn placement // wants everything a new task would queue behind, and a corpse @@ -1718,6 +1696,8 @@ impl SchedPass<'_, '_, H, P, Disposed> { let surplus = self.cpu.rq.fair_len() as u32; handle.publish_load((self.cpu.rq.len() + self.cpu.dying.len()) as u32); handle.publish_surplus(surplus); + self.cpu.rq.refresh(); + handle.publish_weight(self.cpu.rq.weight()); // The stamp goes with them: it is what says they are about now. handle.publish_pass(self.now); self.push_on_surplus(surplus); @@ -1763,12 +1743,16 @@ impl SchedPass<'_, '_, H, P, Disposed> { let kill_due = current.shared().kill_pending() && !self.cpu.aged_grant && matches!(current.shared().state(), TaskState::Running(_)); - let due = self.now >= self.cpu.quantum_end || rt_due || kill_due; + // The instant the last pass armed is when the running request ends or + // an earlier-due group became eligible, and `begin`'s charge has + // already issued the next request; this pass's own queue changes can + // only bring the decision forward. + let due = self.now >= self.cpu.decide_at || self.now >= self.cpu.settle_decision(self.now) || rt_due || kill_due; if !due { return; } let current = self.cpu.running.take().expect("checked above"); - let task = current.preempt(self.cpu.id, self.now); + let mut task = current.preempt(self.cpu.id, self.now); if task.shared().kill_pending() { // Once `Commit::Killed` is `dispose_none` the killed thread keeps // running and unwinds, and its next quantum expiry must not put it @@ -1785,14 +1769,11 @@ impl SchedPass<'_, '_, H, P, Disposed> { // `a_killed_task_that_expires_its_quantum_goes_back_to_the_dying_list` // — the fair queue is where the task would land instead, and that // test asserts it never does. + self.cpu.rq.leave_running(task.key(), task.standing_mut()); self.cpu.keep_dying(task, self.now); return; } - let vruntime = task - .share() - .runnable_vruntime() - .expect("a preempted task's share must be runnable"); - self.cpu.rq.insert(vruntime, task); + self.cpu.rq.put_back(task, false); } /// **RT band, then the dying list, then the fair band — and no kill check @@ -1807,8 +1788,8 @@ impl SchedPass<'_, '_, H, P, Disposed> { /// so `ReadyTask::dispatch`'s note about the kill bit not being asserted /// away is the whole of what remains true. /// - /// A dying task jumps the *fair* queue and the vruntime frontier does not - /// advance for it: it is not spending a share of the CPU, it is finishing. + /// A dying task jumps the *fair* queue and is in no band: it is not + /// spending a share of the CPU, it is finishing. /// /// **It does not jump the RT band, and it is not held off by it for ever /// either — the deferral is bounded, and both absolutes are wrong.** @@ -1857,11 +1838,12 @@ impl SchedPass<'_, '_, H, P, Disposed> { let name = task.ext().name(); self.cpu.running = Some(task.dispatch(self.cpu.id, self.now)); self.cpu.dispatched_at = self.now; + self.cpu.cut = None; // An aged corpse is borrowing the CPU from the RT band, so it // borrows a chunk and not a quantum. With the band empty it is // ordinary normal-band work and gets the ordinary quantum. self.cpu.aged_grant = self.cpu.rq.has_rt(); - self.cpu.quantum_end = self.now.after(if self.cpu.aged_grant { + self.cpu.decide_at = self.now.after(if self.cpu.aged_grant { DYING_CHUNK_NS } else { QUANTUM_NS @@ -1874,14 +1856,20 @@ impl SchedPass<'_, '_, H, P, Disposed> { // A lapsed borrowed window must not demote the task out of the RT band // here: queue time spends none of it, so `ReadyTask::dispatch` re-arms // it instead. The band stays whatever it was at insert. - if let Some((vruntime, task)) = self.cpu.rq.pop_next() { - self.env.frontier.advance(vruntime); + if let Some(task) = self.cpu.rq.pop_next() { let key = task.key(); let name = task.ext().name(); + // The task this CPU stands on, picked again, is still in the run it + // was in: the floor a preemption waits for counts from its start. + if self.cpu.loaded != Loaded::Task(key) { + self.cpu.dispatched_at = self.now; + } self.cpu.running = Some(task.dispatch(self.cpu.id, self.now)); - self.cpu.dispatched_at = self.now; self.cpu.aged_grant = false; - self.cpu.quantum_end = self.now.after(QUANTUM_NS); + self.cpu.cut = None; + // A real-time task's turn; a fair thread's decision is the run + // queue's, settled before the timer is armed. + self.cpu.decide_at = self.now.after(QUANTUM_NS); self.cpu .trace(self.env, self.now, TraceKind::Schedule { task: key, name }); } @@ -1936,7 +1924,7 @@ impl SchedPass<'_, '_, H, P, Disposed> { self.cpu.steal_requests.clear(); return; } - while let Some(&thief) = self.cpu.steal_requests.last() { + while let Some(&(thief, asking)) = self.cpu.steal_requests.last() { if self.stealable() == 0 { if self.cpu.rq.fair_len() > 0 { // The one queued task is the one this CPU still stands on, @@ -1949,13 +1937,44 @@ impl SchedPass<'_, '_, H, P, Disposed> { return; } self.cpu.steal_requests.pop(); - let task = self - .cpu - .rq - .pop_surplus(self.cpu.loaded_key()) - .expect("a stealable task is one `pop_surplus` hands over"); - task.share().leave_runnable(self.env.frontier); - self.cpu.hand_off(task, thief, self.env, self.now); + // A busy asker that no move brings closer goes without. + if let Some(task) = self.cpu.rq.pop_surplus(self.cpu.loaded_key(), asking) { + self.cpu.hand_off(task, thief, self.env, self.now); + } + } + } + + /// Ordinary load balancing between busy CPUs: a CPU running fair work asks + /// the CPU whose job groups weigh most, nearest first among equals, for the + /// thread whose move narrows their difference most — when that CPU weighs + /// more and has a thread to give. Once per [`BALANCE_NS`], on a pass this + /// CPU takes anyway; the answer is [`RunQueue::pop_surplus`]'s. + fn balance_busy(&mut self) { + if !self.env.balance.pulls() || !self.cpu.rq.running_is_fair() || self.now < self.cpu.next_balance { + return; + } + self.cpu.next_balance = self.now.after(BALANCE_NS); + let (cpus, me, now) = (self.env.cpus, self.cpu.id, self.now); + let mine = self.cpu.rq.weight(); + let heaviest = STEAL_RINGS + .into_iter() + .flat_map(|(inner, outer)| cpus.topology().ring(me, inner, outer, 0)) + .filter(|&cpu| cpus.get(cpu).answering(now) && cpus.get(cpu).surplus() >= PUSH_THRESHOLD) + .map(|cpu| (cpus.get(cpu).weight(), cpu)) + .fold(None, |best: Option<(u64, CpuId)>, (weight, cpu)| match best { + Some((most, _)) if most >= weight => best, + _ => Some((weight, cpu)), + }); + let Some((_, victim)) = heaviest.filter(|&(weight, _)| weight > mine) else { + return; + }; + let Some(slot) = self.cpu.steal_probe.claim() else { + return; + }; + if cpus.get(victim).post(slot, Msg::StealRequest { thief: me, load: mine }, Urgency::Normal, self.env.preempt) + == Kick::Send + { + self.env.hw.kick(victim); } } @@ -1984,8 +2003,11 @@ impl SchedPass<'_, '_, H, P, Disposed> { /// deadline. fn apply_timer_no_later_than(&mut self, extra: Option) -> TimerApplied { let deadline = self.cpu.earliest_deadline(); - let quantum = self.cpu.running.as_ref().map(|_| self.cpu.quantum_end); - let plan = TimerPlan::compute(quantum, deadline).no_later_than(extra); + let decision = match self.cpu.running { + Some(_) => Some(self.cpu.settle_decision(self.now)), + None => None, + }; + let plan = TimerPlan::compute(decision, deadline).no_later_than(extra); match plan { TimerPlan::Arm(at) => self.env.hw.set_timer(at), TimerPlan::Stop => self.env.hw.stop_timer(), @@ -2126,7 +2148,7 @@ impl SchedPass<'_, '_, H, P, Disposed> { let thief = self.cpu.id; if self.env.cpus.get(victim).post( slot, - Msg::StealRequest { thief }, + Msg::StealRequest { thief, load: 0 }, Urgency::Normal, self.env.preempt, ) == Kick::Send @@ -2231,7 +2253,10 @@ pub struct CpuHandle { /// with nothing, and sleeps — and the probe is one-shot per idle trip, so /// the genuinely surplus-holding CPU goes unprobed for a whole idle round. surplus: AtomicU32, - /// When the pass that published the two numbers above ran, so that a reader + /// **The summed weight of this CPU's job groups**, published for busy + /// balancing ([`SchedPass::balance_busy`]) and for nothing else. + weight: AtomicU64, + /// When the pass that published the numbers above ran, so that a reader /// can tell a claim about the present from one a stopped CPU left behind. /// See [`CpuHandle::answering`]. last_pass: AtomicU64, @@ -2259,6 +2284,7 @@ impl CpuHandle { doorbell: Doorbell::new(), load: AtomicU32::new(0), surplus: AtomicU32::new(0), + weight: AtomicU64::new(0), last_pass: AtomicU64::new(0), #[cfg(feature = "sched-check")] pass_costs: PassCosts::new(), @@ -2294,7 +2320,15 @@ impl CpuHandle { self.surplus.store(fair, Ordering::Relaxed); } - /// Stamp the two numbers above with the pass that published them. + pub fn weight(&self) -> u64 { + self.weight.load(Ordering::Relaxed) + } + + pub fn publish_weight(&self, weight: u64) { + self.weight.store(weight, Ordering::Relaxed); + } + + /// Stamp the numbers above with the pass that published them. pub fn publish_pass(&self, now: Nanos) { self.last_pass.store(now.0, Ordering::Relaxed); } @@ -2512,26 +2546,17 @@ mod tests { extern crate std; use super::*; - use crate::sched::fair::{FairShare, ShareState}; use crate::sched::hw::{Kicker, Machine}; + use crate::sched::job::{Job, JobKey}; use crate::sched::mailbox::{mailbox, NoPreempt}; - use crate::sched::sync::CellLock; + use crate::sched::queue::SLICE_NS; use crate::sched::task::{RtState, TaskAccounting, TaskBuilder}; use std::sync::Mutex; - struct TestLock(Mutex); - - impl CellLock for TestLock { - fn with(&self, f: impl FnOnce(&mut T) -> R) -> R { - f(&mut self.0.lock().expect("a test never poisons a lock")) - } - } - struct TestPayload; impl SchedPayload for TestPayload { type Ctx = (); - type ShareLock = TestLock; fn name(&self) -> u64 { 0 @@ -2597,7 +2622,6 @@ mod tests { cpus: Vec>, handles: CpuHandles>, hw: TestHw, - frontier: Frontier, preempt: NoPreempt, next_key: u64, /// `Env::balance`. [`Balance::None`] by default because most tests want @@ -2624,7 +2648,6 @@ mod tests { cpus, handles: CpuHandles::new(handles, topology), hw: TestHw::default(), - frontier: Frontier::new(), preempt: NoPreempt, next_key: 1, balance: Balance::None, @@ -2645,7 +2668,6 @@ mod tests { Env { hw: &self.hw, cpus: &self.handles, - frontier: &self.frontier, preempt: &self.preempt, balance: self.balance, }, @@ -2675,42 +2697,41 @@ mod tests { dst: CpuId, rt: RtState, ) -> (TaskKey, Arc>>) { - let share = Arc::new(FairShare::new(TestLock(Mutex::new( - ShareState::NonRunnable { lag: 0 }, - )))); - self.spawn_into(dst, rt, share) + let job = Arc::new(Job::new(JobKey(self.next_key))); + self.spawn_into(dst, rt, job) } - /// A thread of the process whose pot `share` is. + /// A thread of `of`'s job. fn spawn_sibling( &mut self, dst: CpuId, of: TaskKey, ) -> (TaskKey, Arc>>) { - let share = (0..self.cpus.len()) + let job = (0..self.cpus.len()) .find_map(|cpu| { let sched = &self.cpus[cpu]; sched .running() .filter(|t| t.key() == of) - .map(|t| t.share().clone()) - .or_else(|| sched.parked_task(of).map(|t| t.share().clone())) + .map(|t| t.job().clone()) + .or_else(|| sched.parked_task(of).map(|t| t.job().clone())) + .or_else(|| sched.rq().tasks().find(|t| t.key() == of).map(|t| t.job().clone())) }) - .expect("the sibling is running or parked"); - self.spawn_into(dst, RtState::default(), share) + .expect("the sibling is running, parked or queued"); + self.spawn_into(dst, RtState::default(), job) } fn spawn_into( &mut self, dst: CpuId, rt: RtState, - share: Arc>, + job: Arc, ) -> (TaskKey, Arc>>) { let key = TaskKey(self.next_key); self.next_key += 1; let task = TaskBuilder { key, - share, + job, ctx: (), ext: TestPayload, rt, @@ -3301,7 +3322,6 @@ mod tests { { let (cpus, env) = w.split(); let task = cpus[0].rq.remove(key).expect("ready on cpu0"); - task.share().leave_runnable(env.frontier); cpus[0].hand_off(task, C1, env, NOW); } @@ -3331,7 +3351,6 @@ mod tests { { let (cpus, env) = w.split(); let task = cpus[0].rq.remove(key).expect("ready on cpu0"); - task.share().leave_runnable(env.frontier); cpus[0].hand_off(task, C1, env, NOW); } assert_eq!(shared.state(), TaskState::InTransit(C1)); @@ -3820,8 +3839,8 @@ mod tests { /// which one was made. /// /// It reds on a tree without the `loaded` argument: the just-preempted task - /// carries the highest vruntime in the band, so `pop_surplus`'s `next_back` - /// names it first and every run hands over exactly the wrong one. + /// is the newest ready thread of its group, which `pop_surplus` names + /// first, so every run hands over exactly the wrong one. /// /// **Under that mutation the red is [`CpuSched::hand_off`]'s own /// assertion**, which fires inside `run_a_pass_at` before this function's @@ -3844,7 +3863,7 @@ mod tests { .expect("the pick took one of the three"); // An idle sibling's probe, waiting for this CPU's next pass. - w.cpus[1].steal_requests.push(C0); + w.cpus[1].steal_requests.push((C0, 0)); // A quantum later: `preempt_if_due` returns the loaded task to the // band, `pick` takes a fresher one, and `answer_steal_requests` runs @@ -3880,97 +3899,117 @@ mod tests { w.post_claimed_wake(cpu, shared, WakeReason::Woken); } - /// **A wake owed the CPU ahead of the running fair task cuts its quantum to - /// the protected run**, and the pick then serves the wake. - /// - /// The sleeper parked having run nothing, so its key is the frontier; the - /// hog has been charged half the protected run since its dispatch, so its - /// key is above that. The cut lands on the protected run's end, not on now. - #[test] - fn a_wake_with_a_lower_key_cuts_the_running_quantum_to_the_protected_run() { - let mut w = World::new(1); - let (sleeper, sleeper_shared) = w.spawn(C0); - w.run_a_pass(C0); - w.park_running(C0); + /// A hog of a two-thread job on cpu0, its sibling queued on cpu1, so its + /// group there weighs half of a single-thread job's; and `sleepers` + /// single-thread jobs parked on cpu0 with no standing. Answers the hog and + /// the sleepers. + fn hog_beside_sleepers( + w: &mut World, + sleepers: usize, + at: Nanos, + ) -> (TaskKey, Vec>>>) { + let mut parked = Vec::new(); + for _ in 0..sleepers { + let (_, shared) = w.spawn(C0); + w.run_a_pass_at(C0, at); + w.park_running(C0); + parked.push(shared); + } let (hog, _) = w.spawn(C0); - w.run_a_pass(C0); + w.spawn_sibling(C1, hog); + w.run_a_pass_at(C0, at); assert_eq!(w.cpus[0].running().map(|t| t.key()), Some(hog)); + (hog, parked) + } - wake(&w, C0, &sleeper_shared); - w.run_a_pass_at(C0, NOW.after(PROTECTED_RUN_NS / 2)); - assert_eq!( - w.cpus[0].quantum_end(), - NOW.after(PROTECTED_RUN_NS), - "the hog's quantum ends where its protected run does", - ); + /// **A wake the fair band would run ahead of the running thread takes the + /// CPU [`MIN_SLICE_NS`] after that thread's dispatch**, and not before. The + /// sleeper's job weighs twice the hog's group here, so its request is due + /// first. + #[test] + fn a_wake_due_first_takes_the_cpu_once_the_running_thread_has_its_minimum_run() { + let mut w = World::new(2); + let (hog, sleepers) = hog_beside_sleepers(&mut w, 1, NOW); + + wake(&w, C0, &sleepers[0]); + w.run_a_pass_at(C0, NOW.after(MIN_SLICE_NS / 2)); + assert_eq!(w.cpus[0].decide_at(), NOW.after(MIN_SLICE_NS), "the hog keeps its minimum run"); assert_eq!(w.cpus[0].running().map(|t| t.key()), Some(hog), "and not before"); - w.run_a_pass_at(C0, NOW.after(PROTECTED_RUN_NS)); + w.run_a_pass_at(C0, NOW.after(MIN_SLICE_NS)); assert_eq!( w.cpus[0].running().map(|t| t.key()), - Some(sleeper), - "the cut quantum ends and the pick serves the lower key", + Some(sleepers[0].key()), + "the minimum run ends and the band serves the wake", ); w.abandon(); } - /// **A tie is not a debt**: two threads of one process share a pot and so a - /// key, and a sibling's wake cuts nothing, however long the running one has - /// run. + /// **A wake of an equal job with no standing waits for the running + /// request**: its deadline is a whole request after the queue's virtual + /// time, and the running thread's is due first. #[test] - fn a_wake_that_ties_the_running_tasks_key_cuts_nothing() { + fn a_wake_with_no_claim_ahead_waits_for_the_running_request() { let mut w = World::new(1); - let (napper, napper_shared) = w.spawn(C0); + let (sleeper, sleeper_shared) = w.spawn(C0); w.run_a_pass(C0); w.park_running(C0); - let (spinner, _) = w.spawn_sibling(C0, napper); + let (hog, _) = w.spawn(C0); w.run_a_pass(C0); - assert_eq!(w.cpus[0].running().map(|t| t.key()), Some(spinner)); + assert_eq!(w.cpus[0].running().map(|t| t.key()), Some(hog)); - wake(&w, C0, &napper_shared); - w.run_a_pass_at(C0, NOW.after(2 * PROTECTED_RUN_NS)); - assert_eq!( - w.cpus[0].quantum_end(), - NOW.after(QUANTUM_NS), - "the spinner keeps its whole quantum", - ); - assert_eq!(w.cpus[0].running().map(|t| t.key()), Some(spinner)); + wake(&w, C0, &sleeper_shared); + w.run_a_pass_at(C0, NOW.after(2 * MIN_SLICE_NS)); + assert_eq!(w.cpus[0].decide_at(), NOW.after(SLICE_NS), "the hog keeps its whole request"); + assert_eq!(w.cpus[0].running().map(|t| t.key()), Some(hog)); + + w.run_a_pass_at(C0, NOW.after(SLICE_NS)); + assert_eq!(w.cpus[0].running().map(|t| t.key()), Some(sleeper), "and the wake runs at its end"); w.abandon(); } - /// **The protected run is counted from the dispatch, and a cut quantum does - /// not say where that was.** A second owed wake inside the protected run - /// leaves the first cut where it is; reading the dispatch back off the cut - /// `quantum_end` would put it a quantum early and preempt now. + /// **The minimum run is counted from the dispatch, and a cut does not say + /// where that was.** A second wake due first leaves the first cut where it + /// is; reading the dispatch back off the cut would preempt now. #[test] - fn a_second_owed_wake_does_not_cut_inside_the_protected_run() { + fn a_second_wake_due_first_does_not_cut_inside_the_minimum_run() { let at = Nanos(5 * QUANTUM_NS); - let mut w = World::new(1); - let (_, first) = w.spawn(C0); - w.run_a_pass(C0); - w.park_running(C0); - let (_, second) = w.spawn(C0); - w.run_a_pass(C0); - w.park_running(C0); - let (hog, _) = w.spawn(C0); - w.run_a_pass_at(C0, at); - assert_eq!(w.cpus[0].running().map(|t| t.key()), Some(hog)); + let mut w = World::new(2); + let (hog, sleepers) = hog_beside_sleepers(&mut w, 2, at); - wake(&w, C0, &first); - w.run_a_pass_at(C0, at.after(PROTECTED_RUN_NS / 5)); - assert_eq!(w.cpus[0].quantum_end(), at.after(PROTECTED_RUN_NS)); + wake(&w, C0, &sleepers[0]); + w.run_a_pass_at(C0, at.after(MIN_SLICE_NS / 5)); + assert_eq!(w.cpus[0].decide_at(), at.after(MIN_SLICE_NS)); - wake(&w, C0, &second); - w.run_a_pass_at(C0, at.after(PROTECTED_RUN_NS / 2)); + wake(&w, C0, &sleepers[1]); + w.run_a_pass_at(C0, at.after(MIN_SLICE_NS / 2)); assert_eq!( - w.cpus[0].quantum_end(), - at.after(PROTECTED_RUN_NS), - "the second wake moved the cut inside the protected run", + w.cpus[0].decide_at(), + at.after(MIN_SLICE_NS), + "the second wake moved the cut inside the minimum run", ); assert_eq!(w.cpus[0].running().map(|t| t.key()), Some(hog)); w.abandon(); } + /// **A fair thread is decided again at the end of the request armed for + /// it**, though the pass that takes the decision has already issued its + /// next one. A thread alone on its CPU and lent a real-time window while it + /// runs is the case where nothing else would preempt it: at its request's + /// end it goes to the real-time band. + #[test] + fn a_fair_thread_is_decided_again_at_its_armed_request_end() { + let mut w = World::new(1); + let (alone, _) = w.spawn(C0); + w.run_a_pass(C0); + assert_eq!(w.cpus[0].decide_at(), NOW.after(SLICE_NS)); + w.cpus[0].boost_current(NOW, NOW.after(10 * SLICE_NS)); + w.run_a_pass_at(C0, NOW.after(SLICE_NS)); + assert_eq!(w.cpus[0].running().map(|t| t.key()), Some(alone)); + assert!(!w.cpus[0].rq().running_is_fair(), "the lent thread left the fair band at its request end"); + w.abandon(); + } + /// **A wake whose home CPU is running fair work goes to an idle sibling**, /// whatever its key: it displaces nobody there. #[test] @@ -4038,7 +4077,7 @@ mod tests { let (_, queued) = tasks.iter().find(|(key, _)| *key != running).expect("one queues"); assert_eq!(w.handles.get(C1).surplus(), 1, "one queued task is published as surplus"); - w.cpus[1].steal_requests.push(C0); + w.cpus[1].steal_requests.push((C0, 0)); w.run_a_pass_at(C1, Nanos(NOW.0 + 1)); assert_eq!(queued.state(), TaskState::InTransit(C0), "and handed to the probe"); w.abandon(); @@ -4057,12 +4096,12 @@ mod tests { let loaded = w.cpus[1].running().map(|t| t.key()).expect("one was picked"); let (_, loaded_shared) = tasks.iter().find(|(key, _)| *key == loaded).expect("loaded"); - w.cpus[1].steal_requests.push(C0); + w.cpus[1].steal_requests.push((C0, 0)); w.run_a_pass_at(C1, Nanos(NOW.0 + QUANTUM_NS + 1)); assert_ne!(w.cpus[1].running().map(|t| t.key()), Some(loaded), "the quantum ended"); assert_eq!(loaded_shared.state(), TaskState::Ready(C1), "the loaded task stayed"); assert_eq!(w.handles.get(C1).surplus(), 1, "and is published as surplus"); - assert_eq!(w.cpus[1].steal_requests, std::vec![C0], "the probe is kept"); + assert_eq!(w.cpus[1].steal_requests, std::vec![(C0, 0)], "the probe is kept"); assert_eq!(w.hw.state().need_resched, std::vec![C1], "for a pass asked for now"); w.run_a_pass_at(C1, Nanos(NOW.0 + QUANTUM_NS + 2)); diff --git a/kernel/pure/sched/fair.rs b/kernel/pure/sched/fair.rs deleted file mode 100644 index 299474be21..0000000000 --- a/kernel/pure/sched/fair.rs +++ /dev/null @@ -1,369 +0,0 @@ -//! Fairness policy: one pot per *process*, and the vruntime/lag/frontier math -//! over it. The simulator runs this exact arithmetic, so policy changes belong -//! here and are sim-gated. - -use core::num::NonZeroU32; -use core::sync::atomic::{AtomicU64, Ordering}; - -use crate::sched::sync::CellLock; - -/// Clamp for stored lag at the Runnable→NonRunnable transition: how far -/// behind (entitled catch-up) or ahead (throttled on wake) of the frontier a -/// process may be remembered as. 50ms. -pub const MAX_VRUNTIME_LAG_NS: u64 = 50_000_000; - -/// Preemption quantum. 10ms. -pub const QUANTUM_NS: u64 = 10_000_000; - -/// The requested operation needs the `Runnable` arm. A marker, not a -/// recoverable error: callers panic with their own context (the kernel adds -/// the pid) — charging or yield-reading a NonRunnable share is a -/// bookkeeping bug. -#[derive(Clone, Copy, PartialEq, Eq, Debug)] -pub struct NotRunnable; - -/// The global vruntime frontier: a monotonic-non-decreasing baseline from -/// which non-runnable processes' lag is measured (see -/// [`ShareState::NonRunnable`]). One, not one per CPU — a per-CPU frontier -/// needs epoch reconciliation, which is a policy change the simulator gates. -pub struct Frontier(AtomicU64); - -impl Frontier { - /// No `Default` beside it: every frontier in this tree is a `static`, which - /// only a `const fn` can build, and `Default::default` cannot be one. - #[allow(clippy::new_without_default)] - pub const fn new() -> Self { - Self(AtomicU64::new(0)) - } - - /// Advance the frontier to at least `vrt`, at dispatch. `fetch_max` is - /// the only correct semantic on SMP: a plain `store(vrt)` lets a CPU - /// picking a low-vrt task regress the frontier another CPU has already - /// advanced, and lets RT picks (vrt=0) reset it to zero on every - /// preemption. - pub fn advance(&self, vrt: u64) { - self.0.fetch_max(vrt, Ordering::Relaxed); - } - - pub fn get(&self) -> u64 { - self.0.load(Ordering::Relaxed) - } -} - -/// Per-process fair-share state. -/// -/// A non-runnable process stores no vruntime — a stored one would go stale as -/// the frontier moves. It stores `lag = frontier - vruntime` instead, and -/// re-derives `vruntime = frontier - lag` on wake, so it tracks the frontier -/// with no writes while blocked. -pub enum ShareState { - Runnable { - vruntime: u64, - /// Threads of this process in a run queue (ready or current). - /// `NonZeroU32` forbids "Runnable with 0 threads", which would mean a - /// botched leave→NonRunnable transition. - runnable_threads: NonZeroU32, - /// Clamped lag frozen at the most recent NonRunnable→Runnable - /// transition (0 for a process that has never blocked since spawn). - /// - /// This is the *contract* value [`ShareState::lag`] reports: bounded - /// ±[`MAX_VRUNTIME_LAG_NS`] by construction. Live `frontier - - /// vruntime` may exceed that bound between a wake and the pick, which - /// is a diagnostic and not a contract. - lag_at_wake: i64, - }, - NonRunnable { - /// `lag = frontier - vruntime` at the last transition, clamped to - /// ±[`MAX_VRUNTIME_LAG_NS`]. Positive = behind the frontier (entitled - /// to catch up); negative = ran ahead (throttled on wake). - lag: i64, - }, -} - -impl ShareState { - /// First runnable thread of a never-scheduled process: starts at the - /// current frontier with `lag_at_wake = 0`, so its insert vruntime is - /// `frontier` itself. - pub fn new_runnable(frontier: u64) -> Self { - Self::Runnable { - vruntime: frontier, - runnable_threads: NonZeroU32::MIN, - lag_at_wake: 0, - } - } - - /// A thread of this process is becoming runnable; returns the vruntime to - /// insert with. All threads of one process share a vruntime, so a - /// subsequent thread only bumps the refcount and `lag_at_wake` stays - /// frozen at the most recent NonRunnable→Runnable edge. - pub fn enter_runnable(&mut self, frontier: u64) -> u64 { - match self { - Self::Runnable { - vruntime, - runnable_threads, - .. - } => { - *runnable_threads = runnable_threads - .checked_add(1) - .expect("runnable_threads overflow"); - *vruntime - } - Self::NonRunnable { lag } => { - let lag = *lag; - let vrt = vrt_from_lag(frontier, lag); - *self = Self::Runnable { - vruntime: vrt, - runnable_threads: NonZeroU32::MIN, - lag_at_wake: lag, - }; - vrt - } - } - } - - /// A thread of this process is no longer runnable (blocked, exited, - /// killed). Decrements the refcount; on the last runnable thread, - /// stores clamped lag and transitions to NonRunnable. - /// - /// No-op if already NonRunnable — covers removal of a thread that is - /// already blocked (its refcount was decremented when it blocked). - pub fn leave_runnable(&mut self, frontier: u64) { - if let Self::Runnable { - vruntime, - runnable_threads, - .. - } = self - { - if let Some(n) = NonZeroU32::new(runnable_threads.get() - 1) { - *runnable_threads = n; - } else { - let lag = live_lag(frontier, *vruntime) - .clamp(-(MAX_VRUNTIME_LAG_NS as i64), MAX_VRUNTIME_LAG_NS as i64); - *self = Self::NonRunnable { lag }; - } - } - } - - /// Charge `ns` of consumed CPU time to the share's vruntime. - /// `Err(NotRunnable)`: the caller charged a process with no runnable - /// threads — a bookkeeping bug it must report loudly. - pub fn charge(&mut self, ns: u64) -> Result<(), NotRunnable> { - match self { - Self::Runnable { vruntime, .. } => { - *vruntime = vruntime.saturating_add(ns); - Ok(()) - } - Self::NonRunnable { .. } => Err(NotRunnable), - } - } - - /// The stored Runnable vruntime, for the yield re-insert path where the - /// thread stays runnable. `Err(NotRunnable)` means a yielding thread was - /// wrongly counted out — the caller panics with its context. - pub fn runnable_vruntime(&self) -> Result { - match self { - Self::Runnable { vruntime, .. } => Ok(*vruntime), - Self::NonRunnable { .. } => Err(NotRunnable), - } - } - - /// Live vruntime for external readers: the stored value while Runnable, - /// `frontier - lag` while NonRunnable. - pub fn vruntime(&self, frontier: u64) -> u64 { - match self { - Self::Runnable { vruntime, .. } => *vruntime, - Self::NonRunnable { lag } => vrt_from_lag(frontier, *lag), - } - } - - /// Contract lag: the clamped lag established at the most recent - /// transition in either direction. Bounded ±MAX_VRUNTIME_LAG_NS by - /// construction — NOT the live `frontier - vruntime` drift, which can - /// exceed the bound during the wake-to-pick gap on multi-CPU systems. - pub fn lag(&self) -> i64 { - match self { - Self::Runnable { lag_at_wake, .. } => *lag_at_wake, - Self::NonRunnable { lag } => *lag, - } - } -} - -/// `lag = frontier - vruntime`, unclamped; the caller clamps before storing. -/// Wrapping subtraction is deliberate: vruntimes are dense in u64, and the -/// wrapped difference is the signed distance even once the frontier has -/// wrapped past `vrt`. -fn live_lag(frontier: u64, vrt: u64) -> i64 { - (frontier as i64).wrapping_sub(vrt as i64) -} - -/// `vrt = frontier - lag`, saturating at u64 bounds. -fn vrt_from_lag(frontier: u64, lag: i64) -> u64 { - if lag >= 0 { - frontier.saturating_sub(lag as u64) - } else { - frontier.saturating_add((-lag) as u64) - } -} - -/// One process's fair-share pot, reached through any thread that owns it. The -/// cell is supplied by the environment for the reason stated on [`CellLock`]: -/// the kernel's is a word-sized spin, the simulator's a mutex. -pub struct FairShare { - state: L, -} - -impl> FairShare { - pub fn new(state: L) -> Self { - Self { state } - } - - /// A thread of this process is entering a run queue: returns the vruntime - /// to insert with. Called at exactly the transitions that add a task - /// to the Ready+Running pair, which is what keeps `runnable_threads` and - /// the containers in step (invariant I6). - pub fn enter_runnable(&self, frontier: &Frontier) -> u64 { - self.state.with(|s| s.enter_runnable(frontier.get())) - } - - /// A thread left the Ready+Running pair (park, die, migrate away). - pub fn leave_runnable(&self, frontier: &Frontier) { - self.state.with(|s| s.leave_runnable(frontier.get())); - } - - /// The re-insert vruntime for a thread that stays runnable (preempt and - /// yield), which must not re-enter and double-count the refcount. - pub fn runnable_vruntime(&self) -> Result { - self.state.with(|s| s.runnable_vruntime()) - } - - pub fn charge(&self, ns: u64) -> Result<(), NotRunnable> { - self.state.with(|s| s.charge(ns)) - } - - pub fn vruntime(&self, frontier: &Frontier) -> u64 { - self.state.with(|s| s.vruntime(frontier.get())) - } - - pub fn lag(&self) -> i64 { - self.state.with(|s| s.lag()) - } - - /// The refcount invariant I6 compares against the containers. - pub fn runnable_threads(&self) -> u32 { - self.state.with(|s| match s { - ShareState::Runnable { - runnable_threads, .. - } => runnable_threads.get(), - ShareState::NonRunnable { .. } => 0, - }) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - const CLAMP: i64 = MAX_VRUNTIME_LAG_NS as i64; - - fn runnable_count(s: &ShareState) -> u32 { - match s { - ShareState::Runnable { - runnable_threads, .. - } => runnable_threads.get(), - ShareState::NonRunnable { .. } => 0, - } - } - - #[test] - fn new_process_starts_at_frontier_with_zero_lag() { - let s = ShareState::new_runnable(1_000); - assert_eq!(s.vruntime(1_000), 1_000); - assert_eq!(s.lag(), 0); - assert_eq!(runnable_count(&s), 1); - } - - #[test] - fn refcount_tracks_runnable_threads() { - let mut s = ShareState::new_runnable(0); - assert_eq!(s.enter_runnable(0), 0); - assert_eq!(runnable_count(&s), 2); - s.leave_runnable(0); - assert_eq!(runnable_count(&s), 1); - assert!(s.runnable_vruntime().is_ok()); - s.leave_runnable(0); - assert_eq!(runnable_count(&s), 0); - assert_eq!(s.runnable_vruntime(), Err(NotRunnable)); - } - - #[test] - fn charge_accumulates_and_saturates() { - let mut s = ShareState::new_runnable(10); - s.charge(5).unwrap(); - assert_eq!(s.runnable_vruntime(), Ok(15)); - s.charge(u64::MAX).unwrap(); - assert_eq!(s.runnable_vruntime(), Ok(u64::MAX)); - } - - #[test] - fn charge_nonrunnable_is_a_bug() { - let mut s = ShareState::new_runnable(0); - s.leave_runnable(0); - assert_eq!(s.charge(1), Err(NotRunnable)); - } - - #[test] - fn leave_stores_lag_and_wake_rederives_vruntime() { - // Ran 10ms ahead of the frontier, then blocked: lag = -10ms. - let mut s = ShareState::new_runnable(0); - s.charge(10_000_000).unwrap(); - s.leave_runnable(0); - assert_eq!(s.lag(), -10_000_000); - // Frontier advanced to 50ms while blocked: wake resumes 10ms ahead - // of the *current* frontier and freezes the lag as the contract. - let vrt = s.enter_runnable(50_000_000); - assert_eq!(vrt, 60_000_000); - assert_eq!(s.lag(), -10_000_000); - } - - #[test] - fn lag_clamps_in_both_directions() { - // Far behind the frontier: entitled catch-up clamps to +50ms. - let mut s = ShareState::new_runnable(0); - s.leave_runnable(10 * MAX_VRUNTIME_LAG_NS); - assert_eq!(s.lag(), CLAMP); - // Far ahead of the frontier: throttle clamps to -50ms. - let mut s = ShareState::new_runnable(0); - s.charge(10 * MAX_VRUNTIME_LAG_NS).unwrap(); - s.leave_runnable(0); - assert_eq!(s.lag(), -CLAMP); - } - - #[test] - fn nonrunnable_vruntime_tracks_the_live_frontier() { - let mut s = ShareState::new_runnable(1_000); - s.leave_runnable(2_000); // lag = +1000 - assert_eq!(s.vruntime(2_000), 1_000); - assert_eq!(s.vruntime(10_000), 9_000); - // Saturates instead of wrapping below zero. - assert_eq!(s.vruntime(500), 0); - } - - #[test] - fn leave_when_nonrunnable_is_a_noop() { - let mut s = ShareState::new_runnable(0); - s.leave_runnable(1_000); - let lag = s.lag(); - s.leave_runnable(9_999_999); - assert_eq!(s.lag(), lag); - } - - #[test] - fn frontier_is_monotonic() { - let f = Frontier::new(); - f.advance(10); - f.advance(5); - assert_eq!(f.get(), 10); - f.advance(11); - assert_eq!(f.get(), 11); - } -} diff --git a/kernel/pure/sched/invariants.rs b/kernel/pure/sched/invariants.rs index e2df886c2d..aaf11c7cfa 100644 --- a/kernel/pure/sched/invariants.rs +++ b/kernel/pure/sched/invariants.rs @@ -26,9 +26,9 @@ pub enum Container { /// can unwind. Its state word reads `Ready`, because that is what it is — /// a task the pick takes before the fair queue. Dying, - /// Stopped for the life of this machine. Its state word reads `Ready` for - /// the reason a dying task's does — it is still counted into its share — - /// but no pick serves this band, so `Ready` here is where it stays. + /// Stopped for the life of this machine. Its state word reads `Ready`, as a + /// dying task's does, but no pick serves this band, so `Ready` here is + /// where it stays. Stopped, Parked, Zombie, @@ -125,7 +125,7 @@ pub fn check_cpu(cpu: &CpuSched) { } /// Invariant T: outside a pass, the armed deadline is no later than the -/// earliest thing that must happen — the running task's quantum end or the +/// earliest thing that must happen — the running task's next decision or the /// earliest valid parked deadline. And a CPU with nothing loaded and a pending /// deadline must have the timer armed at all. pub fn check_timer(cpu: &CpuSched) { @@ -145,9 +145,9 @@ pub fn check_timer(cpu: &CpuSched) { } fn earliest_event(cpu: &CpuSched) -> Option { - let quantum = cpu.running().map(|_| cpu.quantum_end()); + let decision = cpu.running().map(|_| cpu.decide_at()); let deadline = cpu.parked().filter_map(|p| p.deadline()).min(); - match (quantum, deadline) { + match (decision, deadline) { (Some(q), Some(d)) => Some(q.min(d)), (Some(q), None) => Some(q), (None, d) => d, diff --git a/kernel/pure/sched/job.rs b/kernel/pure/sched/job.rs new file mode 100644 index 0000000000..c8fb823e26 --- /dev/null +++ b/kernel/pure/sched/job.rs @@ -0,0 +1,62 @@ +//! A job: the owner every ordinary thread's CPU time is shared out by. A +//! process's threads and its helpers' threads hold the same job, so a helper +//! buys no entitlement: each CPU weighs the group of the job's threads it +//! holds by that group's share of the job's runnable threads. +//! +//! The job counts its runnable ordinary threads on every CPU; each CPU reads +//! the count when it decides, so a change on one CPU reaches the job's groups +//! elsewhere at their own CPUs' next decisions. + +use crate::sched::sync::{AtomicU32, Ordering}; + +/// Monotonic and never reused: a run queue keys the job's group by it. +#[derive(Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord, Debug)] +pub struct JobKey(pub u64); + +/// Every job's weight. Independent jobs share equally, whoever started them. +pub const JOB_WEIGHT: u64 = 1 << 16; + +pub struct Job { + key: JobKey, + /// Runnable ordinary threads of this job, on every CPU. + runnable: AtomicU32, +} + +impl Job { + pub fn new(key: JobKey) -> Self { + Self { key, runnable: AtomicU32::new(0) } + } + + pub fn key(&self) -> JobKey { + self.key + } + + pub fn runnable(&self) -> u32 { + self.runnable.load(Ordering::Relaxed) + } + + pub(crate) fn enter(&self) { + self.runnable.fetch_add(1, Ordering::Relaxed); + } + + pub(crate) fn leave(&self) { + let before = self.runnable.fetch_sub(1, Ordering::Relaxed); + assert!(before > 0, "job {:?}: a thread left that never entered", self.key); + } + + /// The weight of a group holding `here` of the job's runnable threads. + /// + /// Read on the CPU holding them, whose own entries the count already + /// includes: a count below `here` is a bookkeeping bug. + pub fn group_weight(&self, here: u32) -> u64 { + let all = self.runnable(); + assert!(here <= all, "job {:?}: {here} threads queued here of {all} runnable", self.key); + (JOB_WEIGHT * u64::from(here) / u64::from(all)).max(1) + } + + /// What one of its runnable threads weighs wherever it is queued, which a + /// move from one CPU to another shifts between them. + pub fn thread_weight(&self) -> u64 { + (JOB_WEIGHT / u64::from(self.runnable().max(1))).max(1) + } +} diff --git a/kernel/pure/sched/job_sched/checkpoint.rs b/kernel/pure/sched/job_sched/checkpoint.rs index 619d7cb06f..d86f646b6c 100644 --- a/kernel/pure/sched/job_sched/checkpoint.rs +++ b/kernel/pure/sched/job_sched/checkpoint.rs @@ -508,3 +508,34 @@ fn a_desktop_beside_a_build_runs_every_interactive_wake_within_a_frame() { } } } + +/// The stage-3 metal workload (`tests/guest/rust-tests/src/bin/job_fairness.rs`) +/// on the model: two independent builds of `m` busy helpers each beside a +/// helper-heavy job of `4m`, on both benches' shapes. Every job can fill a third +/// of the machine, so each is owed a third whatever its helper count. Asserts +/// each job's share within the tolerance the guest test applies, which a job +/// per helper — the heavy job owed two thirds — breaks by twice over. +#[test] +fn two_builds_and_a_helper_heavy_job_split_the_machine_in_thirds() { + const TOLERANCE: f64 = 0.1; + let shapes: [Shape; 2] = [("t14", t14), ("amd", amd)]; + for (shape, topo) in shapes { + let mut sim = Sim::new(topo(), Params::DEFAULT); + let m = sim.topo.cpus() as u32; + let jobs: Vec<(JobId, u32)> = [m, m, 4 * m].iter().map(|&n| (sim.job(WEIGHT_UNIT), n)).collect(); + for &(job, threads) in &jobs { + for i in 0..threads { + sim.thread(job, Work::Hog, CpuId(i % m), false); + } + } + sim.run_until(SPAN); + let shares: Vec = jobs.iter().map(|&(job, _)| share(&sim, job) / f64::from(m)).collect(); + println!( + "{shape}: machine shares {:.3} {:.3} {:.3}, migrations {}", + shares[0], shares[1], shares[2], sim.migrations + ); + for (i, s) in shares.iter().enumerate() { + assert!((s - 1.0 / 3.0).abs() <= TOLERANCE, "{shape}: job {i} took {s:.3} of the machine"); + } + } +} diff --git a/kernel/pure/sched/mod.rs b/kernel/pure/sched/mod.rs index 8e941cca49..ac4abc72d8 100644 --- a/kernel/pure/sched/mod.rs +++ b/kernel/pure/sched/mod.rs @@ -14,10 +14,10 @@ pub mod cpu; pub mod eevdf; -pub mod fair; pub mod hw; -pub mod job_sched; pub mod invariants; +pub mod job; +pub mod job_sched; pub mod mailbox; pub mod msg; pub mod park; diff --git a/kernel/pure/sched/msg.rs b/kernel/pure/sched/msg.rs index 03dffa8cc4..e748eaa66f 100644 --- a/kernel/pure/sched/msg.rs +++ b/kernel/pure/sched/msg.rs @@ -21,9 +21,10 @@ pub enum Msg { /// on the node embedded in the transferred task's own record, which is /// why a transfer can never be dropped for lack of queue space. Adopt { task: TransitTask }, - /// "If you are overloaded, send me one". Rides on the thief's - /// single reusable probe node. - StealRequest { thief: CpuId }, + /// "Send me a fair thread": from an idle thief (`load` 0) any, from a busy + /// one whose job groups weigh `load` only one whose move brings the two + /// closer. Rides on the thief's single reusable probe node. + StealRequest { thief: CpuId, load: u64 }, /// Kill protocol. Rides on `TaskShared.retire_node`. Retire { shared: Arc>> }, } @@ -62,7 +63,7 @@ impl core::fmt::Debug for Msg { match self { Msg::Wake { key, cause } => write!(f, "Wake({:?}, {:?})", key, cause.reason), Msg::Adopt { task } => write!(f, "Adopt({:?})", task.key()), - Msg::StealRequest { thief } => write!(f, "StealRequest({thief:?})"), + Msg::StealRequest { thief, load } => write!(f, "StealRequest({thief:?}, {load})"), Msg::Retire { shared } => write!(f, "Retire({:?})", shared.key()), } } diff --git a/kernel/pure/sched/queue.rs b/kernel/pure/sched/queue.rs index 774a9b45bf..9782f41012 100644 --- a/kernel/pure/sched/queue.rs +++ b/kernel/pure/sched/queue.rs @@ -1,169 +1,307 @@ -//! The per-CPU run queue. +//! The per-CPU run queue: a real-time FIFO drained first, and the fair band. //! -//! Two bands: an RT FIFO drained first, and a fair band ordered by -//! `(vruntime, insertion sequence)`. True EEVDF virtual-deadline ordering is a -//! later, sim-gated, `queue.rs`/`fair.rs`-only change. +//! The fair band is EEVDF over the job groups this CPU holds, and EEVDF again +//! over each group's threads ([`Eevdf`] at both levels). A group's weight is +//! its job's weight times the group's share of the job's runnable threads +//! ([`Job::group_weight`]), refreshed at each decision. Lag goes with whatever +//! leaves the queue: a thread's to the next queue it enters, and a dissolving +//! group's with its last thread to the group that thread next enters. There +//! is no cross-CPU service accounting (owner decision J1). //! -//! The queue owns [`ReadyTask`] values. A task in a queue is therefore *not* -//! anywhere else — there is no second owner to construct. +//! The queue owns its [`ReadyTask`] values. The running fair thread's entity +//! stays queued, charged as it runs, while its value is the CPU's `running`. use alloc::collections::{BTreeMap, VecDeque}; -use crate::sched::task::{ReadyTask, SchedPayload, TaskKey}; - -/// How the fair band decides between two ready threads of the *same* share — -/// the tie-break simulator invariant I13 exists to hold. -/// -/// The two broken orderings are reproduced for the simulator's negative gates, -/// behind a feature the kernel does not enable, exactly as -/// [`crate::sched::cpu::CpuSched::set_park_keeps_lapsed_lend`] is. A check that cannot -/// tell these apart from the shipped one is not measuring the rule it names. -#[cfg(feature = "protocol-port")] -#[derive(Clone, Copy, PartialEq, Eq, Debug, Default)] -pub enum FairOrder { - /// What the kernel ships: `(vruntime, monotonic insertion sequence)`. - #[default] - InsertSequence, - /// `(vruntime, TaskKey)` — the identity tie-break the field comment below - /// warns against, ported as literally as it is written. - IdentityTiebreak, - /// That warning made total. Whichever share leads the band, its - /// *lowest-keyed* ready thread is the one dispatched, whatever vruntime each - /// of them was inserted with — so the same thread wins every time and not - /// merely every tie. The share's pot advances at the same rate either way, - /// so the process keeps its half of the machine and invariant I5 sees - /// nothing at all. - IdentityWithinShare, +use crate::sched::eevdf::{Eevdf, Lag}; +use crate::sched::hw::Nanos; +use crate::sched::job::{Job, JobKey}; +use crate::sched::sync::Arc; +use crate::sched::task::{ReadyTask, SchedPayload, Standing, TaskKey}; + +/// The request every fair thread and group issues: the longest a thread runs +/// before its CPU decides again, and what bounds the wait of one that queues +/// behind it. +pub const SLICE_NS: u64 = 4_000_000; + +/// The least a fair thread runs before a woken thread or an earlier-due group +/// takes the CPU from it: what bounds the switching a stream of wakes costs. +pub const MIN_SLICE_NS: u64 = 250_000; + +/// The most lag a thread or a group keeps while it is out of every queue, in +/// either direction: no sleeper credit beyond it, and no more debt. +pub const LAG_BOUND_NS: u64 = 4_000_000; + +/// Every thread weighs the same inside its group. +const THREAD_WEIGHT: u64 = 1 << 16; + +struct Group { + job: Arc, + threads: Eevdf, + /// Every queued thread's value but the running one's. + ready: BTreeMap>, } pub struct RunQueue { rt: VecDeque>, - /// Ordered by `(vruntime, insertion sequence)`. - /// - /// The tie-break must **not** be `TaskKey`, and the sequence must be - /// monotonic: a re-inserted thread has to land *behind* its equal-vruntime - /// siblings, or the same thread can win every tie and the others only run - /// when it blocks. - /// - /// What that is worth is smaller than it reads, and the simulator measured - /// it rather than this comment asserting it. A share's pot is - /// charged for every nanosecond any of its threads runs, so a thread - /// re-inserted after a dispatch already carries a key strictly above every - /// sibling queued before it: the band serves a share's threads in insertion - /// order whatever the tie-break is. Exact ties survive only where no charge - /// separates two inserts — a post that wakes siblings, the spawn burst — and - /// one dispatch dissolves them. [`FairOrder::IdentityTiebreak`] is this - /// field's warning ported literally, and it is invisible to simulator - /// invariant I13. The rule stands because the *pot* is doing the work: a - /// policy that stops charging it once per dispatch — an ordered map of - /// shares each holding a FIFO of its ready threads, say — hands the whole - /// job back here, which is what I13 is in place to guard. - fair: BTreeMap<(u64, u64), ReadyTask>, - insert_seq: u64, - /// Negative-gate escape hatch only; see [`FairOrder`]. + groups: Eevdf, + members: BTreeMap>, + /// The running thread, where it is fair: its entity is queued and its + /// value is not. + running: Option<(JobKey, TaskKey)>, + ready_fair: usize, + /// Negative-gate escape hatches only; see + /// [`Self::set_lowest_key_within_group`] and [`Self::set_double_charge`]. #[cfg(feature = "protocol-port")] - order: FairOrder, + lowest_key_within_group: bool, + #[cfg(feature = "protocol-port")] + double_charge: Option, } impl RunQueue { pub fn new() -> Self { Self { rt: VecDeque::new(), - fair: BTreeMap::new(), - insert_seq: 0, + groups: Eevdf::new(), + members: BTreeMap::new(), + running: None, + ready_fair: 0, + #[cfg(feature = "protocol-port")] + lowest_key_within_group: false, #[cfg(feature = "protocol-port")] - order: FairOrder::InsertSequence, + double_charge: None, } } + /// Serve each group's lowest-keyed ready thread rather than its EEVDF + /// pick, so one thread of a job takes all of the job's time on its CPU + /// while its siblings wait. The simulator's progress invariant must catch + /// it; `scenarios::fair_lowest_key_within_group` is the gate. #[cfg(feature = "protocol-port")] - pub fn set_order(&mut self, order: FairOrder) { - self.order = order; + pub fn set_lowest_key_within_group(&mut self, on: bool) { + self.lowest_key_within_group = on; } - /// `vruntime` orders the fair band and is ignored for RT tasks, which - /// round-robin within their band on the same quantum. - pub fn insert(&mut self, vruntime: u64, task: ReadyTask) { + /// Charge `job`'s group twice for what it runs, so its job is throttled + /// for work it never did. The simulator's I5 must catch it; + /// `scenarios::fair_double_charge` is the gate. + #[cfg(feature = "protocol-port")] + pub fn set_double_charge(&mut self, job: Option) { + self.double_charge = job; + } + + /// Queue a task: the real-time band if it holds real-time treatment, + /// otherwise its job's group, carrying the standing it left its last + /// queue with. + pub fn insert(&mut self, task: ReadyTask) { if task.rt().is_rt() { self.rt.push_back(task); } else { - self.insert_seq += 1; - #[cfg(not(feature = "protocol-port"))] - let tie = self.insert_seq; - #[cfg(feature = "protocol-port")] - let tie = match self.order { - FairOrder::IdentityTiebreak => task.key().0, - FairOrder::InsertSequence | FairOrder::IdentityWithinShare => self.insert_seq, - }; - let previous = self.fair.insert((vruntime, tie), task); - assert!( - previous.is_none(), - "two ready tasks with one (vruntime, sequence)", - ); + self.enter(task); } } - /// RT band first, then the lowest-vruntime fair task. - pub fn pop_next(&mut self) -> Option<(u64, ReadyTask)> { - if let Some(task) = self.rt.pop_front() { - return Some((0, task)); + fn enter(&mut self, mut task: ReadyTask) { + let key = task.key(); + let job = task.job().clone(); + let jk = job.key(); + job.enter(); + let standing = task.standing_mut(); + let carried = core::mem::take(&mut standing.group_lag); + let left = standing.left.take(); + match self.members.get(&jk) { + Some(_) if carried != Lag::ZERO => { + let weight = self.groups.weight(jk); + let mut held = self.groups.leave(jk); + held.lag = held.lag.saturating_add(carried).clamp_ns(LAG_BOUND_NS); + self.groups.rejoin(jk, weight, SLICE_NS, held); + } + Some(_) => {} + None => { + self.members.insert(jk, Group { job, threads: Eevdf::new(), ready: BTreeMap::new() }); + // Weighed below, once the thread is counted here. + self.groups.join(jk, 1, SLICE_NS, carried); + } } + let group = self.members.get_mut(&jk).expect("just made"); + match left { + Some(left) => group.threads.rejoin(key, THREAD_WEIGHT, SLICE_NS, left), + None => group.threads.join(key, THREAD_WEIGHT, SLICE_NS, Lag::ZERO), + }; + group.ready.insert(key, task); + self.ready_fair += 1; + self.reweigh(jk); + } + + /// Take the entity of a fair thread out, writing what it takes with it + /// into `standing`. The value is the caller's. + fn leave(&mut self, jk: JobKey, key: TaskKey, standing: &mut Standing) { + let group = self.members.get_mut(&jk).expect("a queued thread's group exists"); + let mut left = group.threads.leave(key); + left.lag = left.lag.clamp_ns(LAG_BOUND_NS); + standing.left = Some(left); + group.job.leave(); + if group.threads.is_empty() { + self.members.remove(&jk); + standing.group_lag = self.groups.leave(jk).lag.clamp_ns(LAG_BOUND_NS); + } else { + self.reweigh(jk); + } + } + + /// Hold `jk`'s group at its current weight on this CPU. + fn reweigh(&mut self, jk: JobKey) { + let group = &self.members[&jk]; + let weight = group.job.group_weight(group.threads.len() as u32); + self.groups.reweight(jk, weight); + } + + /// Every group at the weight its job's runnable count now gives it: what + /// the job's other CPUs changed since this one last decided. + pub fn refresh(&mut self) { + for (&jk, group) in &self.members { + self.groups.reweight(jk, group.job.group_weight(group.threads.len() as u32)); + } + } + + /// The fair band's pick: a group, then one of its threads. + fn pick_fair(&mut self) -> Option<(JobKey, TaskKey)> { + self.refresh(); + let jk = self.groups.pick()?; + let group = &self.members[&jk]; #[cfg(feature = "protocol-port")] - if self.order == FairOrder::IdentityWithinShare { - return self.pop_lowest_key_of_leading_share(); + if self.lowest_key_within_group { + let key = group.threads.keys().min().expect("a queued group has a thread"); + return Some((jk, key)); } - let key = *self.fair.keys().next()?; - let task = self.fair.remove(&key).expect("key came from the map"); - Some((key.0, task)) + Some((jk, group.threads.pick().expect("a queued group has a thread"))) } - /// [`FairOrder::IdentityWithinShare`]: find the share that leads the band, - /// then serve its lowest-keyed ready thread rather than its earliest-keyed - /// one. Which share runs next is unchanged, so the per-process split is - /// untouched; which *thread* of it runs never varies. - #[cfg(feature = "protocol-port")] - fn pop_lowest_key_of_leading_share(&mut self) -> Option<(u64, ReadyTask)> { - let leader = *self.fair.keys().next()?; - let share = self.fair[&leader].share().clone(); - let chosen = *self - .fair - .iter() - .filter(|(_, task)| crate::sched::sync::Arc::ptr_eq(task.share(), &share)) - .min_by_key(|(_, task)| task.key().0) - .expect("the leader is its own share's member") - .0; - let task = self.fair.remove(&chosen).expect("key came from the map"); - Some((chosen.0, task)) - } - - /// The task a [`crate::sched::msg::Msg::StealRequest`] is answered with: the - /// *last* fair task, i.e. the one whose turn is furthest away. Handing - /// over the next-to-run task instead would trade a cache-warm local - /// dispatch for a two-hop transfer. + /// The real-time band first, then the fair band's pick, which becomes the + /// running thread. Asked only with nothing running. + pub fn pop_next(&mut self) -> Option> { + assert!(self.running.is_none(), "a pick with a fair thread still running"); + if let Some(task) = self.rt.pop_front() { + return Some(task); + } + let (jk, key) = self.pick_fair()?; + let task = self.members.get_mut(&jk).and_then(|g| g.ready.remove(&key)).expect("the pick has its value"); + self.ready_fair -= 1; + self.running = Some((jk, key)); + Some(task) + } + + /// Whether `key`, just queued, is what the fair band would run now in + /// place of the running fair thread. + pub fn would_preempt_for(&mut self, key: TaskKey) -> bool { + self.running.is_some() && self.pick_fair().is_some_and(|(_, picked)| picked == key) + } + + /// Whether the running thread's entity is the fair band's. + pub fn running_is_fair(&self) -> bool { + self.running.is_some() + } + + /// The running fair thread received `ns`. + pub fn charge_running(&mut self, ns: u64) { + if let Some((jk, key)) = self.running { + self.groups.charge(jk, ns); + #[cfg(feature = "protocol-port")] + if self.double_charge == Some(jk) { + self.groups.charge(jk, ns); + } + self.members.get_mut(&jk).expect("the running thread's group").threads.charge(key, ns); + } + } + + /// The running task stops running and stays runnable: back to its band, + /// which may have changed while it ran. A fair thread that `yielded` ends + /// its own and its group's requests where they stand, so whatever was due + /// after them runs first. + pub fn put_back(&mut self, mut task: ReadyTask, yielded: bool) { + match self.running.take() { + Some((jk, key)) => { + assert_eq!(key, task.key(), "the task put back is not the running one"); + if task.rt().is_rt() { + self.leave(jk, key, task.standing_mut()); + self.rt.push_back(task); + } else { + if yielded { + self.groups.end_request(jk); + self.members.get_mut(&jk).expect("the running thread's group").threads.end_request(key); + } + self.members.get_mut(&jk).expect("the running thread's group").ready.insert(key, task); + self.ready_fair += 1; + } + } + None => self.insert(task), + } + } + + /// The running task leaves the fair band without going back to it — it + /// blocks, exits, stops or dies — taking its standing into `standing`. + pub fn leave_running(&mut self, key: TaskKey, standing: &mut Standing) { + if let Some((jk, running)) = self.running.take() { + assert_eq!(running, key, "the task leaving is not the running one"); + self.leave(jk, key, standing); + } + } + + /// When the CPU must decide again for its running fair thread, charged up + /// to `now` and running since `since`: the end of its own or its group's + /// request, or a queued group whose request is due before its group's + /// becoming eligible, once it has run [`MIN_SLICE_NS`]. /// - /// **`loaded` is skipped, and that is a correctness rule and not a policy - /// one** — [`crate::sched::cpu::SchedPass::answer_steal_requests`] carries the - /// derivation. It is also the *most likely* candidate here rather than an - /// unlikely one: a task `preempt_if_due` has just returned to the band was - /// charged for the quantum it spent, so its vruntime is the band's highest - /// and `next_back` names it first. - pub fn pop_surplus(&mut self, loaded: Option) -> Option> { - let key = *self - .fair + /// Within a group no such instant is needed: the group's request is as + /// short as its threads', and deciding sooner only switches siblings. + pub fn decision(&self, now: Nanos, since: Nanos) -> Option { + let (jk, key) = self.running?; + let threads = &self.members[&jk].threads; + let left = self.groups.request_left_ns(jk).min(threads.request_left_ns(key)); + let mut at = now.after(left); + if let Some(ns) = self.groups.overtaken_in(jk) { + at = at.min(now.after(ns).max(since.after(MIN_SLICE_NS))); + } + Some(at) + } + + /// The ready fair task a balance request is best answered with, out of + /// the band: the one whose move from this CPU's summed group weight to the + /// asker's `asking` brings the two closest, never `loaded` — the context + /// this CPU still stands on (`SchedPass::answer_steal_requests`). An idle + /// asker is answered with that task whatever the move does; a busy one only + /// if the move narrows the difference by a quarter of the thread's weight, + /// so that floor rounding cannot hand an odd split back and forth. + pub fn pop_surplus(&mut self, loaded: Option, asking: u64) -> Option> { + self.refresh(); + let diff = self.weight() as i128 - asking as i128; + let (jk, key) = self + .members .iter() - .rev() - .find(|(_, task)| Some(task.key()) != loaded)? - .0; - self.fair.remove(&key) + .filter_map(|(&jk, group)| { + let key = group.ready.keys().rev().copied().find(|&key| Some(key) != loaded)?; + let w = group.job.thread_weight() as i128; + (asking == 0 || w + w / 4 <= diff).then_some(((diff - 2 * w).abs(), jk, key)) + }) + .min() + .map(|(_, jk, key)| (jk, key))?; + let mut task = self.members.get_mut(&jk).and_then(|g| g.ready.remove(&key)).expect("a ready task"); + self.ready_fair -= 1; + self.leave(jk, key, task.standing_mut()); + Some(task) } - /// Retire found the task queued rather than parked. + /// Retire found the task queued rather than parked: out of whichever band + /// it is in. pub fn remove(&mut self, key: TaskKey) -> Option> { if let Some(index) = self.rt.iter().position(|t| t.key() == key) { return self.rt.remove(index); } - let found = *self.fair.iter().find(|(_, t)| t.key() == key)?.0; - self.fair.remove(&found) + let jk = *self.members.iter().find(|(_, g)| g.ready.contains_key(&key))?.0; + let mut task = self.members.get_mut(&jk).and_then(|g| g.ready.remove(&key)).expect("just found"); + self.ready_fair -= 1; + self.leave(jk, key, task.standing_mut()); + Some(task) } /// Is an RT task waiting? The preemption decision in `finish()` and @@ -172,34 +310,44 @@ impl RunQueue { !self.rt.is_empty() } + /// Queued values, both bands. pub fn len(&self) -> usize { - self.rt.len() + self.fair.len() + self.rt.len() + self.ready_fair } + /// Queued fair values: the running fair thread is not one. pub fn fair_len(&self) -> usize { - self.fair.len() + self.ready_fair } /// Is `key` queued in the fair band? pub fn holds_fair(&self, key: TaskKey) -> bool { - self.fair.values().any(|task| task.key() == key) + self.members.values().any(|g| g.ready.contains_key(&key)) } pub fn is_empty(&self) -> bool { - self.rt.is_empty() && self.fair.is_empty() + self.len() == 0 + } + + /// The summed group weights: what busy balancing compares. + pub fn weight(&self) -> u64 { + u64::try_from(self.groups.total_weight()).expect("a CPU's group weights sum past u64") } - /// Residents, for the invariant walks. Order is band-then-vruntime, i.e. - /// pick order. + /// Each job's group here: its key, its weight and how many of its threads + /// it holds, the running one included. + pub fn groups(&self) -> impl Iterator + '_ { + self.members.iter().map(|(&jk, g)| (jk, self.groups.weight(jk), g.threads.len())) + } + + /// Residents, for the invariant walks: the real-time band, then each + /// group's ready threads. pub fn keys(&self) -> impl Iterator + '_ { - self.rt - .iter() - .map(|t| t.key()) - .chain(self.fair.values().map(|t| t.key())) + self.tasks().map(|t| t.key()) } pub fn tasks(&self) -> impl Iterator> + '_ { - self.rt.iter().chain(self.fair.values()) + self.rt.iter().chain(self.members.values().flat_map(|g| g.ready.values())) } } diff --git a/kernel/pure/sched/task.rs b/kernel/pure/sched/task.rs index b461608dd7..1791774bea 100644 --- a/kernel/pure/sched/task.rs +++ b/kernel/pure/sched/task.rs @@ -9,11 +9,12 @@ use alloc::boxed::Box; use core::ptr::addr_of_mut; -use crate::sched::fair::{FairShare, ShareState, QUANTUM_NS}; +use crate::sched::eevdf::{Lag, Left}; use crate::sched::hw::{CpuId, Nanos}; +use crate::sched::job::Job; use crate::sched::mailbox::MailboxNode; use crate::sched::msg::Msg; -use crate::sched::sync::{Arc, AtomicBool, AtomicU64, CellLock, Ordering}; +use crate::sched::sync::{Arc, AtomicBool, AtomicU64, Ordering}; use crate::sched::park::CommittedTicket; /// Monotonic, never reused. Stale messages keyed by `TaskKey` are provably @@ -29,19 +30,27 @@ pub trait SchedPayload: Sized + Send + 'static { /// Saved callee context, restored by `Hw::switch`. type Ctx: Sized + Send; - /// The cell the per-process [`FairShare`] lives in. Supplied by the - /// environment because the scheduler core may not implement a lock itself - /// (see [`CellLock`]). - type ShareLock: CellLock + Send; - /// The environment's own name for the task, which a [`crate::sched::hw::TraceEvent`] /// carries beside its key: a key is never reused and names nothing outside /// the core. fn name(&self) -> u64; } -/// Shorthand for the share type a payload implies. -pub type Share = FairShare<::ShareLock>; +/// The round-robin turn of the real-time band and a dying task's turn, and +/// how long a lent real-time window lasts. +pub const QUANTUM_NS: u64 = 10_000_000; + +/// What a thread took with it out of the last ordinary queue it left, for the +/// one it enters next. +#[derive(Clone, Copy, Debug, Default)] +pub struct Standing { + /// Its lag and unfinished request in its group. + pub(crate) left: Option, + /// The lag of the group it was the last to leave: a group's standing + /// against other jobs goes with its last thread to the group that thread + /// enters next. + pub(crate) group_lag: Lag, +} /// Why a task is being woken, and whether the waker lends it RT priority for /// a bounded window. @@ -771,7 +780,8 @@ pub struct Task(Box>); struct TaskInner { key: TaskKey, shared: Arc>>, - share: Arc>, + job: Arc, + standing: Standing, ctx: X::Ctx, rt: RtState, acct: TaskAccounting, @@ -808,8 +818,8 @@ impl Task { &self.0.shared } - pub fn share(&self) -> &Arc> { - &self.0.share + pub fn job(&self) -> &Arc { + &self.0.job } pub fn rt(&self) -> RtState { @@ -878,8 +888,8 @@ macro_rules! linear_state { self.0.shared() } - pub fn share(&self) -> &Arc> { - self.0.share() + pub fn job(&self) -> &Arc { + self.0.job() } pub fn rt(&self) -> RtState { @@ -947,7 +957,7 @@ linear_state!( /// the destination's queue. pub struct TaskBuilder { pub key: TaskKey, - pub share: Arc>, + pub job: Arc, pub ctx: X::Ctx, pub ext: X, pub rt: RtState, @@ -959,7 +969,8 @@ impl TaskBuilder { TransitTask(Task(Box::new(TaskInner { key: self.key, shared, - share: self.share, + job: self.job, + standing: Standing::default(), ctx: self.ctx, rt: self.rt, acct: TaskAccounting::default(), @@ -995,6 +1006,10 @@ impl TransitTask { } impl ReadyTask { + pub(crate) fn standing_mut(&mut self) -> &mut Standing { + &mut self.0 .0.standing + } + /// Entering the dying list: a borrowed RT window ends here, unconditionally /// and exactly as a park ends one. /// @@ -1054,6 +1069,10 @@ impl ReadyTask { } impl RunningTask { + pub(crate) fn standing_mut(&mut self) -> &mut Standing { + &mut self.0 .0.standing + } + /// A retire found this task running: it will unwind and die on this stack, /// so its borrowed RT window ends now. [`ReadyTask::end_lend`] carries the /// argument; this is the arm where the victim never passes through the @@ -1154,7 +1173,7 @@ impl RunningTask { } /// Time consumed since dispatch or since the last charge, folded into the - /// accounting. The pass charges the share with the same number. + /// accounting. The pass charges the run queue with the same number. pub(crate) fn charge(&mut self, now: Nanos) -> u64 { let elapsed = now.since(self.0 .0.since); self.0.charge_residency(now, Residency::Running); diff --git a/kernel/sim/corpus/old_migrate_kept_the_corpse_i14.trace b/kernel/sim/corpus/old_migrate_kept_the_corpse_i14.trace index 81a648fe28..fa5259c59a 100644 --- a/kernel/sim/corpus/old_migrate_kept_the_corpse_i14.trace +++ b/kernel/sim/corpus/old_migrate_kept_the_corpse_i14.trace @@ -6,6 +6,7 @@ # note: sibling asleep: place() forwards the corpse instead of reaping it. # note: re-shrunk after an awake CPU with nothing loaded came to owe its pass at # note: once, which renumbers every decision, and again after a busy CPU is kicked -# note: and a fair wake behind fair work is forwarded too. This replay's corpse -# note: is forwarded as fair work by place(), not by a probe's answer. -2 0 1 1 1 1 1 1 1 1 1 1 1 1 1 2 1 2 1 2 1 2 0 2 +# note: and a fair wake behind fair work is forwarded too, and again for the +# note: job-group scheduler's request-length decisions. +2 0 1 2 0 2 3 0 3 1 3 1 3 0 0 1 1 0 0 1 0 1 0 1 0 2 0 1 0 0 0 0 +3 0 3 0 2 0 0 0 3 1 1 2 0 0 1 1 2 0 1 2 2 2 3 1 3 1 2 0 1 2 1 diff --git a/kernel/sim/corpus/old_steal_port_i8.trace b/kernel/sim/corpus/old_steal_port_i8.trace index 8eaf939743..575c24e7f0 100644 --- a/kernel/sim/corpus/old_steal_port_i8.trace +++ b/kernel/sim/corpus/old_steal_port_i8.trace @@ -1,4 +1,3 @@ # scenario: old_steal_port # expect: fail -0 0 1 1 0 0 1 1 1 1 0 1 0 1 0 0 0 0 0 1 0 0 0 1 1 1 1 0 0 0 1 1 -1 +1 1 1 0 1 0 1 1 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 1 0 1 diff --git a/kernel/sim/src/explore.rs b/kernel/sim/src/explore.rs index d07881b4cb..b55d574cdb 100644 --- a/kernel/sim/src/explore.rs +++ b/kernel/sim/src/explore.rs @@ -60,27 +60,25 @@ pub struct Outcome { pub unwind_gate_ns: u64, /// Invariant I5's measurement: the widest service spread seen over one /// contention window, and the bound in force when it was seen. A number - /// rather than a verdict, because comparing a per-CPU frontier against the - /// global one needs a number: "both passed" is not a comparison. + /// rather than a verdict, so two runs can be compared. pub fair_spread: u64, pub fair_bound: u64, - /// Worst spread past the *derived* bound, even where the recorded sample - /// allowed the run to pass. Zero means the run met the standard. + /// Worst spread past the *derived* bound, even where a lifted ceiling let + /// the run pass. Zero means the run met the standard. pub fair_over_bound: u64, - /// Virtual nanoseconds I5 had a comparison open for. Its *reach*, which is a - /// separate question from its verdict and one that four different conditions - /// can silently shrink; see [`crate::vm::Vm::fair_covered_ns`]. + /// Virtual nanoseconds I5 had a comparison open for: its *reach*, which + /// several conditions can silently shrink; see + /// [`crate::vm::Vm::fair_covered_ns`]. pub fair_covered_ns: u64, - /// Invariant I13's measurement, in the same three roles: the widest service - /// spread between threads of one share, the bound in force when it was - /// seen, and any crossing of the derived bound the allowance let pass. - pub thread_spread: u64, - pub thread_bound: u64, - pub thread_over_bound: u64, - /// Virtual nanoseconds I13 had a comparison open for. Its *reach*, which is - /// a separate question from its verdict and one a change to the pick or the - /// balance can silently shrink; see [`crate::vm::Vm::thread_covered_ns`]. - pub thread_covered_ns: u64, + /// Invariant I13's measurement: the most service a ready fair thread was + /// owed while it waited, the bound in force when it was seen, and the worst + /// crossing of it. + pub wait_owed: u64, + pub wait_bound: u64, + pub wait_over_bound: u64, + /// Virtual nanoseconds some ready fair thread was owed service: I13's + /// reach; see [`crate::vm::Vm::wait_covered_ns`]. + pub wait_covered_ns: u64, /// Per process: how long its threads waited between being owed a dispatch /// and getting one, split by why they were owed one. The measured policy /// suite's instrument — see [`crate::latency`] and `sim/tests/policy.rs`. @@ -192,7 +190,7 @@ impl Outcome { if self.passed() { return format!( "{}: ok ({} steps, {} ns, {} switches, {} kicks, I5 spread {}/{} ns, \ - I13 spread {}/{} ns, I14 retire {}/{} ns)", + I13 owed {}/{} ns, I14 retire {}/{} ns)", self.scenario, self.steps, self.elapsed, @@ -200,8 +198,8 @@ impl Outcome { self.kicks, self.fair_spread, self.fair_bound, - self.thread_spread, - self.thread_bound, + self.wait_owed, + self.wait_bound, self.retire_latency, self.retire_bound, ); @@ -317,10 +315,10 @@ fn outcome_of(scenario: &'static str, vm: &Vm<'_>, choices: &ChoiceStream) -> Ou fair_bound: vm.fair_bound, fair_over_bound: vm.fair_over_bound, fair_covered_ns: vm.fair_covered_ns, - thread_spread: vm.thread_spread, - thread_bound: vm.thread_bound, - thread_over_bound: vm.thread_over_bound, - thread_covered_ns: vm.thread_covered_ns, + wait_owed: vm.wait_owed, + wait_bound: vm.wait_bound, + wait_over_bound: vm.wait_over_bound, + wait_covered_ns: vm.wait_covered_ns, run_wait: vm.run_wait.clone(), process_service_ns: vm.service_ns.clone(), process_finish_ns: vm.finish_ns.clone(), diff --git a/kernel/sim/src/invariants.rs b/kernel/sim/src/invariants.rs index 57597e0439..7c415ae80e 100644 --- a/kernel/sim/src/invariants.rs +++ b/kernel/sim/src/invariants.rs @@ -11,12 +11,12 @@ //! this file owns the protocol above them and does not model memory ordering //! at all. -use std::collections::{BTreeMap, BTreeSet}; +use std::collections::BTreeMap; use std::sync::Arc as StdArc; -use kernel::sched::fair::{MAX_VRUNTIME_LAG_NS, QUANTUM_NS}; use kernel::sched::invariants::{residents, Container}; -use kernel::sched::task::{TaskKey, TaskState}; +use kernel::sched::queue::{LAG_BOUND_NS, SLICE_NS}; +use kernel::sched::task::{TaskKey, TaskState, QUANTUM_NS}; use kernel::sched::cpu::{Balance, DYING_AGE_NS, DYING_CHUNK_NS}; @@ -74,8 +74,8 @@ fn rt_latency_bound(max_kernel_section: u64) -> u64 { /// when the interrupt lands. /// 3. `QUANTUM_NS` — the pass drains the retire and the victim reaches the /// dying list, but the pick can only take it once the CPU is free to switch: -/// a *running* fair task keeps the CPU until its quantum expires, which is -/// what bounds this hop. +/// a *running* task keeps the CPU until its next decision — a fair thread's +/// request or a real-time task's quantum — which is what bounds this hop. /// 4. **`(1 + peers) × UNWIND_NS × STRETCH`** — the unwind itself, the unwinds /// already queued ahead of it on that CPU, and the real-time band's bounded /// share of the same CPU. See [`rt_deferral_stretch`]. @@ -88,8 +88,7 @@ fn rt_latency_bound(max_kernel_section: u64) -> u64 { /// plus `2 × RUN_CHUNK_NS`, invariant I4's granularity term: the model observes /// each hop up to one execution chunk late. /// -/// **`peers` is a workload-shaped term**, exactly as invariant I5's -/// `(runnable threads + 1)` factor is. Two corpses on one CPU means the second +/// **`peers` is a workload-shaped term.** Two corpses on one CPU means the second /// waits out the first: one CPU cannot run two unwinds at once, and pretending /// otherwise would price the machine rather than the protocol. `peers` is the /// greatest number of *other* corpses that CPU has held since this retire was @@ -124,59 +123,26 @@ pub fn check_all(vm: &mut Vm<'_>) { check_retires(vm); let rt_present = note_rt_service(vm); check_fairness(vm, rt_present); - check_share_refcounts(vm); + check_waits(vm); + check_job_counts(vm); check_address_spaces(vm); check_boost_windows(vm); check_work_conservation(vm); note_run_latency(vm); } -/// What one walk of the containers has to yield for both fairness invariants: -/// the per-process runnable counts I5's member test needs, the same counts -/// broken down per CPU for I13's balance test, and the set of individual -/// threads that are owed service. -/// -/// A task in transit between CPUs is in none of the three. It is in no -/// container — invariant I1 requires its state word to say so — it has left its -/// share's refcount, and it is exactly a thread whose competition is changing, -/// which is the one thing I13 declines to measure across. -fn runnable_now(vm: &Vm<'_>) -> (Vec, Vec, BTreeSet) { +/// Each process's fair-band threads, Ready or Running, per CPU: row `process`, +/// column `cpu`. A task in transit is in no band, nor is a corpse or a stopped +/// task: none of them competes for its job's share. +fn fair_per_cpu(vm: &Vm<'_>) -> Vec { let cpus = vm.scenario.cpus(); - let mut counted = vec![0u32; vm.procs.len()]; - let mut per_cpu = vec![0u32; vm.procs.len() * cpus]; - let mut threads = BTreeSet::new(); - for cpu in 0..cpus { - for (key, container) in residents(&vm.cpus[cpu]) { - if !matches!( - container, - Container::Running | Container::Ready | Container::Dying - ) { - continue; - } - threads.insert(key); + let mut counted = vec![0u32; vm.procs.len() * cpus]; + for (cpu, sched) in vm.cpus.iter().enumerate() { + let running = sched.running().filter(|_| sched.rq().running_is_fair()).map(|t| t.key()); + let ready = sched.rq().tasks().filter(|t| !t.rt().is_rt()).map(|t| t.key()); + for key in running.into_iter().chain(ready) { if let Some(process) = vm.process_of(key) { - counted[process] += 1; - per_cpu[process * cpus + cpu] += 1; - } - } - } - (counted, per_cpu, threads) -} - -/// How many Ready-or-Running tasks each process has right now. Invariants I5 -/// and I6 both need it and it is the more expensive half of either. -fn runnable_per_process(vm: &Vm<'_>) -> Vec { - let mut counted = vec![0u32; vm.procs.len()]; - for cpu in 0..vm.scenario.cpus() { - for (key, container) in residents(&vm.cpus[cpu]) { - if !matches!( - container, - Container::Running | Container::Ready | Container::Dying - ) { - continue; - } - if let Some(process) = vm.process_of(key) { - counted[process] += 1; + counted[process * cpus + cpu] += 1; } } } @@ -408,7 +374,7 @@ fn retire_elapsed(vm: &Vm<'_>, key: TaskKey) -> u64 { fn check_timers(vm: &mut Vm<'_>) { for cpu in 0..vm.scenario.cpus() { let armed = vm.cpus[cpu].armed(); - let quantum = vm.cpus[cpu].running().map(|_| vm.cpus[cpu].quantum_end()); + let quantum = vm.cpus[cpu].running().map(|_| vm.cpus[cpu].decide_at()); let deadline = vm.cpus[cpu].parked().filter_map(|p| p.deadline()).min(); let earliest = match (quantum, deadline) { (Some(q), Some(d)) => Some(q.min(d)), @@ -502,188 +468,62 @@ fn note_rt_service(vm: &mut Vm<'_>) -> bool { occupied } -/// I5: **equal shares receive equal service, to within the granularity the -/// policy chooses.** +/// I5: **equal jobs receive equal service on a machine that holds them alike.** /// /// Fairness is a statement about service, so this measures service — the /// nanoseconds the virtual CPUs actually delivered to each process — and not -/// the vruntime bookkeeping that is supposed to produce it. Checking the -/// bookkeeping against itself is how an instrument stops measuring: a lag that -/// `ShareState::leave_runnable` clamps on the way in satisfies `|lag| ≤ 50 ms` -/// no matter what the scheduler did with the CPU. -/// -/// **The window.** Fairness owes nothing across a block: a process with no -/// runnable thread is not being starved, it is waiting. Nor on an unsaturated -/// machine: a CPU with nothing on it is denying nobody. Nor to a process with -/// fewer runnable threads than its even share of the CPUs, which is limited by -/// its own thread count and not by the scheduler. Nor while the RT band is -/// occupied, because the RT band exists to be unfair and invariant I4 is what -/// bounds it. So service is compared over a *contention window* — a maximal -/// interval during which the same set of fair-band processes was continuously -/// runnable, every CPU had a task loaded, every member could absorb its share, -/// and the RT band was empty — and the comparison restarts the moment any of -/// those stops holding. In a workload where everyone blocks the windows are -/// short and this says little; in `fairness_storm`, where nothing blocks and no -/// RT exists, the window is the whole run and it says everything. -/// -/// **The bound is derived, and it does not move.** Two terms, both of them a -/// granularity the policy picked: -/// -/// * `lag_spread` — the stored lags of the contending shares. A share that -/// parked 50 ms behind the frontier is *entitled* to that much catch-up, so -/// the clamp and the intended service difference are the same number. -/// Asserted against `MAX_VRUNTIME_LAG_NS` here rather than assumed, because -/// the bound is only worth what the clamp is worth. -/// * `(runnable threads + 1) × (QUANTUM + max KernelSection + 2 × RUN_CHUNK)` — -/// the fair band is keyed by the vruntime a task had *when it was inserted*, -/// ties broken by insertion sequence, so a process with T threads carries up -/// to T−1 of its own dispatches' worth of stale keys and can be picked that -/// many times over before its slowest thread comes up. Both sides carry it -/// and the leader is spending one more quantum on top, hence `ΣT_i + 1`. -/// The kernel-section and chunk terms are I9's, for I9's reason: a -/// preempt-off section overruns the quantum it started in, and the model -/// observes the expiry one chunk late. -/// -/// Earlier drafts of this check widened that expression twice, because the -/// shipped scheduler brushed it — 74 ms against 72, then 109 ms against 108. -/// Widening was the wrong move and the near-misses were the signal: a bound -/// calibrated to what the code already does cannot detect the code getting -/// worse. The bound is therefore back to the derived form, and where the shipped -/// scheduler does not meet it, that is recorded as a gap rather than absorbed -/// (`scenarios::FAIRNESS_SAMPLE`, which has the measurements and names the four -/// widths of ten where the standard is currently crossed). -/// -/// **What reds a run** is `max(derived bound, recorded allowance)`, so a -/// scenario with a sample is gated on not regressing against it. The allowance -/// never hides the standard: `Vm::fair_over_bound` records every crossing of the -/// derived bound whatever the allowance permits, and the sweep prints it. +/// the bookkeeping that is supposed to produce it. +/// +/// **The window.** Fairness owes nothing across a block, nor on an unsaturated +/// machine, nor while the RT band is occupied (invariant I4 bounds that). And a +/// job's aggregate share follows its placement (owner decision J1): each CPU +/// splits its own time among the groups it holds, so two jobs placed unalike +/// are owed unalike. So service is compared over a *contention window* — a +/// maximal interval during which the same fair-band processes were runnable, +/// every CPU had a task loaded, the RT band was empty, and every CPU held the +/// same number of each member's runnable threads, which gives every member's +/// group the same weight on every CPU — and restarts when any of those stops +/// holding. +/// +/// **The bound is derived.** On each CPU EEVDF holds every group's lag inside +/// one request plus the granularity a decision is taken late by (Stoica and +/// Abdel-Wahab, Theorem 3): the slice, a preempt-off section that overruns it, +/// and the two chunks the model observes an expiry late by. A window's spread +/// on one CPU is at most twice that, plus the lag a group carries in when it +/// re-forms, and the machine's is the sum over its CPUs. +/// +/// **What reds a run** is `max(derived bound, lifted ceiling)`; only a control +/// measuring a rejected policy lifts it, and `Vm::fair_over_bound` records +/// every crossing of the derived bound regardless. fn check_fairness(vm: &mut Vm<'_>, rt_present: bool) { - // Both cleared here and re-established only by a comparison that actually - // happened, so `Vm::thread_covered_ns` and `Vm::fair_covered_ns` count the - // reach each check has and not the reach it would have if every window - // stayed open. I5's is cleared on the same line as I13's because every one - // of the four conditions below can close it — an RT task, a CPU going idle, - // the member set changing, or a member falling under its even share — and a - // flag cleared anywhere but the top would miss whichever of them returns - // early. vm.fair_window_open = false; - vm.thread_window_open = false; - let (runnable, per_cpu, live_threads) = runnable_now(vm); - let saturated = (0..vm.scenario.cpus()).all(|cpu| vm.cpus[cpu].running().is_some()); + let cpus = vm.scenario.cpus(); + let per_cpu = fair_per_cpu(vm); + let saturated = (0..cpus).all(|cpu| vm.cpus[cpu].running().is_some()); + let row = |p: usize| &per_cpu[p * cpus..(p + 1) * cpus]; let mut members: Vec = if rt_present || !saturated { Vec::new() } else { (0..vm.procs.len()) - .filter(|&p| runnable[p] > 0 && !vm.procs[p].rt_service) + .filter(|&p| row(p).iter().any(|&n| n > 0) && !vm.procs[p].rt_service) .collect() }; - // A process cannot run on more CPUs than it has runnable threads, so one - // with fewer than its even share of the machine is limited by its own thread - // count and not by the scheduler. Measuring it would be measuring the - // workload: at 16 CPUs it is what the late windows of `fairness_storm` are, - // once enough threads have exited that the wide process is the only one that - // can still fill the machine. - if members - .iter() - .any(|&p| runnable[p] as usize * members.len() < vm.scenario.cpus()) - { + if !members.iter().all(|&p| row(p).iter().min() == row(p).iter().max()) { members.clear(); } - - // I13's narrowing: every CPU must carry the same number of each member's - // runnable threads. Two siblings on differently composed CPUs are limited by - // where they were placed and not by the order the fair band picks them in, - // and the policy makes no per-thread placement promise to hold them to. - // Measured rather than assumed, because it is what `fairness_storm`'s - // "balanced by construction" is worth once threads start exiting: at two - // CPUs the shipped scheduler separates two trio threads by 49 ms with one of - // them sharing a CPU with a solo thread that is catching up, which is I5's - // entitlement being honoured and not a sibling being cheated. - let balanced = members.iter().all(|&process| { - let row = &per_cpu[process * vm.scenario.cpus()..(process + 1) * vm.scenario.cpus()]; - row.iter().min() == row.iter().max() - }); - let threads: u32 = members.iter().map(|&p| runnable[p]).sum(); - // What one CPU's fair band holds. Read off CPU 0 rather than divided out of - // the machine-wide total, so it is the quantity itself and not an average - // that happens to equal it; `balanced` is what makes every CPU's the same. - let rivals: u32 = members.iter().map(|&p| per_cpu[p * vm.scenario.cpus()]).sum(); - if members != vm.fair_epoch.members { - vm.fair_epoch = FairEpoch { - members, - base: vm.service_ns.clone(), - threads: 0, - lag_spread: 0, - thread_base: BTreeMap::new(), - thread_rivals: 0, - }; - open_thread_window(vm, balanced, &live_threads); + vm.fair_epoch = FairEpoch { members, base: vm.service_ns.clone() }; return; } - if !balanced { - vm.fair_epoch.thread_base.clear(); - } else if vm.fair_epoch.thread_base.is_empty() { - open_thread_window(vm, balanced, &live_threads); - } else { - // A thread that stopped being runnable is owed nothing further, so it - // drops out of the comparison for the rest of this window rather than - // closing it: a share whose threads exit one by one is still a share - // whose *remaining* threads must be sharing evenly. - vm.fair_epoch - .thread_base - .retain(|key, _| live_threads.contains(key)); - vm.fair_epoch.thread_rivals = vm.fair_epoch.thread_rivals.max(rivals); - check_thread_service(vm, &members); - } - if members.len() < 2 { return; } - vm.fair_epoch.threads = vm.fair_epoch.threads.max(threads); - - let mut lag_low = i64::MAX; - let mut lag_high = i64::MIN; - let mut over_clamp = Vec::new(); - for &process in &members { - for share in &vm.procs[process].shares { - let lag = share.lag(); - if lag.unsigned_abs() > MAX_VRUNTIME_LAG_NS { - over_clamp.push(format!( - "I5: {}'s stored lag is {lag} ns, past the ±{MAX_VRUNTIME_LAG_NS} ns clamp \ - the service bound is derived from", - vm.procs[process].name, - )); - } - lag_low = lag_low.min(lag); - lag_high = lag_high.max(lag); - } - } - for problem in over_clamp { - vm.violate(problem); - } - - vm.fair_epoch.lag_spread = vm - .fair_epoch - .lag_spread - .max(lag_high.saturating_sub(lag_low).unsigned_abs()); - - let bound = vm.fair_epoch.lag_spread - + (vm.fair_epoch.threads as u64 + 1) - * (QUANTUM_NS + vm.max_kernel_section() + 2 * RUN_CHUNK_NS); - let served: Vec = members - .iter() - .map(|&p| vm.service_ns[p] - vm.fair_epoch.base[p]) - .collect(); - let (low, high) = ( - served.iter().copied().min().unwrap_or(0), - served.iter().copied().max().unwrap_or(0), - ); + let bound = cpus as u64 * 2 * (SLICE_NS + LAG_BOUND_NS + vm.max_kernel_section() + 2 * RUN_CHUNK_NS); + let served: Vec = members.iter().map(|&p| vm.service_ns[p] - vm.fair_epoch.base[p]).collect(); + let (low, high) = (served.iter().copied().min().unwrap_or(0), served.iter().copied().max().unwrap_or(0)); let spread = high - low; - // The comparison is made from here on, so this is where the window counts as - // open. Set after the early returns rather than before them: `members` being - // unchanged is not enough — a window that restarted this step has a zero - // baseline and separates nothing yet. + // Set after the early returns: a window that restarted this step has a + // zero baseline and separates nothing yet. vm.fair_window_open = true; if spread > vm.fair_spread { vm.fair_spread = spread; @@ -692,180 +532,51 @@ fn check_fairness(vm: &mut Vm<'_>, rt_present: bool) { if spread > bound { vm.fair_over_bound = vm.fair_over_bound.max(spread); } - // The ceiling is the bound, except where a recorded sample says the shipped - // scheduler does not meet it (`scenarios::FAIRNESS_SAMPLE`). There the run is - // gated on not getting *worse*, and the gap between the two is reported by - // `Outcome::fair_over_bound` on every run rather than left in a document. let ceiling = bound.max(vm.scenario.fair_allowance_ns); if spread > ceiling { - let detail: Vec = members - .iter() - .zip(&served) - .map(|(&p, ns)| format!("{}={ns}", vm.procs[p].name)) - .collect(); + let detail: Vec = + members.iter().zip(&served).map(|(&p, ns)| format!("{}={ns}", vm.procs[p].name)).collect(); vm.violate(format!( - "I5: {spread} ns of service separates equal shares over one contention \ - window (bound {bound} ns, recorded allowance {} ns): {}", + "I5: {spread} ns of service separates equal jobs over one contention window \ + (bound {bound} ns, lifted ceiling {} ns): {}", vm.scenario.fair_allowance_ns, detail.join(" "), )); } } -/// (Re-)baseline invariant I13's measurement at this instant: every runnable -/// thread of a member, with the service it has had so far. A no-op while the -/// members' threads are unevenly spread across the CPUs, because that is -/// exactly the interval the comparison declines to make. -fn open_thread_window(vm: &mut Vm<'_>, balanced: bool, live_threads: &BTreeSet) { - if !balanced { +/// I13: **a ready fair thread is not passed over without bound**, measured as +/// the service its CPU owed it while it waited ([`Vm::owed_waiting`]): its +/// group's weight over the CPU's summed group weight, shared among the group's +/// threads there, for as long as the CPU ran fair work. +/// +/// The bound is the host model's (`kernel/pure/sched/job_sched.rs`): a +/// thread's group waits out the request running ahead of it, the EEVDF lag +/// bound and the lag it carried in, and the thread does the same inside its +/// group — `3 × SLICE + 2 × LAG_BOUND` — plus, at each of the two levels, a +/// preempt-off section and the two chunks a decision is observed late by. +/// Equal-service checks cannot see a group that runs one thread and starves +/// the rest: its job's total is unchanged. This can, and +/// `scenarios::fair_lowest_key_within_group` is its gate. +fn check_waits(vm: &mut Vm<'_>) { + let bound = 3 * SLICE_NS + 2 * LAG_BOUND_NS + 2 * (vm.max_kernel_section() + 2 * RUN_CHUNK_NS); + let Some((&key, &owed)) = vm.owed_waiting.iter().max_by(|a, b| a.1.total_cmp(b.1)) else { return; - } - let base: BTreeMap = vm - .fair_epoch - .members - .iter() - .flat_map(|&process| vm.procs[process].live.iter().copied()) - .filter(|key| live_threads.contains(key)) - .map(|key| (key, vm.thread_service.get(&key).copied().unwrap_or(0))) - .collect(); - vm.fair_epoch.thread_base = base; - vm.fair_epoch.thread_rivals = 0; -} - -/// I13: **threads of one share receive equal service**, measured over the same -/// contention windows invariant I5 measures processes over. -/// -/// I5 is structurally blind to this. A fair share is per *process*, so every -/// thread of a process charges one pot and `service_ns` adds all of -/// them together: a share that runs one thread flat out and never dispatches -/// its siblings delivers exactly the per-process total that a share -/// round-robining them delivers, and I5 reports a perfectly even split while a -/// thread never runs. What prevents that today is the fair band's -/// insertion-sequence tie-break (`queue.rs`), and this check is what measures -/// it. -/// -/// **The window is I5's**, taken from the same `fair_epoch` and opened and -/// closed by the same rules — the runnable set changing, a CPU idling, a member -/// with fewer runnable threads than its even share of the CPUs, an occupied RT -/// band. Two definitions of one word would be a defect rather than a design. -/// I13's is that interval with three changes, two of them narrowings: -/// -/// * **One further narrowing of its own: every CPU must carry the same number -/// of each member's runnable threads.** Two siblings on differently composed -/// CPUs are limited by where they were placed, and the policy makes no -/// per-thread placement promise to hold them to; I13's window re-baselines -/// the instant that stops holding, so it is a sub-interval of I5's rather -/// than a second notion of contention. This is not a hypothetical: at two -/// CPUs the shipped scheduler separates two `trio` threads by 49 ms with one -/// of them sharing a CPU with a `solo` thread that is catching up — I5's -/// entitlement being honoured, not a sibling being cheated. -/// * The measured set is fixed when the window opens and only ever shrinks. A -/// thread that blocks or exits mid-window is owed nothing further and leaves -/// the comparison — it does not close the window, because the siblings still -/// runnable are still owed an even split. -/// * A window with a *single* member is still measured, which is the one -/// relaxation. I5 needs two processes before it has a spread at all; one -/// process with two threads is already a comparison. -/// -/// **The bound is derived, and it does not move.** It is I5's bound with the -/// lag term deleted, and the deletion is the whole content of it: -/// -/// * `(ΣT_i + 1) × (QUANTUM + max KernelSection + 2 × RUN_CHUNK)` — I5's own -/// staleness term, unchanged and computed from the same running maximum. The -/// fair band is keyed by the vruntime a thread held when it was *inserted*, -/// and a dispatched thread is re-inserted with its share's pot as it stands -/// after the charge — strictly above the key it was holding. So every other -/// ready thread in the window, sibling or not, can be picked at most once -/// ahead of a waiting thread on the strength of a key that was already stale -/// when the wait began, and the leader spends one more dispatch on top. It is -/// `ΣT_i` and not the share's own thread count because a thread waits behind -/// the *whole* fair band on its CPU: an earlier draft of this check counted -/// only siblings, which is a derivation that assumes the rest of the machine -/// away, and the shipped scheduler crossed it at every width above one. The -/// running maximum is I13's own (`thread_threads`) because I13's window is -/// the shorter of the two. -/// * **No lag term.** I5 carries `lag_spread` because two different shares may -/// hold stored lags up to ±[`MAX_VRUNTIME_LAG_NS`] apart and are entitled to -/// that much catch-up. [`kernel::sched::fair::ShareState`] holds one vruntime -/// and one lag for every thread of a process, so the intra-share lag spread -/// is identically zero — there is no per-thread entitlement to be behind. -/// That absence is exactly why a per-share check can be tighter than I5, and -/// why it can see what I5 cannot. -/// -/// **What it does not cover is placement.** The insertion-order argument is -/// about one run queue, and two threads of one share on differently loaded CPUs -/// sit in two of them; nothing in the policy equalizes those, and a check that -/// asserted it would be measuring the workload. `scenarios::fairness_storm` -/// hands every CPU the identical mix by construction (its own doc says why), -/// which is what makes the number reported there a statement about the -/// ordering. -/// -/// **What reds a run** is `max(derived bound, recorded allowance)`, and -/// `Vm::thread_over_bound` records every crossing of the derived bound whatever -/// the allowance permits. I5's recording pattern, for I5's reason: an allowance -/// that can quietly become the standard is a gate that has stopped measuring. -fn check_thread_service(vm: &mut Vm<'_>, members: &[usize]) { - let bound = (u64::from(vm.fair_epoch.thread_rivals) + 1) - * (QUANTUM_NS + vm.max_kernel_section() + 2 * RUN_CHUNK_NS); - // Measured without allocating per member: this runs after every step of - // every scenario, and the detail line is built only where one is owed. - let mut measured: Vec<(usize, u64)> = Vec::new(); - for &process in members { - let (mut low, mut high, mut counted) = (u64::MAX, 0u64, 0usize); - for key in &vm.procs[process].live { - let Some(base) = vm.fair_epoch.thread_base.get(key) else { - continue; - }; - let served = vm.thread_service.get(key).copied().unwrap_or(0) - base; - low = low.min(served); - high = high.max(served); - counted += 1; - } - if counted < 2 { - continue; - } - measured.push((process, high - low)); - } - vm.thread_window_open = !measured.is_empty(); - - let mut problems = Vec::new(); - for (process, spread) in measured { - if spread > vm.thread_spread { - vm.thread_spread = spread; - vm.thread_bound = bound; - } - if spread > bound { - vm.thread_over_bound = vm.thread_over_bound.max(spread); - } - if spread <= bound.max(vm.scenario.thread_allowance_ns) { - continue; - } - let detail: Vec = vm.procs[process] - .live - .iter() - .filter_map(|key| { - let base = vm.fair_epoch.thread_base.get(key)?; - let served = vm.thread_service.get(key).copied().unwrap_or(0) - base; - // Which run queue each thread is in, because that is the first - // question a per-thread spread raises and the answer is one - // scan on a path that has already failed. - let at = (0..vm.scenario.cpus()) - .find(|&cpu| residents(&vm.cpus[cpu]).any(|(k, _)| k == *key)) - .map_or_else(|| "?".to_string(), |cpu| cpu.to_string()); - Some(format!("{}@cpu{at}={served}", key.0)) - }) - .collect(); - problems.push(format!( - "I13: {spread} ns of service separates threads of {}'s one share over one \ - contention window (bound {bound} ns, recorded allowance {} ns): {}", - vm.procs[process].name, - vm.scenario.thread_allowance_ns, - detail.join(" "), + }; + let owed = owed as u64; + if owed > vm.wait_owed { + vm.wait_owed = owed; + vm.wait_bound = bound; + } + if owed > bound { + vm.wait_over_bound = vm.wait_over_bound.max(owed); + let at = (0..vm.scenario.cpus()) + .find(|&cpu| residents(&vm.cpus[cpu]).any(|(k, _)| k == key)) + .map_or_else(|| "in transit".to_string(), |cpu| format!("on cpu{cpu}")); + vm.violate(format!( + "I13: {key:?} waited {at} through {owed} ns of service its CPU owed it (bound {bound} ns)", )); } - for problem in problems { - vm.violate(problem); - } } /// I15: **work conservation.** No CPU spends more than @@ -1044,26 +755,29 @@ fn note_run_latency(vm: &mut Vm<'_>) { vm.prev_container = seen; } -/// I6: `FairShare.runnable_threads` equals the actual Ready+Running count of -/// that share. The refcount and the containers are driven by the same linear -/// moves, so a drift means a transition forgot one of the two. -fn check_share_refcounts(vm: &mut Vm<'_>) { - let counted = runnable_per_process(vm); +/// I6: each job's runnable count equals its threads in a fair band, Ready or +/// Running, on every CPU. The count and the bands are driven by the same moves, +/// so a drift means a transition forgot one of the two. +fn check_job_counts(vm: &mut Vm<'_>) { + let mut in_band: BTreeMap = BTreeMap::new(); + for sched in &vm.cpus { + let running = sched.running().filter(|_| sched.rq().running_is_fair()).map(|t| t.job().key()); + for job in running.into_iter().chain(sched.rq().tasks().filter(|t| !t.rt().is_rt()).map(|t| t.job().key())) { + *in_band.entry(job).or_default() += 1; + } + } let mut problems = Vec::new(); - for (process, expected) in counted.iter().enumerate() { - // A sum, because a process holds one share under the shipped policy and - // one per thread under the `PerThread` negative gate. With one share it - // is a single `runnable_threads()` read. - let actual: u32 = vm.procs[process] - .shares - .iter() - .map(|share| share.runnable_threads()) - .sum(); - if actual != *expected { - problems.push(format!( - "I6: {} has {actual} runnable thread(s) counted but {expected} in queues", - vm.procs[process].name, - )); + for process in &vm.procs { + for job in &process.jobs { + let expected = in_band.get(&job.key()).copied().unwrap_or(0); + if job.runnable() != expected { + problems.push(format!( + "I6: {}'s job {:?} counts {} runnable thread(s) but {expected} are in a fair band", + process.name, + job.key(), + job.runnable(), + )); + } } } for problem in problems { @@ -1112,7 +826,7 @@ fn check_boost_windows(vm: &mut Vm<'_>) { // a preempt-off section can overrun the quantum it started inside, and the // model advances the clock in chunks, so the quantum's expiry is observed // one chunk late and the pass that acts on it lands a chunk after that. - let bound = kernel::sched::fair::QUANTUM_NS + vm.max_kernel_section() + 2 * RUN_CHUNK_NS; + let bound = QUANTUM_NS + vm.max_kernel_section() + 2 * RUN_CHUNK_NS; let problems: Vec = vm .boosted_run .iter() diff --git a/kernel/sim/src/main.rs b/kernel/sim/src/main.rs index 4a2f8741c6..88577b9e24 100644 --- a/kernel/sim/src/main.rs +++ b/kernel/sim/src/main.rs @@ -52,12 +52,11 @@ fn main() -> ExitCode { println!("old_rt_starved_the_corpse (negative gate: must fail)"); println!("old_park_kept_the_lend (negative gate: must fail)"); println!("old_preemptible_window (negative gate: must abort)"); - println!("fair_share_per_thread (negative gate: must fail)"); + println!("fair_job_per_thread (negative gate: must fail)"); println!("fair_double_charge (negative gate: must fail)"); - println!("fair_identity_within_share (negative gate: must fail)"); + println!("fair_lowest_key_within_group (negative gate: must fail)"); println!("overlong_pass (negative gate: must abort)"); println!("old_commit_fused (control: passes, and that is the point)"); - println!("fair_identity_tiebreak (control: passes, and that is the point)"); ExitCode::SUCCESS } "run" | "pct" => { @@ -118,9 +117,9 @@ fn main() -> ExitCode { scenarios::old_migrate_kept_the_corpse(), scenarios::old_rt_starved_the_corpse(), scenarios::old_park_kept_the_lend(), - scenarios::fair_share_per_thread(), + scenarios::fair_job_per_thread(), scenarios::fair_double_charge(), - scenarios::fair_identity_within_share(), + scenarios::fair_lowest_key_within_group(), ] { let name = negative.name; let result = sweep::seed_sweep(&negative, budget.min(500), 1); @@ -162,34 +161,22 @@ fn main() -> ExitCode { } clean &= caught.is_some(); } - // And the controls, both of which must come back *clean*: the - // same blocking shape with the two halves fused into one step, - // which is the blind spot this harness used to have, and the - // identity *tie-break*, which is the break `queue.rs` warns - // about and which the shipped pot makes a no-op. - for (control, why) in [ - ( - scenarios::old_commit_fused(), - "clean (the control: no step boundary, no bug in sight)", - ), - ( - scenarios::fair_identity_tiebreak(), - "clean (the control: the pot already orders siblings)", - ), - ] { - let name = control.name; - let result = sweep::seed_sweep(&control, budget.min(500), 1); - println!( - "{name}: {} runs, {}", - result.runs, - if result.passed() { - why - } else { - "FAILED — the control no longer controls for anything" - }, - ); - clean &= result.passed(); - } + // And the control, which must come back *clean*: the same + // blocking shape with the two halves fused into one step, which + // is the blind spot this harness used to have. + let control = scenarios::old_commit_fused(); + let result = sweep::seed_sweep(&control, budget.min(500), 1); + println!( + "{}: {} runs, {}", + control.name, + result.runs, + if result.passed() { + "clean (the control: no step boundary, no bug in sight)" + } else { + "FAILED — the control no longer controls for anything" + }, + ); + clean &= result.passed(); } ok(clean) } diff --git a/kernel/sim/src/payload.rs b/kernel/sim/src/payload.rs index 3cf8343821..88be94476d 100644 --- a/kernel/sim/src/payload.rs +++ b/kernel/sim/src/payload.rs @@ -9,7 +9,6 @@ use std::sync::{Arc, Mutex}; -use kernel::sched::fair::ShareState; use kernel::sched::mailbox::PreemptGuard; use kernel::sched::sync::CellLock; use kernel::sched::task::{SchedPayload, TaskKey}; @@ -36,7 +35,6 @@ impl CellLock for StdLock { } pub type SimWatchList = StdLock>; -pub type SimShareLock = StdLock; /// The simulator's waiters are all threads: no process in it polls, so a /// ring entry has no value to be. @@ -88,7 +86,6 @@ pub struct SimCtx { impl SchedPayload for SimPayload { type Ctx = SimCtx; - type ShareLock = SimShareLock; fn name(&self) -> u64 { self.key.0 diff --git a/kernel/sim/src/scenarios.rs b/kernel/sim/src/scenarios.rs index fdfc5b8886..cbb7269164 100644 --- a/kernel/sim/src/scenarios.rs +++ b/kernel/sim/src/scenarios.rs @@ -6,15 +6,13 @@ //! schedule per second instead of a thousand. use kernel::sched::cpu::Balance; -use kernel::sched::queue::FairOrder; use kernel::sched::task::WaitClass; use crate::topology::Shape; use crate::vm::RUN_CHUNK_NS; use crate::workload::{ - AgeShape, BlockShape, ChargeShape, IrqSpec, MigrateShape, Op, ParkShape, - PlacementShape, ProcSpec, Protocol, QueueSpec, Scenario, Script, ShareShape, SurplusShape, - WindowShape, + AgeShape, BlockShape, ChargeShape, GroupPick, IrqSpec, JobShape, MigrateShape, Op, ParkShape, + PlacementShape, ProcSpec, Protocol, QueueSpec, Scenario, Script, SurplusShape, WindowShape, }; const MS: u64 = 1_000_000; @@ -41,7 +39,7 @@ fn scenario( park: ParkShape::ReleaseLend, migrate: MigrateShape::ReapTheCorpse, age: AgeShape::BoundedDeferral, - share: ShareShape::PerProcess, + job: JobShape::PerProcess, charge: ChargeShape::Honest, placement: PlacementShape::LeastLoadedRotating, stopped: None, @@ -51,10 +49,9 @@ fn scenario( threshold: kernel::sched::cpu::PUSH_THRESHOLD, }, surplus: SurplusShape::Published, - order: FairOrder::InsertSequence, + group_pick: GroupPick::Eevdf, pass_cost_ns: 0, fair_allowance_ns: 0, - thread_allowance_ns: 0, max_steps: 20_000, max_tasks: 32, } @@ -705,106 +702,6 @@ pub fn fork_storm() -> Scenario { ) } -/// **The recorded fairness sample.** What the shipped scheduler's worst -/// invariant-I5 service spread over one contention window actually *is*, per -/// machine width — as opposed to what the derived bound says it should be. -/// -/// The bound in `invariants::check_fairness` is derived from the policy's own -/// granularity and is **not** moved when the shipped code misses it; that would -/// fit the gate to the implementation, and a gate fitted to what the code -/// already does cannot detect the code getting worse. So the derived bound stays -/// the standard, this table records where we are, and the two are compared on -/// every run: `Outcome::fair_over_bound` reports any crossing of the derived -/// bound whatever this table allows, so the allowance can hide a red suite but -/// never the gap. -/// -/// Provenance — every number below came from this command, re-taken over all ten -/// widths at `ae974921`, on one host, with no other measurement running: -/// -/// ```text -/// cargo run --release -p kernel-sim -- measure fairness_storm: 500 -/// ``` -/// -/// The last column is `Outcome::fair_over_bound`, and it is **not** the amount -/// by which the bound was crossed: it is the widest spread seen in a window -/// whose spread exceeded that window's own bound, which is a different quantity -/// from the row's worst spread and from the row's bound. Row 4 is what shows it -/// — 198 ms under a 204 ms bound, and a narrower window crossed anyway. -/// -/// | cpus | worst spread | derived bound | `fair_over_bound` | -/// |---|---|---|---| -/// | 1 | 30 ms | 60 ms | none | -/// | 2 | 84 ms | 108 ms | none | -/// | 3 | 125 ms | 156 ms | none | -/// | 4 | 198 ms | 204 ms | **112 ms** | -/// | 6 | 324 ms | 300 ms | **324 ms** | -/// | 8 | 418 ms | 396 ms | **418 ms** | -/// | 12 | 634 ms | 588 ms | **634 ms** | -/// | 16 | 720 ms | 780 ms | none | -/// | 24 | 1056 ms | 1164 ms | none | -/// | 32 | 1386 ms | 1548 ms | none | -/// -/// Widths 1 and 2 were additionally run at 10 000 seeds — the count `gate` -/// uses — giving 30 ms and 102 ms. One CPU is stable at 30 ms across 400, 500 -/// and 10 000 seeds; two CPUs grows with the sample, which is what a -/// worst-of-N statistic does and why the allowance below carries a margin. -/// -/// Two things this table says out loud. The fair split degrades as the machine -/// widens — 30 ms of spread at one CPU against 1386 ms at 32, which is not -/// noise and is filed as a known issue. And the shipped scheduler sits *on* its -/// own granularity bound at every width, crossing it at four of the ten -/// measured: the mechanism is saturating the limit its insertion-time keys -/// impose, rather than staying comfortably inside it. -/// -/// **It is a bounded offset, not an accumulating drift**, which is the -/// difference between a granularity and a persistent unfair split. Measured by -/// scaling `WORK` and holding the seed count at 200: at one CPU the worst spread -/// is 30 ms at every window length; at eight it is 362 ms, 602 ms and 548 ms as -/// the window doubles and doubles again, so it saturates rather than growing -/// with time. -/// -/// **And it is the policy's, not the model's.** Everything that decides who runs -/// next — `RunQueue`'s insertion-time keys, `FairShare`'s one vruntime pot per -/// process, `CpuSched::pick`, the surplus rule in `answer_steal_requests` — is -/// the shipped core; what the simulator mocks is time, timer, IPI, halt and -/// switch. The width scaling follows from one pot per process: every running -/// thread charges the same pot, so it advances at the process's aggregate rate -/// while each queued thread's key stays frozen at its insert, and -/// one dispatch's worth of staleness therefore buys more wall-clock service the -/// more of that process is running at once. What the *model* contributes is the -/// search: these are worst-of-N figures over adversarially chosen interleavings -/// (seeded and PCT), not the split hardware would show on an average schedule. -const FAIRNESS_SAMPLE: &[(usize, u64)] = &[ - (1, 30 * MS), - (2, 102 * MS), - (3, 125 * MS), - (4, 198 * MS), - (6, 324 * MS), - (8, 418 * MS), - (12, 634 * MS), - (16, 720 * MS), - (24, 1056 * MS), - (32, 1386 * MS), -]; - -/// The ceiling a `fairness_storm` run is *gated* on: the recorded sample plus a -/// quarter. That margin is what makes this a regression test rather than a -/// transcription of one afternoon's tail — the worst-of-N spread grew from 84 ms -/// to 102 ms between 500 and 10 000 seeds at two CPUs, and a ceiling with no -/// headroom would red on sample size alone. -/// -/// What it can detect: a fairness regression that widens the worst spread by -/// more than 25%. Not a subtle one. -/// A width with no recorded sample gets **zero**, so the derived bound governs -/// it — an allowance is a claim that a measurement was taken, and nobody has -/// taken one there. -fn fair_allowance(cpus: usize) -> u64 { - FAIRNESS_SAMPLE - .iter() - .find(|(width, _)| *width == cpus) - .map_or(0, |(_, worst)| worst + worst / 4) -} - /// Invariant I5's workload: two processes of equal entitlement and unequal /// thread count, both pure CPU, neither ever blocking. /// @@ -812,27 +709,20 @@ fn fair_allowance(cpus: usize) -> u64 { /// /// * **Nothing blocks and nothing yields.** Fairness owes nothing across a /// block, so I5 measures over contention windows; a workload with no blocks -/// is one window from the first dispatch to the first exit. Every other -/// scenario gives I5 windows a few milliseconds long, which is to say gives -/// it nothing to measure. -/// * **`solo` has one thread per CPU, `trio` three.** A fair share is per -/// *process*, so they are owed the same CPU. Under any per-thread -/// policy `trio` takes three quarters instead of half — which is the whole -/// distinction, and is `fair_share_per_thread`. +/// is one window from the first dispatch to the first exit. +/// * **`solo` has one thread per CPU, `trio` three.** A job is per *process*, +/// so they are owed the same CPU: on each CPU `solo`'s one thread and +/// `trio`'s three form groups of equal weight. Under a job per thread `trio` +/// takes three quarters instead of half — which is the whole distinction, +/// and is `fair_job_per_thread`. /// * **Thread counts are multiples of `cpus`.** Spawn placement is -/// least-loaded-with-rotation, so each CPU ends up with the identical mix and -/// the run queues are balanced by construction. Balance-by-`StealRequest` -/// only answers a probe from a CPU whose victim has *two* ready tasks, so an -/// odd thread count would leave a standing imbalance and this -/// would be measuring placement rather than fairness. +/// least-loaded-with-rotation, so each CPU ends up with the identical mix, and +/// I5 measures only while it is. /// * **Each `solo` thread carries three times a `trio` thread's work**, so the /// two processes have the *same total* work and, under an even split, finish -/// together. The window I5 measures over closes when the first process stops -/// being runnable, so this is what makes it the whole run rather than the -/// first third of it — and a bound that carries a quantum per thread needs a -/// window many quanta wide before a broken split can clear it. +/// together, which keeps the window open for the whole run. pub fn fairness_storm(cpus: usize) -> Scenario { - let mut scenario = fair_workload( + fair_workload( if cpus == 1 { "fairness_storm" } else { @@ -840,9 +730,7 @@ pub fn fairness_storm(cpus: usize) -> Scenario { }, cpus, WORK, - ); - scenario.fair_allowance_ns = fair_allowance(cpus); - scenario + ) } /// One `trio` thread's work in [`fairness_storm`]: six quanta. `solo`'s threads @@ -852,51 +740,6 @@ const WORK: u64 = 60 * MS; /// The shape both fairness workloads share, with the per-thread work as the one /// parameter — see [`fairness_storm`] for why every other part of it is what it /// is, and [`sibling_storm`] for why one of the two needs longer threads. -/// -/// **The recorded per-thread sample.** What the shipped scheduler's worst -/// invariant-I13 service spread over one contention window actually *is*, per -/// machine width, against the 60 ms the bound derives from the fair band's own -/// granularity (five dispatches of one run queue: four rivals plus the leader, -/// at `QUANTUM + 2 × RUN_CHUNK` each). Every number came from -/// -/// ```text -/// cargo run --release -p kernel-sim -- measure fairness_storm: -/// ``` -/// -/// on `be4b34a` with this change applied, on one host, with no other -/// measurement running: -/// -/// | cpus | seeds | worst I13 spread | worst I5 spread, same runs | -/// |---|---|---|---| -/// | 1 | 10 000 | 10 ms | 30 ms | -/// | 2 | 10 000 | 30 ms | 102 ms | -/// | 3 | 500 | 28 ms | 125 ms | -/// | 4 | 500 | 28 ms | 198 ms | -/// | 6 | 500 | 31 ms | 324 ms | -/// | 8 | 500 | 32 ms | 418 ms | -/// | 12 | 200 | 35 ms | 634 ms | -/// | 16 | 200 | 37 ms | 612 ms | -/// | 24 | 200 | 42 ms | 1046 ms | -/// | 32 | 200 | 50 ms | 1386 ms | -/// -/// `sibling_storm` itself, at 300 seeds: 10 ms. -/// -/// Two things the table says out loud. **No width crosses the derived bound**, -/// so no width has a `thread_allowance_ns` — an allowance is a licence to sit -/// above the standard and nothing here needs one. Handing out the sample plus a -/// quarter regardless, the way [`fair_allowance`] does, would put the 32-CPU -/// ceiling at 62 ms against a derived 60: the allowance quietly becoming the -/// standard, which is the exact failure the two-tier shape exists to prevent. -/// If a width ever does cross, its allowance goes on the scenario and -/// `Outcome::thread_over_bound` reports the gap on every run regardless. -/// -/// And **the per-thread split does not degrade as the machine widens** — 10 ms -/// at one CPU against 50 ms at 32, against I5's 30 ms against 1386 ms over the -/// identical runs. The width degradation filed as a known issue is a -/// *per-process* phenomenon; inside a share the ordering holds flat, which is -/// what the insertion-time keys of one monotone pot are worth. The slow climb -/// is worth watching: at 32 CPUs the spread is 83% of the bound, and no width -/// above 32 has been measured. fn fair_workload(name: &'static str, cpus: usize, work: u64) -> Scenario { let hog = |ns| Script::new(vec![Op::Run(ns)]); let mut scenario = scenario( @@ -910,104 +753,55 @@ fn fair_workload(name: &'static str, cpus: usize, work: u64) -> Scenario { ], ); scenario.max_tasks = 4 * cpus; - // Quadratic in the machine, measured: 440 steps at one CPU, 980 at two, - // 71k at 32 and 265k at 64. The *work* is linear in `cpus` (360 run chunks - // each), so the rest is idle passes, steal probes and IPI deliveries, which - // is a per-CPU cost paid against every other CPU. A safety net sized on the - // linear term alone reports non-termination on a run that was progressing - // fine, which is what the first version of this line did at 64 and 128. - scenario.max_steps = 20_000 + (work / WORK) as usize * 20_000 + 100 * cpus * cpus; + // Quadratic in the machine, measured at seeds 0 to 3: 50k steps at 16 CPUs, + // 110k at 24 and 194k at 32. The *work* is linear in `cpus` (360 run chunks + // each), so the rest is passes, steal probes, balance requests and IPI + // deliveries, which is a per-CPU cost paid against every other CPU. + scenario.max_steps = 20_000 + (work / WORK) as usize * 20_000 + 250 * cpus * cpus; scenario } /// Invariant I13's workload: [`fairness_storm`] at one CPU with threads five -/// times as long. -/// -/// The length is the whole difference, and it is derived rather than picked. -/// I5's bound grows with the machine's thread count, so `fairness_storm`'s -/// six-quantum threads are already many times it; I13's bound is a *per-CPU* -/// constant — 60 ms, five dispatches of one run queue's fair band — and six -/// quanta of work per thread means a completely starved sibling separates from -/// a completely served one by 60 ms and then exits. That is the bound exactly, -/// and a gate that can only reach its own threshold is not a gate. Five times -/// the work makes total starvation a 300 ms separation, so the two negative -/// gates below are caught on the way to the failure rather than at it. -/// -/// One CPU, because the property is a property of one run queue: the fair band -/// orders the threads that are in *it*, and a second CPU adds placement, which -/// I13's window deliberately declines to measure across. +/// times as long, so a sibling the pick never serves is owed many times I13's +/// bound before its rivals run out of work. pub fn sibling_storm() -> Scenario { fair_workload("sibling_storm", 1, 5 * WORK) } -/// Negative gate for invariant I13, first of two: the fair band's tie-break -/// switched from the monotonic insertion sequence to `TaskKey` — the identity -/// tie-break `queue.rs` warns against in the comment beside the field. -/// -/// **It passes, and that is the finding.** The warning is written as though the -/// tie-break were what round-robins a share's threads; it is not. A share's pot -/// is charged for every nanosecond any of its threads runs, so a thread -/// re-inserted after a dispatch carries a key strictly above every sibling -/// queued before it and the ordering is already insertion order *without* the -/// tie-break doing anything. Exact ties survive only where no charge separates -/// two inserts — a post that wakes siblings, or the spawn burst — and one -/// dispatch dissolves them. Measured, not argued: 300 seeds of -/// [`sibling_storm`] under this ordering are clean, and the same 300 under -/// [`fair_identity_within_share`] fail on I13 every time. -/// -/// It is kept for the reason `old_commit_fused` is kept: it is the control that -/// turns "the obvious break is invisible here" from a claim into a measurement, -/// and it is what says the second gate below had to be written the way it is. -pub fn fair_identity_tiebreak() -> Scenario { - let mut scenario = sibling_storm().with_order(FairOrder::IdentityTiebreak); - scenario.name = "fair_identity_tiebreak"; - scenario -} - -/// Negative gate for invariant I13, second of two, and the one with teeth: the -/// `queue.rs` warning made total. Whichever share leads the fair band, its -/// *lowest-keyed* ready thread is dispatched — so the same thread wins every -/// time, not merely every tie. -/// -/// It **must fail**, and it must fail on I13 *alone*. That second half is the -/// point of the whole check: the share's pot advances at exactly the rate it -/// did before, because it is charged for the time the process ran and not for -/// which of its threads ran it. `solo` and `trio` split the machine as evenly as -/// ever and invariant I5 — which measures service per *process* — sees a -/// perfectly fair scheduler while two of `trio`'s three threads never run at -/// all. This is the hole I13 was built for, and the redesign it has to guard -/// (an ordered map of shares, each holding a FIFO of its ready threads) puts -/// precisely this code path at the centre of the scheduler. -pub fn fair_identity_within_share() -> Scenario { - let mut scenario = sibling_storm().with_order(FairOrder::IdentityWithinShare); - scenario.name = "fair_identity_within_share"; +/// Negative gate for invariant I13: each job group serves its lowest-keyed +/// thread every time, not its EEVDF pick. +/// +/// It **must fail**, and on I13 *alone*: the group is charged for the time its +/// job ran, not for which of its threads ran it, so `solo` and `trio` split the +/// machine as evenly as ever and invariant I5 sees a fair scheduler while two of +/// `trio`'s three threads never run. +pub fn fair_lowest_key_within_group() -> Scenario { + let mut scenario = sibling_storm().with_group_pick(GroupPick::LowestKey); + scenario.name = "fair_lowest_key_within_group"; scenario } -/// Negative gate for invariant I5, first of two: the rejected policy, one fair -/// share per *thread* instead of one per process. +/// Negative gate for invariant I5: the rejected policy, one job per *thread* +/// instead of one per process. /// /// It **must fail**. `trio` has three times `solo`'s threads and exactly the -/// same entitlement; under per-thread shares it takes three quarters of the +/// same entitlement; under a job per thread it takes three quarters of the /// machine, and a fairness check that cannot see a 3:1 split of a two-way share /// is not measuring fairness. The control is `fairness_storm` itself, which is /// the identical workload under the shipped policy. -pub fn fair_share_per_thread() -> Scenario { - let mut scenario = fairness_storm(1).with_share(ShareShape::PerThread); - scenario.name = "fair_share_per_thread"; +pub fn fair_job_per_thread() -> Scenario { + let mut scenario = fairness_storm(1).with_job(JobShape::PerThread); + scenario.name = "fair_job_per_thread"; scenario } -/// Negative gate for invariant I5, second of two: `trio`'s share is charged -/// twice for every nanosecond it runs. +/// Negative gate for invariant I5, in the other direction: `trio`'s group is +/// charged twice for every nanosecond it runs. /// -/// It **must fail**, and it fails in the *opposite* direction to -/// `fair_share_per_thread` — a share whose vruntime outruns its service is -/// throttled for work it never did, so `trio` ends up with a third of the -/// machine instead of half. The two gates together are what say I5 measures -/// service against entitlement rather than one side of it: the ordering could -/// be perfect and the charge wrong, or the charge perfect and the shares -/// mis-attributed, and both are unfair. +/// It **must fail**, the opposite way to `fair_job_per_thread`: a group charged +/// past its service is throttled for work it never did, so `trio` ends up with a +/// third of the machine instead of half. The two gates together say I5 measures +/// service against entitlement rather than one side of it. pub fn fair_double_charge() -> Scenario { let mut scenario = fairness_storm(1).with_charge(ChargeShape::Double { process: "trio" }); scenario.name = "fair_double_charge"; @@ -1115,10 +909,9 @@ pub fn by_name(name: &str) -> Option { "old_steal_port" => Some(old_steal_port()), "old_migrate_kept_the_corpse" => Some(old_migrate_kept_the_corpse()), "old_rt_starved_the_corpse" => Some(old_rt_starved_the_corpse()), - "fair_share_per_thread" => Some(fair_share_per_thread()), + "fair_job_per_thread" => Some(fair_job_per_thread()), "fair_double_charge" => Some(fair_double_charge()), - "fair_identity_tiebreak" => Some(fair_identity_tiebreak()), - "fair_identity_within_share" => Some(fair_identity_within_share()), + "fair_lowest_key_within_group" => Some(fair_lowest_key_within_group()), "overlong_pass" => Some(overlong_pass()), "old_park_kept_the_lend" => Some(old_park_kept_the_lend()), "lend_then_block" => Some(lend_then_block()), @@ -1212,19 +1005,17 @@ pub fn lend_then_block() -> Scenario { /// **The share-gain attack, as a workload**: one process with a single runnable /// thread against one with `threads` of them, both pure CPU, on one CPU. /// -/// The policy says "threads execute, processes own fair share", so `solo`'s work -/// must take about twice as long as it would alone whatever `threads` is — -/// `swarm` cannot buy CPU by forking. `sim/tests/policy.rs` measures how long it +/// A job owns its share and its threads divide it, so `solo`'s work must take +/// about twice as long as it would alone whatever `threads` is — `swarm` +/// cannot buy CPU by forking. `sim/tests/policy.rs` measures how long it /// actually takes and what that is worth as a share. /// /// Shape, and why each part of it: /// /// * **One CPU.** A process cannot run on more CPUs than it has runnable /// threads, so a single-threaded `solo` on a wider machine is limited by its -/// own thread count and not by the scheduler — which is the same reason -/// invariant I5's window excludes a member under its even share of the -/// machine. One CPU is where an even split is achievable and therefore where -/// the claim has content. +/// own thread count and not by the scheduler. One CPU is where an even split +/// is achievable and therefore where the claim has content. /// * **Every thread carries the same `work`.** `swarm` then holds `threads` /// times `solo`'s total, so it is still runnable long after `solo` has /// finished under any policy, and `solo`'s completion is a measurement of the @@ -1265,8 +1056,8 @@ pub fn share_gain(threads: usize, work: u64) -> Scenario { /// This is the shape every desktop scheduler is judged on, and the claim under /// test is how long an ordinary wake to a busy CPU waits. The sleeper is under /// its fair share by construction (it uses a quarter of a millisecond per -/// three), so its stored lag is positive and its re-derived vruntime puts it at -/// the head of the fair band, and below the running hog's key. +/// three), and its job's group outweighs each hog's on the CPU it wakes on, so +/// its request is due first. /// /// **The waker is a device and not a thread**, as in [`lost_wake_audio`]: a /// waker thread would be a third competitor for the CPU whose latency is being @@ -1316,20 +1107,9 @@ pub fn interactive_mix(cpus: usize, hogs: usize) -> Scenario { // Measured over the sweep `sim/tests/policy.rs` runs: 220 steps at one hog, // 278 at four, 572 at sixteen, 2,092 at 64. scenario.max_steps = 20_000 + 400 * hogs; - scenario.fair_allowance_ns = INTERACTIVE_I5_SAMPLE - .iter() - .find(|(shape, _)| *shape == (cpus, hogs)) - .map_or(0, |(_, worst)| worst + worst / 4); scenario } -/// Where [`interactive_mix`] crosses invariant I5's derived bound, and the -/// worst spread measured there — gated on as [`FAIRNESS_SAMPLE`] is, with its -/// quarter of margin, and still reported past the derived bound on every run. -/// `issues/interactive-mix-crosses-the-fairness-bound-at-two-cpus.md` is the -/// gap and its exit. -const INTERACTIVE_I5_SAMPLE: &[((usize, usize), u64)] = &[((2, 16), 254 * MS)]; - /// How many times the sleeper of [`interactive_mix`] wakes in one run. It is the /// sample count per seed, and `sim/tests/policy.rs` divides by it: the spawn /// burst delays exactly one of these wakes, so the steady-state claim is about @@ -1339,8 +1119,8 @@ pub const INTERACTIVE_ROUNDS: usize = 20; /// **A thread woken on its busy home CPU, beside a CPU with nothing to do** — /// the state `CpuSched::place`'s fair wake-forward is for. /// -/// One process, so the woken thread's key ties the running thread's and no cut -/// can take the CPU for it: only the forward runs it before the hog's quantum +/// One process, so the woken thread joins the running thread's group and no cut +/// can take the CPU for it: only the forward runs it before the hog's request /// ends. Both threads are spawned onto cpu0 and the balance path is off, so /// the forward is the only thing in a run that moves a task, and every /// migration counted is one. The sleeper's first wake finds its home busy; @@ -1383,12 +1163,11 @@ pub fn busy_home() -> Scenario { pub const BUSY_HOME_ROUNDS: usize = 8; /// **Two threads of one process on one CPU: one that sleeps a microsecond -/// between millisecond bursts, and one that never sleeps** — the shape -/// `CpuSched::place`'s strict key comparison is for. +/// between millisecond bursts, and one that never sleeps.** /// -/// They share a pot and so a key, and a napper whose every wake preempted its -/// sibling would leave the spinner [`kernel::sched::cpu::PROTECTED_RUN_NS`] -/// per burst. The spinner is spawned first, so it is the run's first task key. +/// They are one group, and a napper whose every wake preempted its sibling +/// would leave the spinner [`kernel::sched::queue::MIN_SLICE_NS`] per burst. +/// The spinner is spawned first, so it is the run's first task key. pub fn sleep_spin_siblings() -> Scenario { let mut scenario = scenario( "sleep_spin_siblings", @@ -1506,7 +1285,7 @@ pub const STORM_ROUNDS: usize = 4; /// when it wakes, so any blocking would let the wake path launder the /// adversary's placement into a legal one and the recovery being measured /// would be somebody else's. -/// * **One process.** The threads share a fair share, so what is measured is +/// * **One process.** The threads share one job, so what is measured is /// the *machine* finishing a fixed amount of work and never a split between /// two claimants — `share_gain` is where that question lives. /// * **Every thread carries the same `work`.** The makespan is then a rate: @@ -1526,13 +1305,13 @@ pub fn lopsided_placement(cpus: usize, threads: usize, work: u64) -> Scenario { ) .with_placement(PlacementShape::AllOn(0)); scenario.max_tasks = threads + 1; - // `share_gain`'s term, with `threads` in place of its `threads + 1`: every - // thread's work is chopped into `RUN_CHUNK_NS` execution steps and each - // quantum boundary costs a pass on top. The idle passes and steal probes a - // wide machine spends on top of that are what the fourfold reserve is for — - // measured over the sweep `sim/tests/policy.rs` runs: 677 steps at 2 CPUs - // and 8 threads, 1,681 at 4 and 16, 6,851 at 4 and 64, 8,897 at 8 and 64. - scenario.max_steps = 20_000 + 4 * threads * (work / RUN_CHUNK_NS) as usize; + // Every thread's work is chopped into `RUN_CHUNK_NS` execution steps and + // each request boundary costs a pass on top. The idle passes, steal probes + // and busy-balance requests a wide machine spends on top of that are what + // the tenfold reserve is for — measured at seed 0: 987 steps at 2 CPUs and + // 8 threads, 2,852 at 4 and 16, 11,450 at 4 and 64, 18,811 at 8 and 64, and + // 57,635 at worst over 20 seeds for `group_imbalance`'s 16 CPUs and 24. + scenario.max_steps = 20_000 + 10 * threads * (work / RUN_CHUNK_NS) as usize; scenario } diff --git a/kernel/sim/src/sweep.rs b/kernel/sim/src/sweep.rs index 20a9c7eb47..b4939c32bd 100644 --- a/kernel/sim/src/sweep.rs +++ b/kernel/sim/src/sweep.rs @@ -15,34 +15,27 @@ pub struct SweepResult { pub steps: u64, pub failures: Vec, /// Worst invariant-I5 service spread across the sweep, and the bound in - /// force when it happened. The sweep's fairness *measurement*, which is what - /// compares two frontier implementations against each other; `passed()` - /// only says it stayed under the bound. + /// force when it happened; `passed()` only says it stayed under the bound. pub worst_fair_spread: u64, pub worst_fair_bound: u64, - /// Worst spread past the derived bound anywhere in the sweep. Non-zero is - /// the standard-versus-shipped gap, not a test failure — the run passed on - /// the recorded allowance. + /// Worst spread past the derived bound anywhere in the sweep, which only a + /// lifted ceiling lets pass. pub worst_over_bound: u64, /// I5's *reach* over the whole sweep: virtual nanoseconds it had a - /// comparison open for, against the nanoseconds the sweep executed. Its - /// window has four ways to close — an RT task present, a CPU idle, the - /// member set changing, a member under its even share — and any of them can - /// be narrowed by a change to the pick or the placement without a single - /// verdict going red, so this is the number to A/B across one. + /// comparison open for. Its window closes for an RT task, an idle CPU, a + /// changed member set or an uneven mix, and a change to the pick or the + /// placement can narrow it without a verdict going red, so this is the + /// number to A/B across one. pub fair_covered_ns: u64, - /// Invariant I13's three, in the same three roles: the worst service spread - /// between threads of one share, the bound in force when it happened, and - /// any crossing of the derived per-thread bound the allowance let pass. - pub worst_thread_spread: u64, - pub worst_thread_bound: u64, - pub worst_thread_over_bound: u64, - /// I13's *reach* over the whole sweep: virtual nanoseconds it had a - /// comparison open for, against the nanoseconds the sweep executed. A - /// change to the pick or the balance can close I13's windows instead of - /// failing them, so this is the number to A/B across one — a collapse here - /// means the gate went quiet, which is not the same as the gate passing. - pub thread_covered_ns: u64, + /// Invariant I13's three: the most service a ready fair thread was owed + /// while it waited, the bound in force when it happened, and the worst + /// crossing of it. + pub worst_wait_owed: u64, + pub worst_wait_bound: u64, + pub worst_wait_over_bound: u64, + /// I13's *reach* over the whole sweep: virtual nanoseconds some ready fair + /// thread was owed service. + pub wait_covered_ns: u64, pub elapsed_ns: u64, } @@ -64,10 +57,10 @@ impl SweepResult { worst_fair_bound: 0, worst_over_bound: 0, fair_covered_ns: 0, - worst_thread_spread: 0, - worst_thread_bound: 0, - worst_thread_over_bound: 0, - thread_covered_ns: 0, + worst_wait_owed: 0, + worst_wait_bound: 0, + worst_wait_over_bound: 0, + wait_covered_ns: 0, elapsed_ns: 0, } } @@ -85,12 +78,12 @@ impl SweepResult { } self.worst_over_bound = self.worst_over_bound.max(outcome.fair_over_bound); self.fair_covered_ns += outcome.fair_covered_ns; - if outcome.thread_spread > self.worst_thread_spread { - self.worst_thread_spread = outcome.thread_spread; - self.worst_thread_bound = outcome.thread_bound; + if outcome.wait_owed > self.worst_wait_owed { + self.worst_wait_owed = outcome.wait_owed; + self.worst_wait_bound = outcome.wait_bound; } - self.worst_thread_over_bound = self.worst_thread_over_bound.max(outcome.thread_over_bound); - self.thread_covered_ns += outcome.thread_covered_ns; + self.worst_wait_over_bound = self.worst_wait_over_bound.max(outcome.wait_over_bound); + self.wait_covered_ns += outcome.wait_covered_ns; self.elapsed_ns += outcome.elapsed; } @@ -103,20 +96,20 @@ impl SweepResult { self.fair_covered_ns * 100 / self.elapsed_ns } - /// What fraction of the sweep's executed time I13 had a comparison open - /// for, in percent. The reach, as one number to compare across a change. - pub fn thread_coverage_pct(&self) -> u64 { + /// What fraction of the sweep's executed time some ready fair thread was + /// owed service, in percent: I13's reach. + pub fn wait_coverage_pct(&self) -> u64 { if self.elapsed_ns == 0 { return 0; } - self.thread_covered_ns * 100 / self.elapsed_ns + self.wait_covered_ns * 100 / self.elapsed_ns } pub fn report(&self) -> String { if self.passed() { format!( "{}: {} runs, {} steps, clean (I5 worst spread {}/{} ns{}, I5 reach {}%, \ - I13 worst spread {}/{} ns{}, I13 reach {}%)", + I13 worst owed {}/{} ns{}, I13 reach {}%)", self.scenario, self.runs, self.steps, @@ -124,10 +117,10 @@ impl SweepResult { self.worst_fair_bound, past_the_bound(self.worst_over_bound), self.fair_coverage_pct(), - self.worst_thread_spread, - self.worst_thread_bound, - past_the_bound(self.worst_thread_over_bound), - self.thread_coverage_pct(), + self.worst_wait_owed, + self.worst_wait_bound, + past_the_bound(self.worst_wait_over_bound), + self.wait_coverage_pct(), ) } else { format!( diff --git a/kernel/sim/src/vm.rs b/kernel/sim/src/vm.rs index bef0a9baaa..3967f5091e 100644 --- a/kernel/sim/src/vm.rs +++ b/kernel/sim/src/vm.rs @@ -15,7 +15,7 @@ use std::collections::{BTreeMap, BTreeSet}; use std::sync::Arc as StdArc; use kernel::sched::cpu::{Action, CpuHandle, CpuSched, Env, SchedPass}; -use kernel::sched::fair::{FairShare, Frontier, ShareState}; +use kernel::sched::job::{Job, JobKey}; use kernel::sched::hw::{CpuId, Hw, Kicker, Machine, Nanos, TraceEvent, TraceKind}; use kernel::sched::invariants::Container; use kernel::sched::mailbox::{mailbox, Kick, Urgency}; @@ -33,11 +33,11 @@ use crate::choice::ChoiceStream; use crate::hw_impl::SimHw; use crate::latency::{ReadyCause, RunWait}; use crate::msg::{SimHandles, SimMsg, SimQueue}; -use crate::payload::{MockAddressSpace, SimCtx, SimPayload, SimPreempt, SimShareLock, StdLock}; +use crate::payload::{MockAddressSpace, SimCtx, SimPayload, SimPreempt, StdLock}; use crate::topology::LevelCounts; use crate::workload::{ - AgeShape, BlockShape, ChargeShape, MigrateShape, Op, ParkShape, PlacementShape, - Protocol, Scenario, Script, ShareShape, SurplusShape, WindowShape, + AgeShape, BlockShape, ChargeShape, GroupPick, JobShape, MigrateShape, Op, ParkShape, PlacementShape, + Protocol, Scenario, Script, SurplusShape, WindowShape, }; /// How finely a `Run(ns)` op is chopped. Small enough that a 10 ms quantum @@ -106,12 +106,6 @@ impl QueueState { } } -fn new_share() -> Arc> { - Arc::new(FairShare::new(StdLock::new(ShareState::NonRunnable { - lag: 0, - }))) -} - pub fn build_queues(scenario: &Scenario) -> Vec { scenario .queues @@ -122,11 +116,10 @@ pub fn build_queues(scenario: &Scenario) -> Vec { pub struct ProcState { pub name: &'static str, - /// Every fair share this process's threads hold. Exactly one under - /// [`ShareShape::PerProcess`]; one per spawned thread under the `PerThread` - /// negative gate, which is why invariant I6 sums over the vector rather - /// than reading a single share. - pub shares: Vec>>, + /// Every job this process's threads hold. Exactly one under + /// [`JobShape::PerProcess`]; one per spawned thread under the `PerThread` + /// negative gate. + pub jobs: Vec>, /// The process's own reference to its address space. Dropped when the /// process concludes every one of its threads is gone — under the new /// protocol because they were all finalized, under the old one because a @@ -261,7 +254,6 @@ pub struct Vm<'q> { pub queues: &'q [QueueState], pub hw: SimHw, pub handles: SimHandles, - pub frontier: Frontier, pub cpus: Vec>, pub procs: Vec, pub programs: BTreeMap, @@ -373,69 +365,42 @@ pub struct Vm<'q> { /// summed at the same clock advances `busy_ns` is, so the two are exact /// against each other. pub service_ns: Vec, - /// Invariant I13's measurement: the same nanoseconds, attributed to the - /// *thread* that consumed them. A share is one pot for every thread of a - /// process, so `service_ns` cannot tell a share that round-robins its - /// threads from one that runs a single thread and starves the rest. + /// The same nanoseconds, attributed to the *thread* that consumed them. pub thread_service: BTreeMap, + /// Invariant I13's measurement: per ready fair thread, the service its CPU + /// has owed it since it last ran — its group's weight over the CPU's + /// summed group weight, shared among the group's threads there, for as long + /// as that CPU runs fair work. A thread keeps it across a migration and + /// drops it when it runs or stops being runnable. + pub owed_waiting: BTreeMap, /// The contention window I5 measures service over; see /// [`crate::invariants`]. pub fair_epoch: FairEpoch, /// The widest service spread I5 has seen, and the bound that was in force - /// when it saw it. Reported rather than only asserted, so a per-CPU - /// frontier can be compared against the global one by a number. + /// when it saw it. pub fair_spread: u64, pub fair_bound: u64, - /// Worst spread that exceeded the *derived* bound, whether or not the - /// recorded allowance let the run pass. This is the gap between the standard - /// and the shipped scheduler, surfaced by the instrument on every run. + /// Worst spread that exceeded the *derived* bound, whether or not a lifted + /// ceiling let the run pass. pub fair_over_bound: u64, - /// How many virtual nanoseconds invariant I5 actually had a comparison open - /// for, and whether it has one open right now — [`Self::thread_covered_ns`] - /// for the per-*process* check. - /// - /// **I5 is exposed to the same silent switch-off as I13, through more - /// conditions.** Its window needs a saturated machine, an empty RT band, an - /// unchanged member set, and at least two members that are not limited by - /// their own thread count. Every one of those is a property the pick, the - /// placement or the balance can change, so a change that stops satisfying - /// one makes I5 measure less rather than fail — and I5 is the check the - /// per-process fairness verdict on every other scenario rests on. - /// - /// Its liveness gate asserted only that *some* window opened, which one - /// window a nanosecond wide satisfies. The reach is the quantity that - /// distinguishes those, so it is published here and gated in - /// `the_fairness_storm_is_measured_and_holds` — a collapse read as loudly as - /// a violation. - /// - /// The flag is consulted by `advance`, which runs before the step's checks, - /// so the accounting lags the window by one step — I13's rounding error, for - /// I13's reason. + /// How many virtual nanoseconds invariant I5 had a comparison open for, + /// and whether it has one open right now. Its window needs a saturated + /// machine, an empty RT band, an unchanged member set and every CPU holding + /// the same mix, so a change to the pick, the placement or the balance can + /// make I5 measure less rather than fail: the reach is gated in + /// `the_fairness_storm_is_measured_and_holds`. `advance` reads the flag + /// before the step's checks, so the count lags the window by one step. pub fair_covered_ns: u64, pub fair_window_open: bool, - /// How many virtual nanoseconds invariant I13 actually had a comparison - /// open for, and whether it has one open right now. - /// - /// **A live gate can be switched off by the very change it guards**, and - /// I13 is exposed to exactly that: its window closes when a member's threads - /// stop being evenly spread over the CPUs, so a change that disturbs - /// *placement* makes this check measure less rather than fail. The reach is - /// therefore a published number, to be compared across any change to the - /// pick or the balance — a collapse here has to be read as loudly as a - /// violation. See [`crate::invariants`]. - /// - /// The flag is consulted by `advance`, which runs before the step's checks, - /// so the accounting lags the window by one step. That is a rounding error - /// against a run and not worth a second traversal to remove. - pub thread_covered_ns: u64, - pub thread_window_open: bool, - /// Invariant I13's three numbers, in the same three roles as I5's above: - /// the widest service spread seen *between threads of one share*, the bound - /// in force when it was seen, and the worst crossing of the derived bound - /// whatever the recorded allowance permitted. - pub thread_spread: u64, - pub thread_bound: u64, - pub thread_over_bound: u64, + /// Invariant I13's three numbers: the most service any ready fair thread + /// was owed while it waited, the bound in force when it was seen, and the + /// worst crossing of that bound. + pub wait_owed: u64, + pub wait_bound: u64, + pub wait_over_bound: u64, + /// How many virtual nanoseconds some ready fair thread was owed service: + /// I13's reach, which a change to placement or the pick could shrink. + pub wait_covered_ns: u64, /// Every task currently owed a dispatch and not getting one, with the /// instant it became owed one, why, and the [`Vm::round`] it was in — the /// measured policy suite's instrument (`crate::latency`, @@ -573,40 +538,18 @@ pub struct Vm<'q> { pub const PROBE_WORTH_IT: u32 = kernel::sched::cpu::PUSH_THRESHOLD; /// One contention window: a maximal interval over which the same set of -/// fair-band processes was continuously runnable. +/// fair-band processes was continuously runnable and every CPU held the same +/// number of each one's runnable threads. /// /// Fairness has nothing to say across a window boundary — a process that was -/// blocked was not owed service — so the measurement restarts whenever the set -/// changes, and the bound carries the widest thread count and stored-lag spread -/// seen inside the window. +/// blocked was not owed service, and an unequal placement is owed unequal +/// service (owner decision J1) — so the measurement restarts whenever either +/// changes. #[derive(Default)] pub struct FairEpoch { pub members: Vec, /// Each process's `service_ns` when the window opened. pub base: Vec, - /// Widest total runnable-thread count the window has held, and widest - /// stored-lag spread. Both are terms of I5's bound, and both are running - /// maxima so a thread that exits mid-window cannot shrink the bound under a - /// separation it helped create. - pub threads: u32, - pub lag_spread: u64, - /// Invariant I13's members: every thread that was runnable when *its* - /// window opened, with its `thread_service` at that instant. Entries are - /// only ever removed — a thread that stops being runnable is owed nothing - /// further and never rejoins — so a thread blocking or exiting narrows the - /// comparison instead of restarting it. - /// - /// I13's window is a sub-interval of this one, re-baselined whenever the - /// members' threads stop being spread evenly across the CPUs; see - /// [`crate::invariants`]. - pub thread_base: BTreeMap, - /// The widest count of members' runnable threads that *one CPU* held inside - /// I13's window — how many dispatches a waiting thread can be passed over - /// by before its own key comes up, and the only term of I13's bound. Well - /// defined only because that window requires every CPU to carry the same - /// number of each member's threads, which is the same reason it is a - /// per-CPU count where I5's `threads` is a machine-wide one. - pub thread_rivals: u32, } impl<'q> Vm<'q> { @@ -622,7 +565,12 @@ impl<'q> Vm<'q> { cpu.set_park_keeps_lapsed_lend(scenario.park == ParkShape::KeepLapsedLend); cpu.set_migrate_keeps_the_corpse(scenario.migrate == MigrateShape::KeepTheCorpse); cpu.set_rt_outranks_every_corpse(scenario.age == AgeShape::RtOutranksEveryCorpse); - cpu.set_fair_order(scenario.order); + cpu.set_lowest_key_within_group(scenario.group_pick == GroupPick::LowestKey); + // A process's own job is keyed by its index. + cpu.set_double_charge(match scenario.charge { + ChargeShape::Honest => None, + ChargeShape::Double { process } => scenario.process_index(process).map(|p| JobKey(p as u64)), + }); cpus.push(cpu); } hw.set_pass_cost(scenario.pass_cost_ns); @@ -632,7 +580,7 @@ impl<'q> Vm<'q> { .enumerate() .map(|(index, spec)| ProcState { name: spec.name, - shares: vec![new_share()], + jobs: vec![Arc::new(Job::new(JobKey(index as u64)))], address_space: Some(StdArc::new(MockAddressSpace { process: index as u32, })), @@ -654,7 +602,6 @@ impl<'q> Vm<'q> { queues, hw, handles: SimHandles::new(handles, scenario.topology.topology().clone()), - frontier: Frontier::new(), cpus, procs, programs: BTreeMap::new(), @@ -681,11 +628,11 @@ impl<'q> Vm<'q> { fair_over_bound: 0, fair_covered_ns: 0, fair_window_open: false, - thread_covered_ns: 0, - thread_window_open: false, - thread_spread: 0, - thread_bound: 0, - thread_over_bound: 0, + owed_waiting: BTreeMap::new(), + wait_owed: 0, + wait_bound: 0, + wait_over_bound: 0, + wait_covered_ns: 0, awaiting: BTreeMap::new(), prev_container: BTreeMap::new(), run_wait: vec![RunWait::default(); process_count], @@ -762,12 +709,13 @@ impl<'q> Vm<'q> { .address_space .clone() .expect("spawning into a process whose address space is gone"); - let share = match self.scenario.share { - ShareShape::PerProcess => self.procs[process].shares[0].clone(), - ShareShape::PerThread => { - let share = new_share(); - self.procs[process].shares.push(share.clone()); - share + let job = match self.scenario.job { + JobShape::PerProcess => self.procs[process].jobs[0].clone(), + JobShape::PerThread => { + // Keys past every process's own, never reused. + let job = Arc::new(Job::new(JobKey(self.procs.len() as u64 + key.0))); + self.procs[process].jobs.push(job.clone()); + job } }; let rt = RtState { @@ -796,7 +744,7 @@ impl<'q> Vm<'q> { }; let builder = TaskBuilder { key, - share, + job, ctx: SimCtx { key: Some(key) }, ext: SimPayload { key, @@ -1266,24 +1214,12 @@ impl<'q> Vm<'q> { if delta == 0 { return; } - let doubled = match self.scenario.charge { - ChargeShape::Honest => None, - ChargeShape::Double { process } => self.scenario.process_index(process), - }; for cpu in 0..self.scenario.cpus() { let Some(task) = self.cpus[cpu].running() else { continue; }; self.busy_ns[cpu] += delta; let (key, rt, process) = (task.key(), task.rt(), task.ext().process as usize); - if doubled == Some(process) { - // The second charge for one nanosecond of running: what a - // charge applied at two transitions instead of one looks like - // from outside the core. - task.share() - .charge(delta) - .expect("a running task's share is runnable"); - } self.service_ns[process] += delta; *self.thread_service.entry(key).or_default() += delta; let entry = self.boosted_run.entry(key).or_insert((rt.lends, 0)); @@ -1297,13 +1233,41 @@ impl<'q> Vm<'q> { if self.fair_window_open { self.fair_covered_ns += delta; } - if self.thread_window_open { - self.thread_covered_ns += delta; - } + self.owe_waiting(delta); self.clock = self.clock.after(delta); self.hw.with(|s| s.now = self.clock); } + /// Invariant I13's accumulator: what each CPU running fair work owed + /// every ready fair thread on it over `delta`, by its group's share of the + /// CPU's summed group weight. A running thread is owed nothing; one that + /// is no longer runnable anywhere leaves. + fn owe_waiting(&mut self, delta: u64) { + let mut waiting = false; + for cpu in &self.cpus { + if let Some(running) = cpu.running() { + self.owed_waiting.remove(&running.key()); + } + if !cpu.rq().running_is_fair() { + continue; + } + let load = cpu.rq().weight() as f64; + for task in cpu.rq().tasks().filter(|t| !t.rt().is_rt()) { + let job = task.job().key(); + let (_, weight, here) = cpu.rq().groups().find(|g| g.0 == job).expect("a queued thread's group"); + *self.owed_waiting.entry(task.key()).or_default() += weight as f64 / load / here as f64 * delta as f64; + waiting = true; + } + } + let shared = &self.shared; + self.owed_waiting.retain(|key, _| { + shared.get(key).is_some_and(|s| matches!(s.state(), TaskState::Ready(_) | TaskState::InTransit(_))) + }); + if waiting { + self.wait_covered_ns += delta; + } + } + fn device_irq(&mut self, index: usize) { let spec = self.scenario.irqs[index]; self.next_irq[index] = self.next_irq[index].after(spec.period_ns); @@ -1357,13 +1321,11 @@ impl<'q> Vm<'q> { cpus, hw, handles, - frontier, .. } = self; let env = Env { hw, cpus: handles, - frontier, preempt: &SimPreempt, balance, }; @@ -1540,7 +1502,7 @@ impl<'q> Vm<'q> { self.run_pass(cpu, Dispose::Yield); } Op::SetRt => { - self.cpus[cpu].set_current_rt(true); + self.cpus[cpu].set_current_rt(self.clock, true); self.programs.get_mut(&key).expect("live").pc += 1; } Op::Spawn { template } => { @@ -1782,7 +1744,7 @@ impl<'q> Vm<'q> { fn take_pending_boost(&mut self, cpu: usize, queue: usize) { if let Some(until) = self.queues[queue].boost_until.get() { if until > self.clock { - self.cpus[cpu].boost_current(until); + self.cpus[cpu].boost_current(self.clock, until); } else { self.queues[queue].boost_until.set(None); } diff --git a/kernel/sim/src/workload.rs b/kernel/sim/src/workload.rs index 3aab4ae610..26cc77f665 100644 --- a/kernel/sim/src/workload.rs +++ b/kernel/sim/src/workload.rs @@ -12,7 +12,6 @@ //! one. use kernel::sched::cpu::Balance; -use kernel::sched::queue::FairOrder; use kernel::sched::task::WaitClass; use crate::topology::Shape; @@ -192,41 +191,47 @@ pub enum AgeShape { RtOutranksEveryCorpse, } -/// What a fair share is a share *of*: all threads of one process share a -/// vruntime. +/// What a job is: the owner a thread's CPU time is shared out by. /// /// A scenario dimension rather than a constant for the same reason /// [`BlockShape`] and [`ParkShape`] are, except that the other answer is one -/// the design *rejected* rather than one it shipped: per-thread -/// weight-division fairness was turned down, and per-thread vruntime is that -/// policy in its simplest form. Invariant I5 has to be able to tell the two +/// the design *rejected* rather than one it shipped: a share per thread lets a +/// process buy CPU by forking. Invariant I5 has to be able to tell the two /// apart, or it is not measuring the shipped policy at all. #[derive(Clone, Copy, PartialEq, Eq, Debug)] -pub enum ShareShape { - /// The shipped policy: one [`kernel::sched::fair::FairShare`] per process, - /// reached through every thread of it. +pub enum JobShape { + /// The shipped policy: one [`kernel::sched::job::Job`] per process, + /// held by every thread of it. PerProcess, - /// One share per thread, so a process buys CPU by forking. See - /// `scenarios::fair_share_per_thread`. + /// One job per thread. See `scenarios::fair_job_per_thread`. PerThread, } -/// How much vruntime a running task's share is charged for the time it runs. +/// What a running thread's job group is charged for the time it runs. /// -/// The honest answer is "the time it ran", and it is the core that computes it -/// (`SchedPass::begin`). This dimension lets the VM charge a named process a -/// second time on top, which is the shape of a charge applied at two -/// transitions instead of one — and the shape invariant I5 must notice, -/// because a share whose vruntime outruns its service is a share being -/// throttled for work it never did. +/// The honest answer is "the time it ran", and it is the core that charges it +/// (`SchedPass::begin`). The other charges a named process's group twice, the +/// shape of a charge applied at two transitions instead of one — and the shape +/// invariant I5 must notice, because a job whose group is charged past its +/// service is a job throttled for work it never did. #[derive(Clone, Copy, PartialEq, Eq, Debug)] pub enum ChargeShape { Honest, - /// Charge this process's share twice for every nanosecond it runs. See - /// `scenarios::fair_double_charge`. + /// See `scenarios::fair_double_charge`. Double { process: &'static str }, } +/// Which of a job group's threads the fair band runs. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub enum GroupPick { + /// What ships: EEVDF over the group's threads. + Eevdf, + /// The group's lowest-keyed thread, every time, so one thread takes all of + /// its job's time on its CPU while its siblings wait. See + /// `scenarios::fair_lowest_key_within_group`. + LowestKey, +} + /// Where a spawn is placed. /// /// A scenario dimension rather than a constant because the shipped answer is a @@ -299,7 +304,7 @@ pub struct Scenario { pub park: ParkShape, pub migrate: MigrateShape, pub age: AgeShape, - pub share: ShareShape, + pub job: JobShape, pub charge: ChargeShape, pub placement: PlacementShape, /// A CPU that takes no scheduler pass for the whole run — the machine a @@ -319,32 +324,22 @@ pub struct Scenario { /// policies the decision was priced across ([`Balance::Pull`], the pull /// half alone, and [`Balance::PullWithRearm`], the declined cure). /// - /// **The core's own type rather than a parallel copy of it**, exactly as - /// [`Scenario::order`] carries `FairOrder`: every setting is a policy the - /// core implements, so the simulator drives the real code whichever one a - /// scenario names. + /// **The core's own type rather than a parallel copy of it**: every setting + /// is a policy the core implements, so the simulator drives the real code + /// whichever one a scenario names. pub balance: Balance, pub surplus: SurplusShape, - /// How the fair band picks between two ready threads of one share. A - /// scenario dimension for the same reason [`ShareShape`] is, and the type is - /// the core's own rather than a parallel copy of it: the broken orderings - /// live in `queue.rs` behind `protocol-port`, so the kernel cannot reach - /// them and the simulator drives the real code either way. - pub order: FairOrder, + /// Which of a group's threads the fair band runs. The broken pick lives in + /// `queue.rs` behind `protocol-port`, so the kernel cannot reach it and the + /// simulator drives the real code either way. + pub group_pick: GroupPick, /// What one scheduler pass is modelled to cost. Zero everywhere but /// `scenarios::overlong_pass`; see [`crate::hw_impl::SimHwState`]. pub pass_cost_ns: u64, - /// Invariant I5's *recorded* ceiling, where the shipped scheduler does not - /// meet the derived one — `scenarios::FAIRNESS_SAMPLE`. Zero means the - /// derived bound is the ceiling, which is the only honest default: an - /// allowance is a statement that a measurement was taken, and a scenario - /// nobody has measured has no allowance to offer. + /// Invariant I5's ceiling, lifted above the derived bound only by a control + /// that measures what a rejected policy does rather than asserting that it + /// fails. Zero means the derived bound is the ceiling. pub fair_allowance_ns: u64, - /// Invariant I13's recorded ceiling, in the same role and with the same - /// default as `fair_allowance_ns`: zero unless somebody has measured this - /// scenario and found the shipped scheduler past the derived per-thread - /// bound. - pub thread_allowance_ns: u64, /// Safety net: a run that has not quiesced by here is reported as a /// non-termination failure rather than looping forever. pub max_steps: usize, @@ -412,8 +407,8 @@ impl Scenario { self } - pub fn with_share(mut self, share: ShareShape) -> Self { - self.share = share; + pub fn with_job(mut self, job: JobShape) -> Self { + self.job = job; self } @@ -468,8 +463,8 @@ impl Scenario { self } - pub fn with_order(mut self, order: FairOrder) -> Self { - self.order = order; + pub fn with_group_pick(mut self, pick: GroupPick) -> Self { + self.group_pick = pick; self } diff --git a/kernel/sim/tests/deadline_claim_race.rs b/kernel/sim/tests/deadline_claim_race.rs index 0ee5ef3581..52431b72f1 100644 --- a/kernel/sim/tests/deadline_claim_race.rs +++ b/kernel/sim/tests/deadline_claim_race.rs @@ -19,8 +19,8 @@ use std::sync::Arc as StdArc; use kernel::sched::cpu::{Action, Balance, CpuHandle, CpuHandles, CpuSched, Env, SchedPass}; use kernel::sched::topology::Topology; -use kernel::sched::fair::{FairShare, Frontier, ShareState}; use kernel::sched::hw::{CpuId, Kicker, Nanos}; +use kernel::sched::job::{Job, JobKey}; use kernel::sched::mailbox::{mailbox, Kick, Urgency}; use kernel::sched::msg::Msg; use kernel::sched::sync::Arc; @@ -31,18 +31,17 @@ use kernel::sched::park::{prepare, Cancel, Commit}; use kernel_sim::hw_impl::SimHw; use kernel_sim::msg::SimMsg; -use kernel_sim::payload::{MockAddressSpace, SimCtx, SimPayload, SimPreempt, StdLock}; +use kernel_sim::payload::{MockAddressSpace, SimCtx, SimPayload, SimPreempt}; const CPU0: CpuId = CpuId(0); const KEY: TaskKey = TaskKey(1); -/// Well inside the quantum, so every arming decision this test observes is -/// about the deadline and never about a timeslice running out. -const DEADLINE: Nanos = Nanos(5_000_000); +/// Well inside the running request, so every arming decision this test +/// observes is about the deadline and never about a request running out. +const DEADLINE: Nanos = Nanos(2_000_000); struct Machine1 { hw: SimHw, handles: CpuHandles, - frontier: Frontier, } impl Machine1 { @@ -50,7 +49,6 @@ impl Machine1 { Env { hw: &self.hw, cpus: &self.handles, - frontier: &self.frontier, preempt: &SimPreempt, balance: Balance::None, } @@ -80,15 +78,12 @@ fn boot() -> (Machine1, CpuSched, Arc>) { let m = Machine1 { hw: SimHw::new(1), handles: CpuHandles::new(vec![CpuHandle::new(CPU0, tx)], Topology::flat(1)), - frontier: Frontier::new(), }; let mut cpu = CpuSched::new(CPU0, rx, SimCtx::default()); let task = TaskBuilder { key: KEY, - share: Arc::new(FairShare::new(StdLock::new(ShareState::NonRunnable { - lag: 0, - }))), + job: Arc::new(Job::new(JobKey(0))), ctx: SimCtx { key: Some(KEY) }, ext: SimPayload { key: KEY, diff --git a/kernel/sim/tests/policy.rs b/kernel/sim/tests/policy.rs index 60753c940d..1e528c661b 100644 --- a/kernel/sim/tests/policy.rs +++ b/kernel/sim/tests/policy.rs @@ -2,41 +2,34 @@ //! as numbers, against bounds derived from its own constants. //! //! `scenarios.rs` next door is the exit criterion — every negative gate fails, -//! every scenario passes. That is a suite of *verdicts*, and the external review -//! of 2026-08-20 named what it leaves out: the policy this scheduler states — -//! "threads execute, processes own fair share" — is encoded in implementation -//! intent and was validated nowhere empirically. A verdict cannot say how much -//! share a process actually got, how long an interactive wake actually waited, -//! or how long a runnable task can actually be passed over. Those are the -//! quantities a user feels, and each one below is measured and then asserted. +//! every scenario passes. That is a suite of *verdicts*. The policy this +//! scheduler states — a job owns its share and its threads divide it, and each +//! CPU runs EEVDF over the job groups it holds (owner decision J1) — is about +//! quantities: how much share a process got, how long an interactive wake +//! waited, how long a runnable task was passed over. Each one below is +//! measured and then asserted. //! -//! # The one constant every bound here is made of -//! -//! Every case in this file lands on the same term, and it is not a coincidence: +//! # The constant every bound here is made of //! //! ```text -//! (runnable threads on one CPU + 1) × (QUANTUM_NS + max KernelSection + 2 × RUN_CHUNK_NS) +//! one request = SLICE_NS + 2 × RUN_CHUNK_NS //! ``` //! -//! The fair band is keyed by the vruntime a thread held **when it was inserted** -//! (`queue.rs`), and a share's pot advances only when one of its -//! threads runs. So a thread queued behind `R` others can be passed over by all -//! `R` of them on the strength of keys that were already stale when its own wait -//! began, and the leader spends one more quantum on top. It is invariant I5's -//! staleness term and invariant I13's whole bound, and this file is the finding -//! that it is *also* the interactive wake latency, *also* the starvation bound, -//! and *also* exactly what a thread-count attack is worth. The kernel-section -//! and chunk terms are I9's, for I9's reason: a preempt-off section overruns the -//! quantum it started in, and the model observes an expiry one chunk late. +//! EEVDF holds each entity's lag inside one request plus the granularity a +//! decision is taken late by (Stoica and Abdel-Wahab, Theorem 3), and the model +//! observes a decision up to two chunks late. So a group's service stays within +//! a request of its share, a thread whose group is due waits out at most the +//! request running ahead of it, and a thread in a group of `n` waits out its +//! `n − 1` siblings' requests at the rate its group is served. //! //! # What each case measures, and what it found //! //! | case | measured | derived bound | measured/bound | //! |---|---|---|---| -//! | thread-count asymmetry | `solo` finishes `N+1` quanta late, at every `N` | `(N+2) × 12 ms` | 0.82 at N=64 | -//! | the deficit over time | 50 ms at N=4, 170 ms at N=16, unchanged as the window doubles twice | an offset, not a drift | — | -//! | wakeup storm drain | 27.75 ms for 64 waiters on one CPU, 3.1× the 16-waiter figure | `(per queue+1) × 12 ms` = 780 ms | 0.04 | -//! | starvation | 70 ms at 5 runnable threads, 680 ms at 65 | `(threads+1) × 12 ms` | 0.97 / 0.86 | +//! | thread-count asymmetry | `solo` finishes one slice late at every `N` ≥ 2 | `2 × WORK + 2` requests | 0.94 | +//! | the deficit over time | 4 ms at N=4 and N=16, unchanged as the window doubles twice | an offset, not a drift | — | +//! | interactive wake | 5 ms at worst, 2 ms for every wake but one per run | one request; the minimum run and two chunks | 0.83 / 0.89 | +//! | starvation | 20 ms at 4 runnable threads, 120 ms at 17 | groups × the largest group × one request | 0.55–0.63 | //! | a bounded re-arm | every CPU is reached at every width; 10 ms of probe gap | `every_ns + 3 × RUN_CHUNK` = 13 ms | 0.77 | //! //! The adversarial-placement case is the only one whose bound is a *count* @@ -52,32 +45,30 @@ //! quantity the protocol owns end to end. //! //! Every number came from this file at 16–40 seeds per point: -//! `cargo test -p kernel-sim --test policy -- --nocapture`. The tables in -//! each test carry the full sweeps. +//! `cargo test --release -p kernel-sim --test policy -- --nocapture`, on +//! `e74dbf1` with the job-group scheduler applied. The tables in each test +//! carry the full sweeps. //! //! # What the numbers say, in one paragraph //! -//! **A process cannot buy CPU by forking, and the qualifier is a window -//! length.** `swarm` with N threads delays a single-threaded rival by exactly -//! `N+1` quanta — once — and never again however long the run goes on. Over the -//! 120 ms `solo` would take against one rival that is a fall from a 500‰ share -//! to 77‰ at N=64; over a window four times as long the same deficit is 453‰ at -//! N=4. So the policy's claim holds asymptotically and is a granularity -//! statement, not a guarantee about any particular window — which is the honest -//! form of "processes own fair share", and it is measured here rather than -//! asserted anywhere. +//! **A process cannot buy CPU by forking.** `swarm` with N threads delays a +//! single-threaded rival by one slice — the EEVDF lag of one request — at every +//! N from 2 to 64, once, and never again however long the run goes on. The +//! wait a runnable thread can see grows with its own group's size, not with +//! the machine's thread count: a thread among 16 siblings waits out 15 of their +//! requests at the half of the CPU its job is owed. //! //! **The pull half is one-shot, and the shipped push is what closes it.** An //! idle CPU posts one probe on its way to `hlt` and nothing re-posts it, so //! under plain `Balance::Pull` a CPU that halted before any neighbour published -//! surplus was never probed again: 0 of 20 seeds reach every CPU at eight. A -//! bounded re-arm of the probe and a push on surplus each take that to 20 of 20 -//! at every width, and they cost opposite things — the timer ticks whether or -//! not there is anything to come for, the push fires only where there is. On -//! the audio pipeline that is 154 extra idle wakes per second against **zero**, -//! which is why `Balance::PushOnSurplus` ships (owner decision 2026-08-23) and -//! the re-arm was declined; these three cases are the numbers the decision was -//! made on. +//! surplus was never probed again. A bounded re-arm of the probe and a push on +//! surplus each reach every CPU at every width, and they cost opposite things — +//! the timer ticks whether or not there is anything to come for, the push fires +//! only where there is. That is why `Balance::PushOnSurplus` ships (owner +//! decision 2026-08-23) and the re-arm was declined; these three cases are the +//! numbers the decision was made on. Busy CPUs also balance their job groups' +//! summed weight between themselves (`SchedPass::balance_busy`), at most one +//! request per CPU per `BALANCE_NS`. //! //! # Determinism, seeds, and what varies //! @@ -93,21 +84,22 @@ //! doing what a sweep is for. use kernel::sched::cpu::Balance; -use kernel::sched::cpu::{PROTECTED_RUN_NS, STALE_PASS_NS}; -use kernel::sched::fair::QUANTUM_NS; +use kernel::sched::cpu::{BALANCE_NS, STALE_PASS_NS}; +use kernel::sched::queue::{MIN_SLICE_NS, SLICE_NS}; +use kernel::sched::task::QUANTUM_NS; use kernel_sim::choice::ChoiceStream; use kernel_sim::explore::{run, Outcome}; use kernel_sim::latency::{Latency, ReadyCause}; use kernel_sim::scenarios; use kernel_sim::vm::{IPI_LATENCY_NS, RUN_CHUNK_NS}; -use kernel_sim::workload::{Scenario, ShareShape}; +use kernel_sim::workload::{JobShape, Scenario}; const MS: u64 = 1_000_000; -/// One thread's work in the share-gain cases: six quanta, the same figure -/// `scenarios::WORK` gives a `fairness_storm` thread and for the same reason — -/// a window many quanta wide is what a granularity bound needs before it can -/// separate a fair split from a broken one. +/// One thread's work in the share-gain cases, the figure `scenarios::WORK` +/// gives a `fairness_storm` thread and for the same reason — a window many +/// requests wide is what a granularity bound needs before it can separate a +/// fair split from a broken one. const WORK: u64 = 60 * MS; /// Seeds per point for the one-CPU cases. They are deterministic (see the module @@ -115,13 +107,13 @@ const WORK: u64 = 60 * MS; /// search; the cases that actually explore say so where they set their own. const SEEDS: u64 = 16; -/// The fair band's granularity: one dispatch of one run queue. Every bound in -/// this file is a multiple of it. +/// The fair band's granularity: one request, observed up to two chunks late. +/// Every bound in this file is made of it. /// /// `max KernelSection` is zero in all three policy workloads — none of them runs /// an `Op::KernelSection` — so the term is dropped here rather than carried as a /// zero, and any workload that grows one has to add it back. -const DISPATCH_NS: u64 = QUANTUM_NS + 2 * RUN_CHUNK_NS; +const DISPATCH_NS: u64 = SLICE_NS + 2 * RUN_CHUNK_NS; fn stream(scenario: &Scenario, seed: u64) -> ChoiceStream { if seed.is_multiple_of(2) { @@ -152,33 +144,22 @@ fn sweep(scenario: &Scenario, seeds: u64, mut fold: impl FnMut(&Outcome)) { /// finishes *is* the share it was served at: 120 ms means half the CPU, 240 ms /// means a quarter. /// -/// **What it found.** `solo` finishes exactly `N+1` quanta late at every width -/// above one, and the seeds do not disagree by a nanosecond -/// (`cargo test -p kernel-sim --test policy -- --nocapture`): +/// **What it found.** `solo` finishes one slice late at every width above one, +/// and the seeds do not disagree by a nanosecond +/// (`cargo test --release -p kernel-sim --test policy -- --nocapture`): /// -/// | N | T_solo | deficit | in quanta | `solo`'s share over that window | -/// |---|---|---|---|---| -/// | 1 | 120 ms | 0 ms | 0 | 500‰ | -/// | 2 | 150 ms | 30 ms | 3 | 400‰ | -/// | 4 | 170 ms | 50 ms | 5 | 352‰ | -/// | 8 | 210 ms | 90 ms | 9 | 285‰ | -/// | 16 | 290 ms | 170 ms | 17 | 206‰ | -/// | 32 | 450 ms | 330 ms | 33 | 133‰ | -/// | 64 | 770 ms | 650 ms | 65 | 77‰ | +/// | N | T_solo | deficit | `solo`'s share over that window | +/// |---|---|---|---| +/// | 1 | 120 ms | 0 ms | 500‰ | +/// | 2–64 | 124 ms | 4 ms | 483‰ | /// -/// So the attack **works, once, and by exactly the fair band's staleness**: at -/// t=0 every thread is inserted at vruntime 0, `solo` is dispatched first and -/// re-inserted with its share's pot at 10 ms, and every one of `swarm`'s N -/// threads still holds the stale key it was queued with — so all N run a quantum -/// ahead of `solo`'s second turn. After that burst each swarm thread carries a -/// key its own dispatch set, and the two processes alternate one for one. +/// On one CPU the two jobs' groups weigh the same whatever N is — a group +/// weighs its job's weight times its share of the job's runnable threads, all +/// of which are here — so the swarm's threads divide its half among themselves. /// -/// **The bound is derived and it is not moved.** `(N + 2)` dispatches of the one -/// run queue — the N+1 runnable threads and the leader's extra quantum, which is -/// invariant I5's `(ΣT + 1)` term exactly. The measured deficit fills 0.63 of it -/// at N=2 and climbs to 0.82 at N=64, so the scheduler is closing on its own -/// granularity as the swarm widens rather than living comfortably inside it — -/// which is the same thing `scenarios::FAIRNESS_SAMPLE` records about I5. +/// **The bound is derived and it is not moved.** `solo`'s group's lag stays +/// within one request of zero, so it has had at least half the CPU less one +/// request at every instant, and spends its work at most two requests late. #[test] fn a_process_cannot_buy_cpu_by_forking() { let mut table = Vec::new(); @@ -187,7 +168,7 @@ fn a_process_cannot_buy_cpu_by_forking() { let solo = scenario .process_index("solo") .expect("share_gain has a solo"); - let bound = 2 * WORK + (threads as u64 + 2) * DISPATCH_NS; + let bound = 2 * WORK + 2 * DISPATCH_NS; let (mut worst, mut best) = (0, u64::MAX); sweep(&scenario, SEEDS, |outcome| { let finish = outcome.process_finish_ns[solo] @@ -198,44 +179,39 @@ fn a_process_cannot_buy_cpu_by_forking() { assert!( worst <= bound, "at {threads} rival thread(s) a single-threaded process took {worst} ns to spend \ - {WORK} ns of CPU, against a derived {bound} ns — two quanta of its own plus \ - {} dispatches of stale fair-band keys at {DISPATCH_NS} ns. Past this, forking \ - buys more CPU than the policy says it can.", - threads + 2, + {WORK} ns of CPU, against a derived {bound} ns — two of its own shares plus two \ + requests at {DISPATCH_NS} ns. Past this, forking buys more CPU than the policy \ + says it can.", ); table.push((threads, worst, worst - 2 * WORK, WORK * 1000 / worst, best)); } for &(threads, finish, deficit, permille, best) in &table { println!( - "share_gain N={threads}: T_solo={finish} ns deficit={deficit} ns ({} quanta) \ + "share_gain N={threads}: T_solo={finish} ns deficit={deficit} ns \ share={permille}permille (best seed {best})", - deficit / QUANTUM_NS, ); } - // The measurement has to be a comparison and not an accident of a bound - // nothing could reach: at one CPU the deficit is the band's granularity and - // the scheduler spends nearly all of it. - let (threads, _, deficit, _, _) = table[table.len() - 1]; - let bound = (threads as u64 + 2) * DISPATCH_NS; + // The shape of the finding, asserted rather than only tabulated: the + // deficit does not grow with the rival count, which is what "a helper buys + // no entitlement" means for one CPU. + let deficits: Vec = table.iter().map(|&(_, _, deficit, _, _)| deficit).collect(); + let (two, many) = (deficits[1], deficits[deficits.len() - 1]); assert!( - deficit * 2 > bound, - "at {threads} rivals the deficit is {deficit} ns against a {bound} ns bound — more \ - than a factor of two of slack means the bound has stopped constraining anything and \ - a real regression could hide under it (it sits at 0.82 of the bound today)", + many <= two, + "the deficit went from {two} ns at two rival threads to {many} ns at {} — a deficit \ + that grows with the swarm is a share bought by forking", + table[table.len() - 1].0, ); - - // And the shape of the finding, asserted rather than only tabulated: the - // deficit grows with the rival count, which is what makes this a *bound* on - // an attack rather than a constant nobody can move. - let one = table[0].2; - let many = table[table.len() - 1].2; + // And the measurement has to be a comparison and not an accident of a + // bound nothing could reach. assert!( - one == 0 && many > 0, - "the deficit went from {one} ns at one rival thread to {many} ns at {} — if it no \ - longer grows with the thread count, this case is measuring something else", + many * 4 > 2 * DISPATCH_NS, + "at {} rivals the deficit is {many} ns against a {} ns bound — more than a factor of \ + four of slack means the bound has stopped constraining anything", table[table.len() - 1].0, + 2 * DISPATCH_NS, ); } @@ -243,7 +219,7 @@ fn a_process_cannot_buy_cpu_by_forking() { /// between a granularity and a process that permanently owns more of the machine /// than it is entitled to. /// -/// The case above measures `solo` finishing `N+1` quanta late over a 120 ms +/// The case above measures `solo` finishing one slice late over a 120 ms /// window. That number is only tolerable if it is paid *once*: if it were paid /// per unit of work, `swarm` would be taking a fixed fraction of the machine for /// ever and the policy would simply be false. So the identical scenario is run @@ -254,18 +230,12 @@ fn a_process_cannot_buy_cpu_by_forking() { /// /// | N | work | T_solo | deficit | share | /// |---|---|---|---|---| -/// | 4 | 60 ms | 170 ms | 50 ms | 352‰ | -/// | 4 | 120 ms | 290 ms | 50 ms | 413‰ | -/// | 4 | 240 ms | 530 ms | 50 ms | 452‰ | -/// | 16 | 60 ms | 290 ms | 170 ms | 206‰ | -/// | 16 | 120 ms | 410 ms | 170 ms | 292‰ | -/// | 16 | 240 ms | 650 ms | 170 ms | 369‰ | +/// | 4, 16 | 60 ms | 124 ms | 4 ms | 483‰ | +/// | 4, 16 | 120 ms | 244 ms | 4 ms | 491‰ | +/// | 4, 16 | 240 ms | 484 ms | 4 ms | 495‰ | /// -/// The deficit is the same nanosecond at every window length, so the share -/// climbs toward 500‰ as the window grows — the floor a reader wants is -/// `1/2 − (N+1) × QUANTUM / 2L` for a window of length `L`, and it is a floor -/// independent of N only in the limit. That sentence is the honest form of the -/// policy's claim, and this is the measurement it rests on. +/// The deficit is the same nanosecond at every window length and every swarm, +/// so the share climbs toward 500‰ as the window grows. #[test] fn the_share_deficit_is_an_offset_and_not_a_drift() { for threads in [4usize, 16] { @@ -315,32 +285,32 @@ fn the_share_deficit_is_an_offset_and_not_a_drift() { } } -/// The negative control for both cases above: the **rejected** policy, one fair -/// share per thread instead of one per process. +/// The negative control for both cases above: the **rejected** policy, one job +/// per thread instead of one per process. /// /// Without it, "a process cannot buy CPU by forking" is a number with nothing to -/// compare it to. Under per-thread shares `swarm` is entitled to N times what +/// compare it to. Under a job per thread `swarm` is entitled to N times what /// `solo` is, and the instrument has to see that — not as a slightly worse /// deficit, but as a different machine. Measured: `solo` takes **1,020 ms** to -/// spend the same 60 ms of CPU, against 290 ms under the shipped policy and a -/// 336 ms derived bound, so the case above would red by 3.0×. +/// spend the same 60 ms of CPU, against 124 ms under the shipped policy and a +/// 132 ms derived bound, so the case above would red by 7.7×. /// -/// **Invariant I5's ceiling is lifted for this run, and only for it.** Per-thread -/// shares fail I5 within a few hundred microseconds — that verdict is -/// `scenarios::fair_share_per_thread`'s and is asserted next door — and the +/// **Invariant I5's ceiling is lifted for this run, and only for it.** A job per +/// thread fails I5 within a few hundred microseconds — that verdict is +/// `scenarios::fair_job_per_thread`'s and is asserted next door — and the /// explorer stops at the first violation, so `solo` would never finish and the -/// quantity this file measures could not be read at all. The allowance suppresses -/// the *verdict* and changes no scheduling decision: `ShareShape::PerThread` is -/// still the whole of what differs from the control's control below. +/// quantity this file measures could not be read at all. The lifted ceiling +/// suppresses the *verdict* and changes no scheduling decision: +/// `JobShape::PerThread` is still the whole of what differs from the control. #[test] -fn per_thread_shares_lose_the_floor() { +fn a_job_per_thread_loses_the_floor() { const THREADS: usize = 16; - let mut broken = scenarios::share_gain(THREADS, WORK).with_share(ShareShape::PerThread); + let mut broken = scenarios::share_gain(THREADS, WORK).with_job(JobShape::PerThread); // Past anything this run can produce, so I5 records the spread and reports // no violation; `Vm::fair_over_bound` still counts every crossing. broken = broken.with_fair_allowance(u64::MAX); let solo = broken.process_index("solo").expect("share_gain has a solo"); - let bound = 2 * WORK + (THREADS as u64 + 2) * DISPATCH_NS; + let bound = 2 * WORK + 2 * DISPATCH_NS; let mut broken_finish = 0; sweep(&broken, SEEDS, |outcome| { @@ -361,7 +331,7 @@ fn per_thread_shares_lose_the_floor() { ); assert!( broken_finish > bound, - "under one fair share per *thread* a single-threaded process against {THREADS} rival \ + "under one job per *thread* a single-threaded process against {THREADS} rival \ threads finished its 60 ms of work at {broken_finish} ns — inside the {bound} ns the \ case above allows. A policy that hands `swarm` sixteen times `solo`'s entitlement has \ to break that bound, or the bound is not measuring the policy.", @@ -378,22 +348,19 @@ fn per_thread_shares_lose_the_floor() { /// runs briefly waits for a CPU held by threads that never yield. /// /// The sleeper uses a quarter of a millisecond every three, so it is far under -/// its share, its stored lag is positive and clamped, and -/// `ShareState::enter_runnable` re-derives a vruntime at `frontier − lag` — -/// below every hog that has run recently. It should therefore be picked the -/// instant the running hog gives the CPU up. +/// its share and carries that back in with it: its group joins with no debt +/// and outweighs the hogs' group there, so its request is due first and it +/// takes the CPU once the running hog has had its minimum run. /// -/// **The exception is the spawn burst**: at t=0 every hog thread is queued at -/// vruntime 0 with a key that stays stale until its own first dispatch, so the -/// sleeper's *first* wake waits behind all of them. Every later wake finds a -/// band whose keys its lag beats. +/// **What it found**, 40 seeds per point: the worst wake is 5 ms at every hog +/// count from 1 to 64 and at both widths — at most the hog request running +/// ahead of it — and every wake but one per run at one CPU waits 2 ms at most, +/// the minimum run and the model's two chunks. /// -/// So the bound asserted is the band's granularity — `(rivals + 1)` dispatches, -/// where rivals is what one run queue holds — and the sharper one-quantum claim -/// is asserted at one CPU on the runner-up, where the burst is provably a single -/// wake. +/// So the bound asserted on every wake is one request, and the sharper claim +/// is asserted at one CPU on the runner-up. #[test] -fn an_interactive_wake_waits_out_at_most_the_band_it_is_queued_behind() { +fn an_interactive_wake_waits_out_at_most_one_request() { /// Enough runs that the distribution has a few hundred wakes in it; the /// device interrupt's position among the enabled steps is the freedom the /// seeds explore. @@ -403,10 +370,7 @@ fn an_interactive_wake_waits_out_at_most_the_band_it_is_queued_behind() { let sleeper = scenario .process_index("sleeper") .expect("interactive_mix has a sleeper"); - // What one run queue holds: the hogs spread over the CPUs, plus the - // sleeper itself, plus the leader's extra quantum. - let rivals = hogs.div_ceil(cpus) as u64 + 1; - let bound = (rivals + 1) * DISPATCH_NS; + let bound = DISPATCH_NS; let mut merged = Latency::default(); let mut worst_runner_up = 0; @@ -432,21 +396,19 @@ fn an_interactive_wake_waits_out_at_most_the_band_it_is_queued_behind() { ); assert!( merged.max_ns() <= bound, - "at {cpus} cpu(s) and {hogs} hog(s) an interactive wake waited {} ns against a \ - derived {bound} ns — {rivals} runnable thread(s) on one run queue plus the \ - leader, at {DISPATCH_NS} ns a dispatch. Distribution: {}", + "at {cpus} cpu(s) and {hogs} hog(s) an interactive wake waited {} ns against one \ + request, {bound} ns. Distribution: {}", merged.max_ns(), merged.summary(), ); if cpus == 1 { - let contract = DISPATCH_NS; + let contract = MIN_SLICE_NS + 2 * RUN_CHUNK_NS; assert!( worst_runner_up <= contract, "at {hogs} hog(s) on one CPU, more than one wake per run waited out more \ - than a quantum: the second-worst wake of some run took \ - {worst_runner_up} ns against one quantum plus granularity, {contract} ns. \ - The spawn burst delays the first wake and is measured as the maximum; a \ - second slow wake means the fair band's keys are staying stale.", + than the minimum run: the second-worst wake of some run took \ + {worst_runner_up} ns against {contract} ns. A wake due first is waiting for \ + the running request instead of taking the CPU after the minimum run.", ); } } @@ -462,9 +424,9 @@ const HOP_NS: u64 = IPI_LATENCY_NS + 2 * RUN_CHUNK_NS; /// the sibling. /// /// `scenarios::busy_home` puts a sleeper and a hog of one process on one CPU -/// beside an idle one, with the balance path off — so the woken thread's key -/// ties the hog's, no cut can serve it, and nothing but the forward can run it -/// before the hog's quantum ends. +/// beside an idle one, with the balance path off — so the woken thread joins +/// the hog's group, no cut can serve it, and nothing but the forward can run it +/// before the hog's request ends. /// /// **Only a run that forwarded is measured**, and most must. The model does /// not oblige a CPU with nothing to do to take its first pass, so in some @@ -501,30 +463,28 @@ fn a_wake_to_a_busy_cpu_beside_an_idle_one_runs_within_two_hops() { assert!( merged.max_ns() <= bound, "a wake waited {} ns against two hops, {bound} ns: the busy home's kick and the \ - idle CPU's adopt. Past this the wake is waiting for the hog's quantum. \ + idle CPU's adopt. Past this the wake is waiting for the hog's request. \ Distribution: {}", merged.max_ns(), merged.summary(), ); } -/// **A wake that is owed the CPU ahead of the running task waits out -/// [`PROTECTED_RUN_NS`] and a hop, not a quantum** — on one CPU, against one -/// rival process, every wake but one per run. +/// **A wake due ahead of the running task waits out [`MIN_SLICE_NS`] and a +/// hop, not a request** — on one CPU, against one rival process, every wake +/// but one per run. /// /// `interactive_mix(1, 1)`'s sleeper runs a quarter of a millisecond every -/// three, so its key is below the hog's at nearly every wake, and the one -/// exception is the burst at the start: the frontier is still the spawn's -/// zero when the sleeper first parks, so the lag it stores is measured against -/// a key the hog has long since left. That wake waits its turn in the band; -/// the per-run runner-up is the claim about every other, exactly as the case -/// above reads it. +/// three, so its request is due first at nearly every wake. The exception is +/// the start, where the sleeper's first wake can find the hog's group still +/// ahead; the per-run runner-up is the claim about every other, exactly as the +/// case above reads it. #[test] -fn a_wake_owed_the_cpu_waits_out_the_protected_run_and_no_more() { +fn a_wake_due_first_waits_out_the_minimum_run_and_no_more() { const OWED_SEEDS: u64 = 40; let scenario = scenarios::interactive_mix(1, 1); let sleeper = scenario.process_index("sleeper").expect("interactive_mix has a sleeper"); - let bound = PROTECTED_RUN_NS + HOP_NS; + let bound = MIN_SLICE_NS + HOP_NS; let mut merged = Latency::default(); let mut worst_runner_up = 0; sweep(&scenario, OWED_SEEDS, |outcome| { @@ -544,26 +504,28 @@ fn a_wake_owed_the_cpu_waits_out_the_protected_run_and_no_more() { assert!( worst_runner_up <= bound, "a second wake in one run waited {worst_runner_up} ns against {bound} ns — the \ - protected run and one hop. Past this a wake owed the CPU is waiting for the \ - running task's quantum.", + minimum run and one hop. Past this a wake due first is waiting for the running \ + task's request.", ); } /// **A thread that naps does not preempt the sibling that spins**: two threads -/// of one share tie, and a tie says nothing about which is owed the CPU. +/// of one job are one group, and inside a group the CPU decides only at a +/// request's end. /// /// `scenarios::sleep_spin_siblings` puts them on one CPU. The spinner runs a -/// whole quantum per turn and the napper one burst between turns, so the +/// whole request per turn and the napper one burst between turns, so the /// spinner's [`scenarios::SPINNER_WORK`] is done within -/// `SPINNER_WORK / QUANTUM` turns of `QUANTUM + burst + 2 × RUN_CHUNK`. A -/// napper whose every wake cut the spinner's quantum would hold it to -/// [`PROTECTED_RUN_NS`] a burst, and double the time it takes. +/// `SPINNER_WORK / SLICE` turns of `SLICE + burst + 2 × RUN_CHUNK`. A napper +/// whose every wake cut the spinner's request would hold it to +/// [`MIN_SLICE_NS`] a burst, and double the time it takes. Measured: 148 ms +/// against 175. #[test] fn a_sibling_that_naps_does_not_preempt_the_one_that_spins() { let scenario = scenarios::sleep_spin_siblings(); let spinner = kernel::sched::task::TaskKey(1); - let turns = scenarios::SPINNER_WORK.div_ceil(QUANTUM_NS); - let bound = turns * (QUANTUM_NS + scenarios::NAP_BURST_NS + 2 * RUN_CHUNK_NS); + let turns = scenarios::SPINNER_WORK.div_ceil(SLICE_NS); + let bound = turns * (SLICE_NS + scenarios::NAP_BURST_NS + 2 * RUN_CHUNK_NS); let mut worst = 0; sweep(&scenario, SEEDS, |outcome| { let finish = *outcome @@ -576,7 +538,7 @@ fn a_sibling_that_naps_does_not_preempt_the_one_that_spins() { assert!( worst <= bound, "the spinner took {worst} ns for {} ns of work against {bound} ns — its napping \ - sibling is taking the CPU off it between quanta, which a tied key does not earn", + sibling is taking the CPU off it inside its requests", scenarios::SPINNER_WORK, ); } @@ -688,13 +650,25 @@ const LOPSIDED: [(usize, usize); 4] = [(2, 8), (4, 16), (4, 64), (8, 64)]; /// it runs, and everything above that floor is stealable, one task per /// answered probe. /// -/// It is also the ceiling on the *whole run's* migrations, because in this -/// workload a thief never becomes a victim: a CPU holding one task publishes a -/// surplus of zero, so no task in flight here is ever moved twice. fn stealable(threads: usize) -> u64 { threads as u64 - 1 } +/// The most migrations a run of `elapsed` may make: everything above cpu0's +/// floor once, plus what busy balancing moves between CPUs that are both +/// running work — one request per CPU per [`BALANCE_NS`] at most, which is how +/// a thread can move twice: a thief becomes a victim when the threads it took +/// outlast cpu0's. +fn migration_ceiling(threads: usize, cpus: usize, elapsed: u64) -> u64 { + stealable(threads) + cpus as u64 * (elapsed / BALANCE_NS + 1) +} + +/// The fewest migrations that leave cpu0 holding no more than an even split of +/// the job, which every CPU's threads weigh the same in. +fn even_drain(threads: usize, cpus: usize) -> u64 { + (threads - threads.div_ceil(cpus)) as u64 +} + /// **Adversarial placement**: every thread of a saturating workload spawned onto /// one CPU, and what the balance path does with the machine that leaves. /// @@ -710,10 +684,10 @@ fn stealable(threads: usize) -> u64 { /// [`Balance::PushOnSurplus`] since the owner's 2026-08-23 decision. /// /// **Two findings.** Where the balance path runs at all it drains the loaded -/// CPU *completely*: the best seed at every width moves exactly [`stealable`] -/// tasks — all but the one cpu0 is running — so a machine an adversary piled -/// onto one CPU is emptied to the protocol's own floor, one task per answered -/// probe. +/// CPU to an even split: idle CPUs each take a thread, and then busy CPUs ask +/// for threads by their summed group weight, which for one job is their thread +/// count, until no move narrows it. The best seed at every width moves at least +/// [`even_drain`] tasks. /// /// And the pull alone **only ever reaches the CPUs that are still awake.** The /// probe is posted from an idle pass, one per idle trip, and only against a @@ -750,13 +724,13 @@ fn the_balance_path_drains_the_cpu_an_adversary_loaded() { those CPUs was given work by something that is not the balance path", outcome.migrations, ); - // And the ceiling: no task is moved twice, so the whole run cannot - // exceed what cpu0 held above the surplus floor. + // And the ceiling: what cpu0 has to give, plus what busy balancing + // is rate-limited to. + let ceiling = migration_ceiling(threads, cpus, outcome.elapsed); assert!( - outcome.migrations <= floor, - "at {cpus} cpus the balance path moved {} of {threads} tasks, past the {floor} \ - that stand above cpu0's surplus floor — a task is being migrated more than \ - once, which is a thief that became a victim", + outcome.migrations <= ceiling, + "at {cpus} cpus the balance path moved {} of {threads} tasks, past the {ceiling} \ + that cpu0's surplus and one busy request per CPU per `BALANCE_NS` allow", outcome.migrations, ); best = best.max(outcome.migrations); @@ -768,19 +742,17 @@ fn the_balance_path_drains_the_cpu_an_adversary_loaded() { println!( "lopsided cpus={cpus} threads={threads}: {full}/{LOPSIDED_SEEDS} seeds reached \ every cpu, {reached}/{LOPSIDED_SEEDS} reached a second, widest={widest}, best \ - seed moved {best} of a {floor} floor", + seed moved {best} with {floor} stealable", ); // **The drain, asserted where it is a law.** The best seed is the one in // which a thief was awake for the whole run, and there the balance path - // has to empty cpu0 down to the floor and no further. Anything less is a - // path that gives up while surplus is still published; anything more is - // the ceiling above. - assert_eq!( - best, floor, + // has to bring cpu0 down to an even split. + let even = even_drain(threads, cpus); + assert!( + best >= even, "at {cpus} cpus and {threads} threads on cpu0, the best of {LOPSIDED_SEEDS} \ - schedules moved {best} tasks and the surplus floor leaves {floor} stealable. A \ - machine with a thief awake for the whole run must be drained to that floor: one \ - task per answered probe, while a task is queued", + schedules moved {best} tasks, and leaving cpu0 an even split takes {even}: the \ + balance path gave up while cpu0 still weighed more than its peers", ); // **The shipped default reaches every CPU in every schedule.** This is // the push's whole reason to ship: under plain `Balance::Pull` a CPU @@ -803,7 +775,7 @@ fn the_balance_path_drains_the_cpu_an_adversary_loaded() { for &(cpus, threads, full, reached, best, floor) in &table { println!( "lopsided cpus={cpus} threads={threads}: every-cpu {full}/{LOPSIDED_SEEDS}, \ - second-cpu {reached}/{LOPSIDED_SEEDS}, drained {best}/{floor}", + second-cpu {reached}/{LOPSIDED_SEEDS}, moved {best} with {floor} stealable", ); } } @@ -853,10 +825,9 @@ fn without_the_balance_path_a_lopsided_machine_stays_lopsided() { /// How long a cure waits before probing again, and how many times. /// -/// One quantum, because that is the interval the rest of this scheduler already -/// works in: a loaded CPU reaches a pass at every quantum boundary, so a thief -/// that re-probes faster than that is asking a question whose answer cannot have -/// changed. Four, because the re-arm has to be **bounded** — `times × every_ns` +/// One quantum, because a loaded CPU reaches a pass at least that often — at +/// each fair request's end, or a real-time turn's — so a thief that re-probes +/// faster than that is asking a question whose answer may not have changed. Four, because the re-arm has to be **bounded** — `times × every_ns` /// is the whole window a cure of this shape can see a surplus in, and past it /// the CPU halts for good rather than ticking for ever on a machine with nothing /// to run. @@ -927,12 +898,11 @@ fn wake_rate(outcome: &Outcome) -> f64 { /// Run the lopsided machine under one policy and fold what it did. /// /// The two ceilings the shipped path already obeys are asserted here rather than -/// in each case, because they are claims about the *pull* half and every policy -/// below is built on it: no CPU is given work by anything but the balance path, -/// and no task is moved twice. +/// in each case, because every policy below is built on the same balance path: +/// no CPU is given work by anything but the balance path, and the migrations +/// stay under [`migration_ceiling`]. fn recover(cpus: usize, threads: usize, balance: Balance) -> Recovery { let scenario = scenarios::lopsided_placement(cpus, threads, WORK).with_balance(balance); - let floor = stealable(threads); let mut r = Recovery::default(); sweep(&scenario, LOPSIDED_SEEDS, |outcome| { let ran = outcome.cpus_reached(); @@ -943,11 +913,11 @@ fn recover(cpus: usize, threads: usize, balance: Balance) -> Recovery { those CPUs was given work by something that is not the balance path", outcome.migrations, ); + let ceiling = migration_ceiling(threads, cpus, outcome.elapsed); assert!( - outcome.migrations <= floor, + outcome.migrations <= ceiling, "{balance:?} at {cpus} cpus: the balance path moved {} of {threads} tasks, past the \ - {floor} that stand above cpu0's surplus floor — a task is being migrated more than \ - once, which is a thief that became a victim", + {ceiling} that cpu0's surplus and one busy request per CPU per `BALANCE_NS` allow", outcome.migrations, ); r.full += usize::from(ran == cpus); @@ -1075,14 +1045,14 @@ fn a_bounded_re_arm_reaches_every_cpu_the_pull_path_left_asleep() { the timer is not what is waking the CPU.", cured.gap_ns, ); - // And the drain is unchanged: a cure that reaches every CPU by - // moving more tasks than cpu0 has to give would be a different - // machine, not a repaired one. - assert_eq!( - cured.best_migrations, floor, - "the best of {LOPSIDED_SEEDS} schedules moved {} tasks and the surplus floor \ - leaves {floor} stealable — the re-arm changes when a probe is posted and \ - nothing about what answering one hands over", + // And the drain is the shipped one: the re-arm changes when a probe + // is posted and nothing about what answering one hands over, and + // `recover` holds every run to the ceiling. + let even = even_drain(threads, cpus); + assert!( + cured.best_migrations >= even, + "the best of {LOPSIDED_SEEDS} schedules moved {} tasks, and leaving cpu0 an \ + even split takes {even}", cured.best_migrations, ); table.push((cpus, threads, times, cured, control.full, control.gap_ns)); @@ -1117,8 +1087,8 @@ fn a_bounded_re_arm_reaches_every_cpu_the_pull_path_left_asleep() { /// **The derivation, and it is the assertion.** A pass pushes to one CPU, and /// `SchedPass::push_on_surplus`'s cursor makes consecutive pushes walk the /// machine, so `k` sleeping CPUs are reached in `k` passes of the CPU holding the -/// surplus. That CPU is running a task, so it reaches a pass at each quantum -/// boundary: `k × DISPATCH_NS`. The kick is an IPI ([`IPI_LATENCY_NS`], which the +/// surplus. That CPU is running a task, so it reaches a pass at each request's +/// end: `k × DISPATCH_NS`. The kick is an IPI ([`IPI_LATENCY_NS`], which the /// model enforces by disabling every execution step while a delivery is overdue) /// and the pass it asks for is one chunk of grace plus the chunk that carries the /// clock over it. Here `k` is `cpus − 1`, every CPU but the loaded one, so @@ -1184,10 +1154,11 @@ fn a_push_on_surplus_reaches_every_cpu_the_pull_path_left_asleep() { reaching the same CPU twice instead of walking the machine.", cured.gap_ns, ); - assert_eq!( - cured.best_migrations, floor, - "the best of {LOPSIDED_SEEDS} schedules moved {} tasks and the surplus floor leaves \ - {floor} stealable — the push changes which CPU asks and nothing about what \ + let even = even_drain(threads, cpus); + assert!( + cured.best_migrations >= even, + "the best of {LOPSIDED_SEEDS} schedules moved {} tasks, and leaving cpu0 an even \ + split takes {even} — the push changes which CPU asks and nothing about what \ answering hands over", cured.best_migrations, ); @@ -1245,10 +1216,10 @@ fn the_two_cures_are_priced_against_the_pull_path() { .iter() .position(|p| matches!(p.name, "sleeper" | "waiters" | "client")) .expect("every workload here has a thread whose wake latency is the point"); - // One run queue's worth of rivals plus the leader, exactly as - // `an_interactive_wake_waits_out_at_most_the_band_it_is_queued_behind` - // derives it. `audio_pipeline` carries its own thread count rather than - // a hog parameter, so it is counted from the scenario. + // One request per runnable thread one run queue holds, and one more: a + // bound loose enough for all three workloads, whose own sharper ones + // are the cases above. `audio_pipeline` carries its own thread count + // rather than a hog parameter, so it is counted from the scenario. let rivals = if hogs > 0 { hogs.div_ceil(cpus) as u64 + 1 } else { @@ -1284,8 +1255,8 @@ fn the_two_cures_are_priced_against_the_pull_path() { assert!( woken.max_ns() <= bound, "{label} [{name}]: a wake waited {} ns against a derived {bound} ns — {rivals} \ - runnable thread(s) on one run queue plus the leader, at {DISPATCH_NS} ns a \ - dispatch. A balance policy may cost idle wakes; it may not cost wake latency. \ + runnable thread(s) on one run queue and one more, at {DISPATCH_NS} ns a \ + request. A balance policy may cost idle wakes; it may not cost wake latency. \ Distribution: {}", woken.max_ns(), woken.summary(), @@ -1314,46 +1285,43 @@ fn the_two_cures_are_priced_against_the_pull_path() { /// **The starvation bound**: the worst wait of a runnable task under saturation, /// measured, against the fair band's own granularity. /// -/// This is the same quantity the two cases above measure from their own ends — -/// an interactive wake and a storm drain are both a task waiting for a run queue +/// The same quantity the cases above measure from their own ends — an +/// interactive wake and a storm drain are both a task waiting for a run queue /// to reach it — asked here of *every* task in a workload where nothing ever /// blocks and no CPU is ever idle. There is nothing to wait for but the band. /// /// **What it found**, 20 seeds per point: /// -/// | workload | runnable threads | waits seen | worst wait | in quanta | derived bound | ratio | +/// | workload | groups | largest group | waits seen | worst wait | derived bound | ratio | /// |---|---|---|---|---|---|---| -/// | `fairness_storm(1)` | 4 | 780 | 50 ms | 5 | 60 ms | 0.83 | -/// | `sibling_storm` | 4 | 3,660 | 50 ms | 5 | 60 ms | 0.83 | -/// | `share_gain(4)` | 5 | 680 | 70 ms | 7 | 72 ms | 0.97 | -/// | `share_gain(16)` | 17 | 2,360 | 200 ms | 20 | 216 ms | 0.93 | -/// | `share_gain(64)` | 65 | 9,080 | 680 ms | 68 | 792 ms | 0.86 | +/// | `fairness_storm(1)` | 2 | 3 | 1,860 | 20 ms | 36 ms | 0.56 | +/// | `sibling_storm` | 2 | 3 | 9,060 | 20 ms | 36 ms | 0.56 | +/// | `share_gain(4)` | 2 | 4 | 1,580 | 28 ms | 48 ms | 0.58 | +/// | `share_gain(16)` | 2 | 16 | 5,420 | 120 ms | 192 ms | 0.63 | /// -/// The bound is `(runnable threads + 1) × (QUANTUM + 2 × RUN_CHUNK)`, which is -/// invariant I13's bound with a different name on it, and the measurement is at -/// 0.83–0.97 of it everywhere. **Nothing starves, and the price of that sentence -/// is a linear one**: a task's worst wait is one dispatch per runnable thread on -/// its CPU, so a machine carrying 65 runnable threads makes some task wait 680 -/// ms. That is what "no starvation" is worth here, as a number. +/// A thread in a group of `n` waits for its `n − 1` siblings' requests, served +/// at the rate its group is — one `groups`-th of the CPU — so the bound is +/// `groups × n` requests. **Nothing starves, and the price is set by the +/// thread's own job**: a thread among 16 siblings waits out 15 of their +/// requests, and no number of other jobs' threads makes a job's thread wait +/// longer than its group's turn. #[test] -fn a_runnable_task_waits_at_most_one_dispatch_per_rival() { +fn a_runnable_task_waits_at_most_its_siblings_requests() { const STARVE_SEEDS: u64 = 20; for scenario in [ scenarios::fairness_storm(1), scenarios::sibling_storm(), scenarios::share_gain(4, WORK), scenarios::share_gain(16, WORK), - scenarios::share_gain(64, WORK), ] { let name = scenario.name; // Every thread of these workloads is runnable from the first dispatch to // its own exit, and every one of them is on the one CPU. let threads: u64 = scenario.procs.iter().map(|p| p.initial.len() as u64).sum(); - assert_eq!( - scenario.cpus(), 1, - "{name}: the bound below is one run queue's" - ); - let bound = (threads + 1) * DISPATCH_NS; + let groups = scenario.procs.len() as u64; + let largest = scenario.procs.iter().map(|p| p.initial.len() as u64).max().unwrap_or(0); + assert_eq!(scenario.cpus(), 1, "{name}: the bound below is one run queue's"); + let bound = groups * largest * DISPATCH_NS; let mut worst = 0; let mut samples = 0; @@ -1366,9 +1334,8 @@ fn a_runnable_task_waits_at_most_one_dispatch_per_rival() { .sum::(); }); println!( - "starvation {name} ({threads} runnable threads): worst={worst} ns \ - ({} quanta) bound={bound} ns over {samples} waits", - worst / QUANTUM_NS, + "starvation {name} ({groups} groups, largest {largest}): worst={worst} ns \ + bound={bound} ns over {samples} waits", ); assert!( samples > threads * STARVE_SEEDS, @@ -1379,13 +1346,9 @@ fn a_runnable_task_waits_at_most_one_dispatch_per_rival() { assert!( worst <= bound, "{name}: a runnable task waited {worst} ns for a CPU against a derived {bound} ns \ - — one dispatch per runnable thread on its run queue plus the leader's, at \ - {DISPATCH_NS} ns each. Past this, the fair band is passing some task over more \ - times than its insertion-time keys can account for.", + — its group's siblings' requests at its group's share of the CPU. Past this, the \ + fair band is passing some task over more often than EEVDF can account for.", ); - // Tight enough to constrain: the shipped scheduler sits at 0.83–0.97 of - // this everywhere, so a factor of two of slack would already be a bound - // that had stopped measuring. assert!( worst * 2 > bound, "{name}: the worst wait is {worst} ns against a {bound} ns bound, more than twice \ diff --git a/kernel/sim/tests/scenarios.rs b/kernel/sim/tests/scenarios.rs index 68911f1772..badeb247cb 100644 --- a/kernel/sim/tests/scenarios.rs +++ b/kernel/sim/tests/scenarios.rs @@ -9,7 +9,7 @@ //! decoration. //! 2. **Every scenario passes**, over a seed sweep and a fuzz-byte sweep. //! -//! # The ten negative gates, and the two controls +//! # The negative gates, and the control //! //! Each is a `scenarios::` constructor that *is* a broken scheduler, and each //! has a `#[test]` below asserting both that it is caught and which invariant @@ -24,18 +24,14 @@ //! | `old_migrate_kept_the_corpse` | the balance path handing on a killed task | I14 | //! | `old_rt_starved_the_corpse` | the RT band outranking the dying list without a bound | I14, on every seed | //! | `old_park_kept_the_lend` | commit `9c2fc4d`'s park keeping a lapsed lend | I9 | -//! | `fair_share_per_thread` | one fair share per thread, not per process | I5, and nothing else | -//! | `fair_double_charge` | a share charged twice for what it runs | I5, in the opposite direction | -//! | `fair_identity_within_share` | the lowest-keyed sibling served, not the earliest-inserted | I13, and nothing else | +//! | `fair_job_per_thread` | one job per thread, not per process | I5, and nothing else | +//! | `fair_double_charge` | a job group charged twice for what it runs | I5, in the opposite direction | +//! | `fair_lowest_key_within_group` | a group's lowest-keyed thread served, not its EEVDF pick | I13, and nothing else | //! | `overlong_pass` | a pass costing five times its budget | `cpu::PassCosts`, which records it rather than aborting | //! -//! The controls are `old_commit_fused` and `fair_identity_tiebreak`, and both -//! must come back **clean**. They are what make two of those gates measurements -//! rather than guesses about which break was needed: the first is the same +//! The control is `old_commit_fused`, which must come back **clean**: the same //! blocking bug with no step boundary to expose it, which is the blind spot -//! this harness used to have; the second is the tie-break `queue.rs` warns -//! about, ported literally and invisible to I13 — which is why the gate beside -//! it had to be the stronger `fair_identity_within_share`. +//! this harness used to have. //! //! # The liveness gates, which are a different claim //! @@ -53,7 +49,7 @@ //! //! The budgets here are what a `cargo test` can afford. The full criterion — //! 10⁴ seeds and 10⁷ fuzz steps per scenario class — runs from the CLI, where -//! `gate` carries all ten gates and both controls: +//! `gate` carries every gate and the control: //! //! ```text //! cargo run --release -p kernel-sim -- gate 10000 @@ -201,13 +197,19 @@ fn old_commit_before_pass_is_caught() { *kinds.entry(id).or_default() += 1; } } - assert_eq!( - caught, SEEDS as usize, - "committing the wait ticket before the pass must be caught in every \ - schedule, not merely in some: {caught}/{SEEDS}", + // Not every schedule: the explorer may leave a CPU un-stepped while it + // holds a task, so a consumer's request can end before it took a step, and + // the producer's tokens then outpace the consumers and no block finds the + // queue empty. Measured 415 of 500 on `e74dbf1` with the job-group + // scheduler applied. + assert!( + caught * 4 >= 3 * SEEDS as usize, + "committing the wait ticket before the pass must be caught in at least \ + three schedules of four: {caught}/{SEEDS}", ); + // 50 steps at worst, measured with `caught` above. assert!( - worst_steps < 32, + worst_steps < 64, "the worst schedule took {worst_steps} steps to expose it; this is a \ structural break and should surface almost immediately", ); @@ -646,35 +648,22 @@ fn the_audio_pipeline_holds_on_one_cpu() { /// Invariant I5 is *alive*, which is a separate claim from I5 being green. /// /// A fairness check whose contention windows never open reports "clean" on every -/// scenario in the tree and certifies nothing — which is exactly what would -/// happen here if the window conditions (same runnable set, saturated machine, -/// empty RT band) were -/// one degree stricter than the workload can satisfy. So the spread is required -/// to be *non-zero*: some window has to have opened, stayed open long enough for -/// a real separation to accumulate, and been measured. +/// scenario in the tree and certifies nothing. So the spread is required to be +/// *non-zero* — some window opened, stayed open long enough for a separation to +/// accumulate, and was measured — and within a factor of four of its own bound +/// at some width, or the bound has stopped constraining anything. /// -/// The widths are the two `all()` carries. Any other width runs from the CLI as -/// `measure fairness_storm:`; the sweep here is what `cargo test` can -/// afford. -/// -/// Both widths meet the *derived* bound today — 30/60 ms and 102/108 ms at -/// 10 000 seeds — so `worst_over_bound` must be zero here, and that is asserted: -/// if the recorded allowance ever starts carrying these two, the suite says so -/// rather than passing quietly on it. -/// -/// **And the *reach* is asserted, not only the verdict** — I13's gate, which I5 -/// went without until this test grew one. A non-zero spread says a window opened -/// *somewhere*; it does not say I5 was watching for any meaningful part of the -/// run, and one window a nanosecond wide satisfies it. That gap matters more for -/// I5 than for I13, because I5's window has four ways to close — an RT task -/// present, any CPU idle, the member set changing, a member under its even share -/// — against I13's one, and every one of them is a property the pick or the -/// placement can narrow without a single verdict going red. The recorded figures -/// are 99% and 81% of executed time (`measure fairness_storm: 100`, on -/// `a0729cf` with this change applied); a halving reds. +/// **And the *reach* is asserted, not only the verdict.** I5's window closes for +/// an RT task, an idle CPU, a changed member set or an uneven mix, and the pick +/// or the placement can narrow any of them without a verdict going red. The +/// recorded figures are 99% and 71% of executed time +/// (`measure fairness_storm: 200`, on `e74dbf1` with this change +/// applied), where the worst spreads were 4 ms of 20 and 12 ms of 40; a halving +/// of the reach reds. #[test] fn the_fairness_storm_is_measured_and_holds() { - for (cpus, reach) in [(1, 99), (2, 81)] { + let mut tightest = u64::MAX; + for (cpus, reach) in [(1, 99), (2, 71)] { let result = sweep::seed_sweep(&scenarios::fairness_storm(cpus), FAIR_SEEDS, 3); assert!( result.fair_coverage_pct() * 2 >= reach, @@ -693,38 +682,27 @@ fn the_fairness_storm_is_measured_and_holds() { verdict about nothing: {}", result.report(), ); - assert_eq!( - result.worst_over_bound, 0, - "at {cpus} cpu(s) the shipped scheduler has started crossing the \ - *derived* fairness bound and is passing on the recorded allowance \ - alone. That is a regression against the standard even though the \ - gate is green — see scenarios::FAIRNESS_SAMPLE: {}", - result.report(), - ); - // And the measurement has to be a *comparison*, not an accident of a - // bound so wide nothing could reach it. Both widths sit within a factor - // of two of the derived bound today, so a factor of four here is slack - // and not a target. - assert!( - result.worst_fair_spread * 4 > result.worst_fair_bound, - "at {cpus} cpu(s) the worst spread is more than 4x under its own \ - bound; the bound has stopped constraining anything: {}", - result.report(), - ); + assert_eq!(result.worst_over_bound, 0, "{}", result.report()); + tightest = tightest.min(result.worst_fair_bound / result.worst_fair_spread.max(1)); } + assert!( + tightest <= 4, + "every width sat more than 4x under the fairness bound (closest was \ + {tightest}x); the bound has stopped constraining anything", + ); } -/// The fifth self-validation gate: one fair share per *thread* — the rejected +/// The fifth self-validation gate: one job per *thread* — the rejected /// policy — instead of one per process. /// /// `trio` has three times `solo`'s threads and exactly the same entitlement. -/// Under the shipped policy they split the CPU evenly; under per-thread shares +/// Under the shipped policy they split the CPU evenly; under a job per thread /// `trio` takes three quarters. If I5 cannot see a 3:1 split of a two-way share, /// it is not measuring fairness, and every clean I5 report in this file means /// nothing. #[test] -fn per_thread_fair_shares_are_caught() { - let broken = scenarios::fair_share_per_thread(); +fn a_job_per_thread_is_caught() { + let broken = scenarios::fair_job_per_thread(); let mut caught = 0; let mut kinds: BTreeMap = BTreeMap::new(); for seed in 0..FAIR_SEEDS { @@ -745,16 +723,13 @@ fn per_thread_fair_shares_are_caught() { went undetected in {}/{FAIR_SEEDS} schedules", FAIR_SEEDS as usize - caught, ); - assert!( - kinds.contains_key("I5"), - "expected the service-spread violation; got {kinds:?}", - ); + assert!(kinds.contains_key("I5"), "expected the service-spread violation; got {kinds:?}"); assert_eq!( kinds.len(), 1, - "per-thread shares must be caught by I5 and not as collateral in some \ - other invariant — I6 in particular sums over a process's shares and \ - must stay correct under either shape: {kinds:?}", + "a job per thread must be caught by I5 and not as collateral in some \ + other invariant — I6 counts every job of a process and must stay \ + correct under either shape: {kinds:?}", ); // The control: the identical workload under the shipped policy, or the gate @@ -763,17 +738,15 @@ fn per_thread_fair_shares_are_caught() { assert!(shipped.passed(), "{}", shipped.report()); } -/// The sixth self-validation gate, and I5's other direction: a share charged -/// twice for every nanosecond it runs. +/// The sixth self-validation gate, and I5's other direction: a job group +/// charged twice for every nanosecond it runs. /// -/// It matters that this is a *separate* gate from `per_thread_fair_shares`, and -/// that the two fail in opposite directions. Fairness is service measured +/// It matters that this is a *separate* gate from `a_job_per_thread_is_caught`, +/// and that the two fail in opposite directions. Fairness is service measured /// against entitlement, and there are two ways to get it wrong — mis-attribute -/// the entitlement, or mis-measure the service. A check that saw only one of -/// them would be half an instrument, and the half it was missing would be -/// invisible. +/// the entitlement, or mis-measure the service. #[test] -fn double_charging_a_share_is_caught() { +fn double_charging_a_group_is_caught() { let broken = scenarios::fair_double_charge(); let mut caught = 0; let mut throttled = 0; @@ -806,7 +779,7 @@ fn double_charging_a_share_is_caught() { } assert_eq!( caught, FAIR_SEEDS as usize, - "a share throttled for work it never did went undetected in {}/{FAIR_SEEDS} \ + "a job throttled for work it never did went undetected in {}/{FAIR_SEEDS} \ schedules", FAIR_SEEDS as usize - caught, ); @@ -954,90 +927,66 @@ fn old_park_keeping_the_lend_is_caught() { } } -/// Invariant I13 is *alive*, which is a separate claim from I13 being green — -/// and, unlike I5, it is a claim nothing else in this file would notice was -/// false. I5 measures service per *process*, so it reports a perfectly even -/// split while one thread of a share never runs; if I13's windows never opened, -/// every fairness verdict in this tree would still read clean and sibling -/// starvation would be unguarded. -/// -/// So the spread is required to be non-zero on all three widths of the fairness -/// workload — some window has to have opened, stayed open, and been measured — -/// and to be within a factor of four of its own bound somewhere, or the bound -/// has stopped constraining anything. +/// Invariant I13 is *alive*, which is a separate claim from I13 being green, +/// and one nothing else here would notice was false: I5 measures service per +/// *process*, so a group that runs one thread and starves the rest leaves it +/// clean. /// -/// **And the *reach* is asserted, not only the verdict.** I13's window closes -/// when a member's threads stop being spread evenly over the CPUs, so a change -/// to the pick or the balance can make this check measure less instead of -/// failing — a live gate switched off by the thing it guards. The recorded -/// figures are 96%, 69% and 99% of executed time (`measure 100`, on -/// `be4b34a` with this change applied); a halving reds. The falloff with width -/// is real and worth knowing: 55% at four CPUs and 45% at eight, because -/// threads exit at slightly different moments and unbalance the machine sooner -/// the wider it is. All three meet the *derived* bound today -/// (the recorded sample on `scenarios::fair_workload`), so -/// `worst_thread_over_bound` must be zero, and that is asserted: if the recorded allowance ever starts carrying these, the -/// suite says so rather than passing quietly on it. +/// So the most a waiting thread was owed is required to be non-zero on all three +/// fairness workloads, and within a factor of four of its bound somewhere, or +/// the bound has stopped constraining anything. **The reach is asserted too**: +/// a change to placement or the pick could leave no thread owed service rather +/// than fail. The recorded figures are 100%, 88% and 100% of executed time, and +/// worst owed 3.3, 7.6 and 3.3 ms against 24 (`measure 200`, on +/// `e74dbf1` with this change applied); a halving of the reach reds. #[test] fn invariant_i13_is_measured_and_holds() { let mut tightest = u64::MAX; for (scenario, reach) in [ - (scenarios::fairness_storm(1), 96), - (scenarios::fairness_storm(2), 69), - (scenarios::sibling_storm(), 99), + (scenarios::fairness_storm(1), 100), + (scenarios::fairness_storm(2), 88), + (scenarios::sibling_storm(), 100), ] { let name = scenario.name; let result = sweep::seed_sweep(&scenario, FAIR_SEEDS, 3); assert!( - result.thread_coverage_pct() * 2 >= reach, - "{name}: invariant I13 had a comparison open for {}% of the run \ - against {reach}% recorded. Its *reach* has collapsed, which is not \ - the same as its verdict being clean — read this as loudly as a \ - violation and find what closed the windows: {}", - result.thread_coverage_pct(), + result.wait_coverage_pct() * 2 >= reach, + "{name}: some fair thread was owed service for {}% of the run against \ + {reach}% recorded. I13's *reach* has collapsed, which is not the same \ + as its verdict being clean: {}", + result.wait_coverage_pct(), result.report(), ); assert!(result.passed(), "{}", result.report()); assert!( - result.worst_thread_spread > 0, - "{name}: invariant I13 never opened a contention window it could \ - measure, so its clean verdict on every other scenario is a verdict \ - about nothing: {}", - result.report(), - ); - assert_eq!( - result.worst_thread_over_bound, 0, - "{name}: the shipped scheduler has started crossing the *derived* \ - per-thread bound and is passing on the recorded allowance alone. \ - That is a regression against the standard even though the gate is \ - green — see the recorded sample on scenarios::fair_workload: {}", + result.worst_wait_owed > 0, + "{name}: no waiting thread was ever owed service, so I13's clean \ + verdict on every other scenario is a verdict about nothing: {}", result.report(), ); - tightest = tightest.min(result.worst_thread_bound / result.worst_thread_spread.max(1)); + assert_eq!(result.worst_wait_over_bound, 0, "{}", result.report()); + tightest = tightest.min(result.worst_wait_bound / result.worst_wait_owed.max(1)); } assert!( tightest <= 4, - "every fairness width sat more than 4x under the per-thread bound \ + "every fairness workload sat more than 4x under the wait bound \ (closest was {tightest}x); the bound has stopped constraining anything", ); } /// The eighth self-validation gate, and the one I5 is structurally incapable of -/// standing in for: within a share, the *lowest-keyed* ready thread is -/// dispatched every time instead of the earliest-inserted one. +/// standing in for: each group serves its *lowest-keyed* thread every time +/// instead of its EEVDF pick. /// -/// It **must fail**, and it must fail on I13 and nothing else. That second half -/// is the whole point. A share's pot is charged for the time the *process* ran, -/// not for which of its threads ran it, so this ordering leaves the per-process -/// split exactly as it was: `solo` and `trio` still take half the machine each, -/// I5 still reports an even split, I6's refcounts are still right — and two of -/// `trio`'s three threads never run at all. If I5 ever starts firing here as -/// well, this has stopped being a test of the thing I13 was built for. +/// It **must fail**, and on I13 and nothing else. A group is charged for the +/// time its job ran, not for which of its threads ran it, so the per-process +/// split is exactly as it was: `solo` and `trio` still take half the machine +/// each, I5 still reports an even split, I6's counts are still right — and two +/// of `trio`'s three threads never run. #[test] -fn identity_ordering_within_a_share_is_caught() { - let broken = scenarios::fair_identity_within_share(); +fn the_lowest_key_within_a_group_is_caught() { + let broken = scenarios::fair_lowest_key_within_group(); let mut caught = 0; - let mut starved = 0; let mut kinds: BTreeMap = BTreeMap::new(); for seed in 0..FAIR_SEEDS { let mut choices = ChoiceStream::from_seed(seed); @@ -1046,11 +995,6 @@ fn identity_ordering_within_a_share_is_caught() { continue; } caught += 1; - // A starved sibling, not merely an uneven one: the report has to show a - // thread of the share on zero nanoseconds of service. - if outcome.violations.iter().any(|v| v.contains("=0")) { - starved += 1; - } for violation in &outcome.violations { let id = violation.split(':').next().unwrap_or("?").to_string(); *kinds.entry(id).or_default() += 1; @@ -1058,76 +1002,20 @@ fn identity_ordering_within_a_share_is_caught() { } assert_eq!( caught, FAIR_SEEDS as usize, - "a share serving one of its three threads and starving the other two \ + "a group serving one of its three threads and starving the other two \ went undetected in {}/{FAIR_SEEDS} schedules", FAIR_SEEDS as usize - caught, ); - assert_eq!( - starved, caught, - "the gate fired without a sibling on zero service; it is catching \ - something other than starvation", - ); assert_eq!( kinds.len(), 1, "sibling starvation must be caught by I13 and by I13 alone — I5 sees an \ even per-process split here, which is exactly why I13 exists: {kinds:?}", ); - assert!( - kinds.contains_key("I13"), - "expected the per-thread service violation; got {kinds:?}", - ); + assert!(kinds.contains_key("I13"), "expected the wait violation; got {kinds:?}"); - // The control: the identical workload under the shipped ordering, or the - // gate is detecting the workload rather than the ordering. + // The control: the identical workload under the shipped pick, or the gate + // is detecting the workload rather than the pick. let shipped = sweep::seed_sweep(&scenarios::sibling_storm(), FAIR_SEEDS, 1); assert!(shipped.passed(), "{}", shipped.report()); } - -/// The control that makes `identity_ordering_within_a_share_is_caught` a -/// measurement rather than a guess about which break was needed — and a finding -/// in its own right. -/// -/// `queue.rs` says the fair band's tie-break must not be `TaskKey` because "the -/// same thread wins every tie and the others only run when it blocks". Ported -/// exactly as written, that ordering is **invisible**: a share's pot is charged -/// for every nanosecond any of its threads runs, so a thread re-inserted after -/// a dispatch already carries a key strictly above every sibling queued before -/// it. The ordering is insertion order whatever the tie-break is; exact ties -/// survive only where no charge separates two inserts, and one dispatch -/// dissolves them. -/// -/// So this asserts two things: that the tie-break really is a different -/// scheduler — some seed must produce a different run, or the mode is a no-op -/// and proves nothing — and that I13 nonetheless comes back clean on every one. -#[test] -fn an_identity_tiebreak_changes_the_schedule_and_not_the_sibling_split() { - let shipped = scenarios::sibling_storm(); - let broken = scenarios::fair_identity_tiebreak(); - let mut diverged = 0; - for seed in 0..FAIR_SEEDS { - let mut choices = ChoiceStream::from_seed(seed); - let with = run(broken.clone(), &mut choices); - assert!( - with.passed(), - "the identity tie-break is expected to be invisible to I13; if it \ - now fails, the control has stopped controlling and the second gate \ - may no longer be the one that was needed: {}", - with.report(), - ); - let mut choices = ChoiceStream::from_seed(seed); - let without = run(shipped.clone(), &mut choices); - assert!(without.passed(), "{}", without.report()); - if (with.steps, with.switches, with.fair_spread) - != (without.steps, without.switches, without.fair_spread) - { - diverged += 1; - } - } - assert!( - diverged > 0, - "the identity tie-break produced an identical run on all {FAIR_SEEDS} \ - seeds, so `FairOrder::IdentityTiebreak` is not reaching the queue at \ - all and its clean verdict says nothing", - ); -} diff --git a/kernel/sim/tests/topology.rs b/kernel/sim/tests/topology.rs index 69167e01e0..995da0be8d 100644 --- a/kernel/sim/tests/topology.rs +++ b/kernel/sim/tests/topology.rs @@ -11,7 +11,7 @@ //! ``` use kernel::sched::cpu::{Balance, PUSH_THRESHOLD}; -use kernel::sched::fair::QUANTUM_NS; +use kernel::sched::task::QUANTUM_NS; use kernel_sim::choice::ChoiceStream; use kernel_sim::explore::{run, Outcome}; use kernel_sim::scenarios; diff --git a/kernel/src/arch/aarch64/irqchip.rs b/kernel/src/arch/aarch64/irqchip.rs index 3e2b48b0a4..a61856ab5f 100644 --- a/kernel/src/arch/aarch64/irqchip.rs +++ b/kernel/src/arch/aarch64/irqchip.rs @@ -492,7 +492,7 @@ pub fn comparator() -> u64 { /// far past `armed`. #[cfg(feature = "test-actuators")] pub fn rearmed_a_quantum(armed: u64) -> bool { - comparator() >= armed + crate::clock::counter_ticks(kernel::sched::fair::QUANTUM_NS) + comparator() >= armed + crate::clock::counter_ticks(kernel::sched::task::QUANTUM_NS) } /// Stop the timer: no interrupt until it is armed again. @@ -518,7 +518,7 @@ pub(super) fn rearm() { pub(super) fn rearm_in_kernel() { match percpu::armed_ticks() { 0 => stop_timer_hardware(), - _ => compare_at(cpu::counter() + crate::clock::counter_ticks(kernel::sched::fair::QUANTUM_NS)), + _ => compare_at(cpu::counter() + crate::clock::counter_ticks(kernel::sched::task::QUANTUM_NS)), } } @@ -538,7 +538,7 @@ pub fn floor_selftest() { ctl & TIMER_ISTATUS != 0 }; settles(100, "the timer armed for its floor", due); - let now = arm_within(kernel::sched::fair::QUANTUM_NS); + let now = arm_within(kernel::sched::task::QUANTUM_NS); let cval: u64; // SAFETY: reads the EL1 virtual timer's comparator. unsafe { core::arch::asm!("mrs {}, cntv_cval_el0", out(reg) cval, options(nomem, nostack, preserves_flags)) }; diff --git a/kernel/src/arch/x86_64/apic.rs b/kernel/src/arch/x86_64/apic.rs index 3e1a8389e0..ba3227be11 100644 --- a/kernel/src/arch/x86_64/apic.rs +++ b/kernel/src/arch/x86_64/apic.rs @@ -115,7 +115,7 @@ pub(super) fn msi_message(vector: u8) -> (u32, u32) { /// Calibrated LAPIC timer ticks per 10ms (computed on BSP, reused by APs), /// which is one quantum: what the Ring 0 timer branch re-arms with. static TIMER_TICKS: AtomicU32 = AtomicU32::new(0); -const _: () = assert!(kernel::sched::fair::QUANTUM_NS == 10_000_000); +const _: () = assert!(kernel::sched::task::QUANTUM_NS == 10_000_000); /// The spurious-interrupt vector register, whole (SDM §13.9, APM §16.4.7): the /// APIC enabled, the vector `arch::idt::spurious` gates, and every other bit diff --git a/kernel/src/arch/x86_64/idt/spurious.rs b/kernel/src/arch/x86_64/idt/spurious.rs index 06190a061a..c50fa11572 100644 --- a/kernel/src/arch/x86_64/idt/spurious.rs +++ b/kernel/src/arch/x86_64/idt/spurious.rs @@ -110,7 +110,7 @@ pub fn selftest() { // Confirms the CPU still takes interrupts afterward, by the one it arms here. // The timer's own count and never a total: the hard-lockup sampler's NMI lands whatever `IF` holds. let taken = crate::irq_census::deliveries(cpu, Source::Timer).unwrap_or(0); - apic::arm_within(kernel::sched::fair::QUANTUM_NS); + apic::arm_within(kernel::sched::task::QUANTUM_NS); let ran_on = crate::clock::settles(ARRIVES.nanos(), || { crate::irq_census::deliveries(cpu, Source::Timer).unwrap_or(taken) > taken }); diff --git a/kernel/src/arch/x86_64/idt/unclaimed.rs b/kernel/src/arch/x86_64/idt/unclaimed.rs index cba4458720..f3b0299c30 100644 --- a/kernel/src/arch/x86_64/idt/unclaimed.rs +++ b/kernel/src/arch/x86_64/idt/unclaimed.rs @@ -164,7 +164,7 @@ pub fn selftest() { // The timer's own count and never a total: the hard-lockup sampler's NMI lands whatever `IF` holds. let taken = crate::irq_census::deliveries(cpu, Source::Timer).unwrap_or(0); - apic::arm_within(kernel::sched::fair::QUANTUM_NS); + apic::arm_within(kernel::sched::task::QUANTUM_NS); let ran_on = crate::clock::settles(ARRIVES.nanos(), || { crate::irq_census::deliveries(cpu, Source::Timer).unwrap_or(taken) > taken }); diff --git a/kernel/src/deadline.rs b/kernel/src/deadline.rs index b5e7feddfa..50a27bba79 100644 --- a/kernel/src/deadline.rs +++ b/kernel/src/deadline.rs @@ -314,7 +314,7 @@ fn this_cpu() -> ! { // hard lockup and not the state this control claims. crate::preempt::disable(); let arrived_awake = crate::arch::cpu::interrupts_enabled(); - crate::arch::irqchip::arm_within(kernel::sched::fair::QUANTUM_NS); + crate::arch::irqchip::arm_within(kernel::sched::task::QUANTUM_NS); crate::arch::cpu::enable_interrupts(); log!( "wedge: cpu{} {}", diff --git a/kernel/src/hw.rs b/kernel/src/hw.rs index 29f0e21bb0..06cd961dd9 100644 --- a/kernel/src/hw.rs +++ b/kernel/src/hw.rs @@ -10,7 +10,7 @@ use core::sync::atomic::{AtomicU64, Ordering::Relaxed}; use kernel::sched::cpu::SleepToken; -use kernel::sched::fair::QUANTUM_NS; +use kernel::sched::task::QUANTUM_NS; use kernel::sched::hw::{CpuId, Kicker, Machine, Nanos, TraceEvent}; use crate::arch::{irqchip, percpu}; diff --git a/kernel/src/loader/mod.rs b/kernel/src/loader/mod.rs index f271687603..2d7765eb16 100644 --- a/kernel/src/loader/mod.rs +++ b/kernel/src/loader/mod.rs @@ -163,7 +163,7 @@ fn segment_prot(seg: &toyos_elf::Segment) -> crate::mm::policy::Prot { /// Load a program and place its main thread under `parent`, answering its pid /// and what `commit` left its caller holding of it. `commit` builds the /// child's handle table around the child's handle to itself, once nothing is -/// left to refuse. +/// left to refuse, and names the job it runs in. /// /// `path` is the program, opened here — or, with `image`, the bytes the caller /// read from it itself, and then nothing opens it. `argv[0]` is only the name the child goes by. @@ -174,7 +174,9 @@ fn segment_prot(seg: &toyos_elf::Segment) -> crate::mm::policy::Prot { pub fn spawn( path: &str, argv: &[&str], - commit: impl FnOnce(crate::object::HandleEntry) -> Result<(HandleTable, Endowments, H), crate::object::Refusal>, + commit: impl FnOnce( + crate::object::HandleEntry, + ) -> Result<(HandleTable, Endowments, Arc, H), crate::object::Refusal>, cwd: String, env: Vec, image: Option>, @@ -341,7 +343,7 @@ pub fn spawn( let object = crate::object::process::ProcessObject::new(pid); // The point of no return: every failure above answers the caller with its // table untouched. - let (handles, endowments, held) = commit(start::own_handle(&object))?; + let (handles, endowments, job, held) = commit(start::own_handle(&object))?; let proc_data = Arc::new(Lock::new(ProcessData { handles, cwd, @@ -372,6 +374,7 @@ pub fn spawn( spawn_ns: crate::clock::nanos_since_boot(), accounting: ProcessAccounting::default(), endowments, + job: job.clone(), })); let thread_data = Arc::new(Lock::new(ThreadData { @@ -408,6 +411,7 @@ pub fn spawn( child_pt.clone(), thread_pointer, Some(image), + job, ); table.get_mut(pid).unwrap().threads_mut().get_mut(tid).unwrap().set_sched(sched); }); @@ -460,6 +464,7 @@ pub fn spawn_supervisor() -> Pid { let rights = Rights::DUP .union(Rights::TRANSFER) .union(Rights::DEVICE) + .union(Rights::JOBS) .union(Rights::RT) .union(Rights::LOG) .union(Rights::WAIT) @@ -482,7 +487,7 @@ pub fn spawn_supervisor() -> Pid { // Built by the kernel and owing nobody anything: no table but the supervisor's own holds it. let commit = |own| { start::endow_self(&mut handles, &mut entries, &mut labels, own); - Ok((handles, Endowments::new(entries, labels), ())) + Ok((handles, Endowments::new(entries, labels), crate::object::job::new_job(), ())) }; match spawn(SUPERVISOR_PATH, &[SUPERVISOR_PATH], commit, String::from("/"), Vec::new(), None, Parent::Root) { Ok((pid, ())) => pid, diff --git a/kernel/src/loader/start.rs b/kernel/src/loader/start.rs index 32dc8d8f12..bebdf5342a 100644 --- a/kernel/src/loader/start.rs +++ b/kernel/src/loader/start.rs @@ -8,6 +8,7 @@ use alloc::sync::Arc; use alloc::vec::Vec; +use crate::object::job::JobObject; use crate::object::process::ProcessObject; use crate::object::{ops, HandleEntry, HandleTable, KObjectRef, Refusal}; use crate::process::{ @@ -16,8 +17,9 @@ use crate::process::{ use crate::scheduler; use crate::user_ptr::UserBytes; use toyos_abi::handle::{RawHandle, Rights}; +use kernel::sched::job::Job; use toyos_abi::syscall::{ - EndowEntry, SyscallError, MAX_SPAWN_ENDOWMENTS, MAX_SPAWN_LABELS_LEN, SELF_LABEL, + EndowEntry, SyscallError, JOB_LABEL, MAX_SPAWN_ENDOWMENTS, MAX_SPAWN_LABELS_LEN, SELF_LABEL, }; /// Where a new context first runs. @@ -105,14 +107,16 @@ pub struct PendingHandles { impl PendingHandles { /// Take the endowed handles out of the caller's table and put its handle to the child in it, all under one lock hold: a refusal leaves the table unchanged. - /// `own` is the child's handle to itself, which its table holds under [`SELF_LABEL`] beside the endowments, and the caller's handle is the third answer. - pub fn commit(self, own: HandleEntry) -> Result<(HandleTable, Endowments, RawHandle), Refusal> { + /// `own` is the child's handle to itself, which its table holds under [`SELF_LABEL`] beside the endowments. The child's job is the one endowed under + /// [`JOB_LABEL`], or else the caller's; the caller's handle to the child is the last answer. + pub fn commit(self, own: HandleEntry) -> Result<(HandleTable, Endowments, Arc, RawHandle), Refusal> { let Self { mut table, endow, mut labels } = self; let data_arc = process_data(); let mut data = data_arc.lock(); let count = endow.len() / ENDOW_ENTRY_LEN; let mut moving: Vec<(EndowEntry, RawHandle)> = Vec::with_capacity(count); + let mut endowed_job: Option> = None; for raw in endow.as_chunks::().0 { let label_off = u32::from_ne_bytes([raw[0], raw[1], raw[2], raw[3]]); let label_len = u32::from_ne_bytes([raw[4], raw[5], raw[6], raw[7]]); @@ -123,10 +127,18 @@ impl PendingHandles { if end > labels.len() { return Err(SyscallError::InvalidArgument.into()); } + let label = &labels[label_off as usize..end]; // The kernel's own, which a caller's of that name would shadow in the child's lookup. - if &labels[label_off as usize..end] == SELF_LABEL.as_bytes() { + if label == SELF_LABEL.as_bytes() { return Err(SyscallError::InvalidArgument.into()); } + if label == JOB_LABEL.as_bytes() { + // One owner: a second would be a job the child's lookup cannot name. + if endowed_job.is_some() { + return Err(SyscallError::InvalidArgument.into()); + } + endowed_job = Some(data.handles.get_sent::(handle, Rights::WRITE)?.job().clone()); + } // Checked before any removal, so a missing `TRANSFER` refuses the spawn instead of leaving a hole. let rights = data.handles.rights_of(handle)?; if !rights.contains(Rights::TRANSFER) { @@ -182,10 +194,11 @@ impl PendingHandles { // Minted while `own` is held, and `own` goes into a table no thread reaches until the child lands: another thread of the caller closing this handle never closes the object's last. let held = ops::install(&mut data.handles, own.object().clone()) .expect("a caller's table with verified room refused its child"); + let job = endowed_job.unwrap_or_else(|| data.job.clone()); drop(data); drop(displaced); endow_self(&mut table, &mut entries, &mut labels, own); - Ok((table, Endowments::new(entries, labels), held)) + Ok((table, Endowments::new(entries, labels), job, held)) } } diff --git a/kernel/src/object/job.rs b/kernel/src/object/job.rs new file mode 100644 index 0000000000..7f55a7cef5 --- /dev/null +++ b/kernel/src/object/job.rs @@ -0,0 +1,40 @@ +//! A handle to a job: the owner a process's threads, and its children's unless +//! a spawn names another, share CPU time under (`kernel::sched::job`). A spawn +//! that endows one under `JOB_LABEL` with `WRITE` puts its child in it. +//! +//! The job lives while a thread of it runs or a handle to it is held, and +//! not past both: its launcher ending takes nothing from it. + +use alloc::sync::Arc; +use core::sync::atomic::{AtomicU64, Ordering}; + +use kernel::sched::job::{Job, JobKey}; +use toyos_abi::handle::Rights; + +use super::{KObjectVariant, ObjectCore}; + +pub struct JobObject { + /// Visible to `object`, where `kobject!` generates this type's `core()`. + pub(super) core: ObjectCore, + job: Arc, +} + +impl JobObject { + /// What a handle the kernel mints to a job carries: `WRITE` to place a + /// child in it, `DUP` and `TRANSFER` to hand that on. + pub const RIGHTS: Rights = Rights::WRITE.union(Rights::DUP).union(Rights::TRANSFER); + + pub fn new(job: Arc) -> Arc { + Arc::new(Self { core: Self::new_core(), job }) + } + + pub fn job(&self) -> &Arc { + &self.job + } +} + +/// A job no thread has joined yet. +pub fn new_job() -> Arc { + static NEXT: AtomicU64 = AtomicU64::new(0); + Arc::new(Job::new(JobKey(NEXT.fetch_add(1, Ordering::Relaxed)))) +} diff --git a/kernel/src/object/mod.rs b/kernel/src/object/mod.rs index 47f5976269..65cec5d712 100644 --- a/kernel/src/object/mod.rs +++ b/kernel/src/object/mod.rs @@ -20,6 +20,7 @@ pub mod device; pub mod file; pub mod handle; pub mod inbox; +pub mod job; pub mod namespace; pub mod null; pub mod ops; @@ -254,6 +255,8 @@ kobject! { // The last handle's loss is the loss of the *ability to wait*, not proof the process should stop. immediate Process => process::ProcessObject, immediate Null => null::NullObject, + // A job outlives its handles while a thread of it runs: what ends it is its last reference. + immediate Job => job::JobObject, } /// Objects whose last handle has gone, waiting for release with nothing held; diff --git a/kernel/src/object/ops.rs b/kernel/src/object/ops.rs index eb99be1bbb..f4fb18fd86 100644 --- a/kernel/src/object/ops.rs +++ b/kernel/src/object/ops.rs @@ -59,6 +59,7 @@ pub fn initial_rights(object: &KObjectRef) -> Rights { // A spawner gets everything a child handle offers: exit code, kill, and accounting. KObjectRef::Process(_) => BASE.union(Rights::READ).union(Rights::MANAGE), KObjectRef::Null(_) => super::null::NullObject::RIGHTS, + KObjectRef::Job(_) => super::job::JobObject::RIGHTS, } } @@ -222,7 +223,7 @@ pub fn pipe_read(object: &KObjectRef) -> Option<(PipeId, WaitClass)> { | KObjectRef::Console(_) | KObjectRef::Acceptor(_) | KObjectRef::Inbox(_) | KObjectRef::SysCap(_) | KObjectRef::Connector(_) | KObjectRef::Namespace(_) - | KObjectRef::SharedMem(_) | KObjectRef::Process(_) | KObjectRef::Null(_) => None, + | KObjectRef::SharedMem(_) | KObjectRef::Process(_) | KObjectRef::Job(_) | KObjectRef::Null(_) => None, } } @@ -236,7 +237,7 @@ pub fn pipe_write(object: &KObjectRef) -> Option<(PipeId, WaitClass, Landing)> { | KObjectRef::Console(_) | KObjectRef::Acceptor(_) | KObjectRef::Inbox(_) | KObjectRef::SysCap(_) | KObjectRef::Connector(_) | KObjectRef::Namespace(_) - | KObjectRef::SharedMem(_) | KObjectRef::Process(_) | KObjectRef::Null(_) => None, + | KObjectRef::SharedMem(_) | KObjectRef::Process(_) | KObjectRef::Job(_) | KObjectRef::Null(_) => None, } } @@ -290,7 +291,7 @@ pub fn read_watch(object: &KObjectRef) -> Option { KObjectRef::SysCap(_) => Some(WatchRef::Static(&crate::log::user::WATCH)), KObjectRef::PipeWrite(_) | KObjectRef::File(_) | KObjectRef::Inbox(_) | KObjectRef::Connector(_) | KObjectRef::Namespace(_) - | KObjectRef::SharedMem(_) | KObjectRef::Null(_) => None, + | KObjectRef::SharedMem(_) | KObjectRef::Null(_) | KObjectRef::Job(_) => None, } } @@ -304,7 +305,7 @@ pub fn write_watch(object: &KObjectRef) -> Option { | KObjectRef::Acceptor(_) | KObjectRef::Inbox(_) | KObjectRef::SysCap(_) | KObjectRef::Connector(_) | KObjectRef::Namespace(_) - | KObjectRef::SharedMem(_) | KObjectRef::Process(_) | KObjectRef::Null(_) => None, + | KObjectRef::SharedMem(_) | KObjectRef::Process(_) | KObjectRef::Job(_) | KObjectRef::Null(_) => None, } } @@ -314,7 +315,7 @@ pub fn pipe_end_watch(object: &KObjectRef) -> Option { match object { KObjectRef::PipeRead(r) => pipe::left_watch(r.id(), End::Read).map(WatchRef::Shared), KObjectRef::PipeWrite(w) => pipe::left_watch(w.id(), End::Write).map(WatchRef::Shared), - KObjectRef::Connection(_) | KObjectRef::Acceptor(_) | KObjectRef::Process(_) + KObjectRef::Connection(_) | KObjectRef::Acceptor(_) | KObjectRef::Process(_) | KObjectRef::Job(_) | KObjectRef::Console(_) | KObjectRef::Device(_) | KObjectRef::SysCap(_) | KObjectRef::File(_) | KObjectRef::Inbox(_) | KObjectRef::Connector(_) | KObjectRef::Namespace(_) | KObjectRef::SharedMem(_) | KObjectRef::Null(_) => None, @@ -327,7 +328,7 @@ pub fn other_end_gone(object: &KObjectRef) -> bool { match object { KObjectRef::PipeRead(r) => pipe::other_end_gone(r.id(), End::Read), KObjectRef::PipeWrite(w) => pipe::other_end_gone(w.id(), End::Write), - KObjectRef::Connection(_) | KObjectRef::Acceptor(_) | KObjectRef::Process(_) + KObjectRef::Connection(_) | KObjectRef::Acceptor(_) | KObjectRef::Process(_) | KObjectRef::Job(_) | KObjectRef::Console(_) | KObjectRef::Device(_) | KObjectRef::SysCap(_) | KObjectRef::File(_) | KObjectRef::Inbox(_) | KObjectRef::Connector(_) | KObjectRef::Namespace(_) | KObjectRef::SharedMem(_) | KObjectRef::Null(_) => false, @@ -345,7 +346,7 @@ fn close_ends_polls(object: &KObjectRef) -> bool { match object { KObjectRef::SysCap(_) => false, KObjectRef::Console(_) => false, - KObjectRef::Process(_) => false, + KObjectRef::Process(_) | KObjectRef::Job(_) => false, KObjectRef::Device(d) => match d.class() { device_registry::DeviceType::PciFunction | device_registry::DeviceType::Isa @@ -505,7 +506,7 @@ pub fn try_read(object: &KObjectRef, buf: &mut UserBytesMut) -> Option { KObjectRef::PipeWrite(_) | KObjectRef::Acceptor(_) | KObjectRef::Inbox(_) | KObjectRef::SysCap(_) | KObjectRef::Connector(_) | KObjectRef::Namespace(_) - | KObjectRef::SharedMem(_) | KObjectRef::Process(_) => Some(SyscallError::PermissionDenied.to_u64()), + | KObjectRef::SharedMem(_) | KObjectRef::Process(_) | KObjectRef::Job(_) => Some(SyscallError::PermissionDenied.to_u64()), } } @@ -566,7 +567,7 @@ pub fn try_write(object: &KObjectRef, buf: &UserBytes) -> Option { KObjectRef::PipeRead(_) | KObjectRef::Acceptor(_) | KObjectRef::Inbox(_) | KObjectRef::SharedMem(_) | KObjectRef::SysCap(_) | KObjectRef::Connector(_) | KObjectRef::Namespace(_) - | KObjectRef::Process(_) => { + | KObjectRef::Process(_) | KObjectRef::Job(_) => { Some(SyscallError::PermissionDenied.to_u64()) } } @@ -635,7 +636,7 @@ pub fn fstat(object: &KObjectRef) -> Stat { }, KObjectRef::Inbox(_) | KObjectRef::SysCap(_) | KObjectRef::Connector(_) | KObjectRef::Namespace(_) - | KObjectRef::Process(_) => plain(FileType::Unknown), + | KObjectRef::Process(_) | KObjectRef::Job(_) => plain(FileType::Unknown), KObjectRef::Device(d) => plain(match d.class() { device_registry::DeviceType::Framebuffer => FileType::Framebuffer, device_registry::DeviceType::PciFunction @@ -677,6 +678,7 @@ pub fn has_data(object: &KObjectRef) -> bool { KObjectRef::Console(_) => serial::has_data(), KObjectRef::Acceptor(a) => a.has_pending(), KObjectRef::Process(p) => p.finished(), + KObjectRef::Job(_) => false, KObjectRef::File(_) | KObjectRef::Null(_) => true, KObjectRef::Device(d) => match d.class() { // A claim its last handle has let go has nothing to read: a poll @@ -704,7 +706,7 @@ pub fn read_posts_are_readiness(object: &KObjectRef) -> bool { KObjectRef::Console(_) | KObjectRef::PipeRead(_) | KObjectRef::PipeWrite(_) | KObjectRef::Connection(_) | KObjectRef::Acceptor(_) | KObjectRef::File(_) | KObjectRef::Device(_) | KObjectRef::Inbox(_) | KObjectRef::Connector(_) | KObjectRef::Namespace(_) - | KObjectRef::SharedMem(_) | KObjectRef::Process(_) | KObjectRef::Null(_) => false, + | KObjectRef::SharedMem(_) | KObjectRef::Process(_) | KObjectRef::Job(_) | KObjectRef::Null(_) => false, } } @@ -719,7 +721,7 @@ pub fn has_space(object: &KObjectRef) -> bool { KObjectRef::PipeRead(_) | KObjectRef::Device(_) | KObjectRef::Acceptor(_) | KObjectRef::Inbox(_) | KObjectRef::SysCap(_) | KObjectRef::Connector(_) | KObjectRef::Namespace(_) - | KObjectRef::SharedMem(_) | KObjectRef::Process(_) => false, + | KObjectRef::SharedMem(_) | KObjectRef::Process(_) | KObjectRef::Job(_) => false, } } @@ -738,7 +740,7 @@ pub fn mark_tty(object: &KObjectRef) -> u64 { | KObjectRef::Console(_) | KObjectRef::Acceptor(_) | KObjectRef::Inbox(_) | KObjectRef::SysCap(_) | KObjectRef::SharedMem(_) | KObjectRef::Connector(_) | KObjectRef::Namespace(_) - | KObjectRef::Process(_) | KObjectRef::Null(_) => SyscallError::InvalidArgument.to_u64(), + | KObjectRef::Process(_) | KObjectRef::Job(_) | KObjectRef::Null(_) => SyscallError::InvalidArgument.to_u64(), } } diff --git a/kernel/src/process.rs b/kernel/src/process.rs index 69fe718d49..0bb6fb12d0 100644 --- a/kernel/src/process.rs +++ b/kernel/src/process.rs @@ -445,13 +445,6 @@ impl Processes for ProcessTable { } } -impl Drop for ProcessEntry { - fn drop(&mut self) { - // vruntime must live as long as the table entry: a zombified process's threads still yield inside their own exit path and read it. - scheduler::remove_vruntime(self.pid); - } -} - /// Record of a single demand-paged fault, stored in a ring buffer for crash diagnostics. #[derive(Clone, Copy)] @@ -573,6 +566,9 @@ pub struct ProcessData { pub accounting: ProcessAccounting, /// What this process's parent called each handle it moved in at spawn. pub endowments: Endowments, + /// The job its threads, and its children unless a spawn names another, + /// run in. + pub job: Arc, } /// The labels a parent put on the handles it endowed — names, not authority: the handle is in the table whether or not the child ever asks. @@ -766,7 +762,7 @@ pub fn stats_of( } /// Take every entry whose process has published its exit. -/// Entries come back rather than being dropped here: the caller holds the table lock, and an entry's drop reaches `remove_vruntime`. +/// Entries come back rather than being dropped here: the caller holds the table lock, and an entry's drop releases what its process held. #[must_use = "the reaped entries must be dropped outside the table lock"] pub fn reap_finished(table: &mut ProcessTable, _proof: IdleProof) -> Vec { reap::finished_pids(table).into_iter().filter_map(|pid| table.remove(pid)).collect() @@ -921,9 +917,9 @@ pub fn spawn_thread( .expect("spawn_thread: the spawning thread runs in an address space"); (addr_space, Arc::clone(&proc.process_data)) }; - let (tls_modules, tls) = { + let (tls_modules, tls, job) = { let data = process_data_arc.lock(); - (data.elf.tls_modules.clone(), data.elf.tls) + (data.elf.tls_modules.clone(), data.elf.tls, data.job.clone()) }; // Phase 2: build the TLS block outside any lock, then publish it into the @@ -995,6 +991,7 @@ pub fn spawn_thread( parent_addr_space, thread_pointer, image, + job, ); proc.threads.get_mut(tid).unwrap().set_sched(sched); drop(guard); diff --git a/kernel/src/quiesce.rs b/kernel/src/quiesce.rs index 3098348352..a9bda5dd06 100644 --- a/kernel/src/quiesce.rs +++ b/kernel/src/quiesce.rs @@ -55,7 +55,7 @@ use crate::time::{Budget, Deadline, Duration}; /// again covers one inside a syscall that does not park. A thread can /// therefore outlast this, which is why its expiry is a clause in the record. pub(crate) const PARK: Budget = Budget::of( - Duration::from_nanos(2 * kernel::sched::fair::QUANTUM_NS), + Duration::from_nanos(2 * kernel::sched::task::QUANTUM_NS), "the reset lands wherever the threads that never reached a safe point are, and \ the record names how many", ); diff --git a/kernel/src/sched/driver.rs b/kernel/src/sched/driver.rs index 2da327c1f5..af97309df6 100644 --- a/kernel/src/sched/driver.rs +++ b/kernel/src/sched/driver.rs @@ -12,7 +12,6 @@ use core::ptr; use core::sync::atomic::{AtomicBool, AtomicPtr, AtomicU64, Ordering}; use kernel::sched::cpu::{Action, Balance, CpuHandle, CpuHandles, CpuSched, Env, SchedPass}; -use kernel::sched::fair::Frontier; use kernel::sched::hw::{CpuId, Hw, Kicker, Machine, Nanos}; use kernel::sched::mailbox::{mailbox, Kick, PreemptGuard, Urgency}; use kernel::sched::msg::Msg; @@ -28,7 +27,7 @@ use crate::process::{OwnedAlloc, PageTables, TaskId}; use crate::process::KERNEL_STACK_SIZE; use super::payload::{ - KMsg, KShare, KShared, KernelCtx, KernelPayload, RawTicket, Runs, TaskHandle, ThreadSched, + KMsg, KShared, KernelCtx, KernelPayload, RawTicket, Runs, TaskHandle, ThreadSched, }; /// Proof that preemption is disabled for as long as the borrow lasts. @@ -58,7 +57,6 @@ pub fn irq_off(f: impl FnOnce(&IrqOff) -> R) -> R { } static CPUS: AtomicPtr> = AtomicPtr::new(ptr::null_mut()); -static FRONTIER: Frontier = Frontier::new(); static NEXT_KEY: AtomicU64 = AtomicU64::new(1); @@ -151,10 +149,6 @@ pub fn cpus() -> &'static CpuHandles { unsafe { &*ptr } } -pub fn frontier() -> &'static Frontier { - &FRONTIER -} - /// Monotonic and never reused, so a message names a dead task unambiguously. Not `TaskId`: pids and tids are /// recycled. fn next_key() -> TaskKey { @@ -422,7 +416,8 @@ pub struct NewTask { pub entry_sp: u64, pub address_space: PageTables, pub thread_pointer: u64, - pub share: Arc, + /// The job the thread's time is charged to. + pub job: Arc, /// The process's image; `None` for a kernel thread. pub image: Option>, } @@ -447,7 +442,7 @@ pub fn spawn(new: NewTask) -> (ThreadSched, CpuId) { let handle = Arc::new(TaskHandle::new()); let task = TaskBuilder { key: next_key(), - share: new.share, + job: new.job, ctx, ext: KernelPayload { id: new.id, @@ -500,7 +495,6 @@ fn env(preempt: &PreemptOff) -> Env<'_, crate::hw::KernelHw, PreemptOff> { Env { hw: &HW, cpus: cpus(), - frontier: &FRONTIER, preempt, balance: Balance::PushOnSurplus { threshold: kernel::sched::cpu::PUSH_THRESHOLD, @@ -748,11 +742,11 @@ pub fn with_current_acct( } pub fn set_current_rt(permanent: bool) { - with_cpu(|cpu| cpu.set_current_rt(permanent)); + with_cpu(|cpu| cpu.set_current_rt(HW.now(), permanent)); } pub fn boost_current(until: Nanos) { - with_cpu(|cpu| cpu.boost_current(until)); + with_cpu(|cpu| cpu.boost_current(HW.now(), until)); } pub fn current_is_rt() -> bool { diff --git a/kernel/src/sched/kthread.rs b/kernel/src/sched/kthread.rs index 9758e78c1f..467249ec13 100644 --- a/kernel/src/sched/kthread.rs +++ b/kernel/src/sched/kthread.rs @@ -85,6 +85,8 @@ pub fn spawn(name: &str, body: extern "C" fn(u64) -> !, arg: u64) -> ThreadSched let mut short = [0u8; THREAD_NAME_LEN]; let len = name.len().min(THREAD_NAME_LEN - 1); short[..len].copy_from_slice(&name.as_bytes()[..len]); + // A job of its own: a kernel thread is nobody's helper. + let job = crate::object::job::new_job(); // One hold across insert and place: a visible pid already has its thread scheduled. let mut guard = PROCESS_TABLE.lock(); @@ -92,7 +94,7 @@ pub fn spawn(name: &str, body: extern "C" fn(u64) -> !, arg: u64) -> ThreadSched table.insert(ProcessEntry::new( crate::object::process::ProcessObject::new(pid), short, - Arc::new(Lock::new(kernel_process_data(name))), + Arc::new(Lock::new(kernel_process_data(name, job.clone()))), // No user image: its frames resolve through `symbols::resolve_kernel`. None, ThreadEntry::new(Arc::new(Lock::new(kernel_thread_data()))), @@ -108,6 +110,7 @@ pub fn spawn(name: &str, body: extern "C" fn(u64) -> !, arg: u64) -> ThreadSched crate::mm::paging::kernel().clone(), 0, None, + job, ); table .get_mut(pid) @@ -120,8 +123,8 @@ pub fn spawn(name: &str, body: extern "C" fn(u64) -> !, arg: u64) -> ThreadSched sched } -/// Every field is the empty value: a kernel thread has no user half. -fn kernel_process_data(name: &str) -> ProcessData { +/// Every field is the empty value but its job: a kernel thread has no user half. +fn kernel_process_data(name: &str, job: Arc) -> ProcessData { ProcessData { handles: crate::object::HandleTable::new(), cwd: String::from("/"), @@ -138,6 +141,7 @@ fn kernel_process_data(name: &str) -> ProcessData { spawn_ns: crate::clock::nanos_since_boot(), accounting: ProcessAccounting::default(), endowments: Endowments::empty(), + job, } } diff --git a/kernel/src/sched/payload.rs b/kernel/src/sched/payload.rs index 398f0e29cf..a6aecaf362 100644 --- a/kernel/src/sched/payload.rs +++ b/kernel/src/sched/payload.rs @@ -6,7 +6,6 @@ use alloc::sync::Arc; use alloc::vec::Vec; use core::sync::atomic::{AtomicU32, AtomicU64, Ordering}; -use kernel::sched::fair::{FairShare, ShareState}; use kernel::sched::hw::Nanos; use kernel::sched::msg::Msg; use kernel::sched::sync::CellLock; @@ -36,7 +35,6 @@ impl CellLock for KernelLock { pub type KMsg = Msg; pub type KShared = TaskShared; -pub type KShare = FairShare>; /// The core's wait ticket; blocking sites use `driver::Ticket`, which wraps it in the needed preempt guard. pub type RawTicket = WaitTicket; @@ -88,7 +86,6 @@ pub struct KernelPayload { impl SchedPayload for KernelPayload { type Ctx = KernelCtx; - type ShareLock = KernelLock; /// The thread, as `TaskId::pack` packs it: what a diary record names. fn name(&self) -> u64 { diff --git a/kernel/src/scheduler.rs b/kernel/src/scheduler.rs index d76e7a61cf..6af19fb38a 100644 --- a/kernel/src/scheduler.rs +++ b/kernel/src/scheduler.rs @@ -6,10 +6,8 @@ use alloc::sync::Arc; -use crate::hasher::HashMap; -use kernel::sched::fair::{ShareState, QUANTUM_NS}; use kernel::sched::hw::{CpuId, Machine, Nanos}; -use kernel::sched::task::{Refused, SafePoint, WaitClass}; +use kernel::sched::task::{Refused, SafePoint, WaitClass, QUANTUM_NS}; use crate::arch::percpu; use crate::watch::{self, Cancel}; @@ -17,10 +15,9 @@ use crate::hw::HW; use crate::pipe::PipeId; use crate::process::{self, Pid, Tid}; use crate::sched::driver::{self, cpus, preempt_off, Dispose, NewTask}; -use crate::sched::payload::{KShare, KernelLock, ThreadSched}; +use crate::sched::payload::ThreadSched; use crate::sched::reap_gate::ReapGate; use crate::sched::futex; -use crate::sync::Lock; use crate::time::Deadline; use crate::DirectMap; @@ -105,51 +102,10 @@ impl core::fmt::Display for TaskId { } } -/// Pid → share; the charge path reaches it through the task, never this lock. -static SHARES: Lock>>> = Lock::new(None); - pub fn init() { - *SHARES.lock() = Some(HashMap::default()); driver::init(); } -/// The share a new task of `pid` joins, as `NonRunnable { lag: 0 }` so the -/// adopting CPU's `enter_runnable` reproduces `new_runnable(frontier)`'s state. -fn share_for(pid: Pid) -> Arc { - let mut guard = SHARES.lock(); - let map = guard.as_mut().expect("scheduler not initialized"); - map.entry(pid) - .or_insert_with(|| { - Arc::new(KShare::new(KernelLock::new(ShareState::NonRunnable { - lag: 0, - }))) - }) - .clone() -} - -fn share_of(pid: Pid) -> Option> { - SHARES.lock().as_ref()?.get(&pid).cloned() -} - -/// The process is gone from the table; live tasks keep their `Arc` alive. -pub fn remove_vruntime(pid: Pid) { - if let Some(map) = SHARES.lock().as_mut() { - map.remove(&pid); - } -} - -pub fn process_vruntime(pid: Pid) -> u64 { - share_of(pid).map_or(0, |s| s.vruntime(driver::frontier())) -} - -pub fn process_lag(pid: Pid) -> i64 { - share_of(pid).map_or(0, |s| s.lag()) -} - -pub fn global_min_vruntime() -> u64 { - driver::frontier().get() -} - /// Build and place a new task; returns the CPU it was placed on. pub fn enqueue_new( id: TaskId, @@ -158,6 +114,7 @@ pub fn enqueue_new( address_space: crate::process::PageTables, thread_pointer: u64, image: Option>, + job: Arc, ) -> (ThreadSched, CpuId) { driver::spawn(NewTask { id, @@ -165,7 +122,7 @@ pub fn enqueue_new( entry_sp, address_space, thread_pointer, - share: share_for(id.0), + job, image, }) } @@ -417,7 +374,7 @@ pub(crate) fn reap_finished() { if !REAP_GATE.take() { return; } - // Dropped after the guard: an entry's drop reaches `remove_vruntime`. + // Dropped after the guard: an entry's drop releases what it held. let reaped = { let mut guard = process::PROCESS_TABLE.lock(); let table = guard.as_mut().unwrap(); diff --git a/kernel/src/syscall/dispatch.rs b/kernel/src/syscall/dispatch.rs index 3dbd04988b..5d8fdaa963 100644 --- a/kernel/src/syscall/dispatch.rs +++ b/kernel/src/syscall/dispatch.rs @@ -40,7 +40,7 @@ use super::ipc::{ sys_port_badge, sys_port_create, sys_port_mint, sys_shm_create, sys_shm_map, }; use super::machine::{ - sys_counters, sys_device_inventory, sys_log_read, sys_reboot, sys_sched_info, sys_shutdown, sys_sysinfo, + sys_counters, sys_device_inventory, sys_log_read, sys_reboot, sys_shutdown, sys_sysinfo, sys_trace_read, MAX_INVENTORY_RECORDS, }; @@ -48,7 +48,7 @@ use super::machine::{ use super::machine::lower_sysinfo_bound; use super::proc::{ spawn_place, sys_endowments, sys_exit, sys_nanosleep, sys_process_stats, - sys_process_wait, sys_rt_enter, sys_spawn, sys_thread_exit, sys_thread_join, sys_thread_spawn, + sys_job_create, sys_job_own, sys_process_wait, sys_rt_enter, sys_spawn, sys_thread_exit, sys_thread_join, sys_thread_spawn, }; use super::vm::{shared_image, sys_dlopen, sys_dlsym, sys_mmap, sys_munmap, sys_query_modules, sys_tls_alloc_block}; @@ -392,6 +392,8 @@ pub(crate) fn syscall_dispatch(num: u64, a1: u64, a2: u64, a3: u64, a4: u64) -> } SYS_DEVICE_CLAIM => sys_device_claim(RawHandle(a1 as u32), a2, [a3, a4]), SYS_RT_ENTER => sys_rt_enter(RawHandle(a1 as u32)), + SYS_JOB_CREATE => sys_job_create(RawHandle(a1 as u32)), + SYS_JOB_OWN => sys_job_own(), SYS_LOG_READ => { // checked_mul before mapping: a product that doesn't fit is a bad argument, // not kernel arithmetic wrapping into a window it then trusts. @@ -584,10 +586,6 @@ pub(crate) fn syscall_dispatch(num: u64, a1: u64, a2: u64, a3: u64, a4: u64) -> DA::GLOBAL_LEAF_REWRITE => crate::mm::paging::debug_global_leaf(a2), _ => SyscallError::InvalidArgument.to_u64(), }, - SYS_SCHED_INFO => match ctx.copy_out(UserAddr::new(a1), &sys_sched_info()) { - Ok(()) => 0, - Err(e) => e.to_u64(), - }, SYS_PROCESS_STATS => { let stats_size = core::mem::size_of::() as u64; if a3 < stats_size { return SyscallError::InvalidArgument.to_u64(); } diff --git a/kernel/src/syscall/machine.rs b/kernel/src/syscall/machine.rs index 679e4d8943..89d62cd3da 100644 --- a/kernel/src/syscall/machine.rs +++ b/kernel/src/syscall/machine.rs @@ -2,8 +2,7 @@ //! //! [`sys_log_read`], [`sys_trace_read`], the roster half of [`sys_sysinfo`], and both of //! [`sys_shutdown`] and [`sys_reboot`] each require a `SysCap` bit from -//! `/system/bin/supervisor`'s `system.toml`; `SYS_SYSINFO`'s header is ambient, and -//! [`sys_sched_info`] demands nothing. +//! `/system/bin/supervisor`'s `system.toml`; `SYS_SYSINFO`'s header is ambient. use alloc::vec::Vec; @@ -321,15 +320,6 @@ pub(super) fn sys_sysinfo(syscap: RawHandle, out: &mut UserBytesMut) -> u64 { pos as u64 } -pub(super) fn sys_sched_info() -> toyos_abi::syscall::SchedInfo { - let pid = process::current_process(); - toyos_abi::syscall::SchedInfo { - vruntime: crate::scheduler::process_vruntime(pid), - min_vruntime: crate::scheduler::global_min_vruntime(), - lag: crate::scheduler::process_lag(pid), - } -} - /// Every online CPU's counters into `out`, or the CPU count when `out` is /// empty: requires a `SysCap` carrying `Rights::COUNTERS`, and answers the /// counters that time programs and the power envelope only where it carries diff --git a/kernel/src/syscall/proc.rs b/kernel/src/syscall/proc.rs index 40207ba111..04050dd52c 100644 --- a/kernel/src/syscall/proc.rs +++ b/kernel/src/syscall/proc.rs @@ -108,6 +108,30 @@ pub(super) fn sys_rt_enter(syscap: RawHandle) -> u64 { 0 } +/// Make a new job, gated on a `SysCap` carrying [`Rights::JOBS`], and install a handle to it. +pub(super) fn sys_job_create(syscap: RawHandle) -> u64 { + if let Err(e) = demand_syscap(syscap, Rights::JOBS) { + return e.refuse(); + } + let object = crate::object::KObjectRef::Job(crate::object::job::JobObject::new(crate::object::job::new_job())); + match process::with_process_data(|data| crate::object::ops::install(&mut data.handles, object)) { + Ok(h) => u64::from(h.0), + Err(e) => e.to_u64(), + } +} + +/// Install a handle to the job the caller runs in. +pub(super) fn sys_job_own() -> u64 { + let installed = process::with_process_data(|data| { + let object = crate::object::KObjectRef::Job(crate::object::job::JobObject::new(data.job.clone())); + crate::object::ops::install(&mut data.handles, object) + }); + match installed { + Ok(h) => u64::from(h.0), + Err(e) => e.to_u64(), + } +} + /// Answer this process's endowment table; an empty buffer asks only for its size. pub(super) fn sys_endowments(out: &mut crate::user_ptr::UserBytesMut) -> u64 { let data_arc = process::process_data(); diff --git a/lib/manifest/src/lib.rs b/lib/manifest/src/lib.rs index a230b27c58..ffb9c97ab2 100644 --- a/lib/manifest/src/lib.rs +++ b/lib/manifest/src/lib.rs @@ -210,6 +210,10 @@ const SYSCAP_RIGHTS: &[(&str, Rights)] = &[ // Read what times every other program: each CPU's frequency, its busy // fraction and its wake-ups. An admin tool's, and no applet's. ("trace", Rights::TRACE), + // Start programs as independent jobs, each with its own share of the CPU: + // a launcher's. Every other child joins its parent's job, so this is what + // lets a program gain aggregate CPU by starting many. + ("jobs", Rights::JOBS), ]; /// The whole right set a program's `syscap` list asks for. diff --git a/sdk/abi/src/handle.rs b/sdk/abi/src/handle.rs index 3676402f99..083f2febf9 100644 --- a/sdk/abi/src/handle.rs +++ b/sdk/abi/src/handle.rs @@ -76,10 +76,11 @@ pub enum ObjectKind { SysCap = 11, Process = 12, Null = 13, + Job = 14, } impl ObjectKind { - pub const ALL: [ObjectKind; 14] = [ + pub const ALL: [ObjectKind; 15] = [ Self::PipeRead, Self::PipeWrite, Self::Connection, @@ -94,6 +95,7 @@ impl ObjectKind { Self::SysCap, Self::Process, Self::Null, + Self::Job, ]; pub const fn from_word(word: u32) -> Option { @@ -219,10 +221,18 @@ impl Rights { /// [`SYS_COUNTERS`]: crate::syscall::SYS_COUNTERS /// [`SYS_TRACE_READ`]: crate::syscall::SYS_TRACE_READ pub const TRACE: Rights = Rights(1 << 14); + /// On a `SysCap`: make a new job, with its own share of the CPU + /// ([`SYS_JOB_CREATE`]). Every other process's children join their + /// parent's job, so this is what an independently started program needs a + /// launcher to hold — and what lets the holder gain aggregate CPU by + /// starting many, which is why it rides a bit. + /// + /// [`SYS_JOB_CREATE`]: crate::syscall::SYS_JOB_CREATE + pub const JOBS: Rights = Rights(1 << 15); /// Every bit that has a caller. A wider set than this is a bug in whoever /// composed it, not a right nobody uses. - pub const ALL: Rights = Rights(0x7fff); + pub const ALL: Rights = Rights(0xffff); pub const fn from_bits(bits: u32) -> Option { if bits & !Self::ALL.0 == 0 { Some(Rights(bits)) } else { None } @@ -253,7 +263,7 @@ impl Rights { /// refusal saying which right was missing. impl core::fmt::Debug for Rights { fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { - const NAMES: [(Rights, &str); 15] = [ + const NAMES: [(Rights, &str); 16] = [ (Rights::DUP, "DUP"), (Rights::TRANSFER, "TRANSFER"), (Rights::READ, "READ"), @@ -269,6 +279,7 @@ impl core::fmt::Debug for Rights { (Rights::INVENTORY, "INVENTORY"), (Rights::COUNTERS, "COUNTERS"), (Rights::TRACE, "TRACE"), + (Rights::JOBS, "JOBS"), ]; if self.0 == 0 { return f.write_str("NONE"); diff --git a/sdk/abi/src/syscall.rs b/sdk/abi/src/syscall.rs index fff9a5c515..db5e0e32c4 100644 --- a/sdk/abi/src/syscall.rs +++ b/sdk/abi/src/syscall.rs @@ -122,7 +122,11 @@ pub const SYS_QUERY_MODULES: u64 = 91; /// 0 = kernel panic (triggers panic!() in syscall context) /// 1 = kernel fault (null pointer deref in kernel context) pub const SYS_DEBUG: u64 = 92; -pub const SYS_SCHED_INFO: u64 = 93; +/// Make a new job — the owner its threads share CPU time under — gated by +/// [`Rights::JOBS`] on a `SysCap`. See [`job_create`]. +/// +/// [`Rights::JOBS`]: crate::handle::Rights::JOBS +pub const SYS_JOB_CREATE: u64 = 93; pub const SYS_PROCESS_STATS: u64 = 94; pub const SYS_SET_THREAD_NAME: u64 = 95; /// Read one register of a claimed PCI function's config space. See @@ -282,6 +286,8 @@ pub const SYS_DEVICE_DMA_START: u64 = 132; /// Write a message's bytes to a connection and move the handles it carries, /// together or not at all. See [`message_send`]. pub const SYS_MESSAGE_SEND: u64 = 133; +/// A handle to the job the caller runs in. See [`job_own`]. +pub const SYS_JOB_OWN: u64 = 134; /// Bins in the per-process syscall profile — one for every number this ABI /// issues, and one at the end for every number it does not. @@ -289,13 +295,13 @@ pub const SYS_MESSAGE_SEND: u64 = 133; /// **The profile's parts sum to its total, and that is the whole requirement.** /// A bin array narrower than the ABI reaches drops calls out of the line while /// the total goes on counting them. -pub const SYSCALL_PROFILE_BINS: usize = 135; +pub const SYSCALL_PROFILE_BINS: usize = 136; /// Where a number this ABI does not issue is counted. Merging is a degradation /// a reader can see in the line; dropping is one nobody can. pub const SYSCALL_PROFILE_OTHER: usize = SYSCALL_PROFILE_BINS - 1; -const _: () = assert!(SYS_MESSAGE_SEND < SYSCALL_PROFILE_OTHER as u64); +const _: () = assert!(SYS_JOB_OWN < SYSCALL_PROFILE_OTHER as u64); pub const WNOHANG: u64 = 1; @@ -391,6 +397,14 @@ pub const SVC_LABEL: &str = "svc"; /// `WRITE`, `DUP` and `TRANSFER`: what it hands on for a child to be placed /// under it ([`SpawnArgs::place`]). The kernel puts it in every table. pub const SELF_LABEL: &str = "self"; +/// The label a spawn endows a job under to put its child in it: the owner its +/// threads, and its children's unless a spawn names another, share CPU time +/// under. It takes a handle carrying [`Rights::WRITE`] to a job — the caller's +/// own is [`job_own`]'s, and a new one [`job_create`]'s. A spawn naming none +/// puts the child in its spawner's job. +/// +/// [`Rights::WRITE`]: crate::handle::Rights::WRITE +pub const JOB_LABEL: &str = "job"; /// `serve:`: the acceptor of a machine-wide port this program serves. pub const SERVE_PREFIX: &str = "serve:"; /// `dev:`: the claim for a device class this program was given. @@ -1576,6 +1590,23 @@ pub fn device_claim(syscap: RawHandle, request: DeviceRequest) -> Result Result { + check(syscall(SYS_JOB_CREATE, syscap.0 as u64, 0, 0, 0)).map(|v| RawHandle(v as u32)) +} + +/// A handle to the job the caller runs in, carrying `WRITE`, `DUP` and +/// `TRANSFER`: what a launch carries for its program to join the caller's job. +/// Asks no right: it names only what the caller is already in. +pub fn job_own() -> Result { + check(syscall(SYS_JOB_OWN, 0, 0, 0, 0)).map(|v| RawHandle(v as u32)) +} + /// Enter the real-time scheduling band, presenting a `SysCap` handle that /// carries [`Rights::RT`]. The privilege a device claim was never enough to /// confer. @@ -2022,7 +2053,7 @@ pub fn nanosleep(nanos: u64) { /// source's own set. /// /// A wire encoding of `Option`, decoded at the syscall boundary and -/// never carried inward: `Rights::ALL` is twelve bits, so this value is not a +/// never carried inward: `Rights::ALL` fits in sixteen bits, so this value is not a /// rights set and never becomes one. The two wrappers below are the only /// writers, so no caller ever spells it. pub const RIGHTS_UNCHANGED: u64 = u64::MAX; @@ -2455,32 +2486,6 @@ pub fn modules(answer: &[u8]) -> impl Iterator { }) } -crate::user_safe! { - /// Scheduler info for the calling process. - #[derive(Clone, Copy, Debug)] - pub struct SchedInfo { - /// Current vruntime of this process (nanoseconds of virtual CPU time). - pub vruntime: u64, - /// Global min_vruntime frontier (monotonic non-decreasing). - pub min_vruntime: u64, - /// Signed contract lag, frozen at the most recent runnable-state - /// transition. Positive = process was behind the frontier when it last - /// woke / blocked (entitled to catch up); negative = process ran ahead - /// and will be throttled. Bounded to [-50ms, +50ms] (MAX_VRUNTIME_LAG_NS) - /// by construction. This is the scheduler's contract, not the live - /// `min_vruntime - vruntime` drift that accumulates while running on - /// multi-CPU systems — compute that at the call site if you need it. - pub lag: i64, - } -} - -/// Get scheduler info for the calling process. -pub fn sched_info() -> SchedInfo { - let mut info = SchedInfo { vruntime: 0, min_vruntime: 0, lag: 0 }; - syscall(SYS_SCHED_INFO, &mut info as *mut SchedInfo as u64, 0, 0, 0); - info -} - crate::user_safe! { /// Per-process accounting statistics, as [`SYS_PROCESS_STATS`] answers them. #[derive(Clone, Copy, Default)] diff --git a/sdk/toyos/src/endow.rs b/sdk/toyos/src/endow.rs index 494ce7cabd..f0af251441 100644 --- a/sdk/toyos/src/endow.rs +++ b/sdk/toyos/src/endow.rs @@ -20,6 +20,7 @@ use toyos_abi::syscall::{ }; use crate::ipc::Connection; +use crate::job::Job; use crate::namespace::Namespace; use crate::port::{Acceptor, Connector}; use crate::process::Process; @@ -69,6 +70,7 @@ from_handle! { Connector => Connector, Namespace => Namespace, SysCap => SysCap, + Job => Job, Device => Device, crate::Console => crate::Console, crate::Pipe => crate::Pipe, diff --git a/sdk/toyos/src/job.rs b/sdk/toyos/src/job.rs new file mode 100644 index 0000000000..e40ec70188 --- /dev/null +++ b/sdk/toyos/src/job.rs @@ -0,0 +1,38 @@ +//! A job: the owner a process's threads, and its children's unless a spawn +//! names another, share CPU time under. A spawn endows one under +//! [`JOB_LABEL`] to put its child in it; [`own`] is this process's own, and +//! [`SysCap::new_job`](crate::syscap::SysCap::new_job) makes an independent +//! one. + +use toyos_abi::syscall::{self, SyscallError}; + +pub use toyos_abi::syscall::JOB_LABEL; + +use crate::{AsHandle, OwnedHandle, RawHandle}; + +pub struct Job(pub(crate) OwnedHandle); + +/// A handle to the job this process runs in. +pub fn own() -> Result { + // SAFETY: the kernel installed this handle for this call and no other. + syscall::job_own().map(|raw| Job(unsafe { OwnedHandle::from_raw(raw) })) +} + +impl Job { + /// A second handle to the same job, for a spawn to move. + pub fn duplicate(&self) -> Result { + // SAFETY: the kernel installed this handle for this call and no other. + syscall::dup(self.0.raw()).map(|raw| Job(unsafe { OwnedHandle::from_raw(raw) })) + } + + /// Give up ownership, for a spawn that moves it. + pub fn into_raw(self) -> RawHandle { + self.0.into_raw() + } +} + +impl AsHandle for Job { + fn as_handle(&self) -> RawHandle { + self.0.raw() + } +} diff --git a/sdk/toyos/src/launch.rs b/sdk/toyos/src/launch.rs index f11e3790f0..d110cbfab2 100644 --- a/sdk/toyos/src/launch.rs +++ b/sdk/toyos/src/launch.rs @@ -13,6 +13,11 @@ //! same bound `SYS_SPAWN` has. What it *adds* is that the child's namespace is //! its own manifest row rather than a narrowing of its parent's. //! +//! **A launch names the job its program joins**: the caller's own, or one the +//! caller made with the authority to start an independent job. The supervisor +//! spawns the program in it, so a launched program is its caller's helper unless +//! its caller holds that authority and says otherwise. +//! //! **A launch names its parent**, by one of two words: a place, which is a //! handle to a process carrying `WRITE` — the caller's own `self`, duplicated — //! that the supervisor spawns the program under, so the caller's end takes it down; or @@ -52,7 +57,7 @@ pub const MSG_REFUSED: u32 = 4; pub const MSG_GONE: u32 = 5; /// Handles beyond stdio one launch may carry from its caller: the connectors -/// it transfers, and the place it names. +/// it transfers, the place it names, and its job. /// /// Policy on the primitive, refused by name. Five and not more because the /// batch also carries the three stdio handles and @@ -87,8 +92,8 @@ pub enum Parent { /// `argv` and `env` are exactly the blobs `SYS_SPAWN` takes, so the launcher /// path and the direct path build one thing rather than two. /// -/// **Every handle in `fds`, `extras` and `parent` is the caller's to give, -/// and is named once.** [`launch`] consumes each one that it sends: it is moved +/// **Every handle in `fds`, `extras`, `parent` and `job` is the caller's to +/// give, and is named once.** [`launch`] consumes each one that it sends: it is moved /// to the launcher, or closed when the kernel refuses the move. A launch that /// names one twice, or the connection it is sent on, is a move the kernel /// refuses, and is consumed as one: each number it names is closed once, and @@ -120,6 +125,9 @@ pub struct Launch<'a> { /// Whom the program is placed under. A place is **moved** with the batch, /// and counts against [`MAX_LAUNCH_EXTRAS`]. pub parent: Parent, + /// The job the program joins, a handle carrying `WRITE`. **Moved** with the + /// batch, its last handle, and counts against [`MAX_LAUNCH_EXTRAS`]. + pub job: RawHandle, } /// Why a request would not go on the wire. @@ -136,7 +144,7 @@ type Batch = ([RawHandle; MAX_LAUNCH_EXTRAS + MAX_LAUNCH_FDS], usize); impl Launch<'_> { /// The numbers this launch names, each once and never `conn`: descriptors, then - /// extras, then the place. [`launch`] takes each as its own, and a number + /// extras, then the place, then the job. [`launch`] takes each as its own, and a number /// with two owners is closed twice. /// /// `Err` where one was named twice or is `conn` itself, which is a batch @@ -152,7 +160,7 @@ impl Launch<'_> { }; let fds = self.fds.iter().map(|(_, h)| h); let extras = self.extras.iter().map(|(_, h)| h); - for handle in fds.chain(extras).chain(&place) { + for handle in fds.chain(extras).chain(&place).chain([&self.job]) { if *handle == conn || out[..n].contains(handle) { sendable = false; continue; @@ -169,7 +177,7 @@ impl Launch<'_> { Parent::Place(_) => (1, PARENT_PLACE), Parent::Supervisor => (0, PARENT_SUPERVISOR), }; - if self.extras.len() + place > MAX_LAUNCH_EXTRAS || self.fds.len() > MAX_LAUNCH_FDS { + if self.extras.len() + place + 1 > MAX_LAUNCH_EXTRAS || self.fds.len() > MAX_LAUNCH_FDS { return Err(EncodeError::TooMany); } let names_len: usize = self.extras.iter().map(|(n, _)| n.len() + 1).sum(); @@ -247,7 +255,7 @@ impl<'a> Request<'a> { let (fds_len, program_len, argv_len, env_len, cwd_len, names_len) = (field(2), field(3), field(4), field(5), field(6), field(7)); let parent = field(8) as u32; - if extra_count + usize::from(parent == PARENT_PLACE) > MAX_LAUNCH_EXTRAS + if extra_count + usize::from(parent == PARENT_PLACE) + 1 > MAX_LAUNCH_EXTRAS || fd_count > MAX_LAUNCH_FDS || fds_len != fd_count * 4 { @@ -291,9 +299,10 @@ impl<'a> Request<'a> { } } - /// The handles the batch carries: the descriptors, the extras, then the place. + /// The handles the batch carries: the descriptors, the extras, the place, + /// then the job. pub fn handle_count(&self) -> usize { - self.fd_count() + self.extra_count + usize::from(self.parent == PARENT_PLACE) + self.fd_count() + self.extra_count + usize::from(self.parent == PARENT_PLACE) + 1 } /// The child descriptors the first handles of the batch are for, in order. @@ -406,6 +415,7 @@ mod tests { extras: &[], fds: &[], parent, + job: RawHandle(10), }; for (parent, named) in [(Parent::Place(RawHandle(9)), Parent::Place(())), (Parent::Supervisor, Parent::Supervisor)] { let len = request(parent).encode(&mut buf).expect("encode a launch"); @@ -426,32 +436,45 @@ mod tests { const CONN: RawHandle = RawHandle(3); let h = RawHandle; let batch = |fds: &[(u32, RawHandle)], extras: &[(&str, RawHandle)], parent| { - let request = - Launch { program: "/system/bin/cat", argv: b"", env: b"", cwd: "/", extras, fds, parent }; + let request = Launch { + program: "/system/bin/cat", + argv: b"", + env: b"", + cwd: "/", + extras, + fds, + parent, + job: RawHandle(10), + }; let named = |(handles, count): Batch| handles[..count].to_vec(); request.batch(CONN).map(named).map_err(named) }; assert_eq!( batch(&[(0, h(5)), (1, h(6))], &[("a", h(7)), ("b", h(8))], Parent::Place(h(9))), - Ok(vec![h(5), h(6), h(7), h(8), h(9)]), + Ok(vec![h(5), h(6), h(7), h(8), h(9), h(10)]), ); - assert_eq!(batch(&[], &[("a", h(7)), ("b", h(7))], Parent::Supervisor), Err(vec![h(7)]), "two extras"); - assert_eq!(batch(&[(0, h(5)), (1, h(5))], &[], Parent::Supervisor), Err(vec![h(5)]), "two descriptors"); + assert_eq!(batch(&[], &[("a", h(7)), ("b", h(7))], Parent::Supervisor), Err(vec![h(7), h(10)]), "two extras"); + assert_eq!(batch(&[(0, h(5)), (1, h(5))], &[], Parent::Supervisor), Err(vec![h(5), h(10)]), "two descriptors"); assert_eq!( batch(&[(0, h(7)), (1, h(6))], &[("a", h(7))], Parent::Supervisor), - Err(vec![h(7), h(6)]), + Err(vec![h(7), h(6), h(10)]), "a descriptor and an extra", ); assert_eq!( batch(&[(0, h(5))], &[("a", h(9))], Parent::Place(h(9))), - Err(vec![h(5), h(9)]), + Err(vec![h(5), h(9), h(10)]), "an extra and the place", ); assert_eq!( batch(&[(0, h(5))], &[("a", CONN), ("b", h(7))], Parent::Supervisor), - Err(vec![h(5), h(7)]), + Err(vec![h(5), h(7), h(10)]), "the connection as an extra", ); - assert_eq!(batch(&[], &[("a", h(7))], Parent::Place(CONN)), Err(vec![h(7)]), "the connection as the place"); + assert_eq!( + batch(&[], &[("a", h(7))], Parent::Place(CONN)), + Err(vec![h(7), h(10)]), + "the connection as the place", + ); + assert_eq!(batch(&[(0, h(10))], &[], Parent::Supervisor), Err(vec![h(10)]), "a descriptor and the job"); } } diff --git a/sdk/toyos/src/lib.rs b/sdk/toyos/src/lib.rs index da8a6f3a11..d89a8795a8 100644 --- a/sdk/toyos/src/lib.rs +++ b/sdk/toyos/src/lib.rs @@ -22,6 +22,7 @@ pub mod poller; pub mod power; pub mod wake; pub mod ipc; +pub mod job; pub mod namespace; pub mod launch; pub mod log; diff --git a/sdk/toyos/src/spawn.rs b/sdk/toyos/src/spawn.rs index 6e76c45db1..c09abd0b0e 100644 --- a/sdk/toyos/src/spawn.rs +++ b/sdk/toyos/src/spawn.rs @@ -12,9 +12,13 @@ //! names its image and provides a connector; one that decided what its child //! holds or runs is spawned directly; and one whose launch was not made is //! spawned directly only where it asked nothing a launch alone gives. +//! +//! **The job the child joins is no endowment of that kind**: a launch carries +//! it too. The one endowed under [`JOB_LABEL`], or else this process's own. use toyos_abi::syscall::{ - self, EndowEntry, SpawnArgs, SyscallError, FD_PREFIX, MAX_SPAWN_ENDOWMENTS, MAX_SPAWN_LABELS_LEN, SVC_LABEL, + self, EndowEntry, SpawnArgs, SyscallError, FD_PREFIX, JOB_LABEL, MAX_SPAWN_ENDOWMENTS, MAX_SPAWN_LABELS_LEN, + SVC_LABEL, }; use crate::launch::{self, Launch, LaunchError, Outcome, MAX_LAUNCH_EXTRAS, MAX_LAUNCH_FDS}; @@ -53,7 +57,8 @@ pub struct Spawn<'a> { /// handles, which a spawn duplicates. A launch carries 0 to 2; a /// descriptor past them only a direct spawn gives. pub fds: &'a [(u32, RawHandle)], - /// `(label, handle)` the child is endowed, each moved. + /// `(label, handle)` the child is endowed, each moved. One under + /// [`JOB_LABEL`] is the job the child joins, which a launch carries too. pub endow: &'a [(&'a str, RawHandle)], /// `(name, connector)` put in a launched child's namespace beside its /// row's, each moved. @@ -94,7 +99,7 @@ impl Spawn<'_> { self.refuse_streams()?; let asked = Asked { supervisor: self.parent == Parent::Supervisor, - endowed: !self.endow.is_empty(), + endowed: self.endow.iter().any(|(label, _)| *label != JOB_LABEL), extra_fds: self.fds.iter().any(|&(fd, _)| fd > 2), image_named: self.image.is_some(), provided: !self.provided.is_empty(), @@ -213,8 +218,9 @@ impl Spawn<'_> { /// does not declare — but a child asked of the supervisor is launched or /// refused, never spawned here. /// - /// The stdio handles and the place are **duplicated** before they go: a - /// launch moves what it carries. + /// The stdio handles and the place are **duplicated** before they go, and + /// a handle to this process's own job made: a launch moves what it + /// carries. A job the spawn endowed goes itself, moved as an endowment is. fn ask_launcher<'h>(&self, answer: &'h mut [u8]) -> Result, Refused> { let for_supervisor = self.parent == Parent::Supervisor; let unmade = |why, home| match unmade(why, for_supervisor) { @@ -225,10 +231,11 @@ impl Spawn<'_> { let Some(conn) = crate::endow::launcher().and_then(|held| held.open(launch::LAUNCHER).ok()) else { return unmade(Unmade::NoLauncher { provided: !self.provided.is_empty() }, None); }; - // A place is one of the batch's extras. Refused rather than spawned - // directly: that child would hold this process's namespace, not its row. - if self.provided.len() + usize::from(!for_supervisor) > MAX_LAUNCH_EXTRAS { - return Err(Refused::Invalid("more connectors provided than a launch carries beside its place")); + // The place and the job are two of the batch's extras. Refused rather + // than spawned directly: that child would hold this process's + // namespace, not its row. + if self.provided.len() + usize::from(!for_supervisor) + 1 > MAX_LAUNCH_EXTRAS { + return Err(Refused::Invalid("more connectors provided than a launch carries beside its place and job")); } let stdio = self.fds; if stdio.len() > MAX_LAUNCH_FDS || stdio.iter().any(|&(fd, _)| fd > 2) { @@ -239,11 +246,22 @@ impl Spawn<'_> { Parent::Place(place) => Some(place), Parent::Caller => Some(crate::endow::this_process().as_handle()), }; + let endowed_job = self.endow.iter().find(|(label, _)| *label == JOB_LABEL).map(|&(_, job)| job); + let job_copy = match endowed_job { + Some(_) => None, + None => match crate::job::own() { + Ok(own) => Some(own.into_raw()), + Err(e) => return unmade(Unmade::NotCopied(e), None), + }, + }; let parent = match place { None => launch::Parent::Supervisor, Some(place) => match syscall::dup(place) { Ok(copy) => launch::Parent::Place(copy), - Err(e) => return unmade(Unmade::PlaceNotCopied(e), None), + Err(e) => { + job_copy.into_iter().for_each(syscall::close); + return unmade(Unmade::NotCopied(e), None); + } }, }; let mut fds = [(0u32, RawHandle(0)); MAX_LAUNCH_FDS]; @@ -252,6 +270,7 @@ impl Spawn<'_> { if let launch::Parent::Place(copy) = parent { syscall::close(copy); } + job_copy.into_iter().for_each(syscall::close); }; for (i, &(child, handle)) in stdio.iter().enumerate() { match syscall::dup(handle) { @@ -273,6 +292,7 @@ impl Spawn<'_> { extras: &extras[..self.provided.len()], fds, parent, + job: job_copy.or(endowed_job).expect("a job endowed or copied"), }; // **The send moved them.** Every arm but `NotSent` is past the point // where the duplicates left this table, so closing them would be this @@ -364,8 +384,9 @@ enum Unmade { NotDeclared, /// The launch never left this process. NotSent, - /// The copy of its place a launch carries could not be made. - PlaceNotCopied(SyscallError), + /// The copy of its place or of this process's job a launch carries could + /// not be made. + NotCopied(SyscallError), } /// What a spawn whose launch was not made gets. @@ -383,7 +404,7 @@ enum Unlaunched { /// this process's namespace and not its row's. fn unmade(why: Unmade, supervisor: bool) -> Unlaunched { match why { - Unmade::PlaceNotCopied(e) => Unlaunched::Failed(e), + Unmade::NotCopied(e) => Unlaunched::Failed(e), Unmade::NoLauncher { provided: true } => { Unlaunched::Refused("only a launch carries a provided connector, and this process holds no launcher") } @@ -442,14 +463,15 @@ mod tests { } } - /// A launch carries a copy of its place: one this process cannot copy - /// refuses the launch as the copy was refused, and nothing is spawned - /// directly in its place, whoever the spawn asked for. + /// A launch carries a copy of its place and of this process's job: one + /// this process cannot copy refuses the launch as the copy was refused, + /// and nothing is spawned directly in its place, whoever the spawn asked + /// for. #[test] - fn a_launch_whose_place_cannot_be_copied_is_refused_and_not_spawned_directly() { + fn a_launch_whose_place_or_job_cannot_be_copied_is_refused_and_not_spawned_directly() { for supervisor in [false, true] { let e = SyscallError::PermissionDenied; - assert_eq!(unmade(Unmade::PlaceNotCopied(e), supervisor), Unlaunched::Failed(e)); + assert_eq!(unmade(Unmade::NotCopied(e), supervisor), Unlaunched::Failed(e)); } } diff --git a/sdk/toyos/src/syscap.rs b/sdk/toyos/src/syscap.rs index 0118885a12..8f115e5a63 100644 --- a/sdk/toyos/src/syscap.rs +++ b/sdk/toyos/src/syscap.rs @@ -53,6 +53,14 @@ impl SysCap { self.mint(DeviceRequest::Isa(set)) } + /// Make a new job, with its own share of the CPU: what a launcher endows + /// a program it starts independently. Needs [`Rights::JOBS`]. + pub fn new_job(&self) -> Result { + let raw = syscall::job_create(self.0.raw())?; + // SAFETY: the kernel installed this handle for this call and no other. + Ok(crate::job::Job(unsafe { OwnedHandle::from_raw(raw) })) + } + fn mint(&self, request: DeviceRequest) -> Result { let raw = syscall::device_claim(self.0.raw(), request)?; // SAFETY: the kernel installed this handle in this process's table for diff --git a/system/compositor/src/session.rs b/system/compositor/src/session.rs index aa45dd0cbe..c55b97a839 100644 --- a/system/compositor/src/session.rs +++ b/system/compositor/src/session.rs @@ -9,17 +9,21 @@ //! //! **No clipboard byte is ever logged**: a refusal names a length and a client. +use std::os::toyos::process::CommandExt; use std::process::Command; use std::sync::mpsc; use std::time::{Duration, Instant}; use toyos::endow; +use toyos::job::JOB_LABEL; +use toyos::syscap::SysCap; use toyos::ipc::{Connection, FrameRx, RxStep}; use toyos::poller::{Poller, READABLE}; use toyos::port::Acceptor; use toyos::shm::SharedMemory; use toyos::{ipc, system, AsHandle}; use toyos_abi::input::RawKeyEvent; +use toyos_abi::syscall::SYSCAP_LABEL; use toyos_abi::RawHandle; use toyos_input::wire::{self, Record, CONNECTIONS, MAX_PAYLOAD}; use toyos_input::wire::DEVICES; @@ -2137,13 +2141,29 @@ fn text(from: RawHandle, bytes: Vec) -> Option { /// The thread every launch is made on, and the channel to it: one at a time, /// in order, since a launch waits as long as its question does. +/// +/// **Everything the desktop starts is a job of its own**, made with the +/// `jobs` right this row's system capability carries: a program started from +/// the taskbar is no helper of the compositor's, and must not share its CPU. fn launcher() -> mpsc::Sender { let (launches, queue) = mpsc::channel::(); std::thread::Builder::new() .name("launcher".into()) .spawn(move || { + let jobs = endow::Endowments::get().take::(SYSCAP_LABEL); for program in queue { - if let Err(e) = Command::new(&program).spawn() { + let Some(jobs) = &jobs else { + eprintln!("compositor: {program} did not start: this row holds no `jobs` right to start it as its own job"); + continue; + }; + let job = match jobs.new_job() { + Ok(job) => job, + Err(e) => { + eprintln!("compositor: {program} did not start: no job for it: {e:?}"); + continue; + } + }; + if let Err(e) = Command::new(&program).endow(JOB_LABEL, job.into_raw().0).spawn() { eprintln!("compositor: {program} did not start: {e}"); } } diff --git a/system/supervisor/src/main.rs b/system/supervisor/src/main.rs index f48ecb60ad..3693fe1d3e 100644 --- a/system/supervisor/src/main.rs +++ b/system/supervisor/src/main.rs @@ -125,7 +125,7 @@ use toyos_logstream::{ }; use toyos_abi::inventory::{Arrival, Pci, PciAddr, Record}; use toyos_abi::syscall::{ - DeviceRequest, FileType, SyscallError, DEV_PREFIX, MAX_BADGE, PROVIDE_PREFIX, SERVE_PREFIX, + DeviceRequest, FileType, SyscallError, DEV_PREFIX, JOB_LABEL, MAX_BADGE, PROVIDE_PREFIX, SERVE_PREFIX, SVC_LABEL, SYSCAP_LABEL, }; @@ -1961,9 +1961,11 @@ impl Supervisor<'_> { // can be split. Every part still releases on every path below. let all = held.take(); let (fd_handles, rest) = all.split_at(request.fd_count()); - let (extra_handles, place_handle) = rest.split_at(request.extra_count); + let (extra_handles, rest) = rest.split_at(request.extra_count); + let (place_handle, job_handle) = rest.split_at(rest.len() - 1); let fds = Moved(fd_handles.to_vec()); let place = Moved(place_handle.to_vec()); + let job = Moved(job_handle.to_vec()); // Owned, so they close when the launch is done with: `SYS_NAMESPACE_BUILD` // copies a connector into the namespace and leaves the caller's handle, // and the supervisor's copy of a client's connector has no life beyond @@ -2115,6 +2117,7 @@ impl Supervisor<'_> { caller_fds, _place: place, placed: under.is_some(), + job, extras, cwd: request.cwd.to_string(), }; @@ -2217,7 +2220,8 @@ impl Supervisor<'_> { /// Start `launch`, holding `folder` and in it ([`grants::cwd`]) where it /// holds one, and answer its caller. fn start_launch(&mut self, launch: Launch, folder: Option) { - let Launch { conn, mut program, mut command, session, fds, caller_fds, _place, placed, extras, cwd } = launch; + let Launch { conn, mut program, mut command, session, fds, caller_fds, _place, placed, mut job, extras, cwd } = + launch; if let Some(folder) = folder { // The image `prepare` read is kept: only the working directory is // judged again. @@ -2261,7 +2265,7 @@ impl Supervisor<'_> { &extras, storage, (&self.minted, session), - Output::Launch { log: &mut self.log, fds: &caller_fds }, + Output::Launch { log: &mut self.log, fds: &caller_fds, job: job.take()[0] }, ); drop(fds); match started { @@ -2305,6 +2309,8 @@ struct Launch { _place: Moved, /// Whether it names a place, or the supervisor. placed: bool, + /// The job its caller named, owned until the spawn moves it. + job: Moved, extras: Vec<(String, Connector)>, /// The caller's working directory, which a granted folder's launch keeps /// where it lies inside the folder. @@ -2617,6 +2623,17 @@ fn start<'a>( // class nothing can mint again. A client picks whether a spawn fails — an // unreachable `cwd` is enough — so this is its path as much as a bug's. let mut held = Moved(Vec::new()); + // **Every service the supervisor starts is a job of its own**, made with the + // system capability's `JOBS`; a launch's program joins the job its caller + // named, which is the caller's own unless it holds that right itself. + let job = match &output { + Output::Boot(_) => { + syscap.new_job().map_err(|e| std::io::Error::other(format!("no job for it: {e:?}")))?.into_raw() + } + Output::Launch { job, .. } => *job, + }; + command.endow(JOB_LABEL, job.0); + held.0.push(job); // Acceptors are given *back* rather than closed: a `serves` port whose // acceptor is gone can never be served again. let mut taken: Vec<(&'a str, Acceptor)> = Vec::new(); @@ -2806,7 +2823,7 @@ fn start<'a>( } log } - Output::Launch { log, fds } => { + Output::Launch { log, fds, .. } => { command.stdin(Stdio::null()).stdout(Stdio::null()).stderr(Stdio::null()); for &(fd, handle) in fds { let caller_ring = matches!(fd, 1 | 2) @@ -2871,10 +2888,11 @@ fn start<'a>( } /// Where a started program's output goes: a service's own ring, or a -/// launch's caller's descriptors with any ring among them replaced. +/// launch's caller's descriptors with any ring among them replaced — and, for +/// a launch, the job its caller named, which `start` owns from its first line. enum Output<'l> { Boot(&'l mut Log), - Launch { log: &'l mut Log, fds: &'l [(u32, toyos::RawHandle)] }, + Launch { log: &'l mut Log, fds: &'l [(u32, toyos::RawHandle)], job: toyos::RawHandle }, } /// The namespace this program's `receives` names, [`toyos_swap::PORT`] and diff --git a/tests/checks/metal.rs b/tests/checks/metal.rs index 5cc10ce254..abd49f564c 100644 --- a/tests/checks/metal.rs +++ b/tests/checks/metal.rs @@ -169,7 +169,7 @@ pub fn the_stop_owes_its_record() { /// Each bound's lateness against the period of what polls it, a millisecond /// either way and no further. pub fn a_bound_fires_within_one_period_of_itself() { - let quantum = i64::try_from(kernel::sched::fair::QUANTUM_NS / 1_000_000).expect("ms"); + let quantum = i64::try_from(kernel::sched::task::QUANTUM_NS / 1_000_000).expect("ms"); let deadline = |late: i64| planted(&expired_at(120_060 + late), &armed()).deadline_on_time(); assert_eq!(deadline(4), Ok(())); assert_eq!(deadline(quantum + 1), Ok(())); diff --git a/tests/common/metal.rs b/tests/common/metal.rs index 3e98d9f547..3be402e94e 100644 --- a/tests/common/metal.rs +++ b/tests/common/metal.rs @@ -326,7 +326,7 @@ impl Readback { else { return Ok(()); }; - within_one_period(late?, kernel::sched::fair::QUANTUM_NS) + within_one_period(late?, kernel::sched::task::QUANTUM_NS) .map_err(|why| format!("{}'s boot deadline {why}", self.label)) } diff --git a/tests/guest/rust-tests/src/bin/abuse_page_straddle.rs b/tests/guest/rust-tests/src/bin/abuse_page_straddle.rs index 881e49c51d..bc9b220331 100644 --- a/tests/guest/rust-tests/src/bin/abuse_page_straddle.rs +++ b/tests/guest/rust-tests/src/bin/abuse_page_straddle.rs @@ -15,8 +15,8 @@ use toyos_abi::RawHandle; use toyos_abi::syscall::{ - self, MmapFlags, MmapProt, OpenFlags, ProcessStats, SchedInfo, SpawnArgs, Stat, SyscallError, - SYS_FSTAT, SYS_PROCESS_STATS, SYS_SCHED_INFO, + self, MmapFlags, MmapProt, OpenFlags, ProcessStats, SpawnArgs, Stat, SyscallError, SYS_FSTAT, + SYS_PROCESS_STATS, }; /// Where the child starts: `SpawnArgs` names a working directory or the spawn is refused. @@ -145,25 +145,7 @@ fn main() { let child = unsafe { syscall::spawn(&*fitting) }.expect("spawn a SpawnArgs that fits"); syscall::process_wait(child).expect("wait for the child that fits"); - // 5. `SchedInfo`, which the kernel fills. It used to be reached by - // casting a validated byte slice to `&mut SchedInfo`, which checked - // neither the alignment the cast needs nor the page the write lands in. - let info_len = core::mem::size_of::(); - let straddling = boundary - 8; - poison(straddling, info_len); - let ret = raw(SYS_SCHED_INFO, straddling, 0, 0); - assert!( - intact(boundary, info_len - 8), - "sched_info wrote {} bytes past the end of the physical page it translated", - info_len - 8, - ); - assert!(intact(straddling, 8), "sched_info wrote the near half of a SchedInfo it refused"); - assert_eq!(err(ret), Some(SyscallError::BadAddress), "sched_info took a straddling SchedInfo"); - - let ret = raw(SYS_SCHED_INFO, boundary - info_len as u64, 0, 0); - assert!(err(ret).is_none(), "sched_info refused a SchedInfo that ends at the boundary: {ret:#x}"); - - // 6. `ProcessStats`, and the child waited for above still + // 5. `ProcessStats`, which the kernel fills, and the child waited for above still // answers for one — the numbers are the object's and the handle outlives // the process. let stats_len = core::mem::size_of::(); diff --git a/tests/guest/rust-tests/src/bin/job_fairness.rs b/tests/guest/rust-tests/src/bin/job_fairness.rs new file mode 100644 index 0000000000..0ae406c372 --- /dev/null +++ b/tests/guest/rust-tests/src/bin/job_fairness.rs @@ -0,0 +1,131 @@ +//! **Independent jobs share the machine equally, whatever their helper counts.** +//! +//! Three jobs, each made with the test estate's `jobs` right: a build of one +//! busy helper process per CPU, a build of one process with a busy thread per +//! CPU, and a helper-heavy job of four busy helper processes per CPU. Each can +//! fill a third of the machine, so each is owed a third (owner decision J1). +//! What each receives is the CPU time its processes ran over a window the +//! machine is saturated throughout. +//! +//! The tolerance is a tenth of the machine either way. The host model gives +//! this workload thirds to the thousandth on both benches' shapes +//! (`two_builds_and_a_helper_heavy_job_split_the_machine_in_thirds`), and a +//! share per helper process — what the scheduler had before jobs — gives the +//! heavy job two thirds. +//! +//! Its verdict is its exit code; it prints the shares it measured. + +use std::io::Read; +use std::os::toyos::process::{ChildExt, CommandExt}; +use std::process::{Child, Command, Stdio}; +use std::thread; +use std::time::Duration; + +use toyos::endow::{Endowments, SYSCAP_LABEL}; +use toyos::job::JOB_LABEL; +use toyos::syscap::SysCap; +use toyos_abi::syscall::{self, ProcessStats}; +use toyos_abi::RawHandle; + +const ME: &str = "/system/bin/test_rs_job_fairness"; +/// Long enough for every helper to have been placed and balanced. +const WARMUP: Duration = Duration::from_millis(500); +const WINDOW: Duration = Duration::from_secs(3); +const TOLERANCE: f64 = 0.1; + +fn main() { + match std::env::args().nth(1).as_deref() { + Some("spin") => { + let threads = std::env::args().nth(2).and_then(|n| n.parse().ok()).expect("a thread count"); + spin(threads) + } + _ => run(), + } +} + +/// Wait for the start — stdin closing — then keep `threads` threads busy until +/// killed. +fn spin(threads: usize) -> ! { + let _ = std::io::stdin().read_to_end(&mut Vec::new()); + for _ in 1..threads { + thread::spawn(|| loop { + core::hint::spin_loop(); + }); + } + loop { + core::hint::spin_loop(); + } +} + +fn run() { + let cap: SysCap = Endowments::get().take(SYSCAP_LABEL).expect("test-runner endows the estate's capability"); + let cpus = syscall::cpu_count() as usize; + let shapes = [("build", cpus, 1), ("threaded", 1, cpus), ("heavy", 4 * cpus, 1)]; + let mut jobs: Vec<(&str, Vec)> = Vec::new(); + for (name, helpers, threads) in shapes { + let job = cap.new_job().expect("the estate's capability makes a job"); + let children = (0..helpers) + .map(|_| { + let copy = job.duplicate().expect("a second handle to the job"); + Command::new(ME) + .args(["spin", &threads.to_string()]) + .stdin(Stdio::piped()) + .endow(JOB_LABEL, copy.into_raw().0) + .spawn() + .expect("spawn a helper into its job") + }) + .collect(); + jobs.push((name, children)); + } + for (_, children) in &mut jobs { + children.iter_mut().for_each(|child| drop(child.stdin.take())); + } + + thread::sleep(WARMUP); + let before = cpu_ns(&jobs); + thread::sleep(WINDOW); + let after = cpu_ns(&jobs); + for (_, children) in &mut jobs { + for child in children.iter_mut() { + child.kill().expect("kill a helper"); + child.wait().expect("reap a helper"); + } + } + + let used: Vec = before.iter().zip(&after).map(|(b, a)| a - b).collect(); + let total: u64 = used.iter().sum(); + let shares: Vec = used.iter().map(|&ns| ns as f64 / total as f64).collect(); + let capacity = cpus as u64 * WINDOW.as_nanos() as u64; + println!( + "job_fairness: cpus={cpus} window={}ms ran={}ms of {}ms shares {}", + WINDOW.as_millis(), + total / 1_000_000, + capacity / 1_000_000, + jobs.iter().zip(&shares).map(|((name, _), s)| format!("{name}={s:.3}")).collect::>().join(" "), + ); + // A window the machine was not saturated through measures placement and + // idling, not shares. + assert!(total * 10 >= capacity * 9, "the helpers ran {total} ns of the window's {capacity} ns"); + for ((name, _), share) in jobs.iter().zip(&shares) { + assert!( + (share - 1.0 / 3.0).abs() <= TOLERANCE, + "{name} received {share:.3} of the machine against a third", + ); + } +} + +/// CPU time each job's processes have received so far. +fn cpu_ns(jobs: &[(&str, Vec)]) -> Vec { + jobs.iter() + .map(|(_, children)| { + children + .iter() + .map(|child| { + let mut stats = ProcessStats::default(); + syscall::process_stats(RawHandle(child.as_raw_handle()), &mut stats).expect("a helper's accounting"); + stats.cpu_ns + }) + .sum() + }) + .collect() +} diff --git a/tests/guest/rust-tests/src/bin/sched_stress.rs b/tests/guest/rust-tests/src/bin/sched_stress.rs index 70f7bba32f..e2e46c9ac7 100644 --- a/tests/guest/rust-tests/src/bin/sched_stress.rs +++ b/tests/guest/rust-tests/src/bin/sched_stress.rs @@ -1,45 +1,17 @@ use std::sync::Arc; use std::thread; use std::time::Duration; -use std::process::{Command, Stdio}; use toyos_abi::syscall; use toyos::poller::{Poller, READABLE}; use toyos::{namespace, port}; fn main() { - match std::env::args().nth(1).as_deref() { - Some("burn") => child_burn(), - Some("sched-info") => child_sched_info(), - _ => run_tests(), - } -} - -fn run_tests() { test_acceptor_isolation_io_uring(); - test_min_vruntime_invariant(); test_connect_storm(); println!("all sched_stress tests passed"); } -/// Burns until it is killed. -fn child_burn() -> ! { - loop { - std::hint::spin_loop(); - } -} - -fn child_sched_info() { - // Force a Runnable→NonRunnable→Runnable cycle so the lag we report is - // the freshly-clamped post-wake value, not whatever drift has built up - // since spawn. The kernel clamps lag to ±MAX_VRUNTIME_LAG_NS at the - // transition; reading immediately after wake is the only point where - // the invariant holds deterministically. - thread::sleep(Duration::from_millis(10)); - let info = syscall::sched_info(); - println!("{} {} {}", info.vruntime, info.min_vruntime, info.lag); -} - // Test 1: Acceptor isolation via io_uring POLL_IN /// Two acceptors create io_uring POLL_IN watches on their handles. A connection @@ -80,52 +52,6 @@ fn test_acceptor_isolation_io_uring() { println!(" acceptor isolation (io_uring): ok"); } -/// Verify the post-wake lag invariant: when a process transitions from -/// non-runnable back to runnable, its lag is clamped to ±MAX_VRUNTIME_LAG_NS -/// (50ms). Read immediately after the wake transition. -fn test_min_vruntime_invariant() { - let me = "/system/bin/test_rs_sched_stress"; - - // Spawn 3 CPU burners to drive min_vruntime forward, until they are killed. - let mut burners = Vec::new(); - for _ in 0..3 { - burners.push(Command::new(me).arg("burn").spawn().expect("spawn burner")); - } - - // A process that sleeps briefly (forces Runnable→NonRunnable→Runnable) and - // reports lag immediately after wake, asked again until the burners have - // moved min_vruntime off zero: a min_vruntime never updated is a hang the - // harness ceiling reds. - let (vruntime, min_vruntime, lag) = loop { - let info_out = Command::new(me).arg("sched-info") - .stdout(Stdio::piped()).spawn().expect("spawn sched-info") - .wait_with_output().expect("wait sched-info"); - let output = String::from_utf8_lossy(&info_out.stdout); - let parts: Vec<&str> = output.trim().split_whitespace().collect(); - assert_eq!(parts.len(), 3, "sched-info output should be 'vruntime min_vruntime lag', got: {output:?}"); - let vruntime: u64 = parts[0].parse().expect("parse vruntime"); - let min_vruntime: u64 = parts[1].parse().expect("parse min_vruntime"); - let lag: i64 = parts[2].parse().expect("parse lag"); - if min_vruntime > 0 { - break (vruntime, min_vruntime, lag); - } - }; - - for mut child in burners { - child.kill().expect("kill a burner"); - child.wait().expect("reap a burner"); - } - - println!(" sched_info: vruntime={vruntime} min_vruntime={min_vruntime} lag={lag}"); - - let max_lag_ns: i64 = 50_000_000; - assert!(lag.abs() <= max_lag_ns, - "post-wake lag ({lag}) exceeds ±MAX_VRUNTIME_LAG_NS ({max_lag_ns}) — \ - leave_runnable clamp not enforced or wake is not re-deriving from lag!"); - - println!(" post-wake lag invariant: ok"); -} - fn test_connect_storm() { let num_clients = 8; diff --git a/tests/images/testcases.toml b/tests/images/testcases.toml index 51070fdfc4..a8ba693722 100644 --- a/tests/images/testcases.toml +++ b/tests/images/testcases.toml @@ -63,9 +63,10 @@ syscap = ["rt"] # holds its own line, read back off logkeeper. # `inventory` because a claim names a PCI function by its arrival, which the # jobs that claim one read off the inventory. +# `jobs` because `job_fairness` starts each of its workloads as a job of its own. [programs.test-runner] receives = ["soundserver", "volume", "power", "log"] -syscap = ["device", "dup", "logread", "power", "roster", "counters", "trace", "inventory"] +syscap = ["device", "dup", "logread", "power", "roster", "counters", "trace", "inventory", "jobs"] [programs.toybox] receives = ["soundserver"] diff --git a/tests/toyos.rs b/tests/toyos.rs index 3e3ec8bcea..d8ecd8c67d 100644 --- a/tests/toyos.rs +++ b/tests/toyos.rs @@ -89,6 +89,10 @@ const RUST_SKIP: &[&str] = &[ // Its exit code is a round trip's p99 in microseconds, which a guest's host // sets: the `wake_onto_busy` metal row runs it on the T14. "wake_onto_busy", + // Its verdict is how the CPU time of a saturated machine split between + // jobs, which a guest's host sets: the `job_fairness` metal row runs it on + // each bench. + "job_fairness", // Its product is a cycle count per syscall and the clock rate beside it, // which a guest's host sets: the `syscall_cost` metal row runs it. "syscall_cost", @@ -944,6 +948,13 @@ const METAL: &[(&str, metal::Metal)] = &[ "wake_onto_busy", metal::Metal { arms: TESTCASES, judge: |b| wake_onto_busy(b[0]) }, ), + ( + // Two builds and a helper-heavy job, each its own job, on a saturated + // machine: each takes a third whatever its helper count. Its widths are + // the benches', eight CPUs and sixteen. + "job_fairness", + metal::Metal { arms: TESTCASES, judge: |b| job_fairness(b[0]) }, + ), // ---- tests/images/jobcase.toml, each image armed to end its boot its own way ---- ( // Its own boot, and it must not share one: it is the only arm in this @@ -1363,6 +1374,7 @@ const TESTCASES: &[metal::Arm] = &[metal::once( "test_rs_volume_step", "test_rs_syscall_cost", "test_rs_spawn_cost", + "test_rs_job_fairness", claims::RECLAIM, ], )]; @@ -6509,6 +6521,15 @@ fn wake_onto_busy(boot: &metal::Readback) -> Result<(), String> { Ok(()) } +/// `job_fairness` judges its own shares and exits on the verdict; its line is +/// the measurement. +fn job_fairness(boot: &metal::Readback) -> Result<(), String> { + for said in boot.log().text().lines().filter(|l| l.contains("job_fairness: ")) { + eprintln!(" [jobs] {}", said.trim()); + } + boot.job_passed("test_rs_job_fairness") +} + fn run_debug_mode(c_tests: &[(String, Vec)], rust_bins: &[(String, Vec)]) { let cmd_path = Path::new("/tmp/toyos-debug-cmd"); let result_path = Path::new("/tmp/toyos-debug-result"); From 094c71ca4dd1e9b366b24102e9b7e672a20a77d8 Mon Sep 17 00:00:00 2001 From: japabu Date: Wed, 21 Oct 2026 18:43:20 +0200 Subject: [PATCH 3/4] tests: process_tree's launches name the job their program joins A launch now carries a job, and the two process_tree builds by hand gave none, which only the guest build compiles. Co-Authored-By: Claude Opus 5.5 --- tests/guest/rust-tests/src/bin/process_tree.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/guest/rust-tests/src/bin/process_tree.rs b/tests/guest/rust-tests/src/bin/process_tree.rs index a4472c3dde..963a39dfa1 100644 --- a/tests/guest/rust-tests/src/bin/process_tree.rs +++ b/tests/guest/rust-tests/src/bin/process_tree.rs @@ -196,6 +196,7 @@ fn under_b_is_gone(grown: &Grown) { extras: &[], fds: &[], parent: Parent::Place(place), + job: toyos::job::own().expect("a handle to this process's job").into_raw(), }; let mut home = [0u8; 256]; match launch::launch(&conn, &request, &mut home) { @@ -278,6 +279,7 @@ fn a_pipe_is_no_place() { extras: &[], fds: &[], parent: Parent::Place(place), + job: toyos::job::own().expect("a handle to this process's job").into_raw(), }; let mut home = [0u8; 256]; match launch::launch(&conn, &request, &mut home) { From 731df67c40e987b2261bab44c03269635ec2f886 Mon Sep 17 00:00:00 2001 From: japabu Date: Wed, 21 Oct 2026 19:32:30 +0200 Subject: [PATCH 4/4] sched: bound lopsided moves by their causes, and make the commit gate exact Review r1 of #979. The lopsided placement cases bounded migrations by the run's length, so a ping-pong fifty times worse still passed. A trace of every move over the 20 seeds per width shows two causes before the first exit, neither a bounce. At start-up, an idle CPU steals from a neighbour that busy balancing has just given a second thread. At the end of the drain, cpu0 answers a burst of requests posted in one instant, drops below a CPU that took threads earlier, and that CPU hands one on. Each other CPU sources at most two moves before an exit. After the first exit, only an exit opens a pair more than one thread apart, the quarter-thread margin's threshold. The bounds are now at most even_drain + 2 * (cpus - 1) moves before an exit and at most one move per exit after it. Measured worst: even + 8 before (8 CPUs, 64 threads), and 53 moves after 64 exits (4 CPUs, 64 threads). Dropping pop_surplus's margin fails it: 21 moves after 8 exits at 2 CPUs. old_commit_before_pass passed only on a 3/4 threshold, with an unverified account of the misses. Counting call-site commits that go on to park splits the 500 schedules exactly. All 415 caught schedules commit at least once. None of the 85 others does: every consumer block there found a token waiting (507 at entry, 2 at the re-check). The gate now asserts that a schedule fails if and only if it commits at the call site, and keeps a reach floor so the workload cannot go quiet. Pointing it at the fused shape, which the harness cannot see, fails at seed 1. Also: the decision comment in preempt_if_due no longer claims a pass's queue changes only bring the decision forward. windows.rs no longer cites the deleted Frontier. The scheduler track's three stage-3 leftovers each get an exit. Co-Authored-By: Claude Opus 5.5 --- ...the-scheduler-and-the-clocks-never-talk.md | 41 ++++++------ kernel/pure/sched/cpu.rs | 8 +-- kernel/pure/sched/windows.rs | 4 +- kernel/sim/src/explore.rs | 9 +++ kernel/sim/src/invariants.rs | 3 + kernel/sim/src/vm.rs | 14 +++++ kernel/sim/tests/policy.rs | 62 +++++++++++-------- kernel/sim/tests/scenarios.rs | 36 +++++++---- 8 files changed, 111 insertions(+), 66 deletions(-) diff --git a/issues/the-scheduler-and-the-clocks-never-talk.md b/issues/the-scheduler-and-the-clocks-never-talk.md index 37158f3b5b..c94f808c10 100644 --- a/issues/the-scheduler-and-the-clocks-never-talk.md +++ b/issues/the-scheduler-and-the-clocks-never-talk.md @@ -71,27 +71,22 @@ scales with crowding. These are model figures for the scenarios that ran, not hardware latency guarantees. Stage 3 validates the kernel implementation on the supported machines. -Stage 3 is in the kernel. A job is a kernel object (`kernel/pure/sched/job.rs`, -`kernel/src/object/job.rs`) a process's threads and its children share unless a -spawn endows another under `JOB_LABEL`, and a launch carries its caller's; only a system capability's `JOBS` right -makes one, which the supervisor makes each service with and the compositor -each program it starts. Each CPU -runs the two EEVDF levels (`kernel/pure/sched/queue.rs`) over a deadline-ordered -tree augmented with each subtree's least virtual time, and decides again at a -request's end, at an earlier-due group's eligibility, and at a wake due first — -the last two once the running thread has run `MIN_SLICE_NS`. Every change to a -CPU's queues is made in that CPU's own pass, and every pass ends by arming the -timer for the earliest of these, so a remote steal re-arms it as a wake does. -Without the eligibility decision a job whose request is due first but carries a -little debt waits out every longer request ahead of it: in the model's desktop -mix beside a build the compositor then missed frames, finishing one 9.4 ms after -its release; with it, none on the T14 and AMD shapes. Busy CPUs balance summed -group weight, one request per CPU per `BALANCE_NS`. A voluntary yield ends the -yielder's request at both levels. +Stage 3 is in the kernel (`kernel/pure/sched/queue.rs`, `kernel/pure/sched/job.rs`, +`kernel/src/object/job.rs`). Its remaining work: -Stage 3 leaves three things. Explicit latency requests have no syscall: the -queue takes one request size for every thread until a first caller (the -compositor's or the audio worker's) needs a shorter one. `sshserver` starts its -sessions as its own helpers, since it has no ToyOS dependency to make a job -with. std and libc start a child in a new job only by `CommandExt::endow` under -`JOB_LABEL`. +- **Explicit latency requests.** The queue gives every thread one request + size, and no syscall asks for a shorter one. Deferred until a first caller + needs it (orchestrator, 2026-10-21): stage 4's audio workers are the + expected one. Exit: that caller's stage adds a bounded request that changes + order, not weight, and its test shows the caller meeting its deadline beside + saturating jobs. +- **sshserver sessions.** `sshserver` starts every session as one of its own + helpers, so all remote logins share its job: it has no ToyOS dependency + through which to make one. Exit: sshserver holds `jobs` and starts each session in a job + of its own, through std's endowment of `JOB_LABEL` or the std/libc path + below. +- **A new job from std and libc.** Their spawn reaches a new job only through + `CommandExt::endow(JOB_LABEL, …)`, and libc has no such call. This waits + for the fd-table work that rewrites spawn. Exit: std and libc each take a + job for a child through their ordinary spawn interface, which is then the + route the bullet above uses. diff --git a/kernel/pure/sched/cpu.rs b/kernel/pure/sched/cpu.rs index c0273b80a5..0f4b078bdc 100644 --- a/kernel/pure/sched/cpu.rs +++ b/kernel/pure/sched/cpu.rs @@ -1743,10 +1743,10 @@ impl SchedPass<'_, '_, H, P, Disposed> { let kill_due = current.shared().kill_pending() && !self.cpu.aged_grant && matches!(current.shared().state(), TaskState::Running(_)); - // The instant the last pass armed is when the running request ends or - // an earlier-due group became eligible, and `begin`'s charge has - // already issued the next request; this pass's own queue changes can - // only bring the decision forward. + // Both instants count: `begin`'s charge has already issued the next + // request, so a recomputed decision alone would never reach the end of + // the one the last pass armed. A stale armed instant at worst decides + // early, and the pick keeps `dispatched_at` if it picks this thread again. let due = self.now >= self.cpu.decide_at || self.now >= self.cpu.settle_decision(self.now) || rt_due || kill_due; if !due { return; diff --git a/kernel/pure/sched/windows.rs b/kernel/pure/sched/windows.rs index 6f949cd51d..cdaa96da3e 100644 --- a/kernel/pure/sched/windows.rs +++ b/kernel/pure/sched/windows.rs @@ -75,8 +75,8 @@ impl fmt::Display for Unseen { } impl Windows { - /// No `Default` beside it, for `fair::Frontier::new`'s reason: every - /// record is a `static`, which only a `const fn` can build. + /// No `Default` beside it: every record is a `static`, which only a + /// `const fn` can build. #[cfg(not(feature = "loom"))] #[allow(clippy::new_without_default)] pub const fn new() -> Self { diff --git a/kernel/sim/src/explore.rs b/kernel/sim/src/explore.rs index b55d574cdb..b1a4eb2f1f 100644 --- a/kernel/sim/src/explore.rs +++ b/kernel/sim/src/explore.rs @@ -97,6 +97,12 @@ pub struct Outcome { pub thread_finish_ns: std::collections::BTreeMap, /// How many tasks the balance path moved between CPUs. pub migrations: u64, + /// See [`crate::vm::Vm::migrations_before_exit`]. + pub migrations_before_exit: u64, + /// See [`crate::vm::Vm::call_site_commits`]. + pub call_site_commits: u64, + /// Threads that ran to their exit. + pub exits: u64, /// Per CPU: when it first took an execution step — see /// [`crate::vm::Vm::first_exec_ns`]. pub first_exec_ns: Vec>, @@ -324,6 +330,9 @@ fn outcome_of(scenario: &'static str, vm: &Vm<'_>, choices: &ChoiceStream) -> Ou process_finish_ns: vm.finish_ns.clone(), thread_finish_ns: vm.thread_finish_ns.clone(), migrations: vm.migrations, + migrations_before_exit: vm.migrations_before_exit, + call_site_commits: vm.call_site_commits, + exits: vm.exits, first_exec_ns: vm.first_exec_ns.clone(), never_ran: vm.spawned.difference(&vm.ran).count(), idle_wakes: vm.idle_wakes.clone(), diff --git a/kernel/sim/src/invariants.rs b/kernel/sim/src/invariants.rs index 7c415ae80e..ef71390aa9 100644 --- a/kernel/sim/src/invariants.rs +++ b/kernel/sim/src/invariants.rs @@ -282,6 +282,9 @@ fn check_retires(vm: &mut Vm<'_>) { // case claims a machine drains in parallel, and that claim is only about the // balance path if the balance path moved something. vm.migrations += migrated.len() as u64; + if vm.exits == 0 { + vm.migrations_before_exit += migrated.len() as u64; + } for (key, to) in migrated { if vm.killed.contains_key(&key) { problems.push(format!( diff --git a/kernel/sim/src/vm.rs b/kernel/sim/src/vm.rs index 3967f5091e..367c2b0bef 100644 --- a/kernel/sim/src/vm.rs +++ b/kernel/sim/src/vm.rs @@ -440,6 +440,15 @@ pub struct Vm<'q> { /// than judged: a wakeup storm that drains in parallel across a machine is /// only evidence about the balance path if the balance path ran. pub migrations: u64, + /// Of [`Self::migrations`], those made before any thread exited: the moves + /// the initial placement alone called for. + pub migrations_before_exit: u64, + /// Blocks that committed at the call site and went on to park: under + /// [`BlockShape::CommitAtCallSite`], each is a task whose word says + /// `Blocked` while its CPU still runs it. + pub call_site_commits: u64, + /// Threads that ran to their exit. + pub exits: u64, /// Per CPU: the instant it first took an execution step, or `None` for a /// CPU that never ran anything at all. /// @@ -639,6 +648,9 @@ impl<'q> Vm<'q> { finish_ns: vec![None; process_count], thread_finish_ns: BTreeMap::new(), migrations: 0, + migrations_before_exit: 0, + call_site_commits: 0, + exits: 0, first_exec_ns: vec![None; n], idle_wakes: vec![0; n], late_kick_wakes: vec![0; n], @@ -1624,6 +1636,7 @@ impl<'q> Vm<'q> { BlockShape::CommitAtCallSite | BlockShape::CommitAtCallSiteFused => { match ticket.commit() { Commit::Parked(committed) => { + self.call_site_commits += 1; self.registrations.insert(key, queue); BlockPhase::Committed(committed) } @@ -1805,6 +1818,7 @@ impl<'q> Vm<'q> { } self.programs.remove(&key); self.unwind_left.remove(&key); + self.exits += 1; self.run_pass(cpu, Dispose::Exit); } diff --git a/kernel/sim/tests/policy.rs b/kernel/sim/tests/policy.rs index 1e528c661b..85d1f435b8 100644 --- a/kernel/sim/tests/policy.rs +++ b/kernel/sim/tests/policy.rs @@ -84,7 +84,7 @@ //! doing what a sweep is for. use kernel::sched::cpu::Balance; -use kernel::sched::cpu::{BALANCE_NS, STALE_PASS_NS}; +use kernel::sched::cpu::STALE_PASS_NS; use kernel::sched::queue::{MIN_SLICE_NS, SLICE_NS}; use kernel::sched::task::QUANTUM_NS; use kernel_sim::choice::ChoiceStream; @@ -654,13 +654,39 @@ fn stealable(threads: usize) -> u64 { threads as u64 - 1 } -/// The most migrations a run of `elapsed` may make: everything above cpu0's -/// floor once, plus what busy balancing moves between CPUs that are both -/// running work — one request per CPU per [`BALANCE_NS`] at most, which is how -/// a thread can move twice: a thief becomes a victim when the threads it took -/// outlast cpu0's. -fn migration_ceiling(threads: usize, cpus: usize, elapsed: u64) -> u64 { - stealable(threads) + cpus as u64 * (elapsed / BALANCE_NS + 1) +/// **Every move a lopsided run makes has a cause, and none is a bounce.** +/// +/// Before any thread exits, the counts only change by moves. cpu0 gives up +/// [`even_drain`] tasks net, and each other CPU sources at most two moves of +/// its own: +/// - one at start-up, when an idle CPU's nearest neighbour holding surplus is +/// one busy balancing has just given a second thread, rather than cpu0; +/// - one at the end of the drain, when cpu0 answers a burst of requests posted +/// in the same instant, drops below a CPU that took threads earlier, and that +/// CPU hands one on. +/// +/// After the first exit, a move needs a pair whose weights differ by more than +/// one thread (`pop_surplus`'s quarter-thread margin), and only an exit opens +/// one, so the moves are at most one per exit. Measured over 20 seeds: at most +/// even + 8 before an exit (8 CPUs, 64 threads) and 53 moves after 64 exits +/// (4 CPUs, 64 threads). A pair that bounces a thread with no exit between +/// exceeds one bound or the other. +fn assert_moves_have_causes(outcome: &Outcome, cpus: usize, threads: usize, label: &str) { + let before = even_drain(threads, cpus) + 2 * (cpus as u64 - 1); + assert!( + outcome.migrations_before_exit <= before, + "{label} at {cpus} cpus and {threads} threads: {} moves before any thread exited, past \ + the {before} that an even split, one start-up chain and one end-of-drain correction \ + per other CPU account for", + outcome.migrations_before_exit, + ); + let after = outcome.migrations - outcome.migrations_before_exit; + assert!( + after <= outcome.exits, + "{label} at {cpus} cpus and {threads} threads: {after} moves after the first exit, \ + against {} exits that could each open one imbalance", + outcome.exits, + ); } /// The fewest migrations that leave cpu0 holding no more than an even split of @@ -724,15 +750,7 @@ fn the_balance_path_drains_the_cpu_an_adversary_loaded() { those CPUs was given work by something that is not the balance path", outcome.migrations, ); - // And the ceiling: what cpu0 has to give, plus what busy balancing - // is rate-limited to. - let ceiling = migration_ceiling(threads, cpus, outcome.elapsed); - assert!( - outcome.migrations <= ceiling, - "at {cpus} cpus the balance path moved {} of {threads} tasks, past the {ceiling} \ - that cpu0's surplus and one busy request per CPU per `BALANCE_NS` allow", - outcome.migrations, - ); + assert_moves_have_causes(outcome, cpus, threads, "the shipped policy"); best = best.max(outcome.migrations); widest = widest.max(ran); full += usize::from(ran == cpus); @@ -900,7 +918,7 @@ fn wake_rate(outcome: &Outcome) -> f64 { /// The two ceilings the shipped path already obeys are asserted here rather than /// in each case, because every policy below is built on the same balance path: /// no CPU is given work by anything but the balance path, and the migrations -/// stay under [`migration_ceiling`]. +/// each have a cause ([`assert_moves_have_causes`]). fn recover(cpus: usize, threads: usize, balance: Balance) -> Recovery { let scenario = scenarios::lopsided_placement(cpus, threads, WORK).with_balance(balance); let mut r = Recovery::default(); @@ -913,13 +931,7 @@ fn recover(cpus: usize, threads: usize, balance: Balance) -> Recovery { those CPUs was given work by something that is not the balance path", outcome.migrations, ); - let ceiling = migration_ceiling(threads, cpus, outcome.elapsed); - assert!( - outcome.migrations <= ceiling, - "{balance:?} at {cpus} cpus: the balance path moved {} of {threads} tasks, past the \ - {ceiling} that cpu0's surplus and one busy request per CPU per `BALANCE_NS` allow", - outcome.migrations, - ); + assert_moves_have_causes(outcome, cpus, threads, &format!("{balance:?}")); r.full += usize::from(ran == cpus); r.second += usize::from(ran > 1); r.best_migrations = r.best_migrations.max(outcome.migrations); diff --git a/kernel/sim/tests/scenarios.rs b/kernel/sim/tests/scenarios.rs index badeb247cb..bc33ec5322 100644 --- a/kernel/sim/tests/scenarios.rs +++ b/kernel/sim/tests/scenarios.rs @@ -19,7 +19,7 @@ //! | gate | breaks | caught by | //! |---|---|---| //! | `old_steal_port` | the old steal-and-scan algorithm | I1 **and** I8 | -//! | `old_commit_before_pass` | the pre-`8508b37` blocking shape | I1, on every seed | +//! | `old_commit_before_pass` | the pre-`8508b37` blocking shape | I1, on every seed that commits at the call site | //! | `old_preemptible_window` | preemption left on in the registration window | an abort inside `check_cpu` | //! | `old_migrate_kept_the_corpse` | the balance path handing on a killed task | I14 | //! | `old_rt_starved_the_corpse` | the RT band outranking the dying list without a bound | I14, on every seed | @@ -178,7 +178,7 @@ fn old_steal_port_is_caught() { fn old_commit_before_pass_is_caught() { let scenario = scenarios::old_commit_before_pass(); let mut kinds: BTreeMap = BTreeMap::new(); - let mut caught = 0; + let (mut caught, mut reached) = (0, 0); let mut worst_steps = 0; for seed in 0..SEEDS { let mut choices = if seed % 2 == 0 { @@ -187,6 +187,20 @@ fn old_commit_before_pass_is_caught() { ChoiceStream::pct(seed, scenario.cpus(), 3) }; let outcome = run(scenario.clone(), &mut choices); + // The shape is reached the first time a block commits at the call + // site and goes on to park: its word then reads `Blocked` while its + // CPU still runs it. A schedule in which every block found a token + // waiting never executes the defect. + let reaches = outcome.call_site_commits > 0; + reached += usize::from(reaches); + assert_eq!( + !outcome.passed(), + reaches, + "seed {seed}: {} call-site commit(s), and the run {} — the gate must catch exactly \ + the schedules that commit before the pass", + outcome.call_site_commits, + if outcome.passed() { "passed" } else { "failed" }, + ); if outcome.passed() { continue; } @@ -197,21 +211,19 @@ fn old_commit_before_pass_is_caught() { *kinds.entry(id).or_default() += 1; } } - // Not every schedule: the explorer may leave a CPU un-stepped while it - // holds a task, so a consumer's request can end before it took a step, and - // the producer's tokens then outpace the consumers and no block finds the - // queue empty. Measured 415 of 500 on `e74dbf1` with the job-group - // scheduler applied. + // Coverage, not detection: the workload must keep reaching the window. In + // 85 of 500 schedules the producer stays ahead and every consumer block + // finds a token waiting (507 at entry, 2 at the re-check), so none commits. assert!( - caught * 4 >= 3 * SEEDS as usize, - "committing the wait ticket before the pass must be caught in at least \ - three schedules of four: {caught}/{SEEDS}", + reached * 4 >= 3 * SEEDS as usize, + "only {reached}/{SEEDS} schedules reached a call-site commit; the workload has stopped \ + exercising the window", ); // 50 steps at worst, measured with `caught` above. assert!( worst_steps < 64, - "the worst schedule took {worst_steps} steps to expose it; this is a \ - structural break and should surface almost immediately", + "the worst of {caught} caught schedules took {worst_steps} steps to expose it; this is \ + a structural break and should surface almost immediately", ); assert!( kinds.contains_key("I1"),