From 21b60ba8ba06686bda14eb6ce3f1098b6caa75ca Mon Sep 17 00:00:00 2001 From: japabu Date: Sat, 24 Oct 2026 07:32:37 +0200 Subject: [PATCH 1/7] sched: a run queue's entities live in its arenas, not in maps it rebuilds per wake Every wake made a job group (a BTreeMap entry, an Eevdf with three allocations and a key map) and every block dissolved it again, and each join reweighted the group by a leave and a rejoin. A TCG instruction count of the futex ping-pong put RunQueue::insert and ::leave at 18k of the 58k kernel instructions per round trip, a dozen allocations under the global heap lock among them. The EEVDF arithmetic is unchanged. A Queue is now only a tree root and its sums over entities kept in an Arena that several queues share; a caller names an entity by the Slot its join answered and the entity carries its payload. The run queue holds one arena of group entities, each carrying its job and its threads' Queue, and one of thread entities, each carrying its ReadyTask (None while it runs). A new group joins at the weight it holds once its thread is counted, instead of at 1 and then a reweight. A reweight and a carried group lag readmit the entity in place. The rebase skips its division when the mean is already in its unit. Eevdf keeps the keyed API for job_sched's model and the tests. Co-Authored-By: Claude Opus 5.5 --- kernel/pure/sched/eevdf.rs | 534 +++++++++++++++++++++++++------------ kernel/pure/sched/queue.rs | 236 +++++++++------- 2 files changed, 509 insertions(+), 261 deletions(-) diff --git a/kernel/pure/sched/eevdf.rs b/kernel/pure/sched/eevdf.rs index 99d37125d3b..64a5729be5e 100644 --- a/kernel/pure/sched/eevdf.rs +++ b/kernel/pure/sched/eevdf.rs @@ -20,6 +20,13 @@ //! Entities sit in an AVL tree ordered by deadline, each node holding its //! subtree's least `v`, so a pick, an eligibility query and every update are //! O(log n) in the entities queued. +//! +//! A [`Queue`] is only the tree's root and its sums: its entities live in an +//! [`Arena`], which several queues may share, and a caller names one by the +//! [`Slot`] its join answered. So a queue costs nothing to make or drop, and +//! an entity joins and leaves without a lookup or an allocation once its +//! arena has held as many. [`Eevdf`] is one queue that names its entities by +//! key instead. use alloc::collections::BTreeMap; use alloc::vec::Vec; @@ -57,7 +64,15 @@ impl Lag { const NIL: u32 = u32::MAX; -struct Entity { +/// The deepest an AVL tree of `u32`-indexed nodes can be: `1.44·log2(2^32)`. +const MAX_HEIGHT: usize = 48; + +/// One entity's place in its [`Arena`], valid from the join that answered it +/// to the leave that takes it. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub struct Slot(u32); + +struct Entity { key: K, weight: u64, /// Absolute virtual time, in `2^VSHIFT`-scaled ns per unit weight. @@ -75,11 +90,10 @@ struct Entity { height: u8, /// The least `v` in this entity's subtree. min_v: i128, - /// Where it is in [`Eevdf::order`]. - pos: u32, + payload: P, } -/// What [`Eevdf::leave`] hands back: everything needed to rejoin elsewhere. +/// What [`Queue::leave`] hands back: everything needed to rejoin elsewhere. #[derive(Clone, Copy, PartialEq, Eq, Debug)] pub struct Left { pub lag: Lag, @@ -87,16 +101,16 @@ pub struct Left { pub request_left: i128, } -pub struct Eevdf { - /// Every entity queued, and the slots of those that left, which `free` - /// lists for reuse. - nodes: Vec>, +/// The entities of one or more queues, each carrying its caller's `P`. +pub struct Arena { + nodes: Vec>>, free: Vec, - slots: BTreeMap, - /// The queued slots in join order, a leave moving the last into its place: - /// the order [`Self::keys`] answers in. - order: Vec, +} + +/// One EEVDF queue over entities kept in an [`Arena`]. +pub struct Queue { root: u32, + len: u32, total: i128, /// Origin of `sum`, kept within one virtual unit of `V`. base: i128, @@ -118,59 +132,64 @@ fn fine(ns: u64) -> i128 { (ns as i128) << VSHIFT } -impl Eevdf { - pub fn new() -> Self { - Self { - nodes: Vec::new(), - free: Vec::new(), - slots: BTreeMap::new(), - order: Vec::new(), - root: NIL, - total: 0, - base: 0, - sum: 0, - seq: 0, - } +impl Arena { + pub const fn new() -> Self { + Self { nodes: Vec::new(), free: Vec::new() } } - pub fn len(&self) -> usize { - self.order.len() + fn node(&self, i: u32) -> &Entity { + self.nodes[i as usize].as_ref().expect("eevdf: a slot no entity holds") } - pub fn is_empty(&self) -> bool { - self.order.is_empty() + fn node_mut(&mut self, i: u32) -> &mut Entity { + self.nodes[i as usize].as_mut().expect("eevdf: a slot no entity holds") } - pub fn contains(&self, key: K) -> bool { - self.slots.contains_key(&key) + fn take(&mut self, i: u32) -> Entity { + let e = self.nodes[i as usize].take().expect("eevdf: a slot no entity holds"); + self.free.push(i); + e } - pub fn keys(&self) -> impl Iterator + '_ { - self.order.iter().map(|&i| self.nodes[i as usize].key) + fn put(&mut self, e: Entity) -> u32 { + match self.free.pop() { + Some(i) => { + self.nodes[i as usize] = Some(e); + i + } + None => { + self.nodes.push(Some(e)); + (self.nodes.len() - 1) as u32 + } + } } - pub fn total_weight(&self) -> u128 { - self.total as u128 + pub fn key(&self, s: Slot) -> K { + self.node(s.0).key } - pub fn weight(&self, key: K) -> u64 { - self.get(key).weight + pub fn weight(&self, s: Slot) -> u64 { + self.node(s.0).weight } - fn index(&self, key: K) -> u32 { - *self.slots.get(&key).expect("eevdf: entity is not queued") + pub fn slice(&self, s: Slot) -> u64 { + self.node(s.0).slice } - fn get(&self, key: K) -> &Entity { - &self.nodes[self.index(key) as usize] + pub fn payload(&self, s: Slot) -> &P { + &self.node(s.0).payload } - fn node(&self, i: u32) -> &Entity { - &self.nodes[i as usize] + pub fn payload_mut(&mut self, s: Slot) -> &mut P { + &mut self.node_mut(s.0).payload } - fn node_mut(&mut self, i: u32) -> &mut Entity { - &mut self.nodes[i as usize] + /// Service, in whole ns rounded up, until the entity's current request is done. + pub fn request_left_ns(&self, s: Slot) -> u64 { + let e = self.node(s.0); + let left = mul(e.vd - e.v, e.weight as i128) - e.rem as i128; + let ns = (left + (1 << VSHIFT) - 1) >> VSHIFT; + ns.max(1) as u64 } /// Tree order: deadline, then issue order. @@ -179,12 +198,6 @@ impl Eevdf { (e.vd, e.seq) } - fn eligible(&self, v: i128) -> bool { - mul(self.total, v - self.base) <= self.sum - } - - // ---- The tree ------------------------------------------------------------ - fn height(&self, i: u32) -> u8 { if i == NIL { 0 } else { self.node(i).height } } @@ -308,20 +321,39 @@ impl Eevdf { } self.update(t); } +} - /// Give node `i` a fresh request, moving it to its new place in the tree. - fn reissue(&mut self, i: u32, request: i128) { - let at = self.order_of(i); - self.root = self.remove(self.root, at); - self.issue(i, request); - self.root = self.insert(self.root, i); +impl Default for Arena { + fn default() -> Self { + Self::new() + } +} + +impl Queue { + pub const fn new() -> Self { + Self { root: NIL, len: 0, total: 0, base: 0, sum: 0, seq: 0 } + } + + pub fn len(&self) -> usize { + self.len as usize } - // ---- The queue ----------------------------------------------------------- + pub fn is_empty(&self) -> bool { + self.len == 0 + } + + pub fn total_weight(&self) -> u128 { + self.total as u128 + } + + fn eligible(&self, v: i128) -> bool { + mul(self.total, v - self.base) <= self.sum + } /// Move `base` to `⌊V⌋` so the sums stay small. fn rebase(&mut self) { - if self.total > 0 { + // `0 ≤ sum < total` is `⌊V⌋ = base` already, and the division is spared. + if self.total > 0 && !(0..self.total).contains(&self.sum) { let q = self.sum.div_euclid(self.total); self.base = add(self.base, q); self.sum -= mul(q, self.total); @@ -329,38 +361,51 @@ impl Eevdf { } /// Set a detached node's request. - fn issue(&mut self, i: u32, request: i128) { + fn issue(&mut self, arena: &mut Arena, i: u32, request: i128) { self.seq += 1; let seq = self.seq; - let e = self.node_mut(i); + let e = arena.node_mut(i); e.vd = add(e.v, request.div_euclid(e.weight as i128).max(1)); e.seq = seq; } - /// Enter with `lag` and a fresh request of `slice` ns. Returns the lag the - /// entity really holds afterwards: `lag` itself up to rounding, or zero in - /// an empty queue, where there is no one to owe or be owed. - pub fn join(&mut self, key: K, weight: u64, slice: u64, lag: Lag) -> Lag { - self.join_with_request(key, weight, slice, lag, fine(slice)) + /// Give node `i` a fresh request, moving it to its new place in the tree. + fn reissue(&mut self, arena: &mut Arena, i: u32, request: i128) { + let at = arena.order_of(i); + self.root = arena.remove(self.root, at); + self.issue(arena, i, request); + self.root = arena.insert(self.root, i); } - fn join_with_request(&mut self, key: K, weight: u64, slice: u64, lag: Lag, request: i128) -> Lag { - assert!((1..=MAX_WEIGHT).contains(&weight), "eevdf: weight {weight} out of range"); - assert!(!self.contains(key), "eevdf: entity joined twice"); - self.rebase(); - let w = weight as i128; - let v_rel = if self.total == 0 { - 0 - } else { - // v = V − l/W − l/w makes w·(V' − v) = l once V' includes v. - let l = lag.0; - let num = mul(w, self.sum - l) - mul(l, self.total); - num.div_euclid(mul(self.total, w)) - }; - let entity = Entity { + /// Enter with `lag` and a fresh request of `slice` ns: the entity holds + /// `lag` itself afterwards up to rounding, or zero in an empty queue, where + /// there is no one to owe or be owed. + pub fn join(&mut self, arena: &mut Arena, key: K, weight: u64, slice: u64, lag: Lag, payload: P) -> Slot { + self.join_with_request(arena, key, weight, slice, lag, fine(slice), payload) + } + + /// Enter carrying what [`Self::leave`] returned elsewhere: the lag and the + /// unfinished part of the request. + pub fn rejoin(&mut self, arena: &mut Arena, key: K, weight: u64, slice: u64, left: Left, payload: P) -> Slot { + let request = left.request_left.clamp(1, fine(slice)); + self.join_with_request(arena, key, weight, slice, left.lag, request, payload) + } + + #[allow(clippy::too_many_arguments)] + fn join_with_request( + &mut self, + arena: &mut Arena, + key: K, + weight: u64, + slice: u64, + lag: Lag, + request: i128, + payload: P, + ) -> Slot { + let i = arena.put(Entity { key, weight, - v: add(self.base, v_rel), + v: 0, rem: 0, vd: 0, slice, @@ -369,61 +414,76 @@ impl Eevdf { right: NIL, height: 1, min_v: 0, - pos: self.order.len() as u32, - }; - let i = match self.free.pop() { - Some(i) => { - self.nodes[i as usize] = entity; - i - } - None => { - self.nodes.push(entity); - (self.nodes.len() - 1) as u32 - } + payload, + }); + self.enter(arena, i, lag, request); + Slot(i) + } + + /// Place the detached node `i`, whose weight and slice are set, with + /// `lag` and a fresh request of `request` fine units. + fn enter(&mut self, arena: &mut Arena, i: u32, lag: Lag, request: i128) { + let weight = arena.node(i).weight; + assert!((1..=MAX_WEIGHT).contains(&weight), "eevdf: weight {weight} out of range"); + self.rebase(); + let w = weight as i128; + let v_rel = if self.total == 0 { + 0 + } else { + // v = V − l/W − l/w makes w·(V' − v) = l once V' includes v. + let l = lag.0; + let num = mul(w, self.sum - l) - mul(l, self.total); + num.div_euclid(mul(self.total, w)) }; - self.slots.insert(key, i); - self.order.push(i); - self.issue(i, request); - self.root = self.insert(self.root, i); + let e = arena.node_mut(i); + e.v = add(self.base, v_rel); + e.rem = 0; + self.issue(arena, i, request); + self.root = arena.insert(self.root, i); self.sum = add(self.sum, mul(w, v_rel)); self.total += w; - self.lag(key) + self.len += 1; } - /// Leave, taking the entity's lag and unfinished request with it. - pub fn leave(&mut self, key: K) -> Left { + /// Detach node `i` from the tree and the sums, answering what it takes + /// away; the node stays in its arena. + fn exit(&mut self, arena: &mut Arena, i: u32) -> Left { self.rebase(); - let lag = self.lag(key); - let i = self.slots.remove(&key).expect("eevdf: entity is not queued"); - let at = self.order_of(i); - self.root = self.remove(self.root, at); - let pos = self.node(i).pos as usize; - self.order.swap_remove(pos); - if let Some(&moved) = self.order.get(pos) { - self.node_mut(moved).pos = pos as u32; - } - self.free.push(i); - let e = self.node(i); + let lag = self.lag_of(arena, i); + let at = arena.order_of(i); + self.root = arena.remove(self.root, at); + let e = arena.node(i); let w = e.weight as i128; let (v, vd) = (e.v, e.vd); self.sum -= mul(w, v - self.base); self.total -= w; + self.len -= 1; if self.total == 0 { self.sum = 0; } Left { lag, request_left: mul(vd - v, w) } } - /// Enter carrying what [`Self::leave`] returned elsewhere: the lag and the - /// unfinished part of the request. - pub fn rejoin(&mut self, key: K, weight: u64, slice: u64, left: Left) -> Lag { - let request = left.request_left.clamp(1, fine(slice)); - self.join_with_request(key, weight, slice, left.lag, request) + /// Leave, taking the entity's lag and unfinished request with it, and its + /// payload out of the arena. + pub fn leave(&mut self, arena: &mut Arena, s: Slot) -> (Left, P) { + let left = self.exit(arena, s.0); + (left, arena.take(s.0).payload) } - /// `w·(V − v)`. - pub fn lag(&self, key: K) -> Lag { - let e = self.get(key); + /// Leave and enter again in place at `weight`, carrying the lag and the + /// unfinished request as `adjust` answers them: a reweight, or a lag + /// brought from elsewhere. The slot is the same. + pub fn readmit(&mut self, arena: &mut Arena, s: Slot, weight: u64, adjust: impl FnOnce(Left) -> Left) { + let left = adjust(self.exit(arena, s.0)); + let e = arena.node_mut(s.0); + e.weight = weight; + let request = left.request_left.clamp(1, fine(e.slice)); + self.enter(arena, s.0, left.lag, request); + } + + fn lag_of(&self, arena: &Arena, i: u32) -> Lag { + let e = arena.node(i); if self.total == 0 { return Lag::ZERO; } @@ -432,11 +492,16 @@ impl Eevdf { Lag(lag) } + /// `w·(V − v)`. + pub fn lag(&self, arena: &Arena, s: Slot) -> Lag { + self.lag_of(arena, s.0) + } + /// The running entity received `ns`. A finished request is followed at /// once by the next, whose deadline counts from where service now stands. - pub fn charge(&mut self, key: K, ns: u64) { - let i = self.index(key); - let e = self.node_mut(i); + pub fn charge(&mut self, arena: &mut Arena, s: Slot, ns: u64) { + let i = s.0; + let e = arena.node_mut(i); let total = fine(ns) + e.rem as i128; let w = e.weight as i128; let dv = total / w; @@ -445,22 +510,23 @@ impl Eevdf { let (v, vd, slice) = (e.v, e.vd, e.slice); self.sum = add(self.sum, mul(dv, w)); if v >= vd { - self.reissue(i, fine(slice)); + self.reissue(arena, i, fine(slice)); } else { - self.refresh(self.root, self.order_of(i)); + let at = arena.order_of(i); + arena.refresh(self.root, at); } } /// The eligible entity with the earliest deadline. - pub fn pick(&self) -> Option { + pub fn pick(&self, arena: &Arena) -> Option { let mut t = self.root; while t != NIL { - let e = self.node(t); - if e.left != NIL && self.eligible(self.node(e.left).min_v) { + let e = arena.node(t); + if e.left != NIL && self.eligible(arena.node(e.left).min_v) { t = e.left; } else if self.eligible(e.v) { - return Some(e.key); - } else if e.right != NIL && self.eligible(self.node(e.right).min_v) { + return Some(Slot(t)); + } else if e.right != NIL && self.eligible(arena.node(e.right).min_v) { t = e.right; } else { break; @@ -474,16 +540,16 @@ impl Eevdf { /// whose request is due before `running`'s becomes eligible and so the /// pick: `V` advances by `1 / total` per unit of service, and an entity /// with lag `l < 0` becomes eligible once it has advanced `−l / w`. - pub fn overtaken_in(&self, running: K) -> Option { - let before = self.order_of(self.index(running)); + pub fn overtaken_in(&self, arena: &Arena, running: Slot) -> Option { + let before = arena.order_of(running.0); let mut least: Option = None; let mut t = self.root; while t != NIL { - let e = self.node(t); - if self.order_of(t) < before { + let e = arena.node(t); + if arena.order_of(t) < before { let here = match e.left { NIL => e.v, - l => e.v.min(self.node(l).min_v), + l => e.v.min(arena.node(l).min_v), }; least = Some(least.map_or(here, |m| m.min(here))); t = e.right; @@ -497,46 +563,171 @@ impl Eevdf { }) } - /// Service, in whole ns rounded up, until `key`'s current request is done. - pub fn request_left_ns(&self, key: K) -> u64 { - let e = self.get(key); - let left = mul(e.vd - e.v, e.weight as i128) - e.rem as i128; - let ns = (left + (1 << VSHIFT) - 1) >> VSHIFT; - ns.max(1) as u64 - } - - /// End `key`'s current request where its service stands, and issue the - /// next: what a voluntary yield gives up. - pub fn end_request(&mut self, key: K) { - let i = self.index(key); - let slice = self.node(i).slice; - self.reissue(i, fine(slice)); + /// End the entity's current request where its service stands, and issue + /// the next: what a voluntary yield gives up. + pub fn end_request(&mut self, arena: &mut Arena, s: Slot) { + let slice = arena.node(s.0).slice; + self.reissue(arena, s.0, fine(slice)); } /// Change weight, keeping lag and the service left in the current request. - pub fn reweight(&mut self, key: K, weight: u64) { - let slice = self.get(key).slice; - if self.get(key).weight == weight { - return; + pub fn reweight(&mut self, arena: &mut Arena, s: Slot, weight: u64) { + if arena.node(s.0).weight != weight { + self.readmit(arena, s, weight, |left| left); } - let left = self.leave(key); - self.rejoin(key, weight, slice, left); } /// Change the request size; the current request is replaced by one of the /// new size if that ends sooner. - pub fn set_slice(&mut self, key: K, slice: u64) { - let i = self.index(key); - let e = self.node_mut(i); + pub fn set_slice(&mut self, arena: &mut Arena, s: Slot, slice: u64) { + let i = s.0; + let e = arena.node_mut(i); e.slice = slice; let request = mul(e.vd - e.v, e.weight as i128); if fine(slice) < request { - self.reissue(i, fine(slice)); + self.reissue(arena, i, fine(slice)); } } + /// Every entity queued, in no particular order. + pub fn slots<'a, K: Copy, P>(&self, arena: &'a Arena) -> Slots<'a, K, P> { + let mut stack = [NIL; MAX_HEIGHT]; + assert!(usize::from(arena.height(self.root)) < MAX_HEIGHT, "eevdf: a tree deeper than its index allows"); + let depth = usize::from(self.root != NIL); + stack[0] = self.root; + Slots { arena, stack, depth } + } +} + +impl Default for Queue { + fn default() -> Self { + Self::new() + } +} + +/// [`Queue::slots`]: a walk holding at most the right children along one path. +pub struct Slots<'a, K, P> { + arena: &'a Arena, + stack: [u32; MAX_HEIGHT], + depth: usize, +} + +impl Iterator for Slots<'_, K, P> { + type Item = Slot; + + fn next(&mut self) -> Option { + let t = *self.stack.get(self.depth.checked_sub(1)?)?; + self.depth -= 1; + let e = self.arena.node(t); + for child in [e.right, e.left] { + if child != NIL { + self.stack[self.depth] = child; + self.depth += 1; + } + } + Some(Slot(t)) + } +} + +/// One queue whose entities are named by key, with an arena of its own. +pub struct Eevdf { + arena: Arena, + queue: Queue, + slots: BTreeMap, +} + +impl Eevdf { + pub fn new() -> Self { + Self { arena: Arena::new(), queue: Queue::new(), slots: BTreeMap::new() } + } + + fn slot(&self, key: K) -> Slot { + *self.slots.get(&key).expect("eevdf: entity is not queued") + } + + pub fn len(&self) -> usize { + self.queue.len() + } + + pub fn is_empty(&self) -> bool { + self.queue.is_empty() + } + + pub fn contains(&self, key: K) -> bool { + self.slots.contains_key(&key) + } + + pub fn keys(&self) -> impl Iterator + '_ { + self.slots.keys().copied() + } + + pub fn total_weight(&self) -> u128 { + self.queue.total_weight() + } + + pub fn weight(&self, key: K) -> u64 { + self.arena.weight(self.slot(key)) + } + + /// [`Queue::join`], answering the lag the entity really holds afterwards. + pub fn join(&mut self, key: K, weight: u64, slice: u64, lag: Lag) -> Lag { + assert!(!self.contains(key), "eevdf: entity joined twice"); + let s = self.queue.join(&mut self.arena, key, weight, slice, lag, ()); + self.slots.insert(key, s); + self.queue.lag(&self.arena, s) + } + + pub fn leave(&mut self, key: K) -> Left { + let s = self.slots.remove(&key).expect("eevdf: entity is not queued"); + self.queue.leave(&mut self.arena, s).0 + } + + /// [`Queue::rejoin`], answering the lag the entity really holds afterwards. + pub fn rejoin(&mut self, key: K, weight: u64, slice: u64, left: Left) -> Lag { + assert!(!self.contains(key), "eevdf: entity joined twice"); + let s = self.queue.rejoin(&mut self.arena, key, weight, slice, left, ()); + self.slots.insert(key, s); + self.queue.lag(&self.arena, s) + } + + pub fn lag(&self, key: K) -> Lag { + self.queue.lag(&self.arena, self.slot(key)) + } + + pub fn charge(&mut self, key: K, ns: u64) { + let s = self.slot(key); + self.queue.charge(&mut self.arena, s, ns); + } + + pub fn pick(&self) -> Option { + self.queue.pick(&self.arena).map(|s| self.arena.key(s)) + } + + pub fn overtaken_in(&self, running: K) -> Option { + self.queue.overtaken_in(&self.arena, self.slot(running)) + } + + pub fn request_left_ns(&self, key: K) -> u64 { + self.arena.request_left_ns(self.slot(key)) + } + + pub fn end_request(&mut self, key: K) { + let s = self.slot(key); + self.queue.end_request(&mut self.arena, s); + } + + pub fn reweight(&mut self, key: K, weight: u64) { + let s = self.slot(key); + self.queue.reweight(&mut self.arena, s, weight); + } + + pub fn set_slice(&mut self, key: K, slice: u64) { + let s = self.slot(key); + self.queue.set_slice(&mut self.arena, s, slice); + } + pub fn slice(&self, key: K) -> u64 { - self.get(key).slice + self.arena.slice(self.slot(key)) } /// `Σ lag`, which is zero up to the mean's rounding: the conservation the @@ -735,21 +926,21 @@ mod tests { #[test] fn the_tree_answers_as_a_scan_of_every_entity_does() { fn scan_pick(q: &Eevdf) -> Option { - q.order - .iter() - .map(|&i| q.node(i)) - .filter(|e| q.eligible(e.v)) + q.slots + .values() + .map(|s| q.arena.node(s.0)) + .filter(|e| q.queue.eligible(e.v)) .min_by_key(|e| (e.vd, e.seq)) .map(|e| e.key) } fn scan_overtaken(q: &Eevdf, running: u32) -> Option { - let r = q.get(running); - q.order - .iter() - .map(|&i| q.node(i)) + let r = q.arena.node(q.slot(running).0); + q.slots + .values() + .map(|s| q.arena.node(s.0)) .filter(|e| (e.vd, e.seq) < (r.vd, r.seq)) .map(|e| { - let behind = mul(q.total, e.v - q.base) - q.sum; + let behind = mul(q.queue.total, e.v - q.queue.base) - q.queue.sum; ((behind.max(0) + (1 << VSHIFT) - 1) >> VSHIFT) as u64 }) .min() @@ -759,7 +950,7 @@ mod tests { if t == NIL { return (0, i128::MAX); } - let e = q.node(t); + let e = q.arena.node(t); let (hl, ml) = walk(q, e.left, out); out.push((e.vd, e.seq)); let (hr, mr) = walk(q, e.right, out); @@ -809,7 +1000,12 @@ mod tests { } } let mut ordered = Vec::new(); - walk(&q, q.root, &mut ordered); + walk(&q, q.queue.root, &mut ordered); + let mut walked: Vec = q.queue.slots(&q.arena).map(|s| s.0).collect(); + let mut held: Vec = q.slots.values().map(|s| s.0).collect(); + walked.sort_unstable(); + held.sort_unstable(); + assert_eq!(walked, held, "step {step}: the walk is not every queued slot once"); assert!(ordered.windows(2).all(|w| w[0] < w[1]), "step {step}: out of deadline order"); assert_eq!(ordered.len(), q.len()); assert_eq!(q.pick(), scan_pick(&q), "step {step}: pick"); diff --git a/kernel/pure/sched/queue.rs b/kernel/pure/sched/queue.rs index 9782f410126..be8689f08da 100644 --- a/kernel/pure/sched/queue.rs +++ b/kernel/pure/sched/queue.rs @@ -1,8 +1,10 @@ //! The per-CPU run queue: a real-time FIFO drained first, and the fair band. //! //! The fair band is EEVDF over the job groups this CPU holds, and EEVDF again -//! over each group's threads ([`Eevdf`] at both levels). A group's weight is -//! its job's weight times the group's share of the job's runnable threads +//! over each group's threads ([`Queue`] at both levels). Every entity lives in +//! one of this queue's two arenas, so once they have held as many a wake and a +//! block allocate nothing. A group's weight is its job's weight times the +//! group's share of the job's runnable threads //! ([`Job::group_weight`]), refreshed at each decision. Lag goes with whatever //! leaves the queue: a thread's to the next queue it enters, and a dissolving //! group's with its last thread to the group that thread next enters. There @@ -13,7 +15,7 @@ use alloc::collections::{BTreeMap, VecDeque}; -use crate::sched::eevdf::{Eevdf, Lag}; +use crate::sched::eevdf::{Arena, Lag, Queue, Slot}; use crate::sched::hw::Nanos; use crate::sched::job::{Job, JobKey}; use crate::sched::sync::Arc; @@ -35,20 +37,31 @@ pub const LAG_BOUND_NS: u64 = 4_000_000; /// Every thread weighs the same inside its group. const THREAD_WEIGHT: u64 = 1 << 16; -struct Group { +/// A group entity's payload: its job, and the queue of its threads. +struct Group { job: Arc, - threads: Eevdf, - /// Every queued thread's value but the running one's. - ready: BTreeMap>, + threads: Queue, +} + +/// The two entities a queued fair thread holds: its group's and its own. +#[derive(Clone, Copy)] +struct Place { + group: Slot, + thread: Slot, } pub struct RunQueue { rt: VecDeque>, - groups: Eevdf, - members: BTreeMap>, + groups: Queue, + /// Every queued group's entity. + group_slots: Arena, + /// Every queued thread's entity, whichever group's queue it is in, holding + /// its value: `None` for the running thread's. + threads: Arena>>, + members: BTreeMap, /// The running thread, where it is fair: its entity is queued and its /// value is not. - running: Option<(JobKey, TaskKey)>, + running: Option, ready_fair: usize, /// Negative-gate escape hatches only; see /// [`Self::set_lowest_key_within_group`] and [`Self::set_double_charge`]. @@ -62,7 +75,9 @@ impl RunQueue { pub fn new() -> Self { Self { rt: VecDeque::new(), - groups: Eevdf::new(), + groups: Queue::new(), + group_slots: Arena::new(), + threads: Arena::new(), members: BTreeMap::new(), running: None, ready_fair: 0, @@ -109,72 +124,91 @@ impl RunQueue { let standing = task.standing_mut(); let carried = core::mem::take(&mut standing.group_lag); let left = standing.left.take(); - match self.members.get(&jk) { - Some(_) if carried != Lag::ZERO => { - let weight = self.groups.weight(jk); - let mut held = self.groups.leave(jk); - held.lag = held.lag.saturating_add(carried).clamp_ns(LAG_BOUND_NS); - self.groups.rejoin(jk, weight, SLICE_NS, held); + let group = match self.members.get(&jk) { + Some(&group) => { + if carried != Lag::ZERO { + let weight = self.group_slots.weight(group); + self.groups.readmit(&mut self.group_slots, group, weight, |mut held| { + held.lag = held.lag.saturating_add(carried).clamp_ns(LAG_BOUND_NS); + held + }); + } + group } - Some(_) => {} None => { - self.members.insert(jk, Group { job, threads: Eevdf::new(), ready: BTreeMap::new() }); - // Weighed below, once the thread is counted here. - self.groups.join(jk, 1, SLICE_NS, carried); + // At the weight it holds once this thread is counted here. + let weight = job.group_weight(1); + let group = self.groups.join( + &mut self.group_slots, + jk, + weight, + SLICE_NS, + carried, + Group { job, threads: Queue::new() }, + ); + self.members.insert(jk, group); + group } - } - let group = self.members.get_mut(&jk).expect("just made"); + }; + let threads = &mut self.group_slots.payload_mut(group).threads; match left { - Some(left) => group.threads.rejoin(key, THREAD_WEIGHT, SLICE_NS, left), - None => group.threads.join(key, THREAD_WEIGHT, SLICE_NS, Lag::ZERO), + Some(left) => threads.rejoin(&mut self.threads, key, THREAD_WEIGHT, SLICE_NS, left, Some(task)), + None => threads.join(&mut self.threads, key, THREAD_WEIGHT, SLICE_NS, Lag::ZERO, Some(task)), }; - group.ready.insert(key, task); self.ready_fair += 1; - self.reweigh(jk); + self.reweigh(group); } /// Take the entity of a fair thread out, writing what it takes with it - /// into `standing`. The value is the caller's. - fn leave(&mut self, jk: JobKey, key: TaskKey, standing: &mut Standing) { - let group = self.members.get_mut(&jk).expect("a queued thread's group exists"); - let mut left = group.threads.leave(key); + /// into `standing`. Its value is the caller's, already taken. + fn leave(&mut self, at: Place, standing: &mut Standing) { + let group = self.group_slots.payload_mut(at.group); + let (mut left, value) = group.threads.leave(&mut self.threads, at.thread); + assert!(value.is_none(), "a thread left its queue holding its value"); left.lag = left.lag.clamp_ns(LAG_BOUND_NS); standing.left = Some(left); group.job.leave(); if group.threads.is_empty() { - self.members.remove(&jk); - standing.group_lag = self.groups.leave(jk).lag.clamp_ns(LAG_BOUND_NS); + let (left, group) = self.groups.leave(&mut self.group_slots, at.group); + self.members.remove(&group.job.key()); + standing.group_lag = left.lag.clamp_ns(LAG_BOUND_NS); } else { - self.reweigh(jk); + self.reweigh(at.group); } } - /// Hold `jk`'s group at its current weight on this CPU. - fn reweigh(&mut self, jk: JobKey) { - let group = &self.members[&jk]; - let weight = group.job.group_weight(group.threads.len() as u32); - self.groups.reweight(jk, weight); + /// Hold a group at its current weight on this CPU. + fn reweigh(&mut self, group: Slot) { + let g = self.group_slots.payload(group); + let weight = g.job.group_weight(g.threads.len() as u32); + self.groups.reweight(&mut self.group_slots, group, weight); } /// Every group at the weight its job's runnable count now gives it: what /// the job's other CPUs changed since this one last decided. pub fn refresh(&mut self) { - for (&jk, group) in &self.members { - self.groups.reweight(jk, group.job.group_weight(group.threads.len() as u32)); + for &group in self.members.values() { + let g = self.group_slots.payload(group); + let weight = g.job.group_weight(g.threads.len() as u32); + self.groups.reweight(&mut self.group_slots, group, weight); } } /// The fair band's pick: a group, then one of its threads. - fn pick_fair(&mut self) -> Option<(JobKey, TaskKey)> { + fn pick_fair(&mut self) -> Option { self.refresh(); - let jk = self.groups.pick()?; - let group = &self.members[&jk]; + let group = self.groups.pick(&self.group_slots)?; + let threads = &self.group_slots.payload(group).threads; #[cfg(feature = "protocol-port")] if self.lowest_key_within_group { - let key = group.threads.keys().min().expect("a queued group has a thread"); - return Some((jk, key)); + let thread = threads + .slots(&self.threads) + .min_by_key(|&t| self.threads.key(t)) + .expect("a queued group has a thread"); + return Some(Place { group, thread }); } - Some((jk, group.threads.pick().expect("a queued group has a thread"))) + let thread = threads.pick(&self.threads).expect("a queued group has a thread"); + Some(Place { group, thread }) } /// The real-time band first, then the fair band's pick, which becomes the @@ -184,17 +218,17 @@ impl RunQueue { if let Some(task) = self.rt.pop_front() { return Some(task); } - let (jk, key) = self.pick_fair()?; - let task = self.members.get_mut(&jk).and_then(|g| g.ready.remove(&key)).expect("the pick has its value"); + let at = self.pick_fair()?; + let task = self.threads.payload_mut(at.thread).take().expect("the pick has its value"); self.ready_fair -= 1; - self.running = Some((jk, key)); + self.running = Some(at); Some(task) } /// Whether `key`, just queued, is what the fair band would run now in /// place of the running fair thread. pub fn would_preempt_for(&mut self, key: TaskKey) -> bool { - self.running.is_some() && self.pick_fair().is_some_and(|(_, picked)| picked == key) + self.running.is_some() && self.pick_fair().is_some_and(|at| self.threads.key(at.thread) == key) } /// Whether the running thread's entity is the fair band's. @@ -204,13 +238,13 @@ impl RunQueue { /// The running fair thread received `ns`. pub fn charge_running(&mut self, ns: u64) { - if let Some((jk, key)) = self.running { - self.groups.charge(jk, ns); + if let Some(at) = self.running { + self.groups.charge(&mut self.group_slots, at.group, ns); #[cfg(feature = "protocol-port")] - if self.double_charge == Some(jk) { - self.groups.charge(jk, ns); + if self.double_charge == Some(self.group_slots.key(at.group)) { + self.groups.charge(&mut self.group_slots, at.group, ns); } - self.members.get_mut(&jk).expect("the running thread's group").threads.charge(key, ns); + self.group_slots.payload_mut(at.group).threads.charge(&mut self.threads, at.thread, ns); } } @@ -220,17 +254,17 @@ impl RunQueue { /// after them runs first. pub fn put_back(&mut self, mut task: ReadyTask, yielded: bool) { match self.running.take() { - Some((jk, key)) => { - assert_eq!(key, task.key(), "the task put back is not the running one"); + Some(at) => { + assert_eq!(self.threads.key(at.thread), task.key(), "the task put back is not the running one"); if task.rt().is_rt() { - self.leave(jk, key, task.standing_mut()); + self.leave(at, task.standing_mut()); self.rt.push_back(task); } else { if yielded { - self.groups.end_request(jk); - self.members.get_mut(&jk).expect("the running thread's group").threads.end_request(key); + self.groups.end_request(&mut self.group_slots, at.group); + self.group_slots.payload_mut(at.group).threads.end_request(&mut self.threads, at.thread); } - self.members.get_mut(&jk).expect("the running thread's group").ready.insert(key, task); + *self.threads.payload_mut(at.thread) = Some(task); self.ready_fair += 1; } } @@ -241,9 +275,9 @@ impl RunQueue { /// The running task leaves the fair band without going back to it — it /// blocks, exits, stops or dies — taking its standing into `standing`. pub fn leave_running(&mut self, key: TaskKey, standing: &mut Standing) { - if let Some((jk, running)) = self.running.take() { - assert_eq!(running, key, "the task leaving is not the running one"); - self.leave(jk, key, standing); + if let Some(at) = self.running.take() { + assert_eq!(self.threads.key(at.thread), key, "the task leaving is not the running one"); + self.leave(at, standing); } } @@ -255,14 +289,30 @@ impl RunQueue { /// Within a group no such instant is needed: the group's request is as /// short as its threads', and deciding sooner only switches siblings. pub fn decision(&self, now: Nanos, since: Nanos) -> Option { - let (jk, key) = self.running?; - let threads = &self.members[&jk].threads; - let left = self.groups.request_left_ns(jk).min(threads.request_left_ns(key)); - let mut at = now.after(left); - if let Some(ns) = self.groups.overtaken_in(jk) { - at = at.min(now.after(ns).max(since.after(MIN_SLICE_NS))); + let at = self.running?; + let left = self.group_slots.request_left_ns(at.group).min(self.threads.request_left_ns(at.thread)); + let mut decide = now.after(left); + if let Some(ns) = self.groups.overtaken_in(&self.group_slots, at.group) { + decide = decide.min(now.after(ns).max(since.after(MIN_SLICE_NS))); } - Some(at) + Some(decide) + } + + /// Every queued fair thread whose value is here, with the group it is in. + fn ready(&self) -> impl Iterator)> + '_ { + self.members.values().flat_map(move |&group| { + self.group_slots.payload(group).threads.slots(&self.threads).filter_map(move |thread| { + self.threads.payload(thread).as_ref().map(|task| (Place { group, thread }, task)) + }) + }) + } + + /// Take a ready thread's value and its entity out of the band. + fn take(&mut self, at: Place) -> ReadyTask { + let mut task = self.threads.payload_mut(at.thread).take().expect("a ready task"); + self.ready_fair -= 1; + self.leave(at, task.standing_mut()); + task } /// The ready fair task a balance request is best answered with, out of @@ -271,24 +321,27 @@ impl RunQueue { /// this CPU still stands on (`SchedPass::answer_steal_requests`). An idle /// asker is answered with that task whatever the move does; a busy one only /// if the move narrows the difference by a quarter of the thread's weight, - /// so that floor rounding cannot hand an odd split back and forth. + /// so that floor rounding cannot hand an odd split back and forth. Each + /// group offers its newest ready thread. pub fn pop_surplus(&mut self, loaded: Option, asking: u64) -> Option> { self.refresh(); let diff = self.weight() as i128 - asking as i128; - let (jk, key) = self + let (_, _, _, at) = self .members .iter() - .filter_map(|(&jk, group)| { - let key = group.ready.keys().rev().copied().find(|&key| Some(key) != loaded)?; - let w = group.job.thread_weight() as i128; - (asking == 0 || w + w / 4 <= diff).then_some(((diff - 2 * w).abs(), jk, key)) + .filter_map(|(&jk, &group)| { + let g = self.group_slots.payload(group); + let (key, thread) = g + .threads + .slots(&self.threads) + .filter_map(|t| self.threads.payload(t).as_ref().map(|task| (task.key(), t))) + .filter(|&(key, _)| Some(key) != loaded) + .max_by_key(|&(key, _)| key)?; + let w = g.job.thread_weight() as i128; + (asking == 0 || w + w / 4 <= diff).then_some(((diff - 2 * w).abs(), jk, key, Place { group, thread })) }) - .min() - .map(|(_, jk, key)| (jk, key))?; - let mut task = self.members.get_mut(&jk).and_then(|g| g.ready.remove(&key)).expect("a ready task"); - self.ready_fair -= 1; - self.leave(jk, key, task.standing_mut()); - Some(task) + .min_by_key(|&(score, jk, key, _)| (score, jk, key))?; + Some(self.take(at)) } /// Retire found the task queued rather than parked: out of whichever band @@ -297,11 +350,8 @@ impl RunQueue { if let Some(index) = self.rt.iter().position(|t| t.key() == key) { return self.rt.remove(index); } - let jk = *self.members.iter().find(|(_, g)| g.ready.contains_key(&key))?.0; - let mut task = self.members.get_mut(&jk).and_then(|g| g.ready.remove(&key)).expect("just found"); - self.ready_fair -= 1; - self.leave(jk, key, task.standing_mut()); - Some(task) + let at = self.ready().find(|(_, task)| task.key() == key)?.0; + Some(self.take(at)) } /// Is an RT task waiting? The preemption decision in `finish()` and @@ -322,7 +372,7 @@ impl RunQueue { /// Is `key` queued in the fair band? pub fn holds_fair(&self, key: TaskKey) -> bool { - self.members.values().any(|g| g.ready.contains_key(&key)) + self.ready().any(|(_, task)| task.key() == key) } pub fn is_empty(&self) -> bool { @@ -337,7 +387,9 @@ impl RunQueue { /// Each job's group here: its key, its weight and how many of its threads /// it holds, the running one included. pub fn groups(&self) -> impl Iterator + '_ { - self.members.iter().map(|(&jk, g)| (jk, self.groups.weight(jk), g.threads.len())) + self.members + .iter() + .map(|(&jk, &group)| (jk, self.group_slots.weight(group), self.group_slots.payload(group).threads.len())) } /// Residents, for the invariant walks: the real-time band, then each @@ -347,7 +399,7 @@ impl RunQueue { } pub fn tasks(&self) -> impl Iterator> + '_ { - self.rt.iter().chain(self.members.values().flat_map(|g| g.ready.values())) + self.rt.iter().chain(self.ready().map(|(_, task)| task)) } } From 416cc97841fb7b5d6e0aefd789badf87a9e91064 Mon Sep 17 00:00:00 2001 From: japabu Date: Sat, 24 Oct 2026 07:32:46 +0200 Subject: [PATCH 2/7] sched: an idle pass reads one offering mask, and a CPU's deadlines are kept in order Every idle entry chose a steal victim twice, before and after publishing SLEEPING, by reading the handle of every CPU in four topology rings: lines every other CPU writes each pass. CpuHandles now keeps a bit per CPU whose published surplus reaches PUSH_THRESHOLD, set and cleared by that CPU's own publish_surplus before its pass's balance fence, so a machine with nothing to steal costs a probe one load. The bit decides eligibility and the surplus only ranks candidates, so the push's store-buffer pair is the mask's: loom_push now drives publish_surplus and offers_surplus, and still reds under push-fence-relaxed. The timer arming and the deadline firing each scanned every parked task per pass. Parked keeps the parked entries and, beside them, the (deadline, key) of every entry that has one; only its methods write either. Co-Authored-By: Claude Opus 5.5 --- ...-has-no-migration-cost-or-wake-affinity.md | 10 +- kernel/loom/tests/loom_push.rs | 64 +++---- kernel/pure/sched/cpu.rs | 161 +++++++++++++----- kernel/sim/src/vm.rs | 2 +- 4 files changed, 160 insertions(+), 77 deletions(-) diff --git a/issues/placement-has-no-migration-cost-or-wake-affinity.md b/issues/placement-has-no-migration-cost-or-wake-affinity.md index d44355760e7..a4ae53cb4e0 100644 --- a/issues/placement-has-no-migration-cost-or-wake-affinity.md +++ b/issues/placement-has-no-migration-cost-or-wake-affinity.md @@ -21,10 +21,12 @@ lacks: waker (shared data stays warm — pipe/IPC pairs) and away from it (parallelism). Every wake still ignores the waker. - **An idle-state input**: see `issues/the-scheduler-and-the-clocks-never-talk.md`. -- **Bounded scans.** Each choice walks the rings it needs whole, so an idle - CPU's steal probe on a machine with nothing to steal reads every CPU's - handle, a cost on the idle path that grows with the CPU count. Derived from - the code, not measured; both benches have at most 16 CPUs. +- **Bounded scans.** Each choice walks the rings it needs whole. A steal + probe reads the machine's offering mask first, so a machine with nothing to + steal costs an idle pass one line; once a CPU offers surplus, the probe and + every wake forward still read every handle in their rings, a cost that grows + with the CPU count. Derived from the code, not measured; both benches have + at most 16 CPUs. Composes with the reservation model (placement decides *where*, reservations decide *how much*) and with the scheduler simulator (`kernel/sim`), which runs diff --git a/kernel/loom/tests/loom_push.rs b/kernel/loom/tests/loom_push.rs index 211cd57c5a8..0f2d2707700 100644 --- a/kernel/loom/tests/loom_push.rs +++ b/kernel/loom/tests/loom_push.rs @@ -12,9 +12,9 @@ //! ```text //! pusher sleeper //! ------ ------- -//! surplus.store(2) doorbell.fetch_or(SLEEPING) +//! offering.fetch_or(bit) doorbell.fetch_or(SLEEPING) //! balance_fence() balance_fence() -//! doorbell.load() -> asleep? surplus.load() -> anything to steal? +//! doorbell.load() -> asleep? offering.load() -> anything to steal? //! ``` //! //! Each side stores to one location and loads the other, and with anything @@ -47,9 +47,10 @@ use loom::sync::atomic::{AtomicUsize, Ordering}; use loom::sync::Arc; -use kernel_loom::sched::cpu::{balance_fence, CpuHandle}; +use kernel_loom::sched::cpu::{balance_fence, CpuHandle, CpuHandles}; use kernel_loom::sched::hw::CpuId; use kernel_loom::sched::mailbox::{mailbox, Kick, MailboxConsumer}; +use kernel_loom::sched::topology::Topology; use kernel_loom::sched_model::{model, Msg}; /// The surplus at which a pass pushes and at which an idle pass probes — one @@ -57,14 +58,12 @@ use kernel_loom::sched_model::{model, Msg}; /// stages the shipped threshold. const THRESHOLD: u32 = kernel_loom::sched::cpu::PUSH_THRESHOLD; -/// Two CPUs' shared faces: the one that gains surplus, and the one going to -/// sleep. The `CpuHandle` is the real one, so `publish_surplus`, `surplus`, -/// `doorbell` and `poke` are the shipped code and not a restatement of it. +/// Two CPUs' shared faces: [`VICTIM`], which gains surplus, and [`THIEF`], +/// which is going to sleep. The `CpuHandles` are the real ones, so +/// `publish_surplus`, `offers_surplus`, `doorbell` and `poke` are the shipped +/// code and not a restatement of it. struct World { - /// The CPU that gains surplus. Only its `surplus` matters here. - victim: CpuHandle, - /// The CPU going to sleep. Only its doorbell matters here. - thief: CpuHandle, + cpus: CpuHandles, /// Targeted IPIs the push sent. As in `loom_sleep.rs`, an IPI's only /// relevant effect is "the target cannot stay halted". ipis: AtomicUsize, @@ -74,15 +73,30 @@ struct World { pokes: AtomicUsize, } +const VICTIM: CpuId = CpuId(0); +const THIEF: CpuId = CpuId(1); + impl World { + fn new(victim: CpuHandle, thief: CpuHandle) -> Self { + Self { + cpus: CpuHandles::new(vec![victim, thief], Topology::flat(2)), + ipis: AtomicUsize::new(0), + pokes: AtomicUsize::new(0), + } + } + + fn thief(&self) -> &CpuHandle { + self.cpus.get(THIEF) + } + /// `SchedPass::push_on_surplus`, as the pass runs it: publish, fence, look /// for a sleeper, ring it. fn publish_and_push(&self) { - self.victim.publish_surplus(THRESHOLD); + self.cpus.publish_surplus(VICTIM, THRESHOLD); balance_fence(); - if self.thief.doorbell().sleeping() { + if self.thief().doorbell().sleeping() { self.pokes.fetch_add(1, Ordering::AcqRel); - if self.thief.poke() == Kick::Send { + if self.thief().poke() == Kick::Send { self.ipis.fetch_add(1, Ordering::AcqRel); } } @@ -96,13 +110,13 @@ impl World { /// Returns `true` if the CPU halted **without** having seen the surplus — /// the state the push has to make impossible without a poke behind it. fn try_sleep(&self, rx: &MailboxConsumer) -> bool { - self.thief.doorbell().begin_pass(); + self.thief().doorbell().begin_pass(); // `post_steal_probe`'s read, before SLEEPING is published. A stale // answer here is exactly what the re-read below exists to catch. - let probed = self.victim.surplus() >= THRESHOLD; - let arm = self.thief.doorbell().arm_sleep(); + let probed = self.cpus.offers_surplus(VICTIM); + let arm = self.thief().doorbell().arm_sleep(); balance_fence(); - if self.victim.surplus() >= THRESHOLD { + if self.cpus.offers_surplus(VICTIM) { // Observed: the pass goes round again and posts a probe. arm.abandon(); return false; @@ -126,12 +140,7 @@ fn a_cpu_that_halts_without_seeing_the_surplus_was_pushed() { model(|| { let (victim_tx, _victim_rx) = mailbox::(); let (thief_tx, thief_rx) = mailbox::(); - let world = Arc::new(World { - victim: CpuHandle::new(CpuId(0), victim_tx), - thief: CpuHandle::new(CpuId(1), thief_tx), - ipis: AtomicUsize::new(0), - pokes: AtomicUsize::new(0), - }); + let world = Arc::new(World::new(CpuHandle::new(VICTIM, victim_tx), CpuHandle::new(THIEF, thief_tx))); let pusher = { let world = world.clone(); @@ -163,18 +172,13 @@ fn a_push_rings_nobody_when_nobody_is_asleep() { model(|| { let (victim_tx, _victim_rx) = mailbox::(); let (thief_tx, _thief_rx) = mailbox::(); - let world = Arc::new(World { - victim: CpuHandle::new(CpuId(0), victim_tx), - thief: CpuHandle::new(CpuId(1), thief_tx), - ipis: AtomicUsize::new(0), - pokes: AtomicUsize::new(0), - }); + let world = Arc::new(World::new(CpuHandle::new(VICTIM, victim_tx), CpuHandle::new(THIEF, thief_tx))); // The target runs a pass and stays awake: `begin_pass` clears SLEEPING // and nothing arms one. let target = { let world = world.clone(); - loom::thread::spawn(move || world.thief.doorbell().begin_pass()) + loom::thread::spawn(move || world.thief().doorbell().begin_pass()) }; world.publish_and_push(); target.join().unwrap(); diff --git a/kernel/pure/sched/cpu.rs b/kernel/pure/sched/cpu.rs index 0f4b078bdc3..cef0b980a07 100644 --- a/kernel/pure/sched/cpu.rs +++ b/kernel/pure/sched/cpu.rs @@ -19,7 +19,7 @@ //! to let another CPU restore it ([`SchedPass::answer_steal_requests`]). use alloc::boxed::Box; -use alloc::collections::{BTreeMap, VecDeque}; +use alloc::collections::{BTreeMap, BTreeSet, VecDeque}; use alloc::vec::Vec; use core::marker::PhantomData; @@ -118,16 +118,65 @@ pub enum Action { /// A parked task, plus the two facts that are only meaningful while parked. /// -/// The deadline lives *here* and nowhere else, so a task that is not parked -/// structurally cannot have one, and no second copy can disagree with this one -/// about what the CPU owes. A `since` field here is omitted for the same -/// reason: the residency stamp is in the task record. +/// The deadline lives *here*, so a task that is not parked structurally +/// cannot have one; [`Parked`]'s order of deadlines is written only beside it. +/// A `since` field here is omitted for the same reason: the residency stamp is +/// in the task record. pub struct ParkedEntry { task: BlockedTask, deadline: Option, class: WaitClass, } +/// The tasks parked on a CPU, and their deadlines in the order they fall due: +/// what the CPU owes first is one lookup, however many tasks it holds. Only +/// these methods write either, so the order cannot disagree with an entry. +struct Parked { + entries: BTreeMap>, + due: BTreeSet<(Nanos, TaskKey)>, +} + +impl Parked { + fn new() -> Self { + Self { entries: BTreeMap::new(), due: BTreeSet::new() } + } + + fn insert(&mut self, key: TaskKey, entry: ParkedEntry) { + if let Some(at) = entry.deadline { + self.due.insert((at, key)); + } + assert!(self.entries.insert(key, entry).is_none(), "a task parked twice"); + } + + fn remove(&mut self, key: TaskKey) -> Option> { + let entry = self.entries.remove(&key)?; + if let Some(at) = entry.deadline { + self.due.remove(&(at, key)); + } + Some(entry) + } + + fn get(&self, key: TaskKey) -> Option<&ParkedEntry> { + self.entries.get(&key) + } + + /// The deadline that will not be honoured stops being owed. + fn forget_deadline(&mut self, key: TaskKey) { + if let Some(at) = self.entries.get_mut(&key).and_then(|entry| entry.deadline.take()) { + self.due.remove(&(at, key)); + } + } + + fn earliest(&self) -> Option { + self.due.first().map(|&(at, _)| at) + } + + /// The task whose deadline falls due first, if that is by `now`. + fn next_due(&self, now: Nanos) -> Option { + self.due.first().filter(|&&(at, _)| at <= now).map(|&(_, key)| key) + } +} + /// One parked task as an outside reader sees it. pub struct ParkedView<'a, X: SchedPayload> { key: TaskKey, @@ -179,7 +228,7 @@ pub struct CpuSched { id: CpuId, running: Option>, rq: RunQueue, - parked: BTreeMap>, + parked: Parked, /// Killed tasks that still have a kernel stack to unwind, dispatched /// ahead of the **fair** queue, and behind the RT band only until the head /// of this list has aged ([`DYING_AGE_NS`]). @@ -334,7 +383,7 @@ impl CpuSched { id, running: None, rq: RunQueue::new(), - parked: BTreeMap::new(), + parked: Parked::new(), dying: VecDeque::new(), stopped: VecDeque::new(), zombie: None, @@ -383,11 +432,11 @@ impl CpuSched { } pub fn parked(&self) -> impl Iterator> + '_ { - self.parked.iter().map(|(key, entry)| ParkedView { key: *key, entry }) + self.parked.entries.iter().map(|(key, entry)| ParkedView { key: *key, entry }) } pub fn parked_task(&self, key: TaskKey) -> Option<&BlockedTask> { - self.parked.get(&key).map(|e| &e.task) + self.parked.get(key).map(|e| &e.task) } /// The killed tasks waiting to unwind, for the invariant walks and for a @@ -1037,7 +1086,7 @@ impl CpuSched { env: Env<'_, H, P>, now: Nanos, ) { - let Some(entry) = self.parked.remove(&key) else { + let Some(entry) = self.parked.remove(key) else { // Not parked here any more: a `Retire` already woke it into the // dying list, or its deadline fired first and this wake lost the // arbitration CAS. Keys are never reused, so a stale wake is @@ -1072,18 +1121,17 @@ impl CpuSched { now: Nanos, ) { let key = shared.key(); - if self.parked.contains_key(&key) { + if let Some(entry) = self.parked.get(key) { // **Claim-arbitrated, exactly as `fire_deadlines` is**: // remove-then-convert loses the race. If a remote waker has // already claimed this task its `Msg::Wake` is in flight to this // same CPU, so leaving the entry alone is what keeps the task in // *some* container — `handle_wake` finds it and the wake places // it, into the dying list, because the kill bit is already set. - let entry = self.parked.get_mut(&key).expect("just checked"); match entry.task.shared().claim_wake() { Claim::Parked(cpu) => { assert_eq!(cpu, self.id, "a task parked here claims another CPU"); - let entry = self.parked.remove(&key).expect("still there"); + let entry = self.parked.remove(key).expect("still there"); let task = entry.task.wake( self.id, WakeCause::new(WakeReason::Woken), @@ -1177,14 +1225,7 @@ impl CpuSched { /// long*: a CPU that owes a wake is the wrong one to run unbounded I/O on, /// and the idle loop asks before it flushes (`sched::driver::owes_deadline`). pub fn earliest_deadline(&self) -> Option { - self.parked.values().filter_map(|entry| entry.deadline).min() - } - - fn next_due(&self, now: Nanos) -> Option { - self.parked - .iter() - .find(|(_, entry)| entry.deadline.is_some_and(|at| at <= now)) - .map(|(key, _)| *key) + self.parked.earliest() } /// Fire every deadline that is due, arbitrating with remote wakers @@ -1194,8 +1235,8 @@ impl CpuSched { env: Env<'_, H, P>, now: Nanos, ) { - while let Some(key) = self.next_due(now) { - let entry = self.parked.get_mut(&key).expect("just found"); + while let Some(key) = self.parked.next_due(now) { + let entry = self.parked.get(key).expect("just found"); match entry.task.shared().claim_wake() { Claim::Parked(cpu) => { assert_eq!(cpu, self.id, "a task parked here claims another CPU") @@ -1207,11 +1248,11 @@ impl CpuSched { // also what advances the loop — the deadline that will not be // honoured and the deadline that is not reported are one field. Claim::Lost => { - entry.deadline = None; + self.parked.forget_deadline(key); continue; } } - let entry = self.parked.remove(&key).expect("still there"); + let entry = self.parked.remove(key).expect("still there"); let task = entry.task.wake( self.id, WakeCause::new(WakeReason::Timeout), @@ -1695,7 +1736,7 @@ impl SchedPass<'_, '_, H, P, Disposed> { let handle = self.env.cpus.get(self.cpu.id); let surplus = self.cpu.rq.fair_len() as u32; handle.publish_load((self.cpu.rq.len() + self.cpu.dying.len()) as u32); - handle.publish_surplus(surplus); + self.env.cpus.publish_surplus(self.cpu.id, surplus); self.cpu.rq.refresh(); handle.publish_weight(self.cpu.rq.weight()); // The stamp goes with them: it is what says they are about now. @@ -1955,11 +1996,14 @@ impl SchedPass<'_, '_, H, P, Disposed> { } self.cpu.next_balance = self.now.after(BALANCE_NS); let (cpus, me, now) = (self.env.cpus, self.cpu.id, self.now); + if !cpus.any_offers_surplus() { + return; + } let mine = self.cpu.rq.weight(); let heaviest = STEAL_RINGS .into_iter() .flat_map(|(inner, outer)| cpus.topology().ring(me, inner, outer, 0)) - .filter(|&cpu| cpus.get(cpu).answering(now) && cpus.get(cpu).surplus() >= PUSH_THRESHOLD) + .filter(|&cpu| cpus.offers_surplus(cpu) && cpus.get(cpu).answering(now)) .map(|cpu| (cpus.get(cpu).weight(), cpu)) .fold(None, |best: Option<(u64, CpuId)>, (weight, cpu)| match best { Some((most, _)) if most >= weight => best, @@ -2183,16 +2227,16 @@ impl SchedPass<'_, '_, H, P, Disposed> { /// nearer victim moves the same work for less cache refill. fn best_victim(&self) -> Option { let cpus = self.env.cpus; + if !cpus.any_offers_surplus() { + return None; + } let me = self.cpu.id; STEAL_RINGS.into_iter().find_map(|(inner, outer)| { - let (victim, surplus) = cpus - .topology() - .ring(me, inner, outer, 0) - .filter(|&cpu| cpus.get(cpu).answering(self.now)) - .map(|cpu| (cpu, cpus.get(cpu).surplus())) - .max_by_key(|&(_, surplus)| surplus)?; - (surplus >= PUSH_THRESHOLD).then_some(victim) - }) + cpus.topology() + .ring(me, inner, outer, 0) + .filter(|&cpu| cpus.offers_surplus(cpu) && cpus.get(cpu).answering(self.now)) + .max_by_key(|&cpu| cpus.get(cpu).surplus()) + }) } /// [`Balance::PushOnSurplus`]: this pass has just published `surplus`, so @@ -2242,7 +2286,7 @@ pub struct CpuHandle { /// the fair band, so a task placed here waits for it. load: AtomicU32, /// **How many fair-band tasks this CPU could give away**, published for the - /// steal probe and for nothing else. + /// steal probe and for nothing else, through [`CpuHandles::publish_surplus`]. /// /// A separate number from [`Self::load`] because the two questions have /// different answers, and asking one with the other's is a defect: @@ -2316,10 +2360,6 @@ impl CpuHandle { self.surplus.load(Ordering::Relaxed) } - pub fn publish_surplus(&self, fair: u32) { - self.surplus.store(fair, Ordering::Relaxed); - } - pub fn weight(&self) -> u64 { self.weight.load(Ordering::Relaxed) } @@ -2407,6 +2447,11 @@ impl CpuHandle { pub struct CpuHandles { handles: Box<[CpuHandle]>, topology: Topology, + /// A bit per CPU whose published surplus reaches [`PUSH_THRESHOLD`], set + /// and cleared by its own [`Self::publish_surplus`]: what a choice of a + /// victim reads first, so a machine with nothing to steal costs an idle + /// pass one line instead of every CPU's handle. + offering: Box<[AtomicU64]>, } /// The CPUs an idle CPU is chosen among, nearest first: `me`'s LLC, then the @@ -2434,12 +2479,44 @@ impl CpuHandles { ); } assert_eq!(topology.cpus(), handles.len(), "the topology describes every cpu and no other"); + let offering = (0..handles.len().div_ceil(64)).map(|_| AtomicU64::new(0)).collect(); Self { handles: handles.into_boxed_slice(), topology, + offering, + } + } + + /// Publish how many fair-band tasks `cpu` could give away; called by that + /// CPU's own pass. Its bit in the offering mask changes with the threshold, + /// before the pass's [`balance_fence`], so the mask is what the push's + /// sleeper half reads. + pub fn publish_surplus(&self, cpu: CpuId, fair: u32) { + let handle = self.get(cpu); + let was = handle.surplus() >= PUSH_THRESHOLD; + handle.surplus.store(fair, Ordering::Relaxed); + let (word, bit) = (&self.offering[cpu.0 as usize / 64], 1u64 << (cpu.0 % 64)); + match (was, fair >= PUSH_THRESHOLD) { + (false, true) => { + word.fetch_or(bit, Ordering::Relaxed); + } + (true, false) => { + word.fetch_and(!bit, Ordering::Relaxed); + } + _ => {} } } + /// Whether `cpu` publishes a surplus a probe would be answered from. + pub fn offers_surplus(&self, cpu: CpuId) -> bool { + self.offering[cpu.0 as usize / 64].load(Ordering::Relaxed) & 1u64 << (cpu.0 % 64) != 0 + } + + /// Whether any CPU does. + fn any_offers_surplus(&self) -> bool { + self.offering.iter().any(|word| word.load(Ordering::Relaxed) != 0) + } + pub fn get(&self, cpu: CpuId) -> &CpuHandle { self.handles .get(cpu.0 as usize) @@ -4258,8 +4335,8 @@ mod tests { for cpu in 0..8 { w.handles.get(CpuId(cpu)).publish_pass(NOW); } - w.handles.get(CpuId(near)).publish_surplus(1); - w.handles.get(CpuId(1)).publish_surplus(5); + w.handles.publish_surplus(CpuId(near), 1); + w.handles.publish_surplus(CpuId(1), 5); w.run_a_pass(C0); let asked: Vec = (0..8u32).filter(|&cpu| !w.cpus[cpu as usize].mailbox_is_empty()).collect(); assert_eq!(asked, [victim], "the probe's victim"); diff --git a/kernel/sim/src/vm.rs b/kernel/sim/src/vm.rs index 367c2b0bef2..17b0cf3a89d 100644 --- a/kernel/sim/src/vm.rs +++ b/kernel/sim/src/vm.rs @@ -1435,7 +1435,7 @@ impl<'q> Vm<'q> { self.apply(action); self.hw.leave_pass(); if self.scenario.surplus == SurplusShape::Withheld { - self.handles.get(CpuId(cpu as u32)).publish_surplus(0); + self.handles.publish_surplus(CpuId(cpu as u32), 0); } self.note_kicks(kicks_before); self.note_pass(cpu); From b214ecb87e712abaea0cad0a81285ae0ae268196 Mon Sep 17 00:00:00 2001 From: japabu Date: Sat, 24 Oct 2026 07:33:02 +0200 Subject: [PATCH 3/7] kernel: take the global locks, extra passes and repeated lookups off a round trip Measured by instruction counts on TCG; each item was on every wake, block, syscall or pipe operation. - Pipes own themselves. Every pipe and connection operation on every CPU took the one PIPES lock and searched its map, and the user copy ran under it. A pipe is now an Arc its end handles, objects and waiters hold (PipeRef), with its own lock; the last holder of either end frees the ring page as before. - A syscall's profile is its thread's own. Each syscall took PROCESS_TABLE twice, searched it and locked its ThreadData to count itself, and read the clock through a 128-bit division twice. The counts now live on the thread's TaskHandle, written by that thread with a load and a store, and the time is summed in counter ticks and converted when read. - One tick conversion, without a 128-bit division: toyos_abi::clock's ticks_to_nanos splits the count at 10^6, which is exact for any tick under 18 ms and is the formula the kernel and every clock-page reader share. - A kick owes a pass only when the doorbell was rung since the last pass began. A kick that lands after the pass it was sent for, or one that only asks for a counters round, no longer reschedules. The machine-wide stop and the deadline wedge ring every doorbell, which is what asks a pass. - The LAPIC timer's divide and LVT are programmed once per CPU; an arm writes the count alone, converted without a 128-bit division. - User memory: a buffer's coverage and permissions are one descent of the inventory instead of four range queries, reach finds its range once, and an untimed futex wait no longer re-reads its word after the wait that already did. Co-Authored-By: Claude Opus 5.5 --- kernel/pure/clock.rs | 5 +- kernel/pure/vm.rs | 27 ++- kernel/src/arch/aarch64/trap.rs | 2 +- kernel/src/arch/x86_64/apic.rs | 31 ++- kernel/src/arch/x86_64/idt/kick.rs | 4 +- kernel/src/deadline.rs | 2 +- kernel/src/id_map.rs | 10 - kernel/src/loader/mod.rs | 3 - kernel/src/main.rs | 1 - kernel/src/mm/space.rs | 18 +- kernel/src/object/ops.rs | 49 ++-- kernel/src/object/pipe.rs | 27 +-- kernel/src/object/service.rs | 42 ++-- kernel/src/pipe.rs | 372 +++++++++++++---------------- kernel/src/process.rs | 58 ++--- kernel/src/quiesce.rs | 2 +- kernel/src/sched/driver.rs | 26 ++ kernel/src/sched/kthread.rs | 3 - kernel/src/sched/payload.rs | 48 ++++ kernel/src/scheduler.rs | 22 +- kernel/src/syscall/dispatch.rs | 14 +- kernel/src/syscall/io.rs | 75 +++--- kernel/src/syscall/ipc.rs | 9 +- sdk/abi/src/clock.rs | 38 ++- 24 files changed, 456 insertions(+), 432 deletions(-) diff --git a/kernel/pure/clock.rs b/kernel/pure/clock.rs index 76a991c373c..38dc0e5b9bd 100644 --- a/kernel/pure/clock.rs +++ b/kernel/pure/clock.rs @@ -6,10 +6,7 @@ const NANOS_PER_SEC: u64 = 1_000_000_000; -/// `ticks` of a counter whose tick is `period_fs` femtoseconds, in nanoseconds. -pub const fn ticks_to_nanos(ticks: u64, period_fs: u64) -> u64 { - ((ticks as u128 * period_fs as u128) / 1_000_000) as u64 -} +pub use toyos_abi::clock::ticks_to_nanos; /// Unix seconds at the counter's `boot`, from a reading of `secs` true when the /// counter read `at`, which is after boot where the kernel read the clock and diff --git a/kernel/pure/vm.rs b/kernel/pure/vm.rs index 51b33c6e74e..79fa1b181f7 100644 --- a/kernel/pure/vm.rs +++ b/kernel/pure/vm.rs @@ -214,14 +214,23 @@ impl Inventory { /// Whether entries cover every page of `[start, end)`. pub fn covered(&self, start: u64, end: u64) -> bool { - let mut at = start; - for (s, entry) in self.meeting(start, end) { - if s > at { + self.covered_where(start, end, |_| true) + } + + /// Whether entries cover every page of `[start, end)` and each that meets + /// it answers `ok`: one descent of the tree, walking down from `end`. + pub fn covered_where(&self, start: u64, end: u64, ok: impl Fn(&Entry) -> bool) -> bool { + // `[low, end)` is covered by entries that answered `ok`. + let mut low = end; + let mut below = self.entries.range(..end).rev(); + while low > start { + let Some((&s, entry)) = below.next() else { return false }; + if entry.end < low || !ok(entry) { return false; } - at = entry.end; + low = s; } - at >= end + true } /// Whether any entry meeting `[start, end)` is backed. @@ -561,6 +570,14 @@ mod tests { } } } + // Coverage of the round's range, and of its writable pages, as the map says. + let pages = &map[page(start)..page(end)]; + assert_eq!(v.covered(start, end), pages.iter().all(Option::is_some), "round {round}: covered"); + assert_eq!( + v.covered_where(start, end, |e| e.prot == Prot::ReadWrite), + pages.iter().all(|p| p.is_some_and(|(prot, _)| prot == Prot::ReadWrite)), + "round {round}: covered by writable entries" + ); // Every page as the map says, and no two neighbours that could join. for p in 0..PAGES { let at = FLOOR + p * PAGE; diff --git a/kernel/src/arch/aarch64/trap.rs b/kernel/src/arch/aarch64/trap.rs index ec0aa08bb67..6400a0a2512 100644 --- a/kernel/src/arch/aarch64/trap.rs +++ b/kernel/src/arch/aarch64/trap.rs @@ -215,7 +215,7 @@ fn irq(frame: &Frame, from_el0: bool) -> bool { irqchip::SGI_KICK => { percpu::irq_took(Source::Kick); crate::counters::serve_here(); - true + crate::sched::driver::pass_owed() } #[cfg(feature = "boot-actuators")] irqchip::SGI_STORM => { diff --git a/kernel/src/arch/x86_64/apic.rs b/kernel/src/arch/x86_64/apic.rs index a6b8e5b6ca4..44a40be68d9 100644 --- a/kernel/src/arch/x86_64/apic.rs +++ b/kernel/src/arch/x86_64/apic.rs @@ -136,9 +136,20 @@ fn enable(cpu_id: u32) -> ApicMode { Reg::SVR.write(SVR); let held = Reg::SVR.read(); assert!(held == SVR, "LAPIC: SVR reads {held:#x} after {SVR:#x} was written"); + timer_one_shot(); mode } +/// The timer counting at the bus clock, one-shot on [`TIMER_VECTOR`] and +/// stopped: what every arm after it writes only the count into. The LVT resets +/// masked and nothing here disables the APIC again, so once per CPU holds it. +fn timer_one_shot() { + // Divide by 1 for maximum resolution. + Reg::TIMER_DIVIDE.write(0b1011); + Reg::TIMER_INIT.write(0); + Reg::LVT_TIMER.write(u32::from(TIMER_VECTOR)); +} + /// Enable the BSP's local APIC. pub fn init() { let mode = enable(0); @@ -266,7 +277,7 @@ pub fn kick_cpu(cpu_id: u32) { send(Some(apic_id), 0x4000 | u32::from(super::idt::KICK_VECTOR)); } -// Kicked, and not left to arrive on their own: a CPU halted in the idle path has stopped its own timer, so nothing else brings it to the next scheduler pass. +// Kicked, and not left to arrive on their own: a CPU halted in the idle path has stopped its own timer, so nothing else brings it to the counters round it is asked to serve. pub fn kick_all_but_self() { let me = percpu::cpu_id(); for cpu in 0..crate::smp::cpu_count() { @@ -312,10 +323,7 @@ pub fn stop_other_cpus() { /// Calibrate the LAPIC timer on the BSP (requires HPET); does not start it. pub fn init_timer() { - // Divide by 1 for maximum resolution. - Reg::TIMER_DIVIDE.write(0b1011); - - // Masked one-shot mode for calibration. + // Masked one-shot mode for calibration, at `timer_one_shot`'s divide. Reg::LVT_TIMER.write(1 << 16); Reg::TIMER_INIT.write(0xFFFF_FFFF); @@ -331,7 +339,7 @@ pub fn init_timer() { let ticks_elapsed = 0xFFFF_FFFFu32.wrapping_sub(remaining); let ticks_10ms = (ticks_elapsed as u64 * 10_000_000 / elapsed) as u32; - Reg::TIMER_INIT.write(0); + timer_one_shot(); TIMER_TICKS.store(ticks_10ms, Ordering::Release); // Fallback for any Ring 0 fire before the scheduler arms its first quantum. percpu::set_last_armed_ticks(OneShot::ticks(ticks_10ms as u64).0); @@ -354,15 +362,16 @@ impl OneShot { Self(ticks.clamp(floor, u32::MAX as u64) as u32) } + /// `⌊nanos · per_10ms / 10⁷⌋`, split at 10⁷ ns so both halves stay in 64 + /// bits and divide by a constant (`per_10ms` is a `u32`). fn after(nanos: u64) -> Self { - let per_10ms = TIMER_TICKS.load(Ordering::Relaxed) as u128; - Self::ticks((nanos as u128 * per_10ms / 10_000_000) as u64) + const NS_PER_10MS: u64 = 10_000_000; + let per_10ms = u64::from(TIMER_TICKS.load(Ordering::Relaxed)); + Self::ticks((nanos / NS_PER_10MS).saturating_mul(per_10ms).saturating_add(nanos % NS_PER_10MS * per_10ms / NS_PER_10MS)) } + /// The count alone: [`timer_one_shot`] set the rest on this CPU. fn arm(self) { - Reg::TIMER_DIVIDE.write(0b1011); - // LVT resets masked; an AP may reach here before this register was ever written. - Reg::LVT_TIMER.write(u32::from(TIMER_VECTOR)); percpu::set_last_armed_ticks(self.0); Reg::TIMER_INIT.write(self.0); } diff --git a/kernel/src/arch/x86_64/idt/kick.rs b/kernel/src/arch/x86_64/idt/kick.rs index 0afa1e25882..41390867613 100644 --- a/kernel/src/arch/x86_64/idt/kick.rs +++ b/kernel/src/arch/x86_64/idt/kick.rs @@ -4,7 +4,9 @@ extern "sysv64" fn kick_handler() { crate::arch::percpu::irq_took!(Kick); crate::counters::serve_here(); crate::arch::smi_cmd::serve_here(); - crate::preempt::set_need_resched(); + if crate::sched::driver::pass_owed() { + crate::preempt::set_need_resched(); + } crate::arch::apic::eoi(); } diff --git a/kernel/src/deadline.rs b/kernel/src/deadline.rs index 50a27bba795..07d40d8b44f 100644 --- a/kernel/src/deadline.rs +++ b/kernel/src/deadline.rs @@ -279,7 +279,7 @@ pub fn stage_a_wedge() -> ! { log!("{WEDGE_STAGED}: every CPU stops taking scheduler passes from here"); STAGED.store(true, Relaxed); // A core still asleep is not a core this control has wedged. - crate::arch::irqchip::kick_all_but_self(); + crate::sched::driver::ring_all_but_self(); this_cpu() } diff --git a/kernel/src/id_map.rs b/kernel/src/id_map.rs index 8c9621f6819..826aa72a7da 100644 --- a/kernel/src/id_map.rs +++ b/kernel/src/id_map.rs @@ -35,16 +35,6 @@ impl IdMap { id } - /// Inserts the value `f` builds from the pre-assigned ID, returning it. - /// Avoids ever building the value with an invalid placeholder ID (e.g. `pid: 0`). - pub fn insert_with(&mut self, f: impl FnOnce(K) -> V) -> K { - let id = self.next; - self.next = self.next + K::ONE; - let value = f(id); - self.map.insert(id, value); - id - } - pub fn get(&self, id: K) -> Option<&V> { self.map.get(&id) } diff --git a/kernel/src/loader/mod.rs b/kernel/src/loader/mod.rs index 049c045f819..be39f249f18 100644 --- a/kernel/src/loader/mod.rs +++ b/kernel/src/loader/mod.rs @@ -389,9 +389,6 @@ pub fn spawn( tls: Some(tls_pages), user_stack_base: stack_base, user_stack_size: USER_STACK_SIZE as u64, - syscall_counts: [0; toyos_abi::syscall::SYSCALL_PROFILE_BINS], - syscall_total: 0, - syscall_total_ns: 0, })); #[cfg(feature = "test-actuators")] diff --git a/kernel/src/main.rs b/kernel/src/main.rs index b7a26902b38..a1c30c9a701 100644 --- a/kernel/src/main.rs +++ b/kernel/src/main.rs @@ -446,7 +446,6 @@ pub(crate) unsafe extern "C" fn kernel_main(loader_args: &mut KernelArgs) -> ! { vfs::init(); process::init(); scheduler::init(); - pipe::init(); vfs::lock().mount(&["tmp"], Box::new(crate::tmpfs::TmpFs::new())); diff --git a/kernel/src/mm/space.rs b/kernel/src/mm/space.rs index b026c63f0af..2d6c069e69e 100644 --- a/kernel/src/mm/space.rs +++ b/kernel/src/mm/space.rs @@ -681,23 +681,21 @@ impl AddressSpace { /// Serve a fault at `va` for `access`: populate its page if its range /// allows the access and the page is empty. `false` ends the thread. pub fn fault(&mut self, va: u64, access: Access) -> bool { - self.populate(va, access).is_some() + self.populate(va, access, |_| true).is_some() } /// Where the byte at `va` is, for the kernel to reach through the direct /// map, populating its page where its range allows `access`; `None` for /// an aperture's page, which the direct map does not reach. pub fn reach(&mut self, va: u64, access: Access) -> Option { - if !copyable(self.inventory.find(va)?.1) { - return None; - } - self.populate(va, access) + self.populate(va, access, copyable) } - /// The frame behind `va`, populated where its range allows `access`. - fn populate(&mut self, va: u64, access: Access) -> Option { + /// The frame behind `va`, populated where its range allows `access` and + /// is one `admits`. + fn populate(&mut self, va: u64, access: Access, admits: fn(&SpaceEntry) -> bool) -> Option { let (start, entry) = self.inventory.find(va)?; - if !entry.prot.allows(access) { + if !admits(entry) || !entry.prot.allows(access) { return None; } let page = va & !(PAGE_SIZE - 1); @@ -745,8 +743,8 @@ impl AddressSpace { /// what a copy the kernel makes for a thread asks first. A device's /// registers or firmware's framebuffer, an aperture, are no buffer. pub fn allows(&self, start: u64, end: u64, access: Access) -> bool { - self.inventory.covered(start & !(PAGE_SIZE - 1), end) - && self.inventory.meeting(start, end).all(|(_, entry)| entry.prot.allows(access) && entry.backed() && copyable(entry)) + self.inventory + .covered_where(start & !(PAGE_SIZE - 1), end, |entry| entry.prot.allows(access) && entry.backed() && copyable(entry)) } /// The frame and protection a present leaf maps `va` with, populating diff --git a/kernel/src/object/ops.rs b/kernel/src/object/ops.rs index 21fdba7ceb6..457b85b390c 100644 --- a/kernel/src/object/ops.rs +++ b/kernel/src/object/ops.rs @@ -12,7 +12,7 @@ use kernel::sched::task::WaitClass; use crate::drivers::serial; use crate::file_cache; -use crate::pipe::{self, PipeId}; +use crate::pipe::{self, PipeRef}; use kernel::pipe::{End, Landing}; use crate::user_ptr::{UserBytes, UserBytesMut}; use crate::inbox::PollEntry; @@ -195,19 +195,22 @@ pub fn close_all(table: &mut HandleTable) { drop(core::mem::replace(table, HandleTable::new())); } -pub fn pipe_id_read(object: &KObjectRef) -> Option { - pipe_read(object).map(|(id, _)| id) +/// The pipe a read of `object` took from, held past the handle table's lock +/// to wake its writers. +pub fn read_pipe(object: &KObjectRef) -> Option { + pipe_read(object).map(|(pipe, _)| pipe.clone()) } -pub fn pipe_id_write(object: &KObjectRef) -> Option { - pipe_write(object).map(|(id, _, _)| id) +/// The pipe a write to `object` filled, held the same way to wake its readers. +pub fn written_pipe(object: &KObjectRef) -> Option { + pipe_write(object).map(|(pipe, _, _)| pipe.clone()) } /// The pipe a blocking read of `object` parks on, and the class its wait is /// charged to: a connection's is its peer's answer, which is IPC. -pub fn pipe_read(object: &KObjectRef) -> Option<(PipeId, WaitClass)> { +pub fn pipe_read(object: &KObjectRef) -> Option<(&PipeRef, WaitClass)> { match object { - KObjectRef::PipeRead(r) => Some((r.id(), WaitClass::Pipe)), + KObjectRef::PipeRead(r) => Some((r.pipe(), WaitClass::Pipe)), KObjectRef::Connection(c) => Some((c.rx(), WaitClass::Ipc)), KObjectRef::PipeWrite(_) | KObjectRef::File(_) | KObjectRef::Device(_) | KObjectRef::Console(_) | KObjectRef::Acceptor(_) | KObjectRef::Inbox(_) @@ -220,9 +223,9 @@ pub fn pipe_read(object: &KObjectRef) -> Option<(PipeId, WaitClass)> { /// [`pipe_read`]'s answer for a blocking write, and how a write lands in /// that pipe. -pub fn pipe_write(object: &KObjectRef) -> Option<(PipeId, WaitClass, Landing)> { +pub fn pipe_write(object: &KObjectRef) -> Option<(&PipeRef, WaitClass, Landing)> { match object { - KObjectRef::PipeWrite(w) => Some((w.id(), WaitClass::Pipe, Landing::Pipe)), + KObjectRef::PipeWrite(w) => Some((w.pipe(), WaitClass::Pipe, Landing::Pipe)), KObjectRef::Connection(c) => Some((c.tx(), WaitClass::Ipc, Landing::Partial)), KObjectRef::PipeRead(_) | KObjectRef::File(_) | KObjectRef::Device(_) | KObjectRef::Console(_) | KObjectRef::Acceptor(_) | KObjectRef::Inbox(_) @@ -266,8 +269,8 @@ impl WatchRef { /// readiness of that direction exists. pub fn read_watch(object: &KObjectRef) -> Option { match object { - KObjectRef::PipeRead(r) => pipe::read_watch(r.id()).map(WatchRef::Shared), - KObjectRef::Connection(c) => pipe::read_watch(c.rx()).map(WatchRef::Shared), + KObjectRef::PipeRead(r) => Some(WatchRef::Shared(r.pipe().read_watch().clone())), + KObjectRef::Connection(c) => Some(WatchRef::Shared(c.rx().read_watch().clone())), KObjectRef::Acceptor(a) => Some(WatchRef::Shared(a.watch().clone())), KObjectRef::Process(p) => Some(WatchRef::Shared(p.watch().clone())), KObjectRef::Notice(n) => Some(WatchRef::Shared(n.watch().clone())), @@ -303,8 +306,8 @@ pub fn watched_readable(object: &KObjectRef) { /// The watch a writable poll on this object registers on. pub fn write_watch(object: &KObjectRef) -> Option { match object { - KObjectRef::PipeWrite(w) => pipe::write_watch(w.id()).map(WatchRef::Shared), - KObjectRef::Connection(c) => pipe::write_watch(c.tx()).map(WatchRef::Shared), + KObjectRef::PipeWrite(w) => Some(WatchRef::Shared(w.pipe().write_watch().clone())), + KObjectRef::Connection(c) => Some(WatchRef::Shared(c.tx().write_watch().clone())), KObjectRef::Console(_) => Some(WatchRef::Static(&crate::log::console::SPACE)), KObjectRef::PipeRead(_) | KObjectRef::File(_) | KObjectRef::Device(_) | KObjectRef::Acceptor(_) | KObjectRef::Inbox(_) @@ -319,8 +322,8 @@ pub fn write_watch(object: &KObjectRef) -> Option { /// it lets go; `None` for an object that is no pipe end. pub fn pipe_end_watch(object: &KObjectRef) -> Option { match object { - KObjectRef::PipeRead(r) => pipe::left_watch(r.id(), End::Read).map(WatchRef::Shared), - KObjectRef::PipeWrite(w) => pipe::left_watch(w.id(), End::Write).map(WatchRef::Shared), + KObjectRef::PipeRead(r) => Some(WatchRef::Shared(r.pipe().left_watch(End::Read).clone())), + KObjectRef::PipeWrite(w) => Some(WatchRef::Shared(w.pipe().left_watch(End::Write).clone())), KObjectRef::Connection(_) | KObjectRef::Acceptor(_) | KObjectRef::Process(_) | KObjectRef::Job(_) | KObjectRef::Notice(_) | KObjectRef::Console(_) | KObjectRef::Device(_) | KObjectRef::SysCap(_) @@ -333,8 +336,8 @@ pub fn pipe_end_watch(object: &KObjectRef) -> Option { /// object that is no pipe end. pub fn other_end_gone(object: &KObjectRef) -> bool { match object { - KObjectRef::PipeRead(r) => pipe::other_end_gone(r.id(), End::Read), - KObjectRef::PipeWrite(w) => pipe::other_end_gone(w.id(), End::Write), + KObjectRef::PipeRead(r) => r.pipe().other_end_gone(End::Read), + KObjectRef::PipeWrite(w) => w.pipe().other_end_gone(End::Write), KObjectRef::Connection(_) | KObjectRef::Acceptor(_) | KObjectRef::Process(_) | KObjectRef::Job(_) | KObjectRef::Notice(_) | KObjectRef::Console(_) | KObjectRef::Device(_) | KObjectRef::SysCap(_) @@ -533,7 +536,7 @@ fn record_bytes(record: &toyos_abi::pci::DeviceIrqRecord) -> &[u8] { pub fn try_read(object: &KObjectRef, buf: &mut UserBytesMut) -> Option { match object { KObjectRef::File(f) => read_file(f, buf), - KObjectRef::PipeRead(r) => pipe::try_read(r.id(), buf).map(|n| n as u64), + KObjectRef::PipeRead(r) => r.pipe().try_read(buf).map(|n| n as u64), // A batch pinned where the reader is answers the read, so no byte // after it is read until the batch is taken. KObjectRef::Connection(c) => c.read(buf).map(|read| match read { @@ -576,8 +579,8 @@ pub fn try_read(object: &KObjectRef, buf: &mut UserBytesMut) -> Option { } fn write_pipe(object: &KObjectRef, buf: &UserBytes) -> Option { - let (id, _, landing) = pipe_write(object).expect("a pipe's write end writes its pipe"); - match pipe::write(id, buf, landing) { + let (pipe, _, landing) = pipe_write(object).expect("a pipe's write end writes its pipe"); + match pipe.write(buf, landing) { // The reader is gone, not absent: `NotFound` means the name is not there and nothing else may say so. pipe::PipeWrite::BrokenPipe => Some(SyscallError::Gone.to_u64()), pipe::PipeWrite::NoMemory => Some(SyscallError::ResourceExhausted.to_u64()), @@ -716,7 +719,7 @@ pub fn ftruncate(object: &KObjectRef, size: u64) -> u64 { pub fn has_data(object: &KObjectRef) -> bool { match object { - KObjectRef::PipeRead(r) => pipe::has_data(r.id()), + KObjectRef::PipeRead(r) => r.pipe().has_data(), KObjectRef::Connection(c) => c.has_data(), KObjectRef::Console(_) => serial::has_data(), KObjectRef::Acceptor(a) => a.has_pending(), @@ -759,8 +762,8 @@ pub fn has_space(object: &KObjectRef) -> bool { match object { // Writable is room for any write that lands whole, so a non-blocking // one refused, retried on readiness, lands. - KObjectRef::PipeWrite(w) => pipe::has_space(w.id(), MAX_WHOLE_PIPE_WRITE), - KObjectRef::Connection(c) => pipe::has_space(c.tx(), 1), + KObjectRef::PipeWrite(w) => w.pipe().has_space(MAX_WHOLE_PIPE_WRITE), + KObjectRef::Connection(c) => c.tx().has_space(1), KObjectRef::File(_) | KObjectRef::Null(_) => true, KObjectRef::Console(_) => crate::log::console::has_room(), KObjectRef::PipeRead(_) | KObjectRef::Device(_) | KObjectRef::Acceptor(_) diff --git a/kernel/src/object/pipe.rs b/kernel/src/object/pipe.rs index 3a98103c562..e7b270e1c8d 100644 --- a/kernel/src/object/pipe.rs +++ b/kernel/src/object/pipe.rs @@ -3,30 +3,29 @@ //! [`PipeReadEnd`] has no write path and [`PipeWriteEnd`] has no read path, //! so the wrong direction is refused by the type before any policy runs. //! -//! The ring, its refcounts and `PipeId` stay in `crate::pipe`: adding them -//! here would double their ownership. These two types each own one counted -//! reference, released on the last handle so EOF rides `handle_count`. +//! The ring and its refcounts stay in `crate::pipe`: adding them here would +//! double their ownership. These two types each own one counted reference, +//! released on the last handle so EOF rides `handle_count`. use alloc::sync::Arc; use core::sync::atomic::{AtomicBool, Ordering}; use crate::mm::charge::Exhausted; -use crate::pipe::{PipeId, PipeReader, PipeWriter}; +use crate::pipe::{PipeReader, PipeRef, PipeWriter}; use super::{Held, KObjectVariant, ObjectCore, ZeroHandles}; pub struct PipeReadEnd { pub(super) core: ObjectCore, - /// A plain copy, not an `Arc` clone: the hot read path can't pay for one, - /// and a released `PipeId` is never reissued. - id: PipeId, + /// The pipe, which outlives the counted reference's release. + pipe: PipeRef, tty: AtomicBool, reference: Held, } pub struct PipeWriteEnd { pub(super) core: ObjectCore, - id: PipeId, + pipe: PipeRef, tty: AtomicBool, reference: Held, } @@ -35,14 +34,14 @@ impl PipeReadEnd { pub fn new(reader: PipeReader) -> Result, Exhausted> { Ok(Arc::new(Self { core: Self::new_core(0)?, - id: reader.id(), + pipe: reader.pipe().clone(), tty: AtomicBool::new(false), reference: Held::new(reader), })) } - pub fn id(&self) -> PipeId { - self.id + pub fn pipe(&self) -> &PipeRef { + &self.pipe } pub fn is_tty(&self) -> bool { @@ -60,14 +59,14 @@ impl PipeWriteEnd { pub fn new(writer: PipeWriter) -> Result, Exhausted> { Ok(Arc::new(Self { core: Self::new_core(0)?, - id: writer.id(), + pipe: writer.pipe().clone(), tty: AtomicBool::new(false), reference: Held::new(writer), })) } - pub fn id(&self) -> PipeId { - self.id + pub fn pipe(&self) -> &PipeRef { + &self.pipe } pub fn is_tty(&self) -> bool { diff --git a/kernel/src/object/service.rs b/kernel/src/object/service.rs index 8eeb2b9ad1d..8dc486cd90f 100644 --- a/kernel/src/object/service.rs +++ b/kernel/src/object/service.rs @@ -8,7 +8,7 @@ use kernel::pipe::Landing; use toyos_abi::syscall::{SyscallError, MAX_QUEUED_BATCHES, MAX_TRANSFER_HANDLES}; use crate::mm::charge::{self, Charge, Exhausted}; -use crate::pipe::{self, AtBatch, PipeId, PipeReader, PipeWrite, PipeWriter}; +use crate::pipe::{AtBatch, PipeReader, PipeRef, PipeWrite, PipeWriter}; use crate::user_ptr::{UserBytes, UserBytesMut}; use crate::sync::Lock; @@ -51,7 +51,7 @@ impl HandleQueue { /// Write `bytes` to `tx` and pin `batch` just past them, whole or not at /// all. A refusal hands the batch back: dropped here, it would be /// capabilities destroyed silently. - fn send(&self, tx: PipeId, bytes: &UserBytes, batch: Vec) -> Result<(), (Vec, SendRefused)> { + fn send(&self, tx: &PipeRef, bytes: &UserBytes, batch: Vec) -> Result<(), (Vec, SendRefused)> { let mut guard = self.batches.lock(); let Some(batches) = guard.as_mut() else { return Err((batch, SendRefused::Error(SyscallError::Gone))) }; let carries = !batch.is_empty(); @@ -60,7 +60,7 @@ impl HandleQueue { Err(Refused::Empty) => return Err((batch, SendRefused::Error(SyscallError::InvalidArgument))), Err(Refused::QueueFull) => return Err((batch, SendRefused::Error(SyscallError::ResourceExhausted))), } - match pipe::write(tx, bytes, Landing::Whole) { + match tx.write(bytes, Landing::Whole) { PipeWrite::Wrote { end, .. } => { if carries { batches.pin(end, batch); @@ -74,9 +74,9 @@ impl HandleQueue { } /// A read of `rx`, stopping at the next pinned batch. - fn read(&self, rx: PipeId, buf: &mut UserBytesMut) -> Option> { + fn read(&self, rx: &PipeRef, buf: &mut UserBytesMut) -> Option> { let guard = self.batches.lock(); - pipe::try_read_within(rx, buf, |read, available| match guard.as_ref() { + rx.try_read_within(buf, |read, available| match guard.as_ref() { Some(batches) => batches.window(read, available), None => Window::Bytes(available), }) @@ -84,10 +84,10 @@ impl HandleQueue { /// Take the batch pinned at `rx`'s read position, refusing without taking /// when it is wider than `cap`, and leaving it pinned when `room` refuses. - fn take(&self, rx: PipeId, cap: usize, room: impl FnOnce(usize) -> bool) -> Result>, SyscallError> { + fn take(&self, rx: &PipeRef, cap: usize, room: impl FnOnce(usize) -> bool) -> Result>, SyscallError> { let mut guard = self.batches.lock(); let Some(batches) = guard.as_mut() else { return Ok(None) }; - let Some(read) = pipe::read_position(rx) else { return Ok(None) }; + let read = rx.read_position(); let Some(width) = batches.at(read).map(Vec::len) else { return Ok(None) }; if width > cap { return Err(SyscallError::InvalidArgument); @@ -99,10 +99,10 @@ impl HandleQueue { } /// Whether a batch is pinned where the reader is, which is something to receive. - fn pinned_here(&self, rx: PipeId) -> bool { + fn pinned_here(&self, rx: &PipeRef) -> bool { let guard = self.batches.lock(); let Some(batches) = guard.as_ref() else { return false }; - pipe::read_position(rx).is_some_and(|read| batches.at(read).is_some()) + batches.at(rx.read_position()).is_some() } /// Drops the batches outside the lock: releasing a handle can run another object's hook. @@ -115,8 +115,8 @@ impl HandleQueue { /// One end of a connection: the queues are cross-wired, this end's `outbox` is the peer's `inbox`. pub struct ConnectionEnd { pub(super) core: ObjectCore, - rx: PipeId, - tx: PipeId, + rx: PipeRef, + tx: PipeRef, inbox: Arc, outbox: Arc, reference: Held<(PipeReader, PipeWriter)>, @@ -166,8 +166,8 @@ impl ConnectionEnd { ) -> Arc { Arc::new(Self { core, - rx: rx.id(), - tx: tx.id(), + rx: rx.pipe().clone(), + tx: tx.pipe().clone(), inbox, outbox, reference: Held::new((rx, tx)), @@ -179,32 +179,32 @@ impl ConnectionEnd { self.stamp.as_ref() } - pub fn rx(&self) -> PipeId { - self.rx + pub fn rx(&self) -> &PipeRef { + &self.rx } - pub fn tx(&self) -> PipeId { - self.tx + pub fn tx(&self) -> &PipeRef { + &self.tx } /// See [`HandleQueue::send`]: a refusal comes back with the batch. pub fn send(&self, bytes: &UserBytes, batch: Vec) -> Result<(), (Vec, SendRefused)> { - self.outbox.send(self.tx, bytes, batch) + self.outbox.send(&self.tx, bytes, batch) } /// See [`HandleQueue::read`]. pub fn read(&self, buf: &mut UserBytesMut) -> Option> { - self.inbox.read(self.rx, buf) + self.inbox.read(&self.rx, buf) } /// See [`HandleQueue::take`]. pub fn recv(&self, cap: usize, room: impl FnOnce(usize) -> bool) -> Result>, SyscallError> { - self.inbox.take(self.rx, cap, room) + self.inbox.take(&self.rx, cap, room) } /// Bytes to read, the peer's end, or a batch pinned where the reader is. pub fn has_data(&self) -> bool { - pipe::has_data(self.rx) || self.inbox.pinned_here(self.rx) + self.rx.has_data() || self.inbox.pinned_here(&self.rx) } } diff --git a/kernel/src/pipe.rs b/kernel/src/pipe.rs index f8d78da3bd0..3c8b9649887 100644 --- a/kernel/src/pipe.rs +++ b/kernel/src/pipe.rs @@ -10,90 +10,75 @@ use alloc::sync::Arc; use crate::mm::PAGE_2M; use crate::watch::Watch; -use crate::id_map::{IdKey, IdMap}; use crate::sync::Lock; use crate::user_ptr::{UserBytes, UserBytesMut}; -/// Raw pipe identifier; carries no refcount, not exposed outside the kernel. - -#[derive(Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord, Debug)] -pub struct PipeId(usize); - -impl PipeId { -} - -impl core::ops::Add for PipeId { - type Output = Self; - fn add(self, rhs: Self) -> Self { PipeId(self.0 + rhs.0) } -} - -impl IdKey for PipeId { - const ZERO: Self = PipeId(0); - const ONE: Self = PipeId(1); -} +/// A reference to a pipe that holds neither end: what an end's object, a +/// waiter and a poll keep to reach it. The pipe is its own: nothing else +/// names it, and its lock serializes nothing but its own traffic. +#[derive(Clone)] +pub struct PipeRef(Arc); pub use handle::{PipeReader, PipeWriter}; /// Owned references to a pipe's two ends. /// -/// The id inside each handle is private to this module; only `acquire`, which -/// requires the `&mut Pipe` a `PIPES` lock produces, may look up a pipe and -/// take a counted reference, so no program point exists between the two where -/// the last other end could close and free it. +/// Each counts one holder of its end under the pipe's lock as it is made and +/// lets it go as it drops, so the counts are the live references and nothing +/// else. mod handle { - use super::{release, with_pipes_mut, End, Pipe, PipeId}; + use super::{End, PipeRef}; /// One counted reference to a pipe's read end — `Arc`, for a reader slot. - pub struct PipeReader(PipeId); + pub struct PipeReader(PipeRef); /// One counted reference to a pipe's write end. - pub struct PipeWriter(PipeId); + pub struct PipeWriter(PipeRef); impl PipeReader { /// The only constructor: taking and counting the reference is one statement. - pub(super) fn acquire(pipe: &mut Pipe) -> Self { + pub(super) fn acquire(pipe: &PipeRef) -> Self { pipe.hold(End::Read); - Self(pipe.id) + Self(pipe.clone()) } - pub fn id(&self) -> PipeId { self.0 } + pub fn pipe(&self) -> &PipeRef { + &self.0 + } } impl PipeWriter { - pub(super) fn acquire(pipe: &mut Pipe) -> Self { + pub(super) fn acquire(pipe: &PipeRef) -> Self { pipe.hold(End::Write); - Self(pipe.id) + Self(pipe.clone()) } - pub fn id(&self) -> PipeId { self.0 } + pub fn pipe(&self) -> &PipeRef { + &self.0 + } } - /// A live handle proves the pipe is still in the table, so `expect` here cannot fail on a race. impl Clone for PipeReader { fn clone(&self) -> Self { - with_pipes_mut(|pipes| { - Self::acquire(pipes.get_mut(self.0).expect("a held PipeReader's pipe is in the table")) - }) + Self::acquire(&self.0) } } impl Clone for PipeWriter { fn clone(&self) -> Self { - with_pipes_mut(|pipes| { - Self::acquire(pipes.get_mut(self.0).expect("a held PipeWriter's pipe is in the table")) - }) + Self::acquire(&self.0) } } impl Drop for PipeReader { fn drop(&mut self) { - release(self.0, End::Read); + self.0.release(End::Read); } } impl Drop for PipeWriter { fn drop(&mut self) { - release(self.0, End::Write); + self.0.release(End::Write); } } } @@ -114,59 +99,51 @@ struct Backing { } struct Pipe { - /// Its own key in `PIPES`, so a handle cannot be built naming a different pipe than the one whose count it bumped. - id: PipeId, - /// `None` until first use — allocating eagerly would charge every pending `SYS_CONNECT` 4 MiB before either end sent a byte. - backing: Option, - holders: Holders, + state: Lock, /// The read end's and the write end's watches. Held by `Arc` so a blocking site or a - /// poll registration can clone one out from under the table lock and hold it across its own park. + /// poll registration can clone one out and hold it across its own park. read_watch: Arc, write_watch: Arc, /// Each end's leaving watch, by [`End`]: an `OTHER_END_GONE` poll on that /// end sits on it, and only the other end's last holder posts it — a byte /// moved through the pipe wakes no poll that asked about a leaving. left: [Arc; 2], + /// Its own heap and its watches. + _charge: Charge, +} + +/// What a pipe's lock holds. +struct State { + /// `None` until first use — allocating eagerly would charge every pending + /// `SYS_CONNECT` 4 MiB before either end sent a byte — and again once + /// neither end is held. + backing: Option, + holders: Holders, /// Set when a write should hand the next reader transient RT priority — covers readers already runnable, which the wake-time boost misses. rt_boost_pending: bool, /// Bytes ever written and ever read: the stream positions a connection's /// batches are pinned at (`kernel::message`), which never wrap. written: u64, read: u64, - /// Its place in `PIPES` and its watches. - _charge: Charge, } -// SAFETY: `Backing::ring`'s raw pointer and its owning `PhysPage` move together in the same struct, and the `PIPES` lock is the only way to reach a `Pipe`, serializing all access to it. -unsafe impl Send for Pipe {} +// SAFETY: `Backing::ring`'s raw pointer and its owning `PhysPage` move together in the same struct, and the pipe's lock is the only way to reach a `State`, serializing all access to it. +unsafe impl Send for State {} impl Pipe { - /// The heap one takes: its place in `PIPES` and its four watches. + /// The heap one takes: itself and its four watches. fn heap() -> usize { - charge::map_entry::() + 4 * charge::arc::() - } - - fn new(id: PipeId, charge: Charge) -> Self { - Self { - id, - backing: None, - holders: Holders::new(), - read_watch: Arc::new(Watch::new()), - write_watch: Arc::new(Watch::new()), - left: [Arc::new(Watch::new()), Arc::new(Watch::new())], - rt_boost_pending: false, - written: 0, - read: 0, - _charge: charge, - } + charge::arc::() + 4 * charge::arc::() } +} +impl State { /// Allocate the ring page if this is the first use; `None` where the /// kernel holds back what is left, which userland's write is refused. fn back(&mut self) -> Option<&mut Backing> { if self.backing.is_none() { let page = pmm::alloc_page_for_process()?; - // SAFETY: a fresh 2 MiB page this `Pipe` owns for as long as the `Ring` addresses it. + // SAFETY: a fresh 2 MiB page this `State` owns for as long as the `Ring` addresses it. let ring = unsafe { Ring::new(page.direct_map().as_mut_ptr(), PIPE_SIZE) }; self.backing = Some(Backing { _page: page, ring }); self.publish_ends(); @@ -181,11 +158,6 @@ impl Pipe { if self.holders.held(End::Write) { backing.ring.open_writer() } else { backing.ring.close_writer() } } - fn hold(&mut self, end: End) { - self.holders.hold(end); - self.publish_ends(); - } - fn available(&self) -> u32 { self.backing.as_ref().map_or(0, |b| b.ring.available()) } @@ -196,90 +168,26 @@ impl Pipe { } } -static PIPES: Lock>> = Lock::new(None); - -fn with_pipes(f: impl FnOnce(&IdMap) -> R) -> R { - let guard = PIPES.lock(); - f(guard.as_ref().expect("pipes not initialized")) -} - -fn with_pipes_mut(f: impl FnOnce(&mut IdMap) -> R) -> R { - let mut guard = PIPES.lock(); - f(guard.as_mut().expect("pipes not initialized")) -} - -pub fn init() { - *PIPES.lock() = Some(IdMap::new()); -} - /// Create a new pipe. Returns owned reader + writer references. /// /// A pipe with no traffic owns no physical memory: its ring page is /// allocated at its first `write`, which is refused if none is left. pub fn create() -> Result<(PipeReader, PipeWriter), Exhausted> { let charge = Charge::new(Pipe::heap())?; - Ok(with_pipes_mut(|pipes| { - let id = pipes.insert_with(|id| Pipe::new(id, charge)); - let pipe = pipes.get_mut(id).expect("the pipe just inserted"); - let reader = PipeReader::acquire(pipe); - let writer = PipeWriter::acquire(pipe); - (reader, writer) - })) -} - -pub fn try_read(pipe_id: PipeId, buf: &mut UserBytesMut) -> Option { - try_read_within(pipe_id, buf, |_, available| Window::Bytes(available)) - .map(|read| read.expect("a read with no window stops nowhere")) + let pipe = PipeRef(Arc::new(Pipe { + state: Lock::new(State { backing: None, holders: Holders::new(), rt_boost_pending: false, written: 0, read: 0 }), + read_watch: Arc::new(Watch::new()), + write_watch: Arc::new(Watch::new()), + left: [Arc::new(Watch::new()), Arc::new(Watch::new())], + _charge: charge, + })); + Ok((PipeReader::acquire(&pipe), PipeWriter::acquire(&pipe))) } /// A read stopped at a batch pinned where the reader is. #[derive(Debug)] pub struct AtBatch; -/// [`try_read`], taking only what `window` allows of the bytes the ring -/// holds, given the reader's stream position: [`AtBatch`] where it says the -/// reader is at a pinned batch. -pub fn try_read_within( - pipe_id: PipeId, - buf: &mut UserBytesMut, - window: impl FnOnce(u64, usize) -> Window, -) -> Option> { - let (result, boost) = with_pipes_mut(|pipes| { - let Some(pipe) = pipes.get_mut(pipe_id) else { return (None, false) }; - let limit = match window(pipe.read, pipe.available() as usize) { - Window::AtBatch => return (Some(Err(AtBatch)), false), - Window::Bytes(limit) => limit, - }; - if pipe.available() > 0 && limit > 0 { - let len = buf.len().min(limit); - let n = pipe - .backing - .as_mut() - .expect("available() > 0 implies a ring") - .ring - .read(len, |off, src| buf.write_run(off, &src)); - pipe.read += n as u64; - let boost = pipe.rt_boost_pending; - pipe.rt_boost_pending = false; - (Some(Ok(n)), boost) - } else if !pipe.holders.held(End::Write) && pipe.available() == 0 { - (Some(Ok(0)), false) - } else { - (None, false) - } - }); - if boost { - // Outside the PIPES lock: the scheduler takes its own CPU-queue lock. - crate::scheduler::boost_current_rt_inherited(); - } - result -} - -/// The stream position the reader is at: every byte read so far. -pub fn read_position(pipe_id: PipeId) -> Option { - with_pipes(|pipes| pipes.get(pipe_id).map(|p| p.read)) -} - pub enum PipeWrite { /// This many bytes, the stream ending at `end`. Wrote { n: usize, end: u64 }, @@ -290,10 +198,64 @@ pub enum PipeWrite { NoMemory, } -/// Write `buf` as `landing` has it land, under the one lock hold. -pub fn write(pipe_id: PipeId, buf: &UserBytes, landing: Landing) -> PipeWrite { - with_pipes_mut(|pipes| { - let Some(pipe) = pipes.get_mut(pipe_id) else { return PipeWrite::BrokenPipe }; +impl PipeRef { + fn hold(&self, end: End) { + let mut state = self.0.state.lock(); + state.holders.hold(end); + state.publish_ends(); + } + + pub fn try_read(&self, buf: &mut UserBytesMut) -> Option { + self.try_read_within(buf, |_, available| Window::Bytes(available)) + .map(|read| read.expect("a read with no window stops nowhere")) + } + + /// [`Self::try_read`], taking only what `window` allows of the bytes the + /// ring holds, given the reader's stream position: [`AtBatch`] where it + /// says the reader is at a pinned batch. + pub fn try_read_within( + &self, + buf: &mut UserBytesMut, + window: impl FnOnce(u64, usize) -> Window, + ) -> Option> { + let (result, boost) = { + let mut pipe = self.0.state.lock(); + let limit = match window(pipe.read, pipe.available() as usize) { + Window::AtBatch => return Some(Err(AtBatch)), + Window::Bytes(limit) => limit, + }; + if pipe.available() > 0 && limit > 0 { + let len = buf.len().min(limit); + let n = pipe + .backing + .as_mut() + .expect("available() > 0 implies a ring") + .ring + .read(len, |off, src| buf.write_run(off, &src)); + pipe.read += n as u64; + let boost = core::mem::take(&mut pipe.rt_boost_pending); + (Some(Ok(n)), boost) + } else if !pipe.holders.held(End::Write) && pipe.available() == 0 { + (Some(Ok(0)), false) + } else { + (None, false) + } + }; + if boost { + // Outside the pipe's lock: the scheduler takes its own CPU-queue lock. + crate::scheduler::boost_current_rt_inherited(); + } + result + } + + /// The stream position the reader is at: every byte read so far. + pub fn read_position(&self) -> u64 { + self.0.state.lock().read + } + + /// Write `buf` as `landing` has it land, under the one lock hold. + pub fn write(&self, buf: &UserBytes, landing: Landing) -> PipeWrite { + let mut pipe = self.0.state.lock(); if !pipe.holders.held(End::Read) { return PipeWrite::BrokenPipe; } @@ -303,54 +265,64 @@ pub fn write(pipe_id: PipeId, buf: &UserBytes, landing: Landing) -> PipeWrite { assert_eq!(wrote, n, "pipe: a ring with room for a write took less of it"); pipe.written += n as u64; PipeWrite::Wrote { n, end: pipe.written } - }) -} + } -pub fn has_data(pipe_id: PipeId) -> bool { - with_pipes(|pipes| { - pipes.get(pipe_id).is_some_and(|p| p.holders.readable(p.available())) - }) -} + pub fn has_data(&self) -> bool { + let pipe = self.0.state.lock(); + pipe.holders.readable(pipe.available()) + } -/// Whether a write waiting for `want` bytes of room answers now. -pub fn has_space(pipe_id: PipeId, want: usize) -> bool { - with_pipes(|pipes| pipes.get(pipe_id).is_some_and(|p| p.holders.writable(p.space(), want))) -} + /// Whether a write waiting for `want` bytes of room answers now. + pub fn has_space(&self, want: usize) -> bool { + let pipe = self.0.state.lock(); + pipe.holders.writable(pipe.space(), want) + } -/// Whether the end opposite `end` of this pipe has no holder left. -pub fn other_end_gone(pipe_id: PipeId, end: End) -> bool { - with_pipes(|pipes| pipes.get(pipe_id).is_some_and(|p| p.holders.other_end_gone(end))) -} + /// Whether the end opposite `end` of this pipe has no holder left. + pub fn other_end_gone(&self, end: End) -> bool { + self.0.state.lock().holders.other_end_gone(end) + } -/// Mark the pipe so the next consumer inherits RT priority. -pub fn set_rt_boost_pending(pipe_id: PipeId) { - with_pipes_mut(|pipes| { - if let Some(pipe) = pipes.get_mut(pipe_id) { - pipe.rt_boost_pending = true; - } - }); -} + /// Mark the pipe so the next consumer inherits RT priority. + pub fn set_rt_boost_pending(&self) { + self.0.state.lock().rt_boost_pending = true; + } -/// One reference to `end` lets go; its end's last posts the other end's watch -/// and its leaving watch. -fn release(pipe_id: PipeId, end: End) { - let (released, left) = with_pipes_mut(|pipes| { - let pipe = pipes.get_mut(pipe_id).expect("release: pipe not found"); - let released = pipe.holders.release(end); - pipe.publish_ends(); - let left = pipe.left[slot(end.other())].clone(); - if released == Released::PipeGone { - let pipe = pipes.remove(pipe_id).unwrap(); - free_pipe(pipe); + /// One reference to `end` lets go; its end's last posts the other end's + /// watch and its leaving watch, and the last of either end frees the ring. + fn release(&self, end: End) { + let (released, page) = { + let mut pipe = self.0.state.lock(); + let released = pipe.holders.release(end); + pipe.publish_ends(); + // Dropped outside the lock: the page goes back to the allocator. + let page = (released == Released::PipeGone).then(|| pipe.backing.take()).flatten(); + (released, page) + }; + drop(page); + match (released, end) { + (Released::Held | Released::PipeGone, _) => return, + (Released::EndGone, End::Read) => crate::scheduler::wake_pipe_writers(self), + (Released::EndGone, End::Write) => crate::scheduler::wake_pipe_readers(self), } - (released, left) - }); - match (released, end) { - (Released::Held | Released::PipeGone, _) => return, - (Released::EndGone, End::Read) => crate::scheduler::wake_pipe_writers(pipe_id), - (Released::EndGone, End::Write) => crate::scheduler::wake_pipe_readers(pipe_id), + self.0.left[slot(end.other())].post(); + } + + /// The read end's watch, for a blocking site, a post or a poll to clone and hold on its own stack. + pub fn read_watch(&self) -> &Arc { + &self.0.read_watch + } + + /// The write end's watch, the same way. + pub fn write_watch(&self) -> &Arc { + &self.0.write_watch + } + + /// `end`'s leaving watch, the same way: posted once, by the other end's + /// last holder. + pub fn left_watch(&self, end: End) -> &Arc { + &self.0.left[slot(end)] } - left.post(); } fn slot(end: End) -> usize { @@ -359,23 +331,3 @@ fn slot(end: End) -> usize { End::Write => 1, } } - -fn free_pipe(pipe: Pipe) { - drop(pipe); // PhysPage freed via Drop -} - -/// The read end's watch, cloned out for a blocking site, a post or a poll to hold on its own stack. -pub fn read_watch(pipe_id: PipeId) -> Option> { - with_pipes(|pipes| pipes.get(pipe_id).map(|p| p.read_watch.clone())) -} - -/// The write end's watch, the same way. -pub fn write_watch(pipe_id: PipeId) -> Option> { - with_pipes(|pipes| pipes.get(pipe_id).map(|p| p.write_watch.clone())) -} - -/// `end`'s leaving watch, the same way: posted once, by the other end's last -/// holder. -pub fn left_watch(pipe_id: PipeId, end: End) -> Option> { - with_pipes(|pipes| pipes.get(pipe_id).map(|p| p.left[slot(end)].clone())) -} diff --git a/kernel/src/process.rs b/kernel/src/process.rs index 87d5ffb50e6..8f4df5fa236 100644 --- a/kernel/src/process.rs +++ b/kernel/src/process.rs @@ -145,7 +145,6 @@ impl ThreadEntry { let len = name.len().min(THREAD_NAME_LEN); self.name[..len].copy_from_slice(&name[..len]); } - pub fn thread_data(&self) -> &Arc> { &self.thread_data } } /// A process and all its threads. Removing a ProcessEntry removes all threads. @@ -413,11 +412,6 @@ pub struct ThreadData { pub tls: Option, pub user_stack_base: UserAddr, pub user_stack_size: u64, - /// Syscall counts per syscall number (for profiling) - pub syscall_counts: [u32; toyos_abi::syscall::SYSCALL_PROFILE_BINS], - pub syscall_total: u64, - /// Wall-clock nanoseconds spent in syscall dispatch (includes preemption time) - pub syscall_total_ns: u64, } /// Zero-sized proof of running on the per-CPU idle stack; required by `collect_orphan_zombies` so it never drops the thread entry it runs on. @@ -480,23 +474,17 @@ pub fn stats_of( } let pid = object.pid(); // The other two locks are taken after this one is dropped, so no ordering edge. - let (data_arc, cpu_ns, threads) = { + let (data_arc, cpu_ns, syscall_total, syscall_total_ns) = { let guard = PROCESS_TABLE.lock(); let proc = guard.as_ref()?.get(pid)?; - let mut cpu_ns = 0; - let mut threads = Vec::new(); - for (_, thread) in proc.threads().iter() { - cpu_ns += thread.sched().map_or(0, scheduler::task_cpu_ns); - threads.push(Arc::clone(thread.thread_data())); + let (mut cpu_ns, mut syscall_total, mut syscall_total_ns) = (0, 0, 0); + for sched in proc.threads().iter().filter_map(|(_, thread)| thread.sched()) { + cpu_ns += scheduler::task_cpu_ns(sched); + syscall_total += sched.handle.syscalls().calls(); + syscall_total_ns += sched.handle.syscalls().ns(); } - (Arc::clone(proc.process_data()), cpu_ns, threads) + (Arc::clone(proc.process_data()), cpu_ns, syscall_total, syscall_total_ns) }; - let (mut syscall_total, mut syscall_total_ns) = (0, 0); - for tdata in threads { - let tdata = tdata.lock(); - syscall_total += tdata.syscall_total; - syscall_total_ns += tdata.syscall_total_ns; - } let data = data_arc.lock(); Some(stats_from(&data, pid, cpu_ns, syscall_total, syscall_total_ns)) } @@ -702,9 +690,6 @@ pub fn spawn_thread( tls: Some(tls_alloc), user_stack_base: UserAddr::new(stack_base), user_stack_size: if stack_base > 0 { stack_ptr - stack_base } else { 0 }, - syscall_counts: [0; toyos_abi::syscall::SYSCALL_PROFILE_BINS], - syscall_total: 0, - syscall_total_ns: 0, })); let proc = table.get_mut(parent_process) .expect("spawn_thread: the entry the insert admission just answered for"); @@ -765,16 +750,16 @@ impl core::fmt::Display for Consumed { /// Says nothing of the machine: a reading of every CPU is `crate::census`'s, never one process's end. fn teardown_resources( process_data_arc: &Arc>, - thread_data_arc: &Arc>, + main_thread: &(Arc>, Option), pid: Pid, ) -> Consumed { + let (thread_data_arc, sched) = main_thread; + let syscalls = sched.as_ref().map(|sched| sched.handle.syscalls()); + let syscall_total = syscalls.map_or(0, |s| s.calls()); + let syscall_total_ns = syscalls.map_or(0, |s| s.ns()); + let syscall_counts = syscalls.map_or([0; toyos_abi::syscall::SYSCALL_PROFILE_BINS], |s| s.by_number()); // Never hold ThreadData + ProcessData at once. - let (syscall_total, syscall_total_ns, syscall_counts) = { - let mut tdata = thread_data_arc.lock(); - let stats = (tdata.syscall_total, tdata.syscall_total_ns, tdata.syscall_counts); - tdata.tls.take(); - stats - }; + thread_data_arc.lock().tls.take(); // Before the teardown below: a report spans its process up to here, and the next one this teardown. #[cfg(feature = "mask-windows")] @@ -867,13 +852,14 @@ pub fn stats_from( /// Waits on nothing: it runs on a killed thread whose one cancel may be spent, and at the exit boundary's preempt depth, where a park asserts. /// The section that marks it also lets the teardown's own hold on the publication go: the exit is published here when nothing below it is left, and otherwise by the last end below it. fn teardown(pid: Pid, tid: Tid, end: End, mark: i32, process_data: &Arc>) { - let main_thread_data = { + let main_thread = { let guard = PROCESS_TABLE.lock(); let proc = guard.as_ref().unwrap().get(pid).expect("teardown: the process its last thread is in"); - Arc::clone(&proc.threads.get(proc.main_tid).expect("teardown: a claimed process gives up no thread").thread_data) + let main = proc.threads.get(proc.main_tid).expect("teardown: a claimed process gives up no thread"); + (Arc::clone(&main.thread_data), main.sched().cloned()) }; let releases = crate::object::Releases::open(); - let consumed = teardown_resources(process_data, &main_thread_data, pid); + let consumed = teardown_resources(process_data, &main_thread, pid); let released = releases.run(); let cpu_ns = { let mut guard = PROCESS_TABLE.lock(); @@ -1052,13 +1038,13 @@ pub fn futex_wake(addr: UserAddr, count: u64) -> u64 { } /// Wake processes blocked on reading from a pipe that now has data. -pub fn wake_pipe_readers(pipe_id: pipe::PipeId) { - scheduler::wake_pipe_readers(pipe_id); +pub fn wake_pipe_readers(pipe: &pipe::PipeRef) { + scheduler::wake_pipe_readers(pipe); } /// Wake processes blocked on writing to a pipe that now has space. -pub fn wake_pipe_writers(pipe_id: pipe::PipeId) { - scheduler::wake_pipe_writers(pipe_id); +pub fn wake_pipe_writers(pipe: &pipe::PipeRef) { + scheduler::wake_pipe_writers(pipe); } /// Ask `join` once more; an unsettled one collects under the table lock, which is what makes validating the parent-thread relationship and collecting the zombie one act. diff --git a/kernel/src/quiesce.rs b/kernel/src/quiesce.rs index a9bda5dd063..17cf1676726 100644 --- a/kernel/src/quiesce.rs +++ b/kernel/src/quiesce.rs @@ -157,7 +157,7 @@ pub fn stop() -> (Record, Stopping) { // Last: a gate that sees the stop sees the caller it must not stop. STAGE.open(STOPPING); - crate::arch::irqchip::kick_all_but_self(); + crate::sched::driver::ring_all_but_self(); let cpus = crate::smp::cpu_count(); let began = crate::clock::now(); diff --git a/kernel/src/sched/driver.rs b/kernel/src/sched/driver.rs index af97309df6e..82c7e2f6ac8 100644 --- a/kernel/src/sched/driver.rs +++ b/kernel/src/sched/driver.rs @@ -716,6 +716,32 @@ pub fn current_image() -> Option>> { try_with_cpu(|cpu| cpu.running().map(|t| t.ext().image.clone())).flatten() } +/// Whether this CPU's doorbell was rung since its last pass began: what a +/// kick's handler asks before it owes a pass. A kick that landed after the pass +/// it was sent for began, or that only asks this CPU to serve a counters round, +/// owes none. +pub fn pass_owed() -> bool { + cpus().get(current_cpu()).doorbell().kick_pending() +} + +/// Ring every other CPU's doorbell with nothing behind it and kick each whose +/// edge this raised, so each runs a pass: what a machine-wide stop needs of a +/// CPU halted with its timer stopped, or running a thread that never enters +/// the kernel. +pub fn ring_all_but_self() { + let me = current_cpu(); + for cpu in (0..cpus().len() as u32).map(CpuId).filter(|&cpu| cpu != me) { + if cpus().get(cpu).poke() == Kick::Send { + HW.kick(cpu); + } + } +} + +/// Count the running thread's syscall `num`, which took `ticks` of the counter. +pub fn note_syscall(num: u64, ticks: u64) { + try_with_cpu(|cpu| cpu.running().map(|t| t.ext().handle.syscalls().note(num, ticks))); +} + /// What the running task's marks say it does instead of returning to Ring 3 — one load, no clone, since an `Arc` refcount here is too costly on this path. pub fn current_safe_point(stopping: bool) -> Option { try_with_cpu(|cpu| cpu.running().and_then(|t| t.shared().at_safe_point(stopping))).flatten() diff --git a/kernel/src/sched/kthread.rs b/kernel/src/sched/kthread.rs index 5c756c500b3..be190fcbba6 100644 --- a/kernel/src/sched/kthread.rs +++ b/kernel/src/sched/kthread.rs @@ -146,8 +146,5 @@ fn kernel_thread_data() -> ThreadData { tls: None, user_stack_base: crate::mm::UserAddr::new(0), user_stack_size: 0, - syscall_counts: [0; toyos_abi::syscall::SYSCALL_PROFILE_BINS], - syscall_total: 0, - syscall_total_ns: 0, } } diff --git a/kernel/src/sched/payload.rs b/kernel/src/sched/payload.rs index a6aecaf362e..18ddf352571 100644 --- a/kernel/src/sched/payload.rs +++ b/kernel/src/sched/payload.rs @@ -110,6 +110,49 @@ pub struct TaskHandle { cancels: AtomicU32, /// The releases this thread owns while its process is torn down (`object::Releases`); kept here, as `operation` is, so they follow the thread across a migration. releases: Lock>>, + syscalls: SyscallProfile, +} + +/// What a thread's syscalls cost it. Written only by the thread, at the end of +/// each of its syscalls, so a count is a load and a store; read by anyone. +pub struct SyscallProfile { + calls: AtomicU64, + /// Counter ticks between each syscall's entry and its end, wall-clock: a + /// preemption inside one is counted. + ticks: AtomicU64, + by_number: [AtomicU32; toyos_abi::syscall::SYSCALL_PROFILE_BINS], +} + +impl SyscallProfile { + fn new() -> Self { + Self { + calls: AtomicU64::new(0), + ticks: AtomicU64::new(0), + by_number: [const { AtomicU32::new(0) }; toyos_abi::syscall::SYSCALL_PROFILE_BINS], + } + } + + /// The running thread's syscall `num` took `ticks`. + pub fn note(&self, num: u64, ticks: u64) { + let bump = |count: &AtomicU64, by: u64| count.store(count.load(Ordering::Relaxed) + by, Ordering::Relaxed); + bump(&self.calls, 1); + bump(&self.ticks, ticks); + // Clamped, not guarded: an unissued number still lands in the last bin, not nowhere. + let bin = &self.by_number[(num as usize).min(toyos_abi::syscall::SYSCALL_PROFILE_OTHER)]; + bin.store(bin.load(Ordering::Relaxed) + 1, Ordering::Relaxed); + } + + pub fn calls(&self) -> u64 { + self.calls.load(Ordering::Relaxed) + } + + pub fn ns(&self) -> u64 { + crate::clock::nanos_of_ticks(self.ticks.load(Ordering::Relaxed)) + } + + pub fn by_number(&self) -> [u32; toyos_abi::syscall::SYSCALL_PROFILE_BINS] { + core::array::from_fn(|bin| self.by_number[bin].load(Ordering::Relaxed)) + } } impl TaskHandle { @@ -120,9 +163,14 @@ impl TaskHandle { watch: Watch::new(), cancels: AtomicU32::new(0), releases: Lock::new(None), + syscalls: SyscallProfile::new(), } } + pub fn syscalls(&self) -> &SyscallProfile { + &self.syscalls + } + pub(crate) fn publish(&self, acct: &TaskAccounting, running_since: Option) { self.cpu_ns.store(acct.cpu_ns, Ordering::Relaxed); self.running_since diff --git a/kernel/src/scheduler.rs b/kernel/src/scheduler.rs index 1f11ccd91ba..8962cc508eb 100644 --- a/kernel/src/scheduler.rs +++ b/kernel/src/scheduler.rs @@ -12,7 +12,7 @@ use kernel::sched::task::{Refused, SafePoint, WaitClass, QUANTUM_NS}; use crate::arch::percpu; use crate::watch::{self, Cancel}; use crate::hw::HW; -use crate::pipe::PipeId; +use crate::pipe::PipeRef; use crate::process::{self, Pid, Tid}; use crate::sched::driver::{self, cpus, preempt_off, Dispose, NewTask}; use crate::sched::payload::ThreadSched; @@ -255,12 +255,10 @@ pub fn exit_current() -> ! { /// Wake pipe readers, lending each an RT window if the writer holds one; the /// pipe is also marked, so a runnable reader takes the window too. -pub fn wake_pipe_readers(pipe_id: PipeId) { - let Some(watch) = crate::pipe::read_watch(pipe_id) else { - return; - }; +pub fn wake_pipe_readers(pipe: &PipeRef) { + let watch = pipe.read_watch(); if driver::current_is_rt() { - crate::pipe::set_rt_boost_pending(pipe_id); + pipe.set_rt_boost_pending(); watch.post_boosted(boost_window()); } else { watch.post(); @@ -268,10 +266,8 @@ pub fn wake_pipe_readers(pipe_id: PipeId) { } /// Wake pipe writers, and complete every poll on the write end. -pub fn wake_pipe_writers(pipe_id: PipeId) { - if let Some(watch) = crate::pipe::write_watch(pipe_id) { - watch.post(); - } +pub fn wake_pipe_writers(pipe: &PipeRef) { + pipe.write_watch().post(); } /// How long a lent RT priority lasts: one quantum, a wall-clock bound on time held. @@ -326,8 +322,10 @@ pub fn futex_wait( read, ); // `wait_until` answers `Ok(())` for a satisfied predicate and an expired - // deadline alike, so the word itself is what tells them apart. - if read() { + // deadline alike, so the word itself is what tells them apart. With no + // deadline the word or its frame changed, or a kill ended it, whose answer + // never reaches Ring 3. + if deadline.is_never() || read() { FutexEnd::Changed } else { FutexEnd::Timeout diff --git a/kernel/src/syscall/dispatch.rs b/kernel/src/syscall/dispatch.rs index ff4fb2f1ccb..5f540033640 100644 --- a/kernel/src/syscall/dispatch.rs +++ b/kernel/src/syscall/dispatch.rs @@ -70,13 +70,7 @@ fn task_probes() { pub(crate) fn syscall_dispatch(num: u64, a1: u64, a2: u64, a3: u64, a4: u64) -> u64 { #[cfg(feature = "boot-actuators")] task_probes(); - let t0 = crate::clock::nanos_since_boot(); - - process::with_current_data(|data| { - data.syscall_total += 1; - // Clamped, not guarded: an unissued number still lands in the last bin, not nowhere. - data.syscall_counts[(num as usize).min(toyos_abi::syscall::SYSCALL_PROFILE_OTHER)] += 1; - }); + let entered = crate::arch::cpu::counter(); // SAFETY: current process's page tables remain active for the duration of this call. let ctx = unsafe { SyscallContext::new() }; @@ -662,10 +656,8 @@ pub(crate) fn syscall_dispatch(num: u64, a1: u64, a2: u64, a3: u64, a4: u64) -> crate::object::drain_zero_handles(); // Wall-clock, not CPU time: elapsed includes any preemption. - let elapsed = crate::clock::nanos_since_boot() - t0; - process::with_current_data(|data| { - data.syscall_total_ns += elapsed; - }); + let elapsed = crate::arch::cpu::counter().saturating_sub(entered); + crate::sched::driver::note_syscall(num, elapsed); result } diff --git a/kernel/src/syscall/io.rs b/kernel/src/syscall/io.rs index 4a6a8d560fd..3ce765d996b 100644 --- a/kernel/src/syscall/io.rs +++ b/kernel/src/syscall/io.rs @@ -23,7 +23,7 @@ use super::handles::with_object_ref; /// What `sys_write` does when the object took nothing. enum WriteBlock { /// The pipe, the class of the wait, and the room the write waits for. - Pipe(pipe::PipeId, WaitClass, usize), + Pipe(pipe::PipeRef, WaitClass, usize), Refused(u64), /// Carried out of the process's lock: `HandleError::refuse` may take the /// process down and cannot run under a guard. @@ -32,7 +32,7 @@ enum WriteBlock { /// What `sys_read` parks on when the handle has nothing to give. enum ReadBlock { - Pipe(alloc::sync::Arc, pipe::PipeId, WaitClass), + Pipe(pipe::PipeRef, WaitClass), /// A notice with no reason in it, until a quit puts one there. Notice(alloc::sync::Arc), /// A console, which reads the serial line and nothing else: nothing posts @@ -53,36 +53,26 @@ pub(super) fn sys_write(h: RawHandle, buf: &UserBytes) -> u64 { Err(e) => return Err(WriteBlock::BadHandle(e)), }; match ops::try_write(object, buf) { - Some(n) => Ok((n, ops::pipe_id_write(object))), + Some(n) => Ok((n, ops::written_pipe(object))), None => Err(match ops::pipe_write(object) { - Some((id, class, landing)) => WriteBlock::Pipe(id, class, landing.room(buf.len())), + Some((pipe, class, landing)) => WriteBlock::Pipe(pipe.clone(), class, landing.room(buf.len())), None => WriteBlock::Refused(SyscallError::NotFound.to_u64()), }), } }); match action { - Ok((n, pipe_id)) => { - if let Some(id) = pipe_id { process::wake_pipe_readers(id); } + Ok((n, pipe)) => { + if let Some(pipe) = pipe { process::wake_pipe_readers(&pipe); } return n; } - Err(WriteBlock::Pipe(id, class, room)) => match pipe::write_watch(id) { - Some(end) => { - let parkable = crate::scheduler::Parkable::at_entry(); - if watch::wait_until( - &parkable, - &end, - 0, - class, - Deadline::never(), - || pipe::has_space(id, room), - ) + Err(WriteBlock::Pipe(pipe, class, room)) => { + let parkable = crate::scheduler::Parkable::at_entry(); + if watch::wait_until(&parkable, pipe.write_watch(), 0, class, Deadline::never(), || pipe.has_space(room)) .is_err() - { - return cancelled(); - } + { + return cancelled(); } - None => return SyscallError::NotFound.to_u64(), - }, + } Err(WriteBlock::Refused(word)) => return word, Err(WriteBlock::BadHandle(e)) => return e.refuse(), } @@ -101,9 +91,7 @@ fn read_block(object: &KObjectRef) -> ReadBlock { ); ReadBlock::Console(Deadline::at(crate::clock::now() + CONSOLE_REPOLL.duration())) } - _ => match ops::pipe_read(object).and_then(|(id, class)| { - pipe::read_watch(id).map(|end| ReadBlock::Pipe(end, id, class)) - }) { + _ => match ops::pipe_read(object).map(|(pipe, class)| ReadBlock::Pipe(pipe.clone(), class)) { Some(block) => block, None => ReadBlock::Refused(SyscallError::NotFound.to_u64()), }, @@ -132,26 +120,19 @@ pub(super) fn sys_read(h: RawHandle, buf: &mut UserBytesMut) -> u64 { }; } match ops::try_read(object, buf) { - Some(n) => Ok((n, ops::pipe_id_read(object))), + Some(n) => Ok((n, ops::read_pipe(object))), None => Err(read_block(object)), } }); match action { - Ok((n, pipe_id)) => { - if let Some(id) = pipe_id { process::wake_pipe_writers(id); } + Ok((n, pipe)) => { + if let Some(pipe) = pipe { process::wake_pipe_writers(&pipe); } return n; } - Err(ReadBlock::Pipe(end, id, class)) => { + Err(ReadBlock::Pipe(pipe, class)) => { let parkable = crate::scheduler::Parkable::at_entry(); - if watch::wait_until( - &parkable, - &end, - 0, - class, - Deadline::never(), - || pipe::has_data(id), - ) - .is_err() + if watch::wait_until(&parkable, pipe.read_watch(), 0, class, Deadline::never(), || pipe.has_data()) + .is_err() { return cancelled(); } @@ -205,30 +186,30 @@ pub(super) fn sys_read_nonblock(h: RawHandle, buf: &mut UserBytesMut) -> u64 { .handles .get::(h, Rights::READ) .expect("a Device resolved a moment ago under this same hold"); - return Ok((ops::read_device(&claim, &mut data.handles, buf), None)); + return Ok(ops::read_device(&claim, &mut data.handles, buf).map(|n| (n, None))); } - Ok((ops::try_read(object, buf), ops::pipe_id_read(object))) + Ok(ops::try_read(object, buf).map(|n| (n, ops::read_pipe(object)))) }); match result { - Ok((Some(n), wake)) => { - if let Some(id) = wake { process::wake_pipe_writers(id); } + Ok(Some((n, wake))) => { + if let Some(pipe) = wake { process::wake_pipe_writers(&pipe); } n } - Ok((None, _)) => SyscallError::WouldBlock.to_u64(), + Ok(None) => SyscallError::WouldBlock.to_u64(), Err(e) => e.refuse(), } } pub(super) fn sys_write_nonblock(h: RawHandle, buf: &UserBytes) -> u64 { let result = with_object_ref(h, Rights::WRITE, |object| { - (ops::try_write(object, buf), ops::pipe_id_write(object)) + ops::try_write(object, buf).map(|n| (n, ops::written_pipe(object))) }); match result { - Ok((Some(n), wake)) => { - if let Some(id) = wake { process::wake_pipe_readers(id); } + Ok(Some((n, wake))) => { + if let Some(pipe) = wake { process::wake_pipe_readers(&pipe); } n } - Ok((None, _)) => SyscallError::WouldBlock.to_u64(), + Ok(None) => SyscallError::WouldBlock.to_u64(), Err(e) => e.refuse(), } } diff --git a/kernel/src/syscall/ipc.rs b/kernel/src/syscall/ipc.rs index 41a0669e100..efb4fd791db 100644 --- a/kernel/src/syscall/ipc.rs +++ b/kernel/src/syscall/ipc.rs @@ -404,11 +404,11 @@ pub(super) fn sys_message_send( } // A refusal restores every handle at its own number, so it is invisible. let moved = data.handles.transfer(wanted, |batch| conn.send(bytes, batch)); - Ok::<_, crate::object::Refusal>((conn.tx(), moved)) + Ok::<_, crate::object::Refusal>((conn.tx().clone(), moved)) }); let tx = match sent { Ok((tx, Ok(()))) => { - process::wake_pipe_readers(tx); + process::wake_pipe_readers(&tx); return 0; } Ok((tx, Err(SendRefused::NoRoom))) if !nonblock => tx, @@ -417,10 +417,9 @@ pub(super) fn sys_message_send( Err(e) => return e.refuse(), }; // Outside every lock: room appears as the reader drains, and its read posts this watch. - let Some(watch) = pipe::write_watch(tx) else { return SyscallError::Gone.to_u64() }; let len = bytes.len(); - if watch::wait_until(&crate::scheduler::Parkable::at_entry(), &watch, 0, WaitClass::Ipc, Deadline::never(), || { - pipe::has_space(tx, len) + if watch::wait_until(&crate::scheduler::Parkable::at_entry(), tx.write_watch(), 0, WaitClass::Ipc, Deadline::never(), || { + tx.has_space(len) }) .is_err() { diff --git a/sdk/abi/src/clock.rs b/sdk/abi/src/clock.rs index 83ed14457a4..8a3b6df8d64 100644 --- a/sdk/abi/src/clock.rs +++ b/sdk/abi/src/clock.rs @@ -48,8 +48,20 @@ impl ClockPage { /// Saturating below boot: a counter read on a CPU that trails reads as the /// oldest instant, never as a wrap 584 years ahead. pub const fn nanos_between(counter_at_boot: u64, period_fs: u64, now: u64) -> u64 { - let delta = now.saturating_sub(counter_at_boot); - ((delta as u128 * period_fs as u128) / 1_000_000) as u64 + ticks_to_nanos(now.saturating_sub(counter_at_boot), period_fs) +} + +/// `ticks` of a counter whose tick is `period_fs` femtoseconds, in nanoseconds: +/// `⌊ticks · period_fs / 10⁶⌋` in its low 64 bits, the one formula the kernel +/// and every clock-page reader share. +/// +/// Split at 10⁶ ticks, `⌊(q·10⁶ + r)·p / 10⁶⌋ = q·p + ⌊r·p / 10⁶⌋`, so it +/// takes two 64-bit divisions by a constant, which compile to multiplications, +/// in place of a 128-bit division, which is a library call. Exact for any tick +/// under 18 ms (`10⁶ · period_fs < 2⁶⁴`). +pub const fn ticks_to_nanos(ticks: u64, period_fs: u64) -> u64 { + const FS_PER_NS: u64 = 1_000_000; + (ticks / FS_PER_NS).wrapping_mul(period_fs).wrapping_add(ticks % FS_PER_NS * period_fs / FS_PER_NS) } /// Nanoseconds since boot, off this process's clock page. @@ -103,6 +115,28 @@ mod tests { assert_eq!(nanos_between(0, 1_000_000, u64::MAX), u64::MAX); } + /// The split formula is the 128-bit product's quotient for every period a + /// counter has, at every count: the ends, the split's own boundary, and a + /// spread between. + #[test] + fn the_split_is_the_wide_quotient() { + let wide = |ticks: u64, fs: u64| ((ticks as u128 * fs as u128) / 1_000_000) as u64; + let mut x = 0x2545_f491_4f6c_dd1du64; + let mut next = || { + x = x.wrapping_mul(6_364_136_223_846_793_005).wrapping_add(1_442_695_040_888_963_407); + x + }; + let mut periods = std::vec![1, 999_999, 1_000_000, 1_000_001, 41_666_666, 413_393, 18_000_000_000_000]; + periods.extend((0..64).map(|_| 1 + next() % 100_000_000)); + let mut ticks = std::vec![0, 1, 999_999, 1_000_000, 1_000_001, u64::MAX - 1, u64::MAX]; + ticks.extend((0..256).map(|_| next())); + for &fs in &periods { + for &t in &ticks { + assert_eq!(ticks_to_nanos(t, fs), wide(t, fs), "{t} ticks of {fs} fs"); + } + } + } + #[test] fn the_page_is_four_words() { assert_eq!(core::mem::size_of::(), 32); From 37895ef52f792d81b2dfd9cf23bf95381938f817 Mon Sep 17 00:00:00 2001 From: japabu Date: Sat, 24 Oct 2026 08:26:25 +0200 Subject: [PATCH 4/7] sched: a post kicks after its list lock, once per CPU, and a sub-floor deadline is not slept past The T14 A/B of b214ecb87 against 32046264b read two regressions while every neighbouring row improved. `time.sleep_late.1us` went from 2711 to 11847 ns. The branch's shorter block pass now reaches the idle handshake before a 1 us deadline falls due, and the one-shot is armed for its 10 us floor (`MIN_ONE_SHOT`): 11.8 us is that floor plus a wake. The base's slower pass found the deadline already due in the idle loop's pass. An idle CPU whose armed deadline is nearer than the floor now passes again instead of halting, so the deadline is met by the pass that finds it due, at most a floor's worth of passes later. `wake.storm.call` (one futex wake of 4, 16 or 64 waiters) went to 2.2x, 1.5x and 1.2x while its drain improved. Inferred, not measured: the woken CPUs now clear their doorbell edge before the waker's next post to them, so a later ring raises the edge again and the waker sends a second IPI to a CPU that is already awake, each kick under the futex bucket's lock while the woken threads wait on it to unregister. A post now holds its kicks (`park::Deferred`) and sends them when its list lock is let go; a second ring to a CPU whose kick is still owed elides, so each CPU takes one IPI per post. On TCG the w4 call's IPIs fell from 2.27 to 1.93 per call. Co-Authored-By: Claude Opus 5.5 --- ...s-a-user-copy-and-its-first-allocation.md} | 0 ...-a-syscall-counts-itself-on-its-thread.md} | 0 kernel/pure/sched/park.rs | 46 ++++++++++++++++- kernel/pure/sched/watch.rs | 50 +++++++++++++++++-- kernel/src/sched/driver.rs | 6 ++- 5 files changed, 95 insertions(+), 7 deletions(-) rename issues/{the-global-pipe-lock-spans-a-user-copy.md => a-pipes-lock-spans-a-user-copy-and-its-first-allocation.md} (100%) rename issues/{every-syscall-takes-the-process-table-lock-twice-to-count-itself.md => syscall-cost-is-unread-since-a-syscall-counts-itself-on-its-thread.md} (100%) diff --git a/issues/the-global-pipe-lock-spans-a-user-copy.md b/issues/a-pipes-lock-spans-a-user-copy-and-its-first-allocation.md similarity index 100% rename from issues/the-global-pipe-lock-spans-a-user-copy.md rename to issues/a-pipes-lock-spans-a-user-copy-and-its-first-allocation.md diff --git a/issues/every-syscall-takes-the-process-table-lock-twice-to-count-itself.md b/issues/syscall-cost-is-unread-since-a-syscall-counts-itself-on-its-thread.md similarity index 100% rename from issues/every-syscall-takes-the-process-table-lock-twice-to-count-itself.md rename to issues/syscall-cost-is-unread-since-a-syscall-counts-itself-on-its-thread.md diff --git a/kernel/pure/sched/park.rs b/kernel/pure/sched/park.rs index 45eb91237a6..f0bcc04fe28 100644 --- a/kernel/pure/sched/park.rs +++ b/kernel/pure/sched/park.rs @@ -34,7 +34,7 @@ use core::marker::PhantomData; use crate::sched::cpu::CpuHandles; use crate::sched::hw::{CpuId, Kicker}; use crate::sched::mailbox::{Kick, PreemptGuard, SchedMsg, Urgency}; -use crate::sched::sync::Arc; +use crate::sched::sync::{Arc, AtomicU32, Ordering}; use crate::sched::task::{Gen, Notify, ParkOutcome, Refused, TaskShared, WaitClass, WakeCause}; /// The running task, as the wait path sees it. Only `CpuSched::current_task` @@ -134,6 +134,50 @@ fn deliver( outcome } +/// The kicks one post owes, sent when it is dropped: after the post's list +/// lock is let go, and once per CPU the post's rings found asleep. A post that +/// wakes two threads parked on one CPU rings it twice, and the second ring +/// elides because the first ring's kick is still unsent, so that CPU takes one +/// IPI and is not woken between the two. A kick to the calling CPU, which is +/// its own `need_resched`, and a kick past [`DEFERRED`] go at once. +pub struct Deferred<'k, K: Kicker> { + kicker: &'k K, + owed: [AtomicU32; DEFERRED], + len: AtomicU32, +} + +/// How many CPUs one post holds a kick for. +pub const DEFERRED: usize = 16; + +impl<'k, K: Kicker> Deferred<'k, K> { + pub fn new(kicker: &'k K) -> Self { + Self { kicker, owed: core::array::from_fn(|_| AtomicU32::new(0)), len: AtomicU32::new(0) } + } +} + +impl Kicker for Deferred<'_, K> { + fn kick(&self, target: CpuId) { + let at = self.len.load(Ordering::Relaxed) as usize; + if self.kicker.runs_on(target) || at == DEFERRED { + return self.kicker.kick(target); + } + self.owed[at].store(target.0, Ordering::Relaxed); + self.len.store(at as u32 + 1, Ordering::Relaxed); + } + + fn runs_on(&self, cpu: CpuId) -> bool { + self.kicker.runs_on(cpu) + } +} + +impl Drop for Deferred<'_, K> { + fn drop(&mut self) { + for owed in &self.owed[..self.len.load(Ordering::Relaxed) as usize] { + self.kicker.kick(CpuId(owed.load(Ordering::Relaxed))); + } + } +} + /// Registered on its own word, not yet parked. `!Send` (the registration /// belongs to the CPU that made it) and drop-bombed: exactly one of /// [`WaitTicket::cancel`] and [`WaitTicket::commit`] must consume it. diff --git a/kernel/pure/sched/watch.rs b/kernel/pure/sched/watch.rs index 3d0ec764fc0..a15a8e6ba7c 100644 --- a/kernel/pure/sched/watch.rs +++ b/kernel/pure/sched/watch.rs @@ -38,7 +38,7 @@ use core::marker::PhantomData; use crate::sched::cpu::CpuHandles; use crate::sched::hw::Kicker; use crate::sched::mailbox::{PreemptGuard, SchedMsg}; -use crate::sched::park::{notify, revoke}; +use crate::sched::park::{notify, revoke, Deferred}; use crate::sched::sync::{fence, Arc, AtomicU32, CellLock, Ordering}; use crate::sched::task::{TaskShared, WakeCause}; @@ -156,9 +156,10 @@ impl>> Watch { /// every ring entry, with the list lock let go. pub fn post(&self, cause: WakeCause, env: &Poster<'_, M, K, P>) { let mut few: [Option; FEW] = [const { None }; FEW]; + let kicks = Deferred::new(env.kicker); let many = self.list.with(|w| { for waiter in &w.threads { - notify(&waiter.task, cause, env.cpus, env.kicker, env.preempt); + notify(&waiter.task, cause, env.cpus, &kicks, env.preempt); } // [`FEW`] leave the list's buffer to the next registration. if w.rings.len() > FEW { @@ -171,6 +172,7 @@ impl>> Watch { } Vec::new() }); + drop(kicks); for ring in few.iter().flatten().chain(&many) { ring.fire(Fire::Ready); } @@ -184,9 +186,10 @@ impl>> Watch { cause: WakeCause, env: &Poster<'_, M, K, P>, ) { + let kicks = Deferred::new(env.kicker); self.list.with(|w| { for waiter in &w.threads { - notify(&waiter.task, cause, env.cpus, env.kicker, env.preempt); + notify(&waiter.task, cause, env.cpus, &kicks, env.preempt); } for ring in &w.rings { ring.fire(Fire::Ready); @@ -241,10 +244,11 @@ impl>> Watch { if limit == 0 { return 0; } + let kicks = Deferred::new(env.kicker); self.list.with(|w| { let mut woken = 0; for waiter in w.threads.iter().filter(|t| t.token == token) { - if notify(&waiter.task, cause, env.cpus, env.kicker, env.preempt).woke() { + if notify(&waiter.task, cause, env.cpus, &kicks, env.preempt).woke() { woken += 1; if woken == limit { break; @@ -266,6 +270,7 @@ impl>> Watch { cause: WakeCause, env: &Poster<'_, M, K, P>, ) -> usize { + let kicks = Deferred::new(env.kicker); self.list.with(|w| { let mut ended = 0; let mut at = 0; @@ -275,7 +280,7 @@ impl>> Watch { continue; } let waiter = w.threads.remove(at); - revoke(&waiter.task, cause, env.cpus, env.kicker, env.preempt); + revoke(&waiter.task, cause, env.cpus, &kicks, env.preempt); ended += 1; } ended @@ -533,6 +538,41 @@ mod tests { /// A bounded post spends nothing on a thread that was not parked, and /// reaches the next registered thread instead. + /// Records each kick and whether the list lock was held when it was sent. + struct LockedKicks<'w> { + watch: &'w TestWatch, + kicks: Mutex>, + } + impl Kicker for LockedKicks<'_> { + fn kick(&self, target: CpuId) { + let held = self.watch.list.0.try_lock().is_err(); + self.kicks.lock().unwrap().push((target, held)); + } + fn runs_on(&self, _cpu: CpuId) -> bool { + false + } + } + + /// **A post that wakes two threads parked on one CPU kicks it once, with + /// the list lock let go**: the second ring finds the first's edge raised + /// and its kick still owed. + #[test] + fn a_post_kicks_each_cpu_once_after_its_lock() { + let (handles, mut rx) = cpus(); + let w = watch(); + let kicker = LockedKicks { watch: &w, kicks: Mutex::new(vec![]) }; + let env = Poster { cpus: &handles, kicker: &kicker, preempt: &NoPreempt }; + let (a, b) = (task(1), task(2)); + for t in [&a, &b] { + w.register(t, 7); + parked(t); + } + assert_eq!(w.post_n(7, usize::MAX, woken(), &env), 2); + assert_eq!(*kicker.kicks.lock().unwrap(), vec![(C0, false)]); + assert_eq!(rx.pop(&NoPreempt), Some(Msg::Wake(TaskKey(1)))); + assert_eq!(rx.pop(&NoPreempt), Some(Msg::Wake(TaskKey(2)))); + } + #[test] fn a_bounded_post_skips_a_waiter_that_was_not_parked() { let (handles, mut rx) = cpus(); diff --git a/kernel/src/sched/driver.rs b/kernel/src/sched/driver.rs index 82c7e2f6ac8..df18f57a6b2 100644 --- a/kernel/src/sched/driver.rs +++ b/kernel/src/sched/driver.rs @@ -649,7 +649,11 @@ fn execute(action: Action) { let awake = cpus().get(cpu).doorbell().kick_pending() || crate::preempt::need_resched() // No log condition here: a log to write means a runnable process, covered above. - || !with_cpu(|c| c.mailbox_is_empty()); + || !with_cpu(|c| c.mailbox_is_empty()) + // The one-shot fires no sooner than its floor, so a deadline + // nearer than that is met by passing again, not by sleeping + // past it. + || token.armed().is_some_and(|at| at.0 < HW.now().0.saturating_add(crate::hw::MIN_ONE_SHOT.nanos())); if awake { crate::arch::cpu::enable_interrupts(); drop(token); From 799f4214da405cbd8de8577876311bc93a0c6597 Mon Sep 17 00:00:00 2001 From: japabu Date: Sat, 24 Oct 2026 08:26:26 +0200 Subject: [PATCH 5/7] docs: the push's fence pairs the offering mask, a pipe locks itself, and the records say what changed The fence documentation in `sched::cpu` named the surplus field as the store-buffer pair's location; the mask is. `Ring`'s doc named the deleted `PIPES` lock. The run queue's header claimed more than it holds about allocation, and the clock split's bound is on a tick's period. Issue records this change touched: the pipe-lock defect now describes a pipe's own lock and keeps its copy and allocation halves under a new slug; the per-syscall table lock is gone and only its `syscall_cost` readings are owed, under a new slug; the syscall-profile defect names `SyscallProfile`; the TSC-deadline track says its re-arm figures were the three-write arm's. Co-Authored-By: Claude Opus 5.5 --- ...ns-a-user-copy-and-its-first-allocation.md | 128 ++++-------------- ...processs-syscall-profile-is-one-threads.md | 8 +- ...ves-every-byte-through-two-kernel-pipes.md | 4 +- ...e-a-syscall-counts-itself-on-its-thread.md | 31 ++--- ...e-armed-every-pass-never-a-tsc-deadline.md | 14 +- ...s-what-modern-hardware-offers-for-speed.md | 7 +- kernel/pure/sched/cpu.rs | 16 +-- kernel/pure/sched/queue.rs | 7 +- sdk/abi/src/clock.rs | 2 +- sdk/abi/src/ring.rs | 2 +- 10 files changed, 70 insertions(+), 149 deletions(-) diff --git a/issues/a-pipes-lock-spans-a-user-copy-and-its-first-allocation.md b/issues/a-pipes-lock-spans-a-user-copy-and-its-first-allocation.md index b6d99fa0c33..6a3335d8493 100644 --- a/issues/a-pipes-lock-spans-a-user-copy-and-its-first-allocation.md +++ b/issues/a-pipes-lock-spans-a-user-copy-and-its-first-allocation.md @@ -4,111 +4,41 @@ kind: defect opened: 2026-08-28 --- -# `pipe::try_read`/`write` hold the one global `PIPES` spinlock across a 2 MiB user copy, and a pipe's first write adds a bitmap scan and a 2 MiB zeroing under it +# A pipe's lock spans a 2 MiB user copy, and a pipe's first write allocates and zeroes 2 MiB under it -Every pipe and every IPC `Connection` in the machine lives behind one static, -`PIPES: Lock>>` (`kernel/src/pipe.rs`). -`with_pipes_mut` and `with_pipes` run their whole closure under it, and -`sync::Lock::lock` disables preemption before it spins (`kernel/src/sync.rs`), -so the holder is non-preemptible and every other CPU that wants a pipe spins -with preemption off too. +Each pipe owns its state behind its own `sync::Lock` (`kernel/src/pipe.rs`), +which disables preemption before it spins (`kernel/src/sync.rs`). The bulk copy +runs inside that lock: -## The mechanism +* `PipeRef::try_read_within` runs `ring.read(len, |off, src| buf.write_run(off, &src))` + under it; +* `PipeRef::write` runs `ring.write(n, |off, dst| buf.read_run(off, dst))` in + the same acquisition. -The bulk copy is inside that closure, not outside it. +The copy is bounded only by the ring: `PIPE_SIZE` is `PAGE_2M` less the 64-byte +`RingHeader`, so **2,097,088 bytes** is the largest single copy under the lock, +and nothing above caps it (`SYS_READ`/`SYS_WRITE` pass the caller's length +through). Each 16 pages of it take the caller's space lock and populate any +untouched page, a frame from the allocator and a 4 KiB zeroing. -* `pipe::try_read_within`, which `try_read` calls, runs - `ring.read(len, |off, src| buf.write_run(off, &src))` inside - `with_pipes_mut`. -* `pipe::write` runs `backing.ring.write(n, |off, dst| buf.read_run(off, dst))` - in the same acquisition. -* `Ring::read`/`Ring::write` (`sdk/abi/src/ring.rs`) hand the closure one or two - contiguous runs, and the closure copies each whole. +A pipe's **first** write is worse: `State::back` allocates the ring page lazily +under the same lock, nesting the frame allocator's lock and a 2 MiB zeroing. -The size is bounded only by the ring: `PIPE_SIZE` is `PAGE_2M`, and the -capacity is that less the 64-byte `RingHeader`, so **2,097,088 bytes** is the -largest single copy under the lock. Nothing above caps it: `SYS_READ` and -`SYS_WRITE` pass the caller's length through (`kernel/src/syscall/dispatch.rs`), -and `object::ops::try_read` hands the whole buffer to `pipe::try_read`, which -copies it under the lock. Each 16 pages of that copy take the caller's space -lock and populate any page not yet touched — a frame from the allocator and a -4 KiB zeroing — inside `PIPES`, where the window used to be filled before the -lock was taken. - -A pipe's **first** write is worse, because the page is allocated lazily under the -same lock. `write` calls `Pipe::back`, which calls `pmm::alloc_page`: the frame -allocator's lock nested inside `PIPES` for a free superframe, then a 2 MiB -zeroing (`kernel/src/mm/pmm.rs`), before `Ring::new` and the user -copy that follows it. - -## What queues behind it - -Everything pipe-shaped in the kernel goes through the same static: `create`, -`has_data` and `has_space` — which the blocking read and write -wrappers call per attempt (`kernel/src/syscall/io.rs`) — every -`PipeReader`/`PipeWriter` clone and close, and the read and write watches. A -compositor client, netstack, logkeeper and the shell all contend the same word. - -## Impact, stated at its real size - -This is **not** the device-I/O class. The copy is memory-speed and takes no -device round trip, so it does not approach `sync::Lock`'s 500,000,000-spin -`DEADLOCK` panic the way the kernel's block device lock did before the block -layer left the kernel. - -What it is: a global lock held with preemption disabled for a stretch that -scales with a userland-chosen length up to 2 MiB, on the path that carries every -IPC message in the system. The scheduler's own bar for an uninterruptible -stretch is `MAX_PASS_NS = 200_000` ns (`kernel/pure/sched/cpu.rs`); a 2 MiB copy -plus, on first write, a 2 MiB zeroing and a bitmap scan is on the wrong side of -that at any plausible memory bandwidth. **The hold time is not measured and that -is the first thing owed.** - -Two things make it worth closing rather than noting. The convoy is machine-wide, -not per-pipe: one process reading 2 MiB out of a full pipe stalls an unrelated -process's `SYS_CONNECT` and an unrelated reader's readiness check. And it is -directly userland-driven — the length is the caller's, so a program that wants -the stall only has to ask for it. - -## Precondition / repro - -1. Create a pipe or connection; write until the ring is full (2,097,088 bytes). -2. From the reader, `SYS_READ` with a 2 MiB buffer. -3. From another CPU, time an unrelated `pipe::has_data` or `pipe::create`; it - waits for the whole copy. - -For the allocation half: `SYS_CONNECT` then a single first `SYS_WRITE` of one -byte runs the scan and the 2 MiB zeroing under `PIPES` before one byte moves. +What queues behind it is that one pipe's traffic: its other end's reads and +writes, `has_data`/`has_space` from its blocking wrappers and polls, and its +handles' clones and closes. The scheduler's own bar for an uninterruptible +stretch is `MAX_PASS_NS = 200_000` ns (`kernel/pure/sched/cpu.rs`); a 2 MiB copy, +and on first write a 2 MiB zeroing, is past it at any plausible bandwidth. The +hold time is not measured, and that is owed first. The length is the caller's. ## Fix direction -`PIPES` is doing two jobs and only one of them needs the global. It is the table -lock, and it is also the serialization `Ring` relies on — `ring.rs`'s doc says -"Every access is serialized by the caller's lock (`kernel::pipe`'s `PIPES`)", -which is why the cursors are plain integers rather than atomics. So the copy -cannot simply be lifted out; the ring's data-race argument has to move with it. - -The shape that keeps that argument true is a per-pipe lock under the table lock: -`PIPES` guards the `IdMap` and the refcounts, a `Lock` (or a claimed-range token) -inside `Pipe` guards `Backing`, and the table lock is dropped once the per-pipe -guard is in hand. `try_read`/`write` then hold only the pipe's own lock across -`Ring::read`/`Ring::write`, and `has_data`/`has_space`/`create`/`clone`/`close` -stop queueing behind a stranger's copy. `Ring`'s doc sentence changes with it. - -The lazy allocation should move out from under both: allocate the ring page -before taking the table lock (or, on the miss, drop and retry the way -`file_cache::read_page` does), so neither the bitmap scan nor the 2 MiB zeroing -runs with a global held. - -Sequencing note: this touches `sdk/abi/src/ring.rs`'s doc contract, so it lands -as one commit over `sdk/abi/src` and `kernel/src` together. - -## Two checks +The lock is also the serialization `Ring` relies on (`sdk/abi/src/ring.rs`'s +doc), so the copy cannot simply be lifted out: a claimed-range token per end, +or a copy bounded per acquisition, has to carry the ring's data-race argument. +The lazy allocation moves out from under the lock: allocate before taking it, +or drop and retry on the miss. -* **Negative control:** revert the whole change onto the base and re-run the - contention measurement; the per-CPU wait on an unrelated pipe operation must - return to the pre-change value. -* **Independent oracle:** `kernel-loom`'s lock models for the new two-level - acquisition (table then pipe), which is a different epistemic source from the - kernel's own test suite; plus `audio_tone`'s worst-wake reading, a recorded - real-failure oracle rather than a re-derivation. +**Exit**: no pipe lock is held across more than a bounded copy or any +allocation, the hold time is read before and after, and `audio_tone`'s worst +wake does not move. diff --git a/issues/a-processs-syscall-profile-is-one-threads.md b/issues/a-processs-syscall-profile-is-one-threads.md index 8e9a7673848..acb6a73a511 100644 --- a/issues/a-processs-syscall-profile-is-one-threads.md +++ b/issues/a-processs-syscall-profile-is-one-threads.md @@ -6,10 +6,10 @@ opened: 2026-09-04 # The exit record's `syscalls=` is one thread's counts under a process's name -`ThreadData` holds `syscall_counts`, `syscall_total` and `syscall_total_ns`, -and `teardown_resources` (`kernel/src/process.rs`) reads them from the one -thread's data `teardown` hands it, the main thread's, and they go out as the -process's: in its exit record (`exit: pid=N code=N cpu=Nms peak=… allocs=… +Each thread's `TaskHandle` holds its `SyscallProfile` +(`kernel/src/sched/payload.rs`), and `teardown_resources` +(`kernel/src/process.rs`) reads only the main thread's, which `teardown` hands +it, and sends it out as the process's: in its exit record (`exit: pid=N code=N cpu=Nms peak=… allocs=… frees=… syscalls=N syscall_wall=Nms = …`) and in the `ProcessStats` its exit publishes. Every other thread's calls are silently dropped. diff --git a/issues/netstack-moves-every-byte-through-two-kernel-pipes.md b/issues/netstack-moves-every-byte-through-two-kernel-pipes.md index 11999833818..809d4c84708 100644 --- a/issues/netstack-moves-every-byte-through-two-kernel-pipes.md +++ b/issues/netstack-moves-every-byte-through-two-kernel-pipes.md @@ -9,8 +9,8 @@ opened: 2026-10-11 A received byte is copied three times between the card's buffer and the application's: netstack writes it into its connection's heap ring (`system/netstack/shard/tcp/src/ring.rs`), then into the client's pipe with a -`SYS_WRITE` whose copy runs under the one global `PIPES` lock with preemption -off (`kernel/src/pipe.rs`, `issues/the-global-pipe-lock-spans-a-user-copy.md`), +`SYS_WRITE` whose copy runs under that pipe's lock with preemption off +(`kernel/src/pipe.rs`, `issues/a-pipes-lock-spans-a-user-copy-and-its-first-allocation.md`), and the client's `SYS_READ` copies it out under the same lock. A sent byte is copied into the pipe, out of it into netstack's stack chunk, into the connection's send ring, and into the frame. Both buffers hold the same bytes: a diff --git a/issues/syscall-cost-is-unread-since-a-syscall-counts-itself-on-its-thread.md b/issues/syscall-cost-is-unread-since-a-syscall-counts-itself-on-its-thread.md index bb6de17ffca..ba0b3eba7ca 100644 --- a/issues/syscall-cost-is-unread-since-a-syscall-counts-itself-on-its-thread.md +++ b/issues/syscall-cost-is-unread-since-a-syscall-counts-itself-on-its-thread.md @@ -4,27 +4,16 @@ kind: defect opened: 2026-10-12 --- -# Every syscall takes the process table lock twice to count itself +# `syscall_cost` is unread since a syscall counts itself on its thread -`syscall_dispatch` (`kernel/src/syscall/dispatch.rs`) calls -`process::with_current_data` before the call, to bump `syscall_total` and -`syscall_counts`, and again after it, to add `syscall_total_ns`. Each one takes -`PROCESS_TABLE`'s lock, clones the thread's `Arc`, takes that thread's -`Lock` and drops the `Arc` (`current_data` and `with_current_data` -in `kernel/src/process.rs`). That is twelve locked read-modify-writes on every -syscall, `SYS_GETPID` included: per call, two ticket acquisitions and their two -releases, and the `Arc`'s increment and decrement. It also takes the one -machine-wide table lock, which every thread of every process contends for, to -update a counter that belongs to one thread. +`syscall_dispatch` (`kernel/src/syscall/dispatch.rs`) took `PROCESS_TABLE`'s +lock twice per syscall, cloned the thread's `Arc` and took its `ThreadData` +lock, to update counters that belong to one thread. Since #1061 a syscall +counts itself on its thread's `TaskHandle` (`SyscallProfile`, written by that +thread with a load and a store) and takes no machine-wide lock and no `Arc`. -Counted from the code. Priced at the per-prefix cost the Ring 3 controls of PR -#866's scout measured, 15 TSC cycles on the T14 and 5–6 on the AMD bench -(https://github.com/ToyOSOrg/ToyOS/pull/866#issuecomment-6119438569), the twelve -come to about 180 and 66 cycles. That is an estimate, not a measurement of -these sites. +Read on the T14 in #1061's A/B (one osbench boot per side): `syscall.null` +fell to 0.42× of `main`'s. The AMD bench's reading and the `syscall_cost` +row on both benches are still owed. -Owner: `issues/toyos-uses-what-modern-hardware-offers-for-speed.md`. - -**Exit**: a syscall that returns to its own thread takes no machine-wide lock and -no `Arc` reference to count itself, and `syscall_cost` is read on both benches -before and after. +**Exit**: `syscall_cost` read on both benches before and after, posted. diff --git a/issues/the-timer-is-a-lapic-count-re-armed-every-pass-never-a-tsc-deadline.md b/issues/the-timer-is-a-lapic-count-re-armed-every-pass-never-a-tsc-deadline.md index badc6fa8a18..8624a725226 100644 --- a/issues/the-timer-is-a-lapic-count-re-armed-every-pass-never-a-tsc-deadline.md +++ b/issues/the-timer-is-a-lapic-count-re-armed-every-pass-never-a-tsc-deadline.md @@ -10,9 +10,9 @@ Every scheduler pass ends by handing its timer an absolute deadline (`kernel/pure/sched/cpu.rs:1912-1918`), and the kernel turns it back into a span from now (`kernel/src/hw.rs:47-51`) that `apic::arm_one_shot` converts to LAPIC ticks by the boot's calibration against the TSC-derived clock -(`kernel/src/arch/x86_64/apic.rs:307-336`) and writes as three APIC register -writes, `TIMER_DIVIDE`, `LVT_TIMER` and `TIMER_INIT` (`apic.rs:355-368`): -MMIO stores under xAPIC, `wrmsr` under x2APIC (`apic.rs:49-64`), which the +(`kernel/src/arch/x86_64/apic.rs`) and writes into `TIMER_INIT`, one APIC +register write since #1061 set the divide and the LVT once per CPU: an MMIO +store under xAPIC, a `wrmsr` under x2APIC, which the T14 runs (`apic=X2apic`, https://github.com/ToyOSOrg/ToyOS/pull/861#issuecomment-6113257693). A timer fire from Ring 3 writes `TIMER_INIT` once more before its handler runs @@ -35,7 +35,9 @@ TSC, TSC-deadline or APIC-timer erratum (631123-034, the UP3 line, CPUID 000806C1h). Nothing but speed is its purpose: a deadline removes no path, since a CPU without one keeps the count. -**Measured** (`wt/toyos-scout-tlb` at `d4c2f8ebe`, `main`'s `177e72a9e` plus +**Measured before #1061**, when the arm was three register writes and a u128 +multiply and divide, so these re-arm figures are the old code's and a twin +compares against the arm read again (`wt/toyos-scout-tlb` at `d4c2f8ebe`, `main`'s `177e72a9e` plus the scout; https://github.com/ToyOSOrg/ToyOS/pull/866#issuecomment-6119443198), p50 in TSC ticks per direction of a user ping-pong between CPUs 4 and 5: @@ -51,8 +53,8 @@ which a pass with no idle wake runs too, and 27.8 % and 31.3 % of the switch it sits in; the re-arm and the stop together are 5.5 % and 5.6 % of an idle wake (derived). The scout's re-arm opens at `arm_one_shot`'s first statement, after -`set_timer`'s `now()` (`kernel/src/hw.rs:50`), an `rdtsc` and a u128 multiply -and divide (`kernel/pure/clock.rs:10-11`): the count reads the clock to turn +`set_timer`'s `now()` (`kernel/src/hw.rs:50`), an `rdtsc` and the tick +conversion: the count reads the clock to turn the deadline into a span, and a deadline written as an absolute `IA32_TSC_DEADLINE` value reads none. Of what the segment times, each figure is an upper bound on what deadline mode saves, which still writes once to arm diff --git a/issues/toyos-uses-what-modern-hardware-offers-for-speed.md b/issues/toyos-uses-what-modern-hardware-offers-for-speed.md index 8d802441428..7e5e643e1aa 100644 --- a/issues/toyos-uses-what-modern-hardware-offers-for-speed.md +++ b/issues/toyos-uses-what-modern-hardware-offers-for-speed.md @@ -43,8 +43,9 @@ and contention). - An MWAIT doorbell: the IPI's send and delivery and the kick's handler are at most 14.7 to 14.9 % of an idle wake on the T14 and 22.4 to 22.6 % on the AMD bench (`issues/idle-is-hlt-so-every-wake-of-an-idle-cpu-is-an-ipi.md`). -- A TSC deadline: the re-arm is at most 11.7 to 12.1 % of a pass's pick and - switch on the T14, and at the AMD bench's TSC floor +- A TSC deadline: before #1061 cut the arm to one register write, the + re-arm was at most 11.7 to 12.1 % of a pass's pick and switch on the T14, + and at the AMD bench's TSC floor; it is to be read again (`issues/the-timer-is-a-lapic-count-re-armed-every-pass-never-a-tsc-deadline.md`). **Moot.** PCID on the AMD bench: its CPUs read `pcid=0 invpcid=0`, so there @@ -95,5 +96,5 @@ yet measured. - `issues/fsgsbase-is-taken.md` - `issues/idle-cpus-enter-the-c-states-cst-names.md` - `issues/diskserver-drives-nvme-apst.md` -- `issues/every-syscall-takes-the-process-table-lock-twice-to-count-itself.md` +- `issues/syscall-cost-is-unread-since-a-syscall-counts-itself-on-its-thread.md` - `issues/a-thread-resuming-on-the-cpu-that-holds-its-state-restores-it-again.md` diff --git a/kernel/pure/sched/cpu.rs b/kernel/pure/sched/cpu.rs index cef0b980a07..11d752c4739 100644 --- a/kernel/pure/sched/cpu.rs +++ b/kernel/pure/sched/cpu.rs @@ -743,9 +743,9 @@ pub enum Balance { /// at `threshold:` [`PUSH_THRESHOLD`]. /// /// **It rests on an observation, and the observation is one half of - /// Dekker's pair** — see [`balance_fence`]. The pusher stores its surplus - /// and then loads the sleeper's SLEEPING bit; the sleeper stores SLEEPING - /// and then loads the surplus. Without a `SeqCst` fence between each side's + /// Dekker's pair** — see [`balance_fence`]. The pusher sets its bit in the + /// offering mask and then loads the sleeper's SLEEPING bit; the sleeper + /// stores SLEEPING and then loads the mask. Without a `SeqCst` fence between each side's /// store and its load, both may miss, and the CPU sleeps through a surplus /// exactly as it does under [`Balance::Pull`] — a narrower window on the /// same defect. `kernel/loom/tests/loom_push.rs` is the model. @@ -804,13 +804,13 @@ impl Balance { /// that gained surplus does /// /// ```text -/// surplus.store(n); ... doorbell.load() // "is anybody asleep?" +/// offering.fetch_or(bit); ... doorbell.load() // "is anybody asleep?" /// ``` /// /// and the CPU going to sleep does /// /// ```text -/// doorbell.store(SLEEPING); ... surplus.load() // "is there anything to steal?" +/// doorbell.fetch_or(SLEEPING); ... offering.load() // "is there anything to steal?" /// ``` /// /// Each side stores to one location and loads the other. With plain accesses @@ -2120,7 +2120,7 @@ impl SchedPass<'_, '_, H, P, Disposed> { let arm: SleepArm<'_> = self.env.cpus.get(self.cpu.id).doorbell().arm_sleep(); // [`Balance::PushOnSurplus`]'s other half, and the half that makes the // push an ordering rather than a hope: SLEEPING is published above, so - // reading the surplus *here*, behind the fence, is the load that pairs + // reading the offering mask *here*, behind the fence, is the load that pairs // with the pusher's read of that bit. `post_steal_probe`'s own read is // before the store and answers nothing about it. if self.probe_still_owed() { @@ -2160,8 +2160,8 @@ impl SchedPass<'_, '_, H, P, Disposed> { /// /// Only [`Balance::PushOnSurplus`] asks, and only to close its own Dekker /// window (see [`balance_fence`]): if the pusher missed our SLEEPING bit, we - /// must not also miss its surplus. Answering `true` sends the pass round - /// again, and the second trip's `post_steal_probe` reads the surplus behind + /// must not also miss its offering bit. Answering `true` sends the pass + /// round again, and the second trip's `post_steal_probe` reads the mask behind /// the same fence and posts — which is why this terminates: a probe in /// flight is an answer, so the question is asked at most once per halt. fn probe_still_owed(&self) -> bool { diff --git a/kernel/pure/sched/queue.rs b/kernel/pure/sched/queue.rs index be8689f08da..4ab4b8756aa 100644 --- a/kernel/pure/sched/queue.rs +++ b/kernel/pure/sched/queue.rs @@ -1,10 +1,9 @@ //! The per-CPU run queue: a real-time FIFO drained first, and the fair band. //! //! The fair band is EEVDF over the job groups this CPU holds, and EEVDF again -//! over each group's threads ([`Queue`] at both levels). Every entity lives in -//! one of this queue's two arenas, so once they have held as many a wake and a -//! block allocate nothing. A group's weight is its job's weight times the -//! group's share of the job's runnable threads +//! over each group's threads ([`Queue`] at both levels, their entities in this +//! queue's two arenas). A group's weight is its job's weight times the group's +//! share of the job's runnable threads //! ([`Job::group_weight`]), refreshed at each decision. Lag goes with whatever //! leaves the queue: a thread's to the next queue it enters, and a dissolving //! group's with its last thread to the group that thread next enters. There diff --git a/sdk/abi/src/clock.rs b/sdk/abi/src/clock.rs index 8a3b6df8d64..396aaa632c4 100644 --- a/sdk/abi/src/clock.rs +++ b/sdk/abi/src/clock.rs @@ -58,7 +58,7 @@ pub const fn nanos_between(counter_at_boot: u64, period_fs: u64, now: u64) -> u6 /// Split at 10⁶ ticks, `⌊(q·10⁶ + r)·p / 10⁶⌋ = q·p + ⌊r·p / 10⁶⌋`, so it /// takes two 64-bit divisions by a constant, which compile to multiplications, /// in place of a 128-bit division, which is a library call. Exact for any tick -/// under 18 ms (`10⁶ · period_fs < 2⁶⁴`). +/// period under 18 ms (`10⁶ · period_fs < 2⁶⁴`). pub const fn ticks_to_nanos(ticks: u64, period_fs: u64) -> u64 { const FS_PER_NS: u64 = 1_000_000; (ticks / FS_PER_NS).wrapping_mul(period_fs).wrapping_add(ticks % FS_PER_NS * period_fs / FS_PER_NS) diff --git a/sdk/abi/src/ring.rs b/sdk/abi/src/ring.rs index e2879f1e907..3e4439d73c1 100644 --- a/sdk/abi/src/ring.rs +++ b/sdk/abi/src/ring.rs @@ -104,7 +104,7 @@ impl Dst<'_> { /// Kernel-side owner of one ring region: the cursors, the capacity, and the /// base of the region they address. /// -/// Every access is serialized by the caller's lock (`kernel::pipe`'s `PIPES`), +/// Every access is serialized by the caller's lock (each `kernel::pipe` pipe's own), /// which is why the cursors are plain integers rather than atomics — the /// header's were only ever atomic because they lived in shared memory. pub struct Ring { From 2ff66c842b4156ffbb7c448448f2d09a20b8cda5 Mon Sep 17 00:00:00 2001 From: japabu Date: Sat, 24 Oct 2026 10:46:31 +0200 Subject: [PATCH 6/7] issues: a CPU leaving hlt pays a C1E exit, and malloc_churn t8 fell with the run-queue arenas The T14 diagnosis boots for #1061 placed `wake.storm.call`'s regression in the waker's first microseconds after its own wake from `hlt`: a 20 us spin before the call removes it (w4 10122 -> 3876 ns), and clearing C1E auto-promotion makes `main` and the branch equal (4486 and 4489 ns), with w1 at 841 ns against about 4 us. The firmware sets the promotion (MSR_POWER_CTL 0x24005b). That is the idle policy's cost, recorded under the C-state track that owns it. `malloc_churn.1048576.t8` stays below `main` in every same-session pairing, 7 % with C1E cleared, from the arenas commit on; its cause is not named and is recorded with the measurements and what the change buys. Co-Authored-By: Claude Opus 5.5 --- ...s-its-first-microseconds-slow-under-c1e.md | 35 +++++++++++++++++++ .../idle-cpus-enter-the-c-states-cst-names.md | 3 +- ...-threads-fell-with-the-run-queue-arenas.md | 35 +++++++++++++++++++ 3 files changed, 72 insertions(+), 1 deletion(-) create mode 100644 issues/a-cpu-leaving-hlt-runs-its-first-microseconds-slow-under-c1e.md create mode 100644 issues/malloc-churn-across-eight-threads-fell-with-the-run-queue-arenas.md diff --git a/issues/a-cpu-leaving-hlt-runs-its-first-microseconds-slow-under-c1e.md b/issues/a-cpu-leaving-hlt-runs-its-first-microseconds-slow-under-c1e.md new file mode 100644 index 00000000000..22a3931080f --- /dev/null +++ b/issues/a-cpu-leaving-hlt-runs-its-first-microseconds-slow-under-c1e.md @@ -0,0 +1,35 @@ +--- +status: open +kind: defect +opened: 2026-10-24 +--- + +# A CPU leaving `hlt` runs its first microseconds slow under C1E auto-promotion + +An idle CPU waits in `hlt` (`kernel/src/arch/x86_64/hw.rs`). The T14's firmware +leaves C1E auto-promotion on: every CPU reads `MSR_POWER_CTL = 0x24005b`, bit 1 +set. So every `hlt` is promoted to C1E, and a thread woken there runs its first +microseconds at a clock that has not come back yet. + +**Measured** on the T14 (#1061's diagnosis boots, osbench `wake.storm.call`, p50, ns): + +| | w1 | w4 | w16 | +|---|---|---|---| +| C1E on, the call at once after a 2 ms sleep | 3877–4200 | 10122 | 14152 | +| C1E on, the waker spinning 20 µs in Ring 3 first | — | 3876 | 7206 | +| C1E auto-promotion cleared | 841 | 4489 | 8674 | + +The C1E-cleared row comes from head `799f4214d`; base `32046264b` reads the same +there, with w4 at 4486. A faster wake path reaches the call sooner after its +own wake, so more of its work runs inside the exit. #1061's +`storm.call` regression against `main` is this, and not the call's work: no +diary record shows a pick, preemption or interrupt on the waker's CPU inside a +call. Linux on the same machine reads w4 at 11.6 µs. + +Owner: `issues/idle-cpus-enter-the-c-states-cst-names.md`, which chooses the +idle states; this is the cost of not choosing any. + +**Exit**: an idle policy that does not pay a C1E exit on a short idle, chosen +per Intel's documentation of `MSR_POWER_CTL` and the `_CST` states, measured on +the T14 by `wake.storm.call`, the `time.sleep_late` rows and the idle minute's +package energy against the current policy. diff --git a/issues/idle-cpus-enter-the-c-states-cst-names.md b/issues/idle-cpus-enter-the-c-states-cst-names.md index 9bf02f9a298..a4620ccc2ad 100644 --- a/issues/idle-cpus-enter-the-c-states-cst-names.md +++ b/issues/idle-cpus-enter-the-c-states-cst-names.md @@ -9,7 +9,8 @@ opened: 2026-09-29 The kernel runs no `mwait` and reads no `_CST`. The states come from `_CST`, as Linux takes the T14's, after the AML interpreter `issues/toyos-runs-the-machine-in-acpi-mode-and-interprets-its-aml.md` -records ToyOS lacks. +records ToyOS lacks. What `hlt` alone costs today, firmware's C1E promotion +included, is `issues/a-cpu-leaving-hlt-runs-its-first-microseconds-slow-under-c1e.md`. **Exit**: on the T14, since a KVM guest's idle is its host's: the idle minute's package energy, and timer lateness no worse than `hlt`'s by more than diff --git a/issues/malloc-churn-across-eight-threads-fell-with-the-run-queue-arenas.md b/issues/malloc-churn-across-eight-threads-fell-with-the-run-queue-arenas.md new file mode 100644 index 00000000000..81e9b702cbd --- /dev/null +++ b/issues/malloc-churn-across-eight-threads-fell-with-the-run-queue-arenas.md @@ -0,0 +1,35 @@ +--- +status: open +kind: defect +opened: 2026-10-24 +--- + +# `mem.malloc_churn.1048576.t8` fell with the run-queue arenas + +Eight threads churning 1 MiB blocks through std's allocator, which takes one +futex mutex per call (`sdk/std/sys/alloc.rs`), run fewer operations on the T14 +since #1061. The single-threaded row is flat. Measured in one session each +(kops/s, p50): + +| boot set | `main` 32046264b | run-queue arenas 21b60ba8b | #1061 head | +|---|---|---|---| +| bisect | 3889 | 3536 | 3357 (round-1 head) | +| diary instrument | 4419 | 3406 | 3234 | +| C1E auto-promotion cleared | 3677 | — | 3411 | + +About half of the drop goes with C1E +(`issues/a-cpu-leaving-hlt-runs-its-first-microseconds-slow-under-c1e.md`). +What is left, 7 % with C1E cleared, begins at the arenas commit. That commit +changes no scheduling decision by reading; it makes a wake and a block cheaper. +The diary shows scheduler events at well under one per hundred operations, with +no change in kind between `main` and the branch. The cause is not named. The +candidate is a contended mutex's handoff: a cheaper park leaves less time in which +a waiter finds the lock free before it parks. + +Against this, the same commits halve the futex, pipe and IPC round trips +(#1061). + +**Exit**: the cause named by a measurement, such as futex waits and wakes per +operation on `main` and the branch in one session, and then either the +handoff fixed where it lives or the cost accepted by the owner against what it +buys. From e763004fb7edf377c570e0611abbbb863cca6276 Mon Sep 17 00:00:00 2001 From: japabu Date: Sat, 24 Oct 2026 11:01:56 +0200 Subject: [PATCH 7/7] review r2: a test's doc back on its test, and the C1E and malloc_churn records say what was read The bounded-post test's doc had landed on LockedKicks. The C1E record's w1 cells were wrong: head reads 4393, 4819 and 4722 without the spin and 3877 with it, so the spin removes the slow mode at w4 and w16 only. Main's w16 rose under the C1E control in its one boot, and the record now says so. It also notes that a busy CPU's sub-floor deadline still arms the floor. The malloc_churn record says the branch reads the same with C1E on or cleared, and that the gap narrowed only because main fell. It notes one-boot cells and main's spread across sessions, and names the speed track as owner, which lists it. The syscall_cost record is a measurement owed, so its kind is tooling. Co-Authored-By: Claude Opus 5.5 --- ...s-its-first-microseconds-slow-under-c1e.md | 28 +++++++++++++------ ...-threads-fell-with-the-run-queue-arenas.md | 20 +++++++------ ...e-a-syscall-counts-itself-on-its-thread.md | 11 +++----- ...s-what-modern-hardware-offers-for-speed.md | 1 + kernel/pure/sched/watch.rs | 4 +-- 5 files changed, 38 insertions(+), 26 deletions(-) diff --git a/issues/a-cpu-leaving-hlt-runs-its-first-microseconds-slow-under-c1e.md b/issues/a-cpu-leaving-hlt-runs-its-first-microseconds-slow-under-c1e.md index 22a3931080f..3ad5428f655 100644 --- a/issues/a-cpu-leaving-hlt-runs-its-first-microseconds-slow-under-c1e.md +++ b/issues/a-cpu-leaving-hlt-runs-its-first-microseconds-slow-under-c1e.md @@ -11,20 +11,30 @@ leaves C1E auto-promotion on: every CPU reads `MSR_POWER_CTL = 0x24005b`, bit 1 set. So every `hlt` is promoted to C1E, and a thread woken there runs its first microseconds at a clock that has not come back yet. -**Measured** on the T14 (#1061's diagnosis boots, osbench `wake.storm.call`, p50, ns): +**Measured** on the T14 (#1061's diagnosis boots, osbench `wake.storm.call`, +p50 in ns, head `799f4214d` unless named; each cell is one boot): | | w1 | w4 | w16 | |---|---|---|---| -| C1E on, the call at once after a 2 ms sleep | 3877–4200 | 10122 | 14152 | -| C1E on, the waker spinning 20 µs in Ring 3 first | — | 3876 | 7206 | +| C1E on, the call at once after a 2 ms sleep | 4393, 4819, 4722 | 10122 | 14152 | +| C1E on, the waker spinning 20 µs in Ring 3 first | 3877 | 3876 | 7206 | | C1E auto-promotion cleared | 841 | 4489 | 8674 | +| `main` `32046264b`, C1E auto-promotion cleared | 871 | 4486 | 14481 | -The C1E-cleared row comes from head `799f4214d`; base `32046264b` reads the same -there, with w4 at 4486. A faster wake path reaches the call sooner after its -own wake, so more of its work runs inside the exit. #1061's -`storm.call` regression against `main` is this, and not the call's work: no -diary record shows a pick, preemption or interrupt on the waker's CPU inside a -call. Linux on the same machine reads w4 at 11.6 µs. +The spin removes the slow mode at w4 and w16, but not at w1. There the +C1E-cleared reading, 841 ns, is what the first syscall after a wake costs +without the exit. A faster wake path reaches the call sooner after its own +wake, so more of its work runs inside the exit. #1061's `storm.call` +regression against `main` at w4 is this, and not the call's work: no diary +record shows a pick, preemption or interrupt on the waker's CPU inside a call, +and with C1E cleared `main` and the branch read the same. At w16, `main` *rose* +under the control, from 9948 and 10490 with C1E on to 14481 cleared. That +reading is unexplained and comes from one boot, so the branch's lead there rests +on it. Linux on the same machine reads w4 at 11.6 µs. + +The fix that keeps a sub-floor deadline from being slept past covers idle CPUs +only. A busy CPU whose parked neighbour's deadline is less than the one-shot +floor away still arms the floor, which is unmeasured. Owner: `issues/idle-cpus-enter-the-c-states-cst-names.md`, which chooses the idle states; this is the cost of not choosing any. diff --git a/issues/malloc-churn-across-eight-threads-fell-with-the-run-queue-arenas.md b/issues/malloc-churn-across-eight-threads-fell-with-the-run-queue-arenas.md index 81e9b702cbd..806b90d662d 100644 --- a/issues/malloc-churn-across-eight-threads-fell-with-the-run-queue-arenas.md +++ b/issues/malloc-churn-across-eight-threads-fell-with-the-run-queue-arenas.md @@ -17,18 +17,22 @@ since #1061. The single-threaded row is flat. Measured in one session each | diary instrument | 4419 | 3406 | 3234 | | C1E auto-promotion cleared | 3677 | — | 3411 | -About half of the drop goes with C1E -(`issues/a-cpu-leaving-hlt-runs-its-first-microseconds-slow-under-c1e.md`). -What is left, 7 % with C1E cleared, begins at the arenas commit. That commit -changes no scheduling decision by reading; it makes a wake and a block cheaper. -The diary shows scheduler events at well under one per hundred operations, with -no change in kind between `main` and the branch. The cause is not named. The -candidate is a contended mutex's handoff: a cheaper park leaves less time in which -a waiter finds the lock free before it parks. +Each cell is one boot, and `main` itself spans 3677–4419 across sessions. The +branch reads the same with C1E on or cleared, 3406 and 3413 against 3411. The +gap narrows under the control only because `main` fell, from 3875–3889 to +3677. So C1E is not what costs the branch here. The drop begins at the +arenas commit, which changes no scheduling decision by reading; it makes a wake +and a block cheaper. The diary shows scheduler events at well under one per +hundred operations, with no change in kind between `main` and the branch. The +cause is not named. The candidate is a contended mutex's handoff: a cheaper park +leaves less time in which a waiter finds the lock free before it parks. Against this, the same commits halve the futex, pipe and IPC round trips (#1061). +Owner: `issues/toyos-uses-what-modern-hardware-offers-for-speed.md`, the speed +track the change was made under. + **Exit**: the cause named by a measurement, such as futex waits and wakes per operation on `main` and the branch in one session, and then either the handoff fixed where it lives or the cost accepted by the owner against what it diff --git a/issues/syscall-cost-is-unread-since-a-syscall-counts-itself-on-its-thread.md b/issues/syscall-cost-is-unread-since-a-syscall-counts-itself-on-its-thread.md index ba0b3eba7ca..511b17b27f8 100644 --- a/issues/syscall-cost-is-unread-since-a-syscall-counts-itself-on-its-thread.md +++ b/issues/syscall-cost-is-unread-since-a-syscall-counts-itself-on-its-thread.md @@ -1,18 +1,15 @@ --- status: open -kind: defect +kind: tooling opened: 2026-10-12 --- # `syscall_cost` is unread since a syscall counts itself on its thread -`syscall_dispatch` (`kernel/src/syscall/dispatch.rs`) took `PROCESS_TABLE`'s -lock twice per syscall, cloned the thread's `Arc` and took its `ThreadData` -lock, to update counters that belong to one thread. Since #1061 a syscall -counts itself on its thread's `TaskHandle` (`SyscallProfile`, written by that -thread with a load and a store) and takes no machine-wide lock and no `Arc`. +A syscall counts itself on its thread's `TaskHandle` (`SyscallProfile`, +`kernel/src/sched/payload.rs`) and takes no machine-wide lock to do it. -Read on the T14 in #1061's A/B (one osbench boot per side): `syscall.null` +Read on the T14 in #1061's A/B (two rounds, two boots a side in the second): `syscall.null` fell to 0.42× of `main`'s. The AMD bench's reading and the `syscall_cost` row on both benches are still owed. diff --git a/issues/toyos-uses-what-modern-hardware-offers-for-speed.md b/issues/toyos-uses-what-modern-hardware-offers-for-speed.md index 7e5e643e1aa..8847ae8442c 100644 --- a/issues/toyos-uses-what-modern-hardware-offers-for-speed.md +++ b/issues/toyos-uses-what-modern-hardware-offers-for-speed.md @@ -97,4 +97,5 @@ yet measured. - `issues/idle-cpus-enter-the-c-states-cst-names.md` - `issues/diskserver-drives-nvme-apst.md` - `issues/syscall-cost-is-unread-since-a-syscall-counts-itself-on-its-thread.md` +- `issues/malloc-churn-across-eight-threads-fell-with-the-run-queue-arenas.md` - `issues/a-thread-resuming-on-the-cpu-that-holds-its-state-restores-it-again.md` diff --git a/kernel/pure/sched/watch.rs b/kernel/pure/sched/watch.rs index a15a8e6ba7c..8422756a478 100644 --- a/kernel/pure/sched/watch.rs +++ b/kernel/pure/sched/watch.rs @@ -536,8 +536,6 @@ mod tests { } } - /// A bounded post spends nothing on a thread that was not parked, and - /// reaches the next registered thread instead. /// Records each kick and whether the list lock was held when it was sent. struct LockedKicks<'w> { watch: &'w TestWatch, @@ -573,6 +571,8 @@ mod tests { assert_eq!(rx.pop(&NoPreempt), Some(Msg::Wake(TaskKey(2)))); } + /// A bounded post spends nothing on a thread that was not parked, and + /// reaches the next registered thread instead. #[test] fn a_bounded_post_skips_a_waiter_that_was_not_parked() { let (handles, mut rx) = cpus();