From 927f4817163de352d65e76e283d76e0434ab56df Mon Sep 17 00:00:00 2001 From: Uzair Aftab Date: Tue, 17 Nov 2026 19:50:39 +0100 Subject: [PATCH 1/2] bazel: sweep only generations that changed The collector swept every generation each minute: full listings of its index, objects, receipts, sessions, and markers, a read of each entry, and a usage upsert, then a listing of every Bazel object. Almost every pass recomputed what the last one had found, and with forty generations that was the largest periodic cost in the service. Each store operation that writes a generation's objects now marks it as the operation ends, and wakes the collector. A pass sweeps only the marked generations, no sooner than a minute after the last pass, so a build's writes share sweeps as before while a lone write is observed at once. Management rereads the targets after it creates a Bazel cache or changes a capacity or an organization's limit, and only generations left above their target are swept. With nothing marked, the collector issues no storage request and no statement. A pass over everything remains, one minute after start and then hourly. It covers what this process never saw: another instance's writes, manual changes, and targets moved by other products' usage. It is also the timer for the two-hour graces and for retired generations, which are minimum ages rather than deadlines. Access markers mark nothing: a swept generation is within its target, so a marker alone evicts nothing and changes no usage. --- workers/server/src/app.rs | 12 + workers/server/src/bazel/mod.rs | 238 ++++++++++++++--- workers/server/src/bazel/store.rs | 27 ++ workers/server/src/management.rs | 8 + workers/server/tests/bazel.rs | 413 ++++++++++++++++++++++++++++++ 5 files changed, 664 insertions(+), 34 deletions(-) diff --git a/workers/server/src/app.rs b/workers/server/src/app.rs index 194fbd7..9fdfad4 100644 --- a/workers/server/src/app.rs +++ b/workers/server/src/app.rs @@ -48,6 +48,9 @@ pub struct AppState { pub(crate) finalizations: Arc, /// Sent after a commit that leaves a push job a candidate to publish. pub(crate) publications: Arc, + /// Bazel generations whose objects or targets may have changed since + /// they were swept. + pub(crate) bazel: Arc, /// How long an idle worker sleeps before it looks without being woken. pub(crate) idle_period: Duration, pub(crate) cdn: Option>, @@ -84,6 +87,7 @@ impl AppState { evictions: Arc::default(), finalizations: Arc::default(), publications: Arc::default(), + bazel: Arc::default(), idle_period: crate::wake::FALLBACK, pool, storage, @@ -124,6 +128,14 @@ impl AppState { self } + /// Change, before starting workers or requests, how soon one Bazel sweep + /// pass may follow another, and how often every generation is swept + /// unasked. A minute and an hour by default. + pub fn with_bazel_sweep_periods(mut self, settle: Duration, fallback: Duration) -> Self { + self.bazel = Arc::new(crate::bazel::Sweeps::new(settle, fallback)); + self + } + /// Start the durable multipart consumers once. Tower hosts and tests use /// this same lifecycle as serve(), rather than a request-local shortcut. pub fn start_upload_finalizers(&self) -> CancellationToken { diff --git a/workers/server/src/bazel/mod.rs b/workers/server/src/bazel/mod.rs index 8a71d63..058893f 100644 --- a/workers/server/src/bazel/mod.rs +++ b/workers/server/src/bazel/mod.rs @@ -11,6 +11,11 @@ use axum::{ routing::get, }; use prost::Message; +use std::{ + collections::{HashMap, HashSet}, + sync::Mutex, + time::{Duration, Instant}, +}; use tokio::io::AsyncReadExt; use tokio_util::io::ReaderStream; @@ -20,66 +25,230 @@ use crate::{ cache, error::ApiError, protocol_policy::CallerLimit, + wake::{Pace, Turn, Wake}, }; use proto::build::bazel::remote::execution::v2::{ActionResult, Digest, Tree}; const MAX_METADATA: usize = 4 * 1024 * 1024; +/// The soonest one sweep pass may follow another. A build's writes are swept +/// together, at most this often, as the former fixed interval swept them. +const SETTLE: Duration = Duration::from_secs(60); +/// How often every generation is swept whether or not anything announced it. +const FALLBACK: Duration = Duration::from_secs(60 * 60); + +/// What the collector has been told may have changed. A sweep reads object +/// storage in proportion to a generation's size, so a generation is swept +/// when a sweep could find something new, and otherwise costs nothing: no +/// storage request and no statement. +/// +/// A generation is marked by every `Store` operation that writes its objects +/// (`write`, `write_buffered`, `append`, `write_stream`, `finish_upload`), +/// over HTTP and gRPC alike, as the operation ends, however it ends. The +/// collector marks it again when a sweep fails or leaves it above its target. +/// Targets are reread when management creates a Bazel cache or changes a +/// capacity or an organization's limit; a generation is then swept only if +/// its target fell below the size last observed. +/// +/// Access markers mark nothing. A sweep leaves a generation within its +/// target, so a marker alone evicts nothing and changes no usage; the sweep +/// that next evicts reads the markers then. +/// +/// The fallback sweeps every generation for what this process never saw: +/// another instance's writes, a manual change, and targets moved by the other +/// products' usage. It is also the timer for everything a sweep removes by +/// age (uploads never published, staging chunks, access markers, and the +/// objects of retired generations): two hours is their minimum age, not a +/// deadline, so no sweep is scheduled for it. +pub(crate) struct Sweeps { + wake: Wake, + pending: Mutex, + settle: Duration, + fallback: Duration, +} + +#[derive(Default)] +struct Pending { + generations: HashSet, + targets: bool, +} + +impl Default for Sweeps { + fn default() -> Self { + Self::new(SETTLE, FALLBACK) + } +} + +impl Sweeps { + pub(crate) fn new(settle: Duration, fallback: Duration) -> Self { + Self { + wake: Wake::default(), + pending: Mutex::default(), + settle, + fallback, + } + } + + /// Call after the change, never before: a sweep that looks first records + /// the old state and nothing asks for another. + pub(crate) fn mark(&self, generation: &str) { + let mut pending = self.pending.lock().unwrap(); + let added = !pending.generations.contains(generation) + && pending.generations.insert(generation.to_owned()); + drop(pending); + // Later marks of a waiting generation join the pass already owed. + if added { + self.wake.wake(); + } + } + + /// Call after a commit that may have lowered a retention target. + pub(crate) fn retarget(&self) { + self.pending.lock().unwrap().targets = true; + self.wake.wake(); + } +} + +/// The collector's own record between passes. +struct Collector { + /// No pass starts before this. + next: Instant, + /// The next pass at or after this sweeps everything. + full: Instant, + /// Retained bytes each generation was last observed to hold. + sizes: HashMap, +} + pub(crate) async fn maintain(state: AppState, stopped: tokio_util::sync::CancellationToken) { - let period = std::time::Duration::from_secs(60); - let mut interval = tokio::time::interval_at(tokio::time::Instant::now() + period, period); - interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); - loop { - tokio::select! { + let sweeps = state.bazel.clone(); + let mut pace = Pace::new(&sweeps.wake, &stopped, sweeps.fallback); + // The first pass sweeps everything: nothing before this start was seen. + let start = Instant::now() + sweeps.settle; + let mut collector = Collector { + next: start, + full: start, + sizes: HashMap::new(), + }; + while pace.running() { + let turn = tokio::select! { _ = stopped.cancelled() => break, - _ = interval.tick() => { - tokio::select! { - _ = stopped.cancelled() => break, - result = collect(&state) => { - if result.is_err() { tracing::warn!("Bazel retention sweep failed; retrying next minute"); } - } + turn = collector.turn(&state) => turn, + }; + if turn.is_err() { + tracing::warn!("Bazel retention sweep failed; retrying after the settle period"); + } + pace.after(turn).await; + } +} + +impl Collector { + async fn turn(&mut self, state: &AppState) -> anyhow::Result { + let sweeps = &state.bazel; + let now = Instant::now(); + if now < self.next { + // A wake inside the settle period waits it out. Its marks stay. + return Ok(Turn::Idle(Some(self.next - now))); + } + let full = now >= self.full; + // Take the marks before reading anything, so a change that lands + // during this pass asks for the next one. + let pending = std::mem::take(&mut *sweeps.pending.lock().unwrap()); + if !full && !pending.targets && pending.generations.is_empty() { + return Ok(Turn::Idle(Some(self.full - now))); + } + self.next = now + sweeps.settle; + match collect(state, &mut self.sizes, full, &pending).await { + Ok(_) => { + if full { + self.full = now + sweeps.fallback; } + Ok(Turn::Worked) + } + Err(error) => { + let mut again = sweeps.pending.lock().unwrap(); + again.generations.extend(pending.generations); + again.targets |= pending.targets; + Err(error) } } } } -pub(crate) async fn collect(state: &AppState) -> anyhow::Result { +/// Sweep every generation and retire unknown ones, or with `full` unset only +/// the marked generations and, after a retargeting, those above their target. +async fn collect( + state: &AppState, + sizes: &mut HashMap, + full: bool, + pending: &Pending, +) -> anyhow::Result { use futures_util::{StreamExt, TryStreamExt, stream}; - let active: Vec<(String, i64)> = + let mut active: Vec<(String, i64)> = if full || pending.targets { sqlx::query_as("SELECT generation, target FROM bazel_retention") .fetch_all(&state.pool) - .await?; - let mut generations: std::collections::HashMap = active + .await? + } else { + let marked: Vec<&str> = pending.generations.iter().map(String::as_str).collect(); + sqlx::query_as("SELECT generation, target FROM bazel_retention WHERE generation = ANY($1)") + .bind(marked) + .fetch_all(&state.pool) + .await? + }; + let mut generations: HashMap = active .iter() .map(|(generation, _)| (generation.clone(), true)) .collect(); + if full { + sizes.retain(|generation, _| generations.contains_key(generation)); + } + active.retain(|(generation, target)| { + full || pending.generations.contains(generation) + // Never observed, or observed above what it may now retain. + || sizes.get(generation).is_none_or(|size| *size > *target as u64) + }); let mut deleted = 0; let mut sweeps = stream::iter(active.into_iter().map(|(generation, capacity)| async move { let started = chrono::Utc::now(); let store = store::Store::new(state.storage.clone(), &generation, capacity as u64); - let deleted = store.collect().await?; - let (entries, size) = store.usage().await?; - let (physical, staging) = store.physical_usage().await?; - // Background observations only. A reset cannot inherit the old generation's - // totals, and a slow collector cannot replace a more recent observation. - sqlx::query("INSERT INTO bazel_usage(cache_id,generation,entries,size,physical_size,staging_size,updated_at) - SELECT cache_id,generation,$2,$3,$4,$5,$6 FROM bazel_caches WHERE generation=$1 - ON CONFLICT(cache_id) DO UPDATE SET generation=EXCLUDED.generation,entries=EXCLUDED.entries, - size=EXCLUDED.size,physical_size=EXCLUDED.physical_size,staging_size=EXCLUDED.staging_size, - updated_at=EXCLUDED.updated_at WHERE bazel_usage.updated_at < EXCLUDED.updated_at") - .bind(generation).bind(entries as i64).bind(size as i64).bind(physical as i64) - .bind(staging as i64).bind(started).execute(&state.pool).await?; - Ok::<_, anyhow::Error>(deleted) - })).buffer_unordered(4); - while let Some(sweep) = sweeps.next().await { - match sweep { - Ok(count) => deleted += count, + let swept = async { + let deleted = store.collect().await?; + let (entries, size) = store.usage().await?; + let (physical, staging) = store.physical_usage().await?; + // Background observations only. A reset cannot inherit the old generation's + // totals, and a slow collector cannot replace a more recent observation. + sqlx::query("INSERT INTO bazel_usage(cache_id,generation,entries,size,physical_size,staging_size,updated_at) + SELECT cache_id,generation,$2,$3,$4,$5,$6 FROM bazel_caches WHERE generation=$1 + ON CONFLICT(cache_id) DO UPDATE SET generation=EXCLUDED.generation,entries=EXCLUDED.entries, + size=EXCLUDED.size,physical_size=EXCLUDED.physical_size,staging_size=EXCLUDED.staging_size, + updated_at=EXCLUDED.updated_at WHERE bazel_usage.updated_at < EXCLUDED.updated_at") + .bind(&generation).bind(entries as i64).bind(size as i64).bind(physical as i64) + .bind(staging as i64).bind(started).execute(&state.pool).await?; + Ok::<_, anyhow::Error>((deleted, size)) + } + .await; + (generation, capacity as u64, swept) + })) + .buffer_unordered(4); + while let Some((generation, capacity, swept)) = sweeps.next().await { + match swept { + Ok((count, size)) => { + deleted += count; + // Candidates are bounded and a racing collector may win an + // entry: a large excess drains over successive passes. + if size > capacity { + state.bazel.mark(&generation); + } + sizes.insert(generation, size); + } Err(_) => { - tracing::warn!("Bazel cache sweep failed; retaining its previous usage snapshot") + tracing::warn!("Bazel cache sweep failed; retaining its previous usage snapshot"); + state.bazel.mark(&generation); } } } + if !full { + return Ok(deleted); + } let metadata = state.storage.private_store(); let prefix = object_store::path::Path::from("bazel/"); let mut objects = metadata.list(Some(&prefix)); @@ -194,7 +363,8 @@ async fn open( state.storage.clone(), &target.generation, target.capacity as u64, - ), + ) + .announcing(state.bazel.clone()), target.public, )) } diff --git a/workers/server/src/bazel/store.rs b/workers/server/src/bazel/store.rs index f6771b1..ac007c5 100644 --- a/workers/server/src/bazel/store.rs +++ b/workers/server/src/bazel/store.rs @@ -31,6 +31,20 @@ pub struct Store { metadata: Arc, prefix: String, capacity: u64, + sweeps: Option>, +} + +/// Marks its generation for the collector when dropped: after the writing +/// operation that holds it has ended, whether it published, failed, or was +/// abandoned by its caller. +struct Changed<'a>(&'a Store); + +impl Drop for Changed<'_> { + fn drop(&mut self) { + if let Some(sweeps) = &self.0.sweeps { + sweeps.mark(&self.0.prefix["bazel/".len()..]); + } + } } #[derive(Clone, Serialize, Deserialize)] @@ -65,9 +79,17 @@ impl Store { storage, prefix: format!("bazel/{generation}"), capacity, + sweeps: None, } } + /// Tell the collector of every write through this store. Its own stores, + /// and those of tests that collect by hand, announce nothing. + pub(crate) fn announcing(mut self, sweeps: Arc) -> Self { + self.sweeps = Some(sweeps); + self + } + pub fn max_blob_size(&self) -> u64 { // The negotiated limit keeps resumable-session metadata bounded too. // It is independent of the cache's retained-storage target. @@ -180,6 +202,7 @@ impl Store { } pub async fn write(&self, kind: &str, digest: &str, size: u64, body: Body) -> io::Result<()> { + let _changed = Changed(self); let index = self.entry_key(kind, digest)?; if size > self.max_blob_size() { return Err(invalid("Blob exceeds object-store limit")); @@ -226,6 +249,7 @@ impl Store { bytes: Vec, memory: OwnedSemaphorePermit, ) -> io::Result<()> { + let _changed = Changed(self); let index = self.entry_key("cas", digest)?; let size = bytes.len() as u64; let key = self.verified_candidate(digest, &bytes)?; @@ -546,6 +570,7 @@ impl Store { offset: u64, bytes: Vec, ) -> io::Result { + let _changed = Changed(self); self.entry_key("cas", digest)?; let key = self.session_key(resource); if size > self.max_blob_size() || bytes.len() > 8 * 1024 * 1024 { @@ -593,6 +618,7 @@ impl Store { body: Body, finished: Arc, ) -> io::Result { + let _changed = Changed(self); let index = self.entry_key("cas", digest)?; if size > self.max_blob_size() { return Err(invalid("Upload exceeds capacity")); @@ -648,6 +674,7 @@ impl Store { digest: &str, size: u64, ) -> io::Result<()> { + let _changed = Changed(self); let key = self.session_key(resource); let (mut session, mode) = self.session(&key).await?; if session.expires < Utc::now().timestamp() { diff --git a/workers/server/src/management.rs b/workers/server/src/management.rs index dbb04c9..f55de5b 100644 --- a/workers/server/src/management.rs +++ b/workers/server/src/management.rs @@ -1220,6 +1220,9 @@ async fn put_org( } } tx.commit().await?; + if raw.get("storageLimit").is_some() { + state.bazel.retarget(); + } // Report the row this request just wrote, not the one read before it. org_response(state, id, None, false).await } @@ -1551,6 +1554,11 @@ async fn create_cache( ); set_cache_info(&mut tx, &path(&slug, name)).await?; tx.commit().await?; + if protocols.iter().any(|p| p == "bazel") { + // A new Bazel cache divides its organization's remaining storage + // further, and a capacity may have been lowered. + state.bazel.retarget(); + } Ok(json_response( if created { StatusCode::CREATED diff --git a/workers/server/tests/bazel.rs b/workers/server/tests/bazel.rs index 44eed2c..6dbf4cf 100644 --- a/workers/server/tests/bazel.rs +++ b/workers/server/tests/bazel.rs @@ -1825,3 +1825,416 @@ async fn grpc_through_the_organization_host(pool: PgPool) { tonic::Code::PermissionDenied ); } + +/// Every request the private store received, as `operation key`. +#[derive(Debug)] +struct ObservedStore { + inner: std::sync::Arc, + requests: std::sync::Arc>>, +} + +impl ObservedStore { + fn saw(&self, operation: &str, key: &dyn std::fmt::Display) { + self.requests + .lock() + .unwrap() + .push(format!("{operation} {key}")); + } +} + +impl std::fmt::Display for ObservedStore { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("ObservedStore") + } +} + +#[async_trait::async_trait] +impl object_store::ObjectStore for ObservedStore { + async fn put_opts( + &self, + path: &object_store::path::Path, + bytes: object_store::PutPayload, + options: object_store::PutOptions, + ) -> object_store::Result { + self.saw("put", path); + self.inner.put_opts(path, bytes, options).await + } + async fn put_multipart_opts( + &self, + path: &object_store::path::Path, + options: object_store::PutMultipartOptions, + ) -> object_store::Result> { + self.saw("multipart", path); + self.inner.put_multipart_opts(path, options).await + } + async fn get_opts( + &self, + path: &object_store::path::Path, + options: object_store::GetOptions, + ) -> object_store::Result { + self.saw("get", path); + self.inner.get_opts(path, options).await + } + fn delete_stream( + &self, + paths: stream::BoxStream<'static, object_store::Result>, + ) -> stream::BoxStream<'static, object_store::Result> { + self.saw("delete", &""); + self.inner.delete_stream(paths) + } + fn list( + &self, + prefix: Option<&object_store::path::Path>, + ) -> stream::BoxStream<'static, object_store::Result> { + self.saw("list", prefix.unwrap()); + self.inner.list(prefix) + } + async fn list_with_delimiter( + &self, + prefix: Option<&object_store::path::Path>, + ) -> object_store::Result { + self.saw("list", prefix.unwrap()); + self.inner.list_with_delimiter(prefix).await + } + async fn copy_opts( + &self, + from: &object_store::path::Path, + to: &object_store::path::Path, + options: object_store::CopyOptions, + ) -> object_store::Result<()> { + self.saw("copy", from); + self.inner.copy_opts(from, to, options).await + } +} + +const HOUR: std::time::Duration = std::time::Duration::from_secs(3600); +/// Far below the hour, and far above what a slow runner needs. +const PROMPT: std::time::Duration = std::time::Duration::from_secs(60); + +/// The fixture caches under `serve`, so the collector runs as it does in +/// production, on a pool only the server uses. Every other worker's fallback +/// is an hour. +struct Sweeping { + host: Host, + storage: snowstead_server::storage::Storage, + requests: std::sync::Arc>>, + /// Connections the server's pool handed out: one for each statement sent + /// to the pool, and one for each transaction. + checkouts: std::sync::Arc, + stop: tokio::sync::oneshot::Sender<()>, +} + +async fn sweeping( + pool: &PgPool, + settle: std::time::Duration, + fallback: std::time::Duration, +) -> Sweeping { + use std::sync::{ + Arc, + atomic::{AtomicU64, Ordering}, + }; + let requests = Arc::new(std::sync::Mutex::new(Vec::new())); + let storage = snowstead_server::storage::Storage::from_stores( + Arc::new(object_store::memory::InMemory::new()), + Arc::new(ObservedStore { + inner: Arc::new(object_store::memory::InMemory::new()), + requests: requests.clone(), + }), + ); + let mut host = host(pool, storage.clone()).await; + host.task.abort(); + let checkouts = Arc::new(AtomicU64::new(0)); + let (opened, reused) = (checkouts.clone(), checkouts.clone()); + let server = sqlx::postgres::PgPoolOptions::new() + .max_connections(8) + .after_connect(move |_, _| { + opened.fetch_add(1, Ordering::Relaxed); + Box::pin(async { Ok(()) }) + }) + .before_acquire(move |_, _| { + reused.fetch_add(1, Ordering::Relaxed); + Box::pin(async { Ok(true) }) + }) + .connect_with((*pool.connect_options()).clone()) + .await + .unwrap(); + let state = AppState::new( + server, + Config { + admin_secret: "bazel-sweep-operator".into(), + ..Default::default() + }, + storage.clone(), + reqwest::Client::new(), + ) + .unwrap() + .with_idle_period(HOUR) + .with_bazel_sweep_periods(settle, fallback); + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + host.url = format!("http://{}", listener.local_addr().unwrap()); + let (stop, stopped) = tokio::sync::oneshot::channel(); + host.task = tokio::spawn(async move { + snowstead_server::serve(listener, state, async move { + let _ = stopped.await; + }) + .await + .unwrap(); + }); + Sweeping { + host, + storage, + requests, + checkouts, + stop, + } +} + +impl Sweeping { + /// Wait until the server has issued no statement for a while: startup's + /// passes are over and every worker waits. + async fn quiet(&self) { + use std::sync::atomic::Ordering; + tokio::time::timeout(PROMPT, async { + let mut seen = self.checkouts.load(Ordering::Relaxed); + loop { + tokio::time::sleep(std::time::Duration::from_millis(1500)).await; + let now = self.checkouts.load(Ordering::Relaxed); + if now == seen { + break; + } + seen = now; + } + }) + .await + .expect("the workers never stopped issuing statements"); + } + + /// Storage requests so far that name `generation`, and how many of them + /// began a sweep of it: each sweep lists its publication receipts once. + fn requests(&self, generation: &str) -> (usize, usize) { + let requests = self.requests.lock().unwrap(); + let prefix = format!("bazel/{generation}/"); + ( + requests.iter().filter(|r| r.contains(&prefix)).count(), + requests + .iter() + .filter(|r| **r == format!("list {prefix}published")) + .count(), + ) + } + + async fn put(&self, cache: &str, bytes: &[u8]) { + let response = reqwest::Client::new() + .put(format!( + "{}/_api/v1/bazel/alpha/{cache}/cas/{}", + self.host.url, + digest(bytes).hash + )) + .bearer_auth(&self.host.token) + .body(bytes.to_vec()) + .send() + .await + .unwrap(); + assert_eq!(response.status(), 204); + } + + async fn stop(mut self) { + self.stop.send(()).unwrap(); + tokio::time::timeout(PROMPT, &mut self.host.task) + .await + .expect("workers did not stop") + .unwrap(); + } +} + +async fn generation(pool: &PgPool, cache: &str) -> String { + sqlx::query_scalar( + "SELECT generation FROM bazel_caches b JOIN caches c ON c.id=b.cache_id WHERE c.name=$1", + ) + .bind(cache) + .fetch_one(pool) + .await + .unwrap() +} + +/// The collector's latest observation of a cache's current generation: +/// entries, retained bytes, and when the sweep that made it began. +async fn observed(pool: &PgPool, cache: &str) -> Option<(i64, i64, chrono::DateTime)> { + sqlx::query_as( + "SELECT u.entries, u.size, u.updated_at FROM bazel_usage u + JOIN bazel_caches b ON b.cache_id=u.cache_id AND b.generation=u.generation + JOIN caches c ON c.id=b.cache_id WHERE c.name=$1", + ) + .bind(cache) + .fetch_optional(pool) + .await + .unwrap() +} + +/// Wait for a cache's observation to satisfy `met`, and return it. +async fn observes( + pool: &PgPool, + cache: &str, + why: &str, + met: impl Fn(i64, i64, chrono::DateTime) -> bool, +) -> chrono::DateTime { + tokio::time::timeout(PROMPT, async { + loop { + if let Some((entries, size, at)) = observed(pool, cache).await + && met(entries, size, at) + { + return at; + } + tokio::time::sleep(std::time::Duration::from_millis(20)).await; + } + }) + .await + .unwrap_or_else(|_| panic!("{why}")) +} + +#[sqlx::test(migrator = "snowstead_server::migrations::MIGRATOR")] +async fn only_a_written_generation_is_swept_and_an_idle_collector_does_nothing(pool: PgPool) { + use std::sync::atomic::Ordering; + let settle = std::time::Duration::from_secs(1); + // Only startup's pass and a mark can sweep: the fallback is an hour away. + let server = sweeping(&pool, settle, HOUR).await; + let (public, private) = ( + generation(&pool, "public").await, + generation(&pool, "private").await, + ); + let started = observes(&pool, "public", "startup did not sweep", |_, _, _| true).await; + observes(&pool, "private", "startup did not sweep", |_, _, _| true).await; + server.quiet().await; + + // Several settle periods, in each of which the former collector's interval + // would have swept both generations again. + let requests = server.requests.lock().unwrap().len(); + let checkouts = server.checkouts.load(Ordering::Relaxed); + tokio::time::sleep(3 * settle).await; + assert_eq!(server.requests.lock().unwrap().len(), requests); + assert_eq!(server.checkouts.load(Ordering::Relaxed), checkouts); + + let before = (server.requests(&public), server.requests(&private)); + let wrote = std::time::Instant::now(); + server.put("private", b"first").await; + observes( + &pool, + "private", + "a write was not swept", + |entries, size, _| (entries, size) == (1, 5), + ) + .await; + // The settle period had long passed, so nothing held this sweep back. + assert!(wrote.elapsed() < 10 * settle, "{:?}", wrote.elapsed()); + + // A burst inside one settle period shares the pass that follows it. + for value in 0..5u8 { + server.put("private", &[value; 3]).await; + } + observes( + &pool, + "private", + "a burst was not swept", + |entries, size, _| (entries, size) == (6, 20), + ) + .await; + server.quiet().await; + let sweeps = server.requests(&private).1 - before.1.1; + assert!((1..=3).contains(&sweeps), "{sweeps} sweeps for six writes"); + + // None of that touched the generation nothing wrote to. + assert_eq!(server.requests(&public), before.0); + assert_eq!(observed(&pool, "public").await.unwrap().2, started); + server.stop().await; +} + +#[sqlx::test(migrator = "snowstead_server::migrations::MIGRATOR")] +async fn the_fallback_sweeps_what_no_mark_announced(pool: PgPool) { + let server = sweeping( + &pool, + std::time::Duration::from_millis(200), + std::time::Duration::from_secs(2), + ) + .await; + let started = observes(&pool, "public", "startup did not sweep", |_, _, _| true).await; + observes(&pool, "private", "startup did not sweep", |_, _, _| true).await; + + // As another instance publishes: straight to storage, past this process. + let store = snowstead_server::bazel::store::Store::new( + server.storage.clone(), + &generation(&pool, "private").await, + 33554432, + ); + store + .write( + "cas", + &digest(b"elsewhere").hash, + 9, + axum::body::Body::from("elsewhere"), + ) + .await + .unwrap(); + observes( + &pool, + "private", + "the fallback missed an unannounced write", + |entries, size, _| (entries, size) == (1, 9), + ) + .await; + // A generation with no change at all is swept by the fallback too. + observes( + &pool, + "public", + "the fallback skipped a clean generation", + |_, _, at| at > started, + ) + .await; + server.stop().await; +} + +#[sqlx::test(migrator = "snowstead_server::migrations::MIGRATOR")] +async fn a_lowered_capacity_evicts_without_waiting_for_the_fallback(pool: PgPool) { + let server = sweeping(&pool, std::time::Duration::from_millis(200), HOUR).await; + observes(&pool, "private", "startup did not sweep", |_, _, _| true).await; + for bytes in [b"old!", b"new!"] { + server.put("public", bytes).await; + tokio::time::sleep(std::time::Duration::from_millis(2)).await; + } + observes( + &pool, + "public", + "writes were not swept", + |entries, size, _| (entries, size) == (2, 8), + ) + .await; + server.quiet().await; + let untouched = generation(&pool, "private").await; + let before = server.requests(&untouched); + + let response = reqwest::Client::new() + .post(format!("{}/_api/v1/admin/caches", server.host.url)) + .bearer_auth("bazel-sweep-operator") + .json( + &serde_json::json!({"name":"public","org":"bazel-org","private":false, + "protocols":["bazel"],"bazelCapacity":4}), + ) + .send() + .await + .unwrap(); + let status = response.status(); + assert!( + status.is_success(), + "{status} {}", + response.text().await.unwrap() + ); + observes( + &pool, + "public", + "a lowered capacity did not evict", + |entries, size, _| (entries, size) == (1, 4), + ) + .await; + // Rereading the targets swept only the generation above its own. + assert_eq!(server.requests(&untouched), before); + server.stop().await; +} From 87fbd599b20fd09ee0c4992c880e2c47175d7bf1 Mon Sep 17 00:00:00 2001 From: Uzair Aftab Date: Tue, 17 Nov 2026 23:46:43 +0100 Subject: [PATCH 2/2] bazel: check the sweep's queries at compile time The collector read retention targets and recorded usage through runtime query functions, so a schema change could only fail them when a sweep ran. They now go through the query macros and are described in the offline metadata with the rest of the server's statements. The retention view's columns are marked non-null by hand because Postgres reports every view column as nullable. --- ...8dc919b4038285e3ebf9eed1566eaa92656fa.json | 38 ++++++++++++++++++ ...1499e10e5d86311649a454db007eb1f555640.json | 40 +++++++++++++++++++ ...737da3482e82117c466f8f9581818ec8ba8c6.json | 19 +++++++++ workers/server/src/bazel/mod.rs | 31 ++++++++------ 4 files changed, 116 insertions(+), 12 deletions(-) create mode 100644 workers/server/.sqlx/query-01e1ea581e55343f9db39d110bd8dc919b4038285e3ebf9eed1566eaa92656fa.json create mode 100644 workers/server/.sqlx/query-c6ead782979af8e72a39d9627d71499e10e5d86311649a454db007eb1f555640.json create mode 100644 workers/server/.sqlx/query-c7f8454178759316220c3e8aa41737da3482e82117c466f8f9581818ec8ba8c6.json diff --git a/workers/server/.sqlx/query-01e1ea581e55343f9db39d110bd8dc919b4038285e3ebf9eed1566eaa92656fa.json b/workers/server/.sqlx/query-01e1ea581e55343f9db39d110bd8dc919b4038285e3ebf9eed1566eaa92656fa.json new file mode 100644 index 0000000..7e3e007 --- /dev/null +++ b/workers/server/.sqlx/query-01e1ea581e55343f9db39d110bd8dc919b4038285e3ebf9eed1566eaa92656fa.json @@ -0,0 +1,38 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT generation AS \"generation!\", target AS \"target!\" FROM bazel_retention", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "generation!", + "type_info": "Text", + "origin": { + "Table": { + "table": "bazel_retention", + "name": "generation" + } + } + }, + { + "ordinal": 1, + "name": "target!", + "type_info": "Int8", + "origin": { + "Table": { + "table": "bazel_retention", + "name": "target" + } + } + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + true, + true + ] + }, + "hash": "01e1ea581e55343f9db39d110bd8dc919b4038285e3ebf9eed1566eaa92656fa" +} diff --git a/workers/server/.sqlx/query-c6ead782979af8e72a39d9627d71499e10e5d86311649a454db007eb1f555640.json b/workers/server/.sqlx/query-c6ead782979af8e72a39d9627d71499e10e5d86311649a454db007eb1f555640.json new file mode 100644 index 0000000..e439cbd --- /dev/null +++ b/workers/server/.sqlx/query-c6ead782979af8e72a39d9627d71499e10e5d86311649a454db007eb1f555640.json @@ -0,0 +1,40 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT generation AS \"generation!\", target AS \"target!\"\n FROM bazel_retention WHERE generation = ANY($1)", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "generation!", + "type_info": "Text", + "origin": { + "Table": { + "table": "bazel_retention", + "name": "generation" + } + } + }, + { + "ordinal": 1, + "name": "target!", + "type_info": "Int8", + "origin": { + "Table": { + "table": "bazel_retention", + "name": "target" + } + } + } + ], + "parameters": { + "Left": [ + "TextArray" + ] + }, + "nullable": [ + true, + true + ] + }, + "hash": "c6ead782979af8e72a39d9627d71499e10e5d86311649a454db007eb1f555640" +} diff --git a/workers/server/.sqlx/query-c7f8454178759316220c3e8aa41737da3482e82117c466f8f9581818ec8ba8c6.json b/workers/server/.sqlx/query-c7f8454178759316220c3e8aa41737da3482e82117c466f8f9581818ec8ba8c6.json new file mode 100644 index 0000000..b7ca111 --- /dev/null +++ b/workers/server/.sqlx/query-c7f8454178759316220c3e8aa41737da3482e82117c466f8f9581818ec8ba8c6.json @@ -0,0 +1,19 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO bazel_usage(cache_id,generation,entries,size,physical_size,staging_size,updated_at)\n SELECT cache_id,generation,$2,$3,$4,$5,$6 FROM bazel_caches WHERE generation=$1\n ON CONFLICT(cache_id) DO UPDATE SET generation=EXCLUDED.generation,entries=EXCLUDED.entries,\n size=EXCLUDED.size,physical_size=EXCLUDED.physical_size,staging_size=EXCLUDED.staging_size,\n updated_at=EXCLUDED.updated_at WHERE bazel_usage.updated_at < EXCLUDED.updated_at", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Int8", + "Int8", + "Int8", + "Int8", + "Timestamptz" + ] + }, + "nullable": [] + }, + "hash": "c7f8454178759316220c3e8aa41737da3482e82117c466f8f9581818ec8ba8c6" +} diff --git a/workers/server/src/bazel/mod.rs b/workers/server/src/bazel/mod.rs index 058893f..1747413 100644 --- a/workers/server/src/bazel/mod.rs +++ b/workers/server/src/bazel/mod.rs @@ -184,15 +184,22 @@ async fn collect( ) -> anyhow::Result { use futures_util::{StreamExt, TryStreamExt, stream}; let mut active: Vec<(String, i64)> = if full || pending.targets { - sqlx::query_as("SELECT generation, target FROM bazel_retention") - .fetch_all(&state.pool) - .await? + sqlx::query!( + r#"SELECT generation AS "generation!", target AS "target!" FROM bazel_retention"# + ) + .map(|row| (row.generation, row.target)) + .fetch_all(&state.pool) + .await? } else { - let marked: Vec<&str> = pending.generations.iter().map(String::as_str).collect(); - sqlx::query_as("SELECT generation, target FROM bazel_retention WHERE generation = ANY($1)") - .bind(marked) - .fetch_all(&state.pool) - .await? + let marked: Vec = pending.generations.iter().cloned().collect(); + sqlx::query!( + r#"SELECT generation AS "generation!", target AS "target!" + FROM bazel_retention WHERE generation = ANY($1)"#, + &marked, + ) + .map(|row| (row.generation, row.target)) + .fetch_all(&state.pool) + .await? }; let mut generations: HashMap = active .iter() @@ -216,13 +223,13 @@ async fn collect( let (physical, staging) = store.physical_usage().await?; // Background observations only. A reset cannot inherit the old generation's // totals, and a slow collector cannot replace a more recent observation. - sqlx::query("INSERT INTO bazel_usage(cache_id,generation,entries,size,physical_size,staging_size,updated_at) + sqlx::query!("INSERT INTO bazel_usage(cache_id,generation,entries,size,physical_size,staging_size,updated_at) SELECT cache_id,generation,$2,$3,$4,$5,$6 FROM bazel_caches WHERE generation=$1 ON CONFLICT(cache_id) DO UPDATE SET generation=EXCLUDED.generation,entries=EXCLUDED.entries, size=EXCLUDED.size,physical_size=EXCLUDED.physical_size,staging_size=EXCLUDED.staging_size, - updated_at=EXCLUDED.updated_at WHERE bazel_usage.updated_at < EXCLUDED.updated_at") - .bind(&generation).bind(entries as i64).bind(size as i64).bind(physical as i64) - .bind(staging as i64).bind(started).execute(&state.pool).await?; + updated_at=EXCLUDED.updated_at WHERE bazel_usage.updated_at < EXCLUDED.updated_at", + generation, entries as i64, size as i64, physical as i64, staging as i64, started) + .execute(&state.pool).await?; Ok::<_, anyhow::Error>((deleted, size)) } .await;