diff --git a/workers/server/.sqlx/query-01e1ea581e55343f9db39d110bd8dc919b4038285e3ebf9eed1566eaa92656fa.json b/workers/server/.sqlx/query-01e1ea581e55343f9db39d110bd8dc919b4038285e3ebf9eed1566eaa92656fa.json new file mode 100644 index 0000000..7e3e007 --- /dev/null +++ b/workers/server/.sqlx/query-01e1ea581e55343f9db39d110bd8dc919b4038285e3ebf9eed1566eaa92656fa.json @@ -0,0 +1,38 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT generation AS \"generation!\", target AS \"target!\" FROM bazel_retention", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "generation!", + "type_info": "Text", + "origin": { + "Table": { + "table": "bazel_retention", + "name": "generation" + } + } + }, + { + "ordinal": 1, + "name": "target!", + "type_info": "Int8", + "origin": { + "Table": { + "table": "bazel_retention", + "name": "target" + } + } + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + true, + true + ] + }, + "hash": "01e1ea581e55343f9db39d110bd8dc919b4038285e3ebf9eed1566eaa92656fa" +} diff --git a/workers/server/.sqlx/query-c6ead782979af8e72a39d9627d71499e10e5d86311649a454db007eb1f555640.json b/workers/server/.sqlx/query-c6ead782979af8e72a39d9627d71499e10e5d86311649a454db007eb1f555640.json new file mode 100644 index 0000000..e439cbd --- /dev/null +++ b/workers/server/.sqlx/query-c6ead782979af8e72a39d9627d71499e10e5d86311649a454db007eb1f555640.json @@ -0,0 +1,40 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT generation AS \"generation!\", target AS \"target!\"\n FROM bazel_retention WHERE generation = ANY($1)", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "generation!", + "type_info": "Text", + "origin": { + "Table": { + "table": "bazel_retention", + "name": "generation" + } + } + }, + { + "ordinal": 1, + "name": "target!", + "type_info": "Int8", + "origin": { + "Table": { + "table": "bazel_retention", + "name": "target" + } + } + } + ], + "parameters": { + "Left": [ + "TextArray" + ] + }, + "nullable": [ + true, + true + ] + }, + "hash": "c6ead782979af8e72a39d9627d71499e10e5d86311649a454db007eb1f555640" +} diff --git a/workers/server/.sqlx/query-c7f8454178759316220c3e8aa41737da3482e82117c466f8f9581818ec8ba8c6.json b/workers/server/.sqlx/query-c7f8454178759316220c3e8aa41737da3482e82117c466f8f9581818ec8ba8c6.json new file mode 100644 index 0000000..b7ca111 --- /dev/null +++ b/workers/server/.sqlx/query-c7f8454178759316220c3e8aa41737da3482e82117c466f8f9581818ec8ba8c6.json @@ -0,0 +1,19 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO bazel_usage(cache_id,generation,entries,size,physical_size,staging_size,updated_at)\n SELECT cache_id,generation,$2,$3,$4,$5,$6 FROM bazel_caches WHERE generation=$1\n ON CONFLICT(cache_id) DO UPDATE SET generation=EXCLUDED.generation,entries=EXCLUDED.entries,\n size=EXCLUDED.size,physical_size=EXCLUDED.physical_size,staging_size=EXCLUDED.staging_size,\n updated_at=EXCLUDED.updated_at WHERE bazel_usage.updated_at < EXCLUDED.updated_at", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Int8", + "Int8", + "Int8", + "Int8", + "Timestamptz" + ] + }, + "nullable": [] + }, + "hash": "c7f8454178759316220c3e8aa41737da3482e82117c466f8f9581818ec8ba8c6" +} diff --git a/workers/server/src/app.rs b/workers/server/src/app.rs index 194fbd7..9fdfad4 100644 --- a/workers/server/src/app.rs +++ b/workers/server/src/app.rs @@ -48,6 +48,9 @@ pub struct AppState { pub(crate) finalizations: Arc, /// Sent after a commit that leaves a push job a candidate to publish. pub(crate) publications: Arc, + /// Bazel generations whose objects or targets may have changed since + /// they were swept. + pub(crate) bazel: Arc, /// How long an idle worker sleeps before it looks without being woken. pub(crate) idle_period: Duration, pub(crate) cdn: Option>, @@ -84,6 +87,7 @@ impl AppState { evictions: Arc::default(), finalizations: Arc::default(), publications: Arc::default(), + bazel: Arc::default(), idle_period: crate::wake::FALLBACK, pool, storage, @@ -124,6 +128,14 @@ impl AppState { self } + /// Change, before starting workers or requests, how soon one Bazel sweep + /// pass may follow another, and how often every generation is swept + /// unasked. A minute and an hour by default. + pub fn with_bazel_sweep_periods(mut self, settle: Duration, fallback: Duration) -> Self { + self.bazel = Arc::new(crate::bazel::Sweeps::new(settle, fallback)); + self + } + /// Start the durable multipart consumers once. Tower hosts and tests use /// this same lifecycle as serve(), rather than a request-local shortcut. pub fn start_upload_finalizers(&self) -> CancellationToken { diff --git a/workers/server/src/bazel/mod.rs b/workers/server/src/bazel/mod.rs index 8a71d63..1747413 100644 --- a/workers/server/src/bazel/mod.rs +++ b/workers/server/src/bazel/mod.rs @@ -11,6 +11,11 @@ use axum::{ routing::get, }; use prost::Message; +use std::{ + collections::{HashMap, HashSet}, + sync::Mutex, + time::{Duration, Instant}, +}; use tokio::io::AsyncReadExt; use tokio_util::io::ReaderStream; @@ -20,66 +25,237 @@ use crate::{ cache, error::ApiError, protocol_policy::CallerLimit, + wake::{Pace, Turn, Wake}, }; use proto::build::bazel::remote::execution::v2::{ActionResult, Digest, Tree}; const MAX_METADATA: usize = 4 * 1024 * 1024; +/// The soonest one sweep pass may follow another. A build's writes are swept +/// together, at most this often, as the former fixed interval swept them. +const SETTLE: Duration = Duration::from_secs(60); +/// How often every generation is swept whether or not anything announced it. +const FALLBACK: Duration = Duration::from_secs(60 * 60); + +/// What the collector has been told may have changed. A sweep reads object +/// storage in proportion to a generation's size, so a generation is swept +/// when a sweep could find something new, and otherwise costs nothing: no +/// storage request and no statement. +/// +/// A generation is marked by every `Store` operation that writes its objects +/// (`write`, `write_buffered`, `append`, `write_stream`, `finish_upload`), +/// over HTTP and gRPC alike, as the operation ends, however it ends. The +/// collector marks it again when a sweep fails or leaves it above its target. +/// Targets are reread when management creates a Bazel cache or changes a +/// capacity or an organization's limit; a generation is then swept only if +/// its target fell below the size last observed. +/// +/// Access markers mark nothing. A sweep leaves a generation within its +/// target, so a marker alone evicts nothing and changes no usage; the sweep +/// that next evicts reads the markers then. +/// +/// The fallback sweeps every generation for what this process never saw: +/// another instance's writes, a manual change, and targets moved by the other +/// products' usage. It is also the timer for everything a sweep removes by +/// age (uploads never published, staging chunks, access markers, and the +/// objects of retired generations): two hours is their minimum age, not a +/// deadline, so no sweep is scheduled for it. +pub(crate) struct Sweeps { + wake: Wake, + pending: Mutex, + settle: Duration, + fallback: Duration, +} + +#[derive(Default)] +struct Pending { + generations: HashSet, + targets: bool, +} + +impl Default for Sweeps { + fn default() -> Self { + Self::new(SETTLE, FALLBACK) + } +} + +impl Sweeps { + pub(crate) fn new(settle: Duration, fallback: Duration) -> Self { + Self { + wake: Wake::default(), + pending: Mutex::default(), + settle, + fallback, + } + } + + /// Call after the change, never before: a sweep that looks first records + /// the old state and nothing asks for another. + pub(crate) fn mark(&self, generation: &str) { + let mut pending = self.pending.lock().unwrap(); + let added = !pending.generations.contains(generation) + && pending.generations.insert(generation.to_owned()); + drop(pending); + // Later marks of a waiting generation join the pass already owed. + if added { + self.wake.wake(); + } + } + + /// Call after a commit that may have lowered a retention target. + pub(crate) fn retarget(&self) { + self.pending.lock().unwrap().targets = true; + self.wake.wake(); + } +} + +/// The collector's own record between passes. +struct Collector { + /// No pass starts before this. + next: Instant, + /// The next pass at or after this sweeps everything. + full: Instant, + /// Retained bytes each generation was last observed to hold. + sizes: HashMap, +} + pub(crate) async fn maintain(state: AppState, stopped: tokio_util::sync::CancellationToken) { - let period = std::time::Duration::from_secs(60); - let mut interval = tokio::time::interval_at(tokio::time::Instant::now() + period, period); - interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); - loop { - tokio::select! { + let sweeps = state.bazel.clone(); + let mut pace = Pace::new(&sweeps.wake, &stopped, sweeps.fallback); + // The first pass sweeps everything: nothing before this start was seen. + let start = Instant::now() + sweeps.settle; + let mut collector = Collector { + next: start, + full: start, + sizes: HashMap::new(), + }; + while pace.running() { + let turn = tokio::select! { _ = stopped.cancelled() => break, - _ = interval.tick() => { - tokio::select! { - _ = stopped.cancelled() => break, - result = collect(&state) => { - if result.is_err() { tracing::warn!("Bazel retention sweep failed; retrying next minute"); } - } + turn = collector.turn(&state) => turn, + }; + if turn.is_err() { + tracing::warn!("Bazel retention sweep failed; retrying after the settle period"); + } + pace.after(turn).await; + } +} + +impl Collector { + async fn turn(&mut self, state: &AppState) -> anyhow::Result { + let sweeps = &state.bazel; + let now = Instant::now(); + if now < self.next { + // A wake inside the settle period waits it out. Its marks stay. + return Ok(Turn::Idle(Some(self.next - now))); + } + let full = now >= self.full; + // Take the marks before reading anything, so a change that lands + // during this pass asks for the next one. + let pending = std::mem::take(&mut *sweeps.pending.lock().unwrap()); + if !full && !pending.targets && pending.generations.is_empty() { + return Ok(Turn::Idle(Some(self.full - now))); + } + self.next = now + sweeps.settle; + match collect(state, &mut self.sizes, full, &pending).await { + Ok(_) => { + if full { + self.full = now + sweeps.fallback; } + Ok(Turn::Worked) + } + Err(error) => { + let mut again = sweeps.pending.lock().unwrap(); + again.generations.extend(pending.generations); + again.targets |= pending.targets; + Err(error) } } } } -pub(crate) async fn collect(state: &AppState) -> anyhow::Result { +/// Sweep every generation and retire unknown ones, or with `full` unset only +/// the marked generations and, after a retargeting, those above their target. +async fn collect( + state: &AppState, + sizes: &mut HashMap, + full: bool, + pending: &Pending, +) -> anyhow::Result { use futures_util::{StreamExt, TryStreamExt, stream}; - let active: Vec<(String, i64)> = - sqlx::query_as("SELECT generation, target FROM bazel_retention") - .fetch_all(&state.pool) - .await?; - let mut generations: std::collections::HashMap = active + let mut active: Vec<(String, i64)> = if full || pending.targets { + sqlx::query!( + r#"SELECT generation AS "generation!", target AS "target!" FROM bazel_retention"# + ) + .map(|row| (row.generation, row.target)) + .fetch_all(&state.pool) + .await? + } else { + let marked: Vec = pending.generations.iter().cloned().collect(); + sqlx::query!( + r#"SELECT generation AS "generation!", target AS "target!" + FROM bazel_retention WHERE generation = ANY($1)"#, + &marked, + ) + .map(|row| (row.generation, row.target)) + .fetch_all(&state.pool) + .await? + }; + let mut generations: HashMap = active .iter() .map(|(generation, _)| (generation.clone(), true)) .collect(); + if full { + sizes.retain(|generation, _| generations.contains_key(generation)); + } + active.retain(|(generation, target)| { + full || pending.generations.contains(generation) + // Never observed, or observed above what it may now retain. + || sizes.get(generation).is_none_or(|size| *size > *target as u64) + }); let mut deleted = 0; let mut sweeps = stream::iter(active.into_iter().map(|(generation, capacity)| async move { let started = chrono::Utc::now(); let store = store::Store::new(state.storage.clone(), &generation, capacity as u64); - let deleted = store.collect().await?; - let (entries, size) = store.usage().await?; - let (physical, staging) = store.physical_usage().await?; - // Background observations only. A reset cannot inherit the old generation's - // totals, and a slow collector cannot replace a more recent observation. - sqlx::query("INSERT INTO bazel_usage(cache_id,generation,entries,size,physical_size,staging_size,updated_at) - SELECT cache_id,generation,$2,$3,$4,$5,$6 FROM bazel_caches WHERE generation=$1 - ON CONFLICT(cache_id) DO UPDATE SET generation=EXCLUDED.generation,entries=EXCLUDED.entries, - size=EXCLUDED.size,physical_size=EXCLUDED.physical_size,staging_size=EXCLUDED.staging_size, - updated_at=EXCLUDED.updated_at WHERE bazel_usage.updated_at < EXCLUDED.updated_at") - .bind(generation).bind(entries as i64).bind(size as i64).bind(physical as i64) - .bind(staging as i64).bind(started).execute(&state.pool).await?; - Ok::<_, anyhow::Error>(deleted) - })).buffer_unordered(4); - while let Some(sweep) = sweeps.next().await { - match sweep { - Ok(count) => deleted += count, + let swept = async { + let deleted = store.collect().await?; + let (entries, size) = store.usage().await?; + let (physical, staging) = store.physical_usage().await?; + // Background observations only. A reset cannot inherit the old generation's + // totals, and a slow collector cannot replace a more recent observation. + sqlx::query!("INSERT INTO bazel_usage(cache_id,generation,entries,size,physical_size,staging_size,updated_at) + SELECT cache_id,generation,$2,$3,$4,$5,$6 FROM bazel_caches WHERE generation=$1 + ON CONFLICT(cache_id) DO UPDATE SET generation=EXCLUDED.generation,entries=EXCLUDED.entries, + size=EXCLUDED.size,physical_size=EXCLUDED.physical_size,staging_size=EXCLUDED.staging_size, + updated_at=EXCLUDED.updated_at WHERE bazel_usage.updated_at < EXCLUDED.updated_at", + generation, entries as i64, size as i64, physical as i64, staging as i64, started) + .execute(&state.pool).await?; + Ok::<_, anyhow::Error>((deleted, size)) + } + .await; + (generation, capacity as u64, swept) + })) + .buffer_unordered(4); + while let Some((generation, capacity, swept)) = sweeps.next().await { + match swept { + Ok((count, size)) => { + deleted += count; + // Candidates are bounded and a racing collector may win an + // entry: a large excess drains over successive passes. + if size > capacity { + state.bazel.mark(&generation); + } + sizes.insert(generation, size); + } Err(_) => { - tracing::warn!("Bazel cache sweep failed; retaining its previous usage snapshot") + tracing::warn!("Bazel cache sweep failed; retaining its previous usage snapshot"); + state.bazel.mark(&generation); } } } + if !full { + return Ok(deleted); + } let metadata = state.storage.private_store(); let prefix = object_store::path::Path::from("bazel/"); let mut objects = metadata.list(Some(&prefix)); @@ -194,7 +370,8 @@ async fn open( state.storage.clone(), &target.generation, target.capacity as u64, - ), + ) + .announcing(state.bazel.clone()), target.public, )) } diff --git a/workers/server/src/bazel/store.rs b/workers/server/src/bazel/store.rs index f6771b1..ac007c5 100644 --- a/workers/server/src/bazel/store.rs +++ b/workers/server/src/bazel/store.rs @@ -31,6 +31,20 @@ pub struct Store { metadata: Arc, prefix: String, capacity: u64, + sweeps: Option>, +} + +/// Marks its generation for the collector when dropped: after the writing +/// operation that holds it has ended, whether it published, failed, or was +/// abandoned by its caller. +struct Changed<'a>(&'a Store); + +impl Drop for Changed<'_> { + fn drop(&mut self) { + if let Some(sweeps) = &self.0.sweeps { + sweeps.mark(&self.0.prefix["bazel/".len()..]); + } + } } #[derive(Clone, Serialize, Deserialize)] @@ -65,9 +79,17 @@ impl Store { storage, prefix: format!("bazel/{generation}"), capacity, + sweeps: None, } } + /// Tell the collector of every write through this store. Its own stores, + /// and those of tests that collect by hand, announce nothing. + pub(crate) fn announcing(mut self, sweeps: Arc) -> Self { + self.sweeps = Some(sweeps); + self + } + pub fn max_blob_size(&self) -> u64 { // The negotiated limit keeps resumable-session metadata bounded too. // It is independent of the cache's retained-storage target. @@ -180,6 +202,7 @@ impl Store { } pub async fn write(&self, kind: &str, digest: &str, size: u64, body: Body) -> io::Result<()> { + let _changed = Changed(self); let index = self.entry_key(kind, digest)?; if size > self.max_blob_size() { return Err(invalid("Blob exceeds object-store limit")); @@ -226,6 +249,7 @@ impl Store { bytes: Vec, memory: OwnedSemaphorePermit, ) -> io::Result<()> { + let _changed = Changed(self); let index = self.entry_key("cas", digest)?; let size = bytes.len() as u64; let key = self.verified_candidate(digest, &bytes)?; @@ -546,6 +570,7 @@ impl Store { offset: u64, bytes: Vec, ) -> io::Result { + let _changed = Changed(self); self.entry_key("cas", digest)?; let key = self.session_key(resource); if size > self.max_blob_size() || bytes.len() > 8 * 1024 * 1024 { @@ -593,6 +618,7 @@ impl Store { body: Body, finished: Arc, ) -> io::Result { + let _changed = Changed(self); let index = self.entry_key("cas", digest)?; if size > self.max_blob_size() { return Err(invalid("Upload exceeds capacity")); @@ -648,6 +674,7 @@ impl Store { digest: &str, size: u64, ) -> io::Result<()> { + let _changed = Changed(self); let key = self.session_key(resource); let (mut session, mode) = self.session(&key).await?; if session.expires < Utc::now().timestamp() { diff --git a/workers/server/src/management.rs b/workers/server/src/management.rs index dbb04c9..f55de5b 100644 --- a/workers/server/src/management.rs +++ b/workers/server/src/management.rs @@ -1220,6 +1220,9 @@ async fn put_org( } } tx.commit().await?; + if raw.get("storageLimit").is_some() { + state.bazel.retarget(); + } // Report the row this request just wrote, not the one read before it. org_response(state, id, None, false).await } @@ -1551,6 +1554,11 @@ async fn create_cache( ); set_cache_info(&mut tx, &path(&slug, name)).await?; tx.commit().await?; + if protocols.iter().any(|p| p == "bazel") { + // A new Bazel cache divides its organization's remaining storage + // further, and a capacity may have been lowered. + state.bazel.retarget(); + } Ok(json_response( if created { StatusCode::CREATED diff --git a/workers/server/tests/bazel.rs b/workers/server/tests/bazel.rs index 44eed2c..6dbf4cf 100644 --- a/workers/server/tests/bazel.rs +++ b/workers/server/tests/bazel.rs @@ -1825,3 +1825,416 @@ async fn grpc_through_the_organization_host(pool: PgPool) { tonic::Code::PermissionDenied ); } + +/// Every request the private store received, as `operation key`. +#[derive(Debug)] +struct ObservedStore { + inner: std::sync::Arc, + requests: std::sync::Arc>>, +} + +impl ObservedStore { + fn saw(&self, operation: &str, key: &dyn std::fmt::Display) { + self.requests + .lock() + .unwrap() + .push(format!("{operation} {key}")); + } +} + +impl std::fmt::Display for ObservedStore { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("ObservedStore") + } +} + +#[async_trait::async_trait] +impl object_store::ObjectStore for ObservedStore { + async fn put_opts( + &self, + path: &object_store::path::Path, + bytes: object_store::PutPayload, + options: object_store::PutOptions, + ) -> object_store::Result { + self.saw("put", path); + self.inner.put_opts(path, bytes, options).await + } + async fn put_multipart_opts( + &self, + path: &object_store::path::Path, + options: object_store::PutMultipartOptions, + ) -> object_store::Result> { + self.saw("multipart", path); + self.inner.put_multipart_opts(path, options).await + } + async fn get_opts( + &self, + path: &object_store::path::Path, + options: object_store::GetOptions, + ) -> object_store::Result { + self.saw("get", path); + self.inner.get_opts(path, options).await + } + fn delete_stream( + &self, + paths: stream::BoxStream<'static, object_store::Result>, + ) -> stream::BoxStream<'static, object_store::Result> { + self.saw("delete", &""); + self.inner.delete_stream(paths) + } + fn list( + &self, + prefix: Option<&object_store::path::Path>, + ) -> stream::BoxStream<'static, object_store::Result> { + self.saw("list", prefix.unwrap()); + self.inner.list(prefix) + } + async fn list_with_delimiter( + &self, + prefix: Option<&object_store::path::Path>, + ) -> object_store::Result { + self.saw("list", prefix.unwrap()); + self.inner.list_with_delimiter(prefix).await + } + async fn copy_opts( + &self, + from: &object_store::path::Path, + to: &object_store::path::Path, + options: object_store::CopyOptions, + ) -> object_store::Result<()> { + self.saw("copy", from); + self.inner.copy_opts(from, to, options).await + } +} + +const HOUR: std::time::Duration = std::time::Duration::from_secs(3600); +/// Far below the hour, and far above what a slow runner needs. +const PROMPT: std::time::Duration = std::time::Duration::from_secs(60); + +/// The fixture caches under `serve`, so the collector runs as it does in +/// production, on a pool only the server uses. Every other worker's fallback +/// is an hour. +struct Sweeping { + host: Host, + storage: snowstead_server::storage::Storage, + requests: std::sync::Arc>>, + /// Connections the server's pool handed out: one for each statement sent + /// to the pool, and one for each transaction. + checkouts: std::sync::Arc, + stop: tokio::sync::oneshot::Sender<()>, +} + +async fn sweeping( + pool: &PgPool, + settle: std::time::Duration, + fallback: std::time::Duration, +) -> Sweeping { + use std::sync::{ + Arc, + atomic::{AtomicU64, Ordering}, + }; + let requests = Arc::new(std::sync::Mutex::new(Vec::new())); + let storage = snowstead_server::storage::Storage::from_stores( + Arc::new(object_store::memory::InMemory::new()), + Arc::new(ObservedStore { + inner: Arc::new(object_store::memory::InMemory::new()), + requests: requests.clone(), + }), + ); + let mut host = host(pool, storage.clone()).await; + host.task.abort(); + let checkouts = Arc::new(AtomicU64::new(0)); + let (opened, reused) = (checkouts.clone(), checkouts.clone()); + let server = sqlx::postgres::PgPoolOptions::new() + .max_connections(8) + .after_connect(move |_, _| { + opened.fetch_add(1, Ordering::Relaxed); + Box::pin(async { Ok(()) }) + }) + .before_acquire(move |_, _| { + reused.fetch_add(1, Ordering::Relaxed); + Box::pin(async { Ok(true) }) + }) + .connect_with((*pool.connect_options()).clone()) + .await + .unwrap(); + let state = AppState::new( + server, + Config { + admin_secret: "bazel-sweep-operator".into(), + ..Default::default() + }, + storage.clone(), + reqwest::Client::new(), + ) + .unwrap() + .with_idle_period(HOUR) + .with_bazel_sweep_periods(settle, fallback); + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + host.url = format!("http://{}", listener.local_addr().unwrap()); + let (stop, stopped) = tokio::sync::oneshot::channel(); + host.task = tokio::spawn(async move { + snowstead_server::serve(listener, state, async move { + let _ = stopped.await; + }) + .await + .unwrap(); + }); + Sweeping { + host, + storage, + requests, + checkouts, + stop, + } +} + +impl Sweeping { + /// Wait until the server has issued no statement for a while: startup's + /// passes are over and every worker waits. + async fn quiet(&self) { + use std::sync::atomic::Ordering; + tokio::time::timeout(PROMPT, async { + let mut seen = self.checkouts.load(Ordering::Relaxed); + loop { + tokio::time::sleep(std::time::Duration::from_millis(1500)).await; + let now = self.checkouts.load(Ordering::Relaxed); + if now == seen { + break; + } + seen = now; + } + }) + .await + .expect("the workers never stopped issuing statements"); + } + + /// Storage requests so far that name `generation`, and how many of them + /// began a sweep of it: each sweep lists its publication receipts once. + fn requests(&self, generation: &str) -> (usize, usize) { + let requests = self.requests.lock().unwrap(); + let prefix = format!("bazel/{generation}/"); + ( + requests.iter().filter(|r| r.contains(&prefix)).count(), + requests + .iter() + .filter(|r| **r == format!("list {prefix}published")) + .count(), + ) + } + + async fn put(&self, cache: &str, bytes: &[u8]) { + let response = reqwest::Client::new() + .put(format!( + "{}/_api/v1/bazel/alpha/{cache}/cas/{}", + self.host.url, + digest(bytes).hash + )) + .bearer_auth(&self.host.token) + .body(bytes.to_vec()) + .send() + .await + .unwrap(); + assert_eq!(response.status(), 204); + } + + async fn stop(mut self) { + self.stop.send(()).unwrap(); + tokio::time::timeout(PROMPT, &mut self.host.task) + .await + .expect("workers did not stop") + .unwrap(); + } +} + +async fn generation(pool: &PgPool, cache: &str) -> String { + sqlx::query_scalar( + "SELECT generation FROM bazel_caches b JOIN caches c ON c.id=b.cache_id WHERE c.name=$1", + ) + .bind(cache) + .fetch_one(pool) + .await + .unwrap() +} + +/// The collector's latest observation of a cache's current generation: +/// entries, retained bytes, and when the sweep that made it began. +async fn observed(pool: &PgPool, cache: &str) -> Option<(i64, i64, chrono::DateTime)> { + sqlx::query_as( + "SELECT u.entries, u.size, u.updated_at FROM bazel_usage u + JOIN bazel_caches b ON b.cache_id=u.cache_id AND b.generation=u.generation + JOIN caches c ON c.id=b.cache_id WHERE c.name=$1", + ) + .bind(cache) + .fetch_optional(pool) + .await + .unwrap() +} + +/// Wait for a cache's observation to satisfy `met`, and return it. +async fn observes( + pool: &PgPool, + cache: &str, + why: &str, + met: impl Fn(i64, i64, chrono::DateTime) -> bool, +) -> chrono::DateTime { + tokio::time::timeout(PROMPT, async { + loop { + if let Some((entries, size, at)) = observed(pool, cache).await + && met(entries, size, at) + { + return at; + } + tokio::time::sleep(std::time::Duration::from_millis(20)).await; + } + }) + .await + .unwrap_or_else(|_| panic!("{why}")) +} + +#[sqlx::test(migrator = "snowstead_server::migrations::MIGRATOR")] +async fn only_a_written_generation_is_swept_and_an_idle_collector_does_nothing(pool: PgPool) { + use std::sync::atomic::Ordering; + let settle = std::time::Duration::from_secs(1); + // Only startup's pass and a mark can sweep: the fallback is an hour away. + let server = sweeping(&pool, settle, HOUR).await; + let (public, private) = ( + generation(&pool, "public").await, + generation(&pool, "private").await, + ); + let started = observes(&pool, "public", "startup did not sweep", |_, _, _| true).await; + observes(&pool, "private", "startup did not sweep", |_, _, _| true).await; + server.quiet().await; + + // Several settle periods, in each of which the former collector's interval + // would have swept both generations again. + let requests = server.requests.lock().unwrap().len(); + let checkouts = server.checkouts.load(Ordering::Relaxed); + tokio::time::sleep(3 * settle).await; + assert_eq!(server.requests.lock().unwrap().len(), requests); + assert_eq!(server.checkouts.load(Ordering::Relaxed), checkouts); + + let before = (server.requests(&public), server.requests(&private)); + let wrote = std::time::Instant::now(); + server.put("private", b"first").await; + observes( + &pool, + "private", + "a write was not swept", + |entries, size, _| (entries, size) == (1, 5), + ) + .await; + // The settle period had long passed, so nothing held this sweep back. + assert!(wrote.elapsed() < 10 * settle, "{:?}", wrote.elapsed()); + + // A burst inside one settle period shares the pass that follows it. + for value in 0..5u8 { + server.put("private", &[value; 3]).await; + } + observes( + &pool, + "private", + "a burst was not swept", + |entries, size, _| (entries, size) == (6, 20), + ) + .await; + server.quiet().await; + let sweeps = server.requests(&private).1 - before.1.1; + assert!((1..=3).contains(&sweeps), "{sweeps} sweeps for six writes"); + + // None of that touched the generation nothing wrote to. + assert_eq!(server.requests(&public), before.0); + assert_eq!(observed(&pool, "public").await.unwrap().2, started); + server.stop().await; +} + +#[sqlx::test(migrator = "snowstead_server::migrations::MIGRATOR")] +async fn the_fallback_sweeps_what_no_mark_announced(pool: PgPool) { + let server = sweeping( + &pool, + std::time::Duration::from_millis(200), + std::time::Duration::from_secs(2), + ) + .await; + let started = observes(&pool, "public", "startup did not sweep", |_, _, _| true).await; + observes(&pool, "private", "startup did not sweep", |_, _, _| true).await; + + // As another instance publishes: straight to storage, past this process. + let store = snowstead_server::bazel::store::Store::new( + server.storage.clone(), + &generation(&pool, "private").await, + 33554432, + ); + store + .write( + "cas", + &digest(b"elsewhere").hash, + 9, + axum::body::Body::from("elsewhere"), + ) + .await + .unwrap(); + observes( + &pool, + "private", + "the fallback missed an unannounced write", + |entries, size, _| (entries, size) == (1, 9), + ) + .await; + // A generation with no change at all is swept by the fallback too. + observes( + &pool, + "public", + "the fallback skipped a clean generation", + |_, _, at| at > started, + ) + .await; + server.stop().await; +} + +#[sqlx::test(migrator = "snowstead_server::migrations::MIGRATOR")] +async fn a_lowered_capacity_evicts_without_waiting_for_the_fallback(pool: PgPool) { + let server = sweeping(&pool, std::time::Duration::from_millis(200), HOUR).await; + observes(&pool, "private", "startup did not sweep", |_, _, _| true).await; + for bytes in [b"old!", b"new!"] { + server.put("public", bytes).await; + tokio::time::sleep(std::time::Duration::from_millis(2)).await; + } + observes( + &pool, + "public", + "writes were not swept", + |entries, size, _| (entries, size) == (2, 8), + ) + .await; + server.quiet().await; + let untouched = generation(&pool, "private").await; + let before = server.requests(&untouched); + + let response = reqwest::Client::new() + .post(format!("{}/_api/v1/admin/caches", server.host.url)) + .bearer_auth("bazel-sweep-operator") + .json( + &serde_json::json!({"name":"public","org":"bazel-org","private":false, + "protocols":["bazel"],"bazelCapacity":4}), + ) + .send() + .await + .unwrap(); + let status = response.status(); + assert!( + status.is_success(), + "{status} {}", + response.text().await.unwrap() + ); + observes( + &pool, + "public", + "a lowered capacity did not evict", + |entries, size, _| (entries, size) == (1, 4), + ) + .await; + // Rereading the targets swept only the generation above its own. + assert_eq!(server.requests(&untouched), before); + server.stop().await; +}