From 9f589e0c16a1c148ce40536961e6f8db70b17201 Mon Sep 17 00:00:00 2001 From: Quentin de Quelen Date: Thu, 12 Nov 2026 13:13:40 +0100 Subject: [PATCH 01/10] perf(api): enqueue a single lab event with one auto-commit insert PgOutbox::enqueue and SqliteOutbox::enqueue wrapped every insert in BEGIN / INSERT / COMMIT. Every successful /scrape, /map and /search enqueues exactly one usage event on the request path, so that was three round trips for one row. One event is now a single auto-commit INSERT (atomic by itself). Several events keep the transaction, so a batch is still stored entirely or not at all; a test covers it on both stores. Refs SCR-90 Co-Authored-By: Claude Fable 5.2 --- bins/scrapix-api/src/lab_events.rs | 120 +++++++++++++++++++++-------- 1 file changed, 88 insertions(+), 32 deletions(-) diff --git a/bins/scrapix-api/src/lab_events.rs b/bins/scrapix-api/src/lab_events.rs index ad062191..a9e7b283 100644 --- a/bins/scrapix-api/src/lab_events.rs +++ b/bins/scrapix-api/src/lab_events.rs @@ -166,26 +166,36 @@ fn db(e: impl std::fmt::Display) -> StoreError { StoreError::Other(e.to_string()) } +async fn pg_insert<'e, E: sqlx::PgExecutor<'e>>(ex: E, e: &LabEvent) -> Result<(), StoreError> { + let account: Uuid = e + .account_id + .parse() + .map_err(|_| db(format!("invalid account_id {}", e.account_id)))?; + sqlx::query( + "INSERT INTO lab_events (id, type, account_id, payload) \ + VALUES ($1, $2, $3, $4) ON CONFLICT (id) DO NOTHING", + ) + .bind(e.id) + .bind(&e.kind) + .bind(account) + .bind(serde_json::to_value(e).map_err(db)?) + .execute(ex) + .await + .map_err(db)?; + Ok(()) +} + #[async_trait::async_trait] impl LabOutbox for PgOutbox { async fn enqueue(&self, events: &[LabEvent]) -> Result<(), StoreError> { + // One event (every /scrape, /map and /search) is one auto-commit + // statement: atomic by itself, one round trip instead of three. + if let [e] = events { + return pg_insert(&self.pool, e).await; + } let mut tx = self.pool.begin().await.map_err(db)?; for e in events { - let account: Uuid = e - .account_id - .parse() - .map_err(|_| db(format!("invalid account_id {}", e.account_id)))?; - sqlx::query( - "INSERT INTO lab_events (id, type, account_id, payload) \ - VALUES ($1, $2, $3, $4) ON CONFLICT (id) DO NOTHING", - ) - .bind(e.id) - .bind(&e.kind) - .bind(account) - .bind(serde_json::to_value(e).map_err(db)?) - .execute(&mut *tx) - .await - .map_err(db)?; + pg_insert(&mut *tx, e).await?; } tx.commit().await.map_err(db) } @@ -291,29 +301,42 @@ fn sqlite_ts(t: DateTime) -> String { crate::job_store::sqlite::ts(Some(t)).unwrap_or_default() } +async fn sqlite_insert<'e, E: sqlx::SqliteExecutor<'e>>( + ex: E, + e: &LabEvent, + now: &str, +) -> Result<(), StoreError> { + let account: Uuid = e + .account_id + .parse() + .map_err(|_| db(format!("invalid account_id {}", e.account_id)))?; + sqlx::query( + "INSERT INTO lab_events (id, type, account_id, payload, created_at, next_attempt_at) \ + VALUES (?, ?, ?, ?, ?, ?) ON CONFLICT (id) DO NOTHING", + ) + .bind(e.id.to_string()) + .bind(&e.kind) + .bind(account.to_string()) + .bind(serde_json::to_string(e).map_err(db)?) + .bind(now) + .bind(now) + .execute(ex) + .await + .map_err(db)?; + Ok(()) +} + #[async_trait::async_trait] impl LabOutbox for SqliteOutbox { async fn enqueue(&self, events: &[LabEvent]) -> Result<(), StoreError> { let now = sqlite_ts(Utc::now()); + // As in `PgOutbox`: one event is one auto-commit statement. + if let [e] = events { + return sqlite_insert(&self.pool, e, &now).await; + } let mut tx = self.pool.begin().await.map_err(db)?; for e in events { - let account: Uuid = e - .account_id - .parse() - .map_err(|_| db(format!("invalid account_id {}", e.account_id)))?; - sqlx::query( - "INSERT INTO lab_events (id, type, account_id, payload, created_at, next_attempt_at) \ - VALUES (?, ?, ?, ?, ?, ?) ON CONFLICT (id) DO NOTHING", - ) - .bind(e.id.to_string()) - .bind(&e.kind) - .bind(account.to_string()) - .bind(serde_json::to_string(e).map_err(db)?) - .bind(&now) - .bind(&now) - .execute(&mut *tx) - .await - .map_err(db)?; + sqlite_insert(&mut *tx, e, &now).await?; } tx.commit().await.map_err(db) } @@ -887,6 +910,39 @@ mod tests { assert_eq!(o.pending_stats().await.unwrap().0, 0); } + /// A single event is one auto-commit insert; several stay one + /// transaction: a bad event anywhere in the batch stores none of it. + async fn several_events_are_all_or_nothing(o: &dyn LabOutbox) { + let acct = "7f1c2a8e-0000-4000-8000-000000000001"; + let good = LabEvent::usage(acct, None, "map", 2, json!({}), "m".into(), None); + let bad = LabEvent::usage("acc", None, "map", 2, json!({}), "m".into(), None); + assert!(o.enqueue(&[good.clone(), bad]).await.is_err()); + assert_eq!( + o.pending_stats().await.unwrap().0, + 0, + "the valid event of a failed batch is rolled back" + ); + o.enqueue(std::slice::from_ref(&good)).await.unwrap(); + o.enqueue(std::slice::from_ref(&good)).await.unwrap(); + assert_eq!(o.due(10).await.unwrap(), vec![good.clone()]); + o.abandon(&[good.id]).await.unwrap(); + } + + #[tokio::test] + async fn sqlite_outbox_keeps_several_events_atomic() { + let (_dir, pool) = crate::job_store::sqlite::test_sqlite_pool().await; + several_events_are_all_or_nothing(&SqliteOutbox::new(pool)).await; + } + + #[tokio::test] + async fn pg_outbox_keeps_several_events_atomic() { + let Some(pool) = crate::job_store::postgres::test_pg_pool().await else { + eprintln!("skipped"); + return; + }; + several_events_are_all_or_nothing(&PgOutbox::new(pool)).await; + } + #[tokio::test] async fn sqlite_reschedule_caps_backoff_and_survives_huge_attempt_counts() { let (_dir, pool) = crate::job_store::sqlite::test_sqlite_pool().await; From 86c31007e7a0ac8859f921c568e836ee9a5f4e46 Mon Sep 17 00:00:00 2001 From: Quentin de Quelen Date: Thu, 12 Nov 2026 13:13:47 +0100 Subject: [PATCH 02/10] perf(lab): share one Lab call per cache key and refresh busy keys ahead On a TTL miss every concurrent request for a key called the Lab: at each expiry (every 30 s for a busy API key) all its in-flight requests hit the Lab at once, bounded only by the 32 slots, and a slot wait over 1 s became a 503 'Lab client saturated'. A cached lookup that has to ask the Lab is now a flight: one detached task per cache key, whose answer every caller of that key waits for. A caller that goes away neither cancels the call nor strands the others; a flight that panics leaves the map before its channel closes, and its waiters start one replacement, then fail Unavailable. A fresh entry past 4/5 of its ttl is served and refreshed by a background flight: on access only, at most once per entry, through the same slots and metrics, and capped at a quarter of max_in_flight. A failed refresh changes nothing, so an entry is never served past the ttl the Lab gave for it and the stale rule is as before: a revoked credential still works for at most cache_ttl. cache_ttl 0 stays strict: a key whose last answer was 'do not cache' is asked by every caller for itself, and a caller that joined someone else's flight does not use a cache_ttl 0 answer from it but asks again. LabClient now wraps an Arc of its state, which the flights hold; its public API is unchanged. Refs SCR-90 Co-Authored-By: Claude Fable 5.2 --- crates/scrapix-lab/src/client.rs | 999 ++++++++++++++++++++++++++----- 1 file changed, 856 insertions(+), 143 deletions(-) diff --git a/crates/scrapix-lab/src/client.rs b/crates/scrapix-lab/src/client.rs index 6360cd3f..55dd35a0 100644 --- a/crates/scrapix-lab/src/client.rs +++ b/crates/scrapix-lab/src/client.rs @@ -10,9 +10,18 @@ //! just served stale is not retried for `stale_retry`, and negative answers //! live in their own cache so a spray of unknown credentials cannot evict //! the positive ones. +//! +//! A cached lookup that has to ask the Lab is a *flight*: one detached task +//! per cache key, whose answer every caller of that key waits for, so an +//! expiring entry costs one Lab call however many requests want it. An +//! entry past [`REFRESH_AHEAD`] of its ttl is still served and refreshed by +//! a background flight (once per entry, on access only), so a key in +//! constant use never waits for the Lab. An entry is never served past the +//! ttl the Lab gave for it because of a refresh: a failed one changes +//! nothing. use std::collections::HashMap; -use std::sync::Mutex; +use std::sync::{Arc, Mutex}; use std::time::{Duration, Instant}; use bytes::Bytes; @@ -212,12 +221,25 @@ struct Entry { ttl: Duration, /// Set when this entry was served stale: don't ask the Lab again before. retry_at: Option, + /// A refresh-ahead was started for this entry (at most one per entry: a + /// failed one is not repeated, the entry just expires on time). + refresh_started: bool, } +/// How far into its ttl a fresh entry is refreshed in the background (4/5: +/// with the default 30 s ttl, from 24 s on). +const REFRESH_AHEAD: (u32, u32) = (4, 5); + impl Entry { fn fresh(&self) -> bool { self.fetched.elapsed() < self.ttl } + /// Fresh, but close enough to its expiry to be refreshed ahead. + fn refresh_due(&self) -> bool { + !self.refresh_started + && !self.ttl.is_zero() + && self.fetched.elapsed() >= self.ttl / REFRESH_AHEAD.1 * REFRESH_AHEAD.0 + } /// Servable while the Lab is unreachable: within `grace` past its ttl, /// and only if it was cached with a positive ttl. A `cache_ttl: 0` (or /// negative) answer — an expiring session or OAuth token, "do not @@ -290,16 +312,28 @@ impl Cache { fetched: Instant::now(), ttl, retry_at: None, + refresh_started: false, }, ); } } +/// What a Lab lookup answers: the value (`None` = a negative answer) and how +/// long it may be cached. +type Fetched = Result<(Option, Duration), LabError>; + +/// The waiting side of a flight: `None` until the answer lands. A sender +/// dropped without an answer (the flight panicked or was aborted) closes it. +type FlightRx = tokio::sync::watch::Receiver>>; + /// One lookup's cache: positive answers (servable stale) and negative ones -/// (never stale), each with its own capacity. +/// (never stale), each with its own capacity, and the flights in the air +/// (at most one per key; under the same lock as the answers, so "no fresh +/// answer and no flight" is decided atomically). struct Lookups { positive: Cache, negative: Cache<()>, + flights: HashMap>, } impl Lookups { @@ -307,17 +341,103 @@ impl Lookups { Self { positive: Cache::new(timing.capacity, timing.stale_grace), negative: Cache::new(timing.negative_capacity, Duration::ZERO), + flights: HashMap::new(), + } + } + + /// Record a Lab answer asked at `started`. An error changes nothing: in + /// particular a failed refresh leaves the entry it was refreshing with + /// its original `fetched` and ttl. + fn store(&mut self, key: &str, started: Instant, result: &Fetched) { + match result { + Ok((Some(value), ttl)) => { + // An answer cached after this one was asked for (the AI key + // re-resolution writes directly) is the newer one. + if self + .positive + .map + .get(key) + .is_some_and(|e| e.fetched > started) + { + return; + } + self.negative.remove(key); + self.positive.put(key.to_string(), value.clone(), *ttl); + } + Ok((None, ttl)) => { + self.positive.remove(key); + self.negative.put(key.to_string(), (), *ttl); + } + Err(_) => {} + } + } +} + +/// Selects one of the client's caches (a plain `fn`, so a detached flight +/// can find its cache again without borrowing the caller). +type CacheOf = fn(&Inner) -> &Mutex>; + +fn identities(i: &Inner) -> &Mutex> { + &i.identities +} +fn meili(i: &Inner) -> &Mutex> { + &i.meili +} +fn ai(i: &Inner) -> &Mutex> { + &i.ai +} + +/// The landing side of a flight, owned by its task. However the task ends +/// (answer, panic, abort), the flight leaves the map *before* the sender is +/// dropped, so a waiter that wakes on a closed channel never finds the dead +/// flight again and can start a new one. +struct Landing { + inner: Arc, + cache: CacheOf, + key: String, + started: Instant, + tx: tokio::sync::watch::Sender>>, + landed: bool, +} + +impl Landing { + fn land(mut self, result: Fetched) { + { + let mut c = (self.cache)(&self.inner).lock().unwrap(); + c.flights.remove(&self.key); + c.store(&self.key, self.started, &result); } + self.landed = true; + self.tx.send_replace(Some(result)); } } +impl Drop for Landing { + fn drop(&mut self) { + if !self.landed { + if let Ok(mut c) = (self.cache)(&self.inner).lock() { + c.flights.remove(&self.key); + } + } + } +} + +/// The engine's client for the Lab's internal API. Cheap to share: the +/// state lives behind one `Arc`, which the detached flights also hold. pub struct LabClient { + inner: Arc, +} + +struct Inner { base: String, instance_id: String, secret: String, http: reqwest::Client, timing: Timing, permits: tokio::sync::Semaphore, + /// Background refreshes in the air: a quarter of `max_in_flight`, so + /// they never take every Lab slot from the requests that are waiting. + refresh_slots: Arc, identities: Mutex>, meili: Mutex>, ai: Mutex>, @@ -375,17 +495,23 @@ impl LabClient { .redirect(reqwest::redirect::Policy::none()) .build() .expect("reqwest client"); - Self { + let inner = Inner { base: base_url.trim_end_matches('/').to_string(), instance_id: instance_id.to_string(), secret: instance_secret.to_string(), http, permits: tokio::sync::Semaphore::new(timing.max_in_flight.max(1)), + refresh_slots: Arc::new(tokio::sync::Semaphore::new( + (timing.max_in_flight / 4).max(1), + )), identities: Mutex::new(Lookups::new(&timing)), meili: Mutex::new(Lookups::new(&timing)), ai: Mutex::new(Lookups::new(&timing)), balances: Mutex::new(HashMap::new()), timing, + }; + Self { + inner: Arc::new(inner), } } @@ -397,6 +523,90 @@ impl LabClient { .to_string() } + /// `GET /internal/instances/me`: confirms the instance credentials and + /// tells which deployment they belong to (checked once at boot). + pub async fn instances_me(&self) -> Result { + self.inner.instances_me().await + } + + pub async fn introspect( + &self, + kind: CredentialKind, + credential: &str, + account_id: Option<&str>, + ) -> Result, LabError> { + self.inner.introspect(kind, credential, account_id).await + } + + pub async fn account(&self, account_id: &str) -> Result, LabError> { + self.inner.account(account_id).await + } + + /// Spendable credits as the Lab last reported them (from a fresh + /// snapshot, or a new one). + pub async fn available_credits(&self, account_id: &str) -> Result, LabError> { + self.inner.available_credits(account_id).await + } + + /// Take a new balance snapshot now (directly, not through the identity + /// cache: the balance must be current). While the Lab is unavailable the + /// last snapshot is served for `stale_grace` past its expiry (unless it + /// came from a `cache_ttl: 0` answer), asking the Lab again at most + /// every `stale_retry`. + pub async fn refresh_credits(&self, account_id: &str) -> Result, LabError> { + self.inner.refresh_credits(account_id).await + } + + pub async fn meilisearch( + &self, + account_id: &str, + url: Option<&str>, + ) -> Result, LabError> { + self.inner.meilisearch(account_id, url).await + } + + /// `GET /internal/accounts/{id}/ai`: the Lumen target for this account's + /// LLM calls; `None` when the Lab has none for it (404). A + /// `rejected_key_id` (the key Lumen just refused) skips the cache, asks + /// the Lab for a replacement and makes that the cached answer. + pub async fn ai( + &self, + account_id: &str, + rejected_key_id: Option<&str>, + ) -> Result, LabError> { + self.inner.ai(account_id, rejected_key_id).await + } + + #[cfg(test)] + pub fn debug_cache_keys(&self) -> Vec { + let c = self.inner.identities.lock().unwrap(); + c.positive + .map + .keys() + .chain(c.negative.map.keys()) + .cloned() + .collect() + } + + #[cfg(test)] + pub fn cached_ttl_for_test( + &self, + kind: CredentialKind, + credential: &str, + account_id: Option<&str>, + ) -> Option { + let key = hashed(&["i", kind.as_str(), credential, account_id.unwrap_or("")]); + self.inner + .identities + .lock() + .unwrap() + .positive + .get(&key) + .map(|e| e.ttl) + } +} + +impl Inner { fn ttl_from(&self, secs: Option) -> Duration { match secs { None => self.timing.default_ttl, @@ -496,68 +706,146 @@ impl LabClient { /// only a *positive* cached answer with a positive ttl is ever served /// stale (never a `cache_ttl: 0` one), and a key just served stale is /// not retried for `stale_retry`. + /// + /// Asking the Lab is single-flight per key: the first caller starts a + /// flight (a detached task, so a caller that goes away neither cancels + /// the call nor strands the others) and everyone asking for that key + /// meanwhile waits for its answer. Two exceptions keep "do not cache" + /// strict: a key whose last answer had `cache_ttl: 0` is asked by every + /// caller for itself, and a caller that joined a flight someone else + /// started does not use a `cache_ttl: 0` answer from it (the Lab may + /// have computed it before this caller asked) but asks again. + /// + /// A fresh entry past [`REFRESH_AHEAD`] of its ttl is returned at once + /// and refreshed by a background flight: at most one per entry, only on + /// access, and only while a refresh slot is free. async fn cached( - &self, + self: &Arc, endpoint: &'static str, - cache: &Mutex>, + cache: CacheOf, key: String, fetch: F, ) -> Result, LabError> where - T: Clone, - F: FnOnce() -> Fut, - Fut: std::future::Future, Duration), LabError>>, + T: Clone + Send + Sync + 'static, + F: Fn() -> Fut, + Fut: std::future::Future> + Send + 'static, { let grace = self.timing.stale_grace; - let usable = |e: &Entry| e.servable_stale(grace); - let hit = { - let c = cache.lock().unwrap(); - if c.negative.get(&key).is_some_and(|e| e.fresh()) { - return Ok(None); - } - let hit = c.positive.get(&key); - if let Some(e) = &hit { - if e.fresh() { - return Ok(Some(e.value.clone())); - } - if e.in_backoff() && usable(e) { - count(endpoint, "stale"); - return Ok(Some(e.value.clone())); + // A flight that ends without an answer (it panicked, or the runtime + // is shutting down) is replaced once. + for _ in 0..2 { + let (hit, flight) = { + let mut c = cache(self).lock().unwrap(); + if c.negative.get(&key).is_some_and(|e| e.fresh()) { + return Ok(None); } - } - hit - }; - match fetch().await { - Ok((Some(value), ttl)) => { - let mut c = cache.lock().unwrap(); - c.negative.remove(&key); - c.positive.put(key, value.clone(), ttl); - Ok(Some(value)) - } - Ok((None, ttl)) => { - let mut c = cache.lock().unwrap(); - c.positive.remove(&key); - c.negative.put(key, (), ttl); - Ok(None) - } - Err(LabError::Unavailable(m)) => match hit { - Some(e) if usable(&e) => { - tracing::warn!(error = %m, "Lab unreachable; serving a cached answer"); - count(endpoint, "stale"); - let mut c = cache.lock().unwrap(); - if let Some(entry) = c.positive.map.get_mut(&key) { - // Only the entry we served (not one refreshed meanwhile); - // `fetched` is kept, so the stale window still ends on time. - if entry.fetched == e.fetched { - entry.retry_at = Some(Instant::now() + self.timing.stale_retry); + let hit = c.positive.get(&key); + if let Some(e) = &hit { + if e.fresh() { + if e.refresh_due() && !c.flights.contains_key(&key) { + if let Ok(slot) = self.refresh_slots.clone().try_acquire_owned() { + if let Some(entry) = c.positive.map.get_mut(&key) { + entry.refresh_started = true; + } + self.take_off(&mut c, cache, &key, fetch(), Some(slot)); + } } + return Ok(Some(e.value.clone())); + } + if e.in_backoff() && e.servable_stale(grace) { + count(endpoint, "stale"); + return Ok(Some(e.value.clone())); } - Ok(Some(e.value)) } - _ => Err(LabError::Unavailable(m)), - }, - Err(e) => Err(e), + let flight = if hit.as_ref().is_some_and(|e| e.ttl.is_zero()) { + None + } else if let Some(rx) = c.flights.get(&key) { + Some((rx.clone(), false)) + } else { + Some((self.take_off(&mut c, cache, &key, fetch(), None), true)) + }; + (hit, flight) + }; + let shared = match flight { + None => None, + Some((mut rx, mine)) => { + let landed = rx + .wait_for(|answer| answer.is_some()) + .await + .ok() + .and_then(|answer| answer.clone()); + match landed { + None => continue, + Some(Ok((Some(_), ttl))) if !mine && ttl.is_zero() => None, + Some(result) => Some(result), + } + } + }; + let result = match shared { + Some(result) => result, + None => { + let started = Instant::now(); + let result = fetch().await; + cache(self).lock().unwrap().store(&key, started, &result); + result + } + }; + return match result { + Ok((value, _)) => Ok(value), + Err(LabError::Unavailable(m)) => match hit { + Some(e) if e.servable_stale(grace) => { + tracing::warn!(error = %m, "Lab unreachable; serving a cached answer"); + count(endpoint, "stale"); + let mut c = cache(self).lock().unwrap(); + if let Some(entry) = c.positive.map.get_mut(&key) { + // Only the entry we served (not one refreshed meanwhile); + // `fetched` is kept, so the stale window still ends on time. + if entry.fetched == e.fetched { + entry.retry_at = Some(Instant::now() + self.timing.stale_retry); + } + } + Ok(Some(e.value)) + } + _ => Err(LabError::Unavailable(m)), + }, + Err(e) => Err(e), + }; } + Err(LabError::Unavailable("Lab lookup interrupted".into())) + } + + /// Start the flight for `key` (the caller holds the cache lock and saw + /// none in the air). `slot` is a background refresh's slot, held until + /// it lands. + fn take_off( + self: &Arc, + c: &mut Lookups, + cache: CacheOf, + key: &str, + fetch: Fut, + slot: Option, + ) -> FlightRx + where + T: Clone + Send + Sync + 'static, + Fut: std::future::Future> + Send + 'static, + { + let (tx, rx) = tokio::sync::watch::channel(None); + c.flights.insert(key.to_string(), rx.clone()); + let landing = Landing { + inner: self.clone(), + cache, + key: key.to_string(), + started: Instant::now(), + tx, + landed: false, + }; + tokio::spawn(async move { + let _slot = slot; + let result = fetch.await; + landing.land(result); + }); + rx } async fn to_identity(&self, endpoint: &'static str, a: Answer) -> (Option, Duration) { @@ -581,9 +869,7 @@ impl LabClient { ) } - /// `GET /internal/instances/me`: confirms the instance credentials and - /// tells which deployment they belong to (checked once at boot). - pub async fn instances_me(&self) -> Result { + async fn instances_me(&self) -> Result { let (status, body) = self .request( "instances_me", @@ -603,35 +889,48 @@ impl LabClient { Ok(me) } - pub async fn introspect( - &self, + async fn introspect( + self: &Arc, kind: CredentialKind, credential: &str, account_id: Option<&str>, ) -> Result, LabError> { let key = hashed(&["i", kind.as_str(), credential, account_id.unwrap_or("")]); - self.cached("introspect", &self.identities, key, || async { - let body = serde_json::json!({"kind": kind.as_str(), "credential": credential, "account_id": account_id}); - let a = self.answer("introspect", self.http.post(format!("{}/internal/auth/introspect", self.base)).json(&body)).await?; - Ok(self.to_identity("introspect", a).await) - }).await + let body = serde_json::json!({"kind": kind.as_str(), "credential": credential, "account_id": account_id}); + self.cached("introspect", identities, key, || { + let inner = self.clone(); + let req = self + .http + .post(format!("{}/internal/auth/introspect", self.base)) + .json(&body); + async move { + let a = inner.answer("introspect", req).await?; + Ok(inner.to_identity("introspect", a).await) + } + }) + .await } - pub async fn account(&self, account_id: &str) -> Result, LabError> { + async fn account(self: &Arc, account_id: &str) -> Result, LabError> { let key = hashed(&["a", account_id]); - self.cached("account", &self.identities, key, || async { - let a = self - .answer( - "account", - self.http - .get(format!("{}/internal/accounts/{account_id}", self.base)), - ) - .await?; - let (identity, ttl) = self.to_identity("account", a).await; - if identity.is_none() { - self.forget_balance(account_id); + self.cached("account", identities, key, || { + let inner = self.clone(); + let account_id = account_id.to_string(); + async move { + let a = inner + .answer( + "account", + inner + .http + .get(format!("{}/internal/accounts/{account_id}", inner.base)), + ) + .await?; + let (identity, ttl) = inner.to_identity("account", a).await; + if identity.is_none() { + inner.forget_balance(&account_id); + } + Ok((identity, ttl)) } - Ok((identity, ttl)) }) .await } @@ -641,9 +940,7 @@ impl LabClient { self.balances.lock().unwrap().remove(account_id); } - /// Spendable credits as the Lab last reported them (from a fresh - /// snapshot, or a new one). - pub async fn available_credits(&self, account_id: &str) -> Result, LabError> { + async fn available_credits(&self, account_id: &str) -> Result, LabError> { if let Some(b) = self.balances.lock().unwrap().get(account_id) { if b.fetched.elapsed() < b.ttl { return Ok(Some(Available { @@ -655,12 +952,7 @@ impl LabClient { self.refresh_credits(account_id).await } - /// Take a new balance snapshot now (directly, not through the identity - /// cache: the balance must be current). While the Lab is unavailable the - /// last snapshot is served for `stale_grace` past its expiry (unless it - /// came from a `cache_ttl: 0` answer), asking the Lab again at most - /// every `stale_retry`. - pub async fn refresh_credits(&self, account_id: &str) -> Result, LabError> { + async fn refresh_credits(&self, account_id: &str) -> Result, LabError> { let stale = |b: &Balance| b.servable_stale(self.timing.stale_grace); let served = |credits| { Ok(Some(Available { @@ -708,8 +1000,8 @@ impl LabClient { } } - pub async fn meilisearch( - &self, + async fn meilisearch( + self: &Arc, account_id: &str, url: Option<&str>, ) -> Result, LabError> { @@ -720,7 +1012,8 @@ impl LabClient { ]); let meili_ttl = self.timing.meili_ttl; let negative_ttl = self.timing.negative_ttl; - self.cached("meilisearch", &self.meili, key, || async move { + self.cached("meilisearch", meili, key, || { + let inner = self.clone(); let mut req = self.http.get(format!( "{}/internal/accounts/{account_id}/meilisearch", self.base @@ -728,49 +1021,53 @@ impl LabClient { if let Some(u) = url { req = req.query(&[("url", u)]); } - let (status, body) = self.request("meilisearch", req).await?; - if status == 404 { - // "No engine" is a negative answer: re-asked after 5 s, so - // an engine added in Settings is picked up quickly. - count("meilisearch", "inactive"); - return Ok((None, negative_ttl)); - } - if !(200..300).contains(&status) { - count("meilisearch", "unavailable"); - return Err(LabError::Unavailable(format!("HTTP {status}"))); + async move { + let (status, body) = inner.request("meilisearch", req).await?; + if status == 404 { + // "No engine" is a negative answer: re-asked after 5 s, so + // an engine added in Settings is picked up quickly. + count("meilisearch", "inactive"); + return Ok((None, negative_ttl)); + } + if !(200..300).contains(&status) { + count("meilisearch", "unavailable"); + return Err(LabError::Unavailable(format!("HTTP {status}"))); + } + let m: MeiliAnswer = serde_json::from_slice(&body).map_err(|e| { + count("meilisearch", "unavailable"); + LabError::Unavailable(format!("malformed: {e}")) + })?; + count("meilisearch", "ok"); + Ok(( + Some(MeiliTarget { + url: m.url.trim_end_matches('/').to_string(), + api_key: m.api_key.filter(|k| !k.is_empty()), + }), + meili_ttl, + )) } - let m: MeiliAnswer = serde_json::from_slice(&body).map_err(|e| { - count("meilisearch", "unavailable"); - LabError::Unavailable(format!("malformed: {e}")) - })?; - count("meilisearch", "ok"); - Ok(( - Some(MeiliTarget { - url: m.url.trim_end_matches('/').to_string(), - api_key: m.api_key.filter(|k| !k.is_empty()), - }), - meili_ttl, - )) }) .await } - /// `GET /internal/accounts/{id}/ai`: the Lumen target for this account's - /// LLM calls; `None` when the Lab has none for it (404). A - /// `rejected_key_id` (the key Lumen just refused) skips the cache, asks - /// the Lab for a replacement and makes that the cached answer. - pub async fn ai( - &self, + async fn ai( + self: &Arc, account_id: &str, rejected_key_id: Option<&str>, ) -> Result, LabError> { let key = hashed(&["ai", account_id]); let Some(rejected) = rejected_key_id else { return self - .cached("ai", &self.ai, key, || self.fetch_ai(account_id, None)) + .cached("ai", ai, key, || { + let inner = self.clone(); + let account_id = account_id.to_string(); + async move { inner.fetch_ai(&account_id, None).await } + }) .await; }; // Never served stale: the cached entry is the key that was refused. + // Not a flight either: the answer depends on the refused key, and it + // replaces whatever a flight asked for earlier lands with. let (target, ttl) = self.fetch_ai(account_id, Some(rejected)).await?; let mut c = self.ai.lock().unwrap(); match &target { @@ -824,33 +1121,6 @@ impl LabClient { self.ttl_from(Some(a.cache_ttl)), )) } - - #[cfg(test)] - pub fn debug_cache_keys(&self) -> Vec { - let c = self.identities.lock().unwrap(); - c.positive - .map - .keys() - .chain(c.negative.map.keys()) - .cloned() - .collect() - } - - #[cfg(test)] - pub fn cached_ttl_for_test( - &self, - kind: CredentialKind, - credential: &str, - account_id: Option<&str>, - ) -> Option { - let key = hashed(&["i", kind.as_str(), credential, account_id.unwrap_or("")]); - self.identities - .lock() - .unwrap() - .positive - .get(&key) - .map(|e| e.ttl) - } } #[cfg(any(test, feature = "testing"))] @@ -1901,6 +2171,449 @@ mod tests { assert_eq!(lab.calls(), 0, "the redirect target was never called"); } + // ---- single-flight and refresh-ahead (SCR-90) ---- + + fn slow(lab: &FakeLab, ms: usize) { + lab.state + .slow_ms + .store(ms, std::sync::atomic::Ordering::SeqCst); + } + + fn flights_in_the_air(c: &LabClient) -> usize { + c.inner.identities.lock().unwrap().flights.len() + } + + /// `n` concurrent introspections of the same key. + async fn herd( + c: &Arc, + kind: CredentialKind, + credential: &'static str, + n: usize, + ) -> Vec, LabError>> { + let tasks: Vec<_> = (0..n) + .map(|_| { + let c = c.clone(); + tokio::spawn(async move { c.introspect(kind, credential, None).await }) + }) + .collect(); + let mut out = Vec::new(); + for t in tasks { + out.push(t.await.unwrap()); + } + out + } + + #[tokio::test] + async fn concurrent_callers_of_a_cold_or_expired_key_share_one_lab_call() { + let (lab, c) = setup().await; // ttl 200 ms + let c = Arc::new(c); + lab.set_credential("api_key", "sk_live_a", FakeLab::identity(ACCT, "pro", 50)); + slow(&lab, 50); + for answer in herd(&c, CredentialKind::ApiKey, "sk_live_a", 20).await { + assert_eq!(answer.unwrap().unwrap().account_id, ACCT); + } + assert_eq!(lab.calls(), 1, "a cold key: one call for 20 callers"); + tokio::time::sleep(Duration::from_millis(250)).await; // past ttl + for answer in herd(&c, CredentialKind::ApiKey, "sk_live_a", 20).await { + assert!(answer.unwrap().is_some()); + } + assert_eq!(lab.calls(), 2, "an expired key: one more"); + // Negative answers are shared the same way. + for answer in herd(&c, CredentialKind::ApiKey, "sk_live_unknown", 20).await { + assert_eq!(answer, Ok(None)); + } + assert_eq!(lab.calls(), 3); + assert_eq!(flights_in_the_air(&c), 0, "no flight is left behind"); + } + + #[tokio::test] + async fn a_failed_flight_fails_its_waiters_and_the_key_stays_usable() { + let (lab, c) = setup().await; + let c = Arc::new(c); + lab.set_credential("api_key", "sk_live_a", FakeLab::identity(ACCT, "pro", 50)); + lab.set_down(true); + slow(&lab, 50); + for answer in herd(&c, CredentialKind::ApiKey, "sk_live_a", 10).await { + assert!( + matches!(answer, Err(LabError::Unavailable(_))), + "unknown credential while the Lab is down: {answer:?}" + ); + } + assert_eq!( + lab.calls(), + 1, + "the error is shared, not retried per caller" + ); + assert_eq!(flights_in_the_air(&c), 0); + lab.set_down(false); + assert!(c + .introspect(CredentialKind::ApiKey, "sk_live_a", None) + .await + .unwrap() + .is_some()); + assert_eq!(lab.calls(), 2, "the error was not cached"); + } + + /// A stale-servable entry is still served to every waiter of a failed + /// flight. + #[tokio::test] + async fn waiters_of_a_failed_flight_are_served_stale() { + let (lab, c) = setup().await; // ttl 200, grace 400 (ms) + let c = Arc::new(c); + lab.set_credential("api_key", "sk_live_a", FakeLab::identity(ACCT, "pro", 50)); + c.introspect(CredentialKind::ApiKey, "sk_live_a", None) + .await + .unwrap(); + lab.set_down(true); + slow(&lab, 30); + tokio::time::sleep(Duration::from_millis(250)).await; // past ttl, inside grace + for answer in herd(&c, CredentialKind::ApiKey, "sk_live_a", 10).await { + assert!(answer.unwrap().is_some()); + } + assert_eq!(lab.calls(), 2); + } + + /// The caller that started a flight goes away (an HTTP client that + /// disconnected): the others still get the answer, from the same call. + #[tokio::test] + async fn a_dropped_first_caller_does_not_hang_the_others() { + let (lab, c) = setup().await; // request_timeout 300 ms + let c = Arc::new(c); + lab.set_credential("api_key", "sk_live_a", FakeLab::identity(ACCT, "pro", 50)); + slow(&lab, 150); + let first = { + let c = c.clone(); + tokio::spawn(async move { + c.introspect(CredentialKind::ApiKey, "sk_live_a", None) + .await + }) + }; + tokio::time::sleep(Duration::from_millis(30)).await; + let second = { + let c = c.clone(); + tokio::spawn(async move { + c.introspect(CredentialKind::ApiKey, "sk_live_a", None) + .await + }) + }; + tokio::time::sleep(Duration::from_millis(30)).await; + first.abort(); + assert!(first.await.unwrap_err().is_cancelled()); + let answer = tokio::time::timeout(Duration::from_secs(1), second) + .await + .expect("the second caller is not left waiting") + .unwrap(); + assert!(answer.unwrap().is_some()); + assert_eq!(lab.calls(), 1); + assert_eq!(flights_in_the_air(&c), 0); + + // Every caller gone: the call still completes and its answer is cached. + lab.set_credential("api_key", "sk_live_b", FakeLab::identity(ACCT, "pro", 50)); + let only = { + let c = c.clone(); + tokio::spawn(async move { + c.introspect(CredentialKind::ApiKey, "sk_live_b", None) + .await + }) + }; + tokio::time::sleep(Duration::from_millis(30)).await; + only.abort(); + tokio::time::sleep(Duration::from_millis(180)).await; // the flight has landed + assert_eq!(flights_in_the_air(&c), 0); + slow(&lab, 0); + assert!(c + .introspect(CredentialKind::ApiKey, "sk_live_b", None) + .await + .unwrap() + .is_some()); + assert_eq!(lab.calls(), 2, "served from the abandoned flight's answer"); + } + + fn some_identity() -> Identity { + Identity { + account_id: ACCT.into(), + tier: "pro".into(), + role: None, + api_key_id: None, + limits: None, + } + } + + /// A flight that panics neither strands its waiters nor poisons the key: + /// they start a new one (once), then fail `Unavailable`. + #[tokio::test] + async fn a_panicking_flight_is_replaced_once_and_leaves_no_trace() { + use std::sync::atomic::{AtomicUsize, Ordering}; + let (_lab, c) = setup().await; + let inner = c.inner.clone(); + let calls = Arc::new(AtomicUsize::new(0)); + let lookup = |panics: usize| { + let (inner, calls) = (inner.clone(), calls.clone()); + async move { + inner + .cached("introspect", identities, "k".to_string(), || { + let calls = calls.clone(); + async move { + tokio::time::sleep(Duration::from_millis(30)).await; + if calls.fetch_add(1, Ordering::SeqCst) < panics { + panic!("flight down (expected in this test)"); + } + Ok((Some(some_identity()), Duration::from_secs(60))) + } + }) + .await + } + }; + let herd: Vec<_> = (0..5).map(|_| tokio::spawn(lookup(1))).collect(); + for t in herd { + let answer = tokio::time::timeout(Duration::from_secs(1), t) + .await + .expect("no waiter hangs") + .unwrap(); + assert_eq!(answer, Ok(Some(some_identity()))); + } + assert_eq!( + calls.load(Ordering::SeqCst), + 2, + "the panicked flight and one replacement" + ); + assert_eq!(flights_in_the_air(&c), 0); + + // A lookup that keeps panicking is an error, not a hang or a loop. + c.inner.identities.lock().unwrap().positive.remove("k"); + calls.store(0, Ordering::SeqCst); + assert_eq!( + lookup(usize::MAX).await, + Err(LabError::Unavailable("Lab lookup interrupted".into())) + ); + assert_eq!(calls.load(Ordering::SeqCst), 2); + assert_eq!(flights_in_the_air(&c), 0); + // And the key works again. + calls.store(0, Ordering::SeqCst); + assert_eq!(lookup(0).await, Ok(Some(some_identity()))); + } + + /// ttl 2 s: refresh-ahead from 1.6 s on. + async fn setup_refresh(stale_grace: Duration) -> (FakeLab, Arc) { + let lab = FakeLab::start().await; + let timing = Timing { + default_ttl: Duration::from_secs(2), + stale_grace, + request_timeout: Duration::from_secs(1), + ..fast() + }; + let c = Arc::new(LabClient::with_timing( + &lab.url, + INSTANCE_ID, + SECRET, + timing, + )); + lab.set_credential("api_key", "sk_live_a", FakeLab::identity(ACCT, "pro", 50)); + c.introspect(CredentialKind::ApiKey, "sk_live_a", None) + .await + .unwrap() + .unwrap(); + (lab, c) + } + + #[tokio::test] + async fn an_entry_close_to_expiry_is_served_and_refreshed_in_the_background() { + let (lab, c) = setup_refresh(Duration::from_millis(400)).await; + tokio::time::sleep(Duration::from_millis(1000)).await; + c.introspect(CredentialKind::ApiKey, "sk_live_a", None) + .await + .unwrap(); + tokio::time::sleep(Duration::from_millis(100)).await; + assert_eq!(lab.calls(), 1, "halfway through its ttl: no refresh"); + + tokio::time::sleep(Duration::from_millis(600)).await; // 1.7 s: past 4/5 + lab.set_credential("api_key", "sk_live_a", FakeLab::identity(ACCT, "free", 50)); + slow(&lab, 300); + let started = Instant::now(); + for answer in herd(&c, CredentialKind::ApiKey, "sk_live_a", 10).await { + assert_eq!(answer.unwrap().unwrap().tier, "pro", "the cached answer"); + } + assert!( + started.elapsed() < Duration::from_millis(200), + "nobody waited for the Lab" + ); + tokio::time::sleep(Duration::from_millis(500)).await; // 2.2 s: the refresh landed + assert_eq!(lab.calls(), 2, "exactly one background call"); + assert_eq!(flights_in_the_air(&c), 0); + // Past the first answer's ttl, the refreshed one is served, without + // a Lab call on the request path. + let started = Instant::now(); + let id = c + .introspect(CredentialKind::ApiKey, "sk_live_a", None) + .await + .unwrap() + .unwrap(); + assert_eq!(id.tier, "free"); + assert!(started.elapsed() < Duration::from_millis(200)); + assert_eq!(lab.calls(), 2); + } + + /// The Lab is asked on access only: an entry nobody reads is never + /// refreshed. + #[tokio::test] + async fn an_unused_entry_is_never_refreshed() { + let (lab, _c) = setup_refresh(Duration::from_millis(400)).await; + tokio::time::sleep(Duration::from_millis(2200)).await; + assert_eq!(lab.calls(), 1); + } + + /// A refresh that fails changes nothing: the entry is served until the + /// end of the ttl the Lab gave for it and not a moment longer, and the + /// failed refresh is not repeated. + #[tokio::test] + async fn a_failed_refresh_does_not_extend_the_entry() { + let (lab, c) = setup_refresh(Duration::ZERO).await; // no stale window + tokio::time::sleep(Duration::from_millis(1700)).await; + lab.set_down(true); + for _ in 0..3 { + assert!(c + .introspect(CredentialKind::ApiKey, "sk_live_a", None) + .await + .unwrap() + .is_some()); + tokio::time::sleep(Duration::from_millis(50)).await; + } + assert_eq!(lab.calls(), 2, "one refresh per entry, even when it fails"); + tokio::time::sleep(Duration::from_millis(250)).await; // 2.1 s: past the ttl + assert!( + matches!( + c.introspect(CredentialKind::ApiKey, "sk_live_a", None) + .await, + Err(LabError::Unavailable(_)) + ), + "the failed refresh did not prolong the entry" + ); + } + + /// With a stale window, the failed refresh leaves the stale rule as it + /// was: the window still ends `stale_grace` after the original expiry. + #[tokio::test] + async fn a_failed_refresh_leaves_the_stale_window_unchanged() { + let (lab, c) = setup_refresh(Duration::from_millis(400)).await; + tokio::time::sleep(Duration::from_millis(1700)).await; + lab.set_down(true); + c.introspect(CredentialKind::ApiKey, "sk_live_a", None) + .await + .unwrap() + .unwrap(); + tokio::time::sleep(Duration::from_millis(400)).await; // 2.1 s: stale + assert!(c + .introspect(CredentialKind::ApiKey, "sk_live_a", None) + .await + .unwrap() + .is_some()); + tokio::time::sleep(Duration::from_millis(400)).await; // 2.5 s: past ttl + grace + assert!(matches!( + c.introspect(CredentialKind::ApiKey, "sk_live_a", None) + .await, + Err(LabError::Unavailable(_)) + )); + } + + /// A refresh that learns the credential was revoked stops serving it at + /// once, before the end of its ttl. + #[tokio::test] + async fn a_refresh_that_finds_the_credential_revoked_drops_it_early() { + let (lab, c) = setup_refresh(Duration::from_millis(400)).await; + tokio::time::sleep(Duration::from_millis(1700)).await; + lab.set_credential("api_key", "sk_live_a", json!({"active": false})); + assert!(c + .introspect(CredentialKind::ApiKey, "sk_live_a", None) + .await + .unwrap() + .is_some()); + tokio::time::sleep(Duration::from_millis(50)).await; // 1.75 s: refreshed + assert_eq!( + c.introspect(CredentialKind::ApiKey, "sk_live_a", None) + .await, + Ok(None), + "still inside the first answer's ttl" + ); + assert_eq!(lab.calls(), 2); + } + + /// Background refreshes are capped (a quarter of `max_in_flight`): past + /// the cap an entry is simply not refreshed ahead. + #[tokio::test] + async fn background_refreshes_are_capped() { + let lab = FakeLab::start().await; + let timing = Timing { + default_ttl: Duration::from_secs(2), + request_timeout: Duration::from_secs(1), + capacity: 10, + max_in_flight: 8, // 2 refresh slots + ..fast() + }; + let c = LabClient::with_timing(&lab.url, INSTANCE_ID, SECRET, timing); + let keys: Vec = (0..5).map(|i| format!("sk_live_{i}")).collect(); + for k in &keys { + lab.set_credential("api_key", k, FakeLab::identity(ACCT, "pro", 50)); + c.introspect(CredentialKind::ApiKey, k, None).await.unwrap(); + } + tokio::time::sleep(Duration::from_millis(1700)).await; + slow(&lab, 200); + for k in &keys { + assert!(c + .introspect(CredentialKind::ApiKey, k, None) + .await + .unwrap() + .is_some()); + } + assert_eq!(flights_in_the_air(&c), 2); + tokio::time::sleep(Duration::from_millis(100)).await; + assert_eq!(lab.calls(), 5 + 2, "two refreshes in the air, no more"); + } + + /// `cache_ttl: 0` ("do not cache": an expiring session or token) stays + /// strict under concurrency: a caller never gets an answer the Lab may + /// have computed before it asked. + #[tokio::test] + async fn a_cache_ttl_0_answer_is_fetched_by_every_caller() { + let (lab, c) = setup().await; + let c = Arc::new(c); + let mut v = FakeLab::identity(ACCT, "pro", 50); + v["cache_ttl"] = json!(0); + lab.set_credential("session", "jwt_expiring", v); + slow(&lab, 50); + // Cold: the callers wait for one flight (its ttl is not known yet), + // then all but the one that started it ask again for themselves. + for answer in herd(&c, CredentialKind::Session, "jwt_expiring", 5).await { + assert!(answer.unwrap().is_some()); + } + assert_eq!(lab.calls(), 1 + 4); + // Known to be uncacheable: every caller asks, concurrent or not. + for answer in herd(&c, CredentialKind::Session, "jwt_expiring", 5).await { + assert!(answer.unwrap().is_some()); + } + assert_eq!(lab.calls(), 5 + 5); + c.introspect(CredentialKind::Session, "jwt_expiring", None) + .await + .unwrap(); + assert_eq!(lab.calls(), 11, "a later caller fetches again"); + assert_eq!(flights_in_the_air(&c), 0); + } + + /// A flight asked before the AI key was re-resolved lands after it: the + /// replacement key stays the cached answer. + #[test] + fn a_flight_never_overwrites_a_newer_answer() { + let mut cache: Lookups = Lookups::new(&fast()); + let asked = Instant::now(); + std::thread::sleep(Duration::from_millis(5)); + cache.positive.put("k".into(), 2, Duration::from_secs(60)); + cache.store("k", asked, &Ok((Some(1), Duration::from_secs(60)))); + assert_eq!(cache.positive.get("k").unwrap().value, 2); + cache.store("k", Instant::now(), &Ok((Some(3), Duration::from_secs(60)))); + assert_eq!(cache.positive.get("k").unwrap().value, 3); + cache.store("k", asked, &Ok((None, Duration::from_secs(60)))); + assert!(cache.positive.get("k").is_none(), "a negative answer wins"); + } + #[test] fn base_from_events_url() { assert_eq!( From 5d5c08b3f6c6e96ce6cec65e9d6edd9bdc973e85 Mon Sep 17 00:00:00 2001 From: Quentin de Quelen Date: Thu, 12 Nov 2026 13:25:43 +0100 Subject: [PATCH 03/10] perf(crawler): share the fetcher's DNS cache and connections with robots.txt, fetch it once per origin The robots.txt cache built its own HTTP client: no DNS cache and a separate connection pool, so every new origin paid a second DNS lookup and a second TCP+TLS handshake before its first page. The first URLs of a new origin (up to CONCURRENT_PER_DOMAIN, more across jobs) also each fetched robots.txt, and each held a worker slot for the 10 s timeout when the host hung. - `HttpFetcherBuilder::build_with_robots` / `HttpFetcher::new_with_robots` build the fetcher and its robots.txt cache together (`HttpFetcher::robots_cache`). robots.txt is fetched through the fetcher's default client, so the connection opened for robots.txt is reused for the origin's pages, when that client applies the robots client's policy exactly: same `allow_private_ips`, redirects followed with the default cap, certificates verified, no fetcher-level custom headers or cookies. Otherwise the cache keeps a client of its own, which resolves through the fetcher's DNS cache (`RobotsCache::with_dns_cache`). Both clients come from `safe_client_builder`, so the SSRF resolver, the raw-IP checks and the redirect policy are unchanged. The robots.txt request sets its own timeout, `User-Agent` and `Accept: */*`; on the shared client it also carries the fetcher's `Accept-Language`. The crawler worker uses it; `RobotsCache::new(config)` is unchanged for every other caller. - Single-flight per origin: callers that miss the cache at the same time await one shared fetch. The fetch is not owned by the caller that started it: it runs as long as one caller awaits it and is dropped with the last one, and it leaves the in-flight map when it completes or is dropped. No lock is held across an await. - The body is an `Arc`: a check no longer copies it. The `robotstxt` crate matches while it parses and has no parsed form to keep, so each check still re-reads the text. - The body is read streamed and capped at 500 KiB (truncated at the last complete line, never an error); it is decoded as before (`Response::text`: the `Content-Type` charset, UTF-8 by default). - The cache is bounded: expired entries are dropped on insert (at most once a minute, and when full), and it keeps at most 100 000 origins. What is allowed or disallowed, the TTL, how 404 / other statuses / network errors are read (no rules), `Crawl-delay`, the user agent, `respect_robots: false` and the SSRF refusals are unchanged. Refs SCR-90 Co-Authored-By: Claude Fable 5.2 --- Cargo.lock | 1 + bins/scrapix-worker-crawler/src/lib.rs | 12 +- crates/scrapix-crawler/Cargo.toml | 2 + crates/scrapix-crawler/src/fetcher.rs | 91 +- crates/scrapix-crawler/src/robots.rs | 790 ++++++++++++++---- .../tests/robots_shared_client.rs | 223 +++++ 6 files changed, 952 insertions(+), 167 deletions(-) create mode 100644 crates/scrapix-crawler/tests/robots_shared_client.rs diff --git a/Cargo.lock b/Cargo.lock index 6af12bf8..006951bc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5670,6 +5670,7 @@ dependencies = [ "flate2", "futures", "hickory-resolver", + "hyper 1.8.1", "parking_lot", "quick-xml 0.37.5", "reqwest 0.12.28", diff --git a/bins/scrapix-worker-crawler/src/lib.rs b/bins/scrapix-worker-crawler/src/lib.rs index 72b51580..b6a5252d 100644 --- a/bins/scrapix-worker-crawler/src/lib.rs +++ b/bins/scrapix-worker-crawler/src/lib.rs @@ -41,7 +41,7 @@ use tracing::{debug, info, warn}; use scrapix_core::{CrawlUrl, UrlPatterns}; use scrapix_crawler::{ is_non_page_url_for, url_allowed, ExtractorConfig, HttpFetcher, HttpFetcherBuilder, - RobotsCache, RobotsConfig, SitemapConfig, SitemapParser, UrlExtractor, + RobotsConfig, SitemapConfig, SitemapParser, UrlExtractor, }; #[cfg(feature = "browser")] use scrapix_crawler::{CdpRenderer, CdpRendererBuilder, LightpandaConfig, LightpandaRenderer}; @@ -562,9 +562,6 @@ impl CrawlerWorker { allow_private_ips: args.allow_private_ips, }; - // Create simple in-memory robots cache for the HTTP fetcher - let fetcher_robots_cache = Arc::new(RobotsCache::new(robots_config.clone())?); - // Create sitemap parser if enabled let sitemap_parser = if args.sitemap_discovery { info!("Sitemap discovery enabled"); @@ -602,7 +599,12 @@ impl CrawlerWorker { info!(ttl_secs = args.dns_cache_ttl, "DNS caching enabled"); } - let fetcher = fetcher_builder.build(fetcher_robots_cache.clone())?; + // The robots.txt cache is built with the fetcher: robots.txt goes + // through the fetcher's DNS cache and connection pool, so a new + // origin costs one lookup and one handshake, not two. + let fetcher = fetcher_builder.build_with_robots(robots_config)?; + #[cfg(feature = "browser")] + let fetcher_robots_cache = fetcher.robots_cache(); // Create URL extractor let extractor_config = ExtractorConfig { diff --git a/crates/scrapix-crawler/Cargo.toml b/crates/scrapix-crawler/Cargo.toml index 1d3b563c..cb843d8a 100644 --- a/crates/scrapix-crawler/Cargo.toml +++ b/crates/scrapix-crawler/Cargo.toml @@ -24,6 +24,8 @@ url = { workspace = true } # HTML parsing (for link extraction) scraper = { workspace = true } +# `http::Response`, to decode a size-capped robots.txt body as reqwest does +hyper = { workspace = true } # Robots.txt robotstxt = { workspace = true } diff --git a/crates/scrapix-crawler/src/fetcher.rs b/crates/scrapix-crawler/src/fetcher.rs index f7946b74..191d7ecb 100644 --- a/crates/scrapix-crawler/src/fetcher.rs +++ b/crates/scrapix-crawler/src/fetcher.rs @@ -151,7 +151,7 @@ impl FetchOptions { } use crate::dns::{CachingDnsResolver, DnsCacheStats, DnsConfig}; -use crate::robots::RobotsCache; +use crate::robots::{RobotsCache, RobotsConfig}; use crate::safe_client::{reject_ip_host, safe_client_builder, safe_redirect_policy}; use crate::safe_dns::{validate_proxy_url, NonPublicAddress}; @@ -329,13 +329,7 @@ impl HttpFetcher { robots_cache: Arc, dns_resolver: Option>, ) -> Result { - let cookie_jar = (config.cookie_store || !config.initial_cookies.is_empty()).then(|| { - let jar = reqwest::cookie::Jar::default(); - for (cookie, url) in &config.initial_cookies { - jar.add_cookie_str(cookie, url); - } - Arc::new(jar) - }); + let cookie_jar = Self::cookie_jar(&config); let client = Self::build_client(&config, dns_resolver.clone(), None, cookie_jar.clone())?; Ok(Self { @@ -348,6 +342,74 @@ impl HttpFetcher { }) } + /// Create a fetcher together with its robots.txt cache, so robots.txt + /// is fetched through the fetcher's DNS cache and, when both would + /// build the same kind of client, through the fetcher's own client: the + /// connection opened for an origin's robots.txt is then reused for its + /// pages (one DNS lookup and one TCP+TLS handshake per new origin + /// instead of two). + /// + /// The client is shared only when it applies the robots client's policy + /// exactly (see [`Self::robots_can_share_client`]); otherwise the cache + /// gets its own client, as with [`RobotsCache::new`], resolving through + /// the fetcher's DNS cache when there is one. + pub fn new_with_robots( + config: FetcherConfig, + robots_config: RobotsConfig, + dns_resolver: Option>, + ) -> Result { + let cookie_jar = Self::cookie_jar(&config); + let client = Self::build_client(&config, dns_resolver.clone(), None, cookie_jar.clone())?; + let robots_cache = if Self::robots_can_share_client(&config, &robots_config) { + RobotsCache::with_client(robots_config, client.clone())? + } else if let Some(ref dns) = dns_resolver { + RobotsCache::with_dns_cache(robots_config, dns.clone())? + } else { + RobotsCache::new(robots_config)? + }; + + Ok(Self { + client, + config, + robots_cache: Arc::new(robots_cache), + dns_resolver, + proxy_clients: dashmap::DashMap::new(), + cookie_jar, + }) + } + + /// Whether a robots.txt request sent through this fetcher's default + /// client is the request a dedicated robots client would send: same + /// SSRF policy (`allow_private_ips`), same redirect policy (followed, + /// same cap), certificates verified, and nothing of the fetcher's added + /// to it (custom headers, cookies). The robots request sets its own + /// timeout, `User-Agent` and `Accept`; the fetcher's `Accept-Language` + /// is the one header it gains. + fn robots_can_share_client(config: &FetcherConfig, robots_config: &RobotsConfig) -> bool { + config.allow_private_ips == robots_config.allow_private_ips + && config.follow_redirects + && config.max_redirects == crate::safe_client::DEFAULT_MAX_REDIRECTS + && !config.accept_invalid_certs + && config.custom_headers.is_empty() + && !config.cookie_store + && config.initial_cookies.is_empty() + } + + /// The robots.txt cache this fetcher checks. + pub fn robots_cache(&self) -> Arc { + self.robots_cache.clone() + } + + fn cookie_jar(config: &FetcherConfig) -> Option> { + (config.cookie_store || !config.initial_cookies.is_empty()).then(|| { + let jar = reqwest::cookie::Jar::default(); + for (cookie, url) in &config.initial_cookies { + jar.add_cookie_str(cookie, url); + } + Arc::new(jar) + }) + } + /// Build a `reqwest::Client` from `config`. Always starts from /// [`safe_client_builder`] (SSRF resolver, raw-IP-refusing redirect /// policy, `.no_proxy()`); when `proxy` is given, the explicit job proxy @@ -1153,6 +1215,19 @@ impl HttpFetcherBuilder { }; HttpFetcher::new_with_dns(self.config, robots_cache, dns_resolver) } + + /// Build the fetcher and its robots.txt cache together (see + /// [`HttpFetcher::new_with_robots`]): robots.txt goes through the + /// fetcher's DNS cache and, when possible, its connection pool. The + /// cache is [`HttpFetcher::robots_cache`]. + pub fn build_with_robots(self, robots_config: RobotsConfig) -> Result { + let dns_resolver = if let Some(dns_config) = self.dns_config { + Some(Arc::new(CachingDnsResolver::new(dns_config)?)) + } else { + None + }; + HttpFetcher::new_with_robots(self.config, robots_config, dns_resolver) + } } impl Default for HttpFetcherBuilder { diff --git a/crates/scrapix-crawler/src/robots.rs b/crates/scrapix-crawler/src/robots.rs index 29e79271..17fd11bb 100644 --- a/crates/scrapix-crawler/src/robots.rs +++ b/crates/scrapix-crawler/src/robots.rs @@ -5,10 +5,12 @@ //! across worker instances. use std::collections::HashMap; +use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::Arc; use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; -use parking_lot::RwLock; +use futures::future::{BoxFuture, FutureExt, Shared, WeakShared}; +use parking_lot::{Mutex, RwLock}; use reqwest::Client; use robotstxt::DefaultMatcher; use serde::{Deserialize, Serialize}; @@ -17,6 +19,8 @@ use url::Url; use scrapix_core::{Result, ScrapixError}; +use crate::dns::CachingDnsResolver; + /// Cache key for a robots.txt entry: `scheme://host:port` of `url`. /// /// robots.txt applies per origin (RFC 9309 §2.3), so `http://a.test`, @@ -72,37 +76,168 @@ impl Default for RobotsConfig { } } +/// Largest robots.txt body kept, in bytes (500 KiB, the limit Google +/// applies). A longer body is truncated at the last complete line before +/// the limit; the rest is never read. +pub const MAX_ROBOTS_BODY_BYTES: usize = 500 * 1024; + +/// Most origins [`RobotsCache`] keeps. Past it, expired entries go first, +/// then arbitrary ones. +const MAX_CACHED_ORIGINS: usize = 100_000; + +/// How often an insert also drops the expired entries. +const SWEEP_INTERVAL: Duration = Duration::from_secs(60); + /// Cached robots.txt entry struct CachedRobots { - /// Raw robots.txt content - content: String, + /// Raw robots.txt content, shared with every check (never copied). + content: Arc, /// When this entry was cached cached_at: Instant, /// Crawl delay from robots.txt (in milliseconds) crawl_delay_ms: Option, } -/// Cache for robots.txt files -pub struct RobotsCache { +/// The cached origins and when expired ones were last dropped. +struct CacheState { + entries: HashMap, + last_sweep: Instant, +} + +impl CacheState { + fn new() -> Self { + Self { + entries: HashMap::new(), + last_sweep: Instant::now(), + } + } + + /// Insert `entry`, keeping at most `max` origins: expired entries are + /// dropped when the cache is full (and every [`SWEEP_INTERVAL`]), then + /// arbitrary ones (about 1% at a time, so a full cache does not scan on + /// every insert). + fn insert_bounded(&mut self, key: String, entry: CachedRobots, ttl: Duration, max: usize) { + let full = self.entries.len() >= max && !self.entries.contains_key(&key); + if full || self.last_sweep.elapsed() >= SWEEP_INTERVAL { + self.entries.retain(|_, e| e.cached_at.elapsed() < ttl); + self.last_sweep = Instant::now(); + } + if self.entries.len() >= max && !self.entries.contains_key(&key) { + let excess = self.entries.len() + 1 - max; + let victims: Vec = self + .entries + .keys() + .take(excess.max(max / 100)) + .cloned() + .collect(); + for victim in victims { + self.entries.remove(&victim); + } + } + self.entries.insert(key, entry); + } +} + +/// One robots.txt fetch, shared by every caller waiting for that origin. +type RobotsFetch = Shared>>; + +/// A [`RobotsFetch`] that does not keep the fetch alive. +type WeakRobotsFetch = WeakShared>>; + +/// State shared by the cache and its in-flight fetches. +struct RobotsShared { config: RobotsConfig, client: Client, - cache: RwLock>, + cache: RwLock, + /// The fetch in progress for an origin (cache key), with its id. Held + /// weakly: the fetch lives as long as one caller awaits it, and removes + /// its entry when it completes or is dropped ([`FlightGuard`]). + in_flight: Mutex>, + next_flight_id: AtomicU64, +} + +impl RobotsShared { + /// The cached content of `key`, unless missing or expired. + fn fresh(&self, key: &str) -> Option> { + let cache = self.cache.read(); + let entry = cache.entries.get(key)?; + (entry.cached_at.elapsed() < self.config.cache_ttl).then(|| entry.content.clone()) + } +} + +/// Removes an in-flight entry when its fetch ends, however it ends: +/// completion, or every waiting caller dropped (cancelled) before that. +struct FlightGuard { + shared: Arc, + key: String, + id: u64, +} + +impl Drop for FlightGuard { + fn drop(&mut self) { + let mut in_flight = self.shared.in_flight.lock(); + if in_flight + .get(&self.key) + .is_some_and(|(id, _)| *id == self.id) + { + in_flight.remove(&self.key); + } + } +} + +/// Cache for robots.txt files +pub struct RobotsCache { + shared: Arc, } impl RobotsCache { - /// Create a new robots.txt cache + /// Create a new robots.txt cache with its own HTTP client (own + /// connection pool, system DNS). pub fn new(config: RobotsConfig) -> Result { - let client = crate::safe_client::safe_client_builder(None, config.allow_private_ips) + Self::with_dns(config, None) + } + + /// Create a robots.txt cache whose client resolves hosts through + /// `dns_cache` (the page fetcher's resolver), so a new origin is + /// resolved once for robots.txt and its pages. Same SSRF policy as + /// [`Self::new`]. + pub fn with_dns_cache( + config: RobotsConfig, + dns_cache: Arc, + ) -> Result { + Self::with_dns(config, Some(dns_cache)) + } + + fn with_dns(config: RobotsConfig, dns_cache: Option>) -> Result { + let client = crate::safe_client::safe_client_builder(dns_cache, config.allow_private_ips) .timeout(config.fetch_timeout) .user_agent(&config.user_agent) .build() .map_err(|e| ScrapixError::Crawl(format!("Failed to build robots client: {}", e)))?; + Ok(Self::from_parts(config, client)) + } - Ok(Self { - config, - client, - cache: RwLock::new(HashMap::new()), - }) + /// Create a robots.txt cache that fetches through `client`, sharing its + /// connection pool and resolver. The caller guarantees `client` comes + /// from [`crate::safe_client::safe_client_builder`] with + /// `config.allow_private_ips` and the default redirect policy (see + /// `HttpFetcherBuilder::build_with_robots`, the only caller). + pub(crate) fn with_client(config: RobotsConfig, client: Client) -> Result { + reqwest::header::HeaderValue::from_str(&config.user_agent) + .map_err(|e| ScrapixError::Crawl(format!("Failed to build robots client: {}", e)))?; + Ok(Self::from_parts(config, client)) + } + + fn from_parts(config: RobotsConfig, client: Client) -> Self { + Self { + shared: Arc::new(RobotsShared { + config, + client, + cache: RwLock::new(CacheState::new()), + in_flight: Mutex::new(HashMap::new()), + next_flight_id: AtomicU64::new(0), + }), + } } /// Create a new robots.txt cache with default configuration @@ -113,7 +248,8 @@ impl RobotsCache { /// Check if a URL is allowed by robots.txt #[instrument(skip(self))] pub async fn is_allowed(&self, url: &str) -> Result { - if !self.config.respect_robots { + let config = &self.shared.config; + if !config.respect_robots { return Ok(true); } @@ -124,10 +260,11 @@ impl RobotsCache { let robots_content = self.get_robots(domain, &parsed).await?; let path = parsed.path(); - // Use robotstxt crate to check if allowed + // The robotstxt crate has no parsed form to keep: it matches while + // it parses, so each check re-reads the (shared, uncopied) body. let mut matcher = DefaultMatcher::default(); let allowed = - matcher.one_agent_allowed_by_robots(&robots_content, &self.config.user_agent, path); + matcher.one_agent_allowed_by_robots(&robots_content, &config.user_agent, path); debug!(url, allowed, "Robots.txt check"); Ok(allowed) @@ -140,21 +277,22 @@ impl RobotsCache { pub fn cached_crawl_delay(&self, url: &str) -> Option> { let parsed = Url::parse(url).ok()?; let key = robots_cache_key(&parsed).ok()?; - let cache = self.cache.read(); - let entry = cache.get(&key)?; - (entry.cached_at.elapsed() < self.config.cache_ttl).then_some(entry.crawl_delay_ms) + let cache = self.shared.cache.read(); + let entry = cache.entries.get(&key)?; + (entry.cached_at.elapsed() < self.shared.config.cache_ttl).then_some(entry.crawl_delay_ms) } /// Get crawl delay for a domain pub async fn get_crawl_delay(&self, domain: &str) -> Result> { + let config = &self.shared.config; let (robots_url, key) = domain_robots_url(domain)?; let domain = key.as_str(); // Check cache first { - let cache = self.cache.read(); - if let Some(entry) = cache.get(domain) { - if entry.cached_at.elapsed() < self.config.cache_ttl { - return Ok(entry.crawl_delay_ms.or(self.config.default_crawl_delay_ms)); + let cache = self.shared.cache.read(); + if let Some(entry) = cache.entries.get(domain) { + if entry.cached_at.elapsed() < config.cache_ttl { + return Ok(entry.crawl_delay_ms.or(config.default_crawl_delay_ms)); } } } @@ -162,139 +300,242 @@ impl RobotsCache { let _ = self.get_robots(domain, &robots_url).await?; // Now check cache again - let cache = self.cache.read(); - if let Some(entry) = cache.get(domain) { - return Ok(entry.crawl_delay_ms.or(self.config.default_crawl_delay_ms)); + let cache = self.shared.cache.read(); + if let Some(entry) = cache.entries.get(domain) { + return Ok(entry.crawl_delay_ms.or(config.default_crawl_delay_ms)); } - Ok(self.config.default_crawl_delay_ms) + Ok(config.default_crawl_delay_ms) } /// Get robots.txt content for an origin, fetching if needed. `domain` /// is the cache key from [`robots_cache_key`]. - async fn get_robots(&self, domain: &str, url: &Url) -> Result { - // Check cache first - { - let cache = self.cache.read(); - if let Some(entry) = cache.get(domain) { - if entry.cached_at.elapsed() < self.config.cache_ttl { - return Ok(entry.content.clone()); - } - } - } - - // Fetch robots.txt - let content = self.fetch_robots(url).await?; - - // Parse crawl delay - let crawl_delay_ms = self.parse_crawl_delay(&content); - - // Cache the result - { - let mut cache = self.cache.write(); - cache.insert( - domain.to_string(), - CachedRobots { - content: content.clone(), - cached_at: Instant::now(), - crawl_delay_ms, - }, - ); + /// + /// Callers that miss the cache for the same origin at the same time + /// share one fetch: the first starts it, the others await its result. + /// The fetch is not tied to the caller that started it: it runs as long + /// as one caller awaits it, and is dropped when the last one is. + async fn get_robots(&self, domain: &str, url: &Url) -> Result> { + let shared = &self.shared; + if let Some(content) = shared.fresh(domain) { + return Ok(content); } - Ok(content) - } - - /// Fetch robots.txt from a URL - async fn fetch_robots(&self, url: &Url) -> Result { // Raw-IP hosts bypass `SafeResolver` entirely (reqwest never calls // the DNS resolver for a URL whose host is already an address), so // they must be refused explicitly here too, exactly like raw-IP // seed URLs in `HttpFetcher::fetch_inner`. crate::safe_client::reject_ip_host(url)?; - - // Keep a non-default port: robots.txt lives on the same authority as - // the page (RFC 9309 §2.3). - let robots_url = format!( - "{}://{}{}/robots.txt", - url.scheme(), - url.host_str().unwrap_or(""), - url.port().map(|p| format!(":{p}")).unwrap_or_default() - ); - - debug!(robots_url, "Fetching robots.txt"); - - match self.client.get(&robots_url).send().await { - Ok(response) => { - if response.status().is_success() { - let content = response.text().await.unwrap_or_else(|_| String::new()); - Ok(content) - } else if response.status().as_u16() == 404 { - // No robots.txt means everything is allowed - debug!(robots_url, "No robots.txt found (404)"); - Ok(String::new()) - } else { - warn!( - robots_url, - status = response.status().as_u16(), - "Failed to fetch robots.txt" - ); - // On error, be permissive - Ok(String::new()) - } + let robots_url = robots_txt_url(url); + + let fetch: RobotsFetch = { + let mut in_flight = shared.in_flight.lock(); + // A fetch stores its result before leaving `in_flight`, so with + // the lock held an origin is cached, in flight, or neither. + if let Some(content) = shared.fresh(domain) { + return Ok(content); } - Err(e) => { - warn!(robots_url, error = %e, "Error fetching robots.txt"); - // On network error, be permissive - Ok(String::new()) + match in_flight.get(domain).and_then(|(_, fetch)| fetch.upgrade()) { + Some(fetch) => fetch, + None => { + let id = shared.next_flight_id.fetch_add(1, Ordering::Relaxed); + let fetch = fetch_and_cache(shared.clone(), domain.to_string(), robots_url, id) + .boxed() + .shared(); + if let Some(weak) = fetch.downgrade() { + in_flight.insert(domain.to_string(), (id, weak)); + } + fetch + } } - } + }; + + Ok(fetch.await) } /// Parse crawl delay from robots.txt content + #[cfg(test)] fn parse_crawl_delay(&self, content: &str) -> Option { - let mut in_user_agent_section = false; - let user_agent_lower = self.config.user_agent.to_lowercase(); - - for line in content.lines() { - let line = line.trim().to_lowercase(); - - if line.starts_with("user-agent:") { - let agent = line.trim_start_matches("user-agent:").trim(); - in_user_agent_section = agent == "*" || agent == user_agent_lower; - } else if in_user_agent_section && line.starts_with("crawl-delay:") { - let delay_str = line.trim_start_matches("crawl-delay:").trim(); - if let Ok(delay) = delay_str.parse::() { - if delay < 0.0 || delay.is_nan() || delay.is_infinite() { - return None; - } - // Convert seconds to milliseconds - return Some((delay * 1000.0) as u64); - } - } - } - - None + parse_crawl_delay(&self.shared.config.user_agent, content) } /// Clear the cache pub fn clear_cache(&self) { - let mut cache = self.cache.write(); - cache.clear(); + let mut cache = self.shared.cache.write(); + cache.entries.clear(); } /// Get cache stats pub fn cache_stats(&self) -> (usize, usize) { - let cache = self.cache.read(); - let total = cache.len(); + let cache = self.shared.cache.read(); + let total = cache.entries.len(); let expired = cache + .entries .values() - .filter(|e| e.cached_at.elapsed() >= self.config.cache_ttl) + .filter(|e| e.cached_at.elapsed() >= self.shared.config.cache_ttl) .count(); (total, total - expired) } } +/// The robots.txt URL of `url`'s origin. Keeps a non-default port: +/// robots.txt lives on the same authority as the page (RFC 9309 §2.3). +fn robots_txt_url(url: &Url) -> String { + format!( + "{}://{}{}/robots.txt", + url.scheme(), + url.host_str().unwrap_or(""), + url.port().map(|p| format!(":{p}")).unwrap_or_default() + ) +} + +/// Fetch `robots_url`, cache the result under `key` and leave the in-flight +/// map. Owns everything it needs, so it outlives the caller that started it. +async fn fetch_and_cache( + shared: Arc, + key: String, + robots_url: String, + id: u64, +) -> Arc { + let _guard = FlightGuard { + shared: shared.clone(), + key: key.clone(), + id, + }; + let content: Arc = fetch_robots_body(&shared.client, &shared.config, &robots_url) + .await + .into(); + let crawl_delay_ms = parse_crawl_delay(&shared.config.user_agent, &content); + shared.cache.write().insert_bounded( + key, + CachedRobots { + content: content.clone(), + cached_at: Instant::now(), + crawl_delay_ms, + }, + shared.config.cache_ttl, + MAX_CACHED_ORIGINS, + ); + content +} + +/// Fetch a robots.txt. Never fails: a 404, another non-2xx status, a +/// network error or an unreadable body all mean "no rules" (permissive). +async fn fetch_robots_body(client: &Client, config: &RobotsConfig, robots_url: &str) -> String { + debug!(robots_url, "Fetching robots.txt"); + + // Set per request, so a client shared with the page fetcher (other + // defaults) sends the same request as a dedicated one. + let request = client + .get(robots_url) + .timeout(config.fetch_timeout) + .header(reqwest::header::USER_AGENT, &config.user_agent) + .header(reqwest::header::ACCEPT, "*/*"); + + match request.send().await { + Ok(response) => { + if response.status().is_success() { + read_capped_text(response).await + } else if response.status().as_u16() == 404 { + // No robots.txt means everything is allowed + debug!(robots_url, "No robots.txt found (404)"); + String::new() + } else { + warn!( + robots_url, + status = response.status().as_u16(), + "Failed to fetch robots.txt" + ); + // On error, be permissive + String::new() + } + } + Err(e) => { + warn!(robots_url, error = %e, "Error fetching robots.txt"); + // On network error, be permissive + String::new() + } + } +} + +/// Read a robots.txt body as text, streamed, keeping at most +/// [`MAX_ROBOTS_BODY_BYTES`]: a longer (or endless) body is cut at the last +/// complete line before the limit and the rest is not read. A read error +/// gives an empty body, as before. Decoding is `Response::text`'s (the +/// `Content-Type` charset, UTF-8 by default). +async fn read_capped_text(mut response: reqwest::Response) -> String { + let content_type = response + .headers() + .get(reqwest::header::CONTENT_TYPE) + .cloned(); + let mut bytes: Vec = Vec::new(); + let mut truncated = false; + loop { + match response.chunk().await { + Ok(Some(chunk)) => { + let room = MAX_ROBOTS_BODY_BYTES - bytes.len(); + if chunk.len() > room { + bytes.extend_from_slice(&chunk[..room]); + truncated = true; + break; + } + bytes.extend_from_slice(&chunk); + } + Ok(None) => break, + Err(_) => return String::new(), + } + } + if truncated { + // Half a line could be a different rule (`Disallow: /` cut from + // `Disallow: /private`): keep whole lines only. + let keep = bytes.iter().rposition(|&b| b == b'\n').map_or(0, |i| i + 1); + bytes.truncate(keep); + warn!( + url = %response.url(), + kept_bytes = bytes.len(), + "robots.txt is over the size limit; the rest is ignored" + ); + } + drop(response); + + let mut decoded = hyper::Response::new(reqwest::Body::from(bytes)); + if let Some(content_type) = content_type { + decoded + .headers_mut() + .insert(reqwest::header::CONTENT_TYPE, content_type); + } + reqwest::Response::from(decoded) + .text() + .await + .unwrap_or_default() +} + +/// Parse the crawl delay `user_agent` gets from robots.txt content +fn parse_crawl_delay(user_agent: &str, content: &str) -> Option { + let mut in_user_agent_section = false; + let user_agent_lower = user_agent.to_lowercase(); + + for line in content.lines() { + let line = line.trim().to_lowercase(); + + if line.starts_with("user-agent:") { + let agent = line.trim_start_matches("user-agent:").trim(); + in_user_agent_section = agent == "*" || agent == user_agent_lower; + } else if in_user_agent_section && line.starts_with("crawl-delay:") { + let delay_str = line.trim_start_matches("crawl-delay:").trim(); + if let Ok(delay) = delay_str.parse::() { + if delay < 0.0 || delay.is_nan() || delay.is_infinite() { + return None; + } + // Convert seconds to milliseconds + return Some((delay * 1000.0) as u64); + } + } + } + + None +} + #[cfg(test)] mod key_tests { use super::*; @@ -383,6 +624,277 @@ mod tests { } } +#[cfg(test)] +mod fetch_tests { + use super::*; + use wiremock::matchers::{method, path}; + use wiremock::{Mock, MockServer, ResponseTemplate}; + + fn config() -> RobotsConfig { + RobotsConfig { + allow_private_ips: true, + ..Default::default() + } + } + + /// wiremock binds 127.0.0.1 and raw-IP hosts are refused: use localhost. + fn page(server: &MockServer, p: &str) -> String { + format!("{}{}", server.uri().replace("127.0.0.1", "localhost"), p) + } + + async fn robots_requests(server: &MockServer) -> usize { + server + .received_requests() + .await + .unwrap() + .iter() + .filter(|r| r.url.path() == "/robots.txt") + .count() + } + + async fn slow_robots(delay: Duration) -> MockServer { + let server = MockServer::start().await; + Mock::given(method("GET")) + .and(path("/robots.txt")) + .respond_with( + ResponseTemplate::new(200) + .set_body_string("User-agent: *\nDisallow: /private\nCrawl-delay: 2\n") + .set_delay(delay), + ) + .mount(&server) + .await; + server + } + + #[tokio::test] + async fn concurrent_checks_of_a_new_origin_share_one_fetch() { + let server = slow_robots(Duration::from_millis(200)).await; + let cache = Arc::new(RobotsCache::new(config()).unwrap()); + + let checks: Vec<_> = (0..16) + .map(|i| { + let cache = cache.clone(); + let url = if i % 2 == 0 { + page(&server, "/private/x") + } else { + page(&server, &format!("/public/{i}")) + }; + tokio::spawn(async move { cache.is_allowed(&url).await.unwrap() }) + }) + .collect(); + for (i, check) in checks.into_iter().enumerate() { + assert_eq!(check.await.unwrap(), i % 2 == 1, "check {i}"); + } + + assert_eq!(robots_requests(&server).await, 1); + assert!(cache.shared.in_flight.lock().is_empty()); + assert_eq!( + cache.cached_crawl_delay(&page(&server, "/")), + Some(Some(2000)) + ); + // Cached from now on. + assert!(!cache.is_allowed(&page(&server, "/private")).await.unwrap()); + assert_eq!(robots_requests(&server).await, 1); + } + + #[tokio::test] + async fn a_cancelled_first_caller_does_not_strand_the_others() { + let server = slow_robots(Duration::from_millis(300)).await; + let cache = Arc::new(RobotsCache::new(config()).unwrap()); + let url = page(&server, "/private/x"); + + let leader = { + let (cache, url) = (cache.clone(), url.clone()); + tokio::spawn(async move { cache.is_allowed(&url).await }) + }; + tokio::time::sleep(Duration::from_millis(50)).await; + let follower = { + let (cache, url) = (cache.clone(), url.clone()); + tokio::spawn(async move { cache.is_allowed(&url).await.unwrap() }) + }; + tokio::time::sleep(Duration::from_millis(50)).await; + leader.abort(); + assert!(leader.await.unwrap_err().is_cancelled()); + + let allowed = tokio::time::timeout(Duration::from_secs(5), follower) + .await + .expect("the follower finishes the shared fetch") + .unwrap(); + assert!(!allowed); + assert_eq!(robots_requests(&server).await, 1); + assert!(cache.shared.in_flight.lock().is_empty()); + } + + #[tokio::test] + async fn a_fetch_nobody_awaits_is_dropped_and_started_again() { + let server = slow_robots(Duration::from_millis(300)).await; + let cache = Arc::new(RobotsCache::new(config()).unwrap()); + let url = page(&server, "/private/x"); + + let callers: Vec<_> = (0..3) + .map(|_| { + let (cache, url) = (cache.clone(), url.clone()); + tokio::spawn(async move { cache.is_allowed(&url).await }) + }) + .collect(); + tokio::time::sleep(Duration::from_millis(50)).await; + assert_eq!(cache.shared.in_flight.lock().len(), 1); + for caller in &callers { + caller.abort(); + } + for caller in callers { + let _ = caller.await; + } + // Nothing left behind, nothing cached from the abandoned fetch. + assert!(cache.shared.in_flight.lock().is_empty()); + assert_eq!(cache.cache_stats(), (0, 0)); + + assert!(!cache.is_allowed(&url).await.unwrap()); + assert!(cache.shared.in_flight.lock().is_empty()); + assert_eq!(cache.cache_stats(), (1, 1)); + } + + #[tokio::test] + async fn status_and_error_handling_is_permissive() { + for status in [404, 403, 500, 503] { + let server = MockServer::start().await; + Mock::given(path("/robots.txt")) + .respond_with(ResponseTemplate::new(status).set_body_string("Disallow: /")) + .mount(&server) + .await; + let cache = RobotsCache::new(config()).unwrap(); + assert!( + cache.is_allowed(&page(&server, "/x")).await.unwrap(), + "{status}" + ); + assert_eq!( + cache.cached_crawl_delay(&page(&server, "/x")), + Some(None), + "{status} is cached as no rules" + ); + } + + // Nothing listens there: a network error allows, and is cached. + let cache = RobotsCache::new(config()).unwrap(); + assert!(cache.is_allowed("http://localhost:1/x").await.unwrap()); + assert_eq!(cache.cache_stats(), (1, 1)); + + // Raw-IP hosts are refused, private addresses too without the opt-in. + assert!(matches!( + cache.is_allowed("http://127.0.0.1:1/x").await, + Err(ScrapixError::Refused(_)) + )); + let server = slow_robots(Duration::ZERO).await; + let strict = RobotsCache::new(RobotsConfig::default()).unwrap(); + assert!(strict + .is_allowed(&page(&server, "/private/x")) + .await + .unwrap()); + assert_eq!(robots_requests(&server).await, 0); + + // `respect_robots: false` never fetches. + let off = RobotsCache::new(RobotsConfig { + respect_robots: false, + ..config() + }) + .unwrap(); + assert!(off.is_allowed(&page(&server, "/private/x")).await.unwrap()); + assert_eq!(robots_requests(&server).await, 0); + } + + #[tokio::test] + async fn an_oversized_body_is_cut_at_a_line_before_the_limit() { + let mut body = String::from("User-agent: *\nDisallow: /early\n"); + let mut i = 0; + while body.len() < MAX_ROBOTS_BODY_BYTES + 200_000 { + body.push_str(&format!("Disallow: /filler/{i:08}/\n")); + i += 1; + } + body.push_str("Disallow: /late\n"); + let server = MockServer::start().await; + Mock::given(path("/robots.txt")) + .respond_with(ResponseTemplate::new(200).set_body_string(body.clone())) + .mount(&server) + .await; + + let cache = RobotsCache::new(config()).unwrap(); + assert!(!cache.is_allowed(&page(&server, "/early")).await.unwrap()); + assert!(!cache + .is_allowed(&page(&server, "/filler/00000001/x")) + .await + .unwrap()); + assert!(cache.is_allowed(&page(&server, "/late")).await.unwrap()); + + let key = robots_cache_key(&Url::parse(&page(&server, "/")).unwrap()).unwrap(); + let kept = cache.shared.fresh(&key).unwrap(); + assert!(kept.len() <= MAX_ROBOTS_BODY_BYTES); + assert!(kept.len() > MAX_ROBOTS_BODY_BYTES - 64); + assert!(kept.ends_with("/\n"), "cut at a line end"); + assert!(body.starts_with(&*kept)); + } + + #[tokio::test] + async fn the_body_is_decoded_with_its_charset() { + let server = MockServer::start().await; + // "Disallow: /caf\xe9" in ISO-8859-1, and a UTF-8 BOM case below. + Mock::given(path("/robots.txt")) + .respond_with(ResponseTemplate::new(200).set_body_raw( + b"User-agent: *\nDisallow: /caf\xe9\n".to_vec(), + "text/plain; charset=iso-8859-1", + )) + .mount(&server) + .await; + let cache = RobotsCache::new(config()).unwrap(); + cache.is_allowed(&page(&server, "/x")).await.unwrap(); + let key = robots_cache_key(&Url::parse(&page(&server, "/")).unwrap()).unwrap(); + assert_eq!( + &*cache.shared.fresh(&key).unwrap(), + "User-agent: *\nDisallow: /caf\u{e9}\n" + ); + } + + #[test] + fn the_cache_is_bounded() { + let ttl = Duration::from_secs(60); + let entry = |age: Duration| CachedRobots { + content: Arc::from(""), + cached_at: Instant::now() - age, + crawl_delay_ms: None, + }; + let mut state = CacheState::new(); + for i in 0..10 { + state.insert_bounded(format!("old{i}"), entry(ttl * 2), ttl, 20); + } + for i in 0..10 { + state.insert_bounded(format!("new{i}"), entry(Duration::ZERO), ttl, 20); + } + assert_eq!(state.entries.len(), 20); + + // Full: the expired entries go first. + state.insert_bounded("x".into(), entry(Duration::ZERO), ttl, 20); + assert_eq!(state.entries.len(), 11); + assert!(state.entries.keys().all(|k| !k.starts_with("old"))); + + // Full of live entries: some are evicted, the new one is kept. + for i in 0..100 { + state.insert_bounded(format!("more{i}"), entry(Duration::ZERO), ttl, 20); + assert!(state.entries.len() <= 20); + assert!(state.entries.contains_key(&format!("more{i}"))); + } + // Replacing a cached origin evicts nothing. + let before = state.entries.len(); + state.insert_bounded("more99".into(), entry(Duration::ZERO), ttl, 20); + assert_eq!(state.entries.len(), before); + + // Expired entries are dropped on a later insert even when not full. + let mut state = CacheState::new(); + state.insert_bounded("old".into(), entry(ttl * 2), ttl, 20); + state.last_sweep = Instant::now() - SWEEP_INTERVAL; + state.insert_bounded("new".into(), entry(Duration::ZERO), ttl, 20); + assert_eq!(state.entries.len(), 1); + } +} + // ============================================================================ // Persistent Robots Cache (RocksDB-backed) // ============================================================================ @@ -721,40 +1233,10 @@ impl PersistentRobotsCache { /// Fetch robots.txt from URL async fn fetch_robots(&self, url: &Url) -> Result { - // See the identical comment in `RobotsCache::fetch_robots`: raw-IP + // See the identical comment in `RobotsCache::get_robots`: raw-IP // hosts bypass `SafeResolver` entirely and must be refused here too. crate::safe_client::reject_ip_host(url)?; - - let robots_url = format!( - "{}://{}{}/robots.txt", - url.scheme(), - url.host_str().unwrap_or(""), - url.port().map(|p| format!(":{p}")).unwrap_or_default() - ); - - debug!(robots_url, "Fetching robots.txt"); - - match self.client.get(&robots_url).send().await { - Ok(response) => { - if response.status().is_success() { - Ok(response.text().await.unwrap_or_default()) - } else if response.status().as_u16() == 404 { - debug!(robots_url, "No robots.txt found (404)"); - Ok(String::new()) - } else { - warn!( - robots_url, - status = response.status().as_u16(), - "Failed to fetch robots.txt" - ); - Ok(String::new()) - } - } - Err(e) => { - warn!(robots_url, error = %e, "Error fetching robots.txt"); - Ok(String::new()) - } - } + Ok(fetch_robots_body(&self.client, &self.config, &robots_txt_url(url)).await) } /// Parse crawl delay from robots.txt diff --git a/crates/scrapix-crawler/tests/robots_shared_client.rs b/crates/scrapix-crawler/tests/robots_shared_client.rs new file mode 100644 index 00000000..6f49f2e5 --- /dev/null +++ b/crates/scrapix-crawler/tests/robots_shared_client.rs @@ -0,0 +1,223 @@ +//! A fetcher built with its robots.txt cache (`build_with_robots`) fetches +//! robots.txt through its own DNS cache and connection pool: one lookup and +//! one connection per new origin instead of two. The robots.txt request +//! itself stays what a dedicated robots client sends, and the client is not +//! shared when the fetcher's would send something else. + +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::Arc; + +use parking_lot::Mutex; +use scrapix_core::CrawlUrl; +use scrapix_crawler::{HttpFetcher, HttpFetcherBuilder, RobotsCache, RobotsConfig}; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; +use tokio::net::TcpListener; + +/// A keep-alive HTTP/1.1 server that counts the connections it accepts and +/// records each request head. +struct Origin { + port: u16, + connections: Arc, + requests: Arc>>, +} + +impl Origin { + async fn start() -> Self { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let port = listener.local_addr().unwrap().port(); + let connections = Arc::new(AtomicUsize::new(0)); + let requests = Arc::new(Mutex::new(Vec::new())); + let (accepted, seen) = (connections.clone(), requests.clone()); + tokio::spawn(async move { + loop { + let Ok((mut socket, _)) = listener.accept().await else { + return; + }; + accepted.fetch_add(1, Ordering::SeqCst); + let seen = seen.clone(); + tokio::spawn(async move { + let mut buffer = Vec::new(); + let mut chunk = [0u8; 4096]; + loop { + while let Some(end) = find(&buffer, b"\r\n\r\n") { + let head = String::from_utf8_lossy(&buffer[..end]).to_lowercase(); + buffer.drain(..end + 4); + let body = if head.starts_with("get /robots.txt ") { + "User-agent: *\nDisallow: /private\n" + } else { + "next" + }; + seen.lock().push(head); + let response = format!( + "HTTP/1.1 200 OK\r\ncontent-type: text/html\r\ncontent-length: {}\r\nconnection: keep-alive\r\n\r\n{}", + body.len(), + body + ); + if socket.write_all(response.as_bytes()).await.is_err() { + return; + } + } + match socket.read(&mut chunk).await { + Ok(0) | Err(_) => return, + Ok(n) => buffer.extend_from_slice(&chunk[..n]), + } + } + }); + } + }); + Self { + port, + connections, + requests, + } + } + + fn url(&self, path: &str) -> CrawlUrl { + CrawlUrl::seed(format!("http://localhost:{}{}", self.port, path)) + } + + fn connections(&self) -> usize { + self.connections.load(Ordering::SeqCst) + } + + /// The recorded request heads (lowercased) for `path`. + fn requests_for(&self, path: &str) -> Vec { + self.requests + .lock() + .iter() + .filter(|head| head.starts_with(&format!("get {path} "))) + .cloned() + .collect() + } +} + +fn find(haystack: &[u8], needle: &[u8]) -> Option { + haystack.windows(needle.len()).position(|w| w == needle) +} + +fn robots_config() -> RobotsConfig { + RobotsConfig { + user_agent: "RobotsBot/1.0".to_string(), + allow_private_ips: true, + ..Default::default() + } +} + +fn builder() -> HttpFetcherBuilder { + HttpFetcherBuilder::new() + .user_agent("PageBot/1.0") + .allow_private_ips(true) + .max_retries(0) +} + +async fn fetch_two_pages(fetcher: &HttpFetcher, origin: &Origin) { + let page = fetcher.fetch(&origin.url("/page")).await.unwrap(); + assert!(page.html.contains("/next")); + let page = fetcher.fetch(&origin.url("/other")).await.unwrap(); + assert_eq!(page.status, 200); + let err = fetcher.fetch(&origin.url("/private/x")).await.unwrap_err(); + assert!(err.to_string().contains("Robots.txt disallowed"), "{err}"); +} + +#[tokio::test] +async fn robots_and_pages_share_one_connection_and_one_lookup() { + let origin = Origin::start().await; + let fetcher = builder() + .with_dns_cache() + .build_with_robots(robots_config()) + .unwrap(); + + fetch_two_pages(&fetcher, &origin).await; + + assert_eq!( + origin.connections(), + 1, + "robots.txt and pages, one connection" + ); + let robots = origin.requests_for("/robots.txt"); + assert_eq!(robots.len(), 1, "robots.txt is fetched once"); + // The robots.txt request keeps its own identity on the shared client. + assert!( + robots[0].contains("user-agent: robotsbot/1.0"), + "{}", + robots[0] + ); + assert!(robots[0].contains("accept: */*"), "{}", robots[0]); + let pages = origin.requests_for("/page"); + assert!(pages[0].contains("user-agent: pagebot/1.0"), "{}", pages[0]); + assert!(pages[0].contains("accept: text/html"), "{}", pages[0]); + + let dns = fetcher.dns_cache_stats().unwrap(); + assert_eq!(dns.misses, 1, "one lookup for the origin: {dns:?}"); + + // The cache the fetcher checks is the one it hands out. + let cache = fetcher.robots_cache(); + assert_eq!( + cache.cached_crawl_delay(&origin.url("/x").url), + Some(None), + "cached by the fetch" + ); + assert!(!cache.is_allowed(&origin.url("/private").url).await.unwrap()); + assert_eq!(origin.requests_for("/robots.txt").len(), 1); +} + +#[tokio::test] +async fn a_separately_built_robots_cache_keeps_its_own_connection() { + let origin = Origin::start().await; + let fetcher = builder() + .build(Arc::new(RobotsCache::new(robots_config()).unwrap())) + .unwrap(); + + fetch_two_pages(&fetcher, &origin).await; + + assert_eq!(origin.connections(), 2, "one for robots.txt, one for pages"); + assert_eq!(origin.requests_for("/robots.txt").len(), 1); +} + +#[tokio::test] +async fn the_client_is_not_shared_when_it_would_change_the_robots_request() { + // A fetcher-level header must not reach robots.txt: own client, but + // still the fetcher's DNS cache. + let origin = Origin::start().await; + let fetcher = builder() + .header("X-Tenant", "secret") + .with_dns_cache() + .build_with_robots(robots_config()) + .unwrap(); + fetch_two_pages(&fetcher, &origin).await; + assert_eq!(origin.connections(), 2); + let robots = origin.requests_for("/robots.txt"); + assert_eq!(robots.len(), 1); + assert!(!robots[0].contains("x-tenant"), "{}", robots[0]); + assert!(origin.requests_for("/page")[0].contains("x-tenant: secret")); + let dns = fetcher.dns_cache_stats().unwrap(); + assert_eq!(dns.misses, 1, "{dns:?}"); + assert!( + dns.hits >= 1, + "the second client resolves from the cache: {dns:?}" + ); + + // A stricter robots SSRF policy than the fetcher's is kept: robots.txt + // on a private address is refused (so no rules), never fetched through + // the fetcher's more permissive client. + let origin = Origin::start().await; + let fetcher = builder() + .build_with_robots(RobotsConfig { + allow_private_ips: false, + ..robots_config() + }) + .unwrap(); + let page = fetcher.fetch(&origin.url("/private/x")).await.unwrap(); + assert_eq!(page.status, 200); + assert!(origin.requests_for("/robots.txt").is_empty()); + + // Redirects not followed by the fetcher: robots.txt still follows its + // own, through its own client. + let origin = Origin::start().await; + let fetcher = builder() + .follow_redirects(false) + .build_with_robots(robots_config()) + .unwrap(); + fetch_two_pages(&fetcher, &origin).await; + assert_eq!(origin.connections(), 2); +} From ab9310f73dfe86aa52edccd39087c65963893b3b Mon Sep 17 00:00:00 2001 From: Quentin de Quelen Date: Thu, 12 Nov 2026 13:24:29 +0100 Subject: [PATCH 04/10] perf(frontier): give a domain the rate its delay allows, whatever the dispatch period A domain was granted one slot per dispatcher pass at most, and only when a full delay had passed since the last grant. The time between a request becoming due and the pass that granted it was lost, so a site got 1 / (ceil(delay / period) x period) requests per second instead of 1 / delay: 30 URLs/s for a 25 ms delay with a pass every 33 ms. The politeness stores now keep a virtual last-request time per domain (GCRA). A request is granted when now >= last + delay, and the grant sets last = max(last + delay, now - credit, pause end). The credit is 100 ms (DOMAIN_SPACING_CREDIT_MS, 0 = the old strict spacing) for delays under 1 s and 0 from 1 s up, so robots Crawl-delay and slow job rates are spaced exactly as before. Any n + 1 consecutive grants span at least n x delay - credit: the average rate never exceeds 1 / delay and a burst is at most 1 + credit / delay grants, within the free in-flight slots of the domain and the job. A pause banks no credit, and a 429 or an error drops the credit banked so far. PolitenessStore::try_acquire_many grants up to one slot per token in one atomic call (one Lua call in Redis) and says what one more request would get. The dispatcher asks once per domain per pass, sends what was granted and parks the domain's other URLs for the reported wait. Redis keeps last_ms as its only spacing field, so instances running the previous script can share the keys during a rolling upgrade. Refs SCR-90 Co-Authored-By: Claude Fable 5.2 --- bins/scrapix-frontier-service/src/lib.rs | 305 +++++-- crates/scrapix-frontier/src/lib.rs | 5 +- crates/scrapix-frontier/src/politeness.rs | 743 ++++++++++++++++-- .../scrapix-frontier/src/politeness_redis.rs | 259 +++--- .../tests/politeness_redis.rs | 442 +++++++++++ 5 files changed, 1531 insertions(+), 223 deletions(-) diff --git a/bins/scrapix-frontier-service/src/lib.rs b/bins/scrapix-frontier-service/src/lib.rs index c4068524..6cc5430e 100644 --- a/bins/scrapix-frontier-service/src/lib.rs +++ b/bins/scrapix-frontier-service/src/lib.rs @@ -68,9 +68,9 @@ pub use timeout_store::{TimeoutStore, STORE_CALL_TIMEOUT}; use scrapix_core::{Ack, CrawlUrl}; use scrapix_frontier::{ extract_domain, Acquire, Admission, CrawlRecord, FetchReport, FetchSignal, FrontierStore, - JobCounters, JobLimits, JobRunState, LinkGraph, LinkGraphConfig, MemoryFrontierStore, + Grants, JobCounters, JobLimits, JobRunState, LinkGraph, LinkGraphConfig, MemoryFrontierStore, PolitenessConfig, PolitenessScheduler, PolitenessStore, RecrawlConfig, RecrawlDecision, - RecrawlScheduler, SlotRequest, UrlHistory, UrlHistoryConfig, + RecrawlScheduler, SlotsRequest, UrlHistory, UrlHistoryConfig, }; use scrapix_queue::{ topic_names, AnyConsumer, AnyProducer, ConsumerBuilder, CrawlEvent, CrawlHistoryMessage, @@ -118,6 +118,10 @@ const DISPATCH_JOB_CONCURRENCY: usize = 32; /// How many politeness slots (one per distinct domain) a batch asks for at /// once. const ACQUIRE_CONCURRENCY: usize = 32; +/// Most slots asked for one domain in one politeness call. The store grants +/// far fewer (its spacing credit and the domain's free slots bound a call); +/// a domain with no delay asks again in the same pass. +const MAX_SLOTS_PER_ASK: usize = 32; /// How many jobs the progress publisher reads and publishes at once. const PROGRESS_CONCURRENCY: usize = 32; /// Pops that found nothing, in a row, before an idle job is polled less @@ -296,7 +300,26 @@ pub async fn build_store(args: &Args) -> anyhow::Result> } } -/// Politeness settings from `args`. +/// `DOMAIN_SPACING_CREDIT_MS`: the spacing credit (ms), i.e. how much of +/// the time lost between a request to a domain becoming due and the +/// dispatch pass that sends it is given back, as requests sent closer +/// together (never more than `1 + credit / delay` at once, and none for +/// delays of 1 s or more). It makes a domain's rate `1 / delay` whatever +/// the dispatch interval. `0` spaces requests strictly from the last one +/// sent. Default 100. Read from the environment rather than `Args`, whose +/// fields other binaries (`scrapix all`) list one by one. +fn spacing_credit_from_env() -> u64 { + let default = PolitenessConfig::default().spacing_credit_ms; + match std::env::var("DOMAIN_SPACING_CREDIT_MS") { + Ok(raw) => raw.trim().parse().unwrap_or_else(|_| { + warn!(value = %raw, default, "Invalid DOMAIN_SPACING_CREDIT_MS; using the default"); + default + }), + Err(_) => default, + } +} + +/// Politeness settings from `args` (and `DOMAIN_SPACING_CREDIT_MS`). pub fn politeness_config(args: &Args) -> PolitenessConfig { PolitenessConfig { default_delay_ms: args.domain_delay_ms, @@ -306,6 +329,8 @@ pub fn politeness_config(args: &Args) -> PolitenessConfig { robots_delay_multiplier: args.robots_delay_multiplier, concurrent_per_domain: args.concurrent_per_domain.max(1), slot_ttl: Duration::from_secs(args.request_timeout_secs.max(1) * 2), + spacing_credit_ms: spacing_credit_from_env(), + ..PolitenessConfig::default() } } @@ -1443,13 +1468,19 @@ impl FrontierService { /// pop in the counters, so nothing is counted twice). Returns whether /// this instance holds the job's dispatch lease. /// - /// Politeness slots are asked for one URL per domain at a time, in pop - /// order, `ACQUIRE_CONCURRENCY` domains at once (the store checks the - /// domain and job caps atomically, so asking concurrently cannot grant - /// more than serially). A domain that was granted a slot with a - /// non-zero delay cannot be granted another one before that delay, so - /// its other URLs of the batch are parked for it without asking; with a - /// zero delay they ask in the next round. The granted URLs' bus sends + /// Politeness slots are asked for one domain at a time, in pop order, + /// `ACQUIRE_CONCURRENCY` domains at once (the store checks the domain + /// and job caps atomically, so asking concurrently cannot grant more + /// than serially). One call asks for a slot per URL of the domain (at + /// most `MAX_SLOTS_PER_ASK`) and the store grants as many as the + /// domain's schedule and the caps allow: one, or several when this pass + /// came later than the domain's delay (its spacing credit, see + /// `scrapix_frontier::politeness`), so the rate a domain gets does not + /// depend on how often the dispatcher passes. The store also says how + /// long the domain's next grant is away: its other URLs of the batch + /// are parked for that long without asking again; only a domain that + /// could be granted more right now asks in the next round. The granted + /// URLs' bus sends /// all run at once and are awaited together at the end of the batch: one /// send waits for the producer's linger and the broker's ack, so sending /// them one after another capped the frontier at a few dozen URLs per @@ -1550,11 +1581,10 @@ impl FrontierService { .flat_map(|(_, urls)| urls) .collect() } - // Each round asks for one slot per domain, for the domain's first - // URL. A domain that then cannot take another URL of this batch - // (busy, waiting, or granted a slot with a delay) has its other URLs - // parked without asking again; one granted with no delay asks for - // its next URL in the next round. + // Each round asks once per domain, for its first URLs. A domain + // that then cannot take another URL of this batch (busy, or its + // next grant is not due) has its other URLs parked without asking + // again; one that could be granted more asks in the next round. 'batch: while !domains.is_empty() { let mut next_round: Vec = Vec::new(); let mut round = domains.into_iter(); @@ -1586,20 +1616,22 @@ impl FrontierService { } } - // The slot token becomes the dispatched message's id, which + // A slot's token becomes the dispatched message's id, which // the crawler echoes back in its FetchFeedback. - let acquired = futures::future::join_all(chunk.iter().map(|(domain, _)| { - let token = uuid::Uuid::new_v4().to_string(); + let acquired = futures::future::join_all(chunk.iter().map(|(domain, urls)| { + let tokens: Vec = (0..urls.len().min(MAX_SLOTS_PER_ASK)) + .map(|_| uuid::Uuid::new_v4().to_string()) + .collect(); async move { - let slot = SlotRequest { + let slots = SlotsRequest { domain, job_id, - token: &token, + tokens: &tokens, limits, }; let acquired = tokio::time::timeout( STORE_CALL_TIMEOUT, - self.politeness.try_acquire_spaced(&slot), + self.politeness.try_acquire_many(&slots), ) .await .unwrap_or_else(|_| { @@ -1607,37 +1639,41 @@ impl FrontierService { "politeness try_acquire timed out".to_string(), )) }); - (token, acquired) + (tokens, acquired) } })) .await; let mut job_busy = false; - for ((domain, mut urls), (token, acquired)) in chunk.into_iter().zip(acquired) { - let Some(url) = urls.pop_front() else { - continue; - }; - // `granted`: the URL is sent. `wait`: how long the - // domain's other URLs (and this one, when not granted) - // are parked, so the next ticks don't pop and requeue - // them over and over; `None`: the domain may be granted - // again right away (zero delay). `by_politeness`: the - // wait is the domain's or the job's, not an error. + for ((domain, mut urls), (tokens, acquired)) in chunk.into_iter().zip(acquired) { + // `granted`: how many of the domain's URLs are sent. + // `wait`: how long its other URLs are parked, so the + // next ticks don't pop and requeue them over and over; + // `None`: the domain may be granted more right away. + // `by_politeness`: the wait is the domain's or the + // job's, not an error. let (granted, wait, by_politeness) = match acquired { - Ok((Acquire::Granted, spacing)) => { - (true, (!spacing.is_zero()).then_some(spacing), true) - } - Ok((Acquire::Wait(wait), _)) => (false, Some(wait), true), - Ok((Acquire::DomainBusy, _)) => (false, Some(BUSY_RETRY), true), - Ok((Acquire::JobBusy, _)) => { - job_busy = true; - (false, Some(BUSY_RETRY), true) + Ok(Grants { granted, next }) => { + let wait = match next { + // A store that grants nothing and refuses + // nothing would be asked forever. + Acquire::Granted if granted == 0 => Some(BUSY_RETRY), + Acquire::Granted => None, + Acquire::Wait(wait) => Some(wait), + Acquire::DomainBusy => Some(BUSY_RETRY), + Acquire::JobBusy => { + job_busy = true; + Some(BUSY_RETRY) + } + }; + (granted.min(tokens.len()), wait, true) } Err(e) => { warn!(job_id = %job_id, domain = %domain, error = %e, "Politeness check failed"); - (false, Some(ERROR_RETRY), false) + (0, Some(ERROR_RETRY), false) } }; + let send: Vec = urls.drain(..granted.min(urls.len())).collect(); let mut parked = 0; match wait { Some(wait) => { @@ -1647,41 +1683,34 @@ impl FrontierService { None if !urls.is_empty() => next_round.push((domain.clone(), urls)), None => {} } - let url = if granted { - Some(url) - } else { - parked += 1; - bounced.push(park(url, wait.unwrap_or_default())); - None - }; self.metrics .urls_delayed .fetch_add(parked as u64, Ordering::Relaxed); if by_politeness { politeness_bounced += parked; } - let Some(url) = url else { - continue; - }; - let mut msg = template.child(url); - msg.message_id = token; - let mut hard_stop = hard_stop.clone(); - sends.push(async move { - let key = msg.partition_key(); - let send = self - .producer - .send(topic_names::URL_PROCESSING, Some(&key), &msg); - // Raced only against the shutdown hard stop (see - // `drain_dispatcher`); normally the send runs to completion. - let sent = tokio::select! { - r = send => r, - _ = hard_stop.wait_for(|stop| *stop) => Err(scrapix_core::ScrapixError::Queue( - "send abandoned at shutdown".to_string(), - )), - }; - (domain, msg, sent) - }); + for (url, token) in send.into_iter().zip(tokens) { + let mut msg = template.child(url); + msg.message_id = token; + let mut hard_stop = hard_stop.clone(); + let domain = domain.clone(); + sends.push(async move { + let key = msg.partition_key(); + let send = + self.producer + .send(topic_names::URL_PROCESSING, Some(&key), &msg); + // Raced only against the shutdown hard stop (see + // `drain_dispatcher`); normally the send runs to completion. + let sent = tokio::select! { + r = send => r, + _ = hard_stop.wait_for(|stop| *stop) => Err(scrapix_core::ScrapixError::Queue( + "send abandoned at shutdown".to_string(), + )), + }; + (domain, msg, sent) + }); + } } if job_busy { // Every other URL of the job would be refused too. @@ -2285,6 +2314,7 @@ pub async fn run_with_bus( mod tests { use super::*; use scrapix_core::JobSpec; + use scrapix_frontier::SlotRequest; use scrapix_queue::{ChannelBus, CrawlEvent, FetchFeedback}; use std::time::Instant; @@ -3075,11 +3105,15 @@ mod tests { "4 rps over 1.5 s: at least 3 dispatches, got {}", at.len() ); + // One at the start, then one per 250 ms: never more than the rate. + assert!(at.len() <= 7, "4 rps over 1.5 s: {} dispatches", at.len()); for w in at.windows(2) { let gap = w[1] - w[0]; - // 250 ms minus receive jitter (dispatch tick + poll interval). + // 250 ms, minus the spacing credit a late pass gives back (at + // most 100 ms) and receive jitter (dispatch tick + poll + // interval): no two requests at once. assert!( - gap >= Duration::from_millis(200), + gap >= Duration::from_millis(120), "dispatches {gap:?} apart" ); } @@ -3758,7 +3792,7 @@ mod tests { assert!(h.service.dispatching.lock().is_empty()); } - /// Counts the slots asked of an in-memory politeness scheduler. + /// Counts the calls asking an in-memory politeness scheduler for slots. struct CountingPoliteness { inner: PolitenessScheduler, asks: AtomicU64, @@ -3776,6 +3810,10 @@ mod tests { self.asks.fetch_add(1, Ordering::SeqCst); self.inner.try_acquire_spaced(req).await } + async fn try_acquire_many(&self, req: &SlotsRequest<'_>) -> scrapix_core::Result { + self.asks.fetch_add(1, Ordering::SeqCst); + self.inner.try_acquire_many(req).await + } async fn release( &self, domain: &str, @@ -3793,7 +3831,7 @@ mod tests { } /// One dispatcher pass over a job with `urls` URLs on one domain: - /// (slots asked, URLs dispatched, URLs back in the store). + /// (politeness calls, URLs dispatched, URLs back in the store). async fn one_pass_on_one_domain(args: Args, urls: usize) -> (u64, u64, u64) { let store: Arc = Arc::new(MemoryFrontierStore::default()); seed_store(&*store, "job-asks", urls).await; @@ -3823,15 +3861,126 @@ mod tests { assert_eq!(one_pass_on_one_domain(args, 10).await, (1, 1, 9)); } - /// With a zero delay a domain can be granted again right away, up to - /// its concurrency: its URLs keep asking until the domain is busy. + /// With a zero delay a domain is granted up to its concurrency, in one + /// call. #[tokio::test] - async fn zero_delay_domain_keeps_asking_until_busy() { + async fn zero_delay_domain_is_granted_until_busy_in_one_call() { let mut args = test_args(); args.domain_delay_ms = 0; args.concurrent_per_domain = 3; - // Three grants, then one `DomainBusy` parks the other six. - assert_eq!(one_pass_on_one_domain(args, 10).await, (4, 3, 7)); + // Three grants and `DomainBusy` for the rest, which is parked. + assert_eq!(one_pass_on_one_domain(args, 10).await, (1, 3, 7)); + } + + /// With a zero delay and more URLs than one call asks for, the domain + /// asks again in the same pass. + #[tokio::test] + async fn zero_delay_domain_asks_again_past_one_call() { + let mut args = test_args(); + args.domain_delay_ms = 0; + args.concurrent_per_domain = 10_000; + let urls = MAX_SLOTS_PER_ASK + 6; + assert!(urls <= INITIAL_POP_SIZE, "one pop takes them all"); + assert_eq!( + one_pass_on_one_domain(args, urls).await, + (2, urls as u64, 0) + ); + } + + /// SCR-90: a pass that comes later than the domain's delay is granted + /// the slots it is owed in one call, up to the spacing credit, and the + /// other URLs are parked without asking again. + #[tokio::test] + async fn late_pass_gets_the_grants_it_is_owed_in_one_call() { + let mut args = test_args(); + args.domain_delay_ms = 25; + args.min_domain_delay_ms = 25; + args.concurrent_per_domain = 8; + let store: Arc = Arc::new(MemoryFrontierStore::default()); + seed_store(&*store, "job-late", 20).await; + let politeness = Arc::new(CountingPoliteness { + inner: PolitenessScheduler::new(politeness_config(&args)), + asks: AtomicU64::new(0), + }); + let bus = ChannelBus::new(); + let mut service = build_service(&bus, &args, store.clone()).await; + service.politeness = politeness.clone(); + let sent = || service.metrics.urls_dispatched.load(Ordering::Relaxed); + + // A domain never requested has no credit: one grant. + service.dispatch_tick().await; + assert_eq!((politeness.asks.load(Ordering::SeqCst), sent()), (1, 1)); + // 60 ms later two delays have passed: two grants, in one call. + tokio::time::sleep(Duration::from_millis(60)).await; + service.dispatch_tick().await; + assert_eq!(politeness.asks.load(Ordering::SeqCst), 2); + // (Three if this pass ran over 75 ms after the first.) + assert!((3..=4).contains(&sent()), "{} sent", sent()); + // After a long gap the credit is capped at 1 + 100 / 25 grants, and + // the domain has 8 slots, 3 (or 4) of them held: 8 sent in all. + tokio::time::sleep(Duration::from_millis(400)).await; + service.dispatch_tick().await; + assert_eq!((politeness.asks.load(Ordering::SeqCst), sent()), (3, 8)); + assert_eq!(store.queued("job-late").await.unwrap(), 12); + } + + /// URLs per second dispatched to one site with a 25 ms delay and a + /// dispatcher pass every 33 ms, measured over two seconds. + async fn single_site_rate(spacing_credit_ms: u64) -> f64 { + let store = Arc::new(FlakyStore::default()); + seed_store(&*store, "job-rate", 500).await; + let mut args = test_args(); + args.domain_delay_ms = 25; + args.min_domain_delay_ms = 25; + args.dispatch_interval_ms = 33; + // No crawler sends feedback here: don't let held slots cap the test. + args.concurrent_per_domain = 10_000; + let bus = ChannelBus::new(); + let mut service = build_service(&bus, &args, store.clone()).await; + service.politeness = Arc::new(PolitenessScheduler::new(PolitenessConfig { + spacing_credit_ms, + ..politeness_config(&args) + })); + let service = Arc::new(service); + let h = Harness { + bus, + handle: tokio::spawn(service.clone().run()), + service, + }; + // Measure from the first dispatch. + let started = Instant::now(); + while h.service.metrics.urls_dispatched.load(Ordering::Relaxed) == 0 { + assert!(started.elapsed() < Duration::from_secs(2), "no dispatch"); + tokio::time::sleep(Duration::from_millis(1)).await; + } + let t0 = Instant::now(); + let before = h.service.metrics.urls_dispatched.load(Ordering::Relaxed); + tokio::time::sleep(Duration::from_millis(2_000)).await; + let after = h.service.metrics.urls_dispatched.load(Ordering::Relaxed); + let secs = t0.elapsed().as_secs_f64(); + h.stop(); + (after - before) as f64 / secs + } + + /// SCR-90: a site's rate is what its delay allows, not what the + /// dispatch interval allows. A 25 ms delay is 40 URLs/s; with a pass + /// every 33 ms, one grant per pass is 30 URLs/s at best. + #[tokio::test] + async fn single_site_rate_follows_the_delay_not_the_dispatch_interval() { + let rate = single_site_rate(100).await; + // Never above the delay's rate (plus the 100 ms credit and the + // sends of the last pass, over two seconds); well above 30/s. + assert!(rate <= 43.5, "{rate:.1} URLs/s for a 25 ms delay"); + assert!(rate >= 36.0, "{rate:.1} URLs/s for a 25 ms delay"); + } + + /// A spacing credit of 0 is the strict spacing of before: one + /// grant per pass. + #[tokio::test] + async fn without_spacing_credit_a_site_gets_one_url_per_pass() { + let rate = single_site_rate(0).await; + assert!(rate <= 31.5, "{rate:.1} URLs/s with a pass every 33 ms"); + assert!(rate >= 20.0, "{rate:.1} URLs/s with a pass every 33 ms"); } /// SCR-89: a job with nothing ready and a paused job are not asked diff --git a/crates/scrapix-frontier/src/lib.rs b/crates/scrapix-frontier/src/lib.rs index 30ec1d43..f96ccd8c 100644 --- a/crates/scrapix-frontier/src/lib.rs +++ b/crates/scrapix-frontier/src/lib.rs @@ -77,8 +77,9 @@ pub use history::{ pub use linkgraph::{LinkGraph, LinkGraphBuilder, LinkGraphConfig, LinkGraphStats}; pub use partition::{extract_domain, DomainGrouper, PartitionConfig, Partitioner}; pub use politeness::{ - adapt_on_success, Acquire, DomainStats, FetchReport, FetchSignal, JobLimits, PolitenessConfig, - PolitenessScheduler, PolitenessStore, RobotsInfo, SlotRequest, + adapt_on_success, spacing_credit, Acquire, DomainStats, FetchReport, FetchSignal, Grants, + JobLimits, PolitenessConfig, PolitenessScheduler, PolitenessStore, RobotsInfo, SlotRequest, + SlotsRequest, }; #[cfg(feature = "redis-store")] pub use politeness_redis::RedisPoliteness; diff --git a/crates/scrapix-frontier/src/politeness.rs b/crates/scrapix-frontier/src/politeness.rs index ee2b4c05..777129bd 100644 --- a/crates/scrapix-frontier/src/politeness.rs +++ b/crates/scrapix-frontier/src/politeness.rs @@ -9,7 +9,8 @@ //! the fetch back (`FetchFeedback`), not when the URL reaches the bus. //! Every slot also expires after [`PolitenessConfig::slot_ttl`], so lost //! feedback cannot wedge a domain. -//! - Two requests to one domain are at least the *effective delay* apart: +//! - Requests to one domain are one *effective delay* apart (on average +//! below 1 s, at least from 1 s up: see "Spacing credit"): //! `max(adaptive delay, job min delay, 1000 / job rps, robots crawl-delay × //! multiplier)`, clamped to `[min(min_delay_ms, default_delay_ms), //! max_delay_ms]`. The adaptive delay starts at `default_delay_ms` (the @@ -27,6 +28,44 @@ //! `feedback timestamp + Retry-After` (ignored if already past). //! - A job has at most `JobLimits::max_in_flight` requests in flight. //! +//! ## Spacing credit (SCR-90) +//! +//! The dispatcher asks for slots in passes (every `DISPATCH_INTERVAL_MS` at +//! best). Spacing requests from the *time of the last grant* loses whatever +//! time passed between the moment a request was due and the pass that +//! granted it, so the achieved rate was `1 / (ceil(delay / period) × +//! period)` instead of `1 / delay`. The stores therefore keep a *virtual* +//! last-request time per domain (GCRA, a leaky bucket in time units): +//! +//! - a request is granted when `now >= last + delay` (`delay` is the +//! effective delay of the asking job, evaluated at each check, so a delay +//! raised since the last grant applies at once); +//! - the grant then sets `last = max(last + delay, now - credit, +//! pause end)`: the domain's schedule advances by one delay, and is at +//! most `credit` behind the clock. `credit` is +//! [`PolitenessConfig::spacing_credit_ms`] (100 ms) when `delay <` +//! [`PolitenessConfig::strict_spacing_ms`] (1 s), else 0. +//! +//! Consequences, for a constant delay `D` and credit `C`: +//! +//! - any `n + 1` consecutive grants span at least `n × D − C`: the long-run +//! rate never exceeds `1 / D`, and a pass that comes late grants what it +//! is owed (several grants for one domain in one call, each taking one +//! in-flight slot); +//! - at most `1 + floor(C / D)` grants at one instant (5 at the 25 ms +//! floor), and never more than the free in-flight slots of the domain and +//! the job; +//! - with `D >= strict_spacing_ms` (a robots `Crawl-delay`, a slow job +//! rate) `C` is 0: `last = now`, two requests are never less than `D` +//! apart, exactly as before; +//! - with `C < D < strict_spacing_ms` there is no burst (one grant per +//! instant), but a grant that came late lets the next one come up to `C` +//! early: gaps are at least `D − C` and average `D`; +//! - no credit builds up during a pause (`Retry-After`, error streak): the +//! schedule restarts at the pause's end. A 429 or an error drops the +//! credit banked so far. +//! - a domain never requested before starts with no credit. +//! //! [`PolitenessScheduler`] keeps this state in memory (one frontier //! instance); `RedisPoliteness` (feature `redis-store`) shares it between //! instances. Both implement [`PolitenessStore`]. @@ -60,6 +99,14 @@ pub struct PolitenessConfig { /// How long an in-flight slot is held without feedback before it is /// considered lost and freed (2 × the crawler request timeout). pub slot_ttl: Duration, + /// How far a domain's schedule may lag behind the clock (milliseconds): + /// time lost between a request becoming due and the dispatcher pass + /// that grants it is kept, up to this much, and granted back (see the + /// module docs). `0` spaces requests from the time of the last grant. + pub spacing_credit_ms: u64, + /// Effective delays at or above this (milliseconds) get no spacing + /// credit: their requests are never less than the delay apart. + pub strict_spacing_ms: u64, } impl Default for PolitenessConfig { @@ -72,6 +119,8 @@ impl Default for PolitenessConfig { robots_delay_multiplier: 1.0, concurrent_per_domain: 2, slot_ttl: Duration::from_secs(60), + spacing_credit_ms: 100, + strict_spacing_ms: 1_000, } } } @@ -161,6 +210,16 @@ pub fn effective_delay( .min(config.max_delay_ms.max(floor)) } +/// The spacing credit of a domain whose effective delay is `delay_ms` (see +/// the module docs): `spacing_credit_ms` below `strict_spacing_ms`, else 0. +pub fn spacing_credit(config: &PolitenessConfig, delay_ms: u64) -> u64 { + if delay_ms >= config.strict_spacing_ms { + 0 + } else { + config.spacing_credit_ms + } +} + /// The adaptive delay after a successful fetch. /// /// With the fetch time known, the delay moves halfway toward @@ -215,6 +274,42 @@ pub struct SlotRequest<'a> { pub limits: JobLimits, } +/// One dispatch pass asking for up to `tokens.len()` slots on one domain. +#[derive(Debug, Clone)] +pub struct SlotsRequest<'a> { + pub domain: &'a str, + pub job_id: &'a str, + /// One token per wanted slot, taken in order (see + /// [`SlotRequest::token`]). + pub tokens: &'a [String], + pub limits: JobLimits, +} + +/// Result of [`PolitenessStore::try_acquire_many`]. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Grants { + /// Slots taken, for the first `granted` tokens of the request. + pub granted: usize, + /// What one more request for the domain would be answered right now: + /// why the store stopped granting (`Wait`: until the domain's next + /// grant is due; `DomainBusy`; `JobBusy`), or `Granted` when every + /// token got a slot and another one is available. + pub next: Acquire, +} + +impl Grants { + /// As the answer to a one-token request: the outcome and, for a grant, + /// the wait before the domain's next one (see + /// [`PolitenessStore::try_acquire_spaced`]). + pub fn into_single(self) -> (Acquire, Duration) { + match (self.granted, self.next) { + (0, next) => (next, Duration::ZERO), + (_, Acquire::Wait(wait)) => (Acquire::Granted, wait), + _ => (Acquire::Granted, Duration::ZERO), + } + } +} + /// What a fetch told us about the domain. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum FetchSignal { @@ -256,16 +351,50 @@ pub trait PolitenessStore: Send + Sync { /// Atomically check the domain's delay/pause, the domain and job /// in-flight caps, and take a slot if all allow it. async fn try_acquire(&self, req: &SlotRequest<'_>) -> Result; - /// [`Self::try_acquire`], plus, for a grant, the delay the domain now - /// imposes on this job before its next request (the effective delay at - /// the grant). The dispatcher parks the domain's other URLs for that - /// long instead of asking again only to be told to wait. - /// `Duration::ZERO` means no such delay is known: the domain may be - /// granted again right away (a zero effective delay, or a store that - /// does not report it). Always zero when the outcome is not a grant. + /// [`Self::try_acquire`], plus, for a grant, how long the domain makes + /// this job wait before its next request (the effective delay, less any + /// spacing credit left). `Duration::ZERO` means no such wait is known: + /// the domain may be granted again right away (a zero delay, credit + /// left, or a store that does not report it) or is busy. Always zero + /// when the outcome is not a grant. async fn try_acquire_spaced(&self, req: &SlotRequest<'_>) -> Result<(Acquire, Duration)> { Ok((self.try_acquire(req).await?, Duration::ZERO)) } + /// Atomically take as many slots as the domain's schedule, the domain + /// cap and the job cap allow right now, at most one per token (see the + /// module docs, "Spacing credit"), and say what one more request would + /// get. The dispatcher sends one URL per granted token and parks the + /// domain's other URLs for the reported wait instead of asking again. + /// + /// The default asks for one slot at a time; a store should override it + /// with a single atomic call. + async fn try_acquire_many(&self, req: &SlotsRequest<'_>) -> Result { + let mut granted = 0; + for token in req.tokens { + let one = SlotRequest { + domain: req.domain, + job_id: req.job_id, + token, + limits: req.limits, + }; + match self.try_acquire_spaced(&one).await? { + (Acquire::Granted, spacing) => { + granted += 1; + if !spacing.is_zero() { + return Ok(Grants { + granted, + next: Acquire::Wait(spacing), + }); + } + } + (next, _) => return Ok(Grants { granted, next }), + } + } + Ok(Grants { + granted, + next: Acquire::Granted, + }) + } /// Free a slot without any accounting (the dispatch never reached the /// domain, e.g. the bus refused it). async fn release(&self, domain: &str, job_id: &str, token: &str) -> Result<()>; @@ -289,7 +418,9 @@ struct Slot { /// Per-domain scheduling state struct DomainState { - /// Last request time (`None`: never requested) + /// Virtual last request time (`None`: never requested): the time of the + /// last grant, or up to the spacing credit earlier when that grant came + /// late (see the module docs) last_request: Option, /// Error-adapted delay (starts at `default_delay_ms`) delay_ms: u64, @@ -328,8 +459,9 @@ impl DomainState { .count() } - fn expire(&mut self, ttl: Duration) { - self.in_flight.retain(|s| s.started.elapsed() < ttl); + fn expire(&mut self, ttl: Duration, now: Instant) { + self.in_flight + .retain(|s| now.saturating_duration_since(s.started) < ttl); } /// Remove the slot for `token` (or the oldest token-less/any slot when @@ -352,20 +484,29 @@ impl DomainState { } /// Remaining pause, if any. - fn pause_left(&self) -> Option { + fn pause_left(&self, now: Instant) -> Option { if self.paused { return Some(Duration::from_secs(60)); } let until = self.paused_until?; - let now = Instant::now(); (until > now).then(|| until - now) } - fn pause_for(&mut self, d: Duration) { - let until = Instant::now() + d; + fn pause_for(&mut self, d: Duration, now: Instant) { + let until = now + d; self.paused_until = Some(self.paused_until.map_or(until, |u| u.max(until))); } + /// Drop the spacing credit banked so far: the next grant is due no + /// earlier than `now` (a 429 or an error must not be followed by a + /// catch-up burst). + fn forfeit_credit(&mut self, now: Instant) { + let floor = now.checked_sub(Duration::from_millis(self.delay_ms)); + if let (Some(last), Some(floor)) = (self.last_request, floor) { + self.last_request = Some(last.max(floor)); + } + } + /// Time left before the delay since the last request elapsed. fn delay_left(&self, delay_ms: u64) -> Duration { let required = Duration::from_millis(delay_ms); @@ -420,7 +561,7 @@ impl PolitenessScheduler { let Some(state) = domains.get(domain) else { return Duration::ZERO; }; - if let Some(left) = state.pause_left() { + if let Some(left) = state.pause_left(Instant::now()) { return left; } if state.live_in_flight(self.config.slot_ttl) >= self.config.concurrent_per_domain { @@ -468,7 +609,7 @@ impl PolitenessScheduler { state.delay_ms = adapt_on_success(&self.config, state.delay_ms, fetch_ms); } - fn on_error(&self, domain: &str, state: &mut DomainState, is_rate_limited: bool) { + fn on_error(&self, domain: &str, state: &mut DomainState, is_rate_limited: bool, now: Instant) { state.consecutive_errors += 1; // Adaptive backoff @@ -486,9 +627,10 @@ impl PolitenessScheduler { // Pause the domain for a while if too many errors (timed, so a // flaky domain cannot wedge a job forever) if state.consecutive_errors >= 10 { - state.pause_for(Duration::from_millis(self.config.max_delay_ms)); + state.pause_for(Duration::from_millis(self.config.max_delay_ms), now); warn!(domain, "Domain paused due to excessive errors"); } + state.forfeit_credit(now); } /// Record that a request completed successfully (frees one slot) @@ -512,7 +654,7 @@ impl PolitenessScheduler { pub fn failed_request(&self, domain: &str, is_rate_limited: bool) { if let Some(state) = self.domains.write().get_mut(domain) { state.free(None); - self.on_error(domain, state, is_rate_limited); + self.on_error(domain, state, is_rate_limited, Instant::now()); } } @@ -521,8 +663,9 @@ impl PolitenessScheduler { pub fn failed_request_with_retry_after(&self, domain: &str, retry_after: Duration) { self.with_state(domain, |state| { state.free(None); - self.on_error(domain, state, true); - state.pause_for(retry_after); + let now = Instant::now(); + self.on_error(domain, state, true, now); + state.pause_for(retry_after, now); }); } @@ -570,7 +713,7 @@ impl PolitenessScheduler { domains.get(domain).map(|state| DomainStats { delay_ms: self.state_delay(state, &state.job_limits), in_flight: state.live_in_flight(self.config.slot_ttl), - paused: state.pause_left().is_some(), + paused: state.pause_left(Instant::now()).is_some(), consecutive_errors: state.consecutive_errors, time_since_last_request_ms: state .last_request @@ -611,21 +754,42 @@ impl PolitenessScheduler { } /// Synchronous core of [`PolitenessStore::try_acquire_spaced`]: the - /// outcome and, for a grant, the effective delay that was checked. + /// outcome and, for a grant, the wait before the domain's next one. pub fn acquire_spaced(&self, req: &SlotRequest<'_>) -> (Acquire, Duration) { + let tokens = [req.token.to_string()]; + self.acquire_many(&SlotsRequest { + domain: req.domain, + job_id: req.job_id, + tokens: &tokens, + limits: req.limits, + }) + .into_single() + } + + /// Synchronous core of [`PolitenessStore::try_acquire_many`]. + pub fn acquire_many(&self, req: &SlotsRequest<'_>) -> Grants { + self.acquire_many_at(req, Instant::now()) + } + + /// [`Self::acquire_many`] at a given time (tests drive the clock). + #[doc(hidden)] + pub fn acquire_many_at(&self, req: &SlotsRequest<'_>, now: Instant) -> Grants { let ttl = self.config.slot_ttl; let mut domains = self.domains.write(); let mut jobs = self.jobs.write(); let state = domains .entry(req.domain.to_string()) .or_insert_with(|| DomainState::new(self.config.default_delay_ms)); - state.expire(ttl); - if let Some(left) = state.pause_left() { - return (Acquire::Wait(left), Duration::ZERO); + state.expire(ttl, now); + if let Some(left) = state.pause_left(now) { + return Grants { + granted: 0, + next: Acquire::Wait(left), + }; } - let job_held = match jobs.get_mut(req.job_id) { + let mut job_held = match jobs.get_mut(req.job_id) { Some(slots) => { - slots.retain(|(_, t)| t.elapsed() < ttl); + slots.retain(|(_, t)| now.saturating_duration_since(*t) < ttl); slots.len() } None => 0, @@ -633,29 +797,47 @@ impl PolitenessScheduler { if job_held == 0 { jobs.remove(req.job_id); } - if let Some(cap) = req.limits.max_in_flight.filter(|c| *c > 0) { - if job_held >= cap as usize { - return (Acquire::JobBusy, Duration::ZERO); - } - } - if state.in_flight.len() >= self.config.concurrent_per_domain { - return (Acquire::DomainBusy, Duration::ZERO); - } + let job_cap = req.limits.max_in_flight.filter(|c| *c > 0); let delay_ms = self.state_delay(state, &req.limits); - let left = state.delay_left(delay_ms); - if !left.is_zero() { - return (Acquire::Wait(left), Duration::ZERO); - } - let now = Instant::now(); - state.in_flight.push(Slot { - token: Some(req.token.to_string()), - started: now, - }); - state.last_request = Some(now); - jobs.entry(req.job_id.to_string()) - .or_default() - .push((req.token.to_string(), now)); - (Acquire::Granted, Duration::from_millis(delay_ms)) + let delay = Duration::from_millis(delay_ms); + let credit = Duration::from_millis(spacing_credit(&self.config, delay_ms)); + // The schedule is at most `credit` behind the clock, and never + // behind the end of a pause. + let floor = match (now.checked_sub(credit), state.paused_until) { + (Some(a), Some(b)) => Some(a.max(b.min(now))), + (a, b) => a.or(b.map(|b| b.min(now))), + }; + let mut granted = 0; + let next = loop { + if job_cap.is_some_and(|cap| job_held >= cap as usize) { + break Acquire::JobBusy; + } + if state.in_flight.len() >= self.config.concurrent_per_domain { + break Acquire::DomainBusy; + } + let due = state.last_request.map(|last| last + delay); + if let Some(due) = due.filter(|due| *due > now) { + break Acquire::Wait(due - now); + } + let Some(token) = req.tokens.get(granted) else { + break Acquire::Granted; + }; + state.last_request = Some(match (due, floor) { + (Some(due), Some(floor)) => due.max(floor), + (Some(due), None) => due, + (None, _) => now, + }); + state.in_flight.push(Slot { + token: Some(token.clone()), + started: now, + }); + jobs.entry(req.job_id.to_string()) + .or_default() + .push((token.clone(), now)); + job_held += 1; + granted += 1; + }; + Grants { granted, next } } /// Free `token`'s slots. Returns whether this instance held the domain @@ -678,6 +860,13 @@ impl PolitenessScheduler { /// Synchronous core of [`PolitenessStore::report`]. pub fn apply_report(&self, r: &FetchReport<'_>) { + self.apply_report_at(r, Instant::now(), chrono::Utc::now().timestamp_millis()) + } + + /// [`Self::apply_report`] at a given time, `now_ms` being the same + /// time in ms since the epoch (tests drive the clock). + #[doc(hidden)] + pub fn apply_report_at(&self, r: &FetchReport<'_>, now: Instant, now_ms: i64) { let owned = self.free_token(r.domain, r.job_id, r.token); self.with_state(r.domain, |state| { if r.crawl_delay_ms.is_some() || r.robots_checked { @@ -691,15 +880,15 @@ impl PolitenessScheduler { if owned { match r.signal { FetchSignal::Success => self.on_success(state, r.fetch_ms), - FetchSignal::RateLimited => self.on_error(r.domain, state, true), - FetchSignal::Error => self.on_error(r.domain, state, false), + FetchSignal::RateLimited => self.on_error(r.domain, state, true, now), + FetchSignal::Error => self.on_error(r.domain, state, false, now), FetchSignal::NoRequest => {} } } if let Some(until) = r.retry_until_ms { - let left = until - chrono::Utc::now().timestamp_millis(); + let left = until - now_ms; if left > 0 { - state.pause_for(Duration::from_millis(left as u64)); + state.pause_for(Duration::from_millis(left as u64), now); } } }); @@ -716,6 +905,10 @@ impl PolitenessStore for PolitenessScheduler { Ok(self.acquire_spaced(req)) } + async fn try_acquire_many(&self, req: &SlotsRequest<'_>) -> Result { + Ok(self.acquire_many(req)) + } + async fn release(&self, domain: &str, job_id: &str, token: &str) -> Result<()> { self.free_token(domain, job_id, token); Ok(()) @@ -1297,4 +1490,446 @@ mod tests { assert_eq!(effective_delay(&c, 25, none, &rate), 200); assert_eq!(effective_delay(&c, 25, none, &JobLimits::default()), 25); } + + // ---- Spacing credit (SCR-90) ---- + + /// One domain with a fixed `delay_ms` and room for any number of + /// requests in flight. + fn spaced(delay_ms: u64, credit_ms: u64) -> PolitenessScheduler { + PolitenessScheduler::new(PolitenessConfig { + default_delay_ms: delay_ms, + min_delay_ms: delay_ms, + concurrent_per_domain: 100_000, + spacing_credit_ms: credit_ms, + ..Default::default() + }) + } + + /// A simulated dispatcher on one domain: tokens are numbered, time is + /// `base + ms`. + struct Sim { + p: PolitenessScheduler, + base: Instant, + limits: JobLimits, + next_token: usize, + /// Time (ms) of every grant, in order. + grants: Vec, + } + + impl Sim { + fn new(p: PolitenessScheduler, limits: JobLimits) -> Self { + Self { + p, + base: Instant::now(), + limits, + next_token: 0, + grants: Vec::new(), + } + } + + /// A pass at `at_ms` asking for `want` slots. + fn pass(&mut self, at_ms: u64, want: usize) -> Grants { + let tokens: Vec = (0..want) + .map(|i| format!("t{}", self.next_token + i)) + .collect(); + let got = self.p.acquire_many_at( + &SlotsRequest { + domain: "a.test", + job_id: "j", + tokens: &tokens, + limits: self.limits, + }, + self.base + Duration::from_millis(at_ms), + ); + self.next_token += got.granted; + self.grants.extend(std::iter::repeat_n(at_ms, got.granted)); + got + } + + /// A pass every `period_ms` over `[from_ms, to_ms)`, each asking + /// for 50 slots. + fn run(&mut self, from_ms: u64, to_ms: u64, period_ms: u64) { + let mut t = from_ms; + while t < to_ms { + self.pass(t, 50); + t += period_ms; + } + } + + fn report(&mut self, token: &str, signal: FetchSignal, at_ms: u64) { + self.report_with(report("a.test", "j", token, signal), at_ms); + } + + fn report_with(&mut self, r: FetchReport<'_>, at_ms: u64) { + // The wall clock of the simulation starts at 0. + self.p + .apply_report_at(&r, self.base + Duration::from_millis(at_ms), at_ms as i64); + } + } + + /// The politeness guarantee: any `n + 1` consecutive grants span at + /// least `n × delay − credit`, so the rate never exceeds `1 / delay` on + /// average and a burst is at most `1 + credit / delay` grants. + fn assert_conforms(grants: &[u64], delay_ms: u64, credit_ms: u64) { + for (i, a) in grants.iter().enumerate() { + for (n, b) in grants[i..].iter().enumerate() { + assert!( + b - a + credit_ms >= n as u64 * delay_ms, + "{} grants within {} ms (delay {delay_ms}, credit {credit_ms})", + n + 1, + b - a + ); + } + } + } + + fn most_at_once(grants: &[u64]) -> usize { + grants + .chunk_by(|a, b| a == b) + .map(|same| same.len()) + .max() + .unwrap_or(0) + } + + /// The rate a domain gets is the one its delay allows, whatever the + /// dispatcher's pass period, and never more. + #[test] + fn rate_follows_the_delay_whatever_the_pass_period() { + for delay in [10u64, 25, 40, 100] { + for period in [20u64, 25, 33] { + let mut sim = Sim::new(spaced(delay, 100), JobLimits::default()); + sim.run(0, 1_000, period); + // Exactly the requests due by the last pass of the second + // (one at 0, then one per delay): no pass loses time. That + // is `1000 / delay`, less what falls due after that pass. + let last_pass = (999 / period) * period; + assert_eq!( + sim.grants.len() as u64, + 1 + last_pass / delay, + "delay {delay} ms, pass every {period} ms" + ); + let short = 1_000 / delay - (sim.grants.len() as u64).min(1_000 / delay); + assert!( + short <= (1_000 - last_pass) / delay, + "delay {delay} ms, pass every {period} ms: {short} grants short" + ); + assert_conforms(&sim.grants, delay, 100); + // Passes are at most 33 ms apart: a pass grants what became + // due since the previous one, not the whole credit. + assert!( + most_at_once(&sim.grants) as u64 <= period.div_ceil(delay).max(1), + "delay {delay} ms, pass every {period} ms: {} at once", + most_at_once(&sim.grants) + ); + } + } + } + + /// Without credit (`spacing_credit_ms = 0`, the rule before SCR-90) the + /// rate is quantized by the pass period: a 25 ms delay with a pass + /// every 33 ms gets one grant per pass, a 40 ms delay one every other. + #[test] + fn without_credit_the_rate_is_quantized_by_the_pass_period() { + let mut sim = Sim::new(spaced(25, 0), JobLimits::default()); + sim.run(0, 1_000, 33); + assert_eq!(sim.grants.len(), 31); + assert_eq!(most_at_once(&sim.grants), 1); + let mut sim = Sim::new(spaced(40, 0), JobLimits::default()); + sim.run(0, 1_000, 33); + assert_eq!(sim.grants.len(), 16); + assert_conforms(&sim.grants, 40, 0); + } + + /// The burst after an idle period is `1 + credit / delay` grants, and a + /// domain never requested starts with one. + #[test] + fn burst_is_bounded_by_the_credit() { + let mut sim = Sim::new(spaced(25, 100), JobLimits::default()); + let first = sim.pass(0, 50); + assert_eq!( + first, + Grants { + granted: 1, + next: Acquire::Wait(Duration::from_millis(25)) + } + ); + // Ten seconds idle bank 100 ms, not ten seconds: 1 + 100 / 25. + let late = sim.pass(10_000, 50); + assert_eq!( + late, + Grants { + granted: 5, + next: Acquire::Wait(Duration::from_millis(25)) + } + ); + // Then the delay again. + assert_eq!(sim.pass(10_010, 50).granted, 0); + assert_eq!(sim.pass(10_025, 50).granted, 1); + assert_conforms(&sim.grants, 25, 100); + + // A pass 60 ms late is owed two grants, not the whole credit. + let mut sim = Sim::new(spaced(25, 100), JobLimits::default()); + sim.pass(0, 50); + assert_eq!( + sim.pass(60, 50), + Grants { + granted: 2, + next: Acquire::Wait(Duration::from_millis(15)) + } + ); + // Asking for fewer than are owed leaves the rest available. + let mut sim = Sim::new(spaced(25, 100), JobLimits::default()); + sim.pass(0, 1); + assert_eq!( + sim.pass(1_000, 2), + Grants { + granted: 2, + next: Acquire::Granted + } + ); + assert_eq!(sim.pass(1_000, 50).granted, 3); + } + + /// A delay of a second or more (a robots `Crawl-delay`, a slow job + /// rate) gets no credit at all: requests are never closer than it. + #[test] + fn long_delays_are_spaced_strictly() { + let robots = spaced(25, 100); + robots.set_delay("a.test", 2_000); + let slow_job = JobLimits { + min_delay_ms: 2_000, + ..JobLimits::default() + }; + let at_threshold = JobLimits { + min_delay_ms: 1_000, + ..JobLimits::default() + }; + for (p, limits, delay) in [ + (robots, JobLimits::default(), 2_000), + (spaced(25, 100), slow_job, 2_000), + (spaced(25, 100), at_threshold, 1_000), + ] { + let mut sim = Sim::new(p, limits); + sim.run(0, 9_000, 33); + // Idle for a while, then passes again: still no catching up. + sim.run(30_000, 39_000, 33); + assert_conforms(&sim.grants, delay, 0); + assert_eq!(most_at_once(&sim.grants), 1); + for w in sim.grants.windows(2) { + assert!(w[1] - w[0] >= delay, "{} ms apart", w[1] - w[0]); + } + // Each grant waits for the first pass after the delay. + assert_eq!(sim.grants.len() as u64, 2 * (9_000 / (delay + 33) + 1)); + } + } + + /// Between the credit and the strict threshold there is no burst: one + /// grant per instant, gaps of at least `delay − credit`, `delay` on + /// average. + #[test] + fn medium_delays_never_burst() { + let limits = JobLimits { + min_delay_ms: 500, + ..JobLimits::default() + }; + let mut sim = Sim::new(spaced(25, 100), limits); + sim.run(0, 10_000, 33); + sim.run(60_000, 70_000, 33); + assert_eq!(most_at_once(&sim.grants), 1); + for w in sim.grants.windows(2) { + assert!(w[1] - w[0] >= 400, "{} ms apart", w[1] - w[0]); + } + assert_conforms(&sim.grants, 500, 100); + // 20 s of passes at 2 per second, give or take one per run. + assert!( + (39..=41).contains(&sim.grants.len()), + "{}", + sim.grants.len() + ); + } + + /// A pause (`Retry-After`) is waited out, banks no credit, and the + /// schedule restarts at its end. + #[test] + fn pause_is_respected_and_banks_no_credit() { + let mut sim = Sim::new(spaced(25, 100), JobLimits::default()); + assert_eq!(sim.pass(0, 50).granted, 1); + sim.report_with( + FetchReport { + retry_until_ms: Some(1_000), + ..report("a.test", "j", "t0", FetchSignal::NoRequest) + }, + 0, + ); + assert_eq!( + sim.pass(400, 50), + Grants { + granted: 0, + next: Acquire::Wait(Duration::from_millis(600)) + } + ); + assert_eq!(sim.pass(999, 50).granted, 0); + // 60 ms after the pause: the requests due at 1000, 1025 and 1050, + // not the five a full credit would give. + assert_eq!( + sim.pass(1_060, 50), + Grants { + granted: 3, + next: Acquire::Wait(Duration::from_millis(15)) + } + ); + // Right at the end of a pause: one. + let mut sim = Sim::new(spaced(25, 100), JobLimits::default()); + sim.pass(0, 50); + sim.report_with( + FetchReport { + retry_until_ms: Some(1_000), + ..report("a.test", "j", "t0", FetchSignal::NoRequest) + }, + 0, + ); + assert_eq!(sim.pass(1_000, 50).granted, 1); + } + + /// A 429 raises the delay (×1.5) and drops the banked credit; the rate + /// then follows the raised delay. + #[test] + fn rate_limit_drops_the_credit_and_the_backoff_holds() { + let mut sim = Sim::new(spaced(25, 100), JobLimits::default()); + assert_eq!(sim.pass(0, 50).granted, 1); + // Idle until 1000 (full credit), then the 429 comes in. + sim.report("t0", FetchSignal::RateLimited, 1_000); + assert_eq!(sim.p.effective_delay_ms("a.test"), 37); + assert_eq!( + sim.pass(1_000, 50), + Grants { + granted: 1, + next: Acquire::Wait(Duration::from_millis(37)) + } + ); + sim.run(1_033, 2_000, 33); + assert_conforms(&sim.grants[1..], 37, 100); + // 1 s at 37 ms: 27 grants, give or take one. + let after = sim.grants.len() as i64 - 1; + assert!((26..=28).contains(&after), "{after} grants"); + + // An error streak pauses the domain: nothing is granted, whatever + // the credit. + let mut sim = Sim::new(spaced(25, 100), JobLimits::default()); + assert_eq!(sim.pass(0, 10).granted, 1); + sim.run(25, 300, 25); + for i in 0..10 { + sim.report(&format!("t{i}"), FetchSignal::Error, 300); + } + let granted = sim.grants.len(); + sim.run(300, 5_000, 33); + assert_eq!(sim.grants.len(), granted, "paused for max_delay_ms"); + } + + /// Credit never grants more than the free in-flight slots of the + /// domain or of the job. + #[test] + fn credit_never_exceeds_the_in_flight_caps() { + let capped = |concurrent: usize| { + PolitenessScheduler::new(PolitenessConfig { + default_delay_ms: 10, + min_delay_ms: 10, + concurrent_per_domain: concurrent, + ..Default::default() + }) + }; + // Domain cap 3: the 11 grants the credit allows become 2 (one slot + // is held by the first request). + let mut sim = Sim::new(capped(3), JobLimits::default()); + assert_eq!(sim.pass(0, 50).granted, 1); + assert_eq!( + sim.pass(5_000, 50), + Grants { + granted: 2, + next: Acquire::DomainBusy + } + ); + assert_eq!(sim.pass(5_033, 50).granted, 0); + // A freed slot is granted at once while credit is left. + sim.report("t1", FetchSignal::Success, 5_040); + assert_eq!( + sim.pass(5_040, 50), + Grants { + granted: 1, + next: Acquire::DomainBusy + } + ); + + // Job cap 2, checked before the domain's. + let limits = JobLimits { + max_in_flight: Some(2), + ..JobLimits::default() + }; + let mut sim = Sim::new(capped(8), limits); + assert_eq!(sim.pass(0, 50).granted, 1); + assert_eq!( + sim.pass(5_000, 50), + Grants { + granted: 1, + next: Acquire::JobBusy + } + ); + assert_eq!(sim.p.job_in_flight("j"), 2); + assert_eq!( + sim.pass(5_100, 50), + Grants { + granted: 0, + next: Acquire::JobBusy + } + ); + } + + /// A store that only implements the single-slot calls still serves + /// `try_acquire_many` (one slot at a time). + #[tokio::test] + async fn default_try_acquire_many_asks_one_slot_at_a_time() { + struct Single(PolitenessScheduler); + #[async_trait] + impl PolitenessStore for Single { + async fn try_acquire(&self, req: &SlotRequest<'_>) -> Result { + Ok(self.0.acquire(req)) + } + async fn release(&self, d: &str, j: &str, t: &str) -> Result<()> { + self.0.release(d, j, t).await + } + async fn report(&self, r: &FetchReport<'_>) -> Result<()> { + self.0.report(r).await + } + fn is_shared(&self) -> bool { + false + } + } + let tokens: Vec = (0..5).map(|i| format!("t{i}")).collect(); + let many = |domain| SlotsRequest { + domain, + job_id: "j", + tokens: &tokens, + limits: JobLimits::default(), + }; + let p = Single(fast(3)); + assert_eq!( + p.try_acquire_many(&many("a.test")).await.unwrap(), + Grants { + granted: 3, + next: Acquire::DomainBusy + } + ); + let p = Single(fast(8)); + assert_eq!( + p.try_acquire_many(&many("a.test")).await.unwrap(), + Grants { + granted: 5, + next: Acquire::Granted + } + ); + let p = Single(spaced(250, 100)); + let got = p.try_acquire_many(&many("a.test")).await.unwrap(); + assert_eq!(got.granted, 1); + assert!(matches!(got.next, Acquire::Wait(_)), "{got:?}"); + } } diff --git a/crates/scrapix-frontier/src/politeness_redis.rs b/crates/scrapix-frontier/src/politeness_redis.rs index 2c671477..1b1261c1 100644 --- a/crates/scrapix-frontier/src/politeness_redis.rs +++ b/crates/scrapix-frontier/src/politeness_redis.rs @@ -10,7 +10,7 @@ //! //! | Key | Type | Contents | //! |-----|------|----------| -//! | `{p}:pol:d:{domain}` | hash | `last_ms`, `delay_ms` (adaptive), `robots_ms` (Crawl-delay, absent when none), `robots_checked` (`1` once robots.txt was fetched), `errors`, `pause_until` | +//! | `{p}:pol:d:{domain}` | hash | `last_ms` (virtual last request: the last grant's time, or up to the spacing credit earlier), `delay_ms` (adaptive), `robots_ms` (Crawl-delay, absent when none), `robots_checked` (`1` once robots.txt was fetched), `errors`, `pause_until` | //! | `{p}:pol:f:{domain}` | zset | in-flight slots: member = token, score = expiry ms | //! | `{p}:pol:j:{job_id}` | zset | the job's in-flight slots, same encoding | //! @@ -18,6 +18,17 @@ //! every check. Domain hashes expire a day after their last use; the zsets //! after `2 × slot_ttl`. Only commands supported by Redis 7 and DragonflyDB //! are used, and every key is declared in `KEYS`. +//! +//! ## Mixed versions +//! +//! `last_ms` is the only spacing state, as before SCR-90, so instances +//! running the previous script can share the keys with this one. An old +//! instance grants when `now - last_ms >= delay` and writes `last_ms = now`: +//! under this script's rule that is a grant that banks no credit. It reads +//! a `last_ms` written here as the last request time, so it may grant up to +//! the credit (100 ms) earlier than a full delay after the real last +//! request, which is what this script would do too. Either way a domain's +//! long-run rate stays at most `1 / delay`. use std::collections::HashMap; use std::time::{Duration, Instant}; @@ -29,7 +40,8 @@ use redis::Script; use scrapix_core::{Result, ScrapixError}; use crate::politeness::{ - Acquire, FetchReport, FetchSignal, PolitenessConfig, PolitenessStore, SlotRequest, + Acquire, FetchReport, FetchSignal, Grants, PolitenessConfig, PolitenessStore, SlotRequest, + SlotsRequest, }; /// Idle domain state is forgotten after a day. @@ -41,26 +53,26 @@ const RECENT_DOMAIN_WINDOW: Duration = Duration::from_secs(600); /// `ARGV`: now, concurrent_per_domain, job cap (0 = none), default delay, /// floor, max delay, job min delay, rps delay, respect robots (0/1), default /// crawl delay (`0` unless the job set no explicit delay/rate), robots -/// multiplier, token, slot ttl ms, state ttl ms. -/// Returns `{status, ms}`: `{0, effective delay}` (granted; the delay is the -/// one this grant was checked against, i.e. the earliest the domain can be -/// granted to this job again), `{1, wait}` (delay or pause not elapsed), -/// `{-1, 0}` (domain busy) or `{-2, 0}` (job busy). +/// multiplier, spacing credit ms, slot ttl ms, state ttl ms, strict spacing +/// ms, then one token per wanted slot. +/// Returns `{status, ms, granted}`: `granted` slots were taken (for the +/// first `granted` tokens), and one more request would get `status`: `0` +/// (a slot, only when every token was granted), `1` (wait `ms`: the delay +/// or a pause has not elapsed), `-1` (domain busy) or `-2` (job busy). /// -/// A second grant for one domain is only possible once `now - last_ms >= -/// effective delay`, whatever `concurrent_per_domain` is (it only caps the -/// in-flight count), so within one instant a domain is granted twice only -/// when the effective delay is 0. +/// The spacing rule is the one of [`crate::politeness`] ("Spacing credit"): +/// a grant needs `now - last_ms >= effective delay` and sets `last_ms = +/// max(last_ms + delay, now - credit, pause_until)` (`now` for a domain +/// never requested), with no credit for a delay at or above the strict +/// spacing threshold. So one call grants at most `1 + floor(credit / +/// delay)` slots, and never more than the domain and the job have free. const ACQUIRE_LUA: &str = r#" local now = tonumber(ARGV[1]) redis.call('ZREMRANGEBYSCORE', KEYS[2], '-inf', now) redis.call('ZREMRANGEBYSCORE', KEYS[3], '-inf', now) local h = redis.call('HMGET', KEYS[1], 'last_ms', 'delay_ms', 'robots_ms', 'pause_until', 'robots_checked') local pause = tonumber(h[4]) or 0 -if pause > now then return {1, pause - now} end -local jobcap = tonumber(ARGV[3]) -if jobcap > 0 and redis.call('ZCARD', KEYS[3]) >= jobcap then return {-2, 0} end -if redis.call('ZCARD', KEYS[2]) >= tonumber(ARGV[2]) then return {-1, 0} end +if pause > now then return {1, pause - now, 0} end local eff = tonumber(h[2]) or tonumber(ARGV[4]) eff = math.max(eff, tonumber(ARGV[7]), tonumber(ARGV[8])) if ARGV[9] == '1' then @@ -70,16 +82,42 @@ if ARGV[9] == '1' then end local floor = tonumber(ARGV[5]) eff = math.min(math.max(eff, floor), math.max(tonumber(ARGV[6]), floor)) -local last = tonumber(h[1]) -if last and now - last < eff then return {1, eff - (now - last)} end +local credit = tonumber(ARGV[12]) +if eff >= tonumber(ARGV[15]) then credit = 0 end +local domcap = tonumber(ARGV[2]) +local jobcap = tonumber(ARGV[3]) +local dom_n = redis.call('ZCARD', KEYS[2]) +local job_n = redis.call('ZCARD', KEYS[3]) local ttl = tonumber(ARGV[13]) -redis.call('ZADD', KEYS[2], now + ttl, ARGV[12]) -redis.call('ZADD', KEYS[3], now + ttl, ARGV[12]) -redis.call('HSET', KEYS[1], 'last_ms', now) -redis.call('PEXPIRE', KEYS[1], ARGV[14]) -redis.call('PEXPIRE', KEYS[2], 2 * ttl) -redis.call('PEXPIRE', KEYS[3], 2 * ttl) -return {0, eff} +local want = #ARGV - 15 +local last = tonumber(h[1]) +local granted = 0 +local status = 0 +local ms = 0 +while true do + if jobcap > 0 and job_n >= jobcap then status = -2 break end + if dom_n >= domcap then status = -1 break end + if last and now - last < eff then + status = 1 + ms = eff - (now - last) + break + end + if granted >= want then break end + if last then last = math.max(last + eff, now - credit, pause) else last = now end + granted = granted + 1 + local token = ARGV[15 + granted] + redis.call('ZADD', KEYS[2], now + ttl, token) + redis.call('ZADD', KEYS[3], now + ttl, token) + dom_n = dom_n + 1 + job_n = job_n + 1 +end +if granted > 0 then + redis.call('HSET', KEYS[1], 'last_ms', last) + redis.call('PEXPIRE', KEYS[1], ARGV[14]) + redis.call('PEXPIRE', KEYS[2], 2 * ttl) + redis.call('PEXPIRE', KEYS[3], 2 * ttl) +end +return {status, ms, granted} "#; /// `KEYS`: domain hash, domain zset, job zset. @@ -90,6 +128,8 @@ return {0, eff} /// On success the delay follows [`crate::politeness::adapt_on_success`]. /// Error/success accounting only applies when the slot was still held /// (duplicate or expired feedback only updates robots and Retry-After). +/// A rate limit or an error also drops the spacing credit banked so far +/// (`last_ms` is raised to `now - delay`). const REPORT_LUA: &str = r#" local owned = redis.call('ZREM', KEYS[2], ARGV[1]) redis.call('ZREM', KEYS[3], ARGV[1]) @@ -116,6 +156,8 @@ elseif sig == 'rate' or sig == 'error' then errors = errors + 1 if sig == 'rate' or errors >= 3 then delay = math.min(math.floor(delay * 1.5), maxd) end if errors >= 10 then pause = math.max(pause, now + maxd) end + local last = tonumber(redis.call('HGET', KEYS[1], 'last_ms')) + if last and last < now - delay then redis.call('HSET', KEYS[1], 'last_ms', now - delay) end end if ARGV[4] ~= '' then pause = math.max(pause, tonumber(ARGV[4])) end redis.call('HSET', KEYS[1], 'delay_ms', delay, 'errors', errors, 'pause_until', pause) @@ -175,6 +217,85 @@ impl RedisPoliteness { format!("{}:pol:j:{}", self.prefix, job_id) } + /// [`PolitenessStore::try_acquire_many`] at a given time, in ms since + /// the epoch (tests drive the clock). + #[doc(hidden)] + pub async fn acquire_many_at(&self, req: &SlotsRequest<'_>, now_ms: i64) -> Result { + let c = &self.config; + let l = &req.limits; + let respect = c.respect_robots_delay && l.respect_robots; + let default_crawl = if l.uses_default_crawl_delay() { + l.default_crawl_delay_ms + } else { + 0 + }; + { + let mut recent = self.recent_domains.lock(); + if recent.len() >= 100_000 { + recent.retain(|_, t| t.elapsed() < RECENT_DOMAIN_WINDOW); + } + recent.insert(req.domain.to_string(), Instant::now()); + } + let mut conn = self.conn.clone(); + let mut call = self.acquire.prepare_invoke(); + call.key(self.domain_key(req.domain)) + .key(self.flight_key(req.domain)) + .key(self.job_key(req.job_id)) + .arg(now_ms) + .arg(c.concurrent_per_domain) + .arg(l.max_in_flight.unwrap_or(0)) + .arg(c.default_delay_ms) + .arg(c.min_delay_ms.min(c.default_delay_ms)) + .arg(c.max_delay_ms) + .arg(l.min_delay_ms) + .arg(l.rps_delay_ms()) + .arg(if respect { "1" } else { "0" }) + .arg(default_crawl) + .arg(c.robots_delay_multiplier) + .arg(c.spacing_credit_ms) + .arg(c.slot_ttl.as_millis() as u64) + .arg(DOMAIN_STATE_TTL_MS) + .arg(c.strict_spacing_ms); + for token in req.tokens { + call.arg(token.as_str()); + } + let (status, ms, granted): (i64, i64, i64) = + call.invoke_async(&mut conn).await.map_err(storage_err)?; + Ok(Grants { + granted: granted.max(0) as usize, + next: match status { + 0 => Acquire::Granted, + -1 => Acquire::DomainBusy, + -2 => Acquire::JobBusy, + _ => Acquire::Wait(Duration::from_millis(ms.max(1) as u64)), + }, + }) + } + + /// [`PolitenessStore::report`] at a given time, in ms since the epoch + /// (tests drive the clock). + #[doc(hidden)] + pub async fn report_at(&self, r: &FetchReport<'_>, now_ms: i64) -> Result<()> { + let signal = match r.signal { + FetchSignal::Success => "ok", + FetchSignal::RateLimited => "rate", + FetchSignal::Error => "error", + FetchSignal::NoRequest => "none", + }; + self.run_report( + r.domain, + r.job_id, + r.token, + signal, + r.crawl_delay_ms, + r.robots_checked, + r.retry_until_ms, + r.fetch_ms, + now_ms, + ) + .await + } + #[allow(clippy::too_many_arguments)] async fn run_report( &self, @@ -186,6 +307,7 @@ impl RedisPoliteness { robots_checked: bool, retry_until_ms: Option, fetch_ms: Option, + now_ms: i64, ) -> Result<()> { let c = &self.config; let mut conn = self.conn.clone(); @@ -198,7 +320,7 @@ impl RedisPoliteness { .arg(signal) .arg(crawl_delay_ms.map(|v| v.to_string()).unwrap_or_default()) .arg(retry_until_ms.map(|v| v.to_string()).unwrap_or_default()) - .arg(now_ms()) + .arg(now_ms) .arg(c.default_delay_ms) .arg(c.max_delay_ms) .arg(DOMAIN_STATE_TTL_MS) @@ -220,80 +342,39 @@ impl PolitenessStore for RedisPoliteness { } async fn try_acquire_spaced(&self, req: &SlotRequest<'_>) -> Result<(Acquire, Duration)> { - let c = &self.config; - let l = &req.limits; - let respect = c.respect_robots_delay && l.respect_robots; - let default_crawl = if l.uses_default_crawl_delay() { - l.default_crawl_delay_ms - } else { - 0 + let tokens = [req.token.to_string()]; + let many = SlotsRequest { + domain: req.domain, + job_id: req.job_id, + tokens: &tokens, + limits: req.limits, }; - { - let mut recent = self.recent_domains.lock(); - if recent.len() >= 100_000 { - recent.retain(|_, t| t.elapsed() < RECENT_DOMAIN_WINDOW); - } - recent.insert(req.domain.to_string(), Instant::now()); - } - let mut conn = self.conn.clone(); - let (status, ms): (i64, i64) = self - .acquire - .key(self.domain_key(req.domain)) - .key(self.flight_key(req.domain)) - .key(self.job_key(req.job_id)) - .arg(now_ms()) - .arg(c.concurrent_per_domain) - .arg(l.max_in_flight.unwrap_or(0)) - .arg(c.default_delay_ms) - .arg(c.min_delay_ms.min(c.default_delay_ms)) - .arg(c.max_delay_ms) - .arg(l.min_delay_ms) - .arg(l.rps_delay_ms()) - .arg(if respect { "1" } else { "0" }) - .arg(default_crawl) - .arg(c.robots_delay_multiplier) - .arg(req.token) - .arg(c.slot_ttl.as_millis() as u64) - .arg(DOMAIN_STATE_TTL_MS) - .invoke_async(&mut conn) - .await - .map_err(storage_err)?; - Ok(match status { - 0 => (Acquire::Granted, Duration::from_millis(ms.max(0) as u64)), - -1 => (Acquire::DomainBusy, Duration::ZERO), - -2 => (Acquire::JobBusy, Duration::ZERO), - _ => ( - Acquire::Wait(Duration::from_millis(ms.max(1) as u64)), - Duration::ZERO, - ), - }) + Ok(self.try_acquire_many(&many).await?.into_single()) } - async fn release(&self, domain: &str, job_id: &str, token: &str) -> Result<()> { - self.run_report(domain, job_id, token, "none", None, false, None, None) - .await + async fn try_acquire_many(&self, req: &SlotsRequest<'_>) -> Result { + self.acquire_many_at(req, now_ms()).await } - async fn report(&self, r: &FetchReport<'_>) -> Result<()> { - let signal = match r.signal { - FetchSignal::Success => "ok", - FetchSignal::RateLimited => "rate", - FetchSignal::Error => "error", - FetchSignal::NoRequest => "none", - }; + async fn release(&self, domain: &str, job_id: &str, token: &str) -> Result<()> { self.run_report( - r.domain, - r.job_id, - r.token, - signal, - r.crawl_delay_ms, - r.robots_checked, - r.retry_until_ms, - r.fetch_ms, + domain, + job_id, + token, + "none", + None, + false, + None, + None, + now_ms(), ) .await } + async fn report(&self, r: &FetchReport<'_>) -> Result<()> { + self.report_at(r, now_ms()).await + } + fn is_shared(&self) -> bool { true } diff --git a/crates/scrapix-frontier/tests/politeness_redis.rs b/crates/scrapix-frontier/tests/politeness_redis.rs index 5cf4bee2..282df795 100644 --- a/crates/scrapix-frontier/tests/politeness_redis.rs +++ b/crates/scrapix-frontier/tests/politeness_redis.rs @@ -440,3 +440,445 @@ async fn default_crawl_delay_needs_checked_robots_and_accounting_needs_the_slot( assert!(a.tracked_domain_count() >= 4); cleanup(&url, &prefix).await; } + +// ---- Spacing credit (SCR-90): Redis and memory apply the same rule ---- + +use scrapix_frontier::{Grants, PolitenessScheduler, SlotsRequest}; + +/// The same requests sent to a `RedisPoliteness` and to a +/// `PolitenessScheduler`, on one simulated clock (`at_ms` after a common +/// start): every answer must be identical. +struct Parity { + redis: RedisPoliteness, + memory: PolitenessScheduler, + start: std::time::Instant, + start_ms: i64, + next_token: usize, + /// Time (ms) of every grant, in order. + grants: Vec, +} + +impl Parity { + async fn new(url: &str, prefix: &str, c: PolitenessConfig) -> Self { + Self { + redis: RedisPoliteness::new(url, prefix, c.clone()).await.unwrap(), + memory: PolitenessScheduler::new(c), + start: std::time::Instant::now(), + start_ms: chrono::Utc::now().timestamp_millis(), + next_token: 0, + grants: Vec::new(), + } + } + + /// Ask both stores for `want` slots on `domain` at `at_ms`. + async fn pass(&mut self, domain: &str, limits: JobLimits, at_ms: u64, want: usize) -> Grants { + let tokens: Vec = (0..want) + .map(|i| format!("t{}", self.next_token + i)) + .collect(); + let req = SlotsRequest { + domain, + job_id: "j", + tokens: &tokens, + limits, + }; + let memory = self + .memory + .acquire_many_at(&req, self.start + Duration::from_millis(at_ms)); + let redis = self + .redis + .acquire_many_at(&req, self.start_ms + at_ms as i64) + .await + .unwrap(); + assert_eq!(redis, memory, "{domain} at {at_ms} ms, {want} wanted"); + self.next_token += redis.granted; + self.grants + .extend(std::iter::repeat_n(at_ms, redis.granted)); + redis + } + + async fn report(&mut self, r: FetchReport<'_>, at_ms: u64) { + let now_ms = self.start_ms + at_ms as i64; + self.memory + .apply_report_at(&r, self.start + Duration::from_millis(at_ms), now_ms); + self.redis.report_at(&r, now_ms).await.unwrap(); + } +} + +fn spaced(delay_ms: u64, concurrent: usize) -> PolitenessConfig { + PolitenessConfig { + default_delay_ms: delay_ms, + min_delay_ms: delay_ms, + concurrent_per_domain: concurrent, + ..Default::default() + } +} + +/// Over a simulated second, Redis grants exactly what the memory scheduler +/// grants, pass by pass, and that is what the delay allows whatever the +/// pass period. +#[tokio::test] +async fn spacing_credit_matches_the_memory_store_pass_by_pass() { + let Some(url) = redis_url() else { return }; + let l = JobLimits::default(); + for delay in [10u64, 25, 40, 100] { + for period in [20u64, 25, 33] { + let prefix = format!("test-pol-{}", uuid::Uuid::new_v4()); + let mut p = Parity::new(&url, &prefix, spaced(delay, 100_000)).await; + let mut t = 0; + while t < 1_000 { + p.pass("a.test", l, t, 50).await; + t += period; + } + let last_pass = (999 / period) * period; + assert_eq!( + p.grants.len() as u64, + 1 + last_pass / delay, + "delay {delay} ms, pass every {period} ms" + ); + // n + 1 consecutive grants span at least n × delay − credit. + for (i, a) in p.grants.iter().enumerate() { + for (n, b) in p.grants[i..].iter().enumerate() { + assert!(b - a + 100 >= n as u64 * delay, "delay {delay}: burst"); + } + } + cleanup(&url, &prefix).await; + } + } +} + +/// Burst bound, strict long delays, pauses, backoff and the in-flight caps: +/// the same answers from both stores. +#[tokio::test] +async fn spacing_credit_bounds_match_the_memory_store() { + let Some(url) = redis_url() else { return }; + let prefix = format!("test-pol-{}", uuid::Uuid::new_v4()); + let l = JobLimits::default(); + let wait = |ms| Acquire::Wait(Duration::from_millis(ms)); + + // Burst after an idle period: 1 + credit / delay, then the delay. + let mut p = Parity::new(&url, &prefix, spaced(25, 100_000)).await; + assert_eq!(p.pass("burst.test", l, 0, 50).await.granted, 1); + assert_eq!( + p.pass("burst.test", l, 10_000, 50).await, + Grants { + granted: 5, + next: wait(25) + } + ); + assert_eq!(p.pass("burst.test", l, 10_010, 50).await.granted, 0); + assert_eq!(p.pass("burst.test", l, 10_025, 50).await.granted, 1); + // A pass 60 ms late is owed two; asking for fewer leaves the rest. + assert_eq!(p.pass("late.test", l, 0, 50).await.granted, 1); + assert_eq!( + p.pass("late.test", l, 60, 1).await, + Grants { + granted: 1, + next: Acquire::Granted + } + ); + assert_eq!( + p.pass("late.test", l, 60, 50).await, + Grants { + granted: 1, + next: wait(15) + } + ); + + // A 2 s delay (job rate, or robots Crawl-delay) is strict: one grant + // per delay however late the pass, never two at once. + let slow = JobLimits { + max_rps: Some(0.5), + ..l + }; + let mut at = Vec::new(); + for t in (0..9_000).step_by(330).chain((30_000..39_000).step_by(330)) { + let got = p.pass("slow.test", slow, t, 50).await; + assert!(got.granted <= 1, "{got:?}"); + if got.granted == 1 { + at.push(t); + } + } + assert_eq!(at.len(), 8); + for w in at.windows(2) { + assert!(w[1] - w[0] >= 2_000, "{} ms apart", w[1] - w[0]); + } + assert_eq!(p.pass("robots.test", l, 0, 50).await.granted, 1); + let robots_at = p.grants.len() - 1; + p.report( + FetchReport { + crawl_delay_ms: Some(2_000), + ..report( + "robots.test", + "j", + &format!("t{robots_at}"), + FetchSignal::Success, + ) + }, + 100, + ) + .await; + assert_eq!( + p.pass("robots.test", l, 1_000, 50).await, + Grants { + granted: 0, + next: wait(1_000) + } + ); + assert_eq!( + p.pass("robots.test", l, 60_000, 50).await, + Grants { + granted: 1, + next: wait(2_000) + } + ); + + // A Retry-After pause banks no credit: 60 ms after its end, the three + // requests due since then. + assert_eq!(p.pass("pause.test", l, 0, 50).await.granted, 1); + let token = format!("t{}", p.grants.len() - 1); + let until = p.start_ms + 1_000; + p.report( + FetchReport { + retry_until_ms: Some(until), + ..report("pause.test", "j", &token, FetchSignal::NoRequest) + }, + 0, + ) + .await; + assert_eq!( + p.pass("pause.test", l, 400, 50).await, + Grants { + granted: 0, + next: wait(600) + } + ); + assert_eq!( + p.pass("pause.test", l, 1_060, 50).await, + Grants { + granted: 3, + next: wait(15) + } + ); + + // A 429 raises the delay and drops the banked credit. + assert_eq!(p.pass("rate.test", l, 0, 50).await.granted, 1); + let token = format!("t{}", p.grants.len() - 1); + p.report( + report("rate.test", "j", &token, FetchSignal::RateLimited), + 1_000, + ) + .await; + assert_eq!( + p.pass("rate.test", l, 1_000, 50).await, + Grants { + granted: 1, + next: wait(37) + } + ); + assert_eq!(p.pass("rate.test", l, 1_036, 50).await.granted, 0); + assert_eq!(p.pass("rate.test", l, 1_080, 50).await.granted, 2); + cleanup(&url, &prefix).await; + + // The domain's free slots cap the burst (3 slots, 1 held)... + let prefix = format!("test-pol-{}", uuid::Uuid::new_v4()); + let mut p = Parity::new(&url, &prefix, spaced(10, 3)).await; + assert_eq!(p.pass("cap.test", l, 0, 50).await.granted, 1); + assert_eq!( + p.pass("cap.test", l, 5_000, 50).await, + Grants { + granted: 2, + next: Acquire::DomainBusy + } + ); + // ...a freed slot is granted at once while credit is left... + p.report(report("cap.test", "j", "t1", FetchSignal::Success), 5_040) + .await; + assert_eq!( + p.pass("cap.test", l, 5_040, 50).await, + Grants { + granted: 1, + next: Acquire::DomainBusy + } + ); + // ...and so does the job's cap, across domains. + let capped = JobLimits { + max_in_flight: Some(5), + ..l + }; + assert_eq!(p.pass("job1.test", capped, 5_040, 50).await.granted, 1); + assert_eq!( + p.pass("job1.test", capped, 9_000, 50).await, + Grants { + granted: 1, + next: Acquire::JobBusy + } + ); + assert_eq!( + p.pass("job2.test", capped, 9_000, 50).await, + Grants { + granted: 0, + next: Acquire::JobBusy + } + ); + cleanup(&url, &prefix).await; +} + +/// The script every instance ran before SCR-90 (one slot per call, spacing +/// from `last_ms = now`), for the mixed-version test. +const OLD_ACQUIRE_LUA: &str = r#" +local now = tonumber(ARGV[1]) +redis.call('ZREMRANGEBYSCORE', KEYS[2], '-inf', now) +redis.call('ZREMRANGEBYSCORE', KEYS[3], '-inf', now) +local h = redis.call('HMGET', KEYS[1], 'last_ms', 'delay_ms', 'robots_ms', 'pause_until', 'robots_checked') +local pause = tonumber(h[4]) or 0 +if pause > now then return {1, pause - now} end +local jobcap = tonumber(ARGV[3]) +if jobcap > 0 and redis.call('ZCARD', KEYS[3]) >= jobcap then return {-2, 0} end +if redis.call('ZCARD', KEYS[2]) >= tonumber(ARGV[2]) then return {-1, 0} end +local eff = tonumber(h[2]) or tonumber(ARGV[4]) +eff = math.max(eff, tonumber(ARGV[7]), tonumber(ARGV[8])) +if ARGV[9] == '1' then + local r = tonumber(h[3]) + if not r and h[5] == '1' then r = tonumber(ARGV[10]) end + if r then eff = math.max(eff, math.floor(r * tonumber(ARGV[11]))) end +end +local floor = tonumber(ARGV[5]) +eff = math.min(math.max(eff, floor), math.max(tonumber(ARGV[6]), floor)) +local last = tonumber(h[1]) +if last and now - last < eff then return {1, eff - (now - last)} end +local ttl = tonumber(ARGV[13]) +redis.call('ZADD', KEYS[2], now + ttl, ARGV[12]) +redis.call('ZADD', KEYS[3], now + ttl, ARGV[12]) +redis.call('HSET', KEYS[1], 'last_ms', now) +redis.call('PEXPIRE', KEYS[1], ARGV[14]) +redis.call('PEXPIRE', KEYS[2], 2 * ttl) +redis.call('PEXPIRE', KEYS[3], 2 * ttl) +return {0, eff} +"#; + +/// Rolling upgrade: an instance running the previous script shares a +/// domain's keys with one running the new script. Each honours the other's +/// grants, and together they never exceed one request per delay (plus the +/// new script's credit). +#[tokio::test] +async fn old_and_new_scripts_share_a_domain() { + let Some(url) = redis_url() else { return }; + let prefix = format!("test-pol-{}", uuid::Uuid::new_v4()); + let c = spaced(25, 100_000); + let new = RedisPoliteness::new(&url, &prefix, c.clone()) + .await + .unwrap(); + let client = redis::Client::open(url.as_str()).unwrap(); + let mut conn = client.get_multiplexed_async_connection().await.unwrap(); + let old_script = redis::Script::new(OLD_ACQUIRE_LUA); + let start_ms = chrono::Utc::now().timestamp_millis(); + let l = JobLimits::default(); + let mut n = 0; + // The old instance asks for one slot at `at_ms`: `(status, ms)`. + let mut old = |at_ms: i64| { + n += 1; + let mut call = old_script.prepare_invoke(); + call.key(format!("{prefix}:pol:d:mix.test")) + .key(format!("{prefix}:pol:f:mix.test")) + .key(format!("{prefix}:pol:j:j")) + .arg(start_ms + at_ms) + .arg(c.concurrent_per_domain) + .arg(0) + .arg(c.default_delay_ms) + .arg(c.min_delay_ms) + .arg(c.max_delay_ms) + .arg(0) + .arg(0) + .arg("1") + .arg(0) + .arg(c.robots_delay_multiplier) + .arg(format!("old{n}")) + .arg(c.slot_ttl.as_millis() as u64) + .arg(86_400_000u64); + let mut conn = conn.clone(); + async move { + let got: (i64, i64) = call.invoke_async(&mut conn).await.unwrap(); + got + } + }; + let mut m = 0; + let mut new_pass = |at_ms: i64, want: usize| { + let tokens: Vec = (0..want).map(|i| format!("new{}", m + i)).collect(); + m += want; + let new = &new; + async move { + let req = SlotsRequest { + domain: "mix.test", + job_id: "j", + tokens: &tokens, + limits: l, + }; + new.acquire_many_at(&req, start_ms + at_ms).await.unwrap() + } + }; + let wait = |ms| Acquire::Wait(Duration::from_millis(ms)); + + // A hash written by the old script only (`last_ms`): the new script + // spaces from it, and banks credit against it. + assert_eq!(old(0).await, (0, 25)); + assert_eq!( + new_pass(10, 5).await, + Grants { + granted: 0, + next: wait(15) + } + ); + assert_eq!( + new_pass(25, 5).await, + Grants { + granted: 1, + next: wait(25) + } + ); + // The old script honours the new one's grant. + assert_eq!(old(40).await, (1, 10)); + assert_eq!(old(50).await, (0, 25)); + assert_eq!(new_pass(60, 5).await.granted, 0); + + // After an idle second the new script uses its credit (5 grants at + // 1050: the schedule is then at 1050)... + assert_eq!( + new_pass(1_050, 50).await, + Grants { + granted: 5, + next: wait(25) + } + ); + // ...which the old script waits out like any delay... + assert_eq!(old(1_060).await, (1, 15)); + assert_eq!(old(1_075).await, (0, 25)); + // ...and an old grant leaves no credit behind for the new script. + assert_eq!( + new_pass(1_090, 50).await, + Grants { + granted: 0, + next: wait(10) + } + ); + // A new grant that came late (schedule at 1100, granted at 1140) lets + // the old script grant up to the credit early, as the new one would: + // 1125 is the next slot of the schedule. + assert_eq!( + new_pass(1_140, 1).await, + Grants { + granted: 1, + next: Acquire::Granted + } + ); + assert_eq!(old(1_141).await, (0, 25)); + assert_eq!(new_pass(1_150, 50).await.granted, 0); + + let last: i64 = redis::cmd("HGET") + .arg(format!("{prefix}:pol:d:mix.test")) + .arg("last_ms") + .query_async(&mut conn) + .await + .unwrap(); + assert_eq!(last - start_ms, 1_141, "the old script wrote its own clock"); + cleanup(&url, &prefix).await; +} From c6e2a95867be6e8ce59628971a388f3411dcb023 Mon Sep 17 00:00:00 2001 From: Quentin de Quelen Date: Thu, 12 Nov 2026 13:30:42 +0100 Subject: [PATCH 05/10] perf(frontier): keep explicit delays strict, credit only the adaptive delay The spacing credit let a job with a 250 ms delay see a 150 ms gap, and a job with a 50 ms delay see several requests at one instant. A job's per_domain_delay_ms and requests_per_second, a robots Crawl-delay and default_crawl_delay_ms are floors on the gap between two requests, not averages. The credit now applies only when the effective delay is the domain's adaptive delay itself. When an explicit term is above it, the call is strict as before SCR-90: one grant, last = now, never less than the delay after the previous request. Each call is judged on its own delay, so jobs with different limits can share a domain. A credited grant advances the schedule only virtually, so the stores also keep the real time of the last grant (last_grant in memory, real_ms in Redis; the real last grant is max(real_ms, last_ms), which is correct for hashes written by the previous script). A strict call is measured from it. A credited call whose job has a lower explicit term is also never granted less than that term after the real last grant. job_rate_limit_spaces_requests_to_a_domain is back to its original assertions. Refs SCR-90 Co-Authored-By: Claude Fable 5.2 --- bins/scrapix-frontier-service/src/lib.rs | 23 +- crates/scrapix-frontier/src/lib.rs | 6 +- crates/scrapix-frontier/src/politeness.rs | 290 +++++++++++++++--- .../scrapix-frontier/src/politeness_redis.rs | 77 +++-- .../tests/politeness_redis.rs | 179 ++++++++++- 5 files changed, 494 insertions(+), 81 deletions(-) diff --git a/bins/scrapix-frontier-service/src/lib.rs b/bins/scrapix-frontier-service/src/lib.rs index 6cc5430e..75fd0e25 100644 --- a/bins/scrapix-frontier-service/src/lib.rs +++ b/bins/scrapix-frontier-service/src/lib.rs @@ -303,10 +303,12 @@ pub async fn build_store(args: &Args) -> anyhow::Result> /// `DOMAIN_SPACING_CREDIT_MS`: the spacing credit (ms), i.e. how much of /// the time lost between a request to a domain becoming due and the /// dispatch pass that sends it is given back, as requests sent closer -/// together (never more than `1 + credit / delay` at once, and none for -/// delays of 1 s or more). It makes a domain's rate `1 / delay` whatever -/// the dispatch interval. `0` spaces requests strictly from the last one -/// sent. Default 100. Read from the environment rather than `Args`, whose +/// together (never more than `1 + credit / delay` at once). Only the +/// adaptive delay is credited: a delay set by the job (`per_domain_delay_ms`, +/// `requests_per_second`), by robots.txt or of 1 s or more is always +/// strict. It makes an adaptively paced domain's rate `1 / delay` whatever +/// the dispatch interval. `0` spaces every request strictly from the last +/// one sent. Default 100. Read from the environment rather than `Args`, whose /// fields other binaries (`scrapix all`) list one by one. fn spacing_credit_from_env() -> u64 { let default = PolitenessConfig::default().spacing_credit_ms; @@ -1474,8 +1476,9 @@ impl FrontierService { /// than serially). One call asks for a slot per URL of the domain (at /// most `MAX_SLOTS_PER_ASK`) and the store grants as many as the /// domain's schedule and the caps allow: one, or several when this pass - /// came later than the domain's delay (its spacing credit, see - /// `scrapix_frontier::politeness`), so the rate a domain gets does not + /// came later than the domain's adaptive delay (its spacing credit, + /// see `scrapix_frontier::politeness`; never for a delay set by the job + /// or robots.txt), so the rate an adaptively paced domain gets does not /// depend on how often the dispatcher passes. The store also says how /// long the domain's next grant is away: its other URLs of the batch /// are parked for that long without asking again; only a domain that @@ -3105,15 +3108,11 @@ mod tests { "4 rps over 1.5 s: at least 3 dispatches, got {}", at.len() ); - // One at the start, then one per 250 ms: never more than the rate. - assert!(at.len() <= 7, "4 rps over 1.5 s: {} dispatches", at.len()); for w in at.windows(2) { let gap = w[1] - w[0]; - // 250 ms, minus the spacing credit a late pass gives back (at - // most 100 ms) and receive jitter (dispatch tick + poll - // interval): no two requests at once. + // 250 ms minus receive jitter (dispatch tick + poll interval). assert!( - gap >= Duration::from_millis(120), + gap >= Duration::from_millis(200), "dispatches {gap:?} apart" ); } diff --git a/crates/scrapix-frontier/src/lib.rs b/crates/scrapix-frontier/src/lib.rs index f96ccd8c..ca2bb711 100644 --- a/crates/scrapix-frontier/src/lib.rs +++ b/crates/scrapix-frontier/src/lib.rs @@ -77,9 +77,9 @@ pub use history::{ pub use linkgraph::{LinkGraph, LinkGraphBuilder, LinkGraphConfig, LinkGraphStats}; pub use partition::{extract_domain, DomainGrouper, PartitionConfig, Partitioner}; pub use politeness::{ - adapt_on_success, spacing_credit, Acquire, DomainStats, FetchReport, FetchSignal, Grants, - JobLimits, PolitenessConfig, PolitenessScheduler, PolitenessStore, RobotsInfo, SlotRequest, - SlotsRequest, + adapt_on_success, spacing, Acquire, DomainStats, FetchReport, FetchSignal, Grants, JobLimits, + PolitenessConfig, PolitenessScheduler, PolitenessStore, RobotsInfo, SlotRequest, SlotsRequest, + Spacing, }; #[cfg(feature = "redis-store")] pub use politeness_redis::RedisPoliteness; diff --git a/crates/scrapix-frontier/src/politeness.rs b/crates/scrapix-frontier/src/politeness.rs index 777129bd..ea6783b0 100644 --- a/crates/scrapix-frontier/src/politeness.rs +++ b/crates/scrapix-frontier/src/politeness.rs @@ -42,11 +42,25 @@ //! raised since the last grant applies at once); //! - the grant then sets `last = max(last + delay, now - credit, //! pause end)`: the domain's schedule advances by one delay, and is at -//! most `credit` behind the clock. `credit` is -//! [`PolitenessConfig::spacing_credit_ms`] (100 ms) when `delay <` -//! [`PolitenessConfig::strict_spacing_ms`] (1 s), else 0. +//! most `credit` behind the clock. //! -//! Consequences, for a constant delay `D` and credit `C`: +//! **Only the adaptive delay gets credit.** `credit` is +//! [`PolitenessConfig::spacing_credit_ms`] (100 ms) when the effective +//! delay is the domain's adaptive delay itself and is below +//! [`PolitenessConfig::strict_spacing_ms`] (1 s). It is 0 whenever an +//! explicit term sets the delay, i.e. is strictly above the adaptive delay: +//! the job's `per_domain_delay_ms`, its `requests_per_second`, the robots +//! `Crawl-delay` or the job's `default_crawl_delay_ms`. Those are promises +//! made to the job's owner or asked by the site, so they stay a floor on +//! the real gap between two requests: such a request also needs `now >= +//! real last grant + delay`, where the real last grant is tracked next to +//! the virtual one, and sets `last = now`. A request whose delay is the +//! adaptive one but whose job also has a (lower) explicit term is credited, +//! and still never comes less than that explicit term after the real last +//! grant. Jobs with different limits share a domain's state; each call is +//! judged on its own delay ([`spacing`]). +//! +//! Consequences, for a constant adaptive delay `D` and credit `C`: //! //! - any `n + 1` consecutive grants span at least `n × D − C`: the long-run //! rate never exceeds `1 / D`, and a pass that comes late grants what it @@ -55,12 +69,10 @@ //! - at most `1 + floor(C / D)` grants at one instant (5 at the 25 ms //! floor), and never more than the free in-flight slots of the domain and //! the job; -//! - with `D >= strict_spacing_ms` (a robots `Crawl-delay`, a slow job -//! rate) `C` is 0: `last = now`, two requests are never less than `D` -//! apart, exactly as before; -//! - with `C < D < strict_spacing_ms` there is no burst (one grant per -//! instant), but a grant that came late lets the next one come up to `C` -//! early: gaps are at least `D − C` and average `D`; +//! - an explicit delay (any value), and any delay of `strict_spacing_ms` or +//! more, is strict: one grant per call, two requests never less than the +//! delay apart, no catching up after an idle period or a late pass, +//! exactly as before SCR-90; //! - no credit builds up during a pause (`Retry-After`, error streak): the //! schedule restarts at the pause's end. A 429 or an error drops the //! credit banked so far. @@ -189,6 +201,36 @@ pub fn effective_delay( robots: RobotsInfo, limits: &JobLimits, ) -> u64 { + spacing(config, adaptive_ms, robots, limits).delay_ms +} + +/// How one job's requests to a domain are spaced right now (see the module +/// docs, "Spacing credit"). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Spacing { + /// The effective delay: a grant needs `now >= virtual last + delay_ms`. + pub delay_ms: u64, + /// How far the domain's schedule may lag behind the clock. 0 (strict + /// spacing) when an explicit term sets the delay or the delay is + /// `strict_spacing_ms` or more. + pub credit_ms: u64, + /// A grant also needs `now >= real last grant + real_gap_ms`: the whole + /// delay when spacing is strict, else the job's explicit terms (0 for a + /// job with none). + pub real_gap_ms: u64, +} + +/// The spacing of a job's requests to a domain whose adaptive delay is +/// `adaptive_ms`. The explicit terms are the job's `min_delay_ms`, its +/// rate, and the robots term (`Crawl-delay` × multiplier, or the job's +/// default crawl delay); the spacing is strict when one of them is above +/// the (clamped) adaptive delay. +pub fn spacing( + config: &PolitenessConfig, + adaptive_ms: u64, + robots: RobotsInfo, + limits: &JobLimits, +) -> Spacing { let robots = if config.respect_robots_delay && limits.respect_robots { let raw = match robots.crawl_delay_ms { Some(d) => d, @@ -202,21 +244,23 @@ pub fn effective_delay( 0 }; let floor = config.min_delay_ms.min(config.default_delay_ms); - adaptive_ms - .max(limits.min_delay_ms) + let cap = config.max_delay_ms.max(floor); + let explicit = limits + .min_delay_ms .max(limits.rps_delay_ms()) .max(robots) - .max(floor) - .min(config.max_delay_ms.max(floor)) -} - -/// The spacing credit of a domain whose effective delay is `delay_ms` (see -/// the module docs): `spacing_credit_ms` below `strict_spacing_ms`, else 0. -pub fn spacing_credit(config: &PolitenessConfig, delay_ms: u64) -> u64 { - if delay_ms >= config.strict_spacing_ms { + .min(cap); + let adaptive = adaptive_ms.max(floor).min(cap); + let delay_ms = adaptive.max(explicit); + let credit_ms = if explicit > adaptive || delay_ms >= config.strict_spacing_ms { 0 } else { config.spacing_credit_ms + }; + Spacing { + delay_ms, + credit_ms, + real_gap_ms: if credit_ms == 0 { delay_ms } else { explicit }, } } @@ -422,6 +466,9 @@ struct DomainState { /// last grant, or up to the spacing credit earlier when that grant came /// late (see the module docs) last_request: Option, + /// Time of the last grant (what explicit delays are measured from); + /// only meaningful while `last_request` is set + last_grant: Option, /// Error-adapted delay (starts at `default_delay_ms`) delay_ms: u64, /// robots.txt `Crawl-delay` / whether robots.txt was fetched @@ -442,6 +489,7 @@ impl DomainState { fn new(delay_ms: u64) -> Self { Self { last_request: None, + last_grant: None, delay_ms, robots: RobotsInfo::default(), job_limits: JobLimits::default(), @@ -600,7 +648,9 @@ impl PolitenessScheduler { token: None, started: Instant::now(), }); - state.last_request = Some(Instant::now()); + let now = Instant::now(); + state.last_request = Some(now); + state.last_grant = Some(now); }); } @@ -798,9 +848,10 @@ impl PolitenessScheduler { jobs.remove(req.job_id); } let job_cap = req.limits.max_in_flight.filter(|c| *c > 0); - let delay_ms = self.state_delay(state, &req.limits); - let delay = Duration::from_millis(delay_ms); - let credit = Duration::from_millis(spacing_credit(&self.config, delay_ms)); + let spacing = spacing(&self.config, state.delay_ms, state.robots, &req.limits); + let delay = Duration::from_millis(spacing.delay_ms); + let credit = Duration::from_millis(spacing.credit_ms); + let real_gap = Duration::from_millis(spacing.real_gap_ms); // The schedule is at most `credit` behind the clock, and never // behind the end of a pause. let floor = match (now.checked_sub(credit), state.paused_until) { @@ -816,7 +867,11 @@ impl PolitenessScheduler { break Acquire::DomainBusy; } let due = state.last_request.map(|last| last + delay); - if let Some(due) = due.filter(|due| *due > now) { + // The real last grant is never before the virtual one. + let real_due = state + .last_request + .map(|last| state.last_grant.map_or(last, |real| real.max(last)) + real_gap); + if let Some(due) = due.max(real_due).filter(|due| *due > now) { break Acquire::Wait(due - now); } let Some(token) = req.tokens.get(granted) else { @@ -827,6 +882,7 @@ impl PolitenessScheduler { (Some(due), None) => due, (None, _) => now, }); + state.last_grant = Some(now); state.in_flight.push(Slot { token: Some(token.clone()), started: now, @@ -1723,26 +1779,186 @@ mod tests { } } - /// Between the credit and the strict threshold there is no burst: one - /// grant per instant, gaps of at least `delay − credit`, `delay` on - /// average. + /// An explicit delay is a floor on the real gap between two requests, + /// whatever its value: a job's `per_domain_delay_ms`, its + /// `requests_per_second`, a robots `Crawl-delay`. No credit, no burst + /// after an idle period, nothing given back after a late pass. + #[test] + fn explicit_delays_are_spaced_strictly() { + let job = |min_delay_ms| JobLimits { + min_delay_ms, + ..JobLimits::default() + }; + let rate = JobLimits { + max_rps: Some(10.0), + ..JobLimits::default() + }; + let robots = spaced(25, 100); + robots.set_delay("a.test", 300); + for (p, limits, delay) in [ + (spaced(25, 100), job(50), 50u64), + (spaced(25, 100), job(250), 250), + (spaced(25, 100), rate, 100), + (robots, JobLimits::default(), 300), + ] { + assert_eq!( + spacing( + p.config(), + 25, + p.domains + .read() + .get("a.test") + .map_or(RobotsInfo::default(), |s| s.robots), + &limits + ), + Spacing { + delay_ms: delay, + credit_ms: 0, + real_gap_ms: delay + } + ); + let mut sim = Sim::new(p, limits); + sim.run(0, 3_000, 33); + // One grant per delay, each at the first pass after it. + let step = delay.div_ceil(33) * 33; + assert_eq!(sim.grants.len() as u64, 1 + 2_970 / step, "delay {delay}"); + // After ten idle seconds: one grant, and the delay again. + assert_eq!( + sim.pass(13_000, 50), + Grants { + granted: 1, + next: Acquire::Wait(Duration::from_millis(delay)) + }, + "delay {delay}" + ); + assert_eq!(sim.pass(13_000 + delay - 1, 50).granted, 0); + // A pass that comes 60 ms late: one grant, and the next one a + // full delay after it, not 60 ms sooner. + assert_eq!(sim.pass(13_000 + delay + 60, 50).granted, 1); + assert_eq!( + sim.pass(13_000 + 2 * delay + 59, 50), + Grants { + granted: 0, + next: Acquire::Wait(Duration::from_millis(1)) + }, + "delay {delay}" + ); + sim.run(20_000, 23_000, 33); + assert_eq!(most_at_once(&sim.grants), 1, "delay {delay}"); + for w in sim.grants.windows(2) { + assert!( + w[1] - w[0] >= delay, + "delay {delay}: {} ms apart", + w[1] - w[0] + ); + } + } + } + + /// Jobs with and without an explicit delay share a domain: each call is + /// judged on its own delay, and the strict one is measured from the + /// real last request, not from the credited job's schedule. #[test] - fn medium_delays_never_burst() { + fn strict_job_is_never_granted_early_on_a_domain_shared_with_a_credited_job() { + let p = spaced(25, 100); + let base = Instant::now(); + let adaptive = JobLimits::default(); + let strict = JobLimits { + min_delay_ms: 50, + ..JobLimits::default() + }; + let mut n = 0; + let mut ask = |job: &str, limits: JobLimits, at_ms: u64, want: usize| { + let tokens: Vec = (0..want).map(|i| format!("t{}", n + i)).collect(); + n += want; + p.acquire_many_at( + &SlotsRequest { + domain: "a.test", + job_id: job, + tokens: &tokens, + limits, + }, + base + Duration::from_millis(at_ms), + ) + }; + let wait = |ms| Acquire::Wait(Duration::from_millis(ms)); + assert_eq!(ask("a", adaptive, 0, 50).granted, 1); + // The adaptive job uses its credit after an idle second... + assert_eq!(ask("a", adaptive, 1_000, 50).granted, 5); + // ...and the strict job waits its 50 ms from that real request. + assert_eq!( + ask("b", strict, 1_010, 50), + Grants { + granted: 0, + next: wait(40) + } + ); + assert_eq!( + ask("b", strict, 1_050, 50), + Grants { + granted: 1, + next: wait(50) + } + ); + // A strict grant leaves no credit behind for the adaptive job. + assert_eq!( + ask("a", adaptive, 1_060, 50), + Grants { + granted: 0, + next: wait(15) + } + ); + // The adaptive job is granted late (due at 1075, granted at 1200): + // its schedule only advances to 1100, but the strict job is spaced + // from 1200. + assert_eq!(ask("a", adaptive, 1_200, 1).granted, 1); + assert_eq!( + ask("b", strict, 1_210, 50), + Grants { + granted: 0, + next: wait(40) + } + ); + assert_eq!(ask("b", strict, 1_250, 50).granted, 1); + } + + /// A job's explicit delay below the adaptive delay does not set the + /// spacing (the adaptive delay does, with its credit), but stays a + /// floor on the real gap: never two requests at once. + #[test] + fn explicit_delay_below_the_adaptive_one_is_still_a_floor() { let limits = JobLimits { - min_delay_ms: 500, + min_delay_ms: 50, ..JobLimits::default() }; - let mut sim = Sim::new(spaced(25, 100), limits); - sim.run(0, 10_000, 33); - sim.run(60_000, 70_000, 33); + let p = spaced(80, 100); + assert_eq!( + spacing(p.config(), 80, RobotsInfo::default(), &limits), + Spacing { + delay_ms: 80, + credit_ms: 100, + real_gap_ms: 50 + } + ); + let mut sim = Sim::new(p, limits); + sim.run(0, 2_000, 33); + // After an idle period: one grant, the next 50 ms later. + assert_eq!( + sim.pass(10_000, 50), + Grants { + granted: 1, + next: Acquire::Wait(Duration::from_millis(50)) + } + ); + sim.run(10_033, 12_000, 33); assert_eq!(most_at_once(&sim.grants), 1); for w in sim.grants.windows(2) { - assert!(w[1] - w[0] >= 400, "{} ms apart", w[1] - w[0]); + assert!(w[1] - w[0] >= 50, "{} ms apart", w[1] - w[0]); } - assert_conforms(&sim.grants, 500, 100); - // 20 s of passes at 2 per second, give or take one per run. + assert_conforms(&sim.grants, 80, 100); + // 4 s of passes at one per 80 ms, give or take a couple. assert!( - (39..=41).contains(&sim.grants.len()), + (48..=52).contains(&sim.grants.len()), "{}", sim.grants.len() ); diff --git a/crates/scrapix-frontier/src/politeness_redis.rs b/crates/scrapix-frontier/src/politeness_redis.rs index 1b1261c1..f1d1656f 100644 --- a/crates/scrapix-frontier/src/politeness_redis.rs +++ b/crates/scrapix-frontier/src/politeness_redis.rs @@ -10,7 +10,7 @@ //! //! | Key | Type | Contents | //! |-----|------|----------| -//! | `{p}:pol:d:{domain}` | hash | `last_ms` (virtual last request: the last grant's time, or up to the spacing credit earlier), `delay_ms` (adaptive), `robots_ms` (Crawl-delay, absent when none), `robots_checked` (`1` once robots.txt was fetched), `errors`, `pause_until` | +//! | `{p}:pol:d:{domain}` | hash | `last_ms` (virtual last request: the last grant's time, or up to the spacing credit earlier), `real_ms` (time of the last grant by this script; the real last grant is `max(real_ms, last_ms)`), `delay_ms` (adaptive), `robots_ms` (Crawl-delay, absent when none), `robots_checked` (`1` once robots.txt was fetched), `errors`, `pause_until` | //! | `{p}:pol:f:{domain}` | zset | in-flight slots: member = token, score = expiry ms | //! | `{p}:pol:j:{job_id}` | zset | the job's in-flight slots, same encoding | //! @@ -21,14 +21,22 @@ //! //! ## Mixed versions //! -//! `last_ms` is the only spacing state, as before SCR-90, so instances -//! running the previous script can share the keys with this one. An old -//! instance grants when `now - last_ms >= delay` and writes `last_ms = now`: -//! under this script's rule that is a grant that banks no credit. It reads -//! a `last_ms` written here as the last request time, so it may grant up to -//! the credit (100 ms) earlier than a full delay after the real last -//! request, which is what this script would do too. Either way a domain's -//! long-run rate stays at most `1 / delay`. +//! Instances running the script of before SCR-90 can share the keys with +//! this one. That script only knows `last_ms`: it grants when `now - +//! last_ms >= delay` and writes `last_ms = now`. +//! +//! - A hash it wrote has no `real_ms` (or a stale one): this script takes +//! the real last grant as `max(real_ms, last_ms)`, which is that grant's +//! time, so both the credited and the strict rule space from it, and it +//! left no credit behind. +//! - It reads a `last_ms` written here as the last request time. After a +//! strict grant that is the real time. After a credited grant it can be +//! up to the credit (100 ms) earlier, so an old instance may then grant +//! up to 100 ms early, including to a job with an explicit delay: until +//! every instance is upgraded, the strict guarantee holds on a domain +//! only while no adaptive-only job shares it. Old instances never burst. +//! +//! Either way a domain's long-run rate stays at most `1 / delay`. use std::collections::HashMap; use std::time::{Duration, Instant}; @@ -60,30 +68,39 @@ const RECENT_DOMAIN_WINDOW: Duration = Duration::from_secs(600); /// (a slot, only when every token was granted), `1` (wait `ms`: the delay /// or a pause has not elapsed), `-1` (domain busy) or `-2` (job busy). /// -/// The spacing rule is the one of [`crate::politeness`] ("Spacing credit"): -/// a grant needs `now - last_ms >= effective delay` and sets `last_ms = -/// max(last_ms + delay, now - credit, pause_until)` (`now` for a domain -/// never requested), with no credit for a delay at or above the strict -/// spacing threshold. So one call grants at most `1 + floor(credit / -/// delay)` slots, and never more than the domain and the job have free. +/// The spacing rule is the one of [`crate::politeness`] ("Spacing credit", +/// [`crate::politeness::spacing`]): with `adp` the clamped adaptive delay +/// and `exp` the job's explicit terms (job delay, rate, robots), the delay +/// is `max(adp, exp)`; the credit is 0 when `exp > adp` or the delay is at +/// or above the strict spacing threshold. A grant needs `now - last_ms >= +/// delay` and `now - real >= gap`, where `real = max(real_ms, last_ms)` is +/// the real last grant and `gap` is the delay without credit, else `exp`. +/// It sets `last_ms = max(last_ms + delay, now - credit, pause_until)` +/// (`now` for a domain never requested) and `real_ms = now`. So one call +/// grants at most `1 + floor(credit / delay)` slots (one when spacing is +/// strict), and never more than the domain and the job have free. const ACQUIRE_LUA: &str = r#" local now = tonumber(ARGV[1]) redis.call('ZREMRANGEBYSCORE', KEYS[2], '-inf', now) redis.call('ZREMRANGEBYSCORE', KEYS[3], '-inf', now) -local h = redis.call('HMGET', KEYS[1], 'last_ms', 'delay_ms', 'robots_ms', 'pause_until', 'robots_checked') +local h = redis.call('HMGET', KEYS[1], 'last_ms', 'delay_ms', 'robots_ms', 'pause_until', 'robots_checked', 'real_ms') local pause = tonumber(h[4]) or 0 if pause > now then return {1, pause - now, 0} end -local eff = tonumber(h[2]) or tonumber(ARGV[4]) -eff = math.max(eff, tonumber(ARGV[7]), tonumber(ARGV[8])) +local floor = tonumber(ARGV[5]) +local cap = math.max(tonumber(ARGV[6]), floor) +local exp = math.max(tonumber(ARGV[7]), tonumber(ARGV[8])) if ARGV[9] == '1' then local r = tonumber(h[3]) if not r and h[5] == '1' then r = tonumber(ARGV[10]) end - if r then eff = math.max(eff, math.floor(r * tonumber(ARGV[11]))) end + if r then exp = math.max(exp, math.floor(r * tonumber(ARGV[11]))) end end -local floor = tonumber(ARGV[5]) -eff = math.min(math.max(eff, floor), math.max(tonumber(ARGV[6]), floor)) +exp = math.min(exp, cap) +local adp = math.min(math.max(tonumber(h[2]) or tonumber(ARGV[4]), floor), cap) +local eff = math.max(adp, exp) local credit = tonumber(ARGV[12]) -if eff >= tonumber(ARGV[15]) then credit = 0 end +if exp > adp or eff >= tonumber(ARGV[15]) then credit = 0 end +local gap = exp +if credit == 0 then gap = eff end local domcap = tonumber(ARGV[2]) local jobcap = tonumber(ARGV[3]) local dom_n = redis.call('ZCARD', KEYS[2]) @@ -91,19 +108,25 @@ local job_n = redis.call('ZCARD', KEYS[3]) local ttl = tonumber(ARGV[13]) local want = #ARGV - 15 local last = tonumber(h[1]) +local real = last +if last then real = math.max(tonumber(h[6]) or last, last) end local granted = 0 local status = 0 local ms = 0 while true do if jobcap > 0 and job_n >= jobcap then status = -2 break end if dom_n >= domcap then status = -1 break end - if last and now - last < eff then - status = 1 - ms = eff - (now - last) - break + if last then + local wait = math.max(eff - (now - last), gap - (now - real)) + if wait > 0 then + status = 1 + ms = wait + break + end end if granted >= want then break end if last then last = math.max(last + eff, now - credit, pause) else last = now end + real = now granted = granted + 1 local token = ARGV[15 + granted] redis.call('ZADD', KEYS[2], now + ttl, token) @@ -112,7 +135,7 @@ while true do job_n = job_n + 1 end if granted > 0 then - redis.call('HSET', KEYS[1], 'last_ms', last) + redis.call('HSET', KEYS[1], 'last_ms', last, 'real_ms', real) redis.call('PEXPIRE', KEYS[1], ARGV[14]) redis.call('PEXPIRE', KEYS[2], 2 * ttl) redis.call('PEXPIRE', KEYS[3], 2 * ttl) diff --git a/crates/scrapix-frontier/tests/politeness_redis.rs b/crates/scrapix-frontier/tests/politeness_redis.rs index 282df795..ceaa8c21 100644 --- a/crates/scrapix-frontier/tests/politeness_redis.rs +++ b/crates/scrapix-frontier/tests/politeness_redis.rs @@ -769,7 +769,7 @@ async fn old_and_new_scripts_share_a_domain() { .await .unwrap(); let client = redis::Client::open(url.as_str()).unwrap(); - let mut conn = client.get_multiplexed_async_connection().await.unwrap(); + let conn = client.get_multiplexed_async_connection().await.unwrap(); let old_script = redis::Script::new(OLD_ACQUIRE_LUA); let start_ms = chrono::Utc::now().timestamp_millis(); let l = JobLimits::default(); @@ -873,12 +873,187 @@ async fn old_and_new_scripts_share_a_domain() { assert_eq!(old(1_141).await, (0, 25)); assert_eq!(new_pass(1_150, 50).await.granted, 0); + // A job with an explicit 50 ms delay is spaced from the old script's + // grant (1141), although `real_ms` still says 1140. + let strict = JobLimits { + min_delay_ms: 50, + ..l + }; + let strict_at = |at_ms: i64, token: &'static str| { + let new = &new; + async move { + let tokens = [token.to_string()]; + let req = SlotsRequest { + domain: "mix.test", + job_id: "k", + tokens: &tokens, + limits: strict, + }; + new.acquire_many_at(&req, start_ms + at_ms).await.unwrap() + } + }; + assert_eq!( + strict_at(1_180, "s1").await, + Grants { + granted: 0, + next: wait(11) + } + ); + let last: i64 = redis::cmd("HGET") .arg(format!("{prefix}:pol:d:mix.test")) .arg("last_ms") - .query_async(&mut conn) + .query_async(&mut conn.clone()) .await .unwrap(); assert_eq!(last - start_ms, 1_141, "the old script wrote its own clock"); + // After a strict grant by the new script, `last_ms` is the real time: + // the old script cannot grant early. + assert_eq!(strict_at(1_191, "s2").await.granted, 1); + assert_eq!(old(1_215).await, (1, 1)); + assert_eq!(old(1_216).await, (0, 25)); + cleanup(&url, &prefix).await; +} + +/// Explicit delays (job delay, job rate, robots `Crawl-delay`) are strict +/// in both stores, whatever their value; only the adaptive delay is +/// credited, also on a domain the two kinds of job share. +#[tokio::test] +async fn explicit_delays_are_strict_like_the_memory_store() { + let Some(url) = redis_url() else { return }; + let prefix = format!("test-pol-{}", uuid::Uuid::new_v4()); + let l = JobLimits::default(); + let wait = |ms| Acquire::Wait(Duration::from_millis(ms)); + let job = |min_delay_ms| JobLimits { min_delay_ms, ..l }; + let rate = JobLimits { + max_rps: Some(10.0), + ..l + }; + let mut p = Parity::new(&url, &prefix, spaced(25, 100_000)).await; + + // robots.txt of `robots300.test` asks for 300 ms. + assert_eq!(p.pass("robots300.test", l, 0, 50).await.granted, 1); + let token = format!("t{}", p.grants.len() - 1); + p.report( + FetchReport { + crawl_delay_ms: Some(300), + ..report("robots300.test", "j", &token, FetchSignal::Success) + }, + 1, + ) + .await; + + for (domain, limits, delay) in [ + ("job50.test", job(50), 50u64), + ("job250.test", job(250), 250), + ("rps.test", rate, 100), + ("robots300.test", l, 300), + ] { + let mut at = Vec::new(); + let mut ask = |t: u64, got: Grants| { + assert!(got.granted <= 1, "{domain} at {t}: {got:?}"); + if got.granted == 1 { + at.push(t); + } + }; + // Passes every 33 ms for 2 s (from 1 s on: robots300 was first + // requested at 0). + for t in (1_000..3_000).step_by(33) { + ask(t, p.pass(domain, limits, t, 50).await); + } + // Ten idle seconds bank nothing: one grant, then the delay. + let got = p.pass(domain, limits, 13_000, 50).await; + assert_eq!( + got, + Grants { + granted: 1, + next: wait(delay) + }, + "{domain}" + ); + ask(13_000, got); + ask( + 13_000 + delay - 1, + p.pass(domain, limits, 13_000 + delay - 1, 50).await, + ); + // A pass 60 ms late gives nothing back. + ask( + 13_000 + delay + 60, + p.pass(domain, limits, 13_000 + delay + 60, 50).await, + ); + let got = p.pass(domain, limits, 13_000 + 2 * delay + 59, 50).await; + assert_eq!( + got, + Grants { + granted: 0, + next: wait(1) + }, + "{domain}" + ); + for t in (20_000..22_000).step_by(33) { + ask(t, p.pass(domain, limits, t, 50).await); + } + let step = delay.div_ceil(33) * 33; + assert_eq!(at.len() as u64, 2 * (1 + 1_980 / step) + 2, "{domain}"); + for w in at.windows(2) { + assert!(w[1] - w[0] >= delay, "{domain}: {} ms apart", w[1] - w[0]); + } + } + + // A purely adaptive domain still gets its credit. + assert_eq!(p.pass("adaptive.test", l, 0, 50).await.granted, 1); + assert_eq!(p.pass("adaptive.test", l, 1_000, 50).await.granted, 5); + + // A domain shared by an adaptive job and a job with a 50 ms delay: the + // strict job is spaced from the real last request. + assert_eq!(p.pass("shared.test", l, 0, 50).await.granted, 1); + assert_eq!(p.pass("shared.test", l, 1_000, 50).await.granted, 5); + assert_eq!( + p.pass("shared.test", job(50), 1_010, 50).await, + Grants { + granted: 0, + next: wait(40) + } + ); + assert_eq!( + p.pass("shared.test", job(50), 1_050, 50).await, + Grants { + granted: 1, + next: wait(50) + } + ); + assert_eq!( + p.pass("shared.test", l, 1_060, 50).await, + Grants { + granted: 0, + next: wait(15) + } + ); + // Granted late (due at 1075): the schedule is at 1100, the real + // request at 1200. + assert_eq!(p.pass("shared.test", l, 1_200, 1).await.granted, 1); + assert_eq!( + p.pass("shared.test", job(50), 1_210, 50).await, + Grants { + granted: 0, + next: wait(40) + } + ); + assert_eq!(p.pass("shared.test", job(50), 1_250, 50).await.granted, 1); + cleanup(&url, &prefix).await; + + // An explicit delay below the adaptive one: credited, but never two + // requests less than it apart. + let prefix = format!("test-pol-{}", uuid::Uuid::new_v4()); + let mut p = Parity::new(&url, &prefix, spaced(80, 100_000)).await; + assert_eq!(p.pass("low.test", job(50), 0, 50).await.granted, 1); + assert_eq!( + p.pass("low.test", job(50), 10_000, 50).await, + Grants { + granted: 1, + next: wait(50) + } + ); + assert_eq!(p.pass("low.test", job(50), 10_050, 50).await.granted, 1); cleanup(&url, &prefix).await; } From b5dbc393aa9c18854e2fa150c1923628ce0bfd92 Mon Sep 17 00:00:00 2001 From: Quentin de Quelen Date: Thu, 12 Nov 2026 13:38:46 +0100 Subject: [PATCH 06/10] perf(queue): drop the producer linger to 0 ms and make it configurable Every Kafka producer used linger.ms=20, and `KafkaProducer::send` awaits the delivery report, so each awaited publish stage waited at least one linger: about 40 ms per page in the crawler (page + links, then PageCrawled), one linger per frontier dispatch pass, and the same for the content worker's outcome events and the API's job controls. Since SCR-89 the sends of a stage are concurrent, so they batch without a linger. The linger is now read from KAFKA_LINGER_MS by `ProducerBuilder::build` and `KafkaProducer::with_brokers` (every producer of every binary goes through the builder; no Args change). It must be an integer in 0..=1000; an invalid value fails producer creation, so the service stops at startup. Unset or blank means the default, now 0. Measured with the new `examples/linger_bench.rs` (real KafkaProducer, acks=all, idempotent, lz4, 1.5 KB JSON messages, local single-node Redpanda v25.3.4, 1-partition topic, quiet machine), linger 0/1/5/10/20: - one awaited send, p50: 0.13 / 1.6 / 6.6 / 12.4 / 22.4 ms - 100 sends awaited together: 0.9 / 2.2 / 7.3 / 13.1 / 23.0 ms, in 3.5-3.9 / 3 / 3 / 3 / 3 produce requests - 20,000 messages, 256 in flight: 181-189k / 173-189k / 169-183k / 158-163k / 157-160k msg/s, 436 / 435 / 435 / 435 / 435 requests (batches fill by size whatever the linger) - 64 slots each awaiting its sends: 148-158k / 137-141k / 111-129k / 84k / 75-94k msg/s - 2,000 msg/s open loop: 0.4 / 1.2 / 3.1 / 5.8 / 10.9 ms per send, 1,170 / 610 / 183 / 94 / 48 produce requests/s On a 12-partition topic the closed loop falls from 127k msg/s at 0 to 8.8k at 5 ms and 2.8k at 20 ms. A linger only buys fewer produce requests at moderate rates, and costs its full length per stage. Not measured: a broker that fsyncs (Redpanda's container image always runs with --unsafe-bypass-fsync). A slower ack makes librdkafka batch more on its own at linger 0, since messages queue while requests are in flight. Refs SCR-90 Co-Authored-By: Claude Fable 5.2 --- crates/scrapix-queue/examples/linger_bench.rs | 292 ++++++++++++++++++ crates/scrapix-queue/src/lib.rs | 5 +- crates/scrapix-queue/src/producer.rs | 101 +++++- 3 files changed, 392 insertions(+), 6 deletions(-) create mode 100644 crates/scrapix-queue/examples/linger_bench.rs diff --git a/crates/scrapix-queue/examples/linger_bench.rs b/crates/scrapix-queue/examples/linger_bench.rs new file mode 100644 index 00000000..b675aa49 --- /dev/null +++ b/crates/scrapix-queue/examples/linger_bench.rs @@ -0,0 +1,292 @@ +//! Producer linger benchmark: how `linger.ms` trades the latency of an +//! awaited send against batching, with the real `KafkaProducer` +//! (`acks=all`, idempotent, lz4). It backs `DEFAULT_LINGER_MS`. +//! +//! Redpanda's container image always runs with `--unsafe-bypass-fsync`, +//! so broker acks here do not include a disk flush. +//! +//! ```bash +//! docker run -d --rm --name scrapix-bench-redpanda -p 19093:19093 -p 19644:9644 \ +//! docker.redpanda.com/redpandadata/redpanda:latest redpanda start \ +//! --mode dev-container --smp 1 \ +//! --kafka-addr internal://0.0.0.0:9092,external://0.0.0.0:19093 \ +//! --advertise-kafka-addr internal://127.0.0.1:9092,external://localhost:19093 +//! +//! SCRAPIX_BENCH_KAFKA_BROKERS=localhost:19093 \ +//! SCRAPIX_BENCH_REDPANDA_METRICS=localhost:19644 \ +//! cargo run --release -p scrapix-queue --example linger_bench +//! ``` +//! +//! Without `SCRAPIX_BENCH_KAFKA_BROKERS` it prints a note and exits. +//! `SCRAPIX_BENCH_REDPANDA_METRICS` (the broker's admin address) is +//! optional: with it the bench reads the broker's produce-request counter +//! around each phase. Phases: (a) one awaited send at a time, (b) 100 sends +//! awaited together, 50 rounds, (c) 20,000 messages with 256 in flight, +//! (d) 64 closed-loop slots of 150 awaited sends each, (e) 2,000 msg/s +//! open loop for 3 s. `SCRAPIX_BENCH_LINGERS` (default `0,1,5,10,20`) and +//! `SCRAPIX_BENCH_TOPIC` override the lingers and the topic. + +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use scrapix_queue::{KafkaProducer, ProducerBuilder}; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; +use tokio::sync::Semaphore; +use tokio::task::JoinSet; + +const SINGLE_SENDS: usize = 200; +const BURST_SIZE: usize = 100; +const BURST_ROUNDS: usize = 50; +const SUSTAINED_MESSAGES: usize = 20_000; +const SUSTAINED_IN_FLIGHT: usize = 256; +const LOOP_WORKERS: usize = 64; +const LOOP_SENDS_PER_WORKER: usize = 150; +const PACED_RATE_PER_SEC: usize = 2_000; +const PACED_SECONDS: usize = 3; + +/// A ~1.5 KB JSON message, shaped like a discovered-URL message. +fn payload(i: usize) -> Vec { + let body = serde_json::json!({ + "job_id": "0190f2a4-bench-7000-8000-000000000000", + "index_uid": "bench", + "account_id": "acct_bench", + "url": format!("https://docs.example.com/guides/section-{}/page-{i}.html", i % 97), + "parent_url": "https://docs.example.com/guides/index.html", + "depth": 3, + "priority": 0.5, + "anchor_text": "A link text of an ordinary length, as found on a documentation page", + "headers": {"accept-language": "en-US,en;q=0.9", "x-request": i}, + // Not compressible to nothing: varies per message. + "padding": (0..36).map(|k| format!("{:08x}{:08x}{:08x}", i * 31 + k, k * 7919 + i, i ^ (k << 9))).collect::>(), + }); + serde_json::to_vec(&body).expect("json") +} + +fn percentile(sorted: &[Duration], p: f64) -> Duration { + let idx = ((sorted.len() as f64 - 1.0) * p).round() as usize; + sorted[idx] +} + +fn ms(d: Duration) -> f64 { + d.as_secs_f64() * 1000.0 +} + +/// The broker's count of produce requests, from Redpanda's public metrics. +async fn produce_requests(admin: Option<&str>) -> Option { + let admin = admin?; + let mut stream = tokio::net::TcpStream::connect(admin).await.ok()?; + let request = + format!("GET /public_metrics HTTP/1.1\r\nHost: {admin}\r\nConnection: close\r\n\r\n"); + stream.write_all(request.as_bytes()).await.ok()?; + let mut body = String::new(); + stream.read_to_string(&mut body).await.ok()?; + body.lines() + .find(|l| { + l.starts_with("redpanda_kafka_request_latency_seconds_count") + && l.contains("redpanda_request=\"produce\"") + }) + .and_then(|l| l.rsplit(' ').next()) + .and_then(|v| v.parse().ok()) +} + +fn requests_between(before: Option, after: Option) -> String { + match (before, after) { + (Some(b), Some(a)) => format!("{:.0}", a - b), + _ => "n/a".to_string(), + } +} + +async fn send_many( + producer: &Arc, + topic: &str, + count: usize, + in_flight: usize, +) -> Duration { + let permits = Arc::new(Semaphore::new(in_flight)); + let mut tasks = JoinSet::new(); + let started = Instant::now(); + for i in 0..count { + let permit = permits.clone().acquire_owned().await.expect("semaphore"); + let producer = producer.clone(); + let topic = topic.to_string(); + tasks.spawn(async move { + let key = format!("host-{}.example.com", i % 16); + producer + .send_raw(&topic, Some(&key), &payload(i), None) + .await + .expect("send"); + drop(permit); + }); + } + while let Some(done) = tasks.join_next().await { + done.expect("task"); + } + started.elapsed() +} + +/// Closed loop: each worker awaits one send before the next, like a +/// crawler slot awaiting a publish stage. Throughput is workers / latency. +async fn closed_loop(producer: &Arc, topic: &str) -> Duration { + let mut tasks = JoinSet::new(); + let started = Instant::now(); + for w in 0..LOOP_WORKERS { + let producer = producer.clone(); + let topic = topic.to_string(); + tasks.spawn(async move { + let key = format!("host-{}.example.com", w % 16); + for i in 0..LOOP_SENDS_PER_WORKER { + producer + .send_raw(&topic, Some(&key), &payload(w * 1000 + i), None) + .await + .expect("send"); + } + }); + } + while let Some(done) = tasks.join_next().await { + done.expect("task"); + } + started.elapsed() +} + +/// Open loop at a moderate, fixed rate (2 sends every millisecond): the +/// shape where a short linger batches least. Returns the mean send latency. +async fn paced(producer: &Arc, topic: &str) -> Duration { + let mut tasks = JoinSet::new(); + let mut tick = tokio::time::interval(Duration::from_millis(1)); + tick.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Burst); + let per_tick = PACED_RATE_PER_SEC / 1000; + for t in 0..PACED_SECONDS * 1000 { + tick.tick().await; + for k in 0..per_tick { + let producer = producer.clone(); + let topic = topic.to_string(); + let i = t * per_tick + k; + tasks.spawn(async move { + let key = format!("host-{}.example.com", i % 16); + let started = Instant::now(); + producer + .send_raw(&topic, Some(&key), &payload(i), None) + .await + .expect("send"); + started.elapsed() + }); + } + } + let mut total = Duration::ZERO; + let mut count = 0u32; + while let Some(done) = tasks.join_next().await { + total += done.expect("task"); + count += 1; + } + total / count +} + +#[tokio::main] +async fn main() { + let Ok(brokers) = std::env::var("SCRAPIX_BENCH_KAFKA_BROKERS") else { + eprintln!("SCRAPIX_BENCH_KAFKA_BROKERS is not set: nothing to do (see the file header)"); + return; + }; + let admin = std::env::var("SCRAPIX_BENCH_REDPANDA_METRICS").ok(); + let admin = admin.as_deref(); + let topic = + std::env::var("SCRAPIX_BENCH_TOPIC").unwrap_or_else(|_| "scrapix.bench.linger".to_string()); + let lingers: Vec = std::env::var("SCRAPIX_BENCH_LINGERS") + .unwrap_or_else(|_| "0,1,5,10,20".to_string()) + .split(',') + .map(|v| v.trim().parse().expect("SCRAPIX_BENCH_LINGERS: integers")) + .collect(); + + println!( + "linger_ms | single p50 ms | single p99 ms | burst100 mean ms | burst100 p99 ms | burst msg/s | burst reqs/round | sustained msg/s | sustained reqs | msgs/req | loop64 msg/s | loop64 msgs/req | paced2k mean ms | paced2k req/s" + ); + for linger in lingers { + let producer = Arc::new( + ProducerBuilder::new(&brokers) + .client_id(format!("scrapix-bench-linger-{linger}")) + .compression("lz4") + .linger_ms(linger) + .build() + .expect("producer"), + ); + // Warm up: connection, metadata, idempotent producer id. + for i in 0..20 { + producer + .send_raw(&topic, Some("warmup"), &payload(i), None) + .await + .expect("warmup send"); + } + + // (a) one awaited send at a time. + let mut single = Vec::with_capacity(SINGLE_SENDS); + for i in 0..SINGLE_SENDS { + let started = Instant::now(); + producer + .send_raw(&topic, Some("host-0.example.com"), &payload(i), None) + .await + .expect("send"); + single.push(started.elapsed()); + } + single.sort(); + + // (b) a page with 100 links: 100 sends awaited together. + let before = produce_requests(admin).await; + let mut bursts = Vec::with_capacity(BURST_ROUNDS); + for _ in 0..BURST_ROUNDS { + bursts.push(send_many(&producer, &topic, BURST_SIZE, BURST_SIZE).await); + } + let after = produce_requests(admin).await; + let burst_total: Duration = bursts.iter().sum(); + bursts.sort(); + let burst_reqs = match (before, after) { + (Some(b), Some(a)) => format!("{:.1}", (a - b) / BURST_ROUNDS as f64), + _ => "n/a".to_string(), + }; + + // (c) sustained load. + let before = produce_requests(admin).await; + let sustained = send_many(&producer, &topic, SUSTAINED_MESSAGES, SUSTAINED_IN_FLIGHT).await; + let after = produce_requests(admin).await; + let per_req = match (before, after) { + (Some(b), Some(a)) if a > b => format!("{:.1}", SUSTAINED_MESSAGES as f64 / (a - b)), + _ => "n/a".to_string(), + }; + + // (d) 64 slots, each awaiting its sends one at a time. + let loop_before = produce_requests(admin).await; + let looped = closed_loop(&producer, &topic).await; + let loop_after = produce_requests(admin).await; + let loop_total = LOOP_WORKERS * LOOP_SENDS_PER_WORKER; + let loop_per_req = match (loop_before, loop_after) { + (Some(b), Some(a)) if a > b => format!("{:.1}", loop_total as f64 / (a - b)), + _ => "n/a".to_string(), + }; + + // (e) 2,000 msg/s, open loop. + let paced_before = produce_requests(admin).await; + let paced_latency = paced(&producer, &topic).await; + let paced_after = produce_requests(admin).await; + let paced_reqs = match (paced_before, paced_after) { + (Some(b), Some(a)) => format!("{:.0}", (a - b) / PACED_SECONDS as f64), + _ => "n/a".to_string(), + }; + + println!( + "{linger:>9} | {:>13.2} | {:>13.2} | {:>16.2} | {:>15.2} | {:>11.0} | {:>16} | {:>15.0} | {:>14} | {:>8} | {:>12.0} | {:>15} | {:>15.2} | {:>13}", + ms(percentile(&single, 0.50)), + ms(percentile(&single, 0.99)), + ms(burst_total) / BURST_ROUNDS as f64, + ms(percentile(&bursts, 0.99)), + (BURST_SIZE * BURST_ROUNDS) as f64 / burst_total.as_secs_f64(), + burst_reqs, + SUSTAINED_MESSAGES as f64 / sustained.as_secs_f64(), + requests_between(before, after), + per_req, + loop_total as f64 / looped.as_secs_f64(), + loop_per_req, + ms(paced_latency), + paced_reqs, + ); + producer.flush(Duration::from_secs(5)); + } +} diff --git a/crates/scrapix-queue/src/lib.rs b/crates/scrapix-queue/src/lib.rs index 049bf38a..aa7f60a5 100644 --- a/crates/scrapix-queue/src/lib.rs +++ b/crates/scrapix-queue/src/lib.rs @@ -65,7 +65,10 @@ pub use accounting::{EventPosition, FrontierSnapshot, JobAccounting}; pub use consumer::{ConsumerBuilder, ConsumerConfig, KafkaConsumer, MessageMetadata}; pub use control::{control_group_id, CancelledJobs, CANCELLED_JOBS_CAP}; pub use offsets::OffsetTracker; -pub use producer::{KafkaProducer, ProducerBuilder, ProducerConfig}; +pub use producer::{ + linger_ms_from_env, parse_linger_ms, KafkaProducer, ProducerBuilder, ProducerConfig, + DEFAULT_LINGER_MS, LINGER_MS_ENV, MAX_LINGER_MS, +}; pub use scrapix_core::Ack; pub use topics::{ names as topic_names, CrawlEvent, CrawlHistoryMessage, DlqMessage, DocumentMessage, diff --git a/crates/scrapix-queue/src/producer.rs b/crates/scrapix-queue/src/producer.rs index 47c1ef71..6d52d76c 100644 --- a/crates/scrapix-queue/src/producer.rs +++ b/crates/scrapix-queue/src/producer.rs @@ -9,10 +9,55 @@ use rdkafka::{ util::Timeout, }; use serde::Serialize; -use tracing::{debug, error, instrument}; +use tracing::{debug, error, info, instrument}; use scrapix_core::{Result, ScrapixError}; +/// Environment variable setting the linger of every Kafka producer (ms). +/// Raise it (e.g. `5`) to trade send latency for fewer produce requests. +pub const LINGER_MS_ENV: &str = "KAFKA_LINGER_MS"; + +/// Default producer linger (ms). Measured with `examples/linger_bench.rs` +/// against a local single-node Redpanda (`acks=all`, idempotent, lz4, +/// 1.5 KB messages, 1-partition topic), linger 0 / 5 / 20 ms: +/// one awaited send p50 0.13 / 6.6 / 22.4 ms; 100 sends awaited together +/// 0.9 / 7.3 / 23 ms in 4 / 3 / 3 produce requests; 20,000 messages with +/// 256 in flight: same throughput and 436 / 435 / 435 requests (batches +/// fill by size, and librdkafka batches what queues up while a request is +/// in flight); 64 slots each awaiting its sends: 150k / 120k / 85k msg/s. +/// The linger only bought fewer requests at moderate rates (2,000 msg/s: +/// 1,170 / 183 / 48 requests/s), at 0.4 / 3.1 / 10.9 ms per send. +pub const DEFAULT_LINGER_MS: u64 = 0; + +/// Highest accepted `KAFKA_LINGER_MS`. +pub const MAX_LINGER_MS: u64 = 1000; + +/// Parse a `KAFKA_LINGER_MS` value: an integer in `0..=MAX_LINGER_MS`. +/// Unset or blank means [`DEFAULT_LINGER_MS`]; anything else is an error, +/// so a typo stops the service at startup instead of being ignored. +pub fn parse_linger_ms(raw: Option<&str>) -> Result { + let Some(raw) = raw.map(str::trim).filter(|v| !v.is_empty()) else { + return Ok(DEFAULT_LINGER_MS); + }; + match raw.parse::() { + Ok(ms) if ms <= MAX_LINGER_MS => Ok(ms), + _ => Err(ScrapixError::Queue(format!( + "Invalid {LINGER_MS_ENV}={raw:?}: expected an integer between 0 and {MAX_LINGER_MS} (milliseconds)" + ))), + } +} + +/// The producer linger from `KAFKA_LINGER_MS` (see [`parse_linger_ms`]). +pub fn linger_ms_from_env() -> Result { + match std::env::var(LINGER_MS_ENV) { + Ok(raw) => parse_linger_ms(Some(&raw)), + Err(std::env::VarError::NotPresent) => parse_linger_ms(None), + Err(std::env::VarError::NotUnicode(_)) => Err(ScrapixError::Queue(format!( + "Invalid {LINGER_MS_ENV}: not valid UTF-8" + ))), + } +} + /// Producer configuration #[derive(Debug, Clone)] pub struct ProducerConfig { @@ -30,7 +75,10 @@ pub struct ProducerConfig { pub compression: String, /// Batch size pub batch_size: usize, - /// Linger time (ms) - how long to wait before sending a batch + /// Linger time (ms): how long a message waits for others before its + /// batch is sent. `send` awaits the delivery report, so every awaited + /// publish stage costs at least one linger. [`ProducerBuilder`] and + /// [`KafkaProducer::with_brokers`] take it from `KAFKA_LINGER_MS`. pub linger_ms: u64, /// Acks required (0, 1, all) pub acks: String, @@ -48,7 +96,7 @@ impl Default for ProducerConfig { idempotent: true, compression: "lz4".to_string(), batch_size: 65536, - linger_ms: 20, + linger_ms: DEFAULT_LINGER_MS, acks: "all".to_string(), max_message_bytes: 10 * 1024 * 1024, // 10MB } @@ -91,13 +139,21 @@ impl KafkaProducer { .create() .map_err(|e| ScrapixError::Queue(format!("Failed to create producer: {}", e)))?; + info!( + client_id = %config.client_id, + linger_ms = config.linger_ms, + "Kafka producer created" + ); + Ok(Self { producer, config }) } - /// Create a producer with default configuration + /// Create a producer with default configuration (linger from + /// `KAFKA_LINGER_MS`) pub fn with_brokers(brokers: impl Into) -> Result { Self::new(ProducerConfig { brokers: brokers.into(), + linger_ms: linger_ms_from_env()?, ..Default::default() }) } @@ -235,6 +291,8 @@ impl KafkaProducer { /// Builder for KafkaProducer pub struct ProducerBuilder { config: ProducerConfig, + /// `linger_ms()` was called: `KAFKA_LINGER_MS` is not read. + linger_set: bool, } impl ProducerBuilder { @@ -244,6 +302,7 @@ impl ProducerBuilder { brokers: brokers.into(), ..Default::default() }, + linger_set: false, } } @@ -267,8 +326,10 @@ impl ProducerBuilder { self } + /// Set the linger explicitly, overriding `KAFKA_LINGER_MS`. pub fn linger_ms(mut self, ms: u64) -> Self { self.config.linger_ms = ms; + self.linger_set = true; self } @@ -282,7 +343,12 @@ impl ProducerBuilder { self } - pub fn build(self) -> Result { + /// Build the producer. Unless `linger_ms()` was called, the linger + /// comes from `KAFKA_LINGER_MS`; an invalid value is an error. + pub fn build(mut self) -> Result { + if !self.linger_set { + self.config.linger_ms = linger_ms_from_env()?; + } KafkaProducer::new(self.config) } } @@ -335,5 +401,30 @@ mod tests { assert_eq!(builder.config.client_id, "test-producer"); assert_eq!(builder.config.compression, "gzip"); assert_eq!(builder.config.batch_size, 32768); + assert!(!builder.linger_set); + assert!(builder.linger_ms(0).linger_set); + } + + #[test] + fn linger_ms_defaults_when_unset_or_blank() { + assert_eq!(parse_linger_ms(None).unwrap(), DEFAULT_LINGER_MS); + assert_eq!(parse_linger_ms(Some("")).unwrap(), DEFAULT_LINGER_MS); + assert_eq!(parse_linger_ms(Some(" ")).unwrap(), DEFAULT_LINGER_MS); + assert_eq!(ProducerConfig::default().linger_ms, DEFAULT_LINGER_MS); + } + + #[test] + fn linger_ms_accepts_the_range() { + assert_eq!(parse_linger_ms(Some("0")).unwrap(), 0); + assert_eq!(parse_linger_ms(Some(" 20 ")).unwrap(), 20); + assert_eq!(parse_linger_ms(Some("1000")).unwrap(), 1000); + } + + #[test] + fn linger_ms_rejects_invalid_values() { + for bad in ["1001", "-1", "5ms", "2.5", "abc"] { + let err = parse_linger_ms(Some(bad)).unwrap_err().to_string(); + assert!(err.contains("KAFKA_LINGER_MS"), "{bad}: {err}"); + } } } From 42c7cd9331133864d0cecfe15b2f68a866ea85d1 Mon Sep 17 00:00:00 2001 From: Quentin de Quelen Date: Thu, 12 Nov 2026 13:38:46 +0100 Subject: [PATCH 07/10] perf(bins): use mimalloc as the global allocator of the binaries No binary set a global allocator, and the workload allocates heavily (html5ever DOMs, String copies, JSON). Each binary crate (scrapix, scrapix-api, scrapix-frontier-service, scrapix-worker-crawler, scrapix-worker-content, scrapix-cli) gets a `mimalloc` cargo feature, on by default, that sets `#[global_allocator]` in its main.rs; `--no-default-features` keeps the system allocator. The unified `scrapix` binary takes the service crates with `default-features = false`, so only its own feature decides, and `cargo build -p scrapix --features browser` gets the allocator. scrapix-mcp (a thin stdio proxy) is unchanged. Measured on the scrapix-parser criterion benches, system allocator vs mimalloc, interleaved rounds, minimum of the per-round medians (mimalloc / system): - Linux arm64 container (rust:1-bookworm, glibc), 6 rounds: html_parsing large 0.97, very large 0.97; parse_only 0.96; parse_and_markdown 0.92; parse_markdown_language 0.92; markdown_conversion large 0.90; content_extraction 0.94-0.96; batch_100_mixed 0.93 (15.9 ms -> 14.8 ms) - macOS arm64, 10 rounds: html_parsing 0.98-0.99; parse_only 0.94; parse_and_markdown 0.92; markdown_conversion large 0.91; content_extraction 0.96-0.97; batch_100_mixed 0.98 The two fastest rounds of each side agree within 2%. The benches are single-threaded; production is Linux x86-64 with many threads, which was not measured. Cost: peak RSS of the whole bench run rose from 38-46 MB to 142-159 MB on Linux (36-50 MB to 113-162 MB on macOS): mimalloc keeps freed memory longer. Refs SCR-90 Co-Authored-By: Claude Fable 5.2 --- Cargo.lock | 24 +++++++++++++++++++++++ Cargo.toml | 3 +++ bins/scrapix-api/Cargo.toml | 8 ++++++++ bins/scrapix-api/src/main.rs | 4 ++++ bins/scrapix-cli/Cargo.toml | 8 ++++++++ bins/scrapix-cli/src/main.rs | 4 ++++ bins/scrapix-frontier-service/Cargo.toml | 8 ++++++++ bins/scrapix-frontier-service/src/main.rs | 4 ++++ bins/scrapix-worker-content/Cargo.toml | 8 ++++++++ bins/scrapix-worker-content/src/main.rs | 4 ++++ bins/scrapix-worker-crawler/Cargo.toml | 7 ++++++- bins/scrapix-worker-crawler/src/main.rs | 4 ++++ bins/scrapix/Cargo.toml | 19 ++++++++++++------ bins/scrapix/src/main.rs | 4 ++++ 14 files changed, 102 insertions(+), 7 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 006951bc..234f14da 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3516,6 +3516,15 @@ version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f9fbbcab51052fe104eb5e5d351cf728d30a5be1fe14d9be8a3b097481fb97de" +[[package]] +name = "libmimalloc-sys" +version = "0.1.49" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a45a52f43e1c16f667ccfe4dd8c85b7f7c204fd5e3bf46c5b0db9a5c3c0b8e9" +dependencies = [ + "cc", +] + [[package]] name = "libredox" version = "0.1.12" @@ -3828,6 +3837,15 @@ version = "2.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f52b00d39961fc5b2736ea853c9cc86238e165017a493d1d5c8eac6bdc4cc273" +[[package]] +name = "mimalloc" +version = "0.1.52" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d4139bb28d14ad1facf21d5eb8825051b326e172d216b39f6d31df53cc97862" +dependencies = [ + "libmimalloc-sys", +] + [[package]] name = "mime" version = "0.3.17" @@ -5502,6 +5520,7 @@ dependencies = [ "anyhow", "axum", "clap", + "mimalloc", "reqwest 0.12.28", "scrapix-api", "scrapix-cli", @@ -5553,6 +5572,7 @@ dependencies = [ "hex", "hmac", "jsonschema", + "mimalloc", "parking_lot", "regex", "reqwest 0.12.28", @@ -5621,6 +5641,7 @@ dependencies = [ "dirs", "futures", "indicatif", + "mimalloc", "reqwest 0.12.28", "scraper", "scrapix-core", @@ -5735,6 +5756,7 @@ dependencies = [ "chrono", "clap", "futures", + "mimalloc", "parking_lot", "scrapix-core", "scrapix-frontier", @@ -5917,6 +5939,7 @@ dependencies = [ "chrono", "clap", "hex", + "mimalloc", "parking_lot", "scrapix-ai", "scrapix-core", @@ -5946,6 +5969,7 @@ dependencies = [ "base64 0.22.1", "chrono", "clap", + "mimalloc", "parking_lot", "regex", "scrapix-core", diff --git a/Cargo.toml b/Cargo.toml index ba826ddb..acd4fa57 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -77,6 +77,9 @@ rdkafka = { version = "0.38", features = ["cmake-build", "ssl"] } # Object storage (S3-compatible: RustFS, MinIO, AWS S3) # Using rust-s3 crate directly in scrapix-storage +# Global allocator of the binaries (feature `mimalloc`, on by default) +mimalloc = "0.1" + # Bloom filter bloomfilter = "1.0" probabilistic-collections = "0.7" diff --git a/bins/scrapix-api/Cargo.toml b/bins/scrapix-api/Cargo.toml index 2e0e8e33..e2453986 100644 --- a/bins/scrapix-api/Cargo.toml +++ b/bins/scrapix-api/Cargo.toml @@ -14,7 +14,15 @@ path = "src/lib.rs" name = "scrapix-api" path = "src/main.rs" +[features] +default = ["mimalloc"] +# `mimalloc`: use mimalloc as the binary's global allocator (set in +# `src/main.rs`). On by default; build with `--no-default-features` for the +# system allocator. +mimalloc = ["dep:mimalloc"] + [dependencies] +mimalloc = { workspace = true, optional = true } scrapix-core = { path = "../../crates/scrapix-core" } scrapix-auth = { path = "../../crates/scrapix-auth" } scrapix-queue = { path = "../../crates/scrapix-queue" } diff --git a/bins/scrapix-api/src/main.rs b/bins/scrapix-api/src/main.rs index e072c421..4f316aeb 100644 --- a/bins/scrapix-api/src/main.rs +++ b/bins/scrapix-api/src/main.rs @@ -2,6 +2,10 @@ use clap::Parser; use scrapix_api::Args; +#[cfg(feature = "mimalloc")] +#[global_allocator] +static GLOBAL: mimalloc::MiMalloc = mimalloc::MiMalloc; + #[tokio::main] async fn main() -> anyhow::Result<()> { let args = Args::parse(); diff --git a/bins/scrapix-cli/Cargo.toml b/bins/scrapix-cli/Cargo.toml index 3a89f0f0..b09f8337 100644 --- a/bins/scrapix-cli/Cargo.toml +++ b/bins/scrapix-cli/Cargo.toml @@ -17,7 +17,15 @@ path = "src/lib.rs" name = "scrapix-cli" path = "src/main.rs" +[features] +default = ["mimalloc"] +# `mimalloc`: use mimalloc as the binary's global allocator (set in +# `src/main.rs`). On by default; build with `--no-default-features` for the +# system allocator. +mimalloc = ["dep:mimalloc"] + [dependencies] +mimalloc = { workspace = true, optional = true } scrapix-core = { path = "../../crates/scrapix-core" } scrapix-parser = { path = "../../crates/scrapix-parser" } scrapix-extractor = { path = "../../crates/scrapix-extractor" } diff --git a/bins/scrapix-cli/src/main.rs b/bins/scrapix-cli/src/main.rs index a5142bf7..1055f2ae 100644 --- a/bins/scrapix-cli/src/main.rs +++ b/bins/scrapix-cli/src/main.rs @@ -2,6 +2,10 @@ use clap::Parser; use scrapix_cli::Cli; +#[cfg(feature = "mimalloc")] +#[global_allocator] +static GLOBAL: mimalloc::MiMalloc = mimalloc::MiMalloc; + #[tokio::main] async fn main() { let cli = Cli::parse(); diff --git a/bins/scrapix-frontier-service/Cargo.toml b/bins/scrapix-frontier-service/Cargo.toml index a22a0301..b39da0f0 100644 --- a/bins/scrapix-frontier-service/Cargo.toml +++ b/bins/scrapix-frontier-service/Cargo.toml @@ -14,7 +14,15 @@ path = "src/lib.rs" name = "scrapix-frontier-service" path = "src/main.rs" +[features] +default = ["mimalloc"] +# `mimalloc`: use mimalloc as the binary's global allocator (set in +# `src/main.rs`). On by default; build with `--no-default-features` for the +# system allocator. +mimalloc = ["dep:mimalloc"] + [dependencies] +mimalloc = { workspace = true, optional = true } scrapix-core = { path = "../../crates/scrapix-core" } scrapix-frontier = { path = "../../crates/scrapix-frontier", features = ["redis-store"] } scrapix-queue = { path = "../../crates/scrapix-queue" } diff --git a/bins/scrapix-frontier-service/src/main.rs b/bins/scrapix-frontier-service/src/main.rs index a8b6a75d..dc36d42c 100644 --- a/bins/scrapix-frontier-service/src/main.rs +++ b/bins/scrapix-frontier-service/src/main.rs @@ -2,6 +2,10 @@ use clap::Parser; use scrapix_frontier_service::Args; +#[cfg(feature = "mimalloc")] +#[global_allocator] +static GLOBAL: mimalloc::MiMalloc = mimalloc::MiMalloc; + #[tokio::main] async fn main() -> anyhow::Result<()> { let args = Args::parse(); diff --git a/bins/scrapix-worker-content/Cargo.toml b/bins/scrapix-worker-content/Cargo.toml index 82affdf2..ddbec77e 100644 --- a/bins/scrapix-worker-content/Cargo.toml +++ b/bins/scrapix-worker-content/Cargo.toml @@ -14,7 +14,15 @@ path = "src/lib.rs" name = "scrapix-worker-content" path = "src/main.rs" +[features] +default = ["mimalloc"] +# `mimalloc`: use mimalloc as the binary's global allocator (set in +# `src/main.rs`). On by default; build with `--no-default-features` for the +# system allocator. +mimalloc = ["dep:mimalloc"] + [dependencies] +mimalloc = { workspace = true, optional = true } scrapix-core = { path = "../../crates/scrapix-core" } scrapix-extractor = { path = "../../crates/scrapix-extractor" } scrapix-frontier = { path = "../../crates/scrapix-frontier" } diff --git a/bins/scrapix-worker-content/src/main.rs b/bins/scrapix-worker-content/src/main.rs index dc5c551f..04a879af 100644 --- a/bins/scrapix-worker-content/src/main.rs +++ b/bins/scrapix-worker-content/src/main.rs @@ -2,6 +2,10 @@ use clap::Parser; use scrapix_worker_content::Args; +#[cfg(feature = "mimalloc")] +#[global_allocator] +static GLOBAL: mimalloc::MiMalloc = mimalloc::MiMalloc; + #[tokio::main] async fn main() -> anyhow::Result<()> { let args = Args::parse(); diff --git a/bins/scrapix-worker-crawler/Cargo.toml b/bins/scrapix-worker-crawler/Cargo.toml index ab1fb8a0..94d40bf0 100644 --- a/bins/scrapix-worker-crawler/Cargo.toml +++ b/bins/scrapix-worker-crawler/Cargo.toml @@ -15,10 +15,15 @@ name = "scrapix-worker-crawler" path = "src/main.rs" [features] -default = [] +default = ["mimalloc"] browser = ["scrapix-crawler/browser-cdp", "scrapix-crawler/browser-lightpanda"] +# `mimalloc`: use mimalloc as the binary's global allocator (set in +# `src/main.rs`). On by default; build with `--no-default-features` for the +# system allocator. +mimalloc = ["dep:mimalloc"] [dependencies] +mimalloc = { workspace = true, optional = true } scrapix-core = { path = "../../crates/scrapix-core" } scrapix-crawler = { path = "../../crates/scrapix-crawler" } # PDF link extraction (`features.pdf.extract_links`) diff --git a/bins/scrapix-worker-crawler/src/main.rs b/bins/scrapix-worker-crawler/src/main.rs index c845408d..e2baf2bb 100644 --- a/bins/scrapix-worker-crawler/src/main.rs +++ b/bins/scrapix-worker-crawler/src/main.rs @@ -2,6 +2,10 @@ use clap::Parser; use scrapix_worker_crawler::Args; +#[cfg(feature = "mimalloc")] +#[global_allocator] +static GLOBAL: mimalloc::MiMalloc = mimalloc::MiMalloc; + #[tokio::main] async fn main() -> anyhow::Result<()> { let args = Args::parse(); diff --git a/bins/scrapix/Cargo.toml b/bins/scrapix/Cargo.toml index 59bf6cd5..022e0b50 100644 --- a/bins/scrapix/Cargo.toml +++ b/bins/scrapix/Cargo.toml @@ -11,16 +11,23 @@ name = "scrapix" path = "src/main.rs" [features] -default = [] +default = ["mimalloc"] browser = ["scrapix-worker-crawler/browser"] +# `mimalloc`: use mimalloc as the binary's global allocator (set in +# `src/main.rs`). On by default; build with `--no-default-features` for the +# system allocator. +mimalloc = ["dep:mimalloc"] [dependencies] -scrapix-api = { path = "../scrapix-api" } -scrapix-worker-crawler = { path = "../scrapix-worker-crawler" } -scrapix-worker-content = { path = "../scrapix-worker-content" } -scrapix-frontier-service = { path = "../scrapix-frontier-service" } -scrapix-cli = { path = "../scrapix-cli" } +# The service crates are used as libraries here: their own `mimalloc` +# feature only concerns their standalone binaries. +scrapix-api = { path = "../scrapix-api", default-features = false } +scrapix-worker-crawler = { path = "../scrapix-worker-crawler", default-features = false } +scrapix-worker-content = { path = "../scrapix-worker-content", default-features = false } +scrapix-frontier-service = { path = "../scrapix-frontier-service", default-features = false } +scrapix-cli = { path = "../scrapix-cli", default-features = false } scrapix-queue = { path = "../../crates/scrapix-queue" } +mimalloc = { workspace = true, optional = true } tokio = { workspace = true } clap = { workspace = true } diff --git a/bins/scrapix/src/main.rs b/bins/scrapix/src/main.rs index 22954ca1..5de624a6 100644 --- a/bins/scrapix/src/main.rs +++ b/bins/scrapix/src/main.rs @@ -17,6 +17,10 @@ use clap::{Parser, Subcommand}; use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt}; /// Scrapix — high-performance web crawler and search indexer +#[cfg(feature = "mimalloc")] +#[global_allocator] +static GLOBAL: mimalloc::MiMalloc = mimalloc::MiMalloc; + #[derive(Parser, Debug)] #[command(name = "scrapix")] #[command(version, about = "Scrapix web crawler — unified binary")] From b4ba494c8b6bb3f2b07fd6833d1a15cb3ffe5ff9 Mon Sep 17 00:00:00 2001 From: Quentin de Quelen Date: Thu, 12 Nov 2026 13:44:43 +0100 Subject: [PATCH 08/10] perf(parser): strip tags and rewrite links without a regex pass per tag and per link MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Markdown conversion is most of a page's parse time. Around html2md, the conversion ran nine lazy regexes over the whole page to drop script, style, noscript, svg, head, nav, footer, header and aside, about seven more over the Markdown, and a capture regex per line to resolve links. The output is indexed and returned by /scrape, so it has to stay the same bytes: this only makes the same pipeline cheaper. - Tag stripping: the page is scanned once for the places where one of the nine names follows `<` or ` after): markdown_conversion/simple_html 6.3, 6.0 µs -> 5.1, 5.5 µs markdown_conversion/medium_html 36.6, 34.9 µs -> 32.1, 30.8 µs markdown_conversion/large_html/100p 868, 870 µs -> 787, 759 µs markdown_conversion/realistic_html/60s 901, 875 µs -> 732, 705 µs markdown_main_content/realistic_html_60s 957, 919 µs -> 648, 626 µs markdown_for_page/medium_html 40.5, 38.5 µs -> 28.6, 27.4 µs markdown_for_page/large_html_100p 1303, 1249 µs -> 838, 815 µs markdown_for_page/realistic_html_60s 1084, 1060 µs -> 688, 667 µs full_parsing_pipeline/parse_and_markdown 58.6, 56.8 µs -> 48.1, 46.1 µs html2md is now 84 to 88% of the conversion (52 to 66% before): what is left is inside the dependency. Refs SCR-90 Co-Authored-By: Claude Fable 5.2 --- .../benches/parser_benchmarks.rs | 137 +- crates/scrapix-parser/src/markdown.rs | 561 ++++++-- .../src/markdown/differential.rs | 1200 +++++++++++++++++ crates/scrapix-parser/src/markdown/legacy.rs | 369 +++++ 4 files changed, 2164 insertions(+), 103 deletions(-) create mode 100644 crates/scrapix-parser/src/markdown/differential.rs create mode 100644 crates/scrapix-parser/src/markdown/legacy.rs diff --git a/crates/scrapix-parser/benches/parser_benchmarks.rs b/crates/scrapix-parser/benches/parser_benchmarks.rs index b927b2b1..6811fa83 100644 --- a/crates/scrapix-parser/benches/parser_benchmarks.rs +++ b/crates/scrapix-parser/benches/parser_benchmarks.rs @@ -13,7 +13,10 @@ use std::collections::HashMap; use chrono::Utc; use criterion::{black_box, criterion_group, criterion_main, BenchmarkId, Criterion, Throughput}; use scrapix_core::RawPage; -use scrapix_parser::{detect_language, extract_content, html_to_markdown, HtmlParserBuilder}; +use scrapix_parser::{ + detect_language, extract_content, html_to_main_content_markdown, html_to_markdown, + html_to_markdown_for_page, HtmlParserBuilder, +}; // ============================================================================= // TEST HTML CONTENT @@ -149,6 +152,86 @@ fn generate_large_html(paragraphs: usize, links: usize) -> String { html } +/// A page shaped like a real site's: inline scripts and styles, an SVG logo, +/// a navigation bar, a sidebar and a footer around an article with links, +/// images, a table and code. The other inputs have no script or style, which +/// is where the tag stripping of the Markdown conversion works. +fn generate_realistic_html(sections: usize) -> String { + let mut html = String::from( + r#" + + + + Realistic Page + + + + + + + + + \n
\n
\n

Realistic Page Title

\n"); + for i in 0..sections { + html.push_str(&format!( + r#"

Section {i}

+

Paragraph {i} explains a relative topic, an + external reference and some + bold, italic and inline_code({i}) text. + Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod + tempor incididunt ut labore et dolore magna aliqua.

+ Figure {i} +"# + )); + if i % 4 == 0 { + html.push_str(&format!( + r#"
const x{i} = await fetch("/api/items?page={i}&size=10");
+if (x{i}.status < 400) {{ console.log("ok"); }}
+ + + + +
OptionDefaultDescription
retries{i}How many times to retry
timeout30000Request timeout in ms
+
  • First point of {i}
  • Second point with a link
+"# + )); + } + if i % 10 == 0 { + html.push_str(&format!( + " \n \n" + )); + } + } + html.push_str( + r#"
+
+ + + + + +"#, + ); + html +} + /// Create a RawPage from HTML content fn create_raw_page(html: &str) -> RawPage { RawPage { @@ -312,6 +395,57 @@ fn bench_markdown_conversion(c: &mut Criterion) { }, ); + // A page with scripts, styles, navigation and links + let realistic_html = generate_realistic_html(60); + group.throughput(Throughput::Bytes(realistic_html.len() as u64)); + group.bench_with_input( + BenchmarkId::new("realistic_html", "60s"), + &realistic_html, + |b, html| { + b.iter(|| { + black_box(html_to_markdown(html)); + }); + }, + ); + + group.finish(); +} + +/// The conversions the crawl and `/scrape` run: main content only, and main +/// content with links resolved against the page URL. +fn bench_markdown_for_page(c: &mut Criterion) { + let large_html = generate_large_html(100, 500); + let realistic_html = generate_realistic_html(60); + let inputs: [(&str, &str); 3] = [ + ("medium_html", MEDIUM_HTML), + ("large_html_100p", &large_html), + ("realistic_html_60s", &realistic_html), + ]; + + let mut group = c.benchmark_group("markdown_main_content"); + for (name, html) in inputs { + group.throughput(Throughput::Bytes(html.len() as u64)); + group.bench_function(name, |b| { + b.iter(|| { + black_box(html_to_main_content_markdown(html)); + }); + }); + } + group.finish(); + + let mut group = c.benchmark_group("markdown_for_page"); + for (name, html) in inputs { + group.throughput(Throughput::Bytes(html.len() as u64)); + group.bench_function(name, |b| { + b.iter(|| { + black_box(html_to_markdown_for_page( + html, + "https://example.com/docs/guide/page", + true, + )); + }); + }); + } group.finish(); } @@ -438,6 +572,7 @@ criterion_group!( criterion_group!( conversion_benches, bench_markdown_conversion, + bench_markdown_for_page, bench_content_extraction, bench_language_detection, ); diff --git a/crates/scrapix-parser/src/markdown.rs b/crates/scrapix-parser/src/markdown.rs index 5f0ae107..d05ce835 100644 --- a/crates/scrapix-parser/src/markdown.rs +++ b/crates/scrapix-parser/src/markdown.rs @@ -48,13 +48,14 @@ pub fn html_to_main_content_markdown(html: &str) -> String { /// Convert HTML to Markdown with custom configuration pub fn html_to_markdown_with_config(html: &str, config: &MarkdownConfig) -> String { - // Strip non-content tags (script, style, etc.) - let mut cleaned_html = strip_non_content_tags(html); - - // Optionally strip boilerplate sections - if config.only_main_content { - cleaned_html = strip_boilerplate_tags(cleaned_html); - } + // Strip non-content tags (script, style, etc.) and, optionally, the + // boilerplate sections + let stripped_tags = if config.only_main_content { + STRIPPED_TAGS.len() + } else { + NON_CONTENT_TAGS + }; + let cleaned_html = strip_tags(Cow::Borrowed(html), 0..stripped_tags); // Use html2md for the basic conversion let markdown = parse_html(&cleaned_html); @@ -154,7 +155,8 @@ pub fn absolutize_markdown_links(markdown: &str, base: &url::Url) -> String { } _ => {} } - if fence.is_some() { + // A link or an image has a `](`: a line without one is kept as it is. + if fence.is_some() || !line.contains("](") { out.push_str(line); continue; } @@ -164,8 +166,8 @@ pub fn absolutize_markdown_links(markdown: &str, base: &url::Url) -> String { if j > 0 { out.push('`'); } - if j % 2 == 0 { - out.push_str(&rewrite_links(segment, base)); + if j % 2 == 0 && segment.contains("](") { + push_rewritten_links(&mut out, segment, base); } else { out.push_str(segment); } @@ -174,89 +176,401 @@ pub fn absolutize_markdown_links(markdown: &str, base: &url::Url) -> String { out } -/// Rewrite the links and images of one line of markdown (no code in it). -fn rewrite_links(line: &str, base: &url::Url) -> String { +/// The pattern of a link or an image: (!?) image marker, [text] (which may +/// hold an image), (target "title"). +fn link_regex() -> &'static Regex { static LINK_RE: OnceLock = OnceLock::new(); - // (!?) image marker, [text] (which may hold an image), (target "title") - let link_re = LINK_RE.get_or_init(|| { + LINK_RE.get_or_init(|| { Regex::new( r#"(!?)\[((?:!\[[^\]]*\]\([^)]*\)|[^\]])*)\]\(\s*([^)\s]*)((?:\s+"[^"]*")?)\s*\)"#, ) .unwrap() - }); - link_re + }) +} + +/// Append one line of markdown (no code in it) to `out`, its links and +/// images rewritten. +/// +/// What is a link is decided by [`link_regex`], and the capture search of +/// the regex engine was a quarter of the conversion of a page with hundreds +/// of links. On a plain line the same matches are found by hand +/// ([`push_rewritten_plain_links`]); every other line goes through the +/// regex. +/// +/// A line is plain when it has no `"` (no link title) and either no `![` +/// (no image) or a single `[`. Then a link text cannot hold an image or a +/// `]`, and a target cannot hold a quoted `)`: each `[` has at most one way +/// to be a link, ending at the first `)` after its `](`, so there is one +/// set of matches whatever the order they are searched in. That matters: +/// on the other lines the regex engine (regex 1.12) does not always return +/// the leftmost match (in `[![](a [b](c)d](e)` it finds `[b](c)`), and the +/// Markdown has to stay what it was. +fn push_rewritten_links(out: &mut String, line: &str, base: &url::Url) { + let plain = !line.contains('"') + && (!line.contains("![") || line.bytes().filter(|&b| b == b'[').count() == 1); + if plain { + push_rewritten_plain_links(out, line, base); + } else { + out.push_str(&rewrite_links_with_regex(line, base)); + } +} + +/// [`push_rewritten_links`] for a plain line: `!?[text](target)`, where +/// the text runs to the first `]` and the target, with whitespace allowed +/// around it, to the `)`. +fn push_rewritten_plain_links(out: &mut String, line: &str, base: &url::Url) { + let bytes = line.as_bytes(); + let find = |byte: u8, from: usize| { + bytes[from..] + .iter() + .position(|&b| b == byte) + .map(|offset| from + offset) + }; + // Up to here the line is in `out`. + let mut copied = 0; + let mut at = 0; + // Where a target that was not followed by `)` ended: a later target + // reaching it ends the same way. + let mut dead_target_end = 0; + while let Some(open) = find(b'[', at) { + // The text of a link at `open`, and of every `[` inside it, ends + // at the first `]`: they all are links or none is. + let Some(close) = find(b']', open + 1) else { + break; + }; + at = close + 1; + if bytes.get(close + 1) != Some(&b'(') { + continue; + } + let target_start = skip_whitespace(line, close + 2); + if target_start < dead_target_end { + continue; + } + let rest = &line[target_start..]; + let target_end = target_start + + rest + .find(|c: char| c == ')' || c.is_whitespace()) + .unwrap_or(rest.len()); + let closing = skip_whitespace(line, target_end); + if bytes.get(closing) != Some(&b')') { + dead_target_end = target_end; + continue; + } + + let is_image = open > copied && bytes[open - 1] == b'!'; + let start = open - usize::from(is_image); + let text = &line[open + 1..close]; + let target = &line[target_start..target_end]; + out.push_str(&line[copied..start]); + if !target.is_empty() { + out.push_str(&line[start..=open]); + out.push_str(text); + out.push_str("]("); + push_resolved_target(out, target, base); + out.push(')'); + } else if !is_image { + // A link with no target is replaced by its text, an image + // with none is removed. + out.push_str(text); + } + copied = closing + 1; + at = copied; + } + out.push_str(&line[copied..]); +} + +/// The offset after the whitespace at `at`. +fn skip_whitespace(line: &str, at: usize) -> usize { + line.len() - line[at..].trim_start().len() +} + +/// Append `target` to `out`, resolved against `base` unless it is absolute +/// or a fragment. +fn push_resolved_target(out: &mut String, target: &str, base: &url::Url) { + // An absolute URL has a scheme, so a `:`: without one the target is + // relative and parsing it alone can only fail. + let is_absolute = + target.starts_with('#') || (target.contains(':') && url::Url::parse(target).is_ok()); + if is_absolute { + out.push_str(target); + return; + } + // `//cdn/x.png` is relative to the scheme: `join` handles it. + match base.join(target) { + Ok(resolved) => out.push_str(resolved.as_str()), + Err(_) => out.push_str(target), + } +} + +/// Rewrite the links and images of one line of markdown (no code in it) +/// with [`link_regex`]. +fn rewrite_links_with_regex(line: &str, base: &url::Url) -> String { + link_regex() .replace_all(line, |caps: ®ex::Captures| { let is_image = !caps[1].is_empty(); // The text of a link may be an image: rewrite it too. - let text = if is_image { + let text = if is_image || !caps[2].contains("](") { caps[2].to_string() } else { - rewrite_links(&caps[2], base) + rewrite_links_with_regex(&caps[2], base) }; let target = &caps[3]; let title = &caps[4]; if target.is_empty() { return if is_image { String::new() } else { text }; } - // `//cdn/x.png` is relative to the scheme: `join` handles it. - let is_absolute = target.starts_with('#') || url::Url::parse(target).is_ok(); - let resolved = if is_absolute { - target.to_string() - } else { - base.join(target) - .map(|u| u.to_string()) - .unwrap_or_else(|_| target.to_string()) - }; + let mut resolved = String::new(); + push_resolved_target(&mut resolved, target, base); format!("{}[{}]({}{})", &caps[1], text, resolved, title) }) .into_owned() } +/// Tags removed with their content, in the order they are removed: the +/// first [`NON_CONTENT_TAGS`] from every page, the rest (nav, footer, +/// header, aside) when only the main content is wanted. +const STRIPPED_TAGS: [&str; 9] = [ + "script", "style", "noscript", "svg", "head", "nav", "footer", "header", "aside", +]; +/// How many of [`STRIPPED_TAGS`] are removed from every page. +const NON_CONTENT_TAGS: usize = 5; + /// Strip script, style, noscript, svg, and head tags (and their content) from HTML. +#[cfg(test)] fn strip_non_content_tags(html: &str) -> Cow<'_, str> { - static PATTERNS: OnceLock> = OnceLock::new(); - let patterns = PATTERNS.get_or_init(|| { - ["script", "style", "noscript", "svg", "head"] - .iter() - .map(|tag| Regex::new(&format!(r"(?si)<{tag}[\s>].*?")).unwrap()) - .collect() - }); - strip_matches(Cow::Borrowed(html), patterns) + strip_tags(Cow::Borrowed(html), 0..NON_CONTENT_TAGS) } /// Strip boilerplate HTML sections: nav, header, footer, aside. +#[cfg(test)] fn strip_boilerplate_tags(html: Cow<'_, str>) -> Cow<'_, str> { + strip_tags(html, NON_CONTENT_TAGS..STRIPPED_TAGS.len()) +} + +/// The regex a tag's blocks are defined by. +fn tag_block_regex(tag: usize) -> &'static Regex { static PATTERNS: OnceLock> = OnceLock::new(); - let patterns = PATTERNS.get_or_init(|| { - ["nav", "footer", "header", "aside"] + &PATTERNS.get_or_init(|| { + STRIPPED_TAGS .iter() .map(|tag| Regex::new(&format!(r"(?si)<{tag}[\s>].*?")).unwrap()) .collect() - }); - strip_matches(html, patterns) + })[tag] } -/// Remove every match of each pattern, one pattern after the other (a -/// later pattern sees the text without the earlier ones' matches). The text -/// is only copied by a pattern that matches. -fn strip_matches<'a>(mut text: Cow<'a, str>, patterns: &[Regex]) -> Cow<'a, str> { - for re in patterns { - let stripped = match re.replace_all(&text, "") { - Cow::Owned(stripped) => Some(stripped), - Cow::Borrowed(_) => None, - }; - if let Some(stripped) = stripped { +/// Remove every `…` block of the tags `STRIPPED_TAGS[tags]`, +/// one tag after the other: a later tag sees the text without the earlier +/// ones' blocks, which is what makes ` c` +/// come out as it does. The text is only copied by a tag that has a block. +/// +/// A block is what the regex `(?si)].*?` matches. Running +/// nine such regexes over the page was a fifth of the conversion, so the +/// page is scanned once for the places where one of the nine names follows +/// a `<` or a `, tags: std::ops::Range) -> Cow<'_, str> { + if text.contains('\u{17f}') { + for tag in tags { + let stripped = match tag_block_regex(tag).replace_all(&text, "") { + Cow::Owned(stripped) => Some(stripped), + Cow::Borrowed(_) => None, + }; + if let Some(stripped) = stripped { + text = Cow::Owned(stripped); + } + } + return text; + } + + let mut marks = tag_marks(text.as_bytes()); + let mut blocks = Vec::new(); + for tag in tags { + if marks.is_empty() { + break; + } + blocks.clear(); + find_tag_blocks(&text, &marks, tag, &mut blocks); + if !blocks.is_empty() { + let stripped = remove_blocks(&text, &blocks); + remap_marks(&mut marks, &blocks, stripped.as_bytes()); + debug_assert_eq!(marks, tag_marks(stripped.as_bytes())); text = Cow::Owned(stripped); } } text } +/// A `<` or ` Option { + let closing = bytes.get(lt + 1) == Some(&b'/'); + let name = lt + 1 + usize::from(closing); + // Every stripped tag starts with one of these letters. + if !matches!( + bytes.get(name).map(u8::to_ascii_lowercase), + Some(b's' | b'n' | b'h' | b'f' | b'a') + ) { + return None; + } + let mut tags = 0u16; + for (i, tag) in STRIPPED_TAGS.iter().enumerate() { + let named = bytes + .get(name..name + tag.len()) + .is_some_and(|found| found.eq_ignore_ascii_case(tag.as_bytes())); + if named { + tags |= 1 << i; + } + } + (tags != 0).then_some(TagMark { lt, closing, tags }) +} + +/// The marks of a text, in order. +fn tag_marks(bytes: &[u8]) -> Vec { + let mut marks = Vec::new(); + for_each_byte(bytes, b'<', |lt| marks.extend(tag_mark(bytes, lt))); + marks +} + +/// Call `found` with the offset of every `byte` of `bytes`, in order. +/// Eight bytes at a time: a page has a `<` every few dozen bytes, where a +/// search restarted after each one spends its time starting. +fn for_each_byte(bytes: &[u8], byte: u8, mut found: impl FnMut(usize)) { + const LOW_BITS: u64 = 0x7f7f_7f7f_7f7f_7f7f; + let pattern = u64::from_ne_bytes([byte; 8]); + let (chunks, remainder) = bytes.as_chunks::<8>(); + let mut offset = 0; + for chunk in chunks { + // Little-endian, so the lowest byte of the word is the first one. + let word = u64::from_le_bytes(*chunk) ^ pattern; + // The high bit of each byte of `word` that is zero, exactly. + let mut zeros = !(((word & LOW_BITS) + LOW_BITS) | word | LOW_BITS); + while zeros != 0 { + found(offset + (zeros.trailing_zeros() / 8) as usize); + zeros &= zeros - 1; + } + offset += 8; + } + for (i, &b) in remainder.iter().enumerate() { + if b == byte { + found(offset + i); + } + } +} + +/// The blocks of `STRIPPED_TAGS[tag]` in `text`, as the regex +/// `(?si)].*?` finds them one after the other: the +/// leftmost ``, up to the first `` after it, then again from there. With no +/// closing tag after an opening one there is none after a later one +/// either, so the search is over. +fn find_tag_blocks(text: &str, marks: &[TagMark], tag: usize, blocks: &mut Vec<(usize, usize)>) { + let name_len = STRIPPED_TAGS[tag].len(); + let bit = 1u16 << tag; + let mut marks = marks.iter().filter(|mark| mark.tags & bit != 0); + let mut from = 0; + loop { + let Some(start) = marks.by_ref().find_map(|mark| { + let opens = !mark.closing + && mark.lt >= from + && text[mark.lt + 1 + name_len..] + .starts_with(|c: char| c == '>' || c.is_whitespace()); + opens.then_some(mark.lt) + }) else { + return; + }; + // No `<` sits between an opening tag's `<` and the end of its + // name and delimiter, so every later mark is after them. + let Some(end) = marks.by_ref().find_map(|mark| { + if !mark.closing { + return None; + } + let after_name = text[mark.lt + 2 + name_len..].trim_start(); + after_name + .starts_with('>') + .then(|| text.len() - after_name.len() + 1) + }) else { + return; + }; + blocks.push((start, end)); + from = end; + } +} + +/// `text` without `blocks` (sorted, not overlapping, not empty). +fn remove_blocks(text: &str, blocks: &[(usize, usize)]) -> String { + let mut out = String::with_capacity(text.len()); + let mut kept_from = 0; + for &(start, end) in blocks { + out.push_str(&text[kept_from..start]); + kept_from = end; + } + out.push_str(&text[kept_from..]); + out +} + +/// Bring `marks` from a text to that text without `blocks`, whose bytes are +/// `stripped`: the marks inside a block go, the ones after it move, and the +/// `<` just before the place a block was are read again, because the text +/// that now follows them can complete a name (`v>`). +fn remap_marks(marks: &mut Vec, blocks: &[(usize, usize)], stripped: &[u8]) { + let mut remapped = Vec::with_capacity(marks.len()); + let mut old = marks.iter().peekable(); + let mut removed = 0; + // Where the previous block was, in `stripped`. + let mut reread_from = 0; + for &(start, end) in blocks { + let seam = start - removed; + let window = seam.saturating_sub(TAG_MARK_REACH).max(reread_from); + // Marks before the window are untouched by this block. + while let Some(mark) = old.next_if(|mark| mark.lt < start) { + let lt = mark.lt - removed; + if lt < window { + remapped.push(TagMark { lt, ..*mark }); + } + } + for lt in window..seam { + if stripped[lt] == b'<' { + remapped.extend(tag_mark(stripped, lt)); + } + } + while old.next_if(|mark| mark.lt < end).is_some() {} + removed += end - start; + reread_from = seam; + } + remapped.extend(old.map(|mark| TagMark { + lt: mark.lt - removed, + ..*mark + })); + *marks = remapped; +} + /// Strip any remaining HTML tags from markdown output. /// /// Converts `` tags to `![alt](src)` before stripping, so image info is preserved. /// All other HTML tags are removed, keeping only their text content. -fn strip_remaining_html(markdown: &str) -> String { +fn strip_remaining_html(markdown: &str) -> Cow<'_, str> { + // Both patterns need a `<` and a `>`: most Markdown has neither. + if !markdown.contains('<') || !markdown.contains('>') { + return Cow::Borrowed(markdown); + } + // Convert tags to markdown images: ![alt](src) static IMG_RE: OnceLock = OnceLock::new(); let img_re = IMG_RE.get_or_init(|| Regex::new(r#"(?i)]*?>"#).unwrap()); @@ -288,37 +602,72 @@ fn strip_remaining_html(markdown: &str) -> String { static TAG_RE: OnceLock = OnceLock::new(); let tag_re = TAG_RE.get_or_init(|| Regex::new(r"<[^>]+>").unwrap()); - tag_re.replace_all(&result, "").into_owned() + let stripped = match tag_re.replace_all(&result, "") { + Cow::Owned(stripped) => Some(stripped), + Cow::Borrowed(_) => None, + }; + match stripped { + Some(stripped) => Cow::Owned(stripped), + None => result, + } +} + +/// Whether some `[` is directly followed by whitespace, which a link text +/// spread over several lines starts with. +fn has_bracket_then_whitespace(markdown: &str) -> bool { + let mut pos = 0; + while let Some(offset) = markdown[pos..].find('[') { + pos += offset + 1; + if markdown[pos..].starts_with(char::is_whitespace) { + return true; + } + } + false } /// Clean up the markdown output fn clean_markdown(markdown: &str) -> String { - // First pass: clean up multiline link/image syntax - // [\n\nText\n\n](url) → [Text](url) - // ![\n\nAlt\n\n](url) → ![Alt](url) - static MULTILINE_LINK_RE: OnceLock = OnceLock::new(); - let multiline_link_re = MULTILINE_LINK_RE - .get_or_init(|| Regex::new(r"(!?\[)\s*\n\s*(.*?)\s*\n\s*\]\(([^)]*)\)").unwrap()); - let markdown = multiline_link_re.replace_all(markdown, "$1$2]($3)"); - - // Remove empty links: [](url) - static EMPTY_LINK_RE: OnceLock = OnceLock::new(); - let empty_link_re = EMPTY_LINK_RE.get_or_init(|| Regex::new(r"\[]\([^)]*\)").unwrap()); - let markdown = empty_link_re.replace_all(&markdown, ""); - - // Remove images with relative/asset URLs (decorative) - static DECORATIVE_IMG_RE: OnceLock = OnceLock::new(); - let decorative_img_re = - DECORATIVE_IMG_RE.get_or_init(|| Regex::new(r"!\[[^\]]*\]\(/_next/[^)]*\)").unwrap()); - let markdown = decorative_img_re.replace_all(&markdown, ""); - - // Remove images with no alt text - static NO_ALT_IMG_RE: OnceLock = OnceLock::new(); - let no_alt_img_re = NO_ALT_IMG_RE.get_or_init(|| Regex::new(r"!\[\]\([^)]*\)").unwrap()); - let markdown = no_alt_img_re.replace_all(&markdown, ""); + let mut markdown = Cow::Borrowed(markdown); + + // First pass: fix up links and images. Each of the four patterns needs + // a `](`, and each is skipped when the text it would see cannot match. + if markdown.contains("](") { + // Clean up multiline link/image syntax + // [\n\nText\n\n](url) → [Text](url) + // ![\n\nAlt\n\n](url) → ![Alt](url) + static MULTILINE_LINK_RE: OnceLock = OnceLock::new(); + if has_bracket_then_whitespace(&markdown) { + let multiline_link_re = MULTILINE_LINK_RE + .get_or_init(|| Regex::new(r"(!?\[)\s*\n\s*(.*?)\s*\n\s*\]\(([^)]*)\)").unwrap()); + replace_in_place(&mut markdown, multiline_link_re, "$1$2]($3)"); + } + + // Remove empty links: [](url) + static EMPTY_LINK_RE: OnceLock = OnceLock::new(); + if markdown.contains("[](") { + let empty_link_re = EMPTY_LINK_RE.get_or_init(|| Regex::new(r"\[]\([^)]*\)").unwrap()); + replace_in_place(&mut markdown, empty_link_re, ""); + } + + // Remove images with relative/asset URLs (decorative) + static DECORATIVE_IMG_RE: OnceLock = OnceLock::new(); + if markdown.contains("](/_next/") { + let decorative_img_re = DECORATIVE_IMG_RE + .get_or_init(|| Regex::new(r"!\[[^\]]*\]\(/_next/[^)]*\)").unwrap()); + replace_in_place(&mut markdown, decorative_img_re, ""); + } + + // Remove images with no alt text + static NO_ALT_IMG_RE: OnceLock = OnceLock::new(); + if markdown.contains("![](") { + let no_alt_img_re = + NO_ALT_IMG_RE.get_or_init(|| Regex::new(r"!\[\]\([^)]*\)").unwrap()); + replace_in_place(&mut markdown, no_alt_img_re, ""); + } + } // Second pass: line-by-line cleanup - let mut result = String::new(); + let mut result = String::with_capacity(markdown.len()); let mut prev_blank = false; let mut in_code_block = false; @@ -350,43 +699,46 @@ fn clean_markdown(markdown: &str) -> String { prev_blank = false; // Clean up the line - let cleaned = clean_line(trimmed); - if !cleaned.is_empty() { - result.push_str(&cleaned); - result.push('\n'); - } + push_clean_line(&mut result, trimmed); + result.push('\n'); } // Trim trailing whitespace - result.trim_end().to_string() + let trimmed_len = result.trim_end().len(); + result.truncate(trimmed_len); + result } -/// Clean a single line of markdown -fn clean_line(line: &str) -> String { - // Collapse consecutive spaces in a single pass - O(n) with one allocation - let mut result = String::with_capacity(line.len()); - let mut prev_space = false; - for ch in line.chars() { - if ch == ' ' { - if !prev_space { - result.push(' '); - } - prev_space = true; - } else { - result.push(ch); - prev_space = false; - } +/// Replace every match of `re` in `text`, copying it only when one matches. +fn replace_in_place(text: &mut Cow<'_, str>, re: &Regex, replacement: &str) { + let replaced = match re.replace_all(text, replacement) { + Cow::Owned(replaced) => Some(replaced), + Cow::Borrowed(_) => None, + }; + if let Some(replaced) = replaced { + *text = Cow::Owned(replaced); } +} +/// Append one cleaned line of markdown to `out`: runs of spaces collapsed +/// to one, and a space after the `#`s of a heading that lacks it. `line` is +/// trimmed and not empty. +fn push_clean_line(out: &mut String, line: &str) { // Fix heading spacing - if result.starts_with('#') { - let hash_end = result.find(|c: char| c != '#').unwrap_or(result.len()); - if hash_end < result.len() && !result[hash_end..].starts_with(' ') { - result.insert(hash_end, ' '); - } + let hashes = line.len() - line.trim_start_matches('#').len(); + let mut rest = line; + if hashes > 0 && hashes < line.len() && line.as_bytes()[hashes] != b' ' { + out.push_str(&line[..hashes]); + out.push(' '); + rest = &line[hashes..]; } - result + // Collapse consecutive spaces + while let Some(i) = rest.find(" ") { + out.push_str(&rest[..=i]); + rest = rest[i + 1..].trim_start_matches(' '); + } + out.push_str(rest); } /// Extract plain text from markdown (strip all formatting) @@ -466,6 +818,11 @@ pub fn markdown_to_text(markdown: &str) -> String { result } +#[cfg(test)] +mod differential; +#[cfg(test)] +mod legacy; + #[cfg(test)] mod tests { use super::*; diff --git a/crates/scrapix-parser/src/markdown/differential.rs b/crates/scrapix-parser/src/markdown/differential.rs new file mode 100644 index 00000000..01af3e31 --- /dev/null +++ b/crates/scrapix-parser/src/markdown/differential.rs @@ -0,0 +1,1200 @@ +//! Differential tests of the Markdown conversion against [`super::legacy`], +//! the implementation before SCR-90. +//! +//! The Markdown is indexed and returned by `/scrape`: a faster conversion +//! must give the same bytes, or a re-crawl rewrites every document. Every +//! public entry point and every internal stage is run through both +//! implementations over a corpus of +//! +//! - the HTML string literals of the workspace's Rust sources (tests, +//! benches, fixtures of every crate), +//! - hand-written pages around the tags the conversion strips, +//! - seeded random pages built from the same vocabulary, +//! - the bench inputs. + +use super::legacy; +use super::*; +use std::collections::BTreeSet; +use std::path::{Path, PathBuf}; + +// ============================================================================= +// Corpus +// ============================================================================= + +/// A small deterministic generator (xorshift64*), so the random pages are +/// the same on every run. +struct Rng(u64); + +impl Rng { + fn next(&mut self) -> u64 { + self.0 ^= self.0 >> 12; + self.0 ^= self.0 << 25; + self.0 ^= self.0 >> 27; + self.0.wrapping_mul(0x2545_F491_4F6C_DD1D) + } + + fn below(&mut self, n: usize) -> usize { + (self.next() % n as u64) as usize + } + + fn pick<'a>(&mut self, items: &[&'a str]) -> &'a str { + items[self.below(items.len())] + } +} + +/// Tags the conversion strips, tags it could confuse with them, and other +/// containers. +const TAG_NAMES: &[&str] = &[ + "script", + "style", + "noscript", + "svg", + "head", + "nav", + "footer", + "header", + "aside", + "iframe", + "form", + "SCRIPT", + "Style", + "NoScript", + "SVG", + "HEAD", + "Nav", + "FOOTER", + "Header", + "ASIDE", + "scripty", + "styles", + "navigation", + "headers", + "heading", + "footers", + "asides", + "svgs", + "\u{17f}cript", + "\u{17f}tyle", + "a\u{17f}ide", + "div", + "main", + "article", + "section", + "p", + "span", +]; + +/// What follows a tag name in an opening tag. +const OPEN_TAILS: &[&str] = &[ + ">", + ">", + ">", + " >", + " class=\"x\">", + "\n>", + "\t id='a'>", + " data-x=\"