diff --git a/workers/server/.sqlx/query-117a7f3d3d7a7f6a1127261838e8a1d9c01a6aced7b469094dd913a2820e271e.json b/workers/server/.sqlx/query-117a7f3d3d7a7f6a1127261838e8a1d9c01a6aced7b469094dd913a2820e271e.json new file mode 100644 index 0000000..8d27e99 --- /dev/null +++ b/workers/server/.sqlx/query-117a7f3d3d7a7f6a1127261838e8a1d9c01a6aced7b469094dd913a2820e271e.json @@ -0,0 +1,30 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT held.ok AS \"held!\",(SELECT count(*) FROM (SELECT FROM upload_sessions WHERE held.ok AND id=ANY($1) ORDER BY id FOR NO KEY UPDATE) locked) AS locked FROM (SELECT COALESCE(bool_and(pg_try_advisory_xact_lock(hashtextextended(key,0))),TRUE) AS ok FROM (SELECT DISTINCT 'upload:'||id AS key FROM UNNEST($1::text[]) AS id ORDER BY key OFFSET 0) ordered) held", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "held!", + "type_info": "Bool", + "origin": "Expression" + }, + { + "ordinal": 1, + "name": "locked", + "type_info": "Int8", + "origin": "Expression" + } + ], + "parameters": { + "Left": [ + "TextArray" + ] + }, + "nullable": [ + null, + null + ] + }, + "hash": "117a7f3d3d7a7f6a1127261838e8a1d9c01a6aced7b469094dd913a2820e271e" +} diff --git a/workers/server/.sqlx/query-39245cbe296e63ec7fafe36015a81777e0ed18f49651d363c3b96437132258d3.json b/workers/server/.sqlx/query-39245cbe296e63ec7fafe36015a81777e0ed18f49651d363c3b96437132258d3.json new file mode 100644 index 0000000..b4a80f0 --- /dev/null +++ b/workers/server/.sqlx/query-39245cbe296e63ec7fafe36015a81777e0ed18f49651d363c3b96437132258d3.json @@ -0,0 +1,61 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT s.staged AS \"staged!\",p.session_id AS \"session_id?\",p.part_number AS \"part_number?\",p.size AS \"size?\" FROM (SELECT COALESCE(SUM(u.file_size),0)::bigint AS staged FROM upload_sessions u JOIN caches c ON c.id=u.cache_id WHERE c.org_id=$3 AND u.expires_at>NOW() AND u.status IN ('streaming','finalizing')) s LEFT JOIN (SELECT p.session_id,p.part_number,p.size FROM upload_parts p JOIN UNNEST($1::text[],$2::integer[]) r(session_id,part_number) USING(session_id,part_number)) p ON TRUE", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "staged!", + "type_info": "Int8", + "origin": "Expression" + }, + { + "ordinal": 1, + "name": "session_id?", + "type_info": "Text", + "origin": { + "Table": { + "table": "upload_parts", + "name": "session_id" + } + } + }, + { + "ordinal": 2, + "name": "part_number?", + "type_info": "Int4", + "origin": { + "Table": { + "table": "upload_parts", + "name": "part_number" + } + } + }, + { + "ordinal": 3, + "name": "size?", + "type_info": "Int8", + "origin": { + "Table": { + "table": "upload_parts", + "name": "size" + } + } + } + ], + "parameters": { + "Left": [ + "TextArray", + "Int4Array", + "Text" + ] + }, + "nullable": [ + null, + true, + true, + true + ] + }, + "hash": "39245cbe296e63ec7fafe36015a81777e0ed18f49651d363c3b96437132258d3" +} diff --git a/workers/server/.sqlx/query-53f59f4552bc7c9b0a40a95327a62e99089022bc39ae6832e6391fb06f916117.json b/workers/server/.sqlx/query-53f59f4552bc7c9b0a40a95327a62e99089022bc39ae6832e6391fb06f916117.json new file mode 100644 index 0000000..1807e49 --- /dev/null +++ b/workers/server/.sqlx/query-53f59f4552bc7c9b0a40a95327a62e99089022bc39ae6832e6391fb06f916117.json @@ -0,0 +1,31 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT e.prepare_only AS existing,(SELECT TRUE FROM upload_finalization_batches WHERE e.prepare_only IS NULL AND id=$2 AND NOT admitted FOR SHARE) AS open FROM (SELECT (SELECT b.prepare_only FROM upload_finalizations f JOIN upload_finalization_batches b ON b.id=f.batch_id WHERE f.session_id=$1) AS prepare_only OFFSET 0) e", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "existing", + "type_info": "Bool", + "origin": "Expression" + }, + { + "ordinal": 1, + "name": "open", + "type_info": "Bool", + "origin": "Expression" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + null, + null + ] + }, + "hash": "53f59f4552bc7c9b0a40a95327a62e99089022bc39ae6832e6391fb06f916117" +} diff --git a/workers/server/.sqlx/query-c9fed8dec34c38292cff2db8577a2fd5abc24a4458947ff3aaba5c70823c0d23.json b/workers/server/.sqlx/query-c9fed8dec34c38292cff2db8577a2fd5abc24a4458947ff3aaba5c70823c0d23.json new file mode 100644 index 0000000..66e944a --- /dev/null +++ b/workers/server/.sqlx/query-c9fed8dec34c38292cff2db8577a2fd5abc24a4458947ff3aaba5c70823c0d23.json @@ -0,0 +1,45 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT COALESCE(b.expires_at>clock_timestamp() AND b.actor=$2 AND (b.job_id IS NULL OR b.grant_generation=(SELECT grant_generation FROM push_jobs WHERE id=b.job_id)) AND (f.result IS NULL OR COALESCE((f.result->>'prepared')::boolean,false)) AND ((b.token_jti=$2 AND b.shared_generation IS NULL) OR b.shared_generation=$3),false) AS \"authorized!\", b.work_id,COALESCE(b.work_id=$4 AND b.lease_expires_at>clock_timestamp() AND b.expires_at>clock_timestamp(),false) AS \"claimed!\" FROM upload_finalizations f JOIN upload_finalization_batches b ON b.id=f.batch_id WHERE f.session_id=$1 FOR SHARE OF b", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "authorized!", + "type_info": "Bool", + "origin": "Expression" + }, + { + "ordinal": 1, + "name": "work_id", + "type_info": "Text", + "origin": { + "Table": { + "table": "upload_finalization_batches", + "name": "work_id" + } + } + }, + { + "ordinal": 2, + "name": "claimed!", + "type_info": "Bool", + "origin": "Expression" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Text", + "Text" + ] + }, + "nullable": [ + null, + true, + null + ] + }, + "hash": "c9fed8dec34c38292cff2db8577a2fd5abc24a4458947ff3aaba5c70823c0d23" +} diff --git a/workers/server/src/uploads.rs b/workers/server/src/uploads.rs index 77c0b41..c036309 100644 --- a/workers/server/src/uploads.rs +++ b/workers/server/src/uploads.rs @@ -12,7 +12,7 @@ use axum::{ Extension, Router, body::{Body, to_bytes}, extract::{Path, Query, Request, State}, - http::{StatusCode, header}, + http::{HeaderValue, StatusCode, header}, middleware, response::Response, routing::post, @@ -363,15 +363,6 @@ async fn load_reservation_rows( Ok(sessions) } -#[cfg_attr(feature = "profiling", hotpath::measure)] -async fn lock(tx: &mut Transaction, key: &str) -> Result<(), ApiError> { - crate::profile_future!( - crate::db::advisory_lock(tx, key), - label = "uploads.advisory_lock_wait" - ) - .await?; - Ok(()) -} #[cfg_attr(feature = "profiling", hotpath::measure)] async fn locked( state: &AppState, @@ -396,13 +387,39 @@ async fn lock_session( s: &Session, identity: &Identity, ) -> Result<(Transaction, Session), ApiError> { - lock(&mut tx, &format!("upload:{}", s.id)).await?; - lock_rows(&mut tx, std::slice::from_ref(&s.id)).await?; + let held = crate::profile_future!( + lock_sessions(&mut tx, std::slice::from_ref(&s.id)), + label = "uploads.advisory_lock_wait" + ) + .await?; + if !held { + return Err(error(503, "Concurrent write; retry") + .header(header::RETRY_AFTER, HeaderValue::from_static("1"))); + } // Re-read and reauthorize after the locks, using the same query as preflight. let current = load(&mut *tx, &s.id, identity).await?; Ok((tx, current)) } +/// Try every session's `upload:` advisory lock, in sorted order, and +/// only if all were free lock the session rows as `lock_rows` does. It is one +/// statement, in that order: the row locks read the outcome of the advisory +/// locks, so none is requested before every advisory lock has been tried, and +/// none at all when one was held. Returns whether the locks were taken. +async fn lock_sessions(tx: &mut Transaction, ids: &[String]) -> Result { + let locks = sqlx::query!( + "SELECT held.ok AS \"held!\",(SELECT count(*) FROM (SELECT FROM upload_sessions \ + WHERE held.ok AND id=ANY($1) ORDER BY id FOR NO KEY UPDATE) locked) AS locked \ + FROM (SELECT COALESCE(bool_and(pg_try_advisory_xact_lock(hashtextextended(key,0))),TRUE) AS ok \ + FROM (SELECT DISTINCT 'upload:'||id AS key FROM UNNEST($1::text[]) AS id \ + ORDER BY key OFFSET 0) ordered) held", + ids, + ) + .fetch_one(&mut **tx) + .await?; + Ok(locks.held) +} + /// Lock session rows in ID order, so two multi-session writers cannot wait on /// each other. Status changes and deletion wait for the holder. async fn lock_rows(tx: &mut Transaction, ids: &[String]) -> Result<(), ApiError> { @@ -763,27 +780,29 @@ async fn reserve_group( .iter() .flat_map(|(_, parts)| parts.iter().map(|p| p.part_number)) .collect(); - let previous: Vec<(String, i32, i64)> = sqlx::query_as( - "SELECT p.session_id,p.part_number,p.size FROM upload_parts p \ + // The staging total, on every row, beside the parts already reserved. + // The callers' session locks hold those parts still, so reading both in + // one snapshot sees what reading them one after the other did. + let previous = sqlx::query!( + "SELECT s.staged AS \"staged!\",p.session_id AS \"session_id?\",\ + p.part_number AS \"part_number?\",p.size AS \"size?\" FROM \ + (SELECT COALESCE(SUM(u.file_size),0)::bigint AS staged FROM upload_sessions u \ + JOIN caches c ON c.id=u.cache_id WHERE c.org_id=$3 AND u.expires_at>NOW() \ + AND u.status IN ('streaming','finalizing')) s \ + LEFT JOIN (SELECT p.session_id,p.part_number,p.size FROM upload_parts p \ JOIN UNNEST($1::text[],$2::integer[]) r(session_id,part_number) \ - USING(session_id,part_number)", + USING(session_id,part_number)) p ON TRUE", + &ids, + &numbers, + &requests[0].0.org_id, ) - .bind(&ids) - .bind(&numbers) .fetch_all(&mut **tx) .await?; + let mut staged = previous[0].staged; let previous: HashMap<_, _> = previous .into_iter() - .map(|(id, number, size)| ((id, number), size)) + .filter_map(|p| Some(((p.session_id?, p.part_number?), p.size?))) .collect(); - let mut staged: i64 = sqlx::query_scalar( - "SELECT COALESCE(SUM(u.file_size),0)::bigint FROM upload_sessions u \ - JOIN caches c ON c.id=u.cache_id WHERE c.org_id=$1 AND u.expires_at>NOW() \ - AND u.status IN ('streaming','finalizing')", - ) - .bind(&requests[0].0.org_id) - .fetch_one(&mut **tx) - .await?; let mut ids = Vec::new(); let mut numbers = Vec::new(); let mut sizes = Vec::new(); @@ -808,20 +827,17 @@ async fn reserve_group( } } if !sessions.is_empty() { + // Neither write reads what the other changes. sqlx::query( - "INSERT INTO upload_parts(session_id,part_number,size) \ + "WITH reserved AS (INSERT INTO upload_parts(session_id,part_number,size) \ SELECT * FROM UNNEST($1::text[],$2::integer[],$3::bigint[]) \ - ON CONFLICT(session_id,part_number) DO NOTHING", + ON CONFLICT(session_id,part_number) DO NOTHING) \ + UPDATE upload_sessions u SET file_size=u.file_size+r.added \ + FROM UNNEST($4::text[],$5::bigint[]) r(id,added) WHERE u.id=r.id", ) .bind(&ids) .bind(&numbers) .bind(&sizes) - .execute(&mut **tx) - .await?; - sqlx::query( - "UPDATE upload_sessions u SET file_size=u.file_size+r.added \ - FROM UNNEST($1::text[],$2::bigint[]) r(id,added) WHERE u.id=r.id", - ) .bind(&sessions) .bind(&additions) .execute(&mut **tx) @@ -956,20 +972,18 @@ async fn reserve_batch( } } } - let mut results = Vec::with_capacity(pending.len()); - for (index, row) in pending.into_iter().enumerate() { - let result = async { + // The sessions are distinct, and each adopts its provider upload in its + // own transaction. Start first reservations' uploads a few at a time. + let initialized = stream::iter(pending.into_iter().zip(reserved).map( + |(row, session)| async move { let (_, parts) = row?; - let session = reserved[index] - .take() - .ok_or_else(|| error(500, "Internal server error"))?; + let session = session.ok_or_else(|| error(500, "Internal server error"))?; let session = initialize_multipart(state, identity, session).await?; grants(state, &session, &parts).await - } - .await; - results.push(result); - } - Ok(results) + }, + )) + .buffered(4); + Ok(initialized.collect().await) } async fn reserve_org( @@ -978,16 +992,10 @@ async fn reserve_org( requests: &[(String, &[Reservation])], ) -> Result>, ApiError> { let mut tx = crate::db::begin(state).await?; - lock_set( - &mut tx, - requests - .iter() - .map(|(id, _)| format!("upload:{id}")) - .collect(), - ) - .await?; let ids: Vec<_> = requests.iter().map(|(id, _)| id.clone()).collect(); - lock_rows(&mut tx, &ids).await?; + if !lock_sessions(&mut tx, &ids).await? { + return Err(error(503, "Concurrent write; retry")); + } let mut sessions: HashMap<_, _> = load_session_rows(&mut *tx, &ids, identity) .await? .into_iter() @@ -1152,13 +1160,40 @@ fn shared_generation(state: &AppState) -> Option { /// orders revocation against it: a revocation in flight is waited for and then /// seen, and a later one waits for this transaction. Never store the bearer /// credential, and never trust middleware's old token snapshot. +/// +/// Returns the session's finalization claim when a delegated caller's batch +/// was read, and so held: that claim cannot change before the commit. async fn finalization_authority( state: &AppState, tx: &mut Transaction, identity: &Identity, session: &Session, delegated: bool, -) -> Result<(), ApiError> { +) -> Result, ApiError> { + let authority = token_authority(state, tx, identity).await?; + session_authority( + state, + tx, + identity, + authority.as_ref(), + session, + delegated, + None, + ) + .await +} + +/// The token half of `finalization_authority`: what the token grants now, or +/// nothing for the shared token. One read serves every session a transaction +/// decides. Until it commits, the share lock keeps the token unrevoked and +/// its `NOW()` does not move; grant rows are only ever inserted with their +/// token or removed with a cache, and a transaction that decides several +/// sessions holds their caches FOR KEY SHARE. +async fn token_authority( + state: &AppState, + tx: &mut Transaction, + identity: &Identity, +) -> Result, ApiError> { use crate::authz::{ApiToken, CacheGrant, Permission}; match identity { Identity::ApiToken(token) => { @@ -1179,7 +1214,7 @@ async fn finalization_authority( AND (t.expires_at IS NULL OR t.expires_at>NOW()) FOR SHARE OF t", ).bind(&token.id).fetch_optional(&mut **tx).await? .ok_or_else(|| error(401, "Token revoked or expired"))?; - let current = Identity::ApiToken(ApiToken { + Ok(Some(Identity::ApiToken(ApiToken { id: token.id.clone(), org: grant.org, expires_at: grant.expires_at, @@ -1194,39 +1229,61 @@ async fn finalization_authority( .map(CacheGrant::Id) .chain(grant.patterns.into_iter().map(CacheGrant::Pattern)) .collect(), - }); - decide( - ¤t, - Action::NixPush, - &Resource::cache(&session.org_id, session.cache_id, &session.name), - ) - .map_err(|_| error(403, "Forbidden"))?; + }))) } - Identity::SharedToken if shared_generation(state).is_some() => {} - _ => return Err(error(403, "Forbidden")), - } - if delegated && session.status != UploadStatus::Completed { - // The share lock on the batch holds what this reads until the caller - // commits: a consumer's claim, a settlement, and a job resume all take - // the batch exclusively, so each runs wholly before or after it. - let active: Option = sqlx::query_scalar( - "SELECT COALESCE(b.expires_at>clock_timestamp() AND b.actor=$2 AND \ - (b.job_id IS NULL OR b.grant_generation=(SELECT grant_generation FROM push_jobs WHERE id=b.job_id)) AND \ - (f.result IS NULL OR COALESCE((f.result->>'prepared')::boolean,false)) AND \ - ((b.token_jti=$2 AND b.shared_generation IS NULL) OR b.shared_generation=$3),false) \ - FROM upload_finalizations f JOIN upload_finalization_batches b ON b.id=f.batch_id \ - WHERE f.session_id=$1 FOR SHARE OF b", + Identity::SharedToken if shared_generation(state).is_some() => Ok(None), + _ => Err(error(403, "Forbidden")), + } +} + +/// The session half of `finalization_authority`, under the `authority` its +/// transaction read. A publisher passes the claim it `expected` to still own. +async fn session_authority( + state: &AppState, + tx: &mut Transaction, + identity: &Identity, + authority: Option<&Identity>, + session: &Session, + delegated: bool, + expected: Option<&str>, +) -> Result, ApiError> { + if let Some(current) = authority { + decide( + current, + Action::NixPush, + &Resource::cache(&session.org_id, session.cache_id, &session.name), ) - .bind(&session.id) - .bind(identity.actor()) - .bind(shared_generation(state)) - .fetch_optional(&mut **tx) - .await?; - if active == Some(false) { - return Err(error(401, "Finalization authorization expired or replaced")); - } + .map_err(|_| error(403, "Forbidden"))?; + } + if !delegated || session.status == UploadStatus::Completed { + return Ok(None); + } + // The share lock on the batch holds what this reads until the caller + // commits: a consumer's claim, a settlement, and a job resume all take + // the batch exclusively, so each runs wholly before or after it. + let batch = sqlx::query!( + "SELECT COALESCE(b.expires_at>clock_timestamp() AND b.actor=$2 AND \ + (b.job_id IS NULL OR b.grant_generation=(SELECT grant_generation FROM push_jobs WHERE id=b.job_id)) AND \ + (f.result IS NULL OR COALESCE((f.result->>'prepared')::boolean,false)) AND \ + ((b.token_jti=$2 AND b.shared_generation IS NULL) OR b.shared_generation=$3),false) AS \"authorized!\", \ + b.work_id,COALESCE(b.work_id=$4 AND b.lease_expires_at>clock_timestamp() \ + AND b.expires_at>clock_timestamp(),false) AS \"claimed!\" \ + FROM upload_finalizations f JOIN upload_finalization_batches b ON b.id=f.batch_id \ + WHERE f.session_id=$1 FOR SHARE OF b", + session.id, + identity.actor(), + shared_generation(state), + expected, + ) + .fetch_optional(&mut **tx) + .await?; + if batch.as_ref().is_some_and(|b| !b.authorized) { + return Err(error(401, "Finalization authorization expired or replaced")); } - Ok(()) + if expected.is_some() && !batch.as_ref().is_some_and(|b| b.claimed) { + return Err(error(409, "Finalization claim expired or replaced")); + } + Ok(batch.and_then(|b| b.work_id)) } #[cfg_attr(feature = "profiling", hotpath::measure)] @@ -1241,32 +1298,31 @@ async fn seal( let inline = bytes.is_some(); let nar_hash = receipt.validate()?; let (mut tx, mut s) = locked(state, session, identity).await?; - finalization_authority(state, &mut tx, identity, &s, admission.is_none()).await?; + let claim = finalization_authority(state, &mut tx, identity, &s, admission.is_none()).await?; if let Some((batch, _, prepare_only)) = admission { - let existing: Option = sqlx::query_scalar( - "SELECT b.prepare_only FROM upload_finalizations f JOIN upload_finalization_batches b \ - ON b.id=f.batch_id WHERE f.session_id=$1", + // Never strand a late member in settled work. Only a session with no + // batch yet looks at the offered one, and the share lock then holds + // its admission open until this append commits: a consumer's claim + // fails its NOWAIT and retries, and a close that got there first is + // waited for and then seen. The session's own membership cannot + // change under its locks, so one statement reads both. + let membership = sqlx::query!( + "SELECT e.prepare_only AS existing,(SELECT TRUE FROM upload_finalization_batches \ + WHERE e.prepare_only IS NULL AND id=$2 AND NOT admitted FOR SHARE) AS open \ + FROM (SELECT (SELECT b.prepare_only FROM upload_finalizations f \ + JOIN upload_finalization_batches b ON b.id=f.batch_id WHERE f.session_id=$1) \ + AS prepare_only OFFSET 0) e", + s.id, + batch, ) - .bind(&s.id) - .fetch_optional(&mut *tx) + .fetch_one(&mut *tx) .await?; + let (existing, open) = (membership.existing, membership.open); if existing.is_some_and(|saved| saved != prepare_only) { return Err(error(409, "Finalization intent cannot change")); } - if existing.is_none() { - // Never strand a late member in settled work. The share lock holds - // admission open until this append commits: a consumer's claim - // fails its NOWAIT and retries, and a close that got there first - // is waited for and then seen. - let open: Option = sqlx::query_scalar( - "SELECT TRUE FROM upload_finalization_batches WHERE id=$1 AND NOT admitted FOR SHARE", - ) - .bind(batch) - .fetch_optional(&mut *tx) - .await?; - if open.is_none() { - return Err(error(503, "Finalization admission closed; retry")); - } + if existing.is_none() && open.is_none() { + return Err(error(503, "Finalization admission closed; retry")); } } if (s.r2_upload_id.is_none()) != inline { @@ -1385,27 +1441,28 @@ async fn seal( s.status = UploadStatus::Finalizing; } if let Some((batch, ordinal, _)) = admission { + // An explicit retry may reconsider changed capacity. It cannot renew + // a deadline, replace the actor, or change immutable verification. + // A session is appended or retried, never both: an appended row has + // no result yet, and a row that exists is not appended again. sqlx::query( - "INSERT INTO upload_finalizations(session_id,batch_id,ordinal,receipt) \ - VALUES($1,$2,$3,$4) ON CONFLICT(session_id) DO NOTHING", + "WITH appended AS (INSERT INTO upload_finalizations(session_id,batch_id,ordinal,receipt) \ + VALUES($1,$3,$4,$5) ON CONFLICT(session_id) DO NOTHING), \ + retried AS (UPDATE upload_finalizations f SET result=NULL \ + FROM upload_finalization_batches b WHERE f.session_id=$1 AND b.id=f.batch_id \ + AND b.actor=$2 AND b.expires_at>clock_timestamp() \ + AND (f.result->>'status')::integer IN (413,507) RETURNING f.batch_id) \ + UPDATE upload_finalization_batches SET work_at=NOW(),failures=0 WHERE id IN (SELECT batch_id FROM retried)", ) .bind(&s.id) + .bind(identity.actor()) .bind(batch) .bind(ordinal as i32) .bind(sqlx::types::Json(receipt)) .execute(&mut *tx) .await?; - // An explicit retry may reconsider changed capacity. It cannot renew - // a deadline, replace the actor, or change immutable verification. - sqlx::query( - "WITH retried AS (UPDATE upload_finalizations f SET result=NULL \ - FROM upload_finalization_batches b WHERE f.session_id=$1 AND b.id=f.batch_id \ - AND b.actor=$2 AND b.expires_at>clock_timestamp() \ - AND (f.result->>'status')::integer IN (413,507) RETURNING f.batch_id) \ - UPDATE upload_finalization_batches SET work_at=NOW(),failures=0 WHERE id IN (SELECT batch_id FROM retried)", - ).bind(&s.id).bind(identity.actor()).execute(&mut *tx).await?; } - let prepared = prepare_sealed(&mut tx, s, stored, bytes).await?; + let prepared = prepare_sealed(&mut tx, s, stored, bytes, claim).await?; tx.commit().await?; Ok(prepared) } @@ -1690,6 +1747,7 @@ async fn publish_group( } let mut results = Vec::with_capacity(ready.len()); let mut incoming = Vec::new(); + let mut authority = None; for p in ready { let s = &p.session; let current = sessions.get(&s.id).ok_or_else(|| error(404, "Not found"))?; @@ -1697,19 +1755,22 @@ async fn publish_group( results.push(completed_result(current)); continue; } - finalization_authority(state, &mut tx, identity, current, true).await?; - // The authority check above share-locked this session's batch, so - // its claim cannot be replaced between this read and the commit. - if let Some(expected) = &p.finalization_claim { - let owned: bool = sqlx::query_scalar( - "SELECT EXISTS(SELECT 1 FROM upload_finalizations f JOIN upload_finalization_batches b \ - ON b.id=f.batch_id WHERE f.session_id=$1 AND b.work_id=$2 \ - AND b.lease_expires_at>clock_timestamp() AND b.expires_at>clock_timestamp())", - ).bind(&s.id).bind(expected).fetch_one(&mut *tx).await?; - if !owned { - return Err(error(409,"Finalization claim expired or replaced")); - } + // The token is read, and share-locked, by the first member that + // needs it. The authority check share-locks this session's batch as + // it reads the claim, which cannot then be replaced before the commit. + if authority.is_none() { + authority = Some(token_authority(state, &mut tx, identity).await?); } + session_authority( + state, + &mut tx, + identity, + authority.as_ref().unwrap().as_ref(), + current, + true, + p.finalization_claim.as_deref(), + ) + .await?; if current.status != UploadStatus::Finalizing || current.file_hash != s.file_hash { return Err(error(409, "Conflict")); } @@ -2120,15 +2181,8 @@ async fn prepare_sealed( s: Session, parts: Vec, inline: Option, + finalization_claim: Option, ) -> Result { - let finalization_claim: Option = sqlx::query_scalar::<_, Option>( - "SELECT b.work_id FROM upload_finalizations f JOIN upload_finalization_batches b \ - ON b.id=f.batch_id WHERE f.session_id=$1", - ) - .bind(&s.id) - .fetch_optional(&mut **tx) - .await? - .flatten(); if s.status == UploadStatus::Completed { return Ok(Prepared { session: s, @@ -2632,20 +2686,28 @@ async fn publish_batch( .filter(|(_, p)| p.session.push_job_id.is_some()) .map(|(index, _)| *index) .collect(); - for (index, p) in ready { - if results[index].is_some() { - continue; - } + let unfinished: Vec<_> = ready + .into_iter() + .filter(|(index, _)| results[*index].is_none()) + .collect(); + // Each member allocates, writes, and confirms its own candidate. + // Do that a few at a time, as `prepare` does, in request order. + let (shared, caller) = (&owned, &identity); + let finished = stream::iter(unfinished.into_iter().map(|(index, p)| async move { let id = p.session.id.clone(); let unconfirmed = p.key.is_empty(); - match finish_preparation(&owned, &identity, p).await { - Ok(p) if options.prepare_only => { - if unconfirmed { - // Confirmed just now, for the job's worker. - owned.publications.wake(); - } - results[index] = Some(prepared_value(&p)); - } + let result = finish_preparation(shared, caller, p).await; + if options.prepare_only && unconfirmed && result.is_ok() { + // Confirmed just now, for the job's worker. + shared.publications.wake(); + } + (index, id, result) + })) + .buffered(4); + tokio::pin!(finished); + while let Some((index, id, result)) = finished.next().await { + match result { + Ok(p) if options.prepare_only => results[index] = Some(prepared_value(&p)), Ok(p) => pending.push((index, p)), Err(e) => results[index] = result_value(&id, Err(e)), } diff --git a/workers/server/src/uploads/finalization.rs b/workers/server/src/uploads/finalization.rs index 1b1ccc5..35a91af 100644 --- a/workers/server/src/uploads/finalization.rs +++ b/workers/server/src/uploads/finalization.rs @@ -1244,6 +1244,46 @@ mod tests { state.finish_background().await; } + #[sqlx::test(migrator = "crate::migrations::MIGRATOR")] + #[ignore = "requires disposable PostgreSQL and S3 via the server test script"] + async fn a_held_upload_lock_is_refused_before_any_session_row_is_locked(pool: sqlx::PgPool) { + let (state, receipt) = fixture(pool.clone()).await; + let ids = [receipt.session_id.clone()]; + let key = format!("upload:{}", ids[0]); + let row = "SELECT FROM upload_sessions WHERE id=$1 FOR UPDATE NOWAIT"; + // Another writer of this session holds its advisory lock. + let mut holder = db::begin(&state).await.unwrap(); + db::advisory_lock(&mut holder, &key).await.unwrap(); + let mut tx = db::begin(&state).await.unwrap(); + assert!(!lock_sessions(&mut tx, &ids).await.unwrap()); + // The refused writer requested no row lock: the holder takes the row + // without waiting, while the refused transaction is still open. + sqlx::query(row) + .bind(&ids[0]) + .execute(&mut *holder) + .await + .unwrap(); + tx.rollback().await.unwrap(); + holder.rollback().await.unwrap(); + // A free session is locked both ways by the one statement. + let mut tx = db::begin(&state).await.unwrap(); + assert!(lock_sessions(&mut tx, &ids).await.unwrap()); + let mut other = db::begin(&state).await.unwrap(); + assert!(!db::try_advisory_lock(&mut other, &key).await.unwrap()); + let blocked = sqlx::query(row) + .bind(&ids[0]) + .execute(&mut *other) + .await + .unwrap_err(); + assert_eq!( + blocked.as_database_error().unwrap().code().as_deref(), + Some("55P03") + ); + other.rollback().await.unwrap(); + tx.rollback().await.unwrap(); + state.finish_background().await; + } + #[sqlx::test(migrator = "crate::migrations::MIGRATOR")] #[ignore = "requires disposable PostgreSQL and S3 via the server test script"] async fn an_append_orders_against_admission_closing(pool: sqlx::PgPool) {