From 3b7fdc3326c98fbf1f5a2e3b53c6c776fde35b72 Mon Sep 17 00:00:00 2001 From: Uzair Aftab Date: Tue, 17 Nov 2026 19:43:24 +0100 Subject: [PATCH 1/4] reads: answer scoped narinfo requests with one statement A scoped read now admits the cache and reads the requested publication in a single statement, joining published_paths and blob_objects onto the cache row by the narinfo hash. A public narinfo hit or miss costs one database round trip. Previously every scoped read first looked up the cache, and a narinfo request then ran a second statement for the publication. Each was its own pool checkout and round trip, and narinfo requests, most of them misses, are the hottest route the service has. The cache row still decides the response on its own. An unknown cache or one without the Nix protocol is a plain miss, a private cache authorizes the caller before anything read about the path is used, and a former organization slug authenticates and then exposes nothing. The publication columns are optional on the row, so nix-cache-info and invalid hashes pass no hash and match no path. --- ...3863dfb96e5d57c1d5f691316c8dd5f4fe5f9.json | 65 ------- ...ee9f9542ed8b3bb8109b9555d27b21315e3f.json} | 143 ++++++++++----- workers/server/src/reads.rs | 168 ++++++++++-------- workers/server/tests/cdn_reads.rs | 103 +++++++++++ 4 files changed, 289 insertions(+), 190 deletions(-) delete mode 100644 workers/server/.sqlx/query-7ea23b0bb327596d1e4fee5e2683863dfb96e5d57c1d5f691316c8dd5f4fe5f9.json rename workers/server/.sqlx/{query-0e542f47ea9dcedf878e15eb085f2cb0b3f4c466c891584548ccbe6c296e9e2c.json => query-9084905c0812ee85bee3cca0acfaee9f9542ed8b3bb8109b9555d27b21315e3f.json} (56%) diff --git a/workers/server/.sqlx/query-7ea23b0bb327596d1e4fee5e2683863dfb96e5d57c1d5f691316c8dd5f4fe5f9.json b/workers/server/.sqlx/query-7ea23b0bb327596d1e4fee5e2683863dfb96e5d57c1d5f691316c8dd5f4fe5f9.json deleted file mode 100644 index df48242..0000000 --- a/workers/server/.sqlx/query-7ea23b0bb327596d1e4fee5e2683863dfb96e5d57c1d5f691316c8dd5f4fe5f9.json +++ /dev/null @@ -1,65 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n SELECT\n c.id,\n c.org_id,\n c.is_public,\n o.slug\n FROM caches c\n JOIN orgs o ON o.id = c.org_id\n WHERE c.name = $1\n AND (o.slug = $2 OR o.moving_from = $2)\n AND EXISTS (\n SELECT 1\n FROM cache_protocols cp\n WHERE cp.cache_id = c.id\n AND cp.protocol = 'nix'\n )\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "id", - "type_info": "Int4", - "origin": { - "Table": { - "table": "caches", - "name": "id" - } - } - }, - { - "ordinal": 1, - "name": "org_id", - "type_info": "Text", - "origin": { - "Table": { - "table": "caches", - "name": "org_id" - } - } - }, - { - "ordinal": 2, - "name": "is_public", - "type_info": "Bool", - "origin": { - "Table": { - "table": "caches", - "name": "is_public" - } - } - }, - { - "ordinal": 3, - "name": "slug", - "type_info": "Text", - "origin": { - "Table": { - "table": "orgs", - "name": "slug" - } - } - } - ], - "parameters": { - "Left": [ - "Text", - "Text" - ] - }, - "nullable": [ - false, - false, - false, - false - ] - }, - "hash": "7ea23b0bb327596d1e4fee5e2683863dfb96e5d57c1d5f691316c8dd5f4fe5f9" -} diff --git a/workers/server/.sqlx/query-0e542f47ea9dcedf878e15eb085f2cb0b3f4c466c891584548ccbe6c296e9e2c.json b/workers/server/.sqlx/query-9084905c0812ee85bee3cca0acfaee9f9542ed8b3bb8109b9555d27b21315e3f.json similarity index 56% rename from workers/server/.sqlx/query-0e542f47ea9dcedf878e15eb085f2cb0b3f4c466c891584548ccbe6c296e9e2c.json rename to workers/server/.sqlx/query-9084905c0812ee85bee3cca0acfaee9f9542ed8b3bb8109b9555d27b21315e3f.json index d061174..7b7df10 100644 --- a/workers/server/.sqlx/query-0e542f47ea9dcedf878e15eb085f2cb0b3f4c466c891584548ccbe6c296e9e2c.json +++ b/workers/server/.sqlx/query-9084905c0812ee85bee3cca0acfaee9f9542ed8b3bb8109b9555d27b21315e3f.json @@ -1,155 +1,200 @@ { "db_name": "PostgreSQL", - "query": "\n SELECT\n p.store_path,\n p.nar_hash,\n p.nar_size,\n p.references_json,\n p.deriver,\n p.system,\n p.signatures_json,\n b.file_hash,\n b.file_size,\n b.compression,\n c.signing_key_name,\n c.signing_public_key,\n c.signing_key_sealed\n FROM published_paths p\n JOIN blob_objects b ON b.id = p.blob_object_id\n JOIN caches c ON c.id = p.cache_id\n WHERE p.cache_id = $1\n AND p.store_path_hash = $2\n ", + "query": "\n SELECT\n c.id,\n c.org_id,\n c.is_public,\n o.slug,\n c.signing_key_name,\n c.signing_public_key,\n c.signing_key_sealed,\n p.store_path AS \"store_path?\",\n p.nar_hash AS \"nar_hash?\",\n p.nar_size AS \"nar_size?\",\n p.references_json AS \"references_json?\",\n p.deriver,\n p.system,\n p.signatures_json AS \"signatures_json?\",\n b.file_hash AS \"file_hash?\",\n b.file_size AS \"file_size?\",\n b.compression AS \"compression?\"\n FROM caches c\n JOIN orgs o ON o.id = c.org_id\n LEFT JOIN published_paths p ON p.cache_id = c.id\n AND p.store_path_hash = $3\n LEFT JOIN blob_objects b ON b.id = p.blob_object_id\n WHERE c.name = $1\n AND (o.slug = $2 OR o.moving_from = $2)\n AND EXISTS (\n SELECT 1\n FROM cache_protocols cp\n WHERE cp.cache_id = c.id\n AND cp.protocol = 'nix'\n )\n ", "describe": { "columns": [ { "ordinal": 0, - "name": "store_path", - "type_info": "Text", + "name": "id", + "type_info": "Int4", "origin": { "Table": { - "table": "published_paths", - "name": "store_path" + "table": "caches", + "name": "id" } } }, { "ordinal": 1, - "name": "nar_hash", + "name": "org_id", "type_info": "Text", "origin": { "Table": { - "table": "published_paths", - "name": "nar_hash" + "table": "caches", + "name": "org_id" } } }, { "ordinal": 2, - "name": "nar_size", - "type_info": "Int8", + "name": "is_public", + "type_info": "Bool", "origin": { "Table": { - "table": "published_paths", - "name": "nar_size" + "table": "caches", + "name": "is_public" } } }, { "ordinal": 3, - "name": "references_json", + "name": "slug", "type_info": "Text", "origin": { "Table": { - "table": "published_paths", - "name": "references_json" + "table": "orgs", + "name": "slug" } } }, { "ordinal": 4, - "name": "deriver", + "name": "signing_key_name", "type_info": "Text", "origin": { "Table": { - "table": "published_paths", - "name": "deriver" + "table": "caches", + "name": "signing_key_name" } } }, { "ordinal": 5, - "name": "system", + "name": "signing_public_key", "type_info": "Text", "origin": { "Table": { - "table": "published_paths", - "name": "system" + "table": "caches", + "name": "signing_public_key" } } }, { "ordinal": 6, - "name": "signatures_json", + "name": "signing_key_sealed", "type_info": "Text", "origin": { "Table": { - "table": "published_paths", - "name": "signatures_json" + "table": "caches", + "name": "signing_key_sealed" } } }, { "ordinal": 7, - "name": "file_hash", + "name": "store_path?", "type_info": "Text", "origin": { "Table": { - "table": "blob_objects", - "name": "file_hash" + "table": "published_paths", + "name": "store_path" } } }, { "ordinal": 8, - "name": "file_size", - "type_info": "Int8", + "name": "nar_hash?", + "type_info": "Text", "origin": { "Table": { - "table": "blob_objects", - "name": "file_size" + "table": "published_paths", + "name": "nar_hash" } } }, { "ordinal": 9, - "name": "compression", - "type_info": "Text", + "name": "nar_size?", + "type_info": "Int8", "origin": { "Table": { - "table": "blob_objects", - "name": "compression" + "table": "published_paths", + "name": "nar_size" } } }, { "ordinal": 10, - "name": "signing_key_name", + "name": "references_json?", "type_info": "Text", "origin": { "Table": { - "table": "caches", - "name": "signing_key_name" + "table": "published_paths", + "name": "references_json" } } }, { "ordinal": 11, - "name": "signing_public_key", + "name": "deriver", "type_info": "Text", "origin": { "Table": { - "table": "caches", - "name": "signing_public_key" + "table": "published_paths", + "name": "deriver" } } }, { "ordinal": 12, - "name": "signing_key_sealed", + "name": "system", "type_info": "Text", "origin": { "Table": { - "table": "caches", - "name": "signing_key_sealed" + "table": "published_paths", + "name": "system" + } + } + }, + { + "ordinal": 13, + "name": "signatures_json?", + "type_info": "Text", + "origin": { + "Table": { + "table": "published_paths", + "name": "signatures_json" + } + } + }, + { + "ordinal": 14, + "name": "file_hash?", + "type_info": "Text", + "origin": { + "Table": { + "table": "blob_objects", + "name": "file_hash" + } + } + }, + { + "ordinal": 15, + "name": "file_size?", + "type_info": "Int8", + "origin": { + "Table": { + "table": "blob_objects", + "name": "file_size" + } + } + }, + { + "ordinal": 16, + "name": "compression?", + "type_info": "Text", + "origin": { + "Table": { + "table": "blob_objects", + "name": "compression" } } } ], "parameters": { "Left": [ - "Int4", + "Text", + "Text", "Text" ] }, @@ -160,14 +205,18 @@ false, true, true, + true, false, false, false, false, true, true, - true + false, + false, + false, + false ] }, - "hash": "0e542f47ea9dcedf878e15eb085f2cb0b3f4c466c891584548ccbe6c296e9e2c" + "hash": "9084905c0812ee85bee3cca0acfaee9f9542ed8b3bb8109b9555d27b21315e3f" } diff --git a/workers/server/src/reads.rs b/workers/server/src/reads.rs index 06867f8..a72e5a8 100644 --- a/workers/server/src/reads.rs +++ b/workers/server/src/reads.rs @@ -234,22 +234,43 @@ pub async fn handle(State(state): State, request: Request) -> Result file + .strip_suffix(".narinfo") + .filter(|hash| hash.len() == 32 && hash.bytes().all(|b| NIX32.contains(&b))), + _ => None, + }; + // One statement admits the cache and reads the requested publication, so + // a public narinfo hit or miss is a single round trip. SQL publication is + // authoritative. Never consult mutable metadata objects: new publications + // no longer write them, and old ones may be stale. Signing columns come + // from the same fresh statement snapshot as publication. let cache = sqlx::query_as!( - Cache, + ScopedCache, r#" SELECT c.id, c.org_id, c.is_public, - o.slug + o.slug, + c.signing_key_name, + c.signing_public_key, + c.signing_key_sealed, + p.store_path AS "store_path?", + p.nar_hash AS "nar_hash?", + p.nar_size AS "nar_size?", + p.references_json AS "references_json?", + p.deriver, + p.system, + p.signatures_json AS "signatures_json?", + b.file_hash AS "file_hash?", + b.file_size AS "file_size?", + b.compression AS "compression?" FROM caches c JOIN orgs o ON o.id = c.org_id + LEFT JOIN published_paths p ON p.cache_id = c.id + AND p.store_path_hash = $3 + LEFT JOIN blob_objects b ON b.id = p.blob_object_id WHERE c.name = $1 AND (o.slug = $2 OR o.moving_from = $2) AND EXISTS ( @@ -261,21 +282,19 @@ pub async fn handle(State(state): State, request: Request) -> Result, request: Request) -> Result, request: Request) -> Result { - let hash = file.strip_suffix(".narinfo").unwrap(); - if hash.len() != 32 || !hash.bytes().all(|b| NIX32.contains(&b)) { + let Some(hash) = narinfo else { return Ok(missing(private)); - } - let response = scoped_narinfo(&state, &request, id, &cache_path, hash, private).await?; + }; + let response = scoped_narinfo(&state, &request, cache, &cache_path, hash)?; if response.status().is_success() { record_access(&state, id, hash); } @@ -363,7 +381,7 @@ pub async fn handle(State(state): State, request: Request) -> Result, - system: Option, - signatures_json: String, - file_hash: String, - file_size: i64, - compression: String, +/// A Nix cache as a scoped read addresses it. The publication columns are +/// present only when the requested narinfo is published in that cache. +struct ScopedCache { + id: i32, + org_id: String, + is_public: bool, + slug: String, signing_key_name: Option, signing_public_key: Option, signing_key_sealed: Option, + store_path: Option, + nar_hash: Option, + nar_size: Option, + references_json: Option, + deriver: Option, + system: Option, + signatures_json: Option, + file_hash: Option, + file_size: Option, + compression: Option, } #[cfg_attr(feature = "profiling", hotpath::measure)] -async fn scoped_narinfo( +fn scoped_narinfo( state: &AppState, request: &axum::http::request::Parts, - cache: i32, + p: ScopedCache, cache_path: &str, hash: &str, - private: bool, ) -> Result { - // SQL publication is authoritative. Never consult mutable metadata objects: - // new publications no longer write them, and old ones may be stale. - // Read signing columns in the same fresh statement snapshot as publication. - let published = sqlx::query_as!( - ScopedNarinfo, - r#" - SELECT - p.store_path, - p.nar_hash, - p.nar_size, - p.references_json, - p.deriver, - p.system, - p.signatures_json, - b.file_hash, - b.file_size, - b.compression, - c.signing_key_name, - c.signing_public_key, - c.signing_key_sealed - FROM published_paths p - JOIN blob_objects b ON b.id = p.blob_object_id - JOIN caches c ON c.id = p.cache_id - WHERE p.cache_id = $1 - AND p.store_path_hash = $2 - "#, - cache, - hash, + let private = !p.is_public; + let ( + Some(store_path), + Some(nar_hash), + Some(nar_size), + Some(references_json), + Some(signatures_json), + Some(file_hash), + Some(file_size), + Some(compression), + ) = ( + p.store_path, + p.nar_hash, + p.nar_size, + p.references_json, + p.signatures_json, + p.file_hash, + p.file_size, + p.compression, ) - .fetch_optional(&state.pool) - .await?; - let Some(p) = published else { + else { return Ok(missing(private)); }; - if !valid_blob(&p.file_hash, &p.compression) { + if !valid_blob(&file_hash, &compression) { return Err(ApiError::new( StatusCode::INTERNAL_SERVER_ERROR, "Invalid stored blob", )); } let nar = signing::Narinfo { - store_path: p.store_path, + store_path, store_path_hash: hash.to_owned(), - nar_hash: p.nar_hash, - nar_size: p.nar_size, - references: serde_json::from_str(&p.references_json).map_err(|_| { + nar_hash, + nar_size, + references: serde_json::from_str(&references_json).map_err(|_| { ApiError::new( StatusCode::INTERNAL_SERVER_ERROR, "Invalid stored references", @@ -521,19 +533,19 @@ async fn scoped_narinfo( })?, deriver: p.deriver, system: p.system, - signatures: serde_json::from_str(&p.signatures_json).map_err(|_| { + signatures: serde_json::from_str(&signatures_json).map_err(|_| { ApiError::new( StatusCode::INTERNAL_SERVER_ERROR, "Invalid stored signatures", ) })?, - file_hash: p.file_hash, - file_size: p.file_size, - compression: p.compression, + file_hash, + file_size, + compression, }; let key = signing::select_key( &state.config, - cache, + p.id, p.signing_key_name.as_deref(), p.signing_public_key.as_deref(), p.signing_key_sealed.as_deref(), diff --git a/workers/server/tests/cdn_reads.rs b/workers/server/tests/cdn_reads.rs index 140179b..478e5a8 100644 --- a/workers/server/tests/cdn_reads.rs +++ b/workers/server/tests/cdn_reads.rs @@ -380,6 +380,109 @@ async fn private_stays_authenticated_and_streamed(pool: PgPool) { } } +#[sqlx::test(migrator = "snowstead_server::migrations::MIGRATOR")] +async fn private_reads_reveal_no_path_before_authorization(pool: PgPool) { + // The cache and the path are read together, so a rejected caller must get + // the same answer whether or not the path is published. + let state = fixture(pool.clone(), CDN).await; + let app = router(state.clone()); + let credential = format!("Bearer {}", token(&pool, "org_alpha", "secret").await); + let wrong_org = format!("Bearer {}", token(&pool, "org_beta", "secret").await); + let wrong_cache = format!("Bearer {}", token(&pool, "org_alpha", "main").await); + let published = format!("/alpha/secret/{}.narinfo", "0".repeat(32)); + let unpublished = format!("/alpha/secret/{}.narinfo", "1".repeat(32)); + let stored = format!("/alpha/secret/nar/{}/zstd.nar", "b".repeat(64)); + let unstored = format!("/alpha/secret/nar/{}/zstd.nar", "c".repeat(64)); + async fn answer(response: Response) -> (StatusCode, Vec<(String, String)>, Bytes) { + let mut headers: Vec<_> = response + .headers() + .iter() + .map(|(name, value)| (name.to_string(), value.to_str().unwrap().to_owned())) + .collect(); + headers.sort(); + let status = response.status(); + ( + status, + headers, + to_bytes(response.into_body(), 4096).await.unwrap(), + ) + } + for (present, absent) in [(&published, &unpublished), (&stored, &unstored)] { + for method in ["GET", "HEAD"] { + for (denied, status) in [ + (None, StatusCode::UNAUTHORIZED), + (Some("Bearer not-a-token"), StatusCode::UNAUTHORIZED), + (Some(wrong_org.as_str()), StatusCode::FORBIDDEN), + (Some(wrong_cache.as_str()), StatusCode::FORBIDDEN), + ] { + let headers: Vec<_> = denied.map(|v| ("authorization", v)).into_iter().collect(); + let hit = answer(read(&app, method, present, &headers).await).await; + assert_eq!(hit.0, status, "{method} {present}"); + assert_eq!( + hit, + answer(read(&app, method, absent, &headers).await).await, + "{method} {absent}" + ); + } + let response = read(&app, method, absent, &[("authorization", &credential)]).await; + assert_eq!(response.status(), StatusCode::NOT_FOUND); + assert_eq!( + response.headers()[header::CACHE_CONTROL], + "private, no-cache" + ); + } + } + assert_eq!( + read(&app, "GET", &published, &[("authorization", &credential)]) + .await + .status(), + StatusCode::OK + ); + // A former address still authenticates, then exposes nothing published. + sqlx::query("UPDATE orgs SET slug='renamed',moving_from='alpha' WHERE id='org_alpha'") + .execute(&pool) + .await + .unwrap(); + for path in [&published, &unpublished, &stored] { + assert_eq!( + read(&app, "GET", path, &[]).await.status(), + StatusCode::UNAUTHORIZED + ); + let response = read(&app, "GET", path, &[("authorization", &credential)]).await; + assert_eq!(response.status(), StatusCode::NOT_FOUND); + assert_eq!( + response.headers()[header::CACHE_CONTROL], + "private, no-cache" + ); + } + for (path, status) in [ + ( + format!("/renamed/secret/{}.narinfo", "0".repeat(32)), + StatusCode::OK, + ), + ( + format!("/alpha/main/{}.narinfo", "0".repeat(32)), + StatusCode::NOT_FOUND, + ), + ( + format!("/renamed/main/{}.narinfo", "0".repeat(32)), + StatusCode::OK, + ), + ( + format!("/renamed/main/{}.narinfo", "1".repeat(32)), + StatusCode::NOT_FOUND, + ), + ( + format!("/renamed/go-only/{}.narinfo", "0".repeat(32)), + StatusCode::NOT_FOUND, + ), + ] { + let response = read(&app, "GET", &path, &[("authorization", &credential)]).await; + assert_eq!(response.status(), status, "{path}"); + } + state.finish_background().await; +} + async fn signing_post(app: &Router, path: &str, body: serde_json::Value) -> serde_json::Value { let (path, credential) = web::dashboard("POST", path, "operator-secret", None); let response = app From c9ee10f736a571bac449eb02b7d1f9d77140e91f Mon Sep 17 00:00:00 2001 From: Uzair Aftab Date: Tue, 17 Nov 2026 19:46:25 +0100 Subject: [PATCH 2/4] reads: send the narinfo access stamp only when due The narinfo statement now also reports whether the publication's access stamp is missing or older than five minutes, and a hit spawns the stamp update only when it is. Previously every successful narinfo read spawned the update. Its predicate skipped the write for a fresh stamp, but the task still took a pool connection and a database round trip per hit to learn that. The update keeps its own guard, so readers that race on a due stamp still write it once, and the bounded semaphore is unchanged. Reads keep reordering LRU eviction at the same five-minute granularity. --- ...17755daf83bcad9ad963c8a071248091dc49.json} | 13 +++- workers/server/src/reads.rs | 13 +++- workers/server/tests/reads.rs | 77 +++++++++++++++++++ 3 files changed, 98 insertions(+), 5 deletions(-) rename workers/server/.sqlx/{query-9084905c0812ee85bee3cca0acfaee9f9542ed8b3bb8109b9555d27b21315e3f.json => query-532378b5713d8ab9a45ee0cdc6e917755daf83bcad9ad963c8a071248091dc49.json} (84%) diff --git a/workers/server/.sqlx/query-9084905c0812ee85bee3cca0acfaee9f9542ed8b3bb8109b9555d27b21315e3f.json b/workers/server/.sqlx/query-532378b5713d8ab9a45ee0cdc6e917755daf83bcad9ad963c8a071248091dc49.json similarity index 84% rename from workers/server/.sqlx/query-9084905c0812ee85bee3cca0acfaee9f9542ed8b3bb8109b9555d27b21315e3f.json rename to workers/server/.sqlx/query-532378b5713d8ab9a45ee0cdc6e917755daf83bcad9ad963c8a071248091dc49.json index 7b7df10..f360c7e 100644 --- a/workers/server/.sqlx/query-9084905c0812ee85bee3cca0acfaee9f9542ed8b3bb8109b9555d27b21315e3f.json +++ b/workers/server/.sqlx/query-532378b5713d8ab9a45ee0cdc6e917755daf83bcad9ad963c8a071248091dc49.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n SELECT\n c.id,\n c.org_id,\n c.is_public,\n o.slug,\n c.signing_key_name,\n c.signing_public_key,\n c.signing_key_sealed,\n p.store_path AS \"store_path?\",\n p.nar_hash AS \"nar_hash?\",\n p.nar_size AS \"nar_size?\",\n p.references_json AS \"references_json?\",\n p.deriver,\n p.system,\n p.signatures_json AS \"signatures_json?\",\n b.file_hash AS \"file_hash?\",\n b.file_size AS \"file_size?\",\n b.compression AS \"compression?\"\n FROM caches c\n JOIN orgs o ON o.id = c.org_id\n LEFT JOIN published_paths p ON p.cache_id = c.id\n AND p.store_path_hash = $3\n LEFT JOIN blob_objects b ON b.id = p.blob_object_id\n WHERE c.name = $1\n AND (o.slug = $2 OR o.moving_from = $2)\n AND EXISTS (\n SELECT 1\n FROM cache_protocols cp\n WHERE cp.cache_id = c.id\n AND cp.protocol = 'nix'\n )\n ", + "query": "\n SELECT\n c.id,\n c.org_id,\n c.is_public,\n o.slug,\n c.signing_key_name,\n c.signing_public_key,\n c.signing_key_sealed,\n p.store_path AS \"store_path?\",\n p.nar_hash AS \"nar_hash?\",\n p.nar_size AS \"nar_size?\",\n p.references_json AS \"references_json?\",\n p.deriver,\n p.system,\n p.signatures_json AS \"signatures_json?\",\n b.file_hash AS \"file_hash?\",\n b.file_size AS \"file_size?\",\n b.compression AS \"compression?\",\n (\n p.last_accessed_at IS NULL\n OR p.last_accessed_at < NOW() - INTERVAL '5 minutes'\n ) AS \"stamp_due!\"\n FROM caches c\n JOIN orgs o ON o.id = c.org_id\n LEFT JOIN published_paths p ON p.cache_id = c.id\n AND p.store_path_hash = $3\n LEFT JOIN blob_objects b ON b.id = p.blob_object_id\n WHERE c.name = $1\n AND (o.slug = $2 OR o.moving_from = $2)\n AND EXISTS (\n SELECT 1\n FROM cache_protocols cp\n WHERE cp.cache_id = c.id\n AND cp.protocol = 'nix'\n )\n ", "describe": { "columns": [ { @@ -189,6 +189,12 @@ "name": "compression" } } + }, + { + "ordinal": 17, + "name": "stamp_due!", + "type_info": "Bool", + "origin": "Expression" } ], "parameters": { @@ -215,8 +221,9 @@ false, false, false, - false + false, + null ] }, - "hash": "9084905c0812ee85bee3cca0acfaee9f9542ed8b3bb8109b9555d27b21315e3f" + "hash": "532378b5713d8ab9a45ee0cdc6e917755daf83bcad9ad963c8a071248091dc49" } diff --git a/workers/server/src/reads.rs b/workers/server/src/reads.rs index a72e5a8..99aed87 100644 --- a/workers/server/src/reads.rs +++ b/workers/server/src/reads.rs @@ -22,6 +22,8 @@ pub(crate) const INFO: &str = "StoreDir: /nix/store\nWantMassQuery: 1\nPriority: const NIX32: &[u8] = b"0123456789abcdfghijklmnpqrsvwxyz"; static ACCESS_WRITES: tokio::sync::Semaphore = tokio::sync::Semaphore::const_new(32); +/// Stamp a publication the narinfo read found due. Readers skip this round +/// trip while the stamp is fresh; the predicate still guards concurrent ones. fn record_access(state: &AppState, cache: i32, hash: &str) { // LRU telemetry is best-effort, not part of delivering bytes. Bound tasks // rather than growing a per-path map or queue behind the database pool. @@ -265,7 +267,11 @@ pub async fn handle(State(state): State, request: Request) -> Result, request: Request) -> Result, file_size: Option, compression: Option, + /// The publication's access stamp is missing or older than five minutes. + stamp_due: bool, } #[cfg_attr(feature = "profiling", hotpath::measure)] diff --git a/workers/server/tests/reads.rs b/workers/server/tests/reads.rs index 97e5084..a3bfe16 100644 --- a/workers/server/tests/reads.rs +++ b/workers/server/tests/reads.rs @@ -322,3 +322,80 @@ async fn nar_delivery_uses_only_admitted_bucket_pointer(pool: PgPool) { ); state.finish_background().await; } + +#[sqlx::test(migrator = "snowstead_server::migrations::MIGRATOR")] +async fn narinfo_reads_send_an_access_stamp_only_when_due(pool: PgPool) { + let state = fixture(&pool, CDN, common::storage()).await; + let app = router(state.clone()); + // A statement trigger sees every UPDATE sent, even one that matches no + // row, so a stamp skipped by the reader differs from one skipped in SQL. + sqlx::raw_sql( + "CREATE TABLE stamp_updates (id SERIAL PRIMARY KEY); + CREATE FUNCTION count_stamp_update() RETURNS trigger LANGUAGE plpgsql AS + $$ BEGIN INSERT INTO stamp_updates DEFAULT VALUES; RETURN NULL; END $$; + CREATE TRIGGER count_stamp_update BEFORE UPDATE ON published_paths + FOR EACH STATEMENT EXECUTE FUNCTION count_stamp_update();", + ) + .execute(&pool) + .await + .unwrap(); + let credential = credential(); + let hit = format!("{}.narinfo", "0".repeat(32)); + let miss = format!("{}.narinfo", "1".repeat(32)); + let reads = async |method: &str, file: &str, status: StatusCode| { + for (cache, headers) in [ + ("main", &[][..]), + ("secret", &[("authorization", credential.as_str())][..]), + ] { + let response = read(&app, method, &format!("/alpha/{cache}/{file}"), headers).await; + assert_eq!(response.status(), status, "{method} {cache}/{file}"); + } + state.finish_background().await; + }; + let sent = async || { + sqlx::query_scalar::<_, i64>( + "WITH sent AS (DELETE FROM stamp_updates RETURNING 1) SELECT count(*) FROM sent", + ) + .fetch_one(&pool) + .await + .unwrap() + }; + let fresh = async || { + sqlx::query_scalar::<_, i64>( + "SELECT count(*) FROM published_paths WHERE last_accessed_at > NOW() - INTERVAL '1 minute'", + ) + .fetch_one(&pool) + .await + .unwrap() + }; + // Never-read paths are due, and a miss has nothing to stamp. + reads("GET", &miss, StatusCode::NOT_FOUND).await; + assert_eq!(sent().await, 0); + reads("GET", &hit, StatusCode::OK).await; + assert_eq!((sent().await, fresh().await), (2, 2)); + // Within five minutes of the last stamp a hit sends nothing. + for age in ["0 minutes", "4 minutes"] { + sqlx::query("UPDATE published_paths SET last_accessed_at=NOW() - $1::interval") + .bind(age) + .execute(&pool) + .await + .unwrap(); + assert_eq!(sent().await, 1); + for method in ["GET", "HEAD"] { + reads(method, &hit, StatusCode::OK).await; + } + assert_eq!(sent().await, 0, "{age}"); + } + // An older stamp is due again, but never for a rejected private read. + sqlx::query("UPDATE published_paths SET last_accessed_at=NOW() - INTERVAL '6 minutes'") + .execute(&pool) + .await + .unwrap(); + assert_eq!((sent().await, fresh().await), (1, 0)); + let denied = read(&app, "GET", &format!("/alpha/secret/{hit}"), &[]).await; + assert_eq!(denied.status(), StatusCode::UNAUTHORIZED); + state.finish_background().await; + assert_eq!(sent().await, 0); + reads("HEAD", &hit, StatusCode::OK).await; + assert_eq!((sent().await, fresh().await), (2, 2)); +} From 96f0c52d1f9257511e2192a2e5fc7e047891b1bb Mon Sep 17 00:00:00 2001 From: Uzair Aftab Date: Tue, 17 Nov 2026 19:54:29 +0100 Subject: [PATCH 3/4] reads: shorten the private nar path to its first byte A scoped NAR request now admits the cache and resolves the object key in one statement, the proxied body is read in 256 KiB frames, and the bandwidth upsert for a private download runs as tracked background work instead of ahead of the response headers. Previously a scoped NAR read looked up the cache and then the key in two round trips, a private download awaited the bandwidth write before sending headers, and the body was framed at the 4 KiB reader default, which costs a wakeup and a frame for every few kilobytes of a large NAR. The Bazel read path already uses 256 KiB frames. A private cache still authorizes the caller before the key is used, so a rejected read answers the same for stored and absent NARs. Metering keeps its meaning: an admitted GET counts for the full object size at its start, HEAD does not count, and graceful shutdown drains the task. The uploads test that reads the meter right after a download now waits for the row, since the write no longer precedes the response. --- workers/server/src/reads.rs | 266 ++++++++++++++++++++------------ workers/server/tests/reads.rs | 51 ++++++ workers/server/tests/uploads.rs | 6 + 3 files changed, 224 insertions(+), 99 deletions(-) diff --git a/workers/server/src/reads.rs b/workers/server/src/reads.rs index 99aed87..e5828eb 100644 --- a/workers/server/src/reads.rs +++ b/workers/server/src/reads.rs @@ -54,6 +54,33 @@ fn record_access(state: &AppState, cache: i32, hash: &str) { }); } +/// Count an admitted private transfer at its start, for its full size. +fn record_transfer(state: &AppState, org: &str, cache: i32, bytes: u64) { + // Metering is not part of delivering bytes, so it stays off the path to + // the first one. Tracked tasks still finish before a graceful shutdown. + let owned = state.clone(); + let org = org.to_owned(); + state.tasks.spawn(async move { + if sqlx::query!( + r#" + INSERT INTO bandwidth_daily (org_id, cache_id, product, day, bytes) + VALUES ($1, $2, 'nix', (NOW() AT TIME ZONE 'UTC')::date, $3) + ON CONFLICT (org_id, cache_id, product, day) + DO UPDATE SET bytes = bandwidth_daily.bytes + EXCLUDED.bytes + "#, + org, + cache, + bytes as i64, + ) + .execute(&owned.pool) + .await + .is_err() + { + tracing::warn!("Recording Nix bandwidth failed"); + } + }); +} + fn missing(private: bool) -> Response { ( StatusCode::NOT_FOUND, @@ -242,56 +269,19 @@ pub async fn handle(State(state): State, request: Request) -> Result None, }; - // One statement admits the cache and reads the requested publication, so - // a public narinfo hit or miss is a single round trip. SQL publication is - // authoritative. Never consult mutable metadata objects: new publications - // no longer write them, and old ones may be stale. Signing columns come - // from the same fresh statement snapshot as publication. - let cache = sqlx::query_as!( - ScopedCache, - r#" - SELECT - c.id, - c.org_id, - c.is_public, - o.slug, - c.signing_key_name, - c.signing_public_key, - c.signing_key_sealed, - p.store_path AS "store_path?", - p.nar_hash AS "nar_hash?", - p.nar_size AS "nar_size?", - p.references_json AS "references_json?", - p.deriver, - p.system, - p.signatures_json AS "signatures_json?", - b.file_hash AS "file_hash?", - b.file_size AS "file_size?", - b.compression AS "compression?", - ( - p.last_accessed_at IS NULL - OR p.last_accessed_at < NOW() - INTERVAL '5 minutes' - ) AS "stamp_due!" - FROM caches c - JOIN orgs o ON o.id = c.org_id - LEFT JOIN published_paths p ON p.cache_id = c.id - AND p.store_path_hash = $3 - LEFT JOIN blob_objects b ON b.id = p.blob_object_id - WHERE c.name = $1 - AND (o.slug = $2 OR o.moving_from = $2) - AND EXISTS ( - SELECT 1 - FROM cache_protocols cp - WHERE cp.cache_id = c.id - AND cp.protocol = 'nix' - ) - "#, - name, - slug, - narinfo, - ) - .fetch_optional(&state.pool) - .await?; + let blob = match rest { + ["nar", hash, file] => file + .strip_suffix(".nar") + .filter(|compression| valid_blob(hash, compression)) + .map(|compression| (*hash, compression)), + _ => None, + }; + // One statement admits the cache and reads what the request names, so a + // public read is a single round trip whether or not the path exists. + let (cache, key) = match blob { + Some((hash, compression)) => scoped_nar(&state, name, slug, hash, compression).await?, + None => (scoped_cache(&state, name, slug, narinfo).await?, None), + }; let Some(cache) = cache else { return Ok(missing(false)); }; @@ -349,35 +339,7 @@ pub async fn handle(State(state): State, request: Request) -> Result { - let compression = file.strip_suffix(".nar").unwrap(); - if !valid_blob(hash, compression) { - return Ok(missing(private)); - } - let key = sqlx::query_scalar::<_, Option>( - r#" - SELECT CASE WHEN $4 THEN - COALESCE(b.private_key, CASE WHEN b.r2_key NOT LIKE '%/objects/%' - THEN 'nars/sha256/' || b.file_hash || '/' || b.compression || '.nar' END) - ELSE - COALESCE(b.public_key, CASE WHEN b.r2_key NOT LIKE '%/objects/%' - THEN 'nar/' || b.file_hash || '/' || b.compression || '.nar' END) - END - FROM published_paths p - JOIN blob_objects b ON b.id = p.blob_object_id - WHERE p.cache_id = $1 - AND b.file_hash = $2 - AND b.compression = $3 - LIMIT 1 - "#, - ) - .bind(id) - .bind(hash) - .bind(compression) - .bind(private) - .fetch_optional(&state.pool) - .await? - .flatten(); + ["nar", _, file] if file.ends_with(".nar") => { let Some(key) = key else { return Ok(missing(private)); }; @@ -396,6 +358,126 @@ pub async fn handle(State(state): State, request: Request) -> Result, +) -> Result, ApiError> { + // SQL publication is authoritative. Never consult mutable metadata objects: + // new publications no longer write them, and old ones may be stale. + // Read signing columns in the same fresh statement snapshot as publication. + let cache = sqlx::query_as!( + ScopedCache, + r#" + SELECT + c.id, + c.org_id, + c.is_public, + o.slug, + c.signing_key_name, + c.signing_public_key, + c.signing_key_sealed, + p.store_path AS "store_path?", + p.nar_hash AS "nar_hash?", + p.nar_size AS "nar_size?", + p.references_json AS "references_json?", + p.deriver, + p.system, + p.signatures_json AS "signatures_json?", + b.file_hash AS "file_hash?", + b.file_size AS "file_size?", + b.compression AS "compression?", + ( + p.last_accessed_at IS NULL + OR p.last_accessed_at < NOW() - INTERVAL '5 minutes' + ) AS "stamp_due!" + FROM caches c + JOIN orgs o ON o.id = c.org_id + LEFT JOIN published_paths p ON p.cache_id = c.id + AND p.store_path_hash = $3 + LEFT JOIN blob_objects b ON b.id = p.blob_object_id + WHERE c.name = $1 + AND (o.slug = $2 OR o.moving_from = $2) + AND EXISTS ( + SELECT 1 + FROM cache_protocols cp + WHERE cp.cache_id = c.id + AND cp.protocol = 'nix' + ) + "#, + name, + slug, + narinfo, + ) + .fetch_optional(&state.pool) + .await?; + Ok(cache) +} + +/// Admit a scoped Nix cache and resolve a published NAR's key in its bucket. +async fn scoped_nar( + state: &AppState, + name: &str, + slug: &str, + hash: &str, + compression: &str, +) -> Result<(Option, Option), ApiError> { + let (cache, key) = sqlx::query_as::<_, (i32, String, bool, String, Option)>( + r#" + SELECT + c.id, + c.org_id, + c.is_public, + o.slug, + ( + SELECT CASE WHEN NOT c.is_public THEN + COALESCE(b.private_key, CASE WHEN b.r2_key NOT LIKE '%/objects/%' + THEN 'nars/sha256/' || b.file_hash || '/' || b.compression || '.nar' END) + ELSE + COALESCE(b.public_key, CASE WHEN b.r2_key NOT LIKE '%/objects/%' + THEN 'nar/' || b.file_hash || '/' || b.compression || '.nar' END) + END + FROM published_paths p + JOIN blob_objects b ON b.id = p.blob_object_id + WHERE p.cache_id = c.id + AND b.file_hash = $3 + AND b.compression = $4 + LIMIT 1 + ) + FROM caches c + JOIN orgs o ON o.id = c.org_id + WHERE c.name = $1 + AND (o.slug = $2 OR o.moving_from = $2) + AND EXISTS ( + SELECT 1 + FROM cache_protocols cp + WHERE cp.cache_id = c.id + AND cp.protocol = 'nix' + ) + "#, + ) + .bind(name) + .bind(slug) + .bind(hash) + .bind(compression) + .fetch_optional(&state.pool) + .await? + .map(|(id, org_id, is_public, slug, key)| { + let cache = ScopedCache { + id, + org_id, + is_public, + slug, + ..Default::default() + }; + (cache, key) + }) + .unzip(); + Ok((cache, key.flatten())) +} + #[cfg_attr(feature = "profiling", hotpath::measure)] async fn nar( state: &AppState, @@ -425,28 +507,12 @@ async fn nar( if let Some((id, org)) = scope && request.method == Method::GET { - // Count the admitted transfer, like the hosted read path; HEADs do not count. - if sqlx::query!( - r#" - INSERT INTO bandwidth_daily (org_id, cache_id, product, day, bytes) - VALUES ($1, $2, 'nix', (NOW() AT TIME ZONE 'UTC')::date, $3) - ON CONFLICT (org_id, cache_id, product, day) - DO UPDATE SET bytes = bandwidth_daily.bytes + EXCLUDED.bytes - "#, - org, - id, - size as i64, - ) - .execute(&state.pool) - .await - .is_err() - { - tracing::warn!("Recording Nix bandwidth failed"); - } + // Like the hosted read path; HEADs do not count. + record_transfer(state, org, id, size); } let body = reader.map_or_else(Body::empty, |reader| { Body::from_stream(crate::profile_stream!( - ReaderStream::new(reader), + ReaderStream::with_capacity(reader, 256 * 1024), label = "reads.nar_body" )) }); @@ -469,7 +535,9 @@ async fn nar( } /// A Nix cache as a scoped read addresses it. The publication columns are -/// present only when the requested narinfo is published in that cache. +/// present only when the requested narinfo is published in that cache; a NAR +/// request reads none of them. +#[derive(Default)] struct ScopedCache { id: i32, org_id: String, diff --git a/workers/server/tests/reads.rs b/workers/server/tests/reads.rs index a3bfe16..4de9ee3 100644 --- a/workers/server/tests/reads.rs +++ b/workers/server/tests/reads.rs @@ -399,3 +399,54 @@ async fn narinfo_reads_send_an_access_stamp_only_when_due(pool: PgPool) { reads("HEAD", &hit, StatusCode::OK).await; assert_eq!((sent().await, fresh().await), (2, 2)); } + +#[sqlx::test(migrator = "snowstead_server::migrations::MIGRATOR")] +async fn private_nar_streams_wide_frames_and_meters_beside_the_response(pool: PgPool) { + use futures_util::StreamExt; + let storage = common::storage(); + let state = fixture(&pool, CDN, storage.clone()).await; + let app = router(state.clone()); + let hash = "a".repeat(64); + let bytes: Vec = (0..(1 << 20) + 1).map(|i| i as u8).collect(); + storage + .put(true, &format!("nars/sha256/{hash}/zstd.nar"), &bytes) + .await + .unwrap(); + let path = format!("/alpha/secret/nar/{hash}/zstd.nar"); + let credential = credential(); + let auth = [("authorization", credential.as_str())]; + // Metering is tracked background work: it is complete once that drains. + let metered = async || { + state.finish_background().await; + sqlx::query_scalar::<_, Option>("SELECT sum(bytes)::bigint FROM bandwidth_daily") + .fetch_one(&pool) + .await + .unwrap() + }; + let response = read(&app, "HEAD", &path, &auth).await; + assert_eq!(response.status(), StatusCode::OK); + assert_eq!( + response.headers()[header::CONTENT_LENGTH], + bytes.len().to_string() + ); + assert_eq!( + read(&app, "GET", &path, &[]).await.status(), + StatusCode::UNAUTHORIZED + ); + assert_eq!(metered().await, None); + let response = read(&app, "GET", &path, &auth).await; + assert_eq!(response.status(), StatusCode::OK); + let mut frames = response.into_body().into_data_stream(); + let (mut body, mut widest) = (Vec::new(), 0); + while let Some(frame) = frames.next().await { + let frame = frame.unwrap(); + widest = widest.max(frame.len()); + body.extend_from_slice(&frame); + } + assert!(body == bytes); + assert_eq!(widest, 256 * 1024); + assert_eq!(metered().await, Some(bytes.len() as i64)); + // The admitted transfer counts in full at its start, read or not. + drop(read(&app, "GET", &path, &auth).await); + assert_eq!(metered().await, Some(2 * bytes.len() as i64)); +} diff --git a/workers/server/tests/uploads.rs b/workers/server/tests/uploads.rs index 8955a3c..40f9a8d 100644 --- a/workers/server/tests/uploads.rs +++ b/workers/server/tests/uploads.rs @@ -3739,6 +3739,12 @@ async fn inline_private_publication_and_read_isolation(pool: PgPool) { .0, StatusCode::NOT_FOUND ); + // Metering runs beside the response, not ahead of its first byte. + wait_sql( + &pool, + "SELECT EXISTS (SELECT 1 FROM bandwidth_daily WHERE org_id='org_a')", + ) + .await; let metered: i64 = sqlx::query_scalar("SELECT bytes FROM bandwidth_daily WHERE org_id='org_a'") .fetch_one(&pool) .await From eadb207c6150477fb89bad6b2d344bad59430eda Mon Sep 17 00:00:00 2001 From: Uzair Aftab Date: Tue, 17 Nov 2026 23:55:44 +0100 Subject: [PATCH 4/4] reads: check the scoped nar lookup at compile time The statement that admits a scoped cache and resolves a NAR's key was the one read-path query still bound at runtime and decoded into a positional tuple. It now uses the query macro like the narinfo lookup beside it, so the build verifies it against the schema. --- ...e00cfc3cc5aeae3126f7fc18733204bfa43f2.json | 74 +++++++++++++++++++ workers/server/src/reads.rs | 24 +++--- 2 files changed, 86 insertions(+), 12 deletions(-) create mode 100644 workers/server/.sqlx/query-f5b2b847f87ba0bd12b38be962ae00cfc3cc5aeae3126f7fc18733204bfa43f2.json diff --git a/workers/server/.sqlx/query-f5b2b847f87ba0bd12b38be962ae00cfc3cc5aeae3126f7fc18733204bfa43f2.json b/workers/server/.sqlx/query-f5b2b847f87ba0bd12b38be962ae00cfc3cc5aeae3126f7fc18733204bfa43f2.json new file mode 100644 index 0000000..49ed379 --- /dev/null +++ b/workers/server/.sqlx/query-f5b2b847f87ba0bd12b38be962ae00cfc3cc5aeae3126f7fc18733204bfa43f2.json @@ -0,0 +1,74 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT\n c.id,\n c.org_id,\n c.is_public,\n o.slug,\n (\n SELECT CASE WHEN NOT c.is_public THEN\n COALESCE(b.private_key, CASE WHEN b.r2_key NOT LIKE '%/objects/%'\n THEN 'nars/sha256/' || b.file_hash || '/' || b.compression || '.nar' END)\n ELSE\n COALESCE(b.public_key, CASE WHEN b.r2_key NOT LIKE '%/objects/%'\n THEN 'nar/' || b.file_hash || '/' || b.compression || '.nar' END)\n END\n FROM published_paths p\n JOIN blob_objects b ON b.id = p.blob_object_id\n WHERE p.cache_id = c.id\n AND b.file_hash = $3\n AND b.compression = $4\n LIMIT 1\n ) AS key\n FROM caches c\n JOIN orgs o ON o.id = c.org_id\n WHERE c.name = $1\n AND (o.slug = $2 OR o.moving_from = $2)\n AND EXISTS (\n SELECT 1\n FROM cache_protocols cp\n WHERE cp.cache_id = c.id\n AND cp.protocol = 'nix'\n )\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Int4", + "origin": { + "Table": { + "table": "caches", + "name": "id" + } + } + }, + { + "ordinal": 1, + "name": "org_id", + "type_info": "Text", + "origin": { + "Table": { + "table": "caches", + "name": "org_id" + } + } + }, + { + "ordinal": 2, + "name": "is_public", + "type_info": "Bool", + "origin": { + "Table": { + "table": "caches", + "name": "is_public" + } + } + }, + { + "ordinal": 3, + "name": "slug", + "type_info": "Text", + "origin": { + "Table": { + "table": "orgs", + "name": "slug" + } + } + }, + { + "ordinal": 4, + "name": "key", + "type_info": "Text", + "origin": "Expression" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Text", + "Text" + ] + }, + "nullable": [ + false, + false, + false, + false, + null + ] + }, + "hash": "f5b2b847f87ba0bd12b38be962ae00cfc3cc5aeae3126f7fc18733204bfa43f2" +} diff --git a/workers/server/src/reads.rs b/workers/server/src/reads.rs index e5828eb..d15639f 100644 --- a/workers/server/src/reads.rs +++ b/workers/server/src/reads.rs @@ -424,7 +424,7 @@ async fn scoped_nar( hash: &str, compression: &str, ) -> Result<(Option, Option), ApiError> { - let (cache, key) = sqlx::query_as::<_, (i32, String, bool, String, Option)>( + let (cache, key) = sqlx::query!( r#" SELECT c.id, @@ -445,7 +445,7 @@ async fn scoped_nar( AND b.file_hash = $3 AND b.compression = $4 LIMIT 1 - ) + ) AS key FROM caches c JOIN orgs o ON o.id = c.org_id WHERE c.name = $1 @@ -457,22 +457,22 @@ async fn scoped_nar( AND cp.protocol = 'nix' ) "#, + name, + slug, + hash, + compression, ) - .bind(name) - .bind(slug) - .bind(hash) - .bind(compression) .fetch_optional(&state.pool) .await? - .map(|(id, org_id, is_public, slug, key)| { + .map(|row| { let cache = ScopedCache { - id, - org_id, - is_public, - slug, + id: row.id, + org_id: row.org_id, + is_public: row.is_public, + slug: row.slug, ..Default::default() }; - (cache, key) + (cache, row.key) }) .unzip(); Ok((cache, key.flatten()))