From 3b70f447fd69f7546cb4829ce4be7d974646fe51 Mon Sep 17 00:00:00 2001 From: Uzair Aftab Date: Wed, 18 Nov 2026 14:30:37 +0100 Subject: [PATCH 1/2] server: serve Go and sccache on organization hosts An organization host answered every request as a Nix read, so the Go and sccache caches were reachable only under /_api/v1/// on the shared host. Customers should configure one address shape, .//. The cache router now registers /{cache}/go and /{cache}/sccache with their trailing forms. On an organization host their handler rewrites the path to the canonical protocol route and dispatches it into a router holding only the Go and sccache routes, so authorization, limits, and storage are the existing handlers' and management or upload routes stay unreachable from an organization host. On any other host the same shape remains a path-based Nix read, which keeps caches named go or sccache working. PROPFIND answers with the path the client used. Two things change shape. On an organization host, a product path with a method the protocol rejects now gets the protocol's JSON 404 or 405 instead of the Nix read's 405. Requests matching the new routes are labelled with them in metrics rather than as cache reads. --- docs.md | 5 +- workers/server/Cargo.toml | 2 +- workers/server/src/app.rs | 1 + workers/server/src/cache_host.rs | 113 +++++- workers/server/src/sccache.rs | 5 +- workers/server/test/compiler.mjs | 180 ++++++---- workers/server/tests/org_host.rs | 598 +++++++++++++++++++++++++++++++ 7 files changed, 829 insertions(+), 75 deletions(-) create mode 100644 workers/server/tests/org_host.rs diff --git a/docs.md b/docs.md index 5b39688..895744c 100644 --- a/docs.md +++ b/docs.md @@ -558,7 +558,10 @@ organization names with `CACHE_RESERVED_NAMES` and exact DNS hosts with `CACHE_RESERVED_HOSTS`; both accept whitespace-separated lowercase values. Reserved hosts take precedence over canonical hosts. Core protocol names remain reserved independently of these settings. Hosted DNS policy belongs -to the embedding application, not the cache core. +to the embedding application, not the cache core. On an organization host, +`//go` and `//sccache/` serve the same handlers as +`/_api/v1///`, which keeps working; every other path +on that host is a Nix read. ### Migrating JWT-era tokens diff --git a/workers/server/Cargo.toml b/workers/server/Cargo.toml index 2fffa65..ce9b279 100644 --- a/workers/server/Cargo.toml +++ b/workers/server/Cargo.toml @@ -51,6 +51,7 @@ tokio = { workspace = true, features = ["io-util", "macros", "net", "rt-multi-th tokio-util = { workspace = true, features = ["rt", "io-util"] } tonic = "0.14" tonic-prost = "0.14" +tower = { workspace = true, features = ["util"] } tracing.workspace = true tracing-subscriber = { workspace = true, features = ["env-filter", "fmt"] } tracing-opentelemetry = { version = "0.34.0", default-features = false, optional = true } @@ -64,5 +65,4 @@ protoc-bin-vendored = "3" [dev-dependencies] hegeltest.workspace = true -tower = { workspace = true, features = ["util"] } zstd = { workspace = true, features = ["pkg-config"] } diff --git a/workers/server/src/app.rs b/workers/server/src/app.rs index 59d76c7..194fbd7 100644 --- a/workers/server/src/app.rs +++ b/workers/server/src/app.rs @@ -271,6 +271,7 @@ pub fn router(state: AppState) -> Router { .merge(crate::go::router()) .merge(crate::sccache::router()) .merge(crate::bazel::router(state.clone())) + .merge(crate::cache_host::product_routes(state.clone())) .merge(management::router(state.clone())) .route("/_api/v1", any(api_not_found)) .route("/_api/v1/{*unmatched}", any(api_not_found)) diff --git a/workers/server/src/cache_host.rs b/workers/server/src/cache_host.rs index ff82329..b00ac95 100644 --- a/workers/server/src/cache_host.rs +++ b/workers/server/src/cache_host.rs @@ -1,16 +1,107 @@ //! Direct organization ingress. Only Host is authoritative, never forwarded headers. use axum::{ - extract::{Request, State}, - http::{StatusCode, header, uri::Authority}, + Router, + extract::{ConnectInfo, MatchedPath, Request, State}, + http::{Extensions, StatusCode, header, uri::Authority}, middleware::Next, response::{IntoResponse, Response}, + routing::any, }; +use std::net::SocketAddr; +use tower::ServiceExt; use crate::{AppState, cache, config::Config}; #[derive(Clone)] pub(crate) struct OrganizationHost(pub String); +/// Products served at `//` on an organization host. +pub(crate) const PRODUCTS: &[&str] = &["go", "sccache"]; + +/// The path the client used on an organization host, before dispatch +/// rewrote it to the protocol's canonical route. +#[derive(Clone)] +pub(crate) struct PublicPath(pub String); + +/// `///…`. A product only on an organization host; on any +/// other host this shape is a path-based Nix read. +pub(crate) fn product_routes(state: AppState) -> Router { + // Protocol routers only: an organization host must never reach + // management, upload, or operator routes. + let protocols: Router = Router::new() + .merge(crate::go::router()) + .merge(crate::sccache::router()) + .fallback(|| async { cache::not_found() }) + .with_state(state); + let mut routes = Router::new(); + for &product in PRODUCTS { + for tail in ["", "/", "/{*rest}"] { + let protocols = protocols.clone(); + routes = routes.route( + &format!("/{{cache}}/{product}{tail}"), + any(move |State(state): State, request: Request| { + dispatch(protocols.clone(), product, state, request) + }), + ); + } + } + routes +} + +async fn dispatch( + protocols: Router, + product: &'static str, + state: AppState, + request: Request, +) -> Response { + let Some(owner) = request + .extensions() + .get::() + .map(|host| host.0.clone()) + else { + return match crate::reads::handle(State(state), request).await { + Ok(response) => response, + Err(error) => error.into_response(), + }; + }; + let public = request.uri().path().to_owned(); + let target = public + .strip_prefix('/') + .and_then(|path| path.split_once('/')) + .and_then(|(name, rest)| { + // The route matched `//` followed by `/` or nothing. + let tail = rest + .strip_prefix(product) + .filter(|tail| tail.is_empty() || tail.starts_with('/'))?; + let query = request + .uri() + .query() + .map(|q| format!("?{q}")) + .unwrap_or_default(); + Some(format!("/_api/v1/{product}/{owner}/{name}{tail}{query}")) + }) + .and_then(|target| target.parse::().ok()); + let Some(target) = target else { + return cache::not_found().into_response(); + }; + // Rebuild the extensions: axum appends inner captures after the outer + // `{cache}` capture, which breaks the protocol handlers' `Path` extractors. + // Go and sccache read no request extensions; the peer address is carried + // for handlers that audit by IP. + let (mut parts, body) = request.into_parts(); + let mut extensions = Extensions::new(); + if let Some(peer) = parts.extensions.get::>() { + extensions.insert(*peer); + } + extensions.insert(PublicPath(public)); + parts.extensions = extensions; + parts.uri = target; + match protocols.oneshot(Request::from_parts(parts, body)).await { + Ok(response) => response, + Err(never) => match never {}, + } +} + /// Names that conflict with the core's path-based protocol namespace. pub(crate) const PROTOCOL_NAMES: &[&str] = &["default", "nar", "private", "_api", "healthz"]; @@ -130,9 +221,21 @@ pub(crate) async fn ingress( request .extensions_mut() .insert(OrganizationHost(owner.to_owned())); - match crate::reads::handle(State(state), request).await { - Ok(response) => response, - Err(error) => error.into_response(), + // Product routes are the only ones registered under `/{cache}/`. + // The cache router must be merged, not nested: a nest would prefix + // the matched path, so product requests would miss this check and + // fall back to Nix reads. + let product = request + .extensions() + .get::() + .is_some_and(|path| path.as_str().starts_with("/{cache}/")); + if product { + next.run(request).await + } else { + match crate::reads::handle(State(state), request).await { + Ok(response) => response, + Err(error) => error.into_response(), + } } }; // Every alias hit must observe current SQL policy, even with a warm CDN. diff --git a/workers/server/src/sccache.rs b/workers/server/src/sccache.rs index 6b14d4e..623da03 100644 --- a/workers/server/src/sccache.rs +++ b/workers/server/src/sccache.rs @@ -140,7 +140,10 @@ async fn propfind(state: AppState, path: DavPath, request: Request) -> Result() + .map_or(request.uri.path(), |path| path.0.as_str()); let xml = format!( "{pathname}{modified}HTTP/1.1 200 OK" ); diff --git a/workers/server/test/compiler.mjs b/workers/server/test/compiler.mjs index 2145820..b333ea3 100644 --- a/workers/server/test/compiler.mjs +++ b/workers/server/test/compiler.mjs @@ -4,6 +4,7 @@ import assert from "node:assert/strict"; import { createHash, randomBytes, randomUUID } from "node:crypto"; import { execFile, spawn } from "node:child_process"; import { once } from "node:events"; +import { createServer, request as httpRequest } from "node:http"; import { mkdir, mkdtemp, readFile, rm, utimes, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; @@ -16,7 +17,7 @@ const exec = promisify(execFile); const serverBinary = new URL("../../../target/debug/snowstead-server", import.meta.url).pathname; const goSource = new URL("../../../apps/go-cache/", import.meta.url).pathname; -test("unchanged compiler clients reuse Rust TCP artifacts with fresh local caches", { timeout: 240_000 }, async t => { +test("unchanged compiler clients reuse Rust TCP artifacts with fresh local caches", { timeout: 480_000 }, async t => { const base = process.env.SNOWSTEAD_TEST_DATABASE_URL; assert.ok(base, "run via scripts/with-postgres.sh inside nix develop"); assert.equal(process.env.R2_ENDPOINT, process.env.SNOWSTEAD_TEST_R2_ENDPOINT, "run via scripts/with-r2.sh"); @@ -54,6 +55,7 @@ test("unchanged compiler clients reuse Rust TCP artifacts with fresh local cache SIGNING_PRIVATE_KEY: "", REQUIRE_CACHE_SIGNING_KEYS: "false", PUBLIC_URL: "", UPSTREAM_CACHES: "", PUBLIC_CACHE_ORIGIN: "", CDN_ZONE_ID: "", CDN_PURGE_TOKEN: "", ANALYTICS_TOKEN: "", TRUSTED_PROXY_IPS: "", + CACHE_HOST_SUFFIX: "cache.test", CACHE_CANONICAL_HOSTS: "", CACHE_RESERVED_NAMES: "", CACHE_RESERVED_HOSTS: "", DATABASE_MAX_CONNECTIONS: "2", LISTEN_ADDR: "127.0.0.1:0", RUST_LOG: "info", }; await exec(serverBinary, ["migrate"], { env, timeout: 30_000 }); @@ -78,6 +80,21 @@ test("unchanged compiler clients reuse Rust TCP artifacts with fresh local cache await db.query("INSERT INTO token_caches(token_jti,cache_id) VALUES ($1,$2)", [jti, id]); tokens[protocol] = `snowstead_tok_${jti}_${tokenSecret}`; } + // A private organization cache reached through its organization host. + await db.query("INSERT INTO orgs(id,slug) VALUES ('org_alpha','alpha')"); + const { rows: [{ id: teamId }] } = await db.query( + "INSERT INTO caches(name,org_id,is_public) VALUES ('team','org_alpha',false) RETURNING id"); + await db.query("INSERT INTO cache_protocols(cache_id,protocol) VALUES ($1,'go'),($1,'sccache')", [teamId]); + const orgJti = randomUUID(); + const orgSecret = randomBytes(32).toString("base64url"); + await db.query( + "INSERT INTO api_tokens(jti,subject,secret_hash,created_by,org_id,expires_at) VALUES ($1,'compiler-fixture',$2,'test','org_alpha',now()+interval '1 hour')", + [orgJti, createHash("sha256").update(orgSecret).digest("hex")]); + await db.query( + "INSERT INTO token_permissions(token_jti,permission,position) VALUES ($1,'go:read',1),($1,'go:write',2),($1,'sccache:read',3),($1,'sccache:write',4)", + [orgJti]); + await db.query("INSERT INTO token_caches(token_jti,cache_id) VALUES ($1,$2)", [orgJti, teamId]); + tokens.org = `snowstead_tok_${orgJti}_${orgSecret}`; } finally { await db.end(); } const token = protocol => tokens[protocol]; server = spawn(serverBinary, [], { env, stdio: ["ignore", "pipe", "inherit"] }); @@ -91,40 +108,64 @@ test("unchanged compiler clients reuse Rust TCP artifacts with fresh local cache const address = /127\.0\.0\.1:\d+/.exec(line)?.[0]; assert.ok(address, line); const url = `http://${address}`; - - // Exercise unmodified Rust and C++ compiler clients with fresh local caches. - const dir = join(home, "sccache"); - await mkdir(dir); - const clientEnv = Object.fromEntries(Object.entries(process.env).filter(([key]) => !key.startsWith("SCCACHE_"))); - Object.assign(clientEnv, { - SCCACHE_SERVER_UDS: join(dir, "server.sock"), SCCACHE_CONF: join(dir, "config"), - SCCACHE_WEBDAV_ENDPOINT: `${url}/_api/v1/sccache/default/sccache/`, - SCCACHE_WEBDAV_TOKEN: token("sccache"), + // Stand in for nginx on the organization host: forward to the Rust service + // with the Host the organization address would carry. The Go helper refuses + // plain HTTP except on loopback, so this stays on 127.0.0.1. + const [upstreamHost, upstreamPort] = address.split(":"); + const front = createServer((incoming, outgoing) => { + const upstream = httpRequest({ host: upstreamHost, port: upstreamPort, method: incoming.method, + path: incoming.url, headers: { ...incoming.headers, host: "alpha.cache.test" } }, response => { + outgoing.writeHead(response.statusCode, response.headers); + response.pipe(outgoing); + }); + upstream.on("error", () => outgoing.destroy()); + incoming.pipe(upstream); }); - const run = args => exec("sccache", args, { env: clientEnv, cwd: dir, timeout: 30_000 }); - try { - await writeFile(join(dir, "config"), ""); - await writeFile(join(dir, "input.cpp"), "int cached(int n) { return n * 7 + 3; }\n"); - await writeFile(join(dir, "input.rs"), "pub fn cached(n: u64) -> u64 { n * 11 + 5 }\n"); - let expected; - for (let pass = 0; pass < 2; pass++) { - clientEnv.SCCACHE_DIR = join(dir, `local-${pass}`); - await run(["--start-server"]); - await run(["c++", "-c", "input.cpp", "-o", "cpp.o", "-O2"]); - await run(["rustc", "--crate-name", "fixture", "--crate-type", "rlib", "--emit=link", "input.rs", "--out-dir", ".", "-O"]); - const stats = JSON.parse((await run(["--show-stats", "--stats-format", "json"])).stdout).stats; - const sum = counts => Object.values(counts).reduce((a, b) => a + b, 0); - assert.equal(sum(stats.cache_hits.counts), pass ? 2 : 0, JSON.stringify(stats)); - assert.equal(sum(stats.cache_misses.counts), pass ? 0 : 2, JSON.stringify(stats)); - assert.equal(stats.cache_write_errors, 0, JSON.stringify(stats)); - const outputs = await Promise.all(["cpp.o", "libfixture.rlib"].map(name => readFile(join(dir, name)))); - if (expected) assert.deepEqual(outputs, expected); - expected = outputs; - console.log(`sccache ${pass ? "warm" : "cold"}: hits=${sum(stats.cache_hits.counts)} misses=${sum(stats.cache_misses.counts)} write_errors=${stats.cache_write_errors} output_bytes=${outputs.map(b => b.length).join(",")}`); - await run(["--stop-server"]); - await Promise.all(["cpp.o", "libfixture.rlib"].map(name => rm(join(dir, name)))); - } - } finally { await run(["--stop-server"]).catch(() => {}); } + front.listen(0, "127.0.0.1"); + await once(front, "listening"); + t.after(() => { front.closeAllConnections(); front.close(); }); + const orgUrl = `http://127.0.0.1:${front.address().port}`; + + // Exercise unmodified Rust and C++ compiler clients with fresh local caches, + // on the shared address and on the organization host. + const sccacheEndpoints = [ + ["shared", `${url}/_api/v1/sccache/default/sccache/`, token("sccache")], + ["org", `${orgUrl}/team/sccache/`, tokens.org], + ]; + for (const [label, endpoint, bearer] of sccacheEndpoints) { + const dir = join(home, `sccache-${label}`); + await mkdir(dir); + const clientEnv = Object.fromEntries(Object.entries(process.env).filter(([key]) => !key.startsWith("SCCACHE_"))); + Object.assign(clientEnv, { + SCCACHE_SERVER_UDS: join(dir, "server.sock"), SCCACHE_CONF: join(dir, "config"), + SCCACHE_WEBDAV_ENDPOINT: endpoint, + SCCACHE_WEBDAV_TOKEN: bearer, + }); + const run = args => exec("sccache", args, { env: clientEnv, cwd: dir, timeout: 30_000 }); + try { + await writeFile(join(dir, "config"), ""); + await writeFile(join(dir, "input.cpp"), "int cached(int n) { return n * 7 + 3; }\n"); + await writeFile(join(dir, "input.rs"), "pub fn cached(n: u64) -> u64 { n * 11 + 5 }\n"); + let expected; + for (let pass = 0; pass < 2; pass++) { + clientEnv.SCCACHE_DIR = join(dir, `local-${pass}`); + await run(["--start-server"]); + await run(["c++", "-c", "input.cpp", "-o", "cpp.o", "-O2"]); + await run(["rustc", "--crate-name", "fixture", "--crate-type", "rlib", "--emit=link", "input.rs", "--out-dir", ".", "-O"]); + const stats = JSON.parse((await run(["--show-stats", "--stats-format", "json"])).stdout).stats; + const sum = counts => Object.values(counts).reduce((a, b) => a + b, 0); + assert.equal(sum(stats.cache_hits.counts), pass ? 2 : 0, JSON.stringify(stats)); + assert.equal(sum(stats.cache_misses.counts), pass ? 0 : 2, JSON.stringify(stats)); + assert.equal(stats.cache_write_errors, 0, JSON.stringify(stats)); + const outputs = await Promise.all(["cpp.o", "libfixture.rlib"].map(name => readFile(join(dir, name)))); + if (expected) assert.deepEqual(outputs, expected); + expected = outputs; + console.log(`sccache ${label} ${pass ? "warm" : "cold"}: hits=${sum(stats.cache_hits.counts)} misses=${sum(stats.cache_misses.counts)} write_errors=${stats.cache_write_errors} output_bytes=${outputs.map(b => b.length).join(",")}`); + await run(["--stop-server"]); + await Promise.all(["cpp.o", "libfixture.rlib"].map(name => rm(join(dir, name)))); + } + } finally { await run(["--stop-server"]).catch(() => {}); } + } // Build the unchanged helper with an isolated Go cache, not the user's cache. const goEnv = Object.fromEntries(Object.entries(process.env).filter(([key]) => !key.startsWith("GO") && !key.startsWith("SNOWSTEAD_GO_"))); @@ -142,39 +183,44 @@ test("unchanged compiler clients reuse Rust TCP artifacts with fresh local cache // Make both passes eligible instead of creating a warm-only index record. const timestamp = new Date(Date.now() - 60_000); for (const name of ["go.mod", "main.go"]) await utimes(join(project, name), timestamp, timestamp); - const goToken = token("go"); - let expectedGo; - for (let pass = 0; pass < 2; pass++) { - const output = join(project, "fixture"); - // This throwaway module has no VCS metadata (nor any need for stamping). - const result = await exec("go", ["build", "-trimpath", "-buildvcs=false", "-o", output, "."], { - cwd: project, timeout: 120_000, maxBuffer: 4 * 1024 * 1024, - env: { ...goEnv, GOCACHE: join(home, `go-local-${pass}`), GOCACHEPROG: helper, - SNOWSTEAD_GO_CACHE_DIR: join(home, `go-helper-${pass}`), - SNOWSTEAD_GO_CACHE_URL: `${url}/_api/v1/go/default/go`, SNOWSTEAD_GO_CACHE_TOKEN: goToken, - SNOWSTEAD_GO_CACHE_READ_ONLY: "false" }, - }); - const summary = result.stderr.split("\n").filter(line => line.startsWith("snowstead-go-cache: local_hits=")); - assert.equal(summary.length, 1, result.stderr); - const stats = Object.fromEntries([...summary[0].matchAll(/(\w+)=(\w+)/g)].map(([, key, value]) => [key, value === "true" ? true : value === "false" ? false : Number(value)])); - for (const key of ["upload_errors", "read_errors", "read_timeouts", "conflicts"]) assert.equal(stats[key], 0, summary[0]); - assert.equal(stats.remote_disabled, false, summary[0]); - if (pass === 0) { - assert.equal(stats.remote_hits, 0, summary[0]); - assert.ok(stats.misses > 0 && stats.uploads > 0 && stats.uploaded_bytes > 0, summary[0]); - } else { - // Go does not request every cold metadata put on a warm build, and - // linking can still put an empty stdout record. Require real artifact - // downloads and no newly uploaded payload, not identical request counts. - assert.ok(stats.remote_hits > 0 && stats.downloaded_bytes >= expectedGo.length, summary[0]); - assert.equal(stats.uploaded_bytes, 0, summary[0]); + const goEndpoints = [ + ["shared", `${url}/_api/v1/go/default/go`, token("go")], + ["org", `${orgUrl}/team/go`, tokens.org], + ]; + for (const [label, endpoint, bearer] of goEndpoints) { + let expectedGo; + for (let pass = 0; pass < 2; pass++) { + const output = join(project, "fixture"); + // This throwaway module has no VCS metadata (nor any need for stamping). + const result = await exec("go", ["build", "-trimpath", "-buildvcs=false", "-o", output, "."], { + cwd: project, timeout: 120_000, maxBuffer: 4 * 1024 * 1024, + env: { ...goEnv, GOCACHE: join(home, `go-local-${label}-${pass}`), GOCACHEPROG: helper, + SNOWSTEAD_GO_CACHE_DIR: join(home, `go-helper-${label}-${pass}`), + SNOWSTEAD_GO_CACHE_URL: endpoint, SNOWSTEAD_GO_CACHE_TOKEN: bearer, + SNOWSTEAD_GO_CACHE_READ_ONLY: "false" }, + }); + const summary = result.stderr.split("\n").filter(line => line.startsWith("snowstead-go-cache: local_hits=")); + assert.equal(summary.length, 1, result.stderr); + const stats = Object.fromEntries([...summary[0].matchAll(/(\w+)=(\w+)/g)].map(([, key, value]) => [key, value === "true" ? true : value === "false" ? false : Number(value)])); + for (const key of ["upload_errors", "read_errors", "read_timeouts", "conflicts"]) assert.equal(stats[key], 0, summary[0]); + assert.equal(stats.remote_disabled, false, summary[0]); + if (pass === 0) { + assert.equal(stats.remote_hits, 0, summary[0]); + assert.ok(stats.misses > 0 && stats.uploads > 0 && stats.uploaded_bytes > 0, summary[0]); + } else { + // Go does not request every cold metadata put on a warm build, and + // linking can still put an empty stdout record. Require real artifact + // downloads and no newly uploaded payload, not identical request counts. + assert.ok(stats.remote_hits > 0 && stats.downloaded_bytes >= expectedGo.length, summary[0]); + assert.equal(stats.uploaded_bytes, 0, summary[0]); + } + const bytes = await readFile(output); + if (expectedGo) assert.deepEqual(bytes, expectedGo); + expectedGo = bytes; + const executed = await exec(output, [], { timeout: 10_000 }); + assert.equal(executed.stderr, "snowstead compiler fixture\n"); + console.log(`Go ${label} ${pass ? "warm" : "cold"}: ${summary[0]} executable_bytes=${bytes.length} sha256=${createHash("sha256").update(bytes).digest("hex")}`); + await rm(output); } - const bytes = await readFile(output); - if (expectedGo) assert.deepEqual(bytes, expectedGo); - expectedGo = bytes; - const executed = await exec(output, [], { timeout: 10_000 }); - assert.equal(executed.stderr, "snowstead compiler fixture\n"); - console.log(`Go ${pass ? "warm" : "cold"}: ${summary[0]} executable_bytes=${bytes.length} sha256=${createHash("sha256").update(bytes).digest("hex")}`); - await rm(output); } }); diff --git a/workers/server/tests/org_host.rs b/workers/server/tests/org_host.rs new file mode 100644 index 0000000..cd02036 --- /dev/null +++ b/workers/server/tests/org_host.rs @@ -0,0 +1,598 @@ +mod common; + +use axum::{ + Router, + body::{Body, to_bytes}, + http::{HeaderMap, Request, StatusCode}, +}; +use base64::{Engine, engine::general_purpose::URL_SAFE_NO_PAD}; +use hmac::{Hmac, KeyInit, Mac}; +use serde_json::json; +use sha2::{Digest, Sha256}; +use snowstead_server::{AppState, config::Config}; +use sqlx::PgPool; +use tower::ServiceExt; + +fn jwt(org: &str, cache: &str, perms: &[&str]) -> String { + let claims = json!({"jti":format!("compiler-{org}-{cache}-{}", perms.join("+")),"sub":"compiler","iss":"snowstead","aud":"snowstead","iat":1,"org":org,"caches":[cache],"perms":perms}); + let input = format!( + "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.{}", + URL_SAFE_NO_PAD.encode(serde_json::to_vec(&claims).unwrap()) + ); + let mut mac = Hmac::::new_from_slice(b"fixture-jwt").unwrap(); + mac.update(input.as_bytes()); + format!( + "{input}.{}", + URL_SAFE_NO_PAD.encode(mac.finalize().into_bytes()) + ) +} + +#[path = "support/tokens.rs"] +mod tokens; + +/// A listed JWT-era token, which works with the scope its row records. +async fn token(pool: &PgPool, org: &str, cache: &str, perms: &[&str]) -> String { + tokens::register(pool, jwt(org, cache, perms)).await +} + +fn config(suffix: &str) -> Config { + Config { + jwt_secret: "fixture-jwt".into(), + cache_host_suffix: suffix.into(), + cache_canonical_hosts: "api.snowstead.com".into(), + ..Default::default() + } +} + +async fn seed(pool: &PgPool) { + sqlx::raw_sql( + "INSERT INTO orgs(id,slug) VALUES ('org_a','alpha'),('org_b','beta'); + INSERT INTO caches(name,org_id,is_public,created_at) VALUES + ('team','org_a',false,'2020-01-02 03:04:05+00'), + ('team','org_b',false,'2020-01-02 03:04:05+00'), + ('go','org_a',true,'2020-01-02 03:04:05+00'), + ('sccache','org_a',true,'2020-01-02 03:04:05+00'); + INSERT INTO cache_protocols(cache_id,protocol) + SELECT id,kind FROM caches CROSS JOIN (VALUES ('go'),('sccache'),('nix')) p(kind);", + ) + .execute(pool) + .await + .unwrap(); +} + +async fn app(pool: &PgPool, suffix: &str) -> Router { + let state = AppState::new( + pool.clone(), + config(suffix), + common::storage(), + reqwest::Client::new(), + ) + .unwrap(); + common::router(state) +} + +async fn send( + app: &Router, + method: &str, + host: Option<&str>, + path: &str, + token: Option<&str>, + headers: &[(&str, &str)], + bytes: &[u8], +) -> (StatusCode, HeaderMap, Vec) { + let mut req = Request::builder().method(method).uri(path); + if let Some(host) = host { + req = req.header("host", host); + } + if let Some(token) = token { + req = req.header("authorization", format!("Bearer {token}")); + } + for (key, value) in headers { + req = req.header(*key, *value); + } + let response = app + .clone() + .oneshot(req.body(Body::from(bytes.to_vec())).unwrap()) + .await + .unwrap(); + let status = response.status(); + let headers = response.headers().clone(); + ( + status, + headers, + to_bytes(response.into_body(), 1024 * 1024) + .await + .unwrap() + .to_vec(), + ) +} + +/// A Go action upload: the body commits to its own SHA-256 output ID. +async fn put_go( + app: &Router, + host: &str, + path: &str, + token: &str, + bytes: &[u8], +) -> (StatusCode, HeaderMap, Vec) { + let length = bytes.len().to_string(); + let output = hex::encode(Sha256::digest(bytes)); + send( + app, + "PUT", + Some(host), + path, + Some(token), + &[ + ("content-length", length.as_str()), + ("x-snowstead-output-id", output.as_str()), + ], + bytes, + ) + .await +} + +async fn put_bytes( + app: &Router, + host: &str, + path: &str, + token: &str, + bytes: &[u8], +) -> (StatusCode, HeaderMap, Vec) { + let length = bytes.len().to_string(); + send( + app, + "PUT", + Some(host), + path, + Some(token), + &[("content-length", length.as_str())], + bytes, + ) + .await +} + +#[sqlx::test(migrator = "snowstead_server::migrations::MIGRATOR")] +async fn go_reads_and_writes_through_the_organization_host(pool: PgPool) { + seed(&pool).await; + let app = app(&pool, "snowstead.com").await; + let token = token(&pool, "org_a", "team", &["go:read", "go:write"]).await; + let first = "a".repeat(64); + let second = "b".repeat(64); + let org_path = format!("/team/go/v1/actions/{first}"); + let legacy_path = format!("/_api/v1/go/alpha/team/v1/actions/{first}"); + let body: &[u8] = b"go action output"; + let second_body: &[u8] = b"second go action output"; + + let (status, headers, _) = send( + &app, + "GET", + Some("alpha.snowstead.com"), + &org_path, + None, + &[], + b"", + ) + .await; + assert_eq!(status, StatusCode::UNAUTHORIZED); + assert_eq!(headers["cloudflare-cdn-cache-control"], "no-store"); + + let (status, headers, _) = put_go(&app, "alpha.snowstead.com", &org_path, &token, body).await; + assert_eq!(status, StatusCode::CREATED); + assert_eq!(headers["cloudflare-cdn-cache-control"], "no-store"); + + let (status, headers, bytes) = send( + &app, + "GET", + Some("alpha.snowstead.com"), + &org_path, + Some(&token), + &[], + b"", + ) + .await; + assert_eq!(status, StatusCode::OK); + assert_eq!(headers["cloudflare-cdn-cache-control"], "no-store"); + assert_eq!(bytes, body); + + let (status, _, bytes) = send( + &app, + "HEAD", + Some("alpha.snowstead.com"), + &org_path, + Some(&token), + &[], + b"", + ) + .await; + assert_eq!(status, StatusCode::OK); + assert!(bytes.is_empty()); + + let with_query = format!("{org_path}?x=1"); + let (status, _, bytes) = send( + &app, + "GET", + Some("alpha.snowstead.com"), + &with_query, + Some(&token), + &[], + b"", + ) + .await; + assert_eq!(status, StatusCode::OK); + assert_eq!(bytes, body); + + let (status, _, bytes) = send( + &app, + "GET", + Some("api.snowstead.com"), + &legacy_path, + Some(&token), + &[], + b"", + ) + .await; + assert_eq!(status, StatusCode::OK); + assert_eq!(bytes, body); + + let legacy_second = format!("/_api/v1/go/alpha/team/v1/actions/{second}"); + let (status, _, _) = put_go( + &app, + "api.snowstead.com", + &legacy_second, + &token, + second_body, + ) + .await; + assert_eq!(status, StatusCode::CREATED); + let org_second = format!("/team/go/v1/actions/{second}"); + let (status, _, bytes) = send( + &app, + "GET", + Some("alpha.snowstead.com"), + &org_second, + Some(&token), + &[], + b"", + ) + .await; + assert_eq!(status, StatusCode::OK); + assert_eq!(bytes, second_body); + + let (status, headers, _) = send( + &app, + "GET", + Some("beta.snowstead.com"), + &org_path, + Some(&token), + &[], + b"", + ) + .await; + assert_eq!(status, StatusCode::FORBIDDEN); + assert_eq!(headers["cloudflare-cdn-cache-control"], "no-store"); + + let (status, _, _) = send( + &app, + "GET", + Some("api.snowstead.com"), + &org_path, + None, + &[], + b"", + ) + .await; + assert_eq!(status, StatusCode::NOT_FOUND); + let (status, _, _) = send(&app, "GET", None, &org_path, None, &[], b"").await; + assert_eq!(status, StatusCode::NOT_FOUND); + + let (status, headers, _) = send( + &app, + "DELETE", + Some("alpha.snowstead.com"), + &org_path, + Some(&token), + &[], + b"", + ) + .await; + assert_eq!(status, StatusCode::METHOD_NOT_ALLOWED); + assert_eq!(headers["allow"], "GET, HEAD, PUT"); + + for path in [ + "/team/go".to_owned(), + "/team/go/".to_owned(), + "/team/go/v1/actions/short".to_owned(), + format!("/te%61m/go/v1/actions/{first}"), + ] { + let (status, _, _) = send( + &app, + "GET", + Some("alpha.snowstead.com"), + &path, + None, + &[], + b"", + ) + .await; + assert_eq!(status, StatusCode::NOT_FOUND, "{path}"); + } + + let (status, headers, _) = send( + &app, + "GET", + Some("a.b.snowstead.com"), + &org_path, + None, + &[], + b"", + ) + .await; + assert_eq!(status, StatusCode::NOT_FOUND); + assert_eq!(headers["cloudflare-cdn-cache-control"], "no-store"); +} + +#[sqlx::test(migrator = "snowstead_server::migrations::MIGRATOR")] +async fn sccache_webdav_through_the_organization_host(pool: PgPool) { + seed(&pool).await; + let app = app(&pool, "snowstead.com").await; + let read = token(&pool, "org_a", "team", &["sccache:read"]).await; + let write = token(&pool, "org_a", "team", &["sccache:read", "sccache:write"]).await; + let artifact = "a".repeat(64); + let artifact_path = format!("/a/a/a/{artifact}"); + let org_artifact = format!("/team/sccache{artifact_path}"); + let legacy_artifact = format!("/_api/v1/sccache/alpha/team{artifact_path}"); + let bytes: &[u8] = b"sccache artifact bytes"; + let org = "alpha.snowstead.com"; + let api = "api.snowstead.com"; + + let (status, _, body) = send( + &app, + "PROPFIND", + Some(org), + "/team/sccache/", + Some(&read), + &[], + b"", + ) + .await; + assert_eq!(status, StatusCode::MULTI_STATUS); + assert!( + std::str::from_utf8(&body) + .unwrap() + .contains("/team/sccache/") + ); + + let (status, _, body) = send( + &app, + "PROPFIND", + Some(api), + "/_api/v1/sccache/alpha/team/", + Some(&read), + &[], + b"", + ) + .await; + assert_eq!(status, StatusCode::MULTI_STATUS); + assert!( + std::str::from_utf8(&body) + .unwrap() + .contains("/_api/v1/sccache/alpha/team/") + ); + + let (status, _, _) = send( + &app, + "MKCOL", + Some(org), + "/team/sccache/a/", + Some(&write), + &[], + b"", + ) + .await; + assert_eq!(status, StatusCode::CREATED); + + let (status, _, _) = put_bytes(&app, org, "/team/sccache/.sccache_check", &write, b"x").await; + assert_eq!(status, StatusCode::NO_CONTENT); + + let (status, _, _) = put_bytes(&app, org, &org_artifact, &write, bytes).await; + assert_eq!(status, StatusCode::CREATED); + + let (status, _, body) = + send(&app, "GET", Some(org), &org_artifact, Some(&read), &[], b"").await; + assert_eq!(status, StatusCode::OK); + assert_eq!(body, bytes); + + let (status, _, body) = send( + &app, + "GET", + Some(api), + &legacy_artifact, + Some(&read), + &[], + b"", + ) + .await; + assert_eq!(status, StatusCode::OK); + assert_eq!(body, bytes); + + let (status, _, _) = send( + &app, + "GET", + Some(org), + "/team/sccache", + Some(&read), + &[], + b"", + ) + .await; + assert_eq!(status, StatusCode::NOT_FOUND); + + let (status, _, _) = send( + &app, + "PROPFIND", + Some(org), + "/team/sccache//a/", + Some(&read), + &[], + b"", + ) + .await; + assert_eq!(status, StatusCode::MULTI_STATUS); + + let (status, _, _) = send( + &app, + "MKCOL", + Some(org), + "/team/sccache/a/b/", + Some(&write), + &[], + b"", + ) + .await; + assert_eq!(status, StatusCode::CREATED); + let (status, _, body) = send( + &app, + "PROPFIND", + Some(org), + "/team/sccache/a/b/", + Some(&read), + &[], + b"", + ) + .await; + assert_eq!(status, StatusCode::MULTI_STATUS); + assert!( + std::str::from_utf8(&body) + .unwrap() + .contains("/team/sccache/a/b/") + ); + + let (status, _, _) = send( + &app, + "GET", + Some("beta.snowstead.com"), + &org_artifact, + Some(&read), + &[], + b"", + ) + .await; + assert_eq!(status, StatusCode::FORBIDDEN); +} + +#[sqlx::test(migrator = "snowstead_server::migrations::MIGRATOR")] +async fn caches_named_after_products_keep_their_nix_addresses(pool: PgPool) { + seed(&pool).await; + let plain = app(&pool, "").await; + let app = app(&pool, "snowstead.com").await; + let go_write = token(&pool, "org_a", "go", &["go:read", "go:write"]).await; + let action = "a".repeat(64); + + for name in ["go", "sccache"] { + let nix = format!("/{name}/nix-cache-info"); + let (status, headers, _) = send( + &app, + "GET", + Some("alpha.snowstead.com"), + &nix, + None, + &[], + b"", + ) + .await; + assert_eq!(status, StatusCode::OK, "{name}"); + assert_eq!(headers["content-type"], "text/x-nix-cache-info", "{name}"); + + let shared = format!("/alpha/{name}/nix-cache-info"); + for (router, host) in [ + (&app, Some("api.snowstead.com")), + (&app, None), + (&plain, Some("api.snowstead.com")), + (&plain, None), + ] { + let (status, _, _) = send(router, "GET", host, &shared, None, &[], b"").await; + assert_eq!(status, StatusCode::OK, "{name} {host:?}"); + } + + let (status, headers, _) = send( + &app, + "PUT", + Some("api.snowstead.com"), + &shared, + None, + &[], + b"", + ) + .await; + assert_eq!(status, StatusCode::METHOD_NOT_ALLOWED, "{name}"); + assert_eq!(headers["allow"], "GET, HEAD", "{name}"); + } + + let (status, _, _) = put_go( + &app, + "alpha.snowstead.com", + &format!("/go/go/v1/actions/{action}"), + &go_write, + b"go cache named go", + ) + .await; + assert_eq!(status, StatusCode::CREATED); + let (status, _, body) = send( + &app, + "GET", + Some("alpha.snowstead.com"), + &format!("/go/go/v1/actions/{action}"), + Some(&go_write), + &[], + b"", + ) + .await; + assert_eq!(status, StatusCode::OK); + assert_eq!(body, b"go cache named go"); +} + +#[sqlx::test(migrator = "snowstead_server::migrations::MIGRATOR")] +async fn other_organization_host_paths_stay_nix_reads(pool: PgPool) { + seed(&pool).await; + let app = app(&pool, "snowstead.com").await; + let action = "a".repeat(64); + + let (status, headers, _) = send( + &app, + "PUT", + Some("alpha.snowstead.com"), + "/team/zig/x", + None, + &[], + b"", + ) + .await; + assert_eq!(status, StatusCode::METHOD_NOT_ALLOWED); + assert_eq!(headers["allow"], "GET, HEAD"); + + let (status, headers, _) = send( + &app, + "PUT", + Some("alpha.snowstead.com"), + "/team/nix-cache-info", + None, + &[], + b"", + ) + .await; + assert_eq!(status, StatusCode::METHOD_NOT_ALLOWED); + assert_eq!(headers["allow"], "GET, HEAD"); + + let (status, _, _) = send( + &app, + "GET", + Some("alpha.snowstead.com"), + &format!("/_api/v1/go/alpha/team/v1/actions/{action}"), + None, + &[], + b"", + ) + .await; + assert_eq!(status, StatusCode::NOT_FOUND); +} From b3269cae3f3052cdf1e6318bd4868d44a18bfc3c Mon Sep 17 00:00:00 2001 From: Uzair Aftab Date: Wed, 18 Nov 2026 14:54:38 +0100 Subject: [PATCH 2/2] server: serve Bazel over HTTP on organization hosts Organization hosts served Go and sccache at //, but Bazel's HTTP cache was still reachable only under /_api/v1/bazel// on the shared host. Bazel joins the product list. Its router is split so the HTTP route can be dispatched from an organization host on its own: the gRPC services stay out of the organization-host router and remain unreachable there. The handler, limits, and store are unchanged, and both addresses read the same entries. Responses on an organization host keep their own Cache-Control but are marked no-store for the CDN, like every other response there, so only anonymous reads of public caches on the shared host are edge-cached. --- docs.md | 6 +- workers/server/src/bazel/mod.rs | 8 +- workers/server/src/cache_host.rs | 7 +- workers/server/tests/bazel.rs | 94 +++++++++++++--- workers/server/tests/org_host.rs | 178 ++++++++++++++++++++++++++++++- 5 files changed, 271 insertions(+), 22 deletions(-) diff --git a/docs.md b/docs.md index 895744c..dfd8d8d 100644 --- a/docs.md +++ b/docs.md @@ -559,9 +559,9 @@ organization names with `CACHE_RESERVED_NAMES` and exact DNS hosts with Reserved hosts take precedence over canonical hosts. Core protocol names remain reserved independently of these settings. Hosted DNS policy belongs to the embedding application, not the cache core. On an organization host, -`//go` and `//sccache/` serve the same handlers as -`/_api/v1///`, which keeps working; every other path -on that host is a Nix read. +`//go`, `//sccache/`, and `//bazel` (Bazel over HTTP) +serve the same handlers as `/_api/v1///`, which +keeps working; every other path on that host is a Nix read. ### Migrating JWT-era tokens diff --git a/workers/server/src/bazel/mod.rs b/workers/server/src/bazel/mod.rs index 1c859bb..67f3032 100644 --- a/workers/server/src/bazel/mod.rs +++ b/workers/server/src/bazel/mod.rs @@ -112,7 +112,13 @@ pub(crate) async fn collect(state: &AppState) -> anyhow::Result { } pub(crate) fn router(state: AppState) -> Router { - grpc::router(state).route( + grpc::router(state).merge(http_router()) +} + +/// Bazel over HTTP alone, without gRPC. Organization hosts route to this +/// router so that the gRPC service never sees their requests. +pub(crate) fn http_router() -> Router { + Router::new().route( "/_api/v1/bazel/{slug}/{cache}/{kind}/{digest}", get(http).head(http).put(http), ) diff --git a/workers/server/src/cache_host.rs b/workers/server/src/cache_host.rs index b00ac95..ac07cb2 100644 --- a/workers/server/src/cache_host.rs +++ b/workers/server/src/cache_host.rs @@ -16,7 +16,7 @@ use crate::{AppState, cache, config::Config}; pub(crate) struct OrganizationHost(pub String); /// Products served at `//` on an organization host. -pub(crate) const PRODUCTS: &[&str] = &["go", "sccache"]; +pub(crate) const PRODUCTS: &[&str] = &["go", "sccache", "bazel"]; /// The path the client used on an organization host, before dispatch /// rewrote it to the protocol's canonical route. @@ -31,6 +31,7 @@ pub(crate) fn product_routes(state: AppState) -> Router { let protocols: Router = Router::new() .merge(crate::go::router()) .merge(crate::sccache::router()) + .merge(crate::bazel::http_router()) .fallback(|| async { cache::not_found() }) .with_state(state); let mut routes = Router::new(); @@ -86,8 +87,8 @@ async fn dispatch( }; // Rebuild the extensions: axum appends inner captures after the outer // `{cache}` capture, which breaks the protocol handlers' `Path` extractors. - // Go and sccache read no request extensions; the peer address is carried - // for handlers that audit by IP. + // Go, sccache, and Bazel over HTTP read no request extensions; the peer + // address is carried for handlers that audit by IP. let (mut parts, body) = request.into_parts(); let mut extensions = Extensions::new(); if let Some(peer) = parts.extensions.get::>() { diff --git a/workers/server/tests/bazel.rs b/workers/server/tests/bazel.rs index f9feeb3..2bec7de 100644 --- a/workers/server/tests/bazel.rs +++ b/workers/server/tests/bazel.rs @@ -30,6 +30,13 @@ impl Drop for Host { } async fn host(pool: &PgPool, storage: snowstead_server::storage::Storage) -> Host { + let token = seed(pool).await; + serve(pool, storage, token, false).await +} + +/// Seeds the `alpha` organization's public and private Bazel caches and a token +/// that reads and writes both. Run once per database. Returns the token. +async fn seed(pool: &PgPool) -> String { sqlx::raw_sql("INSERT INTO orgs(id,slug,private_caches) VALUES ('bazel-org','alpha',true); INSERT INTO caches(name,org_id,is_public) VALUES ('public','bazel-org',true),('private','bazel-org',false); INSERT INTO cache_protocols(cache_id,protocol) SELECT id,'bazel' FROM caches; @@ -45,23 +52,47 @@ async fn host(pool: &PgPool, storage: snowstead_server::storage::Storage) -> Hos .execute(pool) .await .unwrap(); - let state = AppState::new( - pool.clone(), - Config::default(), - storage, - reqwest::Client::new(), - ) - .unwrap(); + format!("snowstead_tok_{id}_fixture") +} + +/// Serves the seeded caches. With `org`, every request arrives with the +/// organization Host `alpha.snowstead.com`; the `/_api/v1/…` addresses are +/// unreachable on this server. +async fn serve( + pool: &PgPool, + storage: snowstead_server::storage::Storage, + token: String, + org: bool, +) -> Host { + let config = if org { + Config { + cache_host_suffix: "snowstead.com".into(), + cache_canonical_hosts: "api.snowstead.com".into(), + ..Default::default() + } + } else { + Config::default() + }; + let state = AppState::new(pool.clone(), config, storage, reqwest::Client::new()).unwrap(); + let app = router(state); + let app = if org { + app.layer(axum::middleware::map_request( + |mut request: axum::extract::Request| async move { + request + .headers_mut() + .insert("host", "alpha.snowstead.com".parse().unwrap()); + request + }, + )) + } else { + app + }; let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); let url = format!("http://{}", listener.local_addr().unwrap()); let task = tokio::spawn(async move { - axum::serve(listener, router(state)).await.unwrap(); + axum::serve(listener, app).await.unwrap(); }); - Host { - url, - token: format!("snowstead_tok_{id}_fixture"), - task, - } + Host { url, token, task } } fn authorized(body: T, host: &Host) -> Request { @@ -787,6 +818,43 @@ async fn native_bazel_upload_and_clean_remote_hit(pool: PgPool) { }).await.unwrap(); } +#[sqlx::test(migrator = "snowstead_server::migrations::MIGRATOR")] +#[ignore = "requires Bazel 8 from the development shell"] +async fn native_bazel_through_the_organization_host(pool: PgPool) { + let token = seed(&pool).await; + let host = serve(&pool, common::s3_storage(), token, true).await; + let bazel = std::env::var("BAZEL").unwrap_or_else(|_| "bazel".to_owned()); + tokio::task::spawn_blocking(move || { + let root = std::env::temp_dir().join(format!("bazel-org-e2e-{}", uuid::Uuid::now_v7())); + std::fs::create_dir_all(&root).unwrap(); + std::fs::write(root.join("MODULE.bazel"), "module(name = 'cache_fixture')\n").unwrap(); + std::fs::write(root.join("BUILD.bazel"), format!( + "genrule(name='artifact', outs=['artifact.bin'], cmd='dd if=/dev/zero of=$@ bs=1048576 count=9; echo executed >> {}/executions')\n", + root.display() + )).unwrap(); + for protocol in ["http"] { + let endpoint = format!("{}/private/bazel", host.url); + for run in 0..2 { + let output = std::process::Command::new(&bazel) + .current_dir(&root) + .arg("--batch") + .arg(format!("--output_user_root={}/{protocol}-{run}", root.display())) + .args(["build", "//:artifact", "--spawn_strategy=local", "--remote_download_outputs=all"]) + .arg(format!("--remote_cache={endpoint}")) + .arg(format!("--remote_header=Authorization=Bearer {}", host.token)) + .output().unwrap(); + assert!(output.status.success(), "{protocol} run {run}: {}", String::from_utf8_lossy(&output.stderr)); + assert_eq!(std::fs::metadata(root.join("bazel-bin/artifact.bin")).unwrap().len(), 9 * 1024 * 1024); + if run == 1 { + assert!(String::from_utf8_lossy(&output.stderr).contains("remote cache hit"), "{}", String::from_utf8_lossy(&output.stderr)); + } + } + } + assert_eq!(std::fs::read_to_string(root.join("executions")).unwrap().lines().count(), 1); + std::fs::remove_dir_all(root).unwrap(); + }).await.unwrap(); +} + #[sqlx::test(migrator = "snowstead_server::migrations::MIGRATOR")] async fn publication_does_not_wait_on_control_plane_locks(pool: PgPool) { let host = host(&pool, common::storage()).await; diff --git a/workers/server/tests/org_host.rs b/workers/server/tests/org_host.rs index cd02036..52feed5 100644 --- a/workers/server/tests/org_host.rs +++ b/workers/server/tests/org_host.rs @@ -53,7 +53,15 @@ async fn seed(pool: &PgPool) { ('go','org_a',true,'2020-01-02 03:04:05+00'), ('sccache','org_a',true,'2020-01-02 03:04:05+00'); INSERT INTO cache_protocols(cache_id,protocol) - SELECT id,kind FROM caches CROSS JOIN (VALUES ('go'),('sccache'),('nix')) p(kind);", + SELECT id,kind FROM caches CROSS JOIN (VALUES ('go'),('sccache'),('nix')) p(kind); + INSERT INTO caches(name,org_id,is_public,created_at) VALUES + ('bazel','org_a',true,'2020-01-02 03:04:05+00'); + INSERT INTO cache_protocols(cache_id,protocol) + SELECT id,kind FROM caches CROSS JOIN (VALUES ('bazel'),('nix')) p(kind) WHERE caches.name='bazel'; + INSERT INTO cache_protocols(cache_id,protocol) + SELECT id,'bazel' FROM caches WHERE name='team'; + INSERT INTO bazel_caches(cache_id,generation,capacity) + SELECT id,'generation-' || id,33554432 FROM caches WHERE name IN ('team','bazel');", ) .execute(pool) .await @@ -481,6 +489,172 @@ async fn sccache_webdav_through_the_organization_host(pool: PgPool) { assert_eq!(status, StatusCode::FORBIDDEN); } +#[sqlx::test(migrator = "snowstead_server::migrations::MIGRATOR")] +async fn bazel_http_through_the_organization_host(pool: PgPool) { + seed(&pool).await; + let app = app(&pool, "snowstead.com").await; + let token = token(&pool, "org_a", "team", &["bazel:read", "bazel:write"]).await; + let bytes: &[u8] = b"build output"; + let hash = hex::encode(Sha256::digest(bytes)); + let second_bytes: &[u8] = b"second build output"; + let second_hash = hex::encode(Sha256::digest(second_bytes)); + let org = "alpha.snowstead.com"; + let api = "api.snowstead.com"; + let org_path = format!("/team/bazel/cas/{hash}"); + let legacy_path = format!("/_api/v1/bazel/alpha/team/cas/{hash}"); + + let (status, _, _) = send(&app, "PUT", Some(org), &org_path, None, &[], bytes).await; + assert_eq!(status, StatusCode::UNAUTHORIZED); + + let (status, headers, _) = put_bytes(&app, org, &org_path, &token, bytes).await; + assert_eq!(status, StatusCode::NO_CONTENT); + assert_eq!(headers["cloudflare-cdn-cache-control"], "no-store"); + + let (status, headers, body) = + send(&app, "GET", Some(org), &org_path, Some(&token), &[], b"").await; + assert_eq!(status, StatusCode::OK); + assert_eq!(headers["cloudflare-cdn-cache-control"], "no-store"); + assert_eq!(body, bytes); + + let (status, _, body) = send(&app, "HEAD", Some(org), &org_path, Some(&token), &[], b"").await; + assert_eq!(status, StatusCode::OK); + assert!(body.is_empty()); + + let (status, _, body) = + send(&app, "GET", Some(api), &legacy_path, Some(&token), &[], b"").await; + assert_eq!(status, StatusCode::OK); + assert_eq!(body, bytes); + + let legacy_second = format!("/_api/v1/bazel/alpha/team/cas/{second_hash}"); + let (status, _, _) = put_bytes(&app, api, &legacy_second, &token, second_bytes).await; + assert_eq!(status, StatusCode::NO_CONTENT); + let org_second = format!("/team/bazel/cas/{second_hash}"); + let (status, _, body) = send(&app, "GET", Some(org), &org_second, Some(&token), &[], b"").await; + assert_eq!(status, StatusCode::OK); + assert_eq!(body, second_bytes); + + for path in [ + "/team/bazel".to_owned(), + "/team/bazel/".to_owned(), + format!("/team/bazel/xx/{hash}"), + ] { + let (status, _, _) = send(&app, "GET", Some(org), &path, Some(&token), &[], b"").await; + assert_eq!(status, StatusCode::NOT_FOUND, "{path}"); + } + + let (status, _, _) = send( + &app, + "GET", + Some("beta.snowstead.com"), + &org_path, + Some(&token), + &[], + b"", + ) + .await; + assert_eq!(status, StatusCode::FORBIDDEN); + + // The public cache is where the CDN-facing headers are observable: the + // organization host must stay uncacheable at the edge, while the shared + // host's CAS reads keep their public revalidation policy. + let public_token = crate::token(&pool, "org_a", "bazel", &["bazel:read", "bazel:write"]).await; + let public_bytes: &[u8] = b"public build output"; + let public_hash = hex::encode(Sha256::digest(public_bytes)); + let public_path = format!("/bazel/bazel/cas/{public_hash}"); + let (status, _, _) = put_bytes(&app, org, &public_path, &public_token, public_bytes).await; + assert_eq!(status, StatusCode::NO_CONTENT); + let (status, headers, body) = send(&app, "GET", Some(org), &public_path, None, &[], b"").await; + assert_eq!(status, StatusCode::OK); + assert_eq!(body, public_bytes); + assert_eq!(headers["cloudflare-cdn-cache-control"], "no-store"); + assert_eq!( + headers["cache-control"], + "public, no-cache, must-revalidate, no-transform" + ); + let shared_public = format!("/_api/v1/bazel/alpha/bazel/cas/{public_hash}"); + let (status, headers, body) = + send(&app, "GET", Some(api), &shared_public, None, &[], b"").await; + assert_eq!(status, StatusCode::OK); + assert_eq!(body, public_bytes); + assert_eq!( + headers["cloudflare-cdn-cache-control"], + "public, no-cache, must-revalidate, no-transform" + ); + + // Isolation: nothing else on the organization host reaches Bazel. + let (status, headers, _) = send( + &app, + "POST", + Some(org), + "/build.bazel.remote.execution.v2.Capabilities/GetCapabilities", + None, + &[("content-type", "application/grpc")], + b"", + ) + .await; + assert_eq!(status, StatusCode::METHOD_NOT_ALLOWED); + assert_eq!(headers["allow"], "GET, HEAD"); + + // The go cache has no Bazel protocol, so its Bazel address does not exist. + let go_token = crate::token(&pool, "org_a", "go", &["bazel:read"]).await; + let (status, _, _) = send( + &app, + "GET", + Some(org), + &format!("/go/bazel/cas/{hash}"), + Some(&go_token), + &[], + b"", + ) + .await; + assert_eq!(status, StatusCode::NOT_FOUND); + + let (status, _, _) = send( + &app, + "GET", + Some(org), + &format!("/team/bazel/cas/{}", hash.to_uppercase()), + Some(&token), + &[], + b"", + ) + .await; + assert_eq!(status, StatusCode::NOT_FOUND); + + let (status, _, _) = send( + &app, + "GET", + Some(org), + &format!("/te%61m/bazel/cas/{hash}"), + Some(&token), + &[], + b"", + ) + .await; + assert_eq!(status, StatusCode::NOT_FOUND); + + let (status, headers, _) = + send(&app, "DELETE", Some(org), &org_path, Some(&token), &[], b"").await; + assert_eq!(status, StatusCode::METHOD_NOT_ALLOWED); + assert_eq!(headers["allow"], "GET,HEAD,PUT"); + + // A body that does not hash to its path is refused and never stored. + let mismatched_path = format!("/team/bazel/cas/{}", "c".repeat(64)); + let (status, _, _) = put_bytes(&app, org, &mismatched_path, &token, b"twelve bytes").await; + assert_eq!(status, StatusCode::BAD_REQUEST); + let (status, _, _) = send( + &app, + "GET", + Some(org), + &mismatched_path, + Some(&token), + &[], + b"", + ) + .await; + assert_eq!(status, StatusCode::NOT_FOUND); +} + #[sqlx::test(migrator = "snowstead_server::migrations::MIGRATOR")] async fn caches_named_after_products_keep_their_nix_addresses(pool: PgPool) { seed(&pool).await; @@ -489,7 +663,7 @@ async fn caches_named_after_products_keep_their_nix_addresses(pool: PgPool) { let go_write = token(&pool, "org_a", "go", &["go:read", "go:write"]).await; let action = "a".repeat(64); - for name in ["go", "sccache"] { + for name in ["go", "sccache", "bazel"] { let nix = format!("/{name}/nix-cache-info"); let (status, headers, _) = send( &app,