From 3b70f447fd69f7546cb4829ce4be7d974646fe51 Mon Sep 17 00:00:00 2001 From: Uzair Aftab Date: Wed, 18 Nov 2026 14:30:37 +0100 Subject: [PATCH] server: serve Go and sccache on organization hosts An organization host answered every request as a Nix read, so the Go and sccache caches were reachable only under /_api/v1/// on the shared host. Customers should configure one address shape, .//. The cache router now registers /{cache}/go and /{cache}/sccache with their trailing forms. On an organization host their handler rewrites the path to the canonical protocol route and dispatches it into a router holding only the Go and sccache routes, so authorization, limits, and storage are the existing handlers' and management or upload routes stay unreachable from an organization host. On any other host the same shape remains a path-based Nix read, which keeps caches named go or sccache working. PROPFIND answers with the path the client used. Two things change shape. On an organization host, a product path with a method the protocol rejects now gets the protocol's JSON 404 or 405 instead of the Nix read's 405. Requests matching the new routes are labelled with them in metrics rather than as cache reads. --- docs.md | 5 +- workers/server/Cargo.toml | 2 +- workers/server/src/app.rs | 1 + workers/server/src/cache_host.rs | 113 +++++- workers/server/src/sccache.rs | 5 +- workers/server/test/compiler.mjs | 180 ++++++---- workers/server/tests/org_host.rs | 598 +++++++++++++++++++++++++++++++ 7 files changed, 829 insertions(+), 75 deletions(-) create mode 100644 workers/server/tests/org_host.rs diff --git a/docs.md b/docs.md index 5b39688..895744c 100644 --- a/docs.md +++ b/docs.md @@ -558,7 +558,10 @@ organization names with `CACHE_RESERVED_NAMES` and exact DNS hosts with `CACHE_RESERVED_HOSTS`; both accept whitespace-separated lowercase values. Reserved hosts take precedence over canonical hosts. Core protocol names remain reserved independently of these settings. Hosted DNS policy belongs -to the embedding application, not the cache core. +to the embedding application, not the cache core. On an organization host, +`//go` and `//sccache/` serve the same handlers as +`/_api/v1///`, which keeps working; every other path +on that host is a Nix read. ### Migrating JWT-era tokens diff --git a/workers/server/Cargo.toml b/workers/server/Cargo.toml index 2fffa65..ce9b279 100644 --- a/workers/server/Cargo.toml +++ b/workers/server/Cargo.toml @@ -51,6 +51,7 @@ tokio = { workspace = true, features = ["io-util", "macros", "net", "rt-multi-th tokio-util = { workspace = true, features = ["rt", "io-util"] } tonic = "0.14" tonic-prost = "0.14" +tower = { workspace = true, features = ["util"] } tracing.workspace = true tracing-subscriber = { workspace = true, features = ["env-filter", "fmt"] } tracing-opentelemetry = { version = "0.34.0", default-features = false, optional = true } @@ -64,5 +65,4 @@ protoc-bin-vendored = "3" [dev-dependencies] hegeltest.workspace = true -tower = { workspace = true, features = ["util"] } zstd = { workspace = true, features = ["pkg-config"] } diff --git a/workers/server/src/app.rs b/workers/server/src/app.rs index 59d76c7..194fbd7 100644 --- a/workers/server/src/app.rs +++ b/workers/server/src/app.rs @@ -271,6 +271,7 @@ pub fn router(state: AppState) -> Router { .merge(crate::go::router()) .merge(crate::sccache::router()) .merge(crate::bazel::router(state.clone())) + .merge(crate::cache_host::product_routes(state.clone())) .merge(management::router(state.clone())) .route("/_api/v1", any(api_not_found)) .route("/_api/v1/{*unmatched}", any(api_not_found)) diff --git a/workers/server/src/cache_host.rs b/workers/server/src/cache_host.rs index ff82329..b00ac95 100644 --- a/workers/server/src/cache_host.rs +++ b/workers/server/src/cache_host.rs @@ -1,16 +1,107 @@ //! Direct organization ingress. Only Host is authoritative, never forwarded headers. use axum::{ - extract::{Request, State}, - http::{StatusCode, header, uri::Authority}, + Router, + extract::{ConnectInfo, MatchedPath, Request, State}, + http::{Extensions, StatusCode, header, uri::Authority}, middleware::Next, response::{IntoResponse, Response}, + routing::any, }; +use std::net::SocketAddr; +use tower::ServiceExt; use crate::{AppState, cache, config::Config}; #[derive(Clone)] pub(crate) struct OrganizationHost(pub String); +/// Products served at `//` on an organization host. +pub(crate) const PRODUCTS: &[&str] = &["go", "sccache"]; + +/// The path the client used on an organization host, before dispatch +/// rewrote it to the protocol's canonical route. +#[derive(Clone)] +pub(crate) struct PublicPath(pub String); + +/// `///…`. A product only on an organization host; on any +/// other host this shape is a path-based Nix read. +pub(crate) fn product_routes(state: AppState) -> Router { + // Protocol routers only: an organization host must never reach + // management, upload, or operator routes. + let protocols: Router = Router::new() + .merge(crate::go::router()) + .merge(crate::sccache::router()) + .fallback(|| async { cache::not_found() }) + .with_state(state); + let mut routes = Router::new(); + for &product in PRODUCTS { + for tail in ["", "/", "/{*rest}"] { + let protocols = protocols.clone(); + routes = routes.route( + &format!("/{{cache}}/{product}{tail}"), + any(move |State(state): State, request: Request| { + dispatch(protocols.clone(), product, state, request) + }), + ); + } + } + routes +} + +async fn dispatch( + protocols: Router, + product: &'static str, + state: AppState, + request: Request, +) -> Response { + let Some(owner) = request + .extensions() + .get::() + .map(|host| host.0.clone()) + else { + return match crate::reads::handle(State(state), request).await { + Ok(response) => response, + Err(error) => error.into_response(), + }; + }; + let public = request.uri().path().to_owned(); + let target = public + .strip_prefix('/') + .and_then(|path| path.split_once('/')) + .and_then(|(name, rest)| { + // The route matched `//` followed by `/` or nothing. + let tail = rest + .strip_prefix(product) + .filter(|tail| tail.is_empty() || tail.starts_with('/'))?; + let query = request + .uri() + .query() + .map(|q| format!("?{q}")) + .unwrap_or_default(); + Some(format!("/_api/v1/{product}/{owner}/{name}{tail}{query}")) + }) + .and_then(|target| target.parse::().ok()); + let Some(target) = target else { + return cache::not_found().into_response(); + }; + // Rebuild the extensions: axum appends inner captures after the outer + // `{cache}` capture, which breaks the protocol handlers' `Path` extractors. + // Go and sccache read no request extensions; the peer address is carried + // for handlers that audit by IP. + let (mut parts, body) = request.into_parts(); + let mut extensions = Extensions::new(); + if let Some(peer) = parts.extensions.get::>() { + extensions.insert(*peer); + } + extensions.insert(PublicPath(public)); + parts.extensions = extensions; + parts.uri = target; + match protocols.oneshot(Request::from_parts(parts, body)).await { + Ok(response) => response, + Err(never) => match never {}, + } +} + /// Names that conflict with the core's path-based protocol namespace. pub(crate) const PROTOCOL_NAMES: &[&str] = &["default", "nar", "private", "_api", "healthz"]; @@ -130,9 +221,21 @@ pub(crate) async fn ingress( request .extensions_mut() .insert(OrganizationHost(owner.to_owned())); - match crate::reads::handle(State(state), request).await { - Ok(response) => response, - Err(error) => error.into_response(), + // Product routes are the only ones registered under `/{cache}/`. + // The cache router must be merged, not nested: a nest would prefix + // the matched path, so product requests would miss this check and + // fall back to Nix reads. + let product = request + .extensions() + .get::() + .is_some_and(|path| path.as_str().starts_with("/{cache}/")); + if product { + next.run(request).await + } else { + match crate::reads::handle(State(state), request).await { + Ok(response) => response, + Err(error) => error.into_response(), + } } }; // Every alias hit must observe current SQL policy, even with a warm CDN. diff --git a/workers/server/src/sccache.rs b/workers/server/src/sccache.rs index 6b14d4e..623da03 100644 --- a/workers/server/src/sccache.rs +++ b/workers/server/src/sccache.rs @@ -140,7 +140,10 @@ async fn propfind(state: AppState, path: DavPath, request: Request) -> Result() + .map_or(request.uri.path(), |path| path.0.as_str()); let xml = format!( "{pathname}{modified}HTTP/1.1 200 OK" ); diff --git a/workers/server/test/compiler.mjs b/workers/server/test/compiler.mjs index 2145820..b333ea3 100644 --- a/workers/server/test/compiler.mjs +++ b/workers/server/test/compiler.mjs @@ -4,6 +4,7 @@ import assert from "node:assert/strict"; import { createHash, randomBytes, randomUUID } from "node:crypto"; import { execFile, spawn } from "node:child_process"; import { once } from "node:events"; +import { createServer, request as httpRequest } from "node:http"; import { mkdir, mkdtemp, readFile, rm, utimes, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; @@ -16,7 +17,7 @@ const exec = promisify(execFile); const serverBinary = new URL("../../../target/debug/snowstead-server", import.meta.url).pathname; const goSource = new URL("../../../apps/go-cache/", import.meta.url).pathname; -test("unchanged compiler clients reuse Rust TCP artifacts with fresh local caches", { timeout: 240_000 }, async t => { +test("unchanged compiler clients reuse Rust TCP artifacts with fresh local caches", { timeout: 480_000 }, async t => { const base = process.env.SNOWSTEAD_TEST_DATABASE_URL; assert.ok(base, "run via scripts/with-postgres.sh inside nix develop"); assert.equal(process.env.R2_ENDPOINT, process.env.SNOWSTEAD_TEST_R2_ENDPOINT, "run via scripts/with-r2.sh"); @@ -54,6 +55,7 @@ test("unchanged compiler clients reuse Rust TCP artifacts with fresh local cache SIGNING_PRIVATE_KEY: "", REQUIRE_CACHE_SIGNING_KEYS: "false", PUBLIC_URL: "", UPSTREAM_CACHES: "", PUBLIC_CACHE_ORIGIN: "", CDN_ZONE_ID: "", CDN_PURGE_TOKEN: "", ANALYTICS_TOKEN: "", TRUSTED_PROXY_IPS: "", + CACHE_HOST_SUFFIX: "cache.test", CACHE_CANONICAL_HOSTS: "", CACHE_RESERVED_NAMES: "", CACHE_RESERVED_HOSTS: "", DATABASE_MAX_CONNECTIONS: "2", LISTEN_ADDR: "127.0.0.1:0", RUST_LOG: "info", }; await exec(serverBinary, ["migrate"], { env, timeout: 30_000 }); @@ -78,6 +80,21 @@ test("unchanged compiler clients reuse Rust TCP artifacts with fresh local cache await db.query("INSERT INTO token_caches(token_jti,cache_id) VALUES ($1,$2)", [jti, id]); tokens[protocol] = `snowstead_tok_${jti}_${tokenSecret}`; } + // A private organization cache reached through its organization host. + await db.query("INSERT INTO orgs(id,slug) VALUES ('org_alpha','alpha')"); + const { rows: [{ id: teamId }] } = await db.query( + "INSERT INTO caches(name,org_id,is_public) VALUES ('team','org_alpha',false) RETURNING id"); + await db.query("INSERT INTO cache_protocols(cache_id,protocol) VALUES ($1,'go'),($1,'sccache')", [teamId]); + const orgJti = randomUUID(); + const orgSecret = randomBytes(32).toString("base64url"); + await db.query( + "INSERT INTO api_tokens(jti,subject,secret_hash,created_by,org_id,expires_at) VALUES ($1,'compiler-fixture',$2,'test','org_alpha',now()+interval '1 hour')", + [orgJti, createHash("sha256").update(orgSecret).digest("hex")]); + await db.query( + "INSERT INTO token_permissions(token_jti,permission,position) VALUES ($1,'go:read',1),($1,'go:write',2),($1,'sccache:read',3),($1,'sccache:write',4)", + [orgJti]); + await db.query("INSERT INTO token_caches(token_jti,cache_id) VALUES ($1,$2)", [orgJti, teamId]); + tokens.org = `snowstead_tok_${orgJti}_${orgSecret}`; } finally { await db.end(); } const token = protocol => tokens[protocol]; server = spawn(serverBinary, [], { env, stdio: ["ignore", "pipe", "inherit"] }); @@ -91,40 +108,64 @@ test("unchanged compiler clients reuse Rust TCP artifacts with fresh local cache const address = /127\.0\.0\.1:\d+/.exec(line)?.[0]; assert.ok(address, line); const url = `http://${address}`; - - // Exercise unmodified Rust and C++ compiler clients with fresh local caches. - const dir = join(home, "sccache"); - await mkdir(dir); - const clientEnv = Object.fromEntries(Object.entries(process.env).filter(([key]) => !key.startsWith("SCCACHE_"))); - Object.assign(clientEnv, { - SCCACHE_SERVER_UDS: join(dir, "server.sock"), SCCACHE_CONF: join(dir, "config"), - SCCACHE_WEBDAV_ENDPOINT: `${url}/_api/v1/sccache/default/sccache/`, - SCCACHE_WEBDAV_TOKEN: token("sccache"), + // Stand in for nginx on the organization host: forward to the Rust service + // with the Host the organization address would carry. The Go helper refuses + // plain HTTP except on loopback, so this stays on 127.0.0.1. + const [upstreamHost, upstreamPort] = address.split(":"); + const front = createServer((incoming, outgoing) => { + const upstream = httpRequest({ host: upstreamHost, port: upstreamPort, method: incoming.method, + path: incoming.url, headers: { ...incoming.headers, host: "alpha.cache.test" } }, response => { + outgoing.writeHead(response.statusCode, response.headers); + response.pipe(outgoing); + }); + upstream.on("error", () => outgoing.destroy()); + incoming.pipe(upstream); }); - const run = args => exec("sccache", args, { env: clientEnv, cwd: dir, timeout: 30_000 }); - try { - await writeFile(join(dir, "config"), ""); - await writeFile(join(dir, "input.cpp"), "int cached(int n) { return n * 7 + 3; }\n"); - await writeFile(join(dir, "input.rs"), "pub fn cached(n: u64) -> u64 { n * 11 + 5 }\n"); - let expected; - for (let pass = 0; pass < 2; pass++) { - clientEnv.SCCACHE_DIR = join(dir, `local-${pass}`); - await run(["--start-server"]); - await run(["c++", "-c", "input.cpp", "-o", "cpp.o", "-O2"]); - await run(["rustc", "--crate-name", "fixture", "--crate-type", "rlib", "--emit=link", "input.rs", "--out-dir", ".", "-O"]); - const stats = JSON.parse((await run(["--show-stats", "--stats-format", "json"])).stdout).stats; - const sum = counts => Object.values(counts).reduce((a, b) => a + b, 0); - assert.equal(sum(stats.cache_hits.counts), pass ? 2 : 0, JSON.stringify(stats)); - assert.equal(sum(stats.cache_misses.counts), pass ? 0 : 2, JSON.stringify(stats)); - assert.equal(stats.cache_write_errors, 0, JSON.stringify(stats)); - const outputs = await Promise.all(["cpp.o", "libfixture.rlib"].map(name => readFile(join(dir, name)))); - if (expected) assert.deepEqual(outputs, expected); - expected = outputs; - console.log(`sccache ${pass ? "warm" : "cold"}: hits=${sum(stats.cache_hits.counts)} misses=${sum(stats.cache_misses.counts)} write_errors=${stats.cache_write_errors} output_bytes=${outputs.map(b => b.length).join(",")}`); - await run(["--stop-server"]); - await Promise.all(["cpp.o", "libfixture.rlib"].map(name => rm(join(dir, name)))); - } - } finally { await run(["--stop-server"]).catch(() => {}); } + front.listen(0, "127.0.0.1"); + await once(front, "listening"); + t.after(() => { front.closeAllConnections(); front.close(); }); + const orgUrl = `http://127.0.0.1:${front.address().port}`; + + // Exercise unmodified Rust and C++ compiler clients with fresh local caches, + // on the shared address and on the organization host. + const sccacheEndpoints = [ + ["shared", `${url}/_api/v1/sccache/default/sccache/`, token("sccache")], + ["org", `${orgUrl}/team/sccache/`, tokens.org], + ]; + for (const [label, endpoint, bearer] of sccacheEndpoints) { + const dir = join(home, `sccache-${label}`); + await mkdir(dir); + const clientEnv = Object.fromEntries(Object.entries(process.env).filter(([key]) => !key.startsWith("SCCACHE_"))); + Object.assign(clientEnv, { + SCCACHE_SERVER_UDS: join(dir, "server.sock"), SCCACHE_CONF: join(dir, "config"), + SCCACHE_WEBDAV_ENDPOINT: endpoint, + SCCACHE_WEBDAV_TOKEN: bearer, + }); + const run = args => exec("sccache", args, { env: clientEnv, cwd: dir, timeout: 30_000 }); + try { + await writeFile(join(dir, "config"), ""); + await writeFile(join(dir, "input.cpp"), "int cached(int n) { return n * 7 + 3; }\n"); + await writeFile(join(dir, "input.rs"), "pub fn cached(n: u64) -> u64 { n * 11 + 5 }\n"); + let expected; + for (let pass = 0; pass < 2; pass++) { + clientEnv.SCCACHE_DIR = join(dir, `local-${pass}`); + await run(["--start-server"]); + await run(["c++", "-c", "input.cpp", "-o", "cpp.o", "-O2"]); + await run(["rustc", "--crate-name", "fixture", "--crate-type", "rlib", "--emit=link", "input.rs", "--out-dir", ".", "-O"]); + const stats = JSON.parse((await run(["--show-stats", "--stats-format", "json"])).stdout).stats; + const sum = counts => Object.values(counts).reduce((a, b) => a + b, 0); + assert.equal(sum(stats.cache_hits.counts), pass ? 2 : 0, JSON.stringify(stats)); + assert.equal(sum(stats.cache_misses.counts), pass ? 0 : 2, JSON.stringify(stats)); + assert.equal(stats.cache_write_errors, 0, JSON.stringify(stats)); + const outputs = await Promise.all(["cpp.o", "libfixture.rlib"].map(name => readFile(join(dir, name)))); + if (expected) assert.deepEqual(outputs, expected); + expected = outputs; + console.log(`sccache ${label} ${pass ? "warm" : "cold"}: hits=${sum(stats.cache_hits.counts)} misses=${sum(stats.cache_misses.counts)} write_errors=${stats.cache_write_errors} output_bytes=${outputs.map(b => b.length).join(",")}`); + await run(["--stop-server"]); + await Promise.all(["cpp.o", "libfixture.rlib"].map(name => rm(join(dir, name)))); + } + } finally { await run(["--stop-server"]).catch(() => {}); } + } // Build the unchanged helper with an isolated Go cache, not the user's cache. const goEnv = Object.fromEntries(Object.entries(process.env).filter(([key]) => !key.startsWith("GO") && !key.startsWith("SNOWSTEAD_GO_"))); @@ -142,39 +183,44 @@ test("unchanged compiler clients reuse Rust TCP artifacts with fresh local cache // Make both passes eligible instead of creating a warm-only index record. const timestamp = new Date(Date.now() - 60_000); for (const name of ["go.mod", "main.go"]) await utimes(join(project, name), timestamp, timestamp); - const goToken = token("go"); - let expectedGo; - for (let pass = 0; pass < 2; pass++) { - const output = join(project, "fixture"); - // This throwaway module has no VCS metadata (nor any need for stamping). - const result = await exec("go", ["build", "-trimpath", "-buildvcs=false", "-o", output, "."], { - cwd: project, timeout: 120_000, maxBuffer: 4 * 1024 * 1024, - env: { ...goEnv, GOCACHE: join(home, `go-local-${pass}`), GOCACHEPROG: helper, - SNOWSTEAD_GO_CACHE_DIR: join(home, `go-helper-${pass}`), - SNOWSTEAD_GO_CACHE_URL: `${url}/_api/v1/go/default/go`, SNOWSTEAD_GO_CACHE_TOKEN: goToken, - SNOWSTEAD_GO_CACHE_READ_ONLY: "false" }, - }); - const summary = result.stderr.split("\n").filter(line => line.startsWith("snowstead-go-cache: local_hits=")); - assert.equal(summary.length, 1, result.stderr); - const stats = Object.fromEntries([...summary[0].matchAll(/(\w+)=(\w+)/g)].map(([, key, value]) => [key, value === "true" ? true : value === "false" ? false : Number(value)])); - for (const key of ["upload_errors", "read_errors", "read_timeouts", "conflicts"]) assert.equal(stats[key], 0, summary[0]); - assert.equal(stats.remote_disabled, false, summary[0]); - if (pass === 0) { - assert.equal(stats.remote_hits, 0, summary[0]); - assert.ok(stats.misses > 0 && stats.uploads > 0 && stats.uploaded_bytes > 0, summary[0]); - } else { - // Go does not request every cold metadata put on a warm build, and - // linking can still put an empty stdout record. Require real artifact - // downloads and no newly uploaded payload, not identical request counts. - assert.ok(stats.remote_hits > 0 && stats.downloaded_bytes >= expectedGo.length, summary[0]); - assert.equal(stats.uploaded_bytes, 0, summary[0]); + const goEndpoints = [ + ["shared", `${url}/_api/v1/go/default/go`, token("go")], + ["org", `${orgUrl}/team/go`, tokens.org], + ]; + for (const [label, endpoint, bearer] of goEndpoints) { + let expectedGo; + for (let pass = 0; pass < 2; pass++) { + const output = join(project, "fixture"); + // This throwaway module has no VCS metadata (nor any need for stamping). + const result = await exec("go", ["build", "-trimpath", "-buildvcs=false", "-o", output, "."], { + cwd: project, timeout: 120_000, maxBuffer: 4 * 1024 * 1024, + env: { ...goEnv, GOCACHE: join(home, `go-local-${label}-${pass}`), GOCACHEPROG: helper, + SNOWSTEAD_GO_CACHE_DIR: join(home, `go-helper-${label}-${pass}`), + SNOWSTEAD_GO_CACHE_URL: endpoint, SNOWSTEAD_GO_CACHE_TOKEN: bearer, + SNOWSTEAD_GO_CACHE_READ_ONLY: "false" }, + }); + const summary = result.stderr.split("\n").filter(line => line.startsWith("snowstead-go-cache: local_hits=")); + assert.equal(summary.length, 1, result.stderr); + const stats = Object.fromEntries([...summary[0].matchAll(/(\w+)=(\w+)/g)].map(([, key, value]) => [key, value === "true" ? true : value === "false" ? false : Number(value)])); + for (const key of ["upload_errors", "read_errors", "read_timeouts", "conflicts"]) assert.equal(stats[key], 0, summary[0]); + assert.equal(stats.remote_disabled, false, summary[0]); + if (pass === 0) { + assert.equal(stats.remote_hits, 0, summary[0]); + assert.ok(stats.misses > 0 && stats.uploads > 0 && stats.uploaded_bytes > 0, summary[0]); + } else { + // Go does not request every cold metadata put on a warm build, and + // linking can still put an empty stdout record. Require real artifact + // downloads and no newly uploaded payload, not identical request counts. + assert.ok(stats.remote_hits > 0 && stats.downloaded_bytes >= expectedGo.length, summary[0]); + assert.equal(stats.uploaded_bytes, 0, summary[0]); + } + const bytes = await readFile(output); + if (expectedGo) assert.deepEqual(bytes, expectedGo); + expectedGo = bytes; + const executed = await exec(output, [], { timeout: 10_000 }); + assert.equal(executed.stderr, "snowstead compiler fixture\n"); + console.log(`Go ${label} ${pass ? "warm" : "cold"}: ${summary[0]} executable_bytes=${bytes.length} sha256=${createHash("sha256").update(bytes).digest("hex")}`); + await rm(output); } - const bytes = await readFile(output); - if (expectedGo) assert.deepEqual(bytes, expectedGo); - expectedGo = bytes; - const executed = await exec(output, [], { timeout: 10_000 }); - assert.equal(executed.stderr, "snowstead compiler fixture\n"); - console.log(`Go ${pass ? "warm" : "cold"}: ${summary[0]} executable_bytes=${bytes.length} sha256=${createHash("sha256").update(bytes).digest("hex")}`); - await rm(output); } }); diff --git a/workers/server/tests/org_host.rs b/workers/server/tests/org_host.rs new file mode 100644 index 0000000..cd02036 --- /dev/null +++ b/workers/server/tests/org_host.rs @@ -0,0 +1,598 @@ +mod common; + +use axum::{ + Router, + body::{Body, to_bytes}, + http::{HeaderMap, Request, StatusCode}, +}; +use base64::{Engine, engine::general_purpose::URL_SAFE_NO_PAD}; +use hmac::{Hmac, KeyInit, Mac}; +use serde_json::json; +use sha2::{Digest, Sha256}; +use snowstead_server::{AppState, config::Config}; +use sqlx::PgPool; +use tower::ServiceExt; + +fn jwt(org: &str, cache: &str, perms: &[&str]) -> String { + let claims = json!({"jti":format!("compiler-{org}-{cache}-{}", perms.join("+")),"sub":"compiler","iss":"snowstead","aud":"snowstead","iat":1,"org":org,"caches":[cache],"perms":perms}); + let input = format!( + "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.{}", + URL_SAFE_NO_PAD.encode(serde_json::to_vec(&claims).unwrap()) + ); + let mut mac = Hmac::::new_from_slice(b"fixture-jwt").unwrap(); + mac.update(input.as_bytes()); + format!( + "{input}.{}", + URL_SAFE_NO_PAD.encode(mac.finalize().into_bytes()) + ) +} + +#[path = "support/tokens.rs"] +mod tokens; + +/// A listed JWT-era token, which works with the scope its row records. +async fn token(pool: &PgPool, org: &str, cache: &str, perms: &[&str]) -> String { + tokens::register(pool, jwt(org, cache, perms)).await +} + +fn config(suffix: &str) -> Config { + Config { + jwt_secret: "fixture-jwt".into(), + cache_host_suffix: suffix.into(), + cache_canonical_hosts: "api.snowstead.com".into(), + ..Default::default() + } +} + +async fn seed(pool: &PgPool) { + sqlx::raw_sql( + "INSERT INTO orgs(id,slug) VALUES ('org_a','alpha'),('org_b','beta'); + INSERT INTO caches(name,org_id,is_public,created_at) VALUES + ('team','org_a',false,'2020-01-02 03:04:05+00'), + ('team','org_b',false,'2020-01-02 03:04:05+00'), + ('go','org_a',true,'2020-01-02 03:04:05+00'), + ('sccache','org_a',true,'2020-01-02 03:04:05+00'); + INSERT INTO cache_protocols(cache_id,protocol) + SELECT id,kind FROM caches CROSS JOIN (VALUES ('go'),('sccache'),('nix')) p(kind);", + ) + .execute(pool) + .await + .unwrap(); +} + +async fn app(pool: &PgPool, suffix: &str) -> Router { + let state = AppState::new( + pool.clone(), + config(suffix), + common::storage(), + reqwest::Client::new(), + ) + .unwrap(); + common::router(state) +} + +async fn send( + app: &Router, + method: &str, + host: Option<&str>, + path: &str, + token: Option<&str>, + headers: &[(&str, &str)], + bytes: &[u8], +) -> (StatusCode, HeaderMap, Vec) { + let mut req = Request::builder().method(method).uri(path); + if let Some(host) = host { + req = req.header("host", host); + } + if let Some(token) = token { + req = req.header("authorization", format!("Bearer {token}")); + } + for (key, value) in headers { + req = req.header(*key, *value); + } + let response = app + .clone() + .oneshot(req.body(Body::from(bytes.to_vec())).unwrap()) + .await + .unwrap(); + let status = response.status(); + let headers = response.headers().clone(); + ( + status, + headers, + to_bytes(response.into_body(), 1024 * 1024) + .await + .unwrap() + .to_vec(), + ) +} + +/// A Go action upload: the body commits to its own SHA-256 output ID. +async fn put_go( + app: &Router, + host: &str, + path: &str, + token: &str, + bytes: &[u8], +) -> (StatusCode, HeaderMap, Vec) { + let length = bytes.len().to_string(); + let output = hex::encode(Sha256::digest(bytes)); + send( + app, + "PUT", + Some(host), + path, + Some(token), + &[ + ("content-length", length.as_str()), + ("x-snowstead-output-id", output.as_str()), + ], + bytes, + ) + .await +} + +async fn put_bytes( + app: &Router, + host: &str, + path: &str, + token: &str, + bytes: &[u8], +) -> (StatusCode, HeaderMap, Vec) { + let length = bytes.len().to_string(); + send( + app, + "PUT", + Some(host), + path, + Some(token), + &[("content-length", length.as_str())], + bytes, + ) + .await +} + +#[sqlx::test(migrator = "snowstead_server::migrations::MIGRATOR")] +async fn go_reads_and_writes_through_the_organization_host(pool: PgPool) { + seed(&pool).await; + let app = app(&pool, "snowstead.com").await; + let token = token(&pool, "org_a", "team", &["go:read", "go:write"]).await; + let first = "a".repeat(64); + let second = "b".repeat(64); + let org_path = format!("/team/go/v1/actions/{first}"); + let legacy_path = format!("/_api/v1/go/alpha/team/v1/actions/{first}"); + let body: &[u8] = b"go action output"; + let second_body: &[u8] = b"second go action output"; + + let (status, headers, _) = send( + &app, + "GET", + Some("alpha.snowstead.com"), + &org_path, + None, + &[], + b"", + ) + .await; + assert_eq!(status, StatusCode::UNAUTHORIZED); + assert_eq!(headers["cloudflare-cdn-cache-control"], "no-store"); + + let (status, headers, _) = put_go(&app, "alpha.snowstead.com", &org_path, &token, body).await; + assert_eq!(status, StatusCode::CREATED); + assert_eq!(headers["cloudflare-cdn-cache-control"], "no-store"); + + let (status, headers, bytes) = send( + &app, + "GET", + Some("alpha.snowstead.com"), + &org_path, + Some(&token), + &[], + b"", + ) + .await; + assert_eq!(status, StatusCode::OK); + assert_eq!(headers["cloudflare-cdn-cache-control"], "no-store"); + assert_eq!(bytes, body); + + let (status, _, bytes) = send( + &app, + "HEAD", + Some("alpha.snowstead.com"), + &org_path, + Some(&token), + &[], + b"", + ) + .await; + assert_eq!(status, StatusCode::OK); + assert!(bytes.is_empty()); + + let with_query = format!("{org_path}?x=1"); + let (status, _, bytes) = send( + &app, + "GET", + Some("alpha.snowstead.com"), + &with_query, + Some(&token), + &[], + b"", + ) + .await; + assert_eq!(status, StatusCode::OK); + assert_eq!(bytes, body); + + let (status, _, bytes) = send( + &app, + "GET", + Some("api.snowstead.com"), + &legacy_path, + Some(&token), + &[], + b"", + ) + .await; + assert_eq!(status, StatusCode::OK); + assert_eq!(bytes, body); + + let legacy_second = format!("/_api/v1/go/alpha/team/v1/actions/{second}"); + let (status, _, _) = put_go( + &app, + "api.snowstead.com", + &legacy_second, + &token, + second_body, + ) + .await; + assert_eq!(status, StatusCode::CREATED); + let org_second = format!("/team/go/v1/actions/{second}"); + let (status, _, bytes) = send( + &app, + "GET", + Some("alpha.snowstead.com"), + &org_second, + Some(&token), + &[], + b"", + ) + .await; + assert_eq!(status, StatusCode::OK); + assert_eq!(bytes, second_body); + + let (status, headers, _) = send( + &app, + "GET", + Some("beta.snowstead.com"), + &org_path, + Some(&token), + &[], + b"", + ) + .await; + assert_eq!(status, StatusCode::FORBIDDEN); + assert_eq!(headers["cloudflare-cdn-cache-control"], "no-store"); + + let (status, _, _) = send( + &app, + "GET", + Some("api.snowstead.com"), + &org_path, + None, + &[], + b"", + ) + .await; + assert_eq!(status, StatusCode::NOT_FOUND); + let (status, _, _) = send(&app, "GET", None, &org_path, None, &[], b"").await; + assert_eq!(status, StatusCode::NOT_FOUND); + + let (status, headers, _) = send( + &app, + "DELETE", + Some("alpha.snowstead.com"), + &org_path, + Some(&token), + &[], + b"", + ) + .await; + assert_eq!(status, StatusCode::METHOD_NOT_ALLOWED); + assert_eq!(headers["allow"], "GET, HEAD, PUT"); + + for path in [ + "/team/go".to_owned(), + "/team/go/".to_owned(), + "/team/go/v1/actions/short".to_owned(), + format!("/te%61m/go/v1/actions/{first}"), + ] { + let (status, _, _) = send( + &app, + "GET", + Some("alpha.snowstead.com"), + &path, + None, + &[], + b"", + ) + .await; + assert_eq!(status, StatusCode::NOT_FOUND, "{path}"); + } + + let (status, headers, _) = send( + &app, + "GET", + Some("a.b.snowstead.com"), + &org_path, + None, + &[], + b"", + ) + .await; + assert_eq!(status, StatusCode::NOT_FOUND); + assert_eq!(headers["cloudflare-cdn-cache-control"], "no-store"); +} + +#[sqlx::test(migrator = "snowstead_server::migrations::MIGRATOR")] +async fn sccache_webdav_through_the_organization_host(pool: PgPool) { + seed(&pool).await; + let app = app(&pool, "snowstead.com").await; + let read = token(&pool, "org_a", "team", &["sccache:read"]).await; + let write = token(&pool, "org_a", "team", &["sccache:read", "sccache:write"]).await; + let artifact = "a".repeat(64); + let artifact_path = format!("/a/a/a/{artifact}"); + let org_artifact = format!("/team/sccache{artifact_path}"); + let legacy_artifact = format!("/_api/v1/sccache/alpha/team{artifact_path}"); + let bytes: &[u8] = b"sccache artifact bytes"; + let org = "alpha.snowstead.com"; + let api = "api.snowstead.com"; + + let (status, _, body) = send( + &app, + "PROPFIND", + Some(org), + "/team/sccache/", + Some(&read), + &[], + b"", + ) + .await; + assert_eq!(status, StatusCode::MULTI_STATUS); + assert!( + std::str::from_utf8(&body) + .unwrap() + .contains("/team/sccache/") + ); + + let (status, _, body) = send( + &app, + "PROPFIND", + Some(api), + "/_api/v1/sccache/alpha/team/", + Some(&read), + &[], + b"", + ) + .await; + assert_eq!(status, StatusCode::MULTI_STATUS); + assert!( + std::str::from_utf8(&body) + .unwrap() + .contains("/_api/v1/sccache/alpha/team/") + ); + + let (status, _, _) = send( + &app, + "MKCOL", + Some(org), + "/team/sccache/a/", + Some(&write), + &[], + b"", + ) + .await; + assert_eq!(status, StatusCode::CREATED); + + let (status, _, _) = put_bytes(&app, org, "/team/sccache/.sccache_check", &write, b"x").await; + assert_eq!(status, StatusCode::NO_CONTENT); + + let (status, _, _) = put_bytes(&app, org, &org_artifact, &write, bytes).await; + assert_eq!(status, StatusCode::CREATED); + + let (status, _, body) = + send(&app, "GET", Some(org), &org_artifact, Some(&read), &[], b"").await; + assert_eq!(status, StatusCode::OK); + assert_eq!(body, bytes); + + let (status, _, body) = send( + &app, + "GET", + Some(api), + &legacy_artifact, + Some(&read), + &[], + b"", + ) + .await; + assert_eq!(status, StatusCode::OK); + assert_eq!(body, bytes); + + let (status, _, _) = send( + &app, + "GET", + Some(org), + "/team/sccache", + Some(&read), + &[], + b"", + ) + .await; + assert_eq!(status, StatusCode::NOT_FOUND); + + let (status, _, _) = send( + &app, + "PROPFIND", + Some(org), + "/team/sccache//a/", + Some(&read), + &[], + b"", + ) + .await; + assert_eq!(status, StatusCode::MULTI_STATUS); + + let (status, _, _) = send( + &app, + "MKCOL", + Some(org), + "/team/sccache/a/b/", + Some(&write), + &[], + b"", + ) + .await; + assert_eq!(status, StatusCode::CREATED); + let (status, _, body) = send( + &app, + "PROPFIND", + Some(org), + "/team/sccache/a/b/", + Some(&read), + &[], + b"", + ) + .await; + assert_eq!(status, StatusCode::MULTI_STATUS); + assert!( + std::str::from_utf8(&body) + .unwrap() + .contains("/team/sccache/a/b/") + ); + + let (status, _, _) = send( + &app, + "GET", + Some("beta.snowstead.com"), + &org_artifact, + Some(&read), + &[], + b"", + ) + .await; + assert_eq!(status, StatusCode::FORBIDDEN); +} + +#[sqlx::test(migrator = "snowstead_server::migrations::MIGRATOR")] +async fn caches_named_after_products_keep_their_nix_addresses(pool: PgPool) { + seed(&pool).await; + let plain = app(&pool, "").await; + let app = app(&pool, "snowstead.com").await; + let go_write = token(&pool, "org_a", "go", &["go:read", "go:write"]).await; + let action = "a".repeat(64); + + for name in ["go", "sccache"] { + let nix = format!("/{name}/nix-cache-info"); + let (status, headers, _) = send( + &app, + "GET", + Some("alpha.snowstead.com"), + &nix, + None, + &[], + b"", + ) + .await; + assert_eq!(status, StatusCode::OK, "{name}"); + assert_eq!(headers["content-type"], "text/x-nix-cache-info", "{name}"); + + let shared = format!("/alpha/{name}/nix-cache-info"); + for (router, host) in [ + (&app, Some("api.snowstead.com")), + (&app, None), + (&plain, Some("api.snowstead.com")), + (&plain, None), + ] { + let (status, _, _) = send(router, "GET", host, &shared, None, &[], b"").await; + assert_eq!(status, StatusCode::OK, "{name} {host:?}"); + } + + let (status, headers, _) = send( + &app, + "PUT", + Some("api.snowstead.com"), + &shared, + None, + &[], + b"", + ) + .await; + assert_eq!(status, StatusCode::METHOD_NOT_ALLOWED, "{name}"); + assert_eq!(headers["allow"], "GET, HEAD", "{name}"); + } + + let (status, _, _) = put_go( + &app, + "alpha.snowstead.com", + &format!("/go/go/v1/actions/{action}"), + &go_write, + b"go cache named go", + ) + .await; + assert_eq!(status, StatusCode::CREATED); + let (status, _, body) = send( + &app, + "GET", + Some("alpha.snowstead.com"), + &format!("/go/go/v1/actions/{action}"), + Some(&go_write), + &[], + b"", + ) + .await; + assert_eq!(status, StatusCode::OK); + assert_eq!(body, b"go cache named go"); +} + +#[sqlx::test(migrator = "snowstead_server::migrations::MIGRATOR")] +async fn other_organization_host_paths_stay_nix_reads(pool: PgPool) { + seed(&pool).await; + let app = app(&pool, "snowstead.com").await; + let action = "a".repeat(64); + + let (status, headers, _) = send( + &app, + "PUT", + Some("alpha.snowstead.com"), + "/team/zig/x", + None, + &[], + b"", + ) + .await; + assert_eq!(status, StatusCode::METHOD_NOT_ALLOWED); + assert_eq!(headers["allow"], "GET, HEAD"); + + let (status, headers, _) = send( + &app, + "PUT", + Some("alpha.snowstead.com"), + "/team/nix-cache-info", + None, + &[], + b"", + ) + .await; + assert_eq!(status, StatusCode::METHOD_NOT_ALLOWED); + assert_eq!(headers["allow"], "GET, HEAD"); + + let (status, _, _) = send( + &app, + "GET", + Some("alpha.snowstead.com"), + &format!("/_api/v1/go/alpha/team/v1/actions/{action}"), + None, + &[], + b"", + ) + .await; + assert_eq!(status, StatusCode::NOT_FOUND); +}