diff --git a/docs.md b/docs.md index 5b39688..9a7d465 100644 --- a/docs.md +++ b/docs.md @@ -558,7 +558,16 @@ organization names with `CACHE_RESERVED_NAMES` and exact DNS hosts with `CACHE_RESERVED_HOSTS`; both accept whitespace-separated lowercase values. Reserved hosts take precedence over canonical hosts. Core protocol names remain reserved independently of these settings. Hosted DNS policy belongs -to the embedding application, not the cache core. +to the embedding application, not the cache core. On an organization host, +`//go`, `//sccache/`, and `//bazel` (Bazel over HTTP) +serve the same handlers as `/_api/v1///`, which +keeps working; every other path on that host, apart from the Bazel gRPC +services, is a Nix read. Bazel over gRPC names the organization by host and +the cache by instance: +`--remote_cache=grpcs://. --remote_instance_name=`. +The organization host rejects the `/` instance shape that the +shared host keeps. Organization hosts answer HTTP/2 requests that send no +`Host` header by their `:authority`. ### Migrating JWT-era tokens diff --git a/workers/server/Cargo.toml b/workers/server/Cargo.toml index 2fffa65..ce9b279 100644 --- a/workers/server/Cargo.toml +++ b/workers/server/Cargo.toml @@ -51,6 +51,7 @@ tokio = { workspace = true, features = ["io-util", "macros", "net", "rt-multi-th tokio-util = { workspace = true, features = ["rt", "io-util"] } tonic = "0.14" tonic-prost = "0.14" +tower = { workspace = true, features = ["util"] } tracing.workspace = true tracing-subscriber = { workspace = true, features = ["env-filter", "fmt"] } tracing-opentelemetry = { version = "0.34.0", default-features = false, optional = true } @@ -64,5 +65,4 @@ protoc-bin-vendored = "3" [dev-dependencies] hegeltest.workspace = true -tower = { workspace = true, features = ["util"] } zstd = { workspace = true, features = ["pkg-config"] } diff --git a/workers/server/src/app.rs b/workers/server/src/app.rs index 59d76c7..194fbd7 100644 --- a/workers/server/src/app.rs +++ b/workers/server/src/app.rs @@ -271,6 +271,7 @@ pub fn router(state: AppState) -> Router { .merge(crate::go::router()) .merge(crate::sccache::router()) .merge(crate::bazel::router(state.clone())) + .merge(crate::cache_host::product_routes(state.clone())) .merge(management::router(state.clone())) .route("/_api/v1", any(api_not_found)) .route("/_api/v1/{*unmatched}", any(api_not_found)) diff --git a/workers/server/src/bazel/grpc.rs b/workers/server/src/bazel/grpc.rs index 3814971..39e1eff 100644 --- a/workers/server/src/bazel/grpc.rs +++ b/workers/server/src/bazel/grpc.rs @@ -11,7 +11,7 @@ use tokio_util::io::ReaderStream; use tonic::{Request, Response, Status}; use super::{MAX_METADATA, blob, complete, failure, open, proto, store::Store, valid_digest}; -use crate::{AppState, error::ApiError}; +use crate::{AppState, cache_host::OrganizationHost, error::ApiError}; use proto::build::bazel::{remote::execution::v2::*, semver::SemVer}; use proto::google::bytestream::{self as bs, byte_stream_server::ByteStream}; @@ -63,6 +63,15 @@ fn wire_status(value: Result<(), Status>) -> Option }) } +/// The organization named by the host, when the request came from an +/// organization host. +fn organization(request: &Request) -> Option<&str> { + request + .extensions() + .get::() + .map(|host| host.0.as_str()) +} + impl Cache { async fn store( &self, @@ -72,6 +81,7 @@ impl Cache { ) -> Result { open( &self.0, + organization(request), instance, &request.metadata().clone().into_headers(), write, @@ -433,21 +443,26 @@ pub(super) fn router(state: AppState) -> axum::Router { ) } -fn resource(name: &str, write: bool) -> Result<(String, Digest), Status> { +/// Parses a ByteStream resource name. On an organization host the name has +/// no owner segment, so `depth` is the number of instance segments before the +/// protocol path: two (`/`) on the shared host, one (``) +/// on an organization host. +fn resource(name: &str, write: bool, org_host: bool) -> Result<(String, Digest), Status> { let parts: Vec<_> = name.split('/').collect(); + let depth = if org_host { 1 } else { 2 }; let (hash, size) = if write - && parts.len() >= 7 - && parts[2] == "uploads" - && parts[4] == "blobs" - && !parts[3].is_empty() - && parts[3].len() <= 128 - && parts[3] + && parts.len() >= depth + 5 + && parts[depth] == "uploads" + && parts[depth + 2] == "blobs" + && !parts[depth + 1].is_empty() + && parts[depth + 1].len() <= 128 + && parts[depth + 1] .bytes() .all(|c| c.is_ascii_alphanumeric() || matches!(c, b'-' | b'_')) { - (parts[5], parts[6]) - } else if !write && parts.len() == 5 && parts[2] == "blobs" { - (parts[3], parts[4]) + (parts[depth + 3], parts[depth + 4]) + } else if !write && parts.len() == depth + 3 && parts[depth] == "blobs" { + (parts[depth + 1], parts[depth + 2]) } else { return Err(Status::invalid_argument("Invalid ByteStream resource name")); }; @@ -460,7 +475,7 @@ fn resource(name: &str, write: bool) -> Result<(String, Digest), Status> { .parse() .map_err(|_| Status::invalid_argument("Invalid blob size"))?, }))?; - Ok((format!("{}/{}", parts[0], parts[1]), digest)) + Ok((parts[..depth].join("/"), digest)) } #[tonic::async_trait] @@ -470,7 +485,11 @@ impl ByteStream for Cache { &self, request: Request, ) -> Result, Status> { - let (instance, digest) = resource(&request.get_ref().resource_name, false)?; + let (instance, digest) = resource( + &request.get_ref().resource_name, + false, + organization(&request).is_some(), + )?; let store = self.store(&request, &instance, false).await?; let request = request.into_inner(); if request.read_offset < 0 || request.read_limit < 0 { @@ -542,14 +561,16 @@ impl ByteStream for Cache { let mut progress_deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(30); let headers = request.metadata().clone().into_headers(); + // Extensions do not survive `into_inner`, so capture the owner first. + let owner = organization(&request).map(str::to_owned); let mut stream = request.into_inner(); let first = tokio::time::timeout_at(progress_deadline, stream.message()) .await .map_err(|_| Status::deadline_exceeded("Upload made no byte progress"))?? .ok_or_else(|| Status::invalid_argument("Write request required"))?; let name = first.resource_name.clone(); - let (instance, digest) = resource(&name, true)?; - let (store, _) = open(&self.0, &instance, &headers, true) + let (instance, digest) = resource(&name, true, owner.is_some())?; + let (store, _) = open(&self.0, owner.as_deref(), &instance, &headers, true) .await .map_err(status)?; let size = digest.size_bytes as u64; @@ -772,7 +793,11 @@ impl ByteStream for Cache { &self, request: Request, ) -> Result, Status> { - let (instance, digest) = resource(&request.get_ref().resource_name, true)?; + let (instance, digest) = resource( + &request.get_ref().resource_name, + true, + organization(&request).is_some(), + )?; let store = self.store(&request, &instance, true).await?; let (committed, complete) = store .upload_status( @@ -794,3 +819,48 @@ impl ByteStream for Cache { })) } } + +#[cfg(test)] +mod tests { + use super::resource; + + fn hash() -> String { + "ab".repeat(32) + } + + #[test] + fn resource_names_follow_the_host() { + let h = hash(); + // Shared host: the instance is `/`. + let (instance, digest) = + resource(&format!("alpha/private/blobs/{h}/3"), false, false).expect("shared read"); + assert_eq!(instance, "alpha/private"); + assert_eq!((digest.hash.as_str(), digest.size_bytes), (h.as_str(), 3)); + let (instance, _) = resource( + &format!("alpha/private/uploads/u-1/blobs/{h}/3"), + true, + false, + ) + .expect("shared write"); + assert_eq!(instance, "alpha/private"); + // The organization host's name has no owner, so the shared shape is refused. + assert!(resource(&format!("alpha/private/blobs/{h}/3"), false, true).is_err()); + assert!( + resource( + &format!("alpha/private/uploads/u-1/blobs/{h}/3"), + true, + true + ) + .is_err() + ); + + // Organization host: the instance is ``. + let (instance, _) = + resource(&format!("private/blobs/{h}/3"), false, true).expect("organization read"); + assert_eq!(instance, "private"); + let (instance, digest) = resource(&format!("private/uploads/u-1/blobs/{h}/3"), true, true) + .expect("organization write"); + assert_eq!(instance, "private"); + assert_eq!(digest.size_bytes, 3); + } +} diff --git a/workers/server/src/bazel/mod.rs b/workers/server/src/bazel/mod.rs index 1c859bb..8a71d63 100644 --- a/workers/server/src/bazel/mod.rs +++ b/workers/server/src/bazel/mod.rs @@ -112,7 +112,13 @@ pub(crate) async fn collect(state: &AppState) -> anyhow::Result { } pub(crate) fn router(state: AppState) -> Router { - grpc::router(state).route( + grpc::router(state).merge(http_router()) +} + +/// Bazel over HTTP alone, without gRPC. Organization hosts route to this +/// router so that the gRPC service never sees their requests. +pub(crate) fn http_router() -> Router { + Router::new().route( "/_api/v1/bazel/{slug}/{cache}/{kind}/{digest}", get(http).head(http).put(http), ) @@ -130,11 +136,17 @@ struct Target { async fn open( state: &AppState, + owner: Option<&str>, instance: &str, headers: &HeaderMap, write: bool, ) -> Result<(store::Store, bool), ApiError> { - let (slug, name) = instance.split_once('/').ok_or_else(cache::not_found)?; + let (slug, name) = match owner { + // On an organization host the host names the organization and the + // instance names only the cache. + Some(owner) => (owner, instance), + None => instance.split_once('/').ok_or_else(cache::not_found)?, + }; if !cache::valid_name(slug, cache::SLUG_MAX) || !cache::valid_name(name, cache::NAME_MAX) { return Err(cache::not_found()); } @@ -374,8 +386,10 @@ async fn http( return Err(cache::not_found()); } let (parts, body) = request.into_parts(); + // The HTTP organization path is already rewritten to the canonical route. let (store, public) = open( &state, + None, &format!("{slug}/{name}"), &parts.headers, method == Method::PUT, diff --git a/workers/server/src/cache_host.rs b/workers/server/src/cache_host.rs index ff82329..7d13212 100644 --- a/workers/server/src/cache_host.rs +++ b/workers/server/src/cache_host.rs @@ -1,16 +1,115 @@ -//! Direct organization ingress. Only Host is authoritative, never forwarded headers. +//! Direct organization ingress. Host is authoritative, or the request URI's authority (`:authority` on HTTP/2) when no Host is sent; never forwarded headers. use axum::{ - extract::{Request, State}, - http::{StatusCode, header, uri::Authority}, + Router, + extract::{ConnectInfo, MatchedPath, Request, State}, + http::{Extensions, StatusCode, header, uri::Authority}, middleware::Next, response::{IntoResponse, Response}, + routing::any, }; +use std::net::SocketAddr; +use tower::ServiceExt; use crate::{AppState, cache, config::Config}; #[derive(Clone)] pub(crate) struct OrganizationHost(pub String); +/// Products served at `//` on an organization host. +pub(crate) const PRODUCTS: &[&str] = &["go", "sccache", "bazel"]; + +/// The path the client used on an organization host, before dispatch +/// rewrote it to the protocol's canonical route. +#[derive(Clone)] +pub(crate) struct PublicPath(pub String); + +/// Fixed gRPC service paths. On an organization host they run directly, and +/// the instance names only the cache. +const GRPC_PREFIXES: &[&str] = &[ + "/build.bazel.remote.execution.v2.", + "/google.bytestream.ByteStream/", +]; + +/// `///…`. A product only on an organization host; on any +/// other host this shape is a path-based Nix read. +pub(crate) fn product_routes(state: AppState) -> Router { + // Protocol routers only: an organization host must never reach + // management, upload, or operator routes. + let protocols: Router = Router::new() + .merge(crate::go::router()) + .merge(crate::sccache::router()) + .merge(crate::bazel::http_router()) + .fallback(|| async { cache::not_found() }) + .with_state(state); + let mut routes = Router::new(); + for &product in PRODUCTS { + for tail in ["", "/", "/{*rest}"] { + let protocols = protocols.clone(); + routes = routes.route( + &format!("/{{cache}}/{product}{tail}"), + any(move |State(state): State, request: Request| { + dispatch(protocols.clone(), product, state, request) + }), + ); + } + } + routes +} + +async fn dispatch( + protocols: Router, + product: &'static str, + state: AppState, + request: Request, +) -> Response { + let Some(owner) = request + .extensions() + .get::() + .map(|host| host.0.clone()) + else { + return match crate::reads::handle(State(state), request).await { + Ok(response) => response, + Err(error) => error.into_response(), + }; + }; + let public = request.uri().path().to_owned(); + let target = public + .strip_prefix('/') + .and_then(|path| path.split_once('/')) + .and_then(|(name, rest)| { + // The route matched `//` followed by `/` or nothing. + let tail = rest + .strip_prefix(product) + .filter(|tail| tail.is_empty() || tail.starts_with('/'))?; + let query = request + .uri() + .query() + .map(|q| format!("?{q}")) + .unwrap_or_default(); + Some(format!("/_api/v1/{product}/{owner}/{name}{tail}{query}")) + }) + .and_then(|target| target.parse::().ok()); + let Some(target) = target else { + return cache::not_found().into_response(); + }; + // Rebuild the extensions: axum appends inner captures after the outer + // `{cache}` capture, which breaks the protocol handlers' `Path` extractors. + // Go, sccache, and Bazel over HTTP read no request extensions; the peer + // address is carried for handlers that audit by IP. + let (mut parts, body) = request.into_parts(); + let mut extensions = Extensions::new(); + if let Some(peer) = parts.extensions.get::>() { + extensions.insert(*peer); + } + extensions.insert(PublicPath(public)); + parts.extensions = extensions; + parts.uri = target; + match protocols.oneshot(Request::from_parts(parts, body)).await { + Ok(response) => response, + Err(never) => match never {}, + } +} + /// Names that conflict with the core's path-based protocol namespace. pub(crate) const PROTOCOL_NAMES: &[&str] = &["default", "nar", "private", "_api", "healthz"]; @@ -65,10 +164,17 @@ pub(crate) async fn ingress( .get(header::HOST) .and_then(|v| v.to_str().ok()) .and_then(|v| v.parse::().ok()) + // HTTP/2 names the host in `:authority`, which hyper puts on the URI. + .or_else(|| { + (!request.headers().contains_key(header::HOST)) + .then(|| request.uri().authority().cloned()) + .flatten() + }) .filter(|v| !v.host().ends_with("..")) .map(|v| v.host().trim_end_matches('.').to_ascii_lowercase()); if request.headers().get_all(header::HOST).iter().count() > 1 - || (request.headers().contains_key(header::HOST) && host.is_none()) + || ((request.headers().contains_key(header::HOST) || request.uri().authority().is_some()) + && host.is_none()) { return ( StatusCode::BAD_REQUEST, @@ -130,9 +236,27 @@ pub(crate) async fn ingress( request .extensions_mut() .insert(OrganizationHost(owner.to_owned())); - match crate::reads::handle(State(state), request).await { - Ok(response) => response, - Err(error) => error.into_response(), + // Product routes are the only ones registered under `/{cache}/`. + // The cache router must be merged, not nested: a nest would prefix + // the matched path, so product requests would miss this check and + // fall back to Nix reads. gRPC services run directly as well: they + // read the organization from this extension, since their instance + // names carry no owner segment on an organization host. + let direct = request + .extensions() + .get::() + .is_some_and(|path| { + let path = path.as_str(); + path.starts_with("/{cache}/") + || GRPC_PREFIXES.iter().any(|prefix| path.starts_with(prefix)) + }); + if direct { + next.run(request).await + } else { + match crate::reads::handle(State(state), request).await { + Ok(response) => response, + Err(error) => error.into_response(), + } } }; // Every alias hit must observe current SQL policy, even with a warm CDN. @@ -215,4 +339,55 @@ mod tests { "no-store" ); } + + #[tokio::test] + async fn http2_authority_names_the_host_when_no_host_header_is_sent() { + let config = Config { + cache_host_suffix: "example.test".into(), + cache_canonical_hosts: "api.example.test".into(), + cache_reserved_hosts: "blocked.example.test".into(), + ..Default::default() + }; + let pool = sqlx::postgres::PgPoolOptions::new() + .connect_lazy("postgres://localhost/unused") + .unwrap(); + let storage = crate::storage::Storage::from_stores( + std::sync::Arc::new(object_store::memory::InMemory::new()), + std::sync::Arc::new(object_store::memory::InMemory::new()), + ); + let state = AppState::new(pool, config, storage, reqwest::Client::new()).unwrap(); + let app = axum::Router::new() + .fallback(|| async { StatusCode::NO_CONTENT }) + .layer(axum::middleware::from_fn_with_state(state, ingress)); + use tower::ServiceExt; + let send = |host: Option<&'static str>| { + let app = app.clone(); + async move { + let mut request = Request::builder().uri("http://blocked.example.test/healthz"); + if let Some(host) = host { + request = request.header("host", host); + } + app.oneshot(request.body(axum::body::Body::empty()).unwrap()) + .await + .unwrap() + .status() + } + }; + // Without a Host header, the `:authority` names a reserved host. + assert_eq!(send(None).await, StatusCode::NOT_FOUND); + // A malformed authority is refused as an invalid Host, as a Host header would be. + let malformed = app + .clone() + .oneshot( + Request::builder() + .uri("http://blocked.example.test../healthz") + .body(axum::body::Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(malformed.status(), StatusCode::BAD_REQUEST); + // A Host header wins over the authority: the canonical host passes through. + assert_eq!(send(Some("api.example.test")).await, StatusCode::NO_CONTENT); + } } diff --git a/workers/server/src/sccache.rs b/workers/server/src/sccache.rs index 6b14d4e..623da03 100644 --- a/workers/server/src/sccache.rs +++ b/workers/server/src/sccache.rs @@ -140,7 +140,10 @@ async fn propfind(state: AppState, path: DavPath, request: Request) -> Result() + .map_or(request.uri.path(), |path| path.0.as_str()); let xml = format!( "{pathname}{modified}HTTP/1.1 200 OK" ); diff --git a/workers/server/test/compiler.mjs b/workers/server/test/compiler.mjs index 2145820..b333ea3 100644 --- a/workers/server/test/compiler.mjs +++ b/workers/server/test/compiler.mjs @@ -4,6 +4,7 @@ import assert from "node:assert/strict"; import { createHash, randomBytes, randomUUID } from "node:crypto"; import { execFile, spawn } from "node:child_process"; import { once } from "node:events"; +import { createServer, request as httpRequest } from "node:http"; import { mkdir, mkdtemp, readFile, rm, utimes, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; @@ -16,7 +17,7 @@ const exec = promisify(execFile); const serverBinary = new URL("../../../target/debug/snowstead-server", import.meta.url).pathname; const goSource = new URL("../../../apps/go-cache/", import.meta.url).pathname; -test("unchanged compiler clients reuse Rust TCP artifacts with fresh local caches", { timeout: 240_000 }, async t => { +test("unchanged compiler clients reuse Rust TCP artifacts with fresh local caches", { timeout: 480_000 }, async t => { const base = process.env.SNOWSTEAD_TEST_DATABASE_URL; assert.ok(base, "run via scripts/with-postgres.sh inside nix develop"); assert.equal(process.env.R2_ENDPOINT, process.env.SNOWSTEAD_TEST_R2_ENDPOINT, "run via scripts/with-r2.sh"); @@ -54,6 +55,7 @@ test("unchanged compiler clients reuse Rust TCP artifacts with fresh local cache SIGNING_PRIVATE_KEY: "", REQUIRE_CACHE_SIGNING_KEYS: "false", PUBLIC_URL: "", UPSTREAM_CACHES: "", PUBLIC_CACHE_ORIGIN: "", CDN_ZONE_ID: "", CDN_PURGE_TOKEN: "", ANALYTICS_TOKEN: "", TRUSTED_PROXY_IPS: "", + CACHE_HOST_SUFFIX: "cache.test", CACHE_CANONICAL_HOSTS: "", CACHE_RESERVED_NAMES: "", CACHE_RESERVED_HOSTS: "", DATABASE_MAX_CONNECTIONS: "2", LISTEN_ADDR: "127.0.0.1:0", RUST_LOG: "info", }; await exec(serverBinary, ["migrate"], { env, timeout: 30_000 }); @@ -78,6 +80,21 @@ test("unchanged compiler clients reuse Rust TCP artifacts with fresh local cache await db.query("INSERT INTO token_caches(token_jti,cache_id) VALUES ($1,$2)", [jti, id]); tokens[protocol] = `snowstead_tok_${jti}_${tokenSecret}`; } + // A private organization cache reached through its organization host. + await db.query("INSERT INTO orgs(id,slug) VALUES ('org_alpha','alpha')"); + const { rows: [{ id: teamId }] } = await db.query( + "INSERT INTO caches(name,org_id,is_public) VALUES ('team','org_alpha',false) RETURNING id"); + await db.query("INSERT INTO cache_protocols(cache_id,protocol) VALUES ($1,'go'),($1,'sccache')", [teamId]); + const orgJti = randomUUID(); + const orgSecret = randomBytes(32).toString("base64url"); + await db.query( + "INSERT INTO api_tokens(jti,subject,secret_hash,created_by,org_id,expires_at) VALUES ($1,'compiler-fixture',$2,'test','org_alpha',now()+interval '1 hour')", + [orgJti, createHash("sha256").update(orgSecret).digest("hex")]); + await db.query( + "INSERT INTO token_permissions(token_jti,permission,position) VALUES ($1,'go:read',1),($1,'go:write',2),($1,'sccache:read',3),($1,'sccache:write',4)", + [orgJti]); + await db.query("INSERT INTO token_caches(token_jti,cache_id) VALUES ($1,$2)", [orgJti, teamId]); + tokens.org = `snowstead_tok_${orgJti}_${orgSecret}`; } finally { await db.end(); } const token = protocol => tokens[protocol]; server = spawn(serverBinary, [], { env, stdio: ["ignore", "pipe", "inherit"] }); @@ -91,40 +108,64 @@ test("unchanged compiler clients reuse Rust TCP artifacts with fresh local cache const address = /127\.0\.0\.1:\d+/.exec(line)?.[0]; assert.ok(address, line); const url = `http://${address}`; - - // Exercise unmodified Rust and C++ compiler clients with fresh local caches. - const dir = join(home, "sccache"); - await mkdir(dir); - const clientEnv = Object.fromEntries(Object.entries(process.env).filter(([key]) => !key.startsWith("SCCACHE_"))); - Object.assign(clientEnv, { - SCCACHE_SERVER_UDS: join(dir, "server.sock"), SCCACHE_CONF: join(dir, "config"), - SCCACHE_WEBDAV_ENDPOINT: `${url}/_api/v1/sccache/default/sccache/`, - SCCACHE_WEBDAV_TOKEN: token("sccache"), + // Stand in for nginx on the organization host: forward to the Rust service + // with the Host the organization address would carry. The Go helper refuses + // plain HTTP except on loopback, so this stays on 127.0.0.1. + const [upstreamHost, upstreamPort] = address.split(":"); + const front = createServer((incoming, outgoing) => { + const upstream = httpRequest({ host: upstreamHost, port: upstreamPort, method: incoming.method, + path: incoming.url, headers: { ...incoming.headers, host: "alpha.cache.test" } }, response => { + outgoing.writeHead(response.statusCode, response.headers); + response.pipe(outgoing); + }); + upstream.on("error", () => outgoing.destroy()); + incoming.pipe(upstream); }); - const run = args => exec("sccache", args, { env: clientEnv, cwd: dir, timeout: 30_000 }); - try { - await writeFile(join(dir, "config"), ""); - await writeFile(join(dir, "input.cpp"), "int cached(int n) { return n * 7 + 3; }\n"); - await writeFile(join(dir, "input.rs"), "pub fn cached(n: u64) -> u64 { n * 11 + 5 }\n"); - let expected; - for (let pass = 0; pass < 2; pass++) { - clientEnv.SCCACHE_DIR = join(dir, `local-${pass}`); - await run(["--start-server"]); - await run(["c++", "-c", "input.cpp", "-o", "cpp.o", "-O2"]); - await run(["rustc", "--crate-name", "fixture", "--crate-type", "rlib", "--emit=link", "input.rs", "--out-dir", ".", "-O"]); - const stats = JSON.parse((await run(["--show-stats", "--stats-format", "json"])).stdout).stats; - const sum = counts => Object.values(counts).reduce((a, b) => a + b, 0); - assert.equal(sum(stats.cache_hits.counts), pass ? 2 : 0, JSON.stringify(stats)); - assert.equal(sum(stats.cache_misses.counts), pass ? 0 : 2, JSON.stringify(stats)); - assert.equal(stats.cache_write_errors, 0, JSON.stringify(stats)); - const outputs = await Promise.all(["cpp.o", "libfixture.rlib"].map(name => readFile(join(dir, name)))); - if (expected) assert.deepEqual(outputs, expected); - expected = outputs; - console.log(`sccache ${pass ? "warm" : "cold"}: hits=${sum(stats.cache_hits.counts)} misses=${sum(stats.cache_misses.counts)} write_errors=${stats.cache_write_errors} output_bytes=${outputs.map(b => b.length).join(",")}`); - await run(["--stop-server"]); - await Promise.all(["cpp.o", "libfixture.rlib"].map(name => rm(join(dir, name)))); - } - } finally { await run(["--stop-server"]).catch(() => {}); } + front.listen(0, "127.0.0.1"); + await once(front, "listening"); + t.after(() => { front.closeAllConnections(); front.close(); }); + const orgUrl = `http://127.0.0.1:${front.address().port}`; + + // Exercise unmodified Rust and C++ compiler clients with fresh local caches, + // on the shared address and on the organization host. + const sccacheEndpoints = [ + ["shared", `${url}/_api/v1/sccache/default/sccache/`, token("sccache")], + ["org", `${orgUrl}/team/sccache/`, tokens.org], + ]; + for (const [label, endpoint, bearer] of sccacheEndpoints) { + const dir = join(home, `sccache-${label}`); + await mkdir(dir); + const clientEnv = Object.fromEntries(Object.entries(process.env).filter(([key]) => !key.startsWith("SCCACHE_"))); + Object.assign(clientEnv, { + SCCACHE_SERVER_UDS: join(dir, "server.sock"), SCCACHE_CONF: join(dir, "config"), + SCCACHE_WEBDAV_ENDPOINT: endpoint, + SCCACHE_WEBDAV_TOKEN: bearer, + }); + const run = args => exec("sccache", args, { env: clientEnv, cwd: dir, timeout: 30_000 }); + try { + await writeFile(join(dir, "config"), ""); + await writeFile(join(dir, "input.cpp"), "int cached(int n) { return n * 7 + 3; }\n"); + await writeFile(join(dir, "input.rs"), "pub fn cached(n: u64) -> u64 { n * 11 + 5 }\n"); + let expected; + for (let pass = 0; pass < 2; pass++) { + clientEnv.SCCACHE_DIR = join(dir, `local-${pass}`); + await run(["--start-server"]); + await run(["c++", "-c", "input.cpp", "-o", "cpp.o", "-O2"]); + await run(["rustc", "--crate-name", "fixture", "--crate-type", "rlib", "--emit=link", "input.rs", "--out-dir", ".", "-O"]); + const stats = JSON.parse((await run(["--show-stats", "--stats-format", "json"])).stdout).stats; + const sum = counts => Object.values(counts).reduce((a, b) => a + b, 0); + assert.equal(sum(stats.cache_hits.counts), pass ? 2 : 0, JSON.stringify(stats)); + assert.equal(sum(stats.cache_misses.counts), pass ? 0 : 2, JSON.stringify(stats)); + assert.equal(stats.cache_write_errors, 0, JSON.stringify(stats)); + const outputs = await Promise.all(["cpp.o", "libfixture.rlib"].map(name => readFile(join(dir, name)))); + if (expected) assert.deepEqual(outputs, expected); + expected = outputs; + console.log(`sccache ${label} ${pass ? "warm" : "cold"}: hits=${sum(stats.cache_hits.counts)} misses=${sum(stats.cache_misses.counts)} write_errors=${stats.cache_write_errors} output_bytes=${outputs.map(b => b.length).join(",")}`); + await run(["--stop-server"]); + await Promise.all(["cpp.o", "libfixture.rlib"].map(name => rm(join(dir, name)))); + } + } finally { await run(["--stop-server"]).catch(() => {}); } + } // Build the unchanged helper with an isolated Go cache, not the user's cache. const goEnv = Object.fromEntries(Object.entries(process.env).filter(([key]) => !key.startsWith("GO") && !key.startsWith("SNOWSTEAD_GO_"))); @@ -142,39 +183,44 @@ test("unchanged compiler clients reuse Rust TCP artifacts with fresh local cache // Make both passes eligible instead of creating a warm-only index record. const timestamp = new Date(Date.now() - 60_000); for (const name of ["go.mod", "main.go"]) await utimes(join(project, name), timestamp, timestamp); - const goToken = token("go"); - let expectedGo; - for (let pass = 0; pass < 2; pass++) { - const output = join(project, "fixture"); - // This throwaway module has no VCS metadata (nor any need for stamping). - const result = await exec("go", ["build", "-trimpath", "-buildvcs=false", "-o", output, "."], { - cwd: project, timeout: 120_000, maxBuffer: 4 * 1024 * 1024, - env: { ...goEnv, GOCACHE: join(home, `go-local-${pass}`), GOCACHEPROG: helper, - SNOWSTEAD_GO_CACHE_DIR: join(home, `go-helper-${pass}`), - SNOWSTEAD_GO_CACHE_URL: `${url}/_api/v1/go/default/go`, SNOWSTEAD_GO_CACHE_TOKEN: goToken, - SNOWSTEAD_GO_CACHE_READ_ONLY: "false" }, - }); - const summary = result.stderr.split("\n").filter(line => line.startsWith("snowstead-go-cache: local_hits=")); - assert.equal(summary.length, 1, result.stderr); - const stats = Object.fromEntries([...summary[0].matchAll(/(\w+)=(\w+)/g)].map(([, key, value]) => [key, value === "true" ? true : value === "false" ? false : Number(value)])); - for (const key of ["upload_errors", "read_errors", "read_timeouts", "conflicts"]) assert.equal(stats[key], 0, summary[0]); - assert.equal(stats.remote_disabled, false, summary[0]); - if (pass === 0) { - assert.equal(stats.remote_hits, 0, summary[0]); - assert.ok(stats.misses > 0 && stats.uploads > 0 && stats.uploaded_bytes > 0, summary[0]); - } else { - // Go does not request every cold metadata put on a warm build, and - // linking can still put an empty stdout record. Require real artifact - // downloads and no newly uploaded payload, not identical request counts. - assert.ok(stats.remote_hits > 0 && stats.downloaded_bytes >= expectedGo.length, summary[0]); - assert.equal(stats.uploaded_bytes, 0, summary[0]); + const goEndpoints = [ + ["shared", `${url}/_api/v1/go/default/go`, token("go")], + ["org", `${orgUrl}/team/go`, tokens.org], + ]; + for (const [label, endpoint, bearer] of goEndpoints) { + let expectedGo; + for (let pass = 0; pass < 2; pass++) { + const output = join(project, "fixture"); + // This throwaway module has no VCS metadata (nor any need for stamping). + const result = await exec("go", ["build", "-trimpath", "-buildvcs=false", "-o", output, "."], { + cwd: project, timeout: 120_000, maxBuffer: 4 * 1024 * 1024, + env: { ...goEnv, GOCACHE: join(home, `go-local-${label}-${pass}`), GOCACHEPROG: helper, + SNOWSTEAD_GO_CACHE_DIR: join(home, `go-helper-${label}-${pass}`), + SNOWSTEAD_GO_CACHE_URL: endpoint, SNOWSTEAD_GO_CACHE_TOKEN: bearer, + SNOWSTEAD_GO_CACHE_READ_ONLY: "false" }, + }); + const summary = result.stderr.split("\n").filter(line => line.startsWith("snowstead-go-cache: local_hits=")); + assert.equal(summary.length, 1, result.stderr); + const stats = Object.fromEntries([...summary[0].matchAll(/(\w+)=(\w+)/g)].map(([, key, value]) => [key, value === "true" ? true : value === "false" ? false : Number(value)])); + for (const key of ["upload_errors", "read_errors", "read_timeouts", "conflicts"]) assert.equal(stats[key], 0, summary[0]); + assert.equal(stats.remote_disabled, false, summary[0]); + if (pass === 0) { + assert.equal(stats.remote_hits, 0, summary[0]); + assert.ok(stats.misses > 0 && stats.uploads > 0 && stats.uploaded_bytes > 0, summary[0]); + } else { + // Go does not request every cold metadata put on a warm build, and + // linking can still put an empty stdout record. Require real artifact + // downloads and no newly uploaded payload, not identical request counts. + assert.ok(stats.remote_hits > 0 && stats.downloaded_bytes >= expectedGo.length, summary[0]); + assert.equal(stats.uploaded_bytes, 0, summary[0]); + } + const bytes = await readFile(output); + if (expectedGo) assert.deepEqual(bytes, expectedGo); + expectedGo = bytes; + const executed = await exec(output, [], { timeout: 10_000 }); + assert.equal(executed.stderr, "snowstead compiler fixture\n"); + console.log(`Go ${label} ${pass ? "warm" : "cold"}: ${summary[0]} executable_bytes=${bytes.length} sha256=${createHash("sha256").update(bytes).digest("hex")}`); + await rm(output); } - const bytes = await readFile(output); - if (expectedGo) assert.deepEqual(bytes, expectedGo); - expectedGo = bytes; - const executed = await exec(output, [], { timeout: 10_000 }); - assert.equal(executed.stderr, "snowstead compiler fixture\n"); - console.log(`Go ${pass ? "warm" : "cold"}: ${summary[0]} executable_bytes=${bytes.length} sha256=${createHash("sha256").update(bytes).digest("hex")}`); - await rm(output); } }); diff --git a/workers/server/tests/bazel.rs b/workers/server/tests/bazel.rs index f9feeb3..44eed2c 100644 --- a/workers/server/tests/bazel.rs +++ b/workers/server/tests/bazel.rs @@ -30,6 +30,24 @@ impl Drop for Host { } async fn host(pool: &PgPool, storage: snowstead_server::storage::Storage) -> Host { + let token = seed(pool).await; + serve(pool, storage, token, Hosting::Shared).await +} + +#[derive(Clone, Copy, PartialEq)] +enum Hosting { + /// Path-based addresses on the shared host. + Shared, + /// Every request arrives with the organization Host, as nginx forwards it. + Forwarded, + /// The organization host is configured; each request's own host or + /// `:authority` names the organization. + Direct, +} + +/// Seeds the `alpha` organization's public and private Bazel caches and a token +/// that reads and writes both. Run once per database. Returns the token. +async fn seed(pool: &PgPool) -> String { sqlx::raw_sql("INSERT INTO orgs(id,slug,private_caches) VALUES ('bazel-org','alpha',true); INSERT INTO caches(name,org_id,is_public) VALUES ('public','bazel-org',true),('private','bazel-org',false); INSERT INTO cache_protocols(cache_id,protocol) SELECT id,'bazel' FROM caches; @@ -45,23 +63,48 @@ async fn host(pool: &PgPool, storage: snowstead_server::storage::Storage) -> Hos .execute(pool) .await .unwrap(); - let state = AppState::new( - pool.clone(), - Config::default(), - storage, - reqwest::Client::new(), - ) - .unwrap(); + format!("snowstead_tok_{id}_fixture") +} + +/// Serves the seeded caches. Unless `Hosting::Shared`, the organization host +/// is configured, and `Hosting::Forwarded` rewrites every Host to +/// `alpha.snowstead.com`; the `/_api/v1/…` addresses are unreachable there. +async fn serve( + pool: &PgPool, + storage: snowstead_server::storage::Storage, + token: String, + hosting: Hosting, +) -> Host { + let org = hosting != Hosting::Shared; + let config = if org { + Config { + cache_host_suffix: "snowstead.com".into(), + cache_canonical_hosts: "api.snowstead.com".into(), + ..Default::default() + } + } else { + Config::default() + }; + let state = AppState::new(pool.clone(), config, storage, reqwest::Client::new()).unwrap(); + let app = router(state); + let app = if hosting == Hosting::Forwarded { + app.layer(axum::middleware::map_request( + |mut request: axum::extract::Request| async move { + request + .headers_mut() + .insert("host", "alpha.snowstead.com".parse().unwrap()); + request + }, + )) + } else { + app + }; let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); let url = format!("http://{}", listener.local_addr().unwrap()); let task = tokio::spawn(async move { - axum::serve(listener, router(state)).await.unwrap(); + axum::serve(listener, app).await.unwrap(); }); - Host { - url, - token: format!("snowstead_tok_{id}_fixture"), - task, - } + Host { url, token, task } } fn authorized(body: T, host: &Host) -> Request { @@ -787,6 +830,54 @@ async fn native_bazel_upload_and_clean_remote_hit(pool: PgPool) { }).await.unwrap(); } +#[sqlx::test(migrator = "snowstead_server::migrations::MIGRATOR")] +#[ignore = "requires Bazel 8 from the development shell"] +async fn native_bazel_through_the_organization_host(pool: PgPool) { + let token = seed(&pool).await; + let host = serve(&pool, common::s3_storage(), token, Hosting::Forwarded).await; + let bazel = std::env::var("BAZEL").unwrap_or_else(|_| "bazel".to_owned()); + tokio::task::spawn_blocking(move || { + let root = std::env::temp_dir().join(format!("bazel-org-e2e-{}", uuid::Uuid::now_v7())); + std::fs::create_dir_all(&root).unwrap(); + std::fs::write(root.join("MODULE.bazel"), "module(name = 'cache_fixture')\n").unwrap(); + std::fs::write(root.join("BUILD.bazel"), format!( + "genrule(name='artifact', outs=['artifact.bin'], cmd='dd if=/dev/zero of=$@ bs=1048576 count=9; echo executed >> {}/executions')\n", + root.display() + )).unwrap(); + for protocol in ["grpc", "http"] { + // The organization host names the organization; gRPC's instance names the cache. + let endpoint = if protocol == "http" { + format!("{}/private/bazel", host.url) + } else { + host.url.replace("http://", "grpc://") + }; + let instance: &[&str] = if protocol == "grpc" { + &["--remote_instance_name=private"] + } else { + &[] + }; + for run in 0..2 { + let output = std::process::Command::new(&bazel) + .current_dir(&root) + .arg("--batch") + .arg(format!("--output_user_root={}/{protocol}-{run}", root.display())) + .args(["build", "//:artifact", "--spawn_strategy=local", "--remote_download_outputs=all"]) + .arg(format!("--remote_cache={endpoint}")) + .args(instance) + .arg(format!("--remote_header=Authorization=Bearer {}", host.token)) + .output().unwrap(); + assert!(output.status.success(), "{protocol} run {run}: {}", String::from_utf8_lossy(&output.stderr)); + assert_eq!(std::fs::metadata(root.join("bazel-bin/artifact.bin")).unwrap().len(), 9 * 1024 * 1024); + if run == 1 { + assert!(String::from_utf8_lossy(&output.stderr).contains("remote cache hit"), "{}", String::from_utf8_lossy(&output.stderr)); + } + } + } + assert_eq!(std::fs::read_to_string(root.join("executions")).unwrap().lines().count(), 1); + std::fs::remove_dir_all(root).unwrap(); + }).await.unwrap(); +} + #[sqlx::test(migrator = "snowstead_server::migrations::MIGRATOR")] async fn publication_does_not_wait_on_control_plane_locks(pool: PgPool) { let host = host(&pool, common::storage()).await; @@ -1542,3 +1633,195 @@ async fn action_tree_validation_deduplicates_and_bounds_total_bytes(pool: PgPool .unwrap_err(); assert_eq!(error.code(), tonic::Code::ResourceExhausted); } + +#[sqlx::test(migrator = "snowstead_server::migrations::MIGRATOR")] +async fn grpc_through_the_organization_host(pool: PgPool) { + let token = seed(&pool).await; + let host = serve(&pool, common::storage(), token, Hosting::Direct).await; + // HTTP/2 sends no Host header here: the organization is named by `:authority`. + let org = Channel::from_shared(host.url.clone()) + .unwrap() + .origin("http://alpha.snowstead.com".parse().unwrap()) + .connect() + .await + .unwrap(); + let mut capabilities = capabilities_client::CapabilitiesClient::new(org.clone()); + let mut cas = + content_addressable_storage_client::ContentAddressableStorageClient::new(org.clone()); + let mut bytestream = byte_stream_client::ByteStreamClient::new(org); + assert!( + capabilities + .get_capabilities(authorized( + GetCapabilitiesRequest { + instance_name: "private".into(), + }, + &host, + )) + .await + .is_ok() + ); + // The shared-host shape is not accepted on the organization host. + let shared_shape = capabilities + .get_capabilities(authorized( + GetCapabilitiesRequest { + instance_name: "alpha/private".into(), + }, + &host, + )) + .await + .unwrap_err(); + assert_eq!(shared_shape.code(), tonic::Code::NotFound); + + let batched = b"organization batch"; + let batched_digest = digest(batched); + let batch = BatchUpdateBlobsRequest { + instance_name: "private".into(), + requests: vec![batch_update_blobs_request::Request { + digest: Some(batched_digest.clone()), + data: batched.to_vec(), + compressor: 0, + }], + ..Default::default() + }; + let written = cas + .batch_update_blobs(authorized(batch, &host)) + .await + .unwrap() + .into_inner(); + assert_eq!(written.responses[0].status.as_ref().unwrap().code, 0); + let missing = cas + .find_missing_blobs(authorized( + FindMissingBlobsRequest { + instance_name: "private".into(), + blob_digests: vec![batched_digest.clone()], + ..Default::default() + }, + &host, + )) + .await + .unwrap() + .into_inner(); + assert!(missing.missing_blob_digests.is_empty()); + + let streamed = b"organization stream"; + let streamed_digest = digest(streamed); + let upload = WriteRequest { + resource_name: format!( + "private/uploads/org-upload/blobs/{}/{}", + streamed_digest.hash, streamed_digest.size_bytes + ), + write_offset: 0, + finish_write: true, + data: streamed.to_vec(), + }; + bytestream + .write(authorized(stream::iter([upload]), &host)) + .await + .unwrap(); + let mut response = bytestream + .read(authorized( + ReadRequest { + resource_name: format!( + "private/blobs/{}/{}", + streamed_digest.hash, streamed_digest.size_bytes + ), + read_offset: 0, + read_limit: 0, + }, + &host, + )) + .await + .unwrap() + .into_inner(); + let mut bytes = Vec::new(); + while let Some(part) = response.message().await.unwrap() { + bytes.extend(part.data); + } + assert_eq!(bytes, streamed); + + // A client without the organization Host reaches the shared-host shape only. + let plain = Channel::from_shared(host.url.clone()) + .unwrap() + .connect() + .await + .unwrap(); + let mut shared = + content_addressable_storage_client::ContentAddressableStorageClient::new(plain); + let missing = shared + .find_missing_blobs(authorized( + FindMissingBlobsRequest { + instance_name: "alpha/private".into(), + blob_digests: vec![batched_digest.clone()], + ..Default::default() + }, + &host, + )) + .await + .unwrap() + .into_inner(); + assert!(missing.missing_blob_digests.is_empty()); + assert_eq!( + shared + .find_missing_blobs(authorized( + FindMissingBlobsRequest { + instance_name: "private".into(), + blob_digests: vec![batched_digest.clone()], + ..Default::default() + }, + &host, + )) + .await + .unwrap_err() + .code(), + tonic::Code::NotFound + ); + + // A second organization with its own private cache named `private`. Alpha's + // token must not reach it through the organization host `beta`. + sqlx::raw_sql( + "INSERT INTO orgs(id,slug,private_caches) VALUES ('beta-org','beta',true); + INSERT INTO caches(name,org_id,is_public) VALUES ('private','beta-org',false); + INSERT INTO cache_protocols(cache_id,protocol) SELECT id,'bazel' FROM caches WHERE org_id='beta-org'; + INSERT INTO bazel_caches(cache_id,generation,capacity) SELECT id,'generation-beta-' || id,33554432 FROM caches WHERE org_id='beta-org';", + ) + .execute(&pool) + .await + .unwrap(); + let beta = Channel::from_shared(host.url.clone()) + .unwrap() + .origin("http://beta.snowstead.com".parse().unwrap()) + .connect() + .await + .unwrap(); + let mut beta = content_addressable_storage_client::ContentAddressableStorageClient::new(beta); + assert_eq!( + beta.find_missing_blobs(authorized( + FindMissingBlobsRequest { + instance_name: "private".into(), + blob_digests: vec![batched_digest.clone()], + ..Default::default() + }, + &host, + )) + .await + .unwrap_err() + .code(), + tonic::Code::PermissionDenied + ); + let beta_write = BatchUpdateBlobsRequest { + instance_name: "private".into(), + requests: vec![batch_update_blobs_request::Request { + digest: Some(batched_digest.clone()), + data: batched.to_vec(), + compressor: 0, + }], + ..Default::default() + }; + assert_eq!( + beta.batch_update_blobs(authorized(beta_write, &host)) + .await + .unwrap_err() + .code(), + tonic::Code::PermissionDenied + ); +} diff --git a/workers/server/tests/org_host.rs b/workers/server/tests/org_host.rs new file mode 100644 index 0000000..22b4000 --- /dev/null +++ b/workers/server/tests/org_host.rs @@ -0,0 +1,784 @@ +mod common; + +use axum::{ + Router, + body::{Body, to_bytes}, + http::{HeaderMap, Request, StatusCode}, +}; +use base64::{Engine, engine::general_purpose::URL_SAFE_NO_PAD}; +use hmac::{Hmac, KeyInit, Mac}; +use serde_json::json; +use sha2::{Digest, Sha256}; +use snowstead_server::{AppState, config::Config}; +use sqlx::PgPool; +use tower::ServiceExt; + +fn jwt(org: &str, cache: &str, perms: &[&str]) -> String { + let claims = json!({"jti":format!("compiler-{org}-{cache}-{}", perms.join("+")),"sub":"compiler","iss":"snowstead","aud":"snowstead","iat":1,"org":org,"caches":[cache],"perms":perms}); + let input = format!( + "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.{}", + URL_SAFE_NO_PAD.encode(serde_json::to_vec(&claims).unwrap()) + ); + let mut mac = Hmac::::new_from_slice(b"fixture-jwt").unwrap(); + mac.update(input.as_bytes()); + format!( + "{input}.{}", + URL_SAFE_NO_PAD.encode(mac.finalize().into_bytes()) + ) +} + +#[path = "support/tokens.rs"] +mod tokens; + +/// A listed JWT-era token, which works with the scope its row records. +async fn token(pool: &PgPool, org: &str, cache: &str, perms: &[&str]) -> String { + tokens::register(pool, jwt(org, cache, perms)).await +} + +fn config(suffix: &str) -> Config { + Config { + jwt_secret: "fixture-jwt".into(), + cache_host_suffix: suffix.into(), + cache_canonical_hosts: "api.snowstead.com".into(), + ..Default::default() + } +} + +async fn seed(pool: &PgPool) { + sqlx::raw_sql( + "INSERT INTO orgs(id,slug) VALUES ('org_a','alpha'),('org_b','beta'); + INSERT INTO caches(name,org_id,is_public,created_at) VALUES + ('team','org_a',false,'2020-01-02 03:04:05+00'), + ('team','org_b',false,'2020-01-02 03:04:05+00'), + ('go','org_a',true,'2020-01-02 03:04:05+00'), + ('sccache','org_a',true,'2020-01-02 03:04:05+00'); + INSERT INTO cache_protocols(cache_id,protocol) + SELECT id,kind FROM caches CROSS JOIN (VALUES ('go'),('sccache'),('nix')) p(kind); + INSERT INTO caches(name,org_id,is_public,created_at) VALUES + ('bazel','org_a',true,'2020-01-02 03:04:05+00'); + INSERT INTO cache_protocols(cache_id,protocol) + SELECT id,kind FROM caches CROSS JOIN (VALUES ('bazel'),('nix')) p(kind) WHERE caches.name='bazel'; + INSERT INTO cache_protocols(cache_id,protocol) + SELECT id,'bazel' FROM caches WHERE name='team'; + INSERT INTO bazel_caches(cache_id,generation,capacity) + SELECT id,'generation-' || id,33554432 FROM caches WHERE name IN ('team','bazel');", + ) + .execute(pool) + .await + .unwrap(); +} + +async fn app(pool: &PgPool, suffix: &str) -> Router { + let state = AppState::new( + pool.clone(), + config(suffix), + common::storage(), + reqwest::Client::new(), + ) + .unwrap(); + common::router(state) +} + +async fn send( + app: &Router, + method: &str, + host: Option<&str>, + path: &str, + token: Option<&str>, + headers: &[(&str, &str)], + bytes: &[u8], +) -> (StatusCode, HeaderMap, Vec) { + let mut req = Request::builder().method(method).uri(path); + if let Some(host) = host { + req = req.header("host", host); + } + if let Some(token) = token { + req = req.header("authorization", format!("Bearer {token}")); + } + for (key, value) in headers { + req = req.header(*key, *value); + } + let response = app + .clone() + .oneshot(req.body(Body::from(bytes.to_vec())).unwrap()) + .await + .unwrap(); + let status = response.status(); + let headers = response.headers().clone(); + ( + status, + headers, + to_bytes(response.into_body(), 1024 * 1024) + .await + .unwrap() + .to_vec(), + ) +} + +/// A Go action upload: the body commits to its own SHA-256 output ID. +async fn put_go( + app: &Router, + host: &str, + path: &str, + token: &str, + bytes: &[u8], +) -> (StatusCode, HeaderMap, Vec) { + let length = bytes.len().to_string(); + let output = hex::encode(Sha256::digest(bytes)); + send( + app, + "PUT", + Some(host), + path, + Some(token), + &[ + ("content-length", length.as_str()), + ("x-snowstead-output-id", output.as_str()), + ], + bytes, + ) + .await +} + +async fn put_bytes( + app: &Router, + host: &str, + path: &str, + token: &str, + bytes: &[u8], +) -> (StatusCode, HeaderMap, Vec) { + let length = bytes.len().to_string(); + send( + app, + "PUT", + Some(host), + path, + Some(token), + &[("content-length", length.as_str())], + bytes, + ) + .await +} + +#[sqlx::test(migrator = "snowstead_server::migrations::MIGRATOR")] +async fn go_reads_and_writes_through_the_organization_host(pool: PgPool) { + seed(&pool).await; + let app = app(&pool, "snowstead.com").await; + let token = token(&pool, "org_a", "team", &["go:read", "go:write"]).await; + let first = "a".repeat(64); + let second = "b".repeat(64); + let org_path = format!("/team/go/v1/actions/{first}"); + let legacy_path = format!("/_api/v1/go/alpha/team/v1/actions/{first}"); + let body: &[u8] = b"go action output"; + let second_body: &[u8] = b"second go action output"; + + let (status, headers, _) = send( + &app, + "GET", + Some("alpha.snowstead.com"), + &org_path, + None, + &[], + b"", + ) + .await; + assert_eq!(status, StatusCode::UNAUTHORIZED); + assert_eq!(headers["cloudflare-cdn-cache-control"], "no-store"); + + let (status, headers, _) = put_go(&app, "alpha.snowstead.com", &org_path, &token, body).await; + assert_eq!(status, StatusCode::CREATED); + assert_eq!(headers["cloudflare-cdn-cache-control"], "no-store"); + + let (status, headers, bytes) = send( + &app, + "GET", + Some("alpha.snowstead.com"), + &org_path, + Some(&token), + &[], + b"", + ) + .await; + assert_eq!(status, StatusCode::OK); + assert_eq!(headers["cloudflare-cdn-cache-control"], "no-store"); + assert_eq!(bytes, body); + + let (status, _, bytes) = send( + &app, + "HEAD", + Some("alpha.snowstead.com"), + &org_path, + Some(&token), + &[], + b"", + ) + .await; + assert_eq!(status, StatusCode::OK); + assert!(bytes.is_empty()); + + let with_query = format!("{org_path}?x=1"); + let (status, _, bytes) = send( + &app, + "GET", + Some("alpha.snowstead.com"), + &with_query, + Some(&token), + &[], + b"", + ) + .await; + assert_eq!(status, StatusCode::OK); + assert_eq!(bytes, body); + + let (status, _, bytes) = send( + &app, + "GET", + Some("api.snowstead.com"), + &legacy_path, + Some(&token), + &[], + b"", + ) + .await; + assert_eq!(status, StatusCode::OK); + assert_eq!(bytes, body); + + let legacy_second = format!("/_api/v1/go/alpha/team/v1/actions/{second}"); + let (status, _, _) = put_go( + &app, + "api.snowstead.com", + &legacy_second, + &token, + second_body, + ) + .await; + assert_eq!(status, StatusCode::CREATED); + let org_second = format!("/team/go/v1/actions/{second}"); + let (status, _, bytes) = send( + &app, + "GET", + Some("alpha.snowstead.com"), + &org_second, + Some(&token), + &[], + b"", + ) + .await; + assert_eq!(status, StatusCode::OK); + assert_eq!(bytes, second_body); + + let (status, headers, _) = send( + &app, + "GET", + Some("beta.snowstead.com"), + &org_path, + Some(&token), + &[], + b"", + ) + .await; + assert_eq!(status, StatusCode::FORBIDDEN); + assert_eq!(headers["cloudflare-cdn-cache-control"], "no-store"); + + let (status, _, _) = send( + &app, + "GET", + Some("api.snowstead.com"), + &org_path, + None, + &[], + b"", + ) + .await; + assert_eq!(status, StatusCode::NOT_FOUND); + let (status, _, _) = send(&app, "GET", None, &org_path, None, &[], b"").await; + assert_eq!(status, StatusCode::NOT_FOUND); + + let (status, headers, _) = send( + &app, + "DELETE", + Some("alpha.snowstead.com"), + &org_path, + Some(&token), + &[], + b"", + ) + .await; + assert_eq!(status, StatusCode::METHOD_NOT_ALLOWED); + assert_eq!(headers["allow"], "GET, HEAD, PUT"); + + for path in [ + "/team/go".to_owned(), + "/team/go/".to_owned(), + "/team/go/v1/actions/short".to_owned(), + format!("/te%61m/go/v1/actions/{first}"), + ] { + let (status, _, _) = send( + &app, + "GET", + Some("alpha.snowstead.com"), + &path, + None, + &[], + b"", + ) + .await; + assert_eq!(status, StatusCode::NOT_FOUND, "{path}"); + } + + let (status, headers, _) = send( + &app, + "GET", + Some("a.b.snowstead.com"), + &org_path, + None, + &[], + b"", + ) + .await; + assert_eq!(status, StatusCode::NOT_FOUND); + assert_eq!(headers["cloudflare-cdn-cache-control"], "no-store"); +} + +#[sqlx::test(migrator = "snowstead_server::migrations::MIGRATOR")] +async fn sccache_webdav_through_the_organization_host(pool: PgPool) { + seed(&pool).await; + let app = app(&pool, "snowstead.com").await; + let read = token(&pool, "org_a", "team", &["sccache:read"]).await; + let write = token(&pool, "org_a", "team", &["sccache:read", "sccache:write"]).await; + let artifact = "a".repeat(64); + let artifact_path = format!("/a/a/a/{artifact}"); + let org_artifact = format!("/team/sccache{artifact_path}"); + let legacy_artifact = format!("/_api/v1/sccache/alpha/team{artifact_path}"); + let bytes: &[u8] = b"sccache artifact bytes"; + let org = "alpha.snowstead.com"; + let api = "api.snowstead.com"; + + let (status, _, body) = send( + &app, + "PROPFIND", + Some(org), + "/team/sccache/", + Some(&read), + &[], + b"", + ) + .await; + assert_eq!(status, StatusCode::MULTI_STATUS); + assert!( + std::str::from_utf8(&body) + .unwrap() + .contains("/team/sccache/") + ); + + let (status, _, body) = send( + &app, + "PROPFIND", + Some(api), + "/_api/v1/sccache/alpha/team/", + Some(&read), + &[], + b"", + ) + .await; + assert_eq!(status, StatusCode::MULTI_STATUS); + assert!( + std::str::from_utf8(&body) + .unwrap() + .contains("/_api/v1/sccache/alpha/team/") + ); + + let (status, _, _) = send( + &app, + "MKCOL", + Some(org), + "/team/sccache/a/", + Some(&write), + &[], + b"", + ) + .await; + assert_eq!(status, StatusCode::CREATED); + + let (status, _, _) = put_bytes(&app, org, "/team/sccache/.sccache_check", &write, b"x").await; + assert_eq!(status, StatusCode::NO_CONTENT); + + let (status, _, _) = put_bytes(&app, org, &org_artifact, &write, bytes).await; + assert_eq!(status, StatusCode::CREATED); + + let (status, _, body) = + send(&app, "GET", Some(org), &org_artifact, Some(&read), &[], b"").await; + assert_eq!(status, StatusCode::OK); + assert_eq!(body, bytes); + + let (status, _, body) = send( + &app, + "GET", + Some(api), + &legacy_artifact, + Some(&read), + &[], + b"", + ) + .await; + assert_eq!(status, StatusCode::OK); + assert_eq!(body, bytes); + + let (status, _, _) = send( + &app, + "GET", + Some(org), + "/team/sccache", + Some(&read), + &[], + b"", + ) + .await; + assert_eq!(status, StatusCode::NOT_FOUND); + + let (status, _, _) = send( + &app, + "PROPFIND", + Some(org), + "/team/sccache//a/", + Some(&read), + &[], + b"", + ) + .await; + assert_eq!(status, StatusCode::MULTI_STATUS); + + let (status, _, _) = send( + &app, + "MKCOL", + Some(org), + "/team/sccache/a/b/", + Some(&write), + &[], + b"", + ) + .await; + assert_eq!(status, StatusCode::CREATED); + let (status, _, body) = send( + &app, + "PROPFIND", + Some(org), + "/team/sccache/a/b/", + Some(&read), + &[], + b"", + ) + .await; + assert_eq!(status, StatusCode::MULTI_STATUS); + assert!( + std::str::from_utf8(&body) + .unwrap() + .contains("/team/sccache/a/b/") + ); + + let (status, _, _) = send( + &app, + "GET", + Some("beta.snowstead.com"), + &org_artifact, + Some(&read), + &[], + b"", + ) + .await; + assert_eq!(status, StatusCode::FORBIDDEN); +} + +#[sqlx::test(migrator = "snowstead_server::migrations::MIGRATOR")] +async fn bazel_http_through_the_organization_host(pool: PgPool) { + seed(&pool).await; + let app = app(&pool, "snowstead.com").await; + let token = token(&pool, "org_a", "team", &["bazel:read", "bazel:write"]).await; + let bytes: &[u8] = b"build output"; + let hash = hex::encode(Sha256::digest(bytes)); + let second_bytes: &[u8] = b"second build output"; + let second_hash = hex::encode(Sha256::digest(second_bytes)); + let org = "alpha.snowstead.com"; + let api = "api.snowstead.com"; + let org_path = format!("/team/bazel/cas/{hash}"); + let legacy_path = format!("/_api/v1/bazel/alpha/team/cas/{hash}"); + + let (status, _, _) = send(&app, "PUT", Some(org), &org_path, None, &[], bytes).await; + assert_eq!(status, StatusCode::UNAUTHORIZED); + + let (status, headers, _) = put_bytes(&app, org, &org_path, &token, bytes).await; + assert_eq!(status, StatusCode::NO_CONTENT); + assert_eq!(headers["cloudflare-cdn-cache-control"], "no-store"); + + let (status, headers, body) = + send(&app, "GET", Some(org), &org_path, Some(&token), &[], b"").await; + assert_eq!(status, StatusCode::OK); + assert_eq!(headers["cloudflare-cdn-cache-control"], "no-store"); + assert_eq!(body, bytes); + + let (status, _, body) = send(&app, "HEAD", Some(org), &org_path, Some(&token), &[], b"").await; + assert_eq!(status, StatusCode::OK); + assert!(body.is_empty()); + + let (status, _, body) = + send(&app, "GET", Some(api), &legacy_path, Some(&token), &[], b"").await; + assert_eq!(status, StatusCode::OK); + assert_eq!(body, bytes); + + let legacy_second = format!("/_api/v1/bazel/alpha/team/cas/{second_hash}"); + let (status, _, _) = put_bytes(&app, api, &legacy_second, &token, second_bytes).await; + assert_eq!(status, StatusCode::NO_CONTENT); + let org_second = format!("/team/bazel/cas/{second_hash}"); + let (status, _, body) = send(&app, "GET", Some(org), &org_second, Some(&token), &[], b"").await; + assert_eq!(status, StatusCode::OK); + assert_eq!(body, second_bytes); + + for path in [ + "/team/bazel".to_owned(), + "/team/bazel/".to_owned(), + format!("/team/bazel/xx/{hash}"), + ] { + let (status, _, _) = send(&app, "GET", Some(org), &path, Some(&token), &[], b"").await; + assert_eq!(status, StatusCode::NOT_FOUND, "{path}"); + } + + let (status, _, _) = send( + &app, + "GET", + Some("beta.snowstead.com"), + &org_path, + Some(&token), + &[], + b"", + ) + .await; + assert_eq!(status, StatusCode::FORBIDDEN); + + // The public cache is where the CDN-facing headers are observable: the + // organization host must stay uncacheable at the edge, while the shared + // host's CAS reads keep their public revalidation policy. + let public_token = crate::token(&pool, "org_a", "bazel", &["bazel:read", "bazel:write"]).await; + let public_bytes: &[u8] = b"public build output"; + let public_hash = hex::encode(Sha256::digest(public_bytes)); + let public_path = format!("/bazel/bazel/cas/{public_hash}"); + let (status, _, _) = put_bytes(&app, org, &public_path, &public_token, public_bytes).await; + assert_eq!(status, StatusCode::NO_CONTENT); + let (status, headers, body) = send(&app, "GET", Some(org), &public_path, None, &[], b"").await; + assert_eq!(status, StatusCode::OK); + assert_eq!(body, public_bytes); + assert_eq!(headers["cloudflare-cdn-cache-control"], "no-store"); + assert_eq!( + headers["cache-control"], + "public, no-cache, must-revalidate, no-transform" + ); + let shared_public = format!("/_api/v1/bazel/alpha/bazel/cas/{public_hash}"); + let (status, headers, body) = + send(&app, "GET", Some(api), &shared_public, None, &[], b"").await; + assert_eq!(status, StatusCode::OK); + assert_eq!(body, public_bytes); + assert_eq!( + headers["cloudflare-cdn-cache-control"], + "public, no-cache, must-revalidate, no-transform" + ); + + // Bazel over gRPC is served on the organization host: the gRPC route runs + // directly, so the request is answered by the gRPC service, not a Nix read. + let (status, headers, _) = send( + &app, + "POST", + Some(org), + "/build.bazel.remote.execution.v2.Capabilities/GetCapabilities", + None, + &[("content-type", "application/grpc")], + b"", + ) + .await; + assert_eq!(status, StatusCode::OK); + assert!( + headers["content-type"] + .to_str() + .unwrap() + .starts_with("application/grpc") + ); + assert!(headers.contains_key("grpc-status")); + assert_eq!(headers["cloudflare-cdn-cache-control"], "no-store"); + // Other unmatched POSTs on the organization host remain Nix reads. + let (status, headers, _) = send(&app, "POST", Some(org), "/team/zig/x", None, &[], b"").await; + assert_eq!(status, StatusCode::METHOD_NOT_ALLOWED); + assert_eq!(headers["allow"], "GET, HEAD"); + + // The go cache has no Bazel protocol, so its Bazel address does not exist. + let go_token = crate::token(&pool, "org_a", "go", &["bazel:read"]).await; + let (status, _, _) = send( + &app, + "GET", + Some(org), + &format!("/go/bazel/cas/{hash}"), + Some(&go_token), + &[], + b"", + ) + .await; + assert_eq!(status, StatusCode::NOT_FOUND); + + let (status, _, _) = send( + &app, + "GET", + Some(org), + &format!("/team/bazel/cas/{}", hash.to_uppercase()), + Some(&token), + &[], + b"", + ) + .await; + assert_eq!(status, StatusCode::NOT_FOUND); + + let (status, _, _) = send( + &app, + "GET", + Some(org), + &format!("/te%61m/bazel/cas/{hash}"), + Some(&token), + &[], + b"", + ) + .await; + assert_eq!(status, StatusCode::NOT_FOUND); + + let (status, headers, _) = + send(&app, "DELETE", Some(org), &org_path, Some(&token), &[], b"").await; + assert_eq!(status, StatusCode::METHOD_NOT_ALLOWED); + assert_eq!(headers["allow"], "GET,HEAD,PUT"); + + // A body that does not hash to its path is refused and never stored. + let mismatched_path = format!("/team/bazel/cas/{}", "c".repeat(64)); + let (status, _, _) = put_bytes(&app, org, &mismatched_path, &token, b"twelve bytes").await; + assert_eq!(status, StatusCode::BAD_REQUEST); + let (status, _, _) = send( + &app, + "GET", + Some(org), + &mismatched_path, + Some(&token), + &[], + b"", + ) + .await; + assert_eq!(status, StatusCode::NOT_FOUND); +} + +#[sqlx::test(migrator = "snowstead_server::migrations::MIGRATOR")] +async fn caches_named_after_products_keep_their_nix_addresses(pool: PgPool) { + seed(&pool).await; + let plain = app(&pool, "").await; + let app = app(&pool, "snowstead.com").await; + let go_write = token(&pool, "org_a", "go", &["go:read", "go:write"]).await; + let action = "a".repeat(64); + + for name in ["go", "sccache", "bazel"] { + let nix = format!("/{name}/nix-cache-info"); + let (status, headers, _) = send( + &app, + "GET", + Some("alpha.snowstead.com"), + &nix, + None, + &[], + b"", + ) + .await; + assert_eq!(status, StatusCode::OK, "{name}"); + assert_eq!(headers["content-type"], "text/x-nix-cache-info", "{name}"); + + let shared = format!("/alpha/{name}/nix-cache-info"); + for (router, host) in [ + (&app, Some("api.snowstead.com")), + (&app, None), + (&plain, Some("api.snowstead.com")), + (&plain, None), + ] { + let (status, _, _) = send(router, "GET", host, &shared, None, &[], b"").await; + assert_eq!(status, StatusCode::OK, "{name} {host:?}"); + } + + let (status, headers, _) = send( + &app, + "PUT", + Some("api.snowstead.com"), + &shared, + None, + &[], + b"", + ) + .await; + assert_eq!(status, StatusCode::METHOD_NOT_ALLOWED, "{name}"); + assert_eq!(headers["allow"], "GET, HEAD", "{name}"); + } + + let (status, _, _) = put_go( + &app, + "alpha.snowstead.com", + &format!("/go/go/v1/actions/{action}"), + &go_write, + b"go cache named go", + ) + .await; + assert_eq!(status, StatusCode::CREATED); + let (status, _, body) = send( + &app, + "GET", + Some("alpha.snowstead.com"), + &format!("/go/go/v1/actions/{action}"), + Some(&go_write), + &[], + b"", + ) + .await; + assert_eq!(status, StatusCode::OK); + assert_eq!(body, b"go cache named go"); +} + +#[sqlx::test(migrator = "snowstead_server::migrations::MIGRATOR")] +async fn other_organization_host_paths_stay_nix_reads(pool: PgPool) { + seed(&pool).await; + let app = app(&pool, "snowstead.com").await; + let action = "a".repeat(64); + + let (status, headers, _) = send( + &app, + "PUT", + Some("alpha.snowstead.com"), + "/team/zig/x", + None, + &[], + b"", + ) + .await; + assert_eq!(status, StatusCode::METHOD_NOT_ALLOWED); + assert_eq!(headers["allow"], "GET, HEAD"); + + let (status, headers, _) = send( + &app, + "PUT", + Some("alpha.snowstead.com"), + "/team/nix-cache-info", + None, + &[], + b"", + ) + .await; + assert_eq!(status, StatusCode::METHOD_NOT_ALLOWED); + assert_eq!(headers["allow"], "GET, HEAD"); + + let (status, _, _) = send( + &app, + "GET", + Some("alpha.snowstead.com"), + &format!("/_api/v1/go/alpha/team/v1/actions/{action}"), + None, + &[], + b"", + ) + .await; + assert_eq!(status, StatusCode::NOT_FOUND); +}