From 73355ad02547d740f107c9cf9ef0ab02034c20be Mon Sep 17 00:00:00 2001 From: Taylor Addison Date: Thu, 15 Oct 2026 21:00:52 -0400 Subject: [PATCH] Close 0.8.0's loose ends: legacy auth checks, ci-enqueue on /api/v1, PR builds off latest - Legacy cancel/retry make the owner check and legacy enqueue the shell/admin check, through one shared module (collector/src/api/policy.ts) that /api/v1 also uses. - ci-enqueue moves to /api/v1 with fleetr's exit codes (4 on skipped); ci-enqueue and ci-upload send FLEET_RUNNER_TOKEN/FLEET_TOKEN as a bearer. - x-artifact-channel: pr; "latest" never resolves to a pr build. Both CI scripts set it on pull_request events or --channel pr. - fleet/package.json bin points at dist/bin/fleet.mjs and packs dist/. - version.mjs writes the desktop app's Cargo.toml and tauri.conf.json (0.8.0). - check-examples runs the API's strict schema and params manifests (spec-check.ts, shared with routes.ts); CHECK_NAMES drops names nothing raises. - FLEET_PEER_TOKENS for the peers proxy; explore mirror frames send FLEET_TOKEN; timer intervals validated (>= 1000 ms); stale discovery comments fixed. - Schedule templates: an empty job_id is dropped with a warnings entry. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 74 ++++++++- CONTRIBUTING.md | 5 +- collector/ci/README.md | 25 ++- collector/ci/example-workflow.yml | 10 ++ collector/ci/publish-build.yml | 4 + collector/scripts/check-auth.ts | 150 ++++++++++++++++-- collector/scripts/check-examples.ts | 42 +++-- collector/scripts/ci-enqueue.ts | 194 +++++++++++++++++------ collector/scripts/ci-upload.ts | 67 ++++++-- collector/scripts/smoke.ts | 79 ++++++++- collector/src/api/mutations.ts | 12 +- collector/src/api/peers.ts | 27 +++- collector/src/api/policy.ts | 52 ++++++ collector/src/api/system.ts | 4 +- collector/src/api/v1/routes.ts | 44 +---- collector/src/api/v1/spec-check.ts | 52 ++++++ collector/src/config.test.ts | 41 +++++ collector/src/config.ts | 84 +++++++++- collector/src/db.ts | 5 + collector/src/discovery.ts | 10 +- collector/src/findings.test.ts | 5 +- collector/src/queue.ts | 11 +- collector/src/server.ts | 65 ++++++-- collector/src/workloads/explore/index.ts | 12 +- collector/src/workloads/explore/types.ts | 18 ++- desktop/src-tauri/Cargo.toml | 2 +- desktop/src-tauri/tauri.conf.json | 2 +- docs/api.md | 21 ++- docs/deploy/alerts.md | 3 +- docs/deploy/auth.md | 24 +-- docs/deploy/index.md | 25 +++ docs/explore/findings.md | 7 +- docs/explore/job.md | 3 +- docs/integration/add-your-app.md | 13 +- docs/integration/cookbook.md | 13 +- docs/integration/first-job.md | 55 ++++--- docs/integration/index.md | 23 +-- docs/integration/publish-on-merge.md | 51 +++++- fleet/build.mjs | 2 +- fleet/package.json | 7 +- fleet/src/version.ts | 2 +- scripts/version.mjs | 6 + 42 files changed, 1089 insertions(+), 262 deletions(-) create mode 100644 collector/src/api/policy.ts create mode 100644 collector/src/api/v1/spec-check.ts create mode 100644 collector/src/config.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index b1b6bbf..cb8ea99 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,13 +11,85 @@ itself is `"schema": 1` and stays there until something breaks compatibility, independently of the version below. Since 0.5.0 that is enforced rather than asserted: `VERSION` at the repository -root is the source, `scripts/version.mjs` writes it into all eleven places, and +root is the source, `scripts/version.mjs` writes it into all thirteen places, and a CI job fails when they disagree. ## [Unreleased] +### Added + +- **A pull request's build no longer becomes the nightly's `latest`.** + `ci-upload` always tagged an upload with its app, and `ci-enqueue` did + whenever the spec named one, so the next nightly asking for `latest` tested + an unmerged branch. An upload can now carry `x-artifact-channel: pr`, which + `"sha256": "latest"` never resolves to; both scripts send it when + `GITHUB_EVENT_NAME` is `pull_request` (or with `--channel pr`). The pull + request's own job still names its build by hash. Builds uploaded before this + count as merged builds, and the same bytes republished from main come off the + channel. The `/api/v1` artifact document does not show the channel yet; + `GET /api/artifacts` does. +- **`FLEET_PEER_TOKENS`: the brain switcher can read a peer with tokens on.** + The peers proxy sent no token, so a peer running `FLEET_AUTH=tokens` + answered 401 through it. Each peer's URL now maps to a `read` token that + peer issued, sent only to that peer. + +### Changed + +- **`ci-enqueue` speaks `/api/v1` and exits like `fleetr run`.** It polled the + deprecated `GET /jobs/:id` and exited 0 on `done`, which includes a job that + skipped and tested nothing, and it sent no token, so it failed against a + collector with tokens on. It now enqueues with `POST /api/v1/jobs`, waits + with `/wait`, and exits 0 passed, 1 failed, 2 invalid spec, 3 still running, + 4 skipped, 5 refused or unreachable. A workflow that read exit 1 as "timed + out" now sees 3. +- **`ci-enqueue` and `ci-upload` send `FLEET_RUNNER_TOKEN` (or + `FLEET_TOKEN`) as a bearer token.** `ci-upload` read only + `FLEET_DASH_TOKEN`, even when the collector needed an `agent` token there. + `FLEET_DASH_TOKEN` is still sent as `x-fleet-token`. `ci-upload` exits 5 + rather than 1 when the upload is refused or the collector is unreachable. +- **The example jobs are checked the way `/api/v1` checks a spec.** + `check-examples.ts` used the loose `job.schema.json`, so an example could + pass the check and be refused by the API it documents. It now runs the API's + strict schema and each workload's params manifest, imported rather than + copied. All 37 pass. +- **Explore's `CHECK_NAMES` lists only checks something raises.** `regression`, + `agent_stuck` and `leash` were accepted by `POST /findings` and raised by + nothing; they are refused now. A refused action is on the mission's steps, + not a finding. Stored findings are not touched, and reading does not + validate the name. +- **The desktop app is versioned with everything else.** `scripts/version.mjs` + now writes `desktop/src-tauri/Cargo.toml` and `tauri.conf.json` (they said + 0.5.0), so the CI version check covers them. Both say 0.8.0. + ### Fixed +- **The older job routes skipped two auth checks.** With `FLEET_AUTH=tokens`, + `POST /api/jobs/:id/cancel` and `/retry` let any `run` token stop or re-run + anyone's job, and `POST /jobs` and `POST /api/jobs` accepted the `shell` + workload from a token without `admin`. Only `/api/v1` asked. Both paths now + call the same two functions (`collector/src/api/policy.ts`). With auth off + nothing changes. +- **`fleet-cli`'s `bin` pointed at a file the build does not produce.** It + named `dist/cli.mjs`; the build writes `dist/bin/fleet.mjs`, and `dist/` was + left out of the package by `.gitignore`, so installing from a tarball gave no + command. Both `fleet` and `fleetr` now point at the bundle, and `files` + includes `dist/`. +- **A mistyped timer interval became a near-zero one.** `FLEET_SWEEP_MS`, + `FLEET_SCHEDULER_TICK_MS` and `FLEET_ALERT_TICK_MS` went through a bare + `Number()`, so `15s` was NaN and `15` (seconds, meant) a sweep every 15 ms. + Anything but whole milliseconds of at least 1000 now falls back to the + default, with a startup warning. +- **Explore's live-view frames were refused with `FLEET_AUTH_PROTOCOL=token`.** + They sent only `FLEET_DASH_TOKEN`; they now also send `FLEET_TOKEN` as a + bearer token. +- **A schedule template's empty `job_id` was replaced without a word.** A + non-empty one was already refused; an empty or `null` one was stored and + overwritten at every firing. It is dropped now and the reply carries a + `warnings` entry, and a stored template that still has one is named in the + log when it fires. +- Two comments said `fleet up` turns mDNS discovery on. It passes + `collector.discovery`, which is off by default. + - **`fleet skill` failed on an installed 0.8.0.** The 0.8.0 archives carry `skills/`, but `install.sh` and `install.ps1` copy a release's directories by name and did not name it, so `fleet skill show` and `fleet skill install` diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 835d0a2..032a884 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -162,9 +162,8 @@ the others. One version covers every component, so a release is one number everywhere. 1. On a branch from `develop`, run `node scripts/version.mjs set X.Y.Z`. It - writes `VERSION` and every component's own version except the desktop - app's (`desktop/src-tauri/Cargo.toml` and `tauri.conf.json` are not among - the places it writes, and still say 0.5.0). Then run + writes `VERSION` and every component's own version, the desktop app's + `Cargo.toml` and `tauri.conf.json` included. Then run `npm --prefix collector run gen:api`, because `openapi.json` carries the version and CI fails when it is stale. Android's `versionCode` is not derived from it; bump that by hand if the Play Store build needs it. diff --git a/collector/ci/README.md b/collector/ci/README.md index 2685fcb..6b8dd0d 100644 --- a/collector/ci/README.md +++ b/collector/ci/README.md @@ -20,11 +20,32 @@ published for an app, its nightly is **skipped rather than failed** — a job pinned to a build that does not exist would fail in a way indistinguishable from the app being broken. +**A pull request's build never moves `latest`.** An upload with +`x-artifact-channel: pr` is stored on the `pr` channel, and `latest` only +considers builds without it, so the nightly keeps testing what was merged. +`ci-upload` and `ci-enqueue` send the header themselves when +`GITHUB_EVENT_NAME` is `pull_request` (or `pull_request_target`); `--channel +pr|main` says it explicitly. The pull request's own job names its build by +hash, so it still tests exactly that build. Bytes republished from main come +off the channel; builds from before the channel existed count as merged. + ## 2. Test a pull request — `example-workflow.yml` Enqueue a job against real hardware and block the PR on the verdict. -`ci-enqueue` uploads the build, enqueues, polls to a verdict, and exits 0/1, so -the check fails on a red suite whether or not commit statuses are armed. +`ci-enqueue` uploads the build, enqueues through `/api/v1`, waits for the +verdict, and exits like `fleetr run`: 0 passed, 1 failed, 2 invalid spec, +3 still running at `--timeout`, 4 skipped (nothing ran), 5 refused or +unreachable. So the check fails on a red or skipped suite whether or not +commit statuses are armed. + +## Tokens + +With `FLEET_AUTH=tokens` on the collector, put a token in a secret and pass it +as `FLEET_RUNNER_TOKEN`: both scripts send it as `Authorization: Bearer`. +`ci-enqueue` needs `run`. The upload is the runner protocol, which is open +unless the collector also sets `FLEET_AUTH_PROTOCOL=token`; then both scripts' +uploads need `agent` as well. `FLEET_DASH_TOKEN` is still sent as +`x-fleet-token`, for a collector guarded only by that. ## Commit statuses are built and dark diff --git a/collector/ci/example-workflow.yml b/collector/ci/example-workflow.yml index d85ee43..613fef8 100644 --- a/collector/ci/example-workflow.yml +++ b/collector/ci/example-workflow.yml @@ -4,6 +4,10 @@ # this goes into an app repo (e.g. your-app-android). Until then this file # documents the contract and nothing reads it. # +# ci-enqueue exits like `fleetr run`: 0 passed, 1 failed, 2 invalid spec, +# 3 still running at --timeout, 4 skipped (nothing ran), 5 refused/unreachable. +# Any non-zero exit fails the check, so a skip is not a green check. +# # Requirements at connect time, none of which exist today by design: # - the runner host joined to the tailnet, FLEET_COLLECTOR_URL secret set # - FLEET_GITHUB_STATUS=1 + FLEET_GITHUB_TOKEN on the collector, so the @@ -23,6 +27,12 @@ jobs: run: ./gradlew :app:assembleDebug - name: Real-device smoke via fleet + env: + # A run token (plus agent if the collector sets FLEET_AUTH_PROTOCOL=token), + # needed only when the collector runs with FLEET_AUTH=tokens. + FLEET_RUNNER_TOKEN: ${{ secrets.FLEET_RUNNER_TOKEN }} + # GITHUB_EVENT_NAME is pull_request here, so the build is uploaded on + # the pr channel and never becomes the nightly's "latest". run: | cat > fleet-job.json <((resolve, reject) => { +const freePort = () => new Promise((resolve, reject) => { const srv = createServer(); srv.on("error", reject); srv.listen(0, "127.0.0.1", () => { @@ -42,6 +42,19 @@ const port = await new Promise((resolve, reject) => { srv.close(() => resolve(a.port)); }); }); +const port = await freePort(); + +async function up(base: string, child: ChildProcess) { + const deadline = Date.now() + 30_000; + for (;;) { + if (child.exitCode !== null) throw new Error(`collector exited with ${child.exitCode}`); + try { + if ((await fetch(`${base}/api/v1/health`)).ok) return; + } catch { /* not yet */ } + if (Date.now() > deadline) throw new Error("collector did not come up"); + await new Promise((r) => setTimeout(r, 300)); + } +} const BASE = `http://127.0.0.1:${port}`; const tok = (name: string, scopes: string[]) => createToken({ name, scopes }, TOKENS).token; @@ -66,6 +79,7 @@ async function call(method: string, url: string, o: Opts = {}) { } let server: ChildProcess | undefined; +let peer: ChildProcess | undefined; let log = ""; try { server = spawn(process.execPath, [TSX, "src/server.ts"], { @@ -87,15 +101,7 @@ try { }); server.stdout?.on("data", (d) => { log += d; }); server.stderr?.on("data", (d) => { log += d; }); - const deadline = Date.now() + 30_000; - for (;;) { - if (server.exitCode !== null) throw new Error(`collector exited with ${server.exitCode}`); - try { - if ((await fetch(`${BASE}/api/v1/health`)).ok) break; - } catch { /* not yet */ } - if (Date.now() > deadline) throw new Error("collector did not come up"); - await new Promise((r) => setTimeout(r, 300)); - } + await up(BASE, server); console.log("public"); { @@ -170,6 +176,124 @@ try { check("and the forwarded enqueue keeps the caller's name", r.body.requested_by === "alice", r.body.requested_by); } + console.log("the older routes make the same two checks"); + { + const j = await call("POST", "/api/v1/jobs", { token: ALICE, body: spec }); + const id = j.body.job_id; + const theft = await call("POST", `/api/jobs/${id}/cancel`, { token: BOB }); + check("bob cannot cancel alice's job through /api/jobs/:id/cancel", theft.status === 403 && typeof theft.body.error === "string", JSON.stringify(theft.body)); + check("nor retry it through /api/jobs/:id/retry", (await call("POST", `/api/jobs/${id}/retry`, { token: BOB, body: {} })).status === 403); + const own = await call("POST", `/api/jobs/${id}/cancel`, { token: ALICE }); + check("alice can, through the same route", own.status === 200, JSON.stringify(own.body)); + const k = await call("POST", "/api/v1/jobs", { token: ALICE, body: spec }); + check("admin can cancel anyone's through it", (await call("POST", `/api/jobs/${k.body.job_id}/cancel`, { token: ADMIN })).status === 200); + + const shellSpec = { schema: 1, workload: "shell", executor: "device", params: { script: "x" }, targets: { device_id: "auth-dev" } }; + const viaJobs = await call("POST", "/jobs", { token: ALICE, body: { ...shellSpec, job_id: "legacy-shell-1" } }); + check("POST /jobs refuses shell to a run token", viaJobs.status === 403 && /admin/.test(viaJobs.body.error), JSON.stringify(viaJobs.body)); + const viaApi = await call("POST", "/api/jobs", { token: ALICE, body: { ...shellSpec, job_id: "legacy-shell-2" } }); + check("POST /api/jobs refuses it too (its forward is internal)", viaApi.status === 403, JSON.stringify(viaApi.body)); + const asAdmin = await call("POST", "/jobs", { token: ADMIN, body: { ...shellSpec, job_id: "legacy-shell-3" } }); + check("admin may queue shell through POST /jobs", asAdmin.status === 201, JSON.stringify(asAdmin.body)); + if (asAdmin.status === 201) await call("POST", "/api/v1/jobs/legacy-shell-3/cancel", { token: ADMIN }); + } + + console.log("ci-enqueue and ci-upload with tokens on"); + { + // A device of its own, so nothing queued above is claimed instead. + const CI_DEV = "auth-ci-dev"; + await call("POST", "/devices/register", { token: AGENT, body: { device_id: CI_DEV, descriptor: { model: "CI Phone", os: "Android 15" }, pools: ["auth-ci"], capabilities: ["benchmark"] } }); + const ciSpec = (id: string) => ({ schema: 1, job_id: id, workload: "benchmark", executor: "device", backend: "synthetic", targets: { device_id: CI_DEV } }); + const specFile = async (id: string, body: unknown = ciSpec(id)) => { + const f = path.join(dir, `${id}.json`); + await writeFile(f, JSON.stringify(body)); + return f; + }; + const runScript = (script: string, args: string[], env: Record) => { + const child = spawn(process.execPath, [TSX, `scripts/${script}`, ...args], { + cwd: ROOT, + env: { ...process.env, FLEET_URL: BASE, FLEET_TOKEN: "", FLEET_RUNNER_TOKEN: "", FLEET_DASH_TOKEN: "", GITHUB_EVENT_NAME: "", ...env }, + stdio: ["ignore", "pipe", "pipe"], + }); + let out = ""; + child.stdout?.on("data", (d) => { out += d; }); + child.stderr?.on("data", (d) => { out += d; }); + return new Promise<{ code: number | null; out: string }>((resolve) => child.on("exit", (code) => resolve({ code, out }))); + }; + const settle = async (id: string, result: Record) => { + for (let i = 0; i < 100; i++) { + if ((await call("GET", `/api/v1/jobs/${id}/summary`, { token: READ })).status === 200) break; + await new Promise((r) => setTimeout(r, 200)); + } + const claimed = await call("GET", `/devices/${CI_DEV}/next-job`, { token: AGENT }); + check(`the device claims ${id}`, claimed.status === 200 && claimed.body?.job_id === id, JSON.stringify(claimed.body)?.slice(0, 200)); + await call("POST", "/results", { token: AGENT, body: { schema: 1, kind: "result", job_id: id, device_id: CI_DEV, iter: 0, final: true, ...result } }); + }; + + const anon = await runScript("ci-enqueue.ts", ["--job", await specFile("ci-anon"), "--timeout", "0"], {}); + check("ci-enqueue without a token exits 5", anon.code === 5, `${anon.code}: ${anon.out.slice(-300)}`); + const queued = await runScript("ci-enqueue.ts", ["--job", await specFile("ci-queued"), "--timeout", "0"], { FLEET_RUNNER_TOKEN: ALICE }); + check("with FLEET_RUNNER_TOKEN it enqueues, and a job still queued at the timeout exits 3", queued.code === 3, `${queued.code}: ${queued.out.slice(-300)}`); + const q = await call("GET", "/api/v1/jobs/ci-queued/summary", { token: READ }); + check("the job is alice's, from ci", q.body?.requested_by === "alice" && q.body?.source === "ci", JSON.stringify([q.body?.requested_by, q.body?.source])); + await call("POST", "/api/v1/jobs/ci-queued/cancel", { token: ALICE }); + const invalid = await runScript("ci-enqueue.ts", ["--job", await specFile("ci-invalid", { schema: 1, job_id: "ci-invalid", workload: "benchmark", executor: "device", target: {} })], { FLEET_TOKEN: ALICE }); + check("an invalid spec exits 2 (FLEET_TOKEN works too)", invalid.code === 2, `${invalid.code}: ${invalid.out.slice(-300)}`); + + const skipping = runScript("ci-enqueue.ts", ["--job", await specFile("ci-skipped"), "--timeout", "60"], { FLEET_RUNNER_TOKEN: ALICE }); + await settle("ci-skipped", { ok: true, error: "skipped: no device attached" }); + const skipped = await skipping; + check("a skipped job exits 4, not 0", skipped.code === 4, `${skipped.code}: ${skipped.out.slice(-300)}`); + const passing = runScript("ci-enqueue.ts", ["--job", await specFile("ci-passed"), "--timeout", "60"], { FLEET_RUNNER_TOKEN: ALICE }); + await settle("ci-passed", { ok: true, metrics: { decode_tps: 1 } }); + const passed = await passing; + check("a passed job exits 0", passed.code === 0, `${passed.code}: ${passed.out.slice(-300)}`); + const failing = runScript("ci-enqueue.ts", ["--job", await specFile("ci-failed"), "--timeout", "60"], { FLEET_RUNNER_TOKEN: ALICE }); + await settle("ci-failed", { ok: false, error: "boom" }); + const failed = await failing; + check("a failed job exits 1", failed.code === 1, `${failed.code}: ${failed.out.slice(-300)}`); + + const apk = path.join(dir, "app.apk"); + await writeFile(apk, `apk-${Date.now()}`); + const upArgs = ["--artifact", apk, "--app", "auth-app", "--build", "1.0"]; + const upAnon = await runScript("ci-upload.ts", upArgs, {}); + check("ci-upload without a token is refused (protocol needs agent) and exits 5", upAnon.code === 5, `${upAnon.code}: ${upAnon.out.slice(-300)}`); + const up = await runScript("ci-upload.ts", upArgs, { FLEET_RUNNER_TOKEN: ADMIN, GITHUB_EVENT_NAME: "pull_request" }); + check("ci-upload sends the bearer token, and a pull_request event publishes on the pr channel", up.code === 0 && /pr channel/.test(up.out), `${up.code}: ${up.out.slice(-300)}`); + } + + console.log("a peer with tokens on, read through another brain"); + { + // A second collector, auth off, that lists this one as a peer. Without a + // token this brain answers its overview with 401; FLEET_PEER_TOKENS is + // what lets the other brain's switcher read it. + const port2 = await freePort(); + const BASE2 = `http://127.0.0.1:${port2}`; + peer = spawn(process.execPath, [TSX, "src/server.ts"], { + cwd: ROOT, + env: { + ...process.env, + FLEET_PORT: String(port2), FLEET_BIND: "127.0.0.1", + FLEET_DATA_DIR: path.join(dir, "data2"), FLEET_ARTIFACT_DIR: path.join(dir, "artifacts2"), + FLEET_LOG_FILE: path.join(dir, "collector2.log"), FLEET_TOKENS_FILE: path.join(dir, "tokens2.json"), + FLEET_AUTH: "", FLEET_AUTH_PROTOCOL: "", + FLEET_SWEEP_MS: "60000", FLEET_SCHEDULER_TICK_MS: "60000", + FLEET_PEERS: `${BASE}/`, + FLEET_PEER_TOKENS: JSON.stringify({ [BASE]: READ }), + }, + stdio: ["ignore", "pipe", "pipe"], + }); + peer.stdout?.on("data", (d) => { log += d; }); + peer.stderr?.on("data", (d) => { log += d; }); + await up(BASE2, peer); + const listed = await (await fetch(`${BASE2}/api/peers`)).json() as { peers: { id: string; reachable: boolean; devices: { total: number } | null }[] }; + const me = listed.peers[0]; + check("the other brain sees this one, with device counts it needed a token for", + !!me && me.reachable && (me.devices?.total ?? 0) >= 1, JSON.stringify(listed)); + const proxied = await fetch(`${BASE2}/api/peers/${me?.id}/devices`); + check("and its proxy reads this brain's devices with the peer token", proxied.status === 200, String(proxied.status)); + } + console.log("delegation"); { const w = await call("GET", "/api/v1/whoami", { token: GATEWAY, headers: { "x-fleet-actor": "carol@wbd.com" } }); @@ -213,8 +337,8 @@ try { console.error(` FAIL ${(e as Error).message}`); } finally { if (failures) console.error(`\n--- collector output ---\n${log.slice(-4000)}`); - const child = server; - if (child && child.exitCode === null) { + for (const child of [server, peer]) { + if (!child || child.exitCode !== null) continue; const exited = new Promise((resolve) => child.once("exit", () => resolve())); child.kill("SIGTERM"); await Promise.race([exited, new Promise((r) => setTimeout(r, 10_000))]); diff --git a/collector/scripts/check-examples.ts b/collector/scripts/check-examples.ts index a9119df..16bcb2e 100644 --- a/collector/scripts/check-examples.ts +++ b/collector/scripts/check-examples.ts @@ -1,6 +1,6 @@ -// Every example job spec in examples/jobs/ is validated against -// schemas/job.schema.json, and the docs include those files rather than -// copies of them. +// Every example job spec in examples/jobs/ is validated against the job schema +// /api/v1 enforces (strict; see below), and the docs include those files +// rather than copies of them. // // This exists because the schema is documentation that nothing enforced. The // collector does not validate POST /jobs against it — it never has — so the @@ -21,26 +21,23 @@ // schema, which is the same argument schemas/metrics.json is checked under. // The examples are the enforcement: they are what the docs show a reader, so // if they stop validating, the docs are lying to somebody. +// +// ## Strict, since 0.9 +// +// The check used to be the loose job.schema.json, which allows keys it does not +// know and has no opinion about a workload's params. `/api/v1` is stricter: an +// unknown key is refused, and params are checked against the workload's +// manifest. An example that passed here could still be refused by the API it +// documents, so this now runs the API's own check -- the same compiled schemas, +// imported from src/api/v1/spec-check.ts, not a copy. import { readFileSync, readdirSync } from "node:fs"; import path from "node:path"; import { fileURLToPath } from "node:url"; -// The named export rather than the default: ajv ships this entry point as CJS, -// and under NodeNext a default import resolves to the module namespace, which -// tsc correctly refuses to call with `new`. -import { Ajv2020 } from "ajv/dist/2020.js"; +import { specProblems } from "../src/api/v1/spec-check.js"; const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); -const SCHEMA = path.join(ROOT, "schemas/job.schema.json"); const DIR = path.join(ROOT, "examples/jobs"); -const schema = JSON.parse(readFileSync(SCHEMA, "utf8")); - -// strict:false because the schema uses `examples` and prose `description` -// alongside `oneOf`, which ajv's strict mode flags as ambiguous authoring -// rather than as an error in the data. -const ajv = new Ajv2020({ allErrors: true, strict: false }); -const validate = ajv.compile(schema); - const files = readdirSync(DIR).filter((f) => f.endsWith(".json")).sort(); if (files.length === 0) { console.error("examples/jobs is empty; refusing to pass"); @@ -59,11 +56,10 @@ for (const file of files) { continue; } - if (!validate(spec)) { - for (const err of validate.errors ?? []) { - problems.push(`${file}${err.instancePath || "/"} ${err.message}`); - } - } + // What POST /api/v1/jobs fills in before it validates, and nothing more. + const raw = spec && typeof spec === "object" && !Array.isArray(spec) ? (spec as Record) : {}; + const asPosted = { ...raw, schema: raw.schema ?? 1, job_id: raw.job_id ?? `example-${file}` }; + for (const err of specProblems(asPosted)) problems.push(`${file}${err.path} ${err.message}`); // The docs promise the filename names the workload it demonstrates, and a // reader looking for `drain` should not have to open every file to find it. @@ -79,9 +75,9 @@ if (problems.length) { for (const p of problems) console.error(` ${p}`); console.error( "\nThese files are included verbatim in docs/integration/cookbook.md, so a\n" + - "spec that does not validate is a documented example that would be refused.\n", + "spec that does not validate is a documented example POST /api/v1/jobs would refuse.\n", ); process.exit(1); } -console.log(` ok — ${files.length} example job specs validate against job.schema.json`); +console.log(` ok — ${files.length} example job specs pass /api/v1's strict schema and their params manifests`); diff --git a/collector/scripts/ci-enqueue.ts b/collector/scripts/ci-enqueue.ts index 326c6cf..fa5d165 100644 --- a/collector/scripts/ci-enqueue.ts +++ b/collector/scripts/ci-enqueue.ts @@ -1,77 +1,173 @@ -// CI entry point: upload a build artifact (optional), enqueue a job, poll to -// completion, exit 0/1 on the verdict. This is what a GitHub Actions step -// runs over Tailscale — see ci/example-workflow.yml. Standalone-testable: -// npx tsx scripts/ci-enqueue.ts --job job.json [--artifact app.apk] [--timeout 900] +// CI entry point: upload a build (optional), enqueue a job through /api/v1, +// wait for the verdict, and exit with the same codes as `fleetr run`: +// +// 0 passed 1 failed, mixed or cancelled 2 usage or invalid spec +// 3 still running when --timeout ran out 4 skipped (nothing ran) +// 5 refused or unreachable +// +// A skip is exit 4, not 0: a job that reported `skipped:` tested nothing, and a +// check that goes green on it is lying. This is what a GitHub Actions step +// runs; see ci/example-workflow.yml. Standalone-testable: +// +// npx tsx scripts/ci-enqueue.ts --job job.json [--artifact app.apk] [--timeout 900] [--channel pr|main] +// +// Auth: FLEET_RUNNER_TOKEN (or FLEET_TOKEN) is sent as a bearer token, which a +// collector with FLEET_AUTH=tokens requires (`run` scope). FLEET_DASH_TOKEN is +// still sent as x-fleet-token for a collector that only has the dashboard's +// speed bump. The upload rides the runner protocol, so with +// FLEET_AUTH_PROTOCOL=token the token needs `agent` as well. +// +// --artifact uploads the build and writes its hash into the spec's app.sha256 +// (or model.sha256), replacing whatever placeholder was there +// ("replaced-by-upload" in the examples). A pull request's build is uploaded on +// the `pr` channel, so it never becomes the `latest` a nightly tests: that is +// automatic when GITHUB_EVENT_NAME is pull_request or pull_request_target, and +// --channel pr|main says it explicitly. import { createHash } from "node:crypto"; import { readFileSync } from "node:fs"; +import { FleetApiError, FleetClient, type JobSummary } from "../src/client/index.js"; + +const EXIT = { passed: 0, failed: 1, usage: 2, pending: 3, skipped: 4, error: 5 } as const; + +const USAGE = + "usage: ci-enqueue --job [--artifact ] [--collector ] [--timeout ] [--channel pr|main]"; + +function die(code: number, msg: string): never { + console.error(`ci-enqueue: ${msg}`); + process.exit(code); +} const args = new Map(); for (let i = 2; i < process.argv.length; i += 2) { - args.set(process.argv[i].replace(/^--/, ""), process.argv[i + 1]); + const flag = process.argv[i]; + const value = process.argv[i + 1]; + if (!flag.startsWith("--") || value === undefined) die(EXIT.usage, `${flag} needs a value\n${USAGE}`); + args.set(flag.replace(/^--/, ""), value); } +const known = new Set(["job", "artifact", "collector", "timeout", "channel"]); +for (const k of args.keys()) if (!known.has(k)) die(EXIT.usage, `unknown flag --${k}\n${USAGE}`); const BASE = (args.get("collector") ?? process.env.FLEET_URL ?? "http://127.0.0.1:8788").replace(/\/$/, ""); const JOB_FILE = args.get("job"); const ARTIFACT = args.get("artifact"); const TIMEOUT_S = Number(args.get("timeout") ?? 900); +const TOKEN = (process.env.FLEET_RUNNER_TOKEN || process.env.FLEET_TOKEN || "").trim() || null; +const DASH_TOKEN = process.env.FLEET_DASH_TOKEN?.trim() || null; -if (!JOB_FILE) { - console.error("usage: ci-enqueue --job [--artifact ] [--collector ] [--timeout ]"); - process.exit(2); -} +if (!JOB_FILE) die(EXIT.usage, USAGE); +if (!Number.isFinite(TIMEOUT_S) || TIMEOUT_S < 0) die(EXIT.usage, "--timeout must be a number of seconds"); -const spec = JSON.parse(readFileSync(JOB_FILE, "utf8")); +/** `pr`, or null for a build that may become `latest`. */ +function channelOf(flag: string | undefined, event: string | undefined): "pr" | null { + if (flag === "pr") return "pr"; + if (flag === "main") return null; + if (flag !== undefined) die(EXIT.usage, `--channel must be pr or main, not ${flag}`); + return event === "pull_request" || event === "pull_request_target" ? "pr" : null; +} +const CHANNEL = channelOf(args.get("channel"), process.env.GITHUB_EVENT_NAME); -function fail(msg: string): never { - console.error(`ci-enqueue: ${msg}`); - process.exit(1); +let spec: Record; +try { + spec = JSON.parse(readFileSync(JOB_FILE, "utf8")); +} catch (e) { + die(EXIT.usage, `cannot read ${JOB_FILE}: ${(e as Error).message}`); } +if (!spec || typeof spec !== "object" || Array.isArray(spec)) die(EXIT.usage, `${JOB_FILE} is not a JSON job spec`); + +const client = new FleetClient({ + baseUrl: BASE, + token: TOKEN, + source: "ci", + headers: DASH_TOKEN ? { "x-fleet-token": DASH_TOKEN } : {}, +}); if (ARTIFACT) { - const body = readFileSync(ARTIFACT); - const res = await fetch(`${BASE}/artifacts`, { - method: "POST", - // Tag the build with the app it belongs to, so a nightly can ask for the - // latest one instead of pinning a hash somebody has to remember to bump. - headers: { - "content-type": "application/octet-stream", - "x-artifact-name": ARTIFACT.split("/").pop()!, - ...(spec.app?.name ? { "x-artifact-app": String(spec.app.name) } : {}), - ...(spec.app?.build ? { "x-artifact-build": String(spec.app.build) } : {}), - ...(spec.app?.platform ? { "x-artifact-platform": String(spec.app.platform) } : {}), - }, - body, - }); - if (!res.ok) fail(`artifact upload -> HTTP ${res.status}`); + let body: Buffer; + try { + body = readFileSync(ARTIFACT); + } catch (e) { + die(EXIT.usage, `cannot read ${ARTIFACT}: ${(e as Error).message}`); + } + let res: Response; + try { + res = await fetch(`${BASE}/artifacts`, { + method: "POST", + // Tag the build with the app it belongs to, so a nightly can ask for the + // latest one instead of pinning a hash somebody has to remember to bump. + headers: { + ...client.headers(), + "content-type": "application/octet-stream", + "x-artifact-name": ARTIFACT.split("/").pop()!, + ...(spec.app?.name ? { "x-artifact-app": String(spec.app.name) } : {}), + ...(spec.app?.build ? { "x-artifact-build": String(spec.app.build) } : {}), + ...(spec.app?.platform ? { "x-artifact-platform": String(spec.app.platform) } : {}), + ...(CHANNEL ? { "x-artifact-channel": CHANNEL } : {}), + }, + body: new Uint8Array(body), + }); + } catch (e) { + die(EXIT.error, `cannot reach ${BASE}: ${(e as Error).message}`); + } + if (!res.ok) die(EXIT.error, `artifact upload -> HTTP ${res.status}: ${(await res.text()).slice(0, 300)}`); const { sha256 } = (await res.json()) as { sha256: string }; const local = createHash("sha256").update(body).digest("hex"); - if (sha256 !== local) fail(`artifact hash mismatch: ${sha256} != ${local}`); + if (sha256 !== local) die(EXIT.error, `artifact hash mismatch: ${sha256} != ${local}`); if (spec.app) spec.app.sha256 = sha256; else if (spec.model) spec.model.sha256 = sha256; - console.log(`artifact uploaded: ${sha256}`); + console.log(`artifact uploaded: ${sha256}${CHANNEL ? " (pr channel: never latest)" : ""}`); } -const enq = await fetch(`${BASE}/jobs`, { - method: "POST", - headers: { "content-type": "application/json" }, - body: JSON.stringify(spec), -}); -if (enq.status === 409) console.log(`job ${spec.job_id} already queued; polling it`); -else if (!enq.ok) fail(`enqueue -> HTTP ${enq.status}: ${await enq.text()}`); -else console.log(`enqueued ${spec.job_id}`); +let jobId: string; +try { + const created = await client.createJob(spec as never); + jobId = created.job_id; + console.log(`enqueued ${jobId}`); +} catch (e) { + if (!(e instanceof FleetApiError)) throw e; + if (e.code === "duplicate_job" && typeof spec.job_id === "string") { + jobId = spec.job_id; + console.log(`job ${jobId} already queued; waiting on it`); + } else { + const schemaErrors = (e.details as { schema_errors?: { path: string; message: string }[] } | undefined)?.schema_errors ?? []; + for (const se of schemaErrors) console.error(` ${se.path} ${se.message}`); + const usage = e.status === 400; + die(usage ? EXIT.usage : EXIT.error, `enqueue refused (${e.status} ${e.code}): ${e.message}${e.hint ? `\n ${e.hint}` : ""}`); + } +} +// Long-poll /wait in slices of at most 120 s (the server's limit) until the +// job finishes or --timeout runs out. A collector that blips is retried until +// the deadline; one that never answers again is exit 5, not 3. const deadline = Date.now() + TIMEOUT_S * 1000; -let status = "queued"; -while (Date.now() < deadline) { - const res = await fetch(`${BASE}/jobs/${spec.job_id}`); - if (res.ok) { - status = ((await res.json()) as { status: string }).status; - if (status === "done" || status === "failed") break; +let summary: JobSummary | null = null; +let lastError: FleetApiError | null = null; +for (;;) { + const left = Math.max(0, Math.floor((deadline - Date.now()) / 1000)); + try { + summary = await client.waitJob(jobId, { timeout_s: Math.min(120, left) }, { timeoutMs: (Math.min(120, left) + 30) * 1000 }); + lastError = null; + if (summary.finished) break; + } catch (e) { + if (!(e instanceof FleetApiError)) throw e; + if (e.status !== 0 && e.status < 500) die(EXIT.error, `wait refused (${e.status} ${e.code}): ${e.message}`); + lastError = e; + await new Promise((r) => setTimeout(r, Math.min(10_000, Math.max(0, deadline - Date.now())))); } - await new Promise((r) => setTimeout(r, 10_000)); + if (Date.now() >= deadline) break; +} + +if (!summary || lastError) die(EXIT.error, lastError?.message ?? `no answer from ${BASE}`); + +console.log(`job ${jobId}: ${summary.finished ? summary.verdict : `still ${summary.verdict}`}`); +for (const d of summary.devices) { + const state = d.skipped ? "skipped" : d.ok === true ? "passed" : d.ok === false ? "FAILED" : d.status; + console.log(` ${d.device_id} ${state}`); + if (d.failing_step) console.log(` failing step: ${d.failing_step}`); + if (d.error && !d.ok) console.log(` ${d.error.split("\n")[0].slice(0, 300)}`); } +console.log(`details: ${BASE}${summary.links.detail}`); -console.log(`job ${spec.job_id}: ${status}`); -if (status === "done") process.exit(0); -if (status === "failed") fail("job failed — per-device details on the collector dashboard"); -fail(`timed out after ${TIMEOUT_S}s (status: ${status})`); +if (!summary.finished) die(EXIT.pending, `still ${summary.verdict} after ${TIMEOUT_S}s; it keeps running. fleetr wait ${jobId}`); +if (summary.verdict === "passed") process.exit(EXIT.passed); +if (summary.verdict === "skipped") die(EXIT.skipped, "skipped: nothing ran, so nothing was tested. This is not a pass."); +die(EXIT.failed, `${summary.verdict}: per-device details above and on the dashboard`); diff --git a/collector/scripts/ci-upload.ts b/collector/scripts/ci-upload.ts index 95d0a4c..8a1aa7a 100644 --- a/collector/scripts/ci-upload.ts +++ b/collector/scripts/ci-upload.ts @@ -9,6 +9,18 @@ // are different events. A merge to main should publish whether or not a device // is free, and the nightly picks it up later. Coupling them means a busy shelf // silently stops builds from being published. +// +// A pull request's build is published on the `pr` channel, which `latest` +// never resolves to, so a branch never becomes what the nightly tests. That is +// automatic when GITHUB_EVENT_NAME is pull_request or pull_request_target; +// --channel pr|main says it explicitly. +// +// Auth: FLEET_RUNNER_TOKEN (or FLEET_TOKEN) is sent as a bearer token. An +// upload is the runner protocol, so a collector with FLEET_AUTH_PROTOCOL=token +// wants a token with the `agent` scope; with the protocol open (the default) +// none is needed. FLEET_DASH_TOKEN is still sent as x-fleet-token. +// +// Exits 0 published, 2 usage, 5 refused or unreachable, like `fleetr`. import { readFileSync } from "node:fs"; const args = new Map(); @@ -21,30 +33,51 @@ const ARTIFACT = args.get("artifact"); const APP = args.get("app"); const BUILD = args.get("build"); const PLATFORM = args.get("platform"); -const TOKEN = process.env.FLEET_DASH_TOKEN; +const CHANNEL_FLAG = args.get("channel"); +const TOKEN = (process.env.FLEET_RUNNER_TOKEN || process.env.FLEET_TOKEN || "").trim() || null; +const DASH_TOKEN = process.env.FLEET_DASH_TOKEN?.trim() || null; if (!ARTIFACT || !APP || !BUILD) { - console.error("usage: ci-upload --artifact --app --build [--platform

] [--collector ]"); + console.error("usage: ci-upload --artifact --app --build [--platform

] [--channel pr|main] [--collector ]"); process.exit(2); } +if (CHANNEL_FLAG !== undefined && CHANNEL_FLAG !== "pr" && CHANNEL_FLAG !== "main") { + console.error(`ci-upload: --channel must be pr or main, not ${CHANNEL_FLAG}`); + process.exit(2); +} +const event = process.env.GITHUB_EVENT_NAME; +const PR = CHANNEL_FLAG ? CHANNEL_FLAG === "pr" : event === "pull_request" || event === "pull_request_target"; const body = readFileSync(ARTIFACT); -const res = await fetch(`${BASE}/artifacts`, { - method: "POST", - headers: { - "content-type": "application/octet-stream", - "x-artifact-name": ARTIFACT.split("/").pop()!, - "x-artifact-app": APP, - "x-artifact-build": BUILD, - ...(PLATFORM ? { "x-artifact-platform": PLATFORM } : {}), - ...(TOKEN ? { "x-fleet-token": TOKEN } : {}), - }, - body, -}); +let res: Response; +try { + res = await fetch(`${BASE}/artifacts`, { + method: "POST", + headers: { + "content-type": "application/octet-stream", + "x-artifact-name": ARTIFACT.split("/").pop()!, + "x-artifact-app": APP, + "x-artifact-build": BUILD, + "x-fleet-source": "ci", + ...(PLATFORM ? { "x-artifact-platform": PLATFORM } : {}), + ...(PR ? { "x-artifact-channel": "pr" } : {}), + ...(TOKEN ? { authorization: `Bearer ${TOKEN}` } : {}), + ...(DASH_TOKEN ? { "x-fleet-token": DASH_TOKEN } : {}), + }, + body: new Uint8Array(body), + }); +} catch (e) { + console.error(`ci-upload: cannot reach ${BASE}: ${(e as Error).message}`); + process.exit(5); +} if (!res.ok) { console.error(`ci-upload: ${res.status} ${await res.text()}`); - process.exit(1); + process.exit(5); } const { sha256, size } = (await res.json()) as { sha256: string; size: number }; -console.log(`published ${APP} ${BUILD} -> ${sha256} (${size} bytes)`); -console.log(`nightlies asking for the latest ${APP} will now test this build`); +console.log(`published ${APP} ${BUILD} -> ${sha256} (${size} bytes)${PR ? " on the pr channel" : ""}`); +console.log( + PR + ? `a pull request's build: jobs that name ${sha256} test it; nightlies asking for the latest ${APP} do not` + : `nightlies asking for the latest ${APP} will now test this build`, +); diff --git a/collector/scripts/smoke.ts b/collector/scripts/smoke.ts index 749931f..c1c7d2a 100644 --- a/collector/scripts/smoke.ts +++ b/collector/scripts/smoke.ts @@ -17,6 +17,7 @@ import { runDeviceParserChecks } from "../src/device-parsers.test.js"; import { runDriverChecks, runDescriptorChecks } from "../src/drivers/drivers.test.js"; import { runZipChecks } from "../src/zip-dir.test.js"; import { runTailnetChecks } from "../src/tailnet.test.js"; +import { runConfigChecks } from "../src/config.test.js"; import { runDbChecks } from "../src/db.test.js"; import { runAlertChecks } from "../src/alerts.test.js"; import { runBrowserChecks } from "../src/browser.test.js"; @@ -1430,8 +1431,81 @@ for (const [workload, device] of [["web-audit", "web:audit"], ["web-unfurl", "we check("a job asking for a build nobody has published is rejected", ghost.status === 400, `status ${ghost.status}`); + // A pull request's build never becomes `latest`: the nightly tests what was + // merged. The PR's own job still names it by hash. + const putPr = async (bytes: string, build: string, channel: string) => + fetch(`${BASE}/artifacts`, { + method: "POST", + headers: { + "content-type": "application/octet-stream", "x-artifact-name": `${build}.apk`, + "x-artifact-app": APP, "x-artifact-build": build, "x-artifact-channel": channel, + }, + body: bytes, + }); + const prRes = await putPr(`pr-${run}`, "pr-42", "pr"); + const pr = (await prRes.json()) as { sha256: string; channel: string | null }; + check("an upload can say it is a pull request's build", prRes.status === 201 && pr.channel === "pr", JSON.stringify(pr)); + const AFTER_PR = `smoke-after-pr-${run}`; + await json("POST", "/jobs", { + schema: 1, job_id: AFTER_PR, workload: "install", executor: "host", + app: { name: APP, build: "latest", sha256: "latest" }, + targets: { executor: `never-${run}` }, lease: { ttl_s: 600, max_attempts: 1 }, + }); + const afterPr = await json("GET", `/api/jobs/${AFTER_PR}`); + check("latest skips a pull request's build", afterPr.body?.spec?.app?.sha256 === older.sha256, + `${afterPr.body?.spec?.app?.sha256} (pr=${pr.sha256} merged=${older.sha256})`); + const PR_JOB = `smoke-pr-job-${run}`; + const prJob = await json("POST", "/jobs", { + schema: 1, job_id: PR_JOB, workload: "install", executor: "host", + app: { name: APP, build: "pr-42", sha256: pr.sha256 }, + targets: { executor: `never-${run}` }, lease: { ttl_s: 600, max_attempts: 1 }, + }); + check("a job may still name a pull request's build by hash", prJob.status === 201, JSON.stringify(prJob.body)); + const listed = await json("GET", `/api/artifacts?app=${encodeURIComponent(APP)}`); + check("the artifact list shows the channel", + (listed.body?.artifacts ?? []).some((a: any) => a.sha256 === pr.sha256 && a.channel === "pr"), JSON.stringify(listed.body?.artifacts?.[0])); + // The same bytes published again from main come off the channel and lead. + await putPr(`pr-${run}`, "1.3.0", ""); + const MERGED = `smoke-merged-${run}`; + await json("POST", "/jobs", { + schema: 1, job_id: MERGED, workload: "install", executor: "host", + app: { name: APP, build: "latest", sha256: "latest" }, + targets: { executor: `never-${run}` }, lease: { ttl_s: 600, max_attempts: 1 }, + }); + const merged = await json("GET", `/api/jobs/${MERGED}`); + check("the same bytes republished from main become latest", merged.body?.spec?.app?.sha256 === pr.sha256, + JSON.stringify(merged.body?.spec?.app)); + // And a PR build matching bytes main already published leaves that row alone. + await putPr(`older-${run}`, "pr-43", "pr"); + const STILL = `smoke-still-${run}`; + await json("POST", "/jobs", { + schema: 1, job_id: STILL, workload: "install", executor: "host", + app: { name: APP, build: "latest", sha256: "latest" }, + targets: { executor: `never-${run}` }, lease: { ttl_s: 600, max_attempts: 1 }, + }); + const still = await json("GET", `/api/jobs/${STILL}`); + check("a PR build matching a merged one neither moves nor demotes it", still.body?.spec?.app?.sha256 === pr.sha256, + JSON.stringify(still.body?.spec?.app)); + const typo = await putPr(`typo-${run}`, "x", "PR-maybe"); + check("an unknown channel is refused, not published", typo.status === 400, String(typo.status)); + for (const id of [SCHED, EMPTY]) await json("DELETE", `/schedules/${id}`, undefined); - for (const j of [jobId, REVERT]) if (j) await json("POST", `/api/jobs/${j}/cancel`, {}); + for (const j of [jobId, REVERT, AFTER_PR, PR_JOB, MERGED, STILL]) if (j) await json("POST", `/api/jobs/${j}/cancel`, {}); +} + +// 31b. a schedule template's job_id is never used, and the response says so +{ + const ID = `smoke-jobid-${run}`; + const template = { schema: 1, workload: "install", executor: "host", app: { name: "x", sha256: "latest" }, targets: { executor: `never-${run}` } }; + const refused = await json("POST", "/schedules", { id: ID, cron: "0 3 * * *", enabled: false, template: { ...template, job_id: "mine" } }); + check("a schedule template with a job_id is refused", refused.status === 400, JSON.stringify(refused.body)); + const warned = await json("POST", "/api/schedules", { id: ID, cron: "0 3 * * *", enabled: false, template: { ...template, job_id: "" } }); + check("an empty job_id is dropped with a warning", warned.status === 201 && (warned.body?.warnings ?? []).some((w: string) => w.includes("job_id")), JSON.stringify(warned.body)); + const stored = ((await json("GET", "/schedules")).body as any[]).find((s) => s.id === ID); + check("and is not stored", stored && !("job_id" in stored.template), JSON.stringify(stored?.template)); + const clean = await json("POST", "/schedules", { id: ID, cron: "0 3 * * *", enabled: false, template }); + check("a clean template has no warnings", clean.status === 201 && clean.body?.warnings === undefined, JSON.stringify(clean.body)); + await json("DELETE", `/schedules/${ID}`, undefined); } // 32. xcodebuild test counts: what ran, not what the exit code implied @@ -2576,6 +2650,9 @@ runZipChecks(check); // working; one that wrongly admits is noticed never. runTailnetChecks(check); +// --- environment knobs that used to be a bare Number() --- +runConfigChecks(check); + // The database layer itself: transactions, the open/close lifecycle, the // 40. an unpinned host job goes to the executor that holds a matching device diff --git a/collector/src/api/mutations.ts b/collector/src/api/mutations.ts index ee5a30c..029adeb 100644 --- a/collector/src/api/mutations.ts +++ b/collector/src/api/mutations.ts @@ -12,7 +12,8 @@ import { db } from "../db.js"; import { powerRetention } from "../power.js"; import { isValidMatch } from "../match.js"; import { requireToken } from "./guard.js"; -import { forward } from "../auth.js"; +import { authOf, forward } from "../auth.js"; +import { needsAdmin, notOwned } from "./policy.js"; import { iso, parse, sha256Refs } from "./shared.js"; /** How long explore keeps a whole night's trajectories before only the findings' remain (plan, phase 5). */ @@ -294,6 +295,9 @@ export function registerMutations(app: FastifyInstance, announce: Announce, matc app.post("/api/jobs/:id/cancel", async (req, reply) => { if (!requireToken(req, reply)) return; const { id } = req.params as { id: string }; + // The same owner check /api/v1 makes: a run token stops only its own jobs. + const foreign = notOwned(authOf(req), [id]); + if (foreign) return reply.code(403).send({ error: foreign.message, hint: foreign.hint }); const reason = ((req.body as { reason?: string } | null)?.reason ?? "cancelled from the dashboard").slice(0, 500); const out = cancelTx(id, reason); @@ -322,6 +326,8 @@ export function registerMutations(app: FastifyInstance, announce: Announce, matc app.post("/api/jobs/:id/retry", async (req, reply) => { if (!requireToken(req, reply)) return; const { id } = req.params as { id: string }; + const foreign = notOwned(authOf(req), [id]); + if (foreign) return reply.code(403).send({ error: foreign.message, hint: foreign.hint }); const body = (req.body ?? {}) as { pool?: string; device_id?: string; priority?: number }; const row = db.prepare("SELECT spec, template_id FROM jobs WHERE job_id = ?").get(id) as @@ -373,6 +379,10 @@ export function registerMutations(app: FastifyInstance, announce: Announce, matc // The composer's enqueue. Guarded, then handed to POST /jobs unchanged. app.post("/api/jobs", async (req, reply) => { if (!requireToken(req, reply)) return; + // Asked here and not only in POST /jobs, because the forward below arrives + // there as an internal request, which has already been decided. + const refused = needsAdmin(authOf(req), (req.body as { workload?: unknown } | null)?.workload); + if (refused) return reply.code(403).send({ error: refused.message, hint: refused.hint }); const res = await forward(app, req, { method: "POST", url: "/jobs", payload: req.body as object }); return reply.code(res.statusCode).send(res.json()); }); diff --git a/collector/src/api/peers.ts b/collector/src/api/peers.ts index 06e482e..dd1bc72 100644 --- a/collector/src/api/peers.ts +++ b/collector/src/api/peers.ts @@ -17,7 +17,7 @@ * * ## Why the collector proxies instead of the browser calling directly * - * Because there is no authentication, by design, and CORS would be the wrong + * Because there is no authentication by default, and CORS would be the wrong * end of the stick. Opening the read API cross-origin would let any website * open in the operator's browser read their entire fleet -- device names, * network prefixes, job history -- from any tab. The proxy keeps one origin, @@ -31,11 +31,19 @@ * brain means switching to it, which navigates to that brain's own dashboard: * one origin, one owner of that queue, and no question about which fleet a * button just acted on. + * + * ## A peer with tokens on + * + * A peer running `FLEET_AUTH=tokens` answers 401 to a request without a token, + * so this collector sends the one FLEET_PEER_TOKENS maps that peer's URL to, as + * a bearer token, on every request to it. One token per peer, because each + * brain issues its own: a token is a hash in that brain's tokens.json. Never + * the caller's token -- the person reading this dashboard holds a token for + * this brain, not for the other one. */ import type { FastifyInstance } from "fastify"; -import { PEERS } from "../config.js"; +import { DATA_DIR, PEER_TOKENS, PEERS, peerKey } from "../config.js"; import { identity } from "../identity.js"; -import { DATA_DIR } from "../config.js"; /** How long to wait on a peer before calling it unreachable. */ const PEER_TIMEOUT_MS = 4_000; @@ -75,10 +83,16 @@ export type PeerStatus = { devices: { total: number; online: number } | null; }; +/** The headers every request to this peer carries: its token, when one is configured. */ +function peerHeaders(base: string): Record { + const token = PEER_TOKENS[peerKey(base)]; + return { accept: "application/json", ...(token ? { authorization: `Bearer ${token}` } : {}) }; +} + async function ask(url: string): Promise { - const base = url.replace(/\/+$/, ""); + const base = peerKey(url); try { - const res = await fetch(`${base}/api/health`, { signal: AbortSignal.timeout(PEER_TIMEOUT_MS) }); + const res = await fetch(`${base}/api/health`, { signal: AbortSignal.timeout(PEER_TIMEOUT_MS), headers: peerHeaders(base) }); if (!res.ok) { return { url: base, id: null, name: null, version: null, reachable: false, error: `HTTP ${res.status}`, devices: null }; } @@ -87,6 +101,7 @@ async function ask(url: string): Promise { try { const overview = (await fetch(`${base}/api/overview`, { signal: AbortSignal.timeout(PEER_TIMEOUT_MS), + headers: peerHeaders(base), }).then((r) => r.json())) as { devices?: { total?: number; online?: number } }; if (overview.devices) { devices = { total: overview.devices.total ?? 0, online: overview.devices.online ?? 0 }; @@ -163,7 +178,7 @@ export function registerPeers(app: FastifyInstance) { try { const res = await fetch(`${peer.url}/api/${rest}${query}`, { signal: AbortSignal.timeout(PEER_TIMEOUT_MS), - headers: { accept: "application/json" }, + headers: peerHeaders(peer.url), }); const body = await res.text(); return reply.code(res.status).type(res.headers.get("content-type") ?? "application/json").send(body); diff --git a/collector/src/api/policy.ts b/collector/src/api/policy.ts new file mode 100644 index 0000000..d47d983 --- /dev/null +++ b/collector/src/api/policy.ts @@ -0,0 +1,52 @@ +/** + * The two decisions a scope alone cannot make, in one place. + * + * The scope check in `auth.ts` answers "may this token enqueue, cancel or + * retry at all". Two finer questions depend on what is being asked for: + * + * - **Whose job is it?** A `run` token may cancel or retry only jobs it + * started; `admin` may act on any. + * - **What would it run?** A dangerous workload (`shell`: a script runner on + * somebody's machine) needs `admin`, however the spec is phrased. + * + * Both are asked by `/api/v1` and by the older routes (`POST /jobs`, + * `POST /api/jobs`, `POST /api/jobs/:id/cancel|retry`). They used to be asked + * by `/api/v1` only, which made the old routes a way round them. Each caller + * turns a refusal into its own error shape -- the control API's + * `{error:{code,...}}`, the older routes' `{error: "sentence"}` -- but the + * decision is this function's. + * + * With `FLEET_AUTH` off nothing here refuses anything, which is the posture a + * collector without tokens has always had. A request forwarded from inside the + * process (`internal`) was already decided by the request that forwarded it. + */ +import type { AuthContext } from "../auth.js"; +import { db } from "../db.js"; +import { catalogEntry } from "../workloads/catalog.js"; + +export type Refusal = { message: string; hint: string }; + +/** Why this caller may not cancel or retry these jobs, or null when it may. */ +export function notOwned(ctx: AuthContext, ids: string[]): Refusal | null { + if (ctx.mode !== "tokens" || ctx.internal || ctx.scopes.has("admin") || ids.length === 0) return null; + const rows = db.prepare(`SELECT job_id, requested_by FROM jobs WHERE job_id IN (${ids.map(() => "?").join(",")})`).all(...ids) as { + job_id: string; requested_by: string | null; + }[]; + const foreign = rows.find((r) => r.requested_by !== ctx.actor); + if (!foreign) return null; + return { + message: `you can stop only jobs you started; ${foreign.job_id} was started by ${foreign.requested_by ?? "someone unrecorded"}`, + hint: "an admin token can act on any job", + }; +} + +/** Why this caller may not queue this workload, or null when it may. */ +export function needsAdmin(ctx: AuthContext, workload: unknown): Refusal | null { + if (ctx.mode !== "tokens" || ctx.internal || ctx.scopes.has("admin")) return null; + const entry = typeof workload === "string" ? catalogEntry(workload) : null; + if (!entry?.dangerous) return null; + return { + message: `the ${entry.name} workload needs the admin scope`, + hint: "ask an operator to run it, or use a token with admin", + }; +} diff --git a/collector/src/api/system.ts b/collector/src/api/system.ts index 862b5ea..8be7033 100644 --- a/collector/src/api/system.ts +++ b/collector/src/api/system.ts @@ -261,14 +261,14 @@ export function registerSystem(app: FastifyInstance) { const total = (db.prepare(`SELECT COUNT(*) AS n FROM artifacts${sql}`).get(...params) as { n: number }).n; const rows = db .prepare( - `SELECT sha256, name, size, created_at, app, build, platform, published_at, pinned, pin_reason + `SELECT sha256, name, size, created_at, app, build, platform, published_at, channel, pinned, pin_reason FROM artifacts${sql} ORDER BY COALESCE(publish_seq, rowid) DESC LIMIT ? OFFSET ?`, ) .all(...params, per_page, offset) as { sha256: string; name: string | null; size: number; created_at: string; app: string | null; build: string | null; platform: string | null; published_at: string | null; - pinned: number; pin_reason: string | null; + channel: string | null; pinned: number; pin_reason: string | null; }[]; const baselineShas = new Set( diff --git a/collector/src/api/v1/routes.ts b/collector/src/api/v1/routes.ts index 4916686..60f749a 100644 --- a/collector/src/api/v1/routes.ts +++ b/collector/src/api/v1/routes.ts @@ -23,7 +23,6 @@ import { createReadStream, existsSync, statSync } from "node:fs"; import { randomBytes } from "node:crypto"; import path from "node:path"; import type { FastifyInstance, FastifyReply, FastifyRequest } from "fastify"; -import { Ajv2020, type ValidateFunction } from "ajv/dist/2020.js"; import { onAnnounce } from "../../announce.js"; import { authOf, forward, type Scope } from "../../auth.js"; import { ARTIFACT_DIR, AUTH, AUTH_PROTOCOL } from "../../config.js"; @@ -33,8 +32,10 @@ import { enqueueJob, WORKLOADS, type ErrorCode, type JobSpec } from "../../queue import { CATALOG, catalogEntry, type CatalogEntry } from "../../workloads/catalog.js"; import { isTextual } from "../../media-type.js"; import { AGE, capabilityMatches, deviceCapabilities, inClause, iso, isExpired, ONLINE_S, paging, parse, sha256Refs } from "../shared.js"; +import { needsAdmin, notOwned } from "../policy.js"; import { schedulesView } from "../system.js"; import { SCHEMAS } from "./schemas.js"; +import { errorsOf, paramsValidator, validateSpec, type SchemaError } from "./spec-check.js"; // --- the table's types ------------------------------------------------------------ @@ -153,30 +154,6 @@ const E409: Response = { status: 409, description: "The state does not allow it // --- spec validation ---------------------------------------------------------------- -const ajv = new Ajv2020({ allErrors: true, strict: false }); -const validateSpec = ajv.compile(SCHEMAS.JobSpecInput); -const paramValidators = new Map(); - -function paramsValidator(entry: CatalogEntry | null): ValidateFunction | null { - if (!entry?.params) return null; - if (!paramValidators.has(entry.name)) { - try { - paramValidators.set(entry.name, ajv.compile(entry.params)); - } catch { - paramValidators.set(entry.name, null); - } - } - return paramValidators.get(entry.name) ?? null; -} - -type SchemaError = { path: string; message: string }; - -const errorsOf = (v: ValidateFunction, prefix = ""): SchemaError[] => - (v.errors ?? []).slice(0, 20).map((e) => ({ - path: `${prefix}${e.instancePath}` || "/", - message: `${e.message ?? "invalid"}${e.params && "additionalProperty" in e.params ? `: ${String(e.params.additionalProperty)}` : ""}${e.params && "allowedValues" in e.params ? ` (${(e.params.allowedValues as unknown[]).join(", ")})` : ""}`, - })); - /** The spec as v1 accepts it: schema and job_id filled in, then checked strictly. */ function prepareSpec(req: FastifyRequest): { spec: JobSpec; entry: CatalogEntry | null } { const body = req.body; @@ -200,9 +177,10 @@ function prepareSpec(req: FastifyRequest): { spec: JobSpec; entry: CatalogEntry throw new ApiError(400, "invalid_spec", `params do not match the ${entry!.name} manifest`, `GET /api/v1/workloads/${entry!.name} has the params schema`, { schema_errors: errorsOf(pv, "/params") }); // A script runner reachable from a chat window is the one thing this API - // will not do for an ordinary token, however the spec is phrased. - if (entry?.dangerous && ctx.mode === "tokens" && !ctx.scopes.has("admin")) - throw new ApiError(403, "forbidden", `the ${entry.name} workload needs the admin scope`, "ask an operator to run it, or use a token with admin"); + // will not do for an ordinary token, however the spec is phrased. The same + // function decides it for the older enqueue routes. + const refused = needsAdmin(ctx, spec.workload); + if (refused) throw new ApiError(403, "forbidden", refused.message, refused.hint); return { spec: spec as JobSpec, entry }; } @@ -419,14 +397,8 @@ function waitFor(id: string, timeoutS: number, reply: FastifyReply) { } function owned(req: FastifyRequest, ids: string[]) { - const ctx = authOf(req); - if (ctx.mode !== "tokens" || ctx.internal || ctx.scopes.has("admin")) return; - const rows = db.prepare(`SELECT job_id, requested_by FROM jobs WHERE job_id IN (${ids.map(() => "?").join(",")})`).all(...ids) as { - job_id: string; requested_by: string | null; - }[]; - const foreign = rows.filter((r) => r.requested_by !== ctx.actor); - if (foreign.length) - throw new ApiError(403, "forbidden", `you can stop only jobs you started; ${foreign[0].job_id} was started by ${foreign[0].requested_by ?? "someone unrecorded"}`, "an admin token can act on any job"); + const refused = notOwned(authOf(req), ids); + if (refused) throw new ApiError(403, "forbidden", refused.message, refused.hint); } // --- results --------------------------------------------------------------------------- diff --git a/collector/src/api/v1/spec-check.ts b/collector/src/api/v1/spec-check.ts new file mode 100644 index 0000000..f272e01 --- /dev/null +++ b/collector/src/api/v1/spec-check.ts @@ -0,0 +1,52 @@ +/** + * The strict job-spec check, on its own so two things can share it. + * + * `/api/v1` refuses a spec that does not match `JobSpecInput` (unknown keys + * included) or whose `params` do not match the workload's manifest. The + * examples in `examples/jobs/` are what the docs show a reader, so + * `scripts/check-examples.ts` holds them to the same check, with the same + * compiled schemas, rather than to the looser `job.schema.json` it used to. + * + * No database, no Fastify: a script can import this without opening a + * collector's data directory. + */ +import { Ajv2020, type ValidateFunction } from "ajv/dist/2020.js"; +import { catalogEntry, type CatalogEntry } from "../../workloads/catalog.js"; +import { SCHEMAS } from "./schemas.js"; + +const ajv = new Ajv2020({ allErrors: true, strict: false }); +export const validateSpec = ajv.compile(SCHEMAS.JobSpecInput); +const paramValidators = new Map(); + +export function paramsValidator(entry: CatalogEntry | null): ValidateFunction | null { + if (!entry?.params) return null; + if (!paramValidators.has(entry.name)) { + try { + paramValidators.set(entry.name, ajv.compile(entry.params)); + } catch { + paramValidators.set(entry.name, null); + } + } + return paramValidators.get(entry.name) ?? null; +} + +export type SchemaError = { path: string; message: string }; + +export const errorsOf = (v: ValidateFunction, prefix = ""): SchemaError[] => + (v.errors ?? []).slice(0, 20).map((e) => ({ + path: `${prefix}${e.instancePath}` || "/", + message: `${e.message ?? "invalid"}${e.params && "additionalProperty" in e.params ? `: ${String(e.params.additionalProperty)}` : ""}${e.params && "allowedValues" in e.params ? ` (${(e.params.allowedValues as unknown[]).join(", ")})` : ""}`, + })); + +/** + * Every problem `/api/v1` would refuse this spec for at validation, by JSON + * pointer: the schema first, then the workload's params manifest. Empty when + * the spec would pass. (Whether anything can run it is a later question.) + */ +export function specProblems(spec: unknown): SchemaError[] { + if (!validateSpec(spec)) return errorsOf(validateSpec); + const entry = catalogEntry(String((spec as { workload?: unknown }).workload)); + const pv = paramsValidator(entry); + if (pv && !pv((spec as { params?: unknown }).params ?? {})) return errorsOf(pv, "/params"); + return []; +} diff --git a/collector/src/config.test.ts b/collector/src/config.test.ts new file mode 100644 index 0000000..84521dd --- /dev/null +++ b/collector/src/config.test.ts @@ -0,0 +1,41 @@ +/** + * The environment knobs that used to go through a bare `Number()`. + * + * Every case is a value somebody could plausibly type. The failure being + * guarded against is quiet: `FLEET_SWEEP_MS=15` (seconds, meant) was a sweep + * every 15 ms, and `15s` was NaN, which `setInterval` treats as 1 ms. + */ +import { intervalMs, intervalWarnings, MIN_INTERVAL_MS, parsePeerTokens, peerKey } from "./config.js"; + +type Check = (name: string, cond: boolean, detail?: string) => void; + +export function runConfigChecks(check: Check) { + const read = (v: string | undefined) => intervalMs({ X: v }, "X", 15_000); + + check("an unset interval is the default, with no warning", read(undefined).ms === 15_000 && read(undefined).warning === null); + check("an empty interval is the default, with no warning", read("").ms === 15_000 && read("").warning === null); + check("a sensible interval is used", read("60000").ms === 60_000 && read("60000").warning === null); + check("the floor itself is allowed", read(String(MIN_INTERVAL_MS)).ms === MIN_INTERVAL_MS); + for (const bad of ["15", "0", "-5000", "15s", "abc", "1500.5", "Infinity"]) { + const r = read(bad); + check(`${JSON.stringify(bad)} falls back to the default and says why`, r.ms === 15_000 && !!r.warning?.includes("X="), JSON.stringify(r)); + } + const w = intervalWarnings({ FLEET_SWEEP_MS: "15", FLEET_SCHEDULER_TICK_MS: "20000", FLEET_ALERT_TICK_MS: "1m" }); + check("startup warns about each bad interval and only those", + w.length === 2 && w.some((x) => x.startsWith("FLEET_SWEEP_MS=15")) && w.some((x) => x.startsWith("FLEET_ALERT_TICK_MS=1m")), JSON.stringify(w)); + check("no warnings for an environment that sets none", intervalWarnings({}).length === 0); + + // FLEET_PEER_TOKENS: one token per peer, keyed the way FLEET_PEERS spells it. + const ok = parsePeerTokens('{"http://fleet-host:8788/": "fr_abc", "http://work:8788": "fr_def"}'); + check("peer tokens are keyed by URL without a trailing slash", + ok.error === null && ok.tokens["http://fleet-host:8788"] === "fr_abc" && ok.tokens[peerKey("http://work:8788/")] === "fr_def", JSON.stringify(ok)); + check("no peer tokens is no error", parsePeerTokens(undefined).error === null && Object.keys(parsePeerTokens("").tokens).length === 0); + const notJson = parsePeerTokens("fr_abc"); + check("a bare token is refused, not sent to every peer", Object.keys(notJson.tokens).length === 0 && !!notJson.error, JSON.stringify(notJson)); + const arr = parsePeerTokens('["fr_abc"]'); + check("an array is refused", Object.keys(arr.tokens).length === 0 && !!arr.error); + const partial = parsePeerTokens('{"http://a:8788": "fr_a", "http://b:8788": 42, "http://c:8788": "has space"}'); + check("unusable entries are named and the rest kept", + partial.tokens["http://a:8788"] === "fr_a" && !partial.tokens["http://b:8788"] && !partial.tokens["http://c:8788"] && + !!partial.error?.includes("http://b:8788") && !!partial.error?.includes("http://c:8788"), JSON.stringify(partial)); +} diff --git a/collector/src/config.ts b/collector/src/config.ts index 99568b6..cebc2fa 100644 --- a/collector/src/config.ts +++ b/collector/src/config.ts @@ -33,7 +33,10 @@ export type Settings = { githubApi: string; logFile: string; dashDist: string; - /** Off by default; `fleet up` turns it on. See src/discovery.ts. */ + /** + * Off by default, and `fleet up` leaves it off unless the config says + * otherwise (`fleet config set collector.discovery true`). See src/discovery.ts. + */ discovery: boolean; /** * Other collectors this one knows about, for the dashboard's all-brains view. @@ -43,6 +46,14 @@ export type Settings = { * design rather than a limitation. */ peers: string[]; + /** + * FLEET_PEER_TOKENS: peer URL -> a bearer token that peer issued, sent when + * this collector reads it. Only a peer running FLEET_AUTH=tokens needs one; + * a `read` token is enough, because nothing proxied writes. + */ + peerTokens: Record; + /** Why FLEET_PEER_TOKENS was ignored, when it was. */ + peerTokensError: string | null; /** This brain's display name. Null means "read it from the data directory". */ name: string | null; /** @@ -102,6 +113,67 @@ export function parseFindingsRepos(raw: string | undefined): { repos: Record= MIN_INTERVAL_MS) return { ms: n, warning: null }; + return { + ms: fallback, + warning: `${name}=${raw} is not a whole number of milliseconds of at least ${MIN_INTERVAL_MS}; using ${fallback}`, + }; +} + +/** The timer intervals and their defaults, so the warnings and the values cannot disagree. */ +const INTERVALS = { FLEET_SWEEP_MS: 15_000, FLEET_SCHEDULER_TICK_MS: 20_000, FLEET_ALERT_TICK_MS: 60_000 } as const; + +/** Why an interval variable was ignored, for each one that was. Logged at startup. */ +export function intervalWarnings(env: NodeJS.ProcessEnv = process.env): string[] { + return Object.entries(INTERVALS).flatMap(([name, d]) => intervalMs(env, name, d).warning ?? []); +} + +/** FLEET_ALERT_TICK_MS, read the same way as the others. */ +export const alertTickMs = (env: NodeJS.ProcessEnv = process.env) => intervalMs(env, "FLEET_ALERT_TICK_MS", INTERVALS.FLEET_ALERT_TICK_MS).ms; + +/** A peer URL as the peer list and the token map both compare it. */ +export const peerKey = (url: string) => url.trim().replace(/\/+$/, ""); + +/** + * FLEET_PEER_TOKENS, read strictly: a JSON object of peer URL -> token. A + * malformed value maps nothing and says why, rather than sending one peer's + * token to another. + */ +export function parsePeerTokens(raw: string | undefined): { tokens: Record; error: string | null } { + if (!raw || !raw.trim()) return { tokens: {}, error: null }; + let parsed: unknown; + try { + parsed = JSON.parse(raw); + } catch (e) { + return { tokens: {}, error: `FLEET_PEER_TOKENS is not JSON: ${(e as Error).message}` }; + } + if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) + return { tokens: {}, error: 'FLEET_PEER_TOKENS must be a JSON object of "peer url" -> "token"' }; + const tokens: Record = {}; + const bad: string[] = []; + for (const [url, token] of Object.entries(parsed as Record)) { + if (typeof token === "string" && /^\S+$/.test(token)) tokens[peerKey(url)] = token; + else bad.push(url); + } + return { tokens, error: bad.length ? `FLEET_PEER_TOKENS: no usable token for ${bad.join(", ")}` : null }; +} + const list = (raw: string | undefined, fallback = "") => (raw ?? fallback) .split(",") @@ -144,8 +216,8 @@ export function fromEnv(env: NodeJS.ProcessEnv = process.env): Settings { dataDir: env.FLEET_DATA_DIR ?? path.resolve("data"), artifactDir: env.FLEET_ARTIFACT_DIR ?? path.resolve("artifacts/store"), powerConfigPath: env.FLEET_POWER_CONFIG ?? path.resolve("power.json"), - sweepMs: Number(env.FLEET_SWEEP_MS ?? 15_000), - schedulerTickMs: Number(env.FLEET_SCHEDULER_TICK_MS ?? 20_000), + sweepMs: intervalMs(env, "FLEET_SWEEP_MS", INTERVALS.FLEET_SWEEP_MS).ms, + schedulerTickMs: intervalMs(env, "FLEET_SCHEDULER_TICK_MS", INTERVALS.FLEET_SCHEDULER_TICK_MS).ms, // CI integration is BUILT BUT OFF. Statuses are recorded (posted=0) unless // both are set: FLEET_GITHUB_STATUS=1 arms posting, FLEET_GITHUB_TOKEN // authenticates it. @@ -163,6 +235,8 @@ export function fromEnv(env: NodeJS.ProcessEnv = process.env): Settings { // not announce itself because somebody upgraded. discovery: env.FLEET_DISCOVERY === "1", peers: list(env.FLEET_PEERS), + peerTokens: parsePeerTokens(env.FLEET_PEER_TOKENS).tokens, + peerTokensError: parsePeerTokens(env.FLEET_PEER_TOKENS).error, name: env.FLEET_NAME ?? null, // The host's own name rather than an address: it is what a phone on the // same network can usually resolve, and a link with a raw 192.168 address @@ -241,6 +315,8 @@ export let LOG_FILE: string; export let DASH_DIST: string; export let DISCOVERY: boolean; export let PEERS: string[]; +export let PEER_TOKENS: Record; +export let PEER_TOKENS_ERROR: string | null; export let NAME: string | null; export let DASH_URL: string; export let FINDINGS_REPOS: Record; @@ -268,6 +344,8 @@ function apply(): void { DASH_DIST = settings.dashDist; DISCOVERY = settings.discovery; PEERS = settings.peers; + PEER_TOKENS = settings.peerTokens; + PEER_TOKENS_ERROR = settings.peerTokensError; NAME = settings.name; DASH_URL = settings.dashUrl; FINDINGS_REPOS = settings.findingsRepos; diff --git a/collector/src/db.ts b/collector/src/db.ts index 89a3b24..bbb9e57 100644 --- a/collector/src/db.ts +++ b/collector/src/db.ts @@ -444,6 +444,11 @@ for (const [column, ddl] of [ // months to diff against. Once build and model-convert start producing // artifacts nightly, GC stops being hypothetical and that blind spot becomes // a deleted baseline and a visual suite with nothing to compare to. + // `pr` for a pull request's build, NULL for everything else. `latest` skips + // `pr` rows, so a branch's test build never becomes what the nightly tests. + // Rows from before the column are NULL: they count as merged builds, which + // is what `latest` already took them to be. + ["channel", "channel TEXT"], ["pinned", "pinned INTEGER NOT NULL DEFAULT 0"], ["pin_reason", "pin_reason TEXT"], ] as const) { diff --git a/collector/src/discovery.ts b/collector/src/discovery.ts index 8cfac31..4e8b97a 100644 --- a/collector/src/discovery.ts +++ b/collector/src/discovery.ts @@ -21,10 +21,12 @@ * * ## Why it is off by default * - * `FLEET_DISCOVERY=1`, and `fleet up` turns it on. A collector should not start - * announcing itself on somebody's office network because they upgraded, and the - * project's whole security posture is that the network is the access control -- - * which means what it announces itself to is a decision, not a default. + * `FLEET_DISCOVERY=1`, or `fleet config set collector.discovery true` for a + * brain started with `fleet up`, which otherwise leaves it off. A collector + * should not start announcing itself on somebody's office network because they + * upgraded, and the project's whole security posture is that the network is the + * access control -- which means what it announces itself to is a decision, not a + * default. * * ## What it does not change * diff --git a/collector/src/findings.test.ts b/collector/src/findings.test.ts index f42c30c..e450ed7 100644 --- a/collector/src/findings.test.ts +++ b/collector/src/findings.test.ts @@ -108,8 +108,9 @@ export async function runFindingsChecks(check: Check) { check("findings: ...including for a crash that sent one", !zeroCrash.ok && zeroCrash.status === 422); const unknown = validateFinding({ ...sample(), check: "vibes" }); check("findings: an unknown check name is 400", !unknown.ok && unknown.status === 400 && /vibes/.test(unknown.error)); - check("findings: the newer check names are accepted", - validateFinding(sample({ check: "dead_control" })).ok && validateFinding(sample({ check: "leash" })).ok); + check("findings: dead_control is accepted", validateFinding(sample({ check: "dead_control" })).ok); + check("findings: names nothing raises are refused (regression, agent_stuck, leash)", + ["regression", "agent_stuck", "leash"].every((c) => { const v = validateFinding(sample({ check: c as never })); return !v.ok && v.status === 400; })); const noFp = validateFinding({ ...sample(), fingerprint: "" }); check("findings: a missing fingerprint is 400", !noFp.ok && noFp.status === 400 && /fingerprint/.test(noFp.error)); const big = validateFinding(sample({ detail: "x".repeat(9000) })); diff --git a/collector/src/queue.ts b/collector/src/queue.ts index 18c2f67..5941e0e 100644 --- a/collector/src/queue.ts +++ b/collector/src/queue.ts @@ -200,6 +200,10 @@ export function workloadServiceableBy(workload: string, backend?: string | null) * six days guarding an APK older than the code it was meant to guard; a * schedule that asks for "latest" tests whatever CI pushed most recently. * + * Builds on the `pr` channel are never `latest`: a pull request's upload is + * tested by the job that names its hash, and the nightly keeps testing what + * was merged. + * * Throws when there is nothing to test, rather than enqueueing a job pinned to * a build that does not exist. */ @@ -209,11 +213,14 @@ export function resolveLatestBuild(spec: JobSpec): JobSpec { if (!appName) throw new Error("app.sha256 is 'latest' but app.name is missing"); const row = db .prepare( - `SELECT sha256, build FROM artifacts WHERE app = ? + `SELECT sha256, build FROM artifacts WHERE app = ? AND channel IS NOT 'pr' ORDER BY COALESCE(publish_seq, rowid) DESC LIMIT 1`, ) .get(appName) as { sha256: string; build: string | null } | undefined; - if (!row) throw new Error(`no build has been uploaded for ${appName}`); + if (!row) { + const prOnly = db.prepare("SELECT 1 FROM artifacts WHERE app = ? AND channel = 'pr' LIMIT 1").get(appName); + throw new Error(`no build has been uploaded for ${appName}${prOnly ? " except from pull requests, which latest never resolves to" : ""}`); + } // Record the build string the artifact carries, so a result says which build // actually ran. Falling back to the template's build would write the literal // word "latest" into the result, which is untraceable to any commit; the diff --git a/collector/src/server.ts b/collector/src/server.ts index 8831165..6833b7b 100644 --- a/collector/src/server.ts +++ b/collector/src/server.ts @@ -41,7 +41,8 @@ import { admit, isTailnetAddress, normaliseIp, whois } from "./tailnet.js"; import { firstArtifactSha, settleWaiters, type DepSettlement } from "./api/mutations.js"; import { announce } from "./announce.js"; import { enqueueJob, matchingDevices, resolveLatestBuild, type JobSpec } from "./queue.js"; -import { authWarnings, provenanceOf, registerAuth } from "./auth.js"; +import { authOf, authWarnings, provenanceOf, registerAuth } from "./auth.js"; +import { needsAdmin } from "./api/policy.js"; import { registerDeprecations } from "./api/v1/deprecations.js"; import { cleanMediaType, sniffMediaType } from "./media-type.js"; import { registerDashStatic } from "./dash-static.js"; @@ -49,7 +50,7 @@ import { registerRunnerWeb } from "./runner-web.js"; import { startPowerSampler } from "./power.js"; import { endMirror } from "./api/mirror.js"; import { digestTick, digestTickOptions } from "./findings.js"; -import { FINDINGS_REPOS_ERROR } from "./config.js"; +import { alertTickMs, FINDINGS_REPOS_ERROR, intervalWarnings, PEER_TOKENS_ERROR } from "./config.js"; /** @@ -589,6 +590,9 @@ app.get("/executor/next-job", async (req, reply) => { // --- jobs --- app.post("/jobs", async (req, reply) => { + // With tokens on, `shell` needs admin here exactly as it does on /api/v1. + const refused = needsAdmin(authOf(req), (req.body as { workload?: unknown } | null)?.workload); + if (refused) return reply.code(403).send({ error: refused.message, hint: refused.hint }); const out = enqueueJob(req.body as JobSpec, { provenance: provenanceOf(req) }); if (!out.ok) return reply.code(out.http).send({ error: out.message }); return reply.code(201).send( @@ -963,14 +967,24 @@ app.post("/schedules", async (req, reply) => { if (!b?.id || !b.cron || !b.template) return reply.code(400).send({ error: "id, cron, and template required" }); if (!isValidCron(b.cron)) return reply.code(400).send({ error: `invalid cron: ${b.cron}` }); + if (typeof b.template !== "object" || Array.isArray(b.template)) + return reply.code(400).send({ error: "template must be a job spec object" }); if ((b.template as { job_id?: string }).job_id) return reply.code(400).send({ error: "template must not carry job_id; the scheduler generates it" }); + // An empty or null job_id used to be stored and then replaced at every firing + // without a word. It is dropped instead, and the response says so. + const warnings: string[] = []; + const template = { ...(b.template as Record) }; + if ("job_id" in template) { + delete template.job_id; + warnings.push(`template.job_id was ignored: each run is named ${b.id}- (or ${b.id}-manual-