diff --git a/collector/dash/src/icons.tsx b/collector/dash/src/icons.tsx index 210ae65..359ebb8 100644 --- a/collector/dash/src/icons.tsx +++ b/collector/dash/src/icons.tsx @@ -53,6 +53,48 @@ const PATHS = { ), + convert: ( + <> + + + ), + dataset: ( + <> + + + + ), + serve: ( + <> + + + + + ), + terminal: ( + <> + + + + ), + bell: ( + <> + + + + ), + camera: ( + <> + + + + ), + desktop: ( + <> + + + + ), hammer: ( <> @@ -360,6 +402,13 @@ const WORKLOAD_ICON: Record = { "locale-shots": "translate", "app-soak": "soak", "a11y-audit": "accessibility", + "model-convert": "convert", + "dataset-prep": "dataset", + serve: "serve", + shell: "terminal", + "push-latency": "bell", + "camera-eval": "camera", + "desktop-ui-test": "desktop", }; /** The workload's name with its icon, for job tables. */ diff --git a/collector/dash/src/pages/JobDetail.tsx b/collector/dash/src/pages/JobDetail.tsx index 66aaf48..f690f78 100644 --- a/collector/dash/src/pages/JobDetail.tsx +++ b/collector/dash/src/pages/JobDetail.tsx @@ -1,3 +1,4 @@ +import { useEffect, useState } from "preact/hooks"; import { useApi, type JobDetail as Detail, type ResultRow } from "../api.js"; import { Workload } from "../icons.js"; import { mutate, useMutation } from "../mutate.js"; @@ -218,6 +219,8 @@ export function JobDetail({ id }: { id: string }) { + +
@@ -353,3 +356,48 @@ export function JobDetail({ id }: { id: string }) { ); } + +/** + * The device screen while a host job drives it. + * + * Polls only for whether a stream exists; the picture itself is an on an + * MJPEG endpoint, so the browser does the streaming and this component holds no + * frames. Rendered only while something is actually producing, because an + * that never paints looks exactly like a device with a black screen. + */ +function Mirror({ jobId, live }: { jobId: string; live: boolean }) { + const [streaming, setStreaming] = useState(false); + useEffect(() => { + if (!live) { + setStreaming(false); + return; + } + let alive = true; + const check = async () => { + try { + const r = await fetch(`/api/jobs/${encodeURIComponent(jobId)}/mirror/status`); + const b = (await r.json()) as { streaming?: boolean }; + if (alive) setStreaming(!!b.streaming); + } catch { + if (alive) setStreaming(false); + } + }; + void check(); + const t = setInterval(() => void check(), 5_000); + return () => { + alive = false; + clearInterval(t); + }; + }, [jobId, live]); + + if (!streaming) return null; + return ( + device screen, not recorded}> + the device screen while this job runs + + ); +} diff --git a/collector/schemas/metrics.json b/collector/schemas/metrics.json index 7bce0a6..e458396 100644 --- a/collector/schemas/metrics.json +++ b/collector/schemas/metrics.json @@ -1,6 +1,7 @@ { "schema": 1, "metrics": [ + "app_state", "artifact_bytes", "avg_position", "avg_rating", @@ -13,6 +14,7 @@ "clock_offset_ms", "clusters", "connect_ms", + "convert_s", "crashes", "ctr_pct", "decode_tok_s", @@ -25,10 +27,12 @@ "elapsed_s", "energy_method", "energy_wh", + "exit_code", "images_per_s", "impressions", "issues_error", "issues_warn", + "items", "jank_pct", "launch_ms", "launch_state", @@ -36,12 +40,15 @@ "locales", "mem_method", "network_type", + "outputs", "p50_ms", "p95_ms", "pages_crawled", "peak_mem_mb", "prefill_tok_s", "pss_mb", + "push_delivered_pct", + "push_latency_ms", "recall_at_1", "recall_at_10", "recall_at_5", diff --git a/collector/schemas/result.schema.json b/collector/schemas/result.schema.json index b038e34..5a490ed 100644 --- a/collector/schemas/result.schema.json +++ b/collector/schemas/result.schema.json @@ -23,6 +23,18 @@ "app_ver": { "type": "string" } } }, + "endpoint": { + "type": "string", + "description": "serve: where the model this job is hosting can be reached, announced so a dependent job can consume it. Bound to loopback or the tailnet, never 0.0.0.0 — an unauthenticated inference server on a laptop's hotel wifi is a different product to the one anybody asked for." + }, + "model_out": { + "type": "object", + "description": "model-convert: what one conversion produced — format, quant, name, sha256 and size. One row per output, so a job converting to three formats is three rows and the sha of each is addressable by a dependent job." + }, + "dataset": { + "type": "object", + "description": "dataset-prep: the prepared set's manifest summary, including its licence and source. A prepared set that loses its licence is one nobody can publish results from, which is why it rides on the row rather than only inside the zip." + }, "metrics": { "type": "object", "description": "Workload metrics; memory is labeled, never laundered.\n\nEvery field here means one thing. Before the vision-eval and drain fields existed, those workloads reused the LLM slots — vision put top-1 accuracy in decode_tok_s, p50 latency in ttft_ms and images/sec in prefill_tok_s; drain put percent-per-hour in decode_tok_s — and top-5 and p95 had nowhere to go at all, so they were only ever in the hand-written eval report. Rows predating these fields still carry that convention, and the dashboard marks anything it had to infer that way. New runners must populate the named fields instead.", @@ -87,6 +99,19 @@ "locales": { "type": "number", "description": "locale-shots: how many locales the run covered" }, "shots": { "type": "number", "description": "locale-shots and a11y-audit: screenshots captured" }, + + "convert_s": { "type": "number", "description": "model-convert: seconds for one output format" }, + "outputs": { "type": "number", "description": "model-convert: how many artifacts the conversion produced" }, + "items": { "type": "number", "description": "dataset-prep: items in the prepared manifest, after dedupe" }, + + "exit_code": { "type": "number", "description": "shell: the script's exit status. Named rather than folded into ok, because a script that exits 3 deliberately is not the same as one that could not start" }, + + "push_latency_ms": { "type": "number", "description": "push-latency: send to arrival, measured on the device that received it" }, + "push_delivered_pct": { "type": "number", "description": "push-latency: fraction of sends that arrived at all, which on Android is the number that varies by vendor" }, + "app_state": { + "enum": ["foreground", "background", "doze", "terminated"], + "description": "push-latency: what the app was doing when the push landed. A delivery latency without this is not comparable to anything, because doze is where the vendor forks differ" + }, "drain_pct_per_h": { "type": "number", "description": "drain: battery percent consumed per hour" }, "diff_pct": { "type": "number", "description": "web-shots: percent of pixels differing from the accepted baseline (100 when the dimensions changed)" }, diff --git a/collector/scripts/smoke.ts b/collector/scripts/smoke.ts index 24302a1..c0cdc74 100644 --- a/collector/scripts/smoke.ts +++ b/collector/scripts/smoke.ts @@ -2391,5 +2391,47 @@ runEvalChecks(check); // ever appear on real hardware. runDeviceParserChecks(check); +// --- live mirror --- + +// Nothing here is persisted, so the assertions are about the states a viewer +// can be in rather than about stored rows. The one that matters is the first: +// a job with no producer must 404 rather than open a stream that never paints, +// because an that never paints is indistinguishable from a device with a +// black screen. +{ + const MJOB = `smoke-mirror-${run}`; + const before = await json("GET", `/api/jobs/${MJOB}/mirror/status`); + check("no stream before a frame arrives", before.body?.streaming === false, JSON.stringify(before.body)); + + const viewer = await fetch(`${BASE}/api/jobs/${MJOB}/mirror`); + check("a viewer on a job with no producer gets 404", viewer.status === 404, `status=${viewer.status}`); + await viewer.body?.cancel().catch(() => {}); + + // The smallest thing that is still a JPEG. + const frame = Buffer.from( + "/9j/4AAQSkZJRgABAQEAYABgAAD/2wBDAAgGBgcGBQgHBwcJCQgKDBQNDAsLDBkSEw8UHRofHh0aHBwgJC4nICIsIxwcKDcpLDAxNDQ0Hyc5PTgyPC4zNDL/wAALCAABAAEBAREA/8QAFAABAAAAAAAAAAAAAAAAAAAACf/EABQQAQAAAAAAAAAAAAAAAAAAAAD/2gAIAQEAAD8AKp//2Q==", + "base64", + ); + const post = await fetch(`${BASE}/api/jobs/${MJOB}/mirror`, { + method: "POST", headers: { "content-type": "image/jpeg" }, body: frame, + }); + check("a frame is accepted", post.status === 204, `status=${post.status}`); + + const after = await json("GET", `/api/jobs/${MJOB}/mirror/status`); + check("the stream exists once a frame has arrived", after.body?.streaming === true, JSON.stringify(after.body)); + + // A frame past the cap is refused rather than truncated: half a JPEG renders + // as a corrupt image and reads as a broken device. + const huge = await fetch(`${BASE}/api/jobs/${MJOB}/mirror`, { + method: "POST", headers: { "content-type": "image/jpeg" }, body: Buffer.alloc(3 * 1024 * 1024), + }); + check("an oversized frame is refused", huge.status === 413, `status=${huge.status}`); + + const empty = await fetch(`${BASE}/api/jobs/${MJOB}/mirror`, { + method: "POST", headers: { "content-type": "image/jpeg" }, body: Buffer.alloc(0), + }); + check("an empty frame is refused", empty.status === 400, `status=${empty.status}`); +} + console.log(failures === 0 ? "\nsmoke: ALL PASS" : `\nsmoke: ${failures} FAILURE(S)`); process.exit(failures === 0 ? 0 : 1); diff --git a/collector/src/api/index.ts b/collector/src/api/index.ts index 4efe0c8..7b08494 100644 --- a/collector/src/api/index.ts +++ b/collector/src/api/index.ts @@ -9,6 +9,7 @@ // predict the shape import type { FastifyInstance } from "fastify"; import { registerDevices } from "./devices.js"; +import { registerMirror } from "./mirror.js"; import { registerEnroll } from "./enroll.js"; import { registerEvals } from "./evals.js"; import { registerJobs } from "./jobs.js"; @@ -33,6 +34,7 @@ type MatchingDevices = ( export function registerApi(app: FastifyInstance, announce: Announce, matchingDevices: MatchingDevices) { registerOverview(app); registerDevices(app); + registerMirror(app); registerEnroll(app); registerJobs(app); registerResults(app); diff --git a/collector/src/api/mirror.ts b/collector/src/api/mirror.ts new file mode 100644 index 0000000..a88904a --- /dev/null +++ b/collector/src/api/mirror.ts @@ -0,0 +1,169 @@ +// Watching a device while a host job drives it. +// +// The UI-test artifact viewer answers "what did it look like when it failed". +// This answers "what is it doing right now", which is a different question and +// the one you actually have while a flow is running. Reading a JUnit file to +// work out where a Maestro flow got stuck is slower than watching it get stuck. +// +// Three decisions shape the whole module. +// +// **Nothing is persisted.** Frames live in a bounded in-memory ring per job and +// are dropped when the job ends. The collector runs on a 2016 MacBook whose +// database already carries every measurement the fleet has ever taken; adding a +// video stream to it would trade a permanent cost for a transient benefit. If a +// run is worth keeping, the executor already uploads a recording as an artifact +// on failure — that is the durable path, and this is not it. +// +// **The producer pushes, the browser pulls.** The executor POSTs JPEG frames as +// it captures them; browsers read an MJPEG stream. MJPEG rather than SSE or a +// WebSocket because an `` tag renders it with no client code at all, it +// degrades to a still frame rather than an error when the producer stalls, and +// the dashboard is a single-operator tool where a dependency-free path is worth +// more than efficiency. +// +// **A viewer must never slow a run down.** Every write to a viewer socket is +// best-effort and a broken pipe only drops that viewer. A closed browser tab +// must not be able to make a device test fail, so nothing here propagates an +// error back to the producer. +import type { FastifyInstance, FastifyReply } from "fastify"; +import { requireToken } from "./guard.js"; + +// One frame at a time is all a viewer can see; a couple more absorb the jitter +// between a producer's capture loop and a viewer's render. Beyond that a slow +// viewer would only ever be shown stale frames, so the ring stays tiny and the +// newest frame always wins. +const RING = 3; +// A producer that stops sending has finished, crashed, or lost its device. +// After this, viewers are closed rather than left watching a frozen image that +// looks like a hung app rather than a hung stream. +const PRODUCER_IDLE_MS = 15_000; +// A cap so a runaway producer cannot exhaust memory: roughly a large phone +// screenshot. Anything bigger is refused rather than truncated, because half a +// JPEG renders as a corrupt image and reads as a broken device. +const MAX_FRAME_BYTES = 2 * 1024 * 1024; +const BOUNDARY = "fleetframe"; + +type Stream = { + frames: Buffer[]; + lastAt: number; + viewers: Set; + timer?: NodeJS.Timeout; +}; + +const streams = new Map(); + +function get(jobId: string): Stream { + let s = streams.get(jobId); + if (!s) { + s = { frames: [], lastAt: Date.now(), viewers: new Set() }; + streams.set(jobId, s); + } + return s; +} + +function writeFrame(reply: FastifyReply, frame: Buffer) { + try { + reply.raw.write( + `--${BOUNDARY}\r\nContent-Type: image/jpeg\r\nContent-Length: ${frame.length}\r\n\r\n`, + ); + reply.raw.write(frame); + reply.raw.write("\r\n"); + } catch { + // The viewer is gone. Its own close handler removes it; nothing propagates + // to the producer, which is mid-test and must not care. + } +} + +/** Close a stream out: end every viewer and forget the job. */ +function close(jobId: string) { + const s = streams.get(jobId); + if (!s) return; + if (s.timer) clearInterval(s.timer); + for (const v of s.viewers) { + try { + v.raw.end(); + } catch { + /* already gone */ + } + } + streams.delete(jobId); +} + +/** Called from the job-close path so a finished job stops streaming. */ +export function endMirror(jobId: string) { + close(jobId); +} + +export function registerMirror(app: FastifyInstance) { + // The producer. Raw JPEG bytes, one frame per request: the executor is + // shelling out to screencap or scrcpy and posting what comes back, and a + // frame-per-request needs no framing protocol on either side. + app.post("/api/jobs/:id/mirror", { bodyLimit: MAX_FRAME_BYTES }, async (req, reply) => { + if (!requireToken(req, reply)) return; + const { id } = req.params as { id: string }; + const body = req.body; + if (!Buffer.isBuffer(body) || body.length === 0) + return reply.code(400).send({ error: "expected raw JPEG bytes" }); + if (body.length > MAX_FRAME_BYTES) + return reply.code(413).send({ error: `frame larger than ${MAX_FRAME_BYTES} bytes` }); + + const s = get(id); + s.frames.push(body); + while (s.frames.length > RING) s.frames.shift(); + s.lastAt = Date.now(); + + for (const v of s.viewers) writeFrame(v, body); + + // Started on first frame rather than on stream creation, so a job that + // never produces one costs nothing. + if (!s.timer) { + s.timer = setInterval(() => { + const cur = streams.get(id); + if (cur && Date.now() - cur.lastAt > PRODUCER_IDLE_MS) close(id); + }, 5_000); + s.timer.unref(); + } + return reply.code(204).send(); + }); + + // The viewer. An renders this directly. + app.get("/api/jobs/:id/mirror", async (req, reply) => { + const { id } = req.params as { id: string }; + const s = streams.get(id); + // 404 rather than an empty stream: the dashboard uses this to decide + // whether to show the panel at all, and an that never paints is + // indistinguishable from a device with a black screen. + if (!s) return reply.code(404).send({ error: "nothing streaming for this job" }); + + reply.raw.writeHead(200, { + "content-type": `multipart/x-mixed-replace; boundary=${BOUNDARY}`, + "cache-control": "no-store", + connection: "close", + // The frames are already stale by the time they arrive; a proxy holding + // them makes the picture wrong rather than merely late. + pragma: "no-cache", + }); + + s.viewers.add(reply); + // Whatever the producer sent most recently, so a viewer joining mid-run + // paints immediately instead of waiting for the next capture. + const latest = s.frames[s.frames.length - 1]; + if (latest) writeFrame(reply, latest); + + const remove = () => s.viewers.delete(reply); + req.raw.on("close", remove); + req.raw.on("error", remove); + return reply; + }); + + // Whether a panel is worth showing, without opening a stream to find out. + app.get("/api/jobs/:id/mirror/status", async (req) => { + const { id } = req.params as { id: string }; + const s = streams.get(id); + return { + streaming: !!s, + viewers: s?.viewers.size ?? 0, + age_ms: s ? Date.now() - s.lastAt : null, + }; + }); +} diff --git a/collector/src/executor.ts b/collector/src/executor.ts index bd8b61d..91d41bd 100644 --- a/collector/src/executor.ts +++ b/collector/src/executor.ts @@ -40,10 +40,12 @@ import { parseXcuiDebugDescription, type A11yGeometry, type A11yNode, } from "./a11y-tree.js"; import { countBySeverity, uploadReport, type Finding } from "./web/shared.js"; +import { ADB, batteryPct, hasApp, launchApp, processAlive } from "./workloads/device.js"; +import { discoverWorkloads, loadRun, type LoadedWorkload } from "./workloads/registry.js"; +import type { Target, WorkloadCtx } from "./workloads/types.js"; const FLOWS_DIR = process.env.FLEET_FLOWS_DIR ?? path.resolve("flows"); const MAESTRO = process.env.MAESTRO_BIN ?? path.join(os.homedir(), ".maestro/bin/maestro"); -const ADB = process.env.ADB_BIN ?? "adb"; export type Job = { job_id: string; @@ -91,7 +93,10 @@ const WEB_SPECS_DIR = process.env.FLEET_WEB_SPECS_DIR ?? path.resolve("web-specs const IOS_PROJECT = process.env.FLEET_IOS_PROJECT ?? path.resolve("../runner-ios/FleetRunner.xcodeproj"); -type Target = { id: string; platform: "android" | "ios"; kind?: "device" | "simulator" }; +// One definition, in workloads/types.ts, so a handler that has moved out and a +// handler that has not are talking about the same thing. Re-exported because +// src/web/* and the workload directories import their types from here. +export type { Target }; // Exclusive jobs hold the collector's device locks while they run, so a // device-executor agent never gets handed work mid-UI-test. @@ -225,69 +230,6 @@ async function listTargets(ios?: IosDeviceInfo[]): Promise { return [...android, ...sims, ...phones.filter((t) => !seen.has(t.id))]; } -async function hasApp(target: Target, appId: string): Promise { - try { - if (target.platform === "android") { - await exec(ADB, ["-s", target.id, "shell", "pm", "path", appId], { timeout: 15_000 }); - } else if (target.kind === "device") { - const out = path.join(mkdtempSync(path.join(os.tmpdir(), "fleet-dc-")), "apps.json"); - await exec("xcrun", ["devicectl", "device", "info", "apps", "--device", target.id, "--json-output", out], { timeout: 30_000 }); - const parsed = JSON.parse(readFileSync(out, "utf8")) as { result?: { apps?: { bundleIdentifier: string }[] } }; - return (parsed.result?.apps ?? []).some((a) => a.bundleIdentifier === appId); - } else { - await exec("xcrun", ["simctl", "get_app_container", target.id, appId], { timeout: 15_000 }); - } - return true; - } catch { - return false; - } -} - -async function runInstall(job: Job) { - const app = job.app; - if (!app) throw new Error("install job needs an app ref"); - const platform = app.platform ?? "android"; - const targets = await selectTargets(job, (await listTargets()).filter((t) => t.platform === platform)); - if (targets.length === 0) throw new Error(`no ${platform} targets matched this job`); - - const dir = mkdtempSync(path.join(os.tmpdir(), "fleet-")); - let installable: string; - if (platform === "android") { - installable = path.join(dir, `${app.name}.apk`); - await fetchArtifact(app.sha256, installable); - } else { - // iOS artifacts are zips of the .app bundle (a directory can't be a raw artifact). - const zip = path.join(dir, `${app.name}.zip`); - await fetchArtifact(app.sha256, zip); - await exec("ditto", ["-x", "-k", zip, dir], { timeout: 120_000 }); - const appDir = readdirSync(dir).find((f) => f.endsWith(".app")); - if (!appDir) throw new Error("no .app bundle inside iOS artifact zip"); - installable = path.join(dir, appDir); - } - - let allOk = true; - for (const target of targets) { - let ok = true; - let error: string | undefined; - try { - if (platform === "android") { - await exec(ADB, ["-s", target.id, "install", "-r", installable], { timeout: 120_000 }); - } else if (target.kind === "device") { - await exec("xcrun", ["devicectl", "device", "install", "app", "--device", target.id, installable], { timeout: 300_000 }); - } else { - await exec("xcrun", ["simctl", "install", target.id, installable], { timeout: 120_000 }); - } - } catch (e) { - ok = false; - allOk = false; - error = (e as Error).message.slice(0, 300); - } - await postResult({ job_id: job.job_id, device_id: target.id, iter: 0, ok, error }); - log(`install ${app.name}@${app.build} on ${target.id} (${platform}): ${ok ? "ok" : "FAILED"}`); - } - await postResult({ job_id: job.job_id, device_id: `host:${NAME}`, iter: 0, final: true, ok: allOk }); -} - function parseJunit(xml: string): { passed: number; failed: number } { const m = /tests="(\d+)"[^>]*failures="(\d+)"/.exec(xml); if (!m) return { passed: 0, failed: 1 }; @@ -739,73 +681,6 @@ async function runUiTest(job: Job) { await postResult({ job_id: job.job_id, device_id: `host:${NAME}`, iter: 0, final: true, ok: allOk }); } -/** - * `args` are process launch arguments, and they exist for locale-shots: on iOS - * they land in the app's NSUserDefaults, which is how `-AppleLanguages (ar)` - * reaches an app without touching the device's own language at all. Android has - * no equivalent -- `am start` extras are not read as preferences -- so the - * Android path ignores them rather than pretending otherwise. - */ -async function launchApp(target: Target, appId: string, args: string[] = []) { - if (target.platform === "ios" && target.kind === "device") { - await exec("xcrun", ["devicectl", "device", "process", "launch", "--device", target.id, appId, ...args], { timeout: 60_000 }); - return; - } - if (target.platform === "android") { - // Resolve the real launcher activity; monkey is a fallback because some - // images (ATD) resolve but throttle monkey events. - try { - const { stdout } = await exec( - ADB, ["-s", target.id, "shell", "cmd", "package", "resolve-activity", "--brief", appId], - { timeout: 15_000 }, - ); - const component = stdout.trim().split("\n").pop()?.trim(); - if (!component || !component.includes("/")) throw new Error(`unresolvable: ${stdout}`); - await exec(ADB, ["-s", target.id, "shell", "am", "start", "-n", component], { timeout: 30_000 }); - } catch { - await exec(ADB, ["-s", target.id, "shell", "monkey", "-p", appId, "-c", - "android.intent.category.LAUNCHER", "1"], { timeout: 30_000 }); - } - } else { - await exec("xcrun", ["simctl", "launch", target.id, appId, ...args], { timeout: 60_000 }); - } -} - -async function processAlive(target: Target, appId: string): Promise { - try { - if (target.platform === "android") { - const { stdout } = await exec(ADB, ["-s", target.id, "shell", "pidof", appId], { timeout: 15_000 }); - return stdout.trim().length > 0; - } - const { stdout } = await exec("xcrun", ["simctl", "spawn", target.id, "launchctl", "list"], { timeout: 15_000 }); - return stdout.includes(appId); - } catch { - return false; - } -} - -async function batteryPct(target: Target): Promise { - try { - if (target.platform === "android") { - const { stdout } = await exec(ADB, ["-s", target.id, "shell", "dumpsys", "battery"], { timeout: 15_000 }); - const m = /level:\s*(\d+)/.exec(stdout); - return m ? Number(m[1]) : null; - } - if (target.kind === "device") { - // devicectl exposes battery via device info; the fleet runner's beacon - // is the primary source on real iPhones — this is the host-side cross-check. - const out = path.join(mkdtempSync(path.join(os.tmpdir(), "fleet-dc-")), "info.json"); - await exec("xcrun", ["devicectl", "device", "info", "details", "--device", target.id, "--json-output", out], { timeout: 30_000 }); - const txt = readFileSync(out, "utf8"); - const m = /"batteryLevel"\s*:\s*([0-9.]+)/.exec(txt); - return m ? Math.round(Number(m[1]) * (Number(m[1]) <= 1 ? 100 : 1)) : null; - } - return null; // simulators have no battery - } catch { - return null; - } -} - // Location replay: feed the device a recorded route so a drain run walks the // same path every night with the real GPS radio on. Simulators take a GPX // file directly; Android uses the mock-location provider (the app under test @@ -2967,8 +2842,94 @@ async function restoreDevicesOnStartup() { } } +// --------------------------------------------------------------------------- +// The workload loader +// --------------------------------------------------------------------------- + +/** + * The one ctx, built once from the real collector client. + * + * This object is the entire reason a moved handler is testable: it is the only + * route a workload has to the collector, the artifact store, the device list + * and the Keychain, so a test can hand the same handler a different one and + * watch what it posts. Nothing here is new behaviour — every field is the + * function the handlers were already calling as a module-level import. + */ +const CTX: WorkloadCtx = { + host: NAME, + log, + postResult, + postBeacon, + fetchArtifact, + uploadArtifact, + // Wrapped rather than passed by reference: listTargets takes an optional + // pre-fetched devicectl list, which only reportAttached has a reason to + // supply. A workload asking "what is attached" should not have to know that. + listTargets: () => listTargets(), + selectTargets, + leaseBudgetS, + secrets: { credentialsFor: resolveCredentials, redact }, +}; + +/** + * The workload directories, scanned once at startup. + * + * At startup rather than per job so that a stranger who mistypes a manifest + * hears about it when they restart the executor, instead of at 3am when the + * nightly that needed it fails. `log` is passed in so the complaints appear in + * this executor's log with everything else. + */ +const LOADED = discoverWorkloads(log); + +/** + * Send a job to whoever handles it. + * + * A lookup on the loaded directories first, then the if/else below for the + * handlers that still live in this file. The fallback is not a transitional + * embarrassment to be tidied away quickly — it is what lets a handler move on + * its own, verified on its own, while the other eleven keep running exactly as + * they did. When the chain is empty the chain goes; until then, a workload is + * dispatched by whichever of the two knows it, and a name in neither gets the + * same "not supported by this executor" row it always did. + */ +async function dispatch(job: Job, loaded: Map): Promise { + const w = loaded.get(job.workload); + if (w) { + const run = await loadRun(w); + await run(job, CTX); + return; + } + + if (job.workload === "ui-test") await runUiTest(job); + // Same implementation, and deliberately so: `soak` with no params.flow is + // exactly the old behaviour, so the existing nightlies keep meaning what + // they meant. + else if (job.workload === "soak" || job.workload === "app-soak") await runAppSoak(job); + else if (job.workload === "locale-shots") await runLocaleShots(job); + else if (job.workload === "a11y-audit") await runA11yAudit(job); + else if (job.workload === "drain") await runDrain(job); + else if (job.workload === "cold-start") await runColdStart(job); + else if (job.workload === "web-test") await runWebTest(job); + else if (job.workload === "web-shots") await runWebShots(job); + else if (job.workload === "web-audit") await runWebAudit(job); + else if (job.workload === "web-unfurl") await runWebUnfurl(job); + else if (job.workload === "archive") await runArchive(job); + else if (job.workload === "digest") await runDigest(job); + else { + await postResult({ + job_id: job.job_id, device_id: `host:${NAME}`, iter: 0, final: true, ok: false, + error: `workload '${job.workload}' not supported by this executor yet`, + }); + } +} + async function main() { log(`polling ${BASE} (flows: ${FLOWS_DIR})`); + log( + LOADED.size > 0 + ? `workloads loaded from src/workloads: ${[...LOADED.keys()].sort().join(", ")}` + : "no workload directories found; every job falls back to the built-in handlers", + ); await restoreDevicesOnStartup(); // The dashboard calls a device online for ONLINE_S seconds after it was last // seen. Refreshing on a timer rather than per poll keeps presence steady @@ -2990,28 +2951,7 @@ async function main() { log(`claimed ${job.job_id} (${job.workload})`); try { - if (job.workload === "install") await runInstall(job); - else if (job.workload === "ui-test") await runUiTest(job); - // Same implementation, and deliberately so: `soak` with no params.flow is - // exactly the old behaviour, so the existing nightlies keep meaning what - // they meant. - else if (job.workload === "soak" || job.workload === "app-soak") await runAppSoak(job); - else if (job.workload === "locale-shots") await runLocaleShots(job); - else if (job.workload === "a11y-audit") await runA11yAudit(job); - else if (job.workload === "drain") await runDrain(job); - else if (job.workload === "cold-start") await runColdStart(job); - else if (job.workload === "web-test") await runWebTest(job); - else if (job.workload === "web-shots") await runWebShots(job); - else if (job.workload === "web-audit") await runWebAudit(job); - else if (job.workload === "web-unfurl") await runWebUnfurl(job); - else if (job.workload === "archive") await runArchive(job); - else if (job.workload === "digest") await runDigest(job); - else { - await postResult({ - job_id: job.job_id, device_id: `host:${NAME}`, iter: 0, final: true, ok: false, - error: `workload '${job.workload}' not supported by this executor yet`, - }); - } + await dispatch(job, LOADED); } catch (e) { await postResult({ job_id: job.job_id, device_id: `host:${NAME}`, iter: 0, final: true, ok: false, diff --git a/collector/src/server.ts b/collector/src/server.ts index 2e9ec70..bc72739 100644 --- a/collector/src/server.ts +++ b/collector/src/server.ts @@ -36,6 +36,7 @@ import { } from "./api/mutations.js"; import { registerDashStatic } from "./dash-static.js"; import { startPowerSampler } from "./power.js"; +import { endMirror } from "./api/mirror.js"; mkdirSync(ARTIFACT_DIR, { recursive: true }); @@ -63,6 +64,9 @@ const LONG_LEASE_TTL_S = 4 * 60 * 60; // workloads chew through a corpus on a phone. const LONG_LEASE_WORKLOADS = new Set([ "drain", "soak", "thermal", "build", "app-soak", "speech-eval", "embed-eval", + // A conversion quantises a multi-GB checkpoint; a dataset is downloaded and + // resized; a served model is up for as long as someone is using it. + "model-convert", "dataset-prep", "serve", "push-latency", ]); const MAX_LEASE_TTL_S = 24 * 60 * 60; // Agents beacon every 60 s. Two missed beacons and a state claim is no longer @@ -76,6 +80,11 @@ const app = Fastify({ logger: { level: process.env.FLEET_LOG ?? "info" } }); // Artifact uploads arrive as raw bytes and are streamed to disk while the // hash is computed — a multi-GB GGUF never lives in collector memory. app.addContentTypeParser("application/octet-stream", (_req, payload, done) => done(null, payload)); +// Mirror frames, unlike artifacts, are small, capped, and consumed immediately +// rather than streamed to disk — so they are buffered here. Kept to its own +// content type precisely so the artifact path above keeps streaming: a +// multi-GB GGUF must never be buffered into collector memory. +app.addContentTypeParser("image/jpeg", { parseAs: "buffer" }, (_req, body, done) => done(null, body)); type JobSpec = { schema: number; @@ -149,6 +158,8 @@ const WORKLOADS = new Set([ // them; the rest are host work. All are listed because the collector renders // their results, and a workload the dashboard draws is one it knows about. "build", "speech-eval", "embed-eval", "vantage", "locale-shots", "app-soak", "a11y-audit", + // Wave 3: pipelines that feed the evals, and the workloads that need hardware. + "model-convert", "dataset-prep", "serve", "shell", "push-latency", "camera-eval", "desktop-ui-test", ]); function touchDevice(deviceId: string) { @@ -902,6 +913,9 @@ app.post("/results", async (req, reply) => { const status = b.ok === false ? "failed" : "done"; const settled = closeJobTx(b.job_id, status); + // A finished job has nothing left to show. Outside the transaction, because + // closing viewer sockets must not be able to roll back a device's result. + endMirror(b.job_id); announce({ type: "job", job_id: b.job_id, status, device_id: b.device_id }); // Announced after the transaction commits: a broken browser pipe must not // roll back a device's result. diff --git a/collector/src/workloads/device.ts b/collector/src/workloads/device.ts new file mode 100644 index 0000000..e743296 --- /dev/null +++ b/collector/src/workloads/device.ts @@ -0,0 +1,119 @@ +/** + * The device plumbing more than one workload needs. + * + * These used to be private functions in executor.ts, which is why they could + * only be used by handlers that also lived in executor.ts. They are here, and + * not in a workload's own directory, for exactly one reason: something other + * than that workload calls them. Anything a single workload uses belongs in + * that workload's directory instead, where its blast radius is one job type. + * + * executor.ts imports them back, so the handlers still waiting to move keep + * calling the same functions they always did. There is one definition of "is + * this app installed", and it answers the same on both sides of the move. + */ +import { mkdtempSync, readFileSync } from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { exec } from "../fleet-client.js"; +import type { Target } from "./types.js"; + +/** + * adb, or wherever this host keeps it. A LaunchAgent's PATH is not a login + * shell's, so the fleet's Android SDK hosts set ADB_BIN rather than hoping. + */ +export const ADB = process.env.ADB_BIN ?? "adb"; + +/** + * Is `appId` installed on this target? + * + * False on any failure, including "no adb on this host", because every caller + * uses it to decide whether to skip a device rather than whether to fail the + * job — and a skipped device with a stated reason is a better answer than a + * job that died asking. + */ +export async function hasApp(target: Target, appId: string): Promise { + try { + if (target.platform === "android") { + await exec(ADB, ["-s", target.id, "shell", "pm", "path", appId], { timeout: 15_000 }); + } else if (target.kind === "device") { + const out = path.join(mkdtempSync(path.join(os.tmpdir(), "fleet-dc-")), "apps.json"); + await exec("xcrun", ["devicectl", "device", "info", "apps", "--device", target.id, "--json-output", out], { timeout: 30_000 }); + const parsed = JSON.parse(readFileSync(out, "utf8")) as { result?: { apps?: { bundleIdentifier: string }[] } }; + return (parsed.result?.apps ?? []).some((a) => a.bundleIdentifier === appId); + } else { + await exec("xcrun", ["simctl", "get_app_container", target.id, appId], { timeout: 15_000 }); + } + return true; + } catch { + return false; + } +} + +/** + * Launch the app, three ways, because three platforms. + * + * `args` reach a simulator or a real iPhone as process arguments, which is how + * a launch argument like -AppleLanguages reaches an app without touching the + * device's own language at all. Android has no equivalent -- `am start` extras + * are not read as preferences -- so the Android path ignores them rather than + * pretending otherwise. + */ +export async function launchApp(target: Target, appId: string, args: string[] = []) { + if (target.platform === "ios" && target.kind === "device") { + await exec("xcrun", ["devicectl", "device", "process", "launch", "--device", target.id, appId, ...args], { timeout: 60_000 }); + return; + } + if (target.platform === "android") { + // Resolve the real launcher activity; monkey is a fallback because some + // images (ATD) resolve but throttle monkey events. + try { + const { stdout } = await exec( + ADB, ["-s", target.id, "shell", "cmd", "package", "resolve-activity", "--brief", appId], + { timeout: 15_000 }, + ); + const component = stdout.trim().split("\n").pop()?.trim(); + if (!component || !component.includes("/")) throw new Error(`unresolvable: ${stdout}`); + await exec(ADB, ["-s", target.id, "shell", "am", "start", "-n", component], { timeout: 30_000 }); + } catch { + await exec(ADB, ["-s", target.id, "shell", "monkey", "-p", appId, "-c", + "android.intent.category.LAUNCHER", "1"], { timeout: 30_000 }); + } + } else { + await exec("xcrun", ["simctl", "launch", target.id, appId, ...args], { timeout: 60_000 }); + } +} + +export async function processAlive(target: Target, appId: string): Promise { + try { + if (target.platform === "android") { + const { stdout } = await exec(ADB, ["-s", target.id, "shell", "pidof", appId], { timeout: 15_000 }); + return stdout.trim().length > 0; + } + const { stdout } = await exec("xcrun", ["simctl", "spawn", target.id, "launchctl", "list"], { timeout: 15_000 }); + return stdout.includes(appId); + } catch { + return false; + } +} + +export async function batteryPct(target: Target): Promise { + try { + if (target.platform === "android") { + const { stdout } = await exec(ADB, ["-s", target.id, "shell", "dumpsys", "battery"], { timeout: 15_000 }); + const m = /level:\s*(\d+)/.exec(stdout); + return m ? Number(m[1]) : null; + } + if (target.kind === "device") { + // devicectl exposes battery via device info; the fleet runner's beacon + // is the primary source on real iPhones — this is the host-side cross-check. + const out = path.join(mkdtempSync(path.join(os.tmpdir(), "fleet-dc-")), "info.json"); + await exec("xcrun", ["devicectl", "device", "info", "details", "--device", target.id, "--json-output", out], { timeout: 30_000 }); + const txt = readFileSync(out, "utf8"); + const m = /"batteryLevel"\s*:\s*([0-9.]+)/.exec(txt); + return m ? Math.round(Number(m[1]) * (Number(m[1]) <= 1 ? 100 : 1)) : null; + } + return null; // simulators have no battery + } catch { + return null; + } +} diff --git a/collector/src/workloads/install/index.ts b/collector/src/workloads/install/index.ts new file mode 100644 index 0000000..ca639b6 --- /dev/null +++ b/collector/src/workloads/install/index.ts @@ -0,0 +1,61 @@ +// install: put a build on every device this job selected. +// +// The simplest workload in the fleet, and the one everything else depends on — +// a ui-test, a drain and a cold-start all assume somebody installed the build +// first. It fetches the artifact once, on the host, and pushes the same file to +// each target rather than downloading per device. +// +// One result row per device, failures included, then a final row that is ok +// only if every install was. A device that refused the install is a result, not +// an exception: the other nine phones installed it and the job should say so. +import { mkdtempSync, readdirSync } from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { exec } from "../../fleet-client.js"; +import { ADB } from "../device.js"; +import type { Job, WorkloadCtx } from "../types.js"; + +export async function run(job: Job, ctx: WorkloadCtx): Promise { + const app = job.app; + if (!app) throw new Error("install job needs an app ref"); + const platform = app.platform ?? "android"; + const targets = await ctx.selectTargets(job, (await ctx.listTargets()).filter((t) => t.platform === platform)); + if (targets.length === 0) throw new Error(`no ${platform} targets matched this job`); + + const dir = mkdtempSync(path.join(os.tmpdir(), "fleet-")); + let installable: string; + if (platform === "android") { + installable = path.join(dir, `${app.name}.apk`); + await ctx.fetchArtifact(app.sha256, installable); + } else { + // iOS artifacts are zips of the .app bundle (a directory can't be a raw artifact). + const zip = path.join(dir, `${app.name}.zip`); + await ctx.fetchArtifact(app.sha256, zip); + await exec("ditto", ["-x", "-k", zip, dir], { timeout: 120_000 }); + const appDir = readdirSync(dir).find((f) => f.endsWith(".app")); + if (!appDir) throw new Error("no .app bundle inside iOS artifact zip"); + installable = path.join(dir, appDir); + } + + let allOk = true; + for (const target of targets) { + let ok = true; + let error: string | undefined; + try { + if (platform === "android") { + await exec(ADB, ["-s", target.id, "install", "-r", installable], { timeout: 120_000 }); + } else if (target.kind === "device") { + await exec("xcrun", ["devicectl", "device", "install", "app", "--device", target.id, installable], { timeout: 300_000 }); + } else { + await exec("xcrun", ["simctl", "install", target.id, installable], { timeout: 120_000 }); + } + } catch (e) { + ok = false; + allOk = false; + error = (e as Error).message.slice(0, 300); + } + await ctx.postResult({ job_id: job.job_id, device_id: target.id, iter: 0, ok, error }); + ctx.log(`install ${app.name}@${app.build} on ${target.id} (${platform}): ${ok ? "ok" : "FAILED"}`); + } + await ctx.postResult({ job_id: job.job_id, device_id: `host:${ctx.host}`, iter: 0, final: true, ok: allOk }); +} diff --git a/collector/src/workloads/install/manifest.json b/collector/src/workloads/install/manifest.json new file mode 100644 index 0000000..fb80632 --- /dev/null +++ b/collector/src/workloads/install/manifest.json @@ -0,0 +1,15 @@ +{ + "name": "install", + "capability": "devices", + "executor": "host", + "description": "Install a build on every attached device and verify it launches.", + "params": { + "type": "object", + "additionalProperties": false, + "properties": {}, + "description": "install takes no params. What it installs comes from `app` (name, build, sha256, platform) and where it lands comes from `targets`." + }, + "example": { + "app": { "name": "", "build": "", "sha256": "" } + } +} diff --git a/collector/src/workloads/registry.ts b/collector/src/workloads/registry.ts new file mode 100644 index 0000000..3e2ba0b --- /dev/null +++ b/collector/src/workloads/registry.ts @@ -0,0 +1,120 @@ +/** + * The loader. Finds workloads by looking, not by being told. + * + * A directory next to this file with a `manifest.json` and an `index.ts` in it + * IS a workload — there is no list to add a name to, which is the whole point: + * the register-it-here line is exactly the line that put every workload author + * into a merge conflict with every other workload author. + * + * Two rules the scan follows, both of them about not making a bad day worse: + * + * 1. A broken workload is skipped, loudly, and the executor keeps running. + * A manifest with a typo in it must not stop a host from claiming the + * twelve jobs that have nothing to do with it. The complaint goes to the + * log, and the job that actually wanted that workload fails with the reason + * on its own result row. + * + * 2. The scan happens once, at startup, but the module is imported lazily on + * first use. Knowing what exists is cheap and worth doing before any job + * arrives (so a stranger's typo is visible immediately rather than at 3am + * when the nightly hits it); loading the code is not, since a host may go + * weeks without ever being handed a web-test. + */ +import { readdirSync, readFileSync } from "node:fs"; +import path from "node:path"; +import { fileURLToPath, pathToFileURL } from "node:url"; +import type { WorkloadManifest, WorkloadRun } from "./types.js"; + +// Resolved from this module, not from cwd: the executor is started by a +// LaunchAgent whose working directory is not this repo, and `path.resolve` on a +// relative name would find nothing there. fileURLToPath rather than stripping +// "file://" by hand — this repo lives under a path with a space in it, and the +// URL form percent-encodes it. +const WORKLOADS_DIR = path.dirname(fileURLToPath(import.meta.url)); + +const CAPABILITIES = new Set(["devices", "browsers", "network"]); + +export type LoadedWorkload = { + manifest: WorkloadManifest; + /** Absolute path to the module, for the dynamic import and for error messages. */ + entry: string; +}; + +/** Why a directory that looked like a workload is not one, or null if it is. */ +function manifestProblem(name: string, m: unknown): string | null { + if (typeof m !== "object" || m === null) return "manifest.json is not an object"; + const o = m as Record; + // The name is duplicated between the directory and the manifest on purpose: + // the directory is what a person sees and the manifest is what is served to + // the dashboard, and a workload answering to two names is the kind of bug + // that is only ever found by someone whose job did not run. + if (o.name !== name) return `manifest.json says name ${JSON.stringify(o.name)}, but the directory is ${JSON.stringify(name)}`; + if (typeof o.capability !== "string" || !CAPABILITIES.has(o.capability)) { + return `capability must be one of ${[...CAPABILITIES].join(", ")}, not ${JSON.stringify(o.capability)}`; + } + if (typeof o.description !== "string" || o.description.length === 0) return "description is missing"; + if (o.executor !== "host" && o.executor !== "device") return "executor must be \"host\" or \"device\""; + if (typeof o.params !== "object" || o.params === null) return "params must be a JSON Schema object"; + if (typeof o.example !== "object" || o.example === null) return "example must be a spec object"; + return null; +} + +/** + * Every workload directory that parses, keyed by name. + * + * `onProblem` rather than a throw or a bare console.log: the executor has its + * own prefixed logger and this module should not decide what the complaint + * looks like, and a test wants to assert on the complaints rather than read + * them. + */ +export function discoverWorkloads( + onProblem: (msg: string) => void = () => {}, + dir: string = WORKLOADS_DIR, +): Map { + const found = new Map(); + let entries; + try { + entries = readdirSync(dir, { withFileTypes: true }); + } catch (e) { + onProblem(`cannot read the workloads directory (${(e as Error).message}); no workloads are loadable`); + return found; + } + for (const e of entries) { + // Files next to the directories — types.ts, this file, the tests — are the + // shared plumbing, not workloads. + if (!e.isDirectory()) continue; + const manifestPath = path.join(dir, e.name, "manifest.json"); + let manifest: WorkloadManifest; + try { + manifest = JSON.parse(readFileSync(manifestPath, "utf8")) as WorkloadManifest; + } catch (err) { + onProblem(`workload "${e.name}" has no readable manifest.json (${(err as Error).message}); skipping it`); + continue; + } + const problem = manifestProblem(e.name, manifest); + if (problem) { + onProblem(`workload "${e.name}" has an invalid manifest.json: ${problem}; skipping it`); + continue; + } + found.set(manifest.name, { manifest, entry: path.join(dir, e.name, "index.js") }); + } + return found; +} + +/** + * The `run` a workload exports, imported on first use. + * + * The `.js` in the entry path is deliberate and is what NodeNext wants: under + * tsx it resolves to the neighbouring index.ts, and after a build it resolves + * to the emitted index.js. pathToFileURL rather than importing the raw path, + * again because of the space in this repo's path — a bare absolute path in a + * dynamic import is interpreted as a URL on some platforms and silently fails + * to resolve. + */ +export async function loadRun(w: LoadedWorkload): Promise { + const mod = (await import(pathToFileURL(w.entry).href)) as { run?: unknown }; + if (typeof mod.run !== "function") { + throw new Error(`workload "${w.manifest.name}" does not export a run function from ${w.entry}`); + } + return mod.run as WorkloadRun; +} diff --git a/collector/src/workloads/types.ts b/collector/src/workloads/types.ts new file mode 100644 index 0000000..8fd2822 --- /dev/null +++ b/collector/src/workloads/types.ts @@ -0,0 +1,149 @@ +/** + * What a workload is, and what it is allowed to reach for. + * + * The executor used to be one file with every handler in it and a chain of + * `else if` at the bottom. That shape had two costs. The module has a `main()` + * loop, so importing it starts polling the collector — which meant no handler + * in it could ever be unit-tested, and the only way to exercise one was to plug + * in a phone. And every new workload was another edit to the file everybody + * else was also editing, so adding one meant a merge conflict with whoever was + * fixing an unrelated handler that week. + * + * A workload is now a directory: a manifest saying what it is and what it + * needs, and an `index.ts` exporting `run`. Nothing registers it; the loader + * finds it because the directory is there. + * + * The `ctx` below is the other half of that. Handlers used to import + * `postResult`, `uploadArtifact` and the rest straight from `fleet-client.js`, + * which made every dependency ambient: you could not tell from a handler's + * signature whether it talked to the collector, moved artifacts, or read the + * Keychain, and a test had no seam to substitute any of it. Passing them in + * makes the dependency list part of the type, and lets a test hand a handler a + * fake collector and watch what it posts. + * + * What ctx deliberately does NOT carry is `exec`. Shelling out to adb, simctl, + * devicectl, xcodebuild and npx is the job, not a dependency worth abstracting: + * a fake `exec` would be a re-implementation of four toolchains, and a test + * written against it would pass while the real thing was broken. So a handler + * is testable exactly up to its first shell-out — which, in practice, covers + * every refusal, precondition and result-shaping path, and those are the ones + * that have actually been wrong. + */ +import type { Job } from "../executor.js"; + +// Type-only, and it has to stay that way: executor.ts calls main() at the +// bottom, so a runtime import of it from here would start a second poll loop +// inside whatever imported the workload. `import type` is erased, which is why +// src/web/* has always imported Job the same way. +export type { Job }; + +/** One thing this executor can drive. Mirrors executor.ts's own Target. */ +export type Target = { + id: string; + platform: "android" | "ios"; + kind?: "device" | "simulator"; +}; + +/** What a suite needs to sign in. `password` is resolved on the host, never carried in a spec. */ +export type SuiteCredentials = { + account: string; + password: string; + emailVar: string; + passwordVar: string; +}; + +/** + * The secrets seam. + * + * Narrow on purpose. A workload can ask for a suite's sign-in details and can + * scrub a string before it becomes an artifact; it cannot enumerate the + * Keychain. `credentialsFor` throws with a remedy rather than returning null on + * failure — null means "this suite did not ask to sign in", which is a + * different thing from "it asked and we could not". + */ +export type WorkloadSecrets = { + credentialsFor( + suite: NonNullable, + who: string, + ): Promise; + redact(text: string, secrets: string[]): string; +}; + +/** + * Everything a handler is given. If it is not here, a handler either does not + * need it or is shelling out for it. + */ +export type WorkloadCtx = { + /** + * This executor's name. Every workload closes with a final row on + * `host:`, because a result row needs a device_id and the host is what + * actually ran when the answer is about the job rather than a device. + */ + host: string; + /** Prefixed, line-per-event logging. The executor's log is read as a narrative. */ + log(msg: string): void; + /** Post one result row. `final: true` closes the job. */ + postResult(row: Record): Promise; + /** Post a beacon. Renews the lease as a side effect, which is often the point. */ + postBeacon(jobId: string, deviceId: string, extra: Record): Promise; + /** Fetch an artifact by hash to `dest`, verifying the hash. */ + fetchArtifact(sha256: string, dest: string): Promise; + /** Upload a file to the artifact store under `name`; returns its sha256. */ + uploadArtifact(file: string, name: string): Promise; + /** Everything attached to this host that the fleet owns. */ + listTargets(): Promise; + /** Narrow those to the ones this job asked for (device_id, kind, match). */ + selectTargets(job: Job, all: Target[]): Promise; + secrets: WorkloadSecrets; + /** + * The per-unit time budget the lease allows, in seconds. A unit of work that + * does not beacon must finish inside this, or the sweep requeues a job that + * is still running and a second executor runs it concurrently. + */ + leaseBudgetS(job: Job, fallbackS?: number): number; +}; + +/** + * The signature every `index.ts` exports. + * + * A handler reports by posting rows, not by returning: a job with ten devices + * has eleven answers and no single return value describes them. It throws only + * when the job cannot start at all — a missing app ref, a spec that names + * nothing — and the executor turns that into the failed final row. + */ +export type WorkloadRun = (job: Job, ctx: WorkloadCtx) => Promise; + +/** + * What a host must have before this workload can honestly run on it. + * + * Three, because there are three real answers on these machines: + * + * devices -- attached hardware or simulators, driven over adb / simctl / + * devicectl. Every workload whose result is about a phone. + * browsers -- a Playwright install. Present on one machine in the fleet, so + * a job needing it and landing elsewhere must say so rather than + * download a browser onto a 2016 laptop mid-nightly. + * network -- a route to the site or API under test and nothing else. These + * run anywhere. + * + * It is advisory today: the loader records it and the dispatcher does not yet + * refuse on it, because the workloads that gate (web-test on FLEET_WEB) still + * do their own gating and this refactor changes no behaviour. It is stated in + * the manifest so the check has one place to move to. + */ +export type WorkloadCapability = "devices" | "browsers" | "network"; + +/** manifest.json, as it is on disk. */ +export type WorkloadManifest = { + /** The `workload` string in a job spec. Must equal the directory name. */ + name: string; + capability: WorkloadCapability; + /** One sentence, in the composer's voice: what this does and what you get. */ + description: string; + /** Which executor claims it. Host workloads run here; device ones run on the phone's agent. */ + executor: "host" | "device"; + /** JSON Schema for `params`. Documentation and, one day, validation. */ + params: Record; + /** A starting-point spec for the composer. Edited by a person, not validated. */ + example: Record; +}; diff --git a/runner-ios/FleetRunner.xcodeproj/project.pbxproj b/runner-ios/FleetRunner.xcodeproj/project.pbxproj index 5cbc825..ac103c3 100644 --- a/runner-ios/FleetRunner.xcodeproj/project.pbxproj +++ b/runner-ios/FleetRunner.xcodeproj/project.pbxproj @@ -9,12 +9,14 @@ /* Begin PBXBuildFile section */ 1094A2006ACBE463AD53F5C6 /* Percentile.swift in Sources */ = {isa = PBXBuildFile; fileRef = 67ED6D6D92FC1E39451CC4B1 /* Percentile.swift */; }; 18549C47E64362A3999CBF54 /* LlamaCppBackend.swift in Sources */ = {isa = PBXBuildFile; fileRef = BEC2D15CC8AA61AF38526ECA /* LlamaCppBackend.swift */; }; + 188C71F066363470DCF89341 /* WebShotsWorkload.swift in Sources */ = {isa = PBXBuildFile; fileRef = 87DC249597590EC4A8FDA5ED /* WebShotsWorkload.swift */; }; 29EE70D2DEBA1D834F56F353 /* SyntheticBackend.swift in Sources */ = {isa = PBXBuildFile; fileRef = D84C3EEBAE6C004B45C56596 /* SyntheticBackend.swift */; }; 2EA020A1111355C03CB7D665 /* CoreMLBackend.swift in Sources */ = {isa = PBXBuildFile; fileRef = D322211CB368EAB10C9D22A9 /* CoreMLBackend.swift */; }; 38E30C23455915A872514955 /* Workloads.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4A0BF52256E0CBF3AC96D589 /* Workloads.swift */; }; 6424FDD0EDCF7806C66447F0 /* FleetSmokeUITests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 09213AE64323164BBF07F61A /* FleetSmokeUITests.swift */; }; 8591C734F0CC29AECDA37140 /* BenchSession.swift in Sources */ = {isa = PBXBuildFile; fileRef = C7DFBDD257E5AD4960202BB7 /* BenchSession.swift */; }; 8C5395E6E4FADC66F7314D0F /* Telemetry.swift in Sources */ = {isa = PBXBuildFile; fileRef = 35252602851DD4FC6838C5CC /* Telemetry.swift */; }; + 8CEFD6BE130409469707F559 /* WebShotsManifest.swift in Sources */ = {isa = PBXBuildFile; fileRef = 87DDA25661F8D19F7570C9BE /* WebShotsManifest.swift */; }; A98911FE185A5563E58375CA /* Protocol.swift in Sources */ = {isa = PBXBuildFile; fileRef = 6BB3BA830E838835C4AE2AD1 /* Protocol.swift */; }; B0BC3C9EA4C05413868B3AC0 /* ThermalWorkload.swift in Sources */ = {isa = PBXBuildFile; fileRef = C8C6B181BF315A65223BCFC1 /* ThermalWorkload.swift */; }; B5E9A656019E3738212E40E7 /* CollectorClient.swift in Sources */ = {isa = PBXBuildFile; fileRef = 31632F0B6302F50DC2BD764E /* CollectorClient.swift */; }; @@ -48,6 +50,8 @@ 50ADC470CF907EE3A603AA44 /* VantageWorkload.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = VantageWorkload.swift; sourceTree = ""; }; 67ED6D6D92FC1E39451CC4B1 /* Percentile.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Percentile.swift; sourceTree = ""; }; 6BB3BA830E838835C4AE2AD1 /* Protocol.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Protocol.swift; sourceTree = ""; }; + 87DC249597590EC4A8FDA5ED /* WebShotsWorkload.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = WebShotsWorkload.swift; sourceTree = ""; }; + 87DDA25661F8D19F7570C9BE /* WebShotsManifest.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = WebShotsManifest.swift; sourceTree = ""; }; 965A118D55FCDC99E6F0349A /* FleetAgent.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = FleetAgent.swift; sourceTree = ""; }; A4C158F9244D20CE6B927532 /* FleetRunnerApp.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = FleetRunnerApp.swift; sourceTree = ""; }; BEC2D15CC8AA61AF38526ECA /* LlamaCppBackend.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = LlamaCppBackend.swift; sourceTree = ""; }; @@ -82,6 +86,8 @@ 086C0F942E2B16C41C3BE0B2 /* Theme.swift */, C8C6B181BF315A65223BCFC1 /* ThermalWorkload.swift */, 50ADC470CF907EE3A603AA44 /* VantageWorkload.swift */, + 87DDA25661F8D19F7570C9BE /* WebShotsManifest.swift */, + 87DC249597590EC4A8FDA5ED /* WebShotsWorkload.swift */, 4A0BF52256E0CBF3AC96D589 /* Workloads.swift */, ); path = FleetRunner; @@ -214,6 +220,8 @@ C0D50E9B5C78CC2307571AAF /* Theme.swift in Sources */, B0BC3C9EA4C05413868B3AC0 /* ThermalWorkload.swift in Sources */, D704AFBF0BF1559D31AE6C9B /* VantageWorkload.swift in Sources */, + 8CEFD6BE130409469707F559 /* WebShotsManifest.swift in Sources */, + 188C71F066363470DCF89341 /* WebShotsWorkload.swift in Sources */, 38E30C23455915A872514955 /* Workloads.swift in Sources */, ); runOnlyForDeploymentPostprocessing = 0; diff --git a/runner-ios/FleetRunner/CollectorClient.swift b/runner-ios/FleetRunner/CollectorClient.swift index af721de..6300626 100644 --- a/runner-ios/FleetRunner/CollectorClient.swift +++ b/runner-ios/FleetRunner/CollectorClient.swift @@ -78,6 +78,25 @@ final class CollectorClient: Sendable { return try JSONDecoder().decode(R.self, from: body).sha256 } + /// The accepted visual baselines for one suite, keyed "page|profile". + /// + /// Fetched once per web-shots job rather than once per page, so every page + /// in one run is judged against the same accepted set even if someone + /// clicks accept from the dashboard halfway through — the same rule the + /// host executor follows, for the same reason. + func visualBaselines(suite: String) async throws -> [String: String] { + let url = base.appendingPathComponent("api/visual/baselines") + .appending(queryItems: [URLQueryItem(name: "suite", value: suite)]) + let (data, res) = try await session.data(from: url) + let code = (res as? HTTPURLResponse)?.statusCode ?? 0 + guard code == 200 else { throw CollectorError.http(code, "api/visual/baselines") } + struct Row: Decodable { let page: String; let profile: String; let sha256: String } + struct Body: Decodable { let baselines: [Row] } + let body = try FleetJSON.decoder.decode(Body.self, from: data) + return Dictionary(body.baselines.map { ("\($0.page)|\($0.profile)", $0.sha256) }, + uniquingKeysWith: { _, last in last }) + } + func publishEvent(topic: String, payload: [String: Any]) async throws { var req = URLRequest(url: base.appendingPathComponent("events/\(topic)")) req.httpMethod = "POST" diff --git a/runner-ios/FleetRunner/FleetAgent.swift b/runner-ios/FleetRunner/FleetAgent.swift index afc4f09..93f07b3 100644 --- a/runner-ios/FleetRunner/FleetAgent.swift +++ b/runner-ios/FleetRunner/FleetAgent.swift @@ -198,6 +198,15 @@ final class FleetAgent: ObservableObject { Route(capability: "vantage", workload: "vantage") { _, job, client, deviceId, _ in await Workloads.runVantage(job: job, client: client, deviceId: deviceId) }, + // Backend-qualified, like batch:coreml, and for the same reason: most + // web-shots jobs are the host executor's Playwright matrix, which this + // runner cannot run. Declaring a bare "web-shots" would have the queue + // hand us those and get them bounced straight back. `webkit` is the + // backend because a WKWebView is what captures — see WebShots.profile + // for why that word is not "safari". + Route(capability: "web-shots:webkit", workload: "web-shots", backend: "webkit") { _, job, client, deviceId, cache in + await Workloads.runWebShots(job: job, client: client, deviceId: deviceId, artifacts: cache) + }, ] /// What this device declares at registration, so the collector's routing diff --git a/runner-ios/FleetRunner/Protocol.swift b/runner-ios/FleetRunner/Protocol.swift index 3b7e3b9..a19b64b 100644 --- a/runner-ios/FleetRunner/Protocol.swift +++ b/runner-ios/FleetRunner/Protocol.swift @@ -15,6 +15,21 @@ struct Targets: Codable { let pool: String? let match: String? let exclusive: Bool? + /// web-shots / web-test: what to point the browser at. No device is + /// involved in the host's use of it; on a device job it is the base every + /// manifest page path is resolved against. + let url: String? +} + +/// ui-test / web-test / web-shots: which suite this job runs. +/// +/// `flows` is the spec directory under the collector's web-specs/ — and for +/// web-shots it is also the suite name that baselines are keyed by, which is +/// why a device capture needs it even though it will never open that +/// directory: without it the shot has no cell to land in. +struct SuiteRef: Codable { + let kind: String? + let flows: String? } struct BenchParams: Codable { @@ -53,6 +68,12 @@ struct BenchParams: Codable { let repeats: Int? /// Per-request ceiling, so one hanging host cannot consume the job. let timeoutS: Int? + // web-shots + /// The manifest inline, for a job written by hand rather than pointed at + /// an uploaded shots.json. Same shape either way — `params.input_sha256` + /// names an artifact holding exactly these bytes — so a suite never has + /// two manifest dialects to keep in step. + let shots: ShotsManifest? } struct Constraints: Codable { @@ -68,6 +89,7 @@ struct JobSpec: Codable { let model: ModelRef? let backend: String? let params: BenchParams? + let suite: SuiteRef? let targets: Targets? let constraints: Constraints? } @@ -146,6 +168,13 @@ struct Metrics: Codable { var ttfbMs: Double? var networkType: String? + // web-shots. The one number a per-page row carries: how far this capture + // drifted from the accepted baseline for its (suite, page, profile). A + // named field, never laundered through a slot that means something else — + // and the only metric name in the collector's closed list that means this, + // so nothing here invents one for "pages captured" (that lives in `test`). + var diffPct: Double? + /// Spelled out because `.convertToSnakeCase` cannot produce three of these /// names. The strategy splits on capitals and a digit is not one, so /// `recallAt1` encodes as **`recall_at1`** — one underscore short of the @@ -164,6 +193,7 @@ struct Metrics: Codable { case recallAt10 = "recall_at_10" case docsPerS, dim case dnsMs, connectMs, tlsMs, ttfbMs, networkType + case diffPct } } @@ -173,6 +203,29 @@ struct BeaconSample: Codable { let thermal: String } +/// ui-test / web-shots pass-fail counts for one row, with whatever the row +/// produced attached. Separate from `artifacts` above: those are a job's +/// outputs, these are the evidence for this row's verdict. +struct TestOutcome: Codable { + var passed: Int + var failed: Int + var artifacts: [String]? +} + +/// web-shots: which cell of the visual matrix a per-page row judges. +/// +/// The collector assembles its review grid from these and never from `iter` +/// order — iter maps to manifest order, and manifests change. `sha256` is +/// absent when the capture failed, which is what makes a cell "missing" +/// rather than "diverged". +struct ShotRef: Codable { + var suite: String + var page: String + var profile: String + var sha256: String? + var diffSha256: String? +} + struct ResultPost: Codable { var schema: Int = 1 var kind: String @@ -186,6 +239,8 @@ struct ResultPost: Codable { var beacon: BeaconSample? var error: String? var artifacts: [String]? + var test: TestOutcome? + var shot: ShotRef? } struct RegisterPost: Codable { diff --git a/runner-ios/FleetRunner/WebShotsManifest.swift b/runner-ios/FleetRunner/WebShotsManifest.swift new file mode 100644 index 0000000..621736b --- /dev/null +++ b/runner-ios/FleetRunner/WebShotsManifest.swift @@ -0,0 +1,246 @@ +import Foundation + +/// The shots manifest and the naming rules around it, kept free of UIKit and +/// WebKit so every judgement this workload makes can be checked without a +/// device: parsing, validation, the profile name, thresholds and URL joining. +/// `WebShotsWorkload.swift` holds everything that needs a screen. +/// +/// The shape mirrors `collector/web-specs//shots.json` exactly — the same +/// file the host executor reads in `runWebShots` — rather than a device-only +/// dialect of it. A device job is handed those same bytes as an artifact +/// (`params.input_sha256`) or inline (`params.shots`), so one manifest can +/// drive the host matrix and the phone half of it without being rewritten, and +/// a page added to the suite reaches both. +/// +/// Note the manifest's own key casing is mixed — `waitFor` and `fullPage` are +/// camelCase, `settle_ms` and `threshold_pct` are snake_case — which is what +/// Playwright's option names did to it. `FleetJSON.decoder`'s +/// `.convertFromSnakeCase` handles both: a key with no underscore is passed +/// through untouched. + +struct ShotsManifest: Codable { + /// The profiles the host matrix captures under. A device job ignores this: + /// its profile is its own screen (see `WebShots.profile`), and there is + /// exactly one of them. + var profiles: [String]? + var thresholdPct: Double? + var freezeTime: String? + var pages: [ShotPage]? +} + +struct ShotPage: Codable { + var name: String? + var path: String? + var waitFor: String? + var mask: [String]? + var fullPage: Bool? + var thresholdPct: Double? + var settleMs: Int? +} + +/// A problem with the job or the manifest, phrased for the `error` field of a +/// result row — which is the only place anyone will read it. +struct WebShotsError: Error, LocalizedError { + let message: String + var errorDescription: String? { message } +} + +enum WebShots { + + // MARK: - the profile name + + /// The profile every capture from this runner is filed under. + /// + /// **This says `webkit`, not `safari`, and that is not a typo.** + /// + /// What captures here is a `WKWebView` embedded in the runner app. It is + /// the same WebKit engine Safari renders with, which is the whole reason + /// this workload is worth having — but it is not Safari. Safari applies + /// reader-mode heuristics, runs the user's content blockers, ships its own + /// chrome (which is in frame in the `ios-sim:` profile's `simctl` + /// screenshots and is not in frame here), and treats a few viewport-meta + /// edge cases differently from an embedded web view. A baseline matrix + /// column labelled `safari` would be read as "this is what a person sees + /// in Safari on this phone", and this capture cannot support that claim. + /// `webkit` says exactly what it is: WebKit rendered these pixels. + /// + /// If you are reading this because the name looked wrong next to + /// `ios-sim-safari`: those two profiles genuinely are different things, and + /// renaming this one to match would make the matrix agree with itself by + /// making it wrong. + /// + /// The device id is part of the name for the same reason the host's + /// profiles are `android:` and `ios-sim:`: baselines are + /// keyed `(suite, page, profile)` and pixels differ per screen, so two + /// phones are two baselines. Collapsing them under a bare `webkit` would + /// diff a 6.1" screen against a 6.7" one and call the hardware difference + /// a regression — and, worse, whichever phone ran last would silently + /// redefine the truth for the other. + static let profilePrefix = "webkit" + + static func profile(deviceId: String) -> String { + let id = deviceId.trimmingCharacters(in: .whitespacesAndNewlines) + return id.isEmpty ? profilePrefix : "\(profilePrefix):\(id)" + } + + // MARK: - manifest + + /// How long to sit on a loaded page before the shutter, when the manifest + /// does not say. + /// + /// Deliberately not the 4 s the host's simulator path uses. That path + /// drives Safari with `simctl openurl` and gets no load event at all, so + /// its `settle_ms` is a blind wait standing in for one. Here the navigation + /// really has finished and `document.fonts.ready` really has resolved + /// before this starts, so this is only covering late layout — and four + /// seconds a page, times a matrix, is a lease. + static let defaultSettleMs = 1000 + + /// Threshold the host applies when neither the page nor the manifest names + /// one, kept identical so the same manifest judges the same way on both. + static let defaultThresholdPct = 0.1 + + /// A full-page capture is the whole scroll height, but a runaway page (an + /// infinite feed, a broken 100vh × N layout) would otherwise ask the device + /// for a bitmap it cannot allocate — and a job that dies of memory reports + /// nothing at all. Past this the capture is the first `maxFullPageHeight` + /// points and the row says so, which is a diffable baseline plus a visible + /// caveat rather than silence. + static let maxFullPageHeight: CGFloat = 8000 + + static func decodeManifest(_ data: Data) throws -> ShotsManifest { + do { + return try FleetJSON.decoder.decode(ShotsManifest.self, from: data) + } catch { + throw WebShotsError(message: "shots manifest is not readable JSON: \(error.localizedDescription)") + } + } + + /// The manifest's pages, validated the way the host validates them. + /// + /// Checked here — before a web view is built — rather than discovered as a + /// half-captured matrix when a file path fails to write. Page names become + /// artifact names and baseline keys, so a name that cannot be one is a + /// job-level error, not a page-level one. + static func pages(of manifest: ShotsManifest) throws -> [ShotPage] { + let pages = manifest.pages ?? [] + if pages.isEmpty { throw WebShotsError(message: "shots manifest lists no pages") } + for p in pages { + guard let name = p.name, isUsableName(name) else { + throw WebShotsError( + message: "shots.json page name unusable as a filename: \(p.name.map { "\"\($0)\"" } ?? "null")") + } + guard let path = p.path, !path.isEmpty else { + throw WebShotsError(message: "shots.json page '\(name)' has no path") + } + } + let names = pages.compactMap(\.name) + if Set(names).count != names.count { + throw WebShotsError(message: "shots.json page names must be unique — they name the artifacts") + } + return pages + } + + /// Same rule as the host's `/^[a-z0-9][a-z0-9_-]*$/i`. + static func isUsableName(_ name: String) -> Bool { + name.range(of: "^[a-z0-9][a-z0-9_-]*$", options: [.regularExpression, .caseInsensitive]) != nil + } + + static func threshold(for page: ShotPage, in manifest: ShotsManifest) -> Double { + page.thresholdPct ?? manifest.thresholdPct ?? defaultThresholdPct + } + + static func settleMs(for page: ShotPage) -> Int { + max(page.settleMs ?? defaultSettleMs, 0) + } + + /// A page's path resolved against the job's `targets.url`, the same join + /// the host does with `new URL(p.path, url)`: an absolute path replaces the + /// base's path, a relative one hangs off it, and an absolute URL wins + /// outright. + static func pageURL(base: String, path: String) -> URL? { + guard let baseURL = URL(string: base) else { return nil } + return URL(string: path, relativeTo: baseURL)?.absoluteURL + } + + /// A Swift string as a JavaScript literal, for the selectors and CSS this + /// workload injects. Built by the JSON encoder rather than by hand because + /// a page name or selector carrying a quote, a backslash or a line + /// separator would otherwise stop being data and start being code. + static func jsLiteral(_ s: String) -> String { + guard let data = try? JSONEncoder().encode(s), + let literal = String(data: data, encoding: .utf8) else { + return "\"\"" + } + // U+2028/U+2029 are legal in JSON strings and are line terminators in + // JavaScript source, so JSON-encoding alone is not enough here. + return literal + .replacingOccurrences(of: "\u{2028}", with: "\\u2028") + .replacingOccurrences(of: "\u{2029}", with: "\\u2029") + } + + // MARK: - determinism + + /// Mirrors the `KILL_MOTION_CSS` the host injects on its android path and + /// the `_shots` spec injects on its browser path. Same three properties, + /// because a caret blinking in a captured input is a 100%-reproducible way + /// to fail a diff at random. + static let killMotionCSS = + "*, *::before, *::after { animation: none !important; transition: none !important; " + + "caret-color: transparent !important; }" + + /// CSS that stands in for Playwright's `mask` option: the masked box is + /// painted flat and everything inside it is hidden, so a clock or an avatar + /// cannot fail a diff for being what it is. + /// + /// Not pixel-identical to Playwright's overlay, and it does not need to be + /// — a baseline is per profile, and nothing ever diffs these pixels against + /// a Playwright-captured column. + static func maskCSS(_ selectors: [String]) -> String { + selectors.map { sel in + "\(sel) { background: #FF00FF !important; color: transparent !important; " + + "box-shadow: none !important; border-color: #FF00FF !important; }\n" + + "\(sel) * { visibility: hidden !important; }" + }.joined(separator: "\n") + } + + /// A script that pins `Date` to a fixed instant, for manifests that set + /// `freeze_time` (the host gets this from Playwright's clock API). + /// + /// Injected at document start so a page that stamps a render time reads the + /// frozen clock rather than the real one. Returns nil when the manifest's + /// string is not a date this runner can parse — a silently unfrozen clock + /// would produce a page that fails its diff once a minute, so the caller + /// turns nil into a visible job error instead. + static func freezeTimeScript(_ iso: String) -> String? { + guard let date = isoDate(iso) else { return nil } + let ms = Int64((date.timeIntervalSince1970 * 1000).rounded()) + return """ + (function () { + var fixed = \(ms); + var Real = Date; + function Frozen(...args) { + if (!(this instanceof Frozen)) return new Real(fixed).toString(); + return args.length === 0 ? new Real(fixed) : new Real(...args); + } + Frozen.prototype = Real.prototype; + Frozen.now = function () { return fixed; }; + Frozen.parse = Real.parse; + Frozen.UTC = Real.UTC; + window.Date = Frozen; + })(); + """ + } + + /// `freeze_time` as the manifest writes it. Both spellings the host's + /// `new Date(...)` accepts for an ISO-8601 string: with and without + /// fractional seconds. + static func isoDate(_ s: String) -> Date? { + let withFraction = ISO8601DateFormatter() + withFraction.formatOptions = [.withInternetDateTime, .withFractionalSeconds] + if let d = withFraction.date(from: s) { return d } + let plain = ISO8601DateFormatter() + plain.formatOptions = [.withInternetDateTime] + return plain.date(from: s) + } +} diff --git a/runner-ios/FleetRunner/WebShotsWorkload.swift b/runner-ios/FleetRunner/WebShotsWorkload.swift new file mode 100644 index 0000000..7614af3 --- /dev/null +++ b/runner-ios/FleetRunner/WebShotsWorkload.swift @@ -0,0 +1,649 @@ +import Foundation +import UIKit +import WebKit + +/// web-shots on a real iPhone: the half of the visual-regression matrix the +/// host executor cannot reach. +/// +/// The host captures real Chrome on real Android hardware over adb, and Safari +/// on iOS *simulators* through `simctl`. A real iPhone had no capture path at +/// all — `collector/docs/operations.md` says so plainly — so the matrix has +/// been advertising real phone screens while half the real phones were missing +/// from it. This closes that as a device job: a `WKWebView` inside the runner +/// app, which is the same WebKit engine Safari renders with, needs no WebDriver +/// session, and is already sitting on the phone. +/// +/// It is a device job rather than a host one on purpose. The device is the +/// machine that renders the pixels, so the device is the only machine whose +/// baselines they can be compared against — the same argument that keeps the +/// host's diffing on the host. See `WebShots.profile` for what the profile is +/// called and why it is not called `safari`. +/// +/// Result shape follows the host's: one row per page carrying the page's +/// `shot` block (suite, page, profile, sha256) and its `diff_pct`, then one +/// closing final row. Unlike the host there is no per-profile summary row — +/// a device is one profile, and a summary row would collide with the final row +/// on (job_id, device_id, iter=0) and silently overwrite it. +extension Workloads { + + static let webShotsLoadTimeoutS: TimeInterval = 30 + static let webShotsWaitForTimeoutS: TimeInterval = 15 + + static func runWebShots(job: JobSpec, client: CollectorClient, deviceId: String, + artifacts: ArtifactCache) async { + func fail(_ m: String) async { + try? await client.postResult( + ResultPost(kind: "result", jobId: job.jobId, deviceId: deviceId, + iter: 0, final: true, ok: false, + device: Telemetry.descriptor(), error: m)) + } + + guard let base = job.targets?.url, !base.isEmpty else { + await fail("web-shots needs targets.url"); return + } + // The suite is the first third of a baseline's key. A capture with no + // suite has no cell in the matrix to land in, so this is refused rather + // than defaulted — a default would file every suite's shots together. + guard let suite = job.suite?.flows, !suite.isEmpty else { + await fail("web-shots needs suite.flows (the web-specs/ directory that names this suite)") + return + } + + let manifest: ShotsManifest + let pages: [ShotPage] + do { + if let sha = job.params?.inputSha256 { + let file = try await artifacts.ensure(sha256: sha) + manifest = try WebShots.decodeManifest(try Data(contentsOf: file)) + } else if let inline = job.params?.shots { + manifest = inline + } else { + throw WebShotsError( + message: "web-shots needs params.input_sha256 (the shots.json artifact) or params.shots") + } + pages = try WebShots.pages(of: manifest) + } catch { + await fail(error.localizedDescription); return + } + + // A manifest that asks for a frozen clock and does not get one produces + // a page that fails its diff whenever the minute rolls over, which + // reads as a flaky site rather than as a manifest this runner could not + // honour. Refuse the job instead. + var freezeScript: String? + if let freeze = manifest.freezeTime { + guard let script = WebShots.freezeTimeScript(freeze) else { + await fail("shots.json freeze_time is not an ISO-8601 instant this runner can parse: \(freeze)") + return + } + freezeScript = script + } + + let profile = WebShots.profile(deviceId: deviceId) + + // The accepted truth for this suite. A failure here fails the job: a + // page that cannot be judged is not a page that passed, and capturing + // anyway would fill the matrix with cells that look accepted-and-clean + // because nothing was ever compared. + let baselines: [String: String] + do { + baselines = try await client.visualBaselines(suite: suite) + } catch { + await fail("baselines fetch failed: \(error.localizedDescription)"); return + } + + let capture: WebShotCapture + do { + capture = try await WebShotCapture.make(freezeScript: freezeScript) + } catch { + await fail(error.localizedDescription); return + } + defer { Task { @MainActor in capture.tearDown() } } + + let batteryStart = Telemetry.batteryPct() + var captured = 0 + var diverged = 0 + var shas: [String] = [] + var report: [[String: Any]] = [] + + for (index, page) in pages.enumerated() { + // Iteration boundary: a beacon may have found the lease gone. The + // rows already posted stay valid; the pages after this never happen. + if CancellationRegistry.shared.isCancelled(job.jobId) { + try? await client.postResult( + ResultPost(kind: "result", jobId: job.jobId, deviceId: deviceId, + iter: 0, final: true, ok: false, + device: Telemetry.descriptor(), error: "cancelled")) + return + } + + let name = page.name ?? "page-\(index + 1)" + var ok = false + var note: String? + var sha: String? + var diffSha: String? + var diffPct: Double? + var rowArtifacts: [String] = [] + var row: [String: Any] = ["iter": index + 1, "page": name] + + guard let pageURL = WebShots.pageURL(base: base, path: page.path ?? "") else { + note = "page '\(name)' path \(page.path.map { "'\($0)'" } ?? "(none)") is not a URL against \(base)" + row["error"] = note! + report.append(row) + await postShotRow(client: client, job: job, deviceId: deviceId, iter: index + 1, + ok: false, suite: suite, page: name, profile: profile, + sha: nil, diffSha: nil, diffPct: nil, artifacts: [], note: note) + continue + } + row["url"] = pageURL.absoluteString + + do { + let shot = try await capture.capture(page: page, url: pageURL) + captured += 1 + ok = true + row["width_px"] = shot.widthPx + row["height_px"] = shot.heightPx + row["bytes"] = shot.png.count + if let truncated = shot.truncatedNote { row["note"] = truncated } + + sha = try await client.uploadArtifact(shot.png, name: "\(job.jobId)-\(profile)-\(name).png") + rowArtifacts.append(sha!) + shas.append(sha!) + + let threshold = WebShots.threshold(for: page, in: manifest) + row["threshold_pct"] = threshold + if let baseline = baselines["\(name)|\(profile)"] { + if baseline == sha { + diffPct = 0 // identical bytes; nothing to decode + } else { + do { + let baselineFile = try await artifacts.ensure(sha256: baseline) + let outcome = try ShotDiff.compare( + captured: shot.png, baseline: try Data(contentsOf: baselineFile)) + diffPct = outcome.diffPct + if outcome.diffPct > threshold { + ok = false + diverged += 1 + note = outcome.note + ?? String(format: "%.2f%% of pixels differ (threshold %g%%)", + outcome.diffPct, threshold) + // The diff image only when it matters: a + // within-threshold pair has nothing worth a + // person's look, and the store is forever. + if let diffPNG = outcome.diffPNG { + diffSha = try await client.uploadArtifact( + diffPNG, name: "\(job.jobId)-\(profile)-\(name)-diff.png") + rowArtifacts.append(diffSha!) + } + } + } catch { + // A baseline the store cannot produce is an + // operational problem, not a visual regression — + // but it must fail, loudly, because a page that + // cannot be judged is not a page that passed. + ok = false + diverged += 1 + note = "baseline \(baseline.prefix(12))… unusable: \(error.localizedDescription)" + } + } + } else { + // Visible, not failing: a new page's first capture is not a + // regression, but it stays flagged until someone accepts it. + note = "new: no baseline — accept this shot to start diffing" + } + } catch { + note = "no screenshot for page '\(name)': \(error.localizedDescription)" + } + + if let diffPct { row["diff_pct"] = diffPct } + if let sha { row["sha256"] = sha } + if let note { row["error"] = note } + row["ok"] = ok + report.append(row) + + await postShotRow(client: client, job: job, deviceId: deviceId, iter: index + 1, + ok: ok, suite: suite, page: name, profile: profile, + sha: sha, diffSha: diffSha, diffPct: diffPct, + artifacts: rowArtifacts, note: note) + } + + // The rows carry no URL and no capture size — the result schema has no + // field for either — so the report is what makes a red matrix + // debuggable without a person guessing which path a page name meant. + var finalArtifacts = shas + let summaryJSON: [String: Any] = [ + "job_id": job.jobId, "device_id": deviceId, + "suite": suite, "profile": profile, "base_url": base, + "pages": pages.count, "captured": captured, "diverged": diverged, + "engine": "WKWebView (WebKit); not Safari — no reader mode, no content blockers", + "screen_pt": ["width": capture.screenSize.width, "height": capture.screenSize.height], + "rows": report, + ] + if let data = try? JSONSerialization.data(withJSONObject: summaryJSON), + let sha = try? await client.uploadArtifact(data, name: "\(job.jobId)-web-shots.json") { + finalArtifacts.append(sha) + } + + let missed = pages.count - captured + let ok = missed == 0 && diverged == 0 + let problems = [ + missed > 0 ? "\(missed) of \(pages.count) pages not captured" : nil, + diverged > 0 ? "\(diverged) page(s) diverged from baseline" : nil, + ].compactMap { $0 }.joined(separator: "; ") + + var summary = Metrics() + summary.peakMemMb = Telemetry.physFootprintMb() + summary.memMethod = "phys_footprint" + summary.thermal = [Telemetry.thermal()] + summary.batteryStartPct = batteryStart + summary.batteryEndPct = Telemetry.batteryPct() + + try? await client.postResult( + ResultPost(kind: "result", jobId: job.jobId, deviceId: deviceId, + iter: 0, final: true, ok: ok, + device: Telemetry.descriptor(), metrics: summary, + error: ok ? nil : problems, + artifacts: finalArtifacts.isEmpty ? nil : finalArtifacts, + test: TestOutcome(passed: pages.count - missed - diverged, + failed: missed + diverged, artifacts: nil))) + } + + /// One cell of the matrix, posted the way the collector's + /// /api/visual/matrix expects to read it back. + /// + /// `diff_pct` is the only metric here and it is a named field — never + /// laundered through a slot that means something else. There is no metric + /// name in the collector's closed list for "pages captured", so that count + /// lives in `test` and in the report artifact rather than being invented. + private static func postShotRow( + client: CollectorClient, job: JobSpec, deviceId: String, iter: Int, ok: Bool, + suite: String, page: String, profile: String, + sha: String?, diffSha: String?, diffPct: Double?, artifacts: [String], note: String? + ) async { + var metrics: Metrics? + if let diffPct { + var m = Metrics() + m.diffPct = (diffPct * 10_000).rounded() / 10_000 + metrics = m + } + try? await client.postResult( + ResultPost(kind: "result", jobId: job.jobId, deviceId: deviceId, + iter: iter, ok: ok, metrics: metrics, error: note, + test: TestOutcome(passed: ok ? 1 : 0, failed: ok ? 0 : 1, + artifacts: artifacts.isEmpty ? nil : artifacts), + shot: ShotRef(suite: suite, page: page, profile: profile, + sha256: sha, diffSha256: diffSha))) + } +} + +// MARK: - capture + +/// One page's pixels, plus what the row needs to say about how they were taken. +struct WebShot { + let png: Data + let widthPx: Int + let heightPx: Int + /// Set when a full-page capture hit `WebShots.maxFullPageHeight`, so a + /// clipped baseline is a visible caveat rather than a quiet one. + let truncatedNote: String? +} + +/// An offscreen `WKWebView`, sized to the device's screen, that captures one +/// page at a time. +/// +/// The web view lives in a `UIWindow` of its own at one level *below* normal, +/// rather than inside the runner's own view hierarchy. WebKit renders lazily +/// for a view with no window, so a detached web view snapshots blank; a window +/// under the app's own opaque one is a real render surface that never appears +/// on screen and never disturbs the screen that is there. It is deliberately +/// not made key: the runner's UI keeps the keyboard and the focus it had. +/// +/// One instance per job, reused across pages: a fresh web view per page would +/// pay process-launch cost per cell of the matrix, and its data store is +/// non-persistent anyway, so nothing carries over that a reload would not. +@MainActor +final class WebShotCapture: NSObject, WKNavigationDelegate { + private var window: UIWindow? + private let webView: WKWebView + private var pending: CheckedContinuation? + private var timeoutTask: Task? + let screenSize: CGSize + + static func make(freezeScript: String?) async throws -> WebShotCapture { + guard let scene = UIApplication.shared.connectedScenes + .compactMap({ $0 as? UIWindowScene }) + .first(where: { $0.activationState != .background }) ?? UIApplication.shared.connectedScenes + .compactMap({ $0 as? UIWindowScene }).first + else { + // Not defaulted to a guessed screen size: the profile's whole + // meaning is "this device's screen", and inventing one would make + // the baseline a fiction. + throw WebShotsError( + message: "no window scene — the runner must be in the foreground to capture web shots") + } + return WebShotCapture(scene: scene, freezeScript: freezeScript) + } + + private init(scene: UIWindowScene, freezeScript: String?) { + let bounds = scene.screen.bounds + screenSize = bounds.size + + let config = WKWebViewConfiguration() + // Non-persistent, so no cache or cookie jar left by an earlier job can + // serve a page that no longer renders that way. + config.websiteDataStore = .nonPersistent() + config.suppressesIncrementalRendering = true + if let freezeScript { + config.userContentController.addUserScript( + WKUserScript(source: freezeScript, injectionTime: .atDocumentStart, forMainFrameOnly: false)) + } + + webView = WKWebView(frame: CGRect(origin: .zero, size: bounds.size), configuration: config) + webView.isOpaque = true + webView.backgroundColor = .white + webView.scrollView.contentInsetAdjustmentBehavior = .never + webView.scrollView.showsVerticalScrollIndicator = false + webView.scrollView.showsHorizontalScrollIndicator = false + super.init() + webView.navigationDelegate = self + + let window = UIWindow(windowScene: scene) + window.frame = bounds + window.windowLevel = .normal - 1 + let host = UIViewController() + host.view.backgroundColor = .white + host.view.addSubview(webView) + window.rootViewController = host + window.isHidden = false + self.window = window + } + + func tearDown() { + finish(.failure(WebShotsError(message: "capture torn down"))) + webView.navigationDelegate = nil + webView.stopLoading() + webView.removeFromSuperview() + window?.isHidden = true + window = nil + } + + /// Load one page and photograph it. + func capture(page: ShotPage, url: URL) async throws -> WebShot { + // Back to the screen's own size before every page: the previous page + // may have grown the view for a full-page shot, and a viewport that + // depends on which page ran before it is not a baseline. + webView.frame = CGRect(origin: .zero, size: screenSize) + webView.layoutIfNeeded() + + webView.load(URLRequest(url: url, cachePolicy: .reloadIgnoringLocalAndRemoteCacheData, + timeoutInterval: Workloads.webShotsLoadTimeoutS)) + try await waitForLoad(timeout: Workloads.webShotsLoadTimeoutS) + + if let selector = page.waitFor, !selector.isEmpty { + try await waitFor(selector: selector, timeout: Workloads.webShotsWaitForTimeoutS) + } + + // Determinism first, then fonts, then settle: injected CSS can change + // layout, and web fonts certainly do, so both must land before the + // settle window rather than during it. + try? await inject(css: WebShots.killMotionCSS) + if let mask = page.mask, !mask.isEmpty { + try? await inject(css: WebShots.maskCSS(mask)) + } + _ = try? await webView.callAsyncJavaScript( + "await document.fonts.ready; return true;", contentWorld: .page) + + let settle = WebShots.settleMs(for: page) + if settle > 0 { try? await Task.sleep(for: .milliseconds(settle)) } + + var truncated: String? + if page.fullPage ?? true { + let full = await contentHeight() + let wanted = max(full, screenSize.height) + let height = min(wanted, WebShots.maxFullPageHeight) + if wanted > WebShots.maxFullPageHeight { + truncated = "full-page capture clipped at \(Int(WebShots.maxFullPageHeight))pt " + + "of \(Int(wanted))pt" + } + if abs(height - webView.frame.height) > 0.5 { + webView.frame = CGRect(origin: .zero, size: CGSize(width: screenSize.width, height: height)) + webView.layoutIfNeeded() + // The resize is a layout change of its own; give the page the + // frame it needs to reflow into it before the shutter. + try? await Task.sleep(for: .milliseconds(250)) + } + } + + let config = WKSnapshotConfiguration() + config.rect = CGRect(origin: .zero, size: webView.bounds.size) + config.afterScreenUpdates = true + let image = try await webView.takeSnapshot(configuration: config) + guard let png = image.pngData() else { + throw WebShotsError(message: "snapshot produced no PNG") + } + return WebShot(png: png, + widthPx: Int(image.size.width * image.scale), + heightPx: Int(image.size.height * image.scale), + truncatedNote: truncated) + } + + // MARK: navigation + + private func waitForLoad(timeout: TimeInterval) async throws { + try await withCheckedThrowingContinuation { (c: CheckedContinuation) in + pending = c + timeoutTask = Task { [weak self] in + try? await Task.sleep(for: .seconds(timeout)) + guard !Task.isCancelled else { return } + self?.finish(.failure(WebShotsError( + message: "load did not finish inside \(Int(timeout))s"))) + } + } + } + + /// Resumes whatever is waiting on a navigation, at most once. Everything + /// here runs on the main actor, so "at most once" needs no lock — but it + /// does need this one funnel, because a timeout and a `didFail` racing to + /// resume the same continuation is a crash, not a failed job. + private func finish(_ result: Result) { + guard let c = pending else { return } + pending = nil + timeoutTask?.cancel() + timeoutTask = nil + c.resume(with: result) + } + + func webView(_ webView: WKWebView, didFinish navigation: WKNavigation!) { + finish(.success(())) + } + + func webView(_ webView: WKWebView, didFail navigation: WKNavigation!, withError error: Error) { + finish(.failure(error)) + } + + func webView(_ webView: WKWebView, didFailProvisionalNavigation navigation: WKNavigation!, + withError error: Error) { + finish(.failure(error)) + } + + // MARK: page helpers + + private func waitFor(selector: String, timeout: TimeInterval) async throws { + let js = "document.querySelector(\(WebShots.jsLiteral(selector))) !== null" + let deadline = Date().addingTimeInterval(timeout) + while Date() < deadline { + if let value = try? await webView.evaluateJavaScript(js), (value as? Bool) == true { return } + try? await Task.sleep(for: .milliseconds(200)) + } + throw WebShotsError(message: "waitFor '\(selector)' never matched inside \(Int(timeout))s") + } + + private func inject(css: String) async throws { + let js = """ + (function () { + var s = document.createElement('style'); + s.textContent = \(WebShots.jsLiteral(css)); + (document.head || document.documentElement).appendChild(s); + return true; + })(); + """ + _ = try await webView.evaluateJavaScript(js) + } + + /// The page's full scroll height in points, falling back to the viewport + /// when the page will not say — a fallback that captures the fold rather + /// than capturing nothing. + private func contentHeight() async -> CGFloat { + let js = """ + Math.ceil(Math.max( + document.body ? document.body.scrollHeight : 0, + document.documentElement ? document.documentElement.scrollHeight : 0)); + """ + if let value = try? await webView.evaluateJavaScript(js), let n = value as? NSNumber { + let h = CGFloat(truncating: n) + if h.isFinite && h > 0 { return h } + } + return screenSize.height + } +} + +// MARK: - diffing + +/// How far a capture drifted from its accepted baseline. +/// +/// Diffing happens on the device for the same reason the host diffs on the +/// host: a baseline is only comparable to pixels rendered by the same +/// renderer on the same screen. Shipping the bytes to the collector to be +/// judged would look tidier and would be wrong — the collector has no idea +/// what this phone's subpixel rendering looks like, and moving the judgement +/// there would quietly make every device's baseline mean "whatever the +/// collector's decoder thinks". +enum ShotDiff { + + struct Outcome { + let diffPct: Double + let diffPNG: Data? + let note: String? + } + + /// pixelmatch's colour distance and its default threshold, so "is this + /// pixel different" means the same thing on the phone as it does on the + /// host. What is *not* ported is pixelmatch's antialiasing detection: it + /// needs each pixel's neighbourhood in both images and is a workload of its + /// own. The practical consequence is that this reads slightly hotter than + /// the host on text edges, which is a reason to set a suite's + /// `threshold_pct` with the device column in mind — not a reason to compare + /// the two columns, which nothing does. + private static let maxDelta = 35215.0 * 0.1 * 0.1 + + static func compare(captured: Data, baseline: Data) throws -> Outcome { + guard let capturedImage = UIImage(data: captured)?.cgImage else { + throw WebShotsError(message: "captured PNG could not be decoded") + } + guard let baselineImage = UIImage(data: baseline)?.cgImage else { + throw WebShotsError(message: "baseline PNG could not be decoded") + } + let w = capturedImage.width, h = capturedImage.height + guard w == baselineImage.width, h == baselineImage.height else { + // A size change short-circuits to 100%, as it does on the host: + // mismatched grids cannot be compared pixel for pixel, and a page + // whose height changed has materially changed however its + // overlapping pixels look. + return Outcome( + diffPct: 100, + diffPNG: nil, + note: "dimensions changed: \(w)×\(h) vs baseline " + + "\(baselineImage.width)×\(baselineImage.height)") + } + guard w > 0, h > 0 else { + throw WebShotsError(message: "captured PNG has no pixels") + } + + let a = try raster(capturedImage) + let b = try raster(baselineImage) + + var diffMask = [Bool](repeating: false, count: w * h) + var differing = 0 + for i in 0..<(w * h) { + let o = i * 4 + if delta(a[o], a[o + 1], a[o + 2], b[o], b[o + 1], b[o + 2]) > maxDelta { + diffMask[i] = true + differing += 1 + } + } + let pct = Double(differing) * 100.0 / Double(w * h) + return Outcome(diffPct: pct, + diffPNG: differing > 0 ? diffImage(mask: diffMask, base: a, width: w, height: h) : nil, + note: nil) + } + + /// Both images drawn into one known format — 8-bit RGBA over white — so + /// the comparison is never actually comparing two colour spaces or two + /// alpha conventions and calling the difference a regression. + private static func raster(_ image: CGImage) throws -> [UInt8] { + let w = image.width, h = image.height + var bytes = [UInt8](repeating: 255, count: w * h * 4) + let space = CGColorSpaceCreateDeviceRGB() + let info = CGImageAlphaInfo.premultipliedLast.rawValue + guard let ctx = bytes.withUnsafeMutableBytes({ raw in + CGContext(data: raw.baseAddress, width: w, height: h, bitsPerComponent: 8, + bytesPerRow: w * 4, space: space, bitmapInfo: info) + }) else { + throw WebShotsError(message: "could not rasterise a \(w)×\(h) image for diffing") + } + ctx.setFillColor(UIColor.white.cgColor) + ctx.fill(CGRect(x: 0, y: 0, width: w, height: h)) + ctx.draw(image, in: CGRect(x: 0, y: 0, width: w, height: h)) + guard let data = ctx.data else { + throw WebShotsError(message: "rasteriser produced no pixels") + } + return [UInt8](UnsafeBufferPointer( + start: data.assumingMemoryBound(to: UInt8.self), count: w * h * 4)) + } + + /// pixelmatch's YIQ colour delta. + private static func delta(_ r1: UInt8, _ g1: UInt8, _ b1: UInt8, + _ r2: UInt8, _ g2: UInt8, _ b2: UInt8) -> Double { + if r1 == r2 && g1 == g2 && b1 == b2 { return 0 } + let r1d = Double(r1), g1d = Double(g1), b1d = Double(b1) + let r2d = Double(r2), g2d = Double(g2), b2d = Double(b2) + let y = (r1d - r2d) * 0.29889531 + (g1d - g2d) * 0.58662247 + (b1d - b2d) * 0.11448223 + let i = (r1d - r2d) * 0.59597799 - (g1d - g2d) * 0.27417610 - (b1d - b2d) * 0.32180189 + let q = (r1d - r2d) * 0.21147017 - (g1d - g2d) * 0.52261711 + (b1d - b2d) * 0.31114694 + return 0.5053 * y * y + 0.299 * i * i + 0.1957 * q * q + } + + /// The captured page, faded, with every differing pixel in red — enough for + /// a person to see *where* it moved, which is the only question a diff + /// image is opened to answer. + private static func diffImage(mask: [Bool], base: [UInt8], width w: Int, height h: Int) -> Data? { + var out = [UInt8](repeating: 255, count: w * h * 4) + for i in 0..<(w * h) { + let o = i * 4 + if mask[i] { + out[o] = 255; out[o + 1] = 0; out[o + 2] = 0; out[o + 3] = 255 + } else { + // Greyscale at low contrast: present enough to place the red, + // faint enough that the red is what the eye lands on. + // Broken into steps deliberately: as one expression, Swift's + // type checker gives up on the literal/Double mix and fails the + // build with "unable to type-check in reasonable time". + let r = Double(base[o]) * 0.299 + let g = Double(base[o + 1]) * 0.587 + let b = Double(base[o + 2]) * 0.114 + let luma: Double = r + g + b + let lifted: Double = 180.0 + luma * 0.075 + let grey = UInt8(min(255.0, lifted)) + out[o] = grey; out[o + 1] = grey; out[o + 2] = grey; out[o + 3] = 255 + } + } + let space = CGColorSpaceCreateDeviceRGB() + let info = CGImageAlphaInfo.premultipliedLast.rawValue + return out.withUnsafeMutableBytes { raw -> Data? in + guard let ctx = CGContext(data: raw.baseAddress, width: w, height: h, bitsPerComponent: 8, + bytesPerRow: w * 4, space: space, bitmapInfo: info), + let cg = ctx.makeImage() else { return nil } + return UIImage(cgImage: cg).pngData() + } + } +} diff --git a/runner-machine/recipes/dir-labels.mjs b/runner-machine/recipes/dir-labels.mjs new file mode 100644 index 0000000..e4efb0f --- /dev/null +++ b/runner-machine/recipes/dir-labels.mjs @@ -0,0 +1,34 @@ +/** + * The generic recipe: one directory per class. + * + * `train/rose/0001.jpg` is a rose; the label is the index of `rose` in the + * sorted list of directory names actually present. Sorted, not + * first-seen — the same archive prepared twice must produce the same label + * for the same class, or two runs of the same eval are not comparable and + * nobody would be able to tell from the numbers. + * + * It states no licence on purpose. A recipe that works for any pile of images + * cannot know the terms of the pile it is pointed at, so `params.license` is + * required and the prep refuses without it. + */ + +/** The directory immediately above the file, which is the class. */ +const classOf = (relPath) => { + const parts = relPath.split("/"); + return parts.length > 1 ? parts[parts.length - 2] : ""; +}; + +export default { + id: "dir-labels", + license: null, + size: 224, + preprocess: "center-crop, resize 224, RGB; ImageNet mean/std applied on-device", + + labels(files) { + const classes = [...new Set(files.map(classOf))].sort(); + return { + classes: classes.length, + items: files.map((f) => ({ label: classes.indexOf(classOf(f)), class: classOf(f) })), + }; + }, +}; diff --git a/runner-machine/recipes/plant-id.mjs b/runner-machine/recipes/plant-id.mjs new file mode 100644 index 0000000..b77fa18 --- /dev/null +++ b/runner-machine/recipes/plant-id.mjs @@ -0,0 +1,45 @@ +/** + * PlantNet-300K, prepared for the plant-ID eval. + * + * The classes are not this recipe's to number: the model has 1081 outputs in + * an order fixed at training time, so the directory name IS the class index + * and inventing a compact 0..n-1 numbering from whichever species happen to be + * in this archive would score every prediction against the wrong label. A + * directory that is not a number in range is dropped rather than guessed at. + * + * The licence is stated here rather than left to the job. PlantNet-300K is + * CC-BY-SA, which is the reason its images ship through the artifact store + * instead of being committed, and a prepared set that arrives without the + * attribution is a set whose eval results cannot be published anywhere. + */ + +const CLASSES = 1081; + +const classOf = (relPath) => { + const parts = relPath.split("/"); + return parts.length > 1 ? parts[parts.length - 2] : ""; +}; + +export default { + id: "plant-id", + license: "CC-BY-SA-4.0", + size: 224, + preprocess: "center-crop, resize 224, RGB; ImageNet mean/std applied on-device", + + labels(files) { + return { + classes: CLASSES, + items: files.map((f) => { + const dir = classOf(f); + const label = /^\d+$/.test(dir) ? Number(dir) : -1; + return { + // -1 rather than 0: an unlabelled image scored against class 0 would + // quietly depress top-1 by however many files were misfiled, whereas + // a label no prediction can equal shows up as a visible miss. + label: label >= 0 && label < CLASSES ? label : -1, + species_dir: dir, + }; + }), + }; + }, +}; diff --git a/runner-machine/src/agent.ts b/runner-machine/src/agent.ts index b1e5508..090c963 100644 --- a/runner-machine/src/agent.ts +++ b/runner-machine/src/agent.ts @@ -16,6 +16,10 @@ import { probeCapabilities } from "./capabilities.js"; import { runBenchmark } from "./workloads/benchmark.js"; import { runBuild } from "./workloads/build.js"; import { runSelfCheck } from "./workloads/selfcheck.js"; +import { runModelConvert } from "./workloads/modelconvert.js"; +import { runDatasetPrep } from "./workloads/datasetprep.js"; +import { runServe } from "./workloads/serve.js"; +import { runShell } from "./workloads/shell.js"; import { SCHEMA, type Descriptor, type JobSpec } from "./protocol.js"; import * as JobCancellation from "./cancellation.js"; @@ -207,6 +211,22 @@ async function dispatch(job: JobSpec, client: CollectorClient): Promise { await runSelfCheck(job, client, deviceId, device); return; } + if (job.workload === "model-convert") { + await runModelConvert(job, client, deviceId, device); + return; + } + if (job.workload === "dataset-prep") { + await runDatasetPrep(job, client, deviceId, device); + return; + } + if (job.workload === "serve") { + await runServe(job, client, deviceId, device); + return; + } + if (job.workload === "shell") { + await runShell(job, client, deviceId, device); + return; + } await client.postResult({ schema: SCHEMA, kind: "result", job_id: job.job_id, device_id: deviceId, iter: 0, final: true, ok: false, device: descriptor ?? undefined, diff --git a/runner-machine/src/allowlist.ts b/runner-machine/src/allowlist.ts new file mode 100644 index 0000000..bbf74a8 --- /dev/null +++ b/runner-machine/src/allowlist.ts @@ -0,0 +1,93 @@ +/** + * The trust boundary for the `shell` workload. + * + * `POST /jobs` on the collector is unauthenticated by design — the README says + * so outright — so anyone who can reach the collector can enqueue a job, and a + * workload that runs an arbitrary script would hand them this machine. The + * allowlist is what makes `shell` safe to declare: a file the machine's owner + * maintains by hand, holding the sha256 of every script this machine is + * willing to execute. + * + * Three properties this file exists to guarantee: + * + * 1. The check happens on the sha the JOB NAMED, before anything is fetched. + * Downloading first and checking second would mean an unlisted script had + * already been pulled onto the machine, and the artifact store is writable + * by the same unauthenticated POST. + * 2. Only an exact 64-hex match counts. No prefixes, no substrings: a sha that + * differs by one character is a different script. + * 3. No allowlist is not an empty allowlist in name only — it means `shell` is + * never declared, so the collector cannot even offer this machine the job. + */ +import { readFile } from "node:fs/promises"; +import path from "node:path"; +import { dataDir } from "./git.js"; + +export const SHA256_RE = /^[0-9a-f]{64}$/; + +/** Where the owner keeps the list. One place, overridable for testing and for + * operators who keep it under configuration management. */ +export function allowlistPath(env: NodeJS.ProcessEnv = process.env): string { + return env.FLEET_SHELL_ALLOWLIST ?? path.join(dataDir(env), "shell-allowlist.txt"); +} + +/** + * The shas in an allowlist file. + * + * `#` comments and blank lines are skipped, and anything after the sha on a + * line is a note for whoever reads the file — a bare list of hashes with no + * room to say what each one is would be a file nobody could maintain, and an + * unmaintainable allowlist gets replaced with a wildcard. + * + * A line that is not a sha is DROPPED, not accepted loosely. The failure this + * avoids is a typo'd or truncated hash silently becoming an entry that can + * never match, which reads to the operator as "I listed it and it still + * refuses" — so `entriesRejected` is reported alongside, and the workload says + * so in its refusal. + */ +export function parseAllowlist(text: string | null | undefined): { allowed: string[]; rejected: string[] } { + const allowed: string[] = []; + const rejected: string[] = []; + for (const raw of (text ?? "").split(/\r?\n/)) { + const line = raw.replace(/#.*$/, "").trim(); + if (line === "") continue; + const first = line.split(/\s+/)[0] ?? ""; + const candidate = first.toLowerCase(); + if (SHA256_RE.test(candidate)) { + if (!allowed.includes(candidate)) allowed.push(candidate); + } else { + rejected.push(first.slice(0, 80)); + } + } + return { allowed, rejected }; +} + +/** + * Is this sha one the owner listed? + * + * Case-insensitive because a hash pasted from `shasum` and one pasted from a + * dashboard can differ in case and are the same hash; nothing else is + * forgiven. A value that is not a well-formed sha256 is never allowed, which + * matters because the job spec is attacker-controlled and `""` or `"*"` must + * not become a match against anything. + */ +export function isAllowlisted(sha256: unknown, allowed: readonly string[]): boolean { + if (typeof sha256 !== "string") return false; + const candidate = sha256.trim().toLowerCase(); + if (!SHA256_RE.test(candidate)) return false; + return allowed.includes(candidate); +} + +/** The list on disk. A missing or unreadable file is an empty list, never a throw. */ +export async function loadAllowlist( + env: NodeJS.ProcessEnv = process.env, +): Promise<{ allowed: string[]; rejected: string[]; file: string; exists: boolean }> { + const file = allowlistPath(env); + let text: string | null = null; + try { + text = await readFile(file, "utf8"); + } catch { + return { allowed: [], rejected: [], file, exists: false }; + } + return { ...parseAllowlist(text), file, exists: true }; +} diff --git a/runner-machine/src/beaconing.ts b/runner-machine/src/beaconing.ts new file mode 100644 index 0000000..dc1f453 --- /dev/null +++ b/runner-machine/src/beaconing.ts @@ -0,0 +1,170 @@ +/** + * Keeping a lease alive across something slow. + * + * `model-convert`, `dataset-prep` and `serve` are all in the collector's + * LONG_LEASE_WORKLOADS, but a long lease is not a self-renewing one: the lease + * is renewed by beacons, and the sweep requeues a claim whose beacons stopped. + * A quantisation that takes forty minutes without a beacon gets swept and run + * again on another machine, which is the expensive kind of wrong. + * + * The `build` workload solved this by tailing its log file, because a compiler + * emits a log worth showing. These two do not: one is a child process that + * prints little, the other is ten thousand tiny image resizes. So the shape + * here is a ticker rather than a tail. + * + * Both helpers treat the collector's answers exactly as build does. Only an + * explicit `lease_renewed: false` stops the work — an unreachable collector + * throws and is swallowed, because a flaky network must never be what kills a + * multi-gigabyte conversion. + */ +import { spawn } from "node:child_process"; +import type { CollectorClient } from "./collector.js"; +import { SCHEMA } from "./protocol.js"; +import { beacon } from "./telemetry.js"; +import * as JobCancellation from "./cancellation.js"; + +export const BEACON_MS = 30_000; +/** Kept of a child's output. Enough to name a failure; not enough to grow unbounded. */ +export const MAX_OUTPUT_BYTES = 1 << 20; + +/** + * A rate-limited beacon that can be called from inside any loop. + * + * `maybe()` is cheap when it is not yet time, so it can go at the top of a + * per-image loop without anyone having to reason about how often that runs. + */ +export class Beaconer { + private last = Date.now(); + + constructor( + private readonly client: CollectorClient, + private readonly deviceId: string, + private readonly jobId: string, + private readonly intervalMs: number = BEACON_MS, + ) {} + + /** False means stop: the claim is gone, or the job was cancelled. */ + async maybe(now: number = Date.now()): Promise { + if (JobCancellation.isCancelled(this.jobId)) return false; + if (now - this.last < this.intervalMs) return true; + this.last = now; + try { + const renewed = await this.client.postBeacon({ + schema: SCHEMA, kind: "beacon", job_id: this.jobId, device_id: this.deviceId, beacon: await beacon(), + }); + if (!renewed) { + JobCancellation.cancel(this.jobId); + return false; + } + } catch { + // Not a cancellation. + } + return true; + } +} + +export type ChildOutcome = { + code: number | null; + signal: NodeJS.Signals | null; + /** stdout and stderr, merged, bounded, in the order they arrived. */ + output: string; + cancelled: boolean; + timedOut: boolean; + spawnError: string | null; +}; + +/** + * Runs one long child process, beaconing while it works. + * + * Merged stdio for the same reason the build workload merges it: a converter's + * progress goes to stdout and its traceback to stderr, and reading them apart + * loses which came first, which is the only thing that makes a traceback + * attributable to a step. + */ +export function runWithBeacons(opts: { + cmd: string; + args: string[]; + cwd: string; + env?: NodeJS.ProcessEnv; + timeoutMs: number; + jobId: string; + deviceId: string; + client: CollectorClient; + onTick?: (output: string) => void; +}): Promise { + return new Promise((resolve) => { + let output = ""; + let bytes = 0; + let cancelled = false; + let timedOut = false; + let spawnError: string | null = null; + let finished = false; + + const child = spawn(opts.cmd, opts.args, { + cwd: opts.cwd, + env: opts.env ?? { ...process.env, TERM: "dumb", PYTHONUNBUFFERED: "1" }, + stdio: ["ignore", "pipe", "pipe"], + }); + const absorb = (b: Buffer) => { + if (bytes >= MAX_OUTPUT_BYTES) return; + bytes += b.length; + output += b.toString("utf8"); + }; + child.stdout?.on("data", absorb); + child.stderr?.on("data", absorb); + + const killer = setTimeout(() => { + if (!finished) { + timedOut = true; + child.kill("SIGKILL"); + } + }, opts.timeoutMs); + + const ticker = setInterval(() => { + void (async () => { + opts.onTick?.(output); + if (JobCancellation.isCancelled(opts.jobId) && !cancelled) { + cancelled = true; + child.kill("SIGTERM"); + setTimeout(() => child.kill("SIGKILL"), 10_000).unref(); + return; + } + try { + const renewed = await opts.client.postBeacon({ + schema: SCHEMA, kind: "beacon", job_id: opts.jobId, device_id: opts.deviceId, beacon: await beacon(), + }); + if (!renewed && !cancelled) { + cancelled = true; + JobCancellation.cancel(opts.jobId); + child.kill("SIGTERM"); + setTimeout(() => child.kill("SIGKILL"), 10_000).unref(); + } + } catch { + // An unreachable collector is not a cancellation. + } + })(); + }, BEACON_MS); + ticker.unref?.(); + + const done = (code: number | null, signal: NodeJS.Signals | null) => { + finished = true; + clearTimeout(killer); + clearInterval(ticker); + resolve({ code, signal, output, cancelled, timedOut, spawnError }); + }; + child.on("error", (e) => { + spawnError = e.message; + done(null, null); + }); + child.on("close", (code, signal) => done(code, signal)); + }); +} + +/** The line of a child's output that says what went wrong, for an error row. */ +export function lastMeaningful(output: string, max = 300): string { + const lines = output.split(/\r?\n/).map((l) => l.trim()).filter(Boolean); + // A python traceback's last line is the exception, which is the answer; a + // shell tool's last line is usually its error too. + const line = lines.at(-1) ?? ""; + return line.length > max ? `${line.slice(0, max - 1)}…` : line; +} diff --git a/runner-machine/src/capabilities.ts b/runner-machine/src/capabilities.ts index bbb0aa0..4985741 100644 --- a/runner-machine/src/capabilities.ts +++ b/runner-machine/src/capabilities.ts @@ -37,6 +37,8 @@ */ import { which, run } from "./probe.js"; import { KIND_BINARY } from "./buildkinds.js"; +import { convertersAvailable, probeConverters } from "./converters.js"; +import { loadAllowlist } from "./allowlist.js"; export type CapabilityFlags = { llamaBench: boolean; @@ -44,6 +46,12 @@ export type CapabilityFlags = { gradle: boolean; xcodebuild: boolean; node: boolean; + /** Which model converters resolved: gguf, coreml, tflite. */ + converters?: string[]; + /** Whether a non-empty shell allowlist exists on this machine. */ + shellAllowlist?: boolean; + /** Whether a llama-server binary resolved, for the serve workload. */ + llamaServer?: boolean; }; /** The list, given the answers. Pure, so the ordering is testable. */ @@ -68,6 +76,22 @@ export function capabilitiesFrom(flags: CapabilityFlags): string[] { // reports a skipped check for whatever is not. A machine that cannot answer // any of its questions still answers "I could not", which is the whole point. caps.push("self-check"); + // Same bare-plus-specific shape as build, for the same reason: a job spec has + // nowhere to put an output format that the collector's capabilityMatches + // would read, so `convert:gguf` alone would be a machine that can convert and + // never claims a conversion. + if (flags.converters && flags.converters.length > 0) { + caps.push("model-convert", ...flags.converters.map((c) => `model-convert:${c}`)); + // dataset-prep needs only Node and the image tooling the converters bring + // along, so it rides on the same answer rather than probing twice. + caps.push("dataset-prep"); + } + if (flags.llamaServer) caps.push("serve"); + // shell is declared ONLY when this machine has a non-empty allowlist. That is + // the trust boundary: POST /jobs is unauthenticated by design, so a machine + // whose owner has pinned nothing must be unable to claim a shell job at all, + // rather than claiming it and refusing it afterwards. + if (flags.shellAllowlist) caps.push("shell"); return caps; } @@ -122,10 +146,16 @@ export async function probeBuildKinds(env: NodeJS.ProcessEnv = process.env): Pro } export async function probeCapabilities(env: NodeJS.ProcessEnv = process.env): Promise { - const [llamaBench, mlx, kinds] = await Promise.all([ + const [llamaBench, mlx, kinds, converters, allowlist, llamaServer] = await Promise.all([ resolveLlamaBench(env), hasMlx(env), probeBuildKinds(env), + probeConverters(env).then(convertersAvailable).catch(() => [] as string[]), + loadAllowlist(env).then((a) => a.allowed.length > 0).catch(() => false), + which("llama-server", env).then((p) => p !== null).catch(() => false), ]); - return capabilitiesFrom({ llamaBench: llamaBench !== null, mlx, ...kinds }); + return capabilitiesFrom({ + llamaBench: llamaBench !== null, mlx, ...kinds, + converters, shellAllowlist: allowlist, llamaServer, + }); } diff --git a/runner-machine/src/converters.ts b/runner-machine/src/converters.ts new file mode 100644 index 0000000..926c80d --- /dev/null +++ b/runner-machine/src/converters.ts @@ -0,0 +1,368 @@ +/** + * What "convert this checkpoint" means, per output format — decided from the + * job's params, not guessed from the file. + * + * The same shape as `buildkinds.ts`: a pure planner that turns a requested + * output into a list of commands, so the dispatch and every refusal are + * testable on a machine with none of the three toolchains installed — which is + * exactly the machine where the refusals matter, since a machine that HAS + * coremltools will never exercise the "coremltools is not installed" path. + * + * The refusals are the interesting half. A conversion that quietly produces + * the wrong thing is worse than one that will not start: an unquantised GGUF + * uploaded under a `Q4_K_M` name is four times the size the schedule budgeted + * for and silently changes what every dependent benchmark measured. + */ +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { access } from "node:fs/promises"; +import { which, run } from "./probe.js"; + +export const CONVERT_FORMATS = ["gguf", "coreml", "tflite"] as const; +export type ConvertFormat = (typeof CONVERT_FORMATS)[number]; + +export function isConvertFormat(v: unknown): v is ConvertFormat { + return typeof v === "string" && (CONVERT_FORMATS as readonly string[]).includes(v); +} + +/** + * A quant name reaches a command line, so it is constrained to the alphabet + * llama-quantize actually uses. `execFile` means there is no shell to inject + * into, but an argument like `--help` arriving where a type name belongs would + * still produce a "successful" run that converted nothing. + */ +export const QUANT_RE = /^[A-Za-z0-9_]{1,20}$/; + +export type OutputSpec = { format: ConvertFormat; quant: string | null }; + +/** + * `params.outputs` as the job wrote it. + * + * Throws with the reason rather than skipping a malformed entry: a job that + * asked for three formats and got two, silently, is a job whose result row + * lies about what the fleet has. + */ +export function parseOutputs(v: unknown): OutputSpec[] { + if (!Array.isArray(v) || v.length === 0) { + return errorOut("model-convert needs params.outputs: a non-empty array of { format, quant }"); + } + const specs: OutputSpec[] = []; + for (const [i, entry] of v.entries()) { + if (typeof entry !== "object" || entry === null || Array.isArray(entry)) { + return errorOut(`params.outputs[${i}] is not an object`); + } + const { format, quant } = entry as { format?: unknown; quant?: unknown }; + if (!isConvertFormat(format)) { + return errorOut( + `params.outputs[${i}].format must be one of ${CONVERT_FORMATS.join(", ")} (got ${JSON.stringify(format)})`, + ); + } + let q: string | null = null; + if (quant !== undefined && quant !== null && quant !== "") { + if (typeof quant !== "string" || !QUANT_RE.test(quant)) { + return errorOut(`params.outputs[${i}].quant is not a quantisation name: ${JSON.stringify(quant)}`); + } + q = quant; + } + // Two entries asking for the same format and quant would race for one + // output filename and upload whichever finished last, twice. + if (specs.some((s) => s.format === format && s.quant === q)) { + return errorOut(`params.outputs asks for ${format}/${q ?? "none"} twice`); + } + specs.push({ format, quant: q }); + } + return specs; +} + +const errorOut = (msg: string): never => { + throw new Error(msg); +}; + +/** + * Where the source checkpoint comes from. + * + * Two forms and no third: a 64-hex artifact sha, which the collector can hand + * over and whose content is verified on the way in, or a HuggingFace repo id. + * Anything else — a URL, a local path — is refused, because a `source` that is + * a path would let an unauthenticated `POST /jobs` name a file on this machine + * and have its bytes uploaded to the artifact store. + */ +export const HF_REPO_RE = /^[A-Za-z0-9][\w.-]*\/[\w.-]+$/; + +export type Source = + | { kind: "artifact"; sha256: string } + | { kind: "hf"; repo: string }; + +export function parseSource(v: unknown): Source { + if (typeof v !== "string" || v.trim() === "") { + throw new Error("model-convert needs params.source: an artifact sha256 or a HuggingFace repo id"); + } + const s = v.trim(); + if (/^[0-9a-fA-F]{64}$/.test(s)) return { kind: "artifact", sha256: s.toLowerCase() }; + if (HF_REPO_RE.test(s) && !s.includes("..")) return { kind: "hf", repo: s }; + throw new Error(`params.source is neither an artifact sha256 nor a HuggingFace repo id: ${s.slice(0, 80)}`); +} + +/** + * The toolchains, as resolved on this machine. Every field is a path or null, + * and null is what every refusal below reads. + */ +export type ConverterTools = { + /** llama.cpp's convert_hf_to_gguf.py. */ + ggufConvert: string | null; + /** llama-quantize, needed only for a gguf output that names a quant. */ + ggufQuantize: string | null; + /** The interpreter that runs convert_hf_to_gguf.py. */ + ggufPython: string | null; + /** An interpreter for which `import coremltools` exits 0. */ + coremlPython: string | null; + /** collector/evals/plant-id-assets/convert_coreml.py. */ + coremlConvert: string | null; + /** xcrun, for `coremlcompiler compile`. */ + xcrun: string | null; + /** An interpreter for which `import ai_edge_litert` exits 0. */ + tflitePython: string | null; + /** collector/evals/plant-id-assets/quantize_int8.py. */ + tfliteQuantize: string | null; +}; + +export const NO_TOOLS: ConverterTools = { + ggufConvert: null, ggufQuantize: null, ggufPython: null, + coremlPython: null, coremlConvert: null, xcrun: null, + tflitePython: null, tfliteQuantize: null, +}; + +/** + * The formats this machine can actually produce. + * + * `llama-quantize` is deliberately NOT required for `convert:gguf`: an f16 + * GGUF is a complete conversion and needs only the script. A job asking for a + * quantised one on a machine without the binary is refused by name at run + * time, which is the honest split — the capability says "I can make GGUFs", + * and the refusal says which quant it could not make. + */ +export function convertersAvailable(t: ConverterTools): ConvertFormat[] { + const formats: ConvertFormat[] = []; + if (t.ggufConvert && t.ggufPython) formats.push("gguf"); + if (t.coremlPython && t.coremlConvert && t.xcrun) formats.push("coreml"); + if (t.tflitePython && t.tfliteQuantize) formats.push("tflite"); + return formats; +} + +export type Step = { cmd: string; args: string[]; cwd: string; label: string }; +export type ConvertPlan = { + format: ConvertFormat; + quant: string | null; + steps: Step[]; + /** The file to upload, relative to the plan's `cwd`. */ + product: string; + /** The `format` a ModelRef would carry for this product. */ + modelFormat: "gguf" | "mlmodelc" | "tflite"; +}; + +/** + * The Core ML script's own output naming. + * + * `collector/evals/plant-id-assets/convert_coreml.py` writes + * `PlantNet300K.mlpackage` and `PlantNet300K-int8.mlpackage` with the names + * hard-coded. Reusing the script rather than rewriting it means inheriting + * that, so the constant lives here where the plan can be read against it + * instead of being a string buried in a command line. + */ +export const COREML_PRODUCT_BASE = "PlantNet300K"; + +export type PlanContext = { + /** The source: a directory for gguf, a checkpoint file for coreml/tflite. */ + source: string; + /** Where the products are written. Every step runs here. */ + outDir: string; + /** The stem for produced filenames, from the job or the source. */ + name: string; + /** tflite only: images to calibrate the quantiser on. */ + calibrationDir?: string | null; +}; + +/** + * Turns one requested output into the commands that make it. + * + * Throws, with the missing toolchain named, rather than returning a plan that + * will fail at spawn time — the caller turns the throw into a result row, and + * "coremltools is not installed on machine-x" is an answer while + * "ENOENT: python3" is a puzzle. + */ +export function planConvert(spec: OutputSpec, tools: ConverterTools, ctx: PlanContext): ConvertPlan { + if (spec.format === "gguf") return planGguf(spec, tools, ctx); + if (spec.format === "coreml") return planCoreml(spec, tools, ctx); + return planTflite(spec, tools, ctx); +} + +function planGguf(spec: OutputSpec, tools: ConverterTools, ctx: PlanContext): ConvertPlan { + if (!tools.ggufConvert || !tools.ggufPython) { + throw new Error( + "gguf: llama.cpp's convert_hf_to_gguf.py does not resolve here (set FLEET_LLAMA_CPP to the checkout, or FLEET_CONVERT_HF_TO_GGUF to the script)", + ); + } + // f16 IS the converter's own output, so asking for it is one step, not two. + const wantsQuant = spec.quant !== null && !/^f(16|32)$/i.test(spec.quant); + const f16 = path.join(ctx.outDir, `${ctx.name}-f16.gguf`); + const steps: Step[] = [ + { + cmd: tools.ggufPython, + args: [tools.ggufConvert, ctx.source, "--outfile", f16, "--outtype", "f16"], + cwd: ctx.outDir, + label: "convert_hf_to_gguf", + }, + ]; + if (!wantsQuant) { + return { format: "gguf", quant: spec.quant, steps, product: path.basename(f16), modelFormat: "gguf" }; + } + if (!tools.ggufQuantize) { + throw new Error( + `gguf/${spec.quant}: llama-quantize does not resolve here, so only an f16 GGUF can be produced (set FLEET_LLAMA_QUANTIZE)`, + ); + } + const quantised = path.join(ctx.outDir, `${ctx.name}-${spec.quant!.toLowerCase()}.gguf`); + steps.push({ + cmd: tools.ggufQuantize, + args: [f16, quantised, spec.quant!], + cwd: ctx.outDir, + label: `llama-quantize ${spec.quant}`, + }); + return { format: "gguf", quant: spec.quant, steps, product: path.basename(quantised), modelFormat: "gguf" }; +} + +function planCoreml(spec: OutputSpec, tools: ConverterTools, ctx: PlanContext): ConvertPlan { + if (!tools.coremlPython || !tools.coremlConvert) { + throw new Error( + "coreml: no interpreter here imports coremltools (set FLEET_COREML_PYTHON to one, e.g. a 3.12 venv — coremltools' BlobWriter has no 3.14 wheel)", + ); + } + if (!tools.xcrun) { + throw new Error("coreml: xcrun does not resolve, so `coremlcompiler compile` cannot run — an .mlpackage is not a loadable model"); + } + // The script writes both variants every run; the quant selects which one is + // compiled and published rather than which one is produced. + const int8 = spec.quant !== null && /^int8$/i.test(spec.quant); + if (spec.quant !== null && !int8 && !/^fp?16$/i.test(spec.quant)) { + throw new Error(`coreml/${spec.quant}: convert_coreml.py produces fp16 weights and an int8 variant, nothing else`); + } + const base = int8 ? `${COREML_PRODUCT_BASE}-int8` : COREML_PRODUCT_BASE; + return { + format: "coreml", + quant: spec.quant, + modelFormat: "mlmodelc", + product: `${base}.mlmodelc.zip`, + steps: [ + { cmd: tools.coremlPython, args: [tools.coremlConvert, ctx.source], cwd: ctx.outDir, label: "convert_coreml.py" }, + { cmd: tools.xcrun, args: ["coremlcompiler", "compile", `${base}.mlpackage`, "."], cwd: ctx.outDir, label: "coremlcompiler compile" }, + // ditto rather than zip: an .mlmodelc is a bundle, and the symlinks and + // permission bits a plain zip drops are the difference between a model + // that loads on the phone and one that does not. Same call the build + // workload makes for a .app. + { cmd: "ditto", args: ["-c", "-k", "--sequesterRsrc", "--keepParent", `${base}.mlmodelc`, `${base}.mlmodelc.zip`], cwd: ctx.outDir, label: "ditto" }, + ], + }; +} + +function planTflite(spec: OutputSpec, tools: ConverterTools, ctx: PlanContext): ConvertPlan { + if (!tools.tflitePython || !tools.tfliteQuantize) { + throw new Error( + "tflite: no interpreter here imports ai_edge_litert (set FLEET_TFLITE_PYTHON to one — `pip install ai-edge-litert Pillow numpy`)", + ); + } + // The LiteRT path this repo owns is post-training quantisation of an + // existing .tflite. Copying a float model to a new name and calling it a + // conversion would publish an artifact whose only change is its filename. + if (spec.quant === null || !/^int8$/i.test(spec.quant)) { + throw new Error( + `tflite/${spec.quant ?? "none"}: the LiteRT converter here is quantize_int8.py, so quant must be int8`, + ); + } + if (!ctx.calibrationDir) { + throw new Error( + "tflite/int8: needs params.calibration_sha256, a zip of ~100 calibration images DISJOINT from the eval set", + ); + } + const out = `${ctx.name}-int8.tflite`; + return { + format: "tflite", + quant: spec.quant, + modelFormat: "tflite", + product: out, + steps: [ + { + cmd: tools.tflitePython, + args: [tools.tfliteQuantize, ctx.source, ctx.calibrationDir, path.join(ctx.outDir, out)], + cwd: ctx.outDir, + label: "quantize_int8.py", + }, + ], + }; +} + +// --- probes ----------------------------------------------------------------- + +/** + * Where the plant-ID conversion scripts are. + * + * They live in the collector's tree, which this agent does not import code + * from — but a python script invoked by path is not an import, it is a tool, + * and rewriting coremltools plumbing that already exists and is known to work + * would be the worse call. `FLEET_EVAL_ASSETS` names the directory when the + * two repos are not checked out side by side. + */ +export function evalAssetsDir(env: NodeJS.ProcessEnv = process.env): string { + if (env.FLEET_EVAL_ASSETS) return env.FLEET_EVAL_ASSETS; + // runner-machine/src/converters.ts -> ../../collector/evals/plant-id-assets. + // fileURLToPath, not `.pathname`: this repo's own checkout sits under a + // directory with a space in it, and the URL form spells that `%20`. + return fileURLToPath(new URL("../../collector/evals/plant-id-assets", import.meta.url)); +} + +/** An interpreter for which `import ` exits 0, or null. */ +export async function pythonThatImports( + mod: string, + candidates: (string | undefined)[], + env: NodeJS.ProcessEnv = process.env, +): Promise { + for (const candidate of candidates) { + if (!candidate) continue; + const resolved = await which(candidate, env); + if (!resolved) continue; + const r = await run(resolved, ["-c", `import ${mod}`], 60_000); + if (r.code === 0) return resolved; + } + return null; +} + +async function fileIfReadable(file: string | undefined | null): Promise { + if (!file) return null; + try { + await access(file); + return file; + } catch { + return null; + } +} + +/** Asks the machine each question the planner will ask of the answers. */ +export async function probeConverters(env: NodeJS.ProcessEnv = process.env): Promise { + const assets = evalAssetsDir(env); + const ggufScript = + env.FLEET_CONVERT_HF_TO_GGUF ?? + (env.FLEET_LLAMA_CPP ? path.join(env.FLEET_LLAMA_CPP, "convert_hf_to_gguf.py") : undefined); + + const [ggufConvert, ggufQuantize, ggufPython, coremlPython, coremlConvert, xcrun, tflitePython, tfliteQuantize] = + await Promise.all([ + fileIfReadable(ggufScript), + which(env.FLEET_LLAMA_QUANTIZE ?? "llama-quantize", env), + which(env.FLEET_PYTHON ?? "python3", env), + pythonThatImports("coremltools", [env.FLEET_COREML_PYTHON, env.FLEET_PYTHON, "python3.12", "python3"], env), + fileIfReadable(path.join(assets, "convert_coreml.py")), + which("xcrun", env), + pythonThatImports("ai_edge_litert", [env.FLEET_TFLITE_PYTHON, env.FLEET_PYTHON, "python3"], env), + fileIfReadable(path.join(assets, "quantize_int8.py")), + ]); + + return { ggufConvert, ggufQuantize, ggufPython, coremlPython, coremlConvert, xcrun, tflitePython, tfliteQuantize }; +} diff --git a/runner-machine/src/images.ts b/runner-machine/src/images.ts new file mode 100644 index 0000000..ac63d9e --- /dev/null +++ b/runner-machine/src/images.ts @@ -0,0 +1,104 @@ +/** + * Resizing an eval image to the shape the on-device models expect. + * + * The eval sets this fleet ships are "center-crop, resize 224, RGB" — that + * string is in the plant-ID manifest and both runners' preprocessing assumes + * it. Getting it wrong is not a crash: it is a model that scores four points + * lower for a reason nobody can see, because a letterboxed image and a + * center-cropped one are both 224×224 JPEGs. + * + * So the arithmetic is here, pure and tested, and the two resizers this agent + * knows about are only ever handed the numbers it produces. + */ +import { which } from "./probe.js"; + +export type Resizer = { kind: "sips" | "magick"; bin: string }; + +/** + * The intermediate size a center-crop needs: scale so the SHORT edge is + * `size`, then take the middle `size`×`size`. + * + * Scaling the LONG edge to `size` instead — which is what `sips -Z` does on + * its own, and the obvious one-liner — leaves the short edge under `size` and + * forces a pad. A padded image is a smaller subject surrounded by background + * the model has to look past, which is a different picture from the one the + * eval set was scored on. + */ +export function resizeToShortEdge(w: number, h: number, size: number): { width: number; height: number } { + if (!Number.isFinite(w) || !Number.isFinite(h) || w <= 0 || h <= 0) { + throw new Error(`cannot resize an image reported as ${w}×${h}`); + } + const scale = size / Math.min(w, h); + return { + // Ceil, not round: a rounded-down edge lands one pixel short of `size` and + // the crop that follows then has to pad the very row it was avoiding. + width: Math.max(size, Math.ceil(w * scale)), + height: Math.max(size, Math.ceil(h * scale)), + }; +} + +/** `sips -g pixelWidth -g pixelHeight` output to numbers. */ +export function parseSipsDims(text: string | null): { w: number; h: number } | null { + if (!text) return null; + const w = /pixelWidth:\s*(\d+)/.exec(text)?.[1]; + const h = /pixelHeight:\s*(\d+)/.exec(text)?.[1]; + if (!w || !h) return null; + const dims = { w: Number(w), h: Number(h) }; + return dims.w > 0 && dims.h > 0 ? dims : null; +} + +export function sipsDimArgs(src: string): string[] { + return ["-g", "pixelWidth", "-g", "pixelHeight", src]; +} + +/** + * sips: resample to the short-edge size, then crop the middle, in one call. + * + * `-c h w` on sips is a CENTERED crop, which is the whole reason the resample + * has to come first — sips has no gravity flag, so the only way to choose what + * survives the crop is to control what goes into it. + */ +export function sipsResizeArgs( + src: string, + dst: string, + size: number, + dims: { w: number; h: number }, +): string[] { + const fit = resizeToShortEdge(dims.w, dims.h, size); + return [ + "-s", "format", "jpeg", + "-s", "formatOptions", "high", + "--resampleHeightWidth", String(fit.height), String(fit.width), + "-c", String(size), String(size), + src, "--out", dst, + ]; +} + +/** + * ImageMagick: the same two operations, and it needs no dimension probe + * because `^` means "fit the SHORT edge" and `-extent` with a center gravity + * is the crop. + */ +export function magickResizeArgs(src: string, dst: string, size: number): string[] { + return [src, "-resize", `${size}x${size}^`, "-gravity", "center", "-extent", `${size}x${size}`, "-quality", "90", dst]; +} + +/** + * The resizer this machine has, or null. + * + * sips first because it is on every Mac and needs nothing installed; magick + * because a Linux box that has it can prepare a set the same way. A machine + * with neither must not declare `dataset-prep` — a "prepared" set whose images + * are still 4000 px wide is one the phone runners will resize themselves, + * inconsistently and at eval time, which is exactly the variable this workload + * exists to remove. + */ +export async function resolveResizer(env: NodeJS.ProcessEnv = process.env): Promise { + const sips = await which("sips", env); + if (sips) return { kind: "sips", bin: sips }; + for (const name of ["magick", "convert"]) { + const bin = await which(name, env); + if (bin) return { kind: "magick", bin }; + } + return null; +} diff --git a/runner-machine/src/protocol.ts b/runner-machine/src/protocol.ts index 19b32b4..7671729 100644 --- a/runner-machine/src/protocol.ts +++ b/runner-machine/src/protocol.ts @@ -109,6 +109,18 @@ export type Metrics = { clock_offset_ms?: number; /** self-check: how many checks failed, so the alert rule needs no parsing. */ checks_failed?: number; + + /** model-convert: seconds for one output format. */ + convert_s?: number; + /** model-convert: how many artifacts the conversion produced. */ + outputs?: number; + /** dataset-prep: items in the prepared manifest, after dedupe. */ + items?: number; + /** shell: the script's exit status. Named rather than folded into `ok`, + * because a script that exits 3 deliberately is not one that could not start. */ + exit_code?: number; + /** serve and any long run: seconds since the job started. */ + elapsed_s?: number; }; /** @@ -138,6 +150,13 @@ export type BuildRow = { }; export type ResultPost = { + /** serve: where the hosted model can be reached. Declared in the collector's + * result schema; see there for why it is never bound to 0.0.0.0. */ + endpoint?: string; + /** model-convert: one produced artifact — format, quant, name, sha256, bytes. */ + model_out?: Record; + /** dataset-prep: the prepared manifest's summary, licence and source included. */ + dataset?: Record; schema: number; kind: "result" | "beacon"; job_id?: string; diff --git a/runner-machine/src/recipes.ts b/runner-machine/src/recipes.ts new file mode 100644 index 0000000..40b3072 --- /dev/null +++ b/runner-machine/src/recipes.ts @@ -0,0 +1,197 @@ +/** + * Recipes: how a downloaded pile of images becomes a labelled eval set. + * + * A recipe is a small ES module under `recipes/`, named by the job. Keeping it + * out of this file is the point — a new dataset is a new file next to the + * others, not a branch in a workload — but it also means the recipe NAME + * arrives from an unauthenticated `POST /jobs`, so `recipePath` is a security + * boundary and not a convenience: `../../etc/something` must never resolve. + * + * The manifest is the other half. Its shape is not this file's invention: the + * Android runner's VisionEvalEngine reads `manifest.json` from the zip root + * and walks `items[].file` / `items[].label`, so those two keys are a contract + * with code in another repo and are asserted in the tests rather than trusted. + * + * `license` and `source` are new here, and they are the reason a prepared set + * is publishable. The plant dataset is CC-BY-SA — which is exactly why its + * images ship as artifacts rather than as commits — and a zip that has lost + * the attribution is a zip whose results can never appear in a write-up. So a + * set with no licence is refused at prep time, where somebody can still fix + * it, rather than discovered at publication time when the provenance is gone. + */ +import path from "node:path"; +import { pathToFileURL, fileURLToPath } from "node:url"; + +/** A recipe name is a filename, never a path. */ +export const RECIPE_NAME_RE = /^[a-z0-9][a-z0-9._-]{0,63}$/; + +export type LabelledItem = { label: number; [k: string]: unknown }; +export type Labelling = { items: LabelledItem[]; classes?: number }; + +export type Recipe = { + id: string; + /** + * The dataset's licence, when the recipe is for one specific dataset. Null + * for a generic recipe, which means the JOB has to say — and a job that does + * not is refused. + */ + license: string | null; + /** Square edge, in pixels, of every image in the prepared set. */ + size: number; + /** Copied into the manifest so the runners' preprocessing is documented beside the data. */ + preprocess?: string; + /** + * Labels for the whole file list at once, in the same order. + * + * Whole-list rather than per-file because a label is usually an INDEX into + * the set of classes present, which no single file can know. Returning fewer + * items than it was given is a refusal the caller reports. + */ + labels(files: string[]): Labelling; +}; + +/** Where the recipes live. Overridable so the tests do not need the real ones. */ +export function recipesDir(env: NodeJS.ProcessEnv = process.env): string { + if (env.FLEET_RECIPES_DIR) return env.FLEET_RECIPES_DIR; + // runner-machine/src/recipes.ts -> runner-machine/recipes. fileURLToPath + // rather than `.pathname`, because this checkout's path has a space in it. + return fileURLToPath(new URL("../recipes", import.meta.url)); +} + +/** + * The file a recipe name refers to. + * + * Throws for anything that is not a plain lowercase name. The name comes from + * a job spec and the collector authenticates nobody, so `../../../..` reaching + * `import()` would be arbitrary code execution with no allowlist in front of + * it — which is the thing the `shell` workload has a whole allowlist to avoid. + */ +export function recipePath(name: unknown, dir: string): string { + if (typeof name !== "string" || !RECIPE_NAME_RE.test(name)) { + throw new Error( + `params.recipe must be a recipe name like 'plant-id' (lowercase, no path separators); got ${JSON.stringify(name)}`, + ); + } + const file = path.join(dir, `${name}.mjs`); + // Belt and braces: the regex already forbids separators, so a resolved path + // that escaped the directory would mean the regex was weakened later. + if (path.dirname(path.resolve(file)) !== path.resolve(dir)) { + throw new Error(`recipe '${name}' does not resolve inside ${dir}`); + } + return file; +} + +/** A loaded module is only a recipe if it answers every question the prep asks. */ +export function asRecipe(mod: unknown, name: string): Recipe { + const r = (mod && typeof mod === "object" && "default" in mod ? (mod as { default: unknown }).default : mod) as + | Partial + | undefined; + if (!r || typeof r !== "object") throw new Error(`recipe '${name}' exports no default object`); + if (typeof r.labels !== "function") throw new Error(`recipe '${name}' has no labels(files) function`); + if (typeof r.size !== "number" || !Number.isInteger(r.size) || r.size < 16 || r.size > 4096) { + throw new Error(`recipe '${name}' has no usable size (got ${JSON.stringify(r.size)})`); + } + if (r.license !== null && typeof r.license !== "string") { + throw new Error(`recipe '${name}' must state a license string, or null to require one from the job`); + } + return { + id: typeof r.id === "string" && r.id !== "" ? r.id : name, + license: r.license, + size: r.size, + preprocess: typeof r.preprocess === "string" ? r.preprocess : undefined, + labels: r.labels.bind(r), + }; +} + +export async function loadRecipe(name: unknown, env: NodeJS.ProcessEnv = process.env): Promise { + const dir = recipesDir(env); + const file = recipePath(name, dir); + let mod: unknown; + try { + mod = await import(pathToFileURL(file).href); + } catch (e) { + throw new Error(`recipe '${String(name)}' does not load from ${dir}: ${(e as Error).message}`); + } + return asRecipe(mod, String(name)); +} + +/** + * The licence the prepared set will carry. + * + * The job may override a generic recipe's null; it may NOT quietly relabel a + * recipe that already knows its dataset's terms, because the recipe is the + * thing somebody checked. An empty answer is a refusal, not a blank field. + */ +export function effectiveLicense(recipe: Recipe, fromJob: unknown): string { + const job = typeof fromJob === "string" ? fromJob.trim() : ""; + if (recipe.license !== null && recipe.license.trim() !== "") { + if (job !== "" && job !== recipe.license) { + throw new Error( + `recipe '${recipe.id}' states its dataset is ${recipe.license}; params.license says ${job}. A prep cannot relicense a dataset.`, + ); + } + return recipe.license; + } + if (job === "") { + throw new Error( + `recipe '${recipe.id}' is generic and states no license, so params.license is required — a prepared set with no licence is one nobody can publish results from`, + ); + } + return job; +} + +export type Manifest = { + recipe: string; + /** Where the images came from, so the set is traceable without this job's row. */ + source: string; + license: string; + classes?: number; + preprocess?: string; + items: LabelledItem[]; +}; + +/** + * The manifest that goes in the zip. + * + * `items[].file` is the name inside the zip and `items[].label` is an integer, + * because that is what VisionEvalEngine reads; everything the recipe attached + * to an item rides along untouched, the way the plant-ID manifest carries + * `species` and `species_id` beside the label. + */ +export function buildManifest(opts: { + recipe: Recipe; + source: string; + license: string; + names: string[]; + labelling: Labelling; +}): Manifest { + const { recipe, names, labelling } = opts; + if (labelling.items.length !== names.length) { + throw new Error( + `recipe '${recipe.id}' labelled ${labelling.items.length} of ${names.length} images; the manifest must describe every file in the zip`, + ); + } + const items = names.map((file, i) => { + const item = labelling.items[i]!; + if (!Number.isInteger(item.label)) { + throw new Error(`recipe '${recipe.id}' gave ${file} a non-integer label (${JSON.stringify(item.label)})`); + } + // `file` last so a recipe cannot overwrite the name of the file it is + // describing — the zip's layout is the prep's to decide, not the recipe's. + return { ...item, label: item.label, file }; + }); + return { + recipe: recipe.id, + source: opts.source, + license: opts.license, + ...(labelling.classes !== undefined ? { classes: labelling.classes } : {}), + ...(recipe.preprocess ? { preprocess: recipe.preprocess } : {}), + items, + }; +} + +/** `000.jpg`, `001.jpg`, … — flat, zero-padded, and in the order they were kept. */ +export function outputName(index: number, total: number): string { + const width = Math.max(3, String(Math.max(total - 1, 0)).length); + return `${String(index).padStart(width, "0")}.jpg`; +} diff --git a/runner-machine/src/redact.ts b/runner-machine/src/redact.ts new file mode 100644 index 0000000..ecc20ce --- /dev/null +++ b/runner-machine/src/redact.ts @@ -0,0 +1,52 @@ +/** + * Scrubbing secrets out of text that is about to become an artifact. + * + * The same rule as the collector's `redact` in collector/src/secrets.ts, and + * deliberately a re-implementation rather than an import — this agent depends + * on the protocol, not on the collector's source tree, exactly as + * `protocol.ts` and `collector.ts` do. + * + * The `shell` workload is why this exists here. A script from the artifact + * store runs with a minimal environment, but "minimal" is a list this file + * maintains and a script can still print something it was handed; the captured + * output is uploaded to a store anyone on the dashboard can download, so a + * token that reaches it has leaked no matter how carefully it was fetched. + */ + +/** + * Environment variable names whose VALUES must never reach an artifact. + * + * Matched on the name, because the value of a token is by definition + * unrecognisable. `_KEY` rather than `KEY` so that `KEYBOARD_LAYOUT` and + * friends do not turn a common word into a redaction pattern. + */ +export const SECRET_ENV_RE = /(^|_)(TOKEN|SECRET|PASSWORD|PASSWD|CREDENTIALS?|API_?KEY)(_|$)|_KEY$/i; + +/** + * Remove secret values from text bound for the artifact store. + * + * Values shorter than four characters are skipped: a two-character "secret" + * matches half of any log, and a log that is mostly `[redacted]` is a log + * nobody can read — which is its own failure, since the log is the answer to + * "why did this script fail". + */ +export function redact(text: string, secrets: readonly string[]): string { + let out = text; + for (const s of secrets) { + if (!s || s.length < 4) continue; + out = out.split(s).join("[redacted]"); + } + return out; +} + +/** The values in this process's environment that must not be published. */ +export function secretsFromEnv(env: NodeJS.ProcessEnv = process.env): string[] { + const values: string[] = []; + for (const [name, value] of Object.entries(env)) { + if (typeof value !== "string" || value.length < 4) continue; + if (SECRET_ENV_RE.test(name)) values.push(value); + } + // Longest first, so a token that contains a shorter one is not left with a + // recognisable prefix after the shorter match has already been replaced. + return values.sort((a, b) => b.length - a.length); +} diff --git a/runner-machine/src/workloads/datasetprep.ts b/runner-machine/src/workloads/datasetprep.ts new file mode 100644 index 0000000..bddaaa7 --- /dev/null +++ b/runner-machine/src/workloads/datasetprep.ts @@ -0,0 +1,289 @@ +/** + * The `dataset-prep` workload: download an image set, resize it, drop the + * duplicates, write a manifest, and publish the zip the eval workloads read. + * + * The output is not this file's invention. The Android runner's + * VisionEvalEngine opens `manifest.json` at the zip root and walks + * `items[].file` / `items[].label`, so that layout is a contract with code in + * another repo — which is why the shape is built by `recipes.ts` and asserted + * in the tests instead of being assembled inline here. + * + * Two things this adds beyond "unzip and hand it over": + * + * 1. **The licence travels with the data.** The plant set is CC-BY-SA, which + * is exactly why its images move through the artifact store rather than + * being committed. A prepared zip whose manifest has lost the attribution + * is a zip whose eval numbers can never be published, and by the time + * anyone notices, the provenance is a URL somebody half-remembers. So a + * prep with no licence is refused before a single image is resized. + * 2. **The download is content-addressed on `source_url`.** Re-preparing the + * same set with a different recipe, or after a crash, must not pull + * gigabytes over somebody's home internet again. + */ +import { createHash } from "node:crypto"; +import { existsSync } from "node:fs"; +import { mkdir, readFile, readdir, rename, rm, stat, writeFile } from "node:fs/promises"; +import path from "node:path"; +import type { CollectorClient } from "../collector.js"; +import type { Descriptor, JobSpec, Metrics } from "../protocol.js"; +import { SCHEMA, compact, intParam, stringParam } from "../protocol.js"; +import { dataDir } from "../git.js"; +import { run, which } from "../probe.js"; +import { Beaconer, lastMeaningful } from "../beaconing.js"; +import { + buildManifest, effectiveLicense, loadRecipe, outputName, type Recipe, +} from "../recipes.js"; +import { + magickResizeArgs, parseSipsDims, resolveResizer, sipsDimArgs, sipsResizeArgs, type Resizer, +} from "../images.js"; + +export const IMAGE_EXTS = [".jpg", ".jpeg", ".png", ".webp"]; +/** A download that has not finished in two hours is a download that will not. */ +const DOWNLOAD_TIMEOUT_MS = 2 * 60 * 60 * 1000; + +/** The cache key. The URL and nothing else: the same URL is the same set. */ +export function cacheKey(sourceUrl: string): string { + return createHash("sha256").update(sourceUrl).digest("hex").slice(0, 32); +} + +/** http(s) only. A `file:` source would let an unauthenticated job read this disk. */ +export function parseSourceUrl(v: unknown): string { + if (typeof v !== "string" || v.trim() === "") throw new Error("dataset-prep needs params.source_url"); + const s = v.trim(); + let u: URL; + try { + u = new URL(s); + } catch { + throw new Error(`params.source_url is not a URL: ${s.slice(0, 120)}`); + } + if (u.protocol !== "http:" && u.protocol !== "https:") { + throw new Error(`params.source_url must be http or https (got ${u.protocol})`); + } + return s; +} + +export async function runDatasetPrep( + job: JobSpec, + client: CollectorClient, + deviceId: string, + descriptor: Descriptor, + env: NodeJS.ProcessEnv = process.env, +): Promise { + const log = (m: string) => console.log(`[${deviceId}] dataset-prep ${job.job_id}: ${m}`); + const fail = async (error: string, metrics?: Metrics) => { + log(`failed: ${error}`); + await client.postResult({ + schema: SCHEMA, kind: "result", job_id: job.job_id, device_id: deviceId, + iter: 0, final: true, ok: false, device: descriptor, error, + ...(metrics ? { metrics } : {}), + }); + }; + + // Everything that can be refused without touching the network, first. + let sourceUrl: string; + let recipe: Recipe; + let license: string; + try { + sourceUrl = parseSourceUrl(job.params?.source_url); + recipe = await loadRecipe(job.params?.recipe, env); + license = effectiveLicense(recipe, job.params?.license); + } catch (e) { + return fail((e as Error).message); + } + + const resizer = await resolveResizer(env); + if (!resizer) return fail("dataset-prep needs sips (macOS) or ImageMagick; neither resolves on this machine"); + const zipper = await which("zip", env); + if (!zipper) return fail("dataset-prep needs `zip` to publish the prepared set"); + + const beaconer = new Beaconer(client, deviceId, job.job_id); + const key = cacheKey(sourceUrl); + const cacheDir = path.join(dataDir(env), "datasets", key); + const staging = path.join(cacheDir, "prepared", safe(job.job_id)); + await rm(staging, { recursive: true, force: true }).catch(() => {}); + await mkdir(staging, { recursive: true }); + + try { + // --- download, content-addressed on the URL --------------------------- + let extracted: string; + try { + extracted = await fetchAndExtract(sourceUrl, cacheDir, log); + } catch (e) { + return fail(`fetching ${sourceUrl} failed: ${(e as Error).message}`); + } + + // --- the images, in a stable order ------------------------------------ + const limit = intParam(job.params, "max_items", 0); + const all = (await walkImages(extracted)).sort(); + if (all.length === 0) return fail(`${sourceUrl} contained no images (looked for ${IMAGE_EXTS.join(", ")})`); + const chosen = limit > 0 ? all.slice(0, limit) : all; + log(`${all.length} image(s) found, preparing ${chosen.length} at ${recipe.size}px`); + + // --- resize, then deduplicate on the PREPARED bytes ------------------- + // + // Deduplicating after the resize rather than before is the whole point: a + // scraped set is full of the same photograph at three resolutions and two + // JPEG qualities, which are different files and identical evidence. + // Scoring a model twice on one image is how an eval set quietly weights + // whatever was duplicated most. + const seen = new Map(); + const kept: string[] = []; + const keptNames: string[] = []; + let duplicates = 0; + let failedImages = 0; + + for (const rel of chosen) { + if (!(await beaconer.maybe())) { + return fail("cancelled"); + } + const src = path.join(extracted, rel); + const tmp = path.join(staging, `.staging-${kept.length}.jpg`); + const ok = await resizeOne(resizer, src, tmp, recipe.size); + if (!ok) { + failedImages += 1; + continue; + } + const digest = await sha256File(tmp); + const first = seen.get(digest); + if (first !== undefined) { + duplicates += 1; + await rm(tmp, { force: true }).catch(() => {}); + continue; + } + seen.set(digest, rel); + kept.push(rel); + keptNames.push(tmp); + } + if (kept.length === 0) { + return fail(`every one of the ${chosen.length} image(s) failed to resize; is ${resizer.bin} working?`); + } + + // Names are assigned only once the final count is known, so the padding + // width is right and the numbering has no gaps where a duplicate was. + const names = kept.map((_, i) => outputName(i, kept.length)); + for (const [i, tmp] of keptNames.entries()) { + await rename(tmp, path.join(staging, names[i]!)); + } + + // --- manifest --------------------------------------------------------- + let manifestJson: string; + try { + const labelling = recipe.labels(kept.map(toPosix)); + const manifest = buildManifest({ recipe, source: sourceUrl, license, names, labelling }); + manifestJson = JSON.stringify(manifest, null, 1); + } catch (e) { + return fail(`recipe '${recipe.id}' could not label the set: ${(e as Error).message}`); + } + await writeFile(path.join(staging, "manifest.json"), manifestJson, "utf8"); + + // --- zip and publish -------------------------------------------------- + const zipFile = path.join(cacheDir, `${safe(job.job_id)}.zip`); + await rm(zipFile, { force: true }).catch(() => {}); + // `-X` drops the extra attributes that make an otherwise identical zip + // differ byte for byte; `.` from inside the staging dir puts manifest.json + // at the zip ROOT, which is where VisionEvalEngine looks for it. + const zipped = await run(zipper, ["-q", "-r", "-X", zipFile, "."], 60 * 60 * 1000); + if (zipped.code !== 0) { + return fail(`zipping the prepared set failed: ${lastMeaningful(zipped.stderr || zipped.stdout) || `exit ${zipped.code}`}`, undefined); + } + + const bytes = (await stat(zipFile)).size; + let sha256: string; + try { + const up = await client.uploadArtifact(zipFile, `${recipe.id}-${kept.length}.zip`); + sha256 = up.sha256; + } catch (e) { + return fail(`upload failed: ${(e as Error).message}`, compact({ items: kept.length, artifact_bytes: bytes })); + } + + log(`published ${kept.length} item(s) as ${sha256} (${bytes} bytes; ${duplicates} duplicate(s), ${failedImages} unreadable)`); + await client.postResult({ + schema: SCHEMA, kind: "result", job_id: job.job_id, device_id: deviceId, + iter: 0, final: true, ok: true, device: descriptor, + metrics: compact({ items: kept.length, artifact_bytes: bytes }), + artifacts: [sha256], + // Not in the result schema, deliberately, exactly as build's `build` + // block: the collector stores a result's whole payload, and a prepared + // set whose row cannot say what licence it carries is a row that puts + // the attribution back where it was — nowhere anyone will look. + dataset: { + recipe: recipe.id, source: sourceUrl, license, sha256, + items: kept.length, duplicates, unreadable: failedImages, size: recipe.size, + }, + }); + } finally { + await rm(staging, { recursive: true, force: true }).catch(() => {}); + } +} + +/** + * The extracted archive for a URL, downloading it only if the cache is cold. + * + * The blob is written to a `.part` and renamed, so an interrupted download can + * never be mistaken for a complete one on the next run — the same discipline + * `fetchArtifact` uses for its hash check. + */ +async function fetchAndExtract(sourceUrl: string, cacheDir: string, log: (m: string) => void): Promise { + const extracted = path.join(cacheDir, "extracted"); + if (existsSync(path.join(cacheDir, ".complete"))) { + log("cache hit; not downloading"); + return extracted; + } + await mkdir(cacheDir, { recursive: true }); + const blob = path.join(cacheDir, "source.bin"); + if (!existsSync(blob)) { + log(`downloading ${sourceUrl}`); + const part = `${blob}.part`; + const r = await run("curl", ["-fsSL", "--retry", "3", "-o", part, sourceUrl], DOWNLOAD_TIMEOUT_MS); + if (r.code !== 0) { + await rm(part, { force: true }).catch(() => {}); + throw new Error(lastMeaningful(r.stderr || r.stdout) || `curl exited ${r.code}`); + } + await rename(part, blob); + } + + await rm(extracted, { recursive: true, force: true }).catch(() => {}); + await mkdir(extracted, { recursive: true }); + const isTar = /\.(tar\.gz|tgz|tar)$/i.test(new URL(sourceUrl).pathname); + const r = isTar + ? await run("tar", ["-xf", blob, "-C", extracted], DOWNLOAD_TIMEOUT_MS) + : await run("unzip", ["-q", "-o", blob, "-d", extracted], DOWNLOAD_TIMEOUT_MS); + if (r.code !== 0) throw new Error(`extracting failed: ${lastMeaningful(r.stderr || r.stdout) || `exit ${r.code}`}`); + await writeFile(path.join(cacheDir, ".complete"), sourceUrl, "utf8"); + return extracted; +} + +/** Every image under a directory, as paths relative to it. */ +export async function walkImages(root: string, rel = "", depth = 0): Promise { + if (depth > 8) return []; + const entries = await readdir(path.join(root, rel), { withFileTypes: true }).catch(() => []); + const out: string[] = []; + for (const e of entries) { + if (e.name.startsWith(".") || e.name === "__MACOSX") continue; + const child = rel === "" ? e.name : path.join(rel, e.name); + if (e.isDirectory()) out.push(...(await walkImages(root, child, depth + 1))); + else if (IMAGE_EXTS.some((x) => e.name.toLowerCase().endsWith(x))) out.push(child); + } + return out; +} + +/** One image, center-cropped to a square. False when the tool could not read it. */ +async function resizeOne(resizer: Resizer, src: string, dst: string, size: number): Promise { + if (resizer.kind === "magick") { + const r = await run(resizer.bin, magickResizeArgs(src, dst, size), 120_000); + return r.code === 0 && existsSync(dst); + } + // sips has no gravity flag, so the crop can only be centred by controlling + // what goes into it — which needs the source's dimensions first. + const dims = parseSipsDims((await run(resizer.bin, sipsDimArgs(src), 60_000)).stdout); + if (!dims) return false; + const r = await run(resizer.bin, sipsResizeArgs(src, dst, size, dims), 120_000); + return r.code === 0 && existsSync(dst); +} + +async function sha256File(file: string): Promise { + return createHash("sha256").update(await readFile(file)).digest("hex"); +} + +const toPosix = (p: string) => p.split(path.sep).join("/"); +const safe = (s: string) => s.replace(/[^A-Za-z0-9._-]+/g, "-").slice(0, 60); diff --git a/runner-machine/src/workloads/modelconvert.ts b/runner-machine/src/workloads/modelconvert.ts new file mode 100644 index 0000000..0b32440 --- /dev/null +++ b/runner-machine/src/workloads/modelconvert.ts @@ -0,0 +1,294 @@ +/** + * The `model-convert` workload: turn one source checkpoint into the formats + * the fleet's three runners can actually load, on the machine that has the + * memory to do it. + * + * The reason this is a fleet workload rather than a script somebody runs is + * the machine, not the code: quantising a multi-gigabyte checkpoint needs more + * RAM than a phone has and more patience than a laptop on battery has, and the + * collector already knows which machine is on mains and idle. What this + * workload adds is that the OUTPUT is addressable — every product is uploaded + * as an artifact and the final row lists every sha, so a dependent job can say + * `${jobs..artifact}` and get the model this conversion made rather than + * whatever was in someone's Downloads folder. + * + * The conversions themselves are other people's tools, deliberately: llama.cpp + * for GGUF, and the plant-ID assets' own `convert_coreml.py` and + * `quantize_int8.py` for Core ML and LiteRT. Those two scripts already work + * and are already the provenance of the models this fleet has benchmarked; + * rewriting their coremltools plumbing here would produce a second answer to a + * question that already has one. + */ +import { mkdir, open, rm, stat, readdir } from "node:fs/promises"; +import { existsSync } from "node:fs"; +import path from "node:path"; +import type { CollectorClient } from "../collector.js"; +import type { Descriptor, JobSpec, Metrics } from "../protocol.js"; +import { SCHEMA, compact, intParam, stringParam } from "../protocol.js"; +import { fileMatchesHash } from "../collector.js"; +import { dataDir } from "../git.js"; +import { run, which } from "../probe.js"; +import { runWithBeacons, lastMeaningful } from "../beaconing.js"; +import { + parseOutputs, parseSource, planConvert, probeConverters, convertersAvailable, + type ConvertPlan, type ConverterTools, type Source, +} from "../converters.js"; + +/** A conversion of a 70B checkpoint is hours; a hung one should still end. */ +const DEFAULT_STEP_TIMEOUT_S = 6 * 60 * 60; + +export async function runModelConvert( + job: JobSpec, + client: CollectorClient, + deviceId: string, + descriptor: Descriptor, + env: NodeJS.ProcessEnv = process.env, +): Promise { + const log = (m: string) => console.log(`[${deviceId}] model-convert ${job.job_id}: ${m}`); + const fail = async (error: string, artifacts?: string[], metrics?: Metrics) => { + log(`failed: ${error}`); + await client.postResult({ + schema: SCHEMA, kind: "result", job_id: job.job_id, device_id: deviceId, + iter: 0, final: true, ok: false, device: descriptor, error, + ...(artifacts?.length ? { artifacts } : {}), + ...(metrics ? { metrics } : {}), + }); + }; + + let source: Source; + let outputs: ReturnType; + try { + source = parseSource(job.params?.source); + outputs = parseOutputs(job.params?.outputs); + } catch (e) { + return fail((e as Error).message); + } + + const tools = await probeConverters(env); + const available = convertersAvailable(tools); + const missing = outputs.filter((o) => !available.includes(o.format)).map((o) => o.format); + if (missing.length === outputs.length) { + // Every requested format is unbuildable here. Refused as one row rather + // than N, and naming what this machine CAN do, because the answer the + // operator needs is which machine to route the job to. + return fail( + `none of the requested formats can be converted on this machine (asked for ${[...new Set(missing)].join(", ")}; ` + + `available here: ${available.length ? available.join(", ") : "none"})`, + ); + } + + const workRoot = path.join(dataDir(env), "convert", safe(job.job_id)); + await rm(workRoot, { recursive: true, force: true }).catch(() => {}); + await mkdir(workRoot, { recursive: true }); + + const t0 = Date.now(); + const artifacts: string[] = []; + let totalBytes = 0; + try { + let sourcePath: string; + try { + sourcePath = await materialiseSource(source, client, env, log); + } catch (e) { + return fail(`fetching the source failed: ${(e as Error).message}`); + } + + let calibrationDir: string | null = null; + const calibSha = stringParam(job.params, "calibration_sha256"); + if (calibSha) { + try { + calibrationDir = await materialiseSource({ kind: "artifact", sha256: calibSha.toLowerCase() }, client, env, log); + } catch (e) { + return fail(`fetching the calibration set failed: ${(e as Error).message}`); + } + } + + const name = stringParam(job.params, "name") ?? defaultName(source); + const stepTimeoutMs = intParam(job.params, "timeout_s", DEFAULT_STEP_TIMEOUT_S) * 1000; + const failures: string[] = []; + + for (const [i, spec] of outputs.entries()) { + const outDir = path.join(workRoot, `${spec.format}-${(spec.quant ?? "none").toLowerCase()}`); + await mkdir(outDir, { recursive: true }); + + let plan: ConvertPlan; + try { + plan = planConvert(spec, tools, { source: sourcePath, outDir, name, calibrationDir }); + } catch (e) { + failures.push((e as Error).message); + log(`refused ${spec.format}/${spec.quant ?? "none"}: ${(e as Error).message}`); + continue; + } + + log(`converting ${spec.format}/${spec.quant ?? "none"} in ${plan.steps.length} step(s)`); + const stepFailure = await runSteps(plan, job, client, deviceId, stepTimeoutMs, log); + if (stepFailure) { + failures.push(`${spec.format}/${spec.quant ?? "none"}: ${stepFailure}`); + continue; + } + + const product = path.join(outDir, plan.product); + if (!existsSync(product)) { + failures.push( + `${spec.format}/${spec.quant ?? "none"}: the converter exited 0 but wrote no ${plan.product} (saw ${(await readdir(outDir).catch(() => [])).join(", ") || "nothing"})`, + ); + continue; + } + const bytes = (await stat(product)).size; + try { + // Uploaded WITHOUT `x-artifact-app`: a converted model is not a build, + // and publishing it under an app name would make it the thing a + // nightly asking for `"sha256": "latest"` picks up. + const up = await client.uploadArtifact(product, plan.product); + artifacts.push(up.sha256); + totalBytes += bytes; + log(`${plan.product} -> ${up.sha256} (${bytes} bytes)`); + // One row per output, so a four-output conversion is legible while it + // is still running rather than only at the end. + await client.postResult({ + schema: SCHEMA, kind: "result", job_id: job.job_id, device_id: deviceId, + iter: i + 1, ok: true, artifacts: [up.sha256], + model_out: { format: plan.modelFormat, quant: plan.quant, name: plan.product, sha256: up.sha256, bytes }, + }); + } catch (e) { + failures.push(`${spec.format}/${spec.quant ?? "none"}: upload failed: ${(e as Error).message}`); + } + } + + const convertS = Math.round((Date.now() - t0) / 100) / 10; + const metrics = compact({ + convert_s: convertS, + outputs: artifacts.length, + artifact_bytes: totalBytes || undefined, + }); + if (failures.length > 0) { + // Partial success is still a failure of the job — a pipeline that asked + // for three formats and got two must not read as green — but the shas + // that DID land are listed, because they are real and a dependent job may + // still want them. + return fail( + `${failures.length} of ${outputs.length} output(s) failed: ${failures.join(" | ").slice(0, 900)}`, + artifacts, + metrics, + ); + } + await client.postResult({ + schema: SCHEMA, kind: "result", job_id: job.job_id, device_id: deviceId, + iter: 0, final: true, ok: true, device: descriptor, + metrics, + // Every sha, in the order the job asked for them, so a dependent job's + // `${jobs..artifact}` resolves against a list that is not a surprise. + artifacts, + }); + log(`converted ${artifacts.length} output(s) in ${convertS}s`); + } finally { + // The products are in the artifact store now; the working copies are just + // tens of gigabytes on somebody's boot disk. + if (!env.FLEET_KEEP_CONVERT_WORK) await rm(workRoot, { recursive: true, force: true }).catch(() => {}); + } +} + +/** Runs a plan's steps in order, stopping at the first that fails. */ +async function runSteps( + plan: ConvertPlan, + job: JobSpec, + client: CollectorClient, + deviceId: string, + timeoutMs: number, + log: (m: string) => void, +): Promise { + for (const step of plan.steps) { + log(` ${step.label}`); + const r = await runWithBeacons({ + cmd: step.cmd, args: step.args, cwd: step.cwd, timeoutMs, + jobId: job.job_id, deviceId, client, + }); + if (r.cancelled) return "cancelled"; + if (r.timedOut) return `${step.label} timed out after ${Math.round(timeoutMs / 1000)}s`; + if (r.spawnError) return `${step.label} could not start: ${r.spawnError}`; + if (r.code !== 0) { + return `${step.label} exited ${r.code ?? `on signal ${r.signal ?? "?"}`}: ${lastMeaningful(r.output) || "no output"}`; + } + } + return null; +} + +/** + * The source on disk: a verified artifact, or a HuggingFace snapshot. + * + * An artifact that is a zip is extracted, because `convert_hf_to_gguf.py` + * takes a DIRECTORY of a HuggingFace repo rather than a single file, and a + * multi-file checkpoint can only reach the artifact store as one blob. + */ +async function materialiseSource( + source: Source, + client: CollectorClient, + env: NodeJS.ProcessEnv, + log: (m: string) => void, +): Promise { + if (source.kind === "hf") { + const cli = (await which("huggingface-cli", env)) ?? (await which("hf", env)); + if (!cli) { + throw new Error( + `params.source is the HuggingFace repo ${source.repo}, but neither huggingface-cli nor hf resolves on this machine`, + ); + } + const dest = path.join(dataDir(env), "hf", source.repo.replace(/[^A-Za-z0-9._-]+/g, "-")); + await mkdir(path.dirname(dest), { recursive: true }); + log(`downloading ${source.repo}`); + const r = await run(cli, ["download", source.repo, "--local-dir", dest], 6 * 60 * 60 * 1000); + if (r.code !== 0) throw new Error(`${path.basename(cli)} download ${source.repo} failed: ${lastMeaningful(r.stderr || r.stdout) || `exit ${r.code}`}`); + return dest; + } + + const cacheDir = path.join(dataDir(env), "sources"); + const blob = path.join(cacheDir, `${source.sha256}.bin`); + if (!(await fileMatchesHash(blob, source.sha256))) { + log(`fetching artifact ${source.sha256.slice(0, 12)}`); + await client.fetchArtifact(source.sha256, blob); + } + if (!(await isZip(blob))) return blob; + + const extracted = path.join(cacheDir, source.sha256); + if (!existsSync(extracted)) { + await mkdir(extracted, { recursive: true }); + const r = await run("unzip", ["-q", "-o", blob, "-d", extracted], 60 * 60 * 1000); + if (r.code !== 0) { + await rm(extracted, { recursive: true, force: true }).catch(() => {}); + throw new Error(`unzipping ${source.sha256.slice(0, 12)} failed: ${lastMeaningful(r.stderr || r.stdout) || `exit ${r.code}`}`); + } + } + // A zip of one directory is the common shape; hand the converter that + // directory rather than its parent, which would look empty of config.json. + const entries = await readdir(extracted, { withFileTypes: true }).catch(() => []); + const dirs = entries.filter((e) => e.isDirectory() && !e.name.startsWith("__MACOSX")); + const files = entries.filter((e) => e.isFile()); + if (dirs.length === 1 && files.length === 0) return path.join(extracted, dirs[0]!.name); + return extracted; +} + +/** The four bytes that make a file a zip. */ +export async function isZip(file: string): Promise { + try { + const fh = await open(file, "r"); + try { + const buf = Buffer.alloc(4); + const { bytesRead } = await fh.read(buf, 0, 4, 0); + return bytesRead === 4 && buf[0] === 0x50 && buf[1] === 0x4b && (buf[2] === 3 || buf[2] === 5 || buf[2] === 7); + } finally { + await fh.close(); + } + } catch { + return false; + } +} + +/** A filename stem for the products, from the job or from the source. */ +export function defaultName(source: Source): string { + const raw = source.kind === "hf" ? (source.repo.split("/").pop() ?? "model") : `model-${source.sha256.slice(0, 12)}`; + return raw.replace(/[^A-Za-z0-9._-]+/g, "-").slice(0, 60) || "model"; +} + +const safe = (s: string) => s.replace(/[^A-Za-z0-9._-]+/g, "-").slice(0, 60); + +/** Kept for the tests: what the planner needs, with nothing installed. */ +export type { ConverterTools }; diff --git a/runner-machine/src/workloads/serve.ts b/runner-machine/src/workloads/serve.ts new file mode 100644 index 0000000..176673f --- /dev/null +++ b/runner-machine/src/workloads/serve.ts @@ -0,0 +1,405 @@ +/** + * The `serve` workload: hold an inference server up for the length of a lease, + * announce where it is, and take it down again — always. + * + * Two things dominate the design. + * + * 1. **Teardown on every exit path.** `llama-server` holds the whole model + * resident; a leaked one costs gigabytes on a machine that will go on + * claiming other jobs, and nothing on the dashboard would ever say so. So + * the supervision loop is a `try/finally` around a teardown that runs once + * and runs for all five endings: the duration elapsing, `lease_renewed: + * false`, a dashboard cancel, the child dying on its own, and an uncaught + * throw anywhere in the loop. + * 2. **Never `0.0.0.0`.** The collector's README says it outright: there is no + * authentication anywhere in this system, the network IS the access + * control, and the posture holds only while every participant is on a + * network the owner chose. A model server bound to every interface on a + * laptop is an open inference endpoint on whatever hotel wifi that laptop + * joins next. Loopback and the tailnet address are the two answers, which + * is the same choice `FLEET_BIND` gives the collector itself. + */ +import { spawn, type ChildProcess } from "node:child_process"; +import net from "node:net"; +import os from "node:os"; +import path from "node:path"; +import type { CollectorClient } from "../collector.js"; +import type { Descriptor, JobSpec, Metrics } from "../protocol.js"; +import { SCHEMA, compact, intParam, stringParam } from "../protocol.js"; +import { beacon } from "../telemetry.js"; +import { which } from "../probe.js"; +import { dataDir } from "../git.js"; +import { fileMatchesHash } from "../collector.js"; +import * as JobCancellation from "../cancellation.js"; + +/** The beacon cadence the task asks for, and the lease renewal with it. */ +export const SERVE_BEACON_MS = 30_000; +/** A serve with no duration still ends: an unbounded one is a leak with a lease on it. */ +export const DEFAULT_DURATION_S = 30 * 60; +/** How long a model has to load before "serving" stops being a plausible claim. */ +export const DEFAULT_READY_TIMEOUT_S = 300; +/** Grace between SIGTERM and SIGKILL. llama-server frees its weights on SIGTERM. */ +export const KILL_GRACE_MS = 5_000; + +// --- binding ---------------------------------------------------------------- + +/** Tailscale hands out 100.64.0.0/10. That range is the tailnet and nothing else is. */ +export function isTailnetV4(addr: string): boolean { + const m = /^(\d{1,3})\.(\d{1,3})\.\d{1,3}\.\d{1,3}$/.exec(addr); + if (!m) return false; + return Number(m[1]) === 100 && Number(m[2]) >= 64 && Number(m[2]) <= 127; +} + +/** This machine's tailnet address, or null when it is not on one. */ +export function tailnetAddress(interfaces: NodeJS.Dict = os.networkInterfaces()): string | null { + for (const addrs of Object.values(interfaces)) { + for (const a of addrs ?? []) { + if (a.family === "IPv4" && !a.internal && isTailnetV4(a.address)) return a.address; + } + } + return null; +} + +/** + * Where the server may listen. + * + * Deliberately a closed set rather than a validated free-form address. A job + * spec arrives through an unauthenticated `POST /jobs`, so "any address that + * parses" would let whoever enqueued the job choose to publish this machine's + * model server on the LAN — and `0.0.0.0` is one keystroke away from every + * plausible spelling of an address anyway. + */ +export function resolveBindHost( + param: unknown, + env: NodeJS.ProcessEnv = process.env, + interfaces: NodeJS.Dict = os.networkInterfaces(), +): string { + const want = (typeof param === "string" && param.trim() !== "" ? param : env.FLEET_SERVE_BIND ?? "loopback").trim(); + if (want === "loopback" || want === "127.0.0.1" || want === "localhost") return "127.0.0.1"; + if (want === "tailnet") { + const addr = tailnetAddress(interfaces); + if (!addr) throw new Error("params.bind is 'tailnet' but this machine has no 100.64.0.0/10 address"); + return addr; + } + if (isTailnetV4(want)) { + // An explicit tailnet address still has to be one of THIS machine's, or + // llama-server would fail to bind and the failure would read as a crash. + const addr = tailnetAddress(interfaces); + if (addr !== want) throw new Error(`params.bind ${want} is not an address on this machine (tailnet address is ${addr ?? "absent"})`); + return want; + } + throw new Error( + `params.bind must be 'loopback' or 'tailnet' (got ${JSON.stringify(want)}). ` + + "The collector has no authentication and the network is the access control, so a model server is never bound to a wildcard or to the LAN.", + ); +} + +/** A port nothing is listening on, on the address the server will bind. */ +export function freePort(host: string): Promise { + return new Promise((resolve, reject) => { + const srv = net.createServer(); + srv.on("error", reject); + srv.listen({ host, port: 0, exclusive: true }, () => { + const addr = srv.address(); + const port = typeof addr === "object" && addr !== null ? addr.port : 0; + srv.close(() => (port > 0 ? resolve(port) : reject(new Error("could not find a free port")))); + }); + }); +} + +// --- supervision ------------------------------------------------------------ + +export type ServeStop = "duration" | "lease" | "cancelled" | "child-exit" | "error"; +export type ChildExit = { code: number | null; signal: NodeJS.Signals | null }; +export type ServeOutcome = { + stop: ServeStop; + error?: string; + exit?: ChildExit; + /** Proof the teardown ran. Asserted by the tests for every stop reason. */ + toreDown: boolean; + elapsedMs: number; +}; + +/** + * Runs the server for its duration, ticking a beacon, and tears it down. + * + * Everything is injected so the five endings can be tested without a model + * server: `exited` is the child's close event, `tick` is the beacon (it names + * the reason to stop, or null to keep going), and `teardown` is the kill. The + * `finally` is the point of the whole function — a throw from `tick` must + * still stop a process holding eight gigabytes of weights. + */ +export async function superviseServe(deps: { + exited: Promise; + teardown: () => Promise | void; + tick: () => Promise<"lease" | "cancelled" | null>; + tickMs: number; + durationMs: number; + now?: () => number; +}): Promise { + const now = deps.now ?? Date.now; + const started = now(); + const deadline = started + deps.durationMs; + let toreDown = false; + const teardownOnce = async () => { + if (toreDown) return; + toreDown = true; + await deps.teardown(); + }; + + let stop: ServeStop = "duration"; + let error: string | undefined; + let exit: ChildExit | undefined; + + try { + const exitedTagged = deps.exited.then((e) => ({ kind: "exit" as const, exit: e })); + for (;;) { + const remaining = deadline - now(); + if (remaining <= 0) { + stop = "duration"; + break; + } + const raced = await Promise.race([ + exitedTagged, + delay(Math.min(deps.tickMs, remaining)).then(() => ({ kind: "tick" as const })), + ]); + if (raced.kind === "exit") { + // The server fell over on its own. That is a failure of the job, not a + // clean end, and the exit code is what says why. + stop = "child-exit"; + exit = raced.exit; + break; + } + const reason = await deps.tick(); + if (reason !== null) { + stop = reason; + break; + } + } + } catch (e) { + stop = "error"; + error = (e as Error).message ?? String(e); + } finally { + await teardownOnce(); + } + + return { stop, error, exit, toreDown, elapsedMs: now() - started }; +} + +function delay(ms: number): Promise { + return new Promise((resolve) => { + const t = setTimeout(resolve, Math.max(0, ms)); + // Unref'd: a pending tick timer must never be the reason this process + // outlives the job it was serving. + t.unref?.(); + }); +} + +// --- the workload ----------------------------------------------------------- + +export async function runServe( + job: JobSpec, + client: CollectorClient, + deviceId: string, + descriptor: Descriptor, + env: NodeJS.ProcessEnv = process.env, +): Promise { + const log = (m: string) => console.log(`[${deviceId}] serve ${job.job_id}: ${m}`); + const fail = async (error: string, metrics?: Metrics) => { + log(`failed: ${error}`); + await client.postResult({ + schema: SCHEMA, kind: "result", job_id: job.job_id, device_id: deviceId, + iter: 0, final: true, ok: false, device: descriptor, error, + ...(metrics ? { metrics } : {}), + }); + }; + + const server = await which(env.FLEET_LLAMA_SERVER ?? "llama-server", env); + if (!server) return fail("serve needs llama-server on PATH (or FLEET_LLAMA_SERVER)"); + + const sha = job.model?.sha256 ?? stringParam(job.params, "model_sha256"); + if (!sha || !/^[0-9a-f]{64}$/i.test(sha)) { + return fail("serve needs a model artifact: job.model.sha256 or params.model_sha256"); + } + + let host: string; + try { + host = resolveBindHost(job.params?.bind, env); + } catch (e) { + return fail((e as Error).message); + } + + const modelFile = path.join(dataDir(env), "models", `${sha.toLowerCase()}.gguf`); + try { + if (!(await fileMatchesHash(modelFile, sha.toLowerCase()))) { + log(`fetching model ${sha.slice(0, 12)}`); + await client.fetchArtifact(sha.toLowerCase(), modelFile); + } + } catch (e) { + return fail(`fetching the model failed: ${(e as Error).message}`); + } + + const durationS = intParam(job.params, "duration_s", DEFAULT_DURATION_S); + const readyTimeoutS = intParam(job.params, "ready_timeout_s", DEFAULT_READY_TIMEOUT_S); + let port: number; + try { + port = intParam(job.params, "port", 0) || (await freePort(host)); + } catch (e) { + return fail(`no free port on ${host}: ${(e as Error).message}`); + } + const endpoint = `http://${host}:${port}`; + + const extra = Array.isArray(job.params?.args) + ? (job.params.args as unknown[]).filter((a): a is string => typeof a === "string") + : []; + const child = spawn(server, ["-m", modelFile, "--host", host, "--port", String(port), ...extra], { + stdio: ["ignore", "ignore", "pipe"], + env: { ...process.env, TERM: "dumb" }, + }); + let stderrTail = ""; + child.stderr?.on("data", (b: Buffer) => { + stderrTail = (stderrTail + b.toString("utf8")).slice(-4000); + }); + + const exited = new Promise((resolve) => { + child.on("error", () => resolve({ code: null, signal: null })); + child.on("close", (code, signal) => resolve({ code, signal })); + }); + let childGone = false; + void exited.then(() => (childGone = true)); + + const teardown = async () => { + if (childGone) return; + log("stopping llama-server"); + try { + child.kill("SIGTERM"); + } catch { + /* already gone */ + } + await Promise.race([exited, delay(KILL_GRACE_MS)]); + if (!childGone) { + try { + child.kill("SIGKILL"); + } catch { + /* already gone */ + } + await Promise.race([exited, delay(KILL_GRACE_MS)]); + } + }; + + // From here every path goes through the finally, including a throw. + try { + const ready = await waitForReady(endpoint, readyTimeoutS * 1000, exited); + if (!ready.ok) { + await teardown(); + return fail( + `llama-server did not answer on ${endpoint} within ${readyTimeoutS}s: ${ready.detail || lastLine(stderrTail) || "no output"}`, + ); + } + log(`serving ${sha.slice(0, 12)} on ${endpoint} for ${durationS}s (pid ${child.pid})`); + + // The announcement, immediately — a serve nobody can find is a serve that + // did not happen, and a dependent job may be waiting on this row. + await client.postResult({ + schema: SCHEMA, kind: "result", job_id: job.job_id, device_id: deviceId, + iter: 1, ok: true, endpoint, + }); + + const outcome = await superviseServe({ + exited, + teardown, + tickMs: SERVE_BEACON_MS, + durationMs: durationS * 1000, + tick: async () => { + if (JobCancellation.isCancelled(job.job_id)) return "cancelled"; + try { + const renewed = await client.postBeacon({ + schema: SCHEMA, kind: "beacon", job_id: job.job_id, device_id: deviceId, + beacon: await beacon(), endpoint, + }); + if (!renewed) { + JobCancellation.cancel(job.job_id); + return "lease"; + } + } catch { + // An unreachable collector is not a cancellation, exactly as in the + // build workload: the lease may lapse, but a flaky network must not + // be what takes a served model down. + } + return null; + }, + }); + + const elapsedS = Math.round(outcome.elapsedMs / 100) / 10; + if (outcome.stop === "duration") { + log(`duration elapsed after ${elapsedS}s`); + await client.postResult({ + schema: SCHEMA, kind: "result", job_id: job.job_id, device_id: deviceId, + iter: 0, final: true, ok: true, device: descriptor, endpoint, + metrics: compact({ elapsed_s: elapsedS }), + }); + return; + } + if (outcome.stop === "child-exit") { + return fail( + `llama-server exited on its own after ${elapsedS}s (${outcome.exit?.code !== null && outcome.exit?.code !== undefined ? `exit ${outcome.exit.code}` : `signal ${outcome.exit?.signal ?? "?"}`}): ${lastLine(stderrTail) || "no output"}`, + compact({ elapsed_s: elapsedS, exit_code: outcome.exit?.code ?? undefined }), + ); + } + return fail( + outcome.stop === "error" + ? `serve supervision failed: ${outcome.error}` + : outcome.stop === "lease" + ? "lease not renewed" + : "cancelled", + compact({ elapsed_s: elapsedS }), + ); + } finally { + // Belt and braces over superviseServe's own finally: a throw between the + // spawn and the supervision — a failed announcement post, say — must not + // leave a model server running. + await teardown(); + } +} + +/** Polls the server's own health endpoint, giving up if the child dies first. */ +async function waitForReady( + endpoint: string, + timeoutMs: number, + exited: Promise, +): Promise<{ ok: boolean; detail: string }> { + const deadline = Date.now() + timeoutMs; + let dead = false; + void exited.then(() => (dead = true)); + let detail = ""; + while (Date.now() < deadline) { + if (dead) return { ok: false, detail: "the process exited before it answered" }; + try { + const res = await fetch(`${endpoint}/health`, { signal: AbortSignal.timeout(5_000) }); + // llama-server answers 503 while the model loads and 200 once it can + // serve, so only 200 is ready; anything else is another second of wait. + if (res.ok) return { ok: true, detail: "" }; + detail = `HTTP ${res.status}`; + } catch (e) { + detail = (e as Error).message; + } + await delay(1000); + } + return { ok: false, detail }; +} + +const lastLine = (s: string): string => + s.split(/\r?\n/).map((l) => l.trim()).filter(Boolean).at(-1)?.slice(0, 300) ?? ""; + +/** + * Is a pid still there? Used by the end-to-end check that a cancelled serve + * actually took its child with it, rather than assuming the kill landed. + */ +export function pidAlive(pid: number): boolean { + try { + process.kill(pid, 0); + return true; + } catch { + return false; + } +} diff --git a/runner-machine/src/workloads/shell.ts b/runner-machine/src/workloads/shell.ts new file mode 100644 index 0000000..f2a96fe --- /dev/null +++ b/runner-machine/src/workloads/shell.ts @@ -0,0 +1,275 @@ +/** + * The `shell` workload: run a script that is an artifact — and only one the + * machine's owner listed by hash. + * + * `POST /jobs` on the collector is unauthenticated by design, and so is + * `POST /artifacts`. That means anyone who can reach the collector can upload + * a script AND enqueue a job naming it, so this workload's allowlist is not + * defence in depth: it is the entire trust boundary, and every other property + * here follows from that. + * + * The order matters more than anything else in the file. The sha is checked + * against the allowlist BEFORE the artifact is fetched. Checking after the + * download would mean an attacker-supplied file had already been written to + * this machine's disk under a name of their choosing, and "we deleted it + * again" is not a defence anybody would accept. + * + * The other three, in the order they bite: + * - a refusal is a result row, never a throw. A job that vanished with a + * stack trace tells the operator nothing about WHY it was refused, and the + * refusal reason is the only feedback loop the allowlist has. + * - a fresh temp dir and a minimal environment. The agent's own process env + * holds FLEET_DASH_TOKEN and a GitHub token; handing those to a script + * would make the allowlist the only thing between a job and this machine's + * credentials, and there is no reason to make the allowlist carry that too. + * - `exit_code` as its own metric. A script that deliberately exits 3 and one + * that could not start are both "not zero" and they are not the same event; + * an alert rule needs to be able to tell them apart without parsing prose. + */ +import { spawn } from "node:child_process"; +import { chmod, mkdtemp, rm, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import type { CollectorClient } from "../collector.js"; +import type { Descriptor, JobSpec, Metrics } from "../protocol.js"; +import { SCHEMA, compact, intParam, stringParam } from "../protocol.js"; +import { isAllowlisted, loadAllowlist, SHA256_RE } from "../allowlist.js"; +import { redact, secretsFromEnv } from "../redact.js"; +import * as JobCancellation from "../cancellation.js"; + +/** A script that has not finished in an hour is a script that is not going to. */ +export const DEFAULT_TIMEOUT_S = 3600; +/** What is kept of the output. Enough to diagnose; not enough to exhaust memory. */ +export const MAX_CAPTURE_BYTES = 4 << 20; + +/** + * The environment a script gets. + * + * An allowlist entry says "this exact script may run here"; it does not say + * "this script may read every credential this agent holds". PATH, HOME and + * TMPDIR are what a script needs to be a script at all, and HOME points at the + * job's own temp dir so a script writing a dotfile does not write it into the + * operator's. + */ +export function minimalEnv( + workDir: string, + job: JobSpec, + env: NodeJS.ProcessEnv = process.env, +): NodeJS.ProcessEnv { + const base: NodeJS.ProcessEnv = { + PATH: env.PATH ?? "/usr/bin:/bin:/usr/sbin:/sbin", + HOME: workDir, + TMPDIR: workDir, + PWD: workDir, + LANG: env.LANG ?? "en_US.UTF-8", + TERM: "dumb", + CI: "1", + FLEET_JOB_ID: job.job_id, + }; + // Job-supplied variables, which are the only way a script is parameterised — + // and they are strings from a job spec, so a name that could shadow PATH or + // reintroduce a credential name is dropped rather than sanitised. + const extra = job.params?.env; + if (extra && typeof extra === "object" && !Array.isArray(extra)) { + for (const [k, v] of Object.entries(extra as Record)) { + if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(k)) continue; + if (k in base) continue; + if (typeof v === "string" || typeof v === "number" || typeof v === "boolean") base[k] = String(v); + } + } + return base; +} + +export async function runShell( + job: JobSpec, + client: CollectorClient, + deviceId: string, + descriptor: Descriptor, + env: NodeJS.ProcessEnv = process.env, +): Promise { + const log = (m: string) => console.log(`[${deviceId}] shell ${job.job_id}: ${m}`); + const post = async (ok: boolean, error?: string, metrics?: Metrics, artifacts?: string[]) => { + await client.postResult({ + schema: SCHEMA, kind: "result", job_id: job.job_id, device_id: deviceId, + iter: 0, final: true, ok, device: descriptor, + ...(error ? { error } : {}), + ...(metrics ? { metrics } : {}), + ...(artifacts?.length ? { artifacts } : {}), + }); + }; + /** Every refusal is a row, never a throw. */ + const refuse = async (reason: string) => { + log(`refused: ${reason}`); + await post(false, reason); + }; + + const requested = stringParam(job.params, "sha256") ?? job.model?.sha256; + if (!requested) return refuse("shell needs params.sha256: the sha256 of an allowlisted script artifact"); + + // --- the gate. Nothing is fetched above this line. ------------------------ + const list = await loadAllowlist(env); + if (!list.exists) { + return refuse(`shell is refused: this machine has no allowlist at ${list.file}`); + } + if (list.allowed.length === 0) { + return refuse(`shell is refused: the allowlist at ${list.file} lists no script hashes`); + } + if (!isAllowlisted(requested, list.allowed)) { + const shape = SHA256_RE.test(requested.trim().toLowerCase()) ? "" : " (which is not a sha256)"; + return refuse( + `shell is refused: ${requested.slice(0, 80)}${shape} is not in ${list.file}` + + (list.rejected.length ? `. Note that ${list.rejected.length} line(s) in that file are not sha256s and were ignored` : ""), + ); + } + const sha = requested.trim().toLowerCase(); + log(`${sha.slice(0, 12)} is allowlisted; fetching`); + + // --- from here it is a script the owner vouched for ---------------------- + const workDir = await mkdtemp(path.join(os.tmpdir(), `fleet-shell-${safe(job.job_id)}-`)); + const secrets = secretsFromEnv(env); + try { + const script = path.join(workDir, "script"); + try { + await client.fetchArtifact(sha, script); + } catch (e) { + return post(false, `fetching the script failed: ${(e as Error).message}`); + } + await chmod(script, 0o700); + + const interpreter = stringParam(job.params, "interpreter") ?? "/bin/sh"; + const args = Array.isArray(job.params?.args) + ? (job.params.args as unknown[]).filter((a): a is string => typeof a === "string") + : []; + const timeoutMs = intParam(job.params, "timeout_s", DEFAULT_TIMEOUT_S) * 1000; + + const started = Date.now(); + const outcome = await spawnCapturing(interpreter, [script, ...args], workDir, minimalEnv(workDir, job, env), timeoutMs, job.job_id); + const elapsedS = Math.round((Date.now() - started) / 100) / 10; + + // Redacted before it is written anywhere, not before it is uploaded: the + // file on disk is in a temp dir this job owns, but the habit of scrubbing + // at the boundary is what stops the next change from leaking. + const captured = redact(outcome.output, secrets); + const logFile = path.join(workDir, "output.txt"); + await writeFile(logFile, captured, "utf8"); + let logRef: string | null = null; + try { + const up = await client.uploadArtifact(logFile, `${safe(job.job_id)}-shell.txt`); + logRef = up.sha256; + } catch (e) { + log(`output upload failed: ${(e as Error).message}`); + } + + if (outcome.spawnError) { + // Could not start. Deliberately NO exit_code: there was no exit. + return post( + false, + `could not run ${interpreter}: ${outcome.spawnError}`, + compact({ elapsed_s: elapsedS }), + logRef ? [logRef] : undefined, + ); + } + const metrics = compact({ exit_code: outcome.code ?? undefined, elapsed_s: elapsedS }); + if (outcome.timedOut) { + return post(false, `script timed out after ${Math.round(timeoutMs / 1000)}s`, metrics, logRef ? [logRef] : undefined); + } + if (outcome.cancelled) { + return post(false, "cancelled", metrics, logRef ? [logRef] : undefined); + } + if (outcome.code !== 0) { + return post( + false, + `script exited ${outcome.code ?? `on signal ${outcome.signal ?? "?"}`}: ${lastLine(captured) || "no output"}`, + metrics, + logRef ? [logRef] : undefined, + ); + } + log(`exited 0 in ${elapsedS}s`); + await post(true, undefined, metrics, logRef ? [logRef] : undefined); + } finally { + // The temp dir is the sandbox; leaving it behind would accumulate one per + // job on a machine that runs this nightly. + await rm(workDir, { recursive: true, force: true }).catch(() => {}); + } +} + +type Capture = { + code: number | null; + signal: NodeJS.Signals | null; + output: string; + timedOut: boolean; + cancelled: boolean; + spawnError: string | null; +}; + +/** + * Runs the script, merging stdout and stderr. + * + * Merged because a script's diagnostics go to stderr and its progress to + * stdout, and reading them apart loses which came first — the same call the + * build workload makes, for the same reason. + */ +async function spawnCapturing( + cmd: string, + args: string[], + cwd: string, + env: NodeJS.ProcessEnv, + timeoutMs: number, + jobId: string, +): Promise { + return new Promise((resolve) => { + let output = ""; + let bytes = 0; + let truncated = false; + let timedOut = false; + let cancelled = false; + let spawnError: string | null = null; + + const child = spawn(cmd, args, { cwd, env, stdio: ["ignore", "pipe", "pipe"] }); + const absorb = (b: Buffer) => { + if (bytes >= MAX_CAPTURE_BYTES) { + if (!truncated) { + truncated = true; + output += "\n[output truncated]\n"; + } + return; + } + bytes += b.length; + output += b.toString("utf8"); + }; + child.stdout?.on("data", absorb); + child.stderr?.on("data", absorb); + + const killer = setTimeout(() => { + timedOut = true; + child.kill("SIGKILL"); + }, timeoutMs); + // A cancelled shell job stops the script rather than waiting it out: unlike + // a benchmark there is no iteration boundary to stop at. + const canceller = setInterval(() => { + if (JobCancellation.isCancelled(jobId) && !cancelled) { + cancelled = true; + child.kill("SIGTERM"); + setTimeout(() => child.kill("SIGKILL"), 5_000).unref(); + } + }, 1_000); + canceller.unref?.(); + + const done = (c: Capture) => { + clearTimeout(killer); + clearInterval(canceller); + resolve(c); + }; + child.on("error", (e) => { + spawnError = e.message; + done({ code: null, signal: null, output, timedOut, cancelled, spawnError }); + }); + child.on("close", (code, signal) => { + done({ code, signal, output, timedOut, cancelled, spawnError }); + }); + }); +} + +const safe = (s: string) => s.replace(/[^A-Za-z0-9._-]+/g, "-").slice(0, 60); +const lastLine = (s: string): string => + s.split(/\r?\n/).map((l) => l.trim()).filter(Boolean).at(-1)?.slice(0, 300) ?? ""; diff --git a/runner-machine/test/allowlist.test.ts b/runner-machine/test/allowlist.test.ts new file mode 100644 index 0000000..92e11ea --- /dev/null +++ b/runner-machine/test/allowlist.test.ts @@ -0,0 +1,108 @@ +/** + * The shell allowlist is the fleet's only trust boundary. + * + * `POST /jobs` is unauthenticated by design — anyone who can reach the + * collector can enqueue work — so "run this script" is the one workload where + * the collector's word cannot be enough. The machine's owner pins a sha by + * hand, and nothing else runs. Every test here is about the refusal, because a + * permissive bug in this file is arbitrary code execution and a restrictive + * one is merely a job that does not run. + */ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { mkdtemp, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { parseAllowlist, isAllowlisted, loadAllowlist } from "../src/allowlist.js"; +import { capabilitiesFrom } from "../src/capabilities.js"; + +const A = "a".repeat(64); +const B = "b".repeat(64); + +test("a listed sha is allowed and an unlisted one is not", () => { + assert.equal(isAllowlisted(A, [A]), true); + assert.equal(isAllowlisted(B, [A]), false); +}); + +test("one character of difference is a different script", () => { + // The whole point of addressing a script by content: an attacker who can + // enqueue jobs can name any sha they like, and only the exact bytes the + // owner pinned may run. + const nearly = `${"a".repeat(63)}b`; + assert.equal(isAllowlisted(nearly, [A]), false); +}); + +test("case differs between shasum and a dashboard, and is the same hash", () => { + assert.equal(isAllowlisted(A.toUpperCase(), [A]), true); +}); + +test("anything that is not a well-formed sha256 is refused", () => { + // These arrive from a job spec, so they are attacker-controlled. A truthy + // non-string must never reach a comparison that could coerce. + for (const bad of ["", "not-a-sha", A.slice(0, 63), `${A}a`, "../../etc/passwd", null, undefined, 42, {}, [A]]) { + assert.equal(isAllowlisted(bad, [A]), false, `refused: ${JSON.stringify(bad)}`); + } +}); + +test("an empty allowlist allows nothing at all", () => { + assert.equal(isAllowlisted(A, []), false); +}); + +test("comments and blank lines are ignored; malformed entries are reported, not allowed", () => { + const { allowed, rejected } = parseAllowlist( + ["# the deploy script", "", `${A} # pinned 2026-09-04`, " ", "nonsense-line", `${B}`].join("\n"), + ); + assert.deepEqual(allowed, [A, B]); + // Surfaced rather than silently dropped: a typo'd sha should look like a + // mistake in the file, not like a script that mysteriously will not run. + assert.deepEqual(rejected, ["nonsense-line"]); +}); + +test("a duplicate sha is listed once", () => { + assert.deepEqual(parseAllowlist([A, A].join("\n")).allowed, [A]); +}); + +test("a missing allowlist file is empty, not an error", async () => { + const dir = await mkdtemp(path.join(tmpdir(), "fleet-allow-")); + const env = { FLEET_SHELL_ALLOWLIST: path.join(dir, "absent.txt") } as NodeJS.ProcessEnv; + const res = await loadAllowlist(env); + assert.equal(res.exists, false); + assert.deepEqual(res.allowed, []); +}); + +test("a file that exists is read", async () => { + const dir = await mkdtemp(path.join(tmpdir(), "fleet-allow-")); + const file = path.join(dir, "shell-allowlist.txt"); + await writeFile(file, `# pinned by hand\n${A}\n`); + const res = await loadAllowlist({ FLEET_SHELL_ALLOWLIST: file } as NodeJS.ProcessEnv); + assert.equal(res.exists, true); + assert.deepEqual(res.allowed, [A]); +}); + +test("shell is declared only when this machine has a non-empty allowlist", () => { + const bare = { llamaBench: false, mlx: false, gradle: false, xcodebuild: false, node: false }; + // A machine whose owner has pinned nothing must be unable to CLAIM a shell + // job at all, rather than claiming it and refusing it afterwards. Refusing + // after the claim would still take the job off the queue. + assert.equal(capabilitiesFrom({ ...bare, shellAllowlist: false }).includes("shell"), false); + assert.equal(capabilitiesFrom({ ...bare, shellAllowlist: true }).includes("shell"), true); +}); + +test("model-convert is declared bare as well as per format", () => { + const bare = { llamaBench: false, mlx: false, gradle: false, xcodebuild: false, node: false }; + const caps = capabilitiesFrom({ ...bare, converters: ["gguf", "tflite"] }); + // Bare, because the collector's capabilityMatches reads workload and backend + // and a job spec has nowhere to put an output format — so the specific + // labels alone would be a machine that can convert and never claims one. + assert.equal(caps.includes("model-convert"), true); + assert.equal(caps.includes("model-convert:gguf"), true); + assert.equal(caps.includes("model-convert:tflite"), true); + assert.equal(caps.includes("model-convert:coreml"), false); + assert.equal(capabilitiesFrom({ ...bare, converters: [] }).includes("model-convert"), false); +}); + +test("serve is declared only where a llama-server resolved", () => { + const bare = { llamaBench: false, mlx: false, gradle: false, xcodebuild: false, node: false }; + assert.equal(capabilitiesFrom({ ...bare, llamaServer: false }).includes("serve"), false); + assert.equal(capabilitiesFrom({ ...bare, llamaServer: true }).includes("serve"), true); +});