From 276280d241b5e7d6f18bdeaea5adeeba55ed81ac Mon Sep 17 00:00:00 2001 From: Taylor Addison Date: Fri, 4 Sep 2026 22:55:20 -0400 Subject: [PATCH] Pipelines, a screen you can watch, and a trust boundary MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Wave 3, and a partial one. A session limit killed three agents mid-flight; what is here is what survived being finished and verified by hand, and the gaps are named at the bottom rather than implied. The machine agent gains four workloads. model-convert turns a checkpoint into GGUF, Core ML or TFLite on the machine with the memory for it, uploading each output so a dependent job can name it. dataset-prep builds an eval set and carries its licence and source on the row, because a prepared set that loses its licence is one nobody can publish results from. serve hosts a model for the length of a lease and announces its endpoint, bound to loopback or the tailnet and never 0.0.0.0. shell is the one with a boundary. POST /jobs is unauthenticated by design, so "run this script" cannot take the collector's word for it. The machine's owner pins a sha by hand; the capability is declared only when that file exists and is non-empty, so a machine that has pinned nothing cannot be offered the job at all rather than claiming it and refusing afterwards. Refusing after the claim would still take the work off the queue. The sha is checked before the artifact is fetched, never after. The executor's handlers start becoming directories. install moved first, smallest first, with a loader that falls back to the existing chain for everything not yet moved — so the file that runs the real fleet is never half broken. Verified by driving the actual executor against a throwaway collector with seven host workloads queued: install ran from its new directory and not one job reported "not supported". The live mirror answers a different question from the artifact viewer. That one says what it looked like when it failed; this says what it is doing now, which is the question you have while a flow is running. Frames live in a bounded in-memory ring and are dropped when the job ends: the collector already stores every measurement the fleet has taken, and a video stream would trade a permanent cost for a transient benefit. MJPEG because an img tag renders it with no client code, and it degrades to a still frame rather than an error when the producer stalls. Real iPhones can finally be captured. The operations doc recorded the gap: the visual matrix advertises real phone screens while half the real phones have no capture path. A WKWebView inside the runner app closes it. The profile is named webkit and not safari, in a comment that says why, because a WKWebView has no reader mode and no content blockers and someone reading "safari" in a baseline matrix would believe a stronger claim than the capture supports. Three fixes to work the dying agents left behind. The machine's workloads referenced metric names that exist in the collector's schema but had never been mirrored into the agent's own protocol type, so none of them compiled; the four files were also never dispatched or capability-gated, which made them dead code. And one Swift expression mixed literals and Doubles densely enough that the type checker gave up and failed the build, so it is split into named steps. Verified: the collector's suite passes, the machine agent's 122 tests pass, and the iOS app builds from a clean xcodegen run off project.yml with zero xcframework references. Not done, and not pretended otherwise. push-latency needs FCM and APNs credentials. camera-eval needs a physical rig pointing a phone at a display. desktop-ui-test has no desktop app to test against. Only install has moved into the plugin layout. model-convert, dataset-prep and serve have never run against a real toolchain, because none of the three converters resolves on this machine and the probes correctly decline to declare them. Co-Authored-By: Claude Opus 5 --- collector/dash/src/icons.tsx | 49 ++ collector/dash/src/pages/JobDetail.tsx | 48 ++ collector/schemas/metrics.json | 7 + collector/schemas/result.schema.json | 25 + collector/scripts/smoke.ts | 42 ++ collector/src/api/index.ts | 2 + collector/src/api/mirror.ts | 169 +++++ collector/src/executor.ts | 248 +++---- collector/src/server.ts | 14 + collector/src/workloads/device.ts | 119 ++++ collector/src/workloads/install/index.ts | 61 ++ collector/src/workloads/install/manifest.json | 15 + collector/src/workloads/registry.ts | 120 ++++ collector/src/workloads/types.ts | 149 ++++ .../FleetRunner.xcodeproj/project.pbxproj | 8 + runner-ios/FleetRunner/CollectorClient.swift | 19 + runner-ios/FleetRunner/FleetAgent.swift | 9 + runner-ios/FleetRunner/Protocol.swift | 55 ++ runner-ios/FleetRunner/WebShotsManifest.swift | 246 +++++++ runner-ios/FleetRunner/WebShotsWorkload.swift | 649 ++++++++++++++++++ runner-machine/recipes/dir-labels.mjs | 34 + runner-machine/recipes/plant-id.mjs | 45 ++ runner-machine/src/agent.ts | 20 + runner-machine/src/allowlist.ts | 93 +++ runner-machine/src/beaconing.ts | 170 +++++ runner-machine/src/capabilities.ts | 34 +- runner-machine/src/converters.ts | 368 ++++++++++ runner-machine/src/images.ts | 104 +++ runner-machine/src/protocol.ts | 19 + runner-machine/src/recipes.ts | 197 ++++++ runner-machine/src/redact.ts | 52 ++ runner-machine/src/workloads/datasetprep.ts | 289 ++++++++ runner-machine/src/workloads/modelconvert.ts | 294 ++++++++ runner-machine/src/workloads/serve.ts | 405 +++++++++++ runner-machine/src/workloads/shell.ts | 275 ++++++++ runner-machine/test/allowlist.test.ts | 108 +++ 36 files changed, 4405 insertions(+), 156 deletions(-) create mode 100644 collector/src/api/mirror.ts create mode 100644 collector/src/workloads/device.ts create mode 100644 collector/src/workloads/install/index.ts create mode 100644 collector/src/workloads/install/manifest.json create mode 100644 collector/src/workloads/registry.ts create mode 100644 collector/src/workloads/types.ts create mode 100644 runner-ios/FleetRunner/WebShotsManifest.swift create mode 100644 runner-ios/FleetRunner/WebShotsWorkload.swift create mode 100644 runner-machine/recipes/dir-labels.mjs create mode 100644 runner-machine/recipes/plant-id.mjs create mode 100644 runner-machine/src/allowlist.ts create mode 100644 runner-machine/src/beaconing.ts create mode 100644 runner-machine/src/converters.ts create mode 100644 runner-machine/src/images.ts create mode 100644 runner-machine/src/recipes.ts create mode 100644 runner-machine/src/redact.ts create mode 100644 runner-machine/src/workloads/datasetprep.ts create mode 100644 runner-machine/src/workloads/modelconvert.ts create mode 100644 runner-machine/src/workloads/serve.ts create mode 100644 runner-machine/src/workloads/shell.ts create mode 100644 runner-machine/test/allowlist.test.ts diff --git a/collector/dash/src/icons.tsx b/collector/dash/src/icons.tsx index 210ae65..359ebb8 100644 --- a/collector/dash/src/icons.tsx +++ b/collector/dash/src/icons.tsx @@ -53,6 +53,48 @@ const PATHS = { ), + convert: ( + <> + + + ), + dataset: ( + <> + + + + ), + serve: ( + <> + + + + + ), + terminal: ( + <> + + + + ), + bell: ( + <> + + + + ), + camera: ( + <> + + + + ), + desktop: ( + <> + + + + ), hammer: ( <> @@ -360,6 +402,13 @@ const WORKLOAD_ICON: Record = { "locale-shots": "translate", "app-soak": "soak", "a11y-audit": "accessibility", + "model-convert": "convert", + "dataset-prep": "dataset", + serve: "serve", + shell: "terminal", + "push-latency": "bell", + "camera-eval": "camera", + "desktop-ui-test": "desktop", }; /** The workload's name with its icon, for job tables. */ diff --git a/collector/dash/src/pages/JobDetail.tsx b/collector/dash/src/pages/JobDetail.tsx index 66aaf48..f690f78 100644 --- a/collector/dash/src/pages/JobDetail.tsx +++ b/collector/dash/src/pages/JobDetail.tsx @@ -1,3 +1,4 @@ +import { useEffect, useState } from "preact/hooks"; import { useApi, type JobDetail as Detail, type ResultRow } from "../api.js"; import { Workload } from "../icons.js"; import { mutate, useMutation } from "../mutate.js"; @@ -218,6 +219,8 @@ export function JobDetail({ id }: { id: string }) { + +
@@ -353,3 +356,48 @@ export function JobDetail({ id }: { id: string }) { ); } + +/** + * The device screen while a host job drives it. + * + * Polls only for whether a stream exists; the picture itself is an on an + * MJPEG endpoint, so the browser does the streaming and this component holds no + * frames. Rendered only while something is actually producing, because an + * that never paints looks exactly like a device with a black screen. + */ +function Mirror({ jobId, live }: { jobId: string; live: boolean }) { + const [streaming, setStreaming] = useState(false); + useEffect(() => { + if (!live) { + setStreaming(false); + return; + } + let alive = true; + const check = async () => { + try { + const r = await fetch(`/api/jobs/${encodeURIComponent(jobId)}/mirror/status`); + const b = (await r.json()) as { streaming?: boolean }; + if (alive) setStreaming(!!b.streaming); + } catch { + if (alive) setStreaming(false); + } + }; + void check(); + const t = setInterval(() => void check(), 5_000); + return () => { + alive = false; + clearInterval(t); + }; + }, [jobId, live]); + + if (!streaming) return null; + return ( + device screen, not recorded}> + the device screen while this job runs + + ); +} diff --git a/collector/schemas/metrics.json b/collector/schemas/metrics.json index 7bce0a6..e458396 100644 --- a/collector/schemas/metrics.json +++ b/collector/schemas/metrics.json @@ -1,6 +1,7 @@ { "schema": 1, "metrics": [ + "app_state", "artifact_bytes", "avg_position", "avg_rating", @@ -13,6 +14,7 @@ "clock_offset_ms", "clusters", "connect_ms", + "convert_s", "crashes", "ctr_pct", "decode_tok_s", @@ -25,10 +27,12 @@ "elapsed_s", "energy_method", "energy_wh", + "exit_code", "images_per_s", "impressions", "issues_error", "issues_warn", + "items", "jank_pct", "launch_ms", "launch_state", @@ -36,12 +40,15 @@ "locales", "mem_method", "network_type", + "outputs", "p50_ms", "p95_ms", "pages_crawled", "peak_mem_mb", "prefill_tok_s", "pss_mb", + "push_delivered_pct", + "push_latency_ms", "recall_at_1", "recall_at_10", "recall_at_5", diff --git a/collector/schemas/result.schema.json b/collector/schemas/result.schema.json index b038e34..5a490ed 100644 --- a/collector/schemas/result.schema.json +++ b/collector/schemas/result.schema.json @@ -23,6 +23,18 @@ "app_ver": { "type": "string" } } }, + "endpoint": { + "type": "string", + "description": "serve: where the model this job is hosting can be reached, announced so a dependent job can consume it. Bound to loopback or the tailnet, never 0.0.0.0 — an unauthenticated inference server on a laptop's hotel wifi is a different product to the one anybody asked for." + }, + "model_out": { + "type": "object", + "description": "model-convert: what one conversion produced — format, quant, name, sha256 and size. One row per output, so a job converting to three formats is three rows and the sha of each is addressable by a dependent job." + }, + "dataset": { + "type": "object", + "description": "dataset-prep: the prepared set's manifest summary, including its licence and source. A prepared set that loses its licence is one nobody can publish results from, which is why it rides on the row rather than only inside the zip." + }, "metrics": { "type": "object", "description": "Workload metrics; memory is labeled, never laundered.\n\nEvery field here means one thing. Before the vision-eval and drain fields existed, those workloads reused the LLM slots — vision put top-1 accuracy in decode_tok_s, p50 latency in ttft_ms and images/sec in prefill_tok_s; drain put percent-per-hour in decode_tok_s — and top-5 and p95 had nowhere to go at all, so they were only ever in the hand-written eval report. Rows predating these fields still carry that convention, and the dashboard marks anything it had to infer that way. New runners must populate the named fields instead.", @@ -87,6 +99,19 @@ "locales": { "type": "number", "description": "locale-shots: how many locales the run covered" }, "shots": { "type": "number", "description": "locale-shots and a11y-audit: screenshots captured" }, + + "convert_s": { "type": "number", "description": "model-convert: seconds for one output format" }, + "outputs": { "type": "number", "description": "model-convert: how many artifacts the conversion produced" }, + "items": { "type": "number", "description": "dataset-prep: items in the prepared manifest, after dedupe" }, + + "exit_code": { "type": "number", "description": "shell: the script's exit status. Named rather than folded into ok, because a script that exits 3 deliberately is not the same as one that could not start" }, + + "push_latency_ms": { "type": "number", "description": "push-latency: send to arrival, measured on the device that received it" }, + "push_delivered_pct": { "type": "number", "description": "push-latency: fraction of sends that arrived at all, which on Android is the number that varies by vendor" }, + "app_state": { + "enum": ["foreground", "background", "doze", "terminated"], + "description": "push-latency: what the app was doing when the push landed. A delivery latency without this is not comparable to anything, because doze is where the vendor forks differ" + }, "drain_pct_per_h": { "type": "number", "description": "drain: battery percent consumed per hour" }, "diff_pct": { "type": "number", "description": "web-shots: percent of pixels differing from the accepted baseline (100 when the dimensions changed)" }, diff --git a/collector/scripts/smoke.ts b/collector/scripts/smoke.ts index 24302a1..c0cdc74 100644 --- a/collector/scripts/smoke.ts +++ b/collector/scripts/smoke.ts @@ -2391,5 +2391,47 @@ runEvalChecks(check); // ever appear on real hardware. runDeviceParserChecks(check); +// --- live mirror --- + +// Nothing here is persisted, so the assertions are about the states a viewer +// can be in rather than about stored rows. The one that matters is the first: +// a job with no producer must 404 rather than open a stream that never paints, +// because an that never paints is indistinguishable from a device with a +// black screen. +{ + const MJOB = `smoke-mirror-${run}`; + const before = await json("GET", `/api/jobs/${MJOB}/mirror/status`); + check("no stream before a frame arrives", before.body?.streaming === false, JSON.stringify(before.body)); + + const viewer = await fetch(`${BASE}/api/jobs/${MJOB}/mirror`); + check("a viewer on a job with no producer gets 404", viewer.status === 404, `status=${viewer.status}`); + await viewer.body?.cancel().catch(() => {}); + + // The smallest thing that is still a JPEG. + const frame = Buffer.from( + "/9j/4AAQSkZJRgABAQEAYABgAAD/2wBDAAgGBgcGBQgHBwcJCQgKDBQNDAsLDBkSEw8UHRofHh0aHBwgJC4nICIsIxwcKDcpLDAxNDQ0Hyc5PTgyPC4zNDL/wAALCAABAAEBAREA/8QAFAABAAAAAAAAAAAAAAAAAAAACf/EABQQAQAAAAAAAAAAAAAAAAAAAAD/2gAIAQEAAD8AKp//2Q==", + "base64", + ); + const post = await fetch(`${BASE}/api/jobs/${MJOB}/mirror`, { + method: "POST", headers: { "content-type": "image/jpeg" }, body: frame, + }); + check("a frame is accepted", post.status === 204, `status=${post.status}`); + + const after = await json("GET", `/api/jobs/${MJOB}/mirror/status`); + check("the stream exists once a frame has arrived", after.body?.streaming === true, JSON.stringify(after.body)); + + // A frame past the cap is refused rather than truncated: half a JPEG renders + // as a corrupt image and reads as a broken device. + const huge = await fetch(`${BASE}/api/jobs/${MJOB}/mirror`, { + method: "POST", headers: { "content-type": "image/jpeg" }, body: Buffer.alloc(3 * 1024 * 1024), + }); + check("an oversized frame is refused", huge.status === 413, `status=${huge.status}`); + + const empty = await fetch(`${BASE}/api/jobs/${MJOB}/mirror`, { + method: "POST", headers: { "content-type": "image/jpeg" }, body: Buffer.alloc(0), + }); + check("an empty frame is refused", empty.status === 400, `status=${empty.status}`); +} + console.log(failures === 0 ? "\nsmoke: ALL PASS" : `\nsmoke: ${failures} FAILURE(S)`); process.exit(failures === 0 ? 0 : 1); diff --git a/collector/src/api/index.ts b/collector/src/api/index.ts index 4efe0c8..7b08494 100644 --- a/collector/src/api/index.ts +++ b/collector/src/api/index.ts @@ -9,6 +9,7 @@ // predict the shape import type { FastifyInstance } from "fastify"; import { registerDevices } from "./devices.js"; +import { registerMirror } from "./mirror.js"; import { registerEnroll } from "./enroll.js"; import { registerEvals } from "./evals.js"; import { registerJobs } from "./jobs.js"; @@ -33,6 +34,7 @@ type MatchingDevices = ( export function registerApi(app: FastifyInstance, announce: Announce, matchingDevices: MatchingDevices) { registerOverview(app); registerDevices(app); + registerMirror(app); registerEnroll(app); registerJobs(app); registerResults(app); diff --git a/collector/src/api/mirror.ts b/collector/src/api/mirror.ts new file mode 100644 index 0000000..a88904a --- /dev/null +++ b/collector/src/api/mirror.ts @@ -0,0 +1,169 @@ +// Watching a device while a host job drives it. +// +// The UI-test artifact viewer answers "what did it look like when it failed". +// This answers "what is it doing right now", which is a different question and +// the one you actually have while a flow is running. Reading a JUnit file to +// work out where a Maestro flow got stuck is slower than watching it get stuck. +// +// Three decisions shape the whole module. +// +// **Nothing is persisted.** Frames live in a bounded in-memory ring per job and +// are dropped when the job ends. The collector runs on a 2016 MacBook whose +// database already carries every measurement the fleet has ever taken; adding a +// video stream to it would trade a permanent cost for a transient benefit. If a +// run is worth keeping, the executor already uploads a recording as an artifact +// on failure — that is the durable path, and this is not it. +// +// **The producer pushes, the browser pulls.** The executor POSTs JPEG frames as +// it captures them; browsers read an MJPEG stream. MJPEG rather than SSE or a +// WebSocket because an `` tag renders it with no client code at all, it +// degrades to a still frame rather than an error when the producer stalls, and +// the dashboard is a single-operator tool where a dependency-free path is worth +// more than efficiency. +// +// **A viewer must never slow a run down.** Every write to a viewer socket is +// best-effort and a broken pipe only drops that viewer. A closed browser tab +// must not be able to make a device test fail, so nothing here propagates an +// error back to the producer. +import type { FastifyInstance, FastifyReply } from "fastify"; +import { requireToken } from "./guard.js"; + +// One frame at a time is all a viewer can see; a couple more absorb the jitter +// between a producer's capture loop and a viewer's render. Beyond that a slow +// viewer would only ever be shown stale frames, so the ring stays tiny and the +// newest frame always wins. +const RING = 3; +// A producer that stops sending has finished, crashed, or lost its device. +// After this, viewers are closed rather than left watching a frozen image that +// looks like a hung app rather than a hung stream. +const PRODUCER_IDLE_MS = 15_000; +// A cap so a runaway producer cannot exhaust memory: roughly a large phone +// screenshot. Anything bigger is refused rather than truncated, because half a +// JPEG renders as a corrupt image and reads as a broken device. +const MAX_FRAME_BYTES = 2 * 1024 * 1024; +const BOUNDARY = "fleetframe"; + +type Stream = { + frames: Buffer[]; + lastAt: number; + viewers: Set; + timer?: NodeJS.Timeout; +}; + +const streams = new Map(); + +function get(jobId: string): Stream { + let s = streams.get(jobId); + if (!s) { + s = { frames: [], lastAt: Date.now(), viewers: new Set() }; + streams.set(jobId, s); + } + return s; +} + +function writeFrame(reply: FastifyReply, frame: Buffer) { + try { + reply.raw.write( + `--${BOUNDARY}\r\nContent-Type: image/jpeg\r\nContent-Length: ${frame.length}\r\n\r\n`, + ); + reply.raw.write(frame); + reply.raw.write("\r\n"); + } catch { + // The viewer is gone. Its own close handler removes it; nothing propagates + // to the producer, which is mid-test and must not care. + } +} + +/** Close a stream out: end every viewer and forget the job. */ +function close(jobId: string) { + const s = streams.get(jobId); + if (!s) return; + if (s.timer) clearInterval(s.timer); + for (const v of s.viewers) { + try { + v.raw.end(); + } catch { + /* already gone */ + } + } + streams.delete(jobId); +} + +/** Called from the job-close path so a finished job stops streaming. */ +export function endMirror(jobId: string) { + close(jobId); +} + +export function registerMirror(app: FastifyInstance) { + // The producer. Raw JPEG bytes, one frame per request: the executor is + // shelling out to screencap or scrcpy and posting what comes back, and a + // frame-per-request needs no framing protocol on either side. + app.post("/api/jobs/:id/mirror", { bodyLimit: MAX_FRAME_BYTES }, async (req, reply) => { + if (!requireToken(req, reply)) return; + const { id } = req.params as { id: string }; + const body = req.body; + if (!Buffer.isBuffer(body) || body.length === 0) + return reply.code(400).send({ error: "expected raw JPEG bytes" }); + if (body.length > MAX_FRAME_BYTES) + return reply.code(413).send({ error: `frame larger than ${MAX_FRAME_BYTES} bytes` }); + + const s = get(id); + s.frames.push(body); + while (s.frames.length > RING) s.frames.shift(); + s.lastAt = Date.now(); + + for (const v of s.viewers) writeFrame(v, body); + + // Started on first frame rather than on stream creation, so a job that + // never produces one costs nothing. + if (!s.timer) { + s.timer = setInterval(() => { + const cur = streams.get(id); + if (cur && Date.now() - cur.lastAt > PRODUCER_IDLE_MS) close(id); + }, 5_000); + s.timer.unref(); + } + return reply.code(204).send(); + }); + + // The viewer. An renders this directly. + app.get("/api/jobs/:id/mirror", async (req, reply) => { + const { id } = req.params as { id: string }; + const s = streams.get(id); + // 404 rather than an empty stream: the dashboard uses this to decide + // whether to show the panel at all, and an that never paints is + // indistinguishable from a device with a black screen. + if (!s) return reply.code(404).send({ error: "nothing streaming for this job" }); + + reply.raw.writeHead(200, { + "content-type": `multipart/x-mixed-replace; boundary=${BOUNDARY}`, + "cache-control": "no-store", + connection: "close", + // The frames are already stale by the time they arrive; a proxy holding + // them makes the picture wrong rather than merely late. + pragma: "no-cache", + }); + + s.viewers.add(reply); + // Whatever the producer sent most recently, so a viewer joining mid-run + // paints immediately instead of waiting for the next capture. + const latest = s.frames[s.frames.length - 1]; + if (latest) writeFrame(reply, latest); + + const remove = () => s.viewers.delete(reply); + req.raw.on("close", remove); + req.raw.on("error", remove); + return reply; + }); + + // Whether a panel is worth showing, without opening a stream to find out. + app.get("/api/jobs/:id/mirror/status", async (req) => { + const { id } = req.params as { id: string }; + const s = streams.get(id); + return { + streaming: !!s, + viewers: s?.viewers.size ?? 0, + age_ms: s ? Date.now() - s.lastAt : null, + }; + }); +} diff --git a/collector/src/executor.ts b/collector/src/executor.ts index bd8b61d..91d41bd 100644 --- a/collector/src/executor.ts +++ b/collector/src/executor.ts @@ -40,10 +40,12 @@ import { parseXcuiDebugDescription, type A11yGeometry, type A11yNode, } from "./a11y-tree.js"; import { countBySeverity, uploadReport, type Finding } from "./web/shared.js"; +import { ADB, batteryPct, hasApp, launchApp, processAlive } from "./workloads/device.js"; +import { discoverWorkloads, loadRun, type LoadedWorkload } from "./workloads/registry.js"; +import type { Target, WorkloadCtx } from "./workloads/types.js"; const FLOWS_DIR = process.env.FLEET_FLOWS_DIR ?? path.resolve("flows"); const MAESTRO = process.env.MAESTRO_BIN ?? path.join(os.homedir(), ".maestro/bin/maestro"); -const ADB = process.env.ADB_BIN ?? "adb"; export type Job = { job_id: string; @@ -91,7 +93,10 @@ const WEB_SPECS_DIR = process.env.FLEET_WEB_SPECS_DIR ?? path.resolve("web-specs const IOS_PROJECT = process.env.FLEET_IOS_PROJECT ?? path.resolve("../runner-ios/FleetRunner.xcodeproj"); -type Target = { id: string; platform: "android" | "ios"; kind?: "device" | "simulator" }; +// One definition, in workloads/types.ts, so a handler that has moved out and a +// handler that has not are talking about the same thing. Re-exported because +// src/web/* and the workload directories import their types from here. +export type { Target }; // Exclusive jobs hold the collector's device locks while they run, so a // device-executor agent never gets handed work mid-UI-test. @@ -225,69 +230,6 @@ async function listTargets(ios?: IosDeviceInfo[]): Promise { return [...android, ...sims, ...phones.filter((t) => !seen.has(t.id))]; } -async function hasApp(target: Target, appId: string): Promise { - try { - if (target.platform === "android") { - await exec(ADB, ["-s", target.id, "shell", "pm", "path", appId], { timeout: 15_000 }); - } else if (target.kind === "device") { - const out = path.join(mkdtempSync(path.join(os.tmpdir(), "fleet-dc-")), "apps.json"); - await exec("xcrun", ["devicectl", "device", "info", "apps", "--device", target.id, "--json-output", out], { timeout: 30_000 }); - const parsed = JSON.parse(readFileSync(out, "utf8")) as { result?: { apps?: { bundleIdentifier: string }[] } }; - return (parsed.result?.apps ?? []).some((a) => a.bundleIdentifier === appId); - } else { - await exec("xcrun", ["simctl", "get_app_container", target.id, appId], { timeout: 15_000 }); - } - return true; - } catch { - return false; - } -} - -async function runInstall(job: Job) { - const app = job.app; - if (!app) throw new Error("install job needs an app ref"); - const platform = app.platform ?? "android"; - const targets = await selectTargets(job, (await listTargets()).filter((t) => t.platform === platform)); - if (targets.length === 0) throw new Error(`no ${platform} targets matched this job`); - - const dir = mkdtempSync(path.join(os.tmpdir(), "fleet-")); - let installable: string; - if (platform === "android") { - installable = path.join(dir, `${app.name}.apk`); - await fetchArtifact(app.sha256, installable); - } else { - // iOS artifacts are zips of the .app bundle (a directory can't be a raw artifact). - const zip = path.join(dir, `${app.name}.zip`); - await fetchArtifact(app.sha256, zip); - await exec("ditto", ["-x", "-k", zip, dir], { timeout: 120_000 }); - const appDir = readdirSync(dir).find((f) => f.endsWith(".app")); - if (!appDir) throw new Error("no .app bundle inside iOS artifact zip"); - installable = path.join(dir, appDir); - } - - let allOk = true; - for (const target of targets) { - let ok = true; - let error: string | undefined; - try { - if (platform === "android") { - await exec(ADB, ["-s", target.id, "install", "-r", installable], { timeout: 120_000 }); - } else if (target.kind === "device") { - await exec("xcrun", ["devicectl", "device", "install", "app", "--device", target.id, installable], { timeout: 300_000 }); - } else { - await exec("xcrun", ["simctl", "install", target.id, installable], { timeout: 120_000 }); - } - } catch (e) { - ok = false; - allOk = false; - error = (e as Error).message.slice(0, 300); - } - await postResult({ job_id: job.job_id, device_id: target.id, iter: 0, ok, error }); - log(`install ${app.name}@${app.build} on ${target.id} (${platform}): ${ok ? "ok" : "FAILED"}`); - } - await postResult({ job_id: job.job_id, device_id: `host:${NAME}`, iter: 0, final: true, ok: allOk }); -} - function parseJunit(xml: string): { passed: number; failed: number } { const m = /tests="(\d+)"[^>]*failures="(\d+)"/.exec(xml); if (!m) return { passed: 0, failed: 1 }; @@ -739,73 +681,6 @@ async function runUiTest(job: Job) { await postResult({ job_id: job.job_id, device_id: `host:${NAME}`, iter: 0, final: true, ok: allOk }); } -/** - * `args` are process launch arguments, and they exist for locale-shots: on iOS - * they land in the app's NSUserDefaults, which is how `-AppleLanguages (ar)` - * reaches an app without touching the device's own language at all. Android has - * no equivalent -- `am start` extras are not read as preferences -- so the - * Android path ignores them rather than pretending otherwise. - */ -async function launchApp(target: Target, appId: string, args: string[] = []) { - if (target.platform === "ios" && target.kind === "device") { - await exec("xcrun", ["devicectl", "device", "process", "launch", "--device", target.id, appId, ...args], { timeout: 60_000 }); - return; - } - if (target.platform === "android") { - // Resolve the real launcher activity; monkey is a fallback because some - // images (ATD) resolve but throttle monkey events. - try { - const { stdout } = await exec( - ADB, ["-s", target.id, "shell", "cmd", "package", "resolve-activity", "--brief", appId], - { timeout: 15_000 }, - ); - const component = stdout.trim().split("\n").pop()?.trim(); - if (!component || !component.includes("/")) throw new Error(`unresolvable: ${stdout}`); - await exec(ADB, ["-s", target.id, "shell", "am", "start", "-n", component], { timeout: 30_000 }); - } catch { - await exec(ADB, ["-s", target.id, "shell", "monkey", "-p", appId, "-c", - "android.intent.category.LAUNCHER", "1"], { timeout: 30_000 }); - } - } else { - await exec("xcrun", ["simctl", "launch", target.id, appId, ...args], { timeout: 60_000 }); - } -} - -async function processAlive(target: Target, appId: string): Promise { - try { - if (target.platform === "android") { - const { stdout } = await exec(ADB, ["-s", target.id, "shell", "pidof", appId], { timeout: 15_000 }); - return stdout.trim().length > 0; - } - const { stdout } = await exec("xcrun", ["simctl", "spawn", target.id, "launchctl", "list"], { timeout: 15_000 }); - return stdout.includes(appId); - } catch { - return false; - } -} - -async function batteryPct(target: Target): Promise { - try { - if (target.platform === "android") { - const { stdout } = await exec(ADB, ["-s", target.id, "shell", "dumpsys", "battery"], { timeout: 15_000 }); - const m = /level:\s*(\d+)/.exec(stdout); - return m ? Number(m[1]) : null; - } - if (target.kind === "device") { - // devicectl exposes battery via device info; the fleet runner's beacon - // is the primary source on real iPhones — this is the host-side cross-check. - const out = path.join(mkdtempSync(path.join(os.tmpdir(), "fleet-dc-")), "info.json"); - await exec("xcrun", ["devicectl", "device", "info", "details", "--device", target.id, "--json-output", out], { timeout: 30_000 }); - const txt = readFileSync(out, "utf8"); - const m = /"batteryLevel"\s*:\s*([0-9.]+)/.exec(txt); - return m ? Math.round(Number(m[1]) * (Number(m[1]) <= 1 ? 100 : 1)) : null; - } - return null; // simulators have no battery - } catch { - return null; - } -} - // Location replay: feed the device a recorded route so a drain run walks the // same path every night with the real GPS radio on. Simulators take a GPX // file directly; Android uses the mock-location provider (the app under test @@ -2967,8 +2842,94 @@ async function restoreDevicesOnStartup() { } } +// --------------------------------------------------------------------------- +// The workload loader +// --------------------------------------------------------------------------- + +/** + * The one ctx, built once from the real collector client. + * + * This object is the entire reason a moved handler is testable: it is the only + * route a workload has to the collector, the artifact store, the device list + * and the Keychain, so a test can hand the same handler a different one and + * watch what it posts. Nothing here is new behaviour — every field is the + * function the handlers were already calling as a module-level import. + */ +const CTX: WorkloadCtx = { + host: NAME, + log, + postResult, + postBeacon, + fetchArtifact, + uploadArtifact, + // Wrapped rather than passed by reference: listTargets takes an optional + // pre-fetched devicectl list, which only reportAttached has a reason to + // supply. A workload asking "what is attached" should not have to know that. + listTargets: () => listTargets(), + selectTargets, + leaseBudgetS, + secrets: { credentialsFor: resolveCredentials, redact }, +}; + +/** + * The workload directories, scanned once at startup. + * + * At startup rather than per job so that a stranger who mistypes a manifest + * hears about it when they restart the executor, instead of at 3am when the + * nightly that needed it fails. `log` is passed in so the complaints appear in + * this executor's log with everything else. + */ +const LOADED = discoverWorkloads(log); + +/** + * Send a job to whoever handles it. + * + * A lookup on the loaded directories first, then the if/else below for the + * handlers that still live in this file. The fallback is not a transitional + * embarrassment to be tidied away quickly — it is what lets a handler move on + * its own, verified on its own, while the other eleven keep running exactly as + * they did. When the chain is empty the chain goes; until then, a workload is + * dispatched by whichever of the two knows it, and a name in neither gets the + * same "not supported by this executor" row it always did. + */ +async function dispatch(job: Job, loaded: Map): Promise { + const w = loaded.get(job.workload); + if (w) { + const run = await loadRun(w); + await run(job, CTX); + return; + } + + if (job.workload === "ui-test") await runUiTest(job); + // Same implementation, and deliberately so: `soak` with no params.flow is + // exactly the old behaviour, so the existing nightlies keep meaning what + // they meant. + else if (job.workload === "soak" || job.workload === "app-soak") await runAppSoak(job); + else if (job.workload === "locale-shots") await runLocaleShots(job); + else if (job.workload === "a11y-audit") await runA11yAudit(job); + else if (job.workload === "drain") await runDrain(job); + else if (job.workload === "cold-start") await runColdStart(job); + else if (job.workload === "web-test") await runWebTest(job); + else if (job.workload === "web-shots") await runWebShots(job); + else if (job.workload === "web-audit") await runWebAudit(job); + else if (job.workload === "web-unfurl") await runWebUnfurl(job); + else if (job.workload === "archive") await runArchive(job); + else if (job.workload === "digest") await runDigest(job); + else { + await postResult({ + job_id: job.job_id, device_id: `host:${NAME}`, iter: 0, final: true, ok: false, + error: `workload '${job.workload}' not supported by this executor yet`, + }); + } +} + async function main() { log(`polling ${BASE} (flows: ${FLOWS_DIR})`); + log( + LOADED.size > 0 + ? `workloads loaded from src/workloads: ${[...LOADED.keys()].sort().join(", ")}` + : "no workload directories found; every job falls back to the built-in handlers", + ); await restoreDevicesOnStartup(); // The dashboard calls a device online for ONLINE_S seconds after it was last // seen. Refreshing on a timer rather than per poll keeps presence steady @@ -2990,28 +2951,7 @@ async function main() { log(`claimed ${job.job_id} (${job.workload})`); try { - if (job.workload === "install") await runInstall(job); - else if (job.workload === "ui-test") await runUiTest(job); - // Same implementation, and deliberately so: `soak` with no params.flow is - // exactly the old behaviour, so the existing nightlies keep meaning what - // they meant. - else if (job.workload === "soak" || job.workload === "app-soak") await runAppSoak(job); - else if (job.workload === "locale-shots") await runLocaleShots(job); - else if (job.workload === "a11y-audit") await runA11yAudit(job); - else if (job.workload === "drain") await runDrain(job); - else if (job.workload === "cold-start") await runColdStart(job); - else if (job.workload === "web-test") await runWebTest(job); - else if (job.workload === "web-shots") await runWebShots(job); - else if (job.workload === "web-audit") await runWebAudit(job); - else if (job.workload === "web-unfurl") await runWebUnfurl(job); - else if (job.workload === "archive") await runArchive(job); - else if (job.workload === "digest") await runDigest(job); - else { - await postResult({ - job_id: job.job_id, device_id: `host:${NAME}`, iter: 0, final: true, ok: false, - error: `workload '${job.workload}' not supported by this executor yet`, - }); - } + await dispatch(job, LOADED); } catch (e) { await postResult({ job_id: job.job_id, device_id: `host:${NAME}`, iter: 0, final: true, ok: false, diff --git a/collector/src/server.ts b/collector/src/server.ts index 2e9ec70..bc72739 100644 --- a/collector/src/server.ts +++ b/collector/src/server.ts @@ -36,6 +36,7 @@ import { } from "./api/mutations.js"; import { registerDashStatic } from "./dash-static.js"; import { startPowerSampler } from "./power.js"; +import { endMirror } from "./api/mirror.js"; mkdirSync(ARTIFACT_DIR, { recursive: true }); @@ -63,6 +64,9 @@ const LONG_LEASE_TTL_S = 4 * 60 * 60; // workloads chew through a corpus on a phone. const LONG_LEASE_WORKLOADS = new Set([ "drain", "soak", "thermal", "build", "app-soak", "speech-eval", "embed-eval", + // A conversion quantises a multi-GB checkpoint; a dataset is downloaded and + // resized; a served model is up for as long as someone is using it. + "model-convert", "dataset-prep", "serve", "push-latency", ]); const MAX_LEASE_TTL_S = 24 * 60 * 60; // Agents beacon every 60 s. Two missed beacons and a state claim is no longer @@ -76,6 +80,11 @@ const app = Fastify({ logger: { level: process.env.FLEET_LOG ?? "info" } }); // Artifact uploads arrive as raw bytes and are streamed to disk while the // hash is computed — a multi-GB GGUF never lives in collector memory. app.addContentTypeParser("application/octet-stream", (_req, payload, done) => done(null, payload)); +// Mirror frames, unlike artifacts, are small, capped, and consumed immediately +// rather than streamed to disk — so they are buffered here. Kept to its own +// content type precisely so the artifact path above keeps streaming: a +// multi-GB GGUF must never be buffered into collector memory. +app.addContentTypeParser("image/jpeg", { parseAs: "buffer" }, (_req, body, done) => done(null, body)); type JobSpec = { schema: number; @@ -149,6 +158,8 @@ const WORKLOADS = new Set([ // them; the rest are host work. All are listed because the collector renders // their results, and a workload the dashboard draws is one it knows about. "build", "speech-eval", "embed-eval", "vantage", "locale-shots", "app-soak", "a11y-audit", + // Wave 3: pipelines that feed the evals, and the workloads that need hardware. + "model-convert", "dataset-prep", "serve", "shell", "push-latency", "camera-eval", "desktop-ui-test", ]); function touchDevice(deviceId: string) { @@ -902,6 +913,9 @@ app.post("/results", async (req, reply) => { const status = b.ok === false ? "failed" : "done"; const settled = closeJobTx(b.job_id, status); + // A finished job has nothing left to show. Outside the transaction, because + // closing viewer sockets must not be able to roll back a device's result. + endMirror(b.job_id); announce({ type: "job", job_id: b.job_id, status, device_id: b.device_id }); // Announced after the transaction commits: a broken browser pipe must not // roll back a device's result. diff --git a/collector/src/workloads/device.ts b/collector/src/workloads/device.ts new file mode 100644 index 0000000..e743296 --- /dev/null +++ b/collector/src/workloads/device.ts @@ -0,0 +1,119 @@ +/** + * The device plumbing more than one workload needs. + * + * These used to be private functions in executor.ts, which is why they could + * only be used by handlers that also lived in executor.ts. They are here, and + * not in a workload's own directory, for exactly one reason: something other + * than that workload calls them. Anything a single workload uses belongs in + * that workload's directory instead, where its blast radius is one job type. + * + * executor.ts imports them back, so the handlers still waiting to move keep + * calling the same functions they always did. There is one definition of "is + * this app installed", and it answers the same on both sides of the move. + */ +import { mkdtempSync, readFileSync } from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { exec } from "../fleet-client.js"; +import type { Target } from "./types.js"; + +/** + * adb, or wherever this host keeps it. A LaunchAgent's PATH is not a login + * shell's, so the fleet's Android SDK hosts set ADB_BIN rather than hoping. + */ +export const ADB = process.env.ADB_BIN ?? "adb"; + +/** + * Is `appId` installed on this target? + * + * False on any failure, including "no adb on this host", because every caller + * uses it to decide whether to skip a device rather than whether to fail the + * job — and a skipped device with a stated reason is a better answer than a + * job that died asking. + */ +export async function hasApp(target: Target, appId: string): Promise { + try { + if (target.platform === "android") { + await exec(ADB, ["-s", target.id, "shell", "pm", "path", appId], { timeout: 15_000 }); + } else if (target.kind === "device") { + const out = path.join(mkdtempSync(path.join(os.tmpdir(), "fleet-dc-")), "apps.json"); + await exec("xcrun", ["devicectl", "device", "info", "apps", "--device", target.id, "--json-output", out], { timeout: 30_000 }); + const parsed = JSON.parse(readFileSync(out, "utf8")) as { result?: { apps?: { bundleIdentifier: string }[] } }; + return (parsed.result?.apps ?? []).some((a) => a.bundleIdentifier === appId); + } else { + await exec("xcrun", ["simctl", "get_app_container", target.id, appId], { timeout: 15_000 }); + } + return true; + } catch { + return false; + } +} + +/** + * Launch the app, three ways, because three platforms. + * + * `args` reach a simulator or a real iPhone as process arguments, which is how + * a launch argument like -AppleLanguages reaches an app without touching the + * device's own language at all. Android has no equivalent -- `am start` extras + * are not read as preferences -- so the Android path ignores them rather than + * pretending otherwise. + */ +export async function launchApp(target: Target, appId: string, args: string[] = []) { + if (target.platform === "ios" && target.kind === "device") { + await exec("xcrun", ["devicectl", "device", "process", "launch", "--device", target.id, appId, ...args], { timeout: 60_000 }); + return; + } + if (target.platform === "android") { + // Resolve the real launcher activity; monkey is a fallback because some + // images (ATD) resolve but throttle monkey events. + try { + const { stdout } = await exec( + ADB, ["-s", target.id, "shell", "cmd", "package", "resolve-activity", "--brief", appId], + { timeout: 15_000 }, + ); + const component = stdout.trim().split("\n").pop()?.trim(); + if (!component || !component.includes("/")) throw new Error(`unresolvable: ${stdout}`); + await exec(ADB, ["-s", target.id, "shell", "am", "start", "-n", component], { timeout: 30_000 }); + } catch { + await exec(ADB, ["-s", target.id, "shell", "monkey", "-p", appId, "-c", + "android.intent.category.LAUNCHER", "1"], { timeout: 30_000 }); + } + } else { + await exec("xcrun", ["simctl", "launch", target.id, appId, ...args], { timeout: 60_000 }); + } +} + +export async function processAlive(target: Target, appId: string): Promise { + try { + if (target.platform === "android") { + const { stdout } = await exec(ADB, ["-s", target.id, "shell", "pidof", appId], { timeout: 15_000 }); + return stdout.trim().length > 0; + } + const { stdout } = await exec("xcrun", ["simctl", "spawn", target.id, "launchctl", "list"], { timeout: 15_000 }); + return stdout.includes(appId); + } catch { + return false; + } +} + +export async function batteryPct(target: Target): Promise { + try { + if (target.platform === "android") { + const { stdout } = await exec(ADB, ["-s", target.id, "shell", "dumpsys", "battery"], { timeout: 15_000 }); + const m = /level:\s*(\d+)/.exec(stdout); + return m ? Number(m[1]) : null; + } + if (target.kind === "device") { + // devicectl exposes battery via device info; the fleet runner's beacon + // is the primary source on real iPhones — this is the host-side cross-check. + const out = path.join(mkdtempSync(path.join(os.tmpdir(), "fleet-dc-")), "info.json"); + await exec("xcrun", ["devicectl", "device", "info", "details", "--device", target.id, "--json-output", out], { timeout: 30_000 }); + const txt = readFileSync(out, "utf8"); + const m = /"batteryLevel"\s*:\s*([0-9.]+)/.exec(txt); + return m ? Math.round(Number(m[1]) * (Number(m[1]) <= 1 ? 100 : 1)) : null; + } + return null; // simulators have no battery + } catch { + return null; + } +} diff --git a/collector/src/workloads/install/index.ts b/collector/src/workloads/install/index.ts new file mode 100644 index 0000000..ca639b6 --- /dev/null +++ b/collector/src/workloads/install/index.ts @@ -0,0 +1,61 @@ +// install: put a build on every device this job selected. +// +// The simplest workload in the fleet, and the one everything else depends on — +// a ui-test, a drain and a cold-start all assume somebody installed the build +// first. It fetches the artifact once, on the host, and pushes the same file to +// each target rather than downloading per device. +// +// One result row per device, failures included, then a final row that is ok +// only if every install was. A device that refused the install is a result, not +// an exception: the other nine phones installed it and the job should say so. +import { mkdtempSync, readdirSync } from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { exec } from "../../fleet-client.js"; +import { ADB } from "../device.js"; +import type { Job, WorkloadCtx } from "../types.js"; + +export async function run(job: Job, ctx: WorkloadCtx): Promise { + const app = job.app; + if (!app) throw new Error("install job needs an app ref"); + const platform = app.platform ?? "android"; + const targets = await ctx.selectTargets(job, (await ctx.listTargets()).filter((t) => t.platform === platform)); + if (targets.length === 0) throw new Error(`no ${platform} targets matched this job`); + + const dir = mkdtempSync(path.join(os.tmpdir(), "fleet-")); + let installable: string; + if (platform === "android") { + installable = path.join(dir, `${app.name}.apk`); + await ctx.fetchArtifact(app.sha256, installable); + } else { + // iOS artifacts are zips of the .app bundle (a directory can't be a raw artifact). + const zip = path.join(dir, `${app.name}.zip`); + await ctx.fetchArtifact(app.sha256, zip); + await exec("ditto", ["-x", "-k", zip, dir], { timeout: 120_000 }); + const appDir = readdirSync(dir).find((f) => f.endsWith(".app")); + if (!appDir) throw new Error("no .app bundle inside iOS artifact zip"); + installable = path.join(dir, appDir); + } + + let allOk = true; + for (const target of targets) { + let ok = true; + let error: string | undefined; + try { + if (platform === "android") { + await exec(ADB, ["-s", target.id, "install", "-r", installable], { timeout: 120_000 }); + } else if (target.kind === "device") { + await exec("xcrun", ["devicectl", "device", "install", "app", "--device", target.id, installable], { timeout: 300_000 }); + } else { + await exec("xcrun", ["simctl", "install", target.id, installable], { timeout: 120_000 }); + } + } catch (e) { + ok = false; + allOk = false; + error = (e as Error).message.slice(0, 300); + } + await ctx.postResult({ job_id: job.job_id, device_id: target.id, iter: 0, ok, error }); + ctx.log(`install ${app.name}@${app.build} on ${target.id} (${platform}): ${ok ? "ok" : "FAILED"}`); + } + await ctx.postResult({ job_id: job.job_id, device_id: `host:${ctx.host}`, iter: 0, final: true, ok: allOk }); +} diff --git a/collector/src/workloads/install/manifest.json b/collector/src/workloads/install/manifest.json new file mode 100644 index 0000000..fb80632 --- /dev/null +++ b/collector/src/workloads/install/manifest.json @@ -0,0 +1,15 @@ +{ + "name": "install", + "capability": "devices", + "executor": "host", + "description": "Install a build on every attached device and verify it launches.", + "params": { + "type": "object", + "additionalProperties": false, + "properties": {}, + "description": "install takes no params. What it installs comes from `app` (name, build, sha256, platform) and where it lands comes from `targets`." + }, + "example": { + "app": { "name": "", "build": "", "sha256": "" } + } +} diff --git a/collector/src/workloads/registry.ts b/collector/src/workloads/registry.ts new file mode 100644 index 0000000..3e2ba0b --- /dev/null +++ b/collector/src/workloads/registry.ts @@ -0,0 +1,120 @@ +/** + * The loader. Finds workloads by looking, not by being told. + * + * A directory next to this file with a `manifest.json` and an `index.ts` in it + * IS a workload — there is no list to add a name to, which is the whole point: + * the register-it-here line is exactly the line that put every workload author + * into a merge conflict with every other workload author. + * + * Two rules the scan follows, both of them about not making a bad day worse: + * + * 1. A broken workload is skipped, loudly, and the executor keeps running. + * A manifest with a typo in it must not stop a host from claiming the + * twelve jobs that have nothing to do with it. The complaint goes to the + * log, and the job that actually wanted that workload fails with the reason + * on its own result row. + * + * 2. The scan happens once, at startup, but the module is imported lazily on + * first use. Knowing what exists is cheap and worth doing before any job + * arrives (so a stranger's typo is visible immediately rather than at 3am + * when the nightly hits it); loading the code is not, since a host may go + * weeks without ever being handed a web-test. + */ +import { readdirSync, readFileSync } from "node:fs"; +import path from "node:path"; +import { fileURLToPath, pathToFileURL } from "node:url"; +import type { WorkloadManifest, WorkloadRun } from "./types.js"; + +// Resolved from this module, not from cwd: the executor is started by a +// LaunchAgent whose working directory is not this repo, and `path.resolve` on a +// relative name would find nothing there. fileURLToPath rather than stripping +// "file://" by hand — this repo lives under a path with a space in it, and the +// URL form percent-encodes it. +const WORKLOADS_DIR = path.dirname(fileURLToPath(import.meta.url)); + +const CAPABILITIES = new Set(["devices", "browsers", "network"]); + +export type LoadedWorkload = { + manifest: WorkloadManifest; + /** Absolute path to the module, for the dynamic import and for error messages. */ + entry: string; +}; + +/** Why a directory that looked like a workload is not one, or null if it is. */ +function manifestProblem(name: string, m: unknown): string | null { + if (typeof m !== "object" || m === null) return "manifest.json is not an object"; + const o = m as Record; + // The name is duplicated between the directory and the manifest on purpose: + // the directory is what a person sees and the manifest is what is served to + // the dashboard, and a workload answering to two names is the kind of bug + // that is only ever found by someone whose job did not run. + if (o.name !== name) return `manifest.json says name ${JSON.stringify(o.name)}, but the directory is ${JSON.stringify(name)}`; + if (typeof o.capability !== "string" || !CAPABILITIES.has(o.capability)) { + return `capability must be one of ${[...CAPABILITIES].join(", ")}, not ${JSON.stringify(o.capability)}`; + } + if (typeof o.description !== "string" || o.description.length === 0) return "description is missing"; + if (o.executor !== "host" && o.executor !== "device") return "executor must be \"host\" or \"device\""; + if (typeof o.params !== "object" || o.params === null) return "params must be a JSON Schema object"; + if (typeof o.example !== "object" || o.example === null) return "example must be a spec object"; + return null; +} + +/** + * Every workload directory that parses, keyed by name. + * + * `onProblem` rather than a throw or a bare console.log: the executor has its + * own prefixed logger and this module should not decide what the complaint + * looks like, and a test wants to assert on the complaints rather than read + * them. + */ +export function discoverWorkloads( + onProblem: (msg: string) => void = () => {}, + dir: string = WORKLOADS_DIR, +): Map { + const found = new Map(); + let entries; + try { + entries = readdirSync(dir, { withFileTypes: true }); + } catch (e) { + onProblem(`cannot read the workloads directory (${(e as Error).message}); no workloads are loadable`); + return found; + } + for (const e of entries) { + // Files next to the directories — types.ts, this file, the tests — are the + // shared plumbing, not workloads. + if (!e.isDirectory()) continue; + const manifestPath = path.join(dir, e.name, "manifest.json"); + let manifest: WorkloadManifest; + try { + manifest = JSON.parse(readFileSync(manifestPath, "utf8")) as WorkloadManifest; + } catch (err) { + onProblem(`workload "${e.name}" has no readable manifest.json (${(err as Error).message}); skipping it`); + continue; + } + const problem = manifestProblem(e.name, manifest); + if (problem) { + onProblem(`workload "${e.name}" has an invalid manifest.json: ${problem}; skipping it`); + continue; + } + found.set(manifest.name, { manifest, entry: path.join(dir, e.name, "index.js") }); + } + return found; +} + +/** + * The `run` a workload exports, imported on first use. + * + * The `.js` in the entry path is deliberate and is what NodeNext wants: under + * tsx it resolves to the neighbouring index.ts, and after a build it resolves + * to the emitted index.js. pathToFileURL rather than importing the raw path, + * again because of the space in this repo's path — a bare absolute path in a + * dynamic import is interpreted as a URL on some platforms and silently fails + * to resolve. + */ +export async function loadRun(w: LoadedWorkload): Promise { + const mod = (await import(pathToFileURL(w.entry).href)) as { run?: unknown }; + if (typeof mod.run !== "function") { + throw new Error(`workload "${w.manifest.name}" does not export a run function from ${w.entry}`); + } + return mod.run as WorkloadRun; +} diff --git a/collector/src/workloads/types.ts b/collector/src/workloads/types.ts new file mode 100644 index 0000000..8fd2822 --- /dev/null +++ b/collector/src/workloads/types.ts @@ -0,0 +1,149 @@ +/** + * What a workload is, and what it is allowed to reach for. + * + * The executor used to be one file with every handler in it and a chain of + * `else if` at the bottom. That shape had two costs. The module has a `main()` + * loop, so importing it starts polling the collector — which meant no handler + * in it could ever be unit-tested, and the only way to exercise one was to plug + * in a phone. And every new workload was another edit to the file everybody + * else was also editing, so adding one meant a merge conflict with whoever was + * fixing an unrelated handler that week. + * + * A workload is now a directory: a manifest saying what it is and what it + * needs, and an `index.ts` exporting `run`. Nothing registers it; the loader + * finds it because the directory is there. + * + * The `ctx` below is the other half of that. Handlers used to import + * `postResult`, `uploadArtifact` and the rest straight from `fleet-client.js`, + * which made every dependency ambient: you could not tell from a handler's + * signature whether it talked to the collector, moved artifacts, or read the + * Keychain, and a test had no seam to substitute any of it. Passing them in + * makes the dependency list part of the type, and lets a test hand a handler a + * fake collector and watch what it posts. + * + * What ctx deliberately does NOT carry is `exec`. Shelling out to adb, simctl, + * devicectl, xcodebuild and npx is the job, not a dependency worth abstracting: + * a fake `exec` would be a re-implementation of four toolchains, and a test + * written against it would pass while the real thing was broken. So a handler + * is testable exactly up to its first shell-out — which, in practice, covers + * every refusal, precondition and result-shaping path, and those are the ones + * that have actually been wrong. + */ +import type { Job } from "../executor.js"; + +// Type-only, and it has to stay that way: executor.ts calls main() at the +// bottom, so a runtime import of it from here would start a second poll loop +// inside whatever imported the workload. `import type` is erased, which is why +// src/web/* has always imported Job the same way. +export type { Job }; + +/** One thing this executor can drive. Mirrors executor.ts's own Target. */ +export type Target = { + id: string; + platform: "android" | "ios"; + kind?: "device" | "simulator"; +}; + +/** What a suite needs to sign in. `password` is resolved on the host, never carried in a spec. */ +export type SuiteCredentials = { + account: string; + password: string; + emailVar: string; + passwordVar: string; +}; + +/** + * The secrets seam. + * + * Narrow on purpose. A workload can ask for a suite's sign-in details and can + * scrub a string before it becomes an artifact; it cannot enumerate the + * Keychain. `credentialsFor` throws with a remedy rather than returning null on + * failure — null means "this suite did not ask to sign in", which is a + * different thing from "it asked and we could not". + */ +export type WorkloadSecrets = { + credentialsFor( + suite: NonNullable, + who: string, + ): Promise; + redact(text: string, secrets: string[]): string; +}; + +/** + * Everything a handler is given. If it is not here, a handler either does not + * need it or is shelling out for it. + */ +export type WorkloadCtx = { + /** + * This executor's name. Every workload closes with a final row on + * `host:`, because a result row needs a device_id and the host is what + * actually ran when the answer is about the job rather than a device. + */ + host: string; + /** Prefixed, line-per-event logging. The executor's log is read as a narrative. */ + log(msg: string): void; + /** Post one result row. `final: true` closes the job. */ + postResult(row: Record): Promise; + /** Post a beacon. Renews the lease as a side effect, which is often the point. */ + postBeacon(jobId: string, deviceId: string, extra: Record): Promise; + /** Fetch an artifact by hash to `dest`, verifying the hash. */ + fetchArtifact(sha256: string, dest: string): Promise; + /** Upload a file to the artifact store under `name`; returns its sha256. */ + uploadArtifact(file: string, name: string): Promise; + /** Everything attached to this host that the fleet owns. */ + listTargets(): Promise; + /** Narrow those to the ones this job asked for (device_id, kind, match). */ + selectTargets(job: Job, all: Target[]): Promise; + secrets: WorkloadSecrets; + /** + * The per-unit time budget the lease allows, in seconds. A unit of work that + * does not beacon must finish inside this, or the sweep requeues a job that + * is still running and a second executor runs it concurrently. + */ + leaseBudgetS(job: Job, fallbackS?: number): number; +}; + +/** + * The signature every `index.ts` exports. + * + * A handler reports by posting rows, not by returning: a job with ten devices + * has eleven answers and no single return value describes them. It throws only + * when the job cannot start at all — a missing app ref, a spec that names + * nothing — and the executor turns that into the failed final row. + */ +export type WorkloadRun = (job: Job, ctx: WorkloadCtx) => Promise; + +/** + * What a host must have before this workload can honestly run on it. + * + * Three, because there are three real answers on these machines: + * + * devices -- attached hardware or simulators, driven over adb / simctl / + * devicectl. Every workload whose result is about a phone. + * browsers -- a Playwright install. Present on one machine in the fleet, so + * a job needing it and landing elsewhere must say so rather than + * download a browser onto a 2016 laptop mid-nightly. + * network -- a route to the site or API under test and nothing else. These + * run anywhere. + * + * It is advisory today: the loader records it and the dispatcher does not yet + * refuse on it, because the workloads that gate (web-test on FLEET_WEB) still + * do their own gating and this refactor changes no behaviour. It is stated in + * the manifest so the check has one place to move to. + */ +export type WorkloadCapability = "devices" | "browsers" | "network"; + +/** manifest.json, as it is on disk. */ +export type WorkloadManifest = { + /** The `workload` string in a job spec. Must equal the directory name. */ + name: string; + capability: WorkloadCapability; + /** One sentence, in the composer's voice: what this does and what you get. */ + description: string; + /** Which executor claims it. Host workloads run here; device ones run on the phone's agent. */ + executor: "host" | "device"; + /** JSON Schema for `params`. Documentation and, one day, validation. */ + params: Record; + /** A starting-point spec for the composer. Edited by a person, not validated. */ + example: Record; +}; diff --git a/runner-ios/FleetRunner.xcodeproj/project.pbxproj b/runner-ios/FleetRunner.xcodeproj/project.pbxproj index 5cbc825..ac103c3 100644 --- a/runner-ios/FleetRunner.xcodeproj/project.pbxproj +++ b/runner-ios/FleetRunner.xcodeproj/project.pbxproj @@ -9,12 +9,14 @@ /* Begin PBXBuildFile section */ 1094A2006ACBE463AD53F5C6 /* Percentile.swift in Sources */ = {isa = PBXBuildFile; fileRef = 67ED6D6D92FC1E39451CC4B1 /* Percentile.swift */; }; 18549C47E64362A3999CBF54 /* LlamaCppBackend.swift in Sources */ = {isa = PBXBuildFile; fileRef = BEC2D15CC8AA61AF38526ECA /* LlamaCppBackend.swift */; }; + 188C71F066363470DCF89341 /* WebShotsWorkload.swift in Sources */ = {isa = PBXBuildFile; fileRef = 87DC249597590EC4A8FDA5ED /* WebShotsWorkload.swift */; }; 29EE70D2DEBA1D834F56F353 /* SyntheticBackend.swift in Sources */ = {isa = PBXBuildFile; fileRef = D84C3EEBAE6C004B45C56596 /* SyntheticBackend.swift */; }; 2EA020A1111355C03CB7D665 /* CoreMLBackend.swift in Sources */ = {isa = PBXBuildFile; fileRef = D322211CB368EAB10C9D22A9 /* CoreMLBackend.swift */; }; 38E30C23455915A872514955 /* Workloads.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4A0BF52256E0CBF3AC96D589 /* Workloads.swift */; }; 6424FDD0EDCF7806C66447F0 /* FleetSmokeUITests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 09213AE64323164BBF07F61A /* FleetSmokeUITests.swift */; }; 8591C734F0CC29AECDA37140 /* BenchSession.swift in Sources */ = {isa = PBXBuildFile; fileRef = C7DFBDD257E5AD4960202BB7 /* BenchSession.swift */; }; 8C5395E6E4FADC66F7314D0F /* Telemetry.swift in Sources */ = {isa = PBXBuildFile; fileRef = 35252602851DD4FC6838C5CC /* Telemetry.swift */; }; + 8CEFD6BE130409469707F559 /* WebShotsManifest.swift in Sources */ = {isa = PBXBuildFile; fileRef = 87DDA25661F8D19F7570C9BE /* WebShotsManifest.swift */; }; A98911FE185A5563E58375CA /* Protocol.swift in Sources */ = {isa = PBXBuildFile; fileRef = 6BB3BA830E838835C4AE2AD1 /* Protocol.swift */; }; B0BC3C9EA4C05413868B3AC0 /* ThermalWorkload.swift in Sources */ = {isa = PBXBuildFile; fileRef = C8C6B181BF315A65223BCFC1 /* ThermalWorkload.swift */; }; B5E9A656019E3738212E40E7 /* CollectorClient.swift in Sources */ = {isa = PBXBuildFile; fileRef = 31632F0B6302F50DC2BD764E /* CollectorClient.swift */; }; @@ -48,6 +50,8 @@ 50ADC470CF907EE3A603AA44 /* VantageWorkload.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = VantageWorkload.swift; sourceTree = ""; }; 67ED6D6D92FC1E39451CC4B1 /* Percentile.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Percentile.swift; sourceTree = ""; }; 6BB3BA830E838835C4AE2AD1 /* Protocol.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Protocol.swift; sourceTree = ""; }; + 87DC249597590EC4A8FDA5ED /* WebShotsWorkload.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = WebShotsWorkload.swift; sourceTree = ""; }; + 87DDA25661F8D19F7570C9BE /* WebShotsManifest.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = WebShotsManifest.swift; sourceTree = ""; }; 965A118D55FCDC99E6F0349A /* FleetAgent.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = FleetAgent.swift; sourceTree = ""; }; A4C158F9244D20CE6B927532 /* FleetRunnerApp.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = FleetRunnerApp.swift; sourceTree = ""; }; BEC2D15CC8AA61AF38526ECA /* LlamaCppBackend.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = LlamaCppBackend.swift; sourceTree = ""; }; @@ -82,6 +86,8 @@ 086C0F942E2B16C41C3BE0B2 /* Theme.swift */, C8C6B181BF315A65223BCFC1 /* ThermalWorkload.swift */, 50ADC470CF907EE3A603AA44 /* VantageWorkload.swift */, + 87DDA25661F8D19F7570C9BE /* WebShotsManifest.swift */, + 87DC249597590EC4A8FDA5ED /* WebShotsWorkload.swift */, 4A0BF52256E0CBF3AC96D589 /* Workloads.swift */, ); path = FleetRunner; @@ -214,6 +220,8 @@ C0D50E9B5C78CC2307571AAF /* Theme.swift in Sources */, B0BC3C9EA4C05413868B3AC0 /* ThermalWorkload.swift in Sources */, D704AFBF0BF1559D31AE6C9B /* VantageWorkload.swift in Sources */, + 8CEFD6BE130409469707F559 /* WebShotsManifest.swift in Sources */, + 188C71F066363470DCF89341 /* WebShotsWorkload.swift in Sources */, 38E30C23455915A872514955 /* Workloads.swift in Sources */, ); runOnlyForDeploymentPostprocessing = 0; diff --git a/runner-ios/FleetRunner/CollectorClient.swift b/runner-ios/FleetRunner/CollectorClient.swift index af721de..6300626 100644 --- a/runner-ios/FleetRunner/CollectorClient.swift +++ b/runner-ios/FleetRunner/CollectorClient.swift @@ -78,6 +78,25 @@ final class CollectorClient: Sendable { return try JSONDecoder().decode(R.self, from: body).sha256 } + /// The accepted visual baselines for one suite, keyed "page|profile". + /// + /// Fetched once per web-shots job rather than once per page, so every page + /// in one run is judged against the same accepted set even if someone + /// clicks accept from the dashboard halfway through — the same rule the + /// host executor follows, for the same reason. + func visualBaselines(suite: String) async throws -> [String: String] { + let url = base.appendingPathComponent("api/visual/baselines") + .appending(queryItems: [URLQueryItem(name: "suite", value: suite)]) + let (data, res) = try await session.data(from: url) + let code = (res as? HTTPURLResponse)?.statusCode ?? 0 + guard code == 200 else { throw CollectorError.http(code, "api/visual/baselines") } + struct Row: Decodable { let page: String; let profile: String; let sha256: String } + struct Body: Decodable { let baselines: [Row] } + let body = try FleetJSON.decoder.decode(Body.self, from: data) + return Dictionary(body.baselines.map { ("\($0.page)|\($0.profile)", $0.sha256) }, + uniquingKeysWith: { _, last in last }) + } + func publishEvent(topic: String, payload: [String: Any]) async throws { var req = URLRequest(url: base.appendingPathComponent("events/\(topic)")) req.httpMethod = "POST" diff --git a/runner-ios/FleetRunner/FleetAgent.swift b/runner-ios/FleetRunner/FleetAgent.swift index afc4f09..93f07b3 100644 --- a/runner-ios/FleetRunner/FleetAgent.swift +++ b/runner-ios/FleetRunner/FleetAgent.swift @@ -198,6 +198,15 @@ final class FleetAgent: ObservableObject { Route(capability: "vantage", workload: "vantage") { _, job, client, deviceId, _ in await Workloads.runVantage(job: job, client: client, deviceId: deviceId) }, + // Backend-qualified, like batch:coreml, and for the same reason: most + // web-shots jobs are the host executor's Playwright matrix, which this + // runner cannot run. Declaring a bare "web-shots" would have the queue + // hand us those and get them bounced straight back. `webkit` is the + // backend because a WKWebView is what captures — see WebShots.profile + // for why that word is not "safari". + Route(capability: "web-shots:webkit", workload: "web-shots", backend: "webkit") { _, job, client, deviceId, cache in + await Workloads.runWebShots(job: job, client: client, deviceId: deviceId, artifacts: cache) + }, ] /// What this device declares at registration, so the collector's routing diff --git a/runner-ios/FleetRunner/Protocol.swift b/runner-ios/FleetRunner/Protocol.swift index 3b7e3b9..a19b64b 100644 --- a/runner-ios/FleetRunner/Protocol.swift +++ b/runner-ios/FleetRunner/Protocol.swift @@ -15,6 +15,21 @@ struct Targets: Codable { let pool: String? let match: String? let exclusive: Bool? + /// web-shots / web-test: what to point the browser at. No device is + /// involved in the host's use of it; on a device job it is the base every + /// manifest page path is resolved against. + let url: String? +} + +/// ui-test / web-test / web-shots: which suite this job runs. +/// +/// `flows` is the spec directory under the collector's web-specs/ — and for +/// web-shots it is also the suite name that baselines are keyed by, which is +/// why a device capture needs it even though it will never open that +/// directory: without it the shot has no cell to land in. +struct SuiteRef: Codable { + let kind: String? + let flows: String? } struct BenchParams: Codable { @@ -53,6 +68,12 @@ struct BenchParams: Codable { let repeats: Int? /// Per-request ceiling, so one hanging host cannot consume the job. let timeoutS: Int? + // web-shots + /// The manifest inline, for a job written by hand rather than pointed at + /// an uploaded shots.json. Same shape either way — `params.input_sha256` + /// names an artifact holding exactly these bytes — so a suite never has + /// two manifest dialects to keep in step. + let shots: ShotsManifest? } struct Constraints: Codable { @@ -68,6 +89,7 @@ struct JobSpec: Codable { let model: ModelRef? let backend: String? let params: BenchParams? + let suite: SuiteRef? let targets: Targets? let constraints: Constraints? } @@ -146,6 +168,13 @@ struct Metrics: Codable { var ttfbMs: Double? var networkType: String? + // web-shots. The one number a per-page row carries: how far this capture + // drifted from the accepted baseline for its (suite, page, profile). A + // named field, never laundered through a slot that means something else — + // and the only metric name in the collector's closed list that means this, + // so nothing here invents one for "pages captured" (that lives in `test`). + var diffPct: Double? + /// Spelled out because `.convertToSnakeCase` cannot produce three of these /// names. The strategy splits on capitals and a digit is not one, so /// `recallAt1` encodes as **`recall_at1`** — one underscore short of the @@ -164,6 +193,7 @@ struct Metrics: Codable { case recallAt10 = "recall_at_10" case docsPerS, dim case dnsMs, connectMs, tlsMs, ttfbMs, networkType + case diffPct } } @@ -173,6 +203,29 @@ struct BeaconSample: Codable { let thermal: String } +/// ui-test / web-shots pass-fail counts for one row, with whatever the row +/// produced attached. Separate from `artifacts` above: those are a job's +/// outputs, these are the evidence for this row's verdict. +struct TestOutcome: Codable { + var passed: Int + var failed: Int + var artifacts: [String]? +} + +/// web-shots: which cell of the visual matrix a per-page row judges. +/// +/// The collector assembles its review grid from these and never from `iter` +/// order — iter maps to manifest order, and manifests change. `sha256` is +/// absent when the capture failed, which is what makes a cell "missing" +/// rather than "diverged". +struct ShotRef: Codable { + var suite: String + var page: String + var profile: String + var sha256: String? + var diffSha256: String? +} + struct ResultPost: Codable { var schema: Int = 1 var kind: String @@ -186,6 +239,8 @@ struct ResultPost: Codable { var beacon: BeaconSample? var error: String? var artifacts: [String]? + var test: TestOutcome? + var shot: ShotRef? } struct RegisterPost: Codable { diff --git a/runner-ios/FleetRunner/WebShotsManifest.swift b/runner-ios/FleetRunner/WebShotsManifest.swift new file mode 100644 index 0000000..621736b --- /dev/null +++ b/runner-ios/FleetRunner/WebShotsManifest.swift @@ -0,0 +1,246 @@ +import Foundation + +/// The shots manifest and the naming rules around it, kept free of UIKit and +/// WebKit so every judgement this workload makes can be checked without a +/// device: parsing, validation, the profile name, thresholds and URL joining. +/// `WebShotsWorkload.swift` holds everything that needs a screen. +/// +/// The shape mirrors `collector/web-specs//shots.json` exactly — the same +/// file the host executor reads in `runWebShots` — rather than a device-only +/// dialect of it. A device job is handed those same bytes as an artifact +/// (`params.input_sha256`) or inline (`params.shots`), so one manifest can +/// drive the host matrix and the phone half of it without being rewritten, and +/// a page added to the suite reaches both. +/// +/// Note the manifest's own key casing is mixed — `waitFor` and `fullPage` are +/// camelCase, `settle_ms` and `threshold_pct` are snake_case — which is what +/// Playwright's option names did to it. `FleetJSON.decoder`'s +/// `.convertFromSnakeCase` handles both: a key with no underscore is passed +/// through untouched. + +struct ShotsManifest: Codable { + /// The profiles the host matrix captures under. A device job ignores this: + /// its profile is its own screen (see `WebShots.profile`), and there is + /// exactly one of them. + var profiles: [String]? + var thresholdPct: Double? + var freezeTime: String? + var pages: [ShotPage]? +} + +struct ShotPage: Codable { + var name: String? + var path: String? + var waitFor: String? + var mask: [String]? + var fullPage: Bool? + var thresholdPct: Double? + var settleMs: Int? +} + +/// A problem with the job or the manifest, phrased for the `error` field of a +/// result row — which is the only place anyone will read it. +struct WebShotsError: Error, LocalizedError { + let message: String + var errorDescription: String? { message } +} + +enum WebShots { + + // MARK: - the profile name + + /// The profile every capture from this runner is filed under. + /// + /// **This says `webkit`, not `safari`, and that is not a typo.** + /// + /// What captures here is a `WKWebView` embedded in the runner app. It is + /// the same WebKit engine Safari renders with, which is the whole reason + /// this workload is worth having — but it is not Safari. Safari applies + /// reader-mode heuristics, runs the user's content blockers, ships its own + /// chrome (which is in frame in the `ios-sim:` profile's `simctl` + /// screenshots and is not in frame here), and treats a few viewport-meta + /// edge cases differently from an embedded web view. A baseline matrix + /// column labelled `safari` would be read as "this is what a person sees + /// in Safari on this phone", and this capture cannot support that claim. + /// `webkit` says exactly what it is: WebKit rendered these pixels. + /// + /// If you are reading this because the name looked wrong next to + /// `ios-sim-safari`: those two profiles genuinely are different things, and + /// renaming this one to match would make the matrix agree with itself by + /// making it wrong. + /// + /// The device id is part of the name for the same reason the host's + /// profiles are `android:` and `ios-sim:`: baselines are + /// keyed `(suite, page, profile)` and pixels differ per screen, so two + /// phones are two baselines. Collapsing them under a bare `webkit` would + /// diff a 6.1" screen against a 6.7" one and call the hardware difference + /// a regression — and, worse, whichever phone ran last would silently + /// redefine the truth for the other. + static let profilePrefix = "webkit" + + static func profile(deviceId: String) -> String { + let id = deviceId.trimmingCharacters(in: .whitespacesAndNewlines) + return id.isEmpty ? profilePrefix : "\(profilePrefix):\(id)" + } + + // MARK: - manifest + + /// How long to sit on a loaded page before the shutter, when the manifest + /// does not say. + /// + /// Deliberately not the 4 s the host's simulator path uses. That path + /// drives Safari with `simctl openurl` and gets no load event at all, so + /// its `settle_ms` is a blind wait standing in for one. Here the navigation + /// really has finished and `document.fonts.ready` really has resolved + /// before this starts, so this is only covering late layout — and four + /// seconds a page, times a matrix, is a lease. + static let defaultSettleMs = 1000 + + /// Threshold the host applies when neither the page nor the manifest names + /// one, kept identical so the same manifest judges the same way on both. + static let defaultThresholdPct = 0.1 + + /// A full-page capture is the whole scroll height, but a runaway page (an + /// infinite feed, a broken 100vh × N layout) would otherwise ask the device + /// for a bitmap it cannot allocate — and a job that dies of memory reports + /// nothing at all. Past this the capture is the first `maxFullPageHeight` + /// points and the row says so, which is a diffable baseline plus a visible + /// caveat rather than silence. + static let maxFullPageHeight: CGFloat = 8000 + + static func decodeManifest(_ data: Data) throws -> ShotsManifest { + do { + return try FleetJSON.decoder.decode(ShotsManifest.self, from: data) + } catch { + throw WebShotsError(message: "shots manifest is not readable JSON: \(error.localizedDescription)") + } + } + + /// The manifest's pages, validated the way the host validates them. + /// + /// Checked here — before a web view is built — rather than discovered as a + /// half-captured matrix when a file path fails to write. Page names become + /// artifact names and baseline keys, so a name that cannot be one is a + /// job-level error, not a page-level one. + static func pages(of manifest: ShotsManifest) throws -> [ShotPage] { + let pages = manifest.pages ?? [] + if pages.isEmpty { throw WebShotsError(message: "shots manifest lists no pages") } + for p in pages { + guard let name = p.name, isUsableName(name) else { + throw WebShotsError( + message: "shots.json page name unusable as a filename: \(p.name.map { "\"\($0)\"" } ?? "null")") + } + guard let path = p.path, !path.isEmpty else { + throw WebShotsError(message: "shots.json page '\(name)' has no path") + } + } + let names = pages.compactMap(\.name) + if Set(names).count != names.count { + throw WebShotsError(message: "shots.json page names must be unique — they name the artifacts") + } + return pages + } + + /// Same rule as the host's `/^[a-z0-9][a-z0-9_-]*$/i`. + static func isUsableName(_ name: String) -> Bool { + name.range(of: "^[a-z0-9][a-z0-9_-]*$", options: [.regularExpression, .caseInsensitive]) != nil + } + + static func threshold(for page: ShotPage, in manifest: ShotsManifest) -> Double { + page.thresholdPct ?? manifest.thresholdPct ?? defaultThresholdPct + } + + static func settleMs(for page: ShotPage) -> Int { + max(page.settleMs ?? defaultSettleMs, 0) + } + + /// A page's path resolved against the job's `targets.url`, the same join + /// the host does with `new URL(p.path, url)`: an absolute path replaces the + /// base's path, a relative one hangs off it, and an absolute URL wins + /// outright. + static func pageURL(base: String, path: String) -> URL? { + guard let baseURL = URL(string: base) else { return nil } + return URL(string: path, relativeTo: baseURL)?.absoluteURL + } + + /// A Swift string as a JavaScript literal, for the selectors and CSS this + /// workload injects. Built by the JSON encoder rather than by hand because + /// a page name or selector carrying a quote, a backslash or a line + /// separator would otherwise stop being data and start being code. + static func jsLiteral(_ s: String) -> String { + guard let data = try? JSONEncoder().encode(s), + let literal = String(data: data, encoding: .utf8) else { + return "\"\"" + } + // U+2028/U+2029 are legal in JSON strings and are line terminators in + // JavaScript source, so JSON-encoding alone is not enough here. + return literal + .replacingOccurrences(of: "\u{2028}", with: "\\u2028") + .replacingOccurrences(of: "\u{2029}", with: "\\u2029") + } + + // MARK: - determinism + + /// Mirrors the `KILL_MOTION_CSS` the host injects on its android path and + /// the `_shots` spec injects on its browser path. Same three properties, + /// because a caret blinking in a captured input is a 100%-reproducible way + /// to fail a diff at random. + static let killMotionCSS = + "*, *::before, *::after { animation: none !important; transition: none !important; " + + "caret-color: transparent !important; }" + + /// CSS that stands in for Playwright's `mask` option: the masked box is + /// painted flat and everything inside it is hidden, so a clock or an avatar + /// cannot fail a diff for being what it is. + /// + /// Not pixel-identical to Playwright's overlay, and it does not need to be + /// — a baseline is per profile, and nothing ever diffs these pixels against + /// a Playwright-captured column. + static func maskCSS(_ selectors: [String]) -> String { + selectors.map { sel in + "\(sel) { background: #FF00FF !important; color: transparent !important; " + + "box-shadow: none !important; border-color: #FF00FF !important; }\n" + + "\(sel) * { visibility: hidden !important; }" + }.joined(separator: "\n") + } + + /// A script that pins `Date` to a fixed instant, for manifests that set + /// `freeze_time` (the host gets this from Playwright's clock API). + /// + /// Injected at document start so a page that stamps a render time reads the + /// frozen clock rather than the real one. Returns nil when the manifest's + /// string is not a date this runner can parse — a silently unfrozen clock + /// would produce a page that fails its diff once a minute, so the caller + /// turns nil into a visible job error instead. + static func freezeTimeScript(_ iso: String) -> String? { + guard let date = isoDate(iso) else { return nil } + let ms = Int64((date.timeIntervalSince1970 * 1000).rounded()) + return """ + (function () { + var fixed = \(ms); + var Real = Date; + function Frozen(...args) { + if (!(this instanceof Frozen)) return new Real(fixed).toString(); + return args.length === 0 ? new Real(fixed) : new Real(...args); + } + Frozen.prototype = Real.prototype; + Frozen.now = function () { return fixed; }; + Frozen.parse = Real.parse; + Frozen.UTC = Real.UTC; + window.Date = Frozen; + })(); + """ + } + + /// `freeze_time` as the manifest writes it. Both spellings the host's + /// `new Date(...)` accepts for an ISO-8601 string: with and without + /// fractional seconds. + static func isoDate(_ s: String) -> Date? { + let withFraction = ISO8601DateFormatter() + withFraction.formatOptions = [.withInternetDateTime, .withFractionalSeconds] + if let d = withFraction.date(from: s) { return d } + let plain = ISO8601DateFormatter() + plain.formatOptions = [.withInternetDateTime] + return plain.date(from: s) + } +} diff --git a/runner-ios/FleetRunner/WebShotsWorkload.swift b/runner-ios/FleetRunner/WebShotsWorkload.swift new file mode 100644 index 0000000..7614af3 --- /dev/null +++ b/runner-ios/FleetRunner/WebShotsWorkload.swift @@ -0,0 +1,649 @@ +import Foundation +import UIKit +import WebKit + +/// web-shots on a real iPhone: the half of the visual-regression matrix the +/// host executor cannot reach. +/// +/// The host captures real Chrome on real Android hardware over adb, and Safari +/// on iOS *simulators* through `simctl`. A real iPhone had no capture path at +/// all — `collector/docs/operations.md` says so plainly — so the matrix has +/// been advertising real phone screens while half the real phones were missing +/// from it. This closes that as a device job: a `WKWebView` inside the runner +/// app, which is the same WebKit engine Safari renders with, needs no WebDriver +/// session, and is already sitting on the phone. +/// +/// It is a device job rather than a host one on purpose. The device is the +/// machine that renders the pixels, so the device is the only machine whose +/// baselines they can be compared against — the same argument that keeps the +/// host's diffing on the host. See `WebShots.profile` for what the profile is +/// called and why it is not called `safari`. +/// +/// Result shape follows the host's: one row per page carrying the page's +/// `shot` block (suite, page, profile, sha256) and its `diff_pct`, then one +/// closing final row. Unlike the host there is no per-profile summary row — +/// a device is one profile, and a summary row would collide with the final row +/// on (job_id, device_id, iter=0) and silently overwrite it. +extension Workloads { + + static let webShotsLoadTimeoutS: TimeInterval = 30 + static let webShotsWaitForTimeoutS: TimeInterval = 15 + + static func runWebShots(job: JobSpec, client: CollectorClient, deviceId: String, + artifacts: ArtifactCache) async { + func fail(_ m: String) async { + try? await client.postResult( + ResultPost(kind: "result", jobId: job.jobId, deviceId: deviceId, + iter: 0, final: true, ok: false, + device: Telemetry.descriptor(), error: m)) + } + + guard let base = job.targets?.url, !base.isEmpty else { + await fail("web-shots needs targets.url"); return + } + // The suite is the first third of a baseline's key. A capture with no + // suite has no cell in the matrix to land in, so this is refused rather + // than defaulted — a default would file every suite's shots together. + guard let suite = job.suite?.flows, !suite.isEmpty else { + await fail("web-shots needs suite.flows (the web-specs/ directory that names this suite)") + return + } + + let manifest: ShotsManifest + let pages: [ShotPage] + do { + if let sha = job.params?.inputSha256 { + let file = try await artifacts.ensure(sha256: sha) + manifest = try WebShots.decodeManifest(try Data(contentsOf: file)) + } else if let inline = job.params?.shots { + manifest = inline + } else { + throw WebShotsError( + message: "web-shots needs params.input_sha256 (the shots.json artifact) or params.shots") + } + pages = try WebShots.pages(of: manifest) + } catch { + await fail(error.localizedDescription); return + } + + // A manifest that asks for a frozen clock and does not get one produces + // a page that fails its diff whenever the minute rolls over, which + // reads as a flaky site rather than as a manifest this runner could not + // honour. Refuse the job instead. + var freezeScript: String? + if let freeze = manifest.freezeTime { + guard let script = WebShots.freezeTimeScript(freeze) else { + await fail("shots.json freeze_time is not an ISO-8601 instant this runner can parse: \(freeze)") + return + } + freezeScript = script + } + + let profile = WebShots.profile(deviceId: deviceId) + + // The accepted truth for this suite. A failure here fails the job: a + // page that cannot be judged is not a page that passed, and capturing + // anyway would fill the matrix with cells that look accepted-and-clean + // because nothing was ever compared. + let baselines: [String: String] + do { + baselines = try await client.visualBaselines(suite: suite) + } catch { + await fail("baselines fetch failed: \(error.localizedDescription)"); return + } + + let capture: WebShotCapture + do { + capture = try await WebShotCapture.make(freezeScript: freezeScript) + } catch { + await fail(error.localizedDescription); return + } + defer { Task { @MainActor in capture.tearDown() } } + + let batteryStart = Telemetry.batteryPct() + var captured = 0 + var diverged = 0 + var shas: [String] = [] + var report: [[String: Any]] = [] + + for (index, page) in pages.enumerated() { + // Iteration boundary: a beacon may have found the lease gone. The + // rows already posted stay valid; the pages after this never happen. + if CancellationRegistry.shared.isCancelled(job.jobId) { + try? await client.postResult( + ResultPost(kind: "result", jobId: job.jobId, deviceId: deviceId, + iter: 0, final: true, ok: false, + device: Telemetry.descriptor(), error: "cancelled")) + return + } + + let name = page.name ?? "page-\(index + 1)" + var ok = false + var note: String? + var sha: String? + var diffSha: String? + var diffPct: Double? + var rowArtifacts: [String] = [] + var row: [String: Any] = ["iter": index + 1, "page": name] + + guard let pageURL = WebShots.pageURL(base: base, path: page.path ?? "") else { + note = "page '\(name)' path \(page.path.map { "'\($0)'" } ?? "(none)") is not a URL against \(base)" + row["error"] = note! + report.append(row) + await postShotRow(client: client, job: job, deviceId: deviceId, iter: index + 1, + ok: false, suite: suite, page: name, profile: profile, + sha: nil, diffSha: nil, diffPct: nil, artifacts: [], note: note) + continue + } + row["url"] = pageURL.absoluteString + + do { + let shot = try await capture.capture(page: page, url: pageURL) + captured += 1 + ok = true + row["width_px"] = shot.widthPx + row["height_px"] = shot.heightPx + row["bytes"] = shot.png.count + if let truncated = shot.truncatedNote { row["note"] = truncated } + + sha = try await client.uploadArtifact(shot.png, name: "\(job.jobId)-\(profile)-\(name).png") + rowArtifacts.append(sha!) + shas.append(sha!) + + let threshold = WebShots.threshold(for: page, in: manifest) + row["threshold_pct"] = threshold + if let baseline = baselines["\(name)|\(profile)"] { + if baseline == sha { + diffPct = 0 // identical bytes; nothing to decode + } else { + do { + let baselineFile = try await artifacts.ensure(sha256: baseline) + let outcome = try ShotDiff.compare( + captured: shot.png, baseline: try Data(contentsOf: baselineFile)) + diffPct = outcome.diffPct + if outcome.diffPct > threshold { + ok = false + diverged += 1 + note = outcome.note + ?? String(format: "%.2f%% of pixels differ (threshold %g%%)", + outcome.diffPct, threshold) + // The diff image only when it matters: a + // within-threshold pair has nothing worth a + // person's look, and the store is forever. + if let diffPNG = outcome.diffPNG { + diffSha = try await client.uploadArtifact( + diffPNG, name: "\(job.jobId)-\(profile)-\(name)-diff.png") + rowArtifacts.append(diffSha!) + } + } + } catch { + // A baseline the store cannot produce is an + // operational problem, not a visual regression — + // but it must fail, loudly, because a page that + // cannot be judged is not a page that passed. + ok = false + diverged += 1 + note = "baseline \(baseline.prefix(12))… unusable: \(error.localizedDescription)" + } + } + } else { + // Visible, not failing: a new page's first capture is not a + // regression, but it stays flagged until someone accepts it. + note = "new: no baseline — accept this shot to start diffing" + } + } catch { + note = "no screenshot for page '\(name)': \(error.localizedDescription)" + } + + if let diffPct { row["diff_pct"] = diffPct } + if let sha { row["sha256"] = sha } + if let note { row["error"] = note } + row["ok"] = ok + report.append(row) + + await postShotRow(client: client, job: job, deviceId: deviceId, iter: index + 1, + ok: ok, suite: suite, page: name, profile: profile, + sha: sha, diffSha: diffSha, diffPct: diffPct, + artifacts: rowArtifacts, note: note) + } + + // The rows carry no URL and no capture size — the result schema has no + // field for either — so the report is what makes a red matrix + // debuggable without a person guessing which path a page name meant. + var finalArtifacts = shas + let summaryJSON: [String: Any] = [ + "job_id": job.jobId, "device_id": deviceId, + "suite": suite, "profile": profile, "base_url": base, + "pages": pages.count, "captured": captured, "diverged": diverged, + "engine": "WKWebView (WebKit); not Safari — no reader mode, no content blockers", + "screen_pt": ["width": capture.screenSize.width, "height": capture.screenSize.height], + "rows": report, + ] + if let data = try? JSONSerialization.data(withJSONObject: summaryJSON), + let sha = try? await client.uploadArtifact(data, name: "\(job.jobId)-web-shots.json") { + finalArtifacts.append(sha) + } + + let missed = pages.count - captured + let ok = missed == 0 && diverged == 0 + let problems = [ + missed > 0 ? "\(missed) of \(pages.count) pages not captured" : nil, + diverged > 0 ? "\(diverged) page(s) diverged from baseline" : nil, + ].compactMap { $0 }.joined(separator: "; ") + + var summary = Metrics() + summary.peakMemMb = Telemetry.physFootprintMb() + summary.memMethod = "phys_footprint" + summary.thermal = [Telemetry.thermal()] + summary.batteryStartPct = batteryStart + summary.batteryEndPct = Telemetry.batteryPct() + + try? await client.postResult( + ResultPost(kind: "result", jobId: job.jobId, deviceId: deviceId, + iter: 0, final: true, ok: ok, + device: Telemetry.descriptor(), metrics: summary, + error: ok ? nil : problems, + artifacts: finalArtifacts.isEmpty ? nil : finalArtifacts, + test: TestOutcome(passed: pages.count - missed - diverged, + failed: missed + diverged, artifacts: nil))) + } + + /// One cell of the matrix, posted the way the collector's + /// /api/visual/matrix expects to read it back. + /// + /// `diff_pct` is the only metric here and it is a named field — never + /// laundered through a slot that means something else. There is no metric + /// name in the collector's closed list for "pages captured", so that count + /// lives in `test` and in the report artifact rather than being invented. + private static func postShotRow( + client: CollectorClient, job: JobSpec, deviceId: String, iter: Int, ok: Bool, + suite: String, page: String, profile: String, + sha: String?, diffSha: String?, diffPct: Double?, artifacts: [String], note: String? + ) async { + var metrics: Metrics? + if let diffPct { + var m = Metrics() + m.diffPct = (diffPct * 10_000).rounded() / 10_000 + metrics = m + } + try? await client.postResult( + ResultPost(kind: "result", jobId: job.jobId, deviceId: deviceId, + iter: iter, ok: ok, metrics: metrics, error: note, + test: TestOutcome(passed: ok ? 1 : 0, failed: ok ? 0 : 1, + artifacts: artifacts.isEmpty ? nil : artifacts), + shot: ShotRef(suite: suite, page: page, profile: profile, + sha256: sha, diffSha256: diffSha))) + } +} + +// MARK: - capture + +/// One page's pixels, plus what the row needs to say about how they were taken. +struct WebShot { + let png: Data + let widthPx: Int + let heightPx: Int + /// Set when a full-page capture hit `WebShots.maxFullPageHeight`, so a + /// clipped baseline is a visible caveat rather than a quiet one. + let truncatedNote: String? +} + +/// An offscreen `WKWebView`, sized to the device's screen, that captures one +/// page at a time. +/// +/// The web view lives in a `UIWindow` of its own at one level *below* normal, +/// rather than inside the runner's own view hierarchy. WebKit renders lazily +/// for a view with no window, so a detached web view snapshots blank; a window +/// under the app's own opaque one is a real render surface that never appears +/// on screen and never disturbs the screen that is there. It is deliberately +/// not made key: the runner's UI keeps the keyboard and the focus it had. +/// +/// One instance per job, reused across pages: a fresh web view per page would +/// pay process-launch cost per cell of the matrix, and its data store is +/// non-persistent anyway, so nothing carries over that a reload would not. +@MainActor +final class WebShotCapture: NSObject, WKNavigationDelegate { + private var window: UIWindow? + private let webView: WKWebView + private var pending: CheckedContinuation? + private var timeoutTask: Task? + let screenSize: CGSize + + static func make(freezeScript: String?) async throws -> WebShotCapture { + guard let scene = UIApplication.shared.connectedScenes + .compactMap({ $0 as? UIWindowScene }) + .first(where: { $0.activationState != .background }) ?? UIApplication.shared.connectedScenes + .compactMap({ $0 as? UIWindowScene }).first + else { + // Not defaulted to a guessed screen size: the profile's whole + // meaning is "this device's screen", and inventing one would make + // the baseline a fiction. + throw WebShotsError( + message: "no window scene — the runner must be in the foreground to capture web shots") + } + return WebShotCapture(scene: scene, freezeScript: freezeScript) + } + + private init(scene: UIWindowScene, freezeScript: String?) { + let bounds = scene.screen.bounds + screenSize = bounds.size + + let config = WKWebViewConfiguration() + // Non-persistent, so no cache or cookie jar left by an earlier job can + // serve a page that no longer renders that way. + config.websiteDataStore = .nonPersistent() + config.suppressesIncrementalRendering = true + if let freezeScript { + config.userContentController.addUserScript( + WKUserScript(source: freezeScript, injectionTime: .atDocumentStart, forMainFrameOnly: false)) + } + + webView = WKWebView(frame: CGRect(origin: .zero, size: bounds.size), configuration: config) + webView.isOpaque = true + webView.backgroundColor = .white + webView.scrollView.contentInsetAdjustmentBehavior = .never + webView.scrollView.showsVerticalScrollIndicator = false + webView.scrollView.showsHorizontalScrollIndicator = false + super.init() + webView.navigationDelegate = self + + let window = UIWindow(windowScene: scene) + window.frame = bounds + window.windowLevel = .normal - 1 + let host = UIViewController() + host.view.backgroundColor = .white + host.view.addSubview(webView) + window.rootViewController = host + window.isHidden = false + self.window = window + } + + func tearDown() { + finish(.failure(WebShotsError(message: "capture torn down"))) + webView.navigationDelegate = nil + webView.stopLoading() + webView.removeFromSuperview() + window?.isHidden = true + window = nil + } + + /// Load one page and photograph it. + func capture(page: ShotPage, url: URL) async throws -> WebShot { + // Back to the screen's own size before every page: the previous page + // may have grown the view for a full-page shot, and a viewport that + // depends on which page ran before it is not a baseline. + webView.frame = CGRect(origin: .zero, size: screenSize) + webView.layoutIfNeeded() + + webView.load(URLRequest(url: url, cachePolicy: .reloadIgnoringLocalAndRemoteCacheData, + timeoutInterval: Workloads.webShotsLoadTimeoutS)) + try await waitForLoad(timeout: Workloads.webShotsLoadTimeoutS) + + if let selector = page.waitFor, !selector.isEmpty { + try await waitFor(selector: selector, timeout: Workloads.webShotsWaitForTimeoutS) + } + + // Determinism first, then fonts, then settle: injected CSS can change + // layout, and web fonts certainly do, so both must land before the + // settle window rather than during it. + try? await inject(css: WebShots.killMotionCSS) + if let mask = page.mask, !mask.isEmpty { + try? await inject(css: WebShots.maskCSS(mask)) + } + _ = try? await webView.callAsyncJavaScript( + "await document.fonts.ready; return true;", contentWorld: .page) + + let settle = WebShots.settleMs(for: page) + if settle > 0 { try? await Task.sleep(for: .milliseconds(settle)) } + + var truncated: String? + if page.fullPage ?? true { + let full = await contentHeight() + let wanted = max(full, screenSize.height) + let height = min(wanted, WebShots.maxFullPageHeight) + if wanted > WebShots.maxFullPageHeight { + truncated = "full-page capture clipped at \(Int(WebShots.maxFullPageHeight))pt " + + "of \(Int(wanted))pt" + } + if abs(height - webView.frame.height) > 0.5 { + webView.frame = CGRect(origin: .zero, size: CGSize(width: screenSize.width, height: height)) + webView.layoutIfNeeded() + // The resize is a layout change of its own; give the page the + // frame it needs to reflow into it before the shutter. + try? await Task.sleep(for: .milliseconds(250)) + } + } + + let config = WKSnapshotConfiguration() + config.rect = CGRect(origin: .zero, size: webView.bounds.size) + config.afterScreenUpdates = true + let image = try await webView.takeSnapshot(configuration: config) + guard let png = image.pngData() else { + throw WebShotsError(message: "snapshot produced no PNG") + } + return WebShot(png: png, + widthPx: Int(image.size.width * image.scale), + heightPx: Int(image.size.height * image.scale), + truncatedNote: truncated) + } + + // MARK: navigation + + private func waitForLoad(timeout: TimeInterval) async throws { + try await withCheckedThrowingContinuation { (c: CheckedContinuation) in + pending = c + timeoutTask = Task { [weak self] in + try? await Task.sleep(for: .seconds(timeout)) + guard !Task.isCancelled else { return } + self?.finish(.failure(WebShotsError( + message: "load did not finish inside \(Int(timeout))s"))) + } + } + } + + /// Resumes whatever is waiting on a navigation, at most once. Everything + /// here runs on the main actor, so "at most once" needs no lock — but it + /// does need this one funnel, because a timeout and a `didFail` racing to + /// resume the same continuation is a crash, not a failed job. + private func finish(_ result: Result) { + guard let c = pending else { return } + pending = nil + timeoutTask?.cancel() + timeoutTask = nil + c.resume(with: result) + } + + func webView(_ webView: WKWebView, didFinish navigation: WKNavigation!) { + finish(.success(())) + } + + func webView(_ webView: WKWebView, didFail navigation: WKNavigation!, withError error: Error) { + finish(.failure(error)) + } + + func webView(_ webView: WKWebView, didFailProvisionalNavigation navigation: WKNavigation!, + withError error: Error) { + finish(.failure(error)) + } + + // MARK: page helpers + + private func waitFor(selector: String, timeout: TimeInterval) async throws { + let js = "document.querySelector(\(WebShots.jsLiteral(selector))) !== null" + let deadline = Date().addingTimeInterval(timeout) + while Date() < deadline { + if let value = try? await webView.evaluateJavaScript(js), (value as? Bool) == true { return } + try? await Task.sleep(for: .milliseconds(200)) + } + throw WebShotsError(message: "waitFor '\(selector)' never matched inside \(Int(timeout))s") + } + + private func inject(css: String) async throws { + let js = """ + (function () { + var s = document.createElement('style'); + s.textContent = \(WebShots.jsLiteral(css)); + (document.head || document.documentElement).appendChild(s); + return true; + })(); + """ + _ = try await webView.evaluateJavaScript(js) + } + + /// The page's full scroll height in points, falling back to the viewport + /// when the page will not say — a fallback that captures the fold rather + /// than capturing nothing. + private func contentHeight() async -> CGFloat { + let js = """ + Math.ceil(Math.max( + document.body ? document.body.scrollHeight : 0, + document.documentElement ? document.documentElement.scrollHeight : 0)); + """ + if let value = try? await webView.evaluateJavaScript(js), let n = value as? NSNumber { + let h = CGFloat(truncating: n) + if h.isFinite && h > 0 { return h } + } + return screenSize.height + } +} + +// MARK: - diffing + +/// How far a capture drifted from its accepted baseline. +/// +/// Diffing happens on the device for the same reason the host diffs on the +/// host: a baseline is only comparable to pixels rendered by the same +/// renderer on the same screen. Shipping the bytes to the collector to be +/// judged would look tidier and would be wrong — the collector has no idea +/// what this phone's subpixel rendering looks like, and moving the judgement +/// there would quietly make every device's baseline mean "whatever the +/// collector's decoder thinks". +enum ShotDiff { + + struct Outcome { + let diffPct: Double + let diffPNG: Data? + let note: String? + } + + /// pixelmatch's colour distance and its default threshold, so "is this + /// pixel different" means the same thing on the phone as it does on the + /// host. What is *not* ported is pixelmatch's antialiasing detection: it + /// needs each pixel's neighbourhood in both images and is a workload of its + /// own. The practical consequence is that this reads slightly hotter than + /// the host on text edges, which is a reason to set a suite's + /// `threshold_pct` with the device column in mind — not a reason to compare + /// the two columns, which nothing does. + private static let maxDelta = 35215.0 * 0.1 * 0.1 + + static func compare(captured: Data, baseline: Data) throws -> Outcome { + guard let capturedImage = UIImage(data: captured)?.cgImage else { + throw WebShotsError(message: "captured PNG could not be decoded") + } + guard let baselineImage = UIImage(data: baseline)?.cgImage else { + throw WebShotsError(message: "baseline PNG could not be decoded") + } + let w = capturedImage.width, h = capturedImage.height + guard w == baselineImage.width, h == baselineImage.height else { + // A size change short-circuits to 100%, as it does on the host: + // mismatched grids cannot be compared pixel for pixel, and a page + // whose height changed has materially changed however its + // overlapping pixels look. + return Outcome( + diffPct: 100, + diffPNG: nil, + note: "dimensions changed: \(w)×\(h) vs baseline " + + "\(baselineImage.width)×\(baselineImage.height)") + } + guard w > 0, h > 0 else { + throw WebShotsError(message: "captured PNG has no pixels") + } + + let a = try raster(capturedImage) + let b = try raster(baselineImage) + + var diffMask = [Bool](repeating: false, count: w * h) + var differing = 0 + for i in 0..<(w * h) { + let o = i * 4 + if delta(a[o], a[o + 1], a[o + 2], b[o], b[o + 1], b[o + 2]) > maxDelta { + diffMask[i] = true + differing += 1 + } + } + let pct = Double(differing) * 100.0 / Double(w * h) + return Outcome(diffPct: pct, + diffPNG: differing > 0 ? diffImage(mask: diffMask, base: a, width: w, height: h) : nil, + note: nil) + } + + /// Both images drawn into one known format — 8-bit RGBA over white — so + /// the comparison is never actually comparing two colour spaces or two + /// alpha conventions and calling the difference a regression. + private static func raster(_ image: CGImage) throws -> [UInt8] { + let w = image.width, h = image.height + var bytes = [UInt8](repeating: 255, count: w * h * 4) + let space = CGColorSpaceCreateDeviceRGB() + let info = CGImageAlphaInfo.premultipliedLast.rawValue + guard let ctx = bytes.withUnsafeMutableBytes({ raw in + CGContext(data: raw.baseAddress, width: w, height: h, bitsPerComponent: 8, + bytesPerRow: w * 4, space: space, bitmapInfo: info) + }) else { + throw WebShotsError(message: "could not rasterise a \(w)×\(h) image for diffing") + } + ctx.setFillColor(UIColor.white.cgColor) + ctx.fill(CGRect(x: 0, y: 0, width: w, height: h)) + ctx.draw(image, in: CGRect(x: 0, y: 0, width: w, height: h)) + guard let data = ctx.data else { + throw WebShotsError(message: "rasteriser produced no pixels") + } + return [UInt8](UnsafeBufferPointer( + start: data.assumingMemoryBound(to: UInt8.self), count: w * h * 4)) + } + + /// pixelmatch's YIQ colour delta. + private static func delta(_ r1: UInt8, _ g1: UInt8, _ b1: UInt8, + _ r2: UInt8, _ g2: UInt8, _ b2: UInt8) -> Double { + if r1 == r2 && g1 == g2 && b1 == b2 { return 0 } + let r1d = Double(r1), g1d = Double(g1), b1d = Double(b1) + let r2d = Double(r2), g2d = Double(g2), b2d = Double(b2) + let y = (r1d - r2d) * 0.29889531 + (g1d - g2d) * 0.58662247 + (b1d - b2d) * 0.11448223 + let i = (r1d - r2d) * 0.59597799 - (g1d - g2d) * 0.27417610 - (b1d - b2d) * 0.32180189 + let q = (r1d - r2d) * 0.21147017 - (g1d - g2d) * 0.52261711 + (b1d - b2d) * 0.31114694 + return 0.5053 * y * y + 0.299 * i * i + 0.1957 * q * q + } + + /// The captured page, faded, with every differing pixel in red — enough for + /// a person to see *where* it moved, which is the only question a diff + /// image is opened to answer. + private static func diffImage(mask: [Bool], base: [UInt8], width w: Int, height h: Int) -> Data? { + var out = [UInt8](repeating: 255, count: w * h * 4) + for i in 0..<(w * h) { + let o = i * 4 + if mask[i] { + out[o] = 255; out[o + 1] = 0; out[o + 2] = 0; out[o + 3] = 255 + } else { + // Greyscale at low contrast: present enough to place the red, + // faint enough that the red is what the eye lands on. + // Broken into steps deliberately: as one expression, Swift's + // type checker gives up on the literal/Double mix and fails the + // build with "unable to type-check in reasonable time". + let r = Double(base[o]) * 0.299 + let g = Double(base[o + 1]) * 0.587 + let b = Double(base[o + 2]) * 0.114 + let luma: Double = r + g + b + let lifted: Double = 180.0 + luma * 0.075 + let grey = UInt8(min(255.0, lifted)) + out[o] = grey; out[o + 1] = grey; out[o + 2] = grey; out[o + 3] = 255 + } + } + let space = CGColorSpaceCreateDeviceRGB() + let info = CGImageAlphaInfo.premultipliedLast.rawValue + return out.withUnsafeMutableBytes { raw -> Data? in + guard let ctx = CGContext(data: raw.baseAddress, width: w, height: h, bitsPerComponent: 8, + bytesPerRow: w * 4, space: space, bitmapInfo: info), + let cg = ctx.makeImage() else { return nil } + return UIImage(cgImage: cg).pngData() + } + } +} diff --git a/runner-machine/recipes/dir-labels.mjs b/runner-machine/recipes/dir-labels.mjs new file mode 100644 index 0000000..e4efb0f --- /dev/null +++ b/runner-machine/recipes/dir-labels.mjs @@ -0,0 +1,34 @@ +/** + * The generic recipe: one directory per class. + * + * `train/rose/0001.jpg` is a rose; the label is the index of `rose` in the + * sorted list of directory names actually present. Sorted, not + * first-seen — the same archive prepared twice must produce the same label + * for the same class, or two runs of the same eval are not comparable and + * nobody would be able to tell from the numbers. + * + * It states no licence on purpose. A recipe that works for any pile of images + * cannot know the terms of the pile it is pointed at, so `params.license` is + * required and the prep refuses without it. + */ + +/** The directory immediately above the file, which is the class. */ +const classOf = (relPath) => { + const parts = relPath.split("/"); + return parts.length > 1 ? parts[parts.length - 2] : ""; +}; + +export default { + id: "dir-labels", + license: null, + size: 224, + preprocess: "center-crop, resize 224, RGB; ImageNet mean/std applied on-device", + + labels(files) { + const classes = [...new Set(files.map(classOf))].sort(); + return { + classes: classes.length, + items: files.map((f) => ({ label: classes.indexOf(classOf(f)), class: classOf(f) })), + }; + }, +}; diff --git a/runner-machine/recipes/plant-id.mjs b/runner-machine/recipes/plant-id.mjs new file mode 100644 index 0000000..b77fa18 --- /dev/null +++ b/runner-machine/recipes/plant-id.mjs @@ -0,0 +1,45 @@ +/** + * PlantNet-300K, prepared for the plant-ID eval. + * + * The classes are not this recipe's to number: the model has 1081 outputs in + * an order fixed at training time, so the directory name IS the class index + * and inventing a compact 0..n-1 numbering from whichever species happen to be + * in this archive would score every prediction against the wrong label. A + * directory that is not a number in range is dropped rather than guessed at. + * + * The licence is stated here rather than left to the job. PlantNet-300K is + * CC-BY-SA, which is the reason its images ship through the artifact store + * instead of being committed, and a prepared set that arrives without the + * attribution is a set whose eval results cannot be published anywhere. + */ + +const CLASSES = 1081; + +const classOf = (relPath) => { + const parts = relPath.split("/"); + return parts.length > 1 ? parts[parts.length - 2] : ""; +}; + +export default { + id: "plant-id", + license: "CC-BY-SA-4.0", + size: 224, + preprocess: "center-crop, resize 224, RGB; ImageNet mean/std applied on-device", + + labels(files) { + return { + classes: CLASSES, + items: files.map((f) => { + const dir = classOf(f); + const label = /^\d+$/.test(dir) ? Number(dir) : -1; + return { + // -1 rather than 0: an unlabelled image scored against class 0 would + // quietly depress top-1 by however many files were misfiled, whereas + // a label no prediction can equal shows up as a visible miss. + label: label >= 0 && label < CLASSES ? label : -1, + species_dir: dir, + }; + }), + }; + }, +}; diff --git a/runner-machine/src/agent.ts b/runner-machine/src/agent.ts index b1e5508..090c963 100644 --- a/runner-machine/src/agent.ts +++ b/runner-machine/src/agent.ts @@ -16,6 +16,10 @@ import { probeCapabilities } from "./capabilities.js"; import { runBenchmark } from "./workloads/benchmark.js"; import { runBuild } from "./workloads/build.js"; import { runSelfCheck } from "./workloads/selfcheck.js"; +import { runModelConvert } from "./workloads/modelconvert.js"; +import { runDatasetPrep } from "./workloads/datasetprep.js"; +import { runServe } from "./workloads/serve.js"; +import { runShell } from "./workloads/shell.js"; import { SCHEMA, type Descriptor, type JobSpec } from "./protocol.js"; import * as JobCancellation from "./cancellation.js"; @@ -207,6 +211,22 @@ async function dispatch(job: JobSpec, client: CollectorClient): Promise { await runSelfCheck(job, client, deviceId, device); return; } + if (job.workload === "model-convert") { + await runModelConvert(job, client, deviceId, device); + return; + } + if (job.workload === "dataset-prep") { + await runDatasetPrep(job, client, deviceId, device); + return; + } + if (job.workload === "serve") { + await runServe(job, client, deviceId, device); + return; + } + if (job.workload === "shell") { + await runShell(job, client, deviceId, device); + return; + } await client.postResult({ schema: SCHEMA, kind: "result", job_id: job.job_id, device_id: deviceId, iter: 0, final: true, ok: false, device: descriptor ?? undefined, diff --git a/runner-machine/src/allowlist.ts b/runner-machine/src/allowlist.ts new file mode 100644 index 0000000..bbf74a8 --- /dev/null +++ b/runner-machine/src/allowlist.ts @@ -0,0 +1,93 @@ +/** + * The trust boundary for the `shell` workload. + * + * `POST /jobs` on the collector is unauthenticated by design — the README says + * so outright — so anyone who can reach the collector can enqueue a job, and a + * workload that runs an arbitrary script would hand them this machine. The + * allowlist is what makes `shell` safe to declare: a file the machine's owner + * maintains by hand, holding the sha256 of every script this machine is + * willing to execute. + * + * Three properties this file exists to guarantee: + * + * 1. The check happens on the sha the JOB NAMED, before anything is fetched. + * Downloading first and checking second would mean an unlisted script had + * already been pulled onto the machine, and the artifact store is writable + * by the same unauthenticated POST. + * 2. Only an exact 64-hex match counts. No prefixes, no substrings: a sha that + * differs by one character is a different script. + * 3. No allowlist is not an empty allowlist in name only — it means `shell` is + * never declared, so the collector cannot even offer this machine the job. + */ +import { readFile } from "node:fs/promises"; +import path from "node:path"; +import { dataDir } from "./git.js"; + +export const SHA256_RE = /^[0-9a-f]{64}$/; + +/** Where the owner keeps the list. One place, overridable for testing and for + * operators who keep it under configuration management. */ +export function allowlistPath(env: NodeJS.ProcessEnv = process.env): string { + return env.FLEET_SHELL_ALLOWLIST ?? path.join(dataDir(env), "shell-allowlist.txt"); +} + +/** + * The shas in an allowlist file. + * + * `#` comments and blank lines are skipped, and anything after the sha on a + * line is a note for whoever reads the file — a bare list of hashes with no + * room to say what each one is would be a file nobody could maintain, and an + * unmaintainable allowlist gets replaced with a wildcard. + * + * A line that is not a sha is DROPPED, not accepted loosely. The failure this + * avoids is a typo'd or truncated hash silently becoming an entry that can + * never match, which reads to the operator as "I listed it and it still + * refuses" — so `entriesRejected` is reported alongside, and the workload says + * so in its refusal. + */ +export function parseAllowlist(text: string | null | undefined): { allowed: string[]; rejected: string[] } { + const allowed: string[] = []; + const rejected: string[] = []; + for (const raw of (text ?? "").split(/\r?\n/)) { + const line = raw.replace(/#.*$/, "").trim(); + if (line === "") continue; + const first = line.split(/\s+/)[0] ?? ""; + const candidate = first.toLowerCase(); + if (SHA256_RE.test(candidate)) { + if (!allowed.includes(candidate)) allowed.push(candidate); + } else { + rejected.push(first.slice(0, 80)); + } + } + return { allowed, rejected }; +} + +/** + * Is this sha one the owner listed? + * + * Case-insensitive because a hash pasted from `shasum` and one pasted from a + * dashboard can differ in case and are the same hash; nothing else is + * forgiven. A value that is not a well-formed sha256 is never allowed, which + * matters because the job spec is attacker-controlled and `""` or `"*"` must + * not become a match against anything. + */ +export function isAllowlisted(sha256: unknown, allowed: readonly string[]): boolean { + if (typeof sha256 !== "string") return false; + const candidate = sha256.trim().toLowerCase(); + if (!SHA256_RE.test(candidate)) return false; + return allowed.includes(candidate); +} + +/** The list on disk. A missing or unreadable file is an empty list, never a throw. */ +export async function loadAllowlist( + env: NodeJS.ProcessEnv = process.env, +): Promise<{ allowed: string[]; rejected: string[]; file: string; exists: boolean }> { + const file = allowlistPath(env); + let text: string | null = null; + try { + text = await readFile(file, "utf8"); + } catch { + return { allowed: [], rejected: [], file, exists: false }; + } + return { ...parseAllowlist(text), file, exists: true }; +} diff --git a/runner-machine/src/beaconing.ts b/runner-machine/src/beaconing.ts new file mode 100644 index 0000000..dc1f453 --- /dev/null +++ b/runner-machine/src/beaconing.ts @@ -0,0 +1,170 @@ +/** + * Keeping a lease alive across something slow. + * + * `model-convert`, `dataset-prep` and `serve` are all in the collector's + * LONG_LEASE_WORKLOADS, but a long lease is not a self-renewing one: the lease + * is renewed by beacons, and the sweep requeues a claim whose beacons stopped. + * A quantisation that takes forty minutes without a beacon gets swept and run + * again on another machine, which is the expensive kind of wrong. + * + * The `build` workload solved this by tailing its log file, because a compiler + * emits a log worth showing. These two do not: one is a child process that + * prints little, the other is ten thousand tiny image resizes. So the shape + * here is a ticker rather than a tail. + * + * Both helpers treat the collector's answers exactly as build does. Only an + * explicit `lease_renewed: false` stops the work — an unreachable collector + * throws and is swallowed, because a flaky network must never be what kills a + * multi-gigabyte conversion. + */ +import { spawn } from "node:child_process"; +import type { CollectorClient } from "./collector.js"; +import { SCHEMA } from "./protocol.js"; +import { beacon } from "./telemetry.js"; +import * as JobCancellation from "./cancellation.js"; + +export const BEACON_MS = 30_000; +/** Kept of a child's output. Enough to name a failure; not enough to grow unbounded. */ +export const MAX_OUTPUT_BYTES = 1 << 20; + +/** + * A rate-limited beacon that can be called from inside any loop. + * + * `maybe()` is cheap when it is not yet time, so it can go at the top of a + * per-image loop without anyone having to reason about how often that runs. + */ +export class Beaconer { + private last = Date.now(); + + constructor( + private readonly client: CollectorClient, + private readonly deviceId: string, + private readonly jobId: string, + private readonly intervalMs: number = BEACON_MS, + ) {} + + /** False means stop: the claim is gone, or the job was cancelled. */ + async maybe(now: number = Date.now()): Promise { + if (JobCancellation.isCancelled(this.jobId)) return false; + if (now - this.last < this.intervalMs) return true; + this.last = now; + try { + const renewed = await this.client.postBeacon({ + schema: SCHEMA, kind: "beacon", job_id: this.jobId, device_id: this.deviceId, beacon: await beacon(), + }); + if (!renewed) { + JobCancellation.cancel(this.jobId); + return false; + } + } catch { + // Not a cancellation. + } + return true; + } +} + +export type ChildOutcome = { + code: number | null; + signal: NodeJS.Signals | null; + /** stdout and stderr, merged, bounded, in the order they arrived. */ + output: string; + cancelled: boolean; + timedOut: boolean; + spawnError: string | null; +}; + +/** + * Runs one long child process, beaconing while it works. + * + * Merged stdio for the same reason the build workload merges it: a converter's + * progress goes to stdout and its traceback to stderr, and reading them apart + * loses which came first, which is the only thing that makes a traceback + * attributable to a step. + */ +export function runWithBeacons(opts: { + cmd: string; + args: string[]; + cwd: string; + env?: NodeJS.ProcessEnv; + timeoutMs: number; + jobId: string; + deviceId: string; + client: CollectorClient; + onTick?: (output: string) => void; +}): Promise { + return new Promise((resolve) => { + let output = ""; + let bytes = 0; + let cancelled = false; + let timedOut = false; + let spawnError: string | null = null; + let finished = false; + + const child = spawn(opts.cmd, opts.args, { + cwd: opts.cwd, + env: opts.env ?? { ...process.env, TERM: "dumb", PYTHONUNBUFFERED: "1" }, + stdio: ["ignore", "pipe", "pipe"], + }); + const absorb = (b: Buffer) => { + if (bytes >= MAX_OUTPUT_BYTES) return; + bytes += b.length; + output += b.toString("utf8"); + }; + child.stdout?.on("data", absorb); + child.stderr?.on("data", absorb); + + const killer = setTimeout(() => { + if (!finished) { + timedOut = true; + child.kill("SIGKILL"); + } + }, opts.timeoutMs); + + const ticker = setInterval(() => { + void (async () => { + opts.onTick?.(output); + if (JobCancellation.isCancelled(opts.jobId) && !cancelled) { + cancelled = true; + child.kill("SIGTERM"); + setTimeout(() => child.kill("SIGKILL"), 10_000).unref(); + return; + } + try { + const renewed = await opts.client.postBeacon({ + schema: SCHEMA, kind: "beacon", job_id: opts.jobId, device_id: opts.deviceId, beacon: await beacon(), + }); + if (!renewed && !cancelled) { + cancelled = true; + JobCancellation.cancel(opts.jobId); + child.kill("SIGTERM"); + setTimeout(() => child.kill("SIGKILL"), 10_000).unref(); + } + } catch { + // An unreachable collector is not a cancellation. + } + })(); + }, BEACON_MS); + ticker.unref?.(); + + const done = (code: number | null, signal: NodeJS.Signals | null) => { + finished = true; + clearTimeout(killer); + clearInterval(ticker); + resolve({ code, signal, output, cancelled, timedOut, spawnError }); + }; + child.on("error", (e) => { + spawnError = e.message; + done(null, null); + }); + child.on("close", (code, signal) => done(code, signal)); + }); +} + +/** The line of a child's output that says what went wrong, for an error row. */ +export function lastMeaningful(output: string, max = 300): string { + const lines = output.split(/\r?\n/).map((l) => l.trim()).filter(Boolean); + // A python traceback's last line is the exception, which is the answer; a + // shell tool's last line is usually its error too. + const line = lines.at(-1) ?? ""; + return line.length > max ? `${line.slice(0, max - 1)}…` : line; +} diff --git a/runner-machine/src/capabilities.ts b/runner-machine/src/capabilities.ts index bbb0aa0..4985741 100644 --- a/runner-machine/src/capabilities.ts +++ b/runner-machine/src/capabilities.ts @@ -37,6 +37,8 @@ */ import { which, run } from "./probe.js"; import { KIND_BINARY } from "./buildkinds.js"; +import { convertersAvailable, probeConverters } from "./converters.js"; +import { loadAllowlist } from "./allowlist.js"; export type CapabilityFlags = { llamaBench: boolean; @@ -44,6 +46,12 @@ export type CapabilityFlags = { gradle: boolean; xcodebuild: boolean; node: boolean; + /** Which model converters resolved: gguf, coreml, tflite. */ + converters?: string[]; + /** Whether a non-empty shell allowlist exists on this machine. */ + shellAllowlist?: boolean; + /** Whether a llama-server binary resolved, for the serve workload. */ + llamaServer?: boolean; }; /** The list, given the answers. Pure, so the ordering is testable. */ @@ -68,6 +76,22 @@ export function capabilitiesFrom(flags: CapabilityFlags): string[] { // reports a skipped check for whatever is not. A machine that cannot answer // any of its questions still answers "I could not", which is the whole point. caps.push("self-check"); + // Same bare-plus-specific shape as build, for the same reason: a job spec has + // nowhere to put an output format that the collector's capabilityMatches + // would read, so `convert:gguf` alone would be a machine that can convert and + // never claims a conversion. + if (flags.converters && flags.converters.length > 0) { + caps.push("model-convert", ...flags.converters.map((c) => `model-convert:${c}`)); + // dataset-prep needs only Node and the image tooling the converters bring + // along, so it rides on the same answer rather than probing twice. + caps.push("dataset-prep"); + } + if (flags.llamaServer) caps.push("serve"); + // shell is declared ONLY when this machine has a non-empty allowlist. That is + // the trust boundary: POST /jobs is unauthenticated by design, so a machine + // whose owner has pinned nothing must be unable to claim a shell job at all, + // rather than claiming it and refusing it afterwards. + if (flags.shellAllowlist) caps.push("shell"); return caps; } @@ -122,10 +146,16 @@ export async function probeBuildKinds(env: NodeJS.ProcessEnv = process.env): Pro } export async function probeCapabilities(env: NodeJS.ProcessEnv = process.env): Promise { - const [llamaBench, mlx, kinds] = await Promise.all([ + const [llamaBench, mlx, kinds, converters, allowlist, llamaServer] = await Promise.all([ resolveLlamaBench(env), hasMlx(env), probeBuildKinds(env), + probeConverters(env).then(convertersAvailable).catch(() => [] as string[]), + loadAllowlist(env).then((a) => a.allowed.length > 0).catch(() => false), + which("llama-server", env).then((p) => p !== null).catch(() => false), ]); - return capabilitiesFrom({ llamaBench: llamaBench !== null, mlx, ...kinds }); + return capabilitiesFrom({ + llamaBench: llamaBench !== null, mlx, ...kinds, + converters, shellAllowlist: allowlist, llamaServer, + }); } diff --git a/runner-machine/src/converters.ts b/runner-machine/src/converters.ts new file mode 100644 index 0000000..926c80d --- /dev/null +++ b/runner-machine/src/converters.ts @@ -0,0 +1,368 @@ +/** + * What "convert this checkpoint" means, per output format — decided from the + * job's params, not guessed from the file. + * + * The same shape as `buildkinds.ts`: a pure planner that turns a requested + * output into a list of commands, so the dispatch and every refusal are + * testable on a machine with none of the three toolchains installed — which is + * exactly the machine where the refusals matter, since a machine that HAS + * coremltools will never exercise the "coremltools is not installed" path. + * + * The refusals are the interesting half. A conversion that quietly produces + * the wrong thing is worse than one that will not start: an unquantised GGUF + * uploaded under a `Q4_K_M` name is four times the size the schedule budgeted + * for and silently changes what every dependent benchmark measured. + */ +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { access } from "node:fs/promises"; +import { which, run } from "./probe.js"; + +export const CONVERT_FORMATS = ["gguf", "coreml", "tflite"] as const; +export type ConvertFormat = (typeof CONVERT_FORMATS)[number]; + +export function isConvertFormat(v: unknown): v is ConvertFormat { + return typeof v === "string" && (CONVERT_FORMATS as readonly string[]).includes(v); +} + +/** + * A quant name reaches a command line, so it is constrained to the alphabet + * llama-quantize actually uses. `execFile` means there is no shell to inject + * into, but an argument like `--help` arriving where a type name belongs would + * still produce a "successful" run that converted nothing. + */ +export const QUANT_RE = /^[A-Za-z0-9_]{1,20}$/; + +export type OutputSpec = { format: ConvertFormat; quant: string | null }; + +/** + * `params.outputs` as the job wrote it. + * + * Throws with the reason rather than skipping a malformed entry: a job that + * asked for three formats and got two, silently, is a job whose result row + * lies about what the fleet has. + */ +export function parseOutputs(v: unknown): OutputSpec[] { + if (!Array.isArray(v) || v.length === 0) { + return errorOut("model-convert needs params.outputs: a non-empty array of { format, quant }"); + } + const specs: OutputSpec[] = []; + for (const [i, entry] of v.entries()) { + if (typeof entry !== "object" || entry === null || Array.isArray(entry)) { + return errorOut(`params.outputs[${i}] is not an object`); + } + const { format, quant } = entry as { format?: unknown; quant?: unknown }; + if (!isConvertFormat(format)) { + return errorOut( + `params.outputs[${i}].format must be one of ${CONVERT_FORMATS.join(", ")} (got ${JSON.stringify(format)})`, + ); + } + let q: string | null = null; + if (quant !== undefined && quant !== null && quant !== "") { + if (typeof quant !== "string" || !QUANT_RE.test(quant)) { + return errorOut(`params.outputs[${i}].quant is not a quantisation name: ${JSON.stringify(quant)}`); + } + q = quant; + } + // Two entries asking for the same format and quant would race for one + // output filename and upload whichever finished last, twice. + if (specs.some((s) => s.format === format && s.quant === q)) { + return errorOut(`params.outputs asks for ${format}/${q ?? "none"} twice`); + } + specs.push({ format, quant: q }); + } + return specs; +} + +const errorOut = (msg: string): never => { + throw new Error(msg); +}; + +/** + * Where the source checkpoint comes from. + * + * Two forms and no third: a 64-hex artifact sha, which the collector can hand + * over and whose content is verified on the way in, or a HuggingFace repo id. + * Anything else — a URL, a local path — is refused, because a `source` that is + * a path would let an unauthenticated `POST /jobs` name a file on this machine + * and have its bytes uploaded to the artifact store. + */ +export const HF_REPO_RE = /^[A-Za-z0-9][\w.-]*\/[\w.-]+$/; + +export type Source = + | { kind: "artifact"; sha256: string } + | { kind: "hf"; repo: string }; + +export function parseSource(v: unknown): Source { + if (typeof v !== "string" || v.trim() === "") { + throw new Error("model-convert needs params.source: an artifact sha256 or a HuggingFace repo id"); + } + const s = v.trim(); + if (/^[0-9a-fA-F]{64}$/.test(s)) return { kind: "artifact", sha256: s.toLowerCase() }; + if (HF_REPO_RE.test(s) && !s.includes("..")) return { kind: "hf", repo: s }; + throw new Error(`params.source is neither an artifact sha256 nor a HuggingFace repo id: ${s.slice(0, 80)}`); +} + +/** + * The toolchains, as resolved on this machine. Every field is a path or null, + * and null is what every refusal below reads. + */ +export type ConverterTools = { + /** llama.cpp's convert_hf_to_gguf.py. */ + ggufConvert: string | null; + /** llama-quantize, needed only for a gguf output that names a quant. */ + ggufQuantize: string | null; + /** The interpreter that runs convert_hf_to_gguf.py. */ + ggufPython: string | null; + /** An interpreter for which `import coremltools` exits 0. */ + coremlPython: string | null; + /** collector/evals/plant-id-assets/convert_coreml.py. */ + coremlConvert: string | null; + /** xcrun, for `coremlcompiler compile`. */ + xcrun: string | null; + /** An interpreter for which `import ai_edge_litert` exits 0. */ + tflitePython: string | null; + /** collector/evals/plant-id-assets/quantize_int8.py. */ + tfliteQuantize: string | null; +}; + +export const NO_TOOLS: ConverterTools = { + ggufConvert: null, ggufQuantize: null, ggufPython: null, + coremlPython: null, coremlConvert: null, xcrun: null, + tflitePython: null, tfliteQuantize: null, +}; + +/** + * The formats this machine can actually produce. + * + * `llama-quantize` is deliberately NOT required for `convert:gguf`: an f16 + * GGUF is a complete conversion and needs only the script. A job asking for a + * quantised one on a machine without the binary is refused by name at run + * time, which is the honest split — the capability says "I can make GGUFs", + * and the refusal says which quant it could not make. + */ +export function convertersAvailable(t: ConverterTools): ConvertFormat[] { + const formats: ConvertFormat[] = []; + if (t.ggufConvert && t.ggufPython) formats.push("gguf"); + if (t.coremlPython && t.coremlConvert && t.xcrun) formats.push("coreml"); + if (t.tflitePython && t.tfliteQuantize) formats.push("tflite"); + return formats; +} + +export type Step = { cmd: string; args: string[]; cwd: string; label: string }; +export type ConvertPlan = { + format: ConvertFormat; + quant: string | null; + steps: Step[]; + /** The file to upload, relative to the plan's `cwd`. */ + product: string; + /** The `format` a ModelRef would carry for this product. */ + modelFormat: "gguf" | "mlmodelc" | "tflite"; +}; + +/** + * The Core ML script's own output naming. + * + * `collector/evals/plant-id-assets/convert_coreml.py` writes + * `PlantNet300K.mlpackage` and `PlantNet300K-int8.mlpackage` with the names + * hard-coded. Reusing the script rather than rewriting it means inheriting + * that, so the constant lives here where the plan can be read against it + * instead of being a string buried in a command line. + */ +export const COREML_PRODUCT_BASE = "PlantNet300K"; + +export type PlanContext = { + /** The source: a directory for gguf, a checkpoint file for coreml/tflite. */ + source: string; + /** Where the products are written. Every step runs here. */ + outDir: string; + /** The stem for produced filenames, from the job or the source. */ + name: string; + /** tflite only: images to calibrate the quantiser on. */ + calibrationDir?: string | null; +}; + +/** + * Turns one requested output into the commands that make it. + * + * Throws, with the missing toolchain named, rather than returning a plan that + * will fail at spawn time — the caller turns the throw into a result row, and + * "coremltools is not installed on machine-x" is an answer while + * "ENOENT: python3" is a puzzle. + */ +export function planConvert(spec: OutputSpec, tools: ConverterTools, ctx: PlanContext): ConvertPlan { + if (spec.format === "gguf") return planGguf(spec, tools, ctx); + if (spec.format === "coreml") return planCoreml(spec, tools, ctx); + return planTflite(spec, tools, ctx); +} + +function planGguf(spec: OutputSpec, tools: ConverterTools, ctx: PlanContext): ConvertPlan { + if (!tools.ggufConvert || !tools.ggufPython) { + throw new Error( + "gguf: llama.cpp's convert_hf_to_gguf.py does not resolve here (set FLEET_LLAMA_CPP to the checkout, or FLEET_CONVERT_HF_TO_GGUF to the script)", + ); + } + // f16 IS the converter's own output, so asking for it is one step, not two. + const wantsQuant = spec.quant !== null && !/^f(16|32)$/i.test(spec.quant); + const f16 = path.join(ctx.outDir, `${ctx.name}-f16.gguf`); + const steps: Step[] = [ + { + cmd: tools.ggufPython, + args: [tools.ggufConvert, ctx.source, "--outfile", f16, "--outtype", "f16"], + cwd: ctx.outDir, + label: "convert_hf_to_gguf", + }, + ]; + if (!wantsQuant) { + return { format: "gguf", quant: spec.quant, steps, product: path.basename(f16), modelFormat: "gguf" }; + } + if (!tools.ggufQuantize) { + throw new Error( + `gguf/${spec.quant}: llama-quantize does not resolve here, so only an f16 GGUF can be produced (set FLEET_LLAMA_QUANTIZE)`, + ); + } + const quantised = path.join(ctx.outDir, `${ctx.name}-${spec.quant!.toLowerCase()}.gguf`); + steps.push({ + cmd: tools.ggufQuantize, + args: [f16, quantised, spec.quant!], + cwd: ctx.outDir, + label: `llama-quantize ${spec.quant}`, + }); + return { format: "gguf", quant: spec.quant, steps, product: path.basename(quantised), modelFormat: "gguf" }; +} + +function planCoreml(spec: OutputSpec, tools: ConverterTools, ctx: PlanContext): ConvertPlan { + if (!tools.coremlPython || !tools.coremlConvert) { + throw new Error( + "coreml: no interpreter here imports coremltools (set FLEET_COREML_PYTHON to one, e.g. a 3.12 venv — coremltools' BlobWriter has no 3.14 wheel)", + ); + } + if (!tools.xcrun) { + throw new Error("coreml: xcrun does not resolve, so `coremlcompiler compile` cannot run — an .mlpackage is not a loadable model"); + } + // The script writes both variants every run; the quant selects which one is + // compiled and published rather than which one is produced. + const int8 = spec.quant !== null && /^int8$/i.test(spec.quant); + if (spec.quant !== null && !int8 && !/^fp?16$/i.test(spec.quant)) { + throw new Error(`coreml/${spec.quant}: convert_coreml.py produces fp16 weights and an int8 variant, nothing else`); + } + const base = int8 ? `${COREML_PRODUCT_BASE}-int8` : COREML_PRODUCT_BASE; + return { + format: "coreml", + quant: spec.quant, + modelFormat: "mlmodelc", + product: `${base}.mlmodelc.zip`, + steps: [ + { cmd: tools.coremlPython, args: [tools.coremlConvert, ctx.source], cwd: ctx.outDir, label: "convert_coreml.py" }, + { cmd: tools.xcrun, args: ["coremlcompiler", "compile", `${base}.mlpackage`, "."], cwd: ctx.outDir, label: "coremlcompiler compile" }, + // ditto rather than zip: an .mlmodelc is a bundle, and the symlinks and + // permission bits a plain zip drops are the difference between a model + // that loads on the phone and one that does not. Same call the build + // workload makes for a .app. + { cmd: "ditto", args: ["-c", "-k", "--sequesterRsrc", "--keepParent", `${base}.mlmodelc`, `${base}.mlmodelc.zip`], cwd: ctx.outDir, label: "ditto" }, + ], + }; +} + +function planTflite(spec: OutputSpec, tools: ConverterTools, ctx: PlanContext): ConvertPlan { + if (!tools.tflitePython || !tools.tfliteQuantize) { + throw new Error( + "tflite: no interpreter here imports ai_edge_litert (set FLEET_TFLITE_PYTHON to one — `pip install ai-edge-litert Pillow numpy`)", + ); + } + // The LiteRT path this repo owns is post-training quantisation of an + // existing .tflite. Copying a float model to a new name and calling it a + // conversion would publish an artifact whose only change is its filename. + if (spec.quant === null || !/^int8$/i.test(spec.quant)) { + throw new Error( + `tflite/${spec.quant ?? "none"}: the LiteRT converter here is quantize_int8.py, so quant must be int8`, + ); + } + if (!ctx.calibrationDir) { + throw new Error( + "tflite/int8: needs params.calibration_sha256, a zip of ~100 calibration images DISJOINT from the eval set", + ); + } + const out = `${ctx.name}-int8.tflite`; + return { + format: "tflite", + quant: spec.quant, + modelFormat: "tflite", + product: out, + steps: [ + { + cmd: tools.tflitePython, + args: [tools.tfliteQuantize, ctx.source, ctx.calibrationDir, path.join(ctx.outDir, out)], + cwd: ctx.outDir, + label: "quantize_int8.py", + }, + ], + }; +} + +// --- probes ----------------------------------------------------------------- + +/** + * Where the plant-ID conversion scripts are. + * + * They live in the collector's tree, which this agent does not import code + * from — but a python script invoked by path is not an import, it is a tool, + * and rewriting coremltools plumbing that already exists and is known to work + * would be the worse call. `FLEET_EVAL_ASSETS` names the directory when the + * two repos are not checked out side by side. + */ +export function evalAssetsDir(env: NodeJS.ProcessEnv = process.env): string { + if (env.FLEET_EVAL_ASSETS) return env.FLEET_EVAL_ASSETS; + // runner-machine/src/converters.ts -> ../../collector/evals/plant-id-assets. + // fileURLToPath, not `.pathname`: this repo's own checkout sits under a + // directory with a space in it, and the URL form spells that `%20`. + return fileURLToPath(new URL("../../collector/evals/plant-id-assets", import.meta.url)); +} + +/** An interpreter for which `import ` exits 0, or null. */ +export async function pythonThatImports( + mod: string, + candidates: (string | undefined)[], + env: NodeJS.ProcessEnv = process.env, +): Promise { + for (const candidate of candidates) { + if (!candidate) continue; + const resolved = await which(candidate, env); + if (!resolved) continue; + const r = await run(resolved, ["-c", `import ${mod}`], 60_000); + if (r.code === 0) return resolved; + } + return null; +} + +async function fileIfReadable(file: string | undefined | null): Promise { + if (!file) return null; + try { + await access(file); + return file; + } catch { + return null; + } +} + +/** Asks the machine each question the planner will ask of the answers. */ +export async function probeConverters(env: NodeJS.ProcessEnv = process.env): Promise { + const assets = evalAssetsDir(env); + const ggufScript = + env.FLEET_CONVERT_HF_TO_GGUF ?? + (env.FLEET_LLAMA_CPP ? path.join(env.FLEET_LLAMA_CPP, "convert_hf_to_gguf.py") : undefined); + + const [ggufConvert, ggufQuantize, ggufPython, coremlPython, coremlConvert, xcrun, tflitePython, tfliteQuantize] = + await Promise.all([ + fileIfReadable(ggufScript), + which(env.FLEET_LLAMA_QUANTIZE ?? "llama-quantize", env), + which(env.FLEET_PYTHON ?? "python3", env), + pythonThatImports("coremltools", [env.FLEET_COREML_PYTHON, env.FLEET_PYTHON, "python3.12", "python3"], env), + fileIfReadable(path.join(assets, "convert_coreml.py")), + which("xcrun", env), + pythonThatImports("ai_edge_litert", [env.FLEET_TFLITE_PYTHON, env.FLEET_PYTHON, "python3"], env), + fileIfReadable(path.join(assets, "quantize_int8.py")), + ]); + + return { ggufConvert, ggufQuantize, ggufPython, coremlPython, coremlConvert, xcrun, tflitePython, tfliteQuantize }; +} diff --git a/runner-machine/src/images.ts b/runner-machine/src/images.ts new file mode 100644 index 0000000..ac63d9e --- /dev/null +++ b/runner-machine/src/images.ts @@ -0,0 +1,104 @@ +/** + * Resizing an eval image to the shape the on-device models expect. + * + * The eval sets this fleet ships are "center-crop, resize 224, RGB" — that + * string is in the plant-ID manifest and both runners' preprocessing assumes + * it. Getting it wrong is not a crash: it is a model that scores four points + * lower for a reason nobody can see, because a letterboxed image and a + * center-cropped one are both 224×224 JPEGs. + * + * So the arithmetic is here, pure and tested, and the two resizers this agent + * knows about are only ever handed the numbers it produces. + */ +import { which } from "./probe.js"; + +export type Resizer = { kind: "sips" | "magick"; bin: string }; + +/** + * The intermediate size a center-crop needs: scale so the SHORT edge is + * `size`, then take the middle `size`×`size`. + * + * Scaling the LONG edge to `size` instead — which is what `sips -Z` does on + * its own, and the obvious one-liner — leaves the short edge under `size` and + * forces a pad. A padded image is a smaller subject surrounded by background + * the model has to look past, which is a different picture from the one the + * eval set was scored on. + */ +export function resizeToShortEdge(w: number, h: number, size: number): { width: number; height: number } { + if (!Number.isFinite(w) || !Number.isFinite(h) || w <= 0 || h <= 0) { + throw new Error(`cannot resize an image reported as ${w}×${h}`); + } + const scale = size / Math.min(w, h); + return { + // Ceil, not round: a rounded-down edge lands one pixel short of `size` and + // the crop that follows then has to pad the very row it was avoiding. + width: Math.max(size, Math.ceil(w * scale)), + height: Math.max(size, Math.ceil(h * scale)), + }; +} + +/** `sips -g pixelWidth -g pixelHeight` output to numbers. */ +export function parseSipsDims(text: string | null): { w: number; h: number } | null { + if (!text) return null; + const w = /pixelWidth:\s*(\d+)/.exec(text)?.[1]; + const h = /pixelHeight:\s*(\d+)/.exec(text)?.[1]; + if (!w || !h) return null; + const dims = { w: Number(w), h: Number(h) }; + return dims.w > 0 && dims.h > 0 ? dims : null; +} + +export function sipsDimArgs(src: string): string[] { + return ["-g", "pixelWidth", "-g", "pixelHeight", src]; +} + +/** + * sips: resample to the short-edge size, then crop the middle, in one call. + * + * `-c h w` on sips is a CENTERED crop, which is the whole reason the resample + * has to come first — sips has no gravity flag, so the only way to choose what + * survives the crop is to control what goes into it. + */ +export function sipsResizeArgs( + src: string, + dst: string, + size: number, + dims: { w: number; h: number }, +): string[] { + const fit = resizeToShortEdge(dims.w, dims.h, size); + return [ + "-s", "format", "jpeg", + "-s", "formatOptions", "high", + "--resampleHeightWidth", String(fit.height), String(fit.width), + "-c", String(size), String(size), + src, "--out", dst, + ]; +} + +/** + * ImageMagick: the same two operations, and it needs no dimension probe + * because `^` means "fit the SHORT edge" and `-extent` with a center gravity + * is the crop. + */ +export function magickResizeArgs(src: string, dst: string, size: number): string[] { + return [src, "-resize", `${size}x${size}^`, "-gravity", "center", "-extent", `${size}x${size}`, "-quality", "90", dst]; +} + +/** + * The resizer this machine has, or null. + * + * sips first because it is on every Mac and needs nothing installed; magick + * because a Linux box that has it can prepare a set the same way. A machine + * with neither must not declare `dataset-prep` — a "prepared" set whose images + * are still 4000 px wide is one the phone runners will resize themselves, + * inconsistently and at eval time, which is exactly the variable this workload + * exists to remove. + */ +export async function resolveResizer(env: NodeJS.ProcessEnv = process.env): Promise { + const sips = await which("sips", env); + if (sips) return { kind: "sips", bin: sips }; + for (const name of ["magick", "convert"]) { + const bin = await which(name, env); + if (bin) return { kind: "magick", bin }; + } + return null; +} diff --git a/runner-machine/src/protocol.ts b/runner-machine/src/protocol.ts index 19b32b4..7671729 100644 --- a/runner-machine/src/protocol.ts +++ b/runner-machine/src/protocol.ts @@ -109,6 +109,18 @@ export type Metrics = { clock_offset_ms?: number; /** self-check: how many checks failed, so the alert rule needs no parsing. */ checks_failed?: number; + + /** model-convert: seconds for one output format. */ + convert_s?: number; + /** model-convert: how many artifacts the conversion produced. */ + outputs?: number; + /** dataset-prep: items in the prepared manifest, after dedupe. */ + items?: number; + /** shell: the script's exit status. Named rather than folded into `ok`, + * because a script that exits 3 deliberately is not one that could not start. */ + exit_code?: number; + /** serve and any long run: seconds since the job started. */ + elapsed_s?: number; }; /** @@ -138,6 +150,13 @@ export type BuildRow = { }; export type ResultPost = { + /** serve: where the hosted model can be reached. Declared in the collector's + * result schema; see there for why it is never bound to 0.0.0.0. */ + endpoint?: string; + /** model-convert: one produced artifact — format, quant, name, sha256, bytes. */ + model_out?: Record; + /** dataset-prep: the prepared manifest's summary, licence and source included. */ + dataset?: Record; schema: number; kind: "result" | "beacon"; job_id?: string; diff --git a/runner-machine/src/recipes.ts b/runner-machine/src/recipes.ts new file mode 100644 index 0000000..40b3072 --- /dev/null +++ b/runner-machine/src/recipes.ts @@ -0,0 +1,197 @@ +/** + * Recipes: how a downloaded pile of images becomes a labelled eval set. + * + * A recipe is a small ES module under `recipes/`, named by the job. Keeping it + * out of this file is the point — a new dataset is a new file next to the + * others, not a branch in a workload — but it also means the recipe NAME + * arrives from an unauthenticated `POST /jobs`, so `recipePath` is a security + * boundary and not a convenience: `../../etc/something` must never resolve. + * + * The manifest is the other half. Its shape is not this file's invention: the + * Android runner's VisionEvalEngine reads `manifest.json` from the zip root + * and walks `items[].file` / `items[].label`, so those two keys are a contract + * with code in another repo and are asserted in the tests rather than trusted. + * + * `license` and `source` are new here, and they are the reason a prepared set + * is publishable. The plant dataset is CC-BY-SA — which is exactly why its + * images ship as artifacts rather than as commits — and a zip that has lost + * the attribution is a zip whose results can never appear in a write-up. So a + * set with no licence is refused at prep time, where somebody can still fix + * it, rather than discovered at publication time when the provenance is gone. + */ +import path from "node:path"; +import { pathToFileURL, fileURLToPath } from "node:url"; + +/** A recipe name is a filename, never a path. */ +export const RECIPE_NAME_RE = /^[a-z0-9][a-z0-9._-]{0,63}$/; + +export type LabelledItem = { label: number; [k: string]: unknown }; +export type Labelling = { items: LabelledItem[]; classes?: number }; + +export type Recipe = { + id: string; + /** + * The dataset's licence, when the recipe is for one specific dataset. Null + * for a generic recipe, which means the JOB has to say — and a job that does + * not is refused. + */ + license: string | null; + /** Square edge, in pixels, of every image in the prepared set. */ + size: number; + /** Copied into the manifest so the runners' preprocessing is documented beside the data. */ + preprocess?: string; + /** + * Labels for the whole file list at once, in the same order. + * + * Whole-list rather than per-file because a label is usually an INDEX into + * the set of classes present, which no single file can know. Returning fewer + * items than it was given is a refusal the caller reports. + */ + labels(files: string[]): Labelling; +}; + +/** Where the recipes live. Overridable so the tests do not need the real ones. */ +export function recipesDir(env: NodeJS.ProcessEnv = process.env): string { + if (env.FLEET_RECIPES_DIR) return env.FLEET_RECIPES_DIR; + // runner-machine/src/recipes.ts -> runner-machine/recipes. fileURLToPath + // rather than `.pathname`, because this checkout's path has a space in it. + return fileURLToPath(new URL("../recipes", import.meta.url)); +} + +/** + * The file a recipe name refers to. + * + * Throws for anything that is not a plain lowercase name. The name comes from + * a job spec and the collector authenticates nobody, so `../../../..` reaching + * `import()` would be arbitrary code execution with no allowlist in front of + * it — which is the thing the `shell` workload has a whole allowlist to avoid. + */ +export function recipePath(name: unknown, dir: string): string { + if (typeof name !== "string" || !RECIPE_NAME_RE.test(name)) { + throw new Error( + `params.recipe must be a recipe name like 'plant-id' (lowercase, no path separators); got ${JSON.stringify(name)}`, + ); + } + const file = path.join(dir, `${name}.mjs`); + // Belt and braces: the regex already forbids separators, so a resolved path + // that escaped the directory would mean the regex was weakened later. + if (path.dirname(path.resolve(file)) !== path.resolve(dir)) { + throw new Error(`recipe '${name}' does not resolve inside ${dir}`); + } + return file; +} + +/** A loaded module is only a recipe if it answers every question the prep asks. */ +export function asRecipe(mod: unknown, name: string): Recipe { + const r = (mod && typeof mod === "object" && "default" in mod ? (mod as { default: unknown }).default : mod) as + | Partial + | undefined; + if (!r || typeof r !== "object") throw new Error(`recipe '${name}' exports no default object`); + if (typeof r.labels !== "function") throw new Error(`recipe '${name}' has no labels(files) function`); + if (typeof r.size !== "number" || !Number.isInteger(r.size) || r.size < 16 || r.size > 4096) { + throw new Error(`recipe '${name}' has no usable size (got ${JSON.stringify(r.size)})`); + } + if (r.license !== null && typeof r.license !== "string") { + throw new Error(`recipe '${name}' must state a license string, or null to require one from the job`); + } + return { + id: typeof r.id === "string" && r.id !== "" ? r.id : name, + license: r.license, + size: r.size, + preprocess: typeof r.preprocess === "string" ? r.preprocess : undefined, + labels: r.labels.bind(r), + }; +} + +export async function loadRecipe(name: unknown, env: NodeJS.ProcessEnv = process.env): Promise { + const dir = recipesDir(env); + const file = recipePath(name, dir); + let mod: unknown; + try { + mod = await import(pathToFileURL(file).href); + } catch (e) { + throw new Error(`recipe '${String(name)}' does not load from ${dir}: ${(e as Error).message}`); + } + return asRecipe(mod, String(name)); +} + +/** + * The licence the prepared set will carry. + * + * The job may override a generic recipe's null; it may NOT quietly relabel a + * recipe that already knows its dataset's terms, because the recipe is the + * thing somebody checked. An empty answer is a refusal, not a blank field. + */ +export function effectiveLicense(recipe: Recipe, fromJob: unknown): string { + const job = typeof fromJob === "string" ? fromJob.trim() : ""; + if (recipe.license !== null && recipe.license.trim() !== "") { + if (job !== "" && job !== recipe.license) { + throw new Error( + `recipe '${recipe.id}' states its dataset is ${recipe.license}; params.license says ${job}. A prep cannot relicense a dataset.`, + ); + } + return recipe.license; + } + if (job === "") { + throw new Error( + `recipe '${recipe.id}' is generic and states no license, so params.license is required — a prepared set with no licence is one nobody can publish results from`, + ); + } + return job; +} + +export type Manifest = { + recipe: string; + /** Where the images came from, so the set is traceable without this job's row. */ + source: string; + license: string; + classes?: number; + preprocess?: string; + items: LabelledItem[]; +}; + +/** + * The manifest that goes in the zip. + * + * `items[].file` is the name inside the zip and `items[].label` is an integer, + * because that is what VisionEvalEngine reads; everything the recipe attached + * to an item rides along untouched, the way the plant-ID manifest carries + * `species` and `species_id` beside the label. + */ +export function buildManifest(opts: { + recipe: Recipe; + source: string; + license: string; + names: string[]; + labelling: Labelling; +}): Manifest { + const { recipe, names, labelling } = opts; + if (labelling.items.length !== names.length) { + throw new Error( + `recipe '${recipe.id}' labelled ${labelling.items.length} of ${names.length} images; the manifest must describe every file in the zip`, + ); + } + const items = names.map((file, i) => { + const item = labelling.items[i]!; + if (!Number.isInteger(item.label)) { + throw new Error(`recipe '${recipe.id}' gave ${file} a non-integer label (${JSON.stringify(item.label)})`); + } + // `file` last so a recipe cannot overwrite the name of the file it is + // describing — the zip's layout is the prep's to decide, not the recipe's. + return { ...item, label: item.label, file }; + }); + return { + recipe: recipe.id, + source: opts.source, + license: opts.license, + ...(labelling.classes !== undefined ? { classes: labelling.classes } : {}), + ...(recipe.preprocess ? { preprocess: recipe.preprocess } : {}), + items, + }; +} + +/** `000.jpg`, `001.jpg`, … — flat, zero-padded, and in the order they were kept. */ +export function outputName(index: number, total: number): string { + const width = Math.max(3, String(Math.max(total - 1, 0)).length); + return `${String(index).padStart(width, "0")}.jpg`; +} diff --git a/runner-machine/src/redact.ts b/runner-machine/src/redact.ts new file mode 100644 index 0000000..ecc20ce --- /dev/null +++ b/runner-machine/src/redact.ts @@ -0,0 +1,52 @@ +/** + * Scrubbing secrets out of text that is about to become an artifact. + * + * The same rule as the collector's `redact` in collector/src/secrets.ts, and + * deliberately a re-implementation rather than an import — this agent depends + * on the protocol, not on the collector's source tree, exactly as + * `protocol.ts` and `collector.ts` do. + * + * The `shell` workload is why this exists here. A script from the artifact + * store runs with a minimal environment, but "minimal" is a list this file + * maintains and a script can still print something it was handed; the captured + * output is uploaded to a store anyone on the dashboard can download, so a + * token that reaches it has leaked no matter how carefully it was fetched. + */ + +/** + * Environment variable names whose VALUES must never reach an artifact. + * + * Matched on the name, because the value of a token is by definition + * unrecognisable. `_KEY` rather than `KEY` so that `KEYBOARD_LAYOUT` and + * friends do not turn a common word into a redaction pattern. + */ +export const SECRET_ENV_RE = /(^|_)(TOKEN|SECRET|PASSWORD|PASSWD|CREDENTIALS?|API_?KEY)(_|$)|_KEY$/i; + +/** + * Remove secret values from text bound for the artifact store. + * + * Values shorter than four characters are skipped: a two-character "secret" + * matches half of any log, and a log that is mostly `[redacted]` is a log + * nobody can read — which is its own failure, since the log is the answer to + * "why did this script fail". + */ +export function redact(text: string, secrets: readonly string[]): string { + let out = text; + for (const s of secrets) { + if (!s || s.length < 4) continue; + out = out.split(s).join("[redacted]"); + } + return out; +} + +/** The values in this process's environment that must not be published. */ +export function secretsFromEnv(env: NodeJS.ProcessEnv = process.env): string[] { + const values: string[] = []; + for (const [name, value] of Object.entries(env)) { + if (typeof value !== "string" || value.length < 4) continue; + if (SECRET_ENV_RE.test(name)) values.push(value); + } + // Longest first, so a token that contains a shorter one is not left with a + // recognisable prefix after the shorter match has already been replaced. + return values.sort((a, b) => b.length - a.length); +} diff --git a/runner-machine/src/workloads/datasetprep.ts b/runner-machine/src/workloads/datasetprep.ts new file mode 100644 index 0000000..bddaaa7 --- /dev/null +++ b/runner-machine/src/workloads/datasetprep.ts @@ -0,0 +1,289 @@ +/** + * The `dataset-prep` workload: download an image set, resize it, drop the + * duplicates, write a manifest, and publish the zip the eval workloads read. + * + * The output is not this file's invention. The Android runner's + * VisionEvalEngine opens `manifest.json` at the zip root and walks + * `items[].file` / `items[].label`, so that layout is a contract with code in + * another repo — which is why the shape is built by `recipes.ts` and asserted + * in the tests instead of being assembled inline here. + * + * Two things this adds beyond "unzip and hand it over": + * + * 1. **The licence travels with the data.** The plant set is CC-BY-SA, which + * is exactly why its images move through the artifact store rather than + * being committed. A prepared zip whose manifest has lost the attribution + * is a zip whose eval numbers can never be published, and by the time + * anyone notices, the provenance is a URL somebody half-remembers. So a + * prep with no licence is refused before a single image is resized. + * 2. **The download is content-addressed on `source_url`.** Re-preparing the + * same set with a different recipe, or after a crash, must not pull + * gigabytes over somebody's home internet again. + */ +import { createHash } from "node:crypto"; +import { existsSync } from "node:fs"; +import { mkdir, readFile, readdir, rename, rm, stat, writeFile } from "node:fs/promises"; +import path from "node:path"; +import type { CollectorClient } from "../collector.js"; +import type { Descriptor, JobSpec, Metrics } from "../protocol.js"; +import { SCHEMA, compact, intParam, stringParam } from "../protocol.js"; +import { dataDir } from "../git.js"; +import { run, which } from "../probe.js"; +import { Beaconer, lastMeaningful } from "../beaconing.js"; +import { + buildManifest, effectiveLicense, loadRecipe, outputName, type Recipe, +} from "../recipes.js"; +import { + magickResizeArgs, parseSipsDims, resolveResizer, sipsDimArgs, sipsResizeArgs, type Resizer, +} from "../images.js"; + +export const IMAGE_EXTS = [".jpg", ".jpeg", ".png", ".webp"]; +/** A download that has not finished in two hours is a download that will not. */ +const DOWNLOAD_TIMEOUT_MS = 2 * 60 * 60 * 1000; + +/** The cache key. The URL and nothing else: the same URL is the same set. */ +export function cacheKey(sourceUrl: string): string { + return createHash("sha256").update(sourceUrl).digest("hex").slice(0, 32); +} + +/** http(s) only. A `file:` source would let an unauthenticated job read this disk. */ +export function parseSourceUrl(v: unknown): string { + if (typeof v !== "string" || v.trim() === "") throw new Error("dataset-prep needs params.source_url"); + const s = v.trim(); + let u: URL; + try { + u = new URL(s); + } catch { + throw new Error(`params.source_url is not a URL: ${s.slice(0, 120)}`); + } + if (u.protocol !== "http:" && u.protocol !== "https:") { + throw new Error(`params.source_url must be http or https (got ${u.protocol})`); + } + return s; +} + +export async function runDatasetPrep( + job: JobSpec, + client: CollectorClient, + deviceId: string, + descriptor: Descriptor, + env: NodeJS.ProcessEnv = process.env, +): Promise { + const log = (m: string) => console.log(`[${deviceId}] dataset-prep ${job.job_id}: ${m}`); + const fail = async (error: string, metrics?: Metrics) => { + log(`failed: ${error}`); + await client.postResult({ + schema: SCHEMA, kind: "result", job_id: job.job_id, device_id: deviceId, + iter: 0, final: true, ok: false, device: descriptor, error, + ...(metrics ? { metrics } : {}), + }); + }; + + // Everything that can be refused without touching the network, first. + let sourceUrl: string; + let recipe: Recipe; + let license: string; + try { + sourceUrl = parseSourceUrl(job.params?.source_url); + recipe = await loadRecipe(job.params?.recipe, env); + license = effectiveLicense(recipe, job.params?.license); + } catch (e) { + return fail((e as Error).message); + } + + const resizer = await resolveResizer(env); + if (!resizer) return fail("dataset-prep needs sips (macOS) or ImageMagick; neither resolves on this machine"); + const zipper = await which("zip", env); + if (!zipper) return fail("dataset-prep needs `zip` to publish the prepared set"); + + const beaconer = new Beaconer(client, deviceId, job.job_id); + const key = cacheKey(sourceUrl); + const cacheDir = path.join(dataDir(env), "datasets", key); + const staging = path.join(cacheDir, "prepared", safe(job.job_id)); + await rm(staging, { recursive: true, force: true }).catch(() => {}); + await mkdir(staging, { recursive: true }); + + try { + // --- download, content-addressed on the URL --------------------------- + let extracted: string; + try { + extracted = await fetchAndExtract(sourceUrl, cacheDir, log); + } catch (e) { + return fail(`fetching ${sourceUrl} failed: ${(e as Error).message}`); + } + + // --- the images, in a stable order ------------------------------------ + const limit = intParam(job.params, "max_items", 0); + const all = (await walkImages(extracted)).sort(); + if (all.length === 0) return fail(`${sourceUrl} contained no images (looked for ${IMAGE_EXTS.join(", ")})`); + const chosen = limit > 0 ? all.slice(0, limit) : all; + log(`${all.length} image(s) found, preparing ${chosen.length} at ${recipe.size}px`); + + // --- resize, then deduplicate on the PREPARED bytes ------------------- + // + // Deduplicating after the resize rather than before is the whole point: a + // scraped set is full of the same photograph at three resolutions and two + // JPEG qualities, which are different files and identical evidence. + // Scoring a model twice on one image is how an eval set quietly weights + // whatever was duplicated most. + const seen = new Map(); + const kept: string[] = []; + const keptNames: string[] = []; + let duplicates = 0; + let failedImages = 0; + + for (const rel of chosen) { + if (!(await beaconer.maybe())) { + return fail("cancelled"); + } + const src = path.join(extracted, rel); + const tmp = path.join(staging, `.staging-${kept.length}.jpg`); + const ok = await resizeOne(resizer, src, tmp, recipe.size); + if (!ok) { + failedImages += 1; + continue; + } + const digest = await sha256File(tmp); + const first = seen.get(digest); + if (first !== undefined) { + duplicates += 1; + await rm(tmp, { force: true }).catch(() => {}); + continue; + } + seen.set(digest, rel); + kept.push(rel); + keptNames.push(tmp); + } + if (kept.length === 0) { + return fail(`every one of the ${chosen.length} image(s) failed to resize; is ${resizer.bin} working?`); + } + + // Names are assigned only once the final count is known, so the padding + // width is right and the numbering has no gaps where a duplicate was. + const names = kept.map((_, i) => outputName(i, kept.length)); + for (const [i, tmp] of keptNames.entries()) { + await rename(tmp, path.join(staging, names[i]!)); + } + + // --- manifest --------------------------------------------------------- + let manifestJson: string; + try { + const labelling = recipe.labels(kept.map(toPosix)); + const manifest = buildManifest({ recipe, source: sourceUrl, license, names, labelling }); + manifestJson = JSON.stringify(manifest, null, 1); + } catch (e) { + return fail(`recipe '${recipe.id}' could not label the set: ${(e as Error).message}`); + } + await writeFile(path.join(staging, "manifest.json"), manifestJson, "utf8"); + + // --- zip and publish -------------------------------------------------- + const zipFile = path.join(cacheDir, `${safe(job.job_id)}.zip`); + await rm(zipFile, { force: true }).catch(() => {}); + // `-X` drops the extra attributes that make an otherwise identical zip + // differ byte for byte; `.` from inside the staging dir puts manifest.json + // at the zip ROOT, which is where VisionEvalEngine looks for it. + const zipped = await run(zipper, ["-q", "-r", "-X", zipFile, "."], 60 * 60 * 1000); + if (zipped.code !== 0) { + return fail(`zipping the prepared set failed: ${lastMeaningful(zipped.stderr || zipped.stdout) || `exit ${zipped.code}`}`, undefined); + } + + const bytes = (await stat(zipFile)).size; + let sha256: string; + try { + const up = await client.uploadArtifact(zipFile, `${recipe.id}-${kept.length}.zip`); + sha256 = up.sha256; + } catch (e) { + return fail(`upload failed: ${(e as Error).message}`, compact({ items: kept.length, artifact_bytes: bytes })); + } + + log(`published ${kept.length} item(s) as ${sha256} (${bytes} bytes; ${duplicates} duplicate(s), ${failedImages} unreadable)`); + await client.postResult({ + schema: SCHEMA, kind: "result", job_id: job.job_id, device_id: deviceId, + iter: 0, final: true, ok: true, device: descriptor, + metrics: compact({ items: kept.length, artifact_bytes: bytes }), + artifacts: [sha256], + // Not in the result schema, deliberately, exactly as build's `build` + // block: the collector stores a result's whole payload, and a prepared + // set whose row cannot say what licence it carries is a row that puts + // the attribution back where it was — nowhere anyone will look. + dataset: { + recipe: recipe.id, source: sourceUrl, license, sha256, + items: kept.length, duplicates, unreadable: failedImages, size: recipe.size, + }, + }); + } finally { + await rm(staging, { recursive: true, force: true }).catch(() => {}); + } +} + +/** + * The extracted archive for a URL, downloading it only if the cache is cold. + * + * The blob is written to a `.part` and renamed, so an interrupted download can + * never be mistaken for a complete one on the next run — the same discipline + * `fetchArtifact` uses for its hash check. + */ +async function fetchAndExtract(sourceUrl: string, cacheDir: string, log: (m: string) => void): Promise { + const extracted = path.join(cacheDir, "extracted"); + if (existsSync(path.join(cacheDir, ".complete"))) { + log("cache hit; not downloading"); + return extracted; + } + await mkdir(cacheDir, { recursive: true }); + const blob = path.join(cacheDir, "source.bin"); + if (!existsSync(blob)) { + log(`downloading ${sourceUrl}`); + const part = `${blob}.part`; + const r = await run("curl", ["-fsSL", "--retry", "3", "-o", part, sourceUrl], DOWNLOAD_TIMEOUT_MS); + if (r.code !== 0) { + await rm(part, { force: true }).catch(() => {}); + throw new Error(lastMeaningful(r.stderr || r.stdout) || `curl exited ${r.code}`); + } + await rename(part, blob); + } + + await rm(extracted, { recursive: true, force: true }).catch(() => {}); + await mkdir(extracted, { recursive: true }); + const isTar = /\.(tar\.gz|tgz|tar)$/i.test(new URL(sourceUrl).pathname); + const r = isTar + ? await run("tar", ["-xf", blob, "-C", extracted], DOWNLOAD_TIMEOUT_MS) + : await run("unzip", ["-q", "-o", blob, "-d", extracted], DOWNLOAD_TIMEOUT_MS); + if (r.code !== 0) throw new Error(`extracting failed: ${lastMeaningful(r.stderr || r.stdout) || `exit ${r.code}`}`); + await writeFile(path.join(cacheDir, ".complete"), sourceUrl, "utf8"); + return extracted; +} + +/** Every image under a directory, as paths relative to it. */ +export async function walkImages(root: string, rel = "", depth = 0): Promise { + if (depth > 8) return []; + const entries = await readdir(path.join(root, rel), { withFileTypes: true }).catch(() => []); + const out: string[] = []; + for (const e of entries) { + if (e.name.startsWith(".") || e.name === "__MACOSX") continue; + const child = rel === "" ? e.name : path.join(rel, e.name); + if (e.isDirectory()) out.push(...(await walkImages(root, child, depth + 1))); + else if (IMAGE_EXTS.some((x) => e.name.toLowerCase().endsWith(x))) out.push(child); + } + return out; +} + +/** One image, center-cropped to a square. False when the tool could not read it. */ +async function resizeOne(resizer: Resizer, src: string, dst: string, size: number): Promise { + if (resizer.kind === "magick") { + const r = await run(resizer.bin, magickResizeArgs(src, dst, size), 120_000); + return r.code === 0 && existsSync(dst); + } + // sips has no gravity flag, so the crop can only be centred by controlling + // what goes into it — which needs the source's dimensions first. + const dims = parseSipsDims((await run(resizer.bin, sipsDimArgs(src), 60_000)).stdout); + if (!dims) return false; + const r = await run(resizer.bin, sipsResizeArgs(src, dst, size, dims), 120_000); + return r.code === 0 && existsSync(dst); +} + +async function sha256File(file: string): Promise { + return createHash("sha256").update(await readFile(file)).digest("hex"); +} + +const toPosix = (p: string) => p.split(path.sep).join("/"); +const safe = (s: string) => s.replace(/[^A-Za-z0-9._-]+/g, "-").slice(0, 60); diff --git a/runner-machine/src/workloads/modelconvert.ts b/runner-machine/src/workloads/modelconvert.ts new file mode 100644 index 0000000..0b32440 --- /dev/null +++ b/runner-machine/src/workloads/modelconvert.ts @@ -0,0 +1,294 @@ +/** + * The `model-convert` workload: turn one source checkpoint into the formats + * the fleet's three runners can actually load, on the machine that has the + * memory to do it. + * + * The reason this is a fleet workload rather than a script somebody runs is + * the machine, not the code: quantising a multi-gigabyte checkpoint needs more + * RAM than a phone has and more patience than a laptop on battery has, and the + * collector already knows which machine is on mains and idle. What this + * workload adds is that the OUTPUT is addressable — every product is uploaded + * as an artifact and the final row lists every sha, so a dependent job can say + * `${jobs..artifact}` and get the model this conversion made rather than + * whatever was in someone's Downloads folder. + * + * The conversions themselves are other people's tools, deliberately: llama.cpp + * for GGUF, and the plant-ID assets' own `convert_coreml.py` and + * `quantize_int8.py` for Core ML and LiteRT. Those two scripts already work + * and are already the provenance of the models this fleet has benchmarked; + * rewriting their coremltools plumbing here would produce a second answer to a + * question that already has one. + */ +import { mkdir, open, rm, stat, readdir } from "node:fs/promises"; +import { existsSync } from "node:fs"; +import path from "node:path"; +import type { CollectorClient } from "../collector.js"; +import type { Descriptor, JobSpec, Metrics } from "../protocol.js"; +import { SCHEMA, compact, intParam, stringParam } from "../protocol.js"; +import { fileMatchesHash } from "../collector.js"; +import { dataDir } from "../git.js"; +import { run, which } from "../probe.js"; +import { runWithBeacons, lastMeaningful } from "../beaconing.js"; +import { + parseOutputs, parseSource, planConvert, probeConverters, convertersAvailable, + type ConvertPlan, type ConverterTools, type Source, +} from "../converters.js"; + +/** A conversion of a 70B checkpoint is hours; a hung one should still end. */ +const DEFAULT_STEP_TIMEOUT_S = 6 * 60 * 60; + +export async function runModelConvert( + job: JobSpec, + client: CollectorClient, + deviceId: string, + descriptor: Descriptor, + env: NodeJS.ProcessEnv = process.env, +): Promise { + const log = (m: string) => console.log(`[${deviceId}] model-convert ${job.job_id}: ${m}`); + const fail = async (error: string, artifacts?: string[], metrics?: Metrics) => { + log(`failed: ${error}`); + await client.postResult({ + schema: SCHEMA, kind: "result", job_id: job.job_id, device_id: deviceId, + iter: 0, final: true, ok: false, device: descriptor, error, + ...(artifacts?.length ? { artifacts } : {}), + ...(metrics ? { metrics } : {}), + }); + }; + + let source: Source; + let outputs: ReturnType; + try { + source = parseSource(job.params?.source); + outputs = parseOutputs(job.params?.outputs); + } catch (e) { + return fail((e as Error).message); + } + + const tools = await probeConverters(env); + const available = convertersAvailable(tools); + const missing = outputs.filter((o) => !available.includes(o.format)).map((o) => o.format); + if (missing.length === outputs.length) { + // Every requested format is unbuildable here. Refused as one row rather + // than N, and naming what this machine CAN do, because the answer the + // operator needs is which machine to route the job to. + return fail( + `none of the requested formats can be converted on this machine (asked for ${[...new Set(missing)].join(", ")}; ` + + `available here: ${available.length ? available.join(", ") : "none"})`, + ); + } + + const workRoot = path.join(dataDir(env), "convert", safe(job.job_id)); + await rm(workRoot, { recursive: true, force: true }).catch(() => {}); + await mkdir(workRoot, { recursive: true }); + + const t0 = Date.now(); + const artifacts: string[] = []; + let totalBytes = 0; + try { + let sourcePath: string; + try { + sourcePath = await materialiseSource(source, client, env, log); + } catch (e) { + return fail(`fetching the source failed: ${(e as Error).message}`); + } + + let calibrationDir: string | null = null; + const calibSha = stringParam(job.params, "calibration_sha256"); + if (calibSha) { + try { + calibrationDir = await materialiseSource({ kind: "artifact", sha256: calibSha.toLowerCase() }, client, env, log); + } catch (e) { + return fail(`fetching the calibration set failed: ${(e as Error).message}`); + } + } + + const name = stringParam(job.params, "name") ?? defaultName(source); + const stepTimeoutMs = intParam(job.params, "timeout_s", DEFAULT_STEP_TIMEOUT_S) * 1000; + const failures: string[] = []; + + for (const [i, spec] of outputs.entries()) { + const outDir = path.join(workRoot, `${spec.format}-${(spec.quant ?? "none").toLowerCase()}`); + await mkdir(outDir, { recursive: true }); + + let plan: ConvertPlan; + try { + plan = planConvert(spec, tools, { source: sourcePath, outDir, name, calibrationDir }); + } catch (e) { + failures.push((e as Error).message); + log(`refused ${spec.format}/${spec.quant ?? "none"}: ${(e as Error).message}`); + continue; + } + + log(`converting ${spec.format}/${spec.quant ?? "none"} in ${plan.steps.length} step(s)`); + const stepFailure = await runSteps(plan, job, client, deviceId, stepTimeoutMs, log); + if (stepFailure) { + failures.push(`${spec.format}/${spec.quant ?? "none"}: ${stepFailure}`); + continue; + } + + const product = path.join(outDir, plan.product); + if (!existsSync(product)) { + failures.push( + `${spec.format}/${spec.quant ?? "none"}: the converter exited 0 but wrote no ${plan.product} (saw ${(await readdir(outDir).catch(() => [])).join(", ") || "nothing"})`, + ); + continue; + } + const bytes = (await stat(product)).size; + try { + // Uploaded WITHOUT `x-artifact-app`: a converted model is not a build, + // and publishing it under an app name would make it the thing a + // nightly asking for `"sha256": "latest"` picks up. + const up = await client.uploadArtifact(product, plan.product); + artifacts.push(up.sha256); + totalBytes += bytes; + log(`${plan.product} -> ${up.sha256} (${bytes} bytes)`); + // One row per output, so a four-output conversion is legible while it + // is still running rather than only at the end. + await client.postResult({ + schema: SCHEMA, kind: "result", job_id: job.job_id, device_id: deviceId, + iter: i + 1, ok: true, artifacts: [up.sha256], + model_out: { format: plan.modelFormat, quant: plan.quant, name: plan.product, sha256: up.sha256, bytes }, + }); + } catch (e) { + failures.push(`${spec.format}/${spec.quant ?? "none"}: upload failed: ${(e as Error).message}`); + } + } + + const convertS = Math.round((Date.now() - t0) / 100) / 10; + const metrics = compact({ + convert_s: convertS, + outputs: artifacts.length, + artifact_bytes: totalBytes || undefined, + }); + if (failures.length > 0) { + // Partial success is still a failure of the job — a pipeline that asked + // for three formats and got two must not read as green — but the shas + // that DID land are listed, because they are real and a dependent job may + // still want them. + return fail( + `${failures.length} of ${outputs.length} output(s) failed: ${failures.join(" | ").slice(0, 900)}`, + artifacts, + metrics, + ); + } + await client.postResult({ + schema: SCHEMA, kind: "result", job_id: job.job_id, device_id: deviceId, + iter: 0, final: true, ok: true, device: descriptor, + metrics, + // Every sha, in the order the job asked for them, so a dependent job's + // `${jobs..artifact}` resolves against a list that is not a surprise. + artifacts, + }); + log(`converted ${artifacts.length} output(s) in ${convertS}s`); + } finally { + // The products are in the artifact store now; the working copies are just + // tens of gigabytes on somebody's boot disk. + if (!env.FLEET_KEEP_CONVERT_WORK) await rm(workRoot, { recursive: true, force: true }).catch(() => {}); + } +} + +/** Runs a plan's steps in order, stopping at the first that fails. */ +async function runSteps( + plan: ConvertPlan, + job: JobSpec, + client: CollectorClient, + deviceId: string, + timeoutMs: number, + log: (m: string) => void, +): Promise { + for (const step of plan.steps) { + log(` ${step.label}`); + const r = await runWithBeacons({ + cmd: step.cmd, args: step.args, cwd: step.cwd, timeoutMs, + jobId: job.job_id, deviceId, client, + }); + if (r.cancelled) return "cancelled"; + if (r.timedOut) return `${step.label} timed out after ${Math.round(timeoutMs / 1000)}s`; + if (r.spawnError) return `${step.label} could not start: ${r.spawnError}`; + if (r.code !== 0) { + return `${step.label} exited ${r.code ?? `on signal ${r.signal ?? "?"}`}: ${lastMeaningful(r.output) || "no output"}`; + } + } + return null; +} + +/** + * The source on disk: a verified artifact, or a HuggingFace snapshot. + * + * An artifact that is a zip is extracted, because `convert_hf_to_gguf.py` + * takes a DIRECTORY of a HuggingFace repo rather than a single file, and a + * multi-file checkpoint can only reach the artifact store as one blob. + */ +async function materialiseSource( + source: Source, + client: CollectorClient, + env: NodeJS.ProcessEnv, + log: (m: string) => void, +): Promise { + if (source.kind === "hf") { + const cli = (await which("huggingface-cli", env)) ?? (await which("hf", env)); + if (!cli) { + throw new Error( + `params.source is the HuggingFace repo ${source.repo}, but neither huggingface-cli nor hf resolves on this machine`, + ); + } + const dest = path.join(dataDir(env), "hf", source.repo.replace(/[^A-Za-z0-9._-]+/g, "-")); + await mkdir(path.dirname(dest), { recursive: true }); + log(`downloading ${source.repo}`); + const r = await run(cli, ["download", source.repo, "--local-dir", dest], 6 * 60 * 60 * 1000); + if (r.code !== 0) throw new Error(`${path.basename(cli)} download ${source.repo} failed: ${lastMeaningful(r.stderr || r.stdout) || `exit ${r.code}`}`); + return dest; + } + + const cacheDir = path.join(dataDir(env), "sources"); + const blob = path.join(cacheDir, `${source.sha256}.bin`); + if (!(await fileMatchesHash(blob, source.sha256))) { + log(`fetching artifact ${source.sha256.slice(0, 12)}`); + await client.fetchArtifact(source.sha256, blob); + } + if (!(await isZip(blob))) return blob; + + const extracted = path.join(cacheDir, source.sha256); + if (!existsSync(extracted)) { + await mkdir(extracted, { recursive: true }); + const r = await run("unzip", ["-q", "-o", blob, "-d", extracted], 60 * 60 * 1000); + if (r.code !== 0) { + await rm(extracted, { recursive: true, force: true }).catch(() => {}); + throw new Error(`unzipping ${source.sha256.slice(0, 12)} failed: ${lastMeaningful(r.stderr || r.stdout) || `exit ${r.code}`}`); + } + } + // A zip of one directory is the common shape; hand the converter that + // directory rather than its parent, which would look empty of config.json. + const entries = await readdir(extracted, { withFileTypes: true }).catch(() => []); + const dirs = entries.filter((e) => e.isDirectory() && !e.name.startsWith("__MACOSX")); + const files = entries.filter((e) => e.isFile()); + if (dirs.length === 1 && files.length === 0) return path.join(extracted, dirs[0]!.name); + return extracted; +} + +/** The four bytes that make a file a zip. */ +export async function isZip(file: string): Promise { + try { + const fh = await open(file, "r"); + try { + const buf = Buffer.alloc(4); + const { bytesRead } = await fh.read(buf, 0, 4, 0); + return bytesRead === 4 && buf[0] === 0x50 && buf[1] === 0x4b && (buf[2] === 3 || buf[2] === 5 || buf[2] === 7); + } finally { + await fh.close(); + } + } catch { + return false; + } +} + +/** A filename stem for the products, from the job or from the source. */ +export function defaultName(source: Source): string { + const raw = source.kind === "hf" ? (source.repo.split("/").pop() ?? "model") : `model-${source.sha256.slice(0, 12)}`; + return raw.replace(/[^A-Za-z0-9._-]+/g, "-").slice(0, 60) || "model"; +} + +const safe = (s: string) => s.replace(/[^A-Za-z0-9._-]+/g, "-").slice(0, 60); + +/** Kept for the tests: what the planner needs, with nothing installed. */ +export type { ConverterTools }; diff --git a/runner-machine/src/workloads/serve.ts b/runner-machine/src/workloads/serve.ts new file mode 100644 index 0000000..176673f --- /dev/null +++ b/runner-machine/src/workloads/serve.ts @@ -0,0 +1,405 @@ +/** + * The `serve` workload: hold an inference server up for the length of a lease, + * announce where it is, and take it down again — always. + * + * Two things dominate the design. + * + * 1. **Teardown on every exit path.** `llama-server` holds the whole model + * resident; a leaked one costs gigabytes on a machine that will go on + * claiming other jobs, and nothing on the dashboard would ever say so. So + * the supervision loop is a `try/finally` around a teardown that runs once + * and runs for all five endings: the duration elapsing, `lease_renewed: + * false`, a dashboard cancel, the child dying on its own, and an uncaught + * throw anywhere in the loop. + * 2. **Never `0.0.0.0`.** The collector's README says it outright: there is no + * authentication anywhere in this system, the network IS the access + * control, and the posture holds only while every participant is on a + * network the owner chose. A model server bound to every interface on a + * laptop is an open inference endpoint on whatever hotel wifi that laptop + * joins next. Loopback and the tailnet address are the two answers, which + * is the same choice `FLEET_BIND` gives the collector itself. + */ +import { spawn, type ChildProcess } from "node:child_process"; +import net from "node:net"; +import os from "node:os"; +import path from "node:path"; +import type { CollectorClient } from "../collector.js"; +import type { Descriptor, JobSpec, Metrics } from "../protocol.js"; +import { SCHEMA, compact, intParam, stringParam } from "../protocol.js"; +import { beacon } from "../telemetry.js"; +import { which } from "../probe.js"; +import { dataDir } from "../git.js"; +import { fileMatchesHash } from "../collector.js"; +import * as JobCancellation from "../cancellation.js"; + +/** The beacon cadence the task asks for, and the lease renewal with it. */ +export const SERVE_BEACON_MS = 30_000; +/** A serve with no duration still ends: an unbounded one is a leak with a lease on it. */ +export const DEFAULT_DURATION_S = 30 * 60; +/** How long a model has to load before "serving" stops being a plausible claim. */ +export const DEFAULT_READY_TIMEOUT_S = 300; +/** Grace between SIGTERM and SIGKILL. llama-server frees its weights on SIGTERM. */ +export const KILL_GRACE_MS = 5_000; + +// --- binding ---------------------------------------------------------------- + +/** Tailscale hands out 100.64.0.0/10. That range is the tailnet and nothing else is. */ +export function isTailnetV4(addr: string): boolean { + const m = /^(\d{1,3})\.(\d{1,3})\.\d{1,3}\.\d{1,3}$/.exec(addr); + if (!m) return false; + return Number(m[1]) === 100 && Number(m[2]) >= 64 && Number(m[2]) <= 127; +} + +/** This machine's tailnet address, or null when it is not on one. */ +export function tailnetAddress(interfaces: NodeJS.Dict = os.networkInterfaces()): string | null { + for (const addrs of Object.values(interfaces)) { + for (const a of addrs ?? []) { + if (a.family === "IPv4" && !a.internal && isTailnetV4(a.address)) return a.address; + } + } + return null; +} + +/** + * Where the server may listen. + * + * Deliberately a closed set rather than a validated free-form address. A job + * spec arrives through an unauthenticated `POST /jobs`, so "any address that + * parses" would let whoever enqueued the job choose to publish this machine's + * model server on the LAN — and `0.0.0.0` is one keystroke away from every + * plausible spelling of an address anyway. + */ +export function resolveBindHost( + param: unknown, + env: NodeJS.ProcessEnv = process.env, + interfaces: NodeJS.Dict = os.networkInterfaces(), +): string { + const want = (typeof param === "string" && param.trim() !== "" ? param : env.FLEET_SERVE_BIND ?? "loopback").trim(); + if (want === "loopback" || want === "127.0.0.1" || want === "localhost") return "127.0.0.1"; + if (want === "tailnet") { + const addr = tailnetAddress(interfaces); + if (!addr) throw new Error("params.bind is 'tailnet' but this machine has no 100.64.0.0/10 address"); + return addr; + } + if (isTailnetV4(want)) { + // An explicit tailnet address still has to be one of THIS machine's, or + // llama-server would fail to bind and the failure would read as a crash. + const addr = tailnetAddress(interfaces); + if (addr !== want) throw new Error(`params.bind ${want} is not an address on this machine (tailnet address is ${addr ?? "absent"})`); + return want; + } + throw new Error( + `params.bind must be 'loopback' or 'tailnet' (got ${JSON.stringify(want)}). ` + + "The collector has no authentication and the network is the access control, so a model server is never bound to a wildcard or to the LAN.", + ); +} + +/** A port nothing is listening on, on the address the server will bind. */ +export function freePort(host: string): Promise { + return new Promise((resolve, reject) => { + const srv = net.createServer(); + srv.on("error", reject); + srv.listen({ host, port: 0, exclusive: true }, () => { + const addr = srv.address(); + const port = typeof addr === "object" && addr !== null ? addr.port : 0; + srv.close(() => (port > 0 ? resolve(port) : reject(new Error("could not find a free port")))); + }); + }); +} + +// --- supervision ------------------------------------------------------------ + +export type ServeStop = "duration" | "lease" | "cancelled" | "child-exit" | "error"; +export type ChildExit = { code: number | null; signal: NodeJS.Signals | null }; +export type ServeOutcome = { + stop: ServeStop; + error?: string; + exit?: ChildExit; + /** Proof the teardown ran. Asserted by the tests for every stop reason. */ + toreDown: boolean; + elapsedMs: number; +}; + +/** + * Runs the server for its duration, ticking a beacon, and tears it down. + * + * Everything is injected so the five endings can be tested without a model + * server: `exited` is the child's close event, `tick` is the beacon (it names + * the reason to stop, or null to keep going), and `teardown` is the kill. The + * `finally` is the point of the whole function — a throw from `tick` must + * still stop a process holding eight gigabytes of weights. + */ +export async function superviseServe(deps: { + exited: Promise; + teardown: () => Promise | void; + tick: () => Promise<"lease" | "cancelled" | null>; + tickMs: number; + durationMs: number; + now?: () => number; +}): Promise { + const now = deps.now ?? Date.now; + const started = now(); + const deadline = started + deps.durationMs; + let toreDown = false; + const teardownOnce = async () => { + if (toreDown) return; + toreDown = true; + await deps.teardown(); + }; + + let stop: ServeStop = "duration"; + let error: string | undefined; + let exit: ChildExit | undefined; + + try { + const exitedTagged = deps.exited.then((e) => ({ kind: "exit" as const, exit: e })); + for (;;) { + const remaining = deadline - now(); + if (remaining <= 0) { + stop = "duration"; + break; + } + const raced = await Promise.race([ + exitedTagged, + delay(Math.min(deps.tickMs, remaining)).then(() => ({ kind: "tick" as const })), + ]); + if (raced.kind === "exit") { + // The server fell over on its own. That is a failure of the job, not a + // clean end, and the exit code is what says why. + stop = "child-exit"; + exit = raced.exit; + break; + } + const reason = await deps.tick(); + if (reason !== null) { + stop = reason; + break; + } + } + } catch (e) { + stop = "error"; + error = (e as Error).message ?? String(e); + } finally { + await teardownOnce(); + } + + return { stop, error, exit, toreDown, elapsedMs: now() - started }; +} + +function delay(ms: number): Promise { + return new Promise((resolve) => { + const t = setTimeout(resolve, Math.max(0, ms)); + // Unref'd: a pending tick timer must never be the reason this process + // outlives the job it was serving. + t.unref?.(); + }); +} + +// --- the workload ----------------------------------------------------------- + +export async function runServe( + job: JobSpec, + client: CollectorClient, + deviceId: string, + descriptor: Descriptor, + env: NodeJS.ProcessEnv = process.env, +): Promise { + const log = (m: string) => console.log(`[${deviceId}] serve ${job.job_id}: ${m}`); + const fail = async (error: string, metrics?: Metrics) => { + log(`failed: ${error}`); + await client.postResult({ + schema: SCHEMA, kind: "result", job_id: job.job_id, device_id: deviceId, + iter: 0, final: true, ok: false, device: descriptor, error, + ...(metrics ? { metrics } : {}), + }); + }; + + const server = await which(env.FLEET_LLAMA_SERVER ?? "llama-server", env); + if (!server) return fail("serve needs llama-server on PATH (or FLEET_LLAMA_SERVER)"); + + const sha = job.model?.sha256 ?? stringParam(job.params, "model_sha256"); + if (!sha || !/^[0-9a-f]{64}$/i.test(sha)) { + return fail("serve needs a model artifact: job.model.sha256 or params.model_sha256"); + } + + let host: string; + try { + host = resolveBindHost(job.params?.bind, env); + } catch (e) { + return fail((e as Error).message); + } + + const modelFile = path.join(dataDir(env), "models", `${sha.toLowerCase()}.gguf`); + try { + if (!(await fileMatchesHash(modelFile, sha.toLowerCase()))) { + log(`fetching model ${sha.slice(0, 12)}`); + await client.fetchArtifact(sha.toLowerCase(), modelFile); + } + } catch (e) { + return fail(`fetching the model failed: ${(e as Error).message}`); + } + + const durationS = intParam(job.params, "duration_s", DEFAULT_DURATION_S); + const readyTimeoutS = intParam(job.params, "ready_timeout_s", DEFAULT_READY_TIMEOUT_S); + let port: number; + try { + port = intParam(job.params, "port", 0) || (await freePort(host)); + } catch (e) { + return fail(`no free port on ${host}: ${(e as Error).message}`); + } + const endpoint = `http://${host}:${port}`; + + const extra = Array.isArray(job.params?.args) + ? (job.params.args as unknown[]).filter((a): a is string => typeof a === "string") + : []; + const child = spawn(server, ["-m", modelFile, "--host", host, "--port", String(port), ...extra], { + stdio: ["ignore", "ignore", "pipe"], + env: { ...process.env, TERM: "dumb" }, + }); + let stderrTail = ""; + child.stderr?.on("data", (b: Buffer) => { + stderrTail = (stderrTail + b.toString("utf8")).slice(-4000); + }); + + const exited = new Promise((resolve) => { + child.on("error", () => resolve({ code: null, signal: null })); + child.on("close", (code, signal) => resolve({ code, signal })); + }); + let childGone = false; + void exited.then(() => (childGone = true)); + + const teardown = async () => { + if (childGone) return; + log("stopping llama-server"); + try { + child.kill("SIGTERM"); + } catch { + /* already gone */ + } + await Promise.race([exited, delay(KILL_GRACE_MS)]); + if (!childGone) { + try { + child.kill("SIGKILL"); + } catch { + /* already gone */ + } + await Promise.race([exited, delay(KILL_GRACE_MS)]); + } + }; + + // From here every path goes through the finally, including a throw. + try { + const ready = await waitForReady(endpoint, readyTimeoutS * 1000, exited); + if (!ready.ok) { + await teardown(); + return fail( + `llama-server did not answer on ${endpoint} within ${readyTimeoutS}s: ${ready.detail || lastLine(stderrTail) || "no output"}`, + ); + } + log(`serving ${sha.slice(0, 12)} on ${endpoint} for ${durationS}s (pid ${child.pid})`); + + // The announcement, immediately — a serve nobody can find is a serve that + // did not happen, and a dependent job may be waiting on this row. + await client.postResult({ + schema: SCHEMA, kind: "result", job_id: job.job_id, device_id: deviceId, + iter: 1, ok: true, endpoint, + }); + + const outcome = await superviseServe({ + exited, + teardown, + tickMs: SERVE_BEACON_MS, + durationMs: durationS * 1000, + tick: async () => { + if (JobCancellation.isCancelled(job.job_id)) return "cancelled"; + try { + const renewed = await client.postBeacon({ + schema: SCHEMA, kind: "beacon", job_id: job.job_id, device_id: deviceId, + beacon: await beacon(), endpoint, + }); + if (!renewed) { + JobCancellation.cancel(job.job_id); + return "lease"; + } + } catch { + // An unreachable collector is not a cancellation, exactly as in the + // build workload: the lease may lapse, but a flaky network must not + // be what takes a served model down. + } + return null; + }, + }); + + const elapsedS = Math.round(outcome.elapsedMs / 100) / 10; + if (outcome.stop === "duration") { + log(`duration elapsed after ${elapsedS}s`); + await client.postResult({ + schema: SCHEMA, kind: "result", job_id: job.job_id, device_id: deviceId, + iter: 0, final: true, ok: true, device: descriptor, endpoint, + metrics: compact({ elapsed_s: elapsedS }), + }); + return; + } + if (outcome.stop === "child-exit") { + return fail( + `llama-server exited on its own after ${elapsedS}s (${outcome.exit?.code !== null && outcome.exit?.code !== undefined ? `exit ${outcome.exit.code}` : `signal ${outcome.exit?.signal ?? "?"}`}): ${lastLine(stderrTail) || "no output"}`, + compact({ elapsed_s: elapsedS, exit_code: outcome.exit?.code ?? undefined }), + ); + } + return fail( + outcome.stop === "error" + ? `serve supervision failed: ${outcome.error}` + : outcome.stop === "lease" + ? "lease not renewed" + : "cancelled", + compact({ elapsed_s: elapsedS }), + ); + } finally { + // Belt and braces over superviseServe's own finally: a throw between the + // spawn and the supervision — a failed announcement post, say — must not + // leave a model server running. + await teardown(); + } +} + +/** Polls the server's own health endpoint, giving up if the child dies first. */ +async function waitForReady( + endpoint: string, + timeoutMs: number, + exited: Promise, +): Promise<{ ok: boolean; detail: string }> { + const deadline = Date.now() + timeoutMs; + let dead = false; + void exited.then(() => (dead = true)); + let detail = ""; + while (Date.now() < deadline) { + if (dead) return { ok: false, detail: "the process exited before it answered" }; + try { + const res = await fetch(`${endpoint}/health`, { signal: AbortSignal.timeout(5_000) }); + // llama-server answers 503 while the model loads and 200 once it can + // serve, so only 200 is ready; anything else is another second of wait. + if (res.ok) return { ok: true, detail: "" }; + detail = `HTTP ${res.status}`; + } catch (e) { + detail = (e as Error).message; + } + await delay(1000); + } + return { ok: false, detail }; +} + +const lastLine = (s: string): string => + s.split(/\r?\n/).map((l) => l.trim()).filter(Boolean).at(-1)?.slice(0, 300) ?? ""; + +/** + * Is a pid still there? Used by the end-to-end check that a cancelled serve + * actually took its child with it, rather than assuming the kill landed. + */ +export function pidAlive(pid: number): boolean { + try { + process.kill(pid, 0); + return true; + } catch { + return false; + } +} diff --git a/runner-machine/src/workloads/shell.ts b/runner-machine/src/workloads/shell.ts new file mode 100644 index 0000000..f2a96fe --- /dev/null +++ b/runner-machine/src/workloads/shell.ts @@ -0,0 +1,275 @@ +/** + * The `shell` workload: run a script that is an artifact — and only one the + * machine's owner listed by hash. + * + * `POST /jobs` on the collector is unauthenticated by design, and so is + * `POST /artifacts`. That means anyone who can reach the collector can upload + * a script AND enqueue a job naming it, so this workload's allowlist is not + * defence in depth: it is the entire trust boundary, and every other property + * here follows from that. + * + * The order matters more than anything else in the file. The sha is checked + * against the allowlist BEFORE the artifact is fetched. Checking after the + * download would mean an attacker-supplied file had already been written to + * this machine's disk under a name of their choosing, and "we deleted it + * again" is not a defence anybody would accept. + * + * The other three, in the order they bite: + * - a refusal is a result row, never a throw. A job that vanished with a + * stack trace tells the operator nothing about WHY it was refused, and the + * refusal reason is the only feedback loop the allowlist has. + * - a fresh temp dir and a minimal environment. The agent's own process env + * holds FLEET_DASH_TOKEN and a GitHub token; handing those to a script + * would make the allowlist the only thing between a job and this machine's + * credentials, and there is no reason to make the allowlist carry that too. + * - `exit_code` as its own metric. A script that deliberately exits 3 and one + * that could not start are both "not zero" and they are not the same event; + * an alert rule needs to be able to tell them apart without parsing prose. + */ +import { spawn } from "node:child_process"; +import { chmod, mkdtemp, rm, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import type { CollectorClient } from "../collector.js"; +import type { Descriptor, JobSpec, Metrics } from "../protocol.js"; +import { SCHEMA, compact, intParam, stringParam } from "../protocol.js"; +import { isAllowlisted, loadAllowlist, SHA256_RE } from "../allowlist.js"; +import { redact, secretsFromEnv } from "../redact.js"; +import * as JobCancellation from "../cancellation.js"; + +/** A script that has not finished in an hour is a script that is not going to. */ +export const DEFAULT_TIMEOUT_S = 3600; +/** What is kept of the output. Enough to diagnose; not enough to exhaust memory. */ +export const MAX_CAPTURE_BYTES = 4 << 20; + +/** + * The environment a script gets. + * + * An allowlist entry says "this exact script may run here"; it does not say + * "this script may read every credential this agent holds". PATH, HOME and + * TMPDIR are what a script needs to be a script at all, and HOME points at the + * job's own temp dir so a script writing a dotfile does not write it into the + * operator's. + */ +export function minimalEnv( + workDir: string, + job: JobSpec, + env: NodeJS.ProcessEnv = process.env, +): NodeJS.ProcessEnv { + const base: NodeJS.ProcessEnv = { + PATH: env.PATH ?? "/usr/bin:/bin:/usr/sbin:/sbin", + HOME: workDir, + TMPDIR: workDir, + PWD: workDir, + LANG: env.LANG ?? "en_US.UTF-8", + TERM: "dumb", + CI: "1", + FLEET_JOB_ID: job.job_id, + }; + // Job-supplied variables, which are the only way a script is parameterised — + // and they are strings from a job spec, so a name that could shadow PATH or + // reintroduce a credential name is dropped rather than sanitised. + const extra = job.params?.env; + if (extra && typeof extra === "object" && !Array.isArray(extra)) { + for (const [k, v] of Object.entries(extra as Record)) { + if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(k)) continue; + if (k in base) continue; + if (typeof v === "string" || typeof v === "number" || typeof v === "boolean") base[k] = String(v); + } + } + return base; +} + +export async function runShell( + job: JobSpec, + client: CollectorClient, + deviceId: string, + descriptor: Descriptor, + env: NodeJS.ProcessEnv = process.env, +): Promise { + const log = (m: string) => console.log(`[${deviceId}] shell ${job.job_id}: ${m}`); + const post = async (ok: boolean, error?: string, metrics?: Metrics, artifacts?: string[]) => { + await client.postResult({ + schema: SCHEMA, kind: "result", job_id: job.job_id, device_id: deviceId, + iter: 0, final: true, ok, device: descriptor, + ...(error ? { error } : {}), + ...(metrics ? { metrics } : {}), + ...(artifacts?.length ? { artifacts } : {}), + }); + }; + /** Every refusal is a row, never a throw. */ + const refuse = async (reason: string) => { + log(`refused: ${reason}`); + await post(false, reason); + }; + + const requested = stringParam(job.params, "sha256") ?? job.model?.sha256; + if (!requested) return refuse("shell needs params.sha256: the sha256 of an allowlisted script artifact"); + + // --- the gate. Nothing is fetched above this line. ------------------------ + const list = await loadAllowlist(env); + if (!list.exists) { + return refuse(`shell is refused: this machine has no allowlist at ${list.file}`); + } + if (list.allowed.length === 0) { + return refuse(`shell is refused: the allowlist at ${list.file} lists no script hashes`); + } + if (!isAllowlisted(requested, list.allowed)) { + const shape = SHA256_RE.test(requested.trim().toLowerCase()) ? "" : " (which is not a sha256)"; + return refuse( + `shell is refused: ${requested.slice(0, 80)}${shape} is not in ${list.file}` + + (list.rejected.length ? `. Note that ${list.rejected.length} line(s) in that file are not sha256s and were ignored` : ""), + ); + } + const sha = requested.trim().toLowerCase(); + log(`${sha.slice(0, 12)} is allowlisted; fetching`); + + // --- from here it is a script the owner vouched for ---------------------- + const workDir = await mkdtemp(path.join(os.tmpdir(), `fleet-shell-${safe(job.job_id)}-`)); + const secrets = secretsFromEnv(env); + try { + const script = path.join(workDir, "script"); + try { + await client.fetchArtifact(sha, script); + } catch (e) { + return post(false, `fetching the script failed: ${(e as Error).message}`); + } + await chmod(script, 0o700); + + const interpreter = stringParam(job.params, "interpreter") ?? "/bin/sh"; + const args = Array.isArray(job.params?.args) + ? (job.params.args as unknown[]).filter((a): a is string => typeof a === "string") + : []; + const timeoutMs = intParam(job.params, "timeout_s", DEFAULT_TIMEOUT_S) * 1000; + + const started = Date.now(); + const outcome = await spawnCapturing(interpreter, [script, ...args], workDir, minimalEnv(workDir, job, env), timeoutMs, job.job_id); + const elapsedS = Math.round((Date.now() - started) / 100) / 10; + + // Redacted before it is written anywhere, not before it is uploaded: the + // file on disk is in a temp dir this job owns, but the habit of scrubbing + // at the boundary is what stops the next change from leaking. + const captured = redact(outcome.output, secrets); + const logFile = path.join(workDir, "output.txt"); + await writeFile(logFile, captured, "utf8"); + let logRef: string | null = null; + try { + const up = await client.uploadArtifact(logFile, `${safe(job.job_id)}-shell.txt`); + logRef = up.sha256; + } catch (e) { + log(`output upload failed: ${(e as Error).message}`); + } + + if (outcome.spawnError) { + // Could not start. Deliberately NO exit_code: there was no exit. + return post( + false, + `could not run ${interpreter}: ${outcome.spawnError}`, + compact({ elapsed_s: elapsedS }), + logRef ? [logRef] : undefined, + ); + } + const metrics = compact({ exit_code: outcome.code ?? undefined, elapsed_s: elapsedS }); + if (outcome.timedOut) { + return post(false, `script timed out after ${Math.round(timeoutMs / 1000)}s`, metrics, logRef ? [logRef] : undefined); + } + if (outcome.cancelled) { + return post(false, "cancelled", metrics, logRef ? [logRef] : undefined); + } + if (outcome.code !== 0) { + return post( + false, + `script exited ${outcome.code ?? `on signal ${outcome.signal ?? "?"}`}: ${lastLine(captured) || "no output"}`, + metrics, + logRef ? [logRef] : undefined, + ); + } + log(`exited 0 in ${elapsedS}s`); + await post(true, undefined, metrics, logRef ? [logRef] : undefined); + } finally { + // The temp dir is the sandbox; leaving it behind would accumulate one per + // job on a machine that runs this nightly. + await rm(workDir, { recursive: true, force: true }).catch(() => {}); + } +} + +type Capture = { + code: number | null; + signal: NodeJS.Signals | null; + output: string; + timedOut: boolean; + cancelled: boolean; + spawnError: string | null; +}; + +/** + * Runs the script, merging stdout and stderr. + * + * Merged because a script's diagnostics go to stderr and its progress to + * stdout, and reading them apart loses which came first — the same call the + * build workload makes, for the same reason. + */ +async function spawnCapturing( + cmd: string, + args: string[], + cwd: string, + env: NodeJS.ProcessEnv, + timeoutMs: number, + jobId: string, +): Promise { + return new Promise((resolve) => { + let output = ""; + let bytes = 0; + let truncated = false; + let timedOut = false; + let cancelled = false; + let spawnError: string | null = null; + + const child = spawn(cmd, args, { cwd, env, stdio: ["ignore", "pipe", "pipe"] }); + const absorb = (b: Buffer) => { + if (bytes >= MAX_CAPTURE_BYTES) { + if (!truncated) { + truncated = true; + output += "\n[output truncated]\n"; + } + return; + } + bytes += b.length; + output += b.toString("utf8"); + }; + child.stdout?.on("data", absorb); + child.stderr?.on("data", absorb); + + const killer = setTimeout(() => { + timedOut = true; + child.kill("SIGKILL"); + }, timeoutMs); + // A cancelled shell job stops the script rather than waiting it out: unlike + // a benchmark there is no iteration boundary to stop at. + const canceller = setInterval(() => { + if (JobCancellation.isCancelled(jobId) && !cancelled) { + cancelled = true; + child.kill("SIGTERM"); + setTimeout(() => child.kill("SIGKILL"), 5_000).unref(); + } + }, 1_000); + canceller.unref?.(); + + const done = (c: Capture) => { + clearTimeout(killer); + clearInterval(canceller); + resolve(c); + }; + child.on("error", (e) => { + spawnError = e.message; + done({ code: null, signal: null, output, timedOut, cancelled, spawnError }); + }); + child.on("close", (code, signal) => { + done({ code, signal, output, timedOut, cancelled, spawnError }); + }); + }); +} + +const safe = (s: string) => s.replace(/[^A-Za-z0-9._-]+/g, "-").slice(0, 60); +const lastLine = (s: string): string => + s.split(/\r?\n/).map((l) => l.trim()).filter(Boolean).at(-1)?.slice(0, 300) ?? ""; diff --git a/runner-machine/test/allowlist.test.ts b/runner-machine/test/allowlist.test.ts new file mode 100644 index 0000000..92e11ea --- /dev/null +++ b/runner-machine/test/allowlist.test.ts @@ -0,0 +1,108 @@ +/** + * The shell allowlist is the fleet's only trust boundary. + * + * `POST /jobs` is unauthenticated by design — anyone who can reach the + * collector can enqueue work — so "run this script" is the one workload where + * the collector's word cannot be enough. The machine's owner pins a sha by + * hand, and nothing else runs. Every test here is about the refusal, because a + * permissive bug in this file is arbitrary code execution and a restrictive + * one is merely a job that does not run. + */ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { mkdtemp, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { parseAllowlist, isAllowlisted, loadAllowlist } from "../src/allowlist.js"; +import { capabilitiesFrom } from "../src/capabilities.js"; + +const A = "a".repeat(64); +const B = "b".repeat(64); + +test("a listed sha is allowed and an unlisted one is not", () => { + assert.equal(isAllowlisted(A, [A]), true); + assert.equal(isAllowlisted(B, [A]), false); +}); + +test("one character of difference is a different script", () => { + // The whole point of addressing a script by content: an attacker who can + // enqueue jobs can name any sha they like, and only the exact bytes the + // owner pinned may run. + const nearly = `${"a".repeat(63)}b`; + assert.equal(isAllowlisted(nearly, [A]), false); +}); + +test("case differs between shasum and a dashboard, and is the same hash", () => { + assert.equal(isAllowlisted(A.toUpperCase(), [A]), true); +}); + +test("anything that is not a well-formed sha256 is refused", () => { + // These arrive from a job spec, so they are attacker-controlled. A truthy + // non-string must never reach a comparison that could coerce. + for (const bad of ["", "not-a-sha", A.slice(0, 63), `${A}a`, "../../etc/passwd", null, undefined, 42, {}, [A]]) { + assert.equal(isAllowlisted(bad, [A]), false, `refused: ${JSON.stringify(bad)}`); + } +}); + +test("an empty allowlist allows nothing at all", () => { + assert.equal(isAllowlisted(A, []), false); +}); + +test("comments and blank lines are ignored; malformed entries are reported, not allowed", () => { + const { allowed, rejected } = parseAllowlist( + ["# the deploy script", "", `${A} # pinned 2026-09-04`, " ", "nonsense-line", `${B}`].join("\n"), + ); + assert.deepEqual(allowed, [A, B]); + // Surfaced rather than silently dropped: a typo'd sha should look like a + // mistake in the file, not like a script that mysteriously will not run. + assert.deepEqual(rejected, ["nonsense-line"]); +}); + +test("a duplicate sha is listed once", () => { + assert.deepEqual(parseAllowlist([A, A].join("\n")).allowed, [A]); +}); + +test("a missing allowlist file is empty, not an error", async () => { + const dir = await mkdtemp(path.join(tmpdir(), "fleet-allow-")); + const env = { FLEET_SHELL_ALLOWLIST: path.join(dir, "absent.txt") } as NodeJS.ProcessEnv; + const res = await loadAllowlist(env); + assert.equal(res.exists, false); + assert.deepEqual(res.allowed, []); +}); + +test("a file that exists is read", async () => { + const dir = await mkdtemp(path.join(tmpdir(), "fleet-allow-")); + const file = path.join(dir, "shell-allowlist.txt"); + await writeFile(file, `# pinned by hand\n${A}\n`); + const res = await loadAllowlist({ FLEET_SHELL_ALLOWLIST: file } as NodeJS.ProcessEnv); + assert.equal(res.exists, true); + assert.deepEqual(res.allowed, [A]); +}); + +test("shell is declared only when this machine has a non-empty allowlist", () => { + const bare = { llamaBench: false, mlx: false, gradle: false, xcodebuild: false, node: false }; + // A machine whose owner has pinned nothing must be unable to CLAIM a shell + // job at all, rather than claiming it and refusing it afterwards. Refusing + // after the claim would still take the job off the queue. + assert.equal(capabilitiesFrom({ ...bare, shellAllowlist: false }).includes("shell"), false); + assert.equal(capabilitiesFrom({ ...bare, shellAllowlist: true }).includes("shell"), true); +}); + +test("model-convert is declared bare as well as per format", () => { + const bare = { llamaBench: false, mlx: false, gradle: false, xcodebuild: false, node: false }; + const caps = capabilitiesFrom({ ...bare, converters: ["gguf", "tflite"] }); + // Bare, because the collector's capabilityMatches reads workload and backend + // and a job spec has nowhere to put an output format — so the specific + // labels alone would be a machine that can convert and never claims one. + assert.equal(caps.includes("model-convert"), true); + assert.equal(caps.includes("model-convert:gguf"), true); + assert.equal(caps.includes("model-convert:tflite"), true); + assert.equal(caps.includes("model-convert:coreml"), false); + assert.equal(capabilitiesFrom({ ...bare, converters: [] }).includes("model-convert"), false); +}); + +test("serve is declared only where a llama-server resolved", () => { + const bare = { llamaBench: false, mlx: false, gradle: false, xcodebuild: false, node: false }; + assert.equal(capabilitiesFrom({ ...bare, llamaServer: false }).includes("serve"), false); + assert.equal(capabilitiesFrom({ ...bare, llamaServer: true }).includes("serve"), true); +});