> = {
+ online: "online",
+ stale: "stale",
+ offline: "offline",
+ queued: "queued",
+ claimed: "claimed",
+ done: "done",
+ failed: "failed",
+ cancelled: "cancelled",
+};
export const Pill = ({ kind, children }: { kind: string; children?: ComponentChildren }) => {
const icon = PILL_ICON[kind];
@@ -62,8 +102,38 @@ export const Pill = ({ kind, children }: { kind: string; children?: ComponentChi
);
};
-export function Loading({ what }: { what: string }) {
- return Loading {what}…
;
+/**
+ * A shape-true placeholder for the panel that is about to arrive.
+ *
+ * "Loading devices…" told you the same thing every skeleton does, but the page
+ * then jumped from one line of text to a full table. A placeholder shaped like
+ * its panel means the layout is already correct when the data lands, so
+ * nothing below it moves.
+ *
+ * `what` is still announced, for anyone who is being read the page rather than
+ * looking at it — the blocks themselves are decoration and stay hidden.
+ */
+export function Loading({ what, shape = "rows" }: { what: string; shape?: "rows" | "stats" | "none" }) {
+ if (shape === "none") return Loading {what}…
;
+ return (
+
+
Loading {what}…
+ {shape === "stats" ? (
+
+ {[0, 1, 2, 3, 4, 5].map((i) => (
+
+ ))}
+
+ ) : (
+
+ )}
+
({
state,
what,
+ shape,
+ empty,
children,
}: {
state: { data: T | null; error: Error | null; loading: boolean };
what: string;
+ /** Shape of the placeholder while the first response is in flight. */
+ shape?: "rows" | "stats" | "none";
+ /** Drawn instead of the plain "No x." sentence when the fetch returns nothing. */
+ empty?: ComponentChildren;
children: (data: T) => ComponentChildren;
}) {
if (state.error) return No {what}. ;
+ if (!state.data)
+ return state.loading ? No {what}. };
return <>{children(state.data)};
}
+/**
+ * A clock that ticks, for anything counting down between polls.
+ *
+ * Lease countdowns used to move only when the 30s refresh landed, so a bar
+ * that should drain smoothly jumped in six-percent steps and a "1m 20s left"
+ * could sit there reading 1m 20s for half a minute. The deadline is already in
+ * the payload — this just lets the page do the subtraction itself, which is
+ * arithmetic on data the collector sent, not a number the page invented.
+ */
+export function useNow(everyMs = 1000): number {
+ const [now, setNow] = useState(() => Date.now());
+ useEffect(() => {
+ // A backgrounded tab has nothing to redraw; setInterval there is pure
+ // wake-ups. It resyncs on the way back in.
+ const id = setInterval(() => {
+ if (document.visibilityState === "visible") setNow(Date.now());
+ }, everyMs);
+ const on = () => setNow(Date.now());
+ addEventListener("visibilitychange", on);
+ return () => {
+ clearInterval(id);
+ removeEventListener("visibilitychange", on);
+ };
+ }, [everyMs]);
+ return now;
+}
+
+/**
+ * A lease, counted down locally from its deadline.
+ *
+ * Returns the same two numbers the API sends — seconds left and the fraction
+ * of the window remaining — recomputed for this instant. The window length is
+ * recovered from the pair the collector sent (remaining ÷ fraction) rather
+ * than assumed, so a job with a non-default lease_ttl_s drains at its own rate.
+ */
+export function leaseNow(
+ job: { lease_deadline: string | null; lease_remaining_s: number | null; lease_fraction: number | null },
+ now: number,
+): { remaining_s: number | null; fraction: number | null } {
+ if (!job.lease_deadline) return { remaining_s: job.lease_remaining_s, fraction: job.lease_fraction };
+ const remaining = Math.max(0, (new Date(job.lease_deadline).getTime() - now) / 1000);
+ const ttl =
+ job.lease_remaining_s != null && job.lease_fraction ? job.lease_remaining_s / job.lease_fraction : null;
+ return {
+ remaining_s: remaining,
+ fraction: ttl && ttl > 0 ? Math.max(0, Math.min(1, remaining / ttl)) : job.lease_fraction,
+ };
+}
+
// --- formatters ---
/** Compact duration: the dashboard is read at a glance, so "3h 12m", not "3:12:44". */
diff --git a/collector/docs/operations.md b/collector/docs/operations.md
index c502e9a..5335209 100644
--- a/collector/docs/operations.md
+++ b/collector/docs/operations.md
@@ -157,10 +157,10 @@ exists — the plist invokes it directly to avoid depending on a login `PATH`.
| `POST /devices/register` | Device checks in with descriptor + pool tags (upsert) |
| `GET /devices/:id/next-job` | Long-poll (~25 s) for `executor: "device"` work; 204 when none |
| `GET /executor/next-job` | Long-poll for `executor: "host"` work (`?name=` labels the claimant) |
-| `POST /jobs` | Enqueue a job spec (curl, CI, or the future scheduler); 409 on duplicate `job_id` |
+| `POST /jobs` | Enqueue a job spec (curl, CI, or the future scheduler); 409 on duplicate `job_id`, and the reply's `status` says whether it was queued or is `waiting` on `depends_on` |
| `GET /jobs/:id` | Job status, including `attempts`, `lease_deadline`, and `last_error` |
| `POST /jobs/sweep` | Force a lease sweep now; returns the `job_id`s requeued and failed |
-| `POST /results` | Result rows (`kind: "result"`, idempotent by job/device/iter) and telemetry (`kind: "beacon"`, which renews the job's lease); `final: true` closes the job |
+| `POST /results` | Result rows (`kind: "result"`, idempotent by job/device/iter) and telemetry (`kind: "beacon"`, which renews the job's lease and answers `preempt`); `final: true` closes the job and promotes whatever waited on it — unless it carries `preempted: true`, which requeues it instead |
| `POST /artifacts` | Upload raw bytes (models or app builds); returns `sha256` |
| `GET /artifacts/:sha256` | Download, supports Range requests |
| `POST /schedules` / `GET /schedules` | Upsert / list cron schedules (5-field cron + job template, `enabled` off by default) |
@@ -606,6 +606,24 @@ the executor's own link to the collector and to adb.
**This path is reasoned through but has not been run.** Adding the anchor and
exercising it once by hand is worth doing before a schedule depends on it.
+## Device state is journalled, not assumed
+
+`network-shape.ts` and `device-state.ts` both change something about a phone and
+must put it back. Both write their intent to a journal **before** touching the
+device — `~/.fleet/network-shape.json` and `~/.fleet/device-state.json` — so a
+crash between the two is recoverable, and both restore every attached device at
+executor startup before the first claim.
+
+That makes the journals operator-visible signals. **A device still listed in
+either file after a run is a device that needs manual attention**: a phone left
+offline, or left in Arabic at the largest dynamic type, looks broken rather than
+configured, and nothing else will tell you which.
+
+Restoration is asymmetric on purpose. Wifi is re-enabled unconditionally, since
+a fleet device off the network is broken by definition. Cellular data is only
+re-enabled when the journal says the executor disabled it, because turning data
+back on for a device deliberately kept off a metered SIM costs real money.
+
## Constraints
`constraints` on a job spec is enforced in two places, and which place matters.
@@ -646,6 +664,76 @@ Host jobs fan out too. A host child is pinned exactly as a device child is, and
the executor's own target selection already honours `targets.device_id`, so an
`install` plus `ui-test` pair can cover the OS matrix with two job specs.
+## Job chains (`depends_on`)
+
+Build, then install what it built, then run the UI suite against it. Until now
+that shape needed something outside the collector sitting on the queue, polling
+for the build to close and posting the next job by hand — which is a piece of
+software with no home, no restart story, and nothing watching *it*.
+
+`depends_on` puts the relationship on the job that waits:
+
+```json
+{ "schema": 1, "job_id": "install-903", "workload": "install", "executor": "host",
+ "depends_on": ["build-903"],
+ "app": { "name": "greenfolio-android", "build": "903",
+ "sha256": "${jobs.build-903.artifact}" } }
+```
+
+A job with an unfinished dependency is inserted as **`waiting`**, which is a
+status of its own rather than a flag on a queued row. The claim loop asks for
+`status = 'queued'`, so a waiting job is invisible to it by construction — there
+is no second predicate that someone could forget to update.
+
+- When the last dependency reaches `done`, the waiter is promoted to `queued`,
+ in the same transaction that closed the dependency. A crash between the two
+ would leave a chain stalled on a job that has already finished, and nothing on
+ the collector would ever come back and fix that.
+- When a dependency **fails or is cancelled**, the waiter is failed with
+ `last_error` naming it, and that failure cascades the whole way down. A broken
+ build must not leave an install and a ui-test sitting in `waiting` until
+ somebody notices next week. Cancelling counts for the same reason failing
+ does: nothing behind a job that will never run can ever be satisfied.
+- A dependency that does not exist, and a cycle, are refused at enqueue with a
+ 400. A chain that names a job nobody ever posted would otherwise sit in
+ `waiting` forever, looking exactly like a chain whose build is merely slow.
+- A dependency that has *already* failed makes the new job fail on arrival
+ rather than park in `waiting`: no promotion event is coming for it.
+
+### Template references
+
+At promotion time the waiter's stored spec is rewritten, so what the runner
+claims is a spec with real values in it rather than one carrying instructions it
+would have to interpret:
+
+| Reference | Resolves to |
+|---|---|
+| `${jobs..artifact}` | the first sha256 in that job's final result row's `artifacts` (or `test.artifacts`) |
+| `${jobs..metrics.}` | that metric from the same row |
+
+A reference that is the *whole* string keeps the value's type — a metric
+substituted into `params.budget_s` stays a number, and a budget silently turned
+into the string `"91"` would compare wrong against every number it meets
+afterwards. A reference embedded in a longer string is interpolated as text.
+
+Two refusals worth knowing:
+
+- A reference may only name a job in this job's own `depends_on` (400 at
+ enqueue). Otherwise it would be filled from whatever that job happened to have
+ produced by the time this one was promoted, which is a race dressed up as a
+ feature.
+- A reference that cannot be resolved at promotion — the dependency finished
+ `done` but uploaded no artifact, or reported no such metric — **fails the
+ waiter** and says so. Promoting it with a blank hash would produce a job that
+ dies at download time, hours later and nowhere near the cause.
+
+Dependencies that are all closed by the time the job is posted queue it
+immediately, with the substitution done at enqueue instead — same result, no
+promotion event to hang it on.
+
+Fan-out children inherit their parent's dependencies, so a whole shelf can hang
+off one build.
+
## Artifact pins
The GC reference scan reads job specs, results, schedules and templates. Two
@@ -664,6 +752,43 @@ The artifacts page shows this in a "Kept" column, because the only honest
reading of "references: 0" without it is "free to delete", which for a baseline
is exactly wrong.
+## Energy
+
+`power.json` maps a pool to a smart plug. A pool that also declares `read_url`,
+`watts_path` and `energy_method` is sampled into `power_samples`, and a job's
+energy is integrated over its claim window.
+
+Three things are deliberate:
+
+- **`energy_method` is declared, never inferred.** A pool that omits it gets no
+ energy figure at all. `plug` means the plug feeds exactly one device;
+ `plug-shared` means several sit behind it, and the figure is **the pool's**.
+ It must not be divided by the device count — the devices are not identical,
+ they are not all busy, and a per-device number arrived at by division would be
+ indistinguishable in storage from a measured one.
+- **Gaps are counted, never bridged.** The integration refuses to extrapolate
+ past its first and last in-window sample and skips gaps over five minutes,
+ reporting `covered_ms` against `window_ms`. An unreachable plug must not
+ become fabricated draw, so fewer than two samples yields null rather than zero.
+- **The number includes charging, and says so.** A phone on a plug is also
+ charging. Subtracting a measured per-pool idle baseline removes the charger
+ brick's standing draw, but nothing can remove battery charging current from a
+ wall measurement. The reported quantity is therefore stated everywhere it
+ appears: watt-hours at the wall, above the pool's idle baseline, over the
+ job's claim window, including any charging during it.
+
+## Evals
+
+`/api/evals` and the Evals page group `batch`, `speech-eval` and `embed-eval`
+results by eval set and model, pivoted by device — so the next eval is a URL
+rather than a hand-written report.
+
+The page **counts and lists the rows it excluded**, with the metric keys each
+one does carry. That is the point rather than a detail: the plant-ID write-up's
+accuracy rode in `decode_tok_s` because vision had no field of its own, and
+those rows are unqueryable today. Silently dropping them would repeat exactly
+the mistake the page exists to prevent.
+
## Alerts
Evaluated every 60 s (`FLEET_ALERT_TICK_MS`). Alerts are **state, not events**:
@@ -730,6 +855,49 @@ Defaults are 600 s and 3 attempts, per job via `lease.ttl_s` / `lease.max_attemp
Pick a TTL longer than the worst-case gap between beacons for that workload — too
short and the collector requeues a job that is still running fine.
+## Preemption
+
+A six-hour `app-soak` holds the only iPhone on the shelf. A one-minute smoke
+test lands behind it. Without preemption the choice is to wait six hours or to
+cancel the soak and throw away everything it has measured — and cancelling is
+what actually happens, which is why the soak never finishes.
+
+`preemptible: true` on a job spec offers a third answer: the job agrees to be
+interrupted.
+
+- **Asking.** The runner's beacon reply gains `preempt: true` when the running
+ job's spec is preemptible *and* a `queued` job with a **strictly higher**
+ priority would be handed to the same claimant. Strictly higher, because equal
+ priority is not a reason to throw away work in progress, and two jobs trading
+ a device back and forth is worse than either of them finishing late. The
+ eligibility question is answered by the same code the claim loop uses, so a
+ runner is only ever asked to stand aside for a job that would really replace
+ it.
+- **Stopping.** The collector never kills anything. `preempt: true` is a request
+ the runner honours on its own schedule: it stops at a point it can resume
+ from, uploads its progress as an artifact, and posts its final row with
+ `preempted: true`.
+- **Standing down.** That row does not close the job. It goes back to `queued`,
+ the attempt it burned is **handed back** (`attempts - 1`), the device lock is
+ released, and the uploaded sha256 is written into the spec as
+ `params.resume_from` so the next claim continues rather than starting the hour
+ again. It is not marked failed, and it does not appear in the failed list —
+ being interrupted by the operator's own priorities is not evidence that a job
+ is flaky, and counting it would exhaust `max_attempts` on a job that never
+ once misbehaved.
+
+The dashboard's `priority +1` on any queued job is the operator-facing half of
+this: raise a queued job past a running preemptible one and the next beacon
+carries the request. The job detail page says whether the running job is
+preemptible, because a priority button that does nothing visible is worse than
+no button.
+
+A runner that ignores `preempt` simply keeps running — the field is additive and
+the old behaviour is the default. `preemptible` is not yet in
+`schemas/job.schema.json`; the schema has no top-level `additionalProperties:
+false`, so the field validates today, but it should be documented there
+alongside `priority`.
+
## Phase 0 scope notes
- Artifact uploads are buffered in memory — fine for smoke tests and APKs,
diff --git a/collector/power.example.json b/collector/power.example.json
index da9e694..153d99a 100644
--- a/collector/power.example.json
+++ b/collector/power.example.json
@@ -1,9 +1,33 @@
{
"//": "Copy to power.json (gitignored). Each pool maps to smart-plug webhooks — Tasmota, Shelly, or Home Assistant all speak this shape. The collector POSTs the URL on /power//; Phase 4's drain workloads will call these to unplug the battery pool before a run and restore the 40-70% charge band after.",
+ "//read": "A pool that also sets read_url + watts_path is METERED: the collector polls it into power_samples and can integrate watt-hours across a job (src/power.ts). watts_path is a path expression, not a vendor name, because Tasmota, Shelly and Kasa each put the number somewhere different. Segments split on '.' and '[…]'; use [\"key\"] when a key contains a dot, and watts_scale for a plug that reports milliwatts.",
+ "//energy_method": "REQUIRED for any energy figure — the collector will not guess it. 'plug' means this plug feeds EXACTLY ONE device. 'plug-shared' means several devices sit behind it, and the watt-hours are the POOL's: they must NOT be divided by the number of devices to get a per-device number. A pool that omits this gets no energy figure at all, which is the intended outcome of not knowing how the shelf is wired.",
+ "//idle_watts": "What this pool draws plugged in and idle — charger bricks, the powered hub, the switch itself. Measure it with the shelf quiet and nothing claimed, then put it here; it is subtracted from every integration. Omit it and the collector estimates it from the pool's own samples and labels the figure as estimated. NOTE: this removes the STANDING load only. A phone on a plug is also charging, and no idle baseline can separate battery charging current from the work — so every result also carries includes_charging, and the dashboard says in words that a charging window's watt-hours include replenishing the battery.",
"pools": {
"battery": {
"on": "http://tasmota-plug.local/cm?cmnd=Power%20On",
- "off": "http://tasmota-plug.local/cm?cmnd=Power%20Off"
+ "off": "http://tasmota-plug.local/cm?cmnd=Power%20Off",
+ "read_url": "http://tasmota-plug.local/cm?cmnd=Status%2010",
+ "watts_path": "StatusSNS.ENERGY.Power",
+ "read_interval_s": 30,
+ "energy_method": "plug",
+ "idle_watts": 1.4
+ },
+ "ml-capable": {
+ "//": "Four phones on one powered hub behind one Shelly. Shared on purpose: the figure below is the whole shelf's.",
+ "on": "http://shelly-shelf.local/relay/0?turn=on",
+ "off": "http://shelly-shelf.local/relay/0?turn=off",
+ "read_url": "http://shelly-shelf.local/status",
+ "watts_path": "meters[0].power",
+ "energy_method": "plug-shared",
+ "idle_watts": 6.2
+ },
+ "kasa-example": {
+ "//": "Shelly Gen2 would be watts_path \"[\\\"switch:0\\\"].apower\" against /rpc/Switch.GetStatus?id=0. Kasa reports milliwatts, hence the scale.",
+ "read_url": "http://kasa-bridge.local/emeter",
+ "watts_path": "emeter.get_realtime.power_mw",
+ "watts_scale": 0.001,
+ "energy_method": "plug"
}
}
}
diff --git a/collector/schemas/job.schema.json b/collector/schemas/job.schema.json
index 795fc50..3f74fba 100644
--- a/collector/schemas/job.schema.json
+++ b/collector/schemas/job.schema.json
@@ -52,7 +52,19 @@
"flows": { "type": "string" }
}
},
- "params": { "type": "object" },
+ "params": {
+ "type": "object",
+ "description": "Workload-specific. Deliberately open, but the fields the shipped workloads read are listed here so they are discoverable: benchmark/thermal prompt_tokens, gen_tokens, warmup_iters, measure_iters, duration_s; batch/vision/speech/embed input_sha256, max_items, max_tokens; cold-start app_id, launches, states; locale-shots locales; app-soak flow, duration_s, interval_s; a11y-audit steps, variants, min_target_pt, font_scale, tree_source; build repo, ref, kind, task, scheme; vantage urls, repeats; any host job network and network_to; a preempted job resume_from."
+ },
+ "depends_on": {
+ "type": "array",
+ "items": { "type": "string" },
+ "description": "Job ids this one waits for. The job is enqueued as 'waiting' and becomes 'queued' when the last of them finishes; if any of them fails or is cancelled, this one fails with last_error naming it. While waiting, a spec may reference a dependency's output: \"${jobs..artifact}\" resolves to the first sha256 in that job's final row, and \"${jobs..metrics.}\" to one of its metrics. A whole-string reference keeps the value's type, so a metric stays a number. Referencing a job not listed here is refused at enqueue."
+ },
+ "preemptible": {
+ "type": "boolean",
+ "description": "Whether a higher-priority job may ask this one to step aside. When true, a beacon is answered with preempt:true once a strictly higher-priority queued job would go to the same claimant; the runner is expected to checkpoint, upload its progress, and post a final row with preempted:true. That requeues the job with params.resume_from set to the uploaded sha256 and hands the attempt back, so stepping aside never counts against max_attempts. Equal priority does not preempt."
+ },
"lease": {
"type": "object",
"description": "How long a claim stays valid without proof of life. The runner renews it by posting POST /results kind=beacon with this job_id; a lapsed lease is requeued by the collector's sweep, and failed once max_attempts is spent. Omit for the defaults: 600 s, or 14400 s for drain/soak, which run for hours between beacons.",
diff --git a/collector/schemas/metrics.json b/collector/schemas/metrics.json
index 932582f..7bce0a6 100644
--- a/collector/schemas/metrics.json
+++ b/collector/schemas/metrics.json
@@ -1,38 +1,60 @@
{
"schema": 1,
"metrics": [
+ "artifact_bytes",
"avg_position",
"avg_rating",
"battery_end_pct",
"battery_start_pct",
+ "build_s",
"checks_failed",
"clicks",
+ "clips",
"clock_offset_ms",
"clusters",
+ "connect_ms",
+ "crashes",
"ctr_pct",
"decode_tok_s",
"diff_pct",
+ "dim",
"disk_free_gb",
+ "dns_ms",
+ "docs_per_s",
"drain_pct_per_h",
"elapsed_s",
+ "energy_method",
+ "energy_wh",
"images_per_s",
"impressions",
"issues_error",
"issues_warn",
+ "jank_pct",
"launch_ms",
"launch_state",
"load_ms",
+ "locales",
"mem_method",
+ "network_type",
"p50_ms",
"p95_ms",
"pages_crawled",
"peak_mem_mb",
"prefill_tok_s",
+ "pss_mb",
+ "recall_at_1",
+ "recall_at_10",
+ "recall_at_5",
"reviews_count",
+ "rtf",
+ "shots",
"thermal",
"thermal_state",
+ "tls_ms",
"top1_pct",
"top5_pct",
- "ttft_ms"
+ "ttfb_ms",
+ "ttft_ms",
+ "wer_pct"
]
}
diff --git a/collector/schemas/result.schema.json b/collector/schemas/result.schema.json
index 175fba3..b038e34 100644
--- a/collector/schemas/result.schema.json
+++ b/collector/schemas/result.schema.json
@@ -55,6 +55,38 @@
"disk_free_gb": { "type": "number", "description": "self-check: free space on the executor host's data volume" },
"clock_offset_ms": { "type": "number", "description": "self-check: offset against NTP. A skewed executor clock makes every duration in the results table wrong in a way nothing else would reveal" },
"checks_failed": { "type": "number", "description": "self-check: how many host checks failed, so the alert rule needs no parsing" },
+
+ "build_s": { "type": "number", "description": "build: wall-clock seconds for the build itself, excluding checkout and upload" },
+ "artifact_bytes": { "type": "number", "description": "build: size of the produced app or package" },
+
+ "wer_pct": { "type": "number", "description": "speech-eval: word error rate after case and punctuation normalisation. Lower is better, and it can exceed 100 when a model inserts more words than the reference contains" },
+ "rtf": { "type": "number", "description": "speech-eval: real-time factor, audio seconds per wall second. Above 1 means the device transcribes faster than the clip plays, which is the shippability line" },
+ "clips": { "type": "number", "description": "speech-eval: how many clips the score covers, so a partial run is never mistaken for a full one" },
+
+ "recall_at_1": { "type": "number", "description": "embed-eval: fraction of queries whose top hit is relevant" },
+ "recall_at_5": { "type": "number" },
+ "recall_at_10": { "type": "number" },
+ "docs_per_s": { "type": "number", "description": "embed-eval: embedding throughput" },
+ "dim": { "type": "number", "description": "embed-eval: embedding width, because recall is only comparable within one" },
+
+ "energy_wh": { "type": "number", "description": "watt-hours drawn across the job, integrated from the pool's smart plug or the machine's own power reporting" },
+ "energy_method": {
+ "enum": ["plug", "plug-shared", "os"],
+ "description": "How energy_wh was obtained. plug-shared means several devices sit behind one plug, so the figure is the pool's and must not be divided per device — the distinction exists because dividing it silently is the obvious wrong thing to do"
+ },
+
+ "dns_ms": { "type": "number", "description": "vantage: name resolution" },
+ "connect_ms": { "type": "number", "description": "vantage: TCP connect" },
+ "tls_ms": { "type": "number", "description": "vantage: TLS handshake" },
+ "ttfb_ms": { "type": "number", "description": "vantage: time to first byte" },
+ "network_type": { "type": "string", "description": "vantage: wifi, cellular, ethernet or unknown, as the agent saw it. A latency figure with no idea what carried it is not comparable to anything" },
+
+ "pss_mb": { "type": "number", "description": "app-soak: proportional set size of the app under test at this sample" },
+ "jank_pct": { "type": "number", "description": "app-soak: percentage of janky frames since the last sample" },
+ "crashes": { "type": "number", "description": "app-soak: crashes observed in this interval" },
+
+ "locales": { "type": "number", "description": "locale-shots: how many locales the run covered" },
+ "shots": { "type": "number", "description": "locale-shots and a11y-audit: screenshots captured" },
"drain_pct_per_h": { "type": "number", "description": "drain: battery percent consumed per hour" },
"diff_pct": { "type": "number", "description": "web-shots: percent of pixels differing from the accepted baseline (100 when the dimensions changed)" },
diff --git a/collector/scripts/smoke.ts b/collector/scripts/smoke.ts
index af2efa1..24302a1 100644
--- a/collector/scripts/smoke.ts
+++ b/collector/scripts/smoke.ts
@@ -11,6 +11,9 @@ import {
import { evalMatch } from "../src/match.js";
import { parseAmStart, amStartProblem } from "../src/am-start.js";
import { parseNetworkProfile } from "../src/network-shape.js";
+import { runPowerChecks } from "../src/power.test.js";
+import { runEvalChecks } from "../src/api/evals.test.js";
+import { runDeviceParserChecks } from "../src/device-parsers.test.js";
import { redact, keychainPassword } from "../src/secrets.js";
const BASE = process.env.FLEET_URL ?? "http://127.0.0.1:8788";
@@ -2125,5 +2128,268 @@ for (const [workload, device] of [["web-audit", "web:audit"], ["web-unfurl", "we
check("network: an unknown profile throws rather than running unshaped", threw);
}
+// --- job chains: depends_on ---
+
+// A build → install → ui-test sequence used to need a script sitting on the
+// queue watching for the build to close. The dependency lives on the waiting
+// job instead — which is also what makes a broken build stop the chain rather
+// than let it test yesterday's APK.
+//
+// The workload is scoped to this run and declared by one device, so the claims
+// below can only ever be offered the jobs this section enqueued: a leftover
+// queued job from an earlier section cannot wander into the assertions.
+{
+ const CHAIN_WL = `smoke-chain-wl-${run}`;
+ const CHAIN_DEV = `smoke-chain-dev-${run}`;
+ const BUILD = `smoke-chain-build-${run}`;
+ const INSTALL = `smoke-chain-install-${run}`;
+ const chainJob = (job_id: string, extra: Record = {}) => ({
+ schema: 1, job_id, workload: CHAIN_WL, executor: "device",
+ targets: { device_id: CHAIN_DEV }, ...extra,
+ });
+
+ await json("POST", "/devices/register", {
+ device_id: CHAIN_DEV, descriptor: { model: "Chainlink", os: "android-14", ram_mb: 8000 },
+ pools: [], capabilities: [CHAIN_WL],
+ });
+
+ const head = await json("POST", "/jobs", chainJob(BUILD));
+ check("chain: the head of a chain queues normally", head.body?.status === "queued", JSON.stringify(head.body));
+
+ const waiter = await json("POST", "/jobs", chainJob(INSTALL, {
+ depends_on: [BUILD],
+ app: { name: "chain-app", build: "smoke", sha256: `\${jobs.${BUILD}.artifact}` },
+ params: { budget_s: `\${jobs.${BUILD}.metrics.build_s}` },
+ }));
+ check("chain: a job whose dependency is unfinished is inserted waiting",
+ waiter.status === 201 && waiter.body?.status === "waiting", JSON.stringify(waiter.body));
+
+ // The whole reason 'waiting' is a status rather than a flag: the claim loop
+ // asks for 'queued' and so cannot see it.
+ const offered = await json("GET", `/devices/${CHAIN_DEV}/next-job`);
+ check("chain: the queue offers the dependency, never the waiter",
+ offered.body?.job_id === BUILD, JSON.stringify(offered.body?.job_id));
+ const parked = await json("GET", `/api/jobs/${INSTALL}`);
+ check("chain: the waiter is still waiting", parked.body?.status === "waiting", parked.body?.status);
+ check("chain: and says what it waits for",
+ JSON.stringify(parked.body?.depends_on) === JSON.stringify([BUILD]), JSON.stringify(parked.body?.depends_on));
+
+ // Finish the build with an artifact and a metric, which is what the waiter's
+ // spec is written against.
+ const apk = Buffer.from(`chain-apk-${run}`.repeat(50));
+ const apkSha = createHash("sha256").update(apk).digest("hex");
+ await fetch(`${BASE}/artifacts`, {
+ method: "POST",
+ headers: { "content-type": "application/octet-stream", "x-artifact-name": "chain.apk" },
+ body: apk,
+ });
+ const closed = await json("POST", "/results", {
+ schema: 1, kind: "result", job_id: BUILD, device_id: CHAIN_DEV, iter: 0, final: true, ok: true,
+ metrics: { build_s: 91 }, test: { passed: 1, failed: 0, artifacts: [apkSha] },
+ });
+ check("chain: closing the dependency promotes the waiter",
+ ((closed.body?.promoted ?? []) as string[]).includes(INSTALL), JSON.stringify(closed.body));
+
+ const promoted = await json("GET", `/api/jobs/${INSTALL}`);
+ check("chain: the promoted job is queued", promoted.body?.status === "queued", promoted.body?.status);
+ check("chain: ${jobs..artifact} resolves into the stored spec",
+ promoted.body?.spec?.app?.sha256 === apkSha, JSON.stringify(promoted.body?.spec?.app));
+ // A metric that is the whole string keeps its type: a budget substituted as
+ // the string "91" would compare against numbers wrong for ever after.
+ check("chain: ${jobs..metrics.} resolves and stays a number",
+ promoted.body?.spec?.params?.budget_s === 91, JSON.stringify(promoted.body?.spec?.params));
+ check("chain: a promoted job is claimable",
+ (await json("GET", `/devices/${CHAIN_DEV}/next-job`)).body?.job_id === INSTALL);
+ await json("POST", `/api/jobs/${INSTALL}/cancel`);
+
+ // A dependency that fails takes its waiter with it. Leaving the waiter in
+ // 'waiting' would be worse than failing it: nothing will ever close that
+ // dependency again, so it would sit there until someone noticed.
+ const FAIL_DEP = `smoke-chain-faildep-${run}`, FAIL_WAIT = `smoke-chain-failwait-${run}`;
+ await json("POST", "/jobs", chainJob(FAIL_DEP));
+ await json("POST", "/jobs", chainJob(FAIL_WAIT, { depends_on: [FAIL_DEP] }));
+ const broke = await json("POST", "/results", {
+ schema: 1, kind: "result", job_id: FAIL_DEP, device_id: CHAIN_DEV, iter: 0, final: true, ok: false,
+ error: "compile error",
+ });
+ check("chain: a failed dependency fails its waiter",
+ ((broke.body?.failed ?? []) as string[]).includes(FAIL_WAIT), JSON.stringify(broke.body));
+ const failedWaiter = await json("GET", `/api/jobs/${FAIL_WAIT}`);
+ check("chain: and the waiter's error names the dependency",
+ failedWaiter.body?.status === "failed" && String(failedWaiter.body?.last_error).includes(FAIL_DEP),
+ JSON.stringify(failedWaiter.body?.last_error));
+
+ // Cancelling is not failing, but it is just as final: nothing behind a
+ // cancelled job will ever be satisfied, and the cascade runs the whole way
+ // down rather than stopping at the first link.
+ const CA = `smoke-chain-ca-${run}`, CB = `smoke-chain-cb-${run}`, CC = `smoke-chain-cc-${run}`;
+ await json("POST", "/jobs", chainJob(CA));
+ await json("POST", "/jobs", chainJob(CB, { depends_on: [CA] }));
+ await json("POST", "/jobs", chainJob(CC, { depends_on: [CB] }));
+ const cancelled = await json("POST", `/api/jobs/${CA}/cancel`, { reason: "smoke" });
+ check("chain: cancelling a job cascades to everything queued behind it",
+ ((cancelled.body?.cascaded ?? []) as string[]).includes(CB) &&
+ ((cancelled.body?.cascaded ?? []) as string[]).includes(CC),
+ JSON.stringify(cancelled.body));
+ const cb = await json("GET", `/api/jobs/${CB}`);
+ const cc = await json("GET", `/api/jobs/${CC}`);
+ check("chain: the direct waiter names the cancellation",
+ cb.body?.status === "failed" && String(cb.body?.last_error).includes("cancelled"), JSON.stringify(cb.body?.last_error));
+ check("chain: and the link beyond it goes too",
+ cc.body?.status === "failed" && String(cc.body?.last_error).includes(CB), JSON.stringify(cc.body?.last_error));
+
+ // Refusals, all at enqueue: a chain that is wrong should never become a row.
+ const cyc = await json("POST", "/jobs", chainJob(`smoke-chain-self-${run}`, {
+ depends_on: [`smoke-chain-self-${run}`],
+ }));
+ check("chain: a cycle is refused at enqueue",
+ cyc.status === 400 && String(cyc.body?.error).includes("cycle"), JSON.stringify(cyc));
+ const ghost = await json("POST", "/jobs", chainJob(`smoke-chain-ghost-${run}`, {
+ depends_on: [`smoke-chain-nobody-${run}`],
+ }));
+ check("chain: a dependency that does not exist is refused at enqueue",
+ ghost.status === 400 && String(ghost.body?.error).includes("does not exist"), JSON.stringify(ghost));
+ const stray = await json("POST", "/jobs", chainJob(`smoke-chain-stray-${run}`, {
+ depends_on: [BUILD],
+ app: { name: "chain-app", build: "smoke", sha256: `\${jobs.${FAIL_DEP}.artifact}` },
+ }));
+ check("chain: a reference to a job outside depends_on is refused",
+ stray.status === 400 && String(stray.body?.error).includes(FAIL_DEP), JSON.stringify(stray));
+
+ // Dependencies already satisfied get no promotion event, so the substitution
+ // promotion would have done has to happen at enqueue instead.
+ const LATE = `smoke-chain-late-${run}`;
+ const late = await json("POST", "/jobs", chainJob(LATE, {
+ depends_on: [BUILD],
+ app: { name: "chain-app", build: "smoke", sha256: `\${jobs.${BUILD}.artifact}` },
+ }));
+ check("chain: a dependency already done queues straight away", late.body?.status === "queued", JSON.stringify(late.body));
+ const lateRow = await json("GET", `/api/jobs/${LATE}`);
+ check("chain: and its references were filled in at enqueue",
+ lateRow.body?.spec?.app?.sha256 === apkSha, JSON.stringify(lateRow.body?.spec?.app));
+ await json("POST", `/api/jobs/${LATE}/cancel`);
+}
+
+// --- preemption: a long job stands aside for higher-priority work ---
+
+// The collector never kills anything. It answers a beacon with preempt:true and
+// the runner decides when to stop; the job goes back on the queue with its
+// progress and without the attempt, because being interrupted by the operator's
+// own priorities is not evidence that a job is flaky.
+{
+ const PRE_WL = `smoke-preempt-wl-${run}`;
+ const PRE_DEV = `smoke-preempt-dev-${run}`;
+ const LONG = `smoke-preempt-long-${run}`;
+ const PEER = `smoke-preempt-peer-${run}`;
+ const URGENT = `smoke-preempt-urgent-${run}`;
+ const beacon = () => json("POST", "/results", {
+ schema: 1, kind: "beacon", device_id: PRE_DEV, job_id: LONG,
+ beacon: { battery_pct: 90, charging: true, thermal: "nominal" },
+ });
+
+ await json("POST", "/devices/register", {
+ device_id: PRE_DEV, descriptor: { model: "Marathon", os: "android-14", ram_mb: 8000 },
+ pools: [], capabilities: [PRE_WL],
+ });
+ await json("POST", "/jobs", {
+ schema: 1, job_id: LONG, workload: PRE_WL, executor: "device",
+ preemptible: true, params: { hours: 6 }, targets: { device_id: PRE_DEV },
+ });
+ const claim = await json("GET", `/devices/${PRE_DEV}/next-job`);
+ check("preempt: the long job is claimed", claim.body?.job_id === LONG, JSON.stringify(claim.body?.job_id));
+ check("preempt: preemptible survives the round trip into the claimed spec", claim.body?.preemptible === true);
+
+ const quiet = await beacon();
+ check("preempt: with nothing queued above it the beacon says carry on",
+ quiet.body?.lease_renewed === true && quiet.body?.preempt === false, JSON.stringify(quiet.body));
+
+ // Equal priority is not a reason to throw away work in progress; two jobs
+ // trading a device back and forth is worse than either finishing late.
+ await json("POST", "/jobs", {
+ schema: 1, job_id: PEER, workload: PRE_WL, executor: "device", targets: { device_id: PRE_DEV },
+ });
+ const peered = await beacon();
+ check("preempt: an equal-priority job does not interrupt anything",
+ peered.body?.preempt === false, JSON.stringify(peered.body));
+
+ await json("POST", "/jobs", {
+ schema: 1, job_id: URGENT, workload: PRE_WL, executor: "device", priority: 9,
+ targets: { device_id: PRE_DEV },
+ });
+ const asked = await beacon();
+ check("preempt: a strictly higher-priority job asks the runner to stand aside",
+ asked.body?.preempt === true, JSON.stringify(asked.body));
+
+ // A job that never said it could be interrupted is never asked to be.
+ const STUBBORN = `smoke-preempt-stubborn-${run}`;
+ const STUB_DEV = `smoke-preempt-stubborn-dev-${run}`;
+ const STUB_WL = `smoke-preempt-stubborn-wl-${run}`;
+ await json("POST", "/devices/register", {
+ device_id: STUB_DEV, descriptor: { model: "Stubborn", os: "android-14" }, pools: [], capabilities: [STUB_WL],
+ });
+ await json("POST", "/jobs", {
+ schema: 1, job_id: STUBBORN, workload: STUB_WL, executor: "device", targets: { device_id: STUB_DEV },
+ });
+ await json("GET", `/devices/${STUB_DEV}/next-job`);
+ await json("POST", "/jobs", {
+ schema: 1, job_id: `${STUBBORN}-jumper`, workload: STUB_WL, executor: "device", priority: 9,
+ targets: { device_id: STUB_DEV },
+ });
+ const stubborn = await json("POST", "/results", {
+ schema: 1, kind: "beacon", device_id: STUB_DEV, job_id: STUBBORN,
+ beacon: { battery_pct: 90, charging: true, thermal: "nominal" },
+ });
+ check("preempt: a job that is not preemptible is never asked to stand aside",
+ stubborn.body?.lease_renewed === true && stubborn.body?.preempt === false, JSON.stringify(stubborn.body));
+
+ // The runner checkpoints and posts a final row saying so.
+ const ckpt = Buffer.from(`preempt-checkpoint-${run}`.repeat(50));
+ const ckptSha = createHash("sha256").update(ckpt).digest("hex");
+ await fetch(`${BASE}/artifacts`, {
+ method: "POST",
+ headers: { "content-type": "application/octet-stream", "x-artifact-name": "checkpoint.bin" },
+ body: ckpt,
+ });
+ const before = await json("GET", `/api/jobs/${LONG}`);
+ const stood = await json("POST", "/results", {
+ schema: 1, kind: "result", job_id: LONG, device_id: PRE_DEV, iter: 0, final: true, ok: true,
+ preempted: true, artifacts: [ckptSha], metrics: { hours_done: 2 },
+ });
+ check("preempt: a preempted final requeues instead of closing the job",
+ stood.body?.preempted === true && stood.body?.resume_from === ckptSha, JSON.stringify(stood.body));
+
+ const after = await json("GET", `/api/jobs/${LONG}`);
+ check("preempt: the job is queued again, not failed", after.body?.status === "queued", after.body?.status);
+ check("preempt: the attempt is handed back",
+ after.body?.attempts === (before.body?.attempts ?? 0) - 1,
+ `before=${before.body?.attempts} after=${after.body?.attempts}`);
+ check("preempt: the checkpoint is recorded for the resumed run",
+ after.body?.spec?.params?.resume_from === ckptSha, JSON.stringify(after.body?.spec?.params));
+ check("preempt: standing aside is not a failure in the failed list",
+ !((await json("GET", "/api/jobs?status=failed&per_page=200")).body?.jobs ?? []).some((j: any) => j.job_id === LONG));
+
+ // And the device is free for the job that displaced it.
+ const next = await json("GET", `/devices/${PRE_DEV}/next-job`);
+ check("preempt: the higher-priority job goes first now", next.body?.job_id === URGENT, JSON.stringify(next.body?.job_id));
+ for (const id of [URGENT, LONG, PEER, STUBBORN, `${STUBBORN}-jumper`]) await json("POST", `/api/jobs/${id}/cancel`);
+}
+
+// --- energy accounting and the evals pivot ---
+
+// Both are pure over their inputs, so they are exercised here for the same
+// reason the match language and the am-start parser are: the interesting
+// mistakes (integrating across a gap, dividing a shared plug's draw, dropping
+// a row instead of reporting it excluded) are arithmetic, not HTTP.
+runPowerChecks(check);
+runEvalChecks(check);
+
+// --- device workloads: parsing what the phone actually said ---
+//
+// Same reason as the am-start parser above: these decide whether a number is
+// stored at all, and the shapes that matter (a tool that printed a warning
+// instead of data, an empty crash buffer versus an unreachable device) only
+// ever appear on real hardware.
+runDeviceParserChecks(check);
+
console.log(failures === 0 ? "\nsmoke: ALL PASS" : `\nsmoke: ${failures} FAILURE(S)`);
process.exit(failures === 0 ? 0 : 1);
diff --git a/collector/src/a11y-tree.ts b/collector/src/a11y-tree.ts
new file mode 100644
index 0000000..e5b48b0
--- /dev/null
+++ b/collector/src/a11y-tree.ts
@@ -0,0 +1,447 @@
+/**
+ * Reading an accessibility tree out of whatever the platform will print, and
+ * judging it.
+ *
+ * A sibling of xcparse.ts and am-start.ts, and pure for the same reason: the
+ * executor is a long-running process with a main() loop and cannot be imported,
+ * so a parser only reachable from inside it is only ever tested by plugging in
+ * a phone. Everything an a11y-audit stores comes out of these functions, so
+ * they are the ones that have to be checkable without hardware.
+ *
+ * Three sources, because no two platforms will print the same thing:
+ *
+ * uiautomator dump -- Android's own XML. Bounds in PIXELS.
+ * maestro hierarchy -- Maestro's normalised JSON, Android and iOS
+ * simulators alike. Bounds in pixels or points
+ * depending on which driver produced them, so the
+ * caller says which.
+ * XCUIApplication.debugDescription -- the element subtree an XCUITest helper
+ * prints. Frames are already in POINTS.
+ *
+ * The judging is deliberately small and deliberately honest. Two checks:
+ *
+ * unlabelled tappables -- an error. A control a screen reader announces as
+ * "button" and nothing else is unusable, and it is
+ * the single most common real defect.
+ * small touch targets -- a warning, and ONLY where the tree gives bounds AND
+ * the caller can say what a bound means. An Android
+ * tree measures in pixels; comparing 40 pixels to 44
+ * points calls every control on a 3x phone a failure
+ * and every control on a 1x phone a pass. Without a
+ * density the check does not run, and the run says so
+ * rather than reporting a number it cannot support.
+ */
+
+import type { Finding } from "./web/shared.js";
+
+/** One element, whatever printed it. */
+export type A11yNode = {
+ /** Widget class or XCUIElement type, verbatim. */
+ cls: string;
+ /** Visible text, if the tree distinguishes it from the label. */
+ text: string;
+ /** What a screen reader would announce: content-desc, accessibilityText, label. */
+ label: string;
+ /** A programmatic id -- resource-id, accessibility identifier. NOT announced. */
+ id: string;
+ /** Current value, which some controls use instead of a label. */
+ value: string;
+ tappable: boolean;
+ enabled: boolean;
+ /** In the tree's own units; see A11yGeometry. */
+ bounds: { x: number; y: number; w: number; h: number } | null;
+ /** Depth in the subtree, for naming a finding usefully. */
+ depth: number;
+};
+
+/**
+ * What one unit of `bounds` means.
+ *
+ * "unknown" is a first-class answer, not a failure mode: it is what an Android
+ * tree is until somebody reads the device's density, and the size check is
+ * skipped rather than guessed.
+ */
+export type A11yGeometry =
+ | { unit: "points" }
+ | { unit: "pixels"; densityDpi: number }
+ | { unit: "unknown" };
+
+const attrs = (tag: string): Record => {
+ const out: Record = {};
+ for (const m of tag.matchAll(/([\w:-]+)="([^"]*)"/g)) out[m[1]] = m[2];
+ return out;
+};
+
+const unescapeXml = (s: string): string =>
+ s.replace(/"/g, '"').replace(/'/g, "'").replace(/</g, "<")
+ .replace(/>/g, ">").replace(/&/g, "&");
+
+/** `[0,0][1080,2400]` -> a rect, or null for anything else. */
+export function parseBoundsRect(s: string | undefined): A11yNode["bounds"] {
+ if (!s) return null;
+ const m = /\[(-?\d+),(-?\d+)\]\[(-?\d+),(-?\d+)\]/.exec(s);
+ if (!m) return null;
+ const [x1, y1, x2, y2] = m.slice(1).map(Number);
+ return { x: x1, y: y1, w: x2 - x1, h: y2 - y1 };
+}
+
+// ---------------------------------------------------------------------------
+// uiautomator dump
+// ---------------------------------------------------------------------------
+
+/**
+ * Android's own hierarchy XML.
+ *
+ * `uiautomator dump` writes the XML to the device and prints only a receipt --
+ * "UI hierchary dumped to: /sdcard/window_dump.xml", misspelled in AOSP since
+ * 2012 -- so the text handed to this function is whatever the caller cat'ed
+ * back. Feeding it the receipt by mistake is a realistic error, and it produces
+ * zero nodes rather than a parse failure, which would then read as a screen with
+ * nothing wrong on it. That is what `problem` is for.
+ */
+export function parseUiautomatorDump(xml: string): { nodes: A11yNode[]; problem: string | null } {
+ const text = xml.replace(/\r/g, "");
+ if (text.trim() === "") return { nodes: [], problem: "uiautomator printed nothing" };
+ // The receipt, or the failure it prints instead of one.
+ if (!/ in the dump (${text.trim().split("\n")[0].slice(0, 140)})`,
+ };
+ }
+
+ const nodes: A11yNode[] = [];
+ let depth = 0;
+ // Walk the tags in order so nesting depth is tracked without building a tree:
+ // depth is only ever used to name a finding, and a tree would be a second
+ // representation to keep correct.
+ for (const m of text.matchAll(/]*?)(\/?)>|<\/node>/g)) {
+ if (m[0] === "") { depth = Math.max(0, depth - 1); continue; }
+ const a = attrs(m[1]);
+ nodes.push({
+ cls: a["class"] ?? "",
+ text: unescapeXml(a["text"] ?? ""),
+ label: unescapeXml(a["content-desc"] ?? ""),
+ id: a["resource-id"] ?? "",
+ value: "",
+ // long-clickable alone is not a tap target; checkable controls are.
+ tappable: a["clickable"] === "true" || a["checkable"] === "true",
+ enabled: a["enabled"] !== "false",
+ bounds: parseBoundsRect(a["bounds"]),
+ depth,
+ });
+ if (m[2] !== "/") depth += 1;
+ }
+ if (nodes.length === 0) return { nodes, problem: "the dump has a but no nodes" };
+ return { nodes, problem: null };
+}
+
+// ---------------------------------------------------------------------------
+// maestro hierarchy
+// ---------------------------------------------------------------------------
+
+type MaestroNode = { attributes?: Record; children?: MaestroNode[] };
+
+/**
+ * Maestro's normalised JSON tree.
+ *
+ * Maestro prints a banner before the JSON on some versions, so the object is
+ * located rather than assumed to start at character zero -- the same defence
+ * runWebProject already applies to Playwright's JSON reporter.
+ *
+ * `accessibilityText` is Maestro's name for the announced label on both
+ * platforms; on iOS it carries the accessibility label, on Android the
+ * content-desc. `hintText` deliberately does NOT count as a label: a text field
+ * whose only description is its placeholder announces nothing once it has
+ * content in it.
+ */
+export function parseMaestroHierarchy(out: string): { nodes: A11yNode[]; problem: string | null } {
+ const text = out.replace(/\r/g, "");
+ const start = text.indexOf("{");
+ if (start < 0) {
+ return {
+ nodes: [],
+ problem: text.trim() === ""
+ ? "maestro hierarchy printed nothing"
+ : `maestro hierarchy printed no JSON (${text.trim().split("\n")[0].slice(0, 140)})`,
+ };
+ }
+ let root: MaestroNode;
+ try {
+ root = JSON.parse(text.slice(start)) as MaestroNode;
+ } catch (e) {
+ return { nodes: [], problem: `maestro hierarchy is not JSON: ${(e as Error).message.slice(0, 120)}` };
+ }
+
+ const nodes: A11yNode[] = [];
+ const str = (a: Record, k: string): string => {
+ const v = a[k];
+ return typeof v === "string" ? v : v === undefined || v === null ? "" : String(v);
+ };
+ const walk = (n: MaestroNode, depth: number) => {
+ const a = n.attributes ?? {};
+ // A node with no attributes at all is a container Maestro emits for
+ // structure; it is not an element and must not be judged as one.
+ if (Object.keys(a).length > 0) {
+ nodes.push({
+ cls: str(a, "class") || str(a, "elementType") || "",
+ text: str(a, "text"),
+ label: str(a, "accessibilityText") || str(a, "label") || str(a, "title"),
+ id: str(a, "resource-id") || str(a, "resourceId") || str(a, "accessibilityIdentifier"),
+ value: str(a, "value"),
+ tappable: str(a, "clickable") === "true" || str(a, "checkable") === "true" ||
+ isTappableType(str(a, "elementType") || str(a, "class")),
+ enabled: str(a, "enabled") !== "false",
+ bounds: parseBoundsRect(str(a, "bounds")) ?? rectFromNumbers(a),
+ depth,
+ });
+ }
+ for (const c of n.children ?? []) walk(c, depth + 1);
+ };
+ walk(root, 0);
+ if (nodes.length === 0) return { nodes, problem: "maestro hierarchy has no elements" };
+ return { nodes, problem: null };
+}
+
+/** Maestro's iOS driver sometimes gives x/y/width/height as numbers instead of a bounds string. */
+function rectFromNumbers(a: Record): A11yNode["bounds"] {
+ const n = (k: string) => (typeof a[k] === "number" ? (a[k] as number) : Number(a[k]));
+ const [x, y, w, h] = ["x", "y", "width", "height"].map(n);
+ if ([x, y, w, h].some((v) => !Number.isFinite(v))) return null;
+ return { x, y, w, h };
+}
+
+// ---------------------------------------------------------------------------
+// XCUIApplication.debugDescription
+// ---------------------------------------------------------------------------
+
+/** XCUIElement types a person is expected to tap. Everything else is chrome. */
+const TAPPABLE_IOS = new Set([
+ "button", "cell", "link", "menuitem", "tab", "tabbar", "switch", "toggle",
+ "slider", "stepper", "segmentedcontrol", "textfield", "securetextfield",
+ "searchfield", "textview", "checkbox", "radiobutton", "popupbutton",
+ "datepicker", "picker", "pickerwheel", "icon", "image",
+]);
+
+function isTappableType(t: string): boolean {
+ return TAPPABLE_IOS.has(t.replace(/^XCUIElementType/, "").toLowerCase());
+}
+
+/**
+ * The element subtree XCUITest prints for an application.
+ *
+ * The shape, one element per line, indentation carrying the nesting:
+ *
+ * →Application, 0x600001, {{0.0, 0.0}, {390.0, 844.0}}, label: 'Aliquant'
+ * Window (Main), 0x600002, {{0.0, 0.0}, {390.0, 844.0}}
+ * Button, 0x600004, {{16.0, 100.0}, {40.0, 40.0}}, label: 'Close'
+ * Button, 0x600006, {{16.0, 200.0}, {30.0, 30.0}}
+ *
+ * Frames are in POINTS, which is the whole reason this source is worth having:
+ * it is the only one of the three where a 44 in the tree and a 44 in the
+ * guideline are the same 44.
+ *
+ * The `, Disabled` suffix and the `(Main)` qualifier after a type are both
+ * ordinary, and both broke the obvious "split on comma, field 0 is the type"
+ * parse -- a window came out as type "Window (Main)" and a disabled button's
+ * label was read as the string "Disabled".
+ */
+export function parseXcuiDebugDescription(out: string): { nodes: A11yNode[]; problem: string | null } {
+ const text = out.replace(/\r/g, "");
+ if (text.trim() === "") return { nodes: [], problem: "the XCUITest helper printed nothing" };
+
+ const nodes: A11yNode[] = [];
+ let indents: number[] = [];
+ for (const raw of text.split("\n")) {
+ // The subtree's arrow marks the queried element; it is part of the
+ // indentation, not of the type name.
+ const line = raw.replace(/→/g, " ");
+ const m = /^(\s*)([A-Z][A-Za-z]*)(?:\s+\([^)]*\))?,\s*0x[0-9a-f]+,\s*(.*)$/.exec(line);
+ if (!m) continue;
+ const indent = m[1].length;
+ const cls = m[2];
+ const rest = m[3];
+
+ // {{x, y}, {w, h}} -- floats, and the frame may be absent on some elements.
+ const f = /\{\{\s*(-?[\d.]+),\s*(-?[\d.]+)\s*\},\s*\{\s*(-?[\d.]+),\s*(-?[\d.]+)\s*\}\}/.exec(rest);
+ const quoted = (key: string): string =>
+ new RegExp(`\\b${key}:\\s*'((?:[^'\\\\]|\\\\.)*)'`).exec(rest)?.[1]?.replace(/\\'/g, "'") ?? "";
+
+ // Depth from indentation, without assuming a fixed step: Xcode's is two
+ // spaces at the top and three deeper in, which a divide-by-two read as a
+ // tree that alternately deepened and flattened.
+ while (indents.length > 0 && indent <= indents[indents.length - 1]) indents.pop();
+ indents.push(indent);
+ const depth = indents.length - 1;
+
+ nodes.push({
+ cls,
+ text: quoted("value"),
+ label: quoted("label"),
+ id: quoted("identifier"),
+ value: quoted("value"),
+ tappable: isTappableType(cls),
+ enabled: !/,\s*Disabled\b/.test(rest),
+ bounds: f ? { x: Number(f[1]), y: Number(f[2]), w: Number(f[3]), h: Number(f[4]) } : null,
+ depth,
+ });
+ }
+ indents = [];
+ if (nodes.length === 0) {
+ return { nodes, problem: `no elements in the XCUITest dump (${text.trim().split("\n")[0].slice(0, 140)})` };
+ }
+ return { nodes, problem: null };
+}
+
+// ---------------------------------------------------------------------------
+// The checks
+// ---------------------------------------------------------------------------
+
+/** How a node reads in a finding, when it has no label to name it by. */
+function nameOf(n: A11yNode): string {
+ const cls = n.cls.replace(/^XCUIElementType/, "").split(".").pop() || "element";
+ const where = n.bounds ? ` at ${n.bounds.x},${n.bounds.y} (${n.bounds.w}x${n.bounds.h})` : "";
+ const id = n.id ? ` id=${n.id.split("/").pop()}` : "";
+ return `${cls}${id}${where}`;
+}
+
+export type A11yCheckOptions = {
+ /** Where these findings were observed, e.g. "home" or "checkout". */
+ step: string;
+ /** The minimum touch target, in points. 44 is Apple's; Android's own is 48dp. */
+ minTargetPt?: number;
+ /** Cap per check, so one pathological screen cannot fill a report. */
+ cap?: number;
+};
+
+/**
+ * Judge one tree.
+ *
+ * Severity follows the fleet's red-means-something rule, the same one
+ * web-audit states: `error` fails the run, `warn` is recorded and does not.
+ * An unlabelled control is an error because it is unusable and unambiguous; a
+ * small touch target is a warning because the measurement is a heuristic --
+ * a 30x30 icon inside a 48x48 padded row is a real pattern and reads as a
+ * violation here.
+ *
+ * The size check is skipped entirely, with a finding saying so, when the
+ * geometry is unknown. That is the point of the function taking geometry at
+ * all: an Android tree is in pixels, and 40 pixels on a 3x phone is 13 points.
+ * Reporting that as a violation, or converting it with an assumed density,
+ * would produce a number that means nothing on the device it came from.
+ */
+export function a11yFindings(
+ nodes: A11yNode[],
+ geometry: A11yGeometry,
+ opts: A11yCheckOptions,
+): Finding[] {
+ const minPt = opts.minTargetPt ?? 44;
+ const cap = opts.cap ?? 25;
+ const found: Finding[] = [];
+
+ const toPoints = (px: number): number | null => {
+ if (geometry.unit === "points") return px;
+ if (geometry.unit === "pixels") {
+ if (!Number.isFinite(geometry.densityDpi) || geometry.densityDpi <= 0) return null;
+ return (px * 160) / geometry.densityDpi;
+ }
+ return null;
+ };
+
+ let unlabelled = 0;
+ let tiny = 0;
+ let sized = 0;
+ const seen = new Set();
+
+ for (const n of nodes) {
+ if (!n.tappable || !n.enabled) continue;
+ // A zero-area node is not on screen; judging it produces findings nobody
+ // can act on, because there is nothing there to look at.
+ if (n.bounds && (n.bounds.w <= 0 || n.bounds.h <= 0)) continue;
+
+ const announced = [n.label, n.text, n.value].some((s) => s.trim() !== "");
+ if (!announced) {
+ unlabelled++;
+ const key = `unlabelled:${nameOf(n)}`;
+ if (!seen.has(key) && found.filter((f) => f.check === "a11y-label").length < cap) {
+ seen.add(key);
+ found.push({
+ severity: "error",
+ check: "a11y-label",
+ page: opts.step,
+ // Say what a screen reader gets, because "add a label" is only
+ // actionable once you know the id it is missing from.
+ detail: `${nameOf(n)} is tappable but announces nothing (no label, text or value)`,
+ });
+ }
+ }
+
+ if (n.bounds) {
+ const w = toPoints(n.bounds.w);
+ const h = toPoints(n.bounds.h);
+ if (w !== null && h !== null) {
+ sized++;
+ // Both dimensions have to be short. A 320x30 row is a perfectly
+ // reachable target; a 30x30 icon is the one worth flagging.
+ if (w < minPt && h < minPt) {
+ tiny++;
+ const key = `tiny:${nameOf(n)}`;
+ if (!seen.has(key) && found.filter((f) => f.check === "a11y-target-size").length < cap) {
+ seen.add(key);
+ found.push({
+ severity: "warn",
+ check: "a11y-target-size",
+ page: opts.step,
+ detail: `${n.label || n.text || nameOf(n)} is ${w.toFixed(0)}x${h.toFixed(0)}pt, under the ${minPt}pt minimum`,
+ });
+ }
+ }
+ }
+ }
+ }
+
+ if (sized === 0 && nodes.some((n) => n.tappable && n.bounds)) {
+ // Not a silent skip. A run that judged no sizes and said nothing is
+ // indistinguishable from one where every target was big enough.
+ found.push({
+ severity: "warn",
+ check: "a11y-coverage",
+ page: opts.step,
+ detail: geometry.unit === "unknown"
+ ? "touch-target sizes were not judged: the tree's bounds are in an unknown unit " +
+ "(an Android dump is in pixels, and converting needs the device's density)"
+ : "touch-target sizes were not judged: no element carried usable bounds",
+ });
+ }
+
+ if (unlabelled > cap) {
+ found.push({
+ severity: "warn",
+ check: "a11y-coverage",
+ page: opts.step,
+ detail: `${unlabelled} unlabelled tappables on this screen; the first ${cap} are listed`,
+ });
+ }
+ if (tiny > cap) {
+ found.push({
+ severity: "warn",
+ check: "a11y-coverage",
+ page: opts.step,
+ detail: `${tiny} undersized targets on this screen; the first ${cap} are listed`,
+ });
+ }
+ return found;
+}
+
+/** `Physical density: 420` / `420` out of `wm density` or `getprop`. */
+export function parseAndroidDensity(out: string): number | null {
+ const text = out.replace(/\r/g, "");
+ // `Override density:` wins when present -- it is what is actually in effect,
+ // and a device with a display-size override set (an accessibility setting in
+ // its own right) reports both.
+ const override = /Override density:\s*(\d+)/.exec(text);
+ const physical = /Physical density:\s*(\d+)/.exec(text);
+ const bare = /^\s*(\d{2,4})\s*$/m.exec(text);
+ const v = Number((override ?? physical ?? bare)?.[1] ?? NaN);
+ return Number.isFinite(v) && v > 0 ? v : null;
+}
diff --git a/collector/src/api/evals.test.ts b/collector/src/api/evals.test.ts
new file mode 100644
index 0000000..8958337
--- /dev/null
+++ b/collector/src/api/evals.test.ts
@@ -0,0 +1,167 @@
+/**
+ * Checks for the pure half of /api/evals: the family rule, the pivot, the
+ * exclusion accounting, and the per-joule column.
+ *
+ * `npx tsx src/api/evals.test.ts`
+ *
+ * The case that matters most is the last one: a plant-ID-shaped row — a vision
+ * batch carrying accuracy in `decode_tok_s` — must land in `excluded` with its
+ * actual metric keys attached, never in a cell and never silently gone.
+ *
+ * Note this file imports src/api/evals.ts, which reaches src/db.ts through
+ * src/power.ts, so running it opens a SQLite file in the current directory.
+ * Point FLEET_DATA_DIR at a scratch directory when running it standalone; the
+ * checks themselves touch no table.
+ */
+import { pathToFileURL } from "node:url";
+import { familyOf, perJoule, pivotEvals, type EvalRow } from "./evals.js";
+
+const SHA_A = "a".repeat(64);
+const SHA_B = "b".repeat(64);
+
+function row(over: Partial): EvalRow {
+ return {
+ job_id: "j1",
+ device_id: "pixel-8",
+ device_model: "Pixel 8",
+ simulator: false,
+ at: "2026-09-01T10:00:00.000Z",
+ input_sha256: SHA_A,
+ workload: "batch",
+ backend: "litert",
+ model: "plantnet-r18",
+ quant: "fp32",
+ accel: "gpu",
+ metrics: {},
+ params: {},
+ energy: null,
+ ...over,
+ };
+}
+
+export function runEvalChecks(check: (name: string, cond: boolean, detail?: string) => void) {
+ // --- families ---------------------------------------------------------
+ check("a litert batch is a vision eval", familyOf("batch", "litert") === "vision");
+ check("a coreml batch is a vision eval", familyOf("batch", "coreml") === "vision");
+ check("a llama.cpp batch is an LLM eval", familyOf("batch", "llama.cpp") === "llm");
+ check("speech-eval is speech whatever the backend", familyOf("speech-eval", "whisper.cpp") === "speech");
+ check("embed-eval is embed", familyOf("embed-eval", null) === "embed");
+
+ // --- the pivot --------------------------------------------------------
+ const rows: EvalRow[] = [
+ row({ job_id: "j-a1", device_id: "pixel-8", metrics: { top1_pct: 71.2, p50_ms: 30 } }),
+ row({ job_id: "j-a2", device_id: "iphone-12", device_model: "iPhone 12 Pro", metrics: { top1_pct: 74.9, p50_ms: 18 } }),
+ row({
+ job_id: "j-a3", device_id: "pixel-8", model: "houseplants-v2",
+ metrics: { top1_pct: 60.0, p50_ms: 12 },
+ }),
+ ];
+ const p = pivotEvals(rows);
+ check("one input_sha256 makes one set", p.sets.length === 1, String(p.sets.length));
+ const set = p.sets[0];
+ check("the set is a vision eval", set.family === "vision");
+ check("two models in the set", set.models.length === 2, String(set.models.length));
+ check("two devices as columns", set.devices.length === 2, JSON.stringify(set.devices.map((d) => d.device_id)));
+ const plantnet = set.models.find((m) => m.model === "plantnet-r18")!;
+ check("the pivot puts each device in its own cell", plantnet.cells["pixel-8"].metrics.top1_pct === 71.2);
+ check("and the other device in the other cell", plantnet.cells["iphone-12"].metrics.top1_pct === 74.9);
+ check("a model that ran on one device has one cell", Object.keys(set.models.find((m) => m.model === "houseplants-v2")!.cells).length === 1);
+ check("named metrics with no value are present and null", plantnet.cells["pixel-8"].metrics.top5_pct === null);
+
+ // Two eval sets never merge, even for the same model on the same device:
+ // different bytes are different questions.
+ const twoSets = pivotEvals([
+ row({ job_id: "s1", input_sha256: SHA_A, metrics: { top1_pct: 70 } }),
+ row({ job_id: "s2", input_sha256: SHA_B, metrics: { top1_pct: 40 } }),
+ ]);
+ check("different input_sha256 stay separate sets", twoSets.sets.length === 2);
+
+ // A re-run replaces that one cell and leaves the rest of the table alone.
+ const rerun = pivotEvals([
+ row({ job_id: "old", device_id: "pixel-8", at: "2026-09-01T10:00:00.000Z", metrics: { top1_pct: 71.2 } }),
+ row({ job_id: "new", device_id: "pixel-8", at: "2026-09-03T10:00:00.000Z", metrics: { top1_pct: 73.5 } }),
+ row({ job_id: "other", device_id: "iphone-12", at: "2026-09-01T10:00:00.000Z", metrics: { top1_pct: 74.9 } }),
+ ]);
+ const cells = rerun.sets[0].models[0].cells;
+ check("the newest run wins its cell", cells["pixel-8"].metrics.top1_pct === 73.5 && cells["pixel-8"].job_id === "new");
+ check("a re-run on one device leaves the others alone", cells["iphone-12"].metrics.top1_pct === 74.9);
+ check("both runs still count toward the set's run total", rerun.sets[0].runs === 3);
+
+ // --- exclusions -------------------------------------------------------
+ // The plant-ID shape: a vision batch whose accuracy is in decode_tok_s and
+ // whose latency is in ttft_ms. It has numbers, but none of them are named
+ // vision metrics, so nothing can be put in a cell.
+ const legacy = pivotEvals([
+ row({ job_id: "planteval-legacy", metrics: { decode_tok_s: 71.2, ttft_ms: 30, prefill_tok_s: 33 } }),
+ ]);
+ check("a pre-schema vision row is excluded, not pivoted", legacy.sets.length === 0 && legacy.excluded.length === 1);
+ check(
+ "the exclusion lists the keys the row does carry",
+ legacy.excluded[0].present.sort().join(",") === "decode_tok_s,prefill_tok_s,ttft_ms",
+ legacy.excluded[0].present.join(","),
+ );
+ check("the exclusion names the fields it wanted", legacy.excluded[0].reason.includes("top1_pct"));
+
+ // The same numbers under an LLM backend are NOT legacy — decode_tok_s is the
+ // named field there. The family decides, and it comes from the spec.
+ const llm = pivotEvals([
+ row({ job_id: "llm-1", backend: "llama.cpp", metrics: { decode_tok_s: 22.4, ttft_ms: 310 } }),
+ ]);
+ check("the same metrics under an LLM backend are named, not legacy", llm.sets.length === 1 && llm.excluded.length === 0);
+
+ // A row with no eval set is excluded for a different, stated reason.
+ const noSet = pivotEvals([row({ job_id: "loose", input_sha256: null, metrics: { top1_pct: 50 } })]);
+ check("a row with no input_sha256 is excluded", noSet.excluded.length === 1 && noSet.sets.length === 0);
+ check("and says so specifically", noSet.excluded[0].reason.includes("input_sha256"));
+
+ // Mixed: the set still renders, and carries its own excluded count so the
+ // page can say it beside that table rather than only in a global total.
+ const mixed = pivotEvals([
+ row({ job_id: "good", metrics: { top1_pct: 71.2 } }),
+ row({ job_id: "legacy", device_id: "iphone-12", metrics: { decode_tok_s: 68.0 } }),
+ ]);
+ check("a set renders alongside its excluded rows", mixed.sets.length === 1 && mixed.excluded.length === 1);
+ check("the set counts its own exclusions", mixed.sets[0].excluded === 1, String(mixed.sets[0].excluded));
+
+ // --- per joule --------------------------------------------------------
+ // 1 Wh = 3600 J. 120 images on 0.05 Wh = 120 / 180 J = 0.666… img/J.
+ const pj = perJoule("vision", {}, { images: 120 }, 0.05);
+ check("images per joule", !!pj && Math.abs(pj.value - 120 / 180) < 1e-9, JSON.stringify(pj));
+ check("and names its unit", pj?.unit === "img/J");
+ const tok = perJoule("llm", {}, { gen_tokens: 128 }, 0.01);
+ check("tokens per joule", !!tok && Math.abs(tok.value - 128 / 36) < 1e-9, JSON.stringify(tok));
+ const clips = perJoule("speech", { clips: 50 }, {}, 0.02);
+ check("clips per joule reads the metric, not the spec", !!clips && clips.unit === "clips/J");
+
+ // Omitted cleanly, never approximated: a rate times a duration would look
+ // like a measurement and would not be one.
+ check("no count means no column", perJoule("vision", { images_per_s: 40 }, {}, 0.05) === null);
+ check("no energy means no column", perJoule("vision", {}, { images: 120 }, null) === null);
+ check("zero energy means no column, not infinity", perJoule("vision", {}, { images: 120 }, 0) === null);
+
+ // Cells carry it only where both halves exist.
+ const withEnergy = pivotEvals([
+ row({
+ job_id: "e1", metrics: { top1_pct: 71.2 }, params: { images: 120 },
+ energy: { wh: 0.05, method: "plug", includes_charging: true, source: "integrated", baseline_source: "config", note: "" },
+ }),
+ row({ job_id: "e2", device_id: "iphone-12", metrics: { top1_pct: 74.9 }, params: { images: 120 } }),
+ ]);
+ const m = withEnergy.sets[0].models[0];
+ check("a cell with energy gets a per-joule figure", m.cells["pixel-8"].per_joule !== null);
+ check("a cell without energy omits it", m.cells["iphone-12"].per_joule === null);
+ check("the set knows it has energy somewhere", withEnergy.sets[0].has_energy === true);
+}
+
+if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
+ let failures = 0;
+ runEvalChecks((name, cond, detail = "") => {
+ if (cond) console.log(` ok ${name}`);
+ else {
+ failures++;
+ console.error(` FAIL ${name}${detail ? ` — ${detail}` : ""}`);
+ }
+ });
+ console.log(failures ? `\n${failures} FAILED` : "\nALL PASS");
+ process.exit(failures ? 1 : 0);
+}
diff --git a/collector/src/api/evals.ts b/collector/src/api/evals.ts
new file mode 100644
index 0000000..abc3901
--- /dev/null
+++ b/collector/src/api/evals.ts
@@ -0,0 +1,427 @@
+/**
+ * GET /api/evals and /api/evals/:input_sha — evals as a query, not a document.
+ *
+ * The plant-ID eval was assembled by hand: numbers copied out of result rows
+ * into a Markdown table, because the rows themselves put top-1 accuracy in
+ * `decode_tok_s` and p50 latency in `ttft_ms`, and top-5 and p95 had nowhere to
+ * live at all. No query reproduces that report, which is the whole reason it had
+ * to be typed. The named metric fields now exist (schemas/result.schema.json),
+ * so the next eval should need no hand-written report — this endpoint is the
+ * thing that makes that true.
+ *
+ * The shape is: an EVAL SET is one `params.input_sha256` — the exact bytes every
+ * device scored. Within a set, results group by MODEL and pivot across DEVICES,
+ * because "which model, on which hardware" is the question an eval is asked to
+ * answer and every other arrangement makes you transpose it yourself.
+ *
+ * ## Excluded rows are reported, never dropped
+ *
+ * A row that carries no named metric for its family cannot be pivoted: there is
+ * nothing to put in the cell. The tempting move is to filter it out, and that
+ * would repeat the exact mistake this page exists to correct — the plant-ID rows
+ * would vanish from a page whose reason for existing is that they could not be
+ * queried. So every excluded row is counted, attributed, and listed with the
+ * metric keys it does carry, and the page says so above the table.
+ */
+import type { FastifyInstance } from "fastify";
+import { db } from "../db.js";
+import { energyForJob, isEnergy, loadPowerConfig, meteredPools, type PowerConfig, type PowerMethod } from "../power.js";
+import { iso, isSimulator, parse } from "./shared.js";
+
+/** The workloads that score a shared input. Everything else is not an eval. */
+export const EVAL_WORKLOADS = ["batch", "speech-eval", "embed-eval"] as const;
+
+/** Same list src/api/results.ts uses to tell a vision batch from an LLM batch. */
+const VISION_BACKENDS = new Set(["litert", "coreml", "tflite", "vision"]);
+
+export type Family = "vision" | "llm" | "speech" | "embed";
+
+export function familyOf(workload: string, backend: string | null | undefined): Family {
+ if (workload === "speech-eval") return "speech";
+ if (workload === "embed-eval") return "embed";
+ return VISION_BACKENDS.has(String(backend ?? "")) ? "vision" : "llm";
+}
+
+/**
+ * What "has named metrics" means, per family. A row carrying at least one of
+ * these is pivotable; a row carrying none is excluded and said so.
+ *
+ * `llm` is the awkward case and deliberately so: `decode_tok_s` IS the named
+ * field for an LLM batch, and is the *misused* field for a vision batch. Which
+ * one a row is depends on the job's backend, not on the row — so the family is
+ * decided from the spec before the metrics are read.
+ */
+export const NAMED_METRICS: Record = {
+ vision: ["top1_pct", "top5_pct", "p50_ms", "p95_ms", "images_per_s"],
+ llm: ["prefill_tok_s", "decode_tok_s", "ttft_ms"],
+ speech: ["wer_pct", "rtf", "clips"],
+ embed: ["recall_at_1", "recall_at_5", "recall_at_10", "docs_per_s", "dim"],
+};
+
+/** The headline number per family — what the bars chart and what sorts a table. */
+export const HEADLINE: Record = {
+ vision: { key: "top1_pct", label: "Top-1", unit: "%", higherIsBetter: true },
+ llm: { key: "decode_tok_s", label: "Decode", unit: "tok/s", higherIsBetter: true },
+ speech: { key: "wer_pct", label: "WER", unit: "%", higherIsBetter: false },
+ embed: { key: "recall_at_1", label: "Recall@1", unit: "", higherIsBetter: true },
+};
+
+// --- energy per unit of work ---------------------------------------------
+
+export type PerJoule = { value: number; unit: string; basis: string };
+
+/**
+ * Work done per joule, or null.
+ *
+ * Only DIRECT counts are used. Deriving a count by multiplying a rate by the
+ * job's wall duration would multiply two uncertainties and produce a number that
+ * looks measured, so a family with no stored count gets no column rather than a
+ * plausible one. `omit it cleanly where it does not [exist]` means omit, not
+ * approximate.
+ */
+export function perJoule(
+ family: Family,
+ metrics: Record,
+ params: Record,
+ energyWh: number | null,
+): PerJoule | null {
+ if (energyWh === null || !Number.isFinite(energyWh) || energyWh <= 0) return null;
+ const joules = energyWh * 3600;
+
+ const n = (v: unknown) => (typeof v === "number" && Number.isFinite(v) && v > 0 ? v : null);
+ let count: number | null = null;
+ let unit = "";
+ let basis = "";
+ if (family === "llm") {
+ count = n(params.gen_tokens);
+ unit = "tok/J";
+ basis = "params.gen_tokens";
+ } else if (family === "vision") {
+ count = n(params.images) ?? n(params.image_count) ?? n(params.count);
+ unit = "img/J";
+ basis = "params.images";
+ } else if (family === "speech") {
+ count = n(metrics.clips);
+ unit = "clips/J";
+ basis = "metrics.clips";
+ } else {
+ count = n(params.docs) ?? n(params.doc_count);
+ unit = "docs/J";
+ basis = "params.docs";
+ }
+ if (count === null) return null;
+ return { value: count / joules, unit, basis };
+}
+
+// --- the pivot (pure) -----------------------------------------------------
+
+export type EnergyView = {
+ wh: number;
+ method: PowerMethod | "os";
+ /** true = the plug also saw the battery charging; null = no beacon covered it. */
+ includes_charging: boolean | null;
+ /** "result" when the runner stored it, "integrated" when this collector computed it. */
+ source: "result" | "integrated";
+ baseline_source: string | null;
+ note: string | null;
+};
+
+export type EvalRow = {
+ job_id: string;
+ device_id: string;
+ device_model: string | null;
+ simulator: boolean;
+ at: string | null;
+ input_sha256: string | null;
+ workload: string;
+ backend: string | null;
+ model: string | null;
+ quant: string | null;
+ accel: string | null;
+ metrics: Record;
+ params: Record;
+ energy: EnergyView | null;
+};
+
+export type Cell = {
+ job_id: string;
+ device_id: string;
+ device_model: string | null;
+ simulator: boolean;
+ at: string | null;
+ metrics: Record;
+ energy: EnergyView | null;
+ per_joule: PerJoule | null;
+};
+
+export type ModelRow = {
+ key: string;
+ model: string | null;
+ quant: string | null;
+ backend: string | null;
+ accel: string | null;
+ cells: Record;
+};
+
+export type Excluded = {
+ job_id: string;
+ device_id: string;
+ at: string | null;
+ workload: string;
+ backend: string | null;
+ input_sha256: string | null;
+ reason: string;
+ /** The metric keys the row does carry, so the exclusion is checkable. */
+ present: string[];
+};
+
+export type EvalSet = {
+ input_sha256: string;
+ family: Family;
+ headline: (typeof HEADLINE)[Family];
+ named_metrics: readonly string[];
+ workloads: string[];
+ /** Column order for the pivot. */
+ devices: { device_id: string; device_model: string | null; simulator: boolean }[];
+ models: ModelRow[];
+ runs: number;
+ latest_at: string | null;
+ excluded: number;
+ has_energy: boolean;
+};
+
+export type Pivot = { sets: EvalSet[]; excluded: Excluded[] };
+
+const modelKey = (r: EvalRow) => `${r.model ?? "?"}|${r.quant ?? ""}|${r.backend ?? "?"}|${r.accel ?? ""}`;
+
+/**
+ * Group rows into eval sets, pivot each by model × device.
+ *
+ * Pure and exported so the arithmetic has a test that needs neither a database
+ * nor hardware. Newer rows win per (set, model, device): an eval re-run on one
+ * device replaces that cell and leaves the rest of the table alone, which is how
+ * you actually add a candidate.
+ */
+export function pivotEvals(rows: EvalRow[]): Pivot {
+ const excluded: Excluded[] = [];
+ const bySet = new Map();
+
+ for (const r of rows) {
+ const family = familyOf(r.workload, r.backend);
+ const present = Object.keys(r.metrics ?? {}).filter((k) => r.metrics[k] !== null && r.metrics[k] !== undefined);
+ if (!r.input_sha256) {
+ excluded.push({
+ job_id: r.job_id, device_id: r.device_id, at: r.at, workload: r.workload, backend: r.backend,
+ input_sha256: null,
+ reason: "no params.input_sha256 — nothing says which eval set this scored, so it cannot be compared to anything",
+ present,
+ });
+ continue;
+ }
+ const named = NAMED_METRICS[family].filter((k) => typeof r.metrics?.[k] === "number");
+ if (named.length === 0) {
+ excluded.push({
+ job_id: r.job_id, device_id: r.device_id, at: r.at, workload: r.workload, backend: r.backend,
+ input_sha256: r.input_sha256,
+ reason: `no named ${family} metric (expected one of ${NAMED_METRICS[family].join(", ")}) — this row predates the schema fields and its numbers live in slots that mean something else`,
+ present,
+ });
+ continue;
+ }
+ const list = bySet.get(r.input_sha256) ?? [];
+ list.push(r);
+ bySet.set(r.input_sha256, list);
+ }
+
+ const sets: EvalSet[] = [];
+ for (const [input_sha256, list] of bySet) {
+ const family = familyOf(list[0].workload, list[0].backend);
+ const keys = NAMED_METRICS[family];
+
+ const models = new Map();
+ const devices = new Map();
+ for (const r of list) {
+ devices.set(r.device_id, { device_id: r.device_id, device_model: r.device_model, simulator: r.simulator });
+ const key = modelKey(r);
+ const row =
+ models.get(key) ??
+ ({ key, model: r.model, quant: r.quant, backend: r.backend, accel: r.accel, cells: {} } as ModelRow);
+ const prior = row.cells[r.device_id];
+ // Newest wins. A null `at` is treated as oldest so an undated row never
+ // displaces a dated one.
+ const t = r.at ? Date.parse(r.at) : -Infinity;
+ const priorT = prior?.at ? Date.parse(prior.at) : prior ? -Infinity : -Infinity;
+ if (!prior || t >= priorT) {
+ const metrics: Record = {};
+ for (const k of keys) metrics[k] = typeof r.metrics?.[k] === "number" ? r.metrics[k] : null;
+ row.cells[r.device_id] = {
+ job_id: r.job_id,
+ device_id: r.device_id,
+ device_model: r.device_model,
+ simulator: r.simulator,
+ at: r.at,
+ metrics,
+ energy: r.energy,
+ per_joule: perJoule(family, r.metrics ?? {}, r.params ?? {}, r.energy?.wh ?? null),
+ };
+ }
+ models.set(key, row);
+ }
+
+ const times = list.map((r) => (r.at ? Date.parse(r.at) : 0));
+ sets.push({
+ input_sha256,
+ family,
+ headline: HEADLINE[family],
+ named_metrics: keys,
+ workloads: [...new Set(list.map((r) => r.workload))].sort(),
+ devices: [...devices.values()].sort((a, b) => a.device_id.localeCompare(b.device_id)),
+ models: [...models.values()].sort((a, b) => (a.model ?? "").localeCompare(b.model ?? "") || a.key.localeCompare(b.key)),
+ runs: list.length,
+ latest_at: times.length ? new Date(Math.max(...times)).toISOString() : null,
+ excluded: excluded.filter((e) => e.input_sha256 === input_sha256).length,
+ has_energy: list.some((r) => r.energy !== null),
+ });
+ }
+
+ sets.sort((a, b) => (b.latest_at ?? "").localeCompare(a.latest_at ?? ""));
+ return { sets, excluded };
+}
+
+// --- database → EvalRow ---------------------------------------------------
+
+type DbRow = { job_id: string; device_id: string; payload: string; created_at: string; workload: string; spec: string };
+
+function evalRows(inputSha: string | null, limit: number): EvalRow[] {
+ const rows = db
+ .prepare(
+ `SELECT r.job_id, r.device_id, r.payload, r.created_at, j.workload, j.spec
+ FROM results r JOIN jobs j ON j.job_id = r.job_id
+ WHERE j.workload IN (${EVAL_WORKLOADS.map(() => "?").join(",")})
+ AND json_extract(r.payload, '$.final') = 1
+ ORDER BY r.created_at DESC LIMIT ?`,
+ )
+ .all(...EVAL_WORKLOADS, limit) as DbRow[];
+
+ const descriptors = new Map(
+ (db.prepare("SELECT device_id, descriptor FROM devices").all() as { device_id: string; descriptor: string }[]).map(
+ (d) => [d.device_id, parse>(d.descriptor, {})],
+ ),
+ );
+
+ // power.json once per request, not once per row.
+ const powerCfg: PowerConfig | null = loadPowerConfig();
+ const metered = meteredPools(powerCfg).length > 0;
+
+ const out: EvalRow[] = [];
+ for (const r of rows) {
+ const spec = parse>(r.spec, {});
+ const payload = parse>(r.payload, {});
+ const params = (spec.params ?? {}) as Record;
+ const sha = typeof params.input_sha256 === "string" && params.input_sha256 ? params.input_sha256 : null;
+ if (inputSha && sha !== inputSha) continue;
+
+ const metrics = (payload.metrics ?? {}) as Record;
+ const descriptor = descriptors.get(r.device_id) ?? {};
+
+ // A stored energy_wh is the runner's own measurement and always wins: it
+ // may come from a machine that reports its own power (`os`), which the plug
+ // knows nothing about. Only when nothing was stored does the collector
+ // integrate the pool's samples itself.
+ let energy: EnergyView | null = null;
+ if (typeof metrics.energy_wh === "number" && Number.isFinite(metrics.energy_wh)) {
+ const m = metrics.energy_method;
+ energy = {
+ wh: metrics.energy_wh,
+ method: m === "plug" || m === "plug-shared" || m === "os" ? m : "os",
+ includes_charging: null,
+ source: "result",
+ baseline_source: null,
+ note:
+ m === "plug-shared"
+ ? "Recorded by the runner. Several devices share this plug: the figure is the pool's and must not be divided per device."
+ : "Recorded by the runner.",
+ };
+ } else if (metered) {
+ const e = energyForJob(r.job_id, r.device_id, powerCfg);
+ if (isEnergy(e))
+ energy = {
+ wh: e.energy_wh,
+ method: e.energy_method,
+ includes_charging: e.includes_charging,
+ source: "integrated",
+ baseline_source: e.baseline_source,
+ note: e.note,
+ };
+ }
+
+ out.push({
+ job_id: r.job_id,
+ device_id: r.device_id,
+ device_model: (descriptor.model as string) ?? null,
+ simulator: isSimulator(descriptor, r.device_id),
+ at: iso(r.created_at),
+ input_sha256: sha,
+ workload: r.workload,
+ backend: spec.backend ?? null,
+ model: spec.model?.name ?? null,
+ quant: spec.model?.quant ?? null,
+ accel: params.accelerator ?? params.delegate ?? null,
+ metrics,
+ params,
+ energy,
+ });
+ }
+ return out;
+}
+
+export function registerEvals(app: FastifyInstance) {
+ app.get("/api/evals", async (req) => {
+ const q = req.query as Record;
+ const limit = Math.min(2000, Math.max(50, Number(q.limit ?? 800) || 800));
+ const { sets, excluded } = pivotEvals(evalRows(null, limit));
+
+ return {
+ // Summaries only; a set's table comes from /api/evals/:input_sha.
+ sets: sets.map((s) => ({
+ input_sha256: s.input_sha256,
+ family: s.family,
+ workloads: s.workloads,
+ models: s.models.length,
+ devices: s.devices.length,
+ runs: s.runs,
+ latest_at: s.latest_at,
+ excluded: s.excluded,
+ has_energy: s.has_energy,
+ })),
+ excluded: {
+ count: excluded.length,
+ // Grouped so the page can say "8 rows from 2 jobs", not list 8 lines.
+ by_reason: Object.entries(
+ excluded.reduce>((acc, e) => {
+ const k = e.input_sha256 === null ? "no eval set" : `no named metrics (${e.workload})`;
+ acc[k] = (acc[k] ?? 0) + 1;
+ return acc;
+ }, {}),
+ ).map(([reason, count]) => ({ reason, count })),
+ rows: excluded.slice(0, 200),
+ },
+ // So a page with no sets can tell "nothing ran" from "everything ran but
+ // nothing was queryable" — two very different states.
+ scanned: limit,
+ energy_configured: meteredPools().length > 0,
+ };
+ });
+
+ app.get("/api/evals/:input_sha", async (req) => {
+ const { input_sha } = req.params as { input_sha: string };
+ const q = req.query as Record;
+ const limit = Math.min(2000, Math.max(50, Number(q.limit ?? 800) || 800));
+ const { sets, excluded } = pivotEvals(evalRows(input_sha, limit));
+ return {
+ set: sets[0] ?? null,
+ excluded,
+ energy_configured: meteredPools().length > 0,
+ };
+ });
+}
diff --git a/collector/src/api/index.ts b/collector/src/api/index.ts
index 5a0b09d..4efe0c8 100644
--- a/collector/src/api/index.ts
+++ b/collector/src/api/index.ts
@@ -10,6 +10,7 @@
import type { FastifyInstance } from "fastify";
import { registerDevices } from "./devices.js";
import { registerEnroll } from "./enroll.js";
+import { registerEvals } from "./evals.js";
import { registerJobs } from "./jobs.js";
import { registerMutations } from "./mutations.js";
import { registerOverview } from "./overview.js";
@@ -35,6 +36,7 @@ export function registerApi(app: FastifyInstance, announce: Announce, matchingDe
registerEnroll(app);
registerJobs(app);
registerResults(app);
+ registerEvals(app);
registerSystem(app);
registerVisual(app);
registerStream(app);
diff --git a/collector/src/api/jobs.ts b/collector/src/api/jobs.ts
index 8d006c2..1d3737c 100644
--- a/collector/src/api/jobs.ts
+++ b/collector/src/api/jobs.ts
@@ -3,7 +3,7 @@ import type { FastifyInstance } from "fastify";
import { db } from "../db.js";
import { AGE, beaconFields, inClause, iso, paging, parse, sha256Refs } from "./shared.js";
-const STATUSES = new Set(["queued", "claimed", "done", "failed", "cancelled"]);
+const STATUSES = new Set(["waiting", "queued", "claimed", "done", "failed", "cancelled"]);
const EXECUTORS = new Set(["device", "host"]);
const SORTS: Record = {
created: "created_at",
@@ -30,13 +30,14 @@ type JobRow = {
priority: number;
parent_job_id: string | null;
template_id: string | null;
+ depends_on: string | null;
lease_remaining_s: number | null;
duration_s: number | null;
};
const JOB_COLUMNS = `job_id, executor, workload, spec, status, created_at, claimed_by, claimed_at,
finished_at, lease_ttl_s, max_attempts, attempts, lease_deadline, last_error,
- priority, parent_job_id, template_id,
+ priority, parent_job_id, template_id, depends_on,
CASE WHEN status = 'claimed' AND lease_deadline IS NOT NULL
THEN CAST(strftime('%s', lease_deadline) - strftime('%s','now') AS INTEGER) END AS lease_remaining_s,
CASE WHEN claimed_at IS NOT NULL
@@ -54,6 +55,9 @@ export function shapeJob(j: JobRow, opts: { spec?: boolean } = {}) {
max_attempts: j.max_attempts,
priority: j.priority,
template_id: j.template_id,
+ // The chain, as the row records it. Empty rather than null so a caller can
+ // ask `.length` without knowing whether dependencies were ever a thing.
+ depends_on: parse(j.depends_on, []),
lease_ttl_s: j.lease_ttl_s,
lease_deadline: iso(j.lease_deadline),
lease_remaining_s: j.lease_remaining_s,
@@ -68,6 +72,9 @@ export function shapeJob(j: JobRow, opts: { spec?: boolean } = {}) {
device_id: spec.targets?.device_id ?? null,
wants_executor: spec.targets?.executor ?? null,
exclusive: spec.targets?.exclusive ?? false,
+ // Whether this job will stand aside for higher-priority work. Pulled out of
+ // the spec for the same reason `exclusive` is: a list view reads it.
+ preemptible: spec.preemptible === true,
backend: spec.backend ?? null,
model: spec.model?.name ?? null,
app: spec.app?.name ?? null,
@@ -262,7 +269,12 @@ export function registerJobs(app: FastifyInstance) {
// stores current state, not an event log, so this is derived — and the
// field name says so rather than implying a record that does not exist.
derived_timeline: [
- { at: iso(row.created_at), what: "queued" },
+ {
+ at: iso(row.created_at),
+ what: row.depends_on
+ ? `enqueued, waiting on ${parse(row.depends_on, []).join(", ")}`
+ : "queued",
+ },
...(row.claimed_at ? [{ at: iso(row.claimed_at), what: `claimed by ${row.claimed_by ?? "?"} (attempt ${row.attempts})` }] : []),
...(beacons.length ? [{ at: beacons[0].ts, what: `first beacon of ${beacons.length}` }] : []),
...(row.finished_at ? [{ at: iso(row.finished_at), what: row.status }] : []),
diff --git a/collector/src/api/mutations.ts b/collector/src/api/mutations.ts
index 6678bf5..af9b809 100644
Binary files a/collector/src/api/mutations.ts and b/collector/src/api/mutations.ts differ
diff --git a/collector/src/api/overview.ts b/collector/src/api/overview.ts
index 94328a6..282ba3a 100644
--- a/collector/src/api/overview.ts
+++ b/collector/src/api/overview.ts
@@ -57,6 +57,11 @@ function build() {
.prepare(
`SELECT
COUNT(*) FILTER (WHERE status = 'queued') AS queued,
+ -- Counted separately, never folded into queued. A waiting job is not
+ -- work the fleet is failing to pick up; it is a chain parked on a
+ -- dependency, and adding it to queue depth would make a healthy
+ -- pipeline look like a backlog.
+ COUNT(*) FILTER (WHERE status = 'waiting') AS waiting,
COUNT(*) FILTER (WHERE status = 'claimed') AS claimed,
COUNT(*) FILTER (WHERE status = 'done' AND finished_at >= datetime('now','-24 hours')) AS done_24h,
COUNT(*) FILTER (WHERE status = 'failed' AND finished_at >= datetime('now','-24 hours')) AS failed_24h,
diff --git a/collector/src/api/shared.ts b/collector/src/api/shared.ts
index 80062a7..265aaa4 100644
--- a/collector/src/api/shared.ts
+++ b/collector/src/api/shared.ts
@@ -158,6 +158,10 @@ export function tableCounts(): Record {
"jobs",
"results",
"beacon_samples",
+ // Sampled every few seconds per pool, so it outgrows beacon_samples fast
+ // and the System page is where anyone would look first when the database
+ // starts growing. Counted here so it cannot grow unseen.
+ "power_samples",
"artifacts",
"schedules",
"device_locks",
diff --git a/collector/src/contact-sheet.ts b/collector/src/contact-sheet.ts
new file mode 100644
index 0000000..69b625f
--- /dev/null
+++ b/collector/src/contact-sheet.ts
@@ -0,0 +1,119 @@
+/**
+ * The `index.html` that goes inside a screenshot bundle.
+ *
+ * Its own module because two workloads produce the same artifact for the same
+ * reason: locale-shots captures one column per locale, a11y-audit one per
+ * display condition, and in both cases the review that matters is a horizontal
+ * scan across one screen -- a string that did not translate, a label clipped by
+ * a longer one, a layout that did not mirror, a button that grew to two lines
+ * at the largest text size. None of that is visible in a directory of PNGs, and
+ * a zip a person has to unpack and open file by file is a zip nobody reviews.
+ *
+ * Pure, so the page can be asserted on without a device attached.
+ */
+
+export type SheetShot = {
+ /** The column this shot belongs in: a locale tag, or a display condition. */
+ column: string;
+ /** Render this column's cells right-to-left. */
+ rtl?: boolean;
+ device: string;
+ /** The screen's name, e.g. "home". Becomes a row. */
+ shot: string;
+ /** Path inside the bundle, or null when this cell has no image. */
+ file: string | null;
+ /** Why there is no image, or anything worth saying about this one. */
+ note?: string;
+};
+
+const esc = (s: string): string =>
+ s.replace(/&/g, "&").replace(//g, ">").replace(/"/g, """);
+
+/**
+ * One row per screen, one column per locale or condition.
+ *
+ * RTL columns are marked in the header AND rendered `dir="rtl"`, because the
+ * most common real localisation defect is a screen that translated correctly
+ * and did not mirror, and that is only obvious next to a header saying it
+ * should have.
+ *
+ * A missing shot is a visible, labelled hole rather than an empty cell. An
+ * empty cell lets a column that captured nothing read as a column with nothing
+ * to report, which is the same failure as a green row that measured nothing.
+ */
+export function contactSheetHtml(
+ title: string,
+ shots: SheetShot[],
+ opts: { columnNoun?: string; subtitle?: string } = {},
+): string {
+ const noun = opts.columnNoun ?? "locale";
+ const columns: string[] = [];
+ const rtlOf = new Map();
+ for (const s of shots) {
+ if (!columns.includes(s.column)) columns.push(s.column);
+ if (s.rtl) rtlOf.set(s.column, true);
+ }
+ // Rows are (device, screen) pairs in first-seen order -- the flow's own
+ // order, which is the order a person walked the app in.
+ const rows: { device: string; shot: string }[] = [];
+ for (const s of shots) {
+ if (!rows.some((r) => r.device === s.device && r.shot === s.shot)) {
+ rows.push({ device: s.device, shot: s.shot });
+ }
+ }
+
+ const cell = (device: string, shot: string, column: string): string => {
+ const s = shots.find((x) => x.device === device && x.shot === shot && x.column === column);
+ if (!s || !s.file) {
+ return `no shot
${
+ s?.note ? `${esc(s.note)}
` : ""}` +
+ ` ` +
+ (s.note ? `${esc(s.note)}
` : "") + "";
+ };
+
+ const missing = shots.filter((s) => !s.file).length;
+ const sub = opts.subtitle ??
+ `${columns.length} ${noun}(s) · ${rows.length} screen(s) · ${shots.length - missing} shot(s)` +
+ (missing ? ` · ${missing} missing ` : "");
+
+ return `
+
+${esc(title)}
+
+${esc(title)}
+${sub}
+
+
+screen ${
+ columns.map((c) => `${esc(c)}${rtlOf.get(c) ? 'RTL
' : ""} `).join("")
+ }
+${rows.map((r) =>
+ `${esc(r.shot)}${esc(r.device)}
` +
+ columns.map((c) => cell(r.device, r.shot, c)).join("") + "
+
+
+
+
+
+`;
+
+/** What uiautomator prints on stdout -- the misspelling is AOSP's, not ours. */
+const UIAUTOMATOR_RECEIPT = "UI hierchary dumped to: /sdcard/window_dump.xml\n";
+
+const MAESTRO_TREE = `Running on 8A2C6F11-3D50-4B2E-9F0A-1C6B7E2D4A88
+{"attributes":{"bounds":"[0,0][390,844]","clickable":"false"},"children":[
+ {"attributes":{"elementType":"Button","accessibilityText":"","bounds":"[16,100][56,140]"},"children":[]},
+ {"attributes":{"elementType":"StaticText","text":"Hello","bounds":"[16,160][374,182]"},"children":[]}
+]}
+`;
+
+const XCUI_TREE = `Attributes: Application, pid: 8123, label: 'Aliquant'
+Element subtree:
+ →Application, 0x600001, {{0.0, 0.0}, {390.0, 844.0}}, label: 'Aliquant'
+ Window (Main), 0x600002, {{0.0, 0.0}, {390.0, 844.0}}
+ Other, 0x600003, {{0.0, 0.0}, {390.0, 844.0}}
+ Button, 0x600004, {{16.0, 100.0}, {40.0, 40.0}}, label: 'Close'
+ StaticText, 0x600005, {{16.0, 160.0}, {358.0, 22.0}}, label: 'Hello'
+ Button, 0x600006, {{344.0, 100.0}, {30.0, 30.0}}, identifier: 'help'
+ Button, 0x600007, {{16.0, 700.0}, {358.0, 50.0}}, label: 'Save', Disabled
+Path to element:
+ →Application, 0x600001
+`;
+
+const DEFAULTS_LANGUAGES = `(
+ "en-US",
+ fr
+)
+`;
+
+const DEFAULTS_MISSING =
+ "2026-09-04 11:02:14.881 defaults[4711:88213] \n" +
+ "The domain/default pair of (kCFPreferencesAnyApplication, AppleLanguages) does not exist\n";
+
+/** simctl's help, which lists option names one per line -- each value-shaped. */
+const SIMCTL_UI_USAGE = `Usage: simctl ui []
+Supported options:
+ appearance
+ content_size
+ increase_contrast
+`;
+
+// ---------------------------------------------------------------------------
+
+export function runDeviceParserChecks(check: Check) {
+ // --- app-soak: dumpsys meminfo ---
+ {
+ const m = parseMeminfo(crlf(MEMINFO_PIXEL));
+ check("meminfo: the App Summary total is the PSS, not the swap total on the same line",
+ m.pssKb === 73200 && m.problem === null, JSON.stringify(m));
+ check("meminfo: the pid is read through CRLF", m.pid === 8123, JSON.stringify(m));
+
+ const legacy = parseMeminfo(crlf(MEMINFO_LEGACY));
+ check("meminfo: a build with no App Summary still reports from the column table",
+ legacy.pssKb === 93052 && legacy.problem === null, JSON.stringify(legacy));
+
+ const gone = parseMeminfo(crlf(MEMINFO_GONE));
+ check("meminfo: a dead app is a named problem, not 0 MB",
+ gone.pssKb === null && (gone.problem ?? "").includes("no process found"), JSON.stringify(gone));
+
+ const empty = parseMeminfo("");
+ check("meminfo: empty output is refused rather than read as zero",
+ empty.pssKb === null && empty.problem !== null, JSON.stringify(empty));
+ }
+
+ // --- app-soak: dumpsys gfxinfo ---
+ {
+ const g = parseGfxinfo(crlf(GFXINFO));
+ check("gfxinfo: the modern Janky frames line wins over the legacy one",
+ g.jankyFrames === 93 && g.totalFrames === 1842, JSON.stringify(g));
+ check("gfxinfo: the percentage is computed from the counts, not read from the parentheses",
+ g.jankPct !== null && Math.abs(g.jankPct - (93 / 1842) * 100) < 1e-9, JSON.stringify(g));
+
+ const idle = parseGfxinfo(crlf(GFXINFO_IDLE));
+ check("gfxinfo: zero frames rendered is no reading, not 0% jank",
+ idle.jankPct === null && (idle.problem ?? "").includes("no frames"), JSON.stringify(idle));
+
+ const none = parseGfxinfo(crlf("No process found for: com.taylab.aliquant\n"));
+ check("gfxinfo: a dead app is a named problem", none.jankPct === null && none.problem !== null, JSON.stringify(none));
+
+ const noisy = parseGfxinfo(crlf("Applications Graphics Acceleration Info:\nUptime: 12 Realtime: 12\n"));
+ check("gfxinfo: a header with no counts is refused",
+ noisy.jankPct === null && noisy.problem !== null, JSON.stringify(noisy));
+ }
+
+ // --- app-soak: logcat -b crash ---
+ {
+ const c = parseCrashLogcat(crlf(CRASH_JAVA), "com.taylab.aliquant");
+ check("logcat: another app's FATAL EXCEPTION is not our crash",
+ c.count === 1, JSON.stringify(c));
+ check("logcat: the exception line becomes the signature",
+ (c.signatures[0] ?? "").includes("IllegalStateException"), JSON.stringify(c.signatures));
+
+ const n = parseCrashLogcat(crlf(CRASH_NATIVE), "com.taylab.aliquant");
+ check("logcat: a tombstone counts as a crash and keeps its signal",
+ n.count === 1 && (n.signatures[0] ?? "").includes("SIGSEGV"), JSON.stringify(n));
+
+ const quiet = parseCrashLogcat(crlf("--------- beginning of crash\n"), "com.taylab.aliquant");
+ check("logcat: an empty crash buffer is zero crashes, not a problem",
+ quiet.count === 0 && quiet.problem === null, JSON.stringify(quiet));
+
+ const broken = parseCrashLogcat("error: device 'ZY22HXXXXX' not found\n", "com.taylab.aliquant");
+ check("logcat: a buffer that could not be read is a problem, never 0 crashes",
+ broken.problem !== null, JSON.stringify(broken));
+ }
+
+ // --- app-soak: simulator crash reports ---
+ {
+ const s = parseSimCrashLog(SIM_CRASH, "com.taylab.aliquant");
+ check("sim log: a saved crash report matches the bundle id by its executable leaf",
+ s.count === 1 && s.problem === null, JSON.stringify(s));
+
+ const other = parseSimCrashLog(SIM_CRASH_OTHER, "com.taylab.aliquant");
+ check("sim log: another app's crash report is not ours", other.count === 0, JSON.stringify(other));
+
+ const bad = parseSimCrashLog("log: Invalid predicate: unexpected token\n", "com.taylab.aliquant");
+ check("sim log: a failed log show is a problem, not zero crashes",
+ bad.problem !== null, JSON.stringify(bad));
+
+ const empty = parseSimCrashLog("", "com.taylab.aliquant");
+ check("sim log: an empty log is zero crashes with no problem",
+ empty.count === 0 && empty.problem === null, JSON.stringify(empty));
+ }
+
+ // --- a11y: uiautomator ---
+ {
+ const t = parseUiautomatorDump(crlf(UIAUTOMATOR));
+ check("uiautomator: the dump parses through CRLF", t.problem === null && t.nodes.length === 4, JSON.stringify(t.problem));
+
+ // 420dpi: 90px -> 34pt and 60px -> 23pt are both under 44; 280px -> 107pt is not.
+ const f = a11yFindings(t.nodes, { unit: "pixels", densityDpi: 420 }, { step: "home" });
+ const errors = f.filter((x) => x.severity === "error");
+ const warns = f.filter((x) => x.severity === "warn");
+ check("a11y: an unlabelled tappable is one error",
+ errors.length === 1 && errors[0].check === "a11y-label", JSON.stringify(errors));
+ check("a11y: the unlabelled finding names the element that has no label",
+ (errors[0]?.detail ?? "").includes("close"), JSON.stringify(errors[0]));
+ check("a11y: pixel bounds converted at 420dpi flag both small targets and neither big one",
+ warns.filter((w) => w.check === "a11y-target-size").length === 2, JSON.stringify(warns));
+
+ const unknown = a11yFindings(t.nodes, { unit: "unknown" }, { step: "home" });
+ check("a11y: without a density no size is judged, and the run says so",
+ unknown.filter((x) => x.check === "a11y-target-size").length === 0 &&
+ unknown.some((x) => x.check === "a11y-coverage" && x.detail.includes("unknown unit")),
+ JSON.stringify(unknown));
+ check("a11y: an unjudgeable size does not hide the label error",
+ unknown.filter((x) => x.severity === "error").length === 1, JSON.stringify(unknown));
+
+ const receipt = parseUiautomatorDump(UIAUTOMATOR_RECEIPT);
+ check("uiautomator: the stdout receipt is not an empty screen with nothing wrong",
+ receipt.nodes.length === 0 && receipt.problem !== null, JSON.stringify(receipt.problem));
+ check("uiautomator: empty output is refused", parseUiautomatorDump("").problem !== null);
+ }
+
+ // --- a11y: maestro hierarchy ---
+ {
+ const t = parseMaestroHierarchy(MAESTRO_TREE);
+ check("maestro: the JSON is found past the banner Maestro prints first",
+ t.problem === null && t.nodes.length === 3, JSON.stringify(t.problem));
+ const f = a11yFindings(t.nodes, { unit: "points" }, { step: "home" });
+ check("maestro: an unlabelled Button is an error and its 40pt box is a warning",
+ f.filter((x) => x.severity === "error").length === 1 &&
+ f.filter((x) => x.check === "a11y-target-size").length === 1, JSON.stringify(f));
+ check("maestro: non-JSON output is refused rather than read as an empty screen",
+ parseMaestroHierarchy("command not found: maestro\n").problem !== null);
+ }
+
+ // --- a11y: XCUITest debugDescription ---
+ {
+ const t = parseXcuiDebugDescription(XCUI_TREE);
+ check("xcui: the element subtree parses and the Path section is not read as elements",
+ t.problem === null && t.nodes.length === 7, `${t.nodes.length} nodes`);
+ check("xcui: `Window (Main)` is a Window, not a type called 'Window (Main)'",
+ t.nodes.some((n) => n.cls === "Window"), JSON.stringify(t.nodes.map((n) => n.cls)));
+ check("xcui: a disabled element is marked disabled rather than labelled 'Disabled'",
+ t.nodes.some((n) => n.label === "Save" && !n.enabled), JSON.stringify(t.nodes.find((n) => n.label === "Save")));
+
+ const f = a11yFindings(t.nodes, { unit: "points" }, { step: "home" });
+ check("xcui: an identifier is not a label -- the help button is unlabelled",
+ f.filter((x) => x.severity === "error").length === 1 &&
+ (f.find((x) => x.severity === "error")?.detail ?? "").includes("help"), JSON.stringify(f));
+ check("xcui: frames are already points, so 40pt and 30pt are both under the minimum",
+ f.filter((x) => x.check === "a11y-target-size").length === 2, JSON.stringify(f));
+ check("xcui: a disabled control is not judged at all",
+ !JSON.stringify(f).includes("Save"), JSON.stringify(f));
+ check("xcui: empty output is refused", parseXcuiDebugDescription("").problem !== null);
+ }
+
+ // --- a11y: density ---
+ {
+ check("density: the override wins over the physical density",
+ parseAndroidDensity(crlf("Physical density: 420\nOverride density: 560\n")) === 560);
+ check("density: a bare getprop value is read",
+ parseAndroidDensity(crlf("420\n")) === 420);
+ check("density: nothing usable is null, so the size check skips itself",
+ parseAndroidDensity("") === null && parseAndroidDensity(crlf("Physical density: unknown\n")) === null);
+ }
+
+ // --- locale-shots: the tag vocabulary ---
+ {
+ const ar = parseLocaleTag("ar-eg");
+ check("locale: case is normalised and Arabic is RTL",
+ ar.tag === "ar-EG" && ar.rtl, JSON.stringify(ar));
+ const zh = parseLocaleTag("zh-hant-tw");
+ check("locale: a script subtag is title-cased and is not RTL",
+ zh.tag === "zh-Hant-TW" && !zh.rtl, JSON.stringify(zh));
+ check("locale: Hebrew is RTL", parseLocaleTag("he").rtl);
+ check("locale: AppleLocale takes the underscored form", appleLocaleOf(ar) === "ar_EG");
+
+ let threw = false;
+ try { parseLocaleTag("ar_EG"); } catch { threw = true; }
+ check("locale: an underscored tag is refused rather than stored and ignored", threw);
+
+ threw = false;
+ try { parseLocaleTag("english"); } catch { threw = true; }
+ check("locale: a non-tag is refused", threw);
+
+ const list = parseLocaleList("en,EN,fr,ar-EG");
+ check("locale: the list dedupes case-insensitively and keeps order",
+ list.length === 3 && list[0].tag === "en" && list[2].tag === "ar-EG", JSON.stringify(list.map((l) => l.tag)));
+ check("locale: a list with an RTL member is detectable", coversRtl(list));
+ check("locale: an all-LTR list is detectable", !coversRtl(parseLocaleList(["en", "fr"])));
+
+ threw = false;
+ try { parseLocaleList(undefined); } catch { threw = true; }
+ check("locale: a missing params.locales throws rather than capturing one language", threw);
+
+ threw = false;
+ try { parseLocaleList([]); } catch { threw = true; }
+ check("locale: an empty params.locales throws", threw);
+ }
+
+ // --- device settings: reading a value back ---
+ {
+ check("settings: the literal null means unset, not the string 'null'",
+ parseAndroidSettingValue(crlf("null\n")) === null);
+ check("settings: a value survives CRLF", parseAndroidSettingValue(crlf("ar-EG\n")) === "ar-EG");
+ check("settings: empty output is unset", parseAndroidSettingValue("") === null);
+ check("settings: uimode reports its state", parseUiModeNight(crlf("Night mode: yes\n")) === "yes");
+ check("settings: an unparseable uimode answer is null", parseUiModeNight("") === null);
+
+ check("defaults: a plist array parses with mixed quoting",
+ JSON.stringify(parseDefaultsArray(DEFAULTS_LANGUAGES)) === JSON.stringify(["en-US", "fr"]),
+ JSON.stringify(parseDefaultsArray(DEFAULTS_LANGUAGES)));
+ check("defaults: an absent domain is null, so a restore deletes rather than writes an empty list",
+ parseDefaultsArray(DEFAULTS_MISSING) === null);
+ check("defaults: a scalar is unquoted", parseDefaultsString(crlf('"ar_EG"\n')) === "ar_EG");
+ check("defaults: an absent scalar is null", parseDefaultsString(DEFAULTS_MISSING) === null);
+
+ check("simctl ui: a current value is read", parseSimctlUiValue("light\n") === "light");
+ check("simctl ui: a usage message is not read as a value -- its option names are value-shaped",
+ parseSimctlUiValue(SIMCTL_UI_USAGE) === null, String(parseSimctlUiValue(SIMCTL_UI_USAGE)));
+ }
+
+ // --- device settings: what cannot be managed at all ---
+ {
+ check("state: a physical iPhone is refused by name",
+ (unmanageableReason({ id: "00008130-001", platform: "ios", kind: "device" }) ?? "").includes("launchArguments"));
+ check("state: a simulator is manageable",
+ unmanageableReason({ id: "AB06-37DA", platform: "ios", kind: "simulator" }) === null);
+ check("state: an Android device is manageable",
+ unmanageableReason({ id: "ZY22HXXXXX", platform: "android", kind: "device" }) === null);
+ }
+
+ // --- a11y-audit: which display conditions each platform can actually produce ---
+ {
+ check("display: Android 14 offers a 2.0 text scale",
+ planVariant("large-text", "android", { sdk: 34 }).settings["android:system.font_scale"] === "2.0");
+ check("display: an older Android tops out where its own UI does",
+ planVariant("large-text", "android", { sdk: 33 }).settings["android:system.font_scale"] === "1.3");
+
+ const bold11 = planVariant("bold-text", "android", { sdk: 30 });
+ check("display: bold text on Android 11 is refused -- the setting writes and nothing reads it",
+ (bold11.unreachable ?? "").includes("consumes"), JSON.stringify(bold11));
+ const bold12 = planVariant("bold-text", "android", { sdk: 31 });
+ check("display: bold text on Android 12 is a real setting",
+ !bold12.unreachable && bold12.settings["android:secure.font_weight_adjustment"] === "300", JSON.stringify(bold12));
+ const boldUnknown = planVariant("bold-text", "android", { sdk: null });
+ check("display: an unreadable API level refuses bold text rather than guessing",
+ boldUnknown.unreachable !== undefined, JSON.stringify(boldUnknown));
+
+ const boldIos = planVariant("bold-text", "ios-sim");
+ check("display: bold text on a simulator is refused, naming simctl's actual options",
+ (boldIos.unreachable ?? "").includes("increase_contrast"), JSON.stringify(boldIos));
+
+ check("display: dark mode is reachable on both platforms",
+ planVariant("dark", "ios-sim").settings["ios:ui.appearance"] === "dark" &&
+ planVariant("dark", "android", { sdk: 33 }).settings["android:uimode.night"] === "yes");
+ check("display: dark mode before Android 10 is refused",
+ planVariant("dark", "android", { sdk: 28 }).unreachable !== undefined);
+ check("display: the baseline pass changes nothing",
+ Object.keys(planVariant("baseline", "android", { sdk: 34 }).settings).length === 0);
+
+ check("display: the default is every condition", parseVariantList(undefined).length === 4);
+ const one = parseVariantList("dark");
+ check("display: asking for one condition still captures the baseline to compare it to",
+ one.length === 2 && one[0] === "baseline" && one[1] === "dark", JSON.stringify(one));
+ let threw = false;
+ try { parseVariantList("huge"); } catch { threw = true; }
+ check("display: an unknown condition throws rather than being skipped", threw);
+
+ check("display: an API level parses, and nonsense is null",
+ parseSdkLevel(crlf("34\n")) === 34 && parseSdkLevel("") === null && parseSdkLevel(crlf("REL\n")) === null);
+ }
+
+ // --- the contact sheet ---
+ {
+ const html = contactSheetHtml("locale-shots · com.x", [
+ { column: "en", device: "ZY22H", shot: "home", file: "en/ZY22H/home.png" },
+ { column: "ar-EG", rtl: true, device: "ZY22H", shot: "home", file: "ar-EG/ZY22H/home.png" },
+ { column: "fr", device: "ZY22H", shot: "home", file: null, note: "flow failed: timed out" },
+ ]);
+ check("sheet: an RTL column is marked and its cells are rendered right-to-left",
+ html.includes(">RTL<") && html.includes(''), "");
+ check("sheet: a missing shot is a visible labelled hole, not an empty cell",
+ html.includes("no shot") && html.includes("flow failed: timed out"), "");
+ check("sheet: the note is escaped into the page rather than injected",
+ !contactSheetHtml("t", [{ column: "en", device: "d", shot: "s", file: null, note: "" }])
+ .includes("