Wave 3 (partial): pipelines, a live screen, and the shell trust boundary

collector/dash/src/icons.tsx

@@ -53,6 +53,48 @@ const PATHS = {
       <path d="M8 7v3.5" />
     </>
   ),
+  convert: (
+    <>
+      <path d="M2.25 5.25h8.5M8.5 3l2.25 2.25L8.5 7.5M13.75 10.75h-8.5M7.5 8.5L5.25 10.75 7.5 13" />
+    </>
+  ),
+  dataset: (
+    <>
+      <ellipse cx="8" cy="3.75" rx="5.25" ry="2" />
+      <path d="M2.75 3.75v8.5c0 1.1 2.35 2 5.25 2s5.25-.9 5.25-2v-8.5M2.75 8c0 1.1 2.35 2 5.25 2s5.25-.9 5.25-2" />
+    </>
+  ),
+  serve: (
+    <>
+      <rect x="2" y="2.75" width="12" height="4.25" rx="1" />
+      <rect x="2" y="9" width="12" height="4.25" rx="1" />
+      <path d="M4.5 4.9h.01M4.5 11.15h.01" />
+    </>
+  ),
+  terminal: (
+    <>
+      <rect x="1.75" y="2.75" width="12.5" height="10.5" rx="1.5" />
+      <path d="M4.5 6.25L6.75 8l-2.25 1.75M8.5 10.25h3" />
+    </>
+  ),
+  bell: (
+    <>
+      <path d="M4.25 6.75a3.75 3.75 0 1 1 7.5 0c0 2.5 1 3.5 1 3.5h-9.5s1-1 1-3.5z" />
+      <path d="M6.75 12.5a1.5 1.5 0 0 0 2.5 0" />
+    </>
+  ),
+  camera: (
+    <>
+      <path d="M1.75 5.5h2.5l1-1.75h5.5l1 1.75h2.5v7.25H1.75z" />
+      <circle cx="8" cy="9" r="2.25" />
+    </>
+  ),
+  desktop: (
+    <>
+      <rect x="1.75" y="2.75" width="12.5" height="8.5" rx="1" />
+      <path d="M6 13.5h4" />
+    </>
+  ),
   hammer: (
     <>
       <path d="M2.75 13.25l5-5M6.5 5.25l2.5-2.5 1.5 1.5 1.75-1 1.5 1.5-1 1.75 1.5 1.5-2.5 2.5z" />
@@ -360,6 +402,13 @@ const WORKLOAD_ICON: Record<string, IconName> = {
   "locale-shots": "translate",
   "app-soak": "soak",
   "a11y-audit": "accessibility",
+  "model-convert": "convert",
+  "dataset-prep": "dataset",
+  serve: "serve",
+  shell: "terminal",
+  "push-latency": "bell",
+  "camera-eval": "camera",
+  "desktop-ui-test": "desktop",
 };
 
 /** The workload's name with its icon, for job tables. */

collector/dash/src/pages/JobDetail.tsx

@@ -1,3 +1,4 @@
+import { useEffect, useState } from "preact/hooks";
 import { useApi, type JobDetail as Detail, type ResultRow } from "../api.js";
 import { Workload } from "../icons.js";
 import { mutate, useMutation } from "../mutate.js";
@@ -218,6 +219,8 @@ export function JobDetail({ id }: { id: string }) {
               <JobActions job={j} onDone={state.reload} />
             </Panel>
 
+            <Mirror jobId={j.job_id} live={j.status === "claimed"} />
+
             <Panel title="Target">
               <div class="scroll">
                 <table>
@@ -353,3 +356,48 @@ export function JobDetail({ id }: { id: string }) {
     </>
   );
 }
+
+/**
+ * The device screen while a host job drives it.
+ *
+ * Polls only for whether a stream exists; the picture itself is an <img> on an
+ * MJPEG endpoint, so the browser does the streaming and this component holds no
+ * frames. Rendered only while something is actually producing, because an <img>
+ * that never paints looks exactly like a device with a black screen.
+ */
+function Mirror({ jobId, live }: { jobId: string; live: boolean }) {
+  const [streaming, setStreaming] = useState(false);
+  useEffect(() => {
+    if (!live) {
+      setStreaming(false);
+      return;
+    }
+    let alive = true;
+    const check = async () => {
+      try {
+        const r = await fetch(`/api/jobs/${encodeURIComponent(jobId)}/mirror/status`);
+        const b = (await r.json()) as { streaming?: boolean };
+        if (alive) setStreaming(!!b.streaming);
+      } catch {
+        if (alive) setStreaming(false);
+      }
+    };
+    void check();
+    const t = setInterval(() => void check(), 5_000);
+    return () => {
+      alive = false;
+      clearInterval(t);
+    };
+  }, [jobId, live]);
+
+  if (!streaming) return null;
+  return (
+    <Panel title="Live" aside={<span class="faint">device screen, not recorded</span>}>
+      <img
+        src={`/api/jobs/${encodeURIComponent(jobId)}/mirror`}
+        alt="the device screen while this job runs"
+        style="max-width:100%;border-radius:4px;display:block"
+      />
+    </Panel>
+  );
+}

collector/schemas/metrics.json

@@ -1,6 +1,7 @@
 {
   "schema": 1,
   "metrics": [
+    "app_state",
     "artifact_bytes",
     "avg_position",
     "avg_rating",
@@ -13,6 +14,7 @@
     "clock_offset_ms",
     "clusters",
     "connect_ms",
+    "convert_s",
     "crashes",
     "ctr_pct",
     "decode_tok_s",
@@ -25,23 +27,28 @@
     "elapsed_s",
     "energy_method",
     "energy_wh",
+    "exit_code",
     "images_per_s",
     "impressions",
     "issues_error",
     "issues_warn",
+    "items",
     "jank_pct",
     "launch_ms",
     "launch_state",
     "load_ms",
     "locales",
     "mem_method",
     "network_type",
+    "outputs",
     "p50_ms",
     "p95_ms",
     "pages_crawled",
     "peak_mem_mb",
     "prefill_tok_s",
     "pss_mb",
+    "push_delivered_pct",
+    "push_latency_ms",
     "recall_at_1",
     "recall_at_10",
     "recall_at_5",

collector/schemas/result.schema.json

@@ -23,6 +23,18 @@
         "app_ver": { "type": "string" }
       }
     },
+    "endpoint": {
+      "type": "string",
+      "description": "serve: where the model this job is hosting can be reached, announced so a dependent job can consume it. Bound to loopback or the tailnet, never 0.0.0.0 — an unauthenticated inference server on a laptop's hotel wifi is a different product to the one anybody asked for."
+    },
+    "model_out": {
+      "type": "object",
+      "description": "model-convert: what one conversion produced — format, quant, name, sha256 and size. One row per output, so a job converting to three formats is three rows and the sha of each is addressable by a dependent job."
+    },
+    "dataset": {
+      "type": "object",
+      "description": "dataset-prep: the prepared set's manifest summary, including its licence and source. A prepared set that loses its licence is one nobody can publish results from, which is why it rides on the row rather than only inside the zip."
+    },
     "metrics": {
       "type": "object",
       "description": "Workload metrics; memory is labeled, never laundered.\n\nEvery field here means one thing. Before the vision-eval and drain fields existed, those workloads reused the LLM slots — vision put top-1 accuracy in decode_tok_s, p50 latency in ttft_ms and images/sec in prefill_tok_s; drain put percent-per-hour in decode_tok_s — and top-5 and p95 had nowhere to go at all, so they were only ever in the hand-written eval report. Rows predating these fields still carry that convention, and the dashboard marks anything it had to infer that way. New runners must populate the named fields instead.",
@@ -87,6 +99,19 @@
 
       "locales": { "type": "number", "description": "locale-shots: how many locales the run covered" },
       "shots": { "type": "number", "description": "locale-shots and a11y-audit: screenshots captured" },
+
+      "convert_s": { "type": "number", "description": "model-convert: seconds for one output format" },
+      "outputs": { "type": "number", "description": "model-convert: how many artifacts the conversion produced" },
+      "items": { "type": "number", "description": "dataset-prep: items in the prepared manifest, after dedupe" },
+
+      "exit_code": { "type": "number", "description": "shell: the script's exit status. Named rather than folded into ok, because a script that exits 3 deliberately is not the same as one that could not start" },
+
+      "push_latency_ms": { "type": "number", "description": "push-latency: send to arrival, measured on the device that received it" },
+      "push_delivered_pct": { "type": "number", "description": "push-latency: fraction of sends that arrived at all, which on Android is the number that varies by vendor" },
+      "app_state": {
+        "enum": ["foreground", "background", "doze", "terminated"],
+        "description": "push-latency: what the app was doing when the push landed. A delivery latency without this is not comparable to anything, because doze is where the vendor forks differ"
+      },
         "drain_pct_per_h": { "type": "number", "description": "drain: battery percent consumed per hour" },
         "diff_pct": { "type": "number", "description": "web-shots: percent of pixels differing from the accepted baseline (100 when the dimensions changed)" },
 

collector/scripts/smoke.ts

@@ -2391,5 +2391,47 @@ runEvalChecks(check);
 // ever appear on real hardware.
 runDeviceParserChecks(check);
 
+// --- live mirror ---
+
+// Nothing here is persisted, so the assertions are about the states a viewer
+// can be in rather than about stored rows. The one that matters is the first:
+// a job with no producer must 404 rather than open a stream that never paints,
+// because an <img> that never paints is indistinguishable from a device with a
+// black screen.
+{
+  const MJOB = `smoke-mirror-${run}`;
+  const before = await json("GET", `/api/jobs/${MJOB}/mirror/status`);
+  check("no stream before a frame arrives", before.body?.streaming === false, JSON.stringify(before.body));
+
+  const viewer = await fetch(`${BASE}/api/jobs/${MJOB}/mirror`);
+  check("a viewer on a job with no producer gets 404", viewer.status === 404, `status=${viewer.status}`);
+  await viewer.body?.cancel().catch(() => {});
+
+  // The smallest thing that is still a JPEG.
+  const frame = Buffer.from(
+    "/9j/4AAQSkZJRgABAQEAYABgAAD/2wBDAAgGBgcGBQgHBwcJCQgKDBQNDAsLDBkSEw8UHRofHh0aHBwgJC4nICIsIxwcKDcpLDAxNDQ0Hyc5PTgyPC4zNDL/wAALCAABAAEBAREA/8QAFAABAAAAAAAAAAAAAAAAAAAACf/EABQQAQAAAAAAAAAAAAAAAAAAAAD/2gAIAQEAAD8AKp//2Q==",
+    "base64",
+  );
+  const post = await fetch(`${BASE}/api/jobs/${MJOB}/mirror`, {
+    method: "POST", headers: { "content-type": "image/jpeg" }, body: frame,
+  });
+  check("a frame is accepted", post.status === 204, `status=${post.status}`);
+
+  const after = await json("GET", `/api/jobs/${MJOB}/mirror/status`);
+  check("the stream exists once a frame has arrived", after.body?.streaming === true, JSON.stringify(after.body));
+
+  // A frame past the cap is refused rather than truncated: half a JPEG renders
+  // as a corrupt image and reads as a broken device.
+  const huge = await fetch(`${BASE}/api/jobs/${MJOB}/mirror`, {
+    method: "POST", headers: { "content-type": "image/jpeg" }, body: Buffer.alloc(3 * 1024 * 1024),
+  });
+  check("an oversized frame is refused", huge.status === 413, `status=${huge.status}`);
+
+  const empty = await fetch(`${BASE}/api/jobs/${MJOB}/mirror`, {
+    method: "POST", headers: { "content-type": "image/jpeg" }, body: Buffer.alloc(0),
+  });
+  check("an empty frame is refused", empty.status === 400, `status=${empty.status}`);
+}
+
 console.log(failures === 0 ? "\nsmoke: ALL PASS" : `\nsmoke: ${failures} FAILURE(S)`);
 process.exit(failures === 0 ? 0 : 1);

collector/src/api/index.ts

@@ -9,6 +9,7 @@
 //     predict the shape
 import type { FastifyInstance } from "fastify";
 import { registerDevices } from "./devices.js";
+import { registerMirror } from "./mirror.js";
 import { registerEnroll } from "./enroll.js";
 import { registerEvals } from "./evals.js";
 import { registerJobs } from "./jobs.js";
@@ -33,6 +34,7 @@ type MatchingDevices = (
 export function registerApi(app: FastifyInstance, announce: Announce, matchingDevices: MatchingDevices) {
   registerOverview(app);
   registerDevices(app);
+  registerMirror(app);
   registerEnroll(app);
   registerJobs(app);
   registerResults(app);