Release v0.2.0

.github/workflows/ci.yml

@@ -0,0 +1,133 @@
+name: ci
+
+# The entry point for every push and pull request, and the only workflow whose
+# name belongs in branch protection.
+#
+# Why this exists at all: the four component suites are path-filtered, because
+# a change to a phone runner has no business spinning up a Node matrix or a
+# macOS runner. But GitHub reports a required check that never ran as *pending
+# forever*, so a path-filtered workflow can never itself be a required check --
+# an iOS-only pull request would sit blocked on the collector's suite until
+# somebody force-merged it.
+#
+# So the suites became reusable workflows, called from here behind a change
+# detector, and `ci-ok` at the bottom always runs and reports. It is the one
+# check to require. A suite that was skipped because nothing it watches changed
+# counts as success; a suite that actually failed fails the gate.
+
+on:
+  push:
+    branches: [main, develop]
+  pull_request:
+
+# A second push to a pull request makes the first one's answer irrelevant.
+concurrency:
+  group: ci-${{ github.workflow }}-${{ github.head_ref || github.ref }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+
+jobs:
+  changes:
+    name: what changed
+    runs-on: ubuntu-latest
+    outputs:
+      collector: ${{ steps.filter.outputs.collector }}
+      android: ${{ steps.filter.outputs.android }}
+      ios: ${{ steps.filter.outputs.ios }}
+      machine: ${{ steps.filter.outputs.machine }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          # The diff below needs both ends of the range in this checkout, and
+          # the default shallow clone has neither.
+          fetch-depth: 0
+
+      - id: filter
+        env:
+          BASE: ${{ github.event_name == 'pull_request' && github.event.pull_request.base.sha || github.event.before }}
+        run: |
+          set -euo pipefail
+
+          # An absent base, the all-zero sha of a branch's first push, or a sha
+          # this checkout does not have all mean the same thing: the range is
+          # unknowable. Run everything. Guessing wrong in the other direction
+          # skips a suite silently, which is the failure this file exists to
+          # prevent.
+          if [ -z "${BASE:-}" ] \
+            || [ "$BASE" = "0000000000000000000000000000000000000000" ] \
+            || ! git cat-file -e "${BASE}^{commit}" 2>/dev/null; then
+            echo "base unusable ('${BASE:-}') -- running every suite"
+            CHANGED="__ALL__"
+          else
+            CHANGED="$(git diff --name-only "$BASE" HEAD)"
+            echo "changed files:"
+            printf '%s\n' "$CHANGED" | sed 's/^/  /'
+          fi
+
+          match() {
+            if [ "$CHANGED" = "__ALL__" ]; then
+              echo true
+            elif printf '%s\n' "$CHANGED" | grep -qE "$1"; then
+              echo true
+            else
+              echo false
+            fi
+          }
+
+          # ci.yml is in every pattern on purpose: a change to the gate itself
+          # should be proved against all four suites, not merged on the word of
+          # a detector it just edited.
+          {
+            echo "collector=$(match '^(collector/|\.github/workflows/(collector|ci)\.yml)')"
+            echo "android=$(match '^(runner-android/|\.github/workflows/(runner-android|ci)\.yml)')"
+            echo "ios=$(match '^(runner-ios/|\.github/workflows/(runner-ios|ci)\.yml)')"
+            echo "machine=$(match '^(runner-machine/|\.github/workflows/(runner-machine|ci)\.yml)')"
+          } | tee -a "$GITHUB_OUTPUT"
+
+  collector:
+    needs: changes
+    if: needs.changes.outputs.collector == 'true'
+    uses: ./.github/workflows/collector.yml
+
+  runner-android:
+    needs: changes
+    if: needs.changes.outputs.android == 'true'
+    uses: ./.github/workflows/runner-android.yml
+
+  runner-ios:
+    needs: changes
+    if: needs.changes.outputs.ios == 'true'
+    uses: ./.github/workflows/runner-ios.yml
+
+  runner-machine:
+    needs: changes
+    if: needs.changes.outputs.machine == 'true'
+    uses: ./.github/workflows/runner-machine.yml
+
+  ci-ok:
+    name: ci-ok
+    needs: [changes, collector, runner-android, runner-ios, runner-machine]
+    # Without this the gate inherits the default "only if every dependency
+    # succeeded", which makes a skipped suite skip the gate -- and a skipped
+    # required check never reports, which is the exact bug being fixed.
+    if: always()
+    runs-on: ubuntu-latest
+    steps:
+      - name: Every suite green or skipped
+        env:
+          RESULTS: ${{ join(needs.*.result, ' ') }}
+        run: |
+          set -euo pipefail
+          echo "suite results: $RESULTS"
+          for result in $RESULTS; do
+            case "$result" in
+              success|skipped) ;;
+              *)
+                echo "::error::a suite reported '$result'"
+                exit 1
+                ;;
+            esac
+          done
+          echo "all suites green or skipped"

.github/workflows/collector.yml

@@ -1,18 +1,15 @@
 name: collector
 
-# Path-filtered: a change to a phone runner should not spin up a Node matrix.
+# Called by ci.yml, which owns the change detection and the `ci-ok` check
+# that branch protection requires. Nothing else triggers this file.
 #
-# The trap this creates is worth knowing about before you add a required check
-# in branch protection. A filtered workflow does not run on a PR that touches
-# nothing it watches, and GitHub reports a required check that never ran as
-# pending forever, so an iOS-only PR would block on the collector's suite. Keep
-# these advisory, or add a merge-gate job that always runs.
+# This suite is still only run when something under collector/ changed -- a
+# change to a phone runner has no business spinning up a Node matrix. What
+# moved to ci.yml is the consequence: a path-filtered workflow can never
+# itself be a required check, because GitHub reports a check that never ran
+# as pending forever. The gate there always runs and reports for all four.
 on:
-  push:
-    branches: [main, develop]
-    paths: ["collector/**", ".github/workflows/collector.yml"]
-  pull_request:
-    paths: ["collector/**", ".github/workflows/collector.yml"]
+  workflow_call:
 
 defaults:
   run:

.github/workflows/runner-android.yml

@@ -1,11 +1,9 @@
 name: runner-android
 
+# Called by ci.yml, which owns the change detection and the `ci-ok` check
+# that branch protection requires. Nothing else triggers this file.
 on:
-  push:
-    branches: [main, develop]
-    paths: ["runner-android/**", ".github/workflows/runner-android.yml"]
-  pull_request:
-    paths: ["runner-android/**", ".github/workflows/runner-android.yml"]
+  workflow_call:
 
 defaults:
   run:

.github/workflows/runner-ios.yml

@@ -1,11 +1,9 @@
 name: runner-ios
 
+# Called by ci.yml, which owns the change detection and the `ci-ok` check
+# that branch protection requires. Nothing else triggers this file.
 on:
-  push:
-    branches: [main, develop]
-    paths: ["runner-ios/**", ".github/workflows/runner-ios.yml"]
-  pull_request:
-    paths: ["runner-ios/**", ".github/workflows/runner-ios.yml"]
+  workflow_call:
 
 defaults:
   run:

.github/workflows/runner-machine.yml

@@ -1,11 +1,9 @@
 name: runner-machine
 
+# Called by ci.yml, which owns the change detection and the `ci-ok` check
+# that branch protection requires. Nothing else triggers this file.
 on:
-  push:
-    branches: [main, develop]
-    paths: ["runner-machine/**", ".github/workflows/runner-machine.yml"]
-  pull_request:
-    paths: ["runner-machine/**", ".github/workflows/runner-machine.yml"]
+  workflow_call:
 
 defaults:
   run:

.gitmodules

@@ -0,0 +1,10 @@
+# At the repository root because that is the only place git reads it from.
+#
+# The migration to a mono repo moved the Android runner into a subdirectory and
+# its .gitmodules moved with it, where git never looks — so the gitlink at
+# runner-android/third_party/llama.cpp survived with no URL registered against
+# it, and `git submodule update --init` in a fresh clone silently found nothing
+# to do. The path below is the only line that had to change.
+[submodule "runner-android/third_party/llama.cpp"]
+	path = runner-android/third_party/llama.cpp
+	url = https://github.com/ggml-org/llama.cpp

CHANGELOG.md

@@ -0,0 +1,121 @@
+# Changelog
+
+Notable changes to Fleet Runner. The format follows
+[Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and the project
+follows [semantic versioning](https://semver.org/spec/v2.0.0.html).
+
+One version covers all four components. They ship no shared code, only a JSON
+protocol — and that protocol is the thing that changes, so it is the thing the
+version tracks. A release note says which components moved. The wire protocol
+itself is `"schema": 1` and stays there until something breaks compatibility,
+independently of the version below.
+
+## [Unreleased]
+
+## [0.2.0] — 2026-09-05
+
+The first release from the mono repo, covering two waves of work plus the
+migration itself.
+
+### Added
+
+- **Job chains.** `depends_on` makes build → install → ui-test a single
+  enqueue. A promoted spec can carry `${jobs.<id>.artifact}` and
+  `${jobs.<id>.metrics.<key>}`, substituted whole so a metric stays a number.
+  Chains parked on a dependency are counted as `waiting`, not `queued`, so a
+  healthy pipeline does not read as a backlog.
+- **Preemption.** A twelve-hour `drain` steps aside for a two-minute
+  `install`, checkpoints, and is requeued with `params.resume_from`. Stepping
+  aside never counts against `max_attempts`, because it is not a failure.
+- **The `build` workload**, on the machine agent: check out a ref, build it,
+  publish the product to the artifact store, and let a later job resolve
+  `"sha256": "latest"` to it.
+- **Nine more workloads.** `speech-eval` and `embed-eval` on the phones;
+  `vantage` from wherever an agent sits; `locale-shots`, `app-soak` and
+  `a11y-audit` on the host; `self-check` on the hosts themselves;
+  `model-convert`, `dataset-prep` and `serve` on the machine agent.
+- **The `shell` workload, gated by a locally pinned allowlist.** `POST /jobs`
+  is unauthenticated by design, so the one workload that runs arbitrary code
+  declares its capability only when the owner has pinned a sha256 by hand in a
+  local file. A machine that has pinned nothing is never offered the job at
+  all, rather than claiming it and refusing afterwards.
+- **Energy measurement**, reported in watt-hours at the wall above a measured
+  idle baseline, or not reported at all. `energy_method` is declared in
+  configuration and never inferred; a `plug-shared` pool reports the pool's
+  draw and refuses to divide it between devices.
+- **An Evals page and `/api/evals`**, pivoting eval sets by model and device
+  with a Markdown export. It counts and lists the rows it excluded, naming the
+  metrics each one carries.
+- **A live job mirror.** MJPEG frames in a bounded in-memory ring, dropped when
+  the job ends, so an `<img>` can answer "what is it doing now" without the
+  permanent storage cost of a video stream.
+- **Real iPhone screen capture**, via a `WKWebView` inside the runner app. The
+  profile is named `webkit` rather than `safari`, because a `WKWebView` has no
+  reader mode and no content blockers.
+- **A status screen and launcher icon for the Android runner**, including the
+  pre-API-26 icon fallback that was missing.
+- `CHANGELOG.md`, and a `ci-ok` merge gate that always reports.
+
+### Changed
+
+- **Four repositories became one.** `fleet-collector`, `fleet-runner-ios`,
+  `fleet-runner-android` and `fleet-runner-machine` are now `collector/`,
+  `runner-ios/`, `runner-android/` and `runner-machine/`. Every commit came
+  across with `git subtree`, so history and authorship survive and
+  `git log --follow` works through the move. The old repositories are archived
+  and still resolve.
+- **The component suites are reusable workflows** called by a single `ci`
+  workflow, which decides which of them need to run. A path-filtered workflow
+  can never be a required check, because GitHub reports a check that never ran
+  as pending forever; `ci-ok` always runs and reports for all four.
+- The host executor's handlers began moving into a directory per workload,
+  starting with `install`, behind a loader that falls back to the existing
+  chain for everything not yet moved.
+
+### Fixed
+
+- **Swift's `convertToSnakeCase` splits on capitals, and a digit is not one**,
+  so `recallAt1` encoded as `recall_at1` — one underscore short of the declared
+  metric name, which meant it silently never arrived. Explicit `CodingKeys` now.
+- **A real-device soak ran `simctl` against a physical iPhone's UDID** and
+  reported the process dead at every check.
+- **The mono-repo migration left `.gitmodules` inside `runner-android/`**,
+  where git never looks, so the gitlink for llama.cpp survived with no URL
+  registered against it and `git submodule update --init` in a fresh clone
+  silently found nothing to do.
+- Four machine workloads referenced metric names that were never mirrored into
+  the agent's own protocol type, so they did not compile; the same four were
+  never dispatched or capability-gated, so they were dead code.
+
+### Known gaps
+
+Stated rather than implied. `push-latency` needs FCM and APNs credentials;
+`camera-eval` needs a physical rig; `desktop-ui-test` has no desktop app to
+test against. `model-convert`, `dataset-prep` and `serve` have never run
+against a real toolchain — none of the converters resolves on the development
+machine, and the capability probes correctly decline to declare them. Only
+`install` has moved into the executor's plugin layout.
+
+## [0.1.0] — 2026-09-03
+
+The first public release, when the project was still four repositories.
+
+### Added
+
+- **The collector**: device registry, job queue with leases, artifact store
+  addressed by content hash, results database, scheduler, alert engine, and a
+  dashboard with live updates over SSE. Node, Fastify and SQLite in WAL mode,
+  with no broker and no cloud.
+- **Runner agents for Android, iOS and desktop**, sharing a JSON protocol and
+  no code. Their synthetic SHA-256 benchmark is identical token for token on
+  every platform, which is what makes a 2019 phone's number comparable to a
+  laptop's.
+- **The first real payload**: an on-device plant-identification evaluation
+  across both platforms, at 77% top-1 and 7 ms per image on a phone.
+- MIT licensing, third-party notices, CI on every component, and a `npm test`
+  that starts a throwaway collector on a spare port so it never touches a live
+  fleet's history.
+
+[Unreleased]: https://github.com/addisdev/fleet-runner/compare/v0.2.0...HEAD
+[0.2.0]: https://github.com/addisdev/fleet-runner/compare/v0.1.0...v0.2.0
+[0.1.0]: https://github.com/addisdev/fleet-runner/releases/tag/v0.1.0

collector/README.md

@@ -89,6 +89,12 @@ so it never touches a real fleet's history.
 | `thermal` | device | The same benchmark run back to back for a quarter hour, so the answer is a curve: does the cold number survive the device getting warm |
 | `cold-start` | host | Launch the installed build from cold, warm and hot; p50 and p95 per state, per device |
 | `drain` / `soak` | host | Battery curve under a replayed GPX track; and whether a runner is still alive hours later |
+| `build` | machine | Check out a ref, build it, publish the product as an artifact the next `install` can name |
+| `speech-eval` / `embed-eval` | device | Word error rate and real-time factor for on-device transcription; recall-at-k and throughput for on-device embeddings |
+| `vantage` | device, machine | DNS, connect, TLS and TTFB to your own sites from wherever each agent is, so "the site is slow" and "my wifi is slow" stop being the same answer |
+| `locale-shots` | host | A screenshot flow under every locale, including RTL, bundled as a store-ready contact sheet |
+| `app-soak` / `a11y-audit` | host | Memory, jank and crashes over hours; and the accessibility tree diffed against a baseline at the largest dynamic type |
+| `self-check` | host, machine | The fleet inspects its own hosts: disk, tool versions, clock drift, agents actually loaded |
 | `archive` / `digest` | host | Pull store reviews and Search Console data; then have the shelf summarize its own reviews using its own models |
 
 Full reference, including every parameter and the failure modes worth knowing:

collector/dash/src/api.ts

@@ -128,7 +128,11 @@ export type Job = {
   job_id: string;
   workload: string;
   executor: "device" | "host";
-  status: "queued" | "claimed" | "done" | "failed" | "cancelled";
+  status: "waiting" | "queued" | "claimed" | "done" | "failed" | "cancelled";
+  /** Job ids this one waits for; it stays `waiting` until they are all done. */
+  depends_on?: string[] | null;
+  /** Whether a higher-priority job may ask this one to checkpoint and step aside. */
+  preemptible?: boolean;
   claimed_by: string | null;
   attempts: number;
   max_attempts: number;