diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 826759c..98a62f5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -105,6 +105,42 @@ jobs: - uses: actions/checkout@v4 - run: node scripts/version.mjs check + # Also unconditional, and for the same reason as `version`: the numbers it + # checks are written in README.md and docs/, so a docs-only pull request is + # exactly where a stale one arrives, and no component suite runs for that. + repo-scripts: + name: repo scripts + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Unit tests for scripts/ + run: node --test scripts/*.test.mjs + - name: Documented counts match the code + run: node scripts/check-counts.mjs + + # A release pull request -- a `release/*` branch, or `develop` into `main` -- + # must mention every pull request merged into develop since the last tag. + # Every other pull request skips this, and skipped counts as green below. + changelog: + name: changelog coverage + if: >- + github.event_name == 'pull_request' && ( + startsWith(github.head_ref, 'release/') || + (github.head_ref == 'develop' && github.base_ref == 'main')) + runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: read + steps: + - uses: actions/checkout@v4 + with: + # The tags and the whole of develop's history since the last one. + fetch-depth: 0 + - name: Every merged pull request is in the changelog + env: + GH_TOKEN: ${{ github.token }} + run: node scripts/changelog-coverage.mjs --head "${{ github.event.pull_request.head.sha }}" + collector: needs: changes if: needs.changes.outputs.collector == 'true' @@ -142,7 +178,7 @@ jobs: ci-ok: name: ci-ok - needs: [changes, version, collector, runner-android, runner-ios, runner-machine, fleet, runner-roku, desktop] + needs: [changes, version, repo-scripts, changelog, collector, runner-android, runner-ios, runner-machine, fleet, runner-roku, desktop] # Without this the gate inherits the default "only if every dependency # succeeded", which makes a skipped suite skip the gate -- and a skipped # required check never reports, which is the exact bug being fixed. diff --git a/.github/workflows/desktop.yml b/.github/workflows/desktop.yml index ed6870c..2945167 100644 --- a/.github/workflows/desktop.yml +++ b/.github/workflows/desktop.yml @@ -27,3 +27,66 @@ jobs: ./desktop/stage-sidecar.sh - working-directory: desktop/src-tauri run: cargo check --all-targets + + launch: + # The first time the app ever ran was this job. `cargo check` proves the + # Tauri calls exist; this proves the binary starts, spawns `fleet up` as its + # sidecar, stays up, and that its settings window loads and answers over + # IPC. Linux rather than macOS because that is where Tauri's WebDriver + # support exists (tauri-driver wraps WebKitWebDriver; there is no macOS + # driver), and because Xvfb gives it a display without a logged-in desktop. + # What it cannot prove is anything macOS-specific: the tray in the menu bar, + # the Local Network grant, signing. See desktop/README.md. + name: launch smoke + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: "22.13" + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + with: + workspaces: desktop/src-tauri + key: linux-launch + # Tauri 2's Linux prerequisites (webkit2gtk 4.1, appindicator for the + # tray), plus what the smoke itself uses: Xvfb for a display, dbus for a + # session bus, WebKitWebDriver for tauri-driver, ImageMagick to capture + # the screen. + - name: Linux dependencies + run: | + sudo apt-get update + sudo apt-get install -y --no-install-recommends \ + libwebkit2gtk-4.1-dev build-essential curl wget file libxdo-dev \ + libssl-dev libayatana-appindicator3-dev librsvg2-dev \ + xvfb x11-utils dbus-x11 webkit2gtk-driver imagemagick jq + - name: Stage the fleet sidecar + run: | + npm --prefix collector ci + npm --prefix fleet ci + ./desktop/stage-sidecar.sh + - name: Build the app + working-directory: desktop/src-tauri + # custom-protocol is what `cargo tauri build` turns on: the frontend is + # embedded and served from the binary, as it is in a release, rather + # than expected from a dev server. + run: cargo build --features custom-protocol + - name: tauri-driver + run: cargo install tauri-driver --locked + - name: Launch it + run: ./desktop/ci/launch-smoke.sh + env: + OUT: ${{ runner.temp }}/desktop-smoke + - if: always() + name: What it printed + run: | + for f in "$RUNNER_TEMP"/desktop-smoke/*.log "$RUNNER_TEMP"/desktop-smoke/fleet-home/logs/*.log; do + [ -f "$f" ] || continue + echo "== $f"; tail -80 "$f" + done + - if: always() + uses: actions/upload-artifact@v4 + with: + name: desktop-launch-smoke + path: ${{ runner.temp }}/desktop-smoke + if-no-files-found: warn diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index dd68603..59aa5a3 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -41,6 +41,16 @@ jobs: - run: pip install -r requirements-docs.txt + # Diagrams are hand-placed figures in docs/figures/, rendered to PNG + # (docs/brand.md says why). mkdocs.yml no longer has a mermaid fence, so + # one would build as a plain code block rather than fail; this makes it fail. + - name: No mermaid + run: | + if grep -rn --include='*.md' '^ *```mermaid' docs/; then + echo "::error::mermaid diagram under docs/: draw it as a figure in docs/figures/ instead (docs/brand.md)" + exit 1 + fi + # The cookbook includes collector/examples/jobs/*.json verbatim via the # snippets extension, so a missing or renamed example fails the build # here as well as failing the collector's own suite. diff --git a/.github/workflows/fleet.yml b/.github/workflows/fleet.yml index 063754c..ef55942 100644 --- a/.github/workflows/fleet.yml +++ b/.github/workflows/fleet.yml @@ -89,8 +89,21 @@ jobs: # run it the way the docs say, and drive it through the CLI from outside: # a container whose port is published but whose collector never answers is # the failure a `docker build` alone would call green. - name: container smoke - runs-on: ubuntu-latest + # + # The arm64 row builds natively on GitHub's arm64 runner rather than under + # QEMU on the amd64 one. The release publishes linux/arm64 from a QEMU + # build, and an image that only ever ran emulated is the one a Raspberry Pi + # owner would be first to start for real. + name: container smoke${{ matrix.suffix }} + runs-on: ${{ matrix.runner }} + strategy: + fail-fast: false + matrix: + include: + - runner: ubuntu-latest + suffix: "" + - runner: ubuntu-24.04-arm + suffix: " (arm64)" steps: - uses: actions/checkout@v4 - uses: actions/setup-node@v4 @@ -99,13 +112,21 @@ jobs: cache: npm cache-dependency-path: fleet/package-lock.json - working-directory: . - run: docker build -f fleet/Dockerfile -t fleet:ci . + run: | + docker build -f fleet/Dockerfile -t fleet:ci . + docker image inspect fleet:ci --format 'built for {{.Os}}/{{.Architecture}}' + - working-directory: . run: docker run -d --name fleet -p 8788:8788 fleet:ci up --role brain,agent - name: The collector answers, and the machine agent registers shell: bash + env: + EXPECT_ARCH: ${{ matrix.runner == 'ubuntu-24.04-arm' && 'aarch64' || 'x86_64' }} run: | set -euo pipefail + arch=$(docker exec fleet uname -m) + echo "container is running on $arch" + [ "$arch" = "$EXPECT_ARCH" ] || { echo "::error::expected $EXPECT_ARCH, the container says $arch"; exit 1; } for i in $(seq 1 60); do curl -fsS http://127.0.0.1:8788/api/v1/health && break sleep 2 @@ -165,3 +186,138 @@ jobs: run: | export XDG_RUNTIME_DIR="/run/user/$(id -u)" journalctl --user -u fleet.service --no-pager | tail -100 || true + + launchd: + # The twin of the systemd smoke for macOS. `fleet service install` writes a + # user LaunchAgent and bootstraps it into the gui/ domain; GitHub's + # macOS runners run their jobs inside a logged-in GUI session, so that + # domain exists here the way it does on a desk. The first step says so + # rather than assuming it, because a runner without one would make every + # later failure look like a fleet bug. + name: launchd smoke + runs-on: macos-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: "22.13" + cache: npm + cache-dependency-path: fleet/package-lock.json + - run: npm ci && npm --prefix ../collector ci && npm run build + - name: The runner has a GUI launchd domain + shell: bash + run: | + set -euo pipefail + echo "uid $(id -u), user $(id -un)" + launchctl print "gui/$(id -u)" | head -15 + - name: Install as a LaunchAgent, it serves, and its agent registers + shell: bash + run: | + set -euo pipefail + mkdir -p "$HOME/.fleet" + cp -R dist/. "$HOME/.fleet/" + fleet() { node "$HOME/.fleet/bin/fleet.mjs" "$@"; } + plist="$HOME/Library/LaunchAgents/com.addisdev.fleet.plist" + fleet service install + plutil -lint "$plist" + launchctl print "gui/$(id -u)/com.addisdev.fleet" | head -20 + for i in $(seq 1 60); do + curl -fsS http://127.0.0.1:8788/api/v1/health && break + sleep 2 + done + curl -fsS http://127.0.0.1:8788/api/v1/health >/dev/null + n=0 + for i in $(seq 1 30); do + n=$(fleet devices --url http://127.0.0.1:8788 --json | node -e 'let s="";process.stdin.on("data",d=>s+=d).on("end",()=>console.log(JSON.parse(s).items.length))') + [ "$n" -gt 0 ] && break + sleep 2 + done + [ "$n" -gt 0 ] || { echo "::error::the agent under launchd never registered"; exit 1; } + fleet devices --url http://127.0.0.1:8788 + fleet service status + # Reinstalling is bootout-then-bootstrap, the race service.ts retries. + # Run it once here so the retry is exercised rather than trusted. + fleet service install + for i in $(seq 1 60); do + curl -fsS http://127.0.0.1:8788/api/v1/health >/dev/null 2>&1 && break + sleep 2 + done + curl -fsS http://127.0.0.1:8788/api/v1/health >/dev/null + fleet service uninstall + sleep 5 + if curl -fsS http://127.0.0.1:8788/api/v1/health 2>/dev/null; then + echo "::error::the collector is still answering after uninstall" + exit 1 + fi + [ ! -e "$plist" ] || { echo "::error::$plist is still there after uninstall"; exit 1; } + if launchctl print "gui/$(id -u)/com.addisdev.fleet" >/dev/null 2>&1; then + echo "::error::launchd still has com.addisdev.fleet loaded after uninstall" + exit 1 + fi + if fleet service status; then + echo "::error::fleet service status exited 0 after uninstall" + exit 1 + fi + - if: failure() + shell: bash + run: | + tail -100 "$HOME/.fleet/logs/fleet.log" || true + for f in "$HOME"/.fleet/logs/*.log; do echo "== $f"; tail -40 "$f"; done || true + + windows-task: + # `fleet service install` on Windows creates a scheduled task that runs + # `fleet up` at logon and starts it at once. Same lifecycle as the systemd + # and launchd smokes: install, it serves, its agent registers, status says + # so, uninstall, and nothing is left answering. + name: windows task smoke + runs-on: windows-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: "22.13" + cache: npm + cache-dependency-path: fleet/package-lock.json + - shell: bash + run: npm ci && npm --prefix ../collector ci && npm run build + - name: Install as a scheduled task, it serves, and its agent registers + shell: bash + run: | + set -euo pipefail + home="$(cygpath -u "$USERPROFILE")/.fleet" + mkdir -p "$home" + cp -R dist/. "$home/" + fleet() { node "$home/bin/fleet.mjs" "$@"; } + fleet service install + schtasks //Query //TN FleetRunner //FO LIST //V + for i in $(seq 1 60); do + curl -fsS http://127.0.0.1:8788/api/v1/health && break + sleep 2 + done + curl -fsS http://127.0.0.1:8788/api/v1/health >/dev/null + n=0 + for i in $(seq 1 30); do + n=$(fleet devices --url http://127.0.0.1:8788 --json | node -e 'let s="";process.stdin.on("data",d=>s+=d).on("end",()=>console.log(JSON.parse(s).items.length))') + [ "$n" -gt 0 ] && break + sleep 2 + done + [ "$n" -gt 0 ] || { echo "::error::the agent under the scheduled task never registered"; exit 1; } + fleet devices --url http://127.0.0.1:8788 + fleet service status + fleet service uninstall + sleep 5 + if curl -fsS http://127.0.0.1:8788/api/v1/health 2>/dev/null; then + echo "::error::the collector is still answering after uninstall" + exit 1 + fi + if schtasks //Query //TN FleetRunner >/dev/null 2>&1; then + echo "::error::the FleetRunner task is still there after uninstall" + exit 1 + fi + - if: failure() + shell: bash + run: | + home="$(cygpath -u "$USERPROFILE")/.fleet" + schtasks //Query //TN FleetRunner //FO LIST //V || true + for f in "$home"/logs/*.log; do echo "== $f"; tail -60 "$f"; done || true + tasklist //FI "IMAGENAME eq node.exe" || true diff --git a/AGENTS.md b/AGENTS.md index a1c68e6..4cc206a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -9,11 +9,14 @@ from an agent, see [`skills/fleet-runner/SKILL.md`](skills/fleet-runner/SKILL.md | Path | What | |---|---| | `collector/` | The brain: Fastify + `node:sqlite`, the queue, the dashboard (`dash/`), the host executor | +| `collector/src/workloads/explore/` | Exploratory QA: a vision model drives an app from mission cards (`collector/examples/missions/`); see `docs/explore/internals.md` | | `collector/src/api/v1/` | The control API. `routes.ts` is the single route table: it registers the routes, generates `openapi.json` and the typed client | -| `collector/src/client/` | The generated client (`api.gen.ts`) and its transport. The CLI and the MCP server use nothing else | +| `collector/src/client/` | The generated client (`api.gen.ts`) and its transport. The CLI and the MCP server use nothing else; the dashboard (`dash/src/client.ts`) uses it for everything `/api/v1` offers, and `scripts/check-dash-requests.ts` fails the suite if it calls a deprecated route | | `fleet/` | The `fleet` / `fleetr` command: runs a brain or a runner, plus the work commands, `fleet mcp` and the remote gateway | -| `runner-machine/`, `runner-android/`, `runner-ios/`, `runner-roku/`, `desktop/` | Runners. They speak the runner protocol (`/devices/register`, `/devices/:id/next-job`, `/results`), not `/api/v1` | +| `desktop/` | A Tauri menu-bar app around `fleet`. A front door, not a runner; compiled in CI on macOS and launched in CI on Linux (`desktop/ci/launch-smoke.sh`), never run on a Mac | +| `runner-machine/`, `runner-android/`, `runner-ios/`, `runner-roku/`, `collector/runner-web/` | Runners. They speak the runner protocol (`/devices/register`, `/devices/:id/next-job`, `/results`), not `/api/v1` | | `skills/fleet-runner/` | The agent skill. Its tables are generated | +| `bug-garden/` | An Android fixture app with twenty planted defects, for scoring exploratory QA | | `docs/` | The mkdocs site. Built with `--strict`, so a broken link fails CI | ## Suites diff --git a/CHANGELOG.md b/CHANGELOG.md index 4506e4e..c12ef72 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,20 +11,384 @@ itself is `"schema": 1` and stays there until something breaks compatibility, independently of the version below. Since 0.5.0 that is enforced rather than asserted: `VERSION` at the repository -root is the source, `scripts/version.mjs` writes it into all eleven places, and +root is the source, `scripts/version.mjs` writes it into all thirteen places, and a CI job fails when they disagree. ## [Unreleased] +## [0.9.0] — 2026-10-15 + +Green that means something. A job that ran nothing is `skipped`, not `done`, +and a schedule that keeps skipping says so; every failure carries its reason. +Exploratory QA's findings reach the code: they are in `/api/v1`, `fleetr` and +MCP, and a finding marked Real and then Fixed becomes a regression test in the +app's nightly. The overnight explore jobs run short instead of not at all and +skip when their build is missing. The older routes make the same scope checks +as `/api/v1`, a pull request's build no longer becomes `latest`, and CI now +runs `fleet service` under launchd and Windows, the arm64 image and the +desktop app, each for the first time. + +### Added + +- **A pull request's build no longer becomes the nightly's `latest`.** + `ci-upload` always tagged an upload with its app, and `ci-enqueue` did + whenever the spec named one, so the next nightly asking for `latest` tested + an unmerged branch. An upload can now carry `x-artifact-channel: pr`, which + `"sha256": "latest"` never resolves to; both scripts send it when + `GITHUB_EVENT_NAME` is `pull_request` (or with `--channel pr`). The pull + request's own job still names its build by hash. Builds uploaded before this + count as merged builds, and the same bytes republished from main come off the + channel. The `/api/v1` artifact document shows it as `channel: pr|main` + since #96. +- **`FLEET_PEER_TOKENS`: the brain switcher can read a peer with tokens on.** + The peers proxy sent no token, so a peer running `FLEET_AUTH=tokens` + answered 401 through it. Each peer's URL now maps to a `read` token that + peer issued, sent only to that peer. ([#92](https://github.com/addisdev/fleet-runner/pull/92)) +- **An explore night admitted late runs short instead of not at all.** + `scripts/explore-night.ts` now knows when the night ends: `--until` + (default 07:30, ultra's night-queue stop) or the job's own `max_minutes`, + read from the night queue's manifest for `$NIGHT_JOB`, whichever is first. + When the specs ask for more than is left, each gets a proportional share, + the workload runs its highest-priority cards in it, and the result says so + on an `ok` row: `short night: ran 3 of 7 cards: 40 minutes left of the 60 + asked`. Under 15 minutes (`--min-minutes`) the night is posted as a skip + rather than started. New metrics `explore_cards_planned` and + `explore_minutes` record what the device was given. The night queue's own + admission wait (120 minutes behind a CI job on 10-14) is outside this + repository and unchanged. Not yet seen on a real night. ([#93](https://github.com/addisdev/fleet-runner/pull/93)) +- **`explore-night.ts --dry-run`** opens the tunnel, runs the build and time + checks against the live brain and prints the plan, without enqueueing, + posting, warming or booting anything. ([#93](https://github.com/addisdev/fleet-runner/pull/93)) +- **A job that ran nothing now has a status of its own: `skipped`.** A runner + that could not run a job says `ok: true` with an error beginning `skipped:`, + and the queue closed that as `done`, so the job list, the status counts and + the digest counted it as a pass: `nightly-tvloop-hardware` and three explore + jobs a night "passed" in two seconds for two weeks while testing nothing. + The queue now closes it as `skipped`, with the reason in `last_error`; a host + row of plain `ok: true` over device rows that all skipped counts the same, + and a fan-out family where no child ran is `skipped`. `skipped` is accepted + by the `status` filter of `GET /api/jobs` and `GET /api/v1/jobs`, counted in + the status facets, `GET /metrics` and the Overview (`skipped 24h`, beside + done rather than in it), drawn as a grey chip on the dashboard, and given its + own line in the morning digest. The v1 summary's verdict, `fleetr`'s exit 4 + and the MCP verdict were already `skipped` and are unchanged. A database + opened by this release is migrated once: every `done` job with a `skipped:` + result is re-labelled by the same rule. Result rows are not touched. ([#94](https://github.com/addisdev/fleet-runner/pull/94)) +- **A job waiting on a skipped job skips too.** A skipped build produced no + artifact, so the install behind it had nothing to install; it used to be + promoted and run against whatever was already on the phone. It now skips + with `last_error` `dependency skipped`, the whole way down the chain, + and a job enqueued behind an already-skipped one is skipped on arrival. A + failed dependency still fails its waiters, and wins over a skipped one. ([#94](https://github.com/addisdev/fleet-runner/pull/94)) +- **A `schedule-skipping` alert.** A schedule whose last three runs + (`FLEET_ALERT_SKIP_STREAK`) all skipped raises one alert naming the reason it + keeps giving, for example "nightly-tvloop-hardware has skipped 3 runs in a + row: no Roku developer password on this host", and resolves on the first run + that is not a skip. The tvloop nightly needed it for two weeks. ([#94](https://github.com/addisdev/fleet-runner/pull/94)) +- **Simulator rows are forgotten after a week offline.** The lease sweep deletes + a row whose descriptor says `kind` or `soc` is `simulator` once it has been + offline for `FLEET_SIMULATOR_TTL_DAYS` (default 7), because simulators are + recreated under new UUIDs and never come back; the live brain listed thirteen + of them. Real devices and Android emulators are never forgotten, a row that + holds a lock is left alone, and results and beacons are kept. `POST + /jobs/sweep` reports what it forgot. ([#94](https://github.com/addisdev/fleet-runner/pull/94)) +- **A changelog coverage check for releases.** On a `release/*` pull request, + and on `develop` into `main`, `changelog coverage` in `ci.yml` lists every + pull request merged into `develop` since the last tag and fails, naming + each, if the section being released does not mention it as `#N` or a + `/pull/N` link. 0.8.0's notes left out work that had shipped in it and had + to be amended the day after; this is the check that would have caught it. + Label a pull request `no-changelog` to exempt it; Dependabot's are exempt. + The logic is `scripts/changelog-coverage.mjs`, with unit tests. + ([#89](https://github.com/addisdev/fleet-runner/pull/89)) +- **The documented workload count is checked.** The README and the docs said + 28 while the catalog grew to 34. Each hand-written count is now marked + `34`, and `repo scripts` in `ci.yml` fails on any + pull request, docs-only ones included, when a marked number disagrees with + the shipped entries in `collector/src/workloads/catalog.ts`. The collector's + suite checks that reading against the real `CATALOG`. ([#89](https://github.com/addisdev/fleet-runner/pull/89)) +- **Findings are in the control API.** `/api/v1` had 22 routes and none was + findings, so a coding agent could run a job but not read the bug list night + QA wrote for it. `GET /api/v1/findings` (filter by `app`, `state`, + `verdict`, `check`, `since`, which takes `24h` as well as ISO 8601), + `GET /api/v1/findings/{id}` (the steps in words, the evidence as links, the + replay flow inline up to 16 KiB, the regression test, the verdict history) + and `POST /api/v1/findings/{id}/verdict` (scope `run`, audited). The legacy + `/api/findings` routes call the same functions and gained the same filters + and fields. ([#96](https://github.com/addisdev/fleet-runner/pull/96)) +- **`fleetr findings`, `fleetr finding`, `fleetr verdict`, and three MCP + tools.** "What did last night find in GreenFolio, and give me the Maestro + flow" is now one call: `fleetr findings --app greenfolio --since 24h`, then + `fleetr finding ID --flow repro.yaml`; `fleet_findings`, `fleet_finding` + (flow text inline) and `fleet_verdict` (not read-only, scope `run`) do the + same over MCP. `fleetr findings --since 24h --json` is the input a scheduled + agent session can start from; its shape is documented in docs/agents.md. + The skill has a section on reading a finding: the replay count first, + `agent_mistake` is not a bug, and never `fixed` without a fix merged. ([#96](https://github.com/addisdev/fleet-runner/pull/96)) +- **A fixed finding becomes a regression test.** A new verdict, `fixed`, comes + only after `real` (a 409 otherwise). Marking a finding real copies its replay + flow into a new `regressions` table as a candidate; fixed makes it active; + duplicate, not a bug or agent's mistake retire it. A nightly `ui-test` with + `params.regressions: ""` pulls the app's active flows from the + collector at run time and runs each after its own flow, so a finding marked + fixed today is in tonight's run without anyone editing a schedule. For a + crash or ANR finding on Android the executor reads `logcat -b crash` and + `am_anr` around the flow; a new crash of the package is `reproduced`, fails + the row and reopens the finding (verdict cleared, seen once more, the build + added, a `regressed on build ...` line in its new verdict history). Every + other check can only be asserted as "replays without error", and the docs + and each run's detail say so; a failing flow fails the row but reopens + nothing. GitHub issues are only ever created, never read back, so closing + one does not mark a finding fixed. The dashboard has a **Fixed** button + (f) and a regression panel. A database opened by this release + rebuilds the findings table once to allow `fixed` and backfills one history + row per existing verdict. Not yet run on a real nightly: covered by the + suite against fake crash readings and a fake Maestro. ([#96](https://github.com/addisdev/fleet-runner/pull/96)) +- **The Findings page reads and judges through `/api/v1`.** Its list, detail + and verdict calls go through the generated client, so its types are the API's + and a dashboard verdict is the same call as `fleetr verdict`. `GET + /api/findings`, `GET /api/findings/{id}` and `POST + /api/findings/{id}/verdict` now carry `Deprecation` and a successor `Link`; + `/api/findings/precision`, `/digest` and `/issues` have no v1 counterpart and + do not. The v1 list carries `facets` (every app and check, and open/triaged + counts) like the device and job lists, and the v1 verdict keeps the + `FLEET_DASH_TOKEN` speed bump the older route had. Part of #83. ([#96](https://github.com/addisdev/fleet-runner/pull/96)) + +### Changed + +- **Explore loads its model at 45% free memory, not 65%.** ultra's resident + models alone hold it near 56% free, so the shelf's default bar meant no + explore night could load Holo4 (~25 GB); 10-15's garden night skipped for + it. The night manifests pass `WARM_MIN_FREE_PCT=45` to the warm-up; the + shelf's default for everything else is unchanged, and it still refuses + under memory pressure. ([#95](https://github.com/addisdev/fleet-runner/pull/95)) +- **`ci-enqueue` speaks `/api/v1` and exits like `fleetr run`.** It polled the + deprecated `GET /jobs/:id` and exited 0 on `done`, which includes a job that + skipped and tested nothing, and it sent no token, so it failed against a + collector with tokens on. It now enqueues with `POST /api/v1/jobs`, waits + with `/wait`, and exits 0 passed, 1 failed, 2 invalid spec, 3 still running, + 4 skipped, 5 refused or unreachable. A workflow that read exit 1 as "timed + out" now sees 3. ([#92](https://github.com/addisdev/fleet-runner/pull/92)) +- **`ci-enqueue` and `ci-upload` send `FLEET_RUNNER_TOKEN` (or + `FLEET_TOKEN`) as a bearer token.** `ci-upload` read only + `FLEET_DASH_TOKEN`, even when the collector needed an `agent` token there. + `FLEET_DASH_TOKEN` is still sent as `x-fleet-token`. `ci-upload` exits 5 + rather than 1 when the upload is refused or the collector is unreachable. ([#92](https://github.com/addisdev/fleet-runner/pull/92)) +- **The example jobs are checked the way `/api/v1` checks a spec.** + `check-examples.ts` used the loose `job.schema.json`, so an example could + pass the check and be refused by the API it documents. It now runs the API's + strict schema and each workload's params manifest, imported rather than + copied. All 37 pass. ([#92](https://github.com/addisdev/fleet-runner/pull/92)) +- **Explore's `CHECK_NAMES` lists only checks something raises.** `regression`, + `agent_stuck` and `leash` were accepted by `POST /findings` and raised by + nothing; they are refused now. A refused action is on the mission's steps, + not a finding. Stored findings are not touched, and reading does not + validate the name. ([#92](https://github.com/addisdev/fleet-runner/pull/92)) +- **The desktop app is versioned with everything else.** `scripts/version.mjs` + now writes `desktop/src-tauri/Cargo.toml` and `tauri.conf.json` (they said + 0.5.0), so the CI version check covers them. Both say 0.8.0. ([#92](https://github.com/addisdev/fleet-runner/pull/92)) +- **The explore docs' three diagrams are drawn, not generated.** The night, + one mission's sequence and the code map were mermaid blocks; they are now + figures in `docs/figures/` rendered to PNG like the rest + (`explore-night`, `explore-step`, `explore-code`). The docs workflow fails on + a mermaid fence under `docs/`, and `mkdocs.yml` no longer configures one. ([#90](https://github.com/addisdev/fleet-runner/pull/90)) +- **The workloads figure shows all 34 shipped workloads.** It had 28; `enrol`, + `explore`, `upgrade-test`, `size-report`, `tvloop` and `llm-eval` are added. ([#90](https://github.com/addisdev/fleet-runner/pull/90)) +- **`devices.png` is the new device drawings, and there is a `findings.png`.** + Both are captured read-only from a running brain by the new + `npm run shoot:live`, filtered to what was online and to the `bug-garden` + fixture app; docs/brand.md says why a throwaway collector cannot make them. ([#90](https://github.com/addisdev/fleet-runner/pull/90)) +- **Explore nights that need tonight's build skip, not fail, when there is + none.** With `--fresh-build` (now on `fleet-explore-apple` and + `fleet-explore-tv`), the night checks through the tunnel that the brain has + a build of each app published tonight before it warms the model or boots a + simulator. If not, the job is posted to the brain as `ok: true`, `skipped: + no greenfolio-ios build for tonight (fleet-explore-builds did not run, ...)`, + and the command exits 0: ultra's night queue has no skip exit code, and a + missing upstream build is not this job's failure. `nightly-builds.sh` now + uploads an unchanged build again (the brain ignores the duplicate bytes and + stamps it published), so a quiet day on `develop` does not look like a + missed build. ([#93](https://github.com/addisdev/fleet-runner/pull/93)) +- **The model warm-up moved inside the night command.** The four night-queue + manifests ran `shelf.sh warm pilot &&` before `explore-night.ts`, so a + warm-up the shelf refused ended the job with `exit 1` before any check ran. + `explore-night.ts --warm ""` runs it after the build and time checks, + retries a refusal for memory every minute for up to `--warm-wait` minutes + (default 10), and then posts the night as `skipped: the model would not + load, no room on this host: ...`. Any other warm-up failure is posted as a + failure with its last line. The manifests in + `collector/deploy/explore/night/` need copying to `~/night/recurring/` on + ultra. ([#93](https://github.com/addisdev/fleet-runner/pull/93)) +- **The dashboard reads devices, jobs, templates and schedules through + `/api/v1`.** It called the deprecated `GET /api/devices`, `/api/jobs` and + their detail routes, and enqueued with `POST /api/jobs`, with response types + written by hand, so the server could change a field and the dashboard would + show a blank column with nothing failing. It now uses the generated client + for every read and change `/api/v1` offers, and its types are the client's. + The `/api/v1` schemas state every field the dashboard reads, so + `check-api-v1` fails when a handler stops sending one, and a new suite step + fails if `dash/src` calls a route the deprecation headers mark. To carry the + Devices and Jobs filter menus, `GET /api/v1/devices` and `/api/v1/jobs` now + return `facets`, and `/api/v1/jobs` takes `has_error=true`. Two things a + person can see: the composer's spec is checked strictly, so an unknown + top-level field or params outside the workload's manifest is refused with + the path named instead of failing later, and its jobs record `source: dash`. + Enqueueing is no longer behind `FLEET_DASH_TOKEN`, as `POST /jobs` never + was. Findings, the Overview, results, system, events, the live stream and the + other changes stay on `/api/*` until `/api/v1` has them. + ([#97](https://github.com/addisdev/fleet-runner/pull/97)) + ### Fixed +- **The OpenAPI document named the wrong status for two enqueue refusals.** + `POST /api/v1/jobs` listed `no_matching_devices` and `no_build` under 422, + but `queue.ts` refuses both with 400. The 400 and 422 entries now name the + codes each really carries. ([#96](https://github.com/addisdev/fleet-runner/pull/96)) +- **The older job routes skipped two auth checks.** With `FLEET_AUTH=tokens`, + `POST /api/jobs/:id/cancel` and `/retry` let any `run` token stop or re-run + anyone's job, and `POST /jobs` and `POST /api/jobs` accepted the `shell` + workload from a token without `admin`. Only `/api/v1` asked. Both paths now + call the same two functions (`collector/src/api/policy.ts`). With auth off + nothing changes. ([#92](https://github.com/addisdev/fleet-runner/pull/92)) +- **`fleet-cli`'s `bin` pointed at a file the build does not produce.** It + named `dist/cli.mjs`; the build writes `dist/bin/fleet.mjs`, and `dist/` was + left out of the package by `.gitignore`, so installing from a tarball gave no + command. Both `fleet` and `fleetr` now point at the bundle, and `files` + includes `dist/`. ([#92](https://github.com/addisdev/fleet-runner/pull/92)) +- **A mistyped timer interval became a near-zero one.** `FLEET_SWEEP_MS`, + `FLEET_SCHEDULER_TICK_MS` and `FLEET_ALERT_TICK_MS` went through a bare + `Number()`, so `15s` was NaN and `15` (seconds, meant) a sweep every 15 ms. + Anything but whole milliseconds of at least 1000 now falls back to the + default, with a startup warning. ([#92](https://github.com/addisdev/fleet-runner/pull/92)) +- **Explore's live-view frames were refused with `FLEET_AUTH_PROTOCOL=token`.** + They sent only `FLEET_DASH_TOKEN`; they now also send `FLEET_TOKEN` as a + bearer token. ([#92](https://github.com/addisdev/fleet-runner/pull/92)) +- **A schedule template's empty `job_id` was replaced without a word.** A + non-empty one was already refused; an empty or `null` one was stored and + overwritten at every firing. It is dropped now and the reply carries a + `warnings` entry, and a stored template that still has one is named in the + log when it fires. ([#92](https://github.com/addisdev/fleet-runner/pull/92)) +- Two comments said `fleet up` turns mDNS discovery on. It passes + `collector.discovery`, which is off by default. ([#92](https://github.com/addisdev/fleet-runner/pull/92)) - **`fleet skill` failed on an installed 0.8.0.** The 0.8.0 archives carry `skills/`, but `install.sh` and `install.ps1` copy a release's directories by name and did not name it, so `fleet skill show` and `fleet skill install` could not find `SKILL.md`. Both installers now copy it, and a test fails if the bundle gains a directory the installers do not copy. The installers are fetched from `main`, so re-running the install command fixes an existing - 0.8.0 install; no new release is needed. + 0.8.0 install; no new release is needed. ([#86](https://github.com/addisdev/fleet-runner/pull/86)) +- **`bug-garden/tools/check-bench.mjs` passed `bench-delete-plant` without + checking the plant was gone.** The card's `check` gained `absent_text`, which + the harness enforces and the standalone checker ignored; it also matched + case-sensitively where the harness does not. It now reads `absent_text` and + matches case-insensitively, as the bug garden's README says. ([#88](https://github.com/addisdev/fleet-runner/pull/88)) +- **`fleet-explore-apple` exited 1 in 0.0 minutes on 10-14.** Not a missing + build: the shelf refused to load Holo4 at 64% free memory against its 65% + threshold, after the night queue had admitted the job on its own measure. + See the warm-up change above; with `--fresh-build` that night would also + have skipped earlier, at "no greenfolio-ios build for tonight". ([#93](https://github.com/addisdev/fleet-runner/pull/93)) +- **tvOS simulators registered as `os: ios`, so the TV explore night tested + nothing.** The executor read simctl's runtime key for `iOS-` only, so an + Apple TV simulator was described as plain `ios`, every `os ~ 'tvos'` job + skipped with "no device matches this job on ultra", and each + `fleet-explore-tv` night "passed" in about a minute. A tvOS simulator is + now `tvos-26.5` (watchOS and visionOS likewise). No TV night has run with + the fix yet. ([#93](https://github.com/addisdev/fleet-runner/pull/93)) +- **A failing `ui-test` now says which step failed.** `nightly-fleet-ui-smoke` + failed on the Galaxy S8+ every night from 10-10 with `failed: 1` and no + error, while `Assertion is false: "Fleet Runner" is visible` sat in the JUnit + report it had uploaded. The executor now reads the failing flow and its + assertion out of the report into the device row's `error` and + `test.failing_flow`; when Maestro produces no report, the error is its exit + code and last stderr line, never its command line, which carries the test + account's password. ([#94](https://github.com/addisdev/fleet-runner/pull/94)) +- **No host result says `ok: false` with nothing beside it.** Every host + workload's rows go through one wrapper in the executor: a device row with + only counts says so ("1 of 1 tests failed; the runner reported no failing + step"), and a final host row with no error of its own gets the failing + devices' reasons. A subprocess failure nobody caught is reported as its exit + code and last stderr line instead of Node's message. The collector also + copies a failure's reason into the job's `last_error`, so the job list and + the `job-failed` alert show it. ([#94](https://github.com/addisdev/fleet-runner/pull/94)) +- **`GET /api/artifacts/` answered "no such endpoint".** It is where + `DELETE` and `/pin` already live, so it is the URL a person reaches for, and + the 404 read as a missing artifact: the ui-smoke JUnit report was in the + store all along. It now redirects (308) to + `/api/v1/artifacts//content`, and says plainly when no artifact by + that hash was ever uploaded. ([#94](https://github.com/addisdev/fleet-runner/pull/94)) +- **The executor registered Rokus as `platform: android`.** Its Roku + descriptor said `os: roku-15.3.4` but no platform, and the collector's + fallback for anything that is not iOS is Android. It now sends + `platform: "roku"`; `os ~ 'roku'` targeting and the tvloop workload are + unaffected. ([#94](https://github.com/addisdev/fleet-runner/pull/94)) +- **A Roku seen first by address and later by serial was listed twice.** A + sweep where the box answered SSDP but not device-info left a + `roku-ip-192-168-50-218` row offline beside `roku-X01600EMFRJ9`. A + serial-named registration now retires the IP-named row at the same address. ([#94](https://github.com/addisdev/fleet-runner/pull/94)) +- **`fleet service install` on Windows had never started anything.** The + scheduled task's command kept the backslashes of a `\"` escaping meant for a + `cmd` prompt -- execFile quotes the argument itself -- so the task ran + `\"C:\...\node.exe\"` and failed with "file not found" every time. Found by + the new Windows smoke on its first run. `fleet service uninstall` and `stop` + also now kill `fleet up`'s process tree before ending the task: deleting a + task does not stop it, and Windows does not take a parent's children down + with it. ([#89](https://github.com/addisdev/fleet-runner/pull/89)) +- **The desktop app left `fleet up` running when it was killed, and its Quit + item could not quit.** SIGTERM (a `kill`, a `killall`, the end of a session) + killed the app outright and orphaned its sidecar on the collector's port. And + `RunEvent::ExitRequested` refused every exit, including the `app.exit(0)` the + Quit item calls, so Quit stopped the fleet and left the app running. SIGTERM, + SIGINT and SIGHUP now take the Quit path -- stop `fleet up`, then exit -- and + only the implicit last-window exit is refused. Found by the first launch of + the app, in CI. ([#89](https://github.com/addisdev/fleet-runner/pull/89)) +- **`/api/v1` cancel and retry were always refused with `FLEET_DASH_TOKEN` + set.** They hand the request to the guarded `/api/jobs/:id/cancel` and + `/retry`, and dropped `X-Fleet-Token` on the way, so with the guard on every + one answered 401 whatever the caller sent. The header is now carried, and + the refusal says `unauthorized` rather than `internal`. + ([#97](https://github.com/addisdev/fleet-runner/pull/97)) + +### Now verified + +Four items from earlier "Not verified" lists, each now checked by +GitHub-hosted CI on every pull request that touches them. ([#89](https://github.com/addisdev/fleet-runner/pull/89)) +- **`fleet service install` on launchd.** `fleet / launchd smoke` installs the + LaunchAgent on `macos-latest` (the runner has a `gui/` domain), waits + for its collector to answer and its agent to register, checks + `fleet service status`, reinstalls once so the bootstrap race is hit and + retried (it was), then uninstalls and checks nothing answers, the plist is + gone and launchd no longer has the job. ([#89](https://github.com/addisdev/fleet-runner/pull/89)) +- **The Windows scheduled task.** `fleet / windows task smoke` runs the same + lifecycle on `windows-latest`. It found the bug above; with it fixed, a + Windows machine agent registered against a real collector for the first + time. ([#89](https://github.com/addisdev/fleet-runner/pull/89)) +- **The arm64 container image.** `fleet / container smoke (arm64)` builds + `fleet/Dockerfile` natively on `ubuntu-24.04-arm`, checks the container + reports `aarch64`, and waits for its agent to register and answer + `fleet devices`, `fleet workloads` and `fleet whoami`. ([#89](https://github.com/addisdev/fleet-runner/pull/89)) +- **The desktop app launches.** `desktop / launch smoke` builds it on Ubuntu + and runs it under Xvfb: its `fleet` sidecar starts a collector whose agent + registers, the app is still up 20 seconds later, and SIGTERM stops both. + Then, under `tauri-driver`, the settings window is driven over WebDriver: + it is titled `Fleet`, `get_status` answers over IPC with the sidecar + running, and a screenshot is uploaded. This was the first time the app ever + ran. It has still never run on a Mac, and the tray menu has never been + clicked. + +Still not verified: any of the three services coming back after a reboot, a +graceful stop on Windows, and the MCP gateway against a real identity provider +(it needs a tenant). ([#89](https://github.com/addisdev/fleet-runner/pull/89)) +### Not verified + +- **No regression flow has run on a device.** The executor side is tested + against a fake crash buffer and a fake Maestro; no app's nightly sets + `params.regressions` yet. +- **No short explore night, no warm-up at 45%, and no tvOS night since the + `os: tvos` fix has been seen on ultra.** The first is the night of + 2026-10-15. +- **The MCP gateway has still not met a real identity provider.** +- **The desktop app on a Mac:** CI launches it on Linux; its tray, menus and + windows on macOS are unrun. ## [0.8.0] — 2026-10-15 @@ -126,6 +490,63 @@ pull request that touches them: unversioned routes with hand-written types ([#83](https://github.com/addisdev/fleet-runner/issues/83)). +### Also in this release + +These merged into `develop` before 0.8.0 was cut and shipped in it, but were +left out of the notes above on release day. + +- **Exploratory QA, the `explore` workload.** A vision model works through + mission cards on tonight's build, driving a device the way a person would, + while the harness checks every step for crashes, ANRs, frozen and blank + screens, dead controls, accessibility and visual defects. Each candidate is + replayed twice on a clean install and only what reproduces is filed; a crash + stands on its log. See [Exploratory QA](docs/explore/index.md). + ([#70](https://github.com/addisdev/fleet-runner/pull/70)) +- **Findings, verdicts and a morning digest.** `POST /findings` merges repeats + by fingerprint; the dashboard's Findings page has Real, Duplicate, Not a bug + and Agent's mistake. A check under 30% precision after ten verdicts is + switched off for that app (crashes and ANRs never are). GitHub issues stay a + dry run unless `FLEET_GITHUB_ISSUES=1` and a token are set, and the collector + only ever creates them. ([#70](https://github.com/addisdev/fleet-runner/pull/70)) +- **What it drives:** Android phones, tablets, Android TV and Fire TV over adb; + iOS and tvOS through FleetDriver, a new XCUITest command server in + `runner-ios`; a Roku through tvloop. The leash refuses delete, purchase, + invite and sign-out controls unless a card allows them and never types into + a password field. ([#70](https://github.com/addisdev/fleet-runner/pull/70)) +- **The bug garden**, an Android fixture app with twenty planted defects, a + scoring rule and scorer, and twenty bench cards for scoring a model without + a judge. Its screenshot walks capture each screen in the base, dark, Spanish + and large-text settings. ([#70](https://github.com/addisdev/fleet-runner/pull/70), + [#74](https://github.com/addisdev/fleet-runner/pull/74)) +- **Nights on ultra.** `scripts/explore-night.ts` is the night-queue command: + it boots the devices, opens its own tunnel to the brain, runs one job and + exits with its result. `nightly-builds.sh` builds GreenFolio iOS and + Dozehound tvOS from `origin/develop`; the tvOS and iOS nights run after the + products' own night jobs. Holo4 35B-A3B measured 100 of 100 on pointing and + a median step of 6.8 s. Fire TV is parked and the TV question is answered on + tvOS. ([#73](https://github.com/addisdev/fleet-runner/pull/73), + [#77](https://github.com/addisdev/fleet-runner/pull/77), + [#78](https://github.com/addisdev/fleet-runner/pull/78)) +- **The dashboard draws each device as the object it is**: 22 drawings + (phones, tablets, TV boxes and sticks, computers, watches, a headset, a + board, a browser) instead of seven silhouettes, chosen from the model string + first because older agents report a Roku as `android` and an Apple TV or a + watch as `ios`. ([#69](https://github.com/addisdev/fleet-runner/pull/69)) +- **The Android runner starts on Android 7.0 and 7.1.** `RunnerService` called + `startForegroundService`, which exists from API 26, so API 24 and 25 threw + `NoSuchMethodError`; it uses `startService` there now. + ([#75](https://github.com/addisdev/fleet-runner/pull/75)) +- **The Android runner is linted in CI.** The `android.hardware.type.watch` + feature line that failed `lintDebug` is gone; the runner still detects a + watch at run time. ([#76](https://github.com/addisdev/fleet-runner/pull/76)) + +Not verified, from those pull requests: a physical iPhone or Apple TV through +FleetDriver; a real Roku (its developer password is not on the host yet); any +model-driven night on an Apple surface (the iOS and tvOS simulators were driven +without a model); the GreenFolio Android cards; the Android 7 fix on a real +API 24 or 25 device (Robolectric only); and any Wear device, before or after +the lint change. + ## [0.7.2] — 2026-09-30 ### Fixed @@ -1047,7 +1468,8 @@ The first public release, when the project was still four repositories. that starts a throwaway collector on a spare port so it never touches a live fleet's history. -[Unreleased]: https://github.com/addisdev/fleet-runner/compare/v0.8.0...HEAD +[Unreleased]: https://github.com/addisdev/fleet-runner/compare/v0.9.0...HEAD +[0.9.0]: https://github.com/addisdev/fleet-runner/compare/v0.8.0...v0.9.0 [0.8.0]: https://github.com/addisdev/fleet-runner/compare/v0.7.2...v0.8.0 [0.7.2]: https://github.com/addisdev/fleet-runner/compare/v0.7.1...v0.7.2 [0.7.1]: https://github.com/addisdev/fleet-runner/compare/v0.7.0...v0.7.1 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f5e7d56..d13e343 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -11,7 +11,7 @@ that is not on this shelf is data this repository does not have. ## Running the suites -Four components, four suites. Only run the one you touched. +Five components, five suites. Only run the one you touched. === "collector" @@ -80,7 +80,7 @@ Four components, four suites. Only run the one you touched. npm install && npm test ``` - 122 tests, about a second. This is the easiest component to contribute to + About a second. This is the easiest component to contribute to and the reference implementation of the protocol. ## Things this project learned the hard way @@ -151,7 +151,16 @@ this is the first rule rather than a nice sentiment. - Run the suite for what you touched, and say in the description what you ran. - Update the docs in the same change. The site builds with `--strict`, so a broken link fails CI. -- Add a `CHANGELOG.md` entry under `## [Unreleased]` for anything user-visible. +- Add a `CHANGELOG.md` entry under `## [Unreleased]` for anything user-visible, + ending with a link to the pull request + (`([#86](https://github.com/addisdev/fleet-runner/pull/86))`) once it has a + number. A pull request nobody using Fleet Runner would notice — CI plumbing, + a typo — gets the `no-changelog` label instead. The release check below holds + both to it. +- A number the docs state by hand (how many workloads there are) is marked + `34`. `repo scripts` in `ci.yml` fails when a marked + number disagrees with `collector/src/workloads/catalog.ts`; fix the number, + keep the marker. `ci-ok` is the only required check. It always reports, and it passes when every component suite is green **or skipped** — a change to one runner does not build @@ -161,8 +170,10 @@ the others. One version covers every component, so a release is one number everywhere. -1. On a branch from `develop`, run `node scripts/version.mjs set X.Y.Z`. It - writes `VERSION` and every component's own version. Then run +1. On a branch named `release/X.Y.Z` from `develop`, run + `node scripts/version.mjs set X.Y.Z`. It writes `VERSION` and every + component's own version, the desktop app's `Cargo.toml` and + `tauri.conf.json` included. Then run `npm --prefix collector run gen:api`, because `openapi.json` carries the version and CI fails when it is stale. Android's `versionCode` is not derived from it; bump that by hand if the Play Store build needs it. @@ -171,6 +182,17 @@ One version covers every component, so a release is one number everywhere. at the bottom. 3. Merge that into `develop`, then open a `develop` → `main` pull request named `Release X.Y.Z` and merge it once `ci-ok` passes. + + On both pull requests `ci.yml` runs `changelog coverage` + (`scripts/changelog-coverage.mjs`). It lists every pull request merged + into `develop` since the last `v*` tag and fails, naming each, if one is + not mentioned as `#N` or a `/pull/N` link between `## [Unreleased]` and the + last release's heading. Label a pull request `no-changelog` to exempt it; + Dependabot's are exempt already. The check only runs when the branch is + `release/*`, or `develop` into `main` — so name the branch that way, or it + will not look. Run it yourself first with + `node scripts/changelog-coverage.mjs --head origin/develop` (it needs `gh` + for the labels). 4. Tag the merge commit on `main` and push the tag: `git tag -a vX.Y.Z -m "vX.Y.Z" && git push origin vX.Y.Z`. diff --git a/README.md b/README.md index 9ab2b35..4f6453e 100644 --- a/README.md +++ b/README.md @@ -13,8 +13,10 @@ One queued job installs a build on every attached device, runs a UI suite across them, benchmarks llama.cpp on real silicon, classifies a few hundred images through Core ML and LiteRT, screenshots a website on two real phone screens and diffs it against a baseline, or drains a battery on purpose and -plots the curve. Everything lands in one results database with one dashboard -in front of it. +plots the curve. Overnight, a local vision model can use tonight's build the +way a person would and leave a short list of bugs that reproduced on a clean +install. Everything lands in one results database with one dashboard in front +of it. Built to answer a question I could not otherwise answer: **would on-device machine learning actually be good enough to ship in my apps, and on which @@ -27,7 +29,7 @@ hardware?** It turned out to be yes, and the fleet is how I know. | **Problem** | Mobile tests and ML benchmarks are only useful when they run on the hardware, under conditions, and with the exact artifacts a product will ship. | | **System** | One collector owns the registry, queue, leases, artifacts, results and dashboard; independently implemented agents claim work through one tested JSON protocol. | | **Reach** | Five runner implementations across TypeScript, Swift, Kotlin, JavaScript and BrightScript, plus host-driven iOS and Android workloads. | -| **Workloads** | 28 workload types spanning UI tests, visual regression, llama.cpp, Core ML, LiteRT, battery experiments, builds and deployment. | +| **Workloads** | 34 workload types spanning UI tests, visual regression, exploratory QA by a vision model, llama.cpp, Core ML, LiteRT, battery experiments, builds and deployment. | | **Proof** | The first production question found an int8 CPU model faster to load and run than its fp32 GPU alternative while preserving useful accuracy. | **What this demonstrates:** distributed systems, mobile platforms, protocol @@ -40,8 +42,8 @@ trade-offs. ![The Overview while one fan-out benchmark is claimed by three agents at once: the queue fills, three rows appear under running now, and the done count climbs as each reports back](docs/img/fanout.gif) -One `POST /jobs` with `"fanout": true`, and every agent the expression matches -claims its own child job. Above: a laptop and two browser runners on a collector +One job with `"fanout": true`, and every agent the expression matches claims +its own child job. Above: a laptop and two browser runners on a collector that did not exist a minute earlier. Nothing there is staged — it is the shipping agents against a real collector, recorded a frame every 250 ms. @@ -63,8 +65,9 @@ curl -fsSL https://raw.githubusercontent.com/addisdev/fleet-runner/main/install. > The installer verifies the download against the release's `SHASUMS256.txt` and > refuses to install if it does not match. It needs Node 22.13 or newer and > carries no runtime of its own. **`fleet up` itself has been watched to work on -> macOS/arm64 only** -- CI runs the suites on Windows and Linux, which is not the -> same as somebody having run a fleet on one. +> macOS only**, on Apple silicon and on a 2016 Intel Mac that runs this lab's +> brain -- CI runs the suites on Windows and Linux, and starts the container and +> the systemd unit, which is not the same as somebody having run a fleet on one. > **[docs/install/](https://addisdev.github.io/fleet-runner/install/)** is honest > per platform. @@ -135,9 +138,9 @@ something a runner can do without the collector shipping a release. ## What it can run -![Twenty-eight workloads in three columns by who claims them: on the device, on a host, and on a machine](docs/img/workloads.png) +![Thirty-four workloads in three columns by who claims them: on the device, on a host, and on a machine](docs/img/workloads.png) -Twenty-eight workloads, and the column a workload sits in is the answer to +Thirty-four workloads, and the column a workload sits in is the answer to "who can physically do this". A benchmark runs inside the app on the phone. An install needs a cable and a Mac. A build needs a checkout and a toolchain. Each has [its own page](https://addisdev.github.io/fleet-runner/workloads/), @@ -145,7 +148,7 @@ saying what it measures and what it refuses to guess. ## What is in here -Seven projects in one repository, in three groups: **one brain**, **five +Eight projects in one repository, in three groups: **one brain**, **five runners** that speak its protocol and share no code with each other or with it, and **two front doors** onto the whole thing. They ship independently but they are versioned together, because the protocol is the thing that breaks and a @@ -155,7 +158,7 @@ change to it touches three of them at once. | | What it is | |---|---| -| **[collector/](collector)** | Device registry, job queue with leases, artifact store, results database, scheduler, alert engine, and the dashboard above. Node + Fastify + SQLite, no broker, no cloud. It also holds the host executor, which drives phones from outside over adb, simctl and devicectl. | +| **[collector/](collector)** | Device registry, job queue with leases, artifact store, results database, scheduler, alert engine, and the dashboard above. Node + Fastify + SQLite, no broker, no cloud. It also holds the host executor, which drives phones from outside over adb, simctl and devicectl, and the [exploratory QA](https://addisdev.github.io/fleet-runner/explore/) workload a vision model drives. | **The runners** -- five hand-written implementations of one protocol, in five languages, sharing not one line @@ -173,7 +176,14 @@ languages, sharing not one line | | What it is | |---|---| | **[fleet/](fleet)** | The CLI. `fleet up` runs this machine's components under a supervisor that backs off and gives up loudly rather than restarting a broken component forever; `fleet doctor` says what the machine can run and why not; `fleet service install` keeps it up. It wraps the three programs above and reimplements none of them. | -| **[desktop/](desktop)** | A Tauri menu-bar app around that CLI: a tray menu, role switches, the dashboard in a window, and a notification when a component gives up. It exists to test one hypothesis about macOS local-network permissions, and **it has never been launched**: CI compiles its Rust and that is all -- read its README before believing anything in it. | +| **[desktop/](desktop)** | A Tauri menu-bar app around that CLI: a tray menu, role switches, the dashboard in a window, and a notification when a component gives up. It exists to test one hypothesis about macOS local-network permissions, and **it has never run on a Mac**: CI compiles it on macOS, and launches it on Linux under a virtual display, where it starts its `fleet` sidecar and its settings window answers over WebDriver. The tray, the Mac and the hypothesis are untested -- read its README before believing anything in it. | + +**Alongside them** + +| | What it is | +|---|---| +| **[skills/fleet-runner/](skills/fleet-runner)** | The agent skill `fleet skill install` copies into a coding agent: which commands to run in what order. Its tool and exit-code tables are generated from the code. | +| **[bug-garden/](bug-garden)** | An Android app with twenty planted defects, so exploratory QA has something with a known answer to be scored against. | Each directory has its own README, its own tests and its own CI job, filtered by path so a change to a phone runner does not build the dashboard. @@ -272,8 +282,9 @@ that is not on this shelf is data this repository does not have. the handful of things this project learned the hard way. Behaviour is covered by the [Contributor Covenant](CODE_OF_CONDUCT.md). -**There is no authentication, by design** — the collector is built for a home -LAN or a tailnet. [`SECURITY.md`](SECURITY.md) says what that does and does not +**By default there is no authentication** — the collector is built for a home +LAN or a tailnet, and [scoped tokens](https://addisdev.github.io/fleet-runner/deploy/auth/) +are opt-in. [`SECURITY.md`](SECURITY.md) says what that does and does not cover before you report it. ## License diff --git a/VERSION b/VERSION index a3df0a6..ac39a10 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.8.0 +0.9.0 diff --git a/bug-garden/README.md b/bug-garden/README.md index a7712d7..a3ef1ae 100644 --- a/bug-garden/README.md +++ b/bug-garden/README.md @@ -21,8 +21,10 @@ mission and before a replay) puts it back exactly where it started. | `android/` | The app. Kotlin, Jetpack Compose, minSdk 26, AndroidX only. | | `defects.json` | The twenty defects: class, which check should catch it, screen, how to reach it, what it looks like, and a match pattern. | | `tools/check-bench.mjs` | Checks a bench mission's `check` block against a `uiautomator dump`, with no model. | +| `tools/verify-on-device.sh` | Walks to every defect on a device and checks each one still fires; see [Proving it on a device](#proving-it-on-a-device). | | `../collector/examples/missions/bug-garden/` | Seven exploration missions and twenty bench missions (`bench-*`). | -| `../collector/examples/flows/bug-garden/` | `sign-in.yaml` (the missions' setup flow), `smoke.yaml`, and `bench/`, a scripted answer to each bench mission. | +| `../collector/examples/flows/bug-garden/` | `sign-in.yaml` (the missions' setup flow), `smoke.yaml`, `bench/` (a scripted answer to each bench mission), `probes/` (a walk to each defect), and the `t1-*.yaml` walks that take one screenshot of each screen in the base, dark, Spanish and large-text settings. | +| `../collector/scripts/explore-bench/garden-score.ts` | Scores a night's findings against `defects.json` by the rule under [Scoring](#scoring). | ## Build and install @@ -89,7 +91,7 @@ in any of the three languages. ## The defects -Twenty, in eight classes. `defects.json` has the steps for each; in brief: +Twenty, in nine classes. `defects.json` has the steps for each; in brief: | Id | Class | Check | Screen | Defect | |---|---|---|---|---| @@ -202,9 +204,15 @@ with quantity 3", "Change the language to Spanish"), defects off (`launch_args: ["garden_defects=false"]`), and a `check` block a harness evaluates against the final `uiautomator dump`: -- `text`: every string is a case-sensitive substring of some node's `text` or - `content-desc`. (Content-desc counts: the Settings icon on Home is labelled - "Ajustes" in Spanish, so the Spanish check passes on Home or on Settings.) +- `text`: every string is a case-insensitive substring of some node's `text` + or `content-desc`. (Content-desc counts: the Settings icon on Home is + labelled "Ajustes" in Spanish, so the Spanish check passes on Home or on + Settings.) +- `absent_text`: no node's `text` or `content-desc` contains any of these + strings. `bench-delete-plant` uses it to check that Aloe Vera is gone, not + just that the list is one plant shorter. +- `focused_label`: the focused node's label contains the string. The TV cards + use it, since a D-pad screen's state is where focus sits. - `screen`: some node has `resource-id="screen_"`. The checks only use text that the seed data cannot produce by itself. No seeded @@ -213,11 +221,10 @@ the Office, so "Qty 3" can only come from the plant the mission added. Where a check names no screen, it holds on both the plant list and the plant's detail page, which is where a model doing the task naturally ends. -One limit: the check block has no way to say text must be absent, so -`bench-delete-plant` checks for "8 plants" (one fewer than the seed) rather -than for Aloe Vera's absence. - -`tools/check-bench.mjs` implements exactly these rules: +`tools/check-bench.mjs` implements these rules. The one difference from the +harness is `screen`: the harness matches the screen map's name for the final +screen, while the tool, which has only a dump, looks for the bug garden's +`screen_` tag. To check a dump by hand: ```sh adb exec-out uiautomator dump /dev/tty > /tmp/dump.xml diff --git a/bug-garden/tools/check-bench.mjs b/bug-garden/tools/check-bench.mjs index 1afe7b1..73a7ff4 100644 --- a/bug-garden/tools/check-bench.mjs +++ b/bug-garden/tools/check-bench.mjs @@ -6,17 +6,19 @@ * adb exec-out uiautomator dump /dev/tty | node check-bench.mjs - * * Exit 0 and "PASS" when every part of the mission's `check` holds, exit 1 and - * "FAIL" with what was missing otherwise. + * "FAIL" with what was wrong otherwise. * * The rules, which are the ones README.md states for the harness: - * text every string must be a substring (case-sensitive) of some + * text every string must be a substring (case-insensitive) of some * node's text OR content-desc. Content-desc counts because an * icon button's label is its content-desc (the Settings icon on * Home is "Ajustes" in Spanish). + * absent_text no node's text or content-desc may contain any of these + * strings (case-insensitive): "the plant is gone". * screen some node has resource-id "screen_". The bug garden * tags every screen's root that way. * focused_label some node with focused="true" has the string in its text or - * content-desc. + * content-desc (case-insensitive). * * Standalone on purpose: no imports from the collector, so it can be pointed * at a dump from any device without building anything. @@ -40,7 +42,7 @@ const nodes = []; for (const m of xml.matchAll(/]*?)\/?>/g)) { const a = {}; for (const kv of m[1].matchAll(/([\w:-]+)="([^"]*)"/g)) a[kv[1]] = unescape(kv[2]); - nodes.push({ text: a.text ?? "", desc: a["content-desc"] ?? "", id: a["resource-id"] ?? "", focused: a.focused === "true" }); + nodes.push({ text: (a.text ?? "").toLowerCase(), desc: (a["content-desc"] ?? "").toLowerCase(), id: a["resource-id"] ?? "", focused: a.focused === "true" }); } if (nodes.length === 0) { console.log(`FAIL ${mission.id}: the dump has no nodes`); @@ -48,20 +50,24 @@ if (nodes.length === 0) { } const check = mission.check ?? {}; +const shows = (n, t) => n.text.includes(t.toLowerCase()) || n.desc.includes(t.toLowerCase()); const missing = []; for (const t of check.text ?? []) { - if (!nodes.some((n) => n.text.includes(t) || n.desc.includes(t))) missing.push(`text ${JSON.stringify(t)}`); + if (!nodes.some((n) => shows(n, t))) missing.push(`missing text ${JSON.stringify(t)}`); +} +for (const t of check.absent_text ?? []) { + if (nodes.some((n) => shows(n, t))) missing.push(`still shows ${JSON.stringify(t)}`); } if (check.screen && !nodes.some((n) => n.id === `screen_${check.screen}`)) { const on = nodes.find((n) => n.id.startsWith("screen_"))?.id ?? "no tagged screen"; - missing.push(`screen ${check.screen} (on ${on})`); + missing.push(`missing screen ${check.screen} (on ${on})`); } -if (check.focused_label && !nodes.some((n) => n.focused && (n.text.includes(check.focused_label) || n.desc.includes(check.focused_label)))) { - missing.push(`focused ${JSON.stringify(check.focused_label)}`); +if (check.focused_label && !nodes.some((n) => n.focused && shows(n, check.focused_label))) { + missing.push(`missing focus on ${JSON.stringify(check.focused_label)}`); } if (missing.length) { - console.log(`FAIL ${mission.id}: missing ${missing.join(", ")}`); + console.log(`FAIL ${mission.id}: ${missing.join(", ")}`); process.exit(1); } console.log(`PASS ${mission.id}`); diff --git a/collector/README.md b/collector/README.md index 6f40b81..dc85912 100644 --- a/collector/README.md +++ b/collector/README.md @@ -1,8 +1,9 @@ # collector *Part of [Fleet Runner](../README.md), alongside the -[Android](../runner-android), [iOS](../runner-ios) and -[desktop](../runner-machine) runners.* +[Android](../runner-android), [iOS](../runner-ios), +[desktop](../runner-machine), [Roku](../runner-roku) and +[web](runner-web) runners.* The brain of **Fleet Runner**: a personal device lab that turns a shelf of old phones into something you can send work to. It holds the device registry, the @@ -12,10 +13,12 @@ One queued job can install a build on every attached device, run a Maestro or XCUITest suite across them, benchmark llama.cpp on real silicon, classify a few hundred images through Core ML and LiteRT, screenshot a website on two real phones and diff it against a baseline, or drain a battery on purpose and -plot the curve. The runners are separate apps — -[iOS](../runner-ios) and -[Android](../runner-android) — that speak a -shared JSON protocol, not shared code. +plot the curve. The runners are separate programs — +[iOS](../runner-ios) (with a tvOS target), +[Android](../runner-android), the [machine](../runner-machine) agent for +laptops and desktops, a [web](runner-web) runner this collector serves at +`/runner`, and a [Roku](../runner-roku) channel — that speak a shared JSON +protocol, not shared code. Node + Fastify + SQLite in WAL mode. No build step for the server, no broker, no cloud. @@ -33,7 +36,7 @@ APK or tapping through a UI test is not something an app can do to itself. ## Run it -Needs Node 22 or newer. Nothing else. +Needs Node 22.13 or newer. Nothing else. ```bash npm install @@ -45,7 +48,7 @@ build a runner from one of the app repos, point it at this address, and enqueue a job: ```bash -curl -X POST http://127.0.0.1:8788/jobs -H 'content-type: application/json' -d '{ +curl -X POST http://127.0.0.1:8788/api/v1/jobs -H 'content-type: application/json' -d '{ "schema": 1, "job_id": "bench-1", "workload": "benchmark", "executor": "device", "backend": "synthetic", "params": { "prompt_tokens": 256, "gen_tokens": 64, "measure_iters": 3 }, @@ -53,6 +56,10 @@ curl -X POST http://127.0.0.1:8788/jobs -H 'content-type: application/json' -d ' }' ``` +Or, with the `fleet` CLI installed, `fleetr run benchmark --backend synthetic +--pool ml-capable --param prompt_tokens=256 ...`, which validates, queues, +waits, and exits with the verdict. + ```bash npm test # typecheck, build the dashboard, run the suite npm run executor # host executor: claims host jobs, drives devices @@ -86,6 +93,15 @@ so it never touches a real fleet's history. | `llm-eval` | machine | Scores what a device generated against an eval set — exact, regex, JSON-shape and model-judged rules — so a quantisation is measured for whether it is still RIGHT, not only how fast it is wrong | | `self-check` | host, machine | The fleet inspects its own hosts: disk, tool versions, clock drift, agents actually loaded | | `archive` / `digest` | host | Pull store reviews and Search Console data; then have the shelf summarize its own reviews using its own models | +| `enrol` | host | Point every attached device's runner at a collector, without anybody typing | +| `explore` | host | A vision model explores a build and files reproduced crashes, hangs, dead controls and accessibility defects as findings | +| `tvloop` | host | tvloop's hardware suite on a Roku the fleet has locked: doctor, sideload, contract tests, a JUnit replay of its flows | +| `model-convert` / `dataset-prep` | machine | A checkpoint into GGUF, Core ML or TFLite; an eval set that carries its licence and source | +| `serve` / `shell` | machine | Host a model for the length of a lease; run a script whose hash the machine's owner pinned by hand | + +That is all 34 shipped workloads. `GET /api/v1/workloads` (or `fleetr +workloads`) lists them with their params and an example spec, plus anything a +registered agent declares that the collector does not know. Full reference, including every parameter and the failure modes worth knowing: **[the documentation](https://addisdev.github.io/fleet-runner/)**. @@ -93,6 +109,16 @@ Start at [Concepts](https://addisdev.github.io/fleet-runner/concepts/) for how the queue thinks, or [Workloads](https://addisdev.github.io/fleet-runner/workloads/) for what it can be asked to do. +## Control API + +Scripts, CI, the `fleet` CLI and AI agents use the versioned API under +`/api/v1`: devices, workloads, jobs (enqueue, validate, wait, cancel, retry), +results and artifacts. `GET /api/v1/openapi.json` describes it, and +`GET /metrics` serves the fleet in Prometheus text format. The older +unversioned routes still work and answer with a `Deprecation` header. Runners +speak the separate runner protocol. See +[the HTTP API](https://addisdev.github.io/fleet-runner/api/). + ## Status and threat model Built and running: every phase of the original plan, a dashboard with live @@ -101,13 +127,20 @@ an [on-device plant-ID evaluation](evals/greenfolio-plant-id.md) across both platforms — 77% top-1 at 7 ms per image on a phone, which answered a product question that had been guesswork. -**There is no authentication, by design.** The collector is reachable on a home -LAN and is meant to stay that way; anyone who can reach it can enqueue a job. -`FLEET_DASH_TOKEN` guards the dashboard's mutations, but that is a speed bump -against a stray browser tab, not an access control — `POST /jobs` stays open so -`curl` and CI keep working. Do not put this on the internet. If you want to, -the honest starting point is that every endpoint would need to be re-thought, -not that a token would need to be added. +**There is no authentication by default.** The collector is reachable on a +home LAN and is meant to stay that way; anyone who can reach it can enqueue a +job. `FLEET_DASH_TOKEN` guards the dashboard's mutations, but that is a speed +bump against a stray browser tab, not an access control — `POST /jobs` stays +open so `curl` and CI keep working. + +Scoped bearer tokens are opt-in: `FLEET_AUTH=tokens` requires one on the +control API, and `FLEET_AUTH_PROTOCOL=token` also on the runner protocol, for +fleets whose runners can send one (the phone, TV and browser runners cannot). +See [Authentication](https://addisdev.github.io/fleet-runner/deploy/auth/). +With tokens on, still serve the collector over TLS or keep it on a private +network: tokens are bearer secrets with no proof of possession, and unless +`FLEET_AUTH_PROTOCOL=token` is set the runner protocol answers anyone who can +reach it. That posture depends on every agent being on the same LAN, which stops being true the moment one of them is a laptop that leaves the house. `FLEET_BIND` diff --git a/collector/ci/README.md b/collector/ci/README.md index 2685fcb..6b8dd0d 100644 --- a/collector/ci/README.md +++ b/collector/ci/README.md @@ -20,11 +20,32 @@ published for an app, its nightly is **skipped rather than failed** — a job pinned to a build that does not exist would fail in a way indistinguishable from the app being broken. +**A pull request's build never moves `latest`.** An upload with +`x-artifact-channel: pr` is stored on the `pr` channel, and `latest` only +considers builds without it, so the nightly keeps testing what was merged. +`ci-upload` and `ci-enqueue` send the header themselves when +`GITHUB_EVENT_NAME` is `pull_request` (or `pull_request_target`); `--channel +pr|main` says it explicitly. The pull request's own job names its build by +hash, so it still tests exactly that build. Bytes republished from main come +off the channel; builds from before the channel existed count as merged. + ## 2. Test a pull request — `example-workflow.yml` Enqueue a job against real hardware and block the PR on the verdict. -`ci-enqueue` uploads the build, enqueues, polls to a verdict, and exits 0/1, so -the check fails on a red suite whether or not commit statuses are armed. +`ci-enqueue` uploads the build, enqueues through `/api/v1`, waits for the +verdict, and exits like `fleetr run`: 0 passed, 1 failed, 2 invalid spec, +3 still running at `--timeout`, 4 skipped (nothing ran), 5 refused or +unreachable. So the check fails on a red or skipped suite whether or not +commit statuses are armed. + +## Tokens + +With `FLEET_AUTH=tokens` on the collector, put a token in a secret and pass it +as `FLEET_RUNNER_TOKEN`: both scripts send it as `Authorization: Bearer`. +`ci-enqueue` needs `run`. The upload is the runner protocol, which is open +unless the collector also sets `FLEET_AUTH_PROTOCOL=token`; then both scripts' +uploads need `agent` as well. `FLEET_DASH_TOKEN` is still sent as +`x-fleet-token`, for a collector guarded only by that. ## Commit statuses are built and dark diff --git a/collector/ci/example-workflow.yml b/collector/ci/example-workflow.yml index d85ee43..613fef8 100644 --- a/collector/ci/example-workflow.yml +++ b/collector/ci/example-workflow.yml @@ -4,6 +4,10 @@ # this goes into an app repo (e.g. your-app-android). Until then this file # documents the contract and nothing reads it. # +# ci-enqueue exits like `fleetr run`: 0 passed, 1 failed, 2 invalid spec, +# 3 still running at --timeout, 4 skipped (nothing ran), 5 refused/unreachable. +# Any non-zero exit fails the check, so a skip is not a green check. +# # Requirements at connect time, none of which exist today by design: # - the runner host joined to the tailnet, FLEET_COLLECTOR_URL secret set # - FLEET_GITHUB_STATUS=1 + FLEET_GITHUB_TOKEN on the collector, so the @@ -23,6 +27,12 @@ jobs: run: ./gradlew :app:assembleDebug - name: Real-device smoke via fleet + env: + # A run token (plus agent if the collector sets FLEET_AUTH_PROTOCOL=token), + # needed only when the collector runs with FLEET_AUTH=tokens. + FLEET_RUNNER_TOKEN: ${{ secrets.FLEET_RUNNER_TOKEN }} + # GITHUB_EVENT_NAME is pull_request here, so the build is uploaded on + # the pr channel and never becomes the nightly's "latest". run: | cat > fleet-job.json <("/api/health", [], 0); + const health = useLoad("health", () => read(fleet.getHealth()), [], 0); const [token, setToken] = useToken(); const [draft, setDraft] = useState(""); diff --git a/collector/dash/src/api.ts b/collector/dash/src/api.ts index 7346c9f..7e13e13 100644 --- a/collector/dash/src/api.ts +++ b/collector/dash/src/api.ts @@ -2,6 +2,7 @@ import { useEffect, useRef, useState } from "preact/hooks"; import { onLive } from "./live.js"; import { tokenHeader } from "./token.js"; +import type { JobDetail } from "./client.js"; export class ApiError extends Error { constructor( @@ -38,18 +39,31 @@ export type Loadable = { data: T | null; error: Error | null; loading: boolea * of "3m ago" and lease countdowns goes stale on its own, with no event to say so. */ export function useApi(path: string | null, topics: string[] = [], refreshMs = 0): Loadable { + return useLoad(path, () => api(path!), topics, refreshMs); +} + +/** + * useApi for anything that returns a promise -- the generated /api/v1 client, + * mostly. `key` stands in for the path: it changes exactly when the request + * would, and null means "not yet". + */ +export function useLoad(key: string | null, load: () => Promise, topics: string[] = [], refreshMs = 0): Loadable { const [data, setData] = useState(null); const [error, setError] = useState(null); - const [loading, setLoading] = useState(path !== null); + const [loading, setLoading] = useState(key !== null); const [nonce, setNonce] = useState(0); // Guards against a slow first response overwriting a fast second one. const seq = useRef(0); + // The loader closes over the render that made it; the effect below runs on + // key changes only, so it reads the newest one from here. + const loader = useRef(load); + loader.current = load; useEffect(() => { - if (path === null) return; + if (key === null) return; const mine = ++seq.current; setLoading(true); - api(path) + loader.current() .then((d) => { if (seq.current === mine) { setData(d); @@ -62,7 +76,7 @@ export function useApi(path: string | null, topics: string[] = [], refreshMs .finally(() => { if (seq.current === mine) setLoading(false); }); - }, [path, nonce]); + }, [key, nonce]); const reload = () => setNonce((n) => n + 1); @@ -88,28 +102,16 @@ export function useApi(path: string | null, topics: string[] = [], refreshMs return { data, error, loading, reload }; } -// --- response shapes (mirrors src/api/*.ts) --- - -export type DeviceStatus = "online" | "stale" | "offline"; +// --- response shapes --- +// +// Devices, jobs and their details come from /api/v1, so their types are the +// generated client's (collector/src/client/api.gen.ts), re-exported here for +// the pages that import them from this file. Do not redeclare them: a copy is +// how the dashboard drifted from the server before. +export type { Device, DeviceDetail, DeviceList, Job, JobDetail, JobList } from "./client.js"; +export type ResultRow = JobDetail["results"][number]; -export type Beacon = { - battery_pct: number | null; - charging: boolean | null; - thermal: string | null; - mem_mb: number | null; - mem_method: string | null; - process_alive: boolean | null; - job_id: string | null; - /** - * Set when this device is running a job for a DIFFERENT collector. - * - * Rendered rather than ignored because "busy for another brain" and "idle, - * and nothing here is offering it work" look identical on this screen and are - * opposite problems -- one is working, the other is a queue that is not - * matching. - */ - busy: { job_id: string | null; collector: string | null } | null; -} | null; +// The rest mirror unversioned /api/* routes that /api/v1 does not offer yet. export type Peer = { url: string; @@ -126,74 +128,6 @@ export type Peers = { configured: number; }; -export type Device = { - device_id: string; - name: string | null; - notes: string | null; - descriptor: Record; - /** Effective pools: the override when set, otherwise what the runner reports. */ - pools: string[]; - /** null = an agent that predates capability routing; it is offered everything. */ - capabilities: string[] | null; - /** /24 the agent last registered from. Distinguishes "offline" from "elsewhere". */ - last_net: string | null; - pools_reported: string[]; - pools_override: string[] | null; - /** - * What the agent says it runs. An open string, not a union: the fleet gained - * a laptop, a TV, a watch and a browser tab after this type was written, and - * a union here would have had to be edited before any of them could show up - * correctly. Agents that predate the field are still reported as ios/android - * by the collector's fallback. - */ - platform: string; - /** Form factor, when the agent declared one: phone, tablet, laptop, tv, watch, sbc, browser… */ - kind: string | null; - /** Seconds of silence an ephemeral agent survives; null for a shelf device. */ - ttl_s: number | null; - /** True once an ephemeral agent's window closed. Such devices are hidden unless asked for. */ - expired: boolean; - simulator: boolean; - status: DeviceStatus; - age_s: number | null; - last_seen: string | null; - beacon: Beacon; - current_job: string | null; - lock: { job_id: string; acquired_at: string | null } | null; -}; - -export type Job = { - job_id: string; - workload: string; - executor: "device" | "host"; - status: "waiting" | "queued" | "claimed" | "done" | "failed" | "cancelled"; - /** Job ids this one waits for; it stays `waiting` until they are all done. */ - depends_on?: string[] | null; - /** Whether a higher-priority job may ask this one to checkpoint and step aside. */ - preemptible?: boolean; - claimed_by: string | null; - attempts: number; - max_attempts: number; - priority: number; - template_id: string | null; - lease_ttl_s: number; - lease_deadline: string | null; - lease_remaining_s: number | null; - duration_s: number | null; - last_error: string | null; - created_at: string | null; - claimed_at: string | null; - finished_at: string | null; - pool: string | null; - match: string | null; - device_id: string | null; - exclusive: boolean; - backend: string | null; - model: string | null; - app: string | null; - report_to: Record | null; -}; - export type Overview = { generated_at: string; devices: { @@ -214,6 +148,7 @@ export type Overview = { claimed: number; done_24h: number; failed_24h: number; + skipped_24h: number; last_attempt: number; oldest_queued_age_s: number | null; }; @@ -249,38 +184,6 @@ export type Overview = { health: Health; }; -export type DeviceList = { - total: number; - /** Facets over every device, not the filtered set, so a filter never empties its own menu. */ - pools: string[]; - platforms: string[]; - kinds: string[]; - devices: Device[]; -}; - -export type DeviceDetail = Device & { - counts: { results: number; beacons: number }; - jobs: { - job_id: string; - workload: string; - executor: string; - status: Job["status"]; - created_at: string | null; - finished_at: string | null; - attempts: number; - max_attempts: number; - last_error: string | null; - }[]; - benchmarks: { - job_id: string; - at: string | null; - model: string; - quant: string | null; - backend: string; - metrics: Record | null; - }[]; -}; - export type BeaconSample = { ts: string | null; battery_pct: number | null; @@ -294,44 +197,6 @@ export type BeaconSample = { export type BeaconHistory = { device_id: string; hours: number; count: number; samples: BeaconSample[] }; -export type JobList = { - page: number; - per_page: number; - total: number; - pages: number; - status_counts: Record; - workloads: string[]; - pools: string[]; - jobs: Job[]; -}; - -export type ResultRow = { - device_id: string; - iter: number; - created_at: string | null; - payload: Record; -}; - -export type JobDetail = Job & { - spec: Record; - parent: string | null; - children: Job[]; - siblings: Job[]; - results: ResultRow[]; - beacons: (BeaconSample & { device_id: string })[]; - artifacts: { - sha256: string; - name: string | null; - size: number | null; - created_at: string | null; - in_store: boolean; - role: "input" | "output"; - }[]; - locks: { device_id: string; acquired_at: string | null }[]; - status_report: { target: string; state: string; posted: boolean; detail: string | null; created_at: string | null } | null; - derived_timeline: { at: string | null; what: string }[]; -}; - export type RecentResults = { results: { job_id: string; @@ -348,6 +213,7 @@ export type RecentResults = { export type Locks = { locks: { device_id: string; job_id: string; acquired_at: string | null; held_s: number }[] }; +/** The health block inside /api/overview. The token banner reads /api/v1/health instead. */ export type Health = { ok: boolean; instance: string; @@ -364,69 +230,12 @@ export type Health = { // --- night-QA findings (mirrors src/findings.ts) --- -export type FindingVerdict = "real" | "duplicate" | "not_a_bug" | "agent_mistake"; - -export type FindingIssue = { - state: "dry_run" | "pending" | "filed" | "failed" | "capped" | "no_repo"; - /** True means nothing was sent to GitHub. */ - dry_run: boolean; - repo: string | null; - title: string; - body: string; - labels: string[]; - url: string | null; - number: number | null; - detail: string; - at: string; -}; - -export type Finding = { - id: number; - fingerprint: string; - app: string; - build: string; - platform: string; - device_id: string; - job_id: string; - last_job_id: string | null; - mission_id: string; - /** An open string, like Device.platform: the workload gains checks before the dashboard learns their names. */ - check: string; - severity: "high" | "medium" | "low"; - title: string; - detail: string; - screen: string; - screen_name: string; - steps: string[]; - replay: { kind: string; sha256: string | null; attempts: number; reproduced: number } | null; - /** "reproduced 2/2", "flaky 1/2", "crash log". */ - replay_label: string; - artifacts: { shot?: string; sheet?: string; trajectory?: string; log?: string; replay?: string }; - first_seen: string | null; - last_seen: string | null; - seen_count: number; - builds_seen: string[]; - verdict: FindingVerdict | null; - verdict_note: string | null; - verdict_at: string | null; - duplicate_of: number | null; - issue: FindingIssue | null; - status: "open" | "triaged"; -}; - -export type FindingDetail = Finding & { - duplicates: { id: number; title: string; seen_count: number }[]; - duplicate_of_title: string | null; -}; - -export type FindingList = { - findings: Finding[]; - counts: { open: number; triaged: number; total: number }; - /** Facets over every finding, not the filtered set. */ - apps: string[]; - checks: string[]; - limit: number; -}; +// Findings, one finding and the verdict come from /api/v1, so their types are +// the generated client's. The precision table is not in /api/v1 and keeps its +// own type below. +export type { Finding, FindingDetail, FindingList } from "./client.js"; +import type { Finding as V1Finding } from "./client.js"; +export type FindingVerdict = NonNullable; export type FindingPrecision = { floor: number; diff --git a/collector/dash/src/client.ts b/collector/dash/src/client.ts new file mode 100644 index 0000000..0be0e04 --- /dev/null +++ b/collector/dash/src/client.ts @@ -0,0 +1,64 @@ +// The dashboard's door into /api/v1: the generated client, with the dashboard's +// own way of carrying a token. +// +// The client and its types are generated from the route table, so a field the +// server renames is a compile error here rather than a blank column. Where the +// dashboard needs something /api/v1 does not offer, the page still calls the +// unversioned /api/* route through api.ts or mutate.ts. +import { FleetApiError, FleetClient } from "../../src/client/index.js"; +import type { DeviceList, JobSpecInput } from "../../src/client/index.js"; +import { getToken, tokenHeader } from "./token.js"; + +export type * from "../../src/client/index.js"; + +/** + * Same origin, and the token as `X-Fleet-Token` read at call time: that is the + * header FLEET_DASH_TOKEN's guard checks, and FLEET_AUTH=tokens accepts it as + * readily as `Authorization: Bearer`. The banner can change the token at any + * moment, so it is not baked into the client. + */ +export const fleet = new FleetClient({ + baseUrl: location.origin, + source: "dash", + fetch: (input, init) => + fetch(input, { + ...init, + headers: { accept: "application/json", ...(init?.headers as Record | undefined), ...tokenHeader() }, + }), +}); + +/** Turn a v1 refusal into the sentence a person should read. */ +function explain(e: unknown, mutation: boolean): Error { + if (!(e instanceof FleetApiError)) return e as Error; + // Naming the missing header is true and useless. Say what to do -- the same + // words mutate.ts uses for the routes not yet in /api/v1. + if (mutation && e.status === 401) + return new FleetApiError(401, e.code, getToken() + ? "That dashboard token was rejected. Check it on the System screen." + : "This collector needs a dashboard token — paste it in the banner above, or on the System screen."); + // A spec the strict schema refuses: say where, not only that. + const errors = (e.details as { schema_errors?: { path: string; message: string }[] } | undefined)?.schema_errors; + if (errors?.length) + return new FleetApiError(e.status, e.code, `${e.message}: ${errors.map((x) => `${x.path || "/"} ${x.message}`).join("; ")}`, e.hint, e.details); + return e; +} + +/** A read through the client, with errors in the dashboard's words. */ +export const read = (p: Promise): Promise => p.catch((e) => Promise.reject(explain(e, false))); + +/** A change through the client: a 401 says to paste a token, as every other button does. */ +export const change = (p: Promise): Promise => p.catch((e) => Promise.reject(explain(e, true))); + +/** + * Every device, however many pages that takes. The shelf is one screen, and + * the unversioned route it replaced answered with all of them at once. + */ +export async function allDevices(query: Parameters[0] = {}): Promise { + const first = await fleet.listDevices({ ...query, per_page: 500 }); + const items = [...first.items]; + for (let page = 2; page <= first.pages; page++) items.push(...(await fleet.listDevices({ ...query, per_page: 500, page })).items); + return { ...first, items, page: 1, per_page: items.length || first.per_page, pages: 1 }; +} + +/** The composer's spec is free-form JSON a person typed; the server is what checks it. */ +export const asSpec = (spec: Record) => spec as JobSpecInput; diff --git a/collector/dash/src/deviceKind.ts b/collector/dash/src/deviceKind.ts index a9aefce..1433201 100644 --- a/collector/dash/src/deviceKind.ts +++ b/collector/dash/src/deviceKind.ts @@ -7,7 +7,8 @@ * The model string comes first, because it is the one field that is right * about every device on the shelf today. `platform` is not: agents that * predate the field are reported as ios/android by the collector's fallback, - * so a Roku arrives as `android` and an Apple TV or Apple Watch as `ios`. + * so an Apple TV or Apple Watch arrives as `ios`, and a Roku registered by an + * executor older than 0.9 as `android` (newer ones say `roku`). * `kind` is the agent naming its own shape and is believed — but only to pick * the family; the model still chooses which member of it (a `tv` that says * "Roku Streaming Stick" is drawn as the stick, not a bare panel). diff --git a/collector/dash/src/keys.ts b/collector/dash/src/keys.ts index 8d4a966..34f2809 100644 --- a/collector/dash/src/keys.ts +++ b/collector/dash/src/keys.ts @@ -21,7 +21,7 @@ const GOTO: Record = { export const SHORTCUTS = [ ...Object.entries(GOTO).map(([k, [, label]]) => ({ keys: `g ${k}`, does: label })), { keys: "/", does: "Focus the search box on this screen" }, - { keys: "r d n a", does: "On a finding: Real, Duplicate, Not a bug, Agent's mistake" }, + { keys: "r d n a f", does: "On a finding: Real, Duplicate, Not a bug, Agent's mistake, Fixed (after Real)" }, { keys: "?", does: "Show this help" }, { keys: "Esc", does: "Close help, or leave the search box" }, ]; diff --git a/collector/dash/src/names.ts b/collector/dash/src/names.ts index 77523ac..61a42fd 100644 --- a/collector/dash/src/names.ts +++ b/collector/dash/src/names.ts @@ -9,6 +9,7 @@ // Every screen that prints a device id goes through here instead. One fetch, // shared, refreshed when a device changes — not one request per table row. import { useEffect, useState } from "preact/hooks"; +import { allDevices } from "./client.js"; import { onLive } from "./live.js"; type Names = Record; @@ -18,10 +19,9 @@ let inflight: Promise | null = null; const watchers = new Set<(n: Names) => void>(); async function load(): Promise { - const res = await fetch("/api/devices"); - const body = (await res.json()) as { devices: { device_id: string; name: string | null }[] }; + const body = await allDevices(); cache = Object.fromEntries( - body.devices.filter((d) => d.name).map((d) => [d.device_id, d.name as string]), + body.items.filter((d) => d.name).map((d) => [d.device_id, d.name as string]), ); for (const w of watchers) w(cache); return cache; diff --git a/collector/dash/src/pages/Alerts.tsx b/collector/dash/src/pages/Alerts.tsx index 20c73dc..b1884f0 100644 --- a/collector/dash/src/pages/Alerts.tsx +++ b/collector/dash/src/pages/Alerts.tsx @@ -1,10 +1,14 @@ // Alerts: what the collector thinks is wrong right now, and what it thought // was wrong earlier. -import { useApi } from "../api.js"; +import { useApi, useLoad } from "../api.js"; +import { fleet, read } from "../client.js"; import { mutate, useMutation } from "../mutate.js"; import { useQuery } from "../router.js"; import { Actions, Button, ErrorBox, Link, Loaded, Panel, Pill, Stat, agoFrom, clock, duration } from "../ui.js"; +// This screen reads /api/alerts, not /api/v1/alerts: it shows the counts per +// state, whether a webhook is set and the thresholds, which v1 does not return. +// The banner below needs only the list, and reads it from v1. export type Alert = { id: number; rule: string; @@ -147,8 +151,8 @@ export function Alerts() { /** The banner every screen carries. Silent unless something is actually open. */ export function AlertBanner() { - const state = useApi("/api/alerts?state=open", ["alert", "job", "device"], 60_000); - const open = state.data?.alerts ?? []; + const state = useLoad("alerts open", () => read(fleet.listAlerts({ state: "open" })), ["alert", "job", "device"], 60_000); + const open = state.data?.items ?? []; if (open.length === 0) return null; const worst = open.some((a) => a.severity === "critical") ? "critical" : "warning"; return ( diff --git a/collector/dash/src/pages/Compose.tsx b/collector/dash/src/pages/Compose.tsx index c053258..54755ff 100644 --- a/collector/dash/src/pages/Compose.tsx +++ b/collector/dash/src/pages/Compose.tsx @@ -5,11 +5,14 @@ // before anything is enqueued, and the equivalent curl is shown throughout — so // the form teaches the API rather than hiding it. import { useEffect, useState } from "preact/hooks"; -import { api, useApi, type DeviceList } from "../api.js"; +import { useLoad } from "../api.js"; +import { allDevices, asSpec, change, fleet, read } from "../client.js"; import { mutate, useMutation } from "../mutate.js"; import { navigate, useQuery } from "../router.js"; import { Actions, Button, ErrorBox, Field, Json, Loaded, Panel, Pill, Select } from "../ui.js"; +// The generated Template says only that spec is an object; the composer reads +// a saved spec the way it reads its own free-form half, as loosely typed JSON. type Template = { id: string; name: string | null; spec: Record }; // Defaults per workload, matching schemas/job.schema.json. These are starting @@ -94,8 +97,12 @@ function stamp(workload: string) { export function Compose() { const [q] = useQuery(); - const devices = useApi("/api/devices", ["device"]); - const templates = useApi<{ templates: Template[] }>("/api/templates", ["template"]); + const devices = useLoad("devices", () => read(allDevices()), ["device"]); + const templates = useLoad( + "templates", + () => read(fleet.listTemplates({ per_page: 500 })).then((t) => ({ templates: t.items as Template[] })), + ["template"], + ); const [workload, setWorkload] = useState(q.get("workload") ?? "benchmark"); const [jobId, setJobId] = useState(stamp(q.get("workload") ?? "benchmark")); @@ -190,7 +197,9 @@ export function Compose() { }, [pool, matchExpr, deviceId, executor, workload]); const enqueue = useMutation(async () => { - const out = await mutate<{ ok: boolean; job_id?: string; fanout?: string[] }>("POST", "/api/jobs", spec); + // /api/v1 checks the spec strictly: an unknown top-level field or params + // the workload's manifest does not allow is refused, with the paths named. + const out = await change(fleet.createJob(asSpec(spec ?? {}))); // Fan-out returns children; a single job returns its id. Land on whichever // page can actually show what just happened. if (out.fanout?.length) navigate(`/jobs?q=${encodeURIComponent(jobId)}`); @@ -208,7 +217,7 @@ export function Compose() { }); const curl = spec - ? `curl -X POST ${location.origin}/jobs \\\n -H 'content-type: application/json' \\\n -d '${JSON.stringify(spec)}'` + ? `curl -X POST ${location.origin}/api/v1/jobs \\\n -H 'content-type: application/json' \\\n -d '${JSON.stringify(spec)}'` : ""; return ( @@ -235,11 +244,11 @@ export function Compose() { {(d) => ( <>
- setQuery({ status: v })} /> - setQuery({ pool: v })} /> {/* Faceted from the response, never hard-coded: the platform set is open, so a literal list here would hide every device the fleet learned to run on after this file was written. */} - setQuery({ kind: v })} /> + setQuery({ kind: v })} /> )} setQuery({ q: v })} />