diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 826759c..98a62f5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -105,6 +105,42 @@ jobs: - uses: actions/checkout@v4 - run: node scripts/version.mjs check + # Also unconditional, and for the same reason as `version`: the numbers it + # checks are written in README.md and docs/, so a docs-only pull request is + # exactly where a stale one arrives, and no component suite runs for that. + repo-scripts: + name: repo scripts + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Unit tests for scripts/ + run: node --test scripts/*.test.mjs + - name: Documented counts match the code + run: node scripts/check-counts.mjs + + # A release pull request -- a `release/*` branch, or `develop` into `main` -- + # must mention every pull request merged into develop since the last tag. + # Every other pull request skips this, and skipped counts as green below. + changelog: + name: changelog coverage + if: >- + github.event_name == 'pull_request' && ( + startsWith(github.head_ref, 'release/') || + (github.head_ref == 'develop' && github.base_ref == 'main')) + runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: read + steps: + - uses: actions/checkout@v4 + with: + # The tags and the whole of develop's history since the last one. + fetch-depth: 0 + - name: Every merged pull request is in the changelog + env: + GH_TOKEN: ${{ github.token }} + run: node scripts/changelog-coverage.mjs --head "${{ github.event.pull_request.head.sha }}" + collector: needs: changes if: needs.changes.outputs.collector == 'true' @@ -142,7 +178,7 @@ jobs: ci-ok: name: ci-ok - needs: [changes, version, collector, runner-android, runner-ios, runner-machine, fleet, runner-roku, desktop] + needs: [changes, version, repo-scripts, changelog, collector, runner-android, runner-ios, runner-machine, fleet, runner-roku, desktop] # Without this the gate inherits the default "only if every dependency # succeeded", which makes a skipped suite skip the gate -- and a skipped # required check never reports, which is the exact bug being fixed. diff --git a/.github/workflows/desktop.yml b/.github/workflows/desktop.yml index ed6870c..2945167 100644 --- a/.github/workflows/desktop.yml +++ b/.github/workflows/desktop.yml @@ -27,3 +27,66 @@ jobs: ./desktop/stage-sidecar.sh - working-directory: desktop/src-tauri run: cargo check --all-targets + + launch: + # The first time the app ever ran was this job. `cargo check` proves the + # Tauri calls exist; this proves the binary starts, spawns `fleet up` as its + # sidecar, stays up, and that its settings window loads and answers over + # IPC. Linux rather than macOS because that is where Tauri's WebDriver + # support exists (tauri-driver wraps WebKitWebDriver; there is no macOS + # driver), and because Xvfb gives it a display without a logged-in desktop. + # What it cannot prove is anything macOS-specific: the tray in the menu bar, + # the Local Network grant, signing. See desktop/README.md. + name: launch smoke + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: "22.13" + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + with: + workspaces: desktop/src-tauri + key: linux-launch + # Tauri 2's Linux prerequisites (webkit2gtk 4.1, appindicator for the + # tray), plus what the smoke itself uses: Xvfb for a display, dbus for a + # session bus, WebKitWebDriver for tauri-driver, ImageMagick to capture + # the screen. + - name: Linux dependencies + run: | + sudo apt-get update + sudo apt-get install -y --no-install-recommends \ + libwebkit2gtk-4.1-dev build-essential curl wget file libxdo-dev \ + libssl-dev libayatana-appindicator3-dev librsvg2-dev \ + xvfb x11-utils dbus-x11 webkit2gtk-driver imagemagick jq + - name: Stage the fleet sidecar + run: | + npm --prefix collector ci + npm --prefix fleet ci + ./desktop/stage-sidecar.sh + - name: Build the app + working-directory: desktop/src-tauri + # custom-protocol is what `cargo tauri build` turns on: the frontend is + # embedded and served from the binary, as it is in a release, rather + # than expected from a dev server. + run: cargo build --features custom-protocol + - name: tauri-driver + run: cargo install tauri-driver --locked + - name: Launch it + run: ./desktop/ci/launch-smoke.sh + env: + OUT: ${{ runner.temp }}/desktop-smoke + - if: always() + name: What it printed + run: | + for f in "$RUNNER_TEMP"/desktop-smoke/*.log "$RUNNER_TEMP"/desktop-smoke/fleet-home/logs/*.log; do + [ -f "$f" ] || continue + echo "== $f"; tail -80 "$f" + done + - if: always() + uses: actions/upload-artifact@v4 + with: + name: desktop-launch-smoke + path: ${{ runner.temp }}/desktop-smoke + if-no-files-found: warn diff --git a/.github/workflows/fleet.yml b/.github/workflows/fleet.yml index 063754c..ef55942 100644 --- a/.github/workflows/fleet.yml +++ b/.github/workflows/fleet.yml @@ -89,8 +89,21 @@ jobs: # run it the way the docs say, and drive it through the CLI from outside: # a container whose port is published but whose collector never answers is # the failure a `docker build` alone would call green. - name: container smoke - runs-on: ubuntu-latest + # + # The arm64 row builds natively on GitHub's arm64 runner rather than under + # QEMU on the amd64 one. The release publishes linux/arm64 from a QEMU + # build, and an image that only ever ran emulated is the one a Raspberry Pi + # owner would be first to start for real. + name: container smoke${{ matrix.suffix }} + runs-on: ${{ matrix.runner }} + strategy: + fail-fast: false + matrix: + include: + - runner: ubuntu-latest + suffix: "" + - runner: ubuntu-24.04-arm + suffix: " (arm64)" steps: - uses: actions/checkout@v4 - uses: actions/setup-node@v4 @@ -99,13 +112,21 @@ jobs: cache: npm cache-dependency-path: fleet/package-lock.json - working-directory: . - run: docker build -f fleet/Dockerfile -t fleet:ci . + run: | + docker build -f fleet/Dockerfile -t fleet:ci . + docker image inspect fleet:ci --format 'built for {{.Os}}/{{.Architecture}}' + - working-directory: . run: docker run -d --name fleet -p 8788:8788 fleet:ci up --role brain,agent - name: The collector answers, and the machine agent registers shell: bash + env: + EXPECT_ARCH: ${{ matrix.runner == 'ubuntu-24.04-arm' && 'aarch64' || 'x86_64' }} run: | set -euo pipefail + arch=$(docker exec fleet uname -m) + echo "container is running on $arch" + [ "$arch" = "$EXPECT_ARCH" ] || { echo "::error::expected $EXPECT_ARCH, the container says $arch"; exit 1; } for i in $(seq 1 60); do curl -fsS http://127.0.0.1:8788/api/v1/health && break sleep 2 @@ -165,3 +186,138 @@ jobs: run: | export XDG_RUNTIME_DIR="/run/user/$(id -u)" journalctl --user -u fleet.service --no-pager | tail -100 || true + + launchd: + # The twin of the systemd smoke for macOS. `fleet service install` writes a + # user LaunchAgent and bootstraps it into the gui/ domain; GitHub's + # macOS runners run their jobs inside a logged-in GUI session, so that + # domain exists here the way it does on a desk. The first step says so + # rather than assuming it, because a runner without one would make every + # later failure look like a fleet bug. + name: launchd smoke + runs-on: macos-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: "22.13" + cache: npm + cache-dependency-path: fleet/package-lock.json + - run: npm ci && npm --prefix ../collector ci && npm run build + - name: The runner has a GUI launchd domain + shell: bash + run: | + set -euo pipefail + echo "uid $(id -u), user $(id -un)" + launchctl print "gui/$(id -u)" | head -15 + - name: Install as a LaunchAgent, it serves, and its agent registers + shell: bash + run: | + set -euo pipefail + mkdir -p "$HOME/.fleet" + cp -R dist/. "$HOME/.fleet/" + fleet() { node "$HOME/.fleet/bin/fleet.mjs" "$@"; } + plist="$HOME/Library/LaunchAgents/com.addisdev.fleet.plist" + fleet service install + plutil -lint "$plist" + launchctl print "gui/$(id -u)/com.addisdev.fleet" | head -20 + for i in $(seq 1 60); do + curl -fsS http://127.0.0.1:8788/api/v1/health && break + sleep 2 + done + curl -fsS http://127.0.0.1:8788/api/v1/health >/dev/null + n=0 + for i in $(seq 1 30); do + n=$(fleet devices --url http://127.0.0.1:8788 --json | node -e 'let s="";process.stdin.on("data",d=>s+=d).on("end",()=>console.log(JSON.parse(s).items.length))') + [ "$n" -gt 0 ] && break + sleep 2 + done + [ "$n" -gt 0 ] || { echo "::error::the agent under launchd never registered"; exit 1; } + fleet devices --url http://127.0.0.1:8788 + fleet service status + # Reinstalling is bootout-then-bootstrap, the race service.ts retries. + # Run it once here so the retry is exercised rather than trusted. + fleet service install + for i in $(seq 1 60); do + curl -fsS http://127.0.0.1:8788/api/v1/health >/dev/null 2>&1 && break + sleep 2 + done + curl -fsS http://127.0.0.1:8788/api/v1/health >/dev/null + fleet service uninstall + sleep 5 + if curl -fsS http://127.0.0.1:8788/api/v1/health 2>/dev/null; then + echo "::error::the collector is still answering after uninstall" + exit 1 + fi + [ ! -e "$plist" ] || { echo "::error::$plist is still there after uninstall"; exit 1; } + if launchctl print "gui/$(id -u)/com.addisdev.fleet" >/dev/null 2>&1; then + echo "::error::launchd still has com.addisdev.fleet loaded after uninstall" + exit 1 + fi + if fleet service status; then + echo "::error::fleet service status exited 0 after uninstall" + exit 1 + fi + - if: failure() + shell: bash + run: | + tail -100 "$HOME/.fleet/logs/fleet.log" || true + for f in "$HOME"/.fleet/logs/*.log; do echo "== $f"; tail -40 "$f"; done || true + + windows-task: + # `fleet service install` on Windows creates a scheduled task that runs + # `fleet up` at logon and starts it at once. Same lifecycle as the systemd + # and launchd smokes: install, it serves, its agent registers, status says + # so, uninstall, and nothing is left answering. + name: windows task smoke + runs-on: windows-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: "22.13" + cache: npm + cache-dependency-path: fleet/package-lock.json + - shell: bash + run: npm ci && npm --prefix ../collector ci && npm run build + - name: Install as a scheduled task, it serves, and its agent registers + shell: bash + run: | + set -euo pipefail + home="$(cygpath -u "$USERPROFILE")/.fleet" + mkdir -p "$home" + cp -R dist/. "$home/" + fleet() { node "$home/bin/fleet.mjs" "$@"; } + fleet service install + schtasks //Query //TN FleetRunner //FO LIST //V + for i in $(seq 1 60); do + curl -fsS http://127.0.0.1:8788/api/v1/health && break + sleep 2 + done + curl -fsS http://127.0.0.1:8788/api/v1/health >/dev/null + n=0 + for i in $(seq 1 30); do + n=$(fleet devices --url http://127.0.0.1:8788 --json | node -e 'let s="";process.stdin.on("data",d=>s+=d).on("end",()=>console.log(JSON.parse(s).items.length))') + [ "$n" -gt 0 ] && break + sleep 2 + done + [ "$n" -gt 0 ] || { echo "::error::the agent under the scheduled task never registered"; exit 1; } + fleet devices --url http://127.0.0.1:8788 + fleet service status + fleet service uninstall + sleep 5 + if curl -fsS http://127.0.0.1:8788/api/v1/health 2>/dev/null; then + echo "::error::the collector is still answering after uninstall" + exit 1 + fi + if schtasks //Query //TN FleetRunner >/dev/null 2>&1; then + echo "::error::the FleetRunner task is still there after uninstall" + exit 1 + fi + - if: failure() + shell: bash + run: | + home="$(cygpath -u "$USERPROFILE")/.fleet" + schtasks //Query //TN FleetRunner //FO LIST //V || true + for f in "$home"/logs/*.log; do echo "== $f"; tail -60 "$f"; done || true + tasklist //FI "IMAGENAME eq node.exe" || true diff --git a/AGENTS.md b/AGENTS.md index 35d170b..17f1eb8 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -13,7 +13,7 @@ from an agent, see [`skills/fleet-runner/SKILL.md`](skills/fleet-runner/SKILL.md | `collector/src/api/v1/` | The control API. `routes.ts` is the single route table: it registers the routes, generates `openapi.json` and the typed client | | `collector/src/client/` | The generated client (`api.gen.ts`) and its transport. The CLI and the MCP server use nothing else | | `fleet/` | The `fleet` / `fleetr` command: runs a brain or a runner, plus the work commands, `fleet mcp` and the remote gateway | -| `desktop/` | A Tauri menu-bar app around `fleet`. A front door, not a runner; compiled in CI and never launched | +| `desktop/` | A Tauri menu-bar app around `fleet`. A front door, not a runner; compiled in CI on macOS and launched in CI on Linux (`desktop/ci/launch-smoke.sh`), never run on a Mac | | `runner-machine/`, `runner-android/`, `runner-ios/`, `runner-roku/`, `collector/runner-web/` | Runners. They speak the runner protocol (`/devices/register`, `/devices/:id/next-job`, `/results`), not `/api/v1` | | `skills/fleet-runner/` | The agent skill. Its tables are generated | | `bug-garden/` | An Android fixture app with twenty planted defects, for scoring exploratory QA | diff --git a/CHANGELOG.md b/CHANGELOG.md index b1b6bbf..9bf48cd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,8 +16,43 @@ a CI job fails when they disagree. ## [Unreleased] +### Added + +- **A changelog coverage check for releases.** On a `release/*` pull request, + and on `develop` into `main`, `changelog coverage` in `ci.yml` lists every + pull request merged into `develop` since the last tag and fails, naming + each, if the section being released does not mention it as `#N` or a + `/pull/N` link. 0.8.0's notes left out work that had shipped in it and had + to be amended the day after; this is the check that would have caught it. + Label a pull request `no-changelog` to exempt it; Dependabot's are exempt. + The logic is `scripts/changelog-coverage.mjs`, with unit tests. + ([#89](https://github.com/addisdev/fleet-runner/pull/89)) +- **The documented workload count is checked.** The README and the docs said + 28 while the catalog grew to 34. Each hand-written count is now marked + `34`, and `repo scripts` in `ci.yml` fails on any + pull request, docs-only ones included, when a marked number disagrees with + the shipped entries in `collector/src/workloads/catalog.ts`. The collector's + suite checks that reading against the real `CATALOG`. ([#89](https://github.com/addisdev/fleet-runner/pull/89)) + ### Fixed +- **`fleet service install` on Windows had never started anything.** The + scheduled task's command kept the backslashes of a `\"` escaping meant for a + `cmd` prompt -- execFile quotes the argument itself -- so the task ran + `\"C:\...\node.exe\"` and failed with "file not found" every time. Found by + the new Windows smoke on its first run. `fleet service uninstall` and `stop` + also now kill `fleet up`'s process tree before ending the task: deleting a + task does not stop it, and Windows does not take a parent's children down + with it. ([#89](https://github.com/addisdev/fleet-runner/pull/89)) +- **The desktop app left `fleet up` running when it was killed, and its Quit + item could not quit.** SIGTERM (a `kill`, a `killall`, the end of a session) + killed the app outright and orphaned its sidecar on the collector's port. And + `RunEvent::ExitRequested` refused every exit, including the `app.exit(0)` the + Quit item calls, so Quit stopped the fleet and left the app running. SIGTERM, + SIGINT and SIGHUP now take the Quit path -- stop `fleet up`, then exit -- and + only the implicit last-window exit is refused. Found by the first launch of + the app, in CI. ([#89](https://github.com/addisdev/fleet-runner/pull/89)) + - **`fleet skill` failed on an installed 0.8.0.** The 0.8.0 archives carry `skills/`, but `install.sh` and `install.ps1` copy a release's directories by name and did not name it, so `fleet skill show` and `fleet skill install` @@ -25,12 +60,46 @@ a CI job fails when they disagree. the bundle gains a directory the installers do not copy. The installers are fetched from `main`, so re-running the install command fixes an existing 0.8.0 install; no new release is needed. + ([#86](https://github.com/addisdev/fleet-runner/pull/86)) - **`bug-garden/tools/check-bench.mjs` passed `bench-delete-plant` without checking the plant was gone.** The card's `check` gained `absent_text`, which the harness enforces and the standalone checker ignored; it also matched case-sensitively where the harness does not. It now reads `absent_text` and matches case-insensitively, as the bug garden's README says. + +### Now verified + +Four items from earlier "Not verified" lists, each now checked by +GitHub-hosted CI on every pull request that touches them. ([#89](https://github.com/addisdev/fleet-runner/pull/89)) + +- **`fleet service install` on launchd.** `fleet / launchd smoke` installs the + LaunchAgent on `macos-latest` (the runner has a `gui/` domain), waits + for its collector to answer and its agent to register, checks + `fleet service status`, reinstalls once so the bootstrap race is hit and + retried (it was), then uninstalls and checks nothing answers, the plist is + gone and launchd no longer has the job. +- **The Windows scheduled task.** `fleet / windows task smoke` runs the same + lifecycle on `windows-latest`. It found the bug above; with it fixed, a + Windows machine agent registered against a real collector for the first + time. +- **The arm64 container image.** `fleet / container smoke (arm64)` builds + `fleet/Dockerfile` natively on `ubuntu-24.04-arm`, checks the container + reports `aarch64`, and waits for its agent to register and answer + `fleet devices`, `fleet workloads` and `fleet whoami`. +- **The desktop app launches.** `desktop / launch smoke` builds it on Ubuntu + and runs it under Xvfb: its `fleet` sidecar starts a collector whose agent + registers, the app is still up 20 seconds later, and SIGTERM stops both. + Then, under `tauri-driver`, the settings window is driven over WebDriver: + it is titled `Fleet`, `get_status` answers over IPC with the sidecar + running, and a screenshot is uploaded. This was the first time the app ever + ran. It has still never run on a Mac, and the tray menu has never been + clicked. + +Still not verified: any of the three services coming back after a reboot, a +graceful stop on Windows, and the MCP gateway against a real identity provider +(it needs a tenant). + ## [0.8.0] — 2026-10-15 The fleet as something a program or an AI agent can drive: a versioned control diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 835d0a2..c86eb27 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -151,7 +151,16 @@ this is the first rule rather than a nice sentiment. - Run the suite for what you touched, and say in the description what you ran. - Update the docs in the same change. The site builds with `--strict`, so a broken link fails CI. -- Add a `CHANGELOG.md` entry under `## [Unreleased]` for anything user-visible. +- Add a `CHANGELOG.md` entry under `## [Unreleased]` for anything user-visible, + ending with a link to the pull request + (`([#86](https://github.com/addisdev/fleet-runner/pull/86))`) once it has a + number. A pull request nobody using Fleet Runner would notice — CI plumbing, + a typo — gets the `no-changelog` label instead. The release check below holds + both to it. +- A number the docs state by hand (how many workloads there are) is marked + `34`. `repo scripts` in `ci.yml` fails when a marked + number disagrees with `collector/src/workloads/catalog.ts`; fix the number, + keep the marker. `ci-ok` is the only required check. It always reports, and it passes when every component suite is green **or skipped** — a change to one runner does not build @@ -161,7 +170,8 @@ the others. One version covers every component, so a release is one number everywhere. -1. On a branch from `develop`, run `node scripts/version.mjs set X.Y.Z`. It +1. On a branch named `release/X.Y.Z` from `develop`, run + `node scripts/version.mjs set X.Y.Z`. It writes `VERSION` and every component's own version except the desktop app's (`desktop/src-tauri/Cargo.toml` and `tauri.conf.json` are not among the places it writes, and still say 0.5.0). Then run @@ -173,6 +183,17 @@ One version covers every component, so a release is one number everywhere. at the bottom. 3. Merge that into `develop`, then open a `develop` → `main` pull request named `Release X.Y.Z` and merge it once `ci-ok` passes. + + On both pull requests `ci.yml` runs `changelog coverage` + (`scripts/changelog-coverage.mjs`). It lists every pull request merged + into `develop` since the last `v*` tag and fails, naming each, if one is + not mentioned as `#N` or a `/pull/N` link between `## [Unreleased]` and the + last release's heading. Label a pull request `no-changelog` to exempt it; + Dependabot's are exempt already. The check only runs when the branch is + `release/*`, or `develop` into `main` — so name the branch that way, or it + will not look. Run it yourself first with + `node scripts/changelog-coverage.mjs --head origin/develop` (it needs `gh` + for the labels). 4. Tag the merge commit on `main` and push the tag: `git tag -a vX.Y.Z -m "vX.Y.Z" && git push origin vX.Y.Z`. diff --git a/README.md b/README.md index c46f42b..9298f91 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,7 @@ hardware?** It turned out to be yes, and the fleet is how I know. | **Problem** | Mobile tests and ML benchmarks are only useful when they run on the hardware, under conditions, and with the exact artifacts a product will ship. | | **System** | One collector owns the registry, queue, leases, artifacts, results and dashboard; independently implemented agents claim work through one tested JSON protocol. | | **Reach** | Five runner implementations across TypeScript, Swift, Kotlin, JavaScript and BrightScript, plus host-driven iOS and Android workloads. | -| **Workloads** | 34 workload types spanning UI tests, visual regression, exploratory QA by a vision model, llama.cpp, Core ML, LiteRT, battery experiments, builds and deployment. | +| **Workloads** | 34 workload types spanning UI tests, visual regression, exploratory QA by a vision model, llama.cpp, Core ML, LiteRT, battery experiments, builds and deployment. | | **Proof** | The first production question found an int8 CPU model faster to load and run than its fp32 GPU alternative while preserving useful accuracy. | **What this demonstrates:** distributed systems, mobile platforms, protocol @@ -140,7 +140,7 @@ something a runner can do without the collector shipping a release. ![Twenty-eight workloads in three columns by who claims them: on the device, on a host, and on a machine](docs/img/workloads.png) -Thirty-four workloads (the figure predates six of them, `explore` and `tvloop` +Thirty-four workloads (the figure predates six of them, `explore` and `tvloop` among them), and the column a workload sits in is the answer to "who can physically do this". A benchmark runs inside the app on the phone. An install needs a cable and a Mac. A build needs a checkout and a toolchain. @@ -177,7 +177,7 @@ languages, sharing not one line | | What it is | |---|---| | **[fleet/](fleet)** | The CLI. `fleet up` runs this machine's components under a supervisor that backs off and gives up loudly rather than restarting a broken component forever; `fleet doctor` says what the machine can run and why not; `fleet service install` keeps it up. It wraps the three programs above and reimplements none of them. | -| **[desktop/](desktop)** | A Tauri menu-bar app around that CLI: a tray menu, role switches, the dashboard in a window, and a notification when a component gives up. It exists to test one hypothesis about macOS local-network permissions, and **it has never been launched**: CI compiles its Rust and that is all -- read its README before believing anything in it. | +| **[desktop/](desktop)** | A Tauri menu-bar app around that CLI: a tray menu, role switches, the dashboard in a window, and a notification when a component gives up. It exists to test one hypothesis about macOS local-network permissions, and **it has never run on a Mac**: CI compiles it on macOS, and launches it on Linux under a virtual display, where it starts its `fleet` sidecar and its settings window answers over WebDriver. The tray, the Mac and the hypothesis are untested -- read its README before believing anything in it. | **Alongside them** diff --git a/collector/README.md b/collector/README.md index f44d9fd..dc85912 100644 --- a/collector/README.md +++ b/collector/README.md @@ -99,7 +99,7 @@ so it never touches a real fleet's history. | `model-convert` / `dataset-prep` | machine | A checkpoint into GGUF, Core ML or TFLite; an eval set that carries its licence and source | | `serve` / `shell` | machine | Host a model for the length of a lease; run a script whose hash the machine's owner pinned by hand | -That is all 34 shipped workloads. `GET /api/v1/workloads` (or `fleetr +That is all 34 shipped workloads. `GET /api/v1/workloads` (or `fleetr workloads`) lists them with their params and an example spec, plus anything a registered agent declares that the collector does not know. diff --git a/collector/scripts/check-counts.ts b/collector/scripts/check-counts.ts new file mode 100644 index 0000000..76a0c37 --- /dev/null +++ b/collector/scripts/check-counts.ts @@ -0,0 +1,25 @@ +/** + * scripts/check-counts.mjs reads the catalog right. + * + * That script counts shipped workloads by reading catalog.ts as text, so it can + * run on every pull request without `npm ci` and check the numbers the README + * and the docs state by hand. Reading TypeScript as text is a guess about its + * shape; this imports the real CATALOG and fails if the guess and the truth + * disagree, so a refactor of catalog.ts cannot quietly turn the docs check into + * a check of the wrong number. + */ +import { execFileSync } from "node:child_process"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { CATALOG } from "../src/workloads/catalog.js"; + +const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../.."); +const truth = CATALOG.filter((w) => w.status !== "reserved").length; +const out = execFileSync(process.execPath, [path.join(ROOT, "scripts/check-counts.mjs"), "--json"], { encoding: "utf8" }); +const parsed = (JSON.parse(out) as { workloads: number }).workloads; + +if (parsed !== truth) { + console.error(` FAIL scripts/check-counts.mjs counts ${parsed} shipped workloads; CATALOG has ${truth}`); + process.exit(1); +} +console.log(` ok scripts/check-counts.mjs and CATALOG agree: ${truth} shipped workloads`); diff --git a/collector/scripts/test.ts b/collector/scripts/test.ts index dd1a2b6..8d98c6a 100644 --- a/collector/scripts/test.ts +++ b/collector/scripts/test.ts @@ -183,6 +183,12 @@ if (!(await run(process.execPath, [TSX, "scripts/check-metrics.ts"]))) failed = step("workload table matches the directories"); if (!(await run(process.execPath, [TSX, "scripts/check-workloads.ts"]))) failed = true; +// The README and the docs state the workload count by hand, marked with +// . ci.yml checks the marks on every pull request by +// reading catalog.ts as text; this is what keeps that reading honest. +step("documented counts read the catalog right"); +if (!(await run(process.execPath, [TSX, "scripts/check-counts.ts"]))) failed = true; + // The shelf picks each device's drawing from its model string; the real // fleet's strings are the fixtures, so a regex change cannot quietly redraw // a device that is actually on the shelf. diff --git a/desktop/README.md b/desktop/README.md index 5dc157d..ddf0970 100644 --- a/desktop/README.md +++ b/desktop/README.md @@ -217,14 +217,41 @@ the first `codesign` invocation. ## Status -Be suspicious of this whole directory. **The Rust compiles, and that is all -that is known about it.** There is no Rust toolchain on the machine it was -written on, so CI does the compiling: `desktop / cargo check` stages the -sidecar and runs `cargo check --all-targets` on macOS for every pull request -that touches `desktop/`. +Be suspicious of this directory, a little less than before. There is no Rust +toolchain on the machine it was written on, so CI does the compiling and, since +the launch smoke, the running. Two jobs in `.github/workflows/desktop.yml` run +for every pull request that touches `desktop/`: + +- **`desktop / cargo check`** stages the sidecar and runs + `cargo check --all-targets` on macOS. +- **`desktop / launch smoke`** builds the app on Ubuntu + (`cargo build --features custom-protocol`, the frontend embedded as in a + release) and runs [`ci/launch-smoke.sh`](ci/launch-smoke.sh) under Xvfb. That + was the first time the app ever ran. What **has** actually been checked: +- **It launches, on Linux.** Started with no config file, the binary spawns + `fleet up` as its sidecar through the staged shim; the collector it starts + answers `/api/v1/health` and the machine agent registers with it, inside a + few seconds. The app is still running, and the sidecar still answering, 20 + seconds later. +- **The settings window loads and talks to the app.** The smoke starts the app + again under `tauri-driver` (Tauri's WebDriver server, wrapping + `WebKitWebDriver`) and drives it over plain W3C WebDriver: the window title + is `Fleet`, `window.__TAURI__` is on the page, `get_status` answers over IPC + with `running: true` and the sidecar's own output, the page renders + `Running: brain, agent · :8788`, and a screenshot of the window is uploaded + as the `desktop-launch-smoke` artifact. So the gold-standard check, not just + a process-level one. +- **SIGTERM stops the sidecar and exits.** The smoke sends the app SIGTERM and + checks that the app exits, `fleet up` is gone and nothing answers on the + port. Its first run found the opposite, and two bugs behind it, both fixed: + the app had no signal handler, so SIGTERM killed it outright and orphaned + `fleet up` on the port; and `RunEvent::ExitRequested` refused *every* exit, + including the `app.exit(0)` the Quit item calls, so Quit would have stopped + the fleet and then not quit. SIGTERM, SIGINT and SIGHUP now take the Quit + path (stop, then exit), and only the implicit last-window exit is refused. - **`cargo check` passes.** Its first run failed in the build script, before any Rust was compiled, because the bundled resource and the sidecar were both named `fleet` in `target/debug`. With the resource renamed `fleet-tree` @@ -237,19 +264,35 @@ What **has** actually been checked: arranged into a fake `Contents/MacOS` + `Contents/Resources` layout, and `fleet version` and `fleet config` were run through the shim under `env -i PATH=/usr/bin:/bin:/usr/sbin:/sbin` — the Finder environment, with no - Homebrew on `PATH`. Both worked. That is the piece most likely to be wrong and - it is the piece that is tested. + Homebrew on `PATH`. Both worked. On Linux it also finds the tree beside a + dev build's binary (`target/debug/fleet-tree`), which is what the launch + smoke runs. - The Rust config struct's serialisation order was compared against real `fleet config` output. It matches. +One thing the launch smoke had to work around, and that a Linux user would hit: +the shim checks `/usr/local/bin/node` before `PATH`, and GitHub's Ubuntu image +keeps an older Node there, below the 22.13 `node:sqlite` needs. The smoke sets +`FLEET_NODE`. The order is right for the Mac it was written for (Finder gives an +app no Homebrew on `PATH`); on Linux it can pick a stale system Node over the +one you meant. + What has **not**: -- **It has never run.** Not the tray, not the settings window, not the sidecar - under Tauri, not a notification, not the dashboard window. Compiling says - the Tauri 2 calls exist with those signatures, not that they do what the - code assumes. The Cargo package is `fleet-desktop` - and `productName` is `Fleet`; if the bundler cannot find its binary, that - mismatch is why, and `mainBinaryName` in `tauri.conf.json` is the knob. +- **It has never run on macOS**, which is the platform it is for. Linux is + where Tauri's WebDriver exists, so that is where it was launched. Nothing + macOS-specific has been seen: not the menu-bar tray, not the + `Accessory` activation policy, not a notification. +- **The tray and the dashboard window, anywhere.** Under Xvfb there is no + status-notifier host for the tray to appear in, so the tray menu -- Open + Dashboard, Start, Stop, Quit -- has never been clicked, and the dashboard + window it opens has never been opened. The Quit item's code + path (stop, then `app.exit(0)`) is the one SIGTERM now takes, and that is + checked; the menu item itself is not. +- **A bundle.** The smoke runs a `cargo build`, not `cargo tauri build`. The + Cargo package is `fleet-desktop` and `productName` is `Fleet`; if the bundler + cannot find its binary, that mismatch is why, and `mainBinaryName` in + `tauri.conf.json` is the knob. - **The Local Network hypothesis.** See The bet, above. Everything this app is *for* rests on it and none of it is tested. - **Info.plist merging.** That Tauri's bundler merges `src-tauri/Info.plist` into @@ -266,11 +309,11 @@ What has **not**: otherwise, `disable-library-validation` because `fleet up` re-execs itself as its own children — but nothing has been signed, and the switch-to-Electron threshold above is measured from here. -- **Windows and Linux.** Untouched. `terminate()` on Windows is a no-op with a - comment saying so, because `TerminateProcess` does not run `fleet up`'s +- **Windows.** Untouched. `terminate()` on Windows is a no-op with a comment + saying so, because `TerminateProcess` does not run `fleet up`'s `process.on("SIGTERM")` handler and faking a graceful stop would be worse than - admitting there is not one. The CLI has never been run on either platform - either. + admitting there is not one. Linux has been launched in CI (above) and used by + nobody. - **Notification permission.** On macOS this needs a signed, bundled app; an unsigned `cargo tauri dev` build may get nothing. The tray headline says the same thing without needing permission from anybody, which is the deliberate diff --git a/desktop/ci/launch-smoke.sh b/desktop/ci/launch-smoke.sh new file mode 100755 index 0000000..6c4f385 --- /dev/null +++ b/desktop/ci/launch-smoke.sh @@ -0,0 +1,225 @@ +#!/usr/bin/env bash +# Launch the desktop app on Linux under a virtual display and prove it ran. +# +# Run by `desktop / launch smoke` in .github/workflows/desktop.yml, after +# `./stage-sidecar.sh` and `cargo build --features custom-protocol` in +# src-tauri/. It needs Xvfb, dbus-launch, ImageMagick's `import`, curl, jq, +# and -- for the second half -- `tauri-driver` and `WebKitWebDriver` on PATH. +# +# Two halves, because they prove different things: +# +# 1. The process. Start the binary as a person would, with no config file, and +# watch for what it is supposed to do at launch: spawn `fleet up` as its +# sidecar, which starts a collector on :8788 whose agent registers. Then +# check the app is still up after a while, because a Tauri app that panics +# in its event loop does so after setup has already returned. +# 2. The window. Start it again under tauri-driver, the WebDriver server Tauri +# publishes, and drive the settings window from outside: that the page +# loaded, that `window.__TAURI__` exists, that a #[tauri::command] answers +# over IPC, and a screenshot of what it drew. +# +# Everything lands in $OUT (default $RUNNER_TEMP/desktop-smoke) for the +# workflow to upload, pass or fail. +set -euo pipefail + +here=$(cd "$(dirname "$0")" && pwd) +tauri="$here/../src-tauri" +bin="$tauri/target/debug" +app="$bin/fleet-desktop" +out="${OUT:-${RUNNER_TEMP:-/tmp}/desktop-smoke}" +mkdir -p "$out" + +port=8788 +base="http://127.0.0.1:$port" + +# A fleet home of its own, so the run cannot read or write anybody's +# ~/.fleet. config.rs and the CLI both honour FLEET_HOME, and the sidecar +# inherits this environment. +export FLEET_HOME="$out/fleet-home" +rm -rf "$FLEET_HOME" + +# The sidecar shim checks /usr/local/bin/node before PATH, on purpose (a +# Finder-launched app has no Homebrew on PATH). GitHub's Ubuntu image keeps an +# older Node there, below the 22.13 node:sqlite needs, so the one setup-node +# installed is named explicitly -- the same escape hatch a person would use. +export FLEET_NODE="${FLEET_NODE:-$(command -v node)}" +echo "node: $FLEET_NODE ($("$FLEET_NODE" --version))" + +fleet() { "$FLEET_NODE" "$tauri/resources/fleet/bin/fleet.mjs" "$@"; } + +fail() { echo "::error::$*"; exit 1; } + +echo "== what cargo build put beside the binary" +ls -la "$bin" | grep -E 'fleet' || true +[ -x "$app" ] || fail "$app was not built" +[ -x "$bin/fleet" ] || fail "the sidecar shim was not copied to $bin/fleet" +[ -f "$bin/fleet-tree/bin/fleet.mjs" ] || fail "the fleet tree was not copied to $bin/fleet-tree" + +echo "== virtual display and session bus" +export DISPLAY=:99 +Xvfb "$DISPLAY" -screen 0 1280x800x24 >"$out/xvfb.log" 2>&1 & +xvfb=$! +for _ in $(seq 1 50); do xdpyinfo >/dev/null 2>&1 && break; sleep 0.2; done +xdpyinfo >/dev/null 2>&1 || fail "Xvfb did not come up" +eval "$(dbus-launch --sh-syntax)" + +health() { curl -fsS --max-time 2 "$base/api/v1/health" >/dev/null 2>&1; } + +device_count() { + local n + n=$(fleet devices --url "$base" --json 2>/dev/null | jq '.items | length' 2>/dev/null || true) + echo "${n:-0}" +} + +wait_for_fleet() { + local pid=$1 + for _ in $(seq 1 90); do + kill -0 "$pid" 2>/dev/null || fail "the app exited before its sidecar answered (see app.log)" + health && break + sleep 1 + done + health || fail "nothing answered $base/api/v1/health within 90s of launch" + local n=0 + for _ in $(seq 1 45); do + n=$(device_count) + [ "$n" -gt 0 ] && break + sleep 1 + done + [ "$n" -gt 0 ] || fail "the sidecar's collector answered but its agent never registered" + echo "sidecar up: collector answers, $n device(s) registered" +} + +# `fleet up` writes its own pid to logs/supervisor.json. Used to clean up +# between the halves -- by pid, never by name -- and to see whether quitting the +# app took the sidecar with it. +sidecar_pid() { jq -r '.pid // empty' "$FLEET_HOME/logs/supervisor.json" 2>/dev/null || true; } + +stop_sidecar() { + local spid + spid=$(sidecar_pid) + if [ -n "$spid" ] && kill -0 "$spid" 2>/dev/null; then + kill -TERM "$spid" || true + for _ in $(seq 1 30); do kill -0 "$spid" 2>/dev/null || break; sleep 0.5; done + fi + for _ in $(seq 1 20); do health || break; sleep 0.5; done + if health; then fail "something still answers $base after stopping the sidecar"; fi +} + +# --- 1. the process --------------------------------------------------------- +echo "== half 1: launch the binary" +health && fail "something is already on $base before launch; the app would refuse to start its sidecar" +"$app" >"$out/app.log" 2>&1 & +apppid=$! +echo "app pid $apppid" +wait_for_fleet "$apppid" +fleet devices --url "$base" | tee "$out/devices.txt" + +settle=${SETTLE_S:-20} +echo "waiting ${settle}s to see that the app stays up" +sleep "$settle" +kill -0 "$apppid" 2>/dev/null || fail "the app died within ${settle}s of launch (see app.log)" +health || fail "the sidecar stopped answering within ${settle}s of launch" +echo "app still running after ${settle}s, sidecar still answering" +import -window root "$out/screen-after-launch.png" || echo "warning: import could not capture the screen" + +# SIGTERM is what a `kill`, a `killall Fleet` or the end of a session sends, +# and the app treats it as the Quit item: stop `fleet up` (it gets SIGTERM and +# up to 15s to drain), then exit. The first run of this smoke found the +# opposite -- the app died on the spot and its sidecar kept the port -- so this +# is asserted, not just printed. +spid=$(sidecar_pid) +[ -n "$spid" ] || fail "fleet up did not write its pid to $FLEET_HOME/logs/supervisor.json" +kill -TERM "$apppid" +for _ in $(seq 1 60); do kill -0 "$apppid" 2>/dev/null || break; sleep 0.5; done +if kill -0 "$apppid" 2>/dev/null; then + stop_sidecar + fail "the app was still running 30s after SIGTERM" +fi +if kill -0 "$spid" 2>/dev/null; then + stop_sidecar + fail "the app exited on SIGTERM and left its sidecar (fleet up, pid $spid) running" +fi +for _ in $(seq 1 20); do health || break; sleep 0.5; done +health && fail "the app and its sidecar are gone but something still answers $base" +echo "SIGTERM: the app stopped its sidecar and exited" +grep -E "fleet up (started|exited)|did not stop" "$out/app.log" || true +stop_sidecar + +# --- 2. the window, under tauri-driver --------------------------------------- +if [ "${SKIP_DRIVER:-}" = "1" ]; then + echo "SKIP_DRIVER=1; not driving the window" + kill "$xvfb" 2>/dev/null || true + exit 0 +fi + +echo "== half 2: drive the settings window through tauri-driver" +command -v tauri-driver >/dev/null || fail "tauri-driver is not on PATH" +command -v WebKitWebDriver >/dev/null || fail "WebKitWebDriver is not on PATH (apt: webkit2gtk-driver)" +tauri-driver --port 4444 >"$out/tauri-driver.log" 2>&1 & +driver=$! +for _ in $(seq 1 50); do curl -fsS http://127.0.0.1:4444/status >/dev/null 2>&1 && break; sleep 0.2; done + +wd() { # method path [json] + local method=$1 path=$2 body=${3:-} + if [ -n "$body" ]; then + curl -sS --max-time 60 -X "$method" -H 'content-type: application/json' -d "$body" "http://127.0.0.1:4444$path" + else + curl -sS --max-time 60 -X "$method" "http://127.0.0.1:4444$path" + fi +} + +caps=$(jq -n --arg app "$(cd "$bin" && pwd)/fleet-desktop" \ + '{capabilities:{alwaysMatch:{"tauri:options":{application:$app}}}}') +session=$(wd POST /session "$caps") +echo "$session" > "$out/webdriver-session.json" +sid=$(echo "$session" | jq -r '.value.sessionId // empty') +[ -n "$sid" ] || fail "tauri-driver did not open a session: $(echo "$session" | head -c 600)" +echo "webdriver session $sid" + +cleanup_driver() { + wd DELETE "/session/$sid" >/dev/null 2>&1 || true + kill "$driver" 2>/dev/null || true +} +trap cleanup_driver EXIT + +# The app under the driver autostarts its sidecar too. +for _ in $(seq 1 90); do health && break; sleep 1; done +health || fail "under tauri-driver, the sidecar never answered" + +title=$(wd GET "/session/$sid/title" | jq -r '.value') +echo "window title: $title" +[ "$title" = "Fleet" ] || fail "expected the settings window titled Fleet, got '$title'" + +has_tauri=$(wd POST "/session/$sid/execute/sync" \ + '{"script":"return typeof window.__TAURI__ === \"object\" && typeof window.__TAURI__.core.invoke === \"function\"","args":[]}' \ + | jq -r '.value') +[ "$has_tauri" = "true" ] || fail "window.__TAURI__ is missing from the settings page (got $has_tauri)" + +status=$(wd POST "/session/$sid/execute/async" \ + '{"script":"const done = arguments[arguments.length - 1]; window.__TAURI__.core.invoke(\"get_status\").then((s) => done(s), (e) => done({ error: String(e) }));","args":[]}') +echo "$status" | jq '.value' > "$out/get_status.json" +cat "$out/get_status.json" +[ "$(echo "$status" | jq -r '.value.running')" = "true" ] \ + || fail "get_status over IPC did not say the sidecar is running" + +# The settings window starts hidden -- the app lives in the tray -- so it is +# shown the way the tray's Settings item would, from inside the page, before +# the screenshot. +wd POST "/session/$sid/execute/async" \ + '{"script":"const done = arguments[arguments.length - 1]; window.__TAURI__.window.getCurrentWindow().show().then(() => done(true), (e) => done(String(e)));","args":[]}' \ + | jq -c '.value' +sleep 2 +headline=$(wd POST "/session/$sid/execute/sync" \ + '{"script":"const h = document.getElementById(\"headline\"); return h ? h.textContent : null","args":[]}' | jq -r '.value') +echo "settings window headline: $headline" +wd GET "/session/$sid/screenshot" | jq -r '.value' | base64 -d > "$out/settings-window.png" \ + || echo "warning: the webdriver screenshot failed" +import -window root "$out/screen-settings.png" || true +file "$out"/*.png || true + +cleanup_driver +trap - EXIT +sleep 3 +stop_sidecar +kill "$xvfb" 2>/dev/null || true +echo "desktop launch smoke: ok" diff --git a/desktop/src-tauri/Cargo.toml b/desktop/src-tauri/Cargo.toml index 1c1cd51..56f79ad 100644 --- a/desktop/src-tauri/Cargo.toml +++ b/desktop/src-tauri/Cargo.toml @@ -28,6 +28,10 @@ serde_json = "1" # collector dies with the database mid-write, which is the exact failure the # supervisor's SIGTERM draining exists to prevent. See sidecar.rs. libc = "0.2" +# SIGTERM, SIGINT and SIGHUP, turned into the Quit menu item. Without a handler +# the default action kills this process outright and `fleet up` is orphaned, +# still holding the collector's port -- which the first launch in CI showed. +signal-hook = "0.3" [features] # Kept because `cargo tauri build` sets it; nothing here reads it. diff --git a/desktop/src-tauri/src/main.rs b/desktop/src-tauri/src/main.rs index f8e9edc..a8f1e6d 100644 --- a/desktop/src-tauri/src/main.rs +++ b/desktop/src-tauri/src/main.rs @@ -223,6 +223,29 @@ fn on_menu(app: &AppHandle, event: MenuEvent) { } } +/// SIGTERM, SIGINT and SIGHUP do what the Quit item does: drain `fleet up`, +/// then exit. +/// +/// A `kill`, a `killall Fleet`, a Ctrl-C in the terminal that launched it, a +/// closed session: none of them goes through the menu, and with no handler the +/// default action ends this process on the spot. The sidecar is a separate +/// process with nothing tying its life to ours, so it kept running -- a +/// collector on the port with no app in front of it, which the next launch then +/// refused to fight. The first launch of this app, in CI, showed exactly that. +#[cfg(unix)] +fn quit_on_signal(app: &AppHandle) -> std::io::Result<()> { + use signal_hook::consts::{SIGHUP, SIGINT, SIGTERM}; + let mut signals = signal_hook::iterator::Signals::new([SIGTERM, SIGINT, SIGHUP])?; + let app = app.clone(); + std::thread::spawn(move || { + if signals.forever().next().is_some() { + sidecar::stop(&app); + app.exit(0); + } + }); + Ok(()) +} + fn main() { tauri::Builder::default() .plugin(tauri_plugin_shell::init()) @@ -258,6 +281,9 @@ fn main() { // any particular Menu. app.on_menu_event(|app, event| on_menu(app, event)); + #[cfg(unix)] + quit_on_signal(&handle)?; + autostart(&handle); Ok(()) }) @@ -279,15 +305,17 @@ fn main() { .expect("Fleet could not start") .run(|app, event| match event { // With no windows visible the app would otherwise exit the moment - // the settings window is hidden, taking the fleet with it. - RunEvent::ExitRequested { api, .. } => api.prevent_exit(), + // the settings window is hidden, taking the fleet with it. Only that + // implicit exit is refused: `code` is None for it and Some for an + // `app.exit()`, which is how the Quit item and a SIGTERM leave. This + // used to refuse every exit, so Quit stopped the fleet and then + // could not quit. + RunEvent::ExitRequested { api, code, .. } if code.is_none() => api.prevent_exit(), - // The last chance to be polite. Anything that gets here has skipped - // the Quit item -- a `killall Fleet`, or a logout -- so there is no - // time to wait for a drain; the SIGTERM is sent and the supervisor - // gets whatever the OS gives it. The Quit item is the path that - // actually waits, which is why it exists rather than relying on - // this. + // The last chance to be polite, for whatever reached here without + // going through `stop` first. The Quit item and the signal handler + // both drain before exiting, so for them the child is already gone + // and this does nothing. RunEvent::Exit => sidecar::terminate_now(app), _ => {} diff --git a/docs/deploy/headless.md b/docs/deploy/headless.md index d149b17..eac4f06 100644 --- a/docs/deploy/headless.md +++ b/docs/deploy/headless.md @@ -133,7 +133,8 @@ in [networking](networking.md#macos-gates-local-network-access-and-launchd-canno Either way it is irrelevant to this page. A headless host has nobody to click the dialog, so **the tunnel stays** for exactly the machines this page is - about. None of the Rust has ever been compiled, either. + about. CI compiles it on macOS and launches it on Linux; it has never run + on a Mac. ## Rebuilding one over SSH diff --git a/docs/getting-started.md b/docs/getting-started.md index 13bf3aa..6f5cd5c 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -382,7 +382,7 @@ it needs on `PATH`. - **[Concepts](concepts.md)** -- leases, capabilities and targeting, which is what you need before writing a job spec that does something interesting. -- **[Workloads](workloads/index.md)** -- the other thirty-odd things it can run. +- **[Workloads](workloads/index.md)** -- all 34 things it can run. - **[Exploratory QA](explore/index.md)** -- a vision model using tonight's build overnight, and the bugs it reproduces. - **[AI agents and automation](agents.md)** -- `fleetr`, the skill and the MCP diff --git a/docs/index.md b/docs/index.md index 109ac8e..93d89c8 100644 --- a/docs/index.md +++ b/docs/index.md @@ -49,7 +49,7 @@ fleet on. The answer is macOS; the suites run on all three. --- - All 34 of them, what each measures, and what each refuses to guess. + All 34 of them, what each measures, and what each refuses to guess. - __[Exploratory QA](explore/index.md)__ @@ -137,8 +137,8 @@ And two front doors: [`fleet/`](https://github.com/addisdev/fleet-runner/tree/main/fleet), the CLI behind `fleet up`, and [`desktop/`](https://github.com/addisdev/fleet-runner/tree/main/desktop), a -menu-bar app around it whose Rust compiles in CI and which has never been -launched. +menu-bar app around it that CI compiles on macOS and launches on Linux, and +that has never run on a Mac. [The history section](history/index.md) keeps the original architecture plan and the design journals as they were written. They are not maintained, and they are diff --git a/docs/install/docker.md b/docs/install/docker.md index 7f07282..2e11a35 100644 --- a/docs/install/docker.md +++ b/docs/install/docker.md @@ -17,8 +17,11 @@ docker run -d --restart unless-stopped -p 8788:8788 \ machine agent to register and checks `fleet devices`, `fleet workloads` and `fleet whoami` against it. Its first run found that the image could not start on a fresh volume (the directory was created root-owned), fixed in - 0.8.0. The arm64 image is built and has not been started. If you run it on - a NAS or a board, an issue saying how it went is welcome either way. + 0.8.0. `container smoke (arm64)` does the same on linux/arm64, built + natively on GitHub's arm64 runner rather than under QEMU, and checks the + container reports `aarch64`. That is a cloud VM, not a board: if you run it + on a NAS or a Raspberry Pi, an issue saying how it went is welcome either + way. `runner-machine/Dockerfile`, the agent-only image, is still **not built at all**: nothing publishes it and nobody has run `docker build` on it. diff --git a/docs/install/index.md b/docs/install/index.md index de6d590..2b292d8 100644 --- a/docs/install/index.md +++ b/docs/install/index.md @@ -28,7 +28,7 @@ can type it. | **[macOS](macos.md)** | `curl -fsSL …/install.sh \| sh` | Installed from a real release on **arm64 and Intel**, checksum verified, and upgraded in place from 0.5.0 to 0.6.0. **A real fleet's brain runs here under `fleet service`** — one launchd unit, three components, on macOS 12 | | **[Windows](windows.md)** | `irm …/install.ps1 \| iex` | `install.ps1` has never been parsed by PowerShell. The CLI's own suite -- supervisor, config, `fleet up` end to end -- passes on `windows-latest` in CI, but **nobody has stood a fleet up on a Windows machine**, and the service backend is untested | | **[Linux](linux.md)** | `curl -fsSL …/install.sh \| sh` | The installer is the same script macOS runs against the real release; it has not been run on Linux. The collector's and the CLI's suites pass on x64 and arm64 Linux in CI. **Nobody has stood a fleet up on one.** The systemd service backend is installed, started and uninstalled by a CI smoke test on Ubuntu, and has not been watched survive a reboot | -| **[Docker](docker.md)** | `docker run ghcr.io/addisdev/fleet` | Published for linux/amd64 and linux/arm64 with every release. CI starts the amd64 image on every pull request and waits for its agent to register; the arm64 image has not been started | +| **[Docker](docker.md)** | `docker run ghcr.io/addisdev/fleet` | Published for linux/amd64 and linux/arm64 with every release. CI starts both on every pull request that touches them and waits for each one's agent to register | | **[Devices](devices.md)** | -- | How a phone, a television, a Roku or a browser joins a fleet that is already up | ## What the installers do, and what they refuse to do diff --git a/docs/install/linux.md b/docs/install/linux.md index 39e4885..ee0408e 100644 --- a/docs/install/linux.md +++ b/docs/install/linux.md @@ -133,7 +133,7 @@ reboots unattended and is never logged into will otherwise sit there running nothing. **`fleet service install` on systemd is checked by CI, not by a deployment.** -On macOS it runs a real fleet's brain. On Linux, `systemd smoke` installs the +On macOS it runs a real fleet's brain, and CI runs it there and on Windows too. On Linux, `systemd smoke` installs the user unit on GitHub's Ubuntu runner, checks it with `systemd-analyze verify`, waits for the collector it starts to answer, and uninstalls it. What that does not cover is a reboot: the job enables lingering and starts the unit, and diff --git a/docs/install/macos.md b/docs/install/macos.md index 59d29a1..ab5f3e5 100644 --- a/docs/install/macos.md +++ b/docs/install/macos.md @@ -75,7 +75,7 @@ and the host executor inside it, beside the browser runner and the dashboard in the same relative layout a checkout has. It runs with no `node_modules` anywhere near it, which is the property a release is for and the one that would have broken silently. Only Playwright stays outside -- 400 MB of browser needed -by four of thirty workloads, made a dynamic import so an executor without it +by four of the 34 workloads, made a dynamic import so an executor without it starts fine and says so. ## `fleet up` @@ -181,10 +181,14 @@ writes **one** launchd unit running `fleet up`, into rather than the five plists the old deployment had, so switching the brain off is a config edit rather than a `launchctl` invocation somebody has to look up. -**This is the one platform `fleet service install` has been run on.** Since -2026-09-19 it runs the brain of the fleet this project was built for: one +**This is the one platform `fleet service install` runs a real fleet on.** +Since 2026-09-19 it runs the brain of the fleet this project was built for: one launchd unit, three components, on an Intel Mac under macOS 12, surviving a -reboot of the supervisor and a `kill -9` of the collector under it. +reboot of the supervisor and a `kill -9` of the collector under it. CI also +runs it on every pull request that touches it: `launchd smoke` installs the +LaunchAgent on GitHub's `macos-latest` runner, waits for the collector to +answer and the agent to register, reinstalls once to hit the bootstrap race +below, and uninstalls. Two things that cost an outage each, both fixed and both worth knowing: `launchctl bootstrap` loses a race with the `bootout` immediately before it and diff --git a/docs/install/windows.md b/docs/install/windows.md index 9f60cc0..2418b6c 100644 --- a/docs/install/windows.md +++ b/docs/install/windows.md @@ -12,9 +12,11 @@ irm https://raw.githubusercontent.com/addisdev/fleet-runner/main/install.ps1 | i **And nobody has run a fleet on Windows.** CI runs the `fleet` CLI's suite on `windows-latest` on every pull request that touches it, including the - supervised `fleet up` end to end, and it passes. That is the whole of it: - the bundle from an archive, `fleet service install`'s scheduled-task - backend, and a fleet left running on a Windows machine are all unwatched. + supervised `fleet up` end to end, and `windows task smoke` installs + `fleet service`'s scheduled task, waits for its collector to answer and + its agent to register, and uninstalls it. That is the whole of it: the + bundle from an archive and a fleet left running on a Windows machine are + unwatched. What *has* been exercised on Windows is [the machine agent](../platforms.md#what-a-cloud-runner-actually-reports): its own suite @@ -130,10 +132,16 @@ On Windows the service backend is **`schtasks`**, registering a task that runs `fleet up` at logon. It is not a Windows Service: a service would need elevation, and nothing else in this project does. -**`fleet service install` has never been run on Windows**, and the scheduled-task -backend is the least exercised of the three. It has now been run on macOS, where -it runs a real fleet's brain; that says the command works and nothing about this -backend. See [headless +**`fleet service install` on Windows is checked by CI, not by a deployment.** +`windows task smoke` runs it on GitHub's `windows-latest` runner: the task is +created and started, its collector answers, its agent registers, `fleet service +status` reports it, and `fleet service uninstall` leaves nothing answering. Its +first run found that the task had never worked -- the command it stored kept +backslashes meant for a `cmd` prompt and failed with "file not found" -- and +that is fixed. Two things it does not cover: a logon after a reboot, and a +graceful stop. `fleet service stop` and `uninstall` kill `fleet up`'s process +tree, because Windows has no signal one process can send another that Node +turns into SIGTERM, so the collector does not drain. See [headless hosts](../deploy/headless.md). ## What is deliberately absent diff --git a/docs/platforms.md b/docs/platforms.md index 094a2a9..3874659 100644 --- a/docs/platforms.md +++ b/docs/platforms.md @@ -14,7 +14,7 @@ that says which ones somebody has watched register is worth something. | Android phone, tablet | `runner-android` | `adb install`, or sideload | **yes** — the original shelf | | iPhone, iPad | `runner-ios` | Xcode, TestFlight internal | **yes** | | iOS simulator | `runner-ios` | `simctl install` | **yes** | -| macOS, Linux, Windows | `runner-machine` | `fleet up`, or `npm start`, or a LaunchAgent / systemd unit | **macOS yes, on both architectures** — an Apple M1 Pro and a 2016 Intel i7 both registered, passed `npm run conformance`, and produce comparable synthetic numbers with matching digests; both run under `fleet service`. Linux and Windows run their suite in CI on x64 and arm64, and CI installs `fleet service` as a systemd user unit on Linux, waits for the collector it starts, and uninstalls it; the launchd unit is only proven by this lab's own Macs and the Windows scheduled task has never been installed. On Linux x64 the agent also registers, inside the Docker image in CI, against the collector beside it; Windows has never registered against a real collector — see [what each one reports](#what-a-cloud-runner-actually-reports) | +| macOS, Linux, Windows | `runner-machine` | `fleet up`, or `npm start`, or a LaunchAgent / systemd unit | **macOS yes, on both architectures** — an Apple M1 Pro and a 2016 Intel i7 both registered, passed `npm run conformance`, and produce comparable synthetic numbers with matching digests; both run under `fleet service`. Linux and Windows run their suite in CI on x64 and arm64. CI also installs `fleet service` on all three on every pull request that touches it -- a systemd user unit on Ubuntu, a LaunchAgent on `macos-latest`, a scheduled task on `windows-latest` -- waits for the collector it starts and, on macOS and Windows, for the agent to register, then uninstalls it. The Windows task's first run is the first time a Windows agent registered against a real collector; it found the task had never started anything, now fixed. On Linux the agent also registers inside the Docker image in CI, on x64 and arm64. No service has been watched to come back after a reboot, and nobody runs a Linux or Windows machine in a fleet — see [what each one reports](#what-a-cloud-runner-actually-reports) | | Android emulator | `runner-android` | `adb install` to the emulator | **yes** | ## What Wave 5 added @@ -28,7 +28,7 @@ that says which ones somebody has watched register is worth something. | **Apple TV (tvOS)** | `runner-ios`, `FleetRunnerTV` target | `devicectl` or `simctl` | **builds** — `xcodebuild -scheme FleetRunnerTV` succeeds. No Apple TV has run it. (Exploratory QA drives tvOS 26.5 simulators through FleetDriver, a separate UI-test bundle) | | **Vision Pro (visionOS)** | `runner-ios`, `FleetRunnerVision` target | `devicectl` or `simctl` | **no** — the target is written and has never been compiled; the visionOS platform is not installed on the machine it was written on | | **Raspberry Pi, Jetson, Steam Deck** | `runner-machine` | `install-agent.sh`, or the Docker image | **no** — but arm64 Linux in CI reports a nearly complete descriptor, so a board should register cleanly. No board has | -| **Anything with Docker** | `fleet/Dockerfile` | `docker run ghcr.io/addisdev/fleet` | **in CI** — every release publishes linux/amd64 and linux/arm64. CI starts the amd64 image on every pull request that touches it and waits for its own machine agent to register; the arm64 image has not been started | +| **Anything with Docker** | `fleet/Dockerfile` | `docker run ghcr.io/addisdev/fleet` | **in CI** — every release publishes linux/amd64 and linux/arm64. CI builds and starts both on every pull request that touches them -- arm64 natively on GitHub's arm64 runner -- and waits for each one's own machine agent to register. No board or NAS has run it | ## What Wave 6 added @@ -37,9 +37,13 @@ that says which ones somebody has watched register is worth something. | **Roku player, Roku TV** | `runner-roku` | `build.sh --install `, then an ECP launch | **no** -- and less verified than anything above it. No Roku was available and no Roku emulator exists, so the channel has never run. CI compiles it with brighterscript's `bsc` on every pull request that touches it; the firmware's own compiler has never seen it. See [Roku: the one that has never run](#roku-the-one-that-has-never-run) | | **Roku, from the executor** | `collector/src/drivers/roku.ts` | SSDP discovery on the LAN | **partly** -- the SSDP and device-info parsers are checked against recorded response shapes in `drivers.test.ts`; the UDP socket path is exercised by nothing, and no Roku has answered it | -The desktop app is not a runner, but it is the other thing here that has never -been run: `desktop / cargo check` compiles its Rust on macOS on every pull -request that touches it, and nobody has launched it. +The desktop app is not a runner, and it has never run on the platform it is +for. On every pull request that touches it, `desktop / cargo check` compiles +its Rust on macOS, and `desktop / launch smoke` builds and launches it on +Ubuntu under a virtual display: its `fleet` sidecar comes up and registers an +agent, its settings window answers over WebDriver, and SIGTERM stops both. The +tray, macOS, and the local-network bet it exists for are untested; see +[its README](https://github.com/addisdev/fleet-runner/tree/main/desktop#status). ## Android: one APK, every shape diff --git a/docs/workloads/index.md b/docs/workloads/index.md index 0f9084c..39516db 100644 --- a/docs/workloads/index.md +++ b/docs/workloads/index.md @@ -6,7 +6,7 @@ spec, and which agent claims it is decided by ![The workloads in three columns by who claims them: on the device, on a host, and on a machine](../img/workloads.png) -The collector ships 34 workloads. `GET /api/v1/workloads` (or +The collector ships 34 workloads. `GET /api/v1/workloads` (or `fleetr workloads`) lists the same set with each one's params and an example spec, and marks three more names as reserved: `push-latency`, `camera-eval` and `desktop-ui-test` are accepted, but no runner in this repository implements diff --git a/fleet/README.md b/fleet/README.md index 4f55b71..8051e13 100644 --- a/fleet/README.md +++ b/fleet/README.md @@ -232,21 +232,40 @@ the collector under it. Two things cost an outage each on the way and are fixed `PATH` from the shell that ran the install, which is how an executor ends up unable to find `adb`. -Since 0.8.0, two CI smoke jobs in `fleet.yml` run what had only been built: +Since 0.8.0, CI smoke jobs in `fleet.yml` run what had only been built. Each +runs on every pull request that touches `fleet/`, `collector/` or +`runner-machine/`: - **systemd.** `systemd smoke` installs the built bundle as a systemd user unit on Ubuntu, checks it with `systemd-analyze verify`, waits for its collector to answer, and uninstalls it. -- **A container.** `container smoke` builds `fleet/Dockerfile`, starts `fleet up` in it, and - waits for its machine agent to register. Its first run found the image could not start on - a fresh volume. +- **launchd.** `launchd smoke` does the same on `macos-latest`: `fleet service install` + writes the LaunchAgent and bootstraps it into the runner's `gui/` domain (GitHub's + macOS runners have a logged-in GUI session, and the job prints it first). It waits for the + collector to answer and the agent to register, runs `fleet service status`, reinstalls once + so the bootout/bootstrap race is hit and retried rather than trusted -- the first run printed + "launchd is still letting go of the old job; waiting" and recovered -- then uninstalls and + checks nothing answers, the plist is gone and launchd no longer has the label. +- **The Windows scheduled task.** `windows task smoke` runs the same lifecycle on + `windows-latest`. Its first run found that the task had never worked: the command it + stored kept the backslashes from a `\"` escaping meant for a `cmd` prompt, so the task + ran `\"C:\...\node.exe\"` and failed with "file not found". Fixed, and `fleet service + uninstall` (and `stop`) now kill `fleet up`'s process tree by the supervisor's pid before + ending the task, because deleting a task does not stop it and Windows does not take a + parent's children down with it. With that, a Windows machine agent registered against a + real collector for the first time. +- **A container, on both architectures.** `container smoke` builds `fleet/Dockerfile`, + starts `fleet up` in it, and waits for its machine agent to register. Its first run found + the image could not start on a fresh volume. `container smoke (arm64)` builds it natively + on GitHub's arm64 runner and checks the container reports `aarch64` before doing the same. What has still not been run: -- **`fleet service install` on Windows**, and the systemd unit surviving a reboot on a real - machine. -- **The arm64 container image.** The release workflow builds and publishes it; nothing has - started it. +- **A service surviving a reboot**, on any of the three. CI installs and starts each unit; + nobody has watched one come back at boot or at logon on a real machine. +- **Windows stopping gracefully.** The task is stopped by killing the process tree, so the + collector does not drain; there is no signal Windows lets one process send another that + Node turns into SIGTERM. - **`install.ps1`.** No PowerShell was available to parse it, let alone run it. `install.sh` *has* been run against the real v0.5.0 release on macOS/arm64: checksum verified, binary installed and reporting its version. diff --git a/fleet/src/service.ts b/fleet/src/service.ts index 1c0b42f..a2d6630 100644 --- a/fleet/src/service.ts +++ b/fleet/src/service.ts @@ -266,10 +266,7 @@ WantedBy=default.target */ async function installWindows(command: string, args: string[]): Promise { const p = paths(); - // schtasks takes one command string, so the whole thing is quoted here rather - // than passed as an argv -- and a path with a space in it is the normal case - // on Windows, not the exception. - const tr = [command, ...args].map((a) => `\\"${a}\\"`).join(" "); + const tr = windowsTaskCommand(command, args); try { await exec("schtasks", [ "/Create", @@ -290,6 +287,64 @@ async function installWindows(command: string, args: string[]): Promise return 0; } +/** + * The `/TR` string: one command line, each part in plain double quotes. + * + * schtasks takes the whole command as one argument, so it is quoted here -- a + * path with a space in it is the normal case on Windows. The quotes must be + * bare. This used to write `\"` around each part, the escaping you would type + * at a `cmd` prompt, but execFile is not a prompt: Node quotes the argument + * itself and escapes every `"` in it, schtasks undoes exactly that, and the + * backslashes typed here survived into the task as part of the path. The task + * stored `\"C:\...\node.exe\"`, ran it, and failed with 0x80070002 (file + * not found) -- so `fleet service install` on Windows had never started + * anything. Found by the first `windows task smoke` run in CI. + */ +export function windowsTaskCommand(command: string, args: string[]): string { + return [command, ...args].map((a) => `"${a}"`).join(" "); +} + +/** + * Stop the running task, and the processes `fleet up` started. + * + * `schtasks /End` stops the task's own process, which is `fleet up`, and + * nothing it started: Windows has no process group that dies with its leader, + * so the collector and the agent would outlive it, keep the port, and keep + * answering after `fleet service uninstall` said it was removed. So the tree is + * killed by the supervisor's pid first, while it is still the parent that ties + * the tree together, and each child by its own pid after that. + * + * Forcefully, because there is no other way: `fleet up`'s graceful shutdown + * hangs off SIGTERM, and Windows has no signal a different process can send to + * a console program that Node turns into one. The collector does not drain + * here; it is killed, as it would be by closing its console window. + * + * Only while the task reports Running, so that a supervisor.json left behind by + * a fleet that has since stopped cannot point taskkill at a pid Windows has + * handed to something else. + */ +async function endWindowsTask(): Promise { + let running = false; + try { + const { stdout } = await exec("schtasks", ["/Query", "/TN", WINDOWS_TASK, "/FO", "LIST"]); + running = /Running/i.test(stdout); + } catch { + return; // not installed + } + if (running) { + try { + const snap = JSON.parse(readFileSync(paths().supervisorStatus, "utf8")) as SupervisorSnapshot; + await exec("taskkill", ["/PID", String(snap.pid), "/T", "/F"]).catch(() => {}); + for (const c of snap.children) { + if (c.pid) await exec("taskkill", ["/PID", String(c.pid), "/T", "/F"]).catch(() => {}); + } + } catch { + /* no status file: /End below is all there is */ + } + } + await exec("schtasks", ["/End", "/TN", WINDOWS_TASK]).catch(() => {}); +} + // --- the rest -------------------------------------------------------------- async function uninstall(): Promise { @@ -301,6 +356,8 @@ async function uninstall(): Promise { return 0; } if (process.platform === "win32") { + // Deleting a task does not stop a running instance of it. + await endWindowsTask(); await exec("schtasks", ["/Delete", "/F", "/TN", WINDOWS_TASK]).catch(() => {}); console.log(`removed scheduled task ${WINDOWS_TASK}`); return 0; @@ -324,7 +381,8 @@ async function startStop(which: "start" | "stop"): Promise { if (which === "start") await bootstrapLaunchd(launchdPlistPath()); else await exec("launchctl", ["bootout", `gui/${uid}/${LABEL}`]); } else if (process.platform === "win32") { - await exec("schtasks", [which === "start" ? "/Run" : "/End", "/TN", WINDOWS_TASK]); + if (which === "start") await exec("schtasks", ["/Run", "/TN", WINDOWS_TASK]); + else await endWindowsTask(); } else { await exec("systemctl", ["--user", which, "fleet.service"]); } diff --git a/fleet/test/service.test.ts b/fleet/test/service.test.ts new file mode 100644 index 0000000..f632532 --- /dev/null +++ b/fleet/test/service.test.ts @@ -0,0 +1,21 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { windowsTaskCommand } from "../src/service.js"; + +// The scheduled task's command line. execFile quotes and escapes this string +// itself when it hands it to schtasks, so any escaping written here survives +// into the task: `\"C:\...\node.exe\"` is what the first CI run found stored, +// and the task failed with "file not found" every time it ran. +test("the schtasks /TR string quotes each part with bare quotes", () => { + const tr = windowsTaskCommand("C:\\Program Files\\nodejs\\node.exe", [ + "C:\\Users\\Jo Smith\\.fleet\\bin\\fleet.mjs", + "up", + "--role", + "brain,agent", + ]); + assert.equal( + tr, + '"C:\\Program Files\\nodejs\\node.exe" "C:\\Users\\Jo Smith\\.fleet\\bin\\fleet.mjs" "up" "--role" "brain,agent"', + ); + assert.ok(!tr.includes('\\"'), "no backslash-escaped quotes"); +}); diff --git a/scripts/changelog-coverage.mjs b/scripts/changelog-coverage.mjs new file mode 100644 index 0000000..8e74370 --- /dev/null +++ b/scripts/changelog-coverage.mjs @@ -0,0 +1,213 @@ +#!/usr/bin/env node +/** + * Every pull request in a release is in its changelog. + * + * node scripts/changelog-coverage.mjs [--head ] [--tag ] + * + * Lists the pull requests merged into `develop` since the last `v*` tag and + * fails, naming each one, if the changelog section being released does not + * mention it as `#N` or `/pull/N`. A pull request labelled `no-changelog`, and + * any opened by Dependabot, is exempt. + * + * ## Why + * + * 0.8.0 shipped with notes that left out work merged before it was cut -- + * exploratory QA among it -- and nobody noticed until an audit read the + * history against the notes a day later. The changelog had to be amended after + * the release. A list of merged pull requests is cheap to compute and a human + * reading a long `[Unreleased]` section is not a reliable diff. + * + * ## What "since the last tag" means + * + * The commits on ``'s first-parent chain that the newest `v*` tag does + * not contain. On a `release/*` branch cut from `develop`, and on the + * `develop` -> `main` pull request, that chain is `develop`'s own history, so + * its merge commits are exactly the pull requests merged into `develop` since + * the tag. Both a merge commit (`Merge pull request #N from ...`) and a squash + * merge (`Title (#N)`) are recognised. + * + * ## What "the section being released" means + * + * Everything from `## [Unreleased]` down to the heading of the last tag's + * version. CONTRIBUTING.md has a release rename `[Unreleased]` to the new + * version and add an empty `[Unreleased]` above it, and before that rename the + * notes are all under `[Unreleased]`; this range is right either way. + * + * Labels and authors are read from the GitHub API with `gh` (GH_TOKEN in CI), + * and only for the pull requests the changelog does not mention, so a + * complete changelog needs no network at all. + * + * Plain Node, no dependencies, like scripts/version.mjs. + */ +import { execFileSync } from "node:child_process"; +import { readFileSync } from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); + +export const EXEMPT_LABEL = "no-changelog"; +/** `dependabot[bot]` from the REST API, `app/dependabot` from `gh pr view`. */ +export const isDependabot = (author) => /(^|\/)dependabot(-preview)?(\[bot\])?$/.test(author ?? ""); + +/** + * Pull request numbers from commit subjects, oldest first, without repeats. + * + * Only subjects that look like GitHub's own merge commits count. A commit that + * merely mentions `#12` in passing ("revert the fix from #12") is not a pull + * request being merged, and listing it would make the check cry wolf. + */ +export function prNumbersFromSubjects(subjects) { + const seen = new Set(); + const out = []; + for (const s of subjects) { + const m = /^Merge pull request #(\d+) from /.exec(s) ?? /\(#(\d+)\)\s*$/.exec(s); + if (!m) continue; + const n = Number(m[1]); + if (!seen.has(n)) { + seen.add(n); + out.push(n); + } + } + return out.reverse(); +} + +/** + * The text from `## [Unreleased]` to the heading for `sinceVersion`. + * + * Throws when either heading is missing: a changelog this cannot read is a + * changelog this cannot check, and passing would be a lie. + */ +export function releasedSection(changelog, sinceVersion) { + const lines = changelog.split(/\r?\n/); + const start = lines.findIndex((l) => /^## \[Unreleased\]/i.test(l)); + if (start < 0) throw new Error("CHANGELOG.md has no `## [Unreleased]` heading"); + const escaped = sinceVersion.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + const endRe = new RegExp(`^## \\[${escaped}\\]`); + const end = lines.findIndex((l, i) => i > start && endRe.test(l)); + if (end < 0) { + throw new Error(`CHANGELOG.md has no \`## [${sinceVersion}]\` heading below [Unreleased]`); + } + return lines.slice(start, end).join("\n"); +} + +/** Pull request numbers a section mentions, as `#N` or as a `/pull/N` link. */ +export function mentionedNumbers(section) { + const found = new Set(); + for (const m of section.matchAll(/(?:^|[^\w&])#(\d+)\b/g)) found.add(Number(m[1])); + for (const m of section.matchAll(/\/pull\/(\d+)\b/g)) found.add(Number(m[1])); + return found; +} + +/** + * The verdict, given everything already fetched. + * + * `meta` maps a pull request number to `{ labels: string[], author: string }` + * for the ones that needed looking up; a number with no entry is treated as + * unlabelled, so a lookup that failed fails the check rather than excusing it. + */ +export function coverage({ merged, mentioned, meta = new Map() }) { + const missing = []; + const exempt = []; + for (const n of merged) { + if (mentioned.has(n)) continue; + const m = meta.get(n); + if (m && (m.labels.includes(EXEMPT_LABEL) || isDependabot(m.author))) { + exempt.push({ number: n, why: m.labels.includes(EXEMPT_LABEL) ? EXEMPT_LABEL : m.author }); + continue; + } + missing.push({ number: n, title: m?.title ?? "" }); + } + return { missing, exempt }; +} + +/** The newest `v*` tag by version order, or null. */ +export function newestTag(tags) { + const parsed = tags + .map((t) => ({ t, m: /^v(\d+)\.(\d+)\.(\d+)(?:-(.+))?$/.exec(t.trim()) })) + .filter((x) => x.m); + parsed.sort((a, b) => { + for (let i = 1; i <= 3; i++) { + const d = Number(b.m[i]) - Number(a.m[i]); + if (d) return d; + } + // 1.0.0 is newer than 1.0.0-rc.1. + if (!a.m[4] !== !b.m[4]) return a.m[4] ? 1 : -1; + return (b.m[4] ?? "").localeCompare(a.m[4] ?? "", undefined, { numeric: true }); + }); + return parsed[0]?.t.trim() ?? null; +} + +// --- the command ------------------------------------------------------------ + +function git(...args) { + return execFileSync("git", args, { cwd: ROOT, encoding: "utf8" }); +} + +function prMeta(numbers) { + const meta = new Map(); + for (const n of numbers) { + try { + const json = execFileSync( + "gh", + ["pr", "view", String(n), "--json", "labels,author,title"], + { cwd: ROOT, encoding: "utf8", stdio: ["ignore", "pipe", "pipe"] }, + ); + const pr = JSON.parse(json); + meta.set(n, { + labels: (pr.labels ?? []).map((l) => l.name), + author: pr.author?.login ?? "", + title: pr.title ?? "", + }); + } catch (e) { + console.error(` could not read #${n} from GitHub (${(e.stderr || e.message || "").toString().trim()}); treating it as unlabelled`); + } + } + return meta; +} + +function main(argv) { + const flag = (name) => { + const i = argv.indexOf(name); + return i >= 0 ? argv[i + 1] : undefined; + }; + const head = flag("--head") ?? "HEAD"; + const tag = flag("--tag") ?? newestTag(git("tag", "--list", "v*").split("\n").filter(Boolean)); + if (!tag) { + console.error("no v* tag in this checkout. In CI, check out with fetch-depth: 0 so the tags come too."); + return 1; + } + const version = tag.replace(/^v/, ""); + + const subjects = git("log", "--first-parent", "--format=%s", `${tag}..${head}`).split("\n").filter(Boolean); + const merged = prNumbersFromSubjects(subjects); + + let section; + try { + section = releasedSection(readFileSync(path.join(ROOT, "CHANGELOG.md"), "utf8"), version); + } catch (e) { + console.error(e.message); + return 1; + } + const mentioned = mentionedNumbers(section); + const unmentioned = merged.filter((n) => !mentioned.has(n)); + const { missing, exempt } = coverage({ merged, mentioned, meta: prMeta(unmentioned) }); + + console.log(`${merged.length} pull request(s) merged since ${tag}: ${merged.map((n) => `#${n}`).join(" ") || "none"}`); + for (const e of exempt) console.log(` exempt #${e.number} (${e.why})`); + if (missing.length === 0) { + console.log(`every one is in the changelog between [Unreleased] and [${version}]`); + return 0; + } + console.error(`\nnot mentioned in CHANGELOG.md between [Unreleased] and [${version}]:`); + for (const m of missing) console.error(` #${m.number}${m.title ? ` ${m.title}` : ""}`); + console.error( + `\nAdd each to the section being released (as #N or a /pull/N link), or label it ` + + `\`${EXEMPT_LABEL}\` if nobody using Fleet Runner would notice it.`, + ); + return 1; +} + +if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { + process.exit(main(process.argv.slice(2))); +} diff --git a/scripts/changelog-coverage.test.mjs b/scripts/changelog-coverage.test.mjs new file mode 100644 index 0000000..39ecaee --- /dev/null +++ b/scripts/changelog-coverage.test.mjs @@ -0,0 +1,89 @@ +// node --test scripts/ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { + coverage, + isDependabot, + mentionedNumbers, + newestTag, + prNumbersFromSubjects, + releasedSection, +} from "./changelog-coverage.mjs"; + +test("merge commits and squash merges both count; passing mentions do not", () => { + const subjects = [ + "Merge pull request #91 from addisdev/ci/never-run-list", + "Release notes for 0.9.0", + "Fix the thing (#90)", + "Revert the fix from #12", + "Merge branch 'develop' into feature", + "Merge pull request #88 from addisdev/docs/0.8-catchup", + "Merge pull request #91 from addisdev/ci/never-run-list", + ]; + // git log is newest first; the list comes back oldest first, once each. + assert.deepEqual(prNumbersFromSubjects(subjects), [88, 90, 91]); +}); + +const CHANGELOG = `# Changelog + +## [Unreleased] + +## [0.9.0] — 2026-11-01 + +### Added + +- **A thing.** ([#90](https://github.com/addisdev/fleet-runner/pull/90)) +- **Another**, see #91. +- Not a PR: ] or issue#94. + +## [0.8.0] — 2026-10-15 + +- **Old.** #88 +`; + +test("the section runs from [Unreleased] down to the last tag's heading", () => { + const s = releasedSection(CHANGELOG, "0.8.0"); + assert.match(s, /^## \[Unreleased\]/); + assert.match(s, /## \[0\.9\.0\]/); + assert.doesNotMatch(s, /Old\./); +}); + +test("a changelog without the headings is an error, not a pass", () => { + assert.throws(() => releasedSection("# Changelog\n", "0.8.0"), /Unreleased/); + assert.throws(() => releasedSection("## [Unreleased]\n- x\n", "0.8.0"), /0\.8\.0/); + // 0.8.0 must not match 0.8.01 or 0x8y0. + assert.throws(() => releasedSection("## [Unreleased]\n## [0x8y0]\n", "0.8.0"), /0\.8\.0/); +}); + +test("#N and /pull/N are mentions; HTML entities and word#N are not", () => { + const m = mentionedNumbers(releasedSection(CHANGELOG, "0.8.0")); + assert.deepEqual([...m].sort(), [90, 91]); +}); + +test("missing PRs are listed; no-changelog and Dependabot are exempt", () => { + const meta = new Map([ + [92, { labels: ["no-changelog"], author: "addisdev", title: "CI tweak" }], + [93, { labels: [], author: "app/dependabot", title: "Bump x" }], + [94, { labels: ["docs"], author: "addisdev", title: "Forgotten" }], + ]); + const { missing, exempt } = coverage({ + merged: [88, 90, 91, 92, 93, 94, 95], + mentioned: new Set([90, 91, 88]), + meta, + }); + assert.deepEqual(missing.map((m) => m.number), [94, 95]); + assert.equal(missing[0].title, "Forgotten"); + assert.deepEqual(exempt.map((e) => e.number), [92, 93]); +}); + +test("Dependabot is recognised however the API spells it", () => { + for (const a of ["dependabot[bot]", "app/dependabot", "dependabot"]) assert.ok(isDependabot(a), a); + for (const a of ["addisdev", "not-dependabot-really", ""]) assert.ok(!isDependabot(a), a); +}); + +test("the newest tag is chosen by version, not by string order", () => { + assert.equal(newestTag(["v0.7.2", "v0.10.0", "v0.9.1"]), "v0.10.0"); + assert.equal(newestTag(["v1.0.0-rc.1", "v1.0.0", "v0.9.0"]), "v1.0.0"); + assert.equal(newestTag(["v1.0.0-rc.2", "v1.0.0-rc.10", "v0.9.0"]), "v1.0.0-rc.10"); + assert.equal(newestTag(["latest", "nightly"]), null); +}); diff --git a/scripts/check-counts.mjs b/scripts/check-counts.mjs new file mode 100644 index 0000000..6024d1b --- /dev/null +++ b/scripts/check-counts.mjs @@ -0,0 +1,229 @@ +#!/usr/bin/env node +/** + * The numbers the docs state by hand agree with the code. + * + * node scripts/check-counts.mjs # exit 1 on any disagreement + * node scripts/check-counts.mjs --json # print the counts and exit + * + * The README and the docs said "28 workloads" while the catalog grew to 34, + * and nothing noticed. A number written in prose cannot be generated without + * turning the prose into a template, so instead each one is marked where it + * stands and this checks the marks: + * + * The collector ships 34 workloads. + * Thirty-four workloads ... + * + * The comment is invisible on GitHub and in the mkdocs site. The number right + * after it -- digits, or words up to ninety-nine -- is compared with the count, + * and a mismatch fails with the file, the line and what it should say. + * + * ## Where the count comes from + * + * `workloads` is the number of entries in `CATALOG` in + * collector/src/workloads/catalog.ts that are not `reserved(...)`. It is read as + * text rather than imported, so this runs with no `npm ci` and can sit in a job + * that runs on every pull request, docs-only ones included -- a docs change is + * exactly where a stale number gets written. The collector's own suite imports + * the real `CATALOG` and checks this parse against it + * (collector/scripts/check-counts.ts), so the text reading cannot drift away + * from what the catalog actually holds without that suite failing. + * + * Plain Node, no dependencies, like scripts/version.mjs. + */ +import { readdirSync, readFileSync, statSync } from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); +const CATALOG = "collector/src/workloads/catalog.ts"; + +/** + * Shipped and reserved entries in the CATALOG array literal. + * + * Each entry is one call at the array's top level: `e(...)` or + * `fromManifest(...)` for shipped, `reserved(...)` for reserved. Anything else + * at that level -- a spread, an inline object -- throws, because a parse that + * silently skips an entry is a check that silently passes. + */ +export function countCatalog(src) { + const start = src.indexOf("export const CATALOG"); + if (start < 0) throw new Error(`no \`export const CATALOG\` in ${CATALOG}`); + const open = src.indexOf("[", src.indexOf("=", start)); + let depth = 0; + let i = open; + let entryStart = -1; + const entries = []; + let inString = null; + for (; i < src.length; i++) { + const c = src[i]; + if (inString) { + if (c === "\\") i++; + else if (c === inString) inString = null; + continue; + } + if (c === '"' || c === "'" || c === "`") { + inString = c; + continue; + } + if (c === "/" && src[i + 1] === "/") { + i = src.indexOf("\n", i); + continue; + } + if (c === "/" && src[i + 1] === "*") { + i = src.indexOf("*/", i) + 1; + continue; + } + if (c === "[" || c === "(" || c === "{") { + depth++; + if (depth === 2 && entryStart < 0) entryStart = i; + continue; + } + if (c === "]" || c === ")" || c === "}") { + depth--; + if (depth === 0) break; + continue; + } + if (depth === 1 && /\S/.test(c) && c !== ",") { + if (entryStart < 0) entryStart = i; + } + if (depth === 1 && c === "," && entryStart >= 0) { + entries.push(src.slice(entryStart, i).trim()); + entryStart = -1; + } + } + if (depth !== 0) throw new Error(`the CATALOG array in ${CATALOG} never closes`); + if (entryStart >= 0) { + const last = src.slice(entryStart, i).trim(); + if (last) entries.push(last); + } + let shipped = 0; + let reserved = 0; + for (const entry of entries) { + const callee = /^([A-Za-z_$][\w$]*)\s*\(/.exec(entry)?.[1]; + if (callee === "reserved") reserved++; + else if (callee === "e" || callee === "fromManifest") shipped++; + else throw new Error(`cannot tell whether this CATALOG entry is shipped: ${entry.slice(0, 80)}`); + } + if (shipped === 0) throw new Error(`counted no shipped workloads in ${CATALOG}; this check needs updating`); + return { shipped, reserved }; +} + +const ONES = ["zero", "one", "two", "three", "four", "five", "six", "seven", "eight", "nine", + "ten", "eleven", "twelve", "thirteen", "fourteen", "fifteen", "sixteen", "seventeen", "eighteen", "nineteen"]; +const TENS = ["", "", "twenty", "thirty", "forty", "fifty", "sixty", "seventy", "eighty", "ninety"]; + +/** `34`, `thirty-four`, `Thirty four` -> 34. Null for anything else. */ +export function parseCount(text) { + const t = text.trim().toLowerCase(); + if (/^\d+$/.test(t)) return Number(t); + const words = t.split(/[-\s]+/); + if (words.length === 1) { + const one = ONES.indexOf(words[0]); + if (one >= 0) return one; + const ten = TENS.indexOf(words[0]); + return ten >= 2 ? ten * 10 : null; + } + if (words.length === 2) { + const ten = TENS.indexOf(words[0]); + const one = ONES.indexOf(words[1]); + if (ten >= 2 && one >= 1 && one <= 9) return ten * 10 + one; + } + return null; +} + +const MARKER = /\s*([0-9]+|[A-Za-z]+(?:[-\s][A-Za-z]+)?)/g; + +/** + * The text with code blanked out, same length, newlines kept. + * + * A marker written inside backticks is an example of the syntax -- in + * CONTRIBUTING.md, in a changelog entry -- not a number standing in prose, and + * it must not start failing the day the count moves. + */ +export function maskCode(text) { + const blank = (s) => s.replace(/[^\n]/g, " "); + return text + .replace(/^(```|~~~)[^\n]*\n[\s\S]*?^\1[^\n]*$/gm, blank) + .replace(/(`+)(?!`)[\s\S]*?[^`]\1(?!`)/g, blank); +} + +/** Every marked number in a file, outside code: `{ key, text, value, line }`. */ +export function findMarkers(text) { + const out = []; + text = maskCode(text); + for (const m of text.matchAll(MARKER)) { + let token = m[2]; + // "thirty-four workloads": keep the second word only if it is a units word. + let value = parseCount(token); + if (value === null && /[-\s]/.test(token)) { + token = token.split(/[-\s]/)[0]; + value = parseCount(token); + } + const line = text.slice(0, m.index).split("\n").length; + out.push({ key: m[1], text: token, value, line }); + } + return out; +} + +/** + * Problems, as strings. `files` is `[{ path, text }]`; `counts` maps a marker + * key to the true number. + */ +export function check(files, counts) { + const problems = []; + let seen = 0; + for (const f of files) { + for (const mark of findMarkers(f.text)) { + seen++; + const where = `${f.path}:${mark.line}`; + if (!(mark.key in counts)) { + problems.push(`${where}: unknown count "${mark.key}" (known: ${Object.keys(counts).join(", ")})`); + } else if (mark.value === null) { + problems.push(`${where}: "${mark.text}" after is not a number`); + } else if (mark.value !== counts[mark.key]) { + problems.push(`${where}: says ${mark.text}, but there are ${counts[mark.key]} ${mark.key}`); + } + } + } + if (seen === 0) problems.push("found no markers at all, so nothing was checked"); + return { problems, seen }; +} + +/** Markdown files worth checking: the READMEs and docs/, never dependencies. */ +function markdownFiles(dir = ROOT, rel = "") { + const out = []; + for (const name of readdirSync(dir)) { + if (name === "node_modules" || name.startsWith(".") || name === "site" || name === "dist" || name === "build") continue; + const abs = path.join(dir, name); + const r = rel ? `${rel}/${name}` : name; + if (statSync(abs).isDirectory()) out.push(...markdownFiles(abs, r)); + else if (name.endsWith(".md")) out.push(r); + } + return out; +} + +export function counts() { + const { shipped } = countCatalog(readFileSync(path.join(ROOT, CATALOG), "utf8")); + return { workloads: shipped }; +} + +function main(argv) { + const c = counts(); + if (argv.includes("--json")) { + console.log(JSON.stringify(c)); + return 0; + } + const files = markdownFiles().map((p) => ({ path: p, text: readFileSync(path.join(ROOT, p), "utf8") })); + const { problems, seen } = check(files, c); + if (problems.length) { + for (const p of problems) console.error(` ${p}`); + console.error(`\n${problems.length} documented count(s) disagree with the code. Fix the number after the marker.`); + return 1; + } + console.log(`ok -- ${seen} marked count(s) agree: ${Object.entries(c).map(([k, v]) => `${v} ${k}`).join(", ")}`); + return 0; +} + +if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { + process.exit(main(process.argv.slice(2))); +} diff --git a/scripts/check-counts.test.mjs b/scripts/check-counts.test.mjs new file mode 100644 index 0000000..0a7f032 --- /dev/null +++ b/scripts/check-counts.test.mjs @@ -0,0 +1,82 @@ +// node --test scripts/ +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import { test } from "node:test"; +import { check, countCatalog, findMarkers, maskCode, parseCount } from "./check-counts.mjs"; + +const CATALOG = ` +const e = () => ({}); +export const CATALOG: CatalogEntry[] = [ + e("benchmark", "device", ["device", "machine"], "Prefill (and) decode, [sic] {x}.", "device.md", benchmark), + // e("commented-out", ...) is not an entry + fromManifest(enrolManifest as Manifest, "host.md"), + /* reserved("also-not", ...) */ + e("web-test", "host", ["host"], "Playwright \\"suites\\", one, two.", "host.md", webTest, { dangerous: true }), + reserved("serve", "host", ["host"], "Serve a model."), +]; +export const OTHER = [e("not-in-catalog")]; +`; + +test("counts shipped and reserved entries, ignoring comments and strings", () => { + assert.deepEqual(countCatalog(CATALOG), { shipped: 3, reserved: 1 }); +}); + +test("an entry it cannot classify is an error, not a skip", () => { + assert.throws(() => countCatalog("export const CATALOG = [\n e('a'),\n ...others,\n];"), /cannot tell/); + assert.throws(() => countCatalog("export const NOPE = [];"), /no `export const CATALOG`/); +}); + +test("the real catalog parses", () => { + const src = readFileSync(new URL("../collector/src/workloads/catalog.ts", import.meta.url), "utf8"); + const { shipped, reserved } = countCatalog(src); + assert.ok(shipped > 0 && reserved >= 0); +}); + +test("numbers in digits and in words", () => { + assert.equal(parseCount("34"), 34); + assert.equal(parseCount("Thirty-four"), 34); + assert.equal(parseCount("thirty four"), 34); + assert.equal(parseCount("twelve"), 12); + assert.equal(parseCount("Forty"), 40); + assert.equal(parseCount("thirty-odd"), null); + assert.equal(parseCount("many"), null); +}); + +test("markers find the number after them, with its line", () => { + const text = "# Title\n\nThe collector ships 34 workloads.\n" + + "Thirty-four workloads, and\n" + + "thirty workloads\n"; + const marks = findMarkers(text); + assert.deepEqual(marks.map((m) => [m.value, m.line]), [[34, 3], [34, 4], [30, 5]]); +}); + +test("a disagreeing, unreadable or unknown marker is a problem", () => { + const files = [ + { path: "README.md", text: "34 here\n28 there\n" }, + { path: "docs/x.md", text: "many\n5\n" }, + ]; + const { problems, seen } = check(files, { workloads: 34 }); + assert.equal(seen, 4); + assert.equal(problems.length, 3); + assert.match(problems[0], /README\.md:2: says 28, but there are 34 workloads/); + assert.match(problems[1], /docs\/x\.md:1: "many" .* is not a number/); + assert.match(problems[2], /unknown count "runners"/); +}); + +test("no markers at all fails, so a deleted marker cannot pass silently", () => { + const { problems } = check([{ path: "README.md", text: "34 workloads" }], { workloads: 34 }); + assert.match(problems[0], /no 34` next to the number.", + "```markdown", + "12 workloads", + "```", + "Real: 34 workloads.", + ].join("\n"); + const marks = findMarkers(text); + assert.deepEqual(marks.map((m) => [m.value, m.line]), [[34, 5]]); + assert.equal(maskCode(text).length, text.length); +});