diff --git a/CHANGELOG.md b/CHANGELOG.md index c12ef72..f6a65d2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,32 @@ a CI job fails when they disagree. ## [Unreleased] +## [0.9.1] — 2026-10-18 + +A phone on the shelf is woken before it is driven, and one with a PIN says so +instead of failing as if the app were broken. The lab gains a regression +nightly on the bug garden. + +### Added + +- **`nightly-bug-garden-regressions`, a regression nightly on the bug + garden.** Bug garden's smoke flow plus every bug-garden finding marked real + and then fixed, replayed on the shelf's Android phone at 02:45 + (`seed-schedules`, off on a fresh collector; the lab has it on). The + garden's defects are planted, so a finding marked fixed comes back and + reopens: proved once on ultra's emulator, where finding #18's crash recurred + and reopened it. ([#100](https://github.com/addisdev/fleet-runner/pull/100)) + +### Fixed + +- **A sleeping phone failed its UI tests as if the app were broken.** Maestro + does not wake a device, so `nightly-fleet-ui-smoke` ran against the Galaxy + S8+'s lock screen and failed `"Fleet Runner" is visible` every night. + `ui-test` and `locale-shots` now wake an Android device and dismiss a lock + screen with no PIN first; a PIN, pattern or password makes that device's row + `skipped` with the reason, since nothing here can get past it. + ([#100](https://github.com/addisdev/fleet-runner/pull/100)) + ## [0.9.0] — 2026-10-15 Green that means something. A job that ran nothing is `skipped`, not `done`, @@ -1468,7 +1494,8 @@ The first public release, when the project was still four repositories. that starts a throwaway collector on a spare port so it never touches a live fleet's history. -[Unreleased]: https://github.com/addisdev/fleet-runner/compare/v0.9.0...HEAD +[Unreleased]: https://github.com/addisdev/fleet-runner/compare/v0.9.1...HEAD +[0.9.1]: https://github.com/addisdev/fleet-runner/compare/v0.9.0...v0.9.1 [0.9.0]: https://github.com/addisdev/fleet-runner/compare/v0.8.0...v0.9.0 [0.8.0]: https://github.com/addisdev/fleet-runner/compare/v0.7.2...v0.8.0 [0.7.2]: https://github.com/addisdev/fleet-runner/compare/v0.7.1...v0.7.2 diff --git a/VERSION b/VERSION index ac39a10..f374f66 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.9.0 +0.9.1 diff --git a/collector/dash/package.json b/collector/dash/package.json index 80fe38a..8b3db14 100644 --- a/collector/dash/package.json +++ b/collector/dash/package.json @@ -1,6 +1,6 @@ { "name": "fleet-dash", - "version": "0.9.0", + "version": "0.9.1", "private": true, "type": "module", "description": "Fleet Runner dashboard SPA. Built to dash/dist and served by the collector at /dash.", diff --git a/collector/openapi.json b/collector/openapi.json index 1d87e49..24deb9f 100644 --- a/collector/openapi.json +++ b/collector/openapi.json @@ -2,7 +2,7 @@ "openapi": "3.1.0", "info": { "title": "Fleet Runner", - "version": "0.9.0", + "version": "0.9.1", "description": "The Fleet Runner control API.\n\nEvery list is `{items, page, per_page, total, pages}`. Every failure is\n`{error: {code, message, hint?}}`; branch on `code`. Timestamps are ISO 8601 UTC.\n\nThe usual sequence for running something: listWorkloads (pick one, copy its\nexample), validateJob (fix what it says), createJob, waitJob until\n`finished`, then read `verdict`. `skipped` means nothing ran; it is not a pass.\n\nAuth is off unless the collector runs with FLEET_AUTH=tokens. Then send\n`Authorization: Bearer `; scopes are read, run (implies read), admin\n(implies run), agent (the runner protocol) and delegate (a gateway naming the\nperson it acts for in X-Fleet-Actor). Each operation's `x-fleet-scope` says\nwhich it needs. Send `X-Fleet-Source` (mcp, cli, ci...) so jobs record where\nthey came from.", "license": { "name": "MIT" diff --git a/collector/package.json b/collector/package.json index 470c164..8953ace 100644 --- a/collector/package.json +++ b/collector/package.json @@ -1,6 +1,6 @@ { "name": "fleet-collector", - "version": "0.9.0", + "version": "0.9.1", "private": true, "type": "module", "engines": { diff --git a/collector/runner-web/index.html b/collector/runner-web/index.html index 014111c..7a37a79 100644 --- a/collector/runner-web/index.html +++ b/collector/runner-web/index.html @@ -144,7 +144,7 @@

Fleet Runner

* shelf forever as an offline device nobody can find. */ -const APP_VER = "0.9.0"; +const APP_VER = "0.9.1"; const SCHEMA = 1; const BLOCK_SIZE = 4096; const ROUNDS_PER_TOKEN = 1000; diff --git a/collector/scripts/seed-schedules.ts b/collector/scripts/seed-schedules.ts index b93e8ac..eb6d430 100644 --- a/collector/scripts/seed-schedules.ts +++ b/collector/scripts/seed-schedules.ts @@ -255,6 +255,27 @@ const SCHEDULES: Record = { } } }, + // Regression tests from fixed findings (src/workloads/regressions.ts), on + // the bug garden: its smoke flow, then every bug-garden finding marked real + // and then fixed, replayed on the shelf's Android phone. The garden's + // defects are planted, so a finding marked fixed comes back on this build + // and the run proves a recurring crash reopens its finding. Off here like + // every schedule; the lab turned it on 2026-10-18. Needs the bug-garden APK + // published under app "bug-garden" and bug-garden/{smoke,sign-in}.yaml in + // the executor's flows directory. + "nightly-bug-garden-regressions": { + "cron": "45 2 * * *", + "template": { + "schema": 1, + "workload": "ui-test", + "executor": "host", + "app": { "name": "bug-garden", "build": "latest", "sha256": "latest" }, + "suite": { "kind": "maestro", "flows": "bug-garden/smoke.yaml", "app_id": "dev.fleetrunner.buggarden" }, + "params": { "regressions": "bug-garden" }, + "targets": { "match": "os ~ 'android'", "executor": "fleet-host", "exclusive": true }, + "lease": { "ttl_s": 1800 } + } + }, // Overnight exploratory QA (src/workloads/explore). The night slot the plan // proposes is 22:00-00:30, before Dozehound's video queue takes the Studio's // memory; the window constraint keeps a late claim from running into it. diff --git a/collector/scripts/smoke.ts b/collector/scripts/smoke.ts index b652a20..5a10b69 100644 --- a/collector/scripts/smoke.ts +++ b/collector/scripts/smoke.ts @@ -31,6 +31,7 @@ import { runTvloopChecks } from "../src/workloads/tvloop/tvloop.test.js"; import { runColdStartChecks } from "../src/workloads/cold-start/cold-start.test.js"; import { runFindingsChecks } from "../src/findings.test.js"; import { runRegressionChecks } from "../src/workloads/regressions.test.js"; +import { runDeviceChecks } from "../src/workloads/device.test.js"; import { referenceDigest } from "./conformance.js"; import { redact, keychainPassword } from "../src/secrets.js"; @@ -2892,6 +2893,7 @@ await runFindingsChecks(check); // The ui-test side of regression tests: which app, a stored flow made // runnable, the crash-buffer judgement, and what each run reports. await runRegressionChecks(check); +runDeviceChecks(check); // --- night-QA findings, over HTTP --- // diff --git a/collector/src/executor.ts b/collector/src/executor.ts index 31c0537..0098347 100644 --- a/collector/src/executor.ts +++ b/collector/src/executor.ts @@ -48,7 +48,7 @@ import { parseXcuiDebugDescription, type A11yGeometry, type A11yNode, } from "./a11y-tree.js"; import { countBySeverity, uploadReport, type Finding } from "./web/shared.js"; -import { ADB, batteryPct, hasApp, launchApp, processAlive } from "./workloads/device.js"; +import { ADB, batteryPct, ensureAwake, hasApp, launchApp, processAlive } from "./workloads/device.js"; // Moved out so a workload directory can run a flow without importing the // executor, which would start a second poll loop. Same move device.ts made. import { FLOWS_DIR, MAESTRO, resolveFlow, runFlow } from "./workloads/flows.js"; @@ -603,6 +603,12 @@ async function runUiTest(job: Job) { log(`ui-test on ${serial}: skipped (${appId} not installed)`); continue; } + const asleep = await ensureAwake(target); + if (asleep) { + await postResult({ job_id: job.job_id, device_id: serial, iter: 0, ok: true, error: asleep }); + log(`ui-test on ${serial}: ${asleep}`); + continue; + } const mineRegs = regressionFlows.filter((f) => f.platform === target.platform); if (!flows && mineRegs.length === 0) { await postResult({ @@ -1350,6 +1356,11 @@ async function runLocaleShots(job: Job) { await postResult({ job_id: job.job_id, device_id: target.id, iter: 0, ok: true, error: `skipped: ${appId} not installed` }); continue; } + const asleep = await ensureAwake(target); + if (asleep) { + await postResult({ job_id: job.job_id, device_id: target.id, iter: 0, ok: true, error: asleep }); + continue; + } const deviceDir = safeName(target.id); const fingerprints = new Map(); diff --git a/collector/src/version.ts b/collector/src/version.ts index 0a7c60a..fa218e7 100644 --- a/collector/src/version.ts +++ b/collector/src/version.ts @@ -10,4 +10,4 @@ * which CI runs on every push. See its header for why one number across four * components is the rule. */ -export const APP_VERSION = "0.9.0"; +export const APP_VERSION = "0.9.1"; diff --git a/collector/src/workloads/device.test.ts b/collector/src/workloads/device.test.ts new file mode 100644 index 0000000..21a374f --- /dev/null +++ b/collector/src/workloads/device.test.ts @@ -0,0 +1,39 @@ +/** + * Reading the lock screen from `dumpsys window`, with no device. + * + * The locked sample is the Galaxy S8+ on fleet-host (Android 9) on 2026-10-18, + * after `input keyevent KEYCODE_WAKEUP` and `wm dismiss-keyguard`: its PIN pad + * kept focus. Run from scripts/smoke.ts like the other decision checks. + */ +import { screenLocked } from "./device.js"; + +type Check = (name: string, cond: boolean, detail?: string) => void; + +const S8_PIN = [ + " mShowingDream=false mDreamingLockscreen=true mDreamingSleepToken=null", + " mStatusBar=Window{a9c474 u0 StatusBar} isStatusBarKeyguard=true", + " mCurrentFocus=Window{7f53de3 u0 Bouncer}", +].join("\n"); + +const UNLOCKED = [ + " mShowingDream=false mDreamingLockscreen=false mDreamingSleepToken=null", + " mStatusBar=Window{a9c474 u0 StatusBar} isStatusBarKeyguard=false", + " mCurrentFocus=Window{1b2c3d4 u0 dev.fleetrunner.buggarden/dev.fleetrunner.buggarden.MainActivity}", +].join("\n"); + +// The notification shade pulled down on an unlocked phone also focuses the +// status bar; that is not a lock. +const SHADE = [ + " mShowingDream=false mDreamingLockscreen=false mDreamingSleepToken=null", + " isStatusBarKeyguard=false", + " mCurrentFocus=Window{a9c474 u0 StatusBar}", +].join("\n"); + +export function runDeviceChecks(check: Check) { + check("lock screen: the S8+ with a PIN reads as locked", screenLocked(S8_PIN)); + check("lock screen: an app in front reads as unlocked", !screenLocked(UNLOCKED)); + check("lock screen: the notification shade is not a lock", !screenLocked(SHADE)); + check("lock screen: a keyguard window in focus alone is enough", screenLocked(" mCurrentFocus=Window{9 u0 Keyguard}")); + check("lock screen: AOSP's showing flag alone is enough", screenLocked(" mShowingLockscreen=true mShowingDream=false")); + check("lock screen: nothing to go on reads as unlocked (an emulator, or another dump format)", !screenLocked("")); +} diff --git a/collector/src/workloads/device.ts b/collector/src/workloads/device.ts index e743296..d1edd11 100644 --- a/collector/src/workloads/device.ts +++ b/collector/src/workloads/device.ts @@ -49,6 +49,56 @@ export async function hasApp(target: Target, appId: string): Promise { } } +/** + * Is the lock screen still up, judging by `dumpsys window`? + * + * Read after waking the screen and asking the keyguard to go away, so "still + * up" means it would not go: a PIN, pattern or password. The signals differ + * across Android versions and vendors, so any one of them counts: the keyguard + * owning the status bar (AOSP, Samsung), the lock screen showing or dreaming, + * or the focused window being the keyguard or its PIN pad (Samsung calls it + * "Bouncer"). Not a focused StatusBar alone: that is also the notification + * shade pulled down on an unlocked phone. + */ +export function screenLocked(dumpsysWindow: string): boolean { + return /isStatusBarKeyguard=true/.test(dumpsysWindow) || + /mShowingLockscreen=true/.test(dumpsysWindow) || + /mDreamingLockscreen=true/.test(dumpsysWindow) || + /mCurrentFocus=Window\{[^}]*\b(Bouncer|Keyguard)\b/i.test(dumpsysWindow); +} + +/** + * Wake an Android device and dismiss a lock screen that has no PIN, before a + * host workload drives its screen. Returns null when the device is ready, or + * the reason it is not, which the caller posts as a skip. + * + * A phone on the shelf sleeps between jobs. Maestro does not wake it, so a + * flow ran against a dark lock screen and failed on its first assertion -- + * the Galaxy S8+'s nightly smoke said `"Fleet Runner" is visible` was false + * every night for a week, which read as the app being broken. A PIN is the one + * thing this cannot get past, and saying so is better than a red run that + * blames the app. Emulators have no lock screen and pass straight through. + */ +export async function ensureAwake(target: Target): Promise { + if (target.platform !== "android") return null; + const adb = (...args: string[]) => exec(ADB, ["-s", target.id, "shell", ...args], { timeout: 15_000 }); + try { + await adb("input", "keyevent", "KEYCODE_WAKEUP"); + await adb("wm", "dismiss-keyguard"); + await new Promise((r) => setTimeout(r, 1_500)); + const { stdout } = await adb("dumpsys", "window"); + if (screenLocked(stdout)) { + return "skipped: the screen is locked with a PIN, pattern or password, which nothing here can get past; " + + "remove the screen lock (Settings > Lock screen > None or Swipe) on this test device"; + } + return null; + } catch { + // A device adb cannot reach this way is left to the steps after this one, + // which say why in their own terms; waking it was a courtesy. + return null; + } +} + /** * Launch the app, three ways, because three platforms. * diff --git a/desktop/src-tauri/Cargo.toml b/desktop/src-tauri/Cargo.toml index 6a5305b..4ad2d75 100644 --- a/desktop/src-tauri/Cargo.toml +++ b/desktop/src-tauri/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "fleet-desktop" -version = "0.9.0" +version = "0.9.1" description = "Menu-bar shell around the fleet CLI" authors = ["addisdev"] license = "MIT" diff --git a/desktop/src-tauri/tauri.conf.json b/desktop/src-tauri/tauri.conf.json index 916f566..a793ca8 100644 --- a/desktop/src-tauri/tauri.conf.json +++ b/desktop/src-tauri/tauri.conf.json @@ -1,7 +1,7 @@ { "$schema": "https://schema.tauri.app/config/2", "productName": "Fleet", - "version": "0.9.0", + "version": "0.9.1", "identifier": "com.addisdev.fleet-desktop", "build": { "frontendDist": "../ui" diff --git a/docs/lab.md b/docs/lab.md index 378f877..69beec6 100644 --- a/docs/lab.md +++ b/docs/lab.md @@ -188,13 +188,14 @@ the copy is the last known-good state rather than the freshest one. ## The schedules -Five enabled. All times local to fleet-host. +Six enabled. All times local to fleet-host. | | When | Runs on | What red means | |---|---|---|---| | `nightly-synthetic-shelf` | 02:00 | fanout, pool `machines` | A machine's throughput moved, or an agent is not claiming. This is the comparison table | | `nightly-self-check` | 02:15 | fanout, pool `machines` | Disk, clock drift, a tool that vanished, or the agent is not supervised | -| `nightly-fleet-ui-smoke` | 02:30 | `fleet-host`, an Android device | **Red since 2026-08-21.** From 09-16 to early October it was `no targets attached`. The Galaxy S8+ is plugged in again and the suite now runs on it, and fails, every night; why is not diagnosed yet | +| `nightly-fleet-ui-smoke` | 02:30 | `fleet-host`, an Android device | **Red since 2026-08-21.** From 09-16 to early October it was `no targets attached`. The Galaxy S8+ is plugged in again but still has a PIN (WB-195): the flow ran against its lock screen and failed `"Fleet Runner" is visible`. From the release after 0.9.0 the executor wakes a phone and swipes its lock screen away, and a PIN makes the run `skipped` with that reason instead | +| `nightly-bug-garden-regressions` | 02:45 | `fleet-host`, an Android device | Bug garden's smoke flow, then every bug-garden finding marked real and then fixed, replayed as a regression test. The garden's defects are planted, so finding #18 (BG-01, marked fixed on purpose) comes back every night and reopens: that is the run working. `skipped:` while the S8+ is locked with a PIN (WB-195). Proved once on ultra's emulator on 2026-10-18 (`bg-regression-proof-1`) | | `nightly-aliquant-shots` | 04:15 | `fleet-host`, `chromium` and `mobile-chrome` | Green. The baseline was re-accepted on 2026-09-22 from fleet-host's own captures, after Aliquant's sign-in page gained a "Forgot password?" link; the old one had diverged 2.7% on desktop and 6.1% on mobile since 09-10. The `webkit` and `mobile-safari` baselines are still the August ones, because nothing runs those projects now | | `nightly-tvloop-hardware` | 05:00 | `fleet-host`, the Roku, exclusive | tvloop's doctor, install, contract tests and flow replay. `skipped:` until the Roku password is in fleet-host's Keychain (status `skipped` from 0.9). Red from doctor means the Roku is asleep, off the network, or out of developer mode | diff --git a/docs/workloads/host.md b/docs/workloads/host.md index 17a67e9..ed6f49b 100644 --- a/docs/workloads/host.md +++ b/docs/workloads/host.md @@ -279,6 +279,16 @@ all, the error is its exit code and the last line it printed to stderr, never its command line, which carries the test account's password. Until 0.9 the reason was in the uploaded report and nowhere else. +### A sleeping phone + +Before it drives an Android device's screen, `ui-test` (and `locale-shots`) +wakes it and asks the lock screen to go away. A phone on the shelf sleeps +between jobs and Maestro does not wake it, so a flow used to run against a dark +lock screen and fail on its first assertion, which read as the app being +broken. A lock screen with a PIN, pattern or password cannot be dismissed from +here: that device's row is `skipped` with the reason, and the fix is to set the +test device's screen lock to None or Swipe. Emulators have no lock screen. + ### Regression tests from fixed findings `params.regressions` adds the app's fixed night-QA findings to the run. Set it @@ -298,7 +308,10 @@ regression flows (`GET /findings/regressions`), and after the suite's own flow runs each one whose finding was on the same platform as the device. A finding joins by being marked Real and then Fixed; nobody edits the schedule. `flows` may be left out to run only the regression flows; a device with none to run is -then skipped, not passed. +then skipped, not passed. The lab's `nightly-bug-garden-regressions` is a +working example: the garden's defects are planted, so a bug-garden finding +marked fixed comes back on every run and reopens, which is the mechanism +proving itself rather than a bug. Each stored flow is adapted before it runs: its `appId` becomes the job's `suite.app_id` when the job sets one (night QA often explores `.debug`; the diff --git a/fleet/package.json b/fleet/package.json index 0363879..c77861e 100644 --- a/fleet/package.json +++ b/fleet/package.json @@ -1,6 +1,6 @@ { "name": "fleet-cli", - "version": "0.9.0", + "version": "0.9.1", "private": true, "type": "module", "description": "Fleet Runner's front door: one command that runs a brain, a runner, or both.", diff --git a/fleet/src/version.ts b/fleet/src/version.ts index d6edd7c..2f07117 100644 --- a/fleet/src/version.ts +++ b/fleet/src/version.ts @@ -6,4 +6,4 @@ * it. `scripts/version.mjs` keeps it honest with every other place -- * see its header for why the number matters at all. */ -export const VERSION = "0.9.0"; +export const VERSION = "0.9.1"; diff --git a/runner-android/app/build.gradle.kts b/runner-android/app/build.gradle.kts index 549bc5a..da1bc90 100644 --- a/runner-android/app/build.gradle.kts +++ b/runner-android/app/build.gradle.kts @@ -27,7 +27,7 @@ android { minSdk = 24 targetSdk = 35 versionCode = 2 - versionName = "0.9.0" + versionName = "0.9.1" ndk { // The fleet's LLM-capable devices are all arm64; 32-bit devices diff --git a/runner-ios/project.yml b/runner-ios/project.yml index f13940b..ff2c0bd 100644 --- a/runner-ios/project.yml +++ b/runner-ios/project.yml @@ -33,7 +33,7 @@ targets: settings: base: PRODUCT_BUNDLE_IDENTIFIER: com.taylab.fleetrunner - MARKETING_VERSION: "0.9.0" + MARKETING_VERSION: "0.9.1" GENERATE_INFOPLIST_FILE: YES ASSETCATALOG_COMPILER_APPICON_NAME: AppIcon INFOPLIST_KEY_UILaunchScreen_Generation: YES @@ -106,7 +106,7 @@ targets: settings: base: PRODUCT_BUNDLE_IDENTIFIER: com.taylab.fleetrunner.tv - MARKETING_VERSION: "0.9.0" + MARKETING_VERSION: "0.9.1" SWIFT_VERSION: "5.9" TARGETED_DEVICE_FAMILY: "3" CODE_SIGN_IDENTITY: "" @@ -164,7 +164,7 @@ targets: settings: base: PRODUCT_BUNDLE_IDENTIFIER: com.taylab.fleetrunner.vision - MARKETING_VERSION: "0.9.0" + MARKETING_VERSION: "0.9.1" SWIFT_VERSION: "5.9" CODE_SIGN_IDENTITY: "" CODE_SIGNING_REQUIRED: NO diff --git a/runner-machine/package.json b/runner-machine/package.json index 37152f9..91081c7 100644 --- a/runner-machine/package.json +++ b/runner-machine/package.json @@ -1,6 +1,6 @@ { "name": "fleet-runner-machine", - "version": "0.9.0", + "version": "0.9.1", "private": true, "type": "module", "description": "Fleet Runner machine agent: turns a laptop or desktop into a device the collector can send work to.", diff --git a/runner-machine/src/descriptor.ts b/runner-machine/src/descriptor.ts index 3a9321e..3b7073f 100644 --- a/runner-machine/src/descriptor.ts +++ b/runner-machine/src/descriptor.ts @@ -16,7 +16,7 @@ import { existsSync } from "node:fs"; import { out, readText, orNull, firstMatch, finite, run } from "./probe.js"; import type { Descriptor } from "./protocol.js"; -export const APP_VER = "0.9.0"; +export const APP_VER = "0.9.1"; const MB = 1024 * 1024; diff --git a/runner-roku/manifest b/runner-roku/manifest index ecc532b..fb0e5bf 100644 --- a/runner-roku/manifest +++ b/runner-roku/manifest @@ -23,7 +23,7 @@ build_version=0 # table next to a released one's. Arbitrary manifest keys are readable at # runtime, so the descriptor's version and the repo's version are one string in # one place rather than a constant in the code that drifts from the manifest. -app_ver=0.9.0 +app_ver=0.9.1 # Channel art. Roku draws the focus icon in the home grid and the splash while # the channel boots; a manifest line pointing at a file that is not in the zip