Explore: survive a dropped model, judge only missions that drove the app, fail fast on Xcode's first launch

CHANGELOG.md

@@ -35,6 +35,44 @@ a CI job fails when they disagree.
   screen with no PIN first; a PIN, pattern or password makes that device's row
   `skipped` with the reason, since nothing here can get past it.
   ([#100](https://github.com/addisdev/fleet-runner/pull/100))
+- **A model unloaded mid-night is loaded again.** On ultra the model shelf
+  unloads an on-demand model it sees idle under memory pressure, and between
+  two missions Holo4 is idle: on the first three 0.9.0 nights it was dropped
+  mid-run four times, and every later mission ended at step 0 on `Invalid
+  model identifier`. The workload now runs the night's warm-up again
+  (`FLEET_EXPLORE_WARM`, set by `explore-night.ts --warm`) and retakes the
+  step, twice a mission at most. The night command also sets `SHELF_HOLD_PID`
+  for its warm-up, which ultra's shelf now records as a hold so it leaves the
+  model loaded for the night. ([#101](https://github.com/addisdev/fleet-runner/pull/101))
+- **No goal findings from missions that never drove the app.** A mission cut
+  off by the model server or the device, or abandoned by the model (four
+  replies without a tool call), is no longer judged: the judge had filed the
+  untouched or abandoned screens as goal findings (#70, #71 and the bug
+  garden's 10-17 goal findings). A cut-off mission is `explore_cut_off`, not
+  `explore_missions`, and a bench card it had not finished is not scored.
+  ([#101](https://github.com/addisdev/fleet-runner/pull/101))
+- **A bench card's end state the app takes away by itself counts.** When
+  nothing touched the device after the model's last look, that look is checked
+  too: Dozehound hides its channel bar six seconds after a press, so
+  `bench-focus-all-channels` failed on a screen the model had already left
+  right. ([#101](https://github.com/addisdev/fleet-runner/pull/101))
+- **An Xcode that needs its first-launch install fails the night up front,
+  with the fix.** `explore-night.ts --sim` asks `xcodebuild
+  -checkFirstLaunchStatus` before warming the model, and FleetDriver stops at
+  the first `CoreSimulator is out of date` instead of spending a minute on each
+  card. ultra's Xcode updated itself to 27.1 on 10-16 and the TV and Apple
+  nights each spent about ten minutes a job finding that out.
+  ([#101](https://github.com/addisdev/fleet-runner/pull/101))
+- **A failed explore job says why on the job.** Its error is the failing
+  device's reason, not "one or more devices completed no mission; see their
+  rows", so the alert and the digest carry it.
+  ([#101](https://github.com/addisdev/fleet-runner/pull/101))
+- **The night command's log lines are in local time**, like the times inside
+  them. ([#101](https://github.com/addisdev/fleet-runner/pull/101))
+- **The changelog coverage check exempts the release's own pull requests**:
+  `release/*` into `develop` and `develop` into `main`. 0.9.0's `release/0.9.0`
+  pull request (#98) failed its own `develop` → `main` check and had to be
+  labelled `no-changelog` by hand. ([#101](https://github.com/addisdev/fleet-runner/pull/101))
 
 ## [0.9.0] — 2026-10-15
 

CONTRIBUTING.md

@@ -188,7 +188,8 @@ One version covers every component, so a release is one number everywhere.
    into `develop` since the last `v*` tag and fails, naming each, if one is
    not mentioned as `#N` or a `/pull/N` link between `## [Unreleased]` and the
    last release's heading. Label a pull request `no-changelog` to exempt it;
-   Dependabot's are exempt already. The check only runs when the branch is
+   Dependabot's are exempt already, and so are the release's own pull
+   requests (`release/*` into `develop`, `develop` into `main`). The check only runs when the branch is
    `release/*`, or `develop` into `main` — so name the branch that way, or it
    will not look. Run it yourself first with
    `node scripts/changelog-coverage.mjs --head origin/develop` (it needs `gh`

collector/schemas/metrics.json

@@ -35,6 +35,7 @@
     "explore_cached_tokens_mean",
     "explore_candidates",
     "explore_cards_planned",
+    "explore_cut_off",
     "explore_findings",
     "explore_findings_new",
     "explore_minutes",

collector/schemas/result.schema.json

@@ -186,6 +186,7 @@
         "explore_not_reproduced": { "type": "integer", "description": "explore: candidates dropped because no replay reproduced them" },
         "explore_misses": { "type": "integer", "description": "explore: taps that landed on nothing tappable (the agent's misses, not app bugs)" },
         "explore_refused": { "type": "integer", "description": "explore: actions the leash refused" },
+        "explore_cut_off": { "type": "integer", "description": "explore: missions the model server or the device cut off; not counted in explore_missions and not judged" },
         "explore_step_ms_p50": { "type": "number", "description": "explore: median wall time of one step (observe, model, act, checks), ms" },
         "explore_model_ms_p50": { "type": "number", "description": "explore: median model latency per step, ms" },
         "explore_prompt_tokens_mean": { "type": "number", "description": "explore: mean prompt tokens per model call" },

collector/scripts/explore-night.ts

@@ -8,9 +8,11 @@
 //
 //   1. opens the tunnel to the brain and checks, before anything expensive,
 //      that each spec has a build to test (`--fresh-build`: one published
-//      tonight) and that the night has time left for it;
+//      tonight), that the night has time left for it, and with `--sim` that
+//      Xcode has had its first-launch install;
 //   2. warms the model (`--warm "<cmd>"`), retrying a warm-up the host refused
-//      for memory for up to `--warm-wait` minutes;
+//      for memory for up to `--warm-wait` minutes, and hands the same command
+//      to the workload to run again if the model is unloaded mid-night;
 //   3. boots the night's emulator or simulator if it is not already up (and
 //      remembers it did);
 //   4. enqueues each spec on the brain with tonight's date in its id, pinned to
@@ -66,7 +68,9 @@ const BASE = (process.env.FLEET_URL ?? "http://127.0.0.1:18788").replace(/\/+$/,
 const NAME = process.env.FLEET_EXECUTOR_NAME ?? "ultra";
 const SDK = process.env.ANDROID_HOME ?? path.join(process.env.HOME ?? "", "Library/Android/sdk");
 const ADB = path.join(SDK, "platform-tools/adb");
-const log = (m: string) => console.log(`[explore-night ${new Date().toISOString().slice(11, 19)}] ${m}`);
+// Local time, like every other time in these lines ("the night ends at 06:14")
+// and in the night queue's own log; UTC here put two clocks in one line.
+const log = (m: string) => console.log(`[explore-night ${new Date().toTimeString().slice(0, 8)}] ${m}`);
 const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
 
 /** The serial of a running emulator with this AVD name, or null. */
@@ -227,6 +231,24 @@ async function postUnrun(item: Item, v: Verdict): Promise<string> {
   return `${item.jobId}: ${v.ok ? "skipped" : "failed"}`;
 }
 
+/**
+ * Why Xcode cannot run a simulator test tonight, or null. After an Xcode
+ * update its first-launch install (CoreSimulator among it) has to run before
+ * xcodebuild can drive a simulator; until then every FleetDriver start fails a
+ * minute in with "CoreSimulator is out of date". ultra's Xcode updated itself
+ * to 27.1 on 2026-10-16 and the TV and Apple nights failed that way after
+ * warming a 25 GB model for nothing. `-checkFirstLaunchStatus` answers in a
+ * second, and only the owner can run the fix (it needs sudo). FLEET_XCODEBUILD
+ * stands in for xcodebuild in the tests, which must not depend on this host's Xcode.
+ */
+async function xcodeProblem(): Promise<string | null> {
+  const dir = await run("xcode-select", ["-p"]).then((r) => r.stdout.trim(), () => "Xcode");
+  const code = await run(process.env.FLEET_XCODEBUILD ?? "xcodebuild", ["-checkFirstLaunchStatus"], { timeout: 60_000 }).then(() => 0, (e: { code?: number }) => e.code ?? 1);
+  if (code === 0) return null;
+  return `Xcode at ${dir.replace(/\/Contents\/Developer$/, "")} needs its first-launch install (xcodebuild -checkFirstLaunchStatus exit ${code}); ` +
+    "run `sudo xcodebuild -runFirstLaunch` on this host";
+}
+
 /** Run the warm-up command, retrying a refusal for memory until `untilMs`. */
 async function warmUp(cmd: string, untilMs: number): Promise<{ ok: true } | { ok: false; refused: boolean; line: string }> {
   for (let attempt = 1; ; attempt++) {
@@ -290,6 +312,13 @@ async function main() {
       log(`short night: ${list.map((i, k) => `${path.basename(i.path)} ${plan.minutes[k]} of ${i.asked} min`).join(", ")}`);
     }
   }
+  // A host problem, not a spec's: a failure with its reason, so the night
+  // queue's check stays red until someone fixes it, but found before the
+  // model is warmed rather than ten minutes into each job.
+  if (sim && process.platform === "darwin" && runnable().length) {
+    const why = await xcodeProblem();
+    if (why) for (const i of runnable()) verdicts.set(i, { ok: false, error: why });
+  }
   for (const [i, v] of verdicts) log(`${path.basename(i.path)}: ${v.ok ? "" : "cannot run: "}${v.error}`);
 
   if (dry) {
@@ -317,6 +346,12 @@ async function main() {
 
   // --- the model ---------------------------------------------------------
   if (warm) {
+    // The shelf on ultra unloads an on-demand model it finds idle under
+    // memory pressure, and between two missions the model is idle. With this
+    // set, its warm-up records a hold for the model that lasts as long as this
+    // process; the workload's warm-up after a drop inherits it too. A warm
+    // command that knows nothing of it ignores it.
+    process.env.SHELF_HOLD_PID = String(process.pid);
     // The wait for room may use the night, but must leave the minimum.
     const w = await warmUp(warm, Math.min(Date.now() + warmWait * 60_000, end.getTime() - (minUseful + NIGHT.safetyMin) * 60_000));
     if (!w.ok) {
@@ -340,6 +375,9 @@ async function main() {
   // so they are settled before the import rather than passed in.
   process.env.FLEET_URL = BASE;
   process.env.FLEET_EXECUTOR_NAME = NAME;
+  // The model shelf may unload the model mid-night; the workload runs this
+  // again when the server says it has no such model.
+  if (warm) process.env.FLEET_EXPLORE_WARM = warm;
   const { startExecutor } = await import("../src/executor.js");
   const executor = await startExecutor();
 

collector/src/workloads/explore/actuators/apple.ts

@@ -56,6 +56,17 @@ import type { Action, Actuator, ActuatorCaps, CrashReport, Key, Observation } fr
 // ---------------------------------------------------------------------------
 
 /** A TV is driven by focus; everything else Apple makes here is touched. */
+/**
+ * xcodebuild's words for an Xcode whose first-launch install has not run:
+ * Xcode was updated (ultra's went from 27.0 to 27.1 on 2026-10-16) and the
+ * CoreSimulator it ships was not installed with it. Nothing a retry changes;
+ * `sudo xcodebuild -runFirstLaunch` does.
+ */
+export const XCODE_FIRST_LAUNCH = /CoreSimulator is out of date|xcodebuild -runFirstLaunch|requires (the )?installation of additional components/i;
+
+/** A host that cannot run FleetDriver at all until someone fixes it. Every further mission would fail the same way. */
+export class HostNotReadyError extends Error {}
+
 export function isTv(target: Pick<Target, "platform">): boolean {
   return target.platform === "tvos";
 }
@@ -559,6 +570,8 @@ export class AppleActuator implements Actuator {
     } catch (e) {
       const out = ((e as { stdout?: string }).stdout ?? "") + ((e as { stderr?: string }).stderr ?? "");
       const errors = out.split("\n").filter((l) => /error:|BUILD FAILED|\*\* .* FAILED \*\*/.test(l)).slice(-12);
+      const firstLaunch = out.split("\n").find((l) => XCODE_FIRST_LAUNCH.test(l));
+      if (firstLaunch) throw new HostNotReadyError(`Xcode on this host needs its first-launch install (${firstLaunch.trim()}); run \`sudo xcodebuild -runFirstLaunch\``);
       throw new Error(`FleetDriver build failed: ${(errors.join("\n") || out.slice(-2000) || (e as Error).message).trim()}`);
     }
     writeFileSync(stamp, hash + "\n");
@@ -621,10 +634,14 @@ export class AppleActuator implements Actuator {
     let lastErr = "";
     while (Date.now() < deadline) {
       if (this.childExit) {
-        throw new Error(
-          `xcodebuild exited (${this.childExit}) before FleetDriver answered on ${this.target.id}. ` +
-          `Last lines:\n${this.tail.slice(-15).join("\n")}`,
-        );
+        const tail = this.tail.slice(-15).join("\n");
+        if (XCODE_FIRST_LAUNCH.test(this.tail.join("\n"))) {
+          throw new HostNotReadyError(
+            `Xcode on this host needs its first-launch install (${this.tail.find((l) => XCODE_FIRST_LAUNCH.test(l))?.trim()}); ` +
+            "run `sudo xcodebuild -runFirstLaunch`",
+          );
+        }
+        throw new Error(`xcodebuild exited (${this.childExit}) before FleetDriver answered on ${this.target.id}. Last lines:\n${tail}`);
       }
       try {
         if (!this.baseUrl) this.baseUrl = await this.resolveBaseUrl();