diff --git a/CHANGELOG.md b/CHANGELOG.md index 030ef31..ecfe512 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -35,6 +35,44 @@ a CI job fails when they disagree. screen with no PIN first; a PIN, pattern or password makes that device's row `skipped` with the reason, since nothing here can get past it. ([#100](https://github.com/addisdev/fleet-runner/pull/100)) +- **A model unloaded mid-night is loaded again.** On ultra the model shelf + unloads an on-demand model it sees idle under memory pressure, and between + two missions Holo4 is idle: on the first three 0.9.0 nights it was dropped + mid-run four times, and every later mission ended at step 0 on `Invalid + model identifier`. The workload now runs the night's warm-up again + (`FLEET_EXPLORE_WARM`, set by `explore-night.ts --warm`) and retakes the + step, twice a mission at most. The night command also sets `SHELF_HOLD_PID` + for its warm-up, which ultra's shelf now records as a hold so it leaves the + model loaded for the night. ([#101](https://github.com/addisdev/fleet-runner/pull/101)) +- **No goal findings from missions that never drove the app.** A mission cut + off by the model server or the device, or abandoned by the model (four + replies without a tool call), is no longer judged: the judge had filed the + untouched or abandoned screens as goal findings (#70, #71 and the bug + garden's 10-17 goal findings). A cut-off mission is `explore_cut_off`, not + `explore_missions`, and a bench card it had not finished is not scored. + ([#101](https://github.com/addisdev/fleet-runner/pull/101)) +- **A bench card's end state the app takes away by itself counts.** When + nothing touched the device after the model's last look, that look is checked + too: Dozehound hides its channel bar six seconds after a press, so + `bench-focus-all-channels` failed on a screen the model had already left + right. ([#101](https://github.com/addisdev/fleet-runner/pull/101)) +- **An Xcode that needs its first-launch install fails the night up front, + with the fix.** `explore-night.ts --sim` asks `xcodebuild + -checkFirstLaunchStatus` before warming the model, and FleetDriver stops at + the first `CoreSimulator is out of date` instead of spending a minute on each + card. ultra's Xcode updated itself to 27.1 on 10-16 and the TV and Apple + nights each spent about ten minutes a job finding that out. + ([#101](https://github.com/addisdev/fleet-runner/pull/101)) +- **A failed explore job says why on the job.** Its error is the failing + device's reason, not "one or more devices completed no mission; see their + rows", so the alert and the digest carry it. + ([#101](https://github.com/addisdev/fleet-runner/pull/101)) +- **The night command's log lines are in local time**, like the times inside + them. ([#101](https://github.com/addisdev/fleet-runner/pull/101)) +- **The changelog coverage check exempts the release's own pull requests**: + `release/*` into `develop` and `develop` into `main`. 0.9.0's `release/0.9.0` + pull request (#98) failed its own `develop` → `main` check and had to be + labelled `no-changelog` by hand. ([#101](https://github.com/addisdev/fleet-runner/pull/101)) ## [0.9.0] — 2026-10-15 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d13e343..ec6b45c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -188,7 +188,8 @@ One version covers every component, so a release is one number everywhere. into `develop` since the last `v*` tag and fails, naming each, if one is not mentioned as `#N` or a `/pull/N` link between `## [Unreleased]` and the last release's heading. Label a pull request `no-changelog` to exempt it; - Dependabot's are exempt already. The check only runs when the branch is + Dependabot's are exempt already, and so are the release's own pull + requests (`release/*` into `develop`, `develop` into `main`). The check only runs when the branch is `release/*`, or `develop` into `main` — so name the branch that way, or it will not look. Run it yourself first with `node scripts/changelog-coverage.mjs --head origin/develop` (it needs `gh` diff --git a/collector/schemas/metrics.json b/collector/schemas/metrics.json index 24d6dbf..6e1f2b2 100644 --- a/collector/schemas/metrics.json +++ b/collector/schemas/metrics.json @@ -35,6 +35,7 @@ "explore_cached_tokens_mean", "explore_candidates", "explore_cards_planned", + "explore_cut_off", "explore_findings", "explore_findings_new", "explore_minutes", diff --git a/collector/schemas/result.schema.json b/collector/schemas/result.schema.json index 27e214e..067ffa9 100644 --- a/collector/schemas/result.schema.json +++ b/collector/schemas/result.schema.json @@ -186,6 +186,7 @@ "explore_not_reproduced": { "type": "integer", "description": "explore: candidates dropped because no replay reproduced them" }, "explore_misses": { "type": "integer", "description": "explore: taps that landed on nothing tappable (the agent's misses, not app bugs)" }, "explore_refused": { "type": "integer", "description": "explore: actions the leash refused" }, + "explore_cut_off": { "type": "integer", "description": "explore: missions the model server or the device cut off; not counted in explore_missions and not judged" }, "explore_step_ms_p50": { "type": "number", "description": "explore: median wall time of one step (observe, model, act, checks), ms" }, "explore_model_ms_p50": { "type": "number", "description": "explore: median model latency per step, ms" }, "explore_prompt_tokens_mean": { "type": "number", "description": "explore: mean prompt tokens per model call" }, diff --git a/collector/scripts/explore-night.ts b/collector/scripts/explore-night.ts index 8a05e5b..919313b 100644 --- a/collector/scripts/explore-night.ts +++ b/collector/scripts/explore-night.ts @@ -8,9 +8,11 @@ // // 1. opens the tunnel to the brain and checks, before anything expensive, // that each spec has a build to test (`--fresh-build`: one published -// tonight) and that the night has time left for it; +// tonight), that the night has time left for it, and with `--sim` that +// Xcode has had its first-launch install; // 2. warms the model (`--warm ""`), retrying a warm-up the host refused -// for memory for up to `--warm-wait` minutes; +// for memory for up to `--warm-wait` minutes, and hands the same command +// to the workload to run again if the model is unloaded mid-night; // 3. boots the night's emulator or simulator if it is not already up (and // remembers it did); // 4. enqueues each spec on the brain with tonight's date in its id, pinned to @@ -66,7 +68,9 @@ const BASE = (process.env.FLEET_URL ?? "http://127.0.0.1:18788").replace(/\/+$/, const NAME = process.env.FLEET_EXECUTOR_NAME ?? "ultra"; const SDK = process.env.ANDROID_HOME ?? path.join(process.env.HOME ?? "", "Library/Android/sdk"); const ADB = path.join(SDK, "platform-tools/adb"); -const log = (m: string) => console.log(`[explore-night ${new Date().toISOString().slice(11, 19)}] ${m}`); +// Local time, like every other time in these lines ("the night ends at 06:14") +// and in the night queue's own log; UTC here put two clocks in one line. +const log = (m: string) => console.log(`[explore-night ${new Date().toTimeString().slice(0, 8)}] ${m}`); const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)); /** The serial of a running emulator with this AVD name, or null. */ @@ -227,6 +231,24 @@ async function postUnrun(item: Item, v: Verdict): Promise { return `${item.jobId}: ${v.ok ? "skipped" : "failed"}`; } +/** + * Why Xcode cannot run a simulator test tonight, or null. After an Xcode + * update its first-launch install (CoreSimulator among it) has to run before + * xcodebuild can drive a simulator; until then every FleetDriver start fails a + * minute in with "CoreSimulator is out of date". ultra's Xcode updated itself + * to 27.1 on 2026-10-16 and the TV and Apple nights failed that way after + * warming a 25 GB model for nothing. `-checkFirstLaunchStatus` answers in a + * second, and only the owner can run the fix (it needs sudo). FLEET_XCODEBUILD + * stands in for xcodebuild in the tests, which must not depend on this host's Xcode. + */ +async function xcodeProblem(): Promise { + const dir = await run("xcode-select", ["-p"]).then((r) => r.stdout.trim(), () => "Xcode"); + const code = await run(process.env.FLEET_XCODEBUILD ?? "xcodebuild", ["-checkFirstLaunchStatus"], { timeout: 60_000 }).then(() => 0, (e: { code?: number }) => e.code ?? 1); + if (code === 0) return null; + return `Xcode at ${dir.replace(/\/Contents\/Developer$/, "")} needs its first-launch install (xcodebuild -checkFirstLaunchStatus exit ${code}); ` + + "run `sudo xcodebuild -runFirstLaunch` on this host"; +} + /** Run the warm-up command, retrying a refusal for memory until `untilMs`. */ async function warmUp(cmd: string, untilMs: number): Promise<{ ok: true } | { ok: false; refused: boolean; line: string }> { for (let attempt = 1; ; attempt++) { @@ -290,6 +312,13 @@ async function main() { log(`short night: ${list.map((i, k) => `${path.basename(i.path)} ${plan.minutes[k]} of ${i.asked} min`).join(", ")}`); } } + // A host problem, not a spec's: a failure with its reason, so the night + // queue's check stays red until someone fixes it, but found before the + // model is warmed rather than ten minutes into each job. + if (sim && process.platform === "darwin" && runnable().length) { + const why = await xcodeProblem(); + if (why) for (const i of runnable()) verdicts.set(i, { ok: false, error: why }); + } for (const [i, v] of verdicts) log(`${path.basename(i.path)}: ${v.ok ? "" : "cannot run: "}${v.error}`); if (dry) { @@ -317,6 +346,12 @@ async function main() { // --- the model --------------------------------------------------------- if (warm) { + // The shelf on ultra unloads an on-demand model it finds idle under + // memory pressure, and between two missions the model is idle. With this + // set, its warm-up records a hold for the model that lasts as long as this + // process; the workload's warm-up after a drop inherits it too. A warm + // command that knows nothing of it ignores it. + process.env.SHELF_HOLD_PID = String(process.pid); // The wait for room may use the night, but must leave the minimum. const w = await warmUp(warm, Math.min(Date.now() + warmWait * 60_000, end.getTime() - (minUseful + NIGHT.safetyMin) * 60_000)); if (!w.ok) { @@ -340,6 +375,9 @@ async function main() { // so they are settled before the import rather than passed in. process.env.FLEET_URL = BASE; process.env.FLEET_EXECUTOR_NAME = NAME; + // The model shelf may unload the model mid-night; the workload runs this + // again when the server says it has no such model. + if (warm) process.env.FLEET_EXPLORE_WARM = warm; const { startExecutor } = await import("../src/executor.js"); const executor = await startExecutor(); diff --git a/collector/src/workloads/explore/actuators/apple.ts b/collector/src/workloads/explore/actuators/apple.ts index 38eb080..2011b6d 100644 --- a/collector/src/workloads/explore/actuators/apple.ts +++ b/collector/src/workloads/explore/actuators/apple.ts @@ -56,6 +56,17 @@ import type { Action, Actuator, ActuatorCaps, CrashReport, Key, Observation } fr // --------------------------------------------------------------------------- /** A TV is driven by focus; everything else Apple makes here is touched. */ +/** + * xcodebuild's words for an Xcode whose first-launch install has not run: + * Xcode was updated (ultra's went from 27.0 to 27.1 on 2026-10-16) and the + * CoreSimulator it ships was not installed with it. Nothing a retry changes; + * `sudo xcodebuild -runFirstLaunch` does. + */ +export const XCODE_FIRST_LAUNCH = /CoreSimulator is out of date|xcodebuild -runFirstLaunch|requires (the )?installation of additional components/i; + +/** A host that cannot run FleetDriver at all until someone fixes it. Every further mission would fail the same way. */ +export class HostNotReadyError extends Error {} + export function isTv(target: Pick): boolean { return target.platform === "tvos"; } @@ -559,6 +570,8 @@ export class AppleActuator implements Actuator { } catch (e) { const out = ((e as { stdout?: string }).stdout ?? "") + ((e as { stderr?: string }).stderr ?? ""); const errors = out.split("\n").filter((l) => /error:|BUILD FAILED|\*\* .* FAILED \*\*/.test(l)).slice(-12); + const firstLaunch = out.split("\n").find((l) => XCODE_FIRST_LAUNCH.test(l)); + if (firstLaunch) throw new HostNotReadyError(`Xcode on this host needs its first-launch install (${firstLaunch.trim()}); run \`sudo xcodebuild -runFirstLaunch\``); throw new Error(`FleetDriver build failed: ${(errors.join("\n") || out.slice(-2000) || (e as Error).message).trim()}`); } writeFileSync(stamp, hash + "\n"); @@ -621,10 +634,14 @@ export class AppleActuator implements Actuator { let lastErr = ""; while (Date.now() < deadline) { if (this.childExit) { - throw new Error( - `xcodebuild exited (${this.childExit}) before FleetDriver answered on ${this.target.id}. ` + - `Last lines:\n${this.tail.slice(-15).join("\n")}`, - ); + const tail = this.tail.slice(-15).join("\n"); + if (XCODE_FIRST_LAUNCH.test(this.tail.join("\n"))) { + throw new HostNotReadyError( + `Xcode on this host needs its first-launch install (${this.tail.find((l) => XCODE_FIRST_LAUNCH.test(l))?.trim()}); ` + + "run `sudo xcodebuild -runFirstLaunch`", + ); + } + throw new Error(`xcodebuild exited (${this.childExit}) before FleetDriver answered on ${this.target.id}. Last lines:\n${tail}`); } try { if (!this.baseUrl) this.baseUrl = await this.resolveBaseUrl(); diff --git a/collector/src/workloads/explore/explore.test.ts b/collector/src/workloads/explore/explore.test.ts index dd94e7f..2f942b4 100644 --- a/collector/src/workloads/explore/explore.test.ts +++ b/collector/src/workloads/explore/explore.test.ts @@ -17,14 +17,14 @@ import os from "node:os"; import path from "node:path"; import { PNG } from "pngjs"; import type { A11yNode } from "../../a11y-tree.js"; -import { unpackApp } from "./actuators/apple.js"; +import { XCODE_FIRST_LAUNCH, unpackApp } from "./actuators/apple.js"; import { escapeInputText, focusLine, launchExtras, parseAnrEvents, parseForeground, parseImeShown, pngSize } from "./actuators/android.js"; import { parseConditions } from "./conditions.js"; import { dhash, hamming, isBlank, shrinkForModel } from "./image.js"; import { fileable, gatewayKey, missingFlowVars, modelConfigs, paramsProblem, pickCandidates } from "./index.js"; import { parseJsonLoose, visualIssuesFrom } from "./judge.js"; import { leash, leftApp, snap } from "./leash.js"; -import { benchCheck, confirm, fingerprint, normalizeMessage, runMission, type LoopDeps } from "./loop.js"; +import { benchCheck, confirm, endpointFailure, fingerprint, modelGone, normalizeMessage, runMission, type LoopDeps } from "./loop.js"; import { changedWords, orderMissions } from "./missions.js"; import { convertCall, elementHints, parseInlineCalls, parseToolCalls } from "./model.js"; import { toMaestro, type Executed } from "./replay.js"; @@ -247,6 +247,17 @@ export async function runExploreChecks(check: Check): Promise { // --- a whole mission against a fake device and a fake model ----------------- await missionChecks(check); + + // --- the model server dropped the model, the host's Xcode needs installing --- + const LMSTUDIO_GONE = 'model endpoint 400: {"error":{"message":"litellm.NotFoundError: Lm_studioException - Invalid model identifier \\"holo4-35b-a3b-gguf\\". Please specify a valid downloaded model'; + check("explore(model): LM Studio's unloaded model reads as gone", modelGone(LMSTUDIO_GONE)); + check("explore(model): a timeout is not a gone model", !modelGone("The operation was aborted due to timeout")); + check("explore(model): a reply without a tool call is the model's, not the endpoint's", + !endpointFailure("the reply had no tool call") && endpointFailure(LMSTUDIO_GONE) && !endpointFailure(undefined)); + check("explore(apple): xcodebuild's CoreSimulator complaint means Xcode's first launch", + XCODE_FIRST_LAUNCH.test("CoreSimulator is out of date. Current version (1171.7.0) is older than build version (1174.9.2).") + && !XCODE_FIRST_LAUNCH.test("** TEST EXECUTE FAILED **")); + await cutOffChecks(check); } /** @@ -371,3 +382,114 @@ async function missionChecks(check: Check) { server.close(); } } + +/** + * A model server that has lost the model until `loaded` is set, the way LM + * Studio answered through LiteLLM on ultra after the model shelf unloaded + * Holo4 mid-night (2026-10-16). `asked` counts the requests it answered. + */ +function droppingModel(state: { loaded: boolean; asked: number }): Promise<{ server: Server; url: string }> { + return new Promise((resolve) => { + const server = createServer((req, res) => { + req.resume(); + req.on("end", () => { + state.asked++; + res.setHeader("content-type", "application/json"); + if (!state.loaded) { + res.statusCode = 400; + res.end(JSON.stringify({ error: { message: 'litellm.NotFoundError: Lm_studioException - Invalid model identifier "holo4-35b-a3b-gguf".' } })); + return; + } + res.end(JSON.stringify({ + choices: [{ message: { role: "assistant", content: "", tool_calls: [{ id: `c${state.asked}`, type: "function", function: { name: "answer", arguments: JSON.stringify({ content: "done" }) } }] } }], + usage: { prompt_tokens: 1000, completion_tokens: 20 }, + })); + }); + }); + server.listen(0, "127.0.0.1", () => { + const a = server.address(); + resolve({ server, url: `http://127.0.0.1:${typeof a === "object" && a ? a.port : 0}` }); + }); + }); +} + +async function cutOffChecks(check: Check) { + const pilot = { loaded: false, asked: 0 }; + const judge = { loaded: true, asked: 0 }; + const p = await droppingModel(pilot); + const j = await droppingModel(judge); + try { + const dir = mkdtempSync(path.join(os.tmpdir(), "explore-cut-")); + const deps = (rewarm?: LoopDeps["rewarm"]): LoopDeps => ({ + actuator: new FakeApp(), appId: "com.fake", appName: "Garden", appFile: null, + model: { baseUrl: p.url, model: "pilot" }, judge: { baseUrl: j.url, model: "judge" }, + map: new ScreenMap("fake", path.join(dir, "map")), runDir: dir, night: "n1", + // Visual judging off, so the judge is asked only about the goal. + log: () => {}, canRunFlows: false, deadline: Date.now() + 60_000, disabledChecks: ["visual"], ...(rewarm ? { rewarm } : {}), + }); + const goal: Mission = { id: "g", app: "fake", title: "Press every button", persona: "a viewer", goal: "press them all", success: "the app is playing", budget: { steps: 8 } }; + const r = await runMission(goal, deps()); + check("explore(cut off): a mission the model endpoint never answered ends as cut off", + r.stats.endedBy === "model" && r.stats.steps === 0 && !!r.cutOff && /Invalid model identifier/.test(r.cutOff), JSON.stringify({ stats: r.stats, cutOff: r.cutOff })); + check("explore(cut off): its untouched end state is not judged, so no goal finding", + judge.asked === 0 && r.goal === null && !r.candidates.some((c) => c.check === "goal"), `judge asked ${judge.asked}`); + + // A model that answers in prose with no tool call, four times running. + const prose = await new Promise<{ server: Server; url: string }>((resolve) => { + const server = createServer((req, res) => { + req.resume(); + req.on("end", () => { + res.setHeader("content-type", "application/json"); + res.end(JSON.stringify({ choices: [{ message: { role: "assistant", content: "Scroll up to explore more settings options." } }], usage: {} })); + }); + }); + server.listen(0, "127.0.0.1", () => { + const a = server.address(); + resolve({ server, url: `http://127.0.0.1:${typeof a === "object" && a ? a.port : 0}` }); + }); + }); + try { + const rp = await runMission(goal, { ...deps(), model: { baseUrl: prose.url, model: "pilot" } }); + check("explore(gave up): a mission the model abandoned is not cut off, but not judged either", + rp.stats.endedBy === "model" && !rp.cutOff && judge.asked === 0 && !rp.candidates.some((c) => c.check === "goal"), + JSON.stringify({ stats: rp.stats, cutOff: rp.cutOff, judged: judge.asked })); + const rpb = await runMission({ id: "pb", app: "fake", title: "Open the guide", persona: "a viewer", goal: "open it", check: { text: ["Guide"] }, budget: { steps: 8 } }, + { ...deps(), model: { baseUrl: prose.url, model: "pilot" } }); + check("explore(gave up): and as a bench card it is the model's fail", rpb.bench?.passed === false && !rpb.cutOff, JSON.stringify(rpb.bench)); + } finally { + prose.server.close(); + } + + const unfinished: Mission = { id: "u", app: "fake", title: "Open the guide", persona: "a viewer", goal: "open the guide", check: { text: ["Guide"] }, budget: { steps: 8 } }; + const ru = await runMission(unfinished, deps()); + check("explore(cut off): a cut-off bench card it had not finished is not scored", ru.bench === null && !!ru.cutOff, JSON.stringify(ru.bench)); + const bench: Mission = { id: "b", app: "fake", title: "Point at Water", persona: "a viewer", goal: "move to Water", check: { text: ["Water"] }, budget: { steps: 8 } }; + const rb = await runMission(bench, deps()); + check("explore(cut off): one whose end state was already reached still counts", rb.bench?.passed === true && !!rb.cutOff, JSON.stringify(rb.bench)); + + let warmed = 0; + pilot.asked = 0; + const rw = await runMission(bench, deps(async () => { warmed++; pilot.loaded = true; return true; })); + check("explore(rewarm): a dropped model is loaded again and the mission carries on", + warmed === 1 && rw.stats.endedBy === "answer" && !rw.cutOff && rw.bench?.passed === true, JSON.stringify({ warmed, stats: rw.stats, bench: rw.bench })); + check("explore(rewarm): the step the outage ate is not spent from the budget", rw.stats.steps === 1, `steps ${rw.stats.steps}`); + + // An end state the app takes away by itself (Dozehound's channel bar + // hides six seconds after the last press): the model answered from a + // screen that met the card, and nothing touched the device after. + class HidingApp extends FakeApp { + looks = 0; + override async observe(): Promise { + const o = await super.observe(); + return ++this.looks > 1 ? { ...o, nodes: (o.nodes ?? []).filter((n) => n.text !== "Water") } : o; + } + } + const hiding = { ...deps(), actuator: new HidingApp() }; + const rh = await runMission(bench, hiding); + check("explore(bench): an end state gone by the final look counts if the model answered from it", + rh.bench?.passed === true && /when the model last looked/.test(rh.bench.detail), JSON.stringify(rh.bench)); + } finally { + p.server.close(); + j.server.close(); + } +} diff --git a/collector/src/workloads/explore/index.ts b/collector/src/workloads/explore/index.ts index 74dd974..05fb171 100644 --- a/collector/src/workloads/explore/index.ts +++ b/collector/src/workloads/explore/index.ts @@ -31,6 +31,7 @@ import { BASE, NoTargetsError } from "../../fleet-client.js"; import { keychainPassword } from "../../secrets.js"; import { resolveFlow } from "../flows.js"; import type { Job, Target, WorkloadCtx } from "../types.js"; +import { HostNotReadyError } from "./actuators/apple.js"; import { actuatorFor } from "./actuators/index.js"; import { conditionFor, parseConditions } from "./conditions.js"; import { toJpeg } from "./image.js"; @@ -45,6 +46,26 @@ import { ScreenMap } from "./screenmap.js"; import type { Actuator, CheckName, ExploreFinding, Mission } from "./types.js"; const run = promisify(execFile); + +/** + * The night command's warm-up (`explore-night.ts --warm`, passed down as + * FLEET_EXPLORE_WARM), run again when the model server drops the model in the + * middle of a night. Undefined when nothing warmed it: a local run's model is + * the caller's business. + */ +function rewarmer(log: (m: string) => void): LoopDeps["rewarm"] { + const cmd = process.env.FLEET_EXPLORE_WARM; + if (!cmd) return undefined; + return async () => { + const out = await run("/bin/bash", ["-c", cmd], { timeout: 10 * 60_000, maxBuffer: 4 << 20 }) + .then((r) => ({ ok: true, text: `${r.stdout}${r.stderr}` }), + (e: { stdout?: string; stderr?: string; message: string }) => ({ ok: false, text: `${e.stdout ?? ""}${e.stderr ?? ""}` || e.message })); + const last = out.text.trim().split("\n").pop() ?? ""; + log(`warm again: ${out.ok ? "" : "failed: "}${last.slice(0, 200)}`); + return out.ok; + }; +} + const STATE_DIR = process.env.FLEET_STATE_DIR ?? path.join(os.homedir(), ".fleet"); type Params = { @@ -301,11 +322,13 @@ export async function run_(job: Job, ctx: WorkloadCtx): Promise { let allOk = targets.length > 0; const notes: string[] = []; + const failures: string[] = []; + const rewarm = rewarmer(ctx.log); try { for (const t of targets) { if (Date.now() > deadline) break; const r = await exploreDevice(t); - if (!r.ok) allOk = false; + if (!r.ok) { allOk = false; failures.push(`${t.id}: ${typeof r.error === "string" ? r.error : "no mission completed"}`); } else if (typeof r.error === "string") notes.push(r.error); await ctx.postResult({ job_id: job.job_id, device_id: t.id, iter: 0, ...r }); } @@ -316,7 +339,9 @@ export async function run_(job: Job, ctx: WorkloadCtx): Promise { // is a device that worked: findings are the output, not the failure. A short // night that ran is ok, and its note rides on the final row too. await ctx.postResult({ job_id: job.job_id, device_id: `host:${ctx.host}`, iter: 0, final: true, ok: allOk, - ...(allOk ? (notes.length ? { error: [...new Set(notes)].join("; ") } : {}) : { error: "one or more devices completed no mission; see their rows" }) }); + // A failed night says why on the job itself, not "see their rows": the + // job's error is what the alert, the digest and `fleetr` show. + ...(allOk ? (notes.length ? { error: [...new Set(notes)].join("; ") } : {}) : { error: failures.length ? failures.join("; ").slice(0, 500) : "one or more devices completed no mission; see their rows" }) }); async function exploreDevice(t: Target): Promise> { let actuator: Actuator; @@ -346,7 +371,7 @@ export async function run_(job: Job, ctx: WorkloadCtx): Promise { return { ok: false, error: `no ${p.bench ? "bench " : ""}missions for ${appKey} on a ${surface} device (looked in ${p.missions_dir ?? "examples/missions"}/${appKey})${problems.length ? `; problems: ${problems.join("; ")}` : ""}` }; } - const totals = { missions: 0, steps: 0, actions: 0, candidates: 0, findings: 0, findingsNew: 0, notReproduced: 0, misses: 0, refused: 0, benchPassed: 0, benchTotal: 0 }; + const totals = { missions: 0, cutOff: 0, steps: 0, actions: 0, candidates: 0, findings: 0, findingsNew: 0, notReproduced: 0, misses: 0, refused: 0, benchPassed: 0, benchTotal: 0 }; const stepMs: number[] = [], modelMs: number[] = [], promptTokens: number[] = [], cachedTokens: number[] = []; const missionLog: Record[] = []; let lastStepAt = Date.now(); @@ -373,7 +398,7 @@ export async function run_(job: Job, ctx: WorkloadCtx): Promise { actuator, appId: p.app_id, appName, appFile, model, judge, map, runDir, night: job.job_id, log: (msg) => ctx.log(`[${t.id}] ${msg}`), disabledVisual: disabled.visual, disabledChecks: disabled.checks, language: cond.language, hints, setupEnv, canRunFlows: t.platform === "android" || t.kind === "simulator", deadline, - backgroundAt: cond.backgroundAt, + backgroundAt: cond.backgroundAt, rewarm, onStep: async (step, obs) => { stepMs.push(Date.now() - lastStepAt); lastStepAt = Date.now(); @@ -390,10 +415,20 @@ export async function run_(job: Job, ctx: WorkloadCtx): Promise { } catch (e) { missionLog.push({ id: m.id, condition: condName, error: (e as Error).message.slice(0, 300) }); ctx.log(`[${t.id}] mission ${m.id} failed: ${(e as Error).message.slice(0, 200)}`); + // Xcode needing its first-launch install fails every mission the + // same way, a minute of xcodebuild each: on 10-17 that was ten + // minutes a job, three jobs a night. Say it once and stop. + if (e instanceof HostNotReadyError) break; continue; } lastRun[m.id] = new Date().toISOString(); - totals.missions++; + // A mission the model server or the device cut off ran, but says + // nothing about the app or the model: not a completed mission and not + // a bench card. Its trajectory is still kept. + if (result.cutOff) { + totals.cutOff++; + ctx.log(`[${t.id}] mission ${m.id} cut off: ${result.cutOff.slice(0, 200)}`); + } else totals.missions++; totals.steps += result.stats.steps; totals.actions += result.stats.actions; totals.misses += result.stats.misses; @@ -424,6 +459,7 @@ export async function run_(job: Job, ctx: WorkloadCtx): Promise { id: m.id, condition: condName, steps: result.stats.steps, ended_by: result.stats.endedBy, screens: result.stats.screensThisRun, new_screens: result.stats.newScreensEver, candidates: result.candidates.length, misses: result.stats.misses, refused: result.stats.refusedByClass, goal: result.goal, bench: result.bench, answer: result.answer?.slice(0, 300), + ...(result.cutOff ? { cut_off: result.cutOff } : {}), artifacts: runShas, }); @@ -465,7 +501,7 @@ export async function run_(job: Job, ctx: WorkloadCtx): Promise { return { ok: totals.missions > 0, ...(totals.missions === 0 - ? { error: `${note ? `${note}; ` : ""}no mission completed: ${missionLog.map((x) => `${x.id}: ${x.error ?? x.skipped ?? "?"}`).join("; ").slice(0, 400)}` } + ? { error: `${note ? `${note}; ` : ""}no mission completed: ${missionLog.map((x) => `${x.id}: ${x.error ?? x.cut_off ?? x.skipped ?? "?"}`).join("; ").slice(0, 400)}` } : note ? { error: note } : {}), metrics: { explore_missions: totals.missions, explore_cards_planned: missions.length, explore_minutes: p.minutes ?? 150, @@ -473,6 +509,7 @@ export async function run_(job: Job, ctx: WorkloadCtx): Promise { explore_screens: map.seenThisRun.length, explore_new_screens: missionLog.reduce((a, x) => a + (Number(x.new_screens) || 0), 0), explore_candidates: totals.candidates, explore_findings: totals.findings, explore_findings_new: totals.findingsNew, explore_not_reproduced: totals.notReproduced, explore_misses: totals.misses, explore_refused: totals.refused, + explore_cut_off: totals.cutOff, explore_step_ms_p50: median(stepMs), explore_model_ms_p50: median(modelMs), explore_prompt_tokens_mean: mean(promptTokens), explore_cached_tokens_mean: mean(cachedTokens), ...(p.bench ? { explore_bench_passed: totals.benchPassed, explore_bench_total: totals.benchTotal } : {}), diff --git a/collector/src/workloads/explore/loop.ts b/collector/src/workloads/explore/loop.ts index e4a818f..edb795d 100644 --- a/collector/src/workloads/explore/loop.ts +++ b/collector/src/workloads/explore/loop.ts @@ -71,6 +71,13 @@ export type LoopDeps = { deadline: number; /** Send the app to the background once, at this share of the step budget (the "background" condition). */ backgroundAt?: number | null; + /** + * Load the model again after the server said it has none by that name. On + * ultra the model shelf unloads an on-demand model it sees idle under memory + * pressure, and between two missions (an app reinstall) the driver is idle. + * Resolves true when the model is back. + */ + rewarm?(why: string): Promise; }; /** A candidate finding: something a check saw, not yet replayed. */ @@ -106,6 +113,12 @@ export type MissionResult = { goal: { met: boolean | null; reason: string } | null; /** For bench missions: did the final screen satisfy `mission.check`? */ bench: { passed: boolean; detail: string } | null; + /** + * Why the mission was cut off by something that is not the app or the + * model's driving: the model endpoint stopped answering, or the device + * stopped being observable. Such a mission is neither judged nor scored. + */ + cutOff?: string; screen: { w: number; h: number }; stats: { steps: number; actions: number; misses: number; refused: number; refusedByClass: Record; @@ -116,6 +129,22 @@ export type MissionResult = { const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)); +/** + * The model server answering that it has no model by that name: LM Studio + * through LiteLLM says `Invalid model identifier`, an OpenAI-style server + * `model_not_found`, LiteLLM itself `NotFoundError`, and a request caught + * mid-unload `Model is unloaded.`. A turn that failed this + * way was not the model's mistake, and loading the model again fixes it. + */ +export function modelGone(error: string): boolean { + return /invalid model identifier|model is unloaded|model_not_found|NotFoundError|no models? loaded|model .* (is )?not (found|loaded)/i.test(error); +} + +/** A turn that failed in transport or at the server, rather than a reply that broke the rules. */ +export function endpointFailure(error: string | undefined): boolean { + return !!error && error !== "the reply had no tool call"; +} + /** Numbers, hex, ids and quoted strings out of a message, so two of the same crash fingerprint alike. */ export function normalizeMessage(s: string): string { return s.toLowerCase() @@ -241,6 +270,11 @@ export async function runMission(m: Mission, d: LoopDeps): Promise missionDeadline) { stats.endedBy = "time"; break; } @@ -256,6 +290,7 @@ export async function runMission(m: Mission, d: LoopDeps): Promise 3) { stats.endedBy = "model"; break; } conv.say("Your last reply had no tool call. Every reply must call at least one tool."); @@ -475,6 +519,7 @@ export async function runMission(m: Mission, d: LoopDeps): Promise lastObs); if (final) { writeFileSync(path.join(shotsDir, "final.png"), final.png); const finalScreen = d.map.identify(final.nodes, dhash(final.png), final.height, d.night).entry; - if (m.check) bench = benchCheck(m.check, final.nodes, finalScreen.name); - if (d.judge && m.success && !m.check && !d.disabledChecks?.includes("goal")) { + if (m.check) { + bench = benchCheck(m.check, final.nodes, finalScreen.name); + // An end state the app takes away by itself: Dozehound hides its + // channel bar six seconds after the last press, and the model's answer + // takes about four, so the final look found focus on the hidden bar's + // "Show channels" catcher while the model had answered, rightly, from a + // screen with All channels focused. If nothing touched the device after + // the model's last look, that look counts too. + if (!bench.passed && lastObs && lastObs !== final && !actedSinceLook) { + const seen = (lastScreenId && d.map.byId(lastScreenId)?.name) || finalScreen.name; + const atAnswer = benchCheck(m.check, lastObs.nodes, seen); + if (atAnswer.passed) bench = { passed: true, detail: `end state reached when the model last looked; gone by the final look (${bench.detail})` }; + } + // A cut-off card that had not got there yet says nothing either way. + if (cutOff && !bench.passed) bench = null; + } + // Nor is a mission the model gave up on (four replies in a row with no + // tool call; Holo4 on 10-17 wrote "Scroll up to explore more" over and + // over from step 1): where it stopped is the agent's failure, and the judge + // would file it as the app's, then "reproduce" it by looking again. + const judgeable = !cutOff && stats.endedBy !== "model"; + if (judgeable && d.judge && m.success && !m.check && !d.disabledChecks?.includes("goal")) { const g = await judgeGoal(d.judge, final.png, m.goal, m.success, final.nodes); goal = { met: g.met, reason: g.reason }; if (g.met === false) { @@ -537,7 +612,7 @@ export async function runMission(m: Mission, d: LoopDeps): Promise&2; exit 3"]; + const deviceAndModel = ["--sim", "fleet-night-test:no.such.runtime:no.such.type", "--warm", "echo warmed-up hold=$SHELF_HOLD_PID >&2; exit 3"]; const runNight = (extra: string[], env: NodeJS.ProcessEnv = {}) => new Promise<{ code: number; out: string }>((resolve) => { const p = spawn(process.execPath, [path.join(root, "node_modules/tsx/dist/cli.mjs"), "scripts/explore-night.ts", "--spec", specFile, ...deviceAndModel, ...extra], { - cwd: root, env: { ...process.env, FLEET_URL: `http://127.0.0.1:${port}`, FLEET_EXECUTOR_NAME: "night-test", NIGHT_JOB: "", ...env }, + cwd: root, env: { ...process.env, FLEET_URL: `http://127.0.0.1:${port}`, FLEET_EXECUTOR_NAME: "night-test", NIGHT_JOB: "", FLEET_XCODEBUILD: "true", ...env }, }); let out = ""; p.stdout.on("data", (c) => (out += c)); @@ -207,8 +207,22 @@ async function runNightCommandChecks(check: Check) { const r4result = posts.find((x) => x.url === "/results")?.body; check("a broken warm-up fails the night (exit 1)", r4.code === 1, r4.out); check("and the brain is told why", r4result?.ok === false && /^the model did not load: warmed-up/.test(String(r4result?.error)), JSON.stringify(r4result)); + check("the warm-up was told to hold the model for this process (SHELF_HOLD_PID)", /warmed-up hold=\d+/.test(r4.out), r4.out); check("the simulator was still not booted", !/booting|creating simulator/.test(r4.out), r4.out); } + + // 5. Xcode updated without its first-launch install (ultra, 10-16 and + // 10-17): a failure that names the fix, found before the model is warmed. + // The check is xcodebuild's, so only a Mac makes it. + if (process.platform === "darwin") { + posts.length = 0; + const r5 = await runNight(roomy, { FLEET_XCODEBUILD: "false" }); + const r5result = posts.find((x) => x.url === "/results")?.body; + check("Xcode needing its first launch fails the night (exit 1)", r5.code === 1, r5.out); + check("and the brain is told the fix", + r5result?.ok === false && /needs its first-launch install .*sudo xcodebuild -runFirstLaunch/.test(String(r5result?.error)), JSON.stringify(r5result)); + check("before warming the model or booting the simulator", !/warmed-up|booting|creating simulator/.test(r5.out), r5.out); + } } finally { server.close(); } diff --git a/docs/deploy/explore-on-ultra.md b/docs/deploy/explore-on-ultra.md index 9ef583d..86bac28 100644 --- a/docs/deploy/explore-on-ultra.md +++ b/docs/deploy/explore-on-ultra.md @@ -98,7 +98,9 @@ and `--sim name:runtime:devicetype`, which creates the simulator if missing, boots it, and shuts it down afterwards if it booted it. Simulator names start with `fleet-` so the executor counts them as fleet devices. FleetDriver builds itself into `~/.fleet/explore/derived` on first use. Xcode builds use -`/Applications/Xcode.app`, `xcodegen` comes from `/opt/homebrew`. +`/Applications/Xcode.app`, `xcodegen` comes from `/opt/homebrew`. After an +Xcode update, `sudo xcodebuild -runFirstLaunch` has to run once before any +simulator test will; the night command checks for it and fails until it has. ### Measured on 8 October 2026 @@ -114,6 +116,75 @@ Its first 40-step bug-garden mission filed BG-01 and BG-03, both reproduced 2/2 on clean installs. The night rehearsal through the brain filed BG-16 (low contrast), reproduced 2/2. +### The first nights on 0.9.0 (15 to 17 October 2026) + +The night of 10-15 was the first with Holo4 allowed at 45% free and with tvOS +simulators registering as `tvos`, so the first time the TV and Apple nights +drove anything. The bug garden at 22:09 still ran the old code (0.9.0 was +deployed at 00:10) and skipped at "57% free, needs 65%". No night ran short: +each was admitted with its whole `max_minutes`. + +**Phase 3, the tvOS bench** (Dozehound stub catalog, Apple TV 4K simulator on +tvOS 26.5, build `develop-ebcd74e46e`, 10 cards per model): + +| Model | Passed, as scored | Cards it actually ran | Median step · model | Prompt, mean | +|---|---|---|---|---| +| Holo4 35B-A3B (`pilot`) | **5 of 10** | 5 of 8 | 4.5 s · 2.4 s | 5.9k tokens | +| Qwen3.8-27B (`vision`) | **5 of 10**; 6 with the end-state fix below, re-scored from its saved trees | 5 of 10 | 5.9 s · 3.4 s | 2.1k tokens | + +Both passed `focus-meadow`, `focus-naptime`, `meadow-cancel`, `meadow-link` +and `pause-resume`. **The bar, half of 20 navigation missions, is met, just:** +10 of 20, and each model alone at half. Phase 3 passes; TVs stay on the model +path rather than scripted replays only. + +What the score hides, each now handled in the workload: + +- **Holo4 did not get two of its cards.** The model shelf unloaded it as "idle + under memory pressure" at 03:50:31, six minutes after the warm-up; Holo4 at + 45% free puts ultra in the warn band, and between two missions (an app + reinstall) the model is idle. `bench-pause` and `bench-play-naptime` then + ended at step 0 on `Invalid model identifier`. It happened again at 04:10, + 22:33 on 10-16 and 22:23 on 10-17, each time mid-run. +- **`focus-all-channels` was failed by the app, not the models.** Dozehound + hides its channel bar six seconds after the last press; the final look came + after that and found focus on the hidden bar's "Show channels" catcher, while + `vision` had answered from a screen with All channels focused. Holo4 failed + the same card on its own: five replies with no tool call. +- **The every-button explorer and GreenFolio's second card never ran** (the + same unloads), and the judge then filed the untouched start screens as goal + findings: #70 (Dozehound) and #71 (GreenFolio) are the harness's, not bugs. + +**Phase 4, GreenFolio iOS** (iPhone simulator on iOS 26.5, build +`develop-74c84f254b`, offline): `offline-gate` ran 38 steps, ended stuck with +its goal met (the sign-in screen says "Couldn't reach the server"), 7 +candidates and 6 misses; `offline-create-account` was cut off at step 0. The +night filed 8 findings, 7 of them seen before; the one new one is #71 above. +**Phase 4's bar cannot be read yet:** it compares bench cards with Android, and +`greenfolio-ios` has no bench cards, only its two explore cards. The tap and +screenshot half works on the simulator; the shelf iPhone is unplugged (WB-21). + +**10-16 and 10-17: the TV and Apple nights failed, and said why only on their +device rows.** Xcode on ultra updated itself to 27.1 on 10-16 at 06:57 without +its first-launch install, so its CoreSimulator (1171.7) is older than the one +it was built against (1174.9.2) and every FleetDriver start exited 70 a minute +in. Each job tried every card in turn, about ten minutes a job, and the job's +own error was "one or more devices completed no mission". The bug garden ran +both nights, but on 10-17 Holo4 answered in prose ("Scroll up to explore +more…") with no tool call from the first step, the shelf unloaded it at 22:23, +and seven missions took two steps between them; the judge filed their end +states as goal findings again. + +What changed because of it (PR below): the night command checks +`xcodebuild -checkFirstLaunchStatus` before warming the model and fails with +the fix named; the workload stops at the first such error; a model the server +has dropped is warmed again and the step retaken; a mission cut off by the +server or abandoned by the model is not judged; a bench card is scored on the +model's last look when nothing touched the device after it; and a failed job's +error is its device's reason. The shelf unloading a model that is between +requests is the harness's: from model-ladder's shelf-hold change, a warm-up run +with `SHELF_HOLD_PID` set holds the model while that process lives, and the +night command sets it to its own pid. + ## Phase 0: measure before building on it **The model.** Holo4 35B-A3B only: Apache-2.0. The 27B is CC BY-NC, and testing diff --git a/docs/explore/job.md b/docs/explore/job.md index 2422f97..802f04a 100644 --- a/docs/explore/job.md +++ b/docs/explore/job.md @@ -122,6 +122,7 @@ Metrics on each device row: | Metric | | |---|---| | `explore_missions` | Missions run | +| `explore_cut_off` | Missions the model server or the device cut off; neither counted in `explore_missions` nor judged | | `explore_cards_planned` | Cards lined up for the device, in priority order. Fewer missions than cards means the budget ran out first | | `explore_minutes` | The minutes the device was given: `params.minutes`, cut to fit on a short night | | `explore_steps` · `explore_actions` | Model steps; actions actually performed | @@ -133,11 +134,31 @@ Metrics on each device row: | `explore_refused` | Actions the leash refused | | `explore_step_ms_p50` · `explore_model_ms_p50` | Median step and model latency | | `explore_prompt_tokens_mean` · `explore_cached_tokens_mean` | Mean prompt size, and how much of it the server's cache served | -| `explore_bench_passed` · `explore_bench_total` | Bench mode only | +| `explore_bench_passed` · `explore_bench_total` | Bench mode only. A card cut off before its end state is left out of both; one cut off after it counts | The row also carries `missions`: per mission, its condition, steps, how it ended, screens, candidates, misses, refusals by class, the goal verdict, the -bench result and the model's answer. +bench result and the model's answer. A mission that was cut off carries +`cut_off` with the reason, and is not judged: its end state is wherever the +outage left it. Nor is a mission the model gave up on (four replies in a row +with no tool call), whose end state is the agent's failure, not the app's; as +a bench card that one still counts as failed. A failed job's own error is the failing device's reason, not +"see their rows". + +**A model unloaded mid-night is loaded again.** When the model server answers +that it has no model by that name (`Invalid model identifier`, `Model is +unloaded.`), the workload runs the night command's warm-up again (it is handed +down as `FLEET_EXPLORE_WARM`) and takes the step again, twice a mission at +most, without spending the step's budget. On ultra the model shelf unloads an +on-demand model it finds idle under memory pressure, and between two missions +the driver is idle. The night command also sets `SHELF_HOLD_PID` to its own +pid before warming, which the shelf's warm-up records as a hold: the model is +then left loaded until that process ends, unless pressure turns critical. + +**A bench card's end state the app takes away by itself** still counts when +nothing touched the device after the model's last look: Dozehound hides its +channel bar six seconds after the last press, so the final look can find focus +somewhere the model never put it. On a short night the device row and the final row are still `ok: true`, and `error` says what ran: `short night: ran 3 of 7 cards: 40 minutes left of the @@ -230,6 +251,14 @@ the cheap things first: the model would not load, no room on this host: ...`. Any other warm-up failure fails the night, with the reason posted. +With `--sim`, before the warm-up, it also asks `xcodebuild +-checkFirstLaunchStatus` whether Xcode has had its first-launch install. After +an Xcode update it has not, and every FleetDriver start fails a minute in with +`CoreSimulator is out of date`. That is a **failure**, not a skip, posted with +the fix (`sudo xcodebuild -runFirstLaunch`), because only someone at the host +can make it go away. Inside a job the same message stops the device's +missions at the first, instead of trying each card in turn. + Only then does it boot the simulator or emulator. A skip is `ok: true` with an error starting `skipped:`, the brain's convention for "nothing ran, and that is not a failure of the code". The command's exit code is 0 when every spec diff --git a/scripts/changelog-coverage.mjs b/scripts/changelog-coverage.mjs index 8e74370..c4767ed 100644 --- a/scripts/changelog-coverage.mjs +++ b/scripts/changelog-coverage.mjs @@ -6,8 +6,11 @@ * * Lists the pull requests merged into `develop` since the last `v*` tag and * fails, naming each one, if the changelog section being released does not - * mention it as `#N` or `/pull/N`. A pull request labelled `no-changelog`, and - * any opened by Dependabot, is exempt. + * mention it as `#N` or `/pull/N`. A pull request labelled `no-changelog`, any + * opened by Dependabot, and the release's own pull requests -- `release/*` + * into `develop`, `develop` into `main` -- are exempt. The last were not, and + * 0.9.0's `release/0.9.0` pull request (#98) failed its own `develop` -> `main` + * check and had to be labelled by hand: a release is not a change to note. * * ## Why * @@ -50,6 +53,30 @@ export const EXEMPT_LABEL = "no-changelog"; /** `dependabot[bot]` from the REST API, `app/dependabot` from `gh pr view`. */ export const isDependabot = (author) => /(^|\/)dependabot(-preview)?(\[bot\])?$/.test(author ?? ""); +/** A pull request that is the release itself: `release/*` into anything, or `develop` into `main`. */ +export const isReleasePr = ({ head, base } = {}) => + /^release\//.test(head ?? "") || (head === "develop" && (base === undefined || base === "main")); + +/** + * The branch a GitHub merge commit merged, from its subject + * (`Merge pull request #98 from addisdev/release/0.9.0` -> `release/0.9.0`), + * or null for a squash merge, whose subject does not say. + */ +export function headFromSubject(subject) { + const m = /^Merge pull request #\d+ from [^/\s]+\/(\S+)/.exec(subject); + return m ? m[1] : null; +} + +/** The release pull requests among merge commit subjects, by number, without asking GitHub. */ +export function releasePrsFromSubjects(subjects) { + const out = new Set(); + for (const s of subjects) { + const head = headFromSubject(s); + if (head && isReleasePr({ head })) out.add(Number(/#(\d+)/.exec(s)[1])); + } + return out; +} + /** * Pull request numbers from commit subjects, oldest first, without repeats. * @@ -102,16 +129,22 @@ export function mentionedNumbers(section) { /** * The verdict, given everything already fetched. * - * `meta` maps a pull request number to `{ labels: string[], author: string }` - * for the ones that needed looking up; a number with no entry is treated as - * unlabelled, so a lookup that failed fails the check rather than excusing it. + * `meta` maps a pull request number to `{ labels: string[], author: string, + * head?: string, base?: string }` for the ones that needed looking up; a + * number with no entry is treated as unlabelled, so a lookup that failed fails + * the check rather than excusing it. `release` holds the numbers already known + * to be release pull requests from their merge commits. */ -export function coverage({ merged, mentioned, meta = new Map() }) { +export function coverage({ merged, mentioned, meta = new Map(), release = new Set() }) { const missing = []; const exempt = []; for (const n of merged) { if (mentioned.has(n)) continue; const m = meta.get(n); + if (release.has(n) || (m && isReleasePr(m))) { + exempt.push({ number: n, why: "release" }); + continue; + } if (m && (m.labels.includes(EXEMPT_LABEL) || isDependabot(m.author))) { exempt.push({ number: n, why: m.labels.includes(EXEMPT_LABEL) ? EXEMPT_LABEL : m.author }); continue; @@ -150,7 +183,7 @@ function prMeta(numbers) { try { const json = execFileSync( "gh", - ["pr", "view", String(n), "--json", "labels,author,title"], + ["pr", "view", String(n), "--json", "labels,author,title,headRefName,baseRefName"], { cwd: ROOT, encoding: "utf8", stdio: ["ignore", "pipe", "pipe"] }, ); const pr = JSON.parse(json); @@ -158,6 +191,8 @@ function prMeta(numbers) { labels: (pr.labels ?? []).map((l) => l.name), author: pr.author?.login ?? "", title: pr.title ?? "", + head: pr.headRefName ?? "", + base: pr.baseRefName ?? "", }); } catch (e) { console.error(` could not read #${n} from GitHub (${(e.stderr || e.message || "").toString().trim()}); treating it as unlabelled`); @@ -190,8 +225,9 @@ function main(argv) { return 1; } const mentioned = mentionedNumbers(section); - const unmentioned = merged.filter((n) => !mentioned.has(n)); - const { missing, exempt } = coverage({ merged, mentioned, meta: prMeta(unmentioned) }); + const release = releasePrsFromSubjects(subjects); + const unmentioned = merged.filter((n) => !mentioned.has(n) && !release.has(n)); + const { missing, exempt } = coverage({ merged, mentioned, meta: prMeta(unmentioned), release }); console.log(`${merged.length} pull request(s) merged since ${tag}: ${merged.map((n) => `#${n}`).join(" ") || "none"}`); for (const e of exempt) console.log(` exempt #${e.number} (${e.why})`); diff --git a/scripts/changelog-coverage.test.mjs b/scripts/changelog-coverage.test.mjs index 39ecaee..251cdc9 100644 --- a/scripts/changelog-coverage.test.mjs +++ b/scripts/changelog-coverage.test.mjs @@ -3,11 +3,14 @@ import assert from "node:assert/strict"; import { test } from "node:test"; import { coverage, + headFromSubject, isDependabot, + isReleasePr, mentionedNumbers, newestTag, prNumbersFromSubjects, releasedSection, + releasePrsFromSubjects, } from "./changelog-coverage.mjs"; test("merge commits and squash merges both count; passing mentions do not", () => { @@ -87,3 +90,38 @@ test("the newest tag is chosen by version, not by string order", () => { assert.equal(newestTag(["v1.0.0-rc.2", "v1.0.0-rc.10", "v0.9.0"]), "v1.0.0-rc.10"); assert.equal(newestTag(["latest", "nightly"]), null); }); + +test("the release's own pull requests are exempt, from the merge subject or from GitHub", () => { + // 0.9.0: #98 was release/0.9.0 into develop and failed the develop -> main check. + const subjects = [ + "Merge pull request #98 from addisdev/release/0.9.0", + "Merge pull request #97 from addisdev/feat/dash-v1-client", + "Merge pull request #80 from addisdev/develop", + "Release 0.9.1 (#101)", + ]; + assert.equal(headFromSubject(subjects[0]), "release/0.9.0"); + assert.equal(headFromSubject(subjects[3]), null); + assert.deepEqual([...releasePrsFromSubjects(subjects)].sort(), [80, 98]); + + // A squash-merged release PR is only known from GitHub's head and base. + const meta = new Map([ + [101, { labels: [], author: "addisdev", title: "Release 0.9.1", head: "release/0.9.1", base: "develop" }], + [102, { labels: [], author: "addisdev", title: "Back-merge", head: "develop", base: "feat/x" }], + ]); + const { missing, exempt } = coverage({ + merged: [97, 98, 80, 101, 102], + mentioned: new Set([97]), + meta, + release: releasePrsFromSubjects(subjects), + }); + assert.deepEqual(exempt.map((e) => [e.number, e.why]), [[98, "release"], [80, "release"], [101, "release"]]); + assert.deepEqual(missing.map((m) => m.number), [102]); +}); + +test("a release pull request is release/* into anything, or develop into main", () => { + assert.ok(isReleasePr({ head: "release/0.9.0", base: "develop" })); + assert.ok(isReleasePr({ head: "develop", base: "main" })); + assert.ok(!isReleasePr({ head: "develop", base: "feat/x" })); + assert.ok(!isReleasePr({ head: "feat/release-notes", base: "develop" })); + assert.ok(!isReleasePr({})); +});