cockpit wait: green only once GitHub has every run and job for the head

CHANGELOG.md

@@ -178,6 +178,22 @@ version numbers (`v0.1.0`, `v0.1.1`, etc.).
 
 ### Fixed
 
+- **`cockpit wait` and `wait_for_checks` no longer go green before a PR's
+  checks have all registered.** On acme/stream-app PR #5 (2026-10-24) a
+  wait that started 8 s after a push returned `1 of 1 green` at once, from the
+  previous head's run: the fleet view learns a new head only from its first
+  run, and the new head's run 40116718009 (nine jobs, six of them behind
+  `needs:`) was still queued. GitHub's own check list had the same gap, since
+  a `needs:`-gated job has no check run until its needs finish. Now GitHub
+  names the head (an open PR's branch ref, which moves before the PR's
+  `head.sha` does), and a green ends the wait only when GitHub has every
+  workflow run for that head (`actions/runs?head_sha=`, newest per workflow)
+  completed, every job of each and every other check passed, and nothing new
+  for 30 s (runs for separate events and `workflow_run` runs register later).
+  Until then the answer is `pending` with GitHub's reason; GitHub not
+  answering is not a yes. A green carries `verified` (runs, jobs, checks
+  covered), and the CLI confirms a green itself with `gh` when an older
+  daemon sends none. The CLI's GitHub fallback uses the same rule.
 - **The Hosts tab no longer lists the coordinator as a second, stale host.**
   `buildHostList` already dropped a heartbeat record under the coordinator's
   own id, but the `/api/hosts` handler and the `/metrics` stale-host count

cockpit/Sources/CockpitCore/API.swift

@@ -68,6 +68,18 @@ public struct V1Checks: Decodable, Sendable, Equatable {
     public var fleetSaid: String?
     public var checks: V1CheckSet?
     public var orphans: [V1Orphan]?
+    /// On a green: what GitHub's side of the check covered. A daemon that
+    /// confirms every green on GitHub always sends it, so a green without it
+    /// came from an older daemon and is confirmed by the client
+    /// (stream-app PR #5, 2026-10-24).
+    public var verified: Verified?
+
+    public struct Verified: Codable, Sendable, Equatable {
+        public var head: String?
+        public var runs: Int
+        public var jobs: Int
+        public var checks: Int?
+    }
 
     public var finished: Bool { state != "pending" }
 

cockpit/Sources/CockpitCore/CheckWait.swift

@@ -21,7 +21,9 @@ public struct CheckWait: Sendable {
     /// Wall-clock deadline, epoch ms.
     public var deadline: Double
     public var ask: Ask
-    /// A PR's checks from GitHub, for while the dashboard is unreachable.
+    /// A PR's checks from GitHub, for while the dashboard is unreachable, and
+    /// to confirm a green from a daemon too old to confirm it itself (its
+    /// answer has no `verified`).
     public var github: (@Sendable () async -> CheckSet?)?
     /// Recovers orphaned runs; returns why it could not, or nil.
     public var recover: (@Sendable ([V1Orphan]) async -> String?)?
@@ -30,6 +32,8 @@ public struct CheckWait: Sendable {
     public var now: @Sendable () -> Double = { Format.nowMs() }
     public var sleep: @Sendable (Double) async -> Void = { s in try? await Task.sleep(nanoseconds: UInt64(s * 1e9)) }
     public var githubEveryS: Double = 60
+    /// How often an unconfirmed green is asked of GitHub again.
+    public var confirmEveryS: Double = 15
     public var retryS: Double = 10
     /// After a recovery, a red made only of the recovered runs, cancelled, is
     /// the force-cancel landing before the re-run starts, not a result.
@@ -44,6 +48,7 @@ public struct CheckWait: Sendable {
         var last: (V1Checks, Data)?
         var said: String?
         var askedGitHub = -Double.infinity
+        var confirmedAt = -Double.infinity
         var recovered: [Int: Double] = [:]
         var reachable = false
         var dropped = false
@@ -71,10 +76,28 @@ public struct CheckWait: Sendable {
             reachable = true
             dropped = false
             last = (a, raw)
-            if let s = a.checks, case let p = s.progress + (s.etaGreenMs.map { ", done in \(Presenter.range($0))" } ?? ""), p != said {
+            if let s = a.checks, case let p = s.progress + (s.etaGreenMs.map { ", done in \(Presenter.range($0))" } ?? "")
+                + (a.state == "pending" ? a.reason.map { " — \($0)" } ?? "" : ""), p != said {
                 progress(p)
                 said = p
             }
+            // A green from a daemon that does not confirm greens on GitHub
+            // (stream-app PR #5, 2026-10-24: its "1 of 1 green" was the
+            // previous head's, 8 s after a push). Not believed until GitHub
+            // agrees; GitHub not answering is not a yes.
+            if a.state == "green", a.verified == nil, let github {
+                if now() - confirmedAt >= confirmEveryS * 1000 {
+                    confirmedAt = now()
+                    let s = await github()
+                    if let s, s.state == .green { return .answer(a, raw) }
+                    if let s, s.state == .red { return .github(s) }
+                    let p = "the dashboard read \(a.checks?.progress ?? "green"), but GitHub "
+                        + (s.map { "has \($0.progress)\($0.why.map { " — \($0)" } ?? "")" } ?? "did not answer")
+                    if p != said { progress(p); said = p }
+                }
+                await sleep(min(retryS, max(0.01, (deadline - now()) / 1000)))
+                continue
+            }
 
             if a.state == "pointless", let recover, let orphans = a.orphans, !orphans.isEmpty,
                orphans.allSatisfy(\.workPassed) {
@@ -114,7 +137,10 @@ public struct CheckWait: Sendable {
         let progress = a.checks?.progress
         switch a.state {
         case "green":
-            out.append("✔ \(label): \(progress ?? "green")\(note)")
+            let covered = a.verified.map { v in
+                " (GitHub: \(v.runs) run\(v.runs == 1 ? "" : "s"), \(v.jobs) job\(v.jobs == 1 ? "" : "s"))"
+            } ?? ""
+            out.append("✔ \(label): \(progress ?? "green")\(covered)\(note)")
         case "red":
             out.append("✖ \(label): \(progress ?? "red")\(note)")
             for f in a.checks?.failed ?? [] {

cockpit/Sources/CockpitCore/Fixtures/v1/checks-green.json

@@ -43,5 +43,11 @@
  },
  "reason": null,
  "hint": null,
- "fleetAgeMs": 0
+ "fleetAgeMs": 0,
+ "verified": {
+  "head": "ec99782000000000000000000000000000000000",
+  "runs": 1,
+  "jobs": 1,
+  "checks": 0
+ }
 }

cockpit/Sources/CockpitCore/GitHubChecks.swift

@@ -1,11 +1,18 @@
 import Foundation
 
 /// GitHub's own view of a PR's checks: `gh pr view <n> --json
-/// headRefOid,statusCheckRollup`. Cockpit's rows are a copy of this, made by
-/// the daemon's fast loop; when that loop stops finishing ticks the copy
-/// freezes, so a wait asks the original as well (shop-ios PR #445 and
-/// shop-android PR #280, 2026-10-07: both green on GitHub while cockpit
-/// still showed them running).
+/// headRefOid,statusCheckRollup`, the branch ref, and the head's workflow
+/// runs. Cockpit's rows are a copy of this, made by the daemon's fast loop;
+/// when that loop stops finishing ticks the copy freezes, so a wait asks the
+/// original as well (shop-ios PR #445 and shop-android PR #280, 2026-10-07:
+/// both green on GitHub while cockpit still showed them running).
+///
+/// The same green rule as the daemon's (dashboard/lib/checks.js
+/// githubVerdict): every workflow run for the head completed, every check
+/// completed and passed, and nothing new for `settleMs`. The rollup lists only
+/// jobs that exist, and a job behind `needs:` exists only once its needs
+/// finish, so a green rollup alone is not the end (stream-app PR #5,
+/// 2026-10-24).
 public struct GitHubChecks: Sendable, Equatable {
     public struct Check: Sendable, Equatable {
         public var name: String
@@ -14,40 +21,110 @@ public struct GitHubChecks: Sendable, Equatable {
         public var status: String
         public var conclusion: String?
         public var url: String?
+        public var completedAt: Double? = nil
     }
 
+    /// One workflow run for the head, from `actions/runs?head_sha=`.
+    public struct Run: Sendable, Equatable {
+        public var id: Int
+        public var workflowId: Int?
+        public var name: String?
+        public var status: String
+        public var conclusion: String?
+        public var createdAt: Double?
+        public var updatedAt: Double?
+        public var url: String?
+    }
+
+    /// How long GitHub must have been quiet about the head before a green.
+    public static let settleMs: Double = 30_000
+
     public var head: String?
     public var checks: [Check]
+    /// The head's workflow runs; nil when not asked (judged by checks alone).
+    public var runs: [Run]? = nil
+    /// When GitHub was asked, epoch ms; the settle window counts from it.
+    public var fetchedAt: Double? = nil
+    /// From the PR view, to find where its branch is now.
+    public var open: Bool = false
+    public var headRepo: String? = nil
+    public var headRef: String? = nil
+
+    /// The newest run of each workflow, as in the fleet's rollup.
+    public var latestRuns: [Run] {
+        var by: [String: Run] = [:]
+        for r in runs ?? [] {
+            let key = r.workflowId.map(String.init) ?? r.name ?? String(r.id)
+            if let held = by[key], held.id > r.id { continue }
+            by[key] = r
+        }
+        return by.values.sorted { $0.id < $1.id }
+    }
+
+    var lastChange: Double {
+        (latestRuns.flatMap { [$0.createdAt, $0.updatedAt] } + (runs ?? []).map(\.createdAt) + checks.map(\.completedAt))
+            .compactMap { $0 }.max() ?? 0
+    }
+
+    /// Why a finished-looking answer is still pending; nil otherwise.
+    public var pendingWhy: String? {
+        let runs = latestRuns
+        if checks.isEmpty && runs.isEmpty { return "GitHub has no runs or checks for the head yet" }
+        if let r = runs.first(where: { $0.status != "completed" }) {
+            return "GitHub still has \(r.name ?? "a run") \(r.status.replacingOccurrences(of: "_", with: " "))"
+        }
+        if let c = checks.first(where: { $0.status != "completed" }) { return "GitHub still has \(c.name) running" }
+        if !failedAny, let fetchedAt, case let quiet = fetchedAt - lastChange, quiet < Self.settleMs {
+            return "settling: GitHub's newest run or job changed \(Int(max(0, quiet / 1000))) s ago, and a green waits \(Int(Self.settleMs / 1000)) s for any run still to register"
+        }
+        return nil
+    }
+
+    var failedAny: Bool {
+        !rows.allSatisfy(\.passed) || !latestRuns.allSatisfy { ["success", "skipped", "neutral"].contains($0.conclusion ?? "") }
+    }
 
-    /// Finished only when GitHub lists at least one check and every one is
-    /// complete. An empty rollup is a head whose workflows have not been
-    /// created yet, not a pass.
     public var state: CheckSet.State {
-        if checks.isEmpty || checks.contains(where: { $0.status != "completed" }) { return .pending }
-        return rows.allSatisfy(\.passed) ? .green : .red
+        if pendingWhy != nil { return .pending }
+        return failedAny ? .red : .green
     }
 
     /// As cockpit rows, so a GitHub answer reports exactly like a cockpit one
-    /// (same green rule: success/skipped/neutral only; cancelled is red).
+    /// (same green rule: success/skipped/neutral only; cancelled is red). A
+    /// run with no job in the rollup yet is one row of its own.
     var rows: [RunRow] {
-        checks.enumerated().map { i, c in
-            RunRow(id: -(i + 1), repo: "", workflow: c.name, status: c.status, conclusion: c.conclusion,
+        let listed = checks.map { c -> RunRow in
+            RunRow(id: 0, repo: "", workflow: c.name, status: c.status, conclusion: c.conclusion,
                    branch: nil, sha: head, prNumber: nil, event: nil, title: nil, url: c.url,
                    startedAt: nil, updatedAt: nil, elapsedMs: nil, expectedMs: nil)
         }
+        let unlisted = latestRuns.filter { r in !checks.contains { $0.url?.contains("/actions/runs/\(r.id)/") == true } }
+            .map { r in
+                RunRow(id: 0, repo: "", workflow: r.name, status: r.status, conclusion: r.status == "completed" ? r.conclusion : nil,
+                       branch: nil, sha: head, prNumber: nil, event: nil, title: nil, url: r.url,
+                       startedAt: nil, updatedAt: nil, elapsedMs: nil, expectedMs: nil)
+            }
+        return (listed + unlisted).enumerated().map { i, r in var r = r; r.id = -(i + 1); return r }
     }
 
     public func checkSet(repo: String, pr: Int?) -> CheckSet {
         let rs = rows.map { r -> RunRow in var r = r; r.repo = repo; r.prNumber = pr; return r }
         let failed = rs.filter(\.failed)
         let done = rs.filter { !$0.isActive }.count
-        return CheckSet(id: "github:\(repo)@\(head ?? "?")", repo: repo, prNumber: pr, branch: nil,
-                        sha: head.map { String($0.prefix(12)) }, title: nil, runs: rs, total: rs.count, done: done,
-                        failed: failed, queued: [], etaGreenMs: nil, state: state)
+        var set = CheckSet(id: "github:\(repo)@\(head ?? "?")", repo: repo, prNumber: pr, branch: nil,
+                           sha: head.map { String($0.prefix(12)) }, title: nil, runs: rs, total: rs.count, done: done,
+                           failed: failed, queued: [], etaGreenMs: nil, state: state)
+        set.why = pendingWhy
+        return set
+    }
+
+    static func ms(_ s: String?) -> Double? {
+        s.flatMap { ISO8601DateFormatter().date(from: $0) }.map { $0.timeIntervalSince1970 * 1000 }
     }
 
-    /// Parses `gh pr view --json headRefOid,statusCheckRollup`. Check runs
-    /// carry status + conclusion; commit statuses carry one `state`.
+    /// Parses `gh pr view --json headRefOid,headRefName,headRepository,
+    /// headRepositoryOwner,state,statusCheckRollup`. Check runs carry status +
+    /// conclusion; commit statuses carry one `state`.
     public static func parse(prView data: Data) throws -> GitHubChecks {
         struct Item: Decodable {
             var __typename: String?
@@ -59,8 +136,18 @@ public struct GitHubChecks: Sendable, Equatable {
             var state: String?
             var detailsUrl: String?
             var targetUrl: String?
+            var completedAt: String?
+            var startedAt: String?
+        }
+        struct Named: Decodable { var name: String?; var login: String? }
+        struct View: Decodable {
+            var headRefOid: String?
+            var headRefName: String?
+            var headRepository: Named?
+            var headRepositoryOwner: Named?
+            var state: String?
+            var statusCheckRollup: [Item]?
         }
-        struct View: Decodable { var headRefOid: String?; var statusCheckRollup: [Item]? }
         let v = try JSONDecoder().decode(View.self, from: data)
         let checks = (v.statusCheckRollup ?? []).map { i -> Check in
             if i.__typename == "StatusContext" || (i.status == nil && i.state != nil) {
@@ -73,21 +160,46 @@ public struct GitHubChecks: Sendable, Equatable {
                 }
                 return Check(name: i.context ?? i.name ?? "status", workflow: nil,
                              status: finished ? "completed" : "in_progress", conclusion: conclusion,
-                             url: i.targetUrl)
+                             url: i.targetUrl, completedAt: finished ? ms(i.startedAt) : nil)
             }
             let status = (i.status ?? "QUEUED").lowercased()
             return Check(name: i.name ?? "check", workflow: i.workflowName,
                          status: status == "completed" ? "completed" : status,
                          conclusion: status == "completed" ? (i.conclusion ?? "").lowercased() : nil,
-                         url: i.detailsUrl)
+                         url: i.detailsUrl, completedAt: ms(i.completedAt))
+        }
+        var c = GitHubChecks(head: v.headRefOid, checks: checks)
+        c.open = v.state?.uppercased() == "OPEN"
+        if let owner = v.headRepositoryOwner?.login, let name = v.headRepository?.name { c.headRepo = "\(owner)/\(name)" }
+        c.headRef = v.headRefName
+        return c
+    }
+
+    /// Parses `gh api repos/<repo>/actions/runs?head_sha=<sha>`.
+    public static func parse(runs data: Data) throws -> [Run] {
+        struct R: Decodable {
+            var id: Int
+            var workflow_id: Int?
+            var name: String?
+            var status: String?
+            var conclusion: String?
+            var created_at: String?
+            var updated_at: String?
+            var html_url: String?
+        }
+        struct Page: Decodable { var workflow_runs: [R]? }
+        return (try JSONDecoder().decode(Page.self, from: data).workflow_runs ?? []).map { r in
+            let status = r.status ?? "queued"
+            return Run(id: r.id, workflowId: r.workflow_id, name: r.name, status: status == "completed" ? "completed" : status,
+                       conclusion: status == "completed" ? (r.conclusion ?? "") : nil,
+                       createdAt: ms(r.created_at), updatedAt: ms(r.updated_at), url: r.html_url)
         }
-        return GitHubChecks(head: v.headRefOid, checks: checks)
     }
 }
 
 /// Asks GitHub through the `gh` CLI. `gh pr view` is GraphQL, whose budget is
 /// separate from the REST budget the daemon's token polls with, and a wait
-/// asks at most twice a minute — far from the bulk calls that once
+/// asks at most a few times a minute — far from the bulk calls that once
 /// rate-penalized the daemon.
 public struct GHChecksProbe: Sendable {
     public var repo: String
@@ -96,10 +208,26 @@ public struct GHChecksProbe: Sendable {
 
     public init(repo: String, pr: Int) { self.repo = repo; self.pr = pr }
 
+    /// nil unless GitHub answered all of it: a part missing is not a yes.
     public func fetch() async -> GitHubChecks? {
-        guard let out = await Self.gh(["pr", "view", String(pr), "-R", repo, "--json", "headRefOid,statusCheckRollup"],
-                                      timeout: timeout) else { return nil }
-        return try? GitHubChecks.parse(prView: out)
+        let asked = Format.nowMs()
+        guard let out = await Self.gh(["pr", "view", String(pr), "-R", repo, "--json",
+                                       "headRefOid,headRefName,headRepository,headRepositoryOwner,state,statusCheckRollup"],
+                                      timeout: timeout),
+              var c = try? GitHubChecks.parse(prView: out) else { return nil }
+        // The PR's head catches up with a push a moment after the branch
+        // moves; until it does, its rollup is the previous head's.
+        if c.open, let from = c.headRepo, let ref = c.headRef,
+           let b = await Self.gh(["api", "repos/\(from)/git/ref/heads/\(ref)", "--jq", ".object.sha"], timeout: timeout) {
+            let sha = String(decoding: b, as: UTF8.self).trimmingCharacters(in: .whitespacesAndNewlines)
+            if sha.count == 40, sha != c.head { c.head = sha; c.checks = [] }
+        }
+        guard let head = c.head,
+              let runs = await Self.gh(["api", "repos/\(repo)/actions/runs?head_sha=\(head)&per_page=100"], timeout: timeout),
+              let parsed = try? GitHubChecks.parse(runs: runs) else { return nil }
+        c.runs = parsed
+        c.fetchedAt = asked
+        return c
     }
 
     /// `owner/name` for a bare repo name: from the glance when it knows the

cockpit/Sources/CockpitCore/Rollup.swift

@@ -22,6 +22,8 @@ public struct CheckSet: Identifiable, Sendable, Equatable {
     /// moved to another commit after they ran and has since come back, so a
     /// new run is due (site-web PR #79, 2026-10-03).
     public var superseded: [RunRow] = []
+    /// GitHub's reason a finished-looking set is still pending ("settling: …").
+    public var why: String? = nil
 
     public var label: String {
         let repoShort = repo.split(separator: "/").last.map(String.init) ?? repo