Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 12 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -178,6 +178,18 @@ version numbers (`v0.1.0`, `v0.1.1`, etc.).

### Fixed

- **An account block stops `cockpit wait` only on a repo GitHub is refusing.**
2026-10-24: GitHub refused hosted jobs on three private repos (petcam-android,
petcam-nvr, enjoyable-languages-site; the account's included minutes were
spent), and `cockpit wait taylab-launch-kit --pr 279` and
`cockpit wait taylab-agent-kit --pr 21` exited 2 at once, though their
self-hosted runs were only queued on a busy ultra. The `account-blocked`
finding now carries `repos` (the repos with refused jobs; a storage quota
repo is not one), and the wait is pointless only when the target is one of
them, or when one of the target's own checks is queued for a GitHub-hosted
runner (a public repo's hosted queue still waits). Both deciders changed: the
daemon's (`lib/checks.js`) and the CLI/MCP's (`Waiter.decide`); with an older
daemon the CLI reads the repos from the finding's `Repos:` evidence line.
- **`cockpit wait` and `wait_for_checks` no longer go green before a PR's
checks have all registered.** On acme/stream-app PR #5 (2026-10-24) a
wait that started 8 s after a push returned `1 of 1 green` at once, from the
Expand Down
4 changes: 4 additions & 0 deletions cockpit/Sources/CockpitCore/Fixtures/accountBlocked.json
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,10 @@
"tone": "warning",
"title": "GitHub is refusing jobs on 2 repos",
"blocking": true,
"repos": [
"acme/ion",
"acme/watchtower"
],
"sentence": "Nothing on this machine is wrong. The account is refusing work: payment or spending limit.",
"evidence": [
"87 jobs refused (billing or spending limit) in the last 6.0 h",
Expand Down
1 change: 1 addition & 0 deletions cockpit/Sources/CockpitCore/Fixtures/live.json
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
"tone": "warning",
"title": "Artifact storage quota is full",
"blocking": false,
"repos": [],
"sentence": "Nothing on this machine is wrong. Jobs ran but could not publish their artifacts or caches.",
"evidence": [
"3 jobs hit a storage quota in the last 6.0 h",
Expand Down
12 changes: 12 additions & 0 deletions cockpit/Sources/CockpitCore/Glance.swift
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,18 @@ public struct Finding: Codable, Sendable, Equatable, Identifiable {
public var next: NextMove?
/// account-blocked only: GitHub refuses to start jobs (not merely a full quota).
public var blocking: Bool?
/// account-blocked only: the repos whose jobs GitHub refused (full names).
/// Nil from a daemon older than the field: see `refusedRepos`.
public var repos: [String]?

/// The repos an account block refused jobs on. An older daemon named them
/// only in its evidence, by short name ("Repos: a, b").
public var refusedRepos: [String] {
if let repos { return repos }
guard let line = evidence?.first(where: { $0.hasPrefix("Repos: ") }) else { return [] }
return line.dropFirst("Repos: ".count).split(separator: ",")
.map { $0.trimmingCharacters(in: .whitespaces) }.filter { !$0.isEmpty }
}

public init(id: String, tone: Tone, title: String, sentence: String?, evidence: [String]?, next: NextMove?) {
self.id = id; self.tone = tone; self.title = title
Expand Down
19 changes: 16 additions & 3 deletions cockpit/Sources/CockpitCore/Rollup.swift
Original file line number Diff line number Diff line change
Expand Up @@ -205,12 +205,25 @@ public enum Waiter {
if infra.contains(verdict.id) {
return .pointless("\(verdict.title): \(verdict.sentence ?? "")")
}
if let blocked = verdict.open.first(where: { $0.id == "account-blocked" && $0.blocking == true }) {
return .pointless("\(blocked.title): jobs are refused before they reach a runner")
}
let repoMatch: (String) -> Bool = { $0 == t.repo || $0.hasSuffix("/" + t.repo) }
// An account block stops a wait only on a repo GitHub is refusing. The
// fleet's self-hosted runners and public repos go on through it
// (2026-10-24: hosted petcam-* jobs refused, while the kits' self-hosted
// PR runs were only queued on a busy host and the wait gave up on them).
let short = t.repo.split(separator: "/").last.map { String($0).lowercased() } ?? t.repo.lowercased()
let blocked = verdict.open.first(where: { $0.id == "account-blocked" && $0.blocking == true })
if let blocked, blocked.refusedRepos.contains(where: {
($0.split(separator: "/").last.map { String($0).lowercased() } ?? $0.lowercased()) == short
}) {
return .pointless("\(blocked.title): \(short)'s jobs are refused before they reach a runner")
}
if let stuck = (set?.queued ?? g.queue.filter { repoMatch($0.repo) })
.first(where: { $0.neverStarts }) {
// This repo's own run queued for a GitHub-hosted runner while the
// account is refused: that is the block, about to reach it too.
if let blocked, stuck.cause == "github-hosted" {
return .pointless("\(blocked.title): \(stuck.workflow ?? "a check") is queued for a GitHub-hosted runner, which the account is refusing")
}
return .pointless("\(stuck.workflow ?? "a check") is queued with cause \(stuck.cause ?? "?") — it will not start on its own. \(stuck.recommended ?? "")")
}
if let dead = g.runners.first(where: { repoMatch($0.repo ?? "") && ($0.state == .dead || $0.state == .offline) }),
Expand Down
54 changes: 50 additions & 4 deletions cockpit/Tests/CockpitCoreTests/RollupTests.swift
Original file line number Diff line number Diff line change
Expand Up @@ -157,11 +157,57 @@ struct WaiterTests {
var g = live
g.runs = [run(1, "ci", status: "in_progress", elapsed: 1, expected: 10)]
if case .pointless = Waiter.decide(g, verdict: live.verdict, target: target) { Issue.record("quota should not stop a wait") }
let blocked = try Fixtures.glance("accountBlocked")
let blocked = try Fixtures.glance("accountBlocked") // refuses acme/ion and acme/watchtower
var g2 = blocked
g2.runs = g.runs
guard case .pointless = Waiter.decide(g2, verdict: blocked.verdict, target: target) else {
Issue.record("billing block should stop a wait"); return
g2.runs = [run(1, "ci", status: "queued", repo: "acme/ion")]
guard case .pointless(let why) = Waiter.decide(g2, verdict: blocked.verdict, target: WaitTarget(repo: "ion", pr: 96)) else {
Issue.record("a billing block on this repo should stop a wait"); return
}
#expect(why.hasPrefix("GitHub is refusing jobs on 2 repos: ion's jobs are refused"))
}

// taylab-launch-kit #279 and taylab-agent-kit #21, 2026-10-24: hosted
// petcam-* and enjoyable-languages-site jobs were refused, and both kit
// waits exited 2 while their self-hosted runs were only queued on ultra.
@Test func aBillingBlockOnOtherReposDoesNotStopASelfHostedWait() throws {
var g = try Fixtures.glance("accountBlocked")
var q = queued(1, cause: "host-saturation")
q.etaBasis = "every runner for this repo is busy"
g.runs = [run(1, "ci", status: "queued")]
g.queue = [q]
guard case .waiting = Waiter.decide(g, verdict: g.verdict, target: target) else {
Issue.record("another repo's billing block must not stop this wait"); return
}
}

@Test func aBillingBlockStopsAWaitQueuedForAHostedRunner() throws {
var g = try Fixtures.glance("accountBlocked")
var q = queued(1, cause: "github-hosted")
q.etaBasis = "github-hosted: will not start on its own"
g.runs = [run(1, "e2e", status: "queued")]
g.queue = [q]
guard case .pointless(let why) = Waiter.decide(g, verdict: g.verdict, target: target) else {
Issue.record("a hosted run under a billing block will not start"); return
}
#expect(why.contains("e2e is queued for a GitHub-hosted runner, which the account is refusing"))
g.queue[0].etaBasis = "waiting for a GitHub-hosted runner (public repo)"
guard case .waiting = Waiter.decide(g, verdict: g.verdict, target: target) else {
Issue.record("a public repo's hosted runs are not refused"); return
}
}

@Test func anOlderDaemonNamesRefusedReposInTheEvidence() throws {
var f = Finding(id: "account-blocked", tone: .warning, title: "GitHub is refusing jobs on 3 repos", sentence: nil,
evidence: ["87 jobs refused", "Repos: enjoyable-languages-site, petcam-android, petcam-nvr"], next: nil)
f.blocking = true
#expect(f.refusedRepos == ["enjoyable-languages-site", "petcam-android", "petcam-nvr"])
var g = try glance(runs: [run(1, "ci", status: "queued")])
g.verdict.open = [f]
guard case .waiting = Waiter.decide(g, verdict: g.verdict, target: target) else {
Issue.record("expected waiting"); return
}
guard case .pointless = Waiter.decide(g, verdict: g.verdict, target: WaitTarget(repo: "addisdev/petcam-nvr", pr: 96)) else {
Issue.record("expected pointless for a refused repo"); return
}
}

Expand Down
2 changes: 1 addition & 1 deletion dashboard/lib/api.js
Original file line number Diff line number Diff line change
Expand Up @@ -148,7 +148,7 @@ export const ROUTES = [
'The checks of a PR (its head commit only, never an older commit\'s result), a commit or a branch, '
+ 'rolled up as green, red, pending or pointless. With wait=N the request is held open up to N seconds '
+ '(at most 300) until the checks finish; pending means the wait ran out, so ask again. pointless means '
+ 'waiting cannot end well: the host is down, the disk floor is holding jobs, GitHub is refusing jobs, '
+ 'waiting cannot end well: the host is down, the disk floor is holding jobs, GitHub is refusing this repo\'s jobs, '
+ 'a queued check will never start, or a job was orphaned; reason says which, and orphans carry their '
+ 'recovery. Cancelled is red, not green. Green means GitHub itself has every workflow run for the head '
+ 'completed with all of its jobs passed, and nothing new for 30 s (verified says what it covered): runs '
Expand Down
28 changes: 27 additions & 1 deletion dashboard/lib/checks.js
Original file line number Diff line number Diff line change
Expand Up @@ -174,6 +174,17 @@ function neverStarts(q) {
return NEVER_STARTS.has(q.cause ?? '');
}

// The repos an account block refused jobs on. A daemon older than the
// `repos` field named them only in the evidence, by short name.
export function refusedRepos(blocked) {
if (Array.isArray(blocked?.repos)) return blocked.repos;
const line = (blocked?.evidence ?? []).find((e) => typeof e === 'string' && e.startsWith('Repos: '));
return line ? line.slice('Repos: '.length).split(',').map((r) => r.trim()).filter(Boolean) : [];
}

const shortName = (r) => String(r ?? '').toLowerCase().split('/').pop();
const repoIn = (repos, repo) => repos.some((r) => sameRepo(r, repo) || shortName(r) === shortName(repo));

// What a wait should do with the fleet as it is now: { state, reason, set }.
// A finished result beats a verdict about the fleet, so green and red are
// decided before anything that makes waiting pointless.
Expand All @@ -182,11 +193,26 @@ export function decide(g, target) {
if (set?.state === 'green' || set?.state === 'red') return { state: set.state, reason: null, set };
const v = g.verdict ?? {};
if (INFRA.has(v.id)) return { state: 'pointless', reason: `${v.title}: ${v.sentence ?? ''}`.trim(), set };
// An account block stops a wait only on a repo GitHub is refusing. The
// fleet's self-hosted runners and public repos go on running through it
// (2026-10-24: hosted petcam-* jobs refused, while the kits' self-hosted PR
// runs were only queued on a busy host and `cockpit wait` gave up on them).
const blocked = (v.open ?? []).find((o) => o.id === 'account-blocked' && o.blocking === true);
if (blocked) return { state: 'pointless', reason: `${blocked.title}: jobs are refused before they reach a runner`, set };
if (blocked && repoIn(refusedRepos(blocked), target.repo)) {
return { state: 'pointless', reason: `${blocked.title}: ${shortName(target.repo)}'s jobs are refused before they reach a runner`, set };
}
const queue = set ? set.queued : (g.queue ?? []).filter((q) => sameRepo(q.repo, target.repo));
const stuck = queue.find(neverStarts);
if (stuck) {
// A private repo's own run queued for GitHub-hosted runners while the
// account is refused: that is the block, about to reach this repo too.
if (blocked && stuck.cause === 'github-hosted') {
return {
state: 'pointless',
reason: `${blocked.title}: ${stuck.workflow ?? 'a check'} is queued for a GitHub-hosted runner, which the account is refusing`,
set,
};
}
return {
state: 'pointless',
reason: `${stuck.workflow ?? 'a check'} is queued with cause ${stuck.cause ?? '?'} — it will not start on its own. ${stuck.recommended ?? ''}`.trim(),
Expand Down
14 changes: 12 additions & 2 deletions dashboard/lib/verdict.js
Original file line number Diff line number Diff line change
Expand Up @@ -173,14 +173,16 @@ export function loadFailureFacts(db, now = Date.now(), opts = {}) {
GROUP BY failure_class, repo`)
.all(new Date(now - o.accountWindowMs).toISOString());
const repos = new Set();
const blockedRepos = new Set();
for (const r of rows) {
if (r.cls === 'account-blocked') facts.account.blocked += r.n;
if (r.cls === 'account-blocked') { facts.account.blocked += r.n; blockedRepos.add(r.repo); }
else facts.account.quota += r.n;
repos.add(r.repo);
const at = Date.parse(r.last) || null;
if (at && (!facts.account.lastAt || at > facts.account.lastAt)) facts.account.lastAt = at;
}
facts.account.repos = [...repos].sort();
facts.account.blockedRepos = [...blockedRepos].sort();
} catch { /* pre-migration db: no facts is the honest answer */ }
return facts;
}
Expand Down Expand Up @@ -508,12 +510,20 @@ export function computeVerdict(snapshot, facts = {}, opts = {}) {
const parts = [];
if (account.blocked) parts.push(`${plural(account.blocked, 'job')} refused (billing or spending limit)`);
if (account.quota) parts.push(`${plural(account.quota, 'job')} hit a storage quota`);
const refused = account.blocked > 0 ? (account.blockedRepos ?? account.repos ?? []) : [];
push('account-blocked', {
// True when GitHub refuses to START jobs; a full storage quota lets
// them run. `cockpit wait` stops waiting only for the first.
blocking: account.blocked > 0,
// The repos whose jobs GitHub refused. A block is per repo in practice:
// the account's included minutes run out for private repos on
// GitHub-hosted runners, while public repos and self-hosted runners go
// on (2026-10-24: petcam-* refused, the kit's self-hosted PRs merely
// queued). `cockpit wait` stops only for these repos, or for a run of
// its own queued on GitHub-hosted runners.
repos: refused,
title: account.blocked
? `GitHub is refusing jobs on ${plural(account.repos.length || 1, 'repo')}`
? `GitHub is refusing jobs on ${plural(refused.length || 1, 'repo')}`
: 'Artifact storage quota is full',
sentence: account.blocked
? 'Nothing on this machine is wrong. The account is refusing work: payment or spending limit.'
Expand Down
43 changes: 40 additions & 3 deletions dashboard/test/checks.test.js
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ import { readFileSync } from 'node:fs';
import { join, dirname } from 'node:path';
import { fileURLToPath } from 'node:url';
import {
checkSets, findCheckSet, decide, waitForChecks, parseGitHubChecks, githubState, githubVerdict, githubSet, runIdFromUrl,
checkSets, findCheckSet, decide, refusedRepos, waitForChecks, parseGitHubChecks, githubState, githubVerdict, githubSet, runIdFromUrl,
} from '../lib/checks.js';

// The same cases as cockpit's RollupTests.swift, against the same glance
Expand Down Expand Up @@ -122,10 +122,47 @@ describe('wait decision', () => {
assert.equal(decide(glance({ runs: [run(4, 'e2e', { status: 'queued' })], queue: [hosted] }), target).state, 'pending');
});

test('a storage quota is not a block, a billing block is', () => {
test('a storage quota is not a block, a billing block on this repo is', () => {
const runs = [run(1, 'ci', { status: 'in_progress', elapsed: 1, expected: 10 })];
assert.equal(decide(glance({ runs }, 'live'), target).state, 'pending');
assert.equal(decide(glance({ runs }, 'accountBlocked'), target).state, 'pointless');
const ion = { repo: 'acme/ion', pr: 96 };
const d = decide(glance({ runs: [run(1, 'ci', { status: 'queued', repo: 'acme/ion' })] }, 'accountBlocked'), ion);
assert.equal(d.state, 'pointless');
assert.match(d.reason, /^GitHub is refusing jobs on 2 repos: ion's jobs are refused/);
assert.equal(decide(glance({ runs: [] }, 'accountBlocked'), { repo: 'ion', pr: 96 }).state, 'pointless', 'a short name matches');
});

// taylab-launch-kit #279 and taylab-agent-kit #21, 2026-10-24: hosted
// petcam-* and enjoyable-languages-site jobs were refused, and both kit
// waits exited 2 while their self-hosted runs were only queued on ultra.
test('a billing block on other repos does not stop a self-hosted wait', () => {
const g = glance({
runs: [run(1, 'ci', { status: 'queued' })],
queue: [{ ...queued(1, 'host-saturation'), etaBasis: 'every runner for this repo is busy' }],
}, 'accountBlocked');
const d = decide(g, target);
assert.equal(d.state, 'pending');
assert.equal(d.reason, null);
});

test('a billing block stops a wait whose own run is queued for a hosted runner', () => {
const hosted = { ...queued(1, 'github-hosted'), etaBasis: 'github-hosted: will not start on its own' };
const d = decide(glance({ runs: [run(1, 'e2e', { status: 'queued' })], queue: [hosted] }, 'accountBlocked'), target);
assert.equal(d.state, 'pointless');
assert.match(d.reason, /e2e is queued for a GitHub-hosted runner, which the account is refusing/);
const pub = { ...hosted, etaBasis: 'waiting for a GitHub-hosted runner (public repo)' };
assert.equal(decide(glance({ runs: [run(1, 'e2e', { status: 'queued' })], queue: [pub] }, 'accountBlocked'), target).state,
'pending', 'a public repo\'s hosted runs are not refused');
});

test('an older daemon names the refused repos only in the evidence', () => {
const blocked = { id: 'account-blocked', blocking: true, title: 'GitHub is refusing jobs on 3 repos',
evidence: ['87 jobs refused', 'Repos: enjoyable-languages-site, petcam-android, petcam-nvr'] };
assert.deepEqual(refusedRepos(blocked), ['enjoyable-languages-site', 'petcam-android', 'petcam-nvr']);
const g = glance({ runs: [run(1, 'ci', { status: 'queued' })] });
g.verdict = { ...g.verdict, open: [blocked] };
assert.equal(decide(g, target).state, 'pending');
assert.equal(decide(g, { repo: 'addisdev/petcam-nvr', pr: 96 }).state, 'pointless');
});

test('a dead runner with nothing else serving the repo is pointless', () => {
Expand Down
9 changes: 9 additions & 0 deletions dashboard/test/verdict.test.js
Original file line number Diff line number Diff line change
Expand Up @@ -93,6 +93,14 @@ describe('what does not count', () => {
assert.equal(computeVerdict(s.snapshot, s.facts, { now: s.now }).verdict.id, 'clear');
});

test('an account block names the refused repos, not the quota ones', () => {
const s = SCENARIOS.accountBlocked();
s.facts.account = { ...s.facts.account, quota: 2, repos: ['acme/ion', 'acme/vault', 'acme/watchtower'], blockedRepos: ['acme/ion', 'acme/watchtower'] };
const f = computeVerdict(s.snapshot, s.facts, { now: s.now }).verdict.open.find((o) => o.id === 'account-blocked');
assert.deepEqual(f.repos, ['acme/ion', 'acme/watchtower']);
assert.equal(f.title, 'GitHub is refusing jobs on 2 repos');
});

test('an account block older than the window has left the glance', () => {
const s = SCENARIOS.accountBlocked();
s.facts.account.lastAt = s.now - 7 * 60 * 60 * 1000;
Expand Down Expand Up @@ -342,6 +350,7 @@ describe('loadFailureFacts', () => {
assert.equal(f.account.blocked, 1);
assert.equal(f.account.quota, 1);
assert.deepEqual(f.account.repos, ['acme/b', 'acme/c']);
assert.deepEqual(f.account.blockedRepos, ['acme/b'], 'a storage quota is not a refusal');
assert.equal(f.account.lastAt, Date.parse(iso(29)));
assert.deepEqual(f.recentFailures.map((x) => x.cls).sort(), ['account-blocked', 'account-quota', 'runner-lost']);
} finally {
Expand Down
Loading
Loading