diff --git a/changelog.d/added-14558.md b/changelog.d/added-14558.md new file mode 100644 index 0000000000..dab8bed68c --- /dev/null +++ b/changelog.d/added-14558.md @@ -0,0 +1 @@ +- Repeat-failure check: with `repeat_failure.enabled` on, each agent's dashboard row shows the check state (on, "not available for this backend", or "not receiving data" after `no_data_after_s` of shell activity without command records) and a "Repeating failure: `` failed N times in M min (exit S)" note when a command keeps failing the same way; nothing is shown while the check is off and the agent is never paused. A code coverage table, mirrored in the backend support docs and checked in CI, lists every backend; none is marked covered yet, pending real-run evidence (#14559) (#14558). diff --git a/src/cmd/hive/agent_runtime_sidecar.go b/src/cmd/hive/agent_runtime_sidecar.go index d5a479b7fb..3dc6a3efac 100644 --- a/src/cmd/hive/agent_runtime_sidecar.go +++ b/src/cmd/hive/agent_runtime_sidecar.go @@ -309,6 +309,9 @@ func (h *sidecarHost) SetAdvisorRolesResolver(func() map[string]cruxconfig.Model func (h *sidecarHost) SetRepeatFailureEnabledResolver(func() bool) { h.inProcessOnly("SetRepeatFailureEnabledResolver") } +func (h *sidecarHost) SetRepeatFailureConfigResolver(func() *cruxconfig.RepeatFailureConfig) { + h.inProcessOnly("SetRepeatFailureConfigResolver") +} func (h *sidecarHost) SetRepoAutoMergeEnabledResolver(fn func(string) bool) { h.setPolicy("SetRepoAutoMergeEnabledResolver", func() { h.autoMerge = fn }) } diff --git a/src/cmd/hive/boot_agents_deps.go b/src/cmd/hive/boot_agents_deps.go index 98fd2d8914..a6e5083c5a 100644 --- a/src/cmd/hive/boot_agents_deps.go +++ b/src/cmd/hive/boot_agents_deps.go @@ -26,7 +26,8 @@ import ( // the rest of the test binary's life are recorded instead of started. type bootAgentsDeps struct { // startAgentLoops starts the per-agent token refresh, the credential - // watchdog and the Copilot session refresh. All three are unconditional + // watchdog, the Copilot session refresh and the repeat-failure monitor + // (which only clears row state while the check is off). All are unconditional // (see the comments at the call site for why each must not gate on App // state at boot). startAgentLoops func(ctx context.Context, mgr crux.Runtime) @@ -92,6 +93,7 @@ func defaultBootAgentsDeps() bootAgentsDeps { go mgr.StartCredentialWatchdog(ctx) go mgr.StartCopilotSessionRefresh(ctx) go mgr.StartTranscriptActivityRefresh(ctx) + go mgr.StartRepeatFailureMonitor(ctx) }, prepareRequestDirs: github.PrepareRequestDirs, startTokenAccessAudit: func(ctx context.Context, logger *slog.Logger) { diff --git a/src/cmd/hive/main.go b/src/cmd/hive/main.go index 7272ea16a2..66ef473230 100644 --- a/src/cmd/hive/main.go +++ b/src/cmd/hive/main.go @@ -2430,6 +2430,9 @@ func (b *boot) bootAgentsWith(deps bootAgentsDeps) { b.agentMgr.SetRepeatFailureEnabledResolver(func() bool { return b.cfg.RepeatFailure.IsEnabled() }) + b.agentMgr.SetRepeatFailureConfigResolver(func() *config.RepeatFailureConfig { + return b.cfg.RepeatFailure + }) b.agentMgr.SetAdvisorRolesResolver(func() map[string]config.ModelRole { return b.cfg.ModelRoles }) diff --git a/src/crux/agent/manager.go b/src/crux/agent/manager.go index 408f6a7401..c46cb4a206 100644 --- a/src/crux/agent/manager.go +++ b/src/crux/agent/manager.go @@ -449,6 +449,10 @@ type AgentProcess struct { // report, or plain "ended". Settled(LastKick) is false while the current // turn is still running. Guarded by m.mu; see kick_outcome.go. KickOutcome KickOutcome + // RepeatFailure is the repeat-failure check's row note and check state + // (#14550), refreshed at each record collection. Zero while the check is + // off. Guarded by m.mu; see repeat_failure_status.go. + RepeatFailure RepeatFailureStatus // sandboxResumeAfterCancel is set when an operator resumes a paused // sandbox agent while the canceled sandbox goroutine is still draining. // The completion handler then turns the expected cancellation into Idle @@ -616,6 +620,12 @@ type Manager struct { // switch (#14550), read at launch so a toggle applies on the next launch. // Same atomic.Pointer discipline as advisorEnabledResolver. repeatFailureEnabledResolver atomic.Pointer[func() bool] + // repeatFailureConfigResolver returns the live repeat_failure block for + // the detector and the not-receiving-data safeguard. Same discipline. + repeatFailureConfigResolver atomic.Pointer[func() *config.RepeatFailureConfig] + // repeatFailure is the repeat-failure detector and safeguard state behind + // AgentProcess.RepeatFailure; see repeat_failure_status.go. + repeatFailure repeatFailureMonitor // advisorRolesResolver returns the live model_roles map the OMP adapter // projects as a --config overlay at launch (#9723). Same discipline as // advisorEnabledResolver. diff --git a/src/crux/agent/manager_pane_signals.go b/src/crux/agent/manager_pane_signals.go index aedf201989..c701aaf610 100644 --- a/src/crux/agent/manager_pane_signals.go +++ b/src/crux/agent/manager_pane_signals.go @@ -572,6 +572,7 @@ func (a *AgentProcess) snapshot() crux.Process { BootstrapOverride: a.BootstrapOverride, LastError: a.LastError, KickOutcome: a.KickOutcome, + RepeatFailure: a.RepeatFailure, NeedsLogin: needsLogin, LoginURL: loginURL, Starting: a.startupLaunchQueued || a.launching, diff --git a/src/crux/agent/repeat_failure_coverage.go b/src/crux/agent/repeat_failure_coverage.go new file mode 100644 index 0000000000..e1001bee62 --- /dev/null +++ b/src/crux/agent/repeat_failure_coverage.go @@ -0,0 +1,121 @@ +package agent + +import ( + "strconv" + "strings" +) + +// Repeat-failure coverage table (hivecommons/hive#14550, C4, C5, C12, C13). +// +// Every backend Hive supports (config.CLIBackends and +// config.InferenceBackends) has exactly one row. A backend is covered only +// when a committed, re-runnable harness made its real CLI produce a correct +// command record, with the CLI's own `--version` output in the evidence +// (C4). Hand-written test data does not count. Every other backend is not +// covered: its agent rows say "repeat-failure check not available for this +// backend", and the row names the reason and the follow-up issue (C5). +// +// The docs table in docs/backend-support-tiers.md ("Repeated-failure check") +// is rendered from this table; TestRepeatFailureCoverageDocsMatchCode fails +// when the two differ or a backend is missing (C13). + +// repeatFailureBackendRoutesIssue tracks the backend-specific routes and the +// real-run evidence still missing for every backend below (#14559). +const repeatFailureBackendRoutesIssue = 14559 + +// repeatFailureSafeguardUnavailable is the shell-activity signal of a covered +// backend that has none, so the not-receiving-data safeguard cannot run (C12). +const repeatFailureSafeguardUnavailable = "safeguard unavailable" + +// RepeatFailureCoverage is one backend's row of the coverage table. +type RepeatFailureCoverage struct { + Backend string + // Covered is true only with committed real-run evidence (C4). + Covered bool + // Route is the producer of the backend's records, or, for a backend that + // is not covered, what exists today and what is planned. + Route string + // Evidence names the committed harness and the CLI `--version` it + // recorded. Empty when not covered. + Evidence string + // EvidencePath is the committed harness evidence, relative to src/. + // Required when Covered. + EvidencePath string + // ShellActivity is the signal the not-receiving-data safeguard watches + // for this backend, or repeatFailureSafeguardUnavailable. Empty when not + // covered. + ShellActivity string + // Reason says why the backend is not covered. Empty when covered. + Reason string + // FollowUp is the issue that tracks covering the backend. + FollowUp int +} + +const ( + rfNeutralRouteNoEvidence = "BASH_ENV route installed while the check is on, but no committed real-run harness evidence yet (C4)" + rfOwnRouteNotBuilt = "commands never reach the BASH_ENV route and the backend-specific route is not built yet" +) + +// repeatFailureCoverageTable lists every backend in config.CLIBackends order, +// then config.InferenceBackends. +var repeatFailureCoverageTable = []RepeatFailureCoverage{ + {Backend: "claude", Route: "BASH_ENV (neutral), unwraps `eval '…'`", FollowUp: repeatFailureBackendRoutesIssue, Reason: rfNeutralRouteNoEvidence}, + {Backend: "copilot", Route: "none yet; planned: `events.jsonl` `tool.execution_complete`", FollowUp: repeatFailureBackendRoutesIssue, Reason: "one long-lived Bash session per shell id; " + rfOwnRouteNotBuilt}, + {Backend: "goose", Route: "BASH_ENV (neutral)", FollowUp: repeatFailureBackendRoutesIssue, Reason: rfNeutralRouteNoEvidence}, + {Backend: "codex", Route: "BASH_ENV (neutral), one record across `exec /bin/bash -c`", FollowUp: repeatFailureBackendRoutesIssue, Reason: rfNeutralRouteNoEvidence}, + {Backend: "pi", Route: "BASH_ENV (neutral)", FollowUp: repeatFailureBackendRoutesIssue, Reason: rfNeutralRouteNoEvidence}, + {Backend: "bob", Route: "BASH_ENV (neutral) with `SHELL=/bin/bash` (documented shell change: dash → bash)", FollowUp: repeatFailureBackendRoutesIssue, Reason: rfNeutralRouteNoEvidence}, + {Backend: "aider", Route: "none yet; planned: guarded `~/.bashrc` line in Hive-managed homes", FollowUp: repeatFailureBackendRoutesIssue, Reason: "interactive `$SHELL -i -c` ignores BASH_ENV; " + rfOwnRouteNotBuilt}, + {Backend: "gemini", Route: "BASH_ENV (neutral), unwraps the `{ …; }; __code=$?` wrapper", FollowUp: repeatFailureBackendRoutesIssue, Reason: rfNeutralRouteNoEvidence}, + {Backend: "agy", Route: "none yet; planned: `agy-turn` `run_command` steps", FollowUp: repeatFailureBackendRoutesIssue, Reason: "closed binary, likely a long-lived terminal; " + rfOwnRouteNotBuilt}, + {Backend: "opencode", Route: "BASH_ENV (neutral) with `SHELL=/bin/bash`", FollowUp: repeatFailureBackendRoutesIssue, Reason: rfNeutralRouteNoEvidence}, + {Backend: "kilo", Route: "BASH_ENV (neutral) with `SHELL=/bin/bash`", FollowUp: repeatFailureBackendRoutesIssue, Reason: rfNeutralRouteNoEvidence}, + {Backend: "muse", Route: "none yet; planned: `PostToolUse` / `PostToolUseFailure` hooks", FollowUp: repeatFailureBackendRoutesIssue, Reason: "sandboxed shell tools in a closed binary; " + rfOwnRouteNotBuilt}, + {Backend: "omp", Route: "none yet; planned: `tool_result` extension event", FollowUp: repeatFailureBackendRoutesIssue, Reason: "embedded shell that strips BASH_ENV; " + rfOwnRouteNotBuilt}, + {Backend: "openhands", Route: "none yet; planned: `PostToolUse` hook or saved events", FollowUp: repeatFailureBackendRoutesIssue, Reason: "one interactive bash kept open in tmux; " + rfOwnRouteNotBuilt}, + {Backend: "omnigent", Route: "none yet; planned: tool-call policy checkpoint (`ALLOW`/`DENY` verdicts) or saved session events", FollowUp: repeatFailureBackendRoutesIssue, Reason: "meta-harness driving Claude Code, Codex or Pi through its own policy engine; " + rfOwnRouteNotBuilt}, + {Backend: "vllm", Route: "follows `claude` (drives Claude Code)", FollowUp: repeatFailureBackendRoutesIssue, Reason: "follows `claude`, which is not covered"}, + {Backend: "llm-d", Route: "follows `claude` (drives Claude Code)", FollowUp: repeatFailureBackendRoutesIssue, Reason: "follows `claude`, which is not covered"}, + {Backend: "litellm", Route: "follows `claude` (drives Claude Code)", FollowUp: repeatFailureBackendRoutesIssue, Reason: "follows `claude`, which is not covered"}, + {Backend: "watsonx", Route: "follows `claude` (drives Claude Code)", FollowUp: repeatFailureBackendRoutesIssue, Reason: "follows `claude`, which is not covered"}, +} + +// RepeatFailureCoverageTable returns a copy of the coverage table. +func RepeatFailureCoverageTable() []RepeatFailureCoverage { + return append([]RepeatFailureCoverage(nil), repeatFailureCoverageTable...) +} + +// RepeatFailureBackendCovered reports whether the check covers backend. An +// unknown backend is not covered. +func RepeatFailureBackendCovered(backend string) bool { + backend = strings.ToLower(strings.TrimSpace(backend)) + for _, row := range repeatFailureCoverageTable { + if row.Backend == backend { + return row.Covered + } + } + return false +} + +// RepeatFailureCoverageMarkdownRows renders the docs table rows, one per +// backend, in table order. The docs carry exactly these lines. +func RepeatFailureCoverageMarkdownRows() []string { + rows := make([]string, 0, len(repeatFailureCoverageTable)) + for _, row := range repeatFailureCoverageTable { + status := "Not covered: " + row.Reason + evidence := "None" + signal := "n/a" + if row.Covered { + status = "Covered" + evidence = row.Evidence + signal = row.ShellActivity + } + followUp := "—" + if row.FollowUp > 0 { + n := strconv.Itoa(row.FollowUp) + followUp = "[#" + n + "](https://github.com/hivecommons/hive/issues/" + n + ")" + } + rows = append(rows, "| `"+row.Backend+"` | "+row.Route+" | "+status+" | "+evidence+" | "+signal+" | "+followUp+" |") + } + return rows +} diff --git a/src/crux/agent/repeat_failure_coverage_test.go b/src/crux/agent/repeat_failure_coverage_test.go new file mode 100644 index 0000000000..87357f74df --- /dev/null +++ b/src/crux/agent/repeat_failure_coverage_test.go @@ -0,0 +1,116 @@ +package agent + +import ( + "os" + "path/filepath" + "slices" + "strings" + "testing" + + "github.com/hivecommons/hive/crux/config" +) + +// TestRepeatFailureCoverageListsEveryBackend is C13/C-AC11: the code table has +// exactly one row for every config.CLIBackends and config.InferenceBackends +// entry, so removing a backend from it fails here. +func TestRepeatFailureCoverageListsEveryBackend(t *testing.T) { + rows := map[string]RepeatFailureCoverage{} + for _, row := range RepeatFailureCoverageTable() { + if _, dup := rows[row.Backend]; dup { + t.Errorf("backend %q has more than one coverage row", row.Backend) + } + rows[row.Backend] = row + } + want := append(append([]string(nil), config.CLIBackends...), config.InferenceBackends...) + for _, backend := range want { + if _, ok := rows[backend]; !ok { + t.Errorf("repeat-failure coverage table is missing backend %q", backend) + } + } + for backend := range rows { + if !slices.Contains(want, backend) { + t.Errorf("coverage row %q is neither a config.CLIBackends nor a config.InferenceBackends entry", backend) + } + } +} + +// TestRepeatFailureCoverageRowsAreHonest is C4/C5: a covered row carries a +// route, committed harness evidence with the CLI's --version and a +// shell-activity signal (or "safeguard unavailable"); a row that is not +// covered carries a reason and a follow-up issue. +func TestRepeatFailureCoverageRowsAreHonest(t *testing.T) { + for _, row := range RepeatFailureCoverageTable() { + if row.FollowUp < 0 { + t.Errorf("%s: negative follow-up issue", row.Backend) + } + if strings.TrimSpace(row.Route) == "" { + t.Errorf("%s: route is empty", row.Backend) + } + if strings.Contains(row.Route+row.Reason+row.Evidence+row.ShellActivity, "|") { + t.Errorf("%s: a cell contains '|', which breaks the docs table", row.Backend) + } + if !row.Covered { + if strings.TrimSpace(row.Reason) == "" || row.FollowUp == 0 { + t.Errorf("%s: not covered needs a reason and a follow-up issue (C5)", row.Backend) + } + if row.Evidence != "" || row.EvidencePath != "" || row.ShellActivity != "" { + t.Errorf("%s: not covered must not carry evidence or a safeguard signal", row.Backend) + } + if RepeatFailureBackendCovered(row.Backend) { + t.Errorf("%s: RepeatFailureBackendCovered disagrees with the table", row.Backend) + } + continue + } + if row.Reason != "" { + t.Errorf("%s: a covered row must not carry a not-covered reason", row.Backend) + } + if !strings.Contains(row.Evidence, "--version") { + t.Errorf("%s: covered evidence must include the CLI's --version output (C4)", row.Backend) + } + if row.EvidencePath == "" { + t.Errorf("%s: covered without committed real-run harness evidence (C4)", row.Backend) + } else if _, err := os.Stat(filepath.Join("..", "..", row.EvidencePath)); err != nil { + t.Errorf("%s: evidence %s is not committed: %v", row.Backend, row.EvidencePath, err) + } + if strings.TrimSpace(row.ShellActivity) == "" { + t.Errorf("%s: covered row must name its shell-activity signal or %q (C12)", row.Backend, repeatFailureSafeguardUnavailable) + } + if !RepeatFailureBackendCovered(row.Backend) { + t.Errorf("%s: RepeatFailureBackendCovered disagrees with the table", row.Backend) + } + } + if RepeatFailureBackendCovered("no-such-backend") { + t.Error("an unknown backend must not be covered") + } +} + +// TestRepeatFailureCoverageDocsMatchCode is C13/C-AC11: the repeated-failure +// table in docs/backend-support-tiers.md is exactly the rendered code table, +// so a row changed only in the docs fails here. +func TestRepeatFailureCoverageDocsMatchCode(t *testing.T) { + raw, err := os.ReadFile("../../docs/backend-support-tiers.md") + if err != nil { + t.Fatal(err) + } + doc := string(raw) + const heading = "## Repeated-failure check" + start := strings.Index(doc, heading) + if start < 0 { + t.Fatalf("backend-support-tiers.md has no %q section", heading) + } + section := doc[start+len(heading):] + if end := strings.Index(section, "\n## "); end >= 0 { + section = section[:end] + } + var got []string + for _, line := range strings.Split(section, "\n") { + if strings.HasPrefix(line, "| `") { + got = append(got, line) + } + } + want := RepeatFailureCoverageMarkdownRows() + if !slices.Equal(got, want) { + t.Errorf("docs repeated-failure table differs from the code table (pkg/agent/repeat_failure_coverage.go).\nwant rows:\n%s\n\ngot rows:\n%s", + strings.Join(want, "\n"), strings.Join(got, "\n")) + } +} diff --git a/src/crux/agent/repeat_failure_status.go b/src/crux/agent/repeat_failure_status.go new file mode 100644 index 0000000000..0069576f4d --- /dev/null +++ b/src/crux/agent/repeat_failure_status.go @@ -0,0 +1,363 @@ +package agent + +import ( + "context" + "fmt" + "sync" + "time" + "unicode/utf8" + + "github.com/hivecommons/hive/crux" + "github.com/hivecommons/hive/crux/config" +) + +// Operator surface of the repeat-failure check (hivecommons/hive#14550, C11, +// C12, C14; issue #14558): the collector feeds scrubbed command records to +// the detector, and every collection refreshes each agent's +// RepeatFailureStatus, the field the dashboard row renders. It is a separate +// field on purpose: watchdog conditions are a fixed set the reconciler owns, +// and BusyCondition is one recomputed string. Monitor only (C10): nothing +// here pauses, kicks or interrupts an agent. + +// RepeatFailureCheckState is an agent's repeat-failure check state. Empty +// means the check is off, and the dashboard shows nothing (C14). Defined in +// package crux so observers (dashboard, hub) read it without importing +// this package. +type RepeatFailureCheckState = crux.RepeatFailureCheckState + +const ( + RepeatFailureCheckOn = crux.RepeatFailureCheckOn + RepeatFailureCheckNotCovered = crux.RepeatFailureCheckNotCovered + RepeatFailureCheckNoData = crux.RepeatFailureCheckNoData +) + +// Row texts for the check states (C5, C11, C12). +const ( + RepeatFailureCheckOnMessage = crux.RepeatFailureCheckOnMessage + RepeatFailureCheckNotCoveredMessage = crux.RepeatFailureCheckNotCoveredMessage + RepeatFailureCheckNoDataMessage = crux.RepeatFailureCheckNoDataMessage +) + +// repeatFailureNoteCommandMax bounds the command shown in a row note, in runes. +const repeatFailureNoteCommandMax = 160 + +// RepeatFailureStatus is what an agent's dashboard row shows for the check. +// The zero value (check off) shows nothing. +type RepeatFailureStatus = crux.RepeatFailureStatus + +// FormatRepeatFailureNote renders the row note for the detector's notes, +// newest first: the most recent command and how many others repeat. +func FormatRepeatFailureNote(notes []RepeatFailureNote, window time.Duration) string { + if len(notes) == 0 { + return "" + } + n := notes[0] + result := "(failed)" + if n.Result.ExitCode != nil { + result = fmt.Sprintf("(exit %d)", *n.Result.ExitCode) + } + minutes := int((window + time.Minute - 1) / time.Minute) + note := fmt.Sprintf("Repeating failure: `%s` failed %d times in %d min %s", truncateRepeatFailureCommand(n.Command, repeatFailureNoteCommandMax), n.Count, minutes, result) + switch others := len(notes) - 1; { + case others == 1: + note += " and 1 other" + case others > 1: + note += fmt.Sprintf(" and %d others", others) + } + return note +} + +func truncateRepeatFailureCommand(s string, limit int) string { + if utf8.RuneCountInString(s) <= limit { + return s + } + r := []rune(s) + return string(r[:limit-1]) + "…" +} + +// RepeatFailureToolKind classifies a tool call a backend reports for the +// not-receiving-data safeguard (C12). Only shell calls count: file reads and +// edits alone are not shell-tool activity. +type RepeatFailureToolKind int + +const ( + RepeatFailureToolShell RepeatFailureToolKind = iota + 1 + RepeatFailureToolFileRead + RepeatFailureToolFileEdit + RepeatFailureToolOther +) + +// repeatFailureAgentState is the safeguard's view of one agent. +type repeatFailureAgentState struct { + lastRecord time.Time + // kick is the turn the shell activity below belongs to. + kick time.Time + // running counts shell calls started and not yet finished in kick. + running int + // pendingSince is the first shell call that finished in kick with no + // command record since. A still-running command never sets it. + pendingSince time.Time +} + +// repeatFailureMonitor holds the detector and the per-agent safeguard state. +// Lock order: Manager.mu before repeatFailureMonitor.mu, never the reverse. +type repeatFailureMonitor struct { + mu sync.Mutex + detector *RepeatFailureDetector + detectorKey string + window time.Duration + agents map[string]*repeatFailureAgentState +} + +// SetRepeatFailureConfigResolver injects the live repeat_failure block so the +// detector and safeguard follow config edits without a restart. A nil +// resolver (or a nil block) uses the defaults. The on/off switch stays with +// SetRepeatFailureEnabledResolver. +func (m *Manager) SetRepeatFailureConfigResolver(fn func() *config.RepeatFailureConfig) { + if fn == nil { + m.repeatFailureConfigResolver.Store(nil) + return + } + m.repeatFailureConfigResolver.Store(&fn) +} + +// repeatFailureConfig returns the live block with the switch forced on; only +// called once repeatFailureEnabled has said the check is on. +func (m *Manager) repeatFailureConfig() *config.RepeatFailureConfig { + cfg := config.RepeatFailureConfig{} + if fnp := m.repeatFailureConfigResolver.Load(); fnp != nil && *fnp != nil { + if live := (*fnp)(); live != nil { + cfg = *live + } + } + cfg.Enabled = true + return &cfg +} + +// StartRepeatFailureMonitor runs RunRepeatFailureMonitor on a ticker at +// RepeatFailureCollectInterval until ctx ends. While the check is off each +// tick only clears the row state, touching no file. +func (m *Manager) StartRepeatFailureMonitor(ctx context.Context) { + ticker := time.NewTicker(RepeatFailureCollectInterval) + defer ticker.Stop() + m.RunRepeatFailureMonitor(ctx, ticker.C) +} + +// RunRepeatFailureMonitor collects command records and refreshes every +// agent's RepeatFailureStatus on each tick until ctx ends. Production passes +// a ticker at RepeatFailureCollectInterval; tests call +// CollectRepeatFailures directly. +func (m *Manager) RunRepeatFailureMonitor(ctx context.Context, tick <-chan time.Time) { + for { + select { + case <-ctx.Done(): + return + case now := <-tick: + m.CollectRepeatFailures(now) + } + } +} + +// CollectRepeatFailures runs one collection and refreshes the row status. With +// the check off it touches no file and clears every status, so no row shows a +// note or check state (C14). +func (m *Manager) CollectRepeatFailures(now time.Time) { + mon := &m.repeatFailure + if !m.repeatFailureEnabled() { + mon.mu.Lock() + mon.detector, mon.detectorKey, mon.agents = nil, "", nil + mon.mu.Unlock() + m.mu.Lock() + for _, a := range m.agents { + a.RepeatFailure = RepeatFailureStatus{} + } + m.mu.Unlock() + return + } + cfg := m.repeatFailureConfig() + mon.configure(cfg) + collector := NewRepeatFailureCollector(cfg, m.repeatFailureRecordPaths, func(rec RepeatFailureRecord) { + m.recordRepeatFailure(rec, now) + }, m.logger) + collector.Collect() + m.refreshRepeatFailureStatus(cfg.EffectiveNoDataAfter(), now) +} + +// configure (re)builds the detector when threshold, window or exclusions +// change. Exclusions are normalized like commands (C9). +func (mon *repeatFailureMonitor) configure(cfg *config.RepeatFailureConfig) { + exclude := make([]string, 0, len(cfg.EffectiveExclude())) + for _, p := range cfg.EffectiveExclude() { + if p = NormalizeRepeatFailureCommand(p); p != "" { + exclude = append(exclude, p) + } + } + key := fmt.Sprintf("%d|%s|%q", cfg.EffectiveThreshold(), cfg.EffectiveWindow(), exclude) + mon.mu.Lock() + defer mon.mu.Unlock() + if mon.detector == nil || mon.detectorKey != key { + mon.detector = NewRepeatFailureDetector(cfg.EffectiveThreshold(), cfg.EffectiveWindow(), exclude) + mon.detectorKey = key + mon.window = cfg.EffectiveWindow() + } + if mon.agents == nil { + mon.agents = make(map[string]*repeatFailureAgentState) + } +} + +// repeatFailureRecordPaths maps each agent to its raw record file. Agents +// that share a HOME (contributor local mode) share a file whose records +// cannot be told apart, so they are skipped rather than misattributed. +func (m *Manager) repeatFailureRecordPaths() map[string]string { + m.mu.RLock() + defer m.mu.RUnlock() + paths := make(map[string]string, len(m.agents)) + owners := make(map[string]int, len(m.agents)) + for name, a := range m.agents { + home := AgentHome(name, a.UID, effectiveBackend(a)) + if home == "" { + continue + } + p := RepeatFailureRecordPath(home) + paths[name] = p + owners[p]++ + } + for name, p := range paths { + if owners[p] > 1 { + delete(paths, name) + } + } + return paths +} + +// recordRepeatFailure feeds one scrubbed record to the detector. A raised +// note is logged once, when it is raised (C11). now is the collection time, so +// records reach the detector in arrival order whatever their own clock says. +func (m *Manager) recordRepeatFailure(rec RepeatFailureRecord, now time.Time) { + command := rec.Command + if command == "" { + command = NormalizeRepeatFailureCommand(rec.RawCommand) + } + result := RepeatFailureResult{ExitCode: rec.ExitCode, Failed: rec.Failed} + mon := &m.repeatFailure + mon.mu.Lock() + st := mon.agentState(rec.Agent) + st.lastRecord = now + st.pendingSince = time.Time{} + var note *RepeatFailureNote + if mon.detector != nil { + note = mon.detector.Record(rec.Agent, command, result, now) + } + window := mon.window + mon.mu.Unlock() + if note != nil { + m.logger.Warn("repeat-failure: agent is repeating a failing command (monitor only; the agent is not paused)", + "agent", rec.Agent, "backend", rec.Backend, "route", rec.Route, + "note", FormatRepeatFailureNote([]RepeatFailureNote{*note}, window)) + } +} + +// agentState returns the safeguard state for agent. Caller holds mon.mu. +func (mon *repeatFailureMonitor) agentState(agent string) *repeatFailureAgentState { + if mon.agents == nil { + mon.agents = make(map[string]*repeatFailureAgentState) + } + st := mon.agents[agent] + if st == nil { + st = &repeatFailureAgentState{} + mon.agents[agent] = st + } + return st +} + +// ObserveRepeatFailureTool reports one tool call of a covered backend's turn +// to the not-receiving-data safeguard (C12): finished false when the call +// starts, true when it ends. Only shell calls count; a still-running command +// does not start the timer. Each covered backend's producer reports the +// shell-activity signal named in its coverage-table row. +func (m *Manager) ObserveRepeatFailureTool(agentName string, kind RepeatFailureToolKind, finished bool, at time.Time) { + if kind != RepeatFailureToolShell || !m.repeatFailureEnabled() { + return + } + m.mu.RLock() + a := m.agents[agentName] + var kick time.Time + if a != nil && a.LastKick != nil { + kick = *a.LastKick + } + m.mu.RUnlock() + if a == nil { + return + } + mon := &m.repeatFailure + mon.mu.Lock() + defer mon.mu.Unlock() + st := mon.agentState(agentName) + if !st.kick.Equal(kick) { + st.kick, st.running, st.pendingSince = kick, 0, time.Time{} + } + if !finished { + st.running++ + return + } + if st.running > 0 { + st.running-- + } + if st.pendingSince.IsZero() && at.After(st.lastRecord) { + st.pendingSince = at + } +} + +// refreshRepeatFailureStatus recomputes every agent's row status and forgets +// agents that are gone. +func (m *Manager) refreshRepeatFailureStatus(noDataAfter time.Duration, now time.Time) { + mon := &m.repeatFailure + m.mu.Lock() + defer m.mu.Unlock() + mon.mu.Lock() + defer mon.mu.Unlock() + for name := range mon.agents { + if _, ok := m.agents[name]; !ok { + delete(mon.agents, name) + if mon.detector != nil { + mon.detector.Forget(name) + } + } + } + for name, a := range m.agents { + var notes []RepeatFailureNote + if mon.detector != nil { + notes = mon.detector.Notes(name, now) + } + a.RepeatFailure = RepeatFailureStatus{ + Check: repeatFailureCheckState(RepeatFailureBackendCovered(effectiveBackend(a)), repeatFailureTurnKick(a), mon.agents[name], noDataAfter, now), + Note: FormatRepeatFailureNote(notes, mon.window), + } + } +} + +// repeatFailureTurnKick returns the kick of the agent's turn in progress, or +// nil when no turn is running. Caller holds m.mu. +func repeatFailureTurnKick(a *AgentProcess) *time.Time { + if a.State != StateRunning || a.LastKick == nil || a.KickOutcome.Settled(a.LastKick) { + return nil + } + return a.LastKick +} + +// repeatFailureCheckState decides an agent's check state (C5, C12). A covered +// agent is "not receiving data" only when its turn in progress finished a +// shell call at least noDataAfter ago, no record has arrived since, and no +// shell call is still running. +func repeatFailureCheckState(covered bool, turnKick *time.Time, st *repeatFailureAgentState, noDataAfter time.Duration, now time.Time) RepeatFailureCheckState { + if !covered { + return RepeatFailureCheckNotCovered + } + if turnKick == nil || st == nil || !st.kick.Equal(*turnKick) || st.running > 0 || st.pendingSince.IsZero() { + return RepeatFailureCheckOn + } + if now.Sub(st.pendingSince) >= noDataAfter { + return RepeatFailureCheckNoData + } + return RepeatFailureCheckOn +} diff --git a/src/crux/agent/repeat_failure_status_test.go b/src/crux/agent/repeat_failure_status_test.go new file mode 100644 index 0000000000..7bdcf459f3 --- /dev/null +++ b/src/crux/agent/repeat_failure_status_test.go @@ -0,0 +1,237 @@ +package agent + +import ( + "strings" + "testing" + "time" + + "github.com/hivecommons/hive/crux/config" +) + +func rfStatusManager(t *testing.T, backend string) *Manager { + t.Helper() + t.Setenv("HOME", t.TempDir()) + m := NewManager(map[string]config.AgentConfig{ + "scanner": {Backend: backend, Model: "m"}, + }, discardLogger(), ProjectContext{}) + m.mu.Lock() + m.agents["scanner"].UID = 0 + m.mu.Unlock() + return m +} + +func rfStatus(m *Manager) RepeatFailureStatus { + m.mu.RLock() + defer m.mu.RUnlock() + return m.agents["scanner"].snapshot().RepeatFailure +} + +func rfWriteFailures(t *testing.T, m *Manager, command string, code, n int) { + t.Helper() + path := m.repeatFailureRecordPaths()["scanner"] + if path == "" { + t.Fatal("no record path for scanner") + } + if err := EnsureRepeatFailureRecordDir(path, 0); err != nil { + t.Fatal(err) + } + for range n { + c := code + rec := RepeatFailureRecord{Agent: "scanner", Backend: "claude", RawCommand: command, ExitCode: &c, Failed: c != 0, Route: RepeatFailureRouteBashEnv} + if err := AppendRepeatFailureRecord(path, rec); err != nil { + t.Fatal(err) + } + } +} + +func TestFormatRepeatFailureNote(t *testing.T) { + code := 1 + now := time.Now() + if got := FormatRepeatFailureNote(nil, 10*time.Minute); got != "" { + t.Errorf("no notes: %q, want empty", got) + } + one := []RepeatFailureNote{{Command: "ls /nonexistent", Result: RepeatFailureResult{ExitCode: &code}, Count: 5, Latest: now}} + if got, want := FormatRepeatFailureNote(one, 10*time.Minute), "Repeating failure: `ls /nonexistent` failed 5 times in 10 min (exit 1)"; got != want { + t.Errorf("got %q, want %q", got, want) + } + noCode := []RepeatFailureNote{ + {Command: "npm test", Result: RepeatFailureResult{Failed: true}, Count: 6, Latest: now}, + {Command: "gh pr view 123", Result: RepeatFailureResult{ExitCode: &code}, Count: 5, Latest: now.Add(-time.Minute)}, + {Command: "gh pr view 456", Result: RepeatFailureResult{ExitCode: &code}, Count: 5, Latest: now.Add(-2 * time.Minute)}, + } + if got, want := FormatRepeatFailureNote(noCode, 10*time.Minute), "Repeating failure: `npm test` failed 6 times in 10 min (failed) and 2 others"; got != want { + t.Errorf("got %q, want %q", got, want) + } + if got := FormatRepeatFailureNote(noCode[:2], 10*time.Minute); !strings.HasSuffix(got, " and 1 other") { + t.Errorf("two notes: %q, want the most recent and 1 other", got) + } + long := []RepeatFailureNote{{Command: strings.Repeat("x", 500), Result: RepeatFailureResult{Failed: true}, Count: 5}} + if got := FormatRepeatFailureNote(long, 10*time.Minute); len([]rune(got)) > repeatFailureNoteCommandMax+80 { + t.Errorf("long command not cut: %d runes", len([]rune(got))) + } +} + +// TestRepeatFailureStatus_OffMeansOff is C-AC12/C14: with the switch off no +// row has a check state or note, also after it was on. +func TestRepeatFailureStatus_OffMeansOff(t *testing.T) { + m := rfStatusManager(t, "claude") + now := time.Now() + m.CollectRepeatFailures(now) + if st := rfStatus(m); st != (RepeatFailureStatus{}) { + t.Fatalf("no resolver: status = %+v, want none", st) + } + on := true + m.SetRepeatFailureEnabledResolver(func() bool { return on }) + rfWriteFailures(t, m, "false", 1, 5) + m.CollectRepeatFailures(now) + if st := rfStatus(m); st.Check == "" || st.Note == "" { + t.Fatalf("on: status = %+v, want a check state and a note", st) + } + on = false + m.CollectRepeatFailures(now.Add(time.Second)) + if st := rfStatus(m); st != (RepeatFailureStatus{}) { + t.Fatalf("switched off: status = %+v, want none", st) + } +} + +// TestRepeatFailureStatus_NotCoveredRowAndNote is C-AC9 and the C11 note: an +// agent on a backend that is not covered says so, and repeated failures that +// still reach Hive raise the note, which clears after the window. +func TestRepeatFailureStatus_NotCoveredRowAndNote(t *testing.T) { + m := rfStatusManager(t, "claude") + m.SetRepeatFailureEnabledResolver(func() bool { return true }) + m.SetRepeatFailureConfigResolver(func() *config.RepeatFailureConfig { return &config.RepeatFailureConfig{Enabled: true} }) + now := time.Now() + + m.CollectRepeatFailures(now) + st := rfStatus(m) + if st.Check != RepeatFailureCheckNotCovered || st.Check.Message() != "repeat-failure check not available for this backend" { + t.Fatalf("claude (not covered): check = %q (%q)", st.Check, st.Check.Message()) + } + if st.Note != "" { + t.Fatalf("no failures yet: note = %q", st.Note) + } + + rfWriteFailures(t, m, "ls /nonexistent", 2, 4) + m.CollectRepeatFailures(now.Add(time.Minute)) + if st := rfStatus(m); st.Note != "" { + t.Fatalf("4 failures raised a note: %q", st.Note) + } + rfWriteFailures(t, m, "ls /nonexistent", 2, 1) + m.CollectRepeatFailures(now.Add(2 * time.Minute)) + if st, want := rfStatus(m), "Repeating failure: `ls /nonexistent` failed 5 times in 10 min (exit 2)"; st.Note != want { + t.Fatalf("note = %q, want %q", st.Note, want) + } + m.mu.RLock() + paused := m.agents["scanner"].Paused + m.mu.RUnlock() + if paused { + t.Fatal("a note must never pause the agent (C10)") + } + m.CollectRepeatFailures(now.Add(13 * time.Minute)) + if st := rfStatus(m); st.Note != "" { + t.Fatalf("note must clear once the last failure is older than the window, got %q", st.Note) + } +} + +func TestRepeatFailureStatus_ExclusionsAndSuccessClear(t *testing.T) { + m := rfStatusManager(t, "claude") + m.SetRepeatFailureEnabledResolver(func() bool { return true }) + now := time.Now() + rfWriteFailures(t, m, "gh pr checks 123 --watch", 1, 10) + m.CollectRepeatFailures(now) + if st := rfStatus(m); st.Note != "" { + t.Fatalf("an excluded waiting loop raised %q", st.Note) + } + rfWriteFailures(t, m, "false", 1, 5) + m.CollectRepeatFailures(now.Add(time.Second)) + if st := rfStatus(m); st.Note == "" { + t.Fatal("5 failures raised no note") + } + rfWriteFailures(t, m, "false", 0, 1) + m.CollectRepeatFailures(now.Add(2 * time.Second)) + if st := rfStatus(m); st.Note != "" { + t.Fatalf("a success must clear the note, got %q", st.Note) + } +} + +// TestRepeatFailureSafeguard is C12/C-AC10, driven through +// ObserveRepeatFailureTool and evaluated as for a covered backend. +func TestRepeatFailureSafeguard(t *testing.T) { + const noData = 15 * time.Minute + kickAt := time.Now() + setup := func(t *testing.T) *Manager { + m := rfStatusManager(t, "claude") + m.SetRepeatFailureEnabledResolver(func() bool { return true }) + m.mu.Lock() + a := m.agents["scanner"] + a.State = StateRunning + k := kickAt + a.LastKick = &k + m.mu.Unlock() + return m + } + state := func(m *Manager, now time.Time) RepeatFailureCheckState { + m.mu.RLock() + defer m.mu.RUnlock() + m.repeatFailure.mu.Lock() + defer m.repeatFailure.mu.Unlock() + return repeatFailureCheckState(true, repeatFailureTurnKick(m.agents["scanner"]), m.repeatFailure.agents["scanner"], noData, now) + } + + t.Run("shell activity without records", func(t *testing.T) { + m := setup(t) + at := kickAt.Add(time.Minute) + m.ObserveRepeatFailureTool("scanner", RepeatFailureToolShell, false, at) + m.ObserveRepeatFailureTool("scanner", RepeatFailureToolShell, true, at) + if got := state(m, at.Add(14*time.Minute)); got != RepeatFailureCheckOn { + t.Errorf("before no_data_after_s: %q, want on", got) + } + if got := state(m, at.Add(noData)); got != RepeatFailureCheckNoData { + t.Errorf("after no_data_after_s: %q, want no_data", got) + } + if RepeatFailureCheckNoData.Message() != "repeat-failure check not receiving data" { + t.Errorf("no_data message = %q", RepeatFailureCheckNoData.Message()) + } + m.recordRepeatFailure(RepeatFailureRecord{Agent: "scanner", RawCommand: "true"}, at.Add(noData)) + if got := state(m, at.Add(noData+time.Minute)); got != RepeatFailureCheckOn { + t.Errorf("a record must reset the safeguard: %q", got) + } + }) + t.Run("reads and edits only", func(t *testing.T) { + m := setup(t) + at := kickAt.Add(time.Minute) + for _, kind := range []RepeatFailureToolKind{RepeatFailureToolFileRead, RepeatFailureToolFileEdit} { + m.ObserveRepeatFailureTool("scanner", kind, false, at) + m.ObserveRepeatFailureTool("scanner", kind, true, at) + } + if got := state(m, at.Add(noData+time.Minute)); got != RepeatFailureCheckOn { + t.Errorf("file reads and edits alone: %q, want on", got) + } + }) + t.Run("one long-running command", func(t *testing.T) { + m := setup(t) + at := kickAt.Add(time.Minute) + m.ObserveRepeatFailureTool("scanner", RepeatFailureToolShell, false, at) + if got := state(m, at.Add(20*time.Minute)); got != RepeatFailureCheckOn { + t.Errorf("a still-running command: %q, want on", got) + } + }) + t.Run("turn ended", func(t *testing.T) { + m := setup(t) + at := kickAt.Add(time.Minute) + m.ObserveRepeatFailureTool("scanner", RepeatFailureToolShell, false, at) + m.ObserveRepeatFailureTool("scanner", RepeatFailureToolShell, true, at) + m.mu.Lock() + m.agents["scanner"].KickOutcome = KickOutcome{Kind: KickOutcomeEnded, KickAt: kickAt, At: at} + m.mu.Unlock() + if got := state(m, at.Add(noData+time.Minute)); got != RepeatFailureCheckOn { + t.Errorf("no turn in progress: %q, want on", got) + } + }) + t.Run("not covered wins", func(t *testing.T) { + if got := repeatFailureCheckState(false, &kickAt, &repeatFailureAgentState{kick: kickAt, pendingSince: kickAt}, noData, kickAt.Add(time.Hour)); got != RepeatFailureCheckNotCovered { + t.Errorf("not covered: %q", got) + } + }) +} diff --git a/src/crux/api/client/client.go b/src/crux/api/client/client.go index 99dc73ac2a..5a4dc39fba 100644 --- a/src/crux/api/client/client.go +++ b/src/crux/api/client/client.go @@ -873,6 +873,7 @@ func (r *Runtime) StartAgentTokenRefresh(context.Context) {} func (r *Runtime) StartCredentialWatchdog(context.Context) {} func (r *Runtime) StartCopilotSessionRefresh(context.Context) {} func (r *Runtime) StartTranscriptActivityRefresh(context.Context) {} +func (r *Runtime) StartRepeatFailureMonitor(context.Context) {} // ---- Authorizer (fail closed) ---- diff --git a/src/crux/api/client/client_test.go b/src/crux/api/client/client_test.go index d6be6497be..915b864b37 100644 --- a/src/crux/api/client/client_test.go +++ b/src/crux/api/client/client_test.go @@ -39,6 +39,7 @@ var localOnly = map[string]bool{ "StartCredentialWatchdog": true, "StartCopilotSessionRefresh": true, "StartTranscriptActivityRefresh": true, + "StartRepeatFailureMonitor": true, "SetKickObserver": true, } @@ -371,6 +372,7 @@ func TestEveryMethodReachesTheRuntime(t *testing.T) { rt.StartCredentialWatchdog(ctx) rt.StartCopilotSessionRefresh(ctx) rt.StartTranscriptActivityRefresh(ctx) + rt.StartRepeatFailureMonitor(ctx) // Authorizer must("AuthorizePROpen", rt.AuthorizePROpen("alpha", 1000)) @@ -399,7 +401,7 @@ func TestEveryMethodReachesTheRuntime(t *testing.T) { if len(missing) > 0 { t.Errorf("methods never reached the runtime: %s", strings.Join(missing, ", ")) } - for _, m := range []string{"StartAgentTokenRefresh", "StartCredentialWatchdog", "StartCopilotSessionRefresh", "StartTranscriptActivityRefresh"} { + for _, m := range []string{"StartAgentTokenRefresh", "StartCredentialWatchdog", "StartCopilotSessionRefresh", "StartTranscriptActivityRefresh", "StartRepeatFailureMonitor"} { if seen[m] { t.Errorf("%s crossed the wire; Background loops are server-side", m) } diff --git a/src/crux/api/runtime.go b/src/crux/api/runtime.go index 50442d46f5..e72b54ceb8 100644 --- a/src/crux/api/runtime.go +++ b/src/crux/api/runtime.go @@ -733,6 +733,7 @@ var RuntimeRoutes = map[string]Route{ "StartCredentialWatchdog": {"GET", PathHealth}, "StartCopilotSessionRefresh": {"GET", PathHealth}, "StartTranscriptActivityRefresh": {"GET", PathHealth}, + "StartRepeatFailureMonitor": {"GET", PathHealth}, // Authorizer "AuthorizePROpen": {"POST", PathAuthorize}, "AuthorizePushBranch": {"POST", PathAuthorize}, @@ -845,4 +846,5 @@ var HostSetters = map[string]HostSetter{ // Repeat-failure BASH_ENV route (v6, #14574): the enabled switch is read // in-process until it rides the hive callback policy. "SetRepeatFailureEnabledResolver": {HostSetterInProcessOnly, "repeat-failure capture switch is in-process only; sidecar follow-up"}, + "SetRepeatFailureConfigResolver": {HostSetterInProcessOnly, "repeat-failure monitor config is in-process only; sidecar follow-up"}, } diff --git a/src/crux/cruxtest/recorder.go b/src/crux/cruxtest/recorder.go index a52252ced7..1cbb36643f 100644 --- a/src/crux/cruxtest/recorder.go +++ b/src/crux/cruxtest/recorder.go @@ -895,6 +895,10 @@ func (r *Recorder) StartTranscriptActivityRefresh(ctx context.Context) { r.background(ctx, "StartTranscriptActivityRefresh") } +func (r *Recorder) StartRepeatFailureMonitor(ctx context.Context) { + r.background(ctx, "StartRepeatFailureMonitor") +} + // Background lists the Background loops that have been started, in order; // safe to call while they run. func (r *Recorder) Background() []string { @@ -978,6 +982,9 @@ func (r *Recorder) SetAdvisorRolesResolver(fn func() map[string]config.ModelRole func (r *Recorder) SetRepeatFailureEnabledResolver(fn func() bool) { r.record("SetRepeatFailureEnabledResolver", fn != nil) } +func (r *Recorder) SetRepeatFailureConfigResolver(fn func() *config.RepeatFailureConfig) { + r.record("SetRepeatFailureConfigResolver", fn != nil) +} func (r *Recorder) SetRepoAutoMergeEnabledResolver(fn func(repo string) bool) { r.record("SetRepoAutoMergeEnabledResolver", fn != nil) } diff --git a/src/crux/host_wiring.go b/src/crux/host_wiring.go index 246b490e51..ff278ed0ce 100644 --- a/src/crux/host_wiring.go +++ b/src/crux/host_wiring.go @@ -50,6 +50,10 @@ type Host interface { // SetRepeatFailureEnabledResolver feeds the neutral BASH_ENV route for // repeat-failure capture (#14574): whether the monitor is on at launch. SetRepeatFailureEnabledResolver(fn func() bool) + // SetRepeatFailureConfigResolver feeds the repeat-failure monitor (#14558) + // the live repeat_failure block so threshold and window changes apply on + // the next tick. + SetRepeatFailureConfigResolver(fn func() *config.RepeatFailureConfig) SetKickLogSourceResolver(fn func() string) SetInferenceCallbacks(setRoute func(agentName, backend, model string), clearRoute func(agentName string)) SetSandboxConfig(cfg config.AgentSandboxConfig) diff --git a/src/crux/process.go b/src/crux/process.go index b1ea412bba..be16a448a7 100644 --- a/src/crux/process.go +++ b/src/crux/process.go @@ -70,6 +70,7 @@ type Process struct { TurnLoss TurnLoss ActionNudges int KickOutcome KickOutcome + RepeatFailure RepeatFailureStatus StartFailureClass string StartFailureReason string StartFailureCount int diff --git a/src/crux/repeat_failure.go b/src/crux/repeat_failure.go new file mode 100644 index 0000000000..c3ce229317 --- /dev/null +++ b/src/crux/repeat_failure.go @@ -0,0 +1,42 @@ +package crux + +// RepeatFailureCheckState is an agent's repeat-failure check state. Empty +// means the check is off, and the dashboard shows nothing. +type RepeatFailureCheckState string + +// Repeat-failure check states. +const ( + RepeatFailureCheckOn RepeatFailureCheckState = "on" + RepeatFailureCheckNotCovered RepeatFailureCheckState = "not_covered" + RepeatFailureCheckNoData RepeatFailureCheckState = "no_data" +) + +// Row texts for the check states. +const ( + RepeatFailureCheckOnMessage = "repeat-failure check on" + RepeatFailureCheckNotCoveredMessage = "repeat-failure check not available for this backend" + RepeatFailureCheckNoDataMessage = "repeat-failure check not receiving data" +) + +// Message returns the row text for the state, "" when the check is off. +func (s RepeatFailureCheckState) Message() string { + switch s { + case RepeatFailureCheckOn: + return RepeatFailureCheckOnMessage + case RepeatFailureCheckNotCovered: + return RepeatFailureCheckNotCoveredMessage + case RepeatFailureCheckNoData: + return RepeatFailureCheckNoDataMessage + } + return "" +} + +// RepeatFailureStatus is what an agent's dashboard row shows for the +// repeat-failure check. The zero value (check off) shows nothing. +type RepeatFailureStatus struct { + Check RepeatFailureCheckState + // Note is "Repeating failure: `` failed N times in M min (exit + // S)", with " and K others" when several commands repeat. Built from + // scrubbed, normalized commands only. Empty when nothing repeats. + Note string +} diff --git a/src/crux/repeat_failure_test.go b/src/crux/repeat_failure_test.go new file mode 100644 index 0000000000..6d195e5b5a --- /dev/null +++ b/src/crux/repeat_failure_test.go @@ -0,0 +1,21 @@ +package crux + +import "testing" + +func TestRepeatFailureCheckStateMessage(t *testing.T) { + cases := map[RepeatFailureCheckState]string{ + RepeatFailureCheckOn: RepeatFailureCheckOnMessage, + RepeatFailureCheckNotCovered: RepeatFailureCheckNotCoveredMessage, + RepeatFailureCheckNoData: RepeatFailureCheckNoDataMessage, + "": "", + "bogus": "", + } + for state, want := range cases { + if got := state.Message(); got != want { + t.Errorf("RepeatFailureCheckState(%q).Message() = %q, want %q", state, got, want) + } + } + if (RepeatFailureStatus{}).Check.Message() != "" { + t.Error("zero RepeatFailureStatus must render nothing") + } +} diff --git a/src/crux/runtime.go b/src/crux/runtime.go index 8d326051e0..c039bd82a9 100644 --- a/src/crux/runtime.go +++ b/src/crux/runtime.go @@ -330,6 +330,7 @@ type Background interface { StartCredentialWatchdog(ctx context.Context) StartCopilotSessionRefresh(ctx context.Context) StartTranscriptActivityRefresh(ctx context.Context) + StartRepeatFailureMonitor(ctx context.Context) } // Authorizer is the per-request gate the host's relay watchers consult diff --git a/src/crux/serve/families.go b/src/crux/serve/families.go index 4312ca3e02..f8a9a5fe8c 100644 --- a/src/crux/serve/families.go +++ b/src/crux/serve/families.go @@ -966,7 +966,7 @@ func (s *Server) handleWatchdogConditions(w http.ResponseWriter, r *http.Request // backgroundLoops are the crux.Background methods, in the order Serve starts // them and Health reports them. -var backgroundLoops = []string{"StartAgentTokenRefresh", "StartCredentialWatchdog", "StartCopilotSessionRefresh", "StartTranscriptActivityRefresh"} +var backgroundLoops = []string{"StartAgentTokenRefresh", "StartCredentialWatchdog", "StartCopilotSessionRefresh", "StartTranscriptActivityRefresh", "StartRepeatFailureMonitor"} // publishData publishes an event whose Data is v's JSON. func (s *Server) publishData(typ, agent string, v any) { @@ -1081,6 +1081,7 @@ func (s *Server) startBackground(ctx context.Context) { go rt.StartCredentialWatchdog(ctx) go rt.StartCopilotSessionRefresh(ctx) go rt.StartTranscriptActivityRefresh(ctx) + go rt.StartRepeatFailureMonitor(ctx) s.mu.Lock() s.background = append([]string(nil), backgroundLoops...) s.mu.Unlock() diff --git a/src/crux/serve/families_test.go b/src/crux/serve/families_test.go index 0312f8782b..b9c4d77172 100644 --- a/src/crux/serve/families_test.go +++ b/src/crux/serve/families_test.go @@ -774,7 +774,7 @@ func TestHostObserversReachTheEventStream(t *testing.T) { return resp } - waitUntil(t, "background loops started", func() bool { return len(rt.Background()) == 4 }) + waitUntil(t, "background loops started", func() bool { return len(rt.Background()) == len(backgroundLoops) }) var h api.Health resp := get(api.PathHealth) if err := json.NewDecoder(resp.Body).Decode(&h); err != nil { diff --git a/src/docs/api-reference.md b/src/docs/api-reference.md index 59c3f0feba..4ab4f6042c 100644 --- a/src/docs/api-reference.md +++ b/src/docs/api-reference.md @@ -13,17 +13,17 @@ History point limits, timestamp cutoffs, incremental cost reads and ETag/304 beh | `GET` | `/api/version` | Dashboard auth/session | Build/version metadata; includes `upgradeMarker` (`target`, `current`, `attempts`, `maxAttempts`, `failed`, `requestedAt`, `lastError`) while a self-upgrade is in flight or has failed ([#6765](https://github.com/hivecommons/hive/issues/6765)), the hub-delivered `upgradePolicy` (`hub_managed`, `spoke_managed`, `schedule`, `schedule_hour`, `schedule_timezone`, `schedule_weekday`, `paused`, `branch`, `channel`, `target_sha`, `target_resolved`, `armed_target`, and `next_update_at` — the hub's RFC3339 ETA for the next promotion into the `stable` channel, omitted when unknown ([#10256](https://github.com/hivecommons/hive/issues/10256))) when available, and an `autoUpdate` object (`enabled`, `state` — one of `disabled`/`up_to_date`/`behind`/`retrying`/`failed`/`unknown` — `healthy`, `period`, `targetBranch`, `targetCommit`, `currentCommit`, `commitsBehind`, `lastAttemptAt`, `lastError`, `detail`, and `nextUpdateAt` — the hub's RFC3339 ETA for the next update, omitted when unknown, disabled or paused ([#10257](https://github.com/hivecommons/hive/issues/10257))) that never reports a failed or unknown update as healthy ([#6962](https://github.com/hivecommons/hive/issues/6962), [#6963](https://github.com/hivecommons/hive/issues/6963)); also `hub_build` (the hub's short SHA) and, once a stdio `hive-admin-mcp` client has called `/api/version`, `/api/status` or `/api/status/summary` with its `hive-admin-mcp/+` User-Agent, `last_seen_stdio_build` and `last_seen_stdio_at` (RFC3339) so the dashboard can flag build skew ([#12318](https://github.com/hivecommons/hive/issues/12318)) | `pkg/dashboard/api.go:43` | | `GET` | `/api/version/release-notes` | Dashboard auth/session | Changelog gap between two revisions: `?from=&to=` returns `{from:{sha,version?}, to:{sha,ref,version?}, sections:[{version,date,categories:{added,changed,fixed,security,deprecated}}], unreleased?, truncated, source, error?}` — the `CHANGELOG.md` release sections present at `to` and absent at `from`, newest first, plus (when `to` is not a release tag) the `changelog.d/` fragments new at `to` under an `Unreleased (in this build)` section. Fetch failures return `source: "unavailable"` with `error` rather than an error status; results are cached per (from, to) for an hour ([#11065](https://github.com/hivecommons/hive/issues/11065)) | `pkg/dashboard/api.go:44` | | `POST` | `/api/release-channel` | Owner only | Hosted spoke self-service release-channel selector; relays `stable`/`candidate`/`edge` to the hub's existing switch-branch endpoint with the spoke dashboard-token proof, and reports the requested channel as pending until the Deployment image lands | `pkg/dashboard/api.go:71` | -| `GET` | `/api/health` | Public | Basic health probe | `pkg/dashboard/server.go:1622` | -| `GET` | `/api/health/deep` | Public | Deep health probe | `pkg/dashboard/server.go:1623` | -| `GET` | `/api/livez` | Public | Kubernetes liveness probe | `pkg/dashboard/server.go:1624` | -| `GET` | `/metrics` | Registered only when `HIVE_METRICS_ENABLED`; requires `Authorization: ****** (403 if the token is unset) | Prometheus metrics | `pkg/dashboard/server.go:1630` | -| `GET` | `/api/status` | Dashboard auth/session | Dashboard aggregate status; `labelPresets[]` (`{key,label,description,labels[],states[]}`) lists the built-in label-filter presets for the Projects cards, led by `needs-human` whose `labels[]` are the effective hard-suppress labels plus `needs-signal`, the hold labels, `stage/071-needs-author` and the reporter-trust awaiting label (#12172); accepts `?fields=a,b` for top-level selection and `?omit=repos` to drop heavy sections, and honors `Accept-Encoding: gzip`; `agents[]` includes busy-pane visibility fields (`kicksUndeliverable`, `busySince`, `lastTranscriptActivity`, `condition`, `conditionMessage`) when a Working pane cannot accept kicks; `sla` (fleet) and `repos[].sla` (per repo) carry the Delivery SLA block derived from the lifecycle timeline with no extra GitHub calls: `windows.{7d,30d}.{open_to_triage,triage_to_start,start_to_pr,pr_to_merge,open_to_merge}` as `{p50_h,p95_h,n}` (hours, nearest-rank), `history[]` of up to 30 daily buckets `{day,start_to_pr_p50,pr_to_merge_p50,open_to_merge_p50,n_started,n_pr,n_merged}` persisted under `/data/sla-history.json`, `stale_in_progress[]` (`{number,repo,started_at,age_h,agent}` — issues carrying `hive/in-progress` longer than the agent claim TTL with no linked PR), `stale_count`, `merged_7d`, `claim_ttl_s` and, on the fleet block, the effective `targets` (`start_to_pr_h`, `pr_to_merge_h`, `open_to_merge_h`); `staleSince` and `repos[].countsStale` / `countsStaleSince` mark totals served from last-good figures — the previous enumeration (#13382) or the ForgeState cache's last-good counts (#13376); every your-turn issue and PR row carries the critical path (#13646): `waits_on[]` (`{kind,ref}`, kind one of `pr`, `issue`, `external`, `base-check` (ref = the check red on the base branch), `approval`, `merge`; ref `#N` in the same repository or `owner/repo#N`), `unblocks` (transitive count of your-turn items waiting on it, cycle-guarded), `may_conflict_with[]` (PR numbers of the same repository sharing a closing issue or a known changed file; never an edge), `action` (`merge`, `release-hold`, `rebase`, `rerun` or `approve`), `action_ref`, `root_cause` and `blocker` (the root cause it collapses under); `repos[].yourTurn` has `issues`, `prs` (your turn, `CLOSE` excluded), `root_issues`, `root_prs`, `root_cause_count`, `order[]` (`{kind,repo,number}` by unblocks then age), `groups[]` (`{blocker,blocker_title,dependents[]}`) and `main_red` (`{check,since,fix,fix_your_turn}`); the top-level `yourTurn` sums the counts | `pkg/dashboard/server.go:1635` | -| `GET` | `/api/status/summary` | Dashboard auth/session | Compact agent/governor/budget status summary for pollers, with the hive-wide `yourTurn` root-cause counts the phone widget reads (#13646); honors `Accept-Encoding: gzip` | `pkg/dashboard/server.go:1636` | -| `GET` | `/api/overview/issues.csv` | Dashboard auth/session | Cached Overview issue-band export as browser-compatible CSV; filters `band`, `repo`/`repos`, `stale`, and `held` are ANDed. | `pkg/dashboard/server.go:1639` | -| `GET` | `/api/overview/issues.json` | Dashboard auth/session | Cached Overview issue-band export as typed JSON with `bands[]` metadata and row fields matching the CSV columns. | `pkg/dashboard/server.go:1640` | -| `GET` | `/api/overview/prs.csv` | Dashboard auth/session | Cached Overview PR-band export as browser-compatible CSV; filters `band`, `repo`/`repos`, `stale`, and `held` are ANDed. | `pkg/dashboard/server.go:1641` | -| `GET` | `/api/overview/prs.json` | Dashboard auth/session | Cached Overview PR-band export as typed JSON with `bands[]` metadata and row fields matching the CSV columns. | `pkg/dashboard/server.go:1642` | -| `GET` | `/api/events` | Dashboard auth/session | Server-sent event stream | `pkg/dashboard/server.go:1643` | +| `GET` | `/api/health` | Public | Basic health probe | `pkg/dashboard/server.go:1630` | +| `GET` | `/api/health/deep` | Public | Deep health probe | `pkg/dashboard/server.go:1631` | +| `GET` | `/api/livez` | Public | Kubernetes liveness probe | `pkg/dashboard/server.go:1632` | +| `GET` | `/metrics` | Registered only when `HIVE_METRICS_ENABLED`; requires `Authorization: ****** (403 if the token is unset) | Prometheus metrics | `pkg/dashboard/server.go:1638` | +| `GET` | `/api/status` | Dashboard auth/session | Dashboard aggregate status; `labelPresets[]` (`{key,label,description,labels[],states[]}`) lists the built-in label-filter presets for the Projects cards, led by `needs-human` whose `labels[]` are the effective hard-suppress labels plus `needs-signal`, the hold labels, `stage/071-needs-author` and the reporter-trust awaiting label (#12172); accepts `?fields=a,b` for top-level selection and `?omit=repos` to drop heavy sections, and honors `Accept-Encoding: gzip`; `agents[]` includes busy-pane visibility fields (`kicksUndeliverable`, `busySince`, `lastTranscriptActivity`, `condition`, `conditionMessage`) when a Working pane cannot accept kicks; `sla` (fleet) and `repos[].sla` (per repo) carry the Delivery SLA block derived from the lifecycle timeline with no extra GitHub calls: `windows.{7d,30d}.{open_to_triage,triage_to_start,start_to_pr,pr_to_merge,open_to_merge}` as `{p50_h,p95_h,n}` (hours, nearest-rank), `history[]` of up to 30 daily buckets `{day,start_to_pr_p50,pr_to_merge_p50,open_to_merge_p50,n_started,n_pr,n_merged}` persisted under `/data/sla-history.json`, `stale_in_progress[]` (`{number,repo,started_at,age_h,agent}` — issues carrying `hive/in-progress` longer than the agent claim TTL with no linked PR), `stale_count`, `merged_7d`, `claim_ttl_s` and, on the fleet block, the effective `targets` (`start_to_pr_h`, `pr_to_merge_h`, `open_to_merge_h`); `staleSince` and `repos[].countsStale` / `countsStaleSince` mark totals served from last-good figures — the previous enumeration (#13382) or the ForgeState cache's last-good counts (#13376); every your-turn issue and PR row carries the critical path (#13646): `waits_on[]` (`{kind,ref}`, kind one of `pr`, `issue`, `external`, `base-check` (ref = the check red on the base branch), `approval`, `merge`; ref `#N` in the same repository or `owner/repo#N`), `unblocks` (transitive count of your-turn items waiting on it, cycle-guarded), `may_conflict_with[]` (PR numbers of the same repository sharing a closing issue or a known changed file; never an edge), `action` (`merge`, `release-hold`, `rebase`, `rerun` or `approve`), `action_ref`, `root_cause` and `blocker` (the root cause it collapses under); `repos[].yourTurn` has `issues`, `prs` (your turn, `CLOSE` excluded), `root_issues`, `root_prs`, `root_cause_count`, `order[]` (`{kind,repo,number}` by unblocks then age), `groups[]` (`{blocker,blocker_title,dependents[]}`) and `main_red` (`{check,since,fix,fix_your_turn}`); the top-level `yourTurn` sums the counts | `pkg/dashboard/server.go:1643` | +| `GET` | `/api/status/summary` | Dashboard auth/session | Compact agent/governor/budget status summary for pollers, with the hive-wide `yourTurn` root-cause counts the phone widget reads (#13646); honors `Accept-Encoding: gzip` | `pkg/dashboard/server.go:1644` | +| `GET` | `/api/overview/issues.csv` | Dashboard auth/session | Cached Overview issue-band export as browser-compatible CSV; filters `band`, `repo`/`repos`, `stale`, and `held` are ANDed. | `pkg/dashboard/server.go:1647` | +| `GET` | `/api/overview/issues.json` | Dashboard auth/session | Cached Overview issue-band export as typed JSON with `bands[]` metadata and row fields matching the CSV columns. | `pkg/dashboard/server.go:1648` | +| `GET` | `/api/overview/prs.csv` | Dashboard auth/session | Cached Overview PR-band export as browser-compatible CSV; filters `band`, `repo`/`repos`, `stale`, and `held` are ANDed. | `pkg/dashboard/server.go:1649` | +| `GET` | `/api/overview/prs.json` | Dashboard auth/session | Cached Overview PR-band export as typed JSON with `bands[]` metadata and row fields matching the CSV columns. | `pkg/dashboard/server.go:1650` | +| `GET` | `/api/events` | Dashboard auth/session | Server-sent event stream | `pkg/dashboard/server.go:1651` | | `GET` | `/api/swarm` | Dashboard auth/session | Current swarm status: display name, duration, active repo, expiry, persisted prep metrics when available, and idle-unlock fields. | `pkg/dashboard/swarm.go:310` | | `POST` | `/api/swarm` | Owner only | Start a 24h repo swarm for a configured repo, returning 409 when another swarm is active and 423 when a previous swarm keeps the next swarm locked until enough agents are idle unless body `{"repo":"owner/name","force":true}` is used; saves the active record before repo prep, persists prep metrics, and announces the start to Discord when configured. | `pkg/dashboard/swarm.go:311` | | `DELETE` | `/api/swarm` | Owner only | End the active swarm, score issues closed and PRs merged in the window, append it to history, and announce the result to Discord when configured. | `pkg/dashboard/swarm.go:312` | @@ -66,7 +66,7 @@ History point limits, timestamp cutoffs, incremental cost reads and ETag/304 beh | Method | Path | Auth | Purpose | Source | |---|---|---|---|---| | `GET` | `/api/style` | Public | Sanitized custom dashboard CSS | `pkg/dashboard/api.go:45` | -| `GET` | `/branding/custom.css` | Dashboard auth/session | Operator branding stylesheet override, read per request (see [branding](branding.md)) | `pkg/dashboard/server.go:1704` | +| `GET` | `/branding/custom.css` | Dashboard auth/session | Operator branding stylesheet override, read per request (see [branding](branding.md)) | `pkg/dashboard/server.go:1712` | | `GET` | `/api/snapshot/frame-ancestors` | Public | Snapshot framing allowlist | `pkg/dashboard/api.go:85` | | `GET` | `/api/snapshot` | Public | Snapshot data | `pkg/dashboard/api.go:86` | | `GET` | `/snapshot` | Public | Public read-only snapshot page | `pkg/dashboard/api.go:87` | @@ -94,9 +94,9 @@ History point limits, timestamp cutoffs, incremental cost reads and ETag/304 beh | `GET` | `/api/openrouter/models` | Dashboard auth/session | Open Router Models | `pkg/dashboard/openrouter.go:47` | | `GET` | `/api/openrouter/credit` | Dashboard auth/session | Open Router Credit | `pkg/dashboard/openrouter.go:48` | | `GET` | `/openrouter/callback` | Public | Open Router Callback | `pkg/dashboard/openrouter.go:49` | -| `POST` | `/api/github-app/recheck` | Dashboard auth/session | GitHub App Recheck | `pkg/dashboard/server.go:1644` | -| `POST` | `/api/github-app/install-clicked` | Dashboard auth/session | GitHub App Install Clicked | `pkg/dashboard/server.go:1645` | -| `GET` | `/gh-setup` | Public | GitHub App Setup Callback | `pkg/dashboard/server.go:1646` | +| `POST` | `/api/github-app/recheck` | Dashboard auth/session | GitHub App Recheck | `pkg/dashboard/server.go:1652` | +| `POST` | `/api/github-app/install-clicked` | Dashboard auth/session | GitHub App Install Clicked | `pkg/dashboard/server.go:1653` | +| `GET` | `/gh-setup` | Public | GitHub App Setup Callback | `pkg/dashboard/server.go:1654` | ## Configuration @@ -629,7 +629,7 @@ always resolved server-side from the validated token. | `POST` | `/api/hives/{id}/heartbeat` | Dashboard auth/session | Hives Heartbeat | `pkg/dashboard/api_contribute.go:287` | | `DELETE` | `/api/hives/{id}` | Owner only | Hives Delete | `pkg/dashboard/api_contribute.go:288` | | `POST` | `/api/hives/onboard` | Dashboard auth/session | Hives Onboard | `pkg/dashboard/api_contribute.go:289` | -| `GET` | `/sso` | Public | SSO | `pkg/dashboard/server.go:1651` | +| `GET` | `/sso` | Public | SSO | `pkg/dashboard/server.go:1659` | ## Hub SaaS diff --git a/src/docs/backend-support-tiers.md b/src/docs/backend-support-tiers.md index 8ecfc11ef2..4e07e37976 100644 --- a/src/docs/backend-support-tiers.md +++ b/src/docs/backend-support-tiers.md @@ -547,6 +547,54 @@ agent spend, and the `advisor_records` admin MCP tool. A backend moves into a "Yes" row only with an adapter that answers the hook in the backend's own format and a test that the launch carries it. +## Repeated-failure check + +The repeated-failure check ([#14550](https://github.com/hivecommons/hive/issues/14550)) +records each shell command an agent runs and notes on the agent's dashboard row +when the same command keeps failing the same way. It never pauses the agent and +is off by default (`repeat_failure.enabled`, see +[Agent-runtime blocks](operator-reference.md#agent-runtime-blocks-configagentruntimeconfig)). +Hive needs a way to see each backend's commands, so support is per backend. + +A backend is **covered** only when a committed, re-runnable harness made its +real CLI produce a correct command record, and the evidence includes the CLI's +own `--version` output. Hand-written test data does not count. Every other +backend is **not covered**: with the check on, its agent rows say +"repeat-failure check not available for this backend". The shell-activity +signal is what the "not receiving data" safeguard watches for a covered +backend, or "safeguard unavailable" where the backend has none. + +This table is generated from the code table in +`src/pkg/agent/repeat_failure_coverage.go`; +`TestRepeatFailureCoverageDocsMatchCode` fails when a backend is missing or a +row here differs from the code. + +| Backend | Route | Status | Evidence | Shell-activity signal | Follow-up issue | +| --- | --- | --- | --- | --- | --- | +| `claude` | BASH_ENV (neutral), unwraps `eval '…'` | Not covered: BASH_ENV route installed while the check is on, but no committed real-run harness evidence yet (C4) | None | n/a | [#14559](https://github.com/hivecommons/hive/issues/14559) | +| `copilot` | none yet; planned: `events.jsonl` `tool.execution_complete` | Not covered: one long-lived Bash session per shell id; commands never reach the BASH_ENV route and the backend-specific route is not built yet | None | n/a | [#14559](https://github.com/hivecommons/hive/issues/14559) | +| `goose` | BASH_ENV (neutral) | Not covered: BASH_ENV route installed while the check is on, but no committed real-run harness evidence yet (C4) | None | n/a | [#14559](https://github.com/hivecommons/hive/issues/14559) | +| `codex` | BASH_ENV (neutral), one record across `exec /bin/bash -c` | Not covered: BASH_ENV route installed while the check is on, but no committed real-run harness evidence yet (C4) | None | n/a | [#14559](https://github.com/hivecommons/hive/issues/14559) | +| `pi` | BASH_ENV (neutral) | Not covered: BASH_ENV route installed while the check is on, but no committed real-run harness evidence yet (C4) | None | n/a | [#14559](https://github.com/hivecommons/hive/issues/14559) | +| `bob` | BASH_ENV (neutral) with `SHELL=/bin/bash` (documented shell change: dash → bash) | Not covered: BASH_ENV route installed while the check is on, but no committed real-run harness evidence yet (C4) | None | n/a | [#14559](https://github.com/hivecommons/hive/issues/14559) | +| `aider` | none yet; planned: guarded `~/.bashrc` line in Hive-managed homes | Not covered: interactive `$SHELL -i -c` ignores BASH_ENV; commands never reach the BASH_ENV route and the backend-specific route is not built yet | None | n/a | [#14559](https://github.com/hivecommons/hive/issues/14559) | +| `gemini` | BASH_ENV (neutral), unwraps the `{ …; }; __code=$?` wrapper | Not covered: BASH_ENV route installed while the check is on, but no committed real-run harness evidence yet (C4) | None | n/a | [#14559](https://github.com/hivecommons/hive/issues/14559) | +| `agy` | none yet; planned: `agy-turn` `run_command` steps | Not covered: closed binary, likely a long-lived terminal; commands never reach the BASH_ENV route and the backend-specific route is not built yet | None | n/a | [#14559](https://github.com/hivecommons/hive/issues/14559) | +| `opencode` | BASH_ENV (neutral) with `SHELL=/bin/bash` | Not covered: BASH_ENV route installed while the check is on, but no committed real-run harness evidence yet (C4) | None | n/a | [#14559](https://github.com/hivecommons/hive/issues/14559) | +| `kilo` | BASH_ENV (neutral) with `SHELL=/bin/bash` | Not covered: BASH_ENV route installed while the check is on, but no committed real-run harness evidence yet (C4) | None | n/a | [#14559](https://github.com/hivecommons/hive/issues/14559) | +| `muse` | none yet; planned: `PostToolUse` / `PostToolUseFailure` hooks | Not covered: sandboxed shell tools in a closed binary; commands never reach the BASH_ENV route and the backend-specific route is not built yet | None | n/a | [#14559](https://github.com/hivecommons/hive/issues/14559) | +| `omp` | none yet; planned: `tool_result` extension event | Not covered: embedded shell that strips BASH_ENV; commands never reach the BASH_ENV route and the backend-specific route is not built yet | None | n/a | [#14559](https://github.com/hivecommons/hive/issues/14559) | +| `openhands` | none yet; planned: `PostToolUse` hook or saved events | Not covered: one interactive bash kept open in tmux; commands never reach the BASH_ENV route and the backend-specific route is not built yet | None | n/a | [#14559](https://github.com/hivecommons/hive/issues/14559) | +| `omnigent` | none yet; planned: tool-call policy checkpoint (`ALLOW`/`DENY` verdicts) or saved session events | Not covered: meta-harness driving Claude Code, Codex or Pi through its own policy engine; commands never reach the BASH_ENV route and the backend-specific route is not built yet | None | n/a | [#14559](https://github.com/hivecommons/hive/issues/14559) | +| `vllm` | follows `claude` (drives Claude Code) | Not covered: follows `claude`, which is not covered | None | n/a | [#14559](https://github.com/hivecommons/hive/issues/14559) | +| `llm-d` | follows `claude` (drives Claude Code) | Not covered: follows `claude`, which is not covered | None | n/a | [#14559](https://github.com/hivecommons/hive/issues/14559) | +| `litellm` | follows `claude` (drives Claude Code) | Not covered: follows `claude`, which is not covered | None | n/a | [#14559](https://github.com/hivecommons/hive/issues/14559) | +| `watsonx` | follows `claude` (drives Claude Code) | Not covered: follows `claude`, which is not covered | None | n/a | [#14559](https://github.com/hivecommons/hive/issues/14559) | + +The BASH_ENV route is described in +[the shell route](agent-configuration.md#repeat-failure-check-the-shell-route-repeat_failure). +A backend moves to "Covered" only together with its committed harness evidence. + ## Deprecation path A backend can be demoted or removed when its upstream CLI breaks Hive's contract diff --git a/src/docs/crux-serve.md b/src/docs/crux-serve.md index bae2905f95..3dc4a1f9c7 100644 --- a/src/docs/crux-serve.md +++ b/src/docs/crux-serve.md @@ -107,7 +107,7 @@ it or are the CLI routes above under another method. | | `POST /v1/agents/{name}/tokens/refresh`, `POST /v1/agents/{name}/nudge` ⚑ | `RefreshAgentTokenFor`, `SendGitHubAPIBudgetNudge` | | | | `GET /v1/backends/{backend}/auth`, `POST /v1/audit`, `GET /v1/trajectory` ⚑ | `BackendAuthAvailable`, `RecordAudit`, `TrajectoryAgents` | | | Restorer | `POST /v1/agents/{name}/seed` ⚑ | `SeedPauseState`, `SeedRestartTelemetry`, `SeedTurnLoss`, `SeedLastKick`, `SeedKickHistory` | one body, each field optional | -| Background | `GET /v1/health` | `StartAgentTokenRefresh`, `StartCredentialWatchdog`, `StartCopilotSessionRefresh`, `StartTranscriptActivityRefresh` | the loops run server-side; the client's calls are no-ops | +| Background | `GET /v1/health` | `StartAgentTokenRefresh`, `StartCredentialWatchdog`, `StartCopilotSessionRefresh`, `StartTranscriptActivityRefresh`, `StartRepeatFailureMonitor` | the loops run server-side; the client's calls are no-ops | | Authorizer | `POST /v1/authorize` ⚑ | `AuthorizePROpen`, `AuthorizePushBranch`, `AuthorizeIssueOpen`, `AuthorizeMerge`, `AuthorizeReviewRequest` | synchronous; `{"action","agent","file_uid","kind","repo"}`; `403 forbidden` carries the refusal, any transport error is a refusal (fail closed) | | Watchdog | `GET /v1/watchdog/agents`, `GET /v1/agents/{name}/watchdog`, `PUT /v1/agents/{name}/watchdog/conditions` ⚑ | `watchdog.Source` (`AgentNames`, `Observe`, `LastProduction`, `SetConditions`) | `501 not_supported` when the runtime has no watchdog source; `crux/runtime.NewWatchdogFleet` builds the fleet over these in sidecar mode | diff --git a/src/docs/dashboard-sections.md b/src/docs/dashboard-sections.md index 60db6bd968..8449c2b593 100644 --- a/src/docs/dashboard-sections.md +++ b/src/docs/dashboard-sections.md @@ -620,6 +620,15 @@ More about the line: - **Updates.** The line updates with each worker update, every 10 seconds by default, so a change shows within about 15 seconds. - **Privacy.** Titles come only from the issue and pull request lists the dashboard already shows you. If it shows you no repositories' issues, the line shows no item numbers either. +**The repeat-failure line.** When the repeat-failure check is on (`repeat_failure.enabled`, off by default), each card and detail panel has one more line under the blockers line. When the check is off the line is not shown at all. The check only watches: it never pauses or interrupts a worker. + +| The line says | What it means | +| --- | --- | +| `repeat-failure check on` | Hive records this worker's shell commands and will note one that keeps failing the same way. | +| `repeat-failure check not available for this backend` | Hive cannot record this backend's shell commands yet, so repeated failures are not noticed. The [repeated-failure table](backend-support-tiers.md#repeated-failure-check) says why and names the follow-up issue. | +| `repeat-failure check not receiving data` | The worker's current round of work ran shell commands, but no command record has arrived for `repeat_failure.no_data_after_s` (15 minutes by default). The backend may have changed how it runs commands. Reading and editing files alone do not count, and a command that is still running (a long build) does not start the wait. | +| ``Repeating failure: `gh pr view 123` failed 5 times in 10 min (exit 1)`` | The same command failed the same way at least `repeat_failure.threshold` times (5) within `repeat_failure.window_s` (10 minutes). It says `(failed)` instead of an exit number when the backend gives none, and `and 2 others` when more commands are repeating; the most recent one is shown. The note clears when that command succeeds or when its last failure is older than the window. Hive logs the note once, when it appears. Commands are shown with secrets removed. | + **The run line.** While a worker is working, a second line under **Now:** reads like `$0.42 · 38.1K tok · 4m 12s · tests ✓ · 3 files`. The same line is pinned in the worker's detail panel header. It is built only from figures Hive already keeps: - **`$` and `tok`** are the estimated cost and tokens of *this* round of work: the worker's cumulative token count × public list price (the same basis as the Cost section), minus what it was when Hive started the round (the last kick). Hive remembers the starting point, so a page reload does not reset the figure. `$–` means the token collector has no figure for this worker yet. @@ -639,6 +648,7 @@ More about the line: - Each card's ⚙️ button opens its settings: schedule, model, tools and permissions. See [Agent configuration](agent-configuration.md). - `governor.claims.enabled` turns reservations on. When it is off, which is the default, the line shows nothing about reservations. - `dashboard.agent_poll_interval_s` sets how often worker updates are sent. The default is 10 seconds. +- `repeat_failure.enabled` turns the repeat-failure line on; `threshold`, `window_s`, `no_data_after_s` and `exclude` tune it. See [Agent-runtime blocks](operator-reference.md#agent-runtime-blocks-configagentruntimeconfig). ## Hive activity monitor diff --git a/src/docs/design/crux.md b/src/docs/design/crux.md index a251568209..00f7365461 100644 --- a/src/docs/design/crux.md +++ b/src/docs/design/crux.md @@ -553,7 +553,7 @@ the six dashboard families and gained nothing new to call. | `Resumer` (new) | `ResumeHandle`, `SendResumeKick`, `SessionID`, `SessionMissing`; sentinels `ErrResumeBusy`, `ErrResumeSessionGone`, `ErrResumeUnsupported`; value `ResumeHandle` | cmd/hive (PR follow-up) | | `Supervisor` (new) | `GovernorMayStart`, `MarkStartupLaunchQueued`, `CheckAndRestartCrashedAgents`, `ConsentWedgedAgents`, `RefreshAgentTokens`, `RefreshAgentTokenFor`, `AgentCredentialRefreshable`, `AgentHasValidCredential`, `AgentCapabilities`, `EffectiveBackend`, `InvocationMetadata`, `RestartTelemetry`, `StartFailureState`, `ProviderErrorBackoffRemaining`, `PauseByCause`, `ArchiveAllKickLogs`, `CountAgentsWithModel`, `CrashOOMSuspected`, `BackendAuthAvailable`, `AgentAuthAvailable`, `SendGitHubAPIBudgetNudge`, `RecordAudit`, `TrajectoryAgents` | cmd/hive (governor, heartbeat, rotation, login scan, trajectory) | | `Restorer` (new) | `SeedPauseState`, `SeedRestartTelemetry`, `SeedTurnLoss`, `SeedLastKick`, `SeedKickHistory`, `RestoreBreaker` | cmd/hive (snapshot restore) | -| `Background` (new) | `StartAgentTokenRefresh`, `StartCredentialWatchdog`, `StartCopilotSessionRefresh`, `StartTranscriptActivityRefresh` | cmd/hive (boot) | +| `Background` (new) | `StartAgentTokenRefresh`, `StartCredentialWatchdog`, `StartCopilotSessionRefresh`, `StartTranscriptActivityRefresh`, `StartRepeatFailureMonitor` | cmd/hive (boot) | | `Authorizer` (new) | `AuthorizePROpen`, `AuthorizePushBranch`, `AuthorizeIssueOpen`, `AuthorizeMerge`, `AuthorizeReviewRequest` | cmd/hive (request relays; method values handed to the watchers at boot) | Two things `cmd/hive` calls are **not** runtime-instance methods and are diff --git a/src/docs/operator-reference.md b/src/docs/operator-reference.md index 76d88fd08a..60686c57fa 100644 --- a/src/docs/operator-reference.md +++ b/src/docs/operator-reference.md @@ -167,7 +167,7 @@ Top-level YAML keys accepted by `config.Config`: | `notifications` | ntfy, Slack, and Discord webhooks. | All optional; see [notifications.md](notifications.md). | | `dashboard` | Web UI port, snapshots, auth token, frame allowlist, authorized users. | `auth_token` can come from `HIVE_DASHBOARD_TOKEN`. | | `agent_sandbox` | Podman-rootless sandbox launcher for hub/pod agents (`pkg/sandbox`). | Opt-in and **two-gate**: this block's own `enabled: true` sandboxes nothing by itself — each agent also needs `sandbox.enabled: true` under `agents.`. The dashboard's Security tab writes only this global flag, so enabling it there alone can leave every agent unconfined — but the tab now shows this: `security.sandboxWarnings` in `GET /api/config/governor` carries `config.AgentSandboxGateWarnings`'s diagnosis (also logged at WARN at boot/reload), rendered both in the page's coherence-warnings box and inline under the toggle, naming the still-unconfined agents and the fixing key (#4918). See [sandbox-isolation.md](sandbox-isolation.md) and the [getting-started confinement section](getting-started.md#where-agents-actually-run-read-this-before-l3). | -| `repeat_failure` | Monitor-only check that notices an agent repeating the same failing shell command ([#14550](https://github.com/hivecommons/hive/issues/14550)). | **Off by default** (`enabled: false`); while off Hive collects no command records and adds nothing to an agent's environment or home. `threshold` (5) failures of the same command with the same result within `window_s` (600) raise a note; `no_data_after_s` (900) is the "not receiving data" safeguard; `exclude` (default `gh pr checks`, `gh run watch`) lists command prefixes that fail by design while waiting. Raw command records live in a per-agent file (`~/.hive/repeat-failure/records.jsonl`, directory 0700, file 0600) that Hive collects at least every 30 s by renaming it first, scrubs with the log scrubber and deletes; only scrubbed text is kept. While on, backends that run commands in a fresh bash get `BASH_ENV` (a Hive-owned script in that directory, chaining any existing `BASH_ENV`) and `SHELL=/bin/bash` at launch, which moves bob from dash to bash; see [the shell route](agent-configuration.md#repeat-failure-check-the-shell-route-repeat_failure). Turning it off removes the script at the next launch. Never pauses an agent. | +| `repeat_failure` | Monitor-only check that notices an agent repeating the same failing shell command ([#14550](https://github.com/hivecommons/hive/issues/14550)). | **Off by default** (`enabled: false`); while off Hive collects no command records and adds nothing to an agent's environment or home. `threshold` (5) failures of the same command with the same result within `window_s` (600) raise a note; `no_data_after_s` (900) is the "not receiving data" safeguard; `exclude` (default `gh pr checks`, `gh run watch`) lists command prefixes that fail by design while waiting. Raw command records live in a per-agent file (`~/.hive/repeat-failure/records.jsonl`, directory 0700, file 0600) that Hive collects at least every 30 s by renaming it first, scrubs with the log scrubber and deletes; only scrubbed text is kept. While on, backends that run commands in a fresh bash get `BASH_ENV` (a Hive-owned script in that directory, chaining any existing `BASH_ENV`) and `SHELL=/bin/bash` at launch, which moves bob from dash to bash; see [the shell route](agent-configuration.md#repeat-failure-check-the-shell-route-repeat_failure). Turning it off removes the script at the next launch. While on, each agent's dashboard row shows the check state (on, not available for this backend, not receiving data) and the repeating-failure note; see [Agents](dashboard-sections.md#agents) and the [repeated-failure table](backend-support-tiers.md#repeated-failure-check). Never pauses an agent. | | `data` | Metrics, logs, session, agent overlay and Nous (`data.nous_dir`) directories. | Defaults are `/data/...` in containers. | | `nous` | Nous (Experiment Lab) runtime knobs: gate poll interval, dispatcher, dependency fetch allowlist. | Durable Nous state is under `data.nous_dir`, not here. | | `knowledge` | Wiki layers, vaults, git/document sources, connectors (`knowledge.connectors`, see [knowledge-connectors.md](knowledge-connectors.md)), primer, curator, bead synthesizer. | Disabled unless `enabled: true`. | diff --git a/src/pkg/dashboard/agent_repeat_failure_status_test.go b/src/pkg/dashboard/agent_repeat_failure_status_test.go new file mode 100644 index 0000000000..2ead9eb6d0 --- /dev/null +++ b/src/pkg/dashboard/agent_repeat_failure_status_test.go @@ -0,0 +1,78 @@ +package dashboard + +import ( + "strings" + "testing" + + "github.com/hivecommons/hive/crux" + "github.com/hivecommons/hive/crux/cruxtest" + "github.com/hivecommons/hive/pkg/config" +) + +// #14550 (C11, C12, C14): the agent row carries the repeat-failure check +// state and note, and nothing at all while the check is off. + +func repeatFailureProc(status crux.RepeatFailureStatus) *crux.Process { + return &crux.Process{ + Name: "scanner", + Config: config.AgentConfig{Backend: "claude", Enabled: true}, + State: crux.StateRunning, + OutputBuffer: cruxtest.NewRingBuffer(10), + RepeatFailure: status, + } +} + +func TestBuildAgents_RepeatFailureOffShowsNothing(t *testing.T) { + a := buildOneAgent(t, repeatFailureProc(crux.RepeatFailureStatus{})) + if a.RepeatFailureCheck != "" || a.RepeatFailureCheckMessage != "" || a.RepeatFailureNote != "" { + t.Fatalf("check off: row shows %q / %q / %q, want nothing", a.RepeatFailureCheck, a.RepeatFailureCheckMessage, a.RepeatFailureNote) + } +} + +func TestBuildAgents_RepeatFailureStates(t *testing.T) { + for _, tc := range []struct { + check crux.RepeatFailureCheckState + msg string + }{ + {crux.RepeatFailureCheckOn, "repeat-failure check on"}, + {crux.RepeatFailureCheckNotCovered, "repeat-failure check not available for this backend"}, + {crux.RepeatFailureCheckNoData, "repeat-failure check not receiving data"}, + } { + a := buildOneAgent(t, repeatFailureProc(crux.RepeatFailureStatus{Check: tc.check})) + if a.RepeatFailureCheck != string(tc.check) || a.RepeatFailureCheckMessage != tc.msg { + t.Errorf("check %q: row = %q / %q, want %q", tc.check, a.RepeatFailureCheck, a.RepeatFailureCheckMessage, tc.msg) + } + } + note := "Repeating failure: `false` failed 5 times in 10 min (exit 1)" + a := buildOneAgent(t, repeatFailureProc(crux.RepeatFailureStatus{Check: crux.RepeatFailureCheckOn, Note: note})) + if a.RepeatFailureNote != note { + t.Errorf("note = %q, want %q", a.RepeatFailureNote, note) + } + if a.Paused || a.StructuredStatus == "BLOCKED" { + t.Error("a repeat-failure note must not pause or block the agent (monitor only)") + } +} + +// TestRepeatFailureLineWiredInDashboard: the card and both detail-panel paths +// render the line, and the line renders nothing without a state. +func TestRepeatFailureLineWiredInDashboard(t *testing.T) { + html := indexHTML(t) + body := jsFunc(t, html, "repeatFailureLineHtml") + for _, want := range []string{ + "if (!a || (!a.repeatFailureCheck && !a.repeatFailureNote)) return '';", + "a.repeatFailureCheckMessage", + "a.repeatFailureNote", + "'no_data'", + "'not_covered'", + } { + if !strings.Contains(body, want) { + t.Errorf("repeatFailureLineHtml is missing %q", want) + } + } + if !strings.Contains(jsFunc(t, html, "renderAgents"), "${repeatFailureLineHtml(a)}") { + t.Error("renderAgents does not render the repeat-failure line on the card") + } + if n := strings.Count(jsFunc(t, html, "ocRenderAgentDetail"), "repeatFailureLineHtml(a)"); n != 2 { + t.Errorf("ocRenderAgentDetail renders the repeat-failure line %d times, want 2 (full and quick-update paths)", n) + } +} diff --git a/src/pkg/dashboard/server.go b/src/pkg/dashboard/server.go index 773e9b2435..c663df45dc 100644 --- a/src/pkg/dashboard/server.go +++ b/src/pkg/dashboard/server.go @@ -917,6 +917,14 @@ type FrontendAgent struct { // the turn is still running. KickOutcomeReason is the deciding pane line. KickOutcome string `json:"kickOutcome,omitempty"` KickOutcomeReason string `json:"kickOutcomeReason,omitempty"` + // RepeatFailureCheck is the repeat-failure check state (#14550): "on", + // "not_covered" or "no_data", with RepeatFailureCheckMessage as the row + // text. RepeatFailureNote is "Repeating failure: `` failed N + // times in M min (exit S)" while a command repeats. All empty while the + // check is off, so the row shows nothing. Monitor only: never pauses. + RepeatFailureCheck string `json:"repeatFailureCheck,omitempty"` + RepeatFailureCheckMessage string `json:"repeatFailureCheckMessage,omitempty"` + RepeatFailureNote string `json:"repeatFailureNote,omitempty"` // Run is the live per-run figure (#14576): this kick's estimated cost, // tokens, elapsed time, last visible test result and files changed. // Present only while Busy is "working". diff --git a/src/pkg/dashboard/settings_parity_allowlist.txt b/src/pkg/dashboard/settings_parity_allowlist.txt index 50258639d4..d28a490075 100644 --- a/src/pkg/dashboard/settings_parity_allowlist.txt +++ b/src/pkg/dashboard/settings_parity_allowlist.txt @@ -393,7 +393,6 @@ providers.status.enabled # TODO #11854 removed_agents[] # TODO #11854 repeat_failure.enabled # yaml-only until the check reaches the dashboard (#14550) repeat_failure.exclude[] # yaml-only until the check reaches the dashboard (#14550) -repeat_failure.no_data_after_s # yaml-only until the check reaches the dashboard (#14550) repeat_failure.threshold # yaml-only until the check reaches the dashboard (#14550) repeat_failure.window_s # yaml-only until the check reaches the dashboard (#14550) retro.lesson_confidence_cap # TODO #11854 diff --git a/src/pkg/dashboard/static/index.html b/src/pkg/dashboard/static/index.html index b7ee23cee2..a75bebeaac 100644 --- a/src/pkg/dashboard/static/index.html +++ b/src/pkg/dashboard/static/index.html @@ -1675,6 +1675,10 @@ .agent-blockers .ab-sep { opacity: 0.35; } .agent-blockers .ab-ok { color: var(--green); } .agent-blockers .ab-bad { color: var(--red); font-weight: 600; } + .agent-repeat-failure { display: flex; flex-wrap: wrap; justify-content: flex-end; gap: var(--sp-1) var(--sp-3); font-size: var(--fs-xs); margin-top: var(--sp-1); color: var(--muted); } + .agent-repeat-failure .rf-key { opacity: 0.65; margin-right: 3px; } + .agent-repeat-failure .rf-warn { color: var(--yellow); font-weight: 600; } + .agent-repeat-failure .rf-note { color: var(--yellow); overflow-wrap: anywhere; } .oc-detail-blockers { margin-top: var(--sp-3); display: flex; flex-wrap: wrap; align-items: center; gap: var(--sp-3); } .never-scheduled-chip { font-size: var(--fs-2xs); font-weight: 700; text-transform: uppercase; letter-spacing: 0.4px; color: var(--yellow); background: none; border: 1px solid var(--yellow); border-radius: 3px; padding: 1px 5px; margin-left: var(--sp-3); cursor: pointer; vertical-align: middle; white-space: nowrap; } .never-scheduled-chip:hover { background: color-mix(in srgb, var(--yellow) 18%, transparent); } @@ -11061,7 +11065,10 @@

Where to get help

const nowEl = detail.querySelector('[data-agent-now]'); if (nowEl) nowEl.outerHTML = agentNowHtml(a); const blockersEl = detail.querySelector('.oc-detail-blockers'); - if (blockersEl) blockersEl.innerHTML = agentBlockerLineHtml(a) + neverScheduledChipHtml(a, a.name); + if (blockersEl) { + blockersEl.innerHTML = agentBlockerLineHtml(a) + neverScheduledChipHtml(a, a.name); + blockersEl.insertAdjacentHTML('beforeend', repeatFailureLineHtml(a)); + } const fieldsEl = detail.querySelector('.oc-detail-fields'); if (fieldsEl) fieldsEl.outerHTML = ocDetailFieldsHtml(a, isPaused, isOff, isOnDemand); const indEl = detail.querySelector('.oc-detail-indicators'); @@ -11201,7 +11208,7 @@

Where to get help

${effortSelectHtml(a)} ${agentNowHtml(a)} -
${agentBlockerLineHtml(a)}${neverScheduledChipHtml(a, a.name)}
+
${agentBlockerLineHtml(a)}${neverScheduledChipHtml(a, a.name)}${repeatFailureLineHtml(a)}
${terminalCopyHintHtml(needsLoginDown, agentSessionUnavailable(a), a.name, a.loginURL, a.backendAuthStatus, a.backendAuthLastError, a.statusEvidence, a.inferenceLastError)} ${ocDetailFieldsHtml(a, isPaused, isOff, isOnDemand)} ${subAgentModelsHtml(a)} @@ -17565,6 +17572,27 @@

All labels

} return ''; } + /* repeatFailureLineHtml renders the repeat-failure check (#14550): the + check state ("on", "not available for this backend", "not receiving + data") and, while a command keeps failing the same way, the note. + Monitor only: the agent is never paused for it. Renders nothing while + the check is off — the hive sends no state then. */ + function repeatFailureLineHtml(a) { + if (!a || (!a.repeatFailureCheck && !a.repeatFailureNote)) return ''; + const warn = a.repeatFailureCheck === 'not_covered' || a.repeatFailureCheck === 'no_data'; + const checkTip = a.repeatFailureCheck === 'no_data' + ? 'A turn in progress ran shell commands, but no command record has arrived for repeat_failure.no_data_after_s. The backend may have changed how it runs commands.' + : a.repeatFailureCheck === 'not_covered' + ? 'Hive cannot record this backend\'s shell commands yet, so repeated failures are not noticed. See the repeated-failure table in the backend support docs.' + : 'Hive records this agent\'s shell commands and notes a command that keeps failing the same way.'; + const check = a.repeatFailureCheckMessage + ? `repeat-failure${esc(a.repeatFailureCheckMessage)}` + : ''; + const note = a.repeatFailureNote + ? `${esc(a.repeatFailureNote)}` + : ''; + return `
${note}${check}
`; + } /* agentPowerSwitchHtml renders the master-power 0/1 rocker for an agent's `enabled` flag. @@ -18291,6 +18319,7 @@

All labels

${agentNowHtml(a)} ${agentRunLineHtml(a)} ${agentBlockerLineHtml(a)} + ${repeatFailureLineHtml(a)} ${terminalCopyHintHtml(needsLogin, agentSessionUnavailable(a), a.name, a.loginURL, a.backendAuthStatus, a.backendAuthLastError, a.statusEvidence, a.inferenceLastError)} ${(() => { // Last upstream/launch error stays on its own line below the state diff --git a/src/pkg/dashboard/status_builder.go b/src/pkg/dashboard/status_builder.go index 2109564ae1..c80e92c9fd 100644 --- a/src/pkg/dashboard/status_builder.go +++ b/src/pkg/dashboard/status_builder.go @@ -1256,6 +1256,11 @@ func buildAgentsWithHidden(statuses map[string]*crux.Process, cfg *config.Config CadenceModes: cadenceModes, } applyAgentLastAction(&a, proc, busy) + // #14550: the repeat-failure note and check state; nothing while + // the check is off. + a.RepeatFailureCheck = string(proc.RepeatFailure.Check) + a.RepeatFailureCheckMessage = proc.RepeatFailure.Check.Message() + a.RepeatFailureNote = redactTokens(proc.RepeatFailure.Note) // #7421: how the last kicked turn ended, so the card can say "asked // the operator what to do" or "stood down" instead of implying work. if proc.KickOutcome.Settled(proc.LastKick) {