✨ feat(agent,dashboard): repeat-failure row note, check states, no-data safeguard and coverage table

changelog.d/added-14558.md

@@ -0,0 +1 @@
+- Repeat-failure check: with `repeat_failure.enabled` on, each agent's dashboard row shows the check state (on, "not available for this backend", or "not receiving data" after `no_data_after_s` of shell activity without command records) and a "Repeating failure: `<command>` failed N times in M min (exit S)" note when a command keeps failing the same way; nothing is shown while the check is off and the agent is never paused. A code coverage table, mirrored in the backend support docs and checked in CI, lists every backend; none is marked covered yet, pending real-run evidence (#14559) (#14558).

src/cmd/hive/agent_runtime_sidecar.go

@@ -309,6 +309,9 @@ func (h *sidecarHost) SetAdvisorRolesResolver(func() map[string]cruxconfig.Model
 func (h *sidecarHost) SetRepeatFailureEnabledResolver(func() bool) {
 	h.inProcessOnly("SetRepeatFailureEnabledResolver")
 }
+func (h *sidecarHost) SetRepeatFailureConfigResolver(func() *cruxconfig.RepeatFailureConfig) {
+	h.inProcessOnly("SetRepeatFailureConfigResolver")
+}
 func (h *sidecarHost) SetRepoAutoMergeEnabledResolver(fn func(string) bool) {
 	h.setPolicy("SetRepoAutoMergeEnabledResolver", func() { h.autoMerge = fn })
 }

src/cmd/hive/boot_agents_deps.go

@@ -1,3 +1,3 @@
 package main
 
 import (
@@ -26,7 +26,8 @@
 // the rest of the test binary's life are recorded instead of started.
 type bootAgentsDeps struct {
 	// startAgentLoops starts the per-agent token refresh, the credential
-	// watchdog and the Copilot session refresh. All three are unconditional
+	// watchdog, the Copilot session refresh and the repeat-failure monitor
+	// (which only clears row state while the check is off). All are unconditional
 	// (see the comments at the call site for why each must not gate on App
 	// state at boot).
 	startAgentLoops func(ctx context.Context, mgr crux.Runtime)
@@ -92,6 +93,7 @@
 			go mgr.StartCredentialWatchdog(ctx)
 			go mgr.StartCopilotSessionRefresh(ctx)
 			go mgr.StartTranscriptActivityRefresh(ctx)
+			go mgr.StartRepeatFailureMonitor(ctx)
 		},
 		prepareRequestDirs: github.PrepareRequestDirs,
 		startTokenAccessAudit: func(ctx context.Context, logger *slog.Logger) {

src/cmd/hive/main.go

@@ -1,3 +1,3 @@
 package main
 
 import (
@@ -2430,6 +2430,9 @@
 	b.agentMgr.SetRepeatFailureEnabledResolver(func() bool {
 		return b.cfg.RepeatFailure.IsEnabled()
 	})
+	b.agentMgr.SetRepeatFailureConfigResolver(func() *config.RepeatFailureConfig {
+		return b.cfg.RepeatFailure
+	})
 	b.agentMgr.SetAdvisorRolesResolver(func() map[string]config.ModelRole {
 		return b.cfg.ModelRoles
 	})

src/crux/agent/manager.go

@@ -449,6 +449,10 @@ type AgentProcess struct {
 	// report, or plain "ended". Settled(LastKick) is false while the current
 	// turn is still running. Guarded by m.mu; see kick_outcome.go.
 	KickOutcome KickOutcome
+	// RepeatFailure is the repeat-failure check's row note and check state
+	// (#14550), refreshed at each record collection. Zero while the check is
+	// off. Guarded by m.mu; see repeat_failure_status.go.
+	RepeatFailure RepeatFailureStatus
 	// sandboxResumeAfterCancel is set when an operator resumes a paused
 	// sandbox agent while the canceled sandbox goroutine is still draining.
 	// The completion handler then turns the expected cancellation into Idle
@@ -616,6 +620,12 @@ type Manager struct {
 	// switch (#14550), read at launch so a toggle applies on the next launch.
 	// Same atomic.Pointer discipline as advisorEnabledResolver.
 	repeatFailureEnabledResolver atomic.Pointer[func() bool]
+	// repeatFailureConfigResolver returns the live repeat_failure block for
+	// the detector and the not-receiving-data safeguard. Same discipline.
+	repeatFailureConfigResolver atomic.Pointer[func() *config.RepeatFailureConfig]
+	// repeatFailure is the repeat-failure detector and safeguard state behind
+	// AgentProcess.RepeatFailure; see repeat_failure_status.go.
+	repeatFailure repeatFailureMonitor
 	// advisorRolesResolver returns the live model_roles map the OMP adapter
 	// projects as a --config overlay at launch (#9723). Same discipline as
 	// advisorEnabledResolver.

src/crux/agent/manager_pane_signals.go

@@ -572,6 +572,7 @@ func (a *AgentProcess) snapshot() crux.Process {
 		BootstrapOverride:         a.BootstrapOverride,
 		LastError:                 a.LastError,
 		KickOutcome:               a.KickOutcome,
+		RepeatFailure:             a.RepeatFailure,
 		NeedsLogin:                needsLogin,
 		LoginURL:                  loginURL,
 		Starting:                  a.startupLaunchQueued || a.launching,

src/crux/agent/repeat_failure_coverage.go

@@ -0,0 +1,121 @@
+package agent
+
+import (
+	"strconv"
+	"strings"
+)
+
+// Repeat-failure coverage table (hivecommons/hive#14550, C4, C5, C12, C13).
+//
+// Every backend Hive supports (config.CLIBackends and
+// config.InferenceBackends) has exactly one row. A backend is covered only
+// when a committed, re-runnable harness made its real CLI produce a correct
+// command record, with the CLI's own `--version` output in the evidence
+// (C4). Hand-written test data does not count. Every other backend is not
+// covered: its agent rows say "repeat-failure check not available for this
+// backend", and the row names the reason and the follow-up issue (C5).
+//
+// The docs table in docs/backend-support-tiers.md ("Repeated-failure check")
+// is rendered from this table; TestRepeatFailureCoverageDocsMatchCode fails
+// when the two differ or a backend is missing (C13).
+
+// repeatFailureBackendRoutesIssue tracks the backend-specific routes and the
+// real-run evidence still missing for every backend below (#14559).
+const repeatFailureBackendRoutesIssue = 14559
+
+// repeatFailureSafeguardUnavailable is the shell-activity signal of a covered
+// backend that has none, so the not-receiving-data safeguard cannot run (C12).
+const repeatFailureSafeguardUnavailable = "safeguard unavailable"
+
+// RepeatFailureCoverage is one backend's row of the coverage table.
+type RepeatFailureCoverage struct {
+	Backend string
+	// Covered is true only with committed real-run evidence (C4).
+	Covered bool
+	// Route is the producer of the backend's records, or, for a backend that
+	// is not covered, what exists today and what is planned.
+	Route string
+	// Evidence names the committed harness and the CLI `--version` it
+	// recorded. Empty when not covered.
+	Evidence string
+	// EvidencePath is the committed harness evidence, relative to src/.
+	// Required when Covered.
+	EvidencePath string
+	// ShellActivity is the signal the not-receiving-data safeguard watches
+	// for this backend, or repeatFailureSafeguardUnavailable. Empty when not
+	// covered.
+	ShellActivity string
+	// Reason says why the backend is not covered. Empty when covered.
+	Reason string
+	// FollowUp is the issue that tracks covering the backend.
+	FollowUp int
+}
+
+const (
+	rfNeutralRouteNoEvidence = "BASH_ENV route installed while the check is on, but no committed real-run harness evidence yet (C4)"
+	rfOwnRouteNotBuilt       = "commands never reach the BASH_ENV route and the backend-specific route is not built yet"
+)
+
+// repeatFailureCoverageTable lists every backend in config.CLIBackends order,
+// then config.InferenceBackends.
+var repeatFailureCoverageTable = []RepeatFailureCoverage{
+	{Backend: "claude", Route: "BASH_ENV (neutral), unwraps `eval '…'`", FollowUp: repeatFailureBackendRoutesIssue, Reason: rfNeutralRouteNoEvidence},
+	{Backend: "copilot", Route: "none yet; planned: `events.jsonl` `tool.execution_complete`", FollowUp: repeatFailureBackendRoutesIssue, Reason: "one long-lived Bash session per shell id; " + rfOwnRouteNotBuilt},
+	{Backend: "goose", Route: "BASH_ENV (neutral)", FollowUp: repeatFailureBackendRoutesIssue, Reason: rfNeutralRouteNoEvidence},
+	{Backend: "codex", Route: "BASH_ENV (neutral), one record across `exec /bin/bash -c`", FollowUp: repeatFailureBackendRoutesIssue, Reason: rfNeutralRouteNoEvidence},
+	{Backend: "pi", Route: "BASH_ENV (neutral)", FollowUp: repeatFailureBackendRoutesIssue, Reason: rfNeutralRouteNoEvidence},
+	{Backend: "bob", Route: "BASH_ENV (neutral) with `SHELL=/bin/bash` (documented shell change: dash → bash)", FollowUp: repeatFailureBackendRoutesIssue, Reason: rfNeutralRouteNoEvidence},
+	{Backend: "aider", Route: "none yet; planned: guarded `~/.bashrc` line in Hive-managed homes", FollowUp: repeatFailureBackendRoutesIssue, Reason: "interactive `$SHELL -i -c` ignores BASH_ENV; " + rfOwnRouteNotBuilt},
+	{Backend: "gemini", Route: "BASH_ENV (neutral), unwraps the `{ …; }; __code=$?` wrapper", FollowUp: repeatFailureBackendRoutesIssue, Reason: rfNeutralRouteNoEvidence},
+	{Backend: "agy", Route: "none yet; planned: `agy-turn` `run_command` steps", FollowUp: repeatFailureBackendRoutesIssue, Reason: "closed binary, likely a long-lived terminal; " + rfOwnRouteNotBuilt},
+	{Backend: "opencode", Route: "BASH_ENV (neutral) with `SHELL=/bin/bash`", FollowUp: repeatFailureBackendRoutesIssue, Reason: rfNeutralRouteNoEvidence},
+	{Backend: "kilo", Route: "BASH_ENV (neutral) with `SHELL=/bin/bash`", FollowUp: repeatFailureBackendRoutesIssue, Reason: rfNeutralRouteNoEvidence},
+	{Backend: "muse", Route: "none yet; planned: `PostToolUse` / `PostToolUseFailure` hooks", FollowUp: repeatFailureBackendRoutesIssue, Reason: "sandboxed shell tools in a closed binary; " + rfOwnRouteNotBuilt},
+	{Backend: "omp", Route: "none yet; planned: `tool_result` extension event", FollowUp: repeatFailureBackendRoutesIssue, Reason: "embedded shell that strips BASH_ENV; " + rfOwnRouteNotBuilt},
+	{Backend: "openhands", Route: "none yet; planned: `PostToolUse` hook or saved events", FollowUp: repeatFailureBackendRoutesIssue, Reason: "one interactive bash kept open in tmux; " + rfOwnRouteNotBuilt},
+	{Backend: "omnigent", Route: "none yet; planned: tool-call policy checkpoint (`ALLOW`/`DENY` verdicts) or saved session events", FollowUp: repeatFailureBackendRoutesIssue, Reason: "meta-harness driving Claude Code, Codex or Pi through its own policy engine; " + rfOwnRouteNotBuilt},
+	{Backend: "vllm", Route: "follows `claude` (drives Claude Code)", FollowUp: repeatFailureBackendRoutesIssue, Reason: "follows `claude`, which is not covered"},
+	{Backend: "llm-d", Route: "follows `claude` (drives Claude Code)", FollowUp: repeatFailureBackendRoutesIssue, Reason: "follows `claude`, which is not covered"},
+	{Backend: "litellm", Route: "follows `claude` (drives Claude Code)", FollowUp: repeatFailureBackendRoutesIssue, Reason: "follows `claude`, which is not covered"},
+	{Backend: "watsonx", Route: "follows `claude` (drives Claude Code)", FollowUp: repeatFailureBackendRoutesIssue, Reason: "follows `claude`, which is not covered"},
+}
+
+// RepeatFailureCoverageTable returns a copy of the coverage table.
+func RepeatFailureCoverageTable() []RepeatFailureCoverage {
+	return append([]RepeatFailureCoverage(nil), repeatFailureCoverageTable...)
+}
+
+// RepeatFailureBackendCovered reports whether the check covers backend. An
+// unknown backend is not covered.
+func RepeatFailureBackendCovered(backend string) bool {
+	backend = strings.ToLower(strings.TrimSpace(backend))
+	for _, row := range repeatFailureCoverageTable {
+		if row.Backend == backend {
+			return row.Covered
+		}
+	}
+	return false
+}
+
+// RepeatFailureCoverageMarkdownRows renders the docs table rows, one per
+// backend, in table order. The docs carry exactly these lines.
+func RepeatFailureCoverageMarkdownRows() []string {
+	rows := make([]string, 0, len(repeatFailureCoverageTable))
+	for _, row := range repeatFailureCoverageTable {
+		status := "Not covered: " + row.Reason
+		evidence := "None"
+		signal := "n/a"
+		if row.Covered {
+			status = "Covered"
+			evidence = row.Evidence
+			signal = row.ShellActivity
+		}
+		followUp := "—"
+		if row.FollowUp > 0 {
+			n := strconv.Itoa(row.FollowUp)
+			followUp = "[#" + n + "](https://github.com/hivecommons/hive/issues/" + n + ")"
+		}
+		rows = append(rows, "| `"+row.Backend+"` | "+row.Route+" | "+status+" | "+evidence+" | "+signal+" | "+followUp+" |")
+	}
+	return rows
+}

src/crux/agent/repeat_failure_coverage_test.go

@@ -0,0 +1,116 @@
+package agent
+
+import (
+	"os"
+	"path/filepath"
+	"slices"
+	"strings"
+	"testing"
+
+	"github.com/hivecommons/hive/crux/config"
+)
+
+// TestRepeatFailureCoverageListsEveryBackend is C13/C-AC11: the code table has
+// exactly one row for every config.CLIBackends and config.InferenceBackends
+// entry, so removing a backend from it fails here.
+func TestRepeatFailureCoverageListsEveryBackend(t *testing.T) {
+	rows := map[string]RepeatFailureCoverage{}
+	for _, row := range RepeatFailureCoverageTable() {
+		if _, dup := rows[row.Backend]; dup {
+			t.Errorf("backend %q has more than one coverage row", row.Backend)
+		}
+		rows[row.Backend] = row
+	}
+	want := append(append([]string(nil), config.CLIBackends...), config.InferenceBackends...)
+	for _, backend := range want {
+		if _, ok := rows[backend]; !ok {
+			t.Errorf("repeat-failure coverage table is missing backend %q", backend)
+		}
+	}
+	for backend := range rows {
+		if !slices.Contains(want, backend) {
+			t.Errorf("coverage row %q is neither a config.CLIBackends nor a config.InferenceBackends entry", backend)
+		}
+	}
+}
+
+// TestRepeatFailureCoverageRowsAreHonest is C4/C5: a covered row carries a
+// route, committed harness evidence with the CLI's --version and a
+// shell-activity signal (or "safeguard unavailable"); a row that is not
+// covered carries a reason and a follow-up issue.
+func TestRepeatFailureCoverageRowsAreHonest(t *testing.T) {
+	for _, row := range RepeatFailureCoverageTable() {
+		if row.FollowUp < 0 {
+			t.Errorf("%s: negative follow-up issue", row.Backend)
+		}
+		if strings.TrimSpace(row.Route) == "" {
+			t.Errorf("%s: route is empty", row.Backend)
+		}
+		if strings.Contains(row.Route+row.Reason+row.Evidence+row.ShellActivity, "|") {
+			t.Errorf("%s: a cell contains '|', which breaks the docs table", row.Backend)
+		}
+		if !row.Covered {
+			if strings.TrimSpace(row.Reason) == "" || row.FollowUp == 0 {
+				t.Errorf("%s: not covered needs a reason and a follow-up issue (C5)", row.Backend)
+			}
+			if row.Evidence != "" || row.EvidencePath != "" || row.ShellActivity != "" {
+				t.Errorf("%s: not covered must not carry evidence or a safeguard signal", row.Backend)
+			}
+			if RepeatFailureBackendCovered(row.Backend) {
+				t.Errorf("%s: RepeatFailureBackendCovered disagrees with the table", row.Backend)
+			}
+			continue
+		}
+		if row.Reason != "" {
+			t.Errorf("%s: a covered row must not carry a not-covered reason", row.Backend)
+		}
+		if !strings.Contains(row.Evidence, "--version") {
+			t.Errorf("%s: covered evidence must include the CLI's --version output (C4)", row.Backend)
+		}
+		if row.EvidencePath == "" {
+			t.Errorf("%s: covered without committed real-run harness evidence (C4)", row.Backend)
+		} else if _, err := os.Stat(filepath.Join("..", "..", row.EvidencePath)); err != nil {
+			t.Errorf("%s: evidence %s is not committed: %v", row.Backend, row.EvidencePath, err)
+		}
+		if strings.TrimSpace(row.ShellActivity) == "" {
+			t.Errorf("%s: covered row must name its shell-activity signal or %q (C12)", row.Backend, repeatFailureSafeguardUnavailable)
+		}
+		if !RepeatFailureBackendCovered(row.Backend) {
+			t.Errorf("%s: RepeatFailureBackendCovered disagrees with the table", row.Backend)
+		}
+	}
+	if RepeatFailureBackendCovered("no-such-backend") {
+		t.Error("an unknown backend must not be covered")
+	}
+}
+
+// TestRepeatFailureCoverageDocsMatchCode is C13/C-AC11: the repeated-failure
+// table in docs/backend-support-tiers.md is exactly the rendered code table,
+// so a row changed only in the docs fails here.
+func TestRepeatFailureCoverageDocsMatchCode(t *testing.T) {
+	raw, err := os.ReadFile("../../docs/backend-support-tiers.md")
+	if err != nil {
+		t.Fatal(err)
+	}
+	doc := string(raw)
+	const heading = "## Repeated-failure check"
+	start := strings.Index(doc, heading)
+	if start < 0 {
+		t.Fatalf("backend-support-tiers.md has no %q section", heading)
+	}
+	section := doc[start+len(heading):]
+	if end := strings.Index(section, "\n## "); end >= 0 {
+		section = section[:end]
+	}
+	var got []string
+	for _, line := range strings.Split(section, "\n") {
+		if strings.HasPrefix(line, "| `") {
+			got = append(got, line)
+		}
+	}
+	want := RepeatFailureCoverageMarkdownRows()
+	if !slices.Equal(got, want) {
+		t.Errorf("docs repeated-failure table differs from the code table (pkg/agent/repeat_failure_coverage.go).\nwant rows:\n%s\n\ngot rows:\n%s",
+			strings.Join(want, "\n"), strings.Join(got, "\n"))
+	}
+}