🐛 fix(agent): keep the copilot transcript activity scan inside its interval

changelog.d/fixed-14649.md

@@ -0,0 +1 @@
+- Agent transcript liveness scans now walk several agents in parallel and the slow-scan warning is classified, so hives with many agents no longer file it as an unclassified error ([#14649](https://github.com/hivecommons/hive/issues/14649)).

src/crux/agent/busy_visibility.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+	"sync"
 	"time"
 
 	"github.com/hivecommons/hive/crux"
@@ -328,22 +329,47 @@ func (m *Manager) pendingTranscriptActivityScansLocked(now time.Time) []transcri
 	return scans
 }
 
-// scanTranscriptActivity stats each agent's sessions. caches is keyed by
-// agent name and may be nil (every session is stat'd).
+// transcriptScanConcurrency bounds how many agents' session trees are walked
+// at once. Each walk is dominated by network-filesystem round trips, so a few
+// in parallel bring a many-agent pass back inside the scan interval without
+// flooding the mount.
+const transcriptScanConcurrency = 4
+
+// scanAgentTranscript walks one agent's session tree; a variable so tests can
+// observe how the passes are scheduled.
+var scanAgentTranscript = func(scan transcriptActivityScan, cache *transcriptSessionCache, now time.Time) (time.Time, bool) {
+	return newestCopilotTranscriptActivityCached(AgentHome(scan.name, scan.uid, "copilot"), cache, now)
+}
+
+// scanTranscriptActivity stats each agent's sessions, up to
+// transcriptScanConcurrency agents at a time. caches is keyed by agent name
+// and may be nil (every session is stat'd). Results keep the order of scans.
 func scanTranscriptActivity(scans []transcriptActivityScan, caches map[string]*transcriptSessionCache, now time.Time) []transcriptActivityResult {
-	results := make([]transcriptActivityResult, 0, len(scans))
-	for _, scan := range scans {
-		var cache *transcriptSessionCache
+	results := make([]transcriptActivityResult, len(scans))
+	perAgent := make([]*transcriptSessionCache, len(scans))
+	for i, scan := range scans {
 		if caches != nil {
-			cache = caches[scan.name]
+			cache := caches[scan.name]
 			if cache == nil {
 				cache = &transcriptSessionCache{}
 				caches[scan.name] = cache
 			}
+			perAgent[i] = cache
 		}
-		at, ok := newestCopilotTranscriptActivityCached(AgentHome(scan.name, scan.uid, "copilot"), cache, now)
-		results = append(results, transcriptActivityResult{name: scan.name, at: at, ok: ok})
 	}
+	sem := make(chan struct{}, transcriptScanConcurrency)
+	var wg sync.WaitGroup
+	for i, scan := range scans {
+		wg.Add(1)
+		sem <- struct{}{}
+		go func() {
+			defer wg.Done()
+			defer func() { <-sem }()
+			at, ok := scanAgentTranscript(scan, perAgent[i], now)
+			results[i] = transcriptActivityResult{name: scan.name, at: at, ok: ok}
+		}()
+	}
+	wg.Wait()
 	return results
 }
 

src/crux/agent/busy_visibility_test.go

@@ -548,3 +548,42 @@ func TestNewestCopilotTranscriptActivityCachedSkipsColdSessions(t *testing.T) {
 		t.Fatalf("cache after full rescan = %+v, want no-events dropped and lastFull=%v", cache, full)
 	}
 }
+
+// A pass over many agents must walk several at once; sequential walks are what
+// pushed a 15-agent pass to 13 s against a 10 s interval.
+func TestScanTranscriptActivityWalksAgentsConcurrently(t *testing.T) {
+	orig := scanAgentTranscript
+	t.Cleanup(func() { scanAgentTranscript = orig })
+
+	started := make(chan struct{}, transcriptScanConcurrency)
+	release := make(chan struct{})
+	scanAgentTranscript = func(scan transcriptActivityScan, _ *transcriptSessionCache, _ time.Time) (time.Time, bool) {
+		started <- struct{}{}
+		<-release
+		return time.Unix(int64(scan.uid), 0), true
+	}
+	var scans []transcriptActivityScan
+	for i := range 2 * transcriptScanConcurrency {
+		scans = append(scans, transcriptActivityScan{name: string(rune('a' + i)), uid: i + 1})
+	}
+	done := make(chan []transcriptActivityResult)
+	go func() { done <- scanTranscriptActivity(scans, map[string]*transcriptSessionCache{}, time.Time{}) }()
+
+	for range transcriptScanConcurrency {
+		select {
+		case <-started:
+		case <-time.After(10 * time.Second):
+			t.Fatal("agent walks did not overlap")
+		}
+	}
+	close(release)
+	results := <-done
+	if len(results) != len(scans) {
+		t.Fatalf("got %d results, want %d", len(results), len(scans))
+	}
+	for i, r := range results {
+		if r.name != scans[i].name || !r.ok || r.at.Unix() != int64(scans[i].uid) {
+			t.Fatalf("result %d = %+v, want agent %s", i, r, scans[i].name)
+		}
+	}
+}

src/docs/fleet-report.md

@@ -157,6 +157,7 @@ title is `[selfreport] <class>: <short message>`.
 | `inference_upstream_error` | a failed or non-2xx response from the configured inference backend (LiteLLM, vLLM/llm-d) on the Anthropic-to-OpenAI translator or MITM reroute path (`inference upstream error` / `inference upstream failed`, `crux/proxy/github_proxy.go`), repeated ≥5 times in 10 min — covers transport failures and statuses other than the dedicated 401/403 auth signal. Log-fed: same repeat threshold and idle clear as `agent_pane_timeout` | medium |
 | `sha_poll_commit_message_unavailable` | the SHA poller's commit-message fetch got a non-200 from GitHub (typically a 403 rate limit or token scope; `SHA poll: commit message fetch non-200`, `pkg/hub/saas_sha_poller.go`). The poller backs off per SHA and keeps serving the SHA without a message. The WARN itself is deduped to once per hour per (SHA, status) (`warnGitHubFetchOnce`), so one stuck commit logs one line and never trips this class; it is reported only when ≥5 distinct SHA/status pairs fail within 10 min (a broad outage such as an exhausted rate limit or a revoked token). Its main job is keeping the residual WARN out of `unclassified_error_repeated`. Log-fed: same repeat threshold and idle clear as `agent_pane_timeout` | low |
 | `tmux_session_check_failed` | the agent's tmux session `has-session` check failed (`tmux has-session failed`, `crux/agent/manager.go`) — typically the session was lost or the tmux server is unhealthy, repeated ≥5 times in 10 min. Log-fed: same repeat threshold and idle clear as `agent_pane_timeout` | medium |
+| `transcript_scan_slow` | a Copilot transcript activity scan took longer than its interval (`transcript activity scan outlived its interval`, `crux/agent/busy_visibility.go`) — agent liveness reads may lag. Log-fed: same ≥5-in-10-min repeat threshold and 20-minute idle clear as `agent_pane_timeout`. Operator diagnostics: check the agent's transcript/session volume; it is bounded by concurrency, so repeated hits indicate slow disk or very large session directories | low |
 | `unclassified_error_repeated` | **generic catch-all**: the same WARN/ERROR log line (numbers, hashes, paths and quoted strings collapsed) repeated ≥5 times in 10 min. Fed by a `slog.Handler` tee (`pkg/fleetreport/slogfeed`) on the hive's root logger, so a repeating defect is noticed even with no hook site. Skips benign lines (read/keepalive i/o timeouts, context cancellation, the permissions watcher's `fixed …` self-healing lines) and any message that already matches another class's keywords. Clears automatically after 20 quiet minutes (2× the window), so a "cleared" comment follows once the symptom stops | low |
 
 The generic feed also skips the exact `review pending expired` message: it

src/pkg/fleetreport/runtime.go

@@ -85,6 +85,7 @@ const (
 	ClassGitHubFetchTransient       = "github_fetch_transient"
 	ClassLabelProvisioningFailed    = "label_provisioning_failed"
 	ClassAgentCLIRestartLoop        = "agent_cli_restart_loop"
+	ClassTranscriptScanSlow         = "transcript_scan_slow"
 	// A restart or normal stop can invalidate a pending bootstrap without
 	// indicating a restart storm; only repeated interruptions are reportable.
 	ClassStartupKickInterrupted = "startup_kick_interrupted"
@@ -196,6 +197,7 @@ var Classes = map[string]ClassSpec{
 	ClassGitHubFetchTransient:       {Component: "github-client", Severity: "low", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"channel commit date: fetch failed"}},
 	ClassLabelProvisioningFailed:    {Component: "github-client", Severity: "low", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"issue stage labels: provisioning failed"}},
 	ClassAgentCLIRestartLoop:        {Component: "agent-runtime", Severity: "medium", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"agent cli crashed or stuck on consent screen"}},
+	ClassTranscriptScanSlow:         {Component: "agent-runtime", Severity: "low", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"transcript activity scan outlived its interval"}},
 }
 
 // LogFedClass returns the LogFed class whose Keyword msg contains, or "".

src/pkg/fleetreport/runtime_test.go

@@ -351,6 +351,7 @@ func TestEveryRuntimeClassTriggerAndNonTriggerThresholds(t *testing.T) {
 		{class: ClassAgentDirSetupFailed, message: "failed to create per-agent XDG dir"},
 		{class: ClassReviewEvidenceWriteFail, message: "review evidence: could not write bundle"},
 		{class: ClassFeedbackIssueRefreshFailed, message: "feedback: issue refresh failed"},
+		{class: ClassTranscriptScanSlow, message: "copilot transcript activity scan outlived its interval; agent liveness reads may lag"},
 		{class: ClassBeadFactIngestFailed, message: "failed to ingest bead fact"},
 		{class: ClassSelfReportClearedFailed, message: "fleet report: cleared comment failed"},
 		{class: ClassGitHubAppRepoNotCovered, message: "github app recheck: installation does not cover every configured repo"},
@@ -961,6 +962,7 @@ func TestRecurringRuntimeMessagesAreClassified(t *testing.T) {
 		"github app recheck: installation does not cover every configured repo":                        ClassGitHubAppRepoNotCovered,
 		"proxy: CONNECT dial failed":                                                                   ClassProxyConnectDialFailed,
 		"inference upstream error":                                                                     ClassInferenceUpstreamError,
+		"copilot transcript activity scan outlived its interval; agent liveness reads may lag":         ClassTranscriptScanSlow,
 		"inference upstream failed":                                                                    ClassInferenceUpstreamError,
 	}
 	for msg, want := range fed {