From bfe9750e51b8b3ea8e6d9dfddc0ed796e25c40e7 Mon Sep 17 00:00:00 2001 From: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Tue, 27 Oct 2026 14:28:37 -0400 Subject: [PATCH 1/4] fix(agent): keep the copilot transcript activity scan inside its interval Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- changelog.d/fixed-14649.md | 1 + src/crux/agent/busy_visibility.go | 42 +++++++++++++++++++++----- src/crux/agent/busy_visibility_test.go | 39 ++++++++++++++++++++++++ src/pkg/fleetreport/runtime.go | 2 ++ src/pkg/fleetreport/runtime_test.go | 2 ++ 5 files changed, 78 insertions(+), 8 deletions(-) create mode 100644 changelog.d/fixed-14649.md diff --git a/changelog.d/fixed-14649.md b/changelog.d/fixed-14649.md new file mode 100644 index 0000000000..2d6febe2bb --- /dev/null +++ b/changelog.d/fixed-14649.md @@ -0,0 +1 @@ +Agent transcript liveness scans now walk several agents in parallel and the slow-scan warning is classified, so hives with many agents no longer file it as an unclassified error ([#14649](https://github.com/hivecommons/hive/issues/14649)). diff --git a/src/crux/agent/busy_visibility.go b/src/crux/agent/busy_visibility.go index ab02dd70be..299319e1e8 100644 --- a/src/crux/agent/busy_visibility.go +++ b/src/crux/agent/busy_visibility.go @@ -5,6 +5,7 @@ import ( "fmt" "os" "path/filepath" + "sync" "time" "github.com/hivecommons/hive/crux" @@ -328,22 +329,47 @@ func (m *Manager) pendingTranscriptActivityScansLocked(now time.Time) []transcri return scans } -// scanTranscriptActivity stats each agent's sessions. caches is keyed by -// agent name and may be nil (every session is stat'd). +// transcriptScanConcurrency bounds how many agents' session trees are walked +// at once. Each walk is dominated by network-filesystem round trips, so a few +// in parallel bring a many-agent pass back inside the scan interval without +// flooding the mount. +const transcriptScanConcurrency = 4 + +// scanAgentTranscript walks one agent's session tree; a variable so tests can +// observe how the passes are scheduled. +var scanAgentTranscript = func(scan transcriptActivityScan, cache *transcriptSessionCache, now time.Time) (time.Time, bool) { + return newestCopilotTranscriptActivityCached(AgentHome(scan.name, scan.uid, "copilot"), cache, now) +} + +// scanTranscriptActivity stats each agent's sessions, up to +// transcriptScanConcurrency agents at a time. caches is keyed by agent name +// and may be nil (every session is stat'd). Results keep the order of scans. func scanTranscriptActivity(scans []transcriptActivityScan, caches map[string]*transcriptSessionCache, now time.Time) []transcriptActivityResult { - results := make([]transcriptActivityResult, 0, len(scans)) - for _, scan := range scans { - var cache *transcriptSessionCache + results := make([]transcriptActivityResult, len(scans)) + perAgent := make([]*transcriptSessionCache, len(scans)) + for i, scan := range scans { if caches != nil { - cache = caches[scan.name] + cache := caches[scan.name] if cache == nil { cache = &transcriptSessionCache{} caches[scan.name] = cache } + perAgent[i] = cache } - at, ok := newestCopilotTranscriptActivityCached(AgentHome(scan.name, scan.uid, "copilot"), cache, now) - results = append(results, transcriptActivityResult{name: scan.name, at: at, ok: ok}) } + sem := make(chan struct{}, transcriptScanConcurrency) + var wg sync.WaitGroup + for i, scan := range scans { + wg.Add(1) + sem <- struct{}{} + go func() { + defer wg.Done() + defer func() { <-sem }() + at, ok := scanAgentTranscript(scan, perAgent[i], now) + results[i] = transcriptActivityResult{name: scan.name, at: at, ok: ok} + }() + } + wg.Wait() return results } diff --git a/src/crux/agent/busy_visibility_test.go b/src/crux/agent/busy_visibility_test.go index a593473a33..c104f86b41 100644 --- a/src/crux/agent/busy_visibility_test.go +++ b/src/crux/agent/busy_visibility_test.go @@ -548,3 +548,42 @@ func TestNewestCopilotTranscriptActivityCachedSkipsColdSessions(t *testing.T) { t.Fatalf("cache after full rescan = %+v, want no-events dropped and lastFull=%v", cache, full) } } + +// A pass over many agents must walk several at once; sequential walks are what +// pushed a 15-agent pass to 13 s against a 10 s interval. +func TestScanTranscriptActivityWalksAgentsConcurrently(t *testing.T) { + orig := scanAgentTranscript + t.Cleanup(func() { scanAgentTranscript = orig }) + + started := make(chan struct{}, transcriptScanConcurrency) + release := make(chan struct{}) + scanAgentTranscript = func(scan transcriptActivityScan, _ *transcriptSessionCache, _ time.Time) (time.Time, bool) { + started <- struct{}{} + <-release + return time.Unix(int64(scan.uid), 0), true + } + var scans []transcriptActivityScan + for i := range 2 * transcriptScanConcurrency { + scans = append(scans, transcriptActivityScan{name: string(rune('a' + i)), uid: i + 1}) + } + done := make(chan []transcriptActivityResult) + go func() { done <- scanTranscriptActivity(scans, map[string]*transcriptSessionCache{}, time.Time{}) }() + + for range transcriptScanConcurrency { + select { + case <-started: + case <-time.After(10 * time.Second): + t.Fatal("agent walks did not overlap") + } + } + close(release) + results := <-done + if len(results) != len(scans) { + t.Fatalf("got %d results, want %d", len(results), len(scans)) + } + for i, r := range results { + if r.name != scans[i].name || !r.ok || r.at.Unix() != int64(scans[i].uid) { + t.Fatalf("result %d = %+v, want agent %s", i, r, scans[i].name) + } + } +} diff --git a/src/pkg/fleetreport/runtime.go b/src/pkg/fleetreport/runtime.go index 6893ffc1e7..f9ccda5916 100644 --- a/src/pkg/fleetreport/runtime.go +++ b/src/pkg/fleetreport/runtime.go @@ -85,6 +85,7 @@ const ( ClassGitHubFetchTransient = "github_fetch_transient" ClassLabelProvisioningFailed = "label_provisioning_failed" ClassAgentCLIRestartLoop = "agent_cli_restart_loop" + ClassTranscriptScanSlow = "transcript_scan_slow" // A restart or normal stop can invalidate a pending bootstrap without // indicating a restart storm; only repeated interruptions are reportable. ClassStartupKickInterrupted = "startup_kick_interrupted" @@ -196,6 +197,7 @@ var Classes = map[string]ClassSpec{ ClassGitHubFetchTransient: {Component: "github-client", Severity: "low", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"channel commit date: fetch failed"}}, ClassLabelProvisioningFailed: {Component: "github-client", Severity: "low", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"issue stage labels: provisioning failed"}}, ClassAgentCLIRestartLoop: {Component: "agent-runtime", Severity: "medium", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"agent cli crashed or stuck on consent screen"}}, + ClassTranscriptScanSlow: {Component: "agent-runtime", Severity: "low", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"transcript activity scan outlived its interval"}}, } // LogFedClass returns the LogFed class whose Keyword msg contains, or "". diff --git a/src/pkg/fleetreport/runtime_test.go b/src/pkg/fleetreport/runtime_test.go index b6ce437af6..6cb05d202a 100644 --- a/src/pkg/fleetreport/runtime_test.go +++ b/src/pkg/fleetreport/runtime_test.go @@ -351,6 +351,7 @@ func TestEveryRuntimeClassTriggerAndNonTriggerThresholds(t *testing.T) { {class: ClassAgentDirSetupFailed, message: "failed to create per-agent XDG dir"}, {class: ClassReviewEvidenceWriteFail, message: "review evidence: could not write bundle"}, {class: ClassFeedbackIssueRefreshFailed, message: "feedback: issue refresh failed"}, + {class: ClassTranscriptScanSlow, message: "copilot transcript activity scan outlived its interval; agent liveness reads may lag"}, {class: ClassBeadFactIngestFailed, message: "failed to ingest bead fact"}, {class: ClassSelfReportClearedFailed, message: "fleet report: cleared comment failed"}, {class: ClassGitHubAppRepoNotCovered, message: "github app recheck: installation does not cover every configured repo"}, @@ -961,6 +962,7 @@ func TestRecurringRuntimeMessagesAreClassified(t *testing.T) { "github app recheck: installation does not cover every configured repo": ClassGitHubAppRepoNotCovered, "proxy: CONNECT dial failed": ClassProxyConnectDialFailed, "inference upstream error": ClassInferenceUpstreamError, + "copilot transcript activity scan outlived its interval; agent liveness reads may lag": ClassTranscriptScanSlow, "inference upstream failed": ClassInferenceUpstreamError, } for msg, want := range fed { From 9f711c17b672258fa9c2a5a78044c4c2b35bf3fb Mon Sep 17 00:00:00 2001 From: "hivecommons-hive[bot]" <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Tue, 27 Oct 2026 18:36:44 +0000 Subject: [PATCH 2/4] changelog: fragment for #14670 Generated by changelog-fragment-guard from the PR title (#5675). Edit the wording freely, or replace this file with the no-changelog label if the change is not user-visible. Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Signed-off-by: hivecommons-hive[bot] <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- changelog.d/fixed-14670.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/fixed-14670.md diff --git a/changelog.d/fixed-14670.md b/changelog.d/fixed-14670.md new file mode 100644 index 0000000000..f5c1cc1507 --- /dev/null +++ b/changelog.d/fixed-14670.md @@ -0,0 +1 @@ +- fix(agent): keep the copilot transcript activity scan inside its interval (#14670) From 0c425c517d8fe7764a0abc965623be92b207a66d Mon Sep 17 00:00:00 2001 From: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Tue, 27 Oct 2026 14:40:48 -0400 Subject: [PATCH 3/4] chore(changelog): make the #14649 fragment an entry bullet and drop the duplicate Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- changelog.d/fixed-14649.md | 2 +- changelog.d/fixed-14670.md | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) delete mode 100644 changelog.d/fixed-14670.md diff --git a/changelog.d/fixed-14649.md b/changelog.d/fixed-14649.md index 2d6febe2bb..ab786c6843 100644 --- a/changelog.d/fixed-14649.md +++ b/changelog.d/fixed-14649.md @@ -1 +1 @@ -Agent transcript liveness scans now walk several agents in parallel and the slow-scan warning is classified, so hives with many agents no longer file it as an unclassified error ([#14649](https://github.com/hivecommons/hive/issues/14649)). +- Agent transcript liveness scans now walk several agents in parallel and the slow-scan warning is classified, so hives with many agents no longer file it as an unclassified error ([#14649](https://github.com/hivecommons/hive/issues/14649)). diff --git a/changelog.d/fixed-14670.md b/changelog.d/fixed-14670.md deleted file mode 100644 index f5c1cc1507..0000000000 --- a/changelog.d/fixed-14670.md +++ /dev/null @@ -1 +0,0 @@ -- fix(agent): keep the copilot transcript activity scan inside its interval (#14670) From 37ec4c26aaac71a98e3d8559efe30e28d75bc0e7 Mon Sep 17 00:00:00 2001 From: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Tue, 27 Oct 2026 16:57:26 -0400 Subject: [PATCH 4/4] docs(fleet-report): document the transcript_scan_slow runtime class Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- src/docs/fleet-report.md | 1 + 1 file changed, 1 insertion(+) diff --git a/src/docs/fleet-report.md b/src/docs/fleet-report.md index 551d1f1948..81a0f837b1 100644 --- a/src/docs/fleet-report.md +++ b/src/docs/fleet-report.md @@ -157,6 +157,7 @@ title is `[selfreport] : `. | `inference_upstream_error` | a failed or non-2xx response from the configured inference backend (LiteLLM, vLLM/llm-d) on the Anthropic-to-OpenAI translator or MITM reroute path (`inference upstream error` / `inference upstream failed`, `crux/proxy/github_proxy.go`), repeated ≥5 times in 10 min — covers transport failures and statuses other than the dedicated 401/403 auth signal. Log-fed: same repeat threshold and idle clear as `agent_pane_timeout` | medium | | `sha_poll_commit_message_unavailable` | the SHA poller's commit-message fetch got a non-200 from GitHub (typically a 403 rate limit or token scope; `SHA poll: commit message fetch non-200`, `pkg/hub/saas_sha_poller.go`). The poller backs off per SHA and keeps serving the SHA without a message. The WARN itself is deduped to once per hour per (SHA, status) (`warnGitHubFetchOnce`), so one stuck commit logs one line and never trips this class; it is reported only when ≥5 distinct SHA/status pairs fail within 10 min (a broad outage such as an exhausted rate limit or a revoked token). Its main job is keeping the residual WARN out of `unclassified_error_repeated`. Log-fed: same repeat threshold and idle clear as `agent_pane_timeout` | low | | `tmux_session_check_failed` | the agent's tmux session `has-session` check failed (`tmux has-session failed`, `crux/agent/manager.go`) — typically the session was lost or the tmux server is unhealthy, repeated ≥5 times in 10 min. Log-fed: same repeat threshold and idle clear as `agent_pane_timeout` | medium | +| `transcript_scan_slow` | a Copilot transcript activity scan took longer than its interval (`transcript activity scan outlived its interval`, `crux/agent/busy_visibility.go`) — agent liveness reads may lag. Log-fed: same ≥5-in-10-min repeat threshold and 20-minute idle clear as `agent_pane_timeout`. Operator diagnostics: check the agent's transcript/session volume; it is bounded by concurrency, so repeated hits indicate slow disk or very large session directories | low | | `unclassified_error_repeated` | **generic catch-all**: the same WARN/ERROR log line (numbers, hashes, paths and quoted strings collapsed) repeated ≥5 times in 10 min. Fed by a `slog.Handler` tee (`pkg/fleetreport/slogfeed`) on the hive's root logger, so a repeating defect is noticed even with no hook site. Skips benign lines (read/keepalive i/o timeouts, context cancellation, the permissions watcher's `fixed …` self-healing lines) and any message that already matches another class's keywords. Clears automatically after 20 quiet minutes (2× the window), so a "cleared" comment follows once the symptom stops | low | The generic feed also skips the exact `review pending expired` message: it