diff --git a/CHANGELOG.md b/CHANGELOG.md index 4f24596..364f89d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,3 +14,26 @@ Initial implementation. model-floor pre-flight, budget warn/degrade/refuse bands, run journals, artifact discovery + schema validation + rendering, and spend declaration. - Read-only report browsing (`report`) and ledger views (`ledger`). + +### Fixes from first field use + +- Budget policy parsing accepts the shipped sequence-form `bands:` + (`- {at: 1.5, action: warn}`) in addition to the map form. +- Harness failures now surface the stderr tail (provider 401s are visible + without opening the journal) and are recorded in `result.json` as + `harness_error`. +- Pre-flight model availability check (crush: verified against + `crush models`): unrunnable models fail fast with actionable guidance + instead of an authentication error two seconds into the run. +- The run monitor no longer looks frozen on quiet harnesses: the status bar + shows how long since the last output, and ctrl+c requires a confirming + second press within six seconds — a single stray ctrl+c cannot kill a + long-running audit. +- Force-quitting the monitor (or any early exit) now tears the harness down + and joins the runner before returning: no more "run produced no result", + and no send-on-closed-channel races. +- Failed, refused and aborted runs still discover and validate their + artifacts — partial output and checkpoint state are visible exactly when a + run dies mid-flight. +- The run summary (artifacts, journal path, harness error, final message) + prints after the interactive monitor exits, not only in plain mode. diff --git a/cmd/run.go b/cmd/run.go index 9fcf2bb..faaeef4 100644 --- a/cmd/run.go +++ b/cmd/run.go @@ -174,7 +174,9 @@ func executeRun(state *bootstrap.State, skill *catalog.Skill, args []string) { if runFlags.json { out, _ := json.MarshalIndent(res, "", " ") fmt.Println(string(out)) - } else if !interactive { + } else { + // The summary prints after the monitor too: artifacts, journal + // location and the failure reason must not vanish with the TUI. printRunSummary(res) } os.Exit(res.ExitCode) @@ -192,6 +194,11 @@ func printRunSummary(res *runner.RunResult) { if res.BudgetCap > 0 { kv("budget", fmt.Sprintf("%s band · cap %d output tokens", res.BudgetBand, res.BudgetCap)) } + if res.HarnessError != "" { + fmt.Println() + fmt.Println(styles.Err.Render("harness error:")) + fmt.Println(indent(res.HarnessError, " ")) + } if len(res.Artifacts) > 0 { fmt.Println() for _, a := range res.Artifacts { diff --git a/internal/harness/adapters_test.go b/internal/harness/adapters_test.go index a625def..24afaf9 100644 --- a/internal/harness/adapters_test.go +++ b/internal/harness/adapters_test.go @@ -135,6 +135,26 @@ func TestBinaryEnvOverride(t *testing.T) { } } +func TestMatchCrushModel(t *testing.T) { + listing := "PriceTag/rits/zai-org/glm-5-3\nPriceTag/Inferact/Qwen3.8-Flash-Next\nanthropic/claude-mythos-5\n" + if err := matchCrushModel(listing, "claude-mythos-5"); err != nil { + t.Errorf("exact provider-prefixed match should pass: %v", err) + } + if err := matchCrushModel(listing, "rits/zai-org/glm-5-3"); err != nil { + t.Errorf("exact match should pass: %v", err) + } + if err := matchCrushModel(listing, "glm-5-3"); err != nil { + t.Errorf("suffix match should pass: %v", err) + } + if err := matchCrushModel(listing, "claude-mythos-6"); err == nil { + t.Error("unknown model should fail") + } + // A suffix must not match mid-segment: glm-5-30 is a different model. + if err := matchCrushModel(listing, "glm-5-30"); err == nil { + t.Error("mid-segment suffix should not match") + } +} + func contains(list []string, want string) bool { for _, v := range list { if v == want { diff --git a/internal/harness/crush.go b/internal/harness/crush.go index 33454a2..72c6e79 100644 --- a/internal/harness/crush.go +++ b/internal/harness/crush.go @@ -2,8 +2,10 @@ package harness import ( "context" + "fmt" "os/exec" "strings" + "time" ) // Crush is the Charm Crush adapter (`crush run`). Crush's run mode emits the @@ -45,3 +47,37 @@ func (c *Crush) Notes() []string { // ParseEvent always returns false: crush run output is unstructured. func (c *Crush) ParseEvent(line []byte) (Event, bool) { return Event{}, false } + +// CheckModel verifies the model appears in `crush models` before launch. +// crush accepts "model" or "provider/model", and silently routes unknown +// ids to its default provider — which turns an unrunnable registry floor +// model into an authentication failure two seconds into a run. Failing +// here, with the list in hand, is the cheap time to find out. +func (c *Crush) CheckModel(ctx context.Context, model string) error { + if model == "" { + return nil + } + cctx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + out, err := exec.CommandContext(cctx, c.Binary(), "models").Output() + if err != nil { + // The listing is advisory: never block a run on it. + return nil + } + return matchCrushModel(string(out), model) +} + +// matchCrushModel reports whether model is runnable per a `crush models` +// listing (exact or provider-prefixed match). +func matchCrushModel(listing, model string) error { + for _, line := range strings.Split(listing, "\n") { + line = strings.TrimSpace(line) + if line == "" { + continue + } + if line == model || strings.HasSuffix(line, "/"+model) { + return nil + } + } + return fmt.Errorf("model %q is not in `crush models`", model) +} diff --git a/internal/harness/harness.go b/internal/harness/harness.go index fe5bd34..7c4d022 100644 --- a/internal/harness/harness.go +++ b/internal/harness/harness.go @@ -100,6 +100,15 @@ type Harness interface { Notes() []string } +// ModelChecker is implemented by harnesses that can verify a model id is +// runnable before the process launches. The runner uses it to fail fast +// with actionable guidance — a registry floor model the local harness +// cannot authenticate to otherwise dies two seconds later with an opaque +// provider 401. +type ModelChecker interface { + CheckModel(ctx context.Context, model string) error +} + // IDs returns the harness priority order. func IDs() []string { return []string{"claude", "codex", "pi", "opencode", "crush", "goose"} diff --git a/internal/runner/budget.go b/internal/runner/budget.go index bac0513..5f22a3c 100644 --- a/internal/runner/budget.go +++ b/internal/runner/budget.go @@ -57,11 +57,10 @@ type rawPolicy struct { Units string `yaml:"units"` PerRunDefaults map[string]int64 `yaml:"per_run_defaults"` PerRunDefault *int64 `yaml:"per_run_default"` - Bands *struct { - Warn *float64 `yaml:"warn"` - Degrade *float64 `yaml:"degrade"` - Refuse *float64 `yaml:"refuse"` - } `yaml:"bands"` + // Bands accepts both shipped shapes: the map form ({warn: 1.5, ...}) + // and the sequence form the harness actually ships + // (- {at: 1.5, action: warn}). + Bands any `yaml:"bands"` } type policyFile struct { @@ -116,17 +115,55 @@ func LoadPolicy(configHome string) (*Policy, error) { delete(p.PerRun, "default") } if rp.Bands != nil { - if rp.Bands.Warn != nil { - p.Warn = *rp.Bands.Warn + applyBands(p, rp.Bands) + } + return p, nil +} + +// applyBands folds the two shipped band shapes into the policy. +func applyBands(p *Policy, bands any) { + set := func(action string, at float64) { + switch action { + case "warn": + p.Warn = at + case "degrade": + p.Degrade = at + case "refuse": + p.Refuse = at } - if rp.Bands.Degrade != nil { - p.Degrade = *rp.Bands.Degrade + } + switch b := bands.(type) { + case []any: // sequence form: - {at: 1.5, action: warn} + for _, entry := range b { + m, ok := entry.(map[string]any) + if !ok { + continue + } + set(toString(m["action"]), toFloat(m["at"])) } - if rp.Bands.Refuse != nil { - p.Refuse = *rp.Bands.Refuse + case map[string]any: // map form: {warn: 1.5, degrade: 2.0, refuse: 3.0} + for action, v := range b { + set(action, toFloat(v)) } } - return p, nil +} + +func toFloat(v any) float64 { + switch n := v.(type) { + case float64: + return n + case int: + return float64(n) + case int64: + return float64(n) + default: + return 0 + } +} + +func toString(v any) string { + s, _ := v.(string) + return s } // CapFor returns the per-run output-token cap for a skill. diff --git a/internal/runner/budget_test.go b/internal/runner/budget_test.go index 6270f73..b95b139 100644 --- a/internal/runner/budget_test.go +++ b/internal/runner/budget_test.go @@ -71,6 +71,36 @@ per_run_defaults: } } +// TestLoadPolicySequenceBands covers the shape the harness actually ships: +// bands as a list of {at, action} entries. +func TestLoadPolicySequenceBands(t *testing.T) { + dir := writePolicy(t, ` +budget_policy: + enforcement: enforced + units: output_tokens + per_run_defaults: + secure-code-audit: 2_500_000 + default: 1_000_000 + bands: + - at: 1.5 + action: warn + - at: 2.0 + action: degrade + - at: 3.0 + action: refuse +`) + p, err := LoadPolicy(dir) + if err != nil { + t.Fatal(err) + } + if p.Warn != 1.5 || p.Degrade != 2.0 || p.Refuse != 3.0 { + t.Errorf("sequence bands = %v/%v/%v", p.Warn, p.Degrade, p.Refuse) + } + if got := p.CapFor("secure-code-audit"); got != 2_500_000 { + t.Errorf("cap = %d", got) + } +} + func TestBandFor(t *testing.T) { p := &Policy{DefaultCap: 1000, Warn: 1.5, Degrade: 2.0, Refuse: 3.0} cases := []struct { diff --git a/internal/runner/runner.go b/internal/runner/runner.go index 04b35aa..237ddae 100644 --- a/internal/runner/runner.go +++ b/internal/runner/runner.go @@ -97,8 +97,12 @@ type RunResult struct { BudgetRefused bool `json:"budget_refused"` Aborted bool `json:"aborted"` HarnessExit int `json:"harness_exit"` - ResultText string `json:"result_text,omitempty"` - ExitCode int `json:"exit_code"` + // HarnessError carries the process failure plus the tail of the + // harness stderr log, so the actual provider error (e.g. a 401) is + // visible without opening the journal by hand. + HarnessError string `json:"harness_error,omitempty"` + ResultText string `json:"result_text,omitempty"` + ExitCode int `json:"exit_code"` } // Run executes the skill through the harness. @@ -150,6 +154,31 @@ func Run(ctx context.Context, opts Options, sink Sink) (*RunResult, error) { return nil, err } res.Dir = journal.Dir + + // Pre-flight model availability: when the harness can verify model ids, + // fail fast with guidance instead of an opaque provider 401 two + // seconds into the run. + if pf.Model != "" { + if mc, ok := h.(harness.ModelChecker); ok { + if err := mc.CheckModel(ctx, pf.Model); err != nil { + msg := fmt.Sprintf("model %q is not runnable in %s (%v)", pf.Model, h.DisplayName(), err) + if pf.Role != "" { + msg += fmt.Sprintf(" — the registry floor for role %q resolved to it; map the role to a model your harness can run in %s (traust registry models resolve %s), or pass --model", pf.Role, filepath.Join(state.ConfigHome, "model-registry.yaml"), pf.Role) + } else { + msg += " — pass a --model your harness can run" + } + res.ExitCode = ExitError + res.HarnessError = msg + sink.Progress(msg) + if jerr := journal.WriteResult(res); jerr != nil { + sink.Progress("warning: could not write run journal: " + jerr.Error()) + } + sink.Done() + return res, nil + } + } + } + before := snapshotArtifacts(root) // Capability caveats, up front. @@ -238,7 +267,11 @@ func Run(ctx context.Context, opts Options, sink Sink) (*RunResult, error) { sink.Progress("run aborted") case waitErr != nil: res.ExitCode = ExitError - sink.Progress(fmt.Sprintf("harness exited with error: %v (stderr: %s)", waitErr, stderrPath)) + res.HarnessError = waitErr.Error() + if tail := fileTail(stderrPath, 12); tail != "" { + res.HarnessError += "\n" + tail + } + sink.Progress("harness exited with error: " + res.HarnessError) case scanErr != nil: res.ExitCode = ExitError sink.Progress(fmt.Sprintf("stream read error: %v", scanErr)) @@ -248,9 +281,11 @@ func Run(ctx context.Context, opts Options, sink Sink) (*RunResult, error) { res.ResultText = strings.TrimRight(plainOut.String(), "\n") } - // Post-run: artifacts, validation, rendering, spend — unless the run was - // aborted or refused before producing trustworthy output. - if res.ExitCode == ExitOK || (res.ExitCode == ExitError && res.HarnessExit == 0) { + // Post-run: artifacts, validation, rendering, spend. Discovery runs even + // for failed, refused or aborted runs — partial outputs are exactly what + // the user needs to see when a run dies mid-flight, and skills checkpoint + // their state for resume. + { after := snapshotArtifacts(root) res.Artifacts = discoverArtifacts(root, before, after) validateArtifacts(ctx, opts.API, res.Artifacts, opts.StrictValidation) @@ -290,6 +325,19 @@ func Run(ctx context.Context, opts Options, sink Sink) (*RunResult, error) { return res, nil } +// fileTail returns the last n lines of a file, or "" when unreadable. +func fileTail(path string, n int) string { + data, err := os.ReadFile(path) + if err != nil { + return "" + } + lines := strings.Split(strings.TrimRight(string(data), "\n"), "\n") + if len(lines) > n { + lines = lines[len(lines)-n:] + } + return strings.Join(lines, "\n") +} + // applyEvent folds a stream event into the run result. Usage is cumulative in // every reporting harness, so tokens are maxed, not summed. func applyEvent(res *RunResult, ev harness.Event) { diff --git a/internal/runner/runner_test.go b/internal/runner/runner_test.go index bffc853..3ba4617 100644 --- a/internal/runner/runner_test.go +++ b/internal/runner/runner_test.go @@ -3,6 +3,7 @@ package runner import ( "context" "encoding/json" + "fmt" "os" "path/filepath" "strings" @@ -341,6 +342,106 @@ func TestPrimaryArtifact(t *testing.T) { } } +// modelCheckHarness wraps the claude adapter with a model check result, to +// exercise the pre-flight availability path without a real harness. +type modelCheckHarness struct { + *harness.Claude + checkErr error +} + +func (m modelCheckHarness) CheckModel(_ context.Context, _ string) error { return m.checkErr } + +func TestRunModelCheckFailsFast(t *testing.T) { + t.Setenv("TRAUST_CLI_CLAUDE_BIN", fakeHarnessPath(t, "fake-claude.sh")) + state := testState(t) + client := &fakeAPI{} + sink := &recSink{} + + res, err := Run(context.Background(), Options{ + State: state, API: client, Harness: modelCheckHarness{ + &harness.Claude{}, fmt.Errorf("model %q is not in `crush models`", "claude-mythos-5"), + }, + Skill: testSkill(), DeclareSpend: true, + }, sink) + if err != nil { + t.Fatal(err) + } + if res.ExitCode != ExitError { + t.Fatalf("exit = %d, want 1", res.ExitCode) + } + if res.Artifacts != nil || res.SpendRecorded || res.HarnessExit != 0 { + t.Errorf("harness should never have launched: %+v", res) + } + if !strings.Contains(res.HarnessError, "not runnable") || !strings.Contains(res.HarnessError, "model-registry.yaml") { + t.Errorf("harness error lacks guidance: %q", res.HarnessError) + } + warned := false + for _, p := range sink.progress { + if strings.Contains(p, "not runnable") { + warned = true + } + } + if !warned { + t.Error("model check failure not surfaced to the sink") + } + if _, err := os.Stat(filepath.Join(res.Dir, "result.json")); err != nil { + t.Errorf("failed pre-flight should still journal: %v", err) + } +} + +func TestRunHarnessErrorSurfacesStderrTail(t *testing.T) { + t.Setenv("TRAUST_CLI_CLAUDE_BIN", fakeHarnessPath(t, "fake-claude.sh")) + t.Setenv("FAKE_CLAUDE_MODE", "fail") + state := testState(t) + sink := &recSink{} + + res, err := Run(context.Background(), Options{ + State: state, API: &fakeAPI{}, Harness: &harness.Claude{}, + Skill: testSkill(), DeclareSpend: true, + }, sink) + if err != nil { + t.Fatal(err) + } + if res.ExitCode != ExitError || res.HarnessExit != 3 { + t.Fatalf("exit = %d, harness exit = %d", res.ExitCode, res.HarnessExit) + } + if !strings.Contains(res.HarnessError, "401 Unauthorized") { + t.Errorf("harness error should carry the stderr tail: %q", res.HarnessError) + } + surfaced := false + for _, p := range sink.progress { + if strings.Contains(p, "401 Unauthorized") { + surfaced = true + } + } + if !surfaced { + t.Error("provider error not surfaced to the sink") + } +} + +func TestRunFailedRunStillDiscoversArtifacts(t *testing.T) { + t.Setenv("TRAUST_CLI_CLAUDE_BIN", fakeHarnessPath(t, "fake-claude.sh")) + t.Setenv("FAKE_CLAUDE_MODE", "partial") + state := testState(t) + + res, err := Run(context.Background(), Options{ + State: state, API: &fakeAPI{}, Harness: &harness.Claude{}, + Skill: testSkill(), DeclareSpend: true, + }, nil) + if err != nil { + t.Fatal(err) + } + if res.ExitCode != ExitError || res.HarnessExit != 1 { + t.Fatalf("exit = %d, harness exit = %d", res.ExitCode, res.HarnessExit) + } + if len(res.Artifacts) != 1 || !strings.HasSuffix(res.Artifacts[0].Path, "fixture-security-audit.json") { + t.Fatalf("partial output should still be discovered: %+v", res.Artifacts) + } + if !res.Artifacts[0].Valid { + t.Errorf("partial artifact should still validate: %+v", res.Artifacts[0]) + } +} + func TestPreflightNoRole(t *testing.T) { skill := testSkill() skill.Standalone = &catalog.StandaloneInfo{MinModelClass: "opus"} diff --git a/internal/tui/monitor.go b/internal/tui/monitor.go index bd34ad7..bc1eb1a 100644 --- a/internal/tui/monitor.go +++ b/internal/tui/monitor.go @@ -40,24 +40,49 @@ type monitorItem struct { text string } -// monitorSink forwards run updates into the monitor's channel. Done closes -// the channel exactly once so the relay terminates even on error paths. +// monitorSink forwards run updates into the monitor's channel. Sends are +// non-blocking (the journal keeps everything; the UI may drop under +// backpressure) and guarded against a close, so the runner goroutine can +// never panic or deadlock when the program exits before the run does. type monitorSink struct { - ch chan monitorItem - close sync.Once + ch chan monitorItem + mu sync.Mutex + closed bool + once sync.Once +} + +func (s *monitorSink) send(it monitorItem) { + s.mu.Lock() + defer s.mu.Unlock() + if s.closed { + return + } + select { + case s.ch <- it: + default: // UI backpressure: drop; the journal keeps everything. + } } func (s *monitorSink) Progress(msg string) { - s.ch <- monitorItem{kind: itemProgress, text: msg} + s.send(monitorItem{kind: itemProgress, text: msg}) } func (s *monitorSink) Event(ev harness.Event) { - s.ch <- monitorItem{kind: itemEvent, ev: ev, text: ev.Text} + s.send(monitorItem{kind: itemEvent, ev: ev, text: ev.Text}) } func (s *monitorSink) Raw(line string) { - s.ch <- monitorItem{kind: itemRaw, text: line} + s.send(monitorItem{kind: itemRaw, text: line}) } -func (s *monitorSink) Done() { - s.close.Do(func() { close(s.ch) }) +func (s *monitorSink) Done() { s.Close() } + +// Close tears the channel down exactly once, whether the runner finished +// cleanly or the program exited first. +func (s *monitorSink) Close() { + s.once.Do(func() { + s.mu.Lock() + s.closed = true + close(s.ch) + s.mu.Unlock() + }) } type tickMsg time.Time @@ -74,10 +99,18 @@ type monitorModel struct { done bool failed bool aborting bool - started time.Time - width, height int + // abortArmed is the ctrl+c confirmation state: the first press asks, + // the second (within armWindow) actually aborts. A single stray ctrl+c + // must not kill a 10-minute audit that merely looks quiet. + abortArmed bool + abortArmedAt time.Time + lastOutput time.Time + started time.Time + width, height int } +const armWindow = 6 * time.Second + func newMonitor(header MonitorHeader, cancel func(), width, height int) monitorModel { vp := viewport.New(width-2, height-8) return monitorModel{ @@ -94,8 +127,13 @@ func (m monitorModel) Init() tea.Cmd { func (m monitorModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) { switch msg := msg.(type) { case tickMsg: + // Disarm the abort confirmation when the window passes. + if m.abortArmed && time.Since(m.abortArmedAt) > armWindow { + m.abortArmed = false + } return m, tea.Tick(time.Second, func(t time.Time) tea.Msg { return tickMsg(t) }) case monitorItem: + m.lastOutput = time.Now() switch msg.kind { case itemEvent: m.applyEvent(msg.ev) @@ -110,6 +148,10 @@ func (m monitorModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) { style = m.styles.Err m.failed = true } + if strings.Contains(msg.text, "is not runnable") { + style = m.styles.Err + m.failed = true + } if rest, ok := strings.CutPrefix(msg.text, "budget band "); ok { if fields := strings.Fields(rest); len(fields) > 0 { m.band = fields[0] @@ -129,14 +171,25 @@ func (m monitorModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) { case tea.KeyMsg: switch msg.String() { case "ctrl+c": + if m.done { + return m, tea.Quit + } if m.aborting { + // Second press while already aborting: leave the view; the + // runner keeps tearing down in the background. return m, tea.Quit } - m.aborting = true - m.pushLine(m.styles.Warn.Render("› aborting — terminating the harness…")) - if m.cancel != nil { - m.cancel() + if m.abortArmed { + m.aborting = true + m.pushLine(m.styles.Warn.Render("› aborting — terminating the harness…")) + if m.cancel != nil { + m.cancel() + } + return m, nil } + m.abortArmed = true + m.abortArmedAt = time.Now() + m.pushLine(m.styles.Warn.Render("› press ctrl+c again to abort the run")) return m, nil case "q", "enter", "esc": if m.done { @@ -207,11 +260,18 @@ func (m monitorModel) View() string { b.WriteString(frame) b.WriteString("\n") - // Status bar: usage + budget bar. + // Status bar: usage + budget bar + output age. The age indicator is the + // answer to "is it stuck?": quiet harnesses (crush run emits one large + // chunk every few minutes) look frozen without it. usage := fmt.Sprintf("in %s · out %s", human(m.tokensIn), human(m.tokensOut)) if m.cost > 0 { usage += fmt.Sprintf(" · $%.2f", m.cost) } + if !m.done && !m.lastOutput.IsZero() { + if age := time.Since(m.lastOutput).Round(time.Second); age > 15*time.Second { + usage += fmt.Sprintf(" · last output %s ago", age) + } + } var bar string if m.header.BudgetCap > 0 { bar = m.budgetBar(m.tokensOut, m.header.BudgetCap) @@ -220,7 +280,13 @@ func (m monitorModel) View() string { b.WriteString(m.styles.StatusBar.Render(usage)) b.WriteString("\n") - footer := "ctrl+c abort" + footer := "ctrl+c×2 abort" + if m.abortArmed && !m.aborting { + footer = "press ctrl+c again to abort" + } + if m.aborting { + footer = "aborting… (ctrl+c again to leave)" + } if m.done { footer = "q quit · journal written" } @@ -255,7 +321,10 @@ func (m monitorModel) budgetBar(out, cap int64) string { } // RunMonitor runs the tea program while runFn executes the pipeline with a -// sink wired to the monitor. cancel is invoked when the user hits ctrl+c. +// sink wired to the monitor. cancel is invoked when the user confirms an +// abort. It only returns after runFn has finished (or timed out), so the +// caller can rely on the run result being complete — force-quitting the +// view tears the harness down first. func RunMonitor(header MonitorHeader, cancel func(), runFn func(sink runner.Sink) error) error { items := make(chan monitorItem, 128) sink := &monitorSink{ch: items} @@ -263,25 +332,39 @@ func RunMonitor(header MonitorHeader, cancel func(), runFn func(sink runner.Sink prog := tea.NewProgram(newMonitor(header, cancel, 80, 24), tea.WithAltScreen(), tea.WithMouseCellMotion()) - var runErr error - var once sync.Once - finish := func() { once.Do(func() { close(items) }) } - + runDone := make(chan error, 1) go func() { - runErr = runFn(sink) - finish() + err := runFn(sink) + sink.Close() // terminate the relay even on early error paths + runDone <- err }() + relayDone := make(chan struct{}) go func() { + defer close(relayDone) for it := range items { prog.Send(it) } prog.Send(monitorItem{kind: itemDone}) }() - if _, err := prog.Run(); err != nil { - return err + _, err := prog.Run() + // The program is gone: make sure a still-running harness is torn down, + // then join the runner so the result is complete before returning. + if cancel != nil { + cancel() + } + select { + case runErr := <-runDone: + if err != nil { + return err + } + return runErr + case <-time.After(30 * time.Second): + if err != nil { + return err + } + return fmt.Errorf("run did not shut down within 30s of the monitor exiting") } - return runErr } func truncateLine(s string, n int) string { diff --git a/testdata/fake-harnesses/fake-claude.sh b/testdata/fake-harnesses/fake-claude.sh index f63ceaf..905f02d 100755 --- a/testdata/fake-harnesses/fake-claude.sh +++ b/testdata/fake-harnesses/fake-claude.sh @@ -3,7 +3,10 @@ # # Consumes the prompt on stdin, emits a canned stream-json transcript, and # writes a fixture report into the working directory (the fake harness root). -# FAKE_CLAUDE_MODE=needs-input emits an AskUserQuestion tool_use instead. +# FAKE_CLAUDE_MODE selects a scenario: +# needs-input emits an AskUserQuestion tool_use +# fail writes an error to stderr and exits 3 +# partial writes a report but exits 1 (failed run, partial output) set -eu cat > /dev/null @@ -12,7 +15,16 @@ if [ "${FAKE_CLAUDE_MODE:-}" = "needs-input" ]; then exit 0 fi +if [ "${FAKE_CLAUDE_MODE:-}" = "fail" ]; then + echo "ERROR Agent processing failed: 401 Unauthorized invalid x-api-key" >&2 + exit 3 +fi + printf '%s\n' '{"type":"assistant","message":{"role":"assistant","content":[{"type":"text","text":"starting audit"}]}}' printf '%s\n' '{"type":"assistant","message":{"content":[{"type":"tool_use","name":"Bash","input":{}}]}}' printf '%s\n' '{"type":"result","subtype":"success","result":"audit complete","usage":{"input_tokens":100,"output_tokens":50},"total_cost_usd":0.42}' printf '{"findings": [], "metadata": {"harness_version": "test"}}' > fixture-security-audit.json + +if [ "${FAKE_CLAUDE_MODE:-}" = "partial" ]; then + exit 1 +fi