From 0d61d8f5e12b060185f09d6c87feedde2ee549c8 Mon Sep 17 00:00:00 2001 From: Gurney Buchanan Date: Wed, 21 Oct 2026 14:56:29 -0400 Subject: [PATCH] Install the secure-code-audit toolchain by default, pre-flight per run MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The init wizard asked users to pick a scanner toolchain profile before they could know what they would run, and the answer is deterministic anyway: the secure-code-audit profile's toolset is a superset of the vuln-scan and triage profiles, so one profile covers every stage of every flow (audit and verify use its scanners; triage and patch need nothing beyond it). init now installs it by default β€” the wizard question is gone, and --toolchain none remains the explicit opt-out. Already-installed profiles are skipped so re-running init stays fast and idempotent. With the profile guaranteed by default, runs can hold it to account: every run (and therefore every flow stage) whose skill has a toolchain profile gets a cheap admin-toolchain-doctor pre-flight, and an incomplete profile surfaces as a non-fatal warning naming the profile, the coverage cost, and the fix β€” a degraded audit now says why it is degraded instead of silently producing thinner evidence. Also: run ids widened to a 6-hex random suffix β€” the 4-hex suffix collided ~8% of the time once a dozen runs shared a second (found by the uniqueness test going flaky in CI-race runs), and a collision meant a shared run directory. Verified live against the real workspace: the pre-flight fired on its incomplete profile, and the default install brought seven of eight tools healthy (one version-pinned upgrade left to the operator). Tests cover the warning path (non-fatal, names the fix), the clean path, and id uniqueness across repeated suites. πŸ’˜ Generated with Crush Assisted-by: Crush:rits/zai-org/glm-5-3 --- CHANGELOG.md | 13 +++++ README.md | 2 +- cmd/background.go | 11 ++-- cmd/init.go | 3 +- cmd/worker_test.go | 2 +- internal/api/api.go | 9 +++ internal/bootstrap/bootstrap.go | 36 ++++++++---- internal/catalog/catalog.go | 3 + internal/catalog/data/standalone.yaml | 8 +++ internal/runner/runner.go | 15 +++++ internal/runner/runner_test.go | 81 ++++++++++++++++++++++++--- internal/tui/init.go | 32 +++-------- 12 files changed, 165 insertions(+), 50 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 672c273..1de1905 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -109,3 +109,16 @@ Initial implementation. recommended choice) and its warning spells out that runs and flows (audit β†’ triage β†’ patch β†’ verify) fail closed without it; doctor's scoped-mode message says the same. +- init installs the secure-code-audit scanner toolchain by default (one + profile covers every flow stage β€” a superset of the vuln-scan and triage + profiles) and no longer asks: the wizard question was a decision users + could not make before knowing what they would run. --toolchain none + stays for opt-out, and already-installed profiles are skipped so re-init + stays fast. +- Runs pre-flight the skill's toolchain profile (admin toolchain doctor) + and surface a non-fatal warning naming the profile, the coverage cost, + and the fix when it is incomplete β€” degraded audits now say why they + are degraded instead of silently producing thinner evidence. +- Run ids widened to a 6-hex random suffix: the 4-hex suffix collided + ~8% of the time once a dozen runs shared a second, and a collision + meant a shared run directory. diff --git a/README.md b/README.md index a53cefd..dbdf75e 100644 --- a/README.md +++ b/README.md @@ -73,7 +73,7 @@ need is an agent harness you are logged into. | Command | What it does | |---|---| -| `init` | Bootstrap the managed workspace: pinned uv (checksum-verified, never `curl \| sh`), traust checkout at a stable tag, `uv sync`, `TRAUST_CONFIG_HOME` from templates via `install_traust`, optional toolchain profile. Idempotent; nothing is written to your shell profile (use `eval "$(traust-cli env)"` if you want it). | +| `init` | Bootstrap the managed workspace: pinned uv (checksum-verified, never `curl \| sh`), traust checkout at a stable tag, `uv sync`, `TRAUST_CONFIG_HOME` from templates via `install_traust`, and the `secure-code-audit` scanner toolchain (covers every flow stage; `--toolchain none` skips, already-installed profiles are skipped). Idempotent; nothing is written to your shell profile (use `eval "$(traust-cli env)"` if you want it). | | `doctor` | CLI checks + `install_traust --doctor`; `--json` for CI. | | `update` | Move the pin to a newer tag and re-sync. | | `skills` | Catalog parsed from the same SKILL.md frontmatter the harness reads; TUI picker on a TTY, table otherwise; `--json`. | diff --git a/cmd/background.go b/cmd/background.go index 31a4cc9..7f5343a 100644 --- a/cmd/background.go +++ b/cmd/background.go @@ -33,8 +33,11 @@ type runRequest struct { } // newRunID mints a collision-safe run id: timestamp + skill + random suffix. +// The suffix is 3 bytes (6 hex chars): a 2-byte suffix collides ~8% of the +// time once a dozen runs share a second (birthday paradox), and a collision +// means a shared run directory. func newRunID(skill string) (string, error) { - buf := make([]byte, 2) + buf := make([]byte, 3) if _, err := rand.Read(buf); err != nil { return "", err } @@ -129,10 +132,10 @@ func shortRunID(runID string) string { return runID } -// isHexSuffix reports whether s is a 4-character hex string (the random -// suffix minted by newRunID). +// isHexSuffix reports whether s is a short hex string (4-6 characters) β€” +// the random suffix minted by newRunID, in either width. func isHexSuffix(s string) bool { - if len(s) != 4 { + if len(s) < 4 || len(s) > 6 { return false } for _, r := range s { diff --git a/cmd/init.go b/cmd/init.go index 2057ae1..616863c 100644 --- a/cmd/init.go +++ b/cmd/init.go @@ -49,7 +49,7 @@ func init() { initCmd.Flags().StringVar(&initOpts.forge, "forge", "", "git URL to clone traust from (default: the public traust repository)") initCmd.Flags().StringVar(&initOpts.ref, "ref", "latest", "tag or branch to pin (or \"latest\" for the newest stable tag)") initCmd.Flags().StringVar(&initOpts.configHome, "config-home", "", "TRAUST_CONFIG_HOME (default /config)") - initCmd.Flags().StringVar(&initOpts.toolchain, "toolchain", "none", "scanner toolchain profile (none, secure-code-audit, vuln-scan, triage, …)") + initCmd.Flags().StringVar(&initOpts.toolchain, "toolchain", "secure-code-audit", "scanner toolchain profile to install (secure-code-audit covers every flow stage; 'none' skips)") initCmd.Flags().StringVar(&initOpts.harness, "harness", "", "default agent harness (default: first detected)") initCmd.Flags().BoolVar(&initOpts.fullPermission, "full-permission", false, "consent to full-permission runs: harness executes with all tools auto-approved (required by the harness's skills)") initCmd.Flags().BoolVar(&initOpts.writeProfile, "write-profile", false, "also record TRAUST_CONFIG_HOME in your shell profile (install_traust --write-profile)") @@ -94,7 +94,6 @@ func runInit(cmd *cobra.Command, args []string) { workspace = answers.Workspace initOpts.forge = answers.Forge initOpts.ref = answers.Ref - initOpts.toolchain = answers.ToolchainProfile initOpts.harness = answers.Harness initOpts.fullPermission = answers.FullPermission } diff --git a/cmd/worker_test.go b/cmd/worker_test.go index 8c92ddc..0efd1b4 100644 --- a/cmd/worker_test.go +++ b/cmd/worker_test.go @@ -252,7 +252,7 @@ func TestNewRunIDUniqueness(t *testing.T) { t.Fatalf("duplicate run id %s", id) } seen[id] = true - if short := shortRunID(id); short == id || len(short) != 4 { + if short := shortRunID(id); short == id || len(short) != 6 { t.Errorf("short form of %s = %q", id, short) } } diff --git a/internal/api/api.go b/internal/api/api.go index 6a2c41e..b0b90f7 100644 --- a/internal/api/api.go +++ b/internal/api/api.go @@ -56,6 +56,10 @@ type TraustAPI interface { // (`traust registry models spend`). Best effort: a missing // progress-tracker configuration is a warning in the engine, not an error. RegistryModelsSpend(ctx context.Context, row SpendRow) error + // ToolchainDoctor checks a scanner toolchain profile + // (`traust admin toolchain doctor --profile

`). A non-zero exit + // means tools are missing, outdated, or their databases are absent. + ToolchainDoctor(ctx context.Context, profile string) (Result, error) } // PythonCLI runs the traust engine CLI through the harness venv's Python. @@ -160,3 +164,8 @@ func (p *PythonCLI) RegistryModelsSpend(ctx context.Context, row SpendRow) error } return nil } + +// ToolchainDoctor implements TraustAPI. +func (p *PythonCLI) ToolchainDoctor(ctx context.Context, profile string) (Result, error) { + return p.run(ctx, []string{"admin", "toolchain", "doctor", "--profile", profile}) +} diff --git a/internal/bootstrap/bootstrap.go b/internal/bootstrap/bootstrap.go index 1b893f9..d2cf135 100644 --- a/internal/bootstrap/bootstrap.go +++ b/internal/bootstrap/bootstrap.go @@ -146,19 +146,33 @@ func Init(ctx context.Context, opts InitOptions, log LogFn) (*State, error) { } } - // 5. Optional scanner toolchain profile. + // 5. Scanner toolchain profile: secure-code-audit by default β€” it + // covers every stage of the default flow (a superset of the vuln-scan + // and triage profiles). Already-installed profiles are skipped so + // re-running init stays fast and idempotent. if opts.ToolchainProfile != "" && opts.ToolchainProfile != "none" { - log("installing toolchain profile %s", opts.ToolchainProfile) - py := state.VenvPython() - tc := exec.CommandContext(ctx, py, "-m", "traust.cli", "admin", "toolchain", "setup", "--profile", opts.ToolchainProfile) - tc.Dir = repo - tc.Env = state.ChildEnv() - tc.Stdout = lineWriter(log) - tc.Stderr = lineWriter(log) - if err := tc.Run(); err != nil { - return nil, fmt.Errorf("toolchain setup (%s): %w", opts.ToolchainProfile, err) + installed := false + for _, p := range state.ToolchainProfiles { + if p == opts.ToolchainProfile { + installed = true + break + } + } + if installed { + log("toolchain profile %s already installed β€” skipping", opts.ToolchainProfile) + } else { + log("installing toolchain profile %s", opts.ToolchainProfile) + py := state.VenvPython() + tc := exec.CommandContext(ctx, py, "-m", "traust.cli", "admin", "toolchain", "setup", "--profile", opts.ToolchainProfile) + tc.Dir = repo + tc.Env = state.ChildEnv() + tc.Stdout = lineWriter(log) + tc.Stderr = lineWriter(log) + if err := tc.Run(); err != nil { + return nil, fmt.Errorf("toolchain setup (%s): %w", opts.ToolchainProfile, err) + } + state.ToolchainProfiles = appendUnique(state.ToolchainProfiles, opts.ToolchainProfile) } - state.ToolchainProfiles = appendUnique(state.ToolchainProfiles, opts.ToolchainProfile) } // 6. Default harness. diff --git a/internal/catalog/catalog.go b/internal/catalog/catalog.go index 0c381df..8dbf5f2 100644 --- a/internal/catalog/catalog.go +++ b/internal/catalog/catalog.go @@ -47,6 +47,9 @@ type StandaloneInfo struct { Role string `json:"role,omitempty" yaml:"role"` // MinModelClass is the documented minimum viable model class. MinModelClass string `json:"min_model_class,omitempty" yaml:"min_model_class"` + // Toolchain is the install-toolchain profile the skill's scanners + // live in (empty = scanner-free); the runner pre-flights it. + Toolchain string `json:"toolchain,omitempty" yaml:"toolchain"` // Env lists extra environment/infrastructure requirements. Env []string `json:"env,omitempty" yaml:"env"` } diff --git a/internal/catalog/data/standalone.yaml b/internal/catalog/data/standalone.yaml index ac5f4f6..1411e3e 100644 --- a/internal/catalog/data/standalone.yaml +++ b/internal/catalog/data/standalone.yaml @@ -6,6 +6,7 @@ # the SKILL.md frontmatter of a traust checkout. # # Fields per skill: +# toolchain: install-toolchain profile the skill's scanners live in # role: model-registry role whose floor applies (empty = none) # min_model_class: documented minimum viable model class # env: extra environment/infrastructure requirements @@ -16,12 +17,15 @@ skills: secure-code-audit: role: deep-audit min_model_class: mythos + toolchain: secure-code-audit secure-rpm-audit: role: deep-audit min_model_class: mythos + toolchain: secure-rpm-audit secure-container-audit: role: deep-audit min_model_class: opus + toolchain: secure-container-audit env: [skopeo, syft, grype] cloud-config-audit: role: deep-audit @@ -43,6 +47,7 @@ skills: vuln-scan: role: vuln-scan min_model_class: opus + toolchain: vuln-scan triage: role: triage-verifier min_model_class: mythos @@ -69,6 +74,7 @@ skills: verify-remediation: role: verification-verdict min_model_class: mythos + toolchain: secure-code-audit generate-team-report: role: render-narrate min_model_class: opus @@ -90,5 +96,7 @@ skills: env: [gh] dependency-watch: min_model_class: opus + toolchain: dependency-watch impact-analysis: min_model_class: opus + toolchain: impact-analysis diff --git a/internal/runner/runner.go b/internal/runner/runner.go index 0886490..3622eb1 100644 --- a/internal/runner/runner.go +++ b/internal/runner/runner.go @@ -193,6 +193,21 @@ func Run(ctx context.Context, opts Options, sink Sink) (*RunResult, error) { before := snapshotArtifacts(root) + // Pre-flight toolchain: skills whose scanners live in a toolchain + // profile degrade (thinner evidence, weaker coverage) when the profile + // is incomplete. The skills still run β€” this is a warning with the fix, + // not a gate. + if skill.Standalone != nil && skill.Standalone.Toolchain != "" { + profile := skill.Standalone.Toolchain + if tcres, err := opts.API.ToolchainDoctor(ctx, profile); err == nil && tcres.ExitCode != 0 { + msg := fmt.Sprintf("toolchain profile %q for %s is incomplete β€” the skill will run with reduced scanner coverage; install it with `traust-cli init` (or `--toolchain %s`)", profile, skill.Name, profile) + if tail := firstLines(tcres.Output(), 3); tail != "" { + msg += "\n" + tail + } + sink.Progress("warning: " + msg) + } + } + // Capability caveats, up front. for _, note := range h.Notes() { sink.Progress("note: " + note) diff --git a/internal/runner/runner_test.go b/internal/runner/runner_test.go index 99481f0..7c7fb3f 100644 --- a/internal/runner/runner_test.go +++ b/internal/runner/runner_test.go @@ -17,13 +17,15 @@ import ( // fakeAPI records every delegated engine call. type fakeAPI struct { - resolveModel string - resolveErr error - validateExit int - validated []string - rendered []string - linted []string - spendRows []api.SpendRow + resolveModel string + resolveErr error + validateExit int + toolchainExit int + toolchainHits []string + validated []string + rendered []string + linted []string + spendRows []api.SpendRow } func (f *fakeAPI) ReportingValidate(_ context.Context, path string, _ bool) (api.Result, error) { @@ -59,6 +61,11 @@ func (f *fakeAPI) RegistryModelsSpend(_ context.Context, row api.SpendRow) error return nil } +func (f *fakeAPI) ToolchainDoctor(_ context.Context, profile string) (api.Result, error) { + f.toolchainHits = append(f.toolchainHits, profile) + return api.Result{ExitCode: f.toolchainExit, Stdout: "MISSING opengrep (not found)\nMISSING gitleaks (not found)"}, nil +} + // recSink records sink traffic. type recSink struct { progress []string @@ -95,7 +102,7 @@ func testSkill() *catalog.Skill { Tier: "primary", Body: "Audit the target at $ARGUMENTS. Validate the report.", AllowedTools: []string{"Read", "Bash(git clone:*)"}, - Standalone: &catalog.StandaloneInfo{Role: "deep-audit", MinModelClass: "mythos"}, + Standalone: &catalog.StandaloneInfo{Role: "deep-audit", MinModelClass: "mythos", Toolchain: "secure-code-audit"}, } } @@ -442,6 +449,64 @@ func TestRunFailedRunStillDiscoversArtifacts(t *testing.T) { } } +// TestRunToolchainPreflightWarns verifies a skill with a toolchain profile +// gets a doctor check whose failure surfaces as a NON-fatal warning naming +// the fix. +func TestRunToolchainPreflightWarns(t *testing.T) { + t.Setenv("TRAUST_CLI_CLAUDE_BIN", fakeHarnessPath(t, "fake-claude.sh")) + state := testState(t) + client := &fakeAPI{toolchainExit: 1} + sink := &recSink{} + + res, err := Run(context.Background(), Options{ + State: state, API: client, Harness: &harness.Claude{}, + Skill: testSkill(), DeclareSpend: false, + }, sink) + if err != nil { + t.Fatal(err) + } + // secure-code-audit's profile was checked… + if len(client.toolchainHits) != 1 || client.toolchainHits[0] != "secure-code-audit" { + t.Errorf("toolchain doctor calls = %v", client.toolchainHits) + } + // …the warning names the profile and the fix… + warned := false + for _, p := range sink.progress { + if strings.Contains(p, `toolchain profile "secure-code-audit"`) && strings.Contains(p, "reduced scanner coverage") && strings.Contains(p, "traust-cli init") { + warned = true + } + } + if !warned { + t.Errorf("toolchain warning not surfaced: %v", sink.progress) + } + // …and the run still completes (degraded, not gated). + if res.ExitCode != ExitOK { + t.Fatalf("exit = %d β€” toolchain preflight must be a warning, not a gate", res.ExitCode) + } +} + +// TestRunToolchainPreflightSkipsCleanProfiles verifies a healthy profile +// produces no warning. +func TestRunToolchainPreflightSkipsCleanProfiles(t *testing.T) { + t.Setenv("TRAUST_CLI_CLAUDE_BIN", fakeHarnessPath(t, "fake-claude.sh")) + state := testState(t) + client := &fakeAPI{toolchainExit: 0} + sink := &recSink{} + + _, err := Run(context.Background(), Options{ + State: state, API: client, Harness: &harness.Claude{}, + Skill: testSkill(), DeclareSpend: false, + }, sink) + if err != nil { + t.Fatal(err) + } + for _, p := range sink.progress { + if strings.Contains(p, "toolchain profile") { + t.Errorf("clean profile should not warn: %q", p) + } + } +} + func TestPreflightNoRole(t *testing.T) { skill := testSkill() skill.Standalone = &catalog.StandaloneInfo{MinModelClass: "opus"} diff --git a/internal/tui/init.go b/internal/tui/init.go index dcd06e0..9b389be 100644 --- a/internal/tui/init.go +++ b/internal/tui/init.go @@ -6,12 +6,11 @@ import ( // InitAnswers is the outcome of the init wizard. type InitAnswers struct { - Workspace string - Forge string - Ref string - ToolchainProfile string - Harness string - FullPermission bool + Workspace string + Forge string + Ref string + Harness string + FullPermission bool } // RunInitWizard asks the interactive setup questions. detected lists the @@ -19,11 +18,10 @@ type InitAnswers struct { // preselected one. func RunInitWizard(defaultWorkspace, defaultForge string, detected []string, defaultHarness string) (*InitAnswers, error) { answers := &InitAnswers{ - Workspace: defaultWorkspace, - Forge: defaultForge, - Ref: "latest", - ToolchainProfile: "none", - Harness: defaultHarness, + Workspace: defaultWorkspace, + Forge: defaultForge, + Ref: "latest", + Harness: defaultHarness, // Full permission is the recommended default: the harness's skills // cannot run unattended without it, so the confirm highlights Yes. FullPermission: true, @@ -55,18 +53,6 @@ func RunInitWizard(defaultWorkspace, defaultForge string, detected []string, def Placeholder("latest"). Value(&answers.Ref), ), - huh.NewGroup( - huh.NewSelect[string](). - Title("Scanner toolchain profile"). - Description("Baseline skills (threat-model, vuln-scan) need nothing; secure-code-audit benefits from its pinned scanners."). - Options( - huh.NewOption("none β€” baseline only (threat-model, vuln-scan)", "none"), - huh.NewOption("secure-code-audit β€” opengrep, gitleaks, osv-scanner, tokei", "secure-code-audit"), - huh.NewOption("vuln-scan", "vuln-scan"), - huh.NewOption("triage", "triage"), - ). - Value(&answers.ToolchainProfile), - ), huh.NewGroup( huh.NewSelect[string](). Title("Default agent harness").