From 041dedffbe8113467c30dccb94b043ab46aca4f1 Mon Sep 17 00:00:00 2001 From: dmmdea Date: Thu, 29 Oct 2026 20:09:50 -0500 Subject: [PATCH 1/4] Split the root main.go into topical files of package main Pure declaration moves, no behaviour change. New files: cli_support.go, task_cmd.go, generate_image_cmd.go, generate_video_cmd.go, generate_audio_cmd.go, delegate_cmd.go, fleet_serve_cmd.go, doctor_cmd.go, report_cmd.go, learn_cmd.go, confhead_cmd.go. main.go keeps the entry point and the functions that source-scanning tests read by file name. The census tool's before/after output is byte-identical (1002 declarations); the file paths named in docs/systems follow, and a changes/unreleased fragment is added. Co-Authored-By: Claude Fable 5.1 --- .../unreleased/refactor-split-root-main.md | 5 + cli_support.go | 336 ++ confhead_cmd.go | 347 ++ delegate_cmd.go | 138 + docs/systems/cache-server.md | 2 +- docs/systems/delegate-contract-file.md | 2 +- docs/systems/delegation-client.md | 2 +- docs/systems/fleet-node.md | 6 +- docs/systems/measurement-gate.md | 2 +- docs/systems/security.md | 4 +- doctor_cmd.go | 528 ++ fleet_serve_cmd.go | 437 ++ generate_audio_cmd.go | 226 + generate_image_cmd.go | 582 ++ generate_video_cmd.go | 452 ++ learn_cmd.go | 495 ++ main.go | 5277 ++--------------- report_cmd.go | 259 + task_cmd.go | 568 ++ 19 files changed, 4939 insertions(+), 4729 deletions(-) create mode 100644 changes/unreleased/refactor-split-root-main.md create mode 100644 cli_support.go create mode 100644 confhead_cmd.go create mode 100644 delegate_cmd.go create mode 100644 doctor_cmd.go create mode 100644 fleet_serve_cmd.go create mode 100644 generate_audio_cmd.go create mode 100644 generate_image_cmd.go create mode 100644 generate_video_cmd.go create mode 100644 learn_cmd.go create mode 100644 report_cmd.go create mode 100644 task_cmd.go diff --git a/changes/unreleased/refactor-split-root-main.md b/changes/unreleased/refactor-split-root-main.md new file mode 100644 index 000000000..27e9777e9 --- /dev/null +++ b/changes/unreleased/refactor-split-root-main.md @@ -0,0 +1,5 @@ +### Changed - the root package's `main.go` is split into topical files of the same package + +- **What changed.** `main.go` (5359 lines) keeps the entry point (`main`, `usage`, `hoistGlobalConfig`, the dispatch) plus the functions that source-scanning tests read by file name (`runTask`, `runFleetServe`, `startVRAMSampler`, `runMCP`, `runDelegate`, `runResearch`, `doctorRunChecked`). Its other declarations moved unchanged into `cli_support.go` (config loading, argument splitting, result printing), `task_cmd.go` (text, vision, speech, nim and forecast verbs), `generate_image_cmd.go`, `generate_video_cmd.go`, `generate_audio_cmd.go`, `delegate_cmd.go`, `fleet_serve_cmd.go`, `doctor_cmd.go`, `report_cmd.go` (ledger, models, eval, stats), `learn_cmd.go` (the self-learning offline verbs) and `confhead_cmd.go`. The file paths named in `docs/systems/` follow. No behaviour changes. +- **Why.** A 5359-line file hid the package's structure and made every verb edit a merge-conflict magnet. +- **Check.** A declaration census (every top-level declaration's source hash, comments stripped, sorted) taken before and after is byte-identical (`diff` prints nothing; 1002 declarations); `go build ./...` and `go vet .` are clean. diff --git a/cli_support.go b/cli_support.go new file mode 100644 index 000000000..1f76ccdad --- /dev/null +++ b/cli_support.go @@ -0,0 +1,336 @@ +// CLI plumbing shared by the subcommands: config loading, argument splitting, +// input reading and result printing. + +package main + +import ( + "encoding/json" + "flag" + "fmt" + "io" + "os" + "strings" + "time" + + "github.com/dmmdea/offload-harness/internal/cache" + "github.com/dmmdea/offload-harness/internal/config" + "github.com/dmmdea/offload-harness/internal/core" + "github.com/dmmdea/offload-harness/internal/delegate" + "github.com/dmmdea/offload-harness/internal/embedmemo" + "github.com/dmmdea/offload-harness/internal/ledger" + "github.com/dmmdea/offload-harness/internal/llamaclient" + "github.com/dmmdea/offload-harness/internal/pairworkloads" + "github.com/dmmdea/offload-harness/internal/pipeline" +) + +func loadCfg(fs *flag.FlagSet) config.Config { + cfg, _ := loadCfgWithSource(fs) + return cfg +} + +// loadCfgWithSource loads the config and DISCLOSES its true source, warning on stderr +// whenever the effective config is built-in defaults — including the two silent-trap +// shapes a review surfaced: an explicit --config/$LOCAL_OFFLOAD_CONFIG path that does not +// exist (Load maps IsNotExist to defaults with a NIL error by design), and a file that +// exists but fails to parse. Silent degradation is the trap (live incident 2026-07-20): +// a box whose real config lived at a non-conventional path served every bare CLI call +// from built-in defaults — empty VisionModel and all — and the only symptom was a +// misleading "no vision model configured" defer in a consumer's pipeline hours later. +func loadCfgWithSource(fs *flag.FlagSet) (config.Config, config.Source) { + cfg, src := config.LoadWithSource(fs.Lookup("config").Value.String()) + config.WarnOnDefaults(src, os.Stderr) + // C-95: every verb that builds a delegator loads its config here (the MCP + // server behind agent_delegate/agent_run, delegate, acceptance, report), so + // the note that a config's own layers shadow its agent_model lives on this + // shared path, once per process, on stderr (MCP stdout is JSON-RPC). + config.WarnOnShadowedAgentModelOnce(src, cfg, os.Stderr) + return cfg, src +} + +// resolveCfgPath picks the config file by precedence: explicit --config flag > +// $LOCAL_OFFLOAD_CONFIG > ./config.json if it exists (LO-4: the README +// quickstart says `cp config.example.json config.json`, which was never read) > +// the conventional ~/.local-offload/config.json if it exists > "" (built-in +// defaults). The conventional-path fallback is the fix for a bare +// `local-offload mcp` spawned by an MCP host that passes neither flag nor +// env (as the registration did): without it the server silently ran on defaults +// with ShadowEnabled=false, so capture never fired and the flywheel starved. +// exists is injected so the precedence is unit-testable without touching disk. +func resolveCfgPath(flagPath, envPath, home string, exists func(string) bool) string { + return config.ResolvePath(flagPath, envPath, home, exists) +} + +func openPipeline(cfg config.Config) (*pipeline.Pipeline, func(), error) { + if err := cfg.EnsureDirs(); err != nil { + return nil, nil, err + } + // WithSeatEndpoints threads the Phase-A per-model overrides (seat_endpoints): + // an overridden seat's completions leave for its remote tailnet base through + // the dial-guarded client; with the key absent this is a no-op and the client + // is byte-identical to a pre-seat build. + client := llamaclient.New(cfg.Endpoint, cfg.CompletionPath, cfg.Model, time.Duration(cfg.RequestTimeoutSec)*time.Second). + WithSeatEndpoints(cfg.SeatEndpoints) + // WithRemoteLanes threads the busy-aware cascade lanes (cascade_remote_lanes, + // roast delta 7): while the machine-wide GPU lease is held, a cascade call + // whose model a lane roster-serves rides that lane instead of queueing behind + // the local card. Guarded so the absent key stays byte-identical (pinned). + if len(cfg.CascadeRemoteLanes) > 0 { + gpuLockPath, stateDir := cfg.GPULockPath, cfg.StateDir + // FleetLaneGates, not RosterResident alone: a lane base may be a + // plain llama-swap OR a fleet node whose own llama-swap binds + // loopback (C-41b). The pair shares one probe, so residency and the + // route can never disagree about which shape a base is. + laneResident, laneRoute := llamaclient.FleetLaneGates(cfg.FleetAuthToken) + client = client.WithRemoteLanes(cfg.CascadeRemoteLanes, + func() bool { return delegate.LocalBusy(gpuLockPath, stateDir) }, + llamaclient.LocalSwapBusy(cfg.Endpoint), + laneResident).WithLaneRoute(laneRoute) + } + // Cache + ledger are bbolt (single-writer, exclusive file lock). When the + // long-running MCP server holds the lock, a CLI invocation degrades to + // cache-less rather than aborting — they speed things up / report savings, + // they are not required for correctness. + // + // 0.121.1 (register D-05): the cache handle is LAZY. openPipeline runs for + // every command and every MCP server start, and the eager open made each of + // them take (or lose) the bbolt lock before knowing whether a cacheable task + // would ever run — which is what produced 49 empty per-process siblings on + // this box. Nothing below opens a file; the first Get/Put does, and prints + // the operator note then, when it is true rather than merely predicted. + var ca *cache.Cache + if cfg.CachePath != "" { // "" = caller opted out of caching (e.g. the confhead A/B, where a shared cache would cross-contaminate arms) + ca = cache.New(cfg.CachePath).WithNotify(func(mode cache.Mode, used string, err error) { + switch mode { + case cache.ModeSibling: + // 0.113.21: the shared file is held by another harness process (the + // MCP server, or a sibling session's); this process keeps its own + // in-loop hits in a per-process sibling instead of running cache-less. + fmt.Fprintln(os.Stderr, "note: result cache is held by another local-offload process; using the per-process cache", used) + case cache.ModeUnavailable: + fmt.Fprintln(os.Stderr, "note: cache unavailable; continuing without cache:", err) + } + }) + } + led, err := ledger.Open(cfg.LedgerPath) + if err != nil { + fmt.Fprintln(os.Stderr, "note: ledger unavailable (held by the MCP server?); continuing without ledger") + led = nil + } + // Every non-delegation tool row becomes one card in NVIDIA PAIR's Jobs + // list (docs/systems/pair-workloads.md); inert unless + // pair_workloads_enabled and PAIR is installed on this box. + pair := pairworkloads.New(pairworkloads.FromConfig(cfg)) + pair.AttachLedger(led) + p := pipeline.New(cfg, client, ca, led) + // A long media call opens its card when it starts (calls.go). + p.SetCallTracker(pair) + // A call routed to a fleet node opens its card and writes its asker row through here + // (D5/D6, internal/pairworkloads/remote.go). + p.SetPairEmitter(pair) + return p, func() { + if ca != nil { + ca.Close() + } + if led != nil { + led.Close() + } + // A one-shot CLI task exits right after this: deliver its PAIR card + // first (each send is bounded at 2 s), or the frame dies in flight. + pair.Wait() + // The embed memo's persisted hit/miss totals are written ONLY here. A + // process that exits without this leaves them at zero, and every reporting + // surface then states — in the one number the Phase 0.4 gate reads — that + // a memo which served thousands of hits was "never consulted". The error + // is surfaced rather than swallowed: losing the counters silently is the + // same fabricated-measurement defect in a quieter form. + if err := embedmemo.CloseShared(); err != nil { + fmt.Fprintln(os.Stderr, "note: embed memo counters may not have been persisted:", err) + } + }, nil +} + +// splitPositionals separates the first n positionals from flags, treating the named value-flags as consuming +// the following token. A positional past the n slots is dropped onto flags (see overflowArg). A bare "--" ends +// the flags. The result always has n slots; missing positionals are "", and an explicit empty token fills none (so +// `summarize "" file.txt` still reads the file). splitArgs, splitTwoArgs, splitThreeArgs +// and splitFourArgs are this loop with a fixed n, kept so each subcommand's shape reads at its call site. +func splitPositionals(args []string, valueFlags map[string]bool, n int) (pos []string, flags []string) { + ended, fenced := false, false + for i := 0; i < len(args); i++ { + a := args[i] + if endsFlags(a, ended) { + ended = true + continue + } + if isFlagArg(a, ended) { // a lone "-" (the stdin indicator) is not a flag: it falls through to the positional rule + flags = append(flags, a) + name := strings.TrimLeft(a, "-") + if strings.ContainsRune(name, '=') { + continue + } + if valueFlags[name] && i+1 < len(args) { + i++ + flags = append(flags, args[i]) + } + continue + } + if len(pos) < n { + if a != "" { // an empty token never fills a slot, as the named splitters always behaved + pos = append(pos, a) + } + } else { + flags = overflowArg(flags, a, ended, &fenced) + } + } + for len(pos) < n { + pos = append(pos, "") + } + return pos, flags +} + +// splitFourArgs separates the FIRST FOUR positionals (out path + ref image + driver video + prompt) from flags. +// Used by animate-character, whose CLI shape is ` "" [flags]`. +func splitFourArgs(args []string, valueFlags map[string]bool) (first, second, third, fourth string, flags []string) { + pos, flags := splitPositionals(args, valueFlags, 4) + return pos[0], pos[1], pos[2], pos[3], flags +} + +// splitThreeArgs separates the FIRST THREE positionals (e.g. out path + still + prompt) from flags. Used by +// generate-video, whose CLI shape is ` "" [flags]`. +func splitThreeArgs(args []string, valueFlags map[string]bool) (first, second, third string, flags []string) { + pos, flags := splitPositionals(args, valueFlags, 3) + return pos[0], pos[1], pos[2], flags +} + +// splitTwoArgs separates the FIRST TWO positionals (e.g. out path + text) from flags. Used by generate-audio, +// whose CLI shape is ` "" [flags]`. +func splitTwoArgs(args []string, valueFlags map[string]bool) (first, second string, flags []string) { + pos, flags := splitPositionals(args, valueFlags, 2) + return pos[0], pos[1], flags +} + +// endsFlags reports whether tok is the bare "--" that ends flag handling. Only the first one does: after it, "--" is a positional. +func endsFlags(tok string, ended bool) bool { return !ended && tok == "--" } + +// isFlagArg reports whether tok is read as a flag by the positional splitters: it starts with "-", it is not the lone "-" (the +// stdin positional) and no bare "--" has ended the flags. +func isFlagArg(tok string, ended bool) bool { + return !ended && strings.HasPrefix(tok, "-") && tok != "-" +} + +// overflowArg hands a positional the splitter has no slot for on to the flags the caller gives fs.Parse +// (as the splitters always did: Parse stops at the first non-flag, so the extra is harmless). Once a bare +// "--" has ended the flags, the extra token may start with a dash, and Parse would read it as a flag, so +// ONE "--" of the splitter's own goes in front of the first such token: Parse stops there and nothing +// after the user's terminator is ever a flag. The user's own "--" is never passed on. fenced is the +// splitter's "my -- is already in" state. +func overflowArg(flags []string, tok string, ended bool, fenced *bool) []string { + if ended && !*fenced { + flags = append(flags, "--") + *fenced = true + } + return append(flags, tok) +} + +func printHuman(res core.Result) { + if res.Deferred { + fmt.Printf("DEFERRED (%s) — handle this yourself.\n", res.Reason) + if res.Partial != "" { + fmt.Println("partial:", truncate(res.Partial, 400)) + } + return + } + var pretty any + _ = json.Unmarshal(res.Data, &pretty) + b, _ := json.MarshalIndent(pretty, "", " ") + fmt.Println(string(b)) + fmt.Printf("[%d in / %d out tok, %.0f tok/s, %dms%s]\n", + res.Meta.TokensIn, res.Meta.TokensOut, res.Meta.TokPerSec, res.Meta.LatencyMs, + map[bool]string{true: ", cache hit", false: ""}[res.Meta.CacheHit]) +} + +// emitResult prints a Result, optionally projecting its Data to the --select +// fields and/or minifying with --compact. This is the harness's fastcontext +// citation pattern at the output layer: the caller keeps only the fields it +// needs (e.g. `transcribe --select gist,srt_path` drops the verbose segments[]). +// selectCSV "" / compact false reproduces the prior plain behavior exactly. +func emitResult(res core.Result, asJSON bool, selectCSV string, compact bool) { + if keys := core.SelectKeys(selectCSV); len(keys) > 0 { + res.Data = core.ProjectFields(res.Data, keys) + } + if asJSON { + var b []byte + if compact { + b, _ = json.Marshal(res) + } else { + b, _ = json.MarshalIndent(res, "", " ") + } + fmt.Println(string(b)) + return + } + printHuman(res) +} + +func readInput(arg string) (string, error) { + if arg == "" || arg == "-" { + b, err := io.ReadAll(os.Stdin) + return string(b), err + } + b, err := os.ReadFile(arg) + return string(b), err +} + +// splitArgs separates the first positional (input path) from flags, treating +// the named value-flags as consuming the following token. This lets the input +// path appear before flags on the command line. +// leftoverArgErr rejects arguments left over after flag parsing. The flag +// package stops at the first non-flag token, so a boolean flag given in space +// form ("--refine false") silently drops every later flag and can promote its +// value into a positional. cmd names the command for the error text. +func leftoverArgErr(fs *flag.FlagSet, cmd string) error { + if fs.NArg() == 0 { + return nil + } + return fmt.Errorf("%s: unexpected argument %q — boolean flags need the =form (e.g. --refine=false), and the prompt must be one quoted argument", cmd, fs.Arg(0)) +} + +// applyBatchRefineDefault propagates the CLI-level --refine=false onto batch +// jobs that carry no per-job "refine" value (job-level wins). --refine=true is +// the inert default, so only an explicit false writes anything. +func applyBatchRefineDefault(jobs []pipeline.ImageBatchJob, refine bool) { + if refine { + return + } + off := false + for i := range jobs { + if jobs[i].Refine == nil { + jobs[i].Refine = &off + } + } +} + +// splitArgs separates the FIRST positional from flags, treating the named value-flags as consuming the +// following token. A lone "-" is the stdin positional. A bare "--" ends the flags: every later token is a +// positional, even one that starts with "-" (see overflowArg for what happens to an extra one). +func splitArgs(args []string, valueFlags map[string]bool) (positional string, flags []string) { + pos, flags := splitPositionals(args, valueFlags, 1) + return pos[0], flags +} + +func splitCSV(s string) []string { + parts := strings.Split(s, ",") + out := make([]string, 0, len(parts)) + for _, p := range parts { + if t := strings.TrimSpace(p); t != "" { + out = append(out, t) + } + } + return out +} + +func truncate(s string, n int) string { + if len(s) > n { + return s[:n] + "…" + } + return s +} diff --git a/confhead_cmd.go b/confhead_cmd.go new file mode 100644 index 000000000..1c22a97ec --- /dev/null +++ b/confhead_cmd.go @@ -0,0 +1,347 @@ +// The confidence-head verbs: train, eval and calibrate. + +package main + +import ( + "encoding/json" + "flag" + "fmt" + "os" + "sort" + "strings" + + "github.com/dmmdea/offload-harness/internal/confhead" + "github.com/dmmdea/offload-harness/internal/eval" + "github.com/dmmdea/offload-harness/internal/ledger" +) + +func runTrainConfHead(args []string) error { + fs := flag.NewFlagSet("train-confhead", flag.ExitOnError) + fs.String("config", "", "config file path") + out := fs.String("out", "", "output confhead-weights.json (default: config confhead_path)") + _ = fs.Parse(args) + cfg := loadCfg(fs) + dst := *out + if dst == "" { + dst = cfg.ConfHeadPath + } + rep, err := confhead.Train(cfg.LedgerPath, cfg.ConfHeadLabelsPath, dst) + if err != nil { + return err + } + fmt.Println(rep) + fmt.Println("wrote", dst) + return nil +} + +// confheadVerdict returns "ADOPT" when the bootstrap CI lower bound strictly +// exceeds zero (the head provably lowers AURC), otherwise "REJECT". +// AUGRC and ECE are DIAGNOSTIC only and must NEVER be passed to or read by +// this function — the verdict depends solely on lo. +func confheadVerdict(lo float64) string { + if lo > 0 { + return "ADOPT" + } + return "REJECT" +} + +// runConfheadEval is the Phase 2 confhead ADOPTION GATE: a rigorous, leakage-free +// validation that decides whether the per-task correctness head is good enough to +// adopt. For each task with enough labeled rows it computes out-of-fold p(correct), +// compares the head's AURC against the incumbent confidence's AURC with a paired +// bootstrap CI, and verdicts ADOPT only if the CI lower bound excludes zero (the +// head PROVABLY lowers AURC). Tasks with too few labels are reported as skipped. +func runConfheadEval(args []string) error { + fs := flag.NewFlagSet("confhead-eval", flag.ExitOnError) + fs.String("config", "", "config file path") + k := fs.Int("k", 5, "out-of-fold folds") + b := fs.Int("bootstrap", 2000, "bootstrap resamples") + minFold := fs.Int("fold-min-rows", 40, "min rows to train a head within a fold") + _ = fs.Parse(args) + cfg := loadCfg(fs) + + // Test, bake and measurement rows carry a fixture's margin, not the box's (register F48): the head is + // judged and calibrated on production rows only, and the omission is stated. + rows, excluded, err := ledger.ReadProduction(cfg.LedgerPath) + if err != nil { + return fmt.Errorf("confhead-eval: read ledger: %w", err) + } + noteExcluded(excluded) + labels, err := ledger.ReadLabelFile(cfg.ConfHeadLabelsPath) + if err != nil { + return fmt.Errorf("confhead-eval: read labels: %w", err) + } + all := append(rows, labels...) + + // Group LABELED, non-cache-hit rows per task. + byTask := map[string][]ledger.Entry{} + for _, e := range all { + if e.CacheHit { + continue + } + if _, ok := confhead.Label(e); !ok { + continue + } + task := strings.ToLower(e.Task) + byTask[task] = append(byTask[task], e) + } + + const seed = 1 + usable := 2 * (*minFold) // need at least this many labeled rows to attempt the gate + + type result struct { + N int `json:"n"` + BaseError *float64 `json:"base_error,omitempty"` + AURCIncumbent *float64 `json:"aurc_incumbent,omitempty"` + AURCHeadOOF *float64 `json:"aurc_head_oof,omitempty"` + AUGRCHead *float64 `json:"augrc_head,omitempty"` + ECEHead *float64 `json:"ece_head,omitempty"` // diagnostic only — never read by verdict + Delta *float64 `json:"delta,omitempty"` + CILo *float64 `json:"ci_lo,omitempty"` + CIHi *float64 `json:"ci_hi,omitempty"` + Verdict string `json:"verdict,omitempty"` + Note string `json:"note,omitempty"` + } + + out := map[string]result{} + tasks := make([]string, 0, len(byTask)) + for t := range byTask { + tasks = append(tasks, t) + } + sort.Strings(tasks) + + for _, task := range tasks { + subset := byTask[task] + n := len(subset) + if n < usable { + out[task] = result{N: n, Note: "insufficient labels"} + continue + } + + // Per-row labels and incumbent confidence (Margin; 0 for summarize/extract + // => no-skill baseline). Captured up front so OOF scoring and the bootstrap + // share the same aligned arrays. + correct := make([]bool, n) + margins := make([]float64, n) + nWrong := 0 + for i, e := range subset { + y, _ := confhead.Label(e) + correct[i] = y == 1 + if !correct[i] { + nWrong++ + } + margins[i] = e.Margin + } + + // Out-of-fold p(correct): for each fold, fit a head on the TRAINING subset + // and score the held-out rows. A Predict sentinel (-1: no head for this fold) + // becomes 0.5 = no-skill, so a fold that fails to train can't fake a signal. + oofHead := eval.KFoldOOF(n, *k, seed, + func(train []int) *confhead.Model { + trainSubset := make([]ledger.Entry, 0, len(train)) + for _, ti := range train { + trainSubset = append(trainSubset, subset[ti]) + } + return confhead.FitWithMinRows(trainSubset, *minFold) + }, + func(m *confhead.Model, i int) float64 { + p := m.Predict(task, confhead.FeatureRow(subset[i])) + if p < 0 { + return 0.5 + } + return p + }, + ) + + incPts := make([]eval.RCPoint, n) + headPts := make([]eval.RCPoint, n) + for i := 0; i < n; i++ { + incPts[i] = eval.RCPoint{Confidence: margins[i], Correct: correct[i]} + headPts[i] = eval.RCPoint{Confidence: oofHead[i], Correct: correct[i]} + } + _, _, aurcInc, _ := eval.RiskCoverage(incPts) + _, _, aurcHead, _ := eval.RiskCoverage(headPts) + augrc := eval.AUGRC(headPts) + ece := eval.ECE(headPts, 10) // diagnostic: Expected Calibration Error (10 bins) + delta, lo, hi := eval.BootstrapDeltaAURC(margins, oofHead, correct, *b, seed) + + // Verdict is driven solely by the CI lower bound — augrc and ece are + // diagnostic outputs and are NEVER passed to confheadVerdict. + verdict := confheadVerdict(lo) + baseErr := float64(nWrong) / float64(n) + out[task] = result{ + N: n, BaseError: &baseErr, + AURCIncumbent: &aurcInc, AURCHeadOOF: &aurcHead, AUGRCHead: &augrc, ECEHead: &ece, + Delta: &delta, CILo: &lo, CIHi: &hi, Verdict: verdict, + } + } + + enc, _ := json.MarshalIndent(out, "", " ") + fmt.Println(string(enc)) + return nil +} + +// runConfheadCalibrate is the Phase 2 Task 3 threshold calibrator. For each task +// with enough labeled rows it computes OUT-OF-FOLD p(correct) (so the chosen +// threshold is not optimistic), then selects the smallest conformal threshold tau +// on p(correct) whose ACCEPTED-set error meets the per-task target. At runtime +// (Task 4, separate) the pipeline escalates a summarize call to the larger tier +// when p(correct) < tau. This job is OFFLINE only: it writes a thresholds file and +// does not touch the live pipeline. Tasks with too few labels are omitted. +func runConfheadCalibrate(args []string) error { + fs := flag.NewFlagSet("confhead-calibrate", flag.ExitOnError) + fs.String("config", "", "config file path") + k := fs.Int("k", 5, "out-of-fold folds") + minFold := fs.Int("fold-min-rows", 40, "min rows to train a head within a fold") + out := fs.String("out", "", "output confhead-thresholds.json (default: config confhead_thresholds_path)") + _ = fs.Parse(args) + cfg := loadCfg(fs) + _ = cfg.EnsureDirs() + dst := *out + if dst == "" { + dst = cfg.ConfHeadThresholdsPath + } + + // Test, bake and measurement rows carry a fixture's margin, not the box's (register F48): the head is + // judged and calibrated on production rows only, and the omission is stated. + rows, excluded, err := ledger.ReadProduction(cfg.LedgerPath) + if err != nil { + return fmt.Errorf("confhead-calibrate: read ledger: %w", err) + } + noteExcluded(excluded) + labels, err := ledger.ReadLabelFile(cfg.ConfHeadLabelsPath) + if err != nil { + return fmt.Errorf("confhead-calibrate: read labels: %w", err) + } + all := append(rows, labels...) + + // Group LABELED, non-cache-hit rows per task. + byTask := map[string][]ledger.Entry{} + for _, e := range all { + if e.CacheHit { + continue + } + if _, ok := confhead.Label(e); !ok { + continue + } + task := strings.ToLower(e.Task) + byTask[task] = append(byTask[task], e) + } + + const ( + seed = 1 + defaultTargetErr = 0.15 // per-task overrides via config target_error_rate + ) + usable := 2 * (*minFold) // need at least this many labeled rows to attempt calibration + + thresholds := map[string]float64{} + tasks := make([]string, 0, len(byTask)) + for t := range byTask { + tasks = append(tasks, t) + } + sort.Strings(tasks) + + type calibDiag struct { + N int `json:"n"` + Tau float64 `json:"tau"` + Target float64 `json:"target"` + RealizedAcceptedErr float64 `json:"realized_accepted_err"` // diagnostic: OOF accepted-set error at tau + BaseErr float64 `json:"base_err"` + EscalationRate float64 `json:"escalation_rate"` + } + diags := map[string]calibDiag{} + + var sb strings.Builder + sb.WriteString("confhead threshold calibration (out-of-fold)\n") + for _, task := range tasks { + subset := byTask[task] + n := len(subset) + if n < usable { + fmt.Fprintf(&sb, " %-12s n=%4d <%d labeled — skipped (omitted from thresholds)\n", task, n, usable) + continue + } + + // Per-row correctness label. + correct := make([]bool, n) + nWrong := 0 + for i, e := range subset { + y, _ := confhead.Label(e) + correct[i] = y == 1 + if !correct[i] { + nWrong++ + } + } + + // Out-of-fold p(correct): for each fold, fit on the TRAINING subset and + // score the held-out rows. A Predict sentinel (-1: no head for this fold) + // becomes 0.5 = no-skill, so a fold that fails to train can't fake a signal. + oof := eval.KFoldOOF(n, *k, seed, + func(train []int) *confhead.Model { + trainSubset := make([]ledger.Entry, 0, len(train)) + for _, ti := range train { + trainSubset = append(trainSubset, subset[ti]) + } + return confhead.FitWithMinRows(trainSubset, *minFold) + }, + func(m *confhead.Model, i int) float64 { + p := m.Predict(task, confhead.FeatureRow(subset[i])) + if p < 0 { + return 0.5 + } + return p + }, + ) + + target := defaultTargetErr + if a, ok := cfg.TargetErrorRate[task]; ok { + target = a + } + tau := confhead.SelectThreshold(oof, correct, target) + thresholds[task] = tau + + // Resulting accepted-error and escalation-rate at tau on the OOF scores. + var nAccepted, nAccWrong int + for i, s := range oof { + if s >= tau { + nAccepted++ + if !correct[i] { + nAccWrong++ + } + } + } + baseErr := float64(nWrong) / float64(n) + accErr := 0.0 + if nAccepted > 0 { + accErr = float64(nAccWrong) / float64(nAccepted) + } + escRate := float64(n-nAccepted) / float64(n) + fmt.Fprintf(&sb, " %-12s n=%4d base_err=%.4f target=%.3f tau=%.4f accepted_err=%.4f escalation_rate=%.4f\n", + task, n, baseErr, target, tau, accErr, escRate) + diags[task] = calibDiag{N: n, Tau: tau, Target: target, RealizedAcceptedErr: accErr, BaseErr: baseErr, EscalationRate: escRate} + } + + raw, err := json.MarshalIndent(thresholds, "", " ") + if err != nil { + return fmt.Errorf("confhead-calibrate: marshal thresholds: %w", err) + } + // Atomic write (P4): the long-running MCP server polls confhead-thresholds.json + // and must only ever read a COMPLETE file. tmp+rename (atomic on the same + // filesystem) so the reloader never sees a half-written threshold map. + tmp := dst + ".tmp" + if err := os.WriteFile(tmp, append(raw, '\n'), 0o644); err != nil { + return fmt.Errorf("confhead-calibrate: write %s: %w", tmp, err) + } + if err := os.Rename(tmp, dst); err != nil { + return fmt.Errorf("confhead-calibrate: rename %s: %w", dst, err) + } + + fmt.Print(sb.String()) + fmt.Println("wrote", dst) + // Print JSON diagnostics (realized_accepted_err vs target_error_rate per task). + // This is a diagnostic-only report; the threshold file written above is unchanged. + if len(diags) > 0 { + enc, _ := json.MarshalIndent(diags, "", " ") + fmt.Println(string(enc)) + } + return nil +} diff --git a/delegate_cmd.go b/delegate_cmd.go new file mode 100644 index 000000000..30cdeabd0 --- /dev/null +++ b/delegate_cmd.go @@ -0,0 +1,138 @@ +// Helpers of the delegate and research verbs: contract-file parsing, per-subtask +// intake, engine errors, placement pins and the exit status. + +package main + +import ( + "encoding/json" + "errors" + "fmt" + "os" + "strings" + + "github.com/dmmdea/offload-harness/internal/delegate" +) + +// repeatedFlag collects a repeatable string flag (--remote a --remote b). +type repeatedFlag []string + +func (r *repeatedFlag) String() string { return strings.Join(*r, ",") } + +func (r *repeatedFlag) Set(v string) error { + *r = append(*r, v) + return nil +} + +// cliIntake prepares a CLI verb's subtasks one by one (register D-111), the verbs' half of what the MCP doors do: a +// subtask that fails intake no longer rejects the valid ones beside it. The returned batch holds the contracts that passed +// (tagged with the verb's door, in input order, each with its input position) and the rejections, and the verb splices +// those back among the engine's results (PreparedBatch.Splice) so the answer holds one result per subtask sent. The lints +// are kept by INPUT position, as the wire builder reads them. The error is only for a call with nothing valid in it, and it +// names every bad subtask: label and ids word it as the MCP doors do ("subtask 2: ...", or "source 4: ..." for a research +// page, named by its place in the url list; nil ids = the input position). +func cliIntake(specs []delegate.SubtaskSpec, readRoot string, capBytes int, door, label string, ids []int) (delegate.PreparedBatch, [][]string, error) { + batch := delegate.PrepareBatchWithCap(specs, readRoot, capBytes) + if batch.AllRejected() { + return batch, nil, fmt.Errorf("%s", batch.FlatReason(label, ids)) + } + lints := make([][]string, len(specs)) + for k := range batch.Contracts { + batch.Contracts[k].Door = door + // Linted on the PREPARED contract (context_paths already inlined: grounding is judged against everything the + // sub-agent will see). + lints[batch.Indices[k]] = delegate.LintAcceptance(batch.Contracts[k]) + } + return batch, lints, nil +} + +// cliEngineErr is the error a verb returns when the engine refuses the whole call, so that nothing ran and there is no result +// to splice a refusal into: the engine's own error (kept wrapped, so errors.Is still sees it), and the subtasks intake refused +// beside it, so fixing the first complaint does not meet the second on the retry (the MCP doors word the same sentence). +func cliEngineErr(err error, batch delegate.PreparedBatch, label string, ids []int) error { + if len(batch.Rejections) == 0 { + return err + } + return fmt.Errorf("%w; also refused at intake: %s", err, batch.FlatReason(label, ids)) +} + +// cliPinOptions builds the engine's options for the delegate and research verbs, the CLI doors that offer pin_reason (ADR +// 0078): a --route local or remote without --pin-reason is a hint that placement may override, exactly as it is on the MCP +// tools; with one it is the pin it always was, and the reason is recorded. One builder, so the rescue and the door's half +// of the pin contract are wired in one place (TestEveryDelegatorSurfaceWiresTheRescue reads it one hop away). +func cliPinOptions(priority int, tenant string, rescue delegate.RescueFunc, pinReason string) *delegate.RunOptions { + return &delegate.RunOptions{Priority: priority, Tenant: tenant, Rescue: rescue, PinReason: pinReason, PinNeedsReason: true} +} + +// delegateExitErr maps a run summary onto the verb's exit code. Two things +// exit non-zero, and both mean "a human has to look": +// +// - Failed: transport/config per-subtask errors (auth rejected, dispatch +// refused, broken wire, a node that never answered). +// - Infrastructure: defers whose class blames the STACK or the config rather +// than the work. A node with a dead llama-swap defers every subtask, and +// without this it exits 0 — a broken fleet reading as a good run is exactly +// the silent failure this contract exists to prevent. +// +// Everything else — honest abstentions, budget defers, CONTRACT-classed defers +// (a contract no healthy node would accept: no output_schema, past the origin +// hop, too big for any advertised ceiling), and failed verification — stays +// exit 0: those are RESULT shapes, reported in the JSON above, matching every +// other verb's defer posture. +// +// BOTH counts are reported when both are non-zero. Returning on the first +// non-zero class named only the failures, so an operator who fixed the +// transport error learned about the broken node only on the next run. +// +// THE TWO SURFACES DIVERGE ON PURPOSE (C-75, ADR 0065). The MCP door's error +// flag (mcpserver.delegateIsError) once matched this rule on the case that +// matters — a subtask LOST to the stack (Summary's LostToStack: its contracted +// output never arrived and a box is why, which includes a finished loop whose +// structured re-pack seat was unreachable) — so one of two subtasks eaten by a +// dead llama-server could not read as a clean call on either surface (R5-2). +// C-75 narrowed the flag: IsError says THE CALL FAILED, the MCP client answers an +// error-flagged body by keeping only its head and tail, and a partial call that +// delivered seven digests did not fail. It is now set only when nothing succeeded +// and something failed, was lost or was skipped; a partial result is a successful +// call whose body says what is missing (summary.lost_to_stack, each result's +// defer_class and reason). This verb keeps the wider rule: an exit code sits +// BESIDE the printed results, so it cannot hide a digest, and a script has no +// other loud signal. A subtask lost to the stack, and a fleet-down LOCAL SUCCESS +// (Infrastructure with no LostToStack — every subtask delivered, but the fleet +// failed its probe), exit non-zero here whether or not a sibling succeeded. +func delegateExitErr(sum delegate.Summary) error { + var parts []string + if sum.Failed > 0 { + parts = append(parts, fmt.Sprintf("%d subtask(s) failed (transport/config) — see results[].reason", sum.Failed)) + } + if sum.Infrastructure > 0 { + parts = append(parts, fmt.Sprintf("%d subtask(s) hit infrastructure/config rather than the work (a defer blaming the stack, or a local fallback taken because the fleet failed its health probe) — see results[].defer_class, reason, and placement", sum.Infrastructure)) + } + if len(parts) == 0 { + return nil + } + return errors.New(strings.Join(parts, "; ")) +} + +// parseContractFile reads a delegation contract file: either ONE subtask +// object or an array of them. Tolerant decode (the wire's reader posture) — +// unknown fields are ignored, so a contract file written for a newer harness +// still parses here. +func parseContractFile(path string) ([]delegate.SubtaskSpec, error) { + b, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("reading contract file: %w", err) + } + trimmed := strings.TrimSpace(string(b)) + if strings.HasPrefix(trimmed, "[") { + var specs []delegate.SubtaskSpec + if err := json.Unmarshal([]byte(trimmed), &specs); err != nil { + return nil, fmt.Errorf("contract file %s: %w", path, err) + } + return specs, nil + } + var spec delegate.SubtaskSpec + if err := json.Unmarshal([]byte(trimmed), &spec); err != nil { + return nil, fmt.Errorf("contract file %s: %w", path, err) + } + return []delegate.SubtaskSpec{spec}, nil +} diff --git a/docs/systems/cache-server.md b/docs/systems/cache-server.md index 99019a7e7..0ddb46832 100644 --- a/docs/systems/cache-server.md +++ b/docs/systems/cache-server.md @@ -316,7 +316,7 @@ Raw link: ~980 MB/s →, ~700 MB/s → (iperf3); | one binding: fields, defaults, validation (`ValidateKVCacheServer`, `privateHost`) | `internal/config/kvcacheserver.go` | | the LIST: legacy-object decode, seat selection, cross-binding rules (`ValidateKVCacheServers`) | `internal/config/kvcacheservers.go`, wired in `internal/config/config.go` (`Config.KVCacheServers`, `Config.VLLMSeats`, `load`) | | status report (`kvCacheServerView`, `kvCacheBindingView`) | `internal/mcpserver/mcpserver.go` (`handleStatus`) | -| the per-seat gate (`writeCacheServerSection`) | `main.go` (`doctorRun`) | +| the per-seat gate (`writeCacheServerSection`) | `doctor_cmd.go` (`doctorRun`) | | the render picks a binding by seat name (`applySeatBinding`) | `install_seatbinding.go`, flagged in `install_vllmseat.go` (`--config`) | | the seat's own half and the binding a tier derives (`ConfigBlock`, `ConfigBinding`, `Bindings`) | `internal/vllmseat/vllmseat.go` | | tests | `internal/config/kvcacheserver_test.go`, `internal/config/kvcacheservers_test.go`, `internal/mcpserver/status_test.go`, `doctor_cacheserver_test.go`, `install_seatbinding_test.go` | diff --git a/docs/systems/delegate-contract-file.md b/docs/systems/delegate-contract-file.md index 150865748..11ae1291e 100644 --- a/docs/systems/delegate-contract-file.md +++ b/docs/systems/delegate-contract-file.md @@ -178,7 +178,7 @@ change to the flag help's `(fields: ...)` list fails the doc test below until th | piece | where | |---|---| | the verb, its flags and the `--contract` help | `runDelegate` in `main.go` | -| the file reader (bare object or array, tolerant decode) | `parseContractFile` in `main.go` | +| the file reader (bare object or array, tolerant decode) | `parseContractFile` in `delegate_cmd.go` | | the subtask object and its intake | `SubtaskSpec`, `PrepareContract` in `internal/delegate/intake.go` | | the subtask cap | `CheckSubtaskCount` in `internal/delegate/intake_batch.go` | | the contract fields and their json names | `AgentContract` in `internal/core/agentwire.go` | diff --git a/docs/systems/delegation-client.md b/docs/systems/delegation-client.md index 80486cef1..0a0572786 100644 --- a/docs/systems/delegation-client.md +++ b/docs/systems/delegation-client.md @@ -100,7 +100,7 @@ then the shorter queue, and names every node it skipped when none can take the j |---|---| | `install_client.go` | `install client`: the config it writes, `hasLocalModel` | | `setup/install.sh`, `setup/install.tests.sh` | `--client`, `--remotes`, `--token-file`, and their dry-run tests | -| `acceptance_cmd.go`, `main.go` (`doctorRun`) | the alias check and the endpoint health line that SKIP with no local model | +| `acceptance_cmd.go`, `doctor_cmd.go` (`doctorRun`) | the alias check and the endpoint health line that SKIP with no local model | | `internal/delegate/run_localseat.go` (`localServesLayer`, `seatless`) | a view that names no agent seat is never a placement | | `internal/composeremote/` | `offload_compose_video`'s `route`: node choice, the template and project doors, fetching the outputs back | | `internal/composebundle/` | packing a project, and the extract and reference checks both sides run | diff --git a/docs/systems/fleet-node.md b/docs/systems/fleet-node.md index 1adabd8da..c28e66cb3 100644 --- a/docs/systems/fleet-node.md +++ b/docs/systems/fleet-node.md @@ -260,7 +260,7 @@ aliases over it so the composite tier's placement guards read cards through the that same 2-second sampler instead of the single-value query, and publishes the full breakdown as `gpu_devices[]` in health — additive, and **always present when nvidia-smi or linux-amdgpu is the resolved source, including a single-GPU box** (a one-element array; there is no single-GPU special case — -`chooseSamplerKind` and `startVRAMSampler` in `main.go` are the exact routing decision, unit-tested in +`chooseSamplerKind` (`fleet_serve_cmd.go`) and `startVRAMSampler` (`main.go`) are the exact routing decision, unit-tested in `fleet_verbs_test.go` and `fleet_sampler_test.go`; a linux-amdgpu row has an empty uuid and no utilisation). It is omitted on windows-generic, which has no per-adapter signal to enumerate, and on linux-meminfo, the SoC's one RAM pool with no cards. A new field is additive in practice too: the fleet-dispatcher decodes health with a @@ -1844,7 +1844,7 @@ More shapes originate on the **delegator**, not the node: a week. Absence of evidence about the fleet is never evidence about the contract. `infrastructure` and `config` defers are counted into the delegator's `summary.infrastructure` -and make `local-offload delegate` exit non-zero (`delegateExitErr` in `main.go`): a broken or +and make `local-offload delegate` exit non-zero (`delegateExitErr` in `delegate_cmd.go`): a broken or misconfigured node must not read as a successful run. `contract` deliberately is not. `summary.infrastructure` also counts a **local placement taken while every configured remote was failing its health probe** (`route=auto`, local GPU busy): the placement is right and the work @@ -2334,7 +2334,7 @@ died is still on the node under its job id; the next identical call from that ma with the command llama-swap spawned for each loaded seat - [`doctor_backends.go`](../../doctor_backends.go) — the `backends:` section and `--probe-completion` of `doctor`, and the tier-table reader the lane plan uses -- [`main.go`](../../main.go) — `fleet-serve` / `fleet-measure` verbs +- [`main.go`](../../main.go), [`fleet_serve_cmd.go`](../../fleet_serve_cmd.go) — `fleet-serve` / `fleet-measure` verbs ## Related docs diff --git a/docs/systems/measurement-gate.md b/docs/systems/measurement-gate.md index 854dedfa4..95b9fe719 100644 --- a/docs/systems/measurement-gate.md +++ b/docs/systems/measurement-gate.md @@ -494,7 +494,7 @@ scores from another judge for the same reason, whatever scale they share. These two call `eval.BootstrapDeltaMean` directly with `B = 2000` and `seed = 42`, read their own interval and are **not migrated** by the gate; they stay as they are: -- the `agent-trajectory-gate` verb (`runAgentTrajectoryGate` in `main.go`); +- the `agent-trajectory-gate` verb (`runAgentTrajectoryGate` in `learn_cmd.go`); - the compaction A/B (`compeval.RunAB` in `internal/compeval/abtest.go`). New callers use the gate. `BootstrapDeltaMean` itself is unchanged, to the bit, and a test pins it. diff --git a/docs/systems/security.md b/docs/systems/security.md index d459726ea..630f883d1 100644 --- a/docs/systems/security.md +++ b/docs/systems/security.md @@ -176,8 +176,8 @@ R1–R6 are MUST, R7–R9 SHOULD. | bare-client gate (G2) | `bare_http_client_lint_test.go`, `bare_http_client_allowlist_test.go` | | mutation proof harness (I5) | `scripts/mutation-check.sh`, `scripts/mutation-check.ps1`; self-test `scripts/mutation-check-selftest.sh`, `scripts/mutation-check-selftest.ps1` over `scripts/testdata/mutfixture` | | untrusted-content fences | `internal/agent/fetchtool.go` (`fenceUntrusted`), `internal/agent/browsetool.go`; the shared rules in `internal/untrusted`, applied to research bodies in `internal/mcpserver` (G13) | -| fleet node listeners and the fleet bearer | `internal/fleetnode/server.go`, `main.go` (`fleetServeParams`) | -| posture report | `main.go` (`doctor`) | +| fleet node listeners and the fleet bearer | `internal/fleetnode/server.go`, `fleet_serve_cmd.go` (`fleetServeParams`) | +| posture report | `doctor_cmd.go`, `main.go` (`doctor`) | ## Related diff --git a/doctor_cmd.go b/doctor_cmd.go new file mode 100644 index 000000000..dcf31042b --- /dev/null +++ b/doctor_cmd.go @@ -0,0 +1,528 @@ +// The doctor verb: its report sections and the config, data-home, fleet and +// media checks it prints. + +package main + +import ( + "flag" + "fmt" + "io" + "os" + "sort" + "strings" + "sync" + "time" + + "github.com/dmmdea/offload-harness/internal/config" + "github.com/dmmdea/offload-harness/internal/datahome" + "github.com/dmmdea/offload-harness/internal/mediacap" + "github.com/dmmdea/offload-harness/internal/netguard" + "github.com/dmmdea/offload-harness/internal/rosterprobe" + "github.com/dmmdea/offload-harness/internal/volumes" +) + +func runDoctor(args []string) error { + fs := flag.NewFlagSet("doctor", flag.ExitOnError) + fs.String("config", "", "config file path") + _ = fs.Parse(args) + cfg, src := loadCfgWithSource(fs) + // Disclose the config source FIRST — and truthfully: SourceLine never credits a file + // that was not actually read. Every binding verdict below is only as good as the file + // that produced it, so a file that FAILED validation is named here, verbatim, before + // anything else is printed. + tainted := doctorConfigRow(src, os.Stdout) + err := doctorRunChecked(cfg, mediacap.RoutesChecked(cfg, mediacap.LiveNodeChecker(doctorComfyAPI(), 3*time.Second)), os.Stdout, liveDataReport(cfg)) + if err != nil { + return err + } + if tainted { + return fmt.Errorf("config at %s %s: %w", src.Path, config.LoadFailure(src.LoadErr), src.LoadErr) + } + return nil +} + +// doctorComfyAPI is the ComfyUI endpoint doctor asks for its node classes: the one the +// render runners use (COMFY_API, else ComfyUI's default port). doctor never starts +// ComfyUI; when nothing answers there, the node classes are decided from +// /custom_nodes on disk (mediacap.LiveNodeChecker). +func doctorComfyAPI() string { + if v := strings.TrimSpace(os.Getenv("COMFY_API")); v != "" { + return v + } + return "http://127.0.0.1:8188" +} + +// doctorConfigRow prints doctor's config-source disclosure and, when the file +// failed validation, the loader's refusal text VERBATIM as a FAIL row. It +// reports whether doctor must exit non-zero because of it. +// +// The refusal text is the only thing that names the KEY and the VALUE. doctor +// kept the Source and printed only the generic one-liner, so on a config the +// loader had refused it printed a vague "failed to load", ran every other check +// against the file's surviving settings, and exited 0 — the one verb an operator +// runs to find out what is wrong stayed silent about the thing that was wrong. +// +// The row goes FIRST and the rest of doctor still runs: the media, cache-server +// and config-findings sections are pure config and stay useful, and an operator +// fixing one key wants to see the others in the same pass. +func doctorConfigRow(src config.Source, w io.Writer) bool { + fmt.Fprintln(w, config.SourceLine(src)) + if src.LoadErr == nil { + return false + } + fmt.Fprintf(w, "config: FAIL %s — %v\n", strings.ToUpper(config.LoadFailure(src.LoadErr)), src.LoadErr) + return true +} + +// aliasCheck pairs a config key with its configured llama-swap model alias. +type aliasCheck struct{ Key, Alias string } + +// modelAliases is the subset of config.ModelRoutes the alias-vs-live-roster gate +// checks — the bindings whose absence from the roster means a route this node +// advertises cannot actually run. The table itself lives in internal/config so +// `doctor`, `acceptance`, `report`, and the MCP server's `offload_status` read +// ONE list; they used to carry two hardcoded ones that had drifted 8 keys to 10. +func modelAliases(cfg config.Config) []aliasCheck { + routes := cfg.ModelRoutes() + out := make([]aliasCheck, 0, len(routes)) + for _, r := range routes { + if r.Diffed { + out = append(out, aliasCheck{Key: r.Key, Alias: r.Configured}) + } + } + return out +} + +// doctorRun checks endpoint health, then diffs the LIVE /v1/models roster +// against every configured model alias (LO-11: doctor used to GET only +// /health, so a renamed/removed llama-swap alias passed doctor yet every call +// deferred). Any missing alias prints FAIL and the command exits non-zero. +// +// It also reports the DERIVED media routes (internal/mediacap): doctor checked +// model aliases only, so on a node whose generate_image deferred on a render +// script that was not on disk it still printed green. Passing them in keeps the +// filesystem probe at the caller's edge, so the check itself stays testable. +func doctorRun(cfg config.Config, routes []mediacap.Route, w io.Writer) error { + return doctorRunChecked(cfg, routes, w, nil) +} + +// Seams for the machine facts the data-volume audit reads: the drive Windows boots +// from and the volumes this host mounts. A test stands in for the machine. +var ( + doctorOSDrive = volumes.OSDriveRoot + doctorVolumes = volumes.List +) + +// liveDataReport audits THIS machine: nil when it has no OS drive letter (the rule is +// about the drive Windows boots from), else the report built from the mounted volumes. +// A volume list that cannot be read is an empty one, which the audit words as "no +// other volume qualifies" rather than guessing a target. +func liveDataReport(cfg config.Config) *datahome.Report { + osDrive := doctorOSDrive() + if osDrive == "" { + return nil + } + // The volume list is only worth enumerating when something sits on the OS drive: a + // clean node must not pay for a drive scan (a dead mapped share can stall one). + r := datahome.Audit(cfg, osDrive, nil) + if len(r.OnOS) == 0 { + return &r + } + vols, err := doctorVolumes() + if err != nil { + vols = nil + } + r = datahome.Audit(cfg, osDrive, vols) + return &r +} + +// writeDataHomeSection prints the data-volume verdict (register C-92) and returns how +// many data locations make doctor FAIL: those on the OS drive while a data volume +// qualifies to take them. The operator rule is that C: holds Windows and program +// installs, never data; the section names every config key whose resolved path sits +// there and the way off it. A node with nowhere to move to gets one note and no FAIL, +// and a clean node (or a host with no OS drive letter) prints nothing at all. +func writeDataHomeSection(w io.Writer, r *datahome.Report) int { + if r == nil || len(r.OnOS) == 0 { + return 0 + } + if r.Target == nil { + fmt.Fprintf(w, "data volume: - %d data location(s) are on the OS drive %s but no other volume qualifies (%s); there is nowhere to move them to\n", + len(r.OnOS), r.OSDrive, r.Why) + return 0 + } + home := datahome.ProposedHome(r.Target.Volume.Root) + fmt.Fprintf(w, "data volume (%s holds Windows and program installs, never data):\n", strings.TrimRight(r.OSDrive, `\/`)) + root, under, others := r.Grouped() + if root != nil { + fmt.Fprintf(w, " FAIL %-28s %s\n", root.Key, root.Path) + if len(under) > 0 { + names := make([]string, len(under)) + for i, l := range under { + names[i] = l.Key + } + fmt.Fprintf(w, " and %d paths under it that follow it: %s\n", len(under), strings.Join(names, ", ")) + } + } + for _, l := range others { + fmt.Fprintf(w, " FAIL %-28s %s\n", l.Key, l.Path) + } + fmt.Fprintf(w, " move to %s (%s). The data is COPIED, never moved and never put behind a junction:\n", r.Target.Volume.Root, r.Target.Because) + fmt.Fprintf(w, " 1. stop fleet-serve and every MCP door (they hold the cache and the ledger open)\n") + fmt.Fprintf(w, " 2. local-offload data migrate --to %s --apply --stopped\n", home) + fmt.Fprintf(w, " 3. in config.json set \"home\": \"%s\" (a key you wrote by hand above keeps its value: change it too)\n", datahome.HomeValue(home)) + fmt.Fprintf(w, " 4. restart the doors and re-run doctor; delete the old tree only after the new home has run clean\n") + return len(r.OnOS) +} + +// writeFleetSkewSection prints one row per fleet remote (delegate_remotes): OK +// when the node runs this binary's release, SKEW with both versions when it +// does not, UNKNOWN when the node publishes no version, UNREACHABLE when its +// health cannot be read. Parity broke twice on 2026-09-30 within an hour (a +// merge landed between deploys); the session-start audit catches it, and now +// so does any session that runs doctor. No remotes prints nothing. +func writeFleetSkewSection(w io.Writer, cfg config.Config, self string, read func(base string) (string, error)) { + members := rosterprobe.Members(cfg.DelegateRemotes) + if len(members) == 0 { + return + } + fmt.Fprintf(w, "fleet versions (this binary %s):\n", self) + for _, m := range members { + v, err := read(m.Base) + base := rosterLabel(m) + switch { + case rosterprobe.IsRefusal(err): + fmt.Fprintf(w, " REFUSED %s — not dialled, %s\n", base, rosterprobe.Scrub(m.Base, err)) + case err != nil: + fmt.Fprintf(w, " UNREACHABLE %s — %s\n", base, rosterprobe.Scrub(m.Base, err)) + case v == "": + fmt.Fprintf(w, " UNKNOWN %s — the node publishes no harness_version\n", base) + case v != self: + fmt.Fprintf(w, " SKEW %s runs %s (this binary %s): redeploy to parity (plans/fleet-parity-deploy.sh)\n", base, v, self) + default: + fmt.Fprintf(w, " OK %s %s\n", base, v) + } + } +} + +// doctorFleetProbeTimeout is how long doctor waits for one node's health, and for its answer to the +// token probe. The roster is read concurrently, so this is the whole section's worst case. +const doctorFleetProbeTimeout = 5 * time.Second + +// rosterLabel is how doctor names a roster entry: its slot in delegate_remotes (the number the config +// findings use, blank slots counted) and its base with whatever a URL can carry that is a secret +// removed, so an operator can find the line to fix and the output is safe to paste. The redaction +// is the roster package's (Member.Shown, rosterprobe.Scrub), shared with the lanes' "probed ..." lines. +func rosterLabel(m rosterprobe.Member) string { + return fmt.Sprintf("delegate_remotes[%d] %s", m.Index, m.Shown()) +} + +// rosterVersionReader answers writeFleetSkewSection from one roster reading: a node's published +// harness_version, or why there is none (the entry was refused and never dialled, or the node did not +// answer). +func rosterVersionReader(roster []rosterprobe.Reading) func(base string) (string, error) { + byBase := make(map[string]rosterprobe.Reading, len(roster)) + for _, r := range roster { + byBase[r.Base] = r + } + return func(base string) (string, error) { + r, ok := byBase[base] + switch { + case !ok: + return "", fmt.Errorf("not read") + case r.Err != nil: + return "", r.Err + } + return r.View.HarnessVersion, nil + } +} + +// writeFleetTokenSection prints whether each reachable node accepts this box's fleet_auth_token, and +// says plainly when this box has none. +// +// /fleet/health ignores the Authorization header, so a token that is missing, wrong or absent on the +// node reads healthy on every other row; the first sign was a 401 on dispatch, which the delegator does +// not re-place. check asks the node the one question health cannot (rosterprobe.CheckToken: an agent +// dispatch with no job_id, which the node answers after its bearer check and before it creates +// anything). Only nodes whose health was read are asked, concurrently; a refused or unreachable +// entry is named UNCHECKED, with the reason the version rows already gave. +// +// Informational, like the version rows: it never changes doctor's exit code, because a node that +// disagrees about a token is a finding about the pair, not a broken local box. The token itself and +// the node's reply text are never printed, only the status class. +func writeFleetTokenSection(w io.Writer, cfg config.Config, roster []rosterprobe.Reading, check func(base string) (status int, body string, err error)) { + if len(roster) == 0 { + return + } + if cfg.FleetAuthToken == "" { + fmt.Fprintln(w, "fleet token: NOT SET here (fleet_auth_token) — every lane of a node requires the bearer (401 when the node has a token, 403 when it listens beyond loopback with none), so this box's calls to a node fail until the same token is set here; only a node on loopback with no token is open") + return + } + type verdict struct { + state rosterprobe.TokenState + status int + err error + } + out := make([]verdict, len(roster)) + var wg sync.WaitGroup + for i, r := range roster { + if r.Err != nil { + continue + } + wg.Add(1) + go func(i int, base string) { + defer wg.Done() + status, body, err := check(base) + out[i] = verdict{state: rosterprobe.ClassifyToken(status, body), status: status, err: err} + }(i, r.Base) + } + wg.Wait() + fmt.Fprintln(w, "fleet token (fleet_auth_token is set here; its value is never printed):") + for i, r := range roster { + base, v := rosterLabel(r.Member), out[i] + switch { + case rosterprobe.IsRefusal(r.Err): + fmt.Fprintf(w, " UNCHECKED %s — not asked: the entry is refused by the tailnet guard (see fleet versions)\n", base) + case r.Err != nil: + fmt.Fprintf(w, " UNCHECKED %s — not asked: the node's health could not be read (see fleet versions)\n", base) + case v.err != nil: + fmt.Fprintf(w, " UNKNOWN %s — the token probe failed: %s\n", base, rosterprobe.Scrub(r.Base, v.err)) + case v.state == rosterprobe.TokenAccepted: + fmt.Fprintf(w, " OK %s — the node accepts this box's token\n", base) + case v.state == rosterprobe.TokenMismatch: + fmt.Fprintf(w, " MISMATCH %s — the node answered 401: its fleet_auth_token is not this box's, so every call to it (every lane) fails until the two match\n", base) + case v.state == rosterprobe.TokenNodeHasNone: + fmt.Fprintf(w, " NO-TOKEN %s — the node answered 403: it listens beyond loopback with no fleet_auth_token, so it refuses every lane for every caller (a current fleet-serve will not start so); set the same token on the node\n", base) + default: + fmt.Fprintf(w, " UNKNOWN %s — the node answered %d to the token probe, which does not say whether the token is right (an older node?)\n", base, v.status) + } + } +} + +// writeConfigFindingsSection prints ONE LINE PER non-fatal configuration finding +// (config.Config.Findings) and returns how many — the number that makes doctor +// exit non-zero. +// +// These are values that LOAD and then cannot do what they say: a fleet remote on +// a port no node listens on, a lane base carrying the /v1 segment the client +// appends itself, a media-lane GPU wait sitting at ten minutes against the 90 s +// its own key documents (register C-33), a retired key the file still carries. +// Every one of them was previously invisible: the retired-key note was a single +// stderr line at startup that scrolls past every command, and the rest were only +// ever observable as a dial timeout or a ten-minute block on a media call. +// +// Findings are reported on the config THIS PROCESS loaded, not a re-read of the +// file, so the rows explain the behaviour the operator is actually seeing — the +// `config:` line above says which file that was. +// +// A clean config prints nothing at all, header included: a green doctor stays as +// short as it is today. +func writeConfigFindingsSection(w io.Writer, cfg config.Config) int { + findings := cfg.Findings() + if len(findings) == 0 { + return 0 + } + fmt.Fprintln(w, "config findings (values in the loaded config that cannot do what they say):") + for _, f := range findings { + fmt.Fprintf(w, " FAIL %s\n", f) + } + return len(findings) +} + +// writeMulticardNotesSection prints a NOTE row for each flat setting the multicard seats do not apply (config.MulticardLaunchNotes): comfy_cuda_device and +// comfy_dynamic_vram "on" still serve the single-card routes, so the row says what the config does and asks for nothing. It prints nothing without one. +func writeMulticardNotesSection(w io.Writer, cfg config.Config) { + for _, n := range cfg.MulticardLaunchNotes() { + fmt.Fprintf(w, "multicard: NOTE %s\n", n) + } +} + +// writeTailnetZonesSection prints the tailnet DNS zones this config admits hostnames under, one +// row, with the key each came from: tailnet_suffix is this operator's own tailnet and each +// tailnet_suffixes entry is another tailnet that shared a node in (ADR 0074). Those are the zones +// netguard.TailnetURL, the cache-store host check and the research lane's refusal all judge, so a +// roster entry that is refused by name is read against this row first. +// +// It prints nothing when no zone is configured, so a green doctor stays as short as it is today: +// loopback, tailnet CGNAT-range address literals and dotless MagicDNS names need no zone, and a dotted name +// that does is reported as a finding on its own row. +func writeTailnetZonesSection(w io.Writer, cfg config.Config) { + var parts []string + seen := map[string]bool{} + add := func(raw, key string) { + z, err := netguard.ParseZone(key, raw) + if err != nil || z == "" || seen[z] { + return // a malformed zone already failed the config row; a repeat is one zone + } + seen[z] = true + parts = append(parts, fmt.Sprintf("%s (%s)", z, key)) + } + add(cfg.TailnetSuffix, "tailnet_suffix") + for i, s := range cfg.TailnetSuffixes { + add(s, fmt.Sprintf("tailnet_suffixes[%d]", i)) + } + if len(parts) == 0 { + return + } + fmt.Fprintf(w, "tailnet zones: %s\n", strings.Join(parts, ", ")) +} + +// writeCacheServerSection prints ONE LINE PER vLLM SEAT of this box and returns how +// many have no binding — the number that makes doctor exit non-zero. +// +// The rule is the operator directive of 2026-09-10: the second device's KV store +// backs EVERY vLLM seat while that device is online. What made that rule +// unenforceable was a config that bound one `kv_cache_server.seat` to a single seat +// name, so a second seat running with no tier looked exactly like a second seat +// nobody had thought about. Here the two are different lines. +// +// An explicit opt-out PASSES: `{"seat":"…","storeless":true,"reason":"…"}` is a +// decision, and a tier that is optional by charter must let a box decline it. What +// does not pass is silence — no binding at all, or a binding switched off with +// nothing saying why. +// +// A box that declares no vllm_seats prints nothing and fails nothing: it runs no +// vLLM seat, so the gate has no subject — and the cascade deliberately STAYS on +// llama.cpp (Gemma-4 hybrids crash LMCache's V2 path, upstream #4263), which is why +// the roster is declared in config rather than sniffed from /v1/models. +func writeCacheServerSection(w io.Writer, cfg config.Config) int { + seats := cfg.VLLMSeats + if len(seats) == 0 { + return 0 + } + fmt.Fprintln(w, "cache server (one binding per vLLM seat; the store backs every seat while the second device is online):") + unbound := 0 + for _, seat := range seats { + b := cfg.KVCacheServers.For(seat) + switch { + case b == nil: + unbound++ + fmt.Fprintf(w, " %-24s FAIL no kv_cache_server binding — give this seat a store, or opt out explicitly with {\"seat\":%q,\"storeless\":true,\"reason\":\"\"}\n", + seat+":", seat) + case b.Storeless: + fmt.Fprintf(w, " %-24s OK storeless by declaration: %s\n", seat+":", b.Reason) + case !b.Enabled: + unbound++ + fmt.Fprintf(w, " %-24s FAIL binding present but enabled:false and no storeless reason — turn the store on, or say why this seat runs without one\n", seat+":") + default: + fmt.Fprintf(w, " %-24s OK %s %s key_prefix=%s l1=%dGB\n", + seat+":", b.StoreName(), b.Address, b.EffectiveKeyPrefix(), b.EffectiveL1StagingGB()) + } + } + for _, seat := range cfg.KVCacheServers.BoundSeatsNotDeclared(seats) { + fmt.Fprintf(w, " %-24s - a binding names this seat, but vllm_seats does not list it (a store nothing reads, or a rename that left the binding behind)\n", seat+":") + } + return unbound +} + +// writeMediaSection prints this machine's DERIVED media capability and returns +// the number of routes bound to something that is absent — the script, a model +// file its graph loads, a custom-node class, or (voice) the TTS python, its +// packages and weights. Only that class is a failure: NOT CONFIGURED is a +// legitimate machine (the task defers by design), while BOUND-BUT-MISSING is a +// config that promises a capability the box cannot deliver — which until now +// surfaced only as a defer at call time. +func writeMediaSection(w io.Writer, routes []mediacap.Route) int { + if len(routes) == 0 { + return 0 + } + fmt.Fprintln(w, "media routes (derived from this config; anything a route loads that is absent — script, model file, custom node class, TTS python/weights — means the task DEFERS when called):") + missing := 0 + for _, r := range routes { + mark := "OK " + switch r.State { + case mediacap.BoundButMissing: + mark = "FAIL" + missing++ + case mediacap.NotConfigured: + mark = "- " + } + fmt.Fprintf(w, " %-22s %s %-18s %-15s %s\n", r.Name+":", mark, string(r.State), r.Engine, r.Detail) + } + return missing +} + +// writeSdcppDeviceSection reports which Vulkan device an sdcpp generate_image +// render will use on this box (mediacap.DetectSdcppDevice mirrors +// render/sdcpp-generate.mjs's own device-resolution order exactly), and flags it +// when that device is an integrated GPU. Prints nothing when sdcpp is not this +// box's image-gen engine or its binary is unbound — writeMediaSection already +// says so. +func writeSdcppDeviceSection(w io.Writer, cfg config.Config) { + if cfg.ImageGenEngine != "sdcpp" || cfg.SdcppBin == "" { + return + } + d := mediacap.DetectSdcppDevice(cfg.SdcppBin, os.Getenv("GGML_VK_VISIBLE_DEVICES"), nil) + switch { + case d.ProbeErr != "" && d.Name == "": + fmt.Fprintf(w, "sdcpp device: ? could not determine the Vulkan device (%s) — GGML_VK_VISIBLE_DEVICES=%s will be used as-is\n", d.ProbeErr, d.Index) + case d.IsIGPU: + src := "auto-selected (no discrete adapter found)" + if d.EnvOverride { + src = "from GGML_VK_VISIBLE_DEVICES" + } + fmt.Fprintf(w, "sdcpp device: WARN Vulkan%s %s is an INTEGRATED GPU (%s) — renders will be slow; set GGML_VK_VISIBLE_DEVICES to the discrete adapter's index to override\n", d.Index, d.Name, src) + default: + how := "auto-selected discrete adapter" + if d.EnvOverride { + how = "from GGML_VK_VISIBLE_DEVICES" + } + fmt.Fprintf(w, "sdcpp device: OK Vulkan%s %s (%s)\n", d.Index, d.Name, how) + } +} + +// writeVideoFamilyBindingsSection prints the RESOLVED per-family video bindings +// (gap: videogen_text_encoder and its siblings used to be a single machine-wide +// value applied to whichever family actually rendered — an explicit per-request +// `model` override could silently inherit the box's OTHER family's transformer/ +// text-encoder/VAEs; bigger-models-2026-09-24.md "Interim Phase 2 round 2"). +// Sorted key order for a stable diff between doctor runs. Purely informational — +// it never changes doctor's exit code, matching the sdcpp device line above. +func writeVideoFamilyBindingsSection(w io.Writer, rows []mediacap.VideoFamilyBindingRow) { + if len(rows) == 0 { + return + } + fmt.Fprintln(w, "video family bindings (what THIS machine resolves for each family, independent of which one a request actually selects):") + for _, r := range rows { + tag := "" + if r.Default { + tag = " (this box's default)" + } + fmt.Fprintf(w, " %s%s:\n", r.Family, tag) + keys := make([]string, 0, len(r.Files)) + for k := range r.Files { + keys = append(keys, k) + } + sort.Strings(keys) + // The declared license pair, as fields (CT-47; operator order 2026-10-01: no + // warning text). A family that declares none prints nothing, like an image + // family's route line. + if r.License != nil && r.CommercialUse != nil { + fmt.Fprintf(w, " %-55s %s (commercial_use %t)\n", "license:", *r.License, *r.CommercialUse) + } + for _, k := range keys { + fmt.Fprintf(w, " %-55s %s\n", k+":", r.Files[k]) + } + } +} + +// writeModelBindingsSection prints one line per configured ComfyUI model name +// with where it was found under /models, and returns how many are +// MISSING or MISPLACED (register F-31). A box with no ComfyUI models directory +// prints nothing: there is nothing to bind there. +func writeModelBindingsSection(w io.Writer, bindings []mediacap.Binding) int { + if len(bindings) == 0 { + return 0 + } + fmt.Fprintln(w, "comfyui model bindings (each name must sit in the class directory its loader node opens; MISSING/MISPLACED = the graph is rejected at render time; INCOMPLETE = a file of this name is there but it is smaller than the size the installer recorded when it downloaded it, e.g. a copy still in progress):") + broken := 0 + for _, b := range bindings { + mark := "OK " + if b.State != mediacap.BindingFound { + mark = "FAIL" + broken++ + } + fmt.Fprintf(w, " %-26s %s %-10s %s\n", b.Key+":", mark, string(b.State), b.Detail) + } + return broken +} diff --git a/fleet_serve_cmd.go b/fleet_serve_cmd.go new file mode 100644 index 000000000..ee5dd08e1 --- /dev/null +++ b/fleet_serve_cmd.go @@ -0,0 +1,437 @@ +// fleet-serve support: nvidia-smi probes, the VRAM sampler chooser, startup +// parameters, drain and await, fleet-measure and the advertised accelerators. + +package main + +import ( + "context" + "encoding/json" + "flag" + "fmt" + "io" + "os" + "strings" + "time" + + "github.com/dmmdea/offload-harness/internal/config" + "github.com/dmmdea/offload-harness/internal/core" + "github.com/dmmdea/offload-harness/internal/fleetnode" + "github.com/dmmdea/offload-harness/internal/gpuprobe" + "github.com/dmmdea/offload-harness/internal/netguard" + "github.com/dmmdea/offload-harness/internal/pairworkloads" + "github.com/dmmdea/offload-harness/internal/winexec" +) + +// nvidiaSmiMemory shells the global VRAM query (MiB CSV) that feeds the +// fleet-serve startup GATE probe (ResolveProvider's "does nvidia-smi work at +// all" check); fleetnode.ParseSmiMemory parses it. The ONGOING 2s health +// sampler uses nvidiaSmiMemoryDevices instead (below) — a multi-GPU box needs +// the per-device breakdown that query can't provide (2 fields, no index/name; +// see ParseSmiMemory's doc comment for why this query/parser pair is frozen +// rather than extended: the pipeline's per-process footprint delta sampler +// also depends on this exact 2-field shape). +func nvidiaSmiMemory() (string, error) { + out, err := winexec.Command("nvidia-smi", "--query-gpu=memory.total,memory.used", "--format=csv,noheader,nounits").Output() + return string(out), err +} + +// nvidiaSmiMemoryDevices shells the PER-DEVICE VRAM query (index, uuid, name, +// memory.total, memory.used — MiB CSV, one line per GPU) that feeds the +// fleet-serve 2s health sampler on a working nvidia-smi node; +// fleetnode.ParseSmiMemoryDevices parses it. This is what actually fixes the +// multi-GPU mis-report: nvidia-smi's own line order is PCI bus order, not +// CUDA device order, so which line is "first" says nothing about which +// device a render will actually use (fleetnode.HeadlineDevice picks the +// headline device from the parsed set instead of trusting enumeration order, +// and fleetnode.SelectHeadlineDevice lets an operator override that pick by +// UUID via the primary_gpu_uuid config key — the uuid field is why it's in +// this query at all: index/total-VRAM alone can't reliably identify a card +// across a reboot or reseat, but the UUID is burned into it). +// utilization.gpu is included to advertise which device is currently busiest, +// helping the PAIR operator optimize placement and scheduling. +// +// Since 0.116.0 the command itself lives in gpuprobe.NvidiaSmiRunner (the +// leaf the composite tier's placement guards read through as well): ONE +// query string, ONE PATH-or-System32 lookup, so the health sampler and the +// display-card guard can never disagree about what nvidia-smi was asked. +func nvidiaSmiMemoryDevices() (string, error) { + return gpuprobe.NvidiaSmiRunner()() +} + +// samplerKind is which VRAM sampler runFleetServe starts for a resolved +// provider Source — the exact seam that decides whether /fleet/health's +// gpu_devices[] is populated or omitted. Pulled out as its own type + pure +// selection function (chooseSamplerKind, below) specifically so that claim is +// unit-tested against the SAME decision runFleetServe ships (main_test.go), +// not a hand-copied approximation of it — runFleetServe calls +// chooseSamplerKind rather than re-implementing the branch inline. +type samplerKind int + +const ( + // samplerKindDevice runs the per-device nvidia-smi query + // (nvidiaSmiMemoryDevices/fleetnode.StartDeviceProbeSampler). gpu_devices[] + // is ALWAYS present on this path — including a single-GPU nvidia-smi box, + // which reports it with exactly one entry. There is no single-vs-multi-GPU + // special case anywhere in this file: nvidia-smi always gets this sampler. + samplerKindDevice samplerKind = iota + // samplerKindSingle runs the plain single-value MemProbe + // (fleetnode.StartProbeSampler) — today only the windows-generic source, + // which has no per-adapter identity to enumerate (vram_windows.go). + // gpu_devices[] is omitted (Snapshot.Devices stays nil) on this path. + samplerKindSingle + // samplerKindAmdgpuDevice runs the per-device amdgpu sysfs query + // (fleetnode.AmdgpuSysfsDeviceProbe) through the same device sampler, so a linux-amdgpu + // node publishes gpu_devices[] the way an nvidia-smi node does (ADR 0053 decision 4). The + // card's identity is its sysfs index and name: there is no UUID to pin a headline to. + samplerKindAmdgpuDevice +) + +// amdgpuSource is the generic provider's Source on a Linux box that reads the amdgpu driver's +// sysfs counters: what genericMemProvider names it and what chooseSamplerKind recognises. +const amdgpuSource = "linux-amdgpu" + +// genericMemProvider picks the generic GPU memory source for this OS and the +// installed tier — pulled out of runFleetServe, like chooseSamplerKind, so the +// selection that ships is the selection that is tested (fleet_verbs_test.go). +// +// Windows gets the WDDM provider and Linux the amdgpu sysfs one. A tier that is a +// unified-memory SoC (rockchip-rk3588) has neither counter, so it reads +// /proc/meminfo less the operator's reserve; procRoot is "/" in production and +// only that provider reads through it. A cpu-profile box means detect found NO +// usable GPU: it must not fleet-serve off iGPU-adjacent counters — only a working +// nvidia-smi may qualify it, so its generic source is nil. +func genericMemProvider(goos, profile string, uma bool, umaReserveGiB float64, procRoot string) fleetnode.GenericProvider { + generic := fleetnode.GenericProvider{Probe: fleetnode.GenericWindowsProbe(uma), Source: "windows-generic"} + if goos == "linux" { + generic = fleetnode.GenericProvider{Probe: fleetnode.AmdgpuSysfsProbe("/sys/class/drm", uma), Source: amdgpuSource} + } + switch profile { + case "rockchip-rk3588": + generic = fleetnode.GenericProvider{Probe: fleetnode.MeminfoUMAProbe(procRoot, umaReserveGiB), Source: fleetnode.MeminfoSource} + case "cpu": + generic.Probe = nil + } + return generic +} + +// chooseSamplerKind picks which sampler runFleetServe starts, given the +// resolved GPU memory provider's Source ("nvidia-smi" | "windows-generic" | +// "linux-amdgpu" | "linux-meminfo"). +// nvidia-smi always gets samplerKindDevice — a single-GPU nvidia-smi box is +// NOT special-cased to the single-value sampler; it runs the per-device query +// and /fleet/health reports gpu_devices[] with one entry, same as any other +// nvidia-smi node. linux-amdgpu gets samplerKindAmdgpuDevice, the same shape from the +// driver's sysfs. Every other source (windows-generic, which has no per-adapter identity, and +// linux-meminfo, one RAM pool with no cards) falls through to samplerKindSingle. +func chooseSamplerKind(source string) samplerKind { + switch source { + case "nvidia-smi": + return samplerKindDevice + case amdgpuSource: + return samplerKindAmdgpuDevice + } + return samplerKindSingle +} + +// gpuArchFromName maps an nvidia-smi product name to the architecture CLASS +// the health payload advertises as gpu_arch — the dispatcher routes on arch +// classes, not product names ("NVIDIA GeForce RTX 3070 Laptop GPU" is not a +// schedulable fact; "ampere" is). Mirrors setup/detect.ps1's Get-GpuArch: +// note "RTX PRO" must match in its own right BEFORE any 50xx logic — the PRO +// branding ("RTX PRO 5000") breaks the "RTX 50" substring, and that branding +// IS the Blackwell pro generation (pre-Blackwell pro cards were "RTX A6000" / +// "RTX 6000 Ada Generation", which match neither rule). Unrecognized products +// (and an empty name from a failed query) fall back to the lowercase vendor +// "nvidia" — an honest "NVIDIA, generation unknown", never "". +func gpuArchFromName(name string) string { + n := strings.ToUpper(name) + switch { + case strings.Contains(n, "RTX PRO"), strings.Contains(n, "BLACKWELL"), strings.Contains(n, "RTX 50"): + return "blackwell" + case strings.Contains(n, "RTX 40"): + return "ada" + case strings.Contains(n, "RTX 30"): + return "ampere" + case strings.Contains(n, "RTX 20"), strings.Contains(n, "GTX 16"): + return "turing" + case strings.Contains(n, "V100"): + return "volta" + default: + return "nvidia" + } +} + +// nvidiaSmiGPUName returns the GPU product name feeding gpuArchFromName +// (best-effort: "" when the query fails — the probe already proved the GPU +// works, and the arch mapper turns "" into the vendor fallback). +func nvidiaSmiGPUName() string { + out, err := winexec.Command("nvidia-smi", "--query-gpu=name", "--format=csv,noheader").Output() + if err != nil { + return "" + } + name := strings.TrimSpace(string(out)) + if i := strings.IndexByte(name, '\n'); i >= 0 { + name = strings.TrimSpace(name[:i]) + } + return name +} + +// fleetServeParams resolves + validates fleet-serve's identity/binding args — +// the testable seam (mirrors runGraphParams): flag > config for the listen +// address and node id, hostname fallback for an empty id, the shared +// netguard loopback refusal unless --listen-trusted-network, and the rule that +// a listener beyond loopback needs a fleet_auth_token. Every lane of a node requires +// the bearer, so a node that listens beyond loopback with none has no credential +// that could authorize any caller: it fails to start, naming the key and the config +// file it read (src), instead of serving a port that refuses everything (or, before +// this rule, serving it open). +func fleetServeParams(listenFlag, nodeIDFlag string, trusted bool, cfg config.Config, src config.Source, hostname func() (string, error)) (listen, nodeID string, err error) { + listen = listenFlag + if listen == "" { + listen = cfg.FleetListen + } + if listen == "" { + listen = "127.0.0.1:18811" // config.Default's value; belt-and-suspenders for a zeroed config + } + if err := netguard.Validate(listen, trusted); err != nil { + return "", "", err + } + if !netguard.LoopbackAddr(listen) && cfg.FleetAuthToken == "" { + where, fix := fleetServeConfigWhere(src) + return "", "", fmt.Errorf("refusing to serve on %q: it is not a loopback address and fleet_auth_token is not set (%s). "+ + "Every fleet lane requires the fleet bearer on a node that listens beyond loopback, so with no token no caller could be "+ + "authorized. %s (the same value on this node and on every delegator that calls it), or listen on a loopback address", listen, where, fix) + } + nodeID = nodeIDFlag + if nodeID == "" { + nodeID = cfg.FleetNodeID + } + if nodeID == "" { + if h, herr := hostname(); herr == nil && h != "" { + nodeID = h + } else { + nodeID = "fleet-node" + } + } + return listen, nodeID, nil +} + +// fleetServeConfigWhere says which config the node read and what to do about it, for the refusal that a listener beyond +// loopback has no fleet_auth_token: a node that will not start must name the file to fix, because a box can hold two +// (LOCAL_OFFLOAD_CONFIG against the default, --config against both). +func fleetServeConfigWhere(src config.Source) (where, fix string) { + switch { + case src.Path == "": + return "no config file was found, so this node runs on built-in defaults", + "Put fleet_auth_token in a config file and point --config or LOCAL_OFFLOAD_CONFIG at it" + case src.NotFound: + return fmt.Sprintf("the config file %s was named but does not exist", src.Path), + fmt.Sprintf("Create %s with fleet_auth_token set, or point --config or LOCAL_OFFLOAD_CONFIG at this node's config", src.Path) + default: + return fmt.Sprintf("the config this node read is %s", src.Path), + fmt.Sprintf("Set fleet_auth_token in %s", src.Path) + } +} + +// fleetServeAuthNote is the startup line that says which lanes require the fleet bearer. Only two states can reach it, +// because fleetServeParams refuses the third (a listener beyond loopback with no token): a loopback node with no +// token, where every lane is open to this box and nobody else, and a node with a token, where every lane requires it. +func fleetServeAuthNote(listen string, cfg config.Config) string { + if cfg.FleetAuthToken == "" { + return fmt.Sprintf("loopback only: every fleet lane is tokenless on %s (set fleet_auth_token to require the bearer)", listen) + } + where := "on loopback" + if !netguard.LoopbackAddr(listen) { + where = fmt.Sprintf("beyond loopback on %q", listen) + } + return fmt.Sprintf("every lane requires the fleet bearer (fleet_auth_token is set; this node listens %s): a request without it is refused with 401; health and the jobs feed answer without it, with their private fields withheld", where) +} + +// fleetServeConfig returns cfg with the node id fleet-serve resolved (fleetServeParams) set as the id the pipeline stamps +// on its result envelopes, so a node names itself one way in /fleet/health, in a queue claim and on every result. +func fleetServeConfig(cfg config.Config, nodeID string) config.Config { + cfg.FleetNodeID = nodeID + return cfg +} + +// reportPipelineJobsSweep writes the startup sweep's outcome to w: up to three +// lines, each reported whether or not the others are, so a sweep that removed +// 22 dirs and could not remove one says both. The removed line does not call +// those dirs a crash's leftovers: a dir is also removed for its age, and the +// sweep logs each of those itself, above this summary. +func reportPipelineJobsSweep(w io.Writer, swept, kept int, serr error) { + if serr != nil { + fmt.Fprintf(w, "[fleet-serve] warning: pipeline-jobs sweep: %v\n", serr) + } + if swept > 0 { + fmt.Fprintf(w, "[fleet-serve] swept %d orphaned pipeline-job dir(s): an earlier instance's leftovers, runs whose delegator exited, or dirs past the age bound (those are logged one by one above)\n", swept) + } + if kept > 0 { + fmt.Fprintf(w, "[fleet-serve] kept %d pipeline-job dir(s) that may belong to local runs still in flight (owned by a live process, or too recent to call orphaned)\n", kept) + } +} + +// fleetServeAwait blocks until fleet-serve stops, and drains on BOTH ways out. An interrupt drains +// BEFORE closing the listener: new dispatches already 503, but pollers can still read states while +// in-flight renders finish. A Serve that returns on its own (the listener failed) leaves the same jobs +// in flight and the same card posts pending, so it drains too and then returns Serve's error: the +// process exits either way, and without the drain the cards of the jobs it was running stay open +// until the next process's orphan sweep closes them failed. +func fleetServeAwait(ctx context.Context, errCh <-chan error, ln io.Closer, jobs *fleetnode.Jobs, pair *pairworkloads.Emitter, timeout time.Duration) error { + select { + case err := <-errCh: + fmt.Fprintf(os.Stderr, "[fleet-serve] server stopped (%v) — draining jobs (up to %s); survivors are marked error:\"interrupted\"\n", err, timeout) + fleetServeDrain(jobs, pair, timeout) + return err + case <-ctx.Done(): + fmt.Fprintf(os.Stderr, "[fleet-serve] interrupt — draining jobs (up to %s); survivors are marked error:\"interrupted\"\n", timeout) + fleetServeDrain(jobs, pair, timeout) + ln.Close() + <-errCh + return nil + } +} + +// fleetServeDrain is fleet-serve's shutdown drain: it stops the job store (in-flight jobs finish, the +// never-started ones are dropped), then waits for the node emitter's background posts. The terminal +// frames of the jobs that just finished, and the failed frames of the dropped ones, are posted from +// goroutines the process would otherwise exit under: the card of a job that completed would be left +// open and closed failed by the next process's orphan sweep. Each post is bounded (2 s), so the wait is. +func fleetServeDrain(jobs *fleetnode.Jobs, pair *pairworkloads.Emitter, timeout time.Duration) { + jobs.DrainAndStop(timeout) + pair.Wait() +} + +// runFleetMeasure primes an empty footprint store: one minimal render per +// configured task through the NORMAL pipeline (so the passive gpugen hook +// records exactly what fleet jobs will), then prints the store's on-disk +// records as JSON. Tasks without a cheap universal probe (voice, run-graph) +// are skipped — their footprints accumulate passively during normal use. +func runFleetMeasure(args []string) error { + fs := flag.NewFlagSet("fleet-measure", flag.ExitOnError) + fs.String("config", "", "config file path") + _ = fs.Parse(args) + cfg := loadCfg(fs) + p, cleanup, err := openPipeline(cfg) + if err != nil { + return err + } + defer cleanup() + ctx := context.Background() + note := func(format string, a ...any) { fmt.Fprintf(os.Stderr, "[fleet-measure] "+format+"\n", a...) } + + // image-gen: a small fast render (512² at 8 steps). Engine-aware gate (J2): + // an sdcpp box has no imagegen_script but absolutely serves image-gen. + var stillPath string + if cfg.ImageRouteConfigured() { + note("image-gen: rendering 512x512 at 8 steps...") + res := p.Run(ctx, core.Request{ + Task: core.TaskGenerateImage, + Door: "cli:fleet-measure", + Input: "fleet-measure probe: a plain gray sphere on a white background", + Params: map[string]any{"width": 512, "height": 512, "steps": 8}, + }) + if res.OK { + var out struct { + ImagePath string `json:"image_path"` + } + _ = json.Unmarshal(res.Data, &out) + stillPath = out.ImagePath + note("image-gen: done (%s)", out.ImagePath) + } else { + note("image-gen: deferred: %s", res.Reason) + } + } else { + note("image-gen: skipped (no image route configured - neither imagegen_script nor the sdcpp engine)") + } + + // video-gen: the FAST (distilled) recipe at the smallest frame count — the + // slow native recipe is not a measurement tool. Reuses the probe image as + // the I2V still when the image step produced one. Engine-aware gate (CT-49), like the + // image step: a box whose only video is the sdcpp engine has no videogen_script but + // absolutely serves video-gen, and its iGPU footprints must be recorded too. + if cfg.VideoGenBound() { + note("video-gen: rendering the fast recipe at 9 frames...") + params := map[string]any{"fast": true, "frames": 9} + req := core.Request{Task: core.TaskGenerateVideo, Door: "cli:fleet-measure", Input: "fleet-measure probe: slow gentle camera pan", Params: params} + if stillPath != "" { + req.Image = stillPath + } + if res := p.Run(ctx, req); res.OK { + note("video-gen: done") + } else { + note("video-gen: deferred: %s", res.Reason) + } + } else { + note("video-gen: skipped (no video route configured - neither videogen_script nor an sdcpp video family)") + } + + // audio-gen: 5s of music (the ComfyUI ACE-Step path or the audio.cpp engine — the GPU-heavy one). + if cfg.MusicGenBound() { + note("audio-gen: rendering 5s of music...") + res := p.Run(ctx, core.Request{ + Task: core.TaskGenerateAudio, + Door: "cli:fleet-measure", + Input: "fleet-measure probe: soft ambient pad, slow tempo", + Params: map[string]any{"kind": "music", "seconds": 5}, + }) + if res.OK { + note("audio-gen: done") + } else { + note("audio-gen: deferred: %s", res.Reason) + } + } else { + note("audio-gen: skipped (no music route configured - neither musicgen_script nor musicgen_engine audiocpp)") + } + note("audio-gen (voice): skipped — voice footprints accumulate passively during normal TTS use") + note("run-graph: skipped — no universal probe graph; footprints accumulate passively per model_family") + + // Print the on-disk records (vram_peak_gb + observed_peak_gb/samples — the + // observed value is what the Afterburner validation procedure compares). + st := p.FootprintStore() + if st == nil { + fmt.Println("[]") + return nil + } + raw, rerr := os.ReadFile(st.Path()) + if rerr != nil { + fmt.Println("[]") + return nil + } + fmt.Println(strings.TrimSpace(string(raw))) + return nil +} + +// fleetServeConfigGate refuses to start a fleet NODE on a config that failed +// validation. +// +// Every other entry point degrades and warns, which is right for them: a +// one-shot CLI verb that half-works still answers, and an MCP server that exits +// takes every tool out of every session with no message anywhere. A fleet node +// is the exception. It ADVERTISES capability to other boxes and then accepts +// their dispatches, so a node whose endpoint dials a dead port does not fail +// alone — it turns every delegator's placement decision into a wasted wall, +// which is the cost this whole change exists to remove. A node that cannot +// prove its own config must not join the fleet. +// +// Only a validation FAILURE refuses. A missing file, or no file at all, is a +// legitimate fresh box; that path already warns loudly. +func fleetServeConfigGate(src config.Source) error { + if src.LoadErr == nil { + return nil + } + return fmt.Errorf("refusing to serve: config at %s %s: %w", src.Path, config.LoadFailure(src.LoadErr), src.LoadErr) +} + +// fleetAccelerators picks the accelerator list health advertises: the installer +// manifest's when it lists any, else the harness config's (Coral D6), minus the +// devices that stay on the box that carries them (register E-08: the standalone +// Hailo-8L is never advertised). The source is chosen first and the filter then +// applies to what it listed, so a manifest naming only a local-only device still +// wins and advertises nothing. Pure, so the test can pin both sources. +func fleetAccelerators(manifest, cfg []string) []string { + if len(manifest) > 0 { + return config.FleetVisibleAccelerators(manifest) + } + return config.FleetVisibleAccelerators(cfg) +} diff --git a/generate_audio_cmd.go b/generate_audio_cmd.go new file mode 100644 index 000000000..3344c491f --- /dev/null +++ b/generate_audio_cmd.go @@ -0,0 +1,226 @@ +// The generate-svg and generate-audio verbs, and the lane-targeting helpers +// the audio and video verbs share. + +package main + +import ( + "context" + "encoding/json" + "flag" + "fmt" + "math" + "os" + "strconv" + "strings" + + "github.com/dmmdea/offload-harness/internal/core" + "github.com/dmmdea/offload-harness/internal/mediaremote" +) + +func runGenerateSVG(args []string) error { + fs := flag.NewFlagSet("generate-svg", flag.ExitOnError) + fs.String("config", "", "config file path") + asJSON := fs.Bool("json", false, "print full result JSON") + spec := fs.String("spec", "", "component spec as a JSON string") + specFile := fs.String("spec-file", "", "path to a JSON file with the component spec") + out := fs.String("out", "", "output .svg path (default under the svg dir)") + compactFlag := fs.Bool("compact", false, "compact (minified) JSON output") + positional, flagArgs := splitArgs(args, map[string]bool{ + "config": true, "spec": true, "spec-file": true, "out": true, + }) + _ = fs.Parse(flagArgs) + if positional == "" || positional == "-" { + return fmt.Errorf("generate-svg requires a kind: local-offload generate-svg --spec ''") + } + specStr := *spec + if specStr == "" && *specFile != "" { + b, rerr := os.ReadFile(*specFile) + if rerr != nil { + return rerr + } + specStr = string(b) + } + if specStr == "" { + specStr = "{}" + } + var specObj map[string]any + if jerr := json.Unmarshal([]byte(specStr), &specObj); jerr != nil { + return fmt.Errorf("generate-svg: invalid --spec JSON: %w", jerr) + } + cfg := loadCfg(fs) + p, cleanup, err := openPipeline(cfg) + if err != nil { + return err + } + defer cleanup() + params := map[string]any{"kind": positional, "spec": specObj} + if *out != "" { + params["out"] = *out + } + res := p.Run(context.Background(), core.Request{Task: core.TaskGenerateSVG, Door: "cli:generate-svg", Params: params}) + emitResult(res, *asJSON, "", *compactFlag) + return nil +} + +// audioFlags is the parsed CLI input for `generate-audio`. Factored out so the +// param-building (buildAudioParams) is unit-testable without a live render. +type audioFlags struct { + node string + kind string + voice string + ttsVoice string // voice=endpoint: server-side voice name + clone string + lang string + out string + seconds int + seed int + reserveVRAM float64 +} + +// buildAudioParams maps the parsed CLI flags to the pipeline params map for a +// generate_audio request. Empty kind defaults to "voice" (matching the pipeline +// default); zero/empty optional flags are omitted so the pipeline applies its own +// defaults. reserve_vram is stringified to match the MCP tool's wire shape. +func buildAudioParams(f audioFlags) map[string]any { + kind := f.kind + if kind == "" { + kind = "voice" + } + params := map[string]any{"kind": kind} + laneTargetParams(params, f.node, nil) + if f.voice != "" { + params["voice"] = f.voice + } + if f.ttsVoice != "" { + params["tts_voice"] = f.ttsVoice + } + if f.clone != "" { + params["clone"] = f.clone + } + if f.lang != "" { + params["lang"] = f.lang + } + if f.out != "" { + params["out"] = f.out + } + if f.seconds > 0 { + params["seconds"] = f.seconds + } + if f.seed > 0 { + params["seed"] = f.seed + } + if f.reserveVRAM > 0 { + params["reserve_vram"] = strconv.FormatFloat(f.reserveVRAM, 'f', -1, 64) + } + return params +} + +// runGenerateAudio handles `local-offload generate-audio "" +// [--kind voice|music] [--clone ref.wav] [--lang es] [--seconds N] [--seed N] +// [--reserve-vram F] [--json]`. The TWO positionals are the output path and the +// text (narration for voice, style prompt for music) — mirroring the raw +// `node render/tts.mjs out.wav "text"` CLI. It synthesizes on the LOCAL GPU for +// free via the same runGenerateAudio pipeline branch the MCP tool uses. +func runGenerateAudio(args []string) error { + c, err := parseGenerateAudio(args, flag.ExitOnError) + if err != nil { + return err + } + + cfg := loadCfg(c.fs) + p, cleanup, err := openPipeline(cfg) + if err != nil { + return err + } + defer cleanup() + + params := buildAudioParams(c.audio) + res := mediaremote.Run(context.Background(), cfg, p, core.Request{ + Task: core.TaskGenerateAudio, + Door: "cli:generate-audio", + Input: c.text, + Params: params, + }, c.route, c.remotes) + emitResult(res, c.asJSON, "", c.compact) + return nil +} + +// generateAudioCLI is one parsed `generate-audio` command line. +type generateAudioCLI struct { + fs *flag.FlagSet + audio audioFlags + text string + asJSON bool + compact bool + route string + remotes []string +} + +// parseGenerateAudio parses the generate-audio command line (factored out of runGenerateAudio, like +// parseGenerateVideo, so the positionals can be tested without a render). +func parseGenerateAudio(args []string, errorHandling flag.ErrorHandling) (generateAudioCLI, error) { + fs := flag.NewFlagSet("generate-audio", errorHandling) + fs.String("config", "", "config file path") + asJSON := fs.Bool("json", false, "print full result JSON") + kind := fs.String("kind", "voice", "voice (Chatterbox TTS) | music (ACE-Step)") + voice := fs.String("voice", "", "voice: generalist | finetuned | endpoint (finetuned needs this machine's voicegen_ft_* config; endpoint renders through tts_endpoint, the configured OpenAI-compatible speech server — the default on a box with no voicegen_script)") + ttsVoice := fs.String("tts-voice", "", "voice=endpoint: the server-side voice/profile name (default: this box's tts_voice, else the server's default)") + clone := fs.String("clone", "", "voice: local path to a reference .wav for zero-shot voice cloning") + lang := fs.String("lang", "", "voice: language code (default es)") + seconds := fs.Int("seconds", 0, "music: clip length in seconds") + seed := fs.Int("seed", 0, "RNG seed for reproducibility") + reserveVRAM := fs.Float64("reserve-vram", 0, "music: VRAM held back for the display") + compactFlag := fs.Bool("compact", false, "compact (minified) JSON output") + route := fs.String("route", "auto", "local | auto (here when this machine has the lane, derived from the files its route loads, else a fleet node from delegate_remotes) | remote (a fleet node; input files travel in a hash-checked bundle and the output is fetched back verified; ADR 0077)") + remotes := new(repeatedFlag) + fs.Var(remotes, "remote", "fleet node base URL for this call (repeatable); must be one of delegate_remotes") + node := fs.String("node", "", nodeFlagUsage) + + // generate-audio takes TWO positionals (out path, text); the rest are flags. + out, text, flagArgs := splitTwoArgs(args, map[string]bool{ + "config": true, "kind": true, "voice": true, "clone": true, "lang": true, + "seconds": true, "seed": true, "reserve-vram": true, "tts-voice": true, + "route": true, "remote": true, "node": true, // generate-audio + }) + if err := fs.Parse(flagArgs); err != nil { + return generateAudioCLI{}, err + } + + if out == "" || text == "" { + return generateAudioCLI{}, fmt.Errorf("generate-audio requires an output path and text: local-offload generate-audio \"\" [--kind voice|music] [--lang es]") + } + return generateAudioCLI{ + fs: fs, + audio: audioFlags{ + node: *node, kind: *kind, voice: *voice, ttsVoice: *ttsVoice, clone: *clone, lang: *lang, out: out, + seconds: *seconds, seed: *seed, reserveVRAM: *reserveVRAM, + }, + text: text, asJSON: *asJSON, compact: *compactFlag, + route: *route, remotes: *remotes, + }, nil +} + +// laneTargetParams puts the --node and --min-vram-gb of a lane verb into the request's params, only when they were given (an +// explicit 0 is a number the caller gave). generate-audio has no VRAM floor and passes nil. +func laneTargetParams(params map[string]any, node string, minVRAM *float64) { + if n := strings.TrimSpace(node); n != "" { + params["node"] = n + } + if minVRAM != nil { + params["min_vram_gb"] = *minVRAM + } +} + +// parseMinVRAMFlag reads --min-vram-gb of a verb that takes it: nil when the flag was not given, else a number of GB, 0 or more +// (run-graph reads its own spelling of the same flag in runGraphParams). +func parseMinVRAMFlag(verb, s string) (*float64, error) { + s = strings.TrimSpace(s) + if s == "" { + return nil, nil + } + v, err := strconv.ParseFloat(s, 64) + if err != nil || v < 0 || math.IsNaN(v) || math.IsInf(v, 0) { + return nil, fmt.Errorf("%s: --min-vram-gb must be a number of GB, 0 or more; got %q", verb, s) + } + return &v, nil +} diff --git a/generate_image_cmd.go b/generate_image_cmd.go new file mode 100644 index 000000000..79e2ac1f8 --- /dev/null +++ b/generate_image_cmd.go @@ -0,0 +1,582 @@ +// The image and graph verbs: generate-image, inpaint, upscale, edit-image, +// run-graph and media, with the temp-file seams the inline graph uses. + +package main + +import ( + "context" + "encoding/json" + "errors" + "flag" + "fmt" + "io" + "math" + "os" + "os/signal" + "strconv" + "strings" + "time" + + "github.com/dmmdea/offload-harness/internal/core" + "github.com/dmmdea/offload-harness/internal/mediaremote" +) + +// runGenerateImage handles `local-offload generate-image "" [--negative ...] +// [--width N] [--height N] [--steps N] [--seed N] [--out path] [--json]`. The positional +// argument is the PROMPT (not stdin). It renders on the LOCAL ComfyUI for free. +func runGenerateImage(args []string) error { + fs := flag.NewFlagSet("generate-image", flag.ExitOnError) + fs.String("config", "", "config file path") + asJSON := fs.Bool("json", false, "print full result JSON") + negative := fs.String("negative", "", "negative prompt (hard exclusions, e.g. 'people, text')") + out := fs.String("out", "", "output PNG path (default under the media dir)") + width := fs.Int("width", 0, "image width px (default 1024)") + height := fs.Int("height", 0, "image height px (default 1024)") + steps := fs.Int("steps", 0, "sampler steps (default: machine binding)") + seed := fs.Int("seed", 0, "RNG seed (default random)") + refine := fs.Bool("refine", true, "set --refine=false (the =form is required) to render the prompt verbatim, skipping this machine's opt-in prompt refiner; on --batch it applies to every job without its own \"refine\" value (no-op unless imagegen_refiner_model is configured)") + compactFlag := fs.Bool("compact", false, "compact (minified) JSON output") + family := fs.String("family", "", "named image family to render with (ADR 0058; offload_status media.image_families / doctor list them); default = this machine's default binding") + transparent := fs.Bool("transparent", false, "keep an alpha channel (RGBA PNG; qwen-image-2.1 families only — any other binding defers)") + batchFile := fs.String("batch", "", "render a JSONL batch of jobs (one line per job: {\"prompt\":...,\"out\"?,\"negative\"?,\"width\"?,\"height\"?,\"steps\"?,\"seed\"?,\"refine\"?}): through ONE warm ComfyUI session when this machine's lane is free, else each job a ticket on the first matching lane that frees (ADR 0083)") + resultsFlag := fs.String("results", "", "with --batch: write one JSON row per job to this file as it ends ({i,out,seed,ok,ms,error,node,family,sha256,verdict}; verdict rendered | skipped-existing | failed | queued)") + resume := fs.Bool("resume", false, "with --batch: skip a job whose out already holds a PNG that decodes whole and has the size the job names; anything doubtful (empty, cut short, another size, missing) is rendered again") + jobsFlag := fs.Int("jobs", 0, "with --batch: the most jobs in flight on the ticket path (default: this machine's lane plus one per roster node that renders the recipe, at most 8)") + route := fs.String("route", "auto", "local | auto (here when this machine has the lane, derived from the files its route loads, else a fleet node from delegate_remotes) | remote (a fleet node; input files travel in a hash-checked bundle and the output is fetched back verified; ADR 0077)") + var remotes repeatedFlag + fs.Var(&remotes, "remote", "fleet node base URL for this call (repeatable); must be one of delegate_remotes") + positional, flagArgs := splitArgs(args, map[string]bool{ + "config": true, "negative": true, "out": true, + "width": true, "height": true, "steps": true, "seed": true, + "batch": true, "family": true, "route": true, "remote": true, "results": true, "jobs": true, + }) + _ = fs.Parse(flagArgs) + // Boolean flags in space form ("--refine false") make the value a stray + // argument: flag.Parse stops there, so later flags are SILENTLY dropped and + // the stray can even become the prompt. Refuse loudly instead. + if err := leftoverArgErr(fs, "generate-image"); err != nil { + return err + } + + if *batchFile != "" { + // A batch renders ONE binding for every job: --transparent (a per-render alpha channel) has no place on it. + if *transparent { + return fmt.Errorf("generate-image: --transparent selects an alpha-channel render of ONE image; a batch renders its jobs opaque (render the jobs one at a time)") + } + if *jobsFlag < 0 { + return fmt.Errorf("generate-image: --jobs %d: the number of jobs in flight cannot be negative (0 is the default)", *jobsFlag) + } + return runImageBatchVerb(fs, imageBatchArgs{ + file: *batchFile, family: *family, route: *route, results: *resultsFlag, resume: *resume, jobs: *jobsFlag, + refine: *refine, remotes: remotes, asJSON: *asJSON, compact: *compactFlag, + }) + } + for _, name := range []string{"results", "resume", "jobs"} { + set := false + fs.Visit(func(f *flag.Flag) { set = set || f.Name == name }) + if set { + return fmt.Errorf("generate-image: --%s belongs to --batch (jobs.jsonl), and no batch was given", name) + } + } + + if positional == "" || positional == "-" { + return fmt.Errorf("generate-image requires a prompt (not stdin): local-offload generate-image \"\" [--width 1024]") + } + + cfg, src := loadCfgWithSource(fs) + p, cleanup, err := openPipeline(cfg) + if err != nil { + return err + } + defer cleanup() + + params := map[string]any{} + if *negative != "" { + params["negative"] = *negative + } + if *out != "" { + params["out"] = *out + } + if *width > 0 { + params["width"] = *width + } + if *height > 0 { + params["height"] = *height + } + if *steps > 0 { + params["steps"] = *steps + } + if *seed > 0 { + params["seed"] = *seed + } + // Only an EXPLICIT --refine=false reaches the pipeline (the opt-in + // refiner's only request-level knob turns it off; the default is inert). + if !*refine { + params["refine"] = false + } + if *family != "" { + params["family"] = *family + } + if *transparent { + params["transparent"] = true + } + // This verb IS the wait: when every lane is busy it becomes a ticket (ADR 0083) and blocks until the first matching lane + // that frees has rendered it, telling the operator where it stands on stderr. The first Ctrl-C cancels a ticket that has + // sent nothing anywhere (a job already on a node cannot be recalled; run the same command to collect it); the second + // interrupt is the default one. + ictx, stopInterrupt := signal.NotifyContext(context.Background(), os.Interrupt) + defer stopInterrupt() + go func() { + <-ictx.Done() + stopInterrupt() + }() + wait := mediaremote.Wait{Window: -1, Notify: func(line string) { fmt.Fprintln(os.Stderr, line) }} + if src.Loaded() { + wait.RosterFile = src.Path + } + res := mediaremote.RunWith(ictx, cfg, p, core.Request{ + Task: core.TaskGenerateImage, + Door: "cli:generate-image", + Input: positional, + Params: params, + }, *route, remotes, wait) + emitResult(res, *asJSON, "", *compactFlag) + return nil +} + +// runInpaintImage handles `local-offload inpaint-image --mask +// --prompt "" [--negative ...] [--denoise F] [--grow-mask N] [--steps N] +// [--seed N] [--out path] [--json] [--compact]`. The positional argument is the +// IMAGE PATH; the mask is a white-on-black image the same size (white = repaint). +// It re-renders ONLY the masked region on the LOCAL ComfyUI for free (SDXL-class +// inpaint_* binding required). +func runInpaintImage(args []string) error { + fs := flag.NewFlagSet("inpaint-image", flag.ExitOnError) + fs.String("config", "", "config file path") + asJSON := fs.Bool("json", false, "print full result JSON") + mask := fs.String("mask", "", "white-on-black mask image path (white = repaint); required unless --auto-text") + autoText := fs.Bool("auto-text", false, "EXPERIMENTAL: replace --mask with a vision-detected mask of rendered-text regions (defers when detection is unreliable)") + prompt := fs.String("prompt", "", "what the masked region should become; required") + negative := fs.String("negative", "", "hard exclusions for the repainted region") + denoise := fs.Float64("denoise", 0, "0-1; default 1.0 (full re-imagination inside the mask)") + growMask := fs.Int("grow-mask", -1, "expand+feather the mask by N px in latent space (default 16; 0 = no dilation)") + steps := fs.Int("steps", 0, "sampler steps (default: machine binding)") + seed := fs.Int("seed", 0, "RNG seed (default random)") + out := fs.String("out", "", "output PNG path (default under the media dir)") + compactFlag := fs.Bool("compact", false, "compact (minified) JSON output") + positional, flagArgs := splitArgs(args, map[string]bool{ + "config": true, "mask": true, "prompt": true, "negative": true, + "denoise": true, "grow-mask": true, "steps": true, "seed": true, "out": true, + }) + _ = fs.Parse(flagArgs) + + if positional == "" || positional == "-" { + return fmt.Errorf("inpaint-image requires an image path (not stdin): local-offload inpaint-image --mask m.png --prompt \"...\"") + } + if (*mask == "" && !*autoText) || *prompt == "" { + return fmt.Errorf("inpaint-image requires --mask (or --auto-text) and --prompt: local-offload inpaint-image --mask m.png --prompt \"...\"") + } + + cfg := loadCfg(fs) + p, cleanup, err := openPipeline(cfg) + if err != nil { + return err + } + defer cleanup() + + params := map[string]any{"image": positional} + if *mask != "" { + params["mask"] = *mask + } + if *autoText { + params["auto_text"] = true + } + if *negative != "" { + params["negative"] = *negative + } + if *denoise > 0 { + params["denoise"] = *denoise + } + if *growMask >= 0 { + params["grow_mask"] = *growMask + } + if *steps > 0 { + params["steps"] = *steps + } + if *seed > 0 { + params["seed"] = *seed + } + if *out != "" { + params["out"] = *out + } + res := p.Run(context.Background(), core.Request{ + Task: core.TaskInpaintImage, + Door: "cli:inpaint-image", + Input: *prompt, + Params: params, + }) + emitResult(res, *asJSON, "", *compactFlag) + return nil +} + +// runUpscaleImage handles `local-offload upscale-image [--scale F] +// [--width N --height N] [--method m] [--model name] [--out path] [--json] +// [--compact]`. The positional argument is the IMAGE PATH. It enlarges the image +// with this machine's ESRGAN-family upscale_model on the LOCAL ComfyUI for free. +func runUpscaleImage(args []string) error { + fs := flag.NewFlagSet("upscale-image", flag.ExitOnError) + fs.String("config", "", "config file path") + asJSON := fs.Bool("json", false, "print full result JSON") + scale := fs.Float64("scale", 0, "overall factor relative to the source, made exact by measuring it (0 = unset: the model's own factor)") + width := fs.Int("width", 0, "exact output width (give with --height; wins over --scale)") + height := fs.Int("height", 0, "exact output height (give with --width)") + method := fs.String("method", "", "resampler for the scale/size step: lanczos (default) | bicubic | bilinear | area | nearest-exact") + model := fs.String("model", "", "override this machine's upscale_model (ComfyUI upscale_models filename)") + out := fs.String("out", "", "output PNG path (default under the media dir)") + compactFlag := fs.Bool("compact", false, "compact (minified) JSON output") + positional, flagArgs := splitArgs(args, map[string]bool{ + "config": true, "scale": true, "width": true, "height": true, "method": true, "model": true, "out": true, + }) + _ = fs.Parse(flagArgs) + + if positional == "" || positional == "-" { + return fmt.Errorf("upscale-image requires an image path (not stdin): local-offload upscale-image [--scale 2]") + } + + cfg := loadCfg(fs) + p, cleanup, err := openPipeline(cfg) + if err != nil { + return err + } + defer cleanup() + + params := map[string]any{"image": positional} + if *scale != 0 { + params["scale"] = *scale + } + if *width != 0 { + params["width"] = *width + } + if *height != 0 { + params["height"] = *height + } + if *method != "" { + params["method"] = *method + } + if *model != "" { + params["model"] = *model + } + if *out != "" { + params["out"] = *out + } + res := p.Run(context.Background(), core.Request{Task: core.TaskUpscaleImage, Door: "cli:upscale-image", Params: params}) + emitResult(res, *asJSON, "", *compactFlag) + return nil +} + +// runRunGraph handles `local-offload run-graph --graph [--graph-json ''] +// [--manifest ] [--manifest-json ''] [--out-dir ] [--reserve-vram F] +// [--json] [--compact]`. It executes an arbitrary ComfyUI API-format graph on the LOCAL +// ComfyUI, satisfying its per-workflow node manifest first — generic, the caller owns all +// graph semantics. Any failure defers (never cloud). +func runRunGraph(args []string) error { + fs := flag.NewFlagSet("run-graph", flag.ExitOnError) + fs.String("config", "", "config file path") + asJSON := fs.Bool("json", false, "print full result JSON") + compactFlag := fs.Bool("compact", false, "compact (minified) JSON output") + // graph/manifest/out-dir/reserve-vram are owned by runGraphParams (the testable arg + // seam); register them here too so this parser accepts them without erroring. + fs.String("graph", "", "path to a ComfyUI API-format graph JSON (required unless --graph-json)") + fs.String("graph-json", "", "inline ComfyUI API-format graph JSON (alternative to --graph)") + fs.String("manifest", "", "path to a node manifest JSON") + fs.String("manifest-json", "", "inline node manifest JSON (alternative to --manifest)") + fs.String("out-dir", "", "directory for output files (default under the media dir)") + fs.String("reserve-vram", "", "ComfyUI --reserve-vram override") + fs.String("min-vram-gb", "", "only a fleet node whose largest card is at least N GB may take the graph (0 lifts the default floor)") + fs.String("model-family", "", "the family this graph belongs to (wan22 for Wan 2.2); only the default --min-vram-gb floor reads it") + route := fs.String("route", "auto", "local | auto (here when this machine has the lane, derived from the files its route loads, else a fleet node from delegate_remotes) | remote (a fleet node; input files travel in a hash-checked bundle and the output is fetched back verified; ADR 0077)") + var remotes repeatedFlag + fs.Var(&remotes, "remote", "fleet node base URL for this call (repeatable); must be one of delegate_remotes") + _ = fs.Parse(args) + + params, cleanupInline, err := runGraphParams(args) + if err != nil { + return err + } + // The temp files an inline --graph-json or --manifest-json became are this call's own, and go when it ends, whichever way it ends: + // the pipeline that cannot open (below), an answer, a deferral. Registered before anything else can return. A graph ticket this + // call made is not left holding them: it keeps its own copy of the bytes (internal/mediaremote/ticket.go writeBodies). + defer cleanupInline() + + cfg, src := loadCfgWithSource(fs) + p, cleanup, err := openPipeline(cfg) + if err != nil { + return err + } + defer cleanup() + + // This verb IS the wait, as generate-image's is (ADR 0083 Amendment 1): when this machine's lane is busy the graph becomes a + // ticket and the verb blocks until the first matching lane that frees has run it, telling the operator where it stands on + // stderr. The first Ctrl-C cancels a ticket that has sent nothing anywhere (a graph already on a node cannot be recalled; run + // the same command to collect it); the second interrupt is the default one. + ictx, stopInterrupt := signal.NotifyContext(context.Background(), os.Interrupt) + defer stopInterrupt() + go func() { + <-ictx.Done() + stopInterrupt() + }() + wait := mediaremote.Wait{Window: -1, Notify: func(line string) { fmt.Fprintln(os.Stderr, line) }} + if src.Loaded() { + wait.RosterFile = src.Path + } + res := mediaremote.RunWith(ictx, cfg, p, core.Request{Task: core.TaskRunGraph, Door: "cli:run-graph", Params: params}, *route, remotes, wait) + emitResult(res, *asJSON, "", *compactFlag) + return nil +} + +// runGraphParams parses the run-graph flags into the pipeline request params map. It is +// the testable validation seam: --graph OR --graph-json is required (else an error), and +// inline JSON is materialized to a temp file whose path is threaded on (the runner reads +// files). Kept independent of runRunGraph's pipeline wiring so the arg contract is unit- +// testable without a ComfyUI/pipeline. +// +// The returned cleanup is never nil and removes the temp files THIS call made for inline JSON (a file +// the caller named with --graph or --manifest is never touched); on an error there is nothing to clean, +// because a half-made pair is removed before the error returns. +func runGraphParams(args []string) (map[string]any, func(), error) { + noop := func() {} + fs := flag.NewFlagSet("run-graph-params", flag.ContinueOnError) + fs.SetOutput(io.Discard) + fs.String("config", "", "config file path") + fs.Bool("json", false, "") + fs.Bool("compact", false, "") + graph := fs.String("graph", "", "") + graphJSON := fs.String("graph-json", "", "") + manifest := fs.String("manifest", "", "") + manifestJSON := fs.String("manifest-json", "", "") + outDir := fs.String("out-dir", "", "") + reserve := fs.String("reserve-vram", "", "") + minVRAM := fs.String("min-vram-gb", "", "") + family := fs.String("model-family", "", "") + fs.String("route", "", "") + fs.Var(new(repeatedFlag), "remote", "") + if err := fs.Parse(args); err != nil { + return nil, noop, err + } + if *graph == "" && *graphJSON == "" { + return nil, noop, fmt.Errorf("run-graph: --graph or --graph-json required") + } + var floor *float64 + if s := strings.TrimSpace(*minVRAM); s != "" { + v, err := strconv.ParseFloat(s, 64) + if err != nil || v < 0 || math.IsNaN(v) || math.IsInf(v, 0) { + return nil, noop, fmt.Errorf("run-graph: --min-vram-gb must be a number of GB, 0 or more; got %q", s) + } + floor = &v + } + graphPath, cleanGraph, err := materializeArg(*graph, *graphJSON, "run-graph-*.json") + if err != nil { + return nil, noop, fmt.Errorf("graph: %w", err) + } + manifestPath, cleanManifest, err := materializeArg(*manifest, *manifestJSON, "run-graph-manifest-*.json") + if err != nil { + cleanGraph() // the graph's file was made for a call that now ends + return nil, noop, fmt.Errorf("manifest: %w", err) + } + cleanup := func() { + cleanGraph() + cleanManifest() + } + params := map[string]any{ + "graph_path": graphPath, + "manifest_path": manifestPath, + "out_dir": *outDir, + "reserve_vram": *reserve, + } + if floor != nil { + params["min_vram_gb"] = *floor + } + if f := strings.TrimSpace(*family); f != "" { + params["model_family"] = f + } + return params, cleanup, nil +} + +// createTemp makes the temp file an inline graph or manifest is written to; a seam, so that a test can make the write fail. +var createTemp = os.CreateTemp + +// removeFile, removeAttempts and removePause are how the temp files an inline graph or manifest became are deleted, and tempWarnings +// is where one that cannot be is named; seams, so that a test can make a delete fail the way a file held for a moment fails on Windows. +var ( + removeFile = os.Remove + removeAttempts = 20 + removePause = 25 * time.Millisecond + tempWarnings io.Writer = os.Stderr +) + +// removeInlineTemp deletes a temp file this call made. A scanner that holds the file for a moment, or a runner that has only just +// exited, makes the delete fail with a sharing violation on Windows, so it is tried again for about half a second, as +// pipeline.removeRunGraphEnvelope does for the envelope; a file that is not there is not a failure. One that still cannot be removed is +// named on stderr, so that the leak this cleanup exists to stop is never silent. +func removeInlineTemp(name string) { + var err error + for i := 0; i < removeAttempts; i++ { + if i > 0 { + time.Sleep(removePause) + } + if err = removeFile(name); err == nil || errors.Is(err, os.ErrNotExist) { + return + } + } + fmt.Fprintf(tempWarnings, "local-offload: run-graph: could not remove the temp file %s (%v); it stays in the OS temp dir until someone deletes it\n", name, err) +} + +// materializeArg returns path if set, else writes inline json to a temp file and returns +// that path (the runner reads files, not inline). Empty+empty → no path and no error. +// +// The cleanup it returns is never nil and removes only a file THIS call made: it does nothing for a path +// the caller named, nor for an empty one. A write that fails removes its own file before the error returns. +func materializeArg(path, inline, pattern string) (string, func(), error) { + noop := func() {} + if path != "" { + return path, noop, nil + } + if inline == "" { + return "", noop, nil + } + f, err := createTemp("", pattern) + if err != nil { + return "", noop, err + } + name := f.Name() + _, werr := f.WriteString(inline) + cerr := f.Close() // closed before any removal: Windows will not delete a file that is still open + if werr == nil { + werr = cerr + } + if werr != nil { + removeInlineTemp(name) + return "", noop, werr + } + return name, func() { removeInlineTemp(name) }, nil +} + +// runGenerateSVG handles `local-offload generate-svg --spec '' [--spec-file path] [--out file.svg] [--json]`. +// kind is the positional (gauge|comparison-bar|chromatogram|icon). It renders locally for free (no model/GPU). +// runEditImage handles `local-offload edit-image --ops '' +// [--out path] [--json]`. Deterministic CPU pipeline (PIL + GIMP flatten_design) +// via internal/mediaops — no GPU lock. +func runEditImage(args []string) error { + fs := flag.NewFlagSet("edit-image", flag.ExitOnError) + fs.String("config", "", "config file path") + asJSON := fs.Bool("json", false, "print full result JSON") + opsStr := fs.String("ops", "", `edit ops as a JSON array, e.g. '[{"op":"crop","x":0,"y":0,"width":100,"height":100},{"op":"resize","width":512}]'`) + opsFile := fs.String("ops-file", "", "path to a JSON file with the ops array") + out := fs.String("out", "", "output image path (default under the media dir)") + renditions := fs.String("renditions", "", `export matrix from the master out, e.g. '[{"width":1080,"format":"webp","suffix":"-ig"},{"width":1920,"format":"jpg","suffix":"-web"}]'`) + compactFlag := fs.Bool("compact", false, "compact (minified) JSON output") + positional, flagArgs := splitArgs(args, map[string]bool{ + "config": true, "ops": true, "ops-file": true, "out": true, "renditions": true, + }) + _ = fs.Parse(flagArgs) + if positional == "" || positional == "-" { + return fmt.Errorf("edit-image requires an input image: local-offload edit-image --ops ''") + } + raw := *opsStr + if raw == "" && *opsFile != "" { + b, rerr := os.ReadFile(*opsFile) + if rerr != nil { + return rerr + } + raw = string(b) + } + if raw == "" { + return fmt.Errorf("edit-image: --ops (or --ops-file) is required") + } + var ops []map[string]any + if jerr := json.Unmarshal([]byte(raw), &ops); jerr != nil { + return fmt.Errorf("edit-image: invalid --ops JSON: %w", jerr) + } + cfg := loadCfg(fs) + p, cleanup, err := openPipeline(cfg) + if err != nil { + return err + } + defer cleanup() + params := map[string]any{"ops": ops} + if *out != "" { + params["out"] = *out + } + if *renditions != "" { + var rends []map[string]any + if jerr := json.Unmarshal([]byte(*renditions), &rends); jerr != nil { + return fmt.Errorf("edit-image: invalid --renditions JSON: %w", jerr) + } + params["renditions"] = rends + } + res := p.Run(context.Background(), core.Request{Task: core.TaskEditImage, Door: "cli:edit-image", Image: positional, Params: params}) + emitResult(res, *asJSON, "", *compactFlag) + return nil +} + +// runMedia handles `local-offload media [--in x] [--inputs a,b] [--out y] +// [op flags] [--json]`. One ffmpeg operation via internal/mediaops — no GPU lock. +func runMedia(args []string) error { + fs := flag.NewFlagSet("media", flag.ExitOnError) + fs.String("config", "", "config file path") + asJSON := fs.Bool("json", false, "print full result JSON") + in := fs.String("in", "", "input media path") + inputs := fs.String("inputs", "", "concat: comma-separated input paths (>=2)") + out := fs.String("out", "", "output path (extract_frames: a directory; default under the media dir)") + start := fs.String("start", "", "trim: start (seconds or hh:mm:ss)") + end := fs.String("end", "", "trim: absolute end time") + duration := fs.String("duration", "", "trim: duration seconds (alternative to --end)") + reencode := fs.Bool("reencode", false, "trim: re-encode for exact cuts (default: keyframe-snapped stream copy)") + fps := fs.Float64("fps", 0, "extract_frames: sampling rate") + count := fs.Int("count", 0, "extract_frames: total frames (resolved via probe)") + audio := fs.String("audio", "", "mux_audio: audio input path") + audioOnly := fs.Bool("audio-only", false, "convert: drop video") + videoOnly := fs.Bool("video-only", false, "convert: drop audio") + compactFlag := fs.Bool("compact", false, "compact (minified) JSON output") + positional, flagArgs := splitArgs(args, map[string]bool{ + "config": true, "in": true, "inputs": true, "out": true, "start": true, + "end": true, "duration": true, "fps": true, "count": true, "audio": true, + }) + _ = fs.Parse(flagArgs) + if positional == "" || positional == "-" { + return fmt.Errorf("media requires an op: local-offload media --in ") + } + cfg := loadCfg(fs) + p, cleanup, err := openPipeline(cfg) + if err != nil { + return err + } + defer cleanup() + params := map[string]any{"op": positional} + for k, v := range map[string]string{"in": *in, "out": *out, "start": *start, "end": *end, "duration": *duration, "audio": *audio} { + if v != "" { + params[k] = v + } + } + if *inputs != "" { + params["inputs"] = strings.Split(*inputs, ",") + } + if *reencode { + params["reencode"] = true + } + if *audioOnly { + params["audio_only"] = true + } + if *videoOnly { + params["video_only"] = true + } + if *fps > 0 { + params["fps"] = *fps + } + if *count > 0 { + params["count"] = *count + } + res := p.Run(context.Background(), core.Request{Task: core.TaskMedia, Door: "cli:media", Params: params}) + emitResult(res, *asJSON, "", *compactFlag) + return nil +} diff --git a/generate_video_cmd.go b/generate_video_cmd.go new file mode 100644 index 000000000..5a820b95e --- /dev/null +++ b/generate_video_cmd.go @@ -0,0 +1,452 @@ +// The video verbs: compose-video, generate-video and animate-character, with +// their flag parsers and parameter builders. + +package main + +import ( + "context" + "encoding/json" + "flag" + "fmt" + "os" + "strconv" + "strings" + + "github.com/dmmdea/offload-harness/internal/composeremote" + "github.com/dmmdea/offload-harness/internal/core" + "github.com/dmmdea/offload-harness/internal/mediaremote" +) + +// composeFlags is the parsed CLI input for `compose-video`. Factored out so the +// param building (buildComposeParams) is unit-testable without a render. +type composeFlags struct { + template, variables, variablesFile, htmlFile, projectDir, composition string + out, format, quality, resolution, snapshots string + fps, workers int + strict bool +} + +// buildComposeParams maps `compose-video` flags onto the params offload_compose_video +// sends, so the CLI and the MCP door hand the pipeline the same shapes. --html reads +// the file: the pipeline's html param is the page itself, never a path. +func buildComposeParams(f composeFlags) (map[string]any, error) { + params := map[string]any{} + for k, v := range map[string]string{"template": f.template, "project_dir": f.projectDir, "composition": f.composition, + "out": f.out, "format": f.format, "quality": f.quality, "resolution": f.resolution} { + if v != "" { + params[k] = v + } + } + if f.htmlFile != "" { + b, err := os.ReadFile(f.htmlFile) + if err != nil { + return nil, fmt.Errorf("compose-video: --html: %w", err) + } + params["html"] = string(b) + } + vars := f.variables + if vars != "" && f.variablesFile != "" { + return nil, fmt.Errorf("compose-video: use --variables or --variables-file, not both") + } + if f.variablesFile != "" { + b, err := os.ReadFile(f.variablesFile) + if err != nil { + return nil, fmt.Errorf("compose-video: --variables-file: %w", err) + } + vars = string(b) + } + if vars != "" { + var obj map[string]any + if err := json.Unmarshal([]byte(vars), &obj); err != nil || obj == nil { + return nil, fmt.Errorf("compose-video: variables must be a JSON object") + } + params["variables"] = obj + } + if f.fps > 0 { + params["fps"] = f.fps + } + if f.workers > 0 { + params["workers"] = f.workers + } + if !f.strict { + params["strict"] = false + } + if f.snapshots != "" { + var ts []float64 + for _, s := range strings.Split(f.snapshots, ",") { + v, err := strconv.ParseFloat(strings.TrimSpace(s), 64) + if err != nil { + return nil, fmt.Errorf("compose-video: --snapshots %q: %w", f.snapshots, err) + } + ts = append(ts, v) + } + params["snapshots"] = ts + } + return params, nil +} + +// runComposeVideo is `local-offload compose-video` — the CLI door of the composition +// lane (ADR 0059), the same pipeline path offload_compose_video takes. +func runComposeVideo(args []string) error { + fs := flag.NewFlagSet("compose-video", flag.ExitOnError) + fs.String("config", "", "config file path") + asJSON := fs.Bool("json", false, "print full result JSON") + compactFlag := fs.Bool("compact", false, "compact (minified) JSON output") + var f composeFlags + fs.StringVar(&f.template, "template", "", "a vetted template on this machine (title-card, lower-third, stat-card, section-title, callout-label, checklist-card, captions-bar)") + fs.StringVar(&f.variables, "variables", "", "template variables as a JSON object") + fs.StringVar(&f.variablesFile, "variables-file", "", "path to a JSON file with the template variables") + fs.StringVar(&f.htmlFile, "html", "", "path to a single-file composition (trusted code only)") + fs.StringVar(&f.projectDir, "project-dir", "", "a local composition directory with index.html (trusted code only)") + fs.StringVar(&f.composition, "composition", "", "a composition file relative to --project-dir") + fs.StringVar(&f.out, "out", "", "output path (a directory for png-sequence; default under the media dir)") + fs.StringVar(&f.format, "format", "", "mp4 (default) | webm | mov | png-sequence | gif") + fs.StringVar(&f.quality, "quality", "", "draft | standard | high (default: compose_quality)") + fs.StringVar(&f.resolution, "resolution", "", "landscape | portrait | landscape-4k | portrait-4k | square | square-4k") + fs.StringVar(&f.snapshots, "snapshots", "", "comma-separated seconds to also save as PNG frames") + fs.IntVar(&f.fps, "fps", 0, "frame rate 1-240 (default: the composition's data-fps)") + fs.IntVar(&f.workers, "workers", 0, "Chrome workers 1-24 (default: compose_workers, else auto)") + fs.BoolVar(&f.strict, "strict", true, "fail on lint errors / a failed check (--strict=false to report only)") + route := fs.String("route", "auto", "local | auto (here when this machine has the composition lane, else a fleet node) | remote (a fleet node from delegate_remotes; ADR 0071)") + _ = fs.Parse(args) + params, perr := buildComposeParams(f) + if perr != nil { + return perr + } + cfg := loadCfg(fs) + p, cleanup, err := openPipeline(cfg) + if err != nil { + return err + } + defer cleanup() + res := composeremote.Run(context.Background(), cfg, p, core.Request{Task: core.TaskComposeVideo, Door: "cli:compose-video", Params: params}, *route) + emitResult(res, *asJSON, "", *compactFlag) + return nil +} + +// videoFlags is the parsed CLI input for `generate-video`. Factored out so the +// param-building (buildVideoParams) is unit-testable without a live render. +type videoFlags struct { + node string + minVRAM *float64 + model string + still string + negative string + out string + transformer string + frames int + width int + height int + steps int + seed int + reserveVRAM float64 + fast bool + hero bool + upscale bool +} + +// buildVideoParams maps the parsed CLI flags to the pipeline params map for a +// generate_video request. An EMPTY model is deliberately omitted so the pipeline +// applies the machine's videogen_family (the seat binding — e.g. ltx25); the old +// forced-"wan" default silently defeated the family on every CLI call, which is +// exactly how the Leg-3 verification rendered a broken Wan/LTX hybrid graph +// (live finding 2026-08-15). A machine with no family still gets wan — the +// render runner's own default. The still path is always carried as "still". +// Zero/empty optional flags are omitted so the pipeline/runner apply their own +// defaults (the workflow builder's settings + the machine's videogen_* config + the +// runner's --reserve-vram). reserve_vram is stringified to match the MCP wire shape. +func buildVideoParams(f videoFlags) map[string]any { + params := map[string]any{} + laneTargetParams(params, f.node, f.minVRAM) + if f.model != "" { + params["model"] = f.model + } + if f.still != "" { + params["still"] = f.still + } + if f.negative != "" { + params["negative"] = f.negative + } + if f.out != "" { + params["out"] = f.out + } + if f.transformer != "" { + params["transformer"] = f.transformer + } + if f.frames > 0 { + params["frames"] = f.frames + } + if f.width > 0 { + params["width"] = f.width + } + if f.height > 0 { + params["height"] = f.height + } + if f.steps > 0 { + params["steps"] = f.steps + } + if f.seed > 0 { + params["seed"] = f.seed + } + if f.reserveVRAM > 0 { + params["reserve_vram"] = strconv.FormatFloat(f.reserveVRAM, 'f', -1, 64) + } + if f.fast { + params["fast"] = true + } + if f.hero { + params["hero"] = true + } + if f.upscale { + params["upscale"] = true + } + return params +} + +// generateVideoCLI is one parsed `generate-video` command line. +type generateVideoCLI struct { + fs *flag.FlagSet + video videoFlags + prompt string + asJSON bool + compact bool + route string + remotes []string +} + +const ( + nodeFlagUsage = "name the ONE fleet node that runs this call: a node_id its /fleet/health publishes (any case) or one of the delegate_remotes URLs. No other node and not this machine is tried; a node that cannot take the call is refused by a word before anything is sent (with --json, data.refusal holds the word); --route local with --node is refused" + minVRAMFlagUsage = "only a fleet node whose LARGEST card is at least this many GB may take the call (a number of GB, 0 or more; a 16 GB card reads 15.92 GiB); filters nodes only" +) + +// generateVideoValueFlags are the flags that consume the next token. Bool flags +// (--fast, --hero, --upscale, --json, --compact) must NOT be listed: splitThreeArgs +// would take the following positional as their value. +var generateVideoValueFlags = map[string]bool{ + "config": true, "model": true, "negative": true, "frames": true, + "width": true, "height": true, "steps": true, "seed": true, "reserve-vram": true, + "transformer": true, "route": true, "remote": true, "node": true, "min-vram-gb": true, +} + +// parseGenerateVideo parses the generate-video command line. The CLI carries every +// option the offload_generate_video MCP tool takes (prompt, still, out, model, +// negative, frames, width, height, steps, seed, reserve_vram, fast, hero, upscale): +// --fast was the missing one, which left the distilled Wan recipe reachable only +// through MCP ( parity audit, 2026-09-23). +func parseGenerateVideo(args []string, errorHandling flag.ErrorHandling) (generateVideoCLI, error) { + fs := flag.NewFlagSet("generate-video", errorHandling) + fs.String("config", "", "config file path") + asJSON := fs.Bool("json", false, "print full result JSON") + model := fs.String("model", "", "family override: wan | ltx25 | h3 | hunyuan (h3: still optional — t2v without). Empty = the machine's videogen_family seat (falls back to wan when no family is configured)") + negative := fs.String("negative", "", "hard exclusions, e.g. 'blurry, distorted'") + transformer := fs.String("transformer", "", "OPTIONAL per-request LTX-2.5 transformer file override (e.g. the bf16 transformer for one hero render — int8 stays this box's default; gap 5, bigger-models-2026-09-24.md). Wins over this box's (or the resolved family's) bound file; a no-op on any other family's graph") + frames := fs.Int("frames", 0, "frame count (16fps; 81 ≈ 5s is the native ceiling)") + width := fs.Int("width", 0, "width px") + height := fs.Int("height", 0, "height px") + steps := fs.Int("steps", 0, "sampler steps (0 = builder default: wan 20 native / 8 fast)") + seed := fs.Int("seed", 0, "RNG seed for reproducibility") + reserveVRAM := fs.Float64("reserve-vram", 0, "VRAM held back for the display (default per-workflow; ~2.0 for Hunyuan/Wan)") + fast := fs.Bool("fast", false, "OPT-IN draft mode: 8-step lightx2v distill (wan; visibly weaker motion). The default is the native quality recipe") + hero := fs.Bool("hero", false, "deprecated: the native quality pass IS the default now; accepted as a no-op") + upscale := fs.Bool("upscale", false, "post-decode upscale using this machine's configured upscale model (e.g. 720p->1080p)") + compactFlag := fs.Bool("compact", false, "compact (minified) JSON output") + route := fs.String("route", "auto", "local | auto (here when this machine has the lane, derived from the files its route loads, else a fleet node from delegate_remotes) | remote (a fleet node; input files travel in a hash-checked bundle and the output is fetched back verified; ADR 0077)") + remotes := new(repeatedFlag) + fs.Var(remotes, "remote", "fleet node base URL for this call (repeatable); must be one of delegate_remotes") + node := fs.String("node", "", nodeFlagUsage) + minVRAM := fs.String("min-vram-gb", "", minVRAMFlagUsage) + + // generate-video takes THREE positionals (out path, still image, prompt); the + // rest are flags. + out, still, prompt, flagArgs := splitThreeArgs(args, generateVideoValueFlags) + if err := fs.Parse(flagArgs); err != nil { + return generateVideoCLI{}, err + } + floor, err := parseMinVRAMFlag("generate-video", *minVRAM) + if err != nil { + return generateVideoCLI{}, err + } + if out == "" || prompt == "" { + return generateVideoCLI{}, fmt.Errorf("generate-video requires an output path, a still image, and a prompt: local-offload generate-video \"\" [--model hunyuan|wan] [--frames 49] [--fast]") + } + return generateVideoCLI{ + fs: fs, + video: videoFlags{ + model: *model, node: *node, minVRAM: floor, still: still, negative: *negative, out: out, transformer: *transformer, + frames: *frames, width: *width, height: *height, steps: *steps, + seed: *seed, reserveVRAM: *reserveVRAM, fast: *fast, hero: *hero, upscale: *upscale, + }, + prompt: prompt, asJSON: *asJSON, compact: *compactFlag, + route: *route, remotes: *remotes, + }, nil +} + +// runGenerateVideo handles `local-offload generate-video +// "" [--model hunyuan|wan] [--frames 49] [--width N] [--height N] +// [--steps N] [--seed N] [--negative "..."] [--reserve-vram F] [--fast] [--upscale] +// [--json]`. The THREE positionals are the output path, the input still (I2V needs +// an image), and the prompt — mirroring the raw `node render/comfy-video.mjs out.mp4 +// still.png "prompt"` CLI. It animates the still into a short clip on the LOCAL +// ComfyUI for free via the same runGenerateVideo pipeline branch the MCP tool uses. +// Shift and VAE temporal tiling are SETTLED inside the workflow builder and are +// intentionally NOT exposed here so a caller can't regress them. +func runGenerateVideo(args []string) error { + c, err := parseGenerateVideo(args, flag.ExitOnError) + if err != nil { + return err + } + + cfg := loadCfg(c.fs) + p, cleanup, err := openPipeline(cfg) + if err != nil { + return err + } + defer cleanup() + + res := mediaremote.Run(context.Background(), cfg, p, core.Request{ + Task: core.TaskGenerateVideo, + Door: "cli:generate-video", + Input: c.prompt, + Image: c.video.still, + Params: buildVideoParams(c.video), + }, c.route, c.remotes) + emitResult(res, c.asJSON, "", c.compact) + return nil +} + +// runAnimateCharacter handles `local-offload animate-character +// "" [--motion-prompt "..."] [--negative "..."] [--width N] +// [--height N] [--frames 81] [--steps N] [--seed N] [--pose-strength F] +// [--ref-strength F] [--reserve-vram F] [--route local|auto|remote] [--remote URL]... [--json]`. The FOUR positionals are the +// output path, the reference character image, the driver video whose motion is +// transferred, and the character/background prompt — mirroring the raw +// `node render/comfy-animate.mjs` CLI. Runs the same runAnimateCharacter pipeline +// branch the MCP tool uses. The pose-cache device is SETTLED inside the workflow +// builder (cpu — the template's gpu/int8 cache hard-kills ComfyUI) and is +// intentionally NOT exposed here so a caller can't regress it. +func runAnimateCharacter(args []string) error { + c, err := parseAnimateCharacter(args, flag.ExitOnError) + if err != nil { + return err + } + + cfg := loadCfg(c.fs) + p, cleanup, err := openPipeline(cfg) + if err != nil { + return err + } + defer cleanup() + + res := mediaremote.Run(context.Background(), cfg, p, core.Request{ + Task: core.TaskAnimateCharacter, + Door: "cli:animate-character", + Input: c.animate.prompt, + Image: c.animate.ref, + Video: c.animate.driver, + Params: buildAnimateParams(c.animate), + }, c.route, c.remotes) + emitResult(res, c.asJSON, "", c.compact) + return nil +} + +// animateFlags is the parsed CLI input for `animate-character`, factored out like videoFlags so the command line and the +// param-building are testable without a render. +type animateFlags struct { + node string + minVRAM *float64 + out, ref, driver, prompt string + motionPrompt, negative string + width, height, frames int + steps, seed int + poseStrength float64 + refStrength float64 + reserveVRAM float64 +} + +// animateCharacterCLI is one parsed `animate-character` command line. +type animateCharacterCLI struct { + fs *flag.FlagSet + animate animateFlags + asJSON bool + compact bool + route string + remotes []string +} + +// parseAnimateCharacter parses the animate-character command line (factored out of runAnimateCharacter, like +// parseGenerateVideo, so the flags can be tested without exiting the process or rendering). +func parseAnimateCharacter(args []string, errorHandling flag.ErrorHandling) (animateCharacterCLI, error) { + fs := flag.NewFlagSet("animate-character", errorHandling) + fs.String("config", "", "config file path") + asJSON := fs.Bool("json", false, "print full result JSON") + motionPrompt := fs.String("motion-prompt", "", "one-line description of the driver video's motion") + negative := fs.String("negative", "", "hard exclusions") + width := fs.Int("width", 0, "working width px (0 = the 482x854 template default)") + height := fs.Int("height", 0, "working height px") + frames := fs.Int("frames", 0, "frame count (default 81 — one native chunk; on an animategen_engine sdcpp box the default is its animategen_frames, else 49)") + steps := fs.Int("steps", 0, "sampler steps (0 = the distilled recipe's 10)") + seed := fs.Int("seed", 0, "RNG seed for reproducibility") + poseStrength := fs.Float64("pose-strength", 0, "0-1: how strongly the driver's pose drives the output (0 = builder default 1.0)") + refStrength := fs.Float64("ref-strength", 0, "0-1: how strongly the reference image pins identity (0 = builder default 1.0)") + reserveVRAM := fs.Float64("reserve-vram", 0, "VRAM held back for the display (per-workflow override)") + compactFlag := fs.Bool("compact", false, "compact (minified) JSON output") + route := fs.String("route", "auto", "local | auto (here when this machine has the lane, derived from the files its route loads, else a fleet node from delegate_remotes) | remote (a fleet node; the reference image and driver video travel in a hash-checked bundle and the clip is fetched back verified; ADR 0077)") + remotes := new(repeatedFlag) + fs.Var(remotes, "remote", "fleet node base URL for this call (repeatable); must be one of delegate_remotes") + node := fs.String("node", "", nodeFlagUsage) + minVRAM := fs.String("min-vram-gb", "", minVRAMFlagUsage) + + out, ref, driver, prompt, flagArgs := splitFourArgs(args, map[string]bool{ + "config": true, "motion-prompt": true, "negative": true, "width": true, + "height": true, "frames": true, "steps": true, "seed": true, + "pose-strength": true, "ref-strength": true, "reserve-vram": true, + "route": true, "remote": true, "node": true, "min-vram-gb": true, // animate-character + }) + if err := fs.Parse(flagArgs); err != nil { + return animateCharacterCLI{}, err + } + floor, err := parseMinVRAMFlag("animate-character", *minVRAM) + if err != nil { + return animateCharacterCLI{}, err + } + + if out == "" || ref == "" || driver == "" || prompt == "" { + return animateCharacterCLI{}, fmt.Errorf("animate-character requires an output path, a reference image, a driver video, and a prompt: local-offload animate-character \"\" [--frames 81]") + } + return animateCharacterCLI{ + fs: fs, + animate: animateFlags{ + out: out, node: *node, minVRAM: floor, ref: ref, driver: driver, prompt: prompt, motionPrompt: *motionPrompt, negative: *negative, + width: *width, height: *height, frames: *frames, steps: *steps, seed: *seed, + poseStrength: *poseStrength, refStrength: *refStrength, reserveVRAM: *reserveVRAM, + }, + asJSON: *asJSON, compact: *compactFlag, route: *route, remotes: *remotes, + }, nil +} + +// buildAnimateParams maps the parsed CLI flags to the pipeline params map for an animate_character request. +func buildAnimateParams(f animateFlags) map[string]any { + params := map[string]any{"ref": f.ref, "driver": f.driver, "out": f.out} + laneTargetParams(params, f.node, f.minVRAM) + if f.motionPrompt != "" { + params["motion_prompt"] = f.motionPrompt + } + if f.negative != "" { + params["negative"] = f.negative + } + for k, v := range map[string]int{"width": f.width, "height": f.height, "frames": f.frames, "steps": f.steps, "seed": f.seed} { + if v > 0 { + params[k] = v + } + } + if f.poseStrength > 0 { + params["pose_strength"] = strconv.FormatFloat(f.poseStrength, 'f', -1, 64) + } + if f.refStrength > 0 { + params["ref_strength"] = strconv.FormatFloat(f.refStrength, 'f', -1, 64) + } + if f.reserveVRAM > 0 { + params["reserve_vram"] = strconv.FormatFloat(f.reserveVRAM, 'f', -1, 64) + } + return params +} diff --git a/learn_cmd.go b/learn_cmd.go new file mode 100644 index 000000000..c3be28d59 --- /dev/null +++ b/learn_cmd.go @@ -0,0 +1,495 @@ +// The self-learning offline subcommands (run by offload-dream.ps1 or by hand): +// calibrate, health, train-router, shadow-label, agent-trajectory, optimize, audit-sample. + +package main + +import ( + "context" + "encoding/json" + "flag" + "fmt" + "os" + "path/filepath" + "strings" + "time" + + "github.com/dmmdea/offload-harness/internal/agent" + "github.com/dmmdea/offload-harness/internal/calibration" + "github.com/dmmdea/offload-harness/internal/core" + "github.com/dmmdea/offload-harness/internal/embedmemo" + "github.com/dmmdea/offload-harness/internal/eval" + "github.com/dmmdea/offload-harness/internal/exemplars" + "github.com/dmmdea/offload-harness/internal/grounding" + "github.com/dmmdea/offload-harness/internal/health" + "github.com/dmmdea/offload-harness/internal/judge" + "github.com/dmmdea/offload-harness/internal/knn" + "github.com/dmmdea/offload-harness/internal/ledger" + "github.com/dmmdea/offload-harness/internal/nimclient" + "github.com/dmmdea/offload-harness/internal/nimoracle" + "github.com/dmmdea/offload-harness/internal/pipeline" + "github.com/dmmdea/offload-harness/internal/router" + "github.com/dmmdea/offload-harness/internal/shadow" + "github.com/dmmdea/offload-harness/internal/trajectory" +) + +// --- self-learning offline subcommands (run by offload-dream.ps1 or by hand) --- + +func runCalibrate(args []string) error { + fs := flag.NewFlagSet("calibrate", flag.ExitOnError) + fs.String("config", "", "config file path") + out := fs.String("out", "", "output thresholds.json (default: config thresholds_path)") + alpha := fs.Float64("alpha", 0.10, "default target error rate (per-task overrides via config target_error_rate)") + _ = fs.Parse(args) + cfg := loadCfg(fs) + _ = cfg.EnsureDirs() + dst := *out + if dst == "" { + dst = cfg.ThresholdsPath + } + // Both label sources (register D-126: the ledger and the confhead labels + // sidecar, where the classify/triage agreement labels land), calibrated on + // the scale this box EMITS (register D-130: a cutoff derived from a mix of + // the two ~10x-apart margin scales belongs to neither). + _, report, err := calibration.RunSources([]string{cfg.LedgerPath, cfg.ConfHeadLabelsPath}, *alpha, cfg.TargetErrorRate, dst, core.MarginScaleOf(cfg.ConfidenceMarginFullDenominator)) + if err != nil { + return err + } + fmt.Println(report) + fmt.Println("wrote", dst) + return nil +} + +func runHealth(args []string) error { + fs := flag.NewFlagSet("health", flag.ExitOnError) + fs.String("config", "", "config file path") + out := fs.String("out", "", "output tier_overrides.json (default: config tier_overrides_path)") + _ = fs.Parse(args) + cfg := loadCfg(fs) + dst := *out + if dst == "" { + dst = cfg.TierOverridesPath + } + rep, err := health.Run(cfg.LedgerPath, dst, core.MarginScaleOf(cfg.ConfidenceMarginFullDenominator)) + if err != nil { + return err + } + b, _ := json.MarshalIndent(rep, "", " ") + fmt.Println(string(b)) + fmt.Println("wrote", dst) + return nil +} + +func runTrainRouter(args []string) error { + fs := flag.NewFlagSet("train-router", flag.ExitOnError) + fs.String("config", "", "config file path") + out := fs.String("out", "", "output router-weights.json (default: config router_weights_path)") + _ = fs.Parse(args) + cfg := loadCfg(fs) + dst := *out + if dst == "" { + dst = cfg.RouterWeightsPath + } + rep, err := router.Train(cfg.LedgerPath, cfg.RouterLabelsPath, dst, cfg.TriageModel) + if err != nil { + return err + } + fmt.Println(rep) + fmt.Println("wrote", dst) + return nil +} + +// runShadowLabel drains the shadow queue, runs the escalation tier on each item +// (counterfactual), judges agreement vs the stored entry output, and appends a +// confhead label row to cfg.ConfHeadLabelsPath. It caps processing at --n items +// (default 200). Only confhead-labels.jsonl is written; the savings ledger is +// never touched. +// +// --oracle nim swaps the counterfactual oracle from the local escalation tier +// to the remote NIM endpoint (cfg.NIMEndpoint / cfg.NIMModel; key from +// $NVIDIA_API_KEY, never config): each item's oracle answer is produced by the +// remote model and adapted to the judge-shaped Data via internal/nimoracle. +// Only the escalation slot goes remote — the B1 E2B router-counterfactual +// stays local — and oracle-judged label rows carry "oracle":"nim" for +// provenance. Deliberate remote experiment: input text leaves the box. +func runShadowLabel(args []string) error { + fs := flag.NewFlagSet("shadow-label", flag.ExitOnError) + fs.String("config", "", "config file path") + n := fs.Int("n", 200, "max items to label this run") + oracle := fs.String("oracle", "local", "counterfactual oracle: local (escalation tier) or nim (remote NIM endpoint; tasks "+strings.Join(nimoracle.SupportedTasks(), "/")+")") + _ = fs.Parse(args) + cfg := loadCfg(fs) + if *oracle != "local" && *oracle != "nim" { + return fmt.Errorf("shadow-label: unknown --oracle %q (want local or nim)", *oracle) + } + + p, cleanup, err := openPipeline(cfg) + if err != nil { + return fmt.Errorf("shadow-label: open pipeline: %w", err) + } + defer cleanup() + + // The NIM client is built and PREFLIGHTED before Drain: Drain destructively + // claims the queue, and per-item chat failures are skipped as un-judgeable — + // so a dead endpoint, bad key, wrong model id, or a model that truncates at + // the configured budget would otherwise burn every queued item for zero + // labels. nimoracle.Preflight runs the SUMMARIZE shape (the structurally + // largest instructed reply) at its real budget through the same Adapt path + // items take — a classify-sized ping would prove auth but pass at budgets + // that then truncate every summarize item. On failure nothing has been + // drained. The paid preflight is skipped entirely when there is no queued + // work to protect (neither a queue file nor a crashed drain's claim file). + runTier := p.RunTier + oracleTag := "" + stats := &nimoracle.Stats{} + if *oracle == "nim" { + if !fileHasContent(cfg.ShadowQueuePath) && !fileHasContent(cfg.ShadowQueuePath+".draining") { + fmt.Println("shadow-label: queue empty") + return nil + } + baseURL := cfg.NIMEndpoint + key := nimclient.KeyForBase(baseURL) // env key only for NVIDIA hosts; never transmitted to a non-NVIDIA base + if key == "" && nimclient.IsHostedNVIDIA(baseURL) { + return fmt.Errorf("shadow-label: NVIDIA_API_KEY (or NGC_API_KEY) is not set — required for the hosted endpoint %s.\n"+ + "Get a free key at build.nvidia.com, then: export NVIDIA_API_KEY=nvapi-...\n"+ + "(A self-hosted NIM via nim_endpoint http://host:8000/v1 needs no key.)", baseURL) + } + timeout := time.Duration(cfg.NIMTimeoutSec) * time.Second + if timeout <= 0 { + timeout = 120 * time.Second + } + client := nimclient.New(baseURL, key, timeout) + if err := nimoracle.Preflight(context.Background(), client.Chat, cfg.NIMModel, cfg.NIMMaxTokens); err != nil { + return fmt.Errorf("shadow-label: NIM preflight failed (nothing drained): %w", err) + } + runTier = nimoracle.WrapRunTier(client.Chat, cfg.NIMModel, cfg.NIMMaxTokens, cfg.EscalationModel, p.RunTier, stats) + oracleTag = "nim" + fmt.Printf("shadow-label: oracle=nim model=%s base=%s (escalation slot only; E2B counterfactual stays local)\n", cfg.NIMModel, baseURL) + } + + items, err := shadow.Drain(cfg.ShadowQueuePath) + if err != nil { + return fmt.Errorf("shadow-label: drain queue: %w", err) + } + if len(items) == 0 { + fmt.Println("shadow-label: queue empty") + return nil + } + + // T2-C: the drain is THE repetitive embedding workload on this box — it + // re-embeds the same stored inputs on every run, and re-scores the same + // reference summaries across every item. Memoizing it is where the memo earns + // most of its keep, and leaving it on a plain embedder was the gap that made + // the package's own stated rationale untrue as shipped. + memoPath, memoEpoch, memoMax := cfg.EmbedMemoSettings() + emb := judge.NewMemoizedEmbedder(cfg.Endpoint, cfg.EmbedModel(), 30*time.Second, + judge.MemoOptions{Path: memoPath, Epoch: memoEpoch, MaxEntries: memoMax}) + if emb.Memo == nil { + fmt.Printf("shadow-label: embed memo not active — %s\n", emb.Reason) + } + defer func() { + if err := embedmemo.CloseShared(); err != nil { + fmt.Fprintln(os.Stderr, "note: embed memo counters may not have been persisted:", err) + } + }() + deps := shadow.LabelDeps{ + Escalation: cfg.EscalationModel, + E2B: cfg.TriageModel, + RunTier: runTier, + Oracle: oracleTag, + AnswersAgree: pipeline.AnswersAgree, + Ground: grounding.Check, + Similar: emb.Similar, + SummarizeSimThreshold: cfg.SummarizeSimThreshold, + AppendLabel: ledger.AppendLabel, + LabelsPath: cfg.ConfHeadLabelsPath, + RouterLabelsPath: cfg.RouterLabelsPath, + } + // meta-router v2: build the kNN entry-tier substrate during the drain, only + // when the feature is enabled (one flag controls the whole feature). + if cfg.KNNPreFilterEnabled { + deps.Embed = emb.Embed + deps.AppendKNN = func(task string, vec []float64, accept bool) error { + return knn.Append(cfg.KNNIndexPath, knn.Row{Task: task, Vec: vec, Accept: accept}) + } + } + routerBefore := countJSONLLines(cfg.RouterLabelsPath) + w := shadow.LabelQueue(context.Background(), items, *n, deps) + routerWritten := countJSONLLines(cfg.RouterLabelsPath) - routerBefore + fmt.Printf("shadow-label: drained %d items, wrote %d confhead labels -> %s\n", len(items), w, cfg.ConfHeadLabelsPath) + fmt.Printf("shadow-label: wrote %d router labels -> %s\n", routerWritten, cfg.RouterLabelsPath) + if cfg.KNNPreFilterEnabled { + fmt.Printf("shadow-label: kNN substrate now %d rows -> %s\n", countJSONLLines(cfg.KNNIndexPath), cfg.KNNIndexPath) + } + if oracleTag != "" { + fmt.Printf("shadow-label: oracle stats: %s\n", stats) + if w == 0 { + // Fail LOUD, and only after every summary line above has printed + // (the router count is a diagnostic — e.g. a broken confhead sidecar + // path yields w==0 with router rows already on disk): a zero-label + // nim run over a non-empty (and destructively drained) queue is a + // systemic failure until proven otherwise — exit non-zero so + // schedulers cannot mistake it for a clean run. + return fmt.Errorf("shadow-label: oracle=nim wrote 0 labels for %d drained items (drain is destructive — those items are gone); %s", len(items), stats) + } + } + return nil +} + +// fileHasContent reports whether path exists with non-zero size. Used to skip +// the paid NIM preflight when there is no queued shadow work to protect. +func fileHasContent(path string) bool { + fi, err := os.Stat(path) + return err == nil && fi.Size() > 0 +} + +// runAgentTrajectoryLabel drains the P6 agent-trajectory capture queue, judges each +// trajectory for GOAL SATISFACTION (a local triage — the raw StopReason "done" only +// means the model stopped calling tools, not that it reached the goal), and appends +// a label per item to the trajectory-label sidecar. The judge runs on a RECORDLESS +// pipeline (nil cache + nil ledger): the savings ledger is never opened or written — +// only the sidecar grows. A single un-judgeable item is skipped, never aborts. +func runAgentTrajectoryLabel(args []string) error { + fs := flag.NewFlagSet("agent-trajectory-label", flag.ExitOnError) + fs.String("config", "", "config file path") + n := fs.Int("n", 500, "max trajectories to label this run (<=0 = no limit)") + _ = fs.Parse(args) + cfg := loadCfg(fs) + + items, err := trajectory.Drain(cfg.AgentTrajectoryQueuePath) + if err != nil { + return fmt.Errorf("agent-trajectory-label: drain queue: %w", err) + } + if len(items) == 0 { + fmt.Println("agent-trajectory-label: trajectory queue empty") + return nil + } + // A recordless pipeline (nil cache + nil ledger): the judge's RunTier calls write + // NOTHING and never open the savings ledger — labels go to the sidecar only. The + // single shared constructor keeps the nil-store invariant from drifting. + ap := pipeline.NewRecordlessPipeline(cfg, time.Duration(cfg.RequestTimeoutSec)*time.Second) + judgeModel := cfg.EscalationModel + if judgeModel == "" { + judgeModel = cfg.Model + } + w := trajectory.LabelQueue(context.Background(), items, *n, trajectory.LabelDeps{ + RunTier: ap.RunTier, + JudgeModel: judgeModel, + AppendLabel: trajectory.AppendLabel, + LabelsPath: cfg.AgentTrajectoryLabelsPath, + }) + fmt.Printf("agent-trajectory-label: drained %d trajectories, wrote %d goal-satisfaction labels -> %s\n", len(items), w, cfg.AgentTrajectoryLabelsPath) + return nil +} + +// runAgentTrajectoryGate is the P6 flywheel adoption-gate DRY RUN: it re-runs each +// labeled READ-ONLY trajectory under a CANDIDATE planner prompt in a side-effect- +// free replay (a read-only agent.Build — no write/fetch/shell tools registered — +// with the candidate prompt), judges goal satisfaction, and reports the paired +// bootstrap delta (candidate − incumbent) + verdict (ADOPT-eligible iff ci_lo>0, +// BLOCK iff ci_hi<0). It ADOPTS NOTHING and is ledger-pristine (recordless +// pipelines throughout). Effectful trajectories (write/shell/fetch) are excluded — +// a side-effect-free replay cannot fairly re-run them. NOTE: the replay loop +// deliberately keeps the ladder rungs OFF and a fixed 8192 window (a frozen +// compaction regime) so candidate-vs-incumbent deltas are never confounded by +// compaction-default changes like ADR 0015's flip. +func runAgentTrajectoryGate(args []string) error { + fs := flag.NewFlagSet("agent-trajectory-gate", flag.ExitOnError) + fs.String("config", "", "config file path") + candidatePath := fs.String("candidate-prompt", "", "path to a candidate planner system-prompt file to evaluate") + root := fs.String("root", ".", "read root the replay agent may read (read-only)") + n := fs.Int("n", 200, "max labeled trajectories to replay (<=0 = all)") + _ = fs.Parse(args) + if *candidatePath == "" { + return fmt.Errorf("agent-trajectory-gate requires --candidate-prompt ") + } + candidate, err := os.ReadFile(*candidatePath) + if err != nil { + return fmt.Errorf("read candidate prompt: %w", err) + } + cfg := loadCfg(fs) + + labels, err := trajectory.ReadLabels(cfg.AgentTrajectoryLabelsPath) + if err != nil { + return fmt.Errorf("read trajectory labels (%s): %w", cfg.AgentTrajectoryLabelsPath, err) + } + // Only READ-ONLY trajectories are fairly replayable side-effect-free. + var gateable []trajectory.Label + for _, l := range labels { + if !hasEffectfulCap(l.Envelope) { + gateable = append(gateable, l) + } + } + if *n > 0 && len(gateable) > *n { + gateable = gateable[:*n] + } + if len(gateable) < 2 { + fmt.Printf("agent-trajectory-gate: need >=2 read-only labeled trajectories to gate; have %d\n", len(gateable)) + return nil + } + + timeout := time.Duration(cfg.RequestTimeoutSec) * time.Second + // The replay agent must score prompts on the model PRODUCTION agents plan + // with — the agent seat, not the bare workhorse (else prompt A/B verdicts + // grade the wrong planner). + plannerModel := cfg.AgentPlannerModel("") + judgeModel := cfg.EscalationModel + if judgeModel == "" { + judgeModel = cfg.Model + } + // The replay agent: READ-ONLY (no write/fetch/shell tools => side-effect-free) + // with the CANDIDATE prompt. Offload + judge run on recordless (nil-store) + // pipelines, so the whole dry run is ledger-pristine. + absRoot, _ := filepath.Abs(*root) + built, err := agent.Build(agent.BuildConfig{ + PlannerBase: cfg.Endpoint, + Model: plannerModel, + Timeout: timeout, + MaxTokens: cfg.AgentMaxTokens, + Thinking: cfg.AgentThinking, + Sampling: cfg.AgentSampling, + SamplingFinal: cfg.AgentSamplingFinal, + ReadRoot: absRoot, + Offload: pipeline.NewRecordlessOffload(cfg, plannerModel, timeout), + NPU: pipeline.NewLoopNPU(cfg), + Accel: pipeline.NewLoopAccel(cfg), + SystemPromptOverride: string(candidate), + }) + if err != nil { + return fmt.Errorf("build replay agent: %w", err) + } + judgePipe := pipeline.NewRecordlessPipeline(cfg, timeout) + + var inc, cand []float64 + incReached, candReached := 0, 0 + ctx := context.Background() + for _, lab := range gateable { + res, rerr := built.Loop.Run(ctx, lab.Goal) + if rerr != nil { + continue // a replay failure drops that pair, never aborts the gate + } + decision, _, ok := trajectory.JudgeGoalReached(ctx, judgePipe.RunTier, judgeModel, lab.Goal, res.Output) + if !ok { + continue + } + reached := decision == "yes" + inc = append(inc, b2f(lab.GoalReached)) + cand = append(cand, b2f(reached)) + if lab.GoalReached { + incReached++ + } + if reached { + candReached++ + } + } + m := len(inc) + if m < 2 { + fmt.Printf("agent-trajectory-gate: only %d comparable pairs after replay; need >=2\n", m) + return nil + } + delta, lo, hi := eval.BootstrapDeltaMean(inc, cand, 2000, 42) + verdict := "INCONCLUSIVE (95% CI spans 0 — do not adopt)" + switch { + case lo > 0: + verdict = "ADOPT-ELIGIBLE (candidate strictly better; ci_lo>0)" + case hi < 0: + verdict = "BLOCK (candidate regresses; ci_hi<0)" + } + fmt.Printf("agent-trajectory-gate DRY RUN (nothing adopted) — %d read-only trajectories replayed under %s\n", m, *candidatePath) + fmt.Printf(" incumbent goal-reached: %d/%d (%.0f%%) | candidate: %d/%d (%.0f%%)\n", + incReached, m, 100*float64(incReached)/float64(m), candReached, m, 100*float64(candReached)/float64(m)) + fmt.Printf(" delta (candidate - incumbent) = %+.3f 95%% CI [%+.3f, %+.3f]\n", delta, lo, hi) + fmt.Printf(" VERDICT: %s\n", verdict) + fmt.Println(" caveats: incumbent = label-time verdict (not re-judged); only read-only trajectories gated; small-N CI is wide") + fmt.Println(" (dry run — no prompt adopted; live adoption is P6c, operator-gated)") + return nil +} + +// hasEffectfulCap reports whether a captured envelope granted a side-effecting +// capability (write/fetch/shell) — such trajectories are excluded from the gate +// because a side-effect-free replay cannot fairly re-run them. +func hasEffectfulCap(envelope []string) bool { + for _, c := range envelope { + if c == "write" || c == "fetch" || c == "shell" { + return true + } + } + return false +} + +func b2f(b bool) float64 { + if b { + return 1 + } + return 0 +} + +// countJSONLLines returns the number of newline-terminated lines in a JSONL file +// (0 if absent/unreadable). Used to report how many router-sidecar labels a +// shadow-label run wrote, since LabelQueue's return value counts only confhead labels. +func countJSONLLines(path string) int { + data, err := os.ReadFile(path) + if err != nil { + return 0 + } + return strings.Count(string(data), "\n") +} + +func runOptimize(args []string) error { + fs := flag.NewFlagSet("optimize", flag.ExitOnError) + fs.String("config", "", "config file path") + _ = fs.Parse(args) + cfg := loadCfg(fs) + for _, t := range []string{"summarize", "classify", "extract", "triage"} { + rep, err := exemplars.Select(cfg.ExemplarsDir, t) + if err != nil { + fmt.Fprintln(os.Stderr, t+":", err) + continue + } + fmt.Println(rep) + } + return nil +} + +func runAuditSample(args []string) error { + fs := flag.NewFlagSet("audit-sample", flag.ExitOnError) + fs.String("config", "", "config file path") + n := fs.Int("n", 20, "number of cases to surface") + hard := fs.Bool("hard", false, "only hard cases (deferred / ungrounded / low margin)") + asJSON := fs.Bool("json", false, "emit JSON (for codex/Opus review)") + includeTraffic := includeTrafficFlag(fs) + _ = fs.Parse(args) + cfg := loadCfg(fs) + entries, err := ledger.ReadAll(cfg.LedgerPath) + if err != nil { + return err + } + entries, excluded := productionEntries(entries, *includeTraffic) + noteExcluded(excluded) + var picked []ledger.Entry + for _, e := range ledger.JobRows(entries) { // a local job is one job (C-62) + if e.CacheHit { + continue + } + isHard := e.Deferred || (e.Grounded != nil && !*e.Grounded) || (e.Margin > 0 && e.Margin < cfg.ConfidenceMarginThreshold) + if !*hard || isHard { + picked = append(picked, e) + } + } + if len(picked) > *n { + picked = picked[len(picked)-*n:] + } + if *asJSON { + b, _ := json.MarshalIndent(picked, "", " ") + fmt.Println(string(b)) + return nil + } + for _, e := range picked { + g := "-" + if e.Grounded != nil { + g = fmt.Sprintf("%v", *e.Grounded) + } + fmt.Printf("[%-9s] tier=%-14s margin=%.2f deferred=%v grounded=%s err=%s\n", e.Task, e.ModelTier, e.Margin, e.Deferred, g, e.ErrClass) + } + fmt.Printf("(%d cases)\n", len(picked)) + return nil +} diff --git a/main.go b/main.go index 46c4ca189..e07512e25 100644 --- a/main.go +++ b/main.go @@ -10,65 +10,35 @@ import ( "errors" "flag" "fmt" - "github.com/dmmdea/offload-harness/internal/gpulease" - "github.com/dmmdea/offload-harness/internal/winexec" "io" "log" - "math" "net" "os" "os/signal" "path/filepath" "runtime" - "sort" - "strconv" "strings" - "sync" "syscall" "time" - "github.com/dmmdea/offload-harness/internal/agent" "github.com/dmmdea/offload-harness/internal/buildinfo" - "github.com/dmmdea/offload-harness/internal/cache" - "github.com/dmmdea/offload-harness/internal/calibration" - "github.com/dmmdea/offload-harness/internal/composeremote" - "github.com/dmmdea/offload-harness/internal/confhead" "github.com/dmmdea/offload-harness/internal/config" "github.com/dmmdea/offload-harness/internal/core" "github.com/dmmdea/offload-harness/internal/datahome" "github.com/dmmdea/offload-harness/internal/delegate" "github.com/dmmdea/offload-harness/internal/embedmemo" - "github.com/dmmdea/offload-harness/internal/eval" - "github.com/dmmdea/offload-harness/internal/exemplars" "github.com/dmmdea/offload-harness/internal/fleetnode" - "github.com/dmmdea/offload-harness/internal/gpuprobe" - "github.com/dmmdea/offload-harness/internal/grounding" - "github.com/dmmdea/offload-harness/internal/health" + "github.com/dmmdea/offload-harness/internal/gpulease" "github.com/dmmdea/offload-harness/internal/hostsample" - "github.com/dmmdea/offload-harness/internal/judge" - "github.com/dmmdea/offload-harness/internal/knn" - "github.com/dmmdea/offload-harness/internal/ledger" "github.com/dmmdea/offload-harness/internal/llamaclient" "github.com/dmmdea/offload-harness/internal/mcpserver" "github.com/dmmdea/offload-harness/internal/mediacap" - "github.com/dmmdea/offload-harness/internal/mediaremote" "github.com/dmmdea/offload-harness/internal/netguard" - "github.com/dmmdea/offload-harness/internal/nimclient" - "github.com/dmmdea/offload-harness/internal/nimoracle" "github.com/dmmdea/offload-harness/internal/pairworkloads" - "github.com/dmmdea/offload-harness/internal/pipeline" - "github.com/dmmdea/offload-harness/internal/report" "github.com/dmmdea/offload-harness/internal/research" "github.com/dmmdea/offload-harness/internal/rosterprobe" - "github.com/dmmdea/offload-harness/internal/router" - "github.com/dmmdea/offload-harness/internal/shadow" "github.com/dmmdea/offload-harness/internal/storesteward" - "github.com/dmmdea/offload-harness/internal/sttremote" "github.com/dmmdea/offload-harness/internal/swapclient" - "github.com/dmmdea/offload-harness/internal/textremote" - "github.com/dmmdea/offload-harness/internal/trajectory" - "github.com/dmmdea/offload-harness/internal/visionremote" - "github.com/dmmdea/offload-harness/internal/volumes" ) // version aliases buildinfo.Version — the const moved to a shared package in @@ -404,132 +374,6 @@ A bare {"field":"string"} map has no usable properties and is deferred. `) } -func loadCfg(fs *flag.FlagSet) config.Config { - cfg, _ := loadCfgWithSource(fs) - return cfg -} - -// loadCfgWithSource loads the config and DISCLOSES its true source, warning on stderr -// whenever the effective config is built-in defaults — including the two silent-trap -// shapes a review surfaced: an explicit --config/$LOCAL_OFFLOAD_CONFIG path that does not -// exist (Load maps IsNotExist to defaults with a NIL error by design), and a file that -// exists but fails to parse. Silent degradation is the trap (live incident 2026-07-20): -// a box whose real config lived at a non-conventional path served every bare CLI call -// from built-in defaults — empty VisionModel and all — and the only symptom was a -// misleading "no vision model configured" defer in a consumer's pipeline hours later. -func loadCfgWithSource(fs *flag.FlagSet) (config.Config, config.Source) { - cfg, src := config.LoadWithSource(fs.Lookup("config").Value.String()) - config.WarnOnDefaults(src, os.Stderr) - // C-95: every verb that builds a delegator loads its config here (the MCP - // server behind agent_delegate/agent_run, delegate, acceptance, report), so - // the note that a config's own layers shadow its agent_model lives on this - // shared path, once per process, on stderr (MCP stdout is JSON-RPC). - config.WarnOnShadowedAgentModelOnce(src, cfg, os.Stderr) - return cfg, src -} - -// resolveCfgPath picks the config file by precedence: explicit --config flag > -// $LOCAL_OFFLOAD_CONFIG > ./config.json if it exists (LO-4: the README -// quickstart says `cp config.example.json config.json`, which was never read) > -// the conventional ~/.local-offload/config.json if it exists > "" (built-in -// defaults). The conventional-path fallback is the fix for a bare -// `local-offload mcp` spawned by an MCP host that passes neither flag nor -// env (as the registration did): without it the server silently ran on defaults -// with ShadowEnabled=false, so capture never fired and the flywheel starved. -// exists is injected so the precedence is unit-testable without touching disk. -func resolveCfgPath(flagPath, envPath, home string, exists func(string) bool) string { - return config.ResolvePath(flagPath, envPath, home, exists) -} - -func openPipeline(cfg config.Config) (*pipeline.Pipeline, func(), error) { - if err := cfg.EnsureDirs(); err != nil { - return nil, nil, err - } - // WithSeatEndpoints threads the Phase-A per-model overrides (seat_endpoints): - // an overridden seat's completions leave for its remote tailnet base through - // the dial-guarded client; with the key absent this is a no-op and the client - // is byte-identical to a pre-seat build. - client := llamaclient.New(cfg.Endpoint, cfg.CompletionPath, cfg.Model, time.Duration(cfg.RequestTimeoutSec)*time.Second). - WithSeatEndpoints(cfg.SeatEndpoints) - // WithRemoteLanes threads the busy-aware cascade lanes (cascade_remote_lanes, - // roast delta 7): while the machine-wide GPU lease is held, a cascade call - // whose model a lane roster-serves rides that lane instead of queueing behind - // the local card. Guarded so the absent key stays byte-identical (pinned). - if len(cfg.CascadeRemoteLanes) > 0 { - gpuLockPath, stateDir := cfg.GPULockPath, cfg.StateDir - // FleetLaneGates, not RosterResident alone: a lane base may be a - // plain llama-swap OR a fleet node whose own llama-swap binds - // loopback (C-41b). The pair shares one probe, so residency and the - // route can never disagree about which shape a base is. - laneResident, laneRoute := llamaclient.FleetLaneGates(cfg.FleetAuthToken) - client = client.WithRemoteLanes(cfg.CascadeRemoteLanes, - func() bool { return delegate.LocalBusy(gpuLockPath, stateDir) }, - llamaclient.LocalSwapBusy(cfg.Endpoint), - laneResident).WithLaneRoute(laneRoute) - } - // Cache + ledger are bbolt (single-writer, exclusive file lock). When the - // long-running MCP server holds the lock, a CLI invocation degrades to - // cache-less rather than aborting — they speed things up / report savings, - // they are not required for correctness. - // - // 0.121.1 (register D-05): the cache handle is LAZY. openPipeline runs for - // every command and every MCP server start, and the eager open made each of - // them take (or lose) the bbolt lock before knowing whether a cacheable task - // would ever run — which is what produced 49 empty per-process siblings on - // this box. Nothing below opens a file; the first Get/Put does, and prints - // the operator note then, when it is true rather than merely predicted. - var ca *cache.Cache - if cfg.CachePath != "" { // "" = caller opted out of caching (e.g. the confhead A/B, where a shared cache would cross-contaminate arms) - ca = cache.New(cfg.CachePath).WithNotify(func(mode cache.Mode, used string, err error) { - switch mode { - case cache.ModeSibling: - // 0.113.21: the shared file is held by another harness process (the - // MCP server, or a sibling session's); this process keeps its own - // in-loop hits in a per-process sibling instead of running cache-less. - fmt.Fprintln(os.Stderr, "note: result cache is held by another local-offload process; using the per-process cache", used) - case cache.ModeUnavailable: - fmt.Fprintln(os.Stderr, "note: cache unavailable; continuing without cache:", err) - } - }) - } - led, err := ledger.Open(cfg.LedgerPath) - if err != nil { - fmt.Fprintln(os.Stderr, "note: ledger unavailable (held by the MCP server?); continuing without ledger") - led = nil - } - // Every non-delegation tool row becomes one card in NVIDIA PAIR's Jobs - // list (docs/systems/pair-workloads.md); inert unless - // pair_workloads_enabled and PAIR is installed on this box. - pair := pairworkloads.New(pairworkloads.FromConfig(cfg)) - pair.AttachLedger(led) - p := pipeline.New(cfg, client, ca, led) - // A long media call opens its card when it starts (calls.go). - p.SetCallTracker(pair) - // A call routed to a fleet node opens its card and writes its asker row through here - // (D5/D6, internal/pairworkloads/remote.go). - p.SetPairEmitter(pair) - return p, func() { - if ca != nil { - ca.Close() - } - if led != nil { - led.Close() - } - // A one-shot CLI task exits right after this: deliver its PAIR card - // first (each send is bounded at 2 s), or the frame dies in flight. - pair.Wait() - // The embed memo's persisted hit/miss totals are written ONLY here. A - // process that exits without this leaves them at zero, and every reporting - // surface then states — in the one number the Phase 0.4 gate reads — that - // a memo which served thousands of hits was "never consulted". The error - // is surfaced rather than swallowed: losing the counters silently is the - // same fabricated-measurement defect in a quieter form. - if err := embedmemo.CloseShared(); err != nil { - fmt.Fprintln(os.Stderr, "note: embed memo counters may not have been persisted:", err) - } - }, nil -} - func runTask(task string, args []string) error { fs := flag.NewFlagSet(task, flag.ExitOnError) fs.String("config", "", "config file path") @@ -592,4650 +436,656 @@ func runTask(task string, args []string) error { return nil } -// textRouteHelp is the --route flag text of classify and extract (0.154.0): the MCP tools carry the same -// vocabulary (textremote). -const textRouteHelp = "classify / extract only. Where the call runs: local (default), auto (a fleet node advertising the text lane when the local GPU lease is held), remote (force a fleet node; defers when none is eligible)" - -// runCLITextTask runs one cascade text task for a CLI verb. classify and extract ride the route lane -// (textremote), as their MCP twins do; with no route that is the in-process pipeline, unchanged. -// summarize and triage have no route on either surface (a fleet node refuses them), so a route given to -// them is an error rather than a silent local run under a route the caller asked for. -func runCLITextTask(ctx context.Context, cfg config.Config, runner textremote.Runner, req core.Request, route string) (core.Result, error) { - switch req.Task { - case core.TaskClassify, core.TaskExtract: - return textremote.Run(ctx, cfg, runner, req, route), nil - } - if strings.TrimSpace(route) != "" { - return core.Result{}, fmt.Errorf("--route applies to classify and extract; %s always runs on this box's cascade (no fleet node serves it)", req.Task) - } - return runner.Run(ctx, req), nil -} - -// sttRouteHelp is the --route flag text of transcribe (ADR 0072). The CLI default is local, like the -// vision verbs: a script that never passes --route is byte-identical to before the route existed (the -// MCP tool defaults to auto). -const sttRouteHelp = "where the whisper model runs: local (default), auto (a fleet node when a render would keep the local whisper waiting), remote (force a fleet node; defers when none is eligible)" - -// visionRouteHelp is the --route flag text shared by vqa / ocr / assess-image -// (0.116.0) — the MCP tools carry the same vocabulary (visionremote). -const visionRouteHelp = "where the vision model runs: local (default), auto (a fleet node when the local GPU lease is held or the local vision seat is busy), remote (force a fleet node; defers when none is eligible)" - -// runVQA handles `local-offload vqa --question "..." [--route local|auto|remote] [--json]`. -// Unlike the text tasks, the positional argument is an IMAGE PATH (or data URI), -// not stdin text; it is passed through Request.Image and resolved in the pipeline. -func runVQA(args []string) error { - fs := flag.NewFlagSet("vqa", flag.ExitOnError) +func runMCP(args []string) error { + // D-36: take the identity of the image this process started from BEFORE anything else runs. The capture hashes the + // executable at the path os.Executable names, and on Windows that path keeps its name after a rename-swap, so a + // deploy landing during openPipeline would otherwise be hashed as the image this process runs. (A short CLI call + // never captures and keeps the lazy hash.) + buildinfo.CaptureStartup() + fs := flag.NewFlagSet("mcp", flag.ExitOnError) fs.String("config", "", "config file path") - asJSON := fs.Bool("json", false, "print full result JSON") - question := fs.String("question", "", "the question to ask about the image") - route := fs.String("route", "", visionRouteHelp) - positional, flagArgs := splitArgs(args, map[string]bool{ - "config": true, "question": true, "route": true, - }) - _ = fs.Parse(flagArgs) - - if positional == "" || positional == "-" { - return fmt.Errorf("vqa requires an image path (not stdin): local-offload vqa --question \"...\"") - } - if *question == "" { - return fmt.Errorf("vqa requires --question \"...\"") - } - - cfg := loadCfg(fs) + _ = fs.Parse(args) + // The MCP server deliberately STARTS on a config that failed validation. A + // server that exits removes every offload_* tool from every session with no + // message on any surface the operator reads — this repo has been bitten by + // that shape before. It starts and says so instead: offload_status carries + // config_error and every other tool defers by name (mcpserver.WithConfigError). + cfg, src := loadCfgWithSource(fs) p, cleanup, err := openPipeline(cfg) if err != nil { return err } defer cleanup() + // A2 hot-reload: the MCP server is long-running, so a nightly retrain that + // rewrites the self-learning artifacts would otherwise never go live without a + // manual restart. Start the background poll reloader HERE (only for the MCP + // server — CLI one-shots never start it, so they stay byte-identical and leak + // no goroutine) and stop it on cleanup. It hot-swaps changed artifacts within + // one tick, fail-open, with zero IO/parse added to the request path. + stopReloader := p.StartReloader(0) // 0 => default interval + defer stopReloader() + + // The MCP server is the ONLY long-running process that accumulates embed-memo + // hits, and deferred functions do not run on SIGINT/SIGTERM — so without a + // signal context its `defer cleanup()` (the sole caller of the memo's Flush) + // never fired, and weeks of hit counters vanished on every stop. fleet-serve + // already installs one; this did not. + ctx, stopSignals := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer stopSignals() + + // A signal handler still loses everything since the last flush on a kill -9, + // and this process can run for weeks. A periodic flush bounds that loss to one + // interval instead of one process lifetime. It is cheap: a no-op when no + // lookups happened, and one small bbolt transaction when they did. + flushDone := make(chan struct{}) + go func() { + defer close(flushDone) + t := time.NewTicker(5 * time.Minute) + defer t.Stop() + for { + select { + case <-ctx.Done(): + return + case <-t.C: + if err := embedmemo.FlushShared(); err != nil { + fmt.Fprintln(os.Stderr, "note: periodic embed-memo flush failed:", err) + } + } + } + }() - res := visionremote.Run(context.Background(), cfg, p, core.Request{ - Task: core.TaskVQA, - Door: "cli:vqa", - Image: positional, - Params: map[string]any{"question": *question}, - }, *route) - if *asJSON { - b, _ := json.MarshalIndent(res, "", " ") - fmt.Println(string(b)) + srv := mcpserver.New(p).WithConfigError(src.LoadErr) + if src.Loaded() { + // The server never reloads its config, so the ticket queue re-reads delegate_remotes from this file when it changes + // (ADR 0083): a node added to the file joins the running session. + srv = srv.WithConfigSource(src.Path) + } + err = srv.Run(ctx, version) + stopSignals() + <-flushDone + // A cancelled context IS the clean shutdown here, not a failure. Returning it + // made main print "error: context canceled" and exit 1 on every SIGTERM, which + // a supervisor reads as a crash. runFleetServe — the precedent this signal + // handling was copied from — returns nil for exactly this reason. + if errors.Is(err, context.Canceled) { return nil } - printHuman(res) - return nil + return err } -// runVideoWatch handles `local-offload video-watch --question "..." -// [--window-sec 8] [--fps 1] [--max-frames N] [--frame-width N] [--start S] -// [--end S] [--no-synthesize] [--json]`: the whole-file sweep (video_watch). -func runVideoWatch(args []string) error { - fs := flag.NewFlagSet("video-watch", flag.ExitOnError) +// runDelegate is the CLI surface onto delegate.Run (multi-node delegation, +// Task 6): same intake, engine, and response JSON as the MCP agent_delegate +// tool, for testing and scripts. Exit code contract: 0 even when subtasks +// DEFER (a defer is a success shape — the result JSON says what happened); +// non-zero only for transport/config errors (bad flags, disabled role, +// unreadable contract file, failed subtasks). +func runDelegate(args []string) error { + fs := flag.NewFlagSet("delegate", flag.ExitOnError) fs.String("config", "", "config file path") - asJSON := fs.Bool("json", false, "print full result JSON") - question := fs.String("question", "", "the question to answer about the whole video") - windowSec := fs.Float64("window-sec", 0, "seconds per window (default 8)") - fps := fs.Float64("fps", 0, "frames per second inside each window (default 1)") - maxFrames := fs.Int("max-frames", 0, "cap on frames per window (default: config video_max_frames)") - frameWidth := fs.Int("frame-width", 0, "frame width px (default: config video_frame_width)") - startS := fs.Float64("start", 0, "start second") - endS := fs.Float64("end", 0, "end second (0 = end of file)") - noSynth := fs.Bool("no-synthesize", false, "return the per-window notes only (skip the text-seat answer)") - selectFlag := fs.String("select", "", "comma-separated top-level result fields to keep") - compactFlag := fs.Bool("compact", false, "compact (minified) JSON output") - positional, flagArgs := splitArgs(args, map[string]bool{ - "config": true, "question": true, "select": true, "window-sec": true, "fps": true, - "max-frames": true, "frame-width": true, "start": true, "end": true, - }) - _ = fs.Parse(flagArgs) - if positional == "" || positional == "-" { - return fmt.Errorf("video-watch requires a video path (not stdin): local-offload video-watch