From 08fbb8ba2f247a63529aa1b02a96700cc41fc728 Mon Sep 17 00:00:00 2001 From: Chris Hoang Date: Mon, 26 Oct 2026 10:41:11 -0700 Subject: [PATCH 1/2] OUTERLOOP_AUTHOR_EFFORT passes --effort to a Claude author The author's effort level, read when each session starts. Judges, the steward and other backends ignore it; author_config_error refuses an unknown level at start, in the tick, and on each wake. --- CHANGELOG.md | 1 + docs/reference/environment.md | 14 ++++++++++++++ scripts/tick_deploy.sh | 3 ++- src/outerloop/attempt.py | 10 ++++++++++ src/outerloop/cli.py | 1 + src/outerloop/harness.py | 22 ++++++++++++++++++++++ src/outerloop/role_runner.py | 3 +++ tests/test_attempt.py | 14 ++++++++++++++ tests/test_author_overrides.py | 5 +++++ tests/test_harness.py | 27 +++++++++++++++++++++++++++ 10 files changed, 99 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 67d5974c..2f1f0edc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ Versions follow [SemVer](https://semver.org). ## [Unreleased] +- `OUTERLOOP_AUTHOR_EFFORT` sets a Claude author's effort level (`low`, `medium`, `high`, `xhigh` or `max`), passed to Claude Code as `--effort`; the pinned release (2.1.285) has the flag. Unset keeps Claude Code's default. Panel judges, the steward, and Codex or Hermes authors ignore it, so a cheaper author does not change how its work is reviewed. Start, the tick and each wake refuse an unknown level. The setting is read when each session starts and is not stored in run records. - Run endings append one `attest` trajectory event: an HMAC-SHA256 over the run id and chain head with a deployment key created owner-only beside the operator settings file, and `outerloop trajectory --verify` reports `attested` or `attestation mismatch`. Older kernels ignore the event; no ledger or PR-body change. - The PR body gains a report-only Number check that lists values in the author's report matching no `measure` event or Experiments result when rounded to the report's decimals; nothing is blocked and no trajectory event is added. - Session end records each tool call from the retained Claude or Codex log as a `tool.call` trajectory event, once per call even when a session resumes. `outerloop trajectory` and a new Process section in the PR body show counts by tool. Older kernels ignore these events. diff --git a/docs/reference/environment.md b/docs/reference/environment.md index 90e00ac2..43ba3d8f 100644 --- a/docs/reference/environment.md +++ b/docs/reference/environment.md @@ -45,6 +45,7 @@ over the inherited environment even when it is empty. | [`OUTERLOOP_IDENTITY`](#outerloop_identity) | `personal` when the kernel posts as you | unset | | [`OUTERLOOP_AUTHOR_BACKEND`](#outerloop_author_backend) | the author's harness | `claude` | | [`OUTERLOOP_AUTHOR_MODEL`](#outerloop_author_model) | the author's model | unset | +| [`OUTERLOOP_AUTHOR_EFFORT`](#outerloop_author_effort) | a Claude author's effort level | unset | | [`OUTERLOOP_CLAUDE_MODEL`](#outerloop_claude_model) | model for every Claude role | unset | | [`OUTERLOOP_AUTHOR_ENDPOINT`](#outerloop_author_endpoint) | endpoint profile for the author | unset | | [`OUTERLOOP_AUTHOR_OVERRIDES`](#outerloop_author_overrides) | per-target author selection | unset | @@ -305,6 +306,19 @@ only. A Hermes author needs it, in your provider's model ID. - Default: unset +### `OUTERLOOP_AUTHOR_EFFORT` + +The effort level a Claude author runs at, passed to Claude Code as +`--effort`. Lower levels spend fewer tokens per turn. Panel judges, the +steward, and Codex or Hermes authors ignore it. A Claude author on an +endpoint profile gets the flag too, so set it there only when the server +accepts effort. Start, the tick and each wake refuse a value outside the +list. Unlike the author's model, it is not stored in the run's record, so a +parked run picks up a change at its next wake. + +- Values: `low`, `medium`, `high`, `xhigh` or `max` +- Default: unset, which keeps Claude Code's own default + ### `OUTERLOOP_CLAUDE_MODEL` The model for every Claude role: the author when its backend is claude, diff --git a/scripts/tick_deploy.sh b/scripts/tick_deploy.sh index 4bc02d71..9cbe275c 100755 --- a/scripts/tick_deploy.sh +++ b/scripts/tick_deploy.sh @@ -155,7 +155,8 @@ if [ -n "$ENV_TRUSTED" ]; then OUTERLOOP_HERMES_REF OUTERLOOP_HERMES_SHA \ OUTERLOOP_CACHE_ROOT REVIEW_BACKEND OUTERLOOP_BRIDGE_RUNTIME \ OUTERLOOP_AUTHOR_ENDPOINT REVIEW_ENDPOINT REVIEW_MODEL \ - OUTERLOOP_AUTHOR_BACKEND OUTERLOOP_AUTHOR_MODEL OUTERLOOP_AUTHOR_OVERRIDES \ + OUTERLOOP_AUTHOR_BACKEND OUTERLOOP_AUTHOR_MODEL OUTERLOOP_AUTHOR_EFFORT \ + OUTERLOOP_AUTHOR_OVERRIDES \ OUTERLOOP_CLAUDE_MODEL \ OUTERLOOP_CLAUDE_BIN OUTERLOOP_CODEX_BIN OUTERLOOP_CODEX_KEY_FILE \ OUTERLOOP_HERMES_KEY_FILE OUTERLOOP_HERMES_RESUME_MAX_CHARS \ diff --git a/src/outerloop/attempt.py b/src/outerloop/attempt.py index bef70d57..7bee5248 100644 --- a/src/outerloop/attempt.py +++ b/src/outerloop/attempt.py @@ -74,6 +74,7 @@ ClaudeModelUnset, Harness, ResumeContextBlocked, + author_effort, default_binary, default_claude_model, redact, @@ -258,6 +259,11 @@ def author_config_error( env = os.environ if environ is None else environ if backend not in ("claude", "codex", "hermes"): return f"unknown author backend {backend!r} (expected 'claude', 'codex' or 'hermes')" + if backend == "claude": + try: + author_effort(env) + except ValueError as exc: + return str(exc) if backend == "hermes": from outerloop.harness import hermes_resume_max_chars from outerloop.hermes_install import hermes_ready @@ -6047,6 +6053,9 @@ def _run_id(value: str) -> str: hermes_repo=Path(os.environ["REVIEW_HERMES_REPO"]) if wake_backend == "hermes" else None, + effort=author_effort() + if wake_backend == "claude" and role_spec is author_spec + else "", ) wake_secrets = tuple(k for k in (*forge.secrets(), *wake_panel_secrets, wake_api_key) if k) if end_requested(Path(args.run_root), args.resume): @@ -6199,6 +6208,7 @@ def _run_id(value: str) -> str: hermes_repo=Path(os.environ["REVIEW_HERMES_REPO"]) if args.author_backend == "hermes" else None, + effort=author_effort() if args.author_backend == "claude" else "", ), spec=spec, panel_lenses=panel_lenses, diff --git a/src/outerloop/cli.py b/src/outerloop/cli.py index b60a11b0..7cbad076 100644 --- a/src/outerloop/cli.py +++ b/src/outerloop/cli.py @@ -70,6 +70,7 @@ "REVIEW_MODEL", "OUTERLOOP_AUTHOR_BACKEND", "OUTERLOOP_AUTHOR_MODEL", + "OUTERLOOP_AUTHOR_EFFORT", "OUTERLOOP_AUTHOR_OVERRIDES", "OUTERLOOP_CLAUDE_MODEL", "OUTERLOOP_CLAUDE_BIN", diff --git a/src/outerloop/harness.py b/src/outerloop/harness.py index d1dfebda..574463c6 100644 --- a/src/outerloop/harness.py +++ b/src/outerloop/harness.py @@ -90,6 +90,24 @@ def default_claude_model() -> str: return model +# The levels Claude Code's `--effort` accepts. +EFFORT_LEVELS = ("low", "medium", "high", "xhigh", "max") + + +def author_effort(env: Mapping[str, str] | None = None) -> str: + """The Claude author's effort level from OUTERLOOP_AUTHOR_EFFORT, passed to + Claude Code as `--effort`. Unset or blank keeps Claude Code's own default. + Raises ValueError for a value outside EFFORT_LEVELS. Judges, the steward and + other backends never read it.""" + value = (os.environ if env is None else env).get("OUTERLOOP_AUTHOR_EFFORT", "") + value = value.strip().lower() + if value and value not in EFFORT_LEVELS: + raise ValueError( + f"OUTERLOOP_AUTHOR_EFFORT must be one of {', '.join(EFFORT_LEVELS)} (got {value!r})" + ) + return value + + @dataclass(frozen=True) class SessionResult: """What happened in one session — everything the orchestrator needs to @@ -657,6 +675,9 @@ class ClaudeCodeHarness: # ignored when set. Contained sessions get the ADC file bind-mounted. vertex: VertexConfig | None = None endpoint: EndpointProfile | None = None + # `--effort` level; "" keeps Claude Code's default. Only the author sets it + # (OUTERLOOP_AUTHOR_EFFORT), so judges keep their default depth. + effort: str = "" CONTAINER_CLAUDE = "/opt/agent/claude" CONTAINER_ADC = "/opt/agent/adc.json" @@ -706,6 +727,7 @@ def run( "--permission-mode", permission_mode, *([("--safe-mode" if oauth else "--bare")] if self.bare else []), + *(["--effort", self.effort] if self.effort else []), *self.extra_args, ] if resume_session_id: diff --git a/src/outerloop/role_runner.py b/src/outerloop/role_runner.py index d319fce7..4d3ff229 100644 --- a/src/outerloop/role_runner.py +++ b/src/outerloop/role_runner.py @@ -98,6 +98,7 @@ def build_harness( hermes_repo: Path | None = None, hermes_provider: str = "", endpoint: str = "", + effort: str = "", ) -> Harness: """Construct the harness for any role on any backend — the ONE deployment wiring (`spec.tools` → native flags, `spec.budget` → turns/walltime, @@ -108,6 +109,7 @@ def build_harness( untrusted checkout, so judges (specs with an `output_schema`) run `--bare` — never loading the tree's CLAUDE.md / hooks as instructions. Editors keep instruction discovery (the target repo's guidance is legitimate for them). + `effort` becomes `--effort`; the other backends ignore it. - codex: `danger-full-access` uniformly — codex's own sandbox needs bubblewrap (absent in the image, unreliable nested in apptainer), so the boundary is the deployment's container or ephemeral runner, exactly as it @@ -185,6 +187,7 @@ def build_harness( # role on every CLI flips together and the API key stays the fallback vertex=None if profile else vertex_from_env(), endpoint=profile, + effort=effort, ) diff --git a/tests/test_attempt.py b/tests/test_attempt.py index 209143f4..870bd577 100644 --- a/tests/test_attempt.py +++ b/tests/test_attempt.py @@ -1114,6 +1114,17 @@ def test_codex_author_config_error() -> None: assert "unknown author backend" in codex_author_config_error("typo", "m", "img.sif") +def test_author_config_error_checks_the_claude_effort() -> None: + from outerloop.attempt import author_config_error + + bad = {"OUTERLOOP_AUTHOR_EFFORT": "minimal"} + assert "OUTERLOOP_AUTHOR_EFFORT" in author_config_error("claude", "claude-x", "", environ=bad) + good = {"OUTERLOOP_AUTHOR_EFFORT": "low"} + assert author_config_error("claude", "claude-x", "", environ=good) == "" + # other backends never read it + assert author_config_error("codex", "gpt-5.6-terra", "img.sif", environ=bad) == "" + + def test_resolve_author_key_file(monkeypatch, tmp_path) -> None: """Per-backend author keys COEXIST and are selected by backend; an explicit path wins, else the per-backend env var, else the default file.""" @@ -1890,6 +1901,9 @@ def test_harness_is_built_from_the_spec() -> None: assert harness.timeout_s == 120 assert harness.container_image == "img.sif" assert harness.bare is False # an editor keeps the target repo's guidance + assert harness.effort == "" # Claude Code's default unless the author sets one + lowered = build_harness("sk-key", spec, effort="low") + assert isinstance(lowered, ClaudeCodeHarness) and lowered.effort == "low" spec = author_spec(max_turns=9, walltime_s=300) harness = build_harness( diff --git a/tests/test_author_overrides.py b/tests/test_author_overrides.py index 86e8832a..2aaf278b 100644 --- a/tests/test_author_overrides.py +++ b/tests/test_author_overrides.py @@ -184,6 +184,7 @@ def test_override_credential_separation(deployment, same_path): @pytest.mark.parametrize("queued", [False, True]) @pytest.mark.parametrize("long_session", [False, True]) def test_binding_survives_setting_change(deployment, monkeypatch, backend, queued, long_session): + monkeypatch.setenv("OUTERLOOP_AUTHOR_EFFORT", "low") monkeypatch.setenv( "OUTERLOOP_AUTHOR_OVERRIDES", json.dumps( @@ -262,6 +263,7 @@ def launch(**kw): ) assert attempt.main() == 0 assert (seen["backend"], seen["model"]) == (backend, "served-model[endpoint=onprem]") + assert seen["effort"] == ("low" if backend == "claude" else "") record = load_record(deployment.run_root, seen["record"].run_id) assert record.author_overridden if long_session: @@ -297,8 +299,11 @@ def launch(**kw): ], ) seen.pop("backend") + # effort is read per session, never bound to the run like the model + monkeypatch.setenv("OUTERLOOP_AUTHOR_EFFORT", "high") assert attempt.main() == 0 assert (seen["backend"], seen["model"]) == (backend, "served-model[endpoint=onprem]") + assert seen["effort"] == ("high" if backend == "claude" else "") if long_session: assert seen["spec"].budget.walltime_s == 180 * 60 diff --git a/tests/test_harness.py b/tests/test_harness.py index 9276e4a7..8f14a3fd 100644 --- a/tests/test_harness.py +++ b/tests/test_harness.py @@ -18,6 +18,7 @@ ClaudeCodeHarness, CodexHarness, SessionResult, + author_effort, budget_exhausted, outage, redact, @@ -165,6 +166,32 @@ def test_argv_carries_the_controls(tmp_path: Path) -> None: assert "--permission-mode acceptEdits" in argv +@pytest.mark.parametrize("effort", ["", "low"]) +def test_effort_reaches_argv_only_when_set(tmp_path: Path, effort: str) -> None: + run_session(tmp_path, effort=effort) + argv = seen(tmp_path, "seen_argv").split() + assert ("--effort" in argv) is bool(effort) + if effort: + assert argv[argv.index("--effort") + 1] == effort + + +@pytest.mark.parametrize( + "env, expected", + [ + ({}, ""), + ({"OUTERLOOP_AUTHOR_EFFORT": " "}, ""), + ({"OUTERLOOP_AUTHOR_EFFORT": " XHigh "}, "xhigh"), + ], +) +def test_author_effort_reads_the_setting(env: dict[str, str], expected: str) -> None: + assert author_effort(env) == expected + + +def test_author_effort_refuses_an_unknown_level() -> None: + with pytest.raises(ValueError, match="OUTERLOOP_AUTHOR_EFFORT must be one of"): + author_effort({"OUTERLOOP_AUTHOR_EFFORT": "minimal"}) + + def test_brief_travels_on_stdin_never_argv(tmp_path: Path) -> None: """argv is world-readable via /proc on shared nodes; briefs carry private research text.""" From 689e55f9c3cfae0458931bf806671ca7599e71ea Mon Sep 17 00:00:00 2001 From: Chris Hoang Date: Mon, 26 Oct 2026 15:44:17 -0700 Subject: [PATCH 2/2] Author effort for every backend: Codex model_reasoning_effort, Hermes agent.reasoning_effort Review: the setting's name is backend-neutral, so a Codex or Hermes author honors it too. Codex gets a kernel-owned -c model_reasoning_effort. Hermes gets agent.reasoning_effort in the seeded config.yaml, and the capture wrapper hands the level to the agent, since run_agent.main builds its agent without a reasoning config and never reads that key. One level list and one check for every backend; judges and the steward keep their defaults. --- CHANGELOG.md | 2 +- docs/reference/environment.md | 21 ++++++++------ src/outerloop/attempt.py | 15 ++++------ src/outerloop/harness.py | 27 +++++++++++++---- src/outerloop/hermes_capture.py | 21 +++++++++++++- src/outerloop/role_runner.py | 7 +++-- tests/test_attempt.py | 13 +++++---- tests/test_author_overrides.py | 4 +-- tests/test_endpoints.py | 51 +++++++++++++++++++++++++++++++++ tests/test_session_evidence.py | 41 ++++++++++++++++++++++++++ 10 files changed, 167 insertions(+), 35 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2f1f0edc..a20f2d96 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,7 @@ Versions follow [SemVer](https://semver.org). ## [Unreleased] -- `OUTERLOOP_AUTHOR_EFFORT` sets a Claude author's effort level (`low`, `medium`, `high`, `xhigh` or `max`), passed to Claude Code as `--effort`; the pinned release (2.1.285) has the flag. Unset keeps Claude Code's default. Panel judges, the steward, and Codex or Hermes authors ignore it, so a cheaper author does not change how its work is reviewed. Start, the tick and each wake refuse an unknown level. The setting is read when each session starts and is not stored in run records. +- `OUTERLOOP_AUTHOR_EFFORT` sets the author's effort level (`low`, `medium`, `high`, `xhigh` or `max`) on every backend: Claude Code's `--effort` (the pinned 2.1.285 has the flag), Codex's `model_reasoning_effort` as a kernel-owned `-c` setting, and Hermes's `agent.reasoning_effort`. For Hermes the setting is written to the seeded `config.yaml`, and the capture wrapper also hands it to the agent, because `run_agent.main` builds its agent without a reasoning config and would otherwise ignore it. Unset keeps each harness's default. Panel judges and the steward ignore it, so a cheaper author does not change how its work is reviewed. Start, the tick and each wake refuse an unknown level. The setting is read when each session starts and is not stored in run records. - Run endings append one `attest` trajectory event: an HMAC-SHA256 over the run id and chain head with a deployment key created owner-only beside the operator settings file, and `outerloop trajectory --verify` reports `attested` or `attestation mismatch`. Older kernels ignore the event; no ledger or PR-body change. - The PR body gains a report-only Number check that lists values in the author's report matching no `measure` event or Experiments result when rounded to the report's decimals; nothing is blocked and no trajectory event is added. - Session end records each tool call from the retained Claude or Codex log as a `tool.call` trajectory event, once per call even when a session resumes. `outerloop trajectory` and a new Process section in the PR body show counts by tool. Older kernels ignore these events. diff --git a/docs/reference/environment.md b/docs/reference/environment.md index 43ba3d8f..4111ecc1 100644 --- a/docs/reference/environment.md +++ b/docs/reference/environment.md @@ -45,7 +45,7 @@ over the inherited environment even when it is empty. | [`OUTERLOOP_IDENTITY`](#outerloop_identity) | `personal` when the kernel posts as you | unset | | [`OUTERLOOP_AUTHOR_BACKEND`](#outerloop_author_backend) | the author's harness | `claude` | | [`OUTERLOOP_AUTHOR_MODEL`](#outerloop_author_model) | the author's model | unset | -| [`OUTERLOOP_AUTHOR_EFFORT`](#outerloop_author_effort) | a Claude author's effort level | unset | +| [`OUTERLOOP_AUTHOR_EFFORT`](#outerloop_author_effort) | the author's effort level | unset | | [`OUTERLOOP_CLAUDE_MODEL`](#outerloop_claude_model) | model for every Claude role | unset | | [`OUTERLOOP_AUTHOR_ENDPOINT`](#outerloop_author_endpoint) | endpoint profile for the author | unset | | [`OUTERLOOP_AUTHOR_OVERRIDES`](#outerloop_author_overrides) | per-target author selection | unset | @@ -308,16 +308,19 @@ only. A Hermes author needs it, in your provider's model ID. ### `OUTERLOOP_AUTHOR_EFFORT` -The effort level a Claude author runs at, passed to Claude Code as -`--effort`. Lower levels spend fewer tokens per turn. Panel judges, the -steward, and Codex or Hermes authors ignore it. A Claude author on an -endpoint profile gets the flag too, so set it there only when the server -accepts effort. Start, the tick and each wake refuse a value outside the -list. Unlike the author's model, it is not stored in the run's record, so a -parked run picks up a change at its next wake. +The effort level the author runs at, on every backend: Claude Code's +`--effort`, Codex's `model_reasoning_effort`, or Hermes's +`agent.reasoning_effort`. Lower levels spend fewer tokens per turn. Panel +judges and the steward ignore it. An author on an endpoint profile gets it +too, so set it there only when the server accepts the backend's effort +parameter. Hermes is the exception: it sends a top-level `reasoning_effort` +field and drops the field for the rest of the session if the server rejects +it. Start, the tick and each wake refuse a value outside the list. Unlike +the author's model, it is not stored in the run's record, so a parked run +picks up a change at its next wake. - Values: `low`, `medium`, `high`, `xhigh` or `max` -- Default: unset, which keeps Claude Code's own default +- Default: unset, which keeps each harness's own default ### `OUTERLOOP_CLAUDE_MODEL` diff --git a/src/outerloop/attempt.py b/src/outerloop/attempt.py index 7bee5248..947e150e 100644 --- a/src/outerloop/attempt.py +++ b/src/outerloop/attempt.py @@ -259,11 +259,10 @@ def author_config_error( env = os.environ if environ is None else environ if backend not in ("claude", "codex", "hermes"): return f"unknown author backend {backend!r} (expected 'claude', 'codex' or 'hermes')" - if backend == "claude": - try: - author_effort(env) - except ValueError as exc: - return str(exc) + try: + author_effort(env) + except ValueError as exc: + return str(exc) if backend == "hermes": from outerloop.harness import hermes_resume_max_chars from outerloop.hermes_install import hermes_ready @@ -6053,9 +6052,7 @@ def _run_id(value: str) -> str: hermes_repo=Path(os.environ["REVIEW_HERMES_REPO"]) if wake_backend == "hermes" else None, - effort=author_effort() - if wake_backend == "claude" and role_spec is author_spec - else "", + effort=author_effort() if role_spec is author_spec else "", ) wake_secrets = tuple(k for k in (*forge.secrets(), *wake_panel_secrets, wake_api_key) if k) if end_requested(Path(args.run_root), args.resume): @@ -6208,7 +6205,7 @@ def _run_id(value: str) -> str: hermes_repo=Path(os.environ["REVIEW_HERMES_REPO"]) if args.author_backend == "hermes" else None, - effort=author_effort() if args.author_backend == "claude" else "", + effort=author_effort(), ), spec=spec, panel_lenses=panel_lenses, diff --git a/src/outerloop/harness.py b/src/outerloop/harness.py index 574463c6..16d1d2a3 100644 --- a/src/outerloop/harness.py +++ b/src/outerloop/harness.py @@ -90,15 +90,16 @@ def default_claude_model() -> str: return model -# The levels Claude Code's `--effort` accepts. +# The effort levels all three author harnesses accept: Claude Code's +# `--effort`, Codex's `model_reasoning_effort` and Hermes's +# `agent.reasoning_effort`. EFFORT_LEVELS = ("low", "medium", "high", "xhigh", "max") def author_effort(env: Mapping[str, str] | None = None) -> str: - """The Claude author's effort level from OUTERLOOP_AUTHOR_EFFORT, passed to - Claude Code as `--effort`. Unset or blank keeps Claude Code's own default. - Raises ValueError for a value outside EFFORT_LEVELS. Judges, the steward and - other backends never read it.""" + """The author's effort level from OUTERLOOP_AUTHOR_EFFORT, for any backend. + Unset or blank keeps the harness's own default. Raises ValueError for a + value outside EFFORT_LEVELS. Judges and the steward never read it.""" value = (os.environ if env is None else env).get("OUTERLOOP_AUTHOR_EFFORT", "") value = value.strip().lower() if value and value not in EFFORT_LEVELS: @@ -1195,6 +1196,9 @@ class CodexHarness: # (the deployment's runner is the boundary) container_image: str = "" apptainer_binary: str = field(default_factory=apptainer_from_env) + # `model_reasoning_effort`; "" keeps Codex's default. Only the author sets + # it (OUTERLOOP_AUTHOR_EFFORT). + effort: str = "" # in-container path the host codex binary is bound to (bind-from-host, like # ClaudeCodeHarness.CONTAINER_CLAUDE — the image stays codex-free, and codex # is updated by swapping one host binary, no rebuild). A bare class attribute @@ -1421,6 +1425,8 @@ def run( kernel_owned: tuple[str, ...] = ("-c", "features.multi_agent=false") if not codex_web_search(bool(self.endpoint)): kernel_owned += ("-c", 'web_search="disabled"') + if self.effort: + kernel_owned += ("-c", f"model_reasoning_effort={json.dumps(self.effort)}") codex_argv = _codex_command( self.CONTAINER_CODEX if self.container_image else self.binary, self.model, @@ -1704,6 +1710,9 @@ class HermesHarness: apptainer_binary: str = field(default_factory=apptainer_from_env) resume_max_chars: int | None = None + # `agent.reasoning_effort`; "" keeps Hermes's default. Only the author sets + # it (OUTERLOOP_AUTHOR_EFFORT). + effort: str = "" @capture_session("hermes") def run( @@ -1771,6 +1780,8 @@ def run( f" key_env: {json.dumps(self.key_env)}\n" " api_mode: chat_completions\n" ) + if self.effort: + config_lines.append(f"agent:\n reasoning_effort: {json.dumps(self.effort)}\n") if self.approvals_deny: config_lines.append("approvals:\n deny:\n") config_lines += [f' - "{glob}"\n' for glob in self.approvals_deny] @@ -1834,12 +1845,18 @@ def run( env = session_env(self.api_key, self.key_env, session_home) env["TERMINAL_CWD"] = str(workspace.resolve()) env["OUTERLOOP_CAPTURE_ID"] = capture_id + if self.effort: + # run_agent.main builds its agent without the config's + # reasoning_effort; hermes_capture applies it (see there) + env["OUTERLOOP_HERMES_EFFORT"] = self.effort if self.container_image: # --cleanenv drops the host env except APPTAINERENV_*: the key # travels via the environment, never argv env[f"APPTAINERENV_{self.key_env}"] = env[self.key_env] env["APPTAINERENV_TERMINAL_CWD"] = env["TERMINAL_CWD"] env["APPTAINERENV_OUTERLOOP_CAPTURE_ID"] = capture_id + if self.effort: + env["APPTAINERENV_OUTERLOOP_HERMES_EFFORT"] = self.effort # cwd is the per-run home, NOT the workspace: --save_sample writes # its trajectory JSON to cwd, and artifacts must never land in the # clone (they would enter the diff). diff --git a/src/outerloop/hermes_capture.py b/src/outerloop/hermes_capture.py index bebd5dd5..2bf9af85 100644 --- a/src/outerloop/hermes_capture.py +++ b/src/outerloop/hermes_capture.py @@ -1,4 +1,10 @@ -"""Run the pinned Hermes CLI unchanged, retaining its reported session evidence. +"""Run the pinned Hermes CLI, retaining its reported session evidence. + +The one behavior change is the author's effort level. `run_agent.main` builds +its agent without a reasoning config, so the `agent.reasoning_effort` the +harness writes to config.yaml would never reach a request. When the harness +sets OUTERLOOP_HERMES_EFFORT, the agent gets the reasoning config Hermes's own +CLI would build from that level. Unset, nothing changes. This file runs with Hermes's Python, without an outerloop installation. """ @@ -18,6 +24,19 @@ def main() -> None: import run_agent from agent.legacy_cli import main as cli_main + if effort := os.environ.get("OUTERLOOP_HERMES_EFFORT", ""): + from hermes_constants import parse_reasoning_effort + + reasoning = parse_reasoning_effort(effort) + original_init = run_agent.AIAgent.__init__ + + def init(agent: Any, *args: Any, **kwargs: Any) -> None: + if kwargs.get("reasoning_config") is None: + kwargs["reasoning_config"] = reasoning + original_init(agent, *args, **kwargs) + + run_agent.AIAgent.__init__ = init + original = run_agent.AIAgent.run_conversation root_agent: Any = None diff --git a/src/outerloop/role_runner.py b/src/outerloop/role_runner.py index 4d3ff229..fc6d3cb0 100644 --- a/src/outerloop/role_runner.py +++ b/src/outerloop/role_runner.py @@ -109,7 +109,6 @@ def build_harness( untrusted checkout, so judges (specs with an `output_schema`) run `--bare` — never loading the tree's CLAUDE.md / hooks as instructions. Editors keep instruction discovery (the target repo's guidance is legitimate for them). - `effort` becomes `--effort`; the other backends ignore it. - codex: `danger-full-access` uniformly — codex's own sandbox needs bubblewrap (absent in the image, unreliable nested in apptainer), so the boundary is the deployment's container or ephemeral runner, exactly as it @@ -120,7 +119,9 @@ def build_harness( Containment is NOT decided here: pass `container_image` where the deployment has a jail (the cluster), pass none where the runner itself is the ephemeral boundary (CI). The tokenless split keeps credentials out of - the session either way.""" + the session either way. `effort` is each backend's own reasoning-effort + setting (`--effort`, `model_reasoning_effort`, `agent.reasoning_effort`); + callers pass it for the author only.""" model, profile = resolve_endpoint(model or "", backend, endpoint) if profile: api_key = profile.key() @@ -137,6 +138,7 @@ def build_harness( timeout_s=spec.budget.walltime_s, container_image=container_image, extra_args=(*codex_extra_args, *web), + effort=effort, ) if backend == "hermes": if hermes_repo is None: @@ -167,6 +169,7 @@ def build_harness( enabled_toolsets=enabled, disabled_toolsets=tuple(t for t in _HERMES_TOOLSETS if t not in enabled), container_image=container_image, + effort=effort, ) if backend != "claude": raise ValueError(f"unknown backend: {backend!r}") diff --git a/tests/test_attempt.py b/tests/test_attempt.py index 870bd577..11611a65 100644 --- a/tests/test_attempt.py +++ b/tests/test_attempt.py @@ -1114,15 +1114,16 @@ def test_codex_author_config_error() -> None: assert "unknown author backend" in codex_author_config_error("typo", "m", "img.sif") -def test_author_config_error_checks_the_claude_effort() -> None: +@pytest.mark.parametrize( + "backend, model, image", [("claude", "claude-x", ""), ("codex", "gpt-5.6-terra", "img.sif")] +) +def test_author_config_error_checks_the_effort(backend: str, model: str, image: str) -> None: from outerloop.attempt import author_config_error - bad = {"OUTERLOOP_AUTHOR_EFFORT": "minimal"} - assert "OUTERLOOP_AUTHOR_EFFORT" in author_config_error("claude", "claude-x", "", environ=bad) + bad = {"OUTERLOOP_AUTHOR_EFFORT": "minimal"} # Codex knows it; not every backend does + assert "OUTERLOOP_AUTHOR_EFFORT" in author_config_error(backend, model, image, environ=bad) good = {"OUTERLOOP_AUTHOR_EFFORT": "low"} - assert author_config_error("claude", "claude-x", "", environ=good) == "" - # other backends never read it - assert author_config_error("codex", "gpt-5.6-terra", "img.sif", environ=bad) == "" + assert author_config_error(backend, model, image, environ=good) == "" def test_resolve_author_key_file(monkeypatch, tmp_path) -> None: diff --git a/tests/test_author_overrides.py b/tests/test_author_overrides.py index 2aaf278b..7606d2c3 100644 --- a/tests/test_author_overrides.py +++ b/tests/test_author_overrides.py @@ -263,7 +263,7 @@ def launch(**kw): ) assert attempt.main() == 0 assert (seen["backend"], seen["model"]) == (backend, "served-model[endpoint=onprem]") - assert seen["effort"] == ("low" if backend == "claude" else "") + assert seen["effort"] == "low" record = load_record(deployment.run_root, seen["record"].run_id) assert record.author_overridden if long_session: @@ -303,7 +303,7 @@ def launch(**kw): monkeypatch.setenv("OUTERLOOP_AUTHOR_EFFORT", "high") assert attempt.main() == 0 assert (seen["backend"], seen["model"]) == (backend, "served-model[endpoint=onprem]") - assert seen["effort"] == ("high" if backend == "claude" else "") + assert seen["effort"] == "high" if long_session: assert seen["spec"].budget.walltime_s == 180 * 60 diff --git a/tests/test_endpoints.py b/tests/test_endpoints.py index 61456329..5e210ca8 100644 --- a/tests/test_endpoints.py +++ b/tests/test_endpoints.py @@ -253,6 +253,57 @@ def communicate(self, **kwargs: Any) -> tuple[str, str]: assert "endpoint-secret" not in seen.get("config", "") +@pytest.mark.parametrize("backend", ["claude", "codex", "hermes"]) +@pytest.mark.parametrize("effort", ["", "low"]) +def test_author_effort_reaches_each_backend( + profile: dict[str, str], + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + backend: str, + effort: str, +) -> None: + seen: dict[str, Any] = {} + workspace = tmp_path / "workspace" + workspace.mkdir() + monkeypatch.setattr(harness_mod, "hermes_ready", lambda _: True) + + class Process: + returncode = 0 + + def __init__(self, command: list[str], **kwargs: Any): + seen.update(argv=command, env=kwargs["env"]) + path = Path(kwargs["env"]["HOME"]) / ".hermes/config.yaml" + if path.is_file(): + seen["config"] = yaml.safe_load(path.read_text()) + + def communicate(self, **kwargs: Any) -> tuple[str, str]: + return '{"type":"result","subtype":"success","result":"done"}', "" + + monkeypatch.setattr(harness_mod.subprocess, "Popen", Process) + harness = build_harness( + "ignored-key", + author_spec(), + backend=backend, + endpoint="local", + binary="/opt/agent-cli", + hermes_repo=tmp_path / "hermes", + effort=effort, + ) + harness.run("brief", workspace) + argv, env = seen["argv"], seen["env"] + if backend == "claude": + assert (argv[argv.index("--effort") + 1] == "low") if effort else "--effort" not in argv + elif backend == "codex": + setting = 'model_reasoning_effort="low"' + assert (setting in argv) is bool(effort) + if effort: # kernel-owned: after the operator's own -c settings + assert argv.index(setting) > argv.index("features.multi_agent=false") + else: + agent = seen["config"].get("agent", {}) + assert agent.get("reasoning_effort") == (effort or None) + assert env.get("OUTERLOOP_HERMES_EFFORT") == (effort or None) + + @pytest.mark.parametrize("backend", ["claude", "codex", "hermes"]) def test_reviewer_endpoint( profile: dict[str, str], monkeypatch: pytest.MonkeyPatch, backend: str diff --git a/tests/test_session_evidence.py b/tests/test_session_evidence.py index 027534d3..e8b61032 100644 --- a/tests/test_session_evidence.py +++ b/tests/test_session_evidence.py @@ -323,6 +323,47 @@ def cli_main(run): assert result.tokens["output_tokens"] == 12 +@pytest.mark.parametrize("effort", ["", "low"]) +def test_hermes_wrapper_applies_the_author_effort(tmp_path, monkeypatch, effort): + import sys + from types import ModuleType + + from outerloop import hermes_capture + + built: list[object] = [] + + class Agent: + model = "test-model" + + def __init__(self, **kwargs): + built.append(kwargs.get("reasoning_config")) + + def run_conversation(self, query): + return {"messages": [], "completed": True} + + module = ModuleType("run_agent") + module.__dict__["AIAgent"] = Agent + module.__dict__["main"] = lambda: Agent(model="test-model").run_conversation("prompt") + cli = ModuleType("agent.legacy_cli") + cli.__dict__["main"] = lambda run: run() and 0 + constants = ModuleType("hermes_constants") + constants.__dict__["parse_reasoning_effort"] = lambda level: {"enabled": True, "effort": level} + monkeypatch.setitem(sys.modules, "run_agent", module) + monkeypatch.setitem(sys.modules, "agent", ModuleType("agent")) + monkeypatch.setitem(sys.modules, "agent.legacy_cli", cli) + monkeypatch.setitem(sys.modules, "hermes_constants", constants) + monkeypatch.setenv("HOME", str(tmp_path)) + monkeypatch.setenv("OUTERLOOP_CAPTURE_ID", "session-1-invocation-1") + if effort: + monkeypatch.setenv("OUTERLOOP_HERMES_EFFORT", effort) + else: + monkeypatch.delenv("OUTERLOOP_HERMES_EFFORT", raising=False) + monkeypatch.setattr(sys, "argv", ["capture.py", str(tmp_path)]) + with pytest.raises(SystemExit): + hermes_capture.main() + assert built == [{"enabled": True, "effort": "low"} if effort else None] + + def test_redaction_at_cap_boundary_and_issued_tokens(tmp_path, monkeypatch): import outerloop.appauth as appauth