From 0eb7839437f7a0d20fcac889813536d3ae7576d8 Mon Sep 17 00:00:00 2001 From: Mengye Ren Date: Sat, 24 Oct 2026 22:25:58 -0400 Subject: [PATCH 1/2] Review panel accepts measured composite changes (#489) --- CHANGELOG.md | 1 + src/outerloop/guidance_pack.py | 4 +++- src/outerloop/verifier.py | 15 ++++++++++++--- tests/test_verifier.py | 15 +++++++++++++++ 4 files changed, 31 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 43e83ece..1f7fbe5e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ Versions follow [SemVer](https://semver.org). ## [Unreleased] +- The pre-PR panel accepts a composite change — a PR that carries several changes — when the report gives each component's isolated measurement and the combination's under the same protocol and the combination is what the kernel measured. A bundle without those isolated measurements stays blocked (#489). - Claude sessions can use a subscription token from `claude setup-token` (`sk-ant-oat...`). The harness sends it as `CLAUDE_CODE_OAUTH_TOKEN`, not `ANTHROPIC_API_KEY`, and `init` accepts it. A panel judge on a token runs `claude --safe-mode`, because `--bare` cannot read OAuth tokens. Both modes ignore the checkout's CLAUDE.md, hooks, plugins and MCP servers. The pinned Claude Code release (2.1.285) has `--safe-mode`. - A session now hears once when a live sibling states a new or changed hypothesis (#360). The note names the sibling, quotes its hypothesis, and points to `siblings` for its launches and score. It is sent only when `channels.siblings` is on. Upgrading: the note uses the existing `note` kind, so an older kernel shows it as ordinary inbox text. - A skills-only plugin for Claude Code and Codex sets up Outerloop, writes and validates the contract, and operates runs from the agent (#562). Install it with `/plugin marketplace add outerloop-science/outerloop` or `codex plugin marketplace add outerloop-science/outerloop`. Its setup skill offers a Claude subscription token from `claude setup-token` as the author credential. diff --git a/src/outerloop/guidance_pack.py b/src/outerloop/guidance_pack.py index 2dce6f5f..737642f7 100644 --- a/src/outerloop/guidance_pack.py +++ b/src/outerloop/guidance_pack.py @@ -16,7 +16,9 @@ AUTHOR = ( "Form ONE concrete hypothesis for why the benchmark underperforms " - "before you implement, and implement that one.\n\n" + "before you implement, and implement that one. A combination whose parts " + "each fall below the improvement floor may ship together if the PR " + "reports each part's isolated measurement and the combination's.\n\n" "Read past reports critically: a negative settles only what was actually " "run. One point in a parameter space, an eval that hit its walltime, or " "an infrastructure failure does not close an idea — vary what went " diff --git a/src/outerloop/verifier.py b/src/outerloop/verifier.py index 1708bae7..055f59ac 100644 --- a/src/outerloop/verifier.py +++ b/src/outerloop/verifier.py @@ -154,7 +154,15 @@ def gather_thread( - unsupported-claim: statements in the report that the PR's own evidence contradicts - measurement-gap: noise floors, seeds, or protocol issues in the - measurement itself + measurement itself, including a bundle with no isolated measurement for + each of its components + +A composite change — a pull request that carries several changes — is +acceptable when the report gives each component's isolated measurement and +the combination's under the same protocol, and the combination is what the +orchestrator measured. A bundle that reports no isolated measurement for +each component stays blocked: the combination's number alone does not show +what produced it. Check the report's evidence; do not re-run anything. Use the contract (the rules), the ruler source (how the eval actually works), the claimed numbers, and the agent's own report. The report's @@ -177,8 +185,9 @@ def gather_thread( contradiction between the claim and the PR's OWN evidence (a mechanism story the report's own numbers refute, or an improvement that a trivial baseline in the report's own tables beats — true even when the measured -delta itself is real), or a scope or measurement fact that makes the -number mislead. Anything else — every research-taste judgment, including +delta itself is real), a bundle that reports no isolated measurement for +each component, or a scope or measurement fact that makes the number +mislead. Anything else — every research-taste judgment, including anything set by repository guidance, when provided below — is advisory: `blocking` false. diff --git a/tests/test_verifier.py b/tests/test_verifier.py index 567dd359..694714fc 100644 --- a/tests/test_verifier.py +++ b/tests/test_verifier.py @@ -270,3 +270,18 @@ def test_taste_categories_live_only_in_the_panel_pack() -> None: # the standards themselves are the pack's, carried as advisory guidance assert "documents each change's own effect" in PANEL assert "picture of the landscape" in PANEL + + +def test_verify_brief_accepts_a_measured_composite_but_blocks_a_bare_bundle() -> None: + # Issue #489: the panel accepts a combination when each component was measured in + # isolation under the same protocol and the combination is what the + # orchestrator gated; a bundle without those measurements stays blocked. + from outerloop.guidance_pack import AUTHOR + from outerloop.verifier import build_verify_agent_brief + + brief = " ".join(build_verify_agent_brief(make_pr(), "contract").split()) + assert "each component's isolated measurement" in brief + assert "the combination is what the orchestrator measured" in brief + assert "A bundle that reports no isolated measurement for each component stays blocked" in brief + # the author's guidance names the same route, beside the one-hypothesis rule + assert "each part's isolated measurement and the combination's" in AUTHOR From fc148f475665355bc31c396300899625bbf27f02 Mon Sep 17 00:00:00 2001 From: Mengye Ren Date: Sat, 24 Oct 2026 22:43:44 -0400 Subject: [PATCH 2/2] Composite components must appear in the kernel's launch ledger; the panel claim carries it --- src/outerloop/attempt.py | 30 ++++++++++++++++++++---------- src/outerloop/orchestrator.py | 6 +++--- src/outerloop/verifier.py | 20 +++++++++++--------- tests/test_attempt.py | 3 +++ tests/test_launchlog.py | 6 +++--- tests/test_verifier.py | 5 +++-- 6 files changed, 43 insertions(+), 27 deletions(-) diff --git a/src/outerloop/attempt.py b/src/outerloop/attempt.py index b96e58d8..41085212 100644 --- a/src/outerloop/attempt.py +++ b/src/outerloop/attempt.py @@ -105,6 +105,7 @@ attempt_once, benchmark_floor, clears_min_delta, + experiments_section, out_of_scope, pr_body, resume_attempt, @@ -3917,20 +3918,23 @@ def _panel_lenses_from_args( return tuple(lenses), tuple(dict.fromkeys(secrets)) -def _panel_claim_body(benchmark: str, baseline: float, candidate: float, report: str) -> str: - """The synthetic claim the panel judges: the kernel's measured claim plus - the author's report and nothing else — this repository's guidance travels - in the judges' own prompt (run_panel's `guidance`), never inside this - body, which every judge prompt fences as untrusted data. The report - budget subtracts the actual header so the body never outgrows the cap - the verify prompt fences it under.""" +def _panel_claim_body( + benchmark: str, baseline: float, candidate: float, report: str, experiments: str = "" +) -> str: + """The synthetic claim the panel judges: the kernel's measured claim, the + run's launches from the kernel's ledger, and the author's report — this + repository's guidance travels in the judges' own prompt (run_panel's + `guidance`), never inside this body, which every judge prompt fences as + untrusted data. The report budget subtracts the header and the ledger + table so the body never outgrows the cap the verify prompt fences it under.""" header = ( f"Automated improvement claim (pre-PR): {benchmark} " - f"{baseline} -> {candidate}, measured by the orchestrator.\n\n" + f"{baseline} -> {candidate}, measured by the orchestrator.\n" + f"{experiments}\n\n" f"## Research report\n\n*Session prose, written before " f"the orchestrator measured.*\n\n" ) - return f"{header}{report[: MAX_CLAIM_CHARS - len(header)]}" + return f"{header}{report[: max(0, MAX_CLAIM_CHARS - len(header))]}" def build_panel_runner( @@ -3968,7 +3972,13 @@ def build_panel_runner( reads = {"n": start_round} render_claim = claim_body or ( lambda baseline, candidate, report: _panel_claim_body( - benchmark, baseline, candidate, report + benchmark, + baseline, + candidate, + report, + # the run's own launches, recorded by the kernel: the evidence a + # composite change's component measurements must appear in + "\n".join(experiments_section(experiments_rows(run_dir))), ) ) diff --git a/src/outerloop/orchestrator.py b/src/outerloop/orchestrator.py index f63b3414..45c0c6cb 100644 --- a/src/outerloop/orchestrator.py +++ b/src/outerloop/orchestrator.py @@ -2786,7 +2786,7 @@ def _ended(row: dict[str, Any]) -> str: return how -def _experiments_section(rows: list[dict[str, Any]]) -> list[str]: +def experiments_section(rows: list[dict[str, Any]]) -> list[str]: """The run's launches as the ledger recorded them: what ran, how each job ended, what it printed last. Empty when the run launched nothing.""" if not rows: @@ -2908,7 +2908,7 @@ def pr_body( "## Research report", "", *report_lines, - *_experiments_section(experiments or []), + *experiments_section(experiments or []), "", "## Measured", "", @@ -2939,7 +2939,7 @@ def pr_body( "## Research report", "", redact(result.submit_report, redact_secrets)[:MAX_REPORT_BODY], - *_experiments_section(experiments or []), + *experiments_section(experiments or []), "", "## Self-reported values", "", diff --git a/src/outerloop/verifier.py b/src/outerloop/verifier.py index 055f59ac..91a5f428 100644 --- a/src/outerloop/verifier.py +++ b/src/outerloop/verifier.py @@ -154,15 +154,17 @@ def gather_thread( - unsupported-claim: statements in the report that the PR's own evidence contradicts - measurement-gap: noise floors, seeds, or protocol issues in the - measurement itself, including a bundle with no isolated measurement for - each of its components + measurement itself, including a bundle that does not show an isolated + measurement for every one of its components A composite change — a pull request that carries several changes — is -acceptable when the report gives each component's isolated measurement and -the combination's under the same protocol, and the combination is what the -orchestrator measured. A bundle that reports no isolated measurement for -each component stays blocked: the combination's number alone does not show -what produced it. Check the report's evidence; do not re-run anything. +acceptable when every component's isolated measurement and the +combination's were run under the same protocol, those component runs appear +among the run's own launches in the kernel's Experiments table (not only in +the report's prose), and the combination is what the orchestrator measured. +A bundle that does not show such a measurement for every component stays +blocked: the combination's number alone does not show what produced it. +Check the evidence; do not re-run anything. Use the contract (the rules), the ruler source (how the eval actually works), the claimed numbers, and the agent's own report. The report's @@ -185,8 +187,8 @@ def gather_thread( contradiction between the claim and the PR's OWN evidence (a mechanism story the report's own numbers refute, or an improvement that a trivial baseline in the report's own tables beats — true even when the measured -delta itself is real), a bundle that reports no isolated measurement for -each component, or a scope or measurement fact that makes the number +delta itself is real), a bundle that does not show a kernel-recorded +isolated measurement for every component, or a scope or measurement fact that makes the number mislead. Anything else — every research-taste judgment, including anything set by repository guidance, when provided below — is advisory: `blocking` false. diff --git a/tests/test_attempt.py b/tests/test_attempt.py index f9b032b5..209143f4 100644 --- a/tests/test_attempt.py +++ b/tests/test_attempt.py @@ -4970,6 +4970,9 @@ def test_guidance_layer_shapes_brief_and_claim(mode: str) -> None: ) claim = _panel_claim_body("tsp", 13.8, 13.1, "report text") assert "measured by the orchestrator" in claim + # the kernel's launch ledger rides ahead of the author's prose (#489) + ledgered = _panel_claim_body("tsp", 13.8, 13.1, "report text", "## Experiments\n| row |") + assert ledgered.index("## Experiments") < ledgered.index("## Research report") # the header and wrapper are priced into the report budget, so a long # report cannot push the body past the fence the verify prompt caps long_claim = _panel_claim_body("tsp", 13.8, 13.1, "r" * (2 * MAX_CLAIM_CHARS)) diff --git a/tests/test_launchlog.py b/tests/test_launchlog.py index ed04f9c2..a49fb35b 100644 --- a/tests/test_launchlog.py +++ b/tests/test_launchlog.py @@ -161,7 +161,7 @@ def test_experiments_rows_come_from_the_ledger_including_each_jobs_last_line( def test_legacy_commit_tolerance_and_retry(tmp_path: Path) -> None: from outerloop.launchlog import experiments_rows - from outerloop.orchestrator import _experiments_section + from outerloop.orchestrator import experiments_section legacy = Path(__file__).parent / "fixtures" / "launches-before-provenance.jsonl" (tmp_path / LEDGER).write_bytes(legacy.read_bytes()) @@ -169,7 +169,7 @@ def test_legacy_commit_tolerance_and_retry(tmp_path: Path) -> None: for _ in range(2): rows = experiments_rows(tmp_path) assert rows[0]["commit"] == "" - assert "| unknown |" in "\n".join(_experiments_section(rows)) + assert "| unknown |" in "\n".join(experiments_section(rows)) # Re-parking a pre-upgrade launch must not invent provenance. append_submitted(tmp_path, sleep=1, launches=launch, job_ids=["1"], at=30, commit="a" * 40) assert (tmp_path / LEDGER).read_bytes() == legacy.read_bytes() @@ -180,5 +180,5 @@ def test_legacy_commit_tolerance_and_retry(tmp_path: Path) -> None: append_submitted(tmp_path, sleep=2, launches=launch, job_ids=["2"], at=40, commit="b" * 40) rows = experiments_rows(tmp_path) assert [r["commit"] for r in rows] == ["", "b" * 40] - rendered = "\n".join(_experiments_section(rows)) + rendered = "\n".join(experiments_section(rows)) assert "| unknown |" in rendered and "| bbbbbbb |" in rendered diff --git a/tests/test_verifier.py b/tests/test_verifier.py index 694714fc..9f94cf4b 100644 --- a/tests/test_verifier.py +++ b/tests/test_verifier.py @@ -280,8 +280,9 @@ def test_verify_brief_accepts_a_measured_composite_but_blocks_a_bare_bundle() -> from outerloop.verifier import build_verify_agent_brief brief = " ".join(build_verify_agent_brief(make_pr(), "contract").split()) - assert "each component's isolated measurement" in brief + assert "every component's isolated measurement" in brief + assert "the kernel's Experiments table (not only in the report's prose)" in brief assert "the combination is what the orchestrator measured" in brief - assert "A bundle that reports no isolated measurement for each component stays blocked" in brief + assert "does not show such a measurement for every component stays blocked" in brief # the author's guidance names the same route, beside the one-hypothesis rule assert "each part's isolated measurement and the combination's" in AUTHOR