Review panel accepts measured composite changes (#489)

CHANGELOG.md

@@ -6,6 +6,7 @@ Versions follow [SemVer](https://semver.org).
 
 ## [Unreleased]
 
+- The pre-PR panel accepts a composite change — a PR that carries several changes — when the report gives each component's isolated measurement and the combination's under the same protocol and the combination is what the kernel measured. A bundle without those isolated measurements stays blocked (#489).
 - Claude sessions can use a subscription token from `claude setup-token` (`sk-ant-oat...`). The harness sends it as `CLAUDE_CODE_OAUTH_TOKEN`, not `ANTHROPIC_API_KEY`, and `init` accepts it. A panel judge on a token runs `claude --safe-mode`, because `--bare` cannot read OAuth tokens. Both modes ignore the checkout's CLAUDE.md, hooks, plugins and MCP servers. The pinned Claude Code release (2.1.285) has `--safe-mode`.
 - A session now hears once when a live sibling states a new or changed hypothesis (#360). The note names the sibling, quotes its hypothesis, and points to `siblings` for its launches and score. It is sent only when `channels.siblings` is on. Upgrading: the note uses the existing `note` kind, so an older kernel shows it as ordinary inbox text.
 - A skills-only plugin for Claude Code and Codex sets up Outerloop, writes and validates the contract, and operates runs from the agent (#562). Install it with `/plugin marketplace add outerloop-science/outerloop` or `codex plugin marketplace add outerloop-science/outerloop`. Its setup skill offers a Claude subscription token from `claude setup-token` as the author credential.

src/outerloop/attempt.py

@@ -105,6 +105,7 @@
     attempt_once,
     benchmark_floor,
     clears_min_delta,
+    experiments_section,
     out_of_scope,
     pr_body,
     resume_attempt,
@@ -3917,20 +3918,23 @@ def _panel_lenses_from_args(
     return tuple(lenses), tuple(dict.fromkeys(secrets))
 
 
-def _panel_claim_body(benchmark: str, baseline: float, candidate: float, report: str) -> str:
-    """The synthetic claim the panel judges: the kernel's measured claim plus
-    the author's report and nothing else — this repository's guidance travels
-    in the judges' own prompt (run_panel's `guidance`), never inside this
-    body, which every judge prompt fences as untrusted data. The report
-    budget subtracts the actual header so the body never outgrows the cap
-    the verify prompt fences it under."""
+def _panel_claim_body(
+    benchmark: str, baseline: float, candidate: float, report: str, experiments: str = ""
+) -> str:
+    """The synthetic claim the panel judges: the kernel's measured claim, the
+    run's launches from the kernel's ledger, and the author's report — this
+    repository's guidance travels in the judges' own prompt (run_panel's
+    `guidance`), never inside this body, which every judge prompt fences as
+    untrusted data. The report budget subtracts the header and the ledger
+    table so the body never outgrows the cap the verify prompt fences it under."""
     header = (
         f"Automated improvement claim (pre-PR): {benchmark} "
-        f"{baseline} -> {candidate}, measured by the orchestrator.\n\n"
+        f"{baseline} -> {candidate}, measured by the orchestrator.\n"
+        f"{experiments}\n\n"
         f"## Research report\n\n*Session prose, written before "
         f"the orchestrator measured.*\n\n"
     )
-    return f"{header}{report[: MAX_CLAIM_CHARS - len(header)]}"
+    return f"{header}{report[: max(0, MAX_CLAIM_CHARS - len(header))]}"
 
 
 def build_panel_runner(
@@ -3968,7 +3972,13 @@ def build_panel_runner(
     reads = {"n": start_round}
     render_claim = claim_body or (
         lambda baseline, candidate, report: _panel_claim_body(
-            benchmark, baseline, candidate, report
+            benchmark,
+            baseline,
+            candidate,
+            report,
+            # the run's own launches, recorded by the kernel: the evidence a
+            # composite change's component measurements must appear in
+            "\n".join(experiments_section(experiments_rows(run_dir))),
         )
     )
 

src/outerloop/guidance_pack.py

@@ -16,7 +16,9 @@
 
 AUTHOR = (
     "Form ONE concrete hypothesis for why the benchmark underperforms "
-    "before you implement, and implement that one.\n\n"
+    "before you implement, and implement that one. A combination whose parts "
+    "each fall below the improvement floor may ship together if the PR "
+    "reports each part's isolated measurement and the combination's.\n\n"
     "Read past reports critically: a negative settles only what was actually "
     "run. One point in a parameter space, an eval that hit its walltime, or "
     "an infrastructure failure does not close an idea — vary what went "

src/outerloop/orchestrator.py

@@ -2786,7 +2786,7 @@ def _ended(row: dict[str, Any]) -> str:
     return how
 
 
-def _experiments_section(rows: list[dict[str, Any]]) -> list[str]:
+def experiments_section(rows: list[dict[str, Any]]) -> list[str]:
     """The run's launches as the ledger recorded them: what ran, how each job
     ended, what it printed last. Empty when the run launched nothing."""
     if not rows:
@@ -2908,7 +2908,7 @@ def pr_body(
             "## Research report",
             "",
             *report_lines,
-            *_experiments_section(experiments or []),
+            *experiments_section(experiments or []),
             "",
             "## Measured",
             "",
@@ -2939,7 +2939,7 @@ def pr_body(
                 "## Research report",
                 "",
                 redact(result.submit_report, redact_secrets)[:MAX_REPORT_BODY],
-                *_experiments_section(experiments or []),
+                *experiments_section(experiments or []),
                 "",
                 "## Self-reported values",
                 "",

src/outerloop/verifier.py

@@ -154,7 +154,17 @@ def gather_thread(
 - unsupported-claim: statements in the report that the PR's own evidence
   contradicts
 - measurement-gap: noise floors, seeds, or protocol issues in the
-  measurement itself
+  measurement itself, including a bundle that does not show an isolated
+  measurement for every one of its components
+
+A composite change — a pull request that carries several changes — is
+acceptable when every component's isolated measurement and the
+combination's were run under the same protocol, those component runs appear
+among the run's own launches in the kernel's Experiments table (not only in
+the report's prose), and the combination is what the orchestrator measured.
+A bundle that does not show such a measurement for every component stays
+blocked: the combination's number alone does not show what produced it.
+Check the evidence; do not re-run anything.
 
 Use the contract (the rules), the ruler source (how the eval actually
 works), the claimed numbers, and the agent's own report. The report's
@@ -177,8 +187,9 @@ def gather_thread(
 contradiction between the claim and the PR's OWN evidence (a mechanism
 story the report's own numbers refute, or an improvement that a trivial
 baseline in the report's own tables beats — true even when the measured
-delta itself is real), or a scope or measurement fact that makes the
-number mislead. Anything else — every research-taste judgment, including
+delta itself is real), a bundle that does not show a kernel-recorded
+isolated measurement for every component, or a scope or measurement fact that makes the number
+mislead. Anything else — every research-taste judgment, including
 anything set by repository guidance, when provided below — is advisory:
 `blocking` false.
 

tests/test_attempt.py

@@ -4970,6 +4970,9 @@ def test_guidance_layer_shapes_brief_and_claim(mode: str) -> None:
     )
     claim = _panel_claim_body("tsp", 13.8, 13.1, "report text")
     assert "measured by the orchestrator" in claim
+    # the kernel's launch ledger rides ahead of the author's prose (#489)
+    ledgered = _panel_claim_body("tsp", 13.8, 13.1, "report text", "## Experiments\n| row |")
+    assert ledgered.index("## Experiments") < ledgered.index("## Research report")
     # the header and wrapper are priced into the report budget, so a long
     # report cannot push the body past the fence the verify prompt caps
     long_claim = _panel_claim_body("tsp", 13.8, 13.1, "r" * (2 * MAX_CLAIM_CHARS))

tests/test_launchlog.py

@@ -161,15 +161,15 @@ def test_experiments_rows_come_from_the_ledger_including_each_jobs_last_line(
 
 def test_legacy_commit_tolerance_and_retry(tmp_path: Path) -> None:
     from outerloop.launchlog import experiments_rows
-    from outerloop.orchestrator import _experiments_section
+    from outerloop.orchestrator import experiments_section
 
     legacy = Path(__file__).parent / "fixtures" / "launches-before-provenance.jsonl"
     (tmp_path / LEDGER).write_bytes(legacy.read_bytes())
     launch = (Launch(name="probe", command="x", minutes=5),)
     for _ in range(2):
         rows = experiments_rows(tmp_path)
         assert rows[0]["commit"] == ""
-        assert "| unknown |" in "\n".join(_experiments_section(rows))
+        assert "| unknown |" in "\n".join(experiments_section(rows))
         # Re-parking a pre-upgrade launch must not invent provenance.
         append_submitted(tmp_path, sleep=1, launches=launch, job_ids=["1"], at=30, commit="a" * 40)
         assert (tmp_path / LEDGER).read_bytes() == legacy.read_bytes()
@@ -180,5 +180,5 @@ def test_legacy_commit_tolerance_and_retry(tmp_path: Path) -> None:
         append_submitted(tmp_path, sleep=2, launches=launch, job_ids=["2"], at=40, commit="b" * 40)
     rows = experiments_rows(tmp_path)
     assert [r["commit"] for r in rows] == ["", "b" * 40]
-    rendered = "\n".join(_experiments_section(rows))
+    rendered = "\n".join(experiments_section(rows))
     assert "| unknown |" in rendered and "| bbbbbbb |" in rendered

tests/test_verifier.py

@@ -270,3 +270,19 @@ def test_taste_categories_live_only_in_the_panel_pack() -> None:
     # the standards themselves are the pack's, carried as advisory guidance
     assert "documents each change's own effect" in PANEL
     assert "picture of the landscape" in PANEL
+
+
+def test_verify_brief_accepts_a_measured_composite_but_blocks_a_bare_bundle() -> None:
+    # Issue #489: the panel accepts a combination when each component was measured in
+    # isolation under the same protocol and the combination is what the
+    # orchestrator gated; a bundle without those measurements stays blocked.
+    from outerloop.guidance_pack import AUTHOR
+    from outerloop.verifier import build_verify_agent_brief
+
+    brief = " ".join(build_verify_agent_brief(make_pr(), "contract").split())
+    assert "every component's isolated measurement" in brief
+    assert "the kernel's Experiments table (not only in the report's prose)" in brief
+    assert "the combination is what the orchestrator measured" in brief
+    assert "does not show such a measurement for every component stays blocked" in brief
+    # the author's guidance names the same route, beside the one-hypothesis rule
+    assert "each part's isolated measurement and the combination's" in AUTHOR