From f212b59743452346f9d56cf9024716689309ba14 Mon Sep 17 00:00:00 2001 From: Mengye Ren Date: Sun, 25 Oct 2026 10:32:33 -0400 Subject: [PATCH 1/3] Report-only number check in the PR body (trajectory step 4) --- CHANGELOG.md | 1 + docs/design/trajectory.md | 7 ++- src/outerloop/attempt.py | 11 +++++ src/outerloop/number_check.py | 85 +++++++++++++++++++++++++++++++++++ src/outerloop/orchestrator.py | 35 ++++++++++++++- tests/test_orchestrator.py | 41 +++++++++++++++++ 6 files changed, 178 insertions(+), 2 deletions(-) create mode 100644 src/outerloop/number_check.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 75e066d1..b5747e79 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ Versions follow [SemVer](https://semver.org). ## [Unreleased] +- The PR body gains a report-only Number check that lists values in the author's report matching no `measure` event or Experiments result when rounded to the report's decimals; nothing is blocked and no trajectory event is added. - Session end records each tool call from the retained Claude or Codex log as a `tool.call` trajectory event, once per call even when a session resumes. `outerloop trajectory` and a new Process section in the PR body show counts by tool. Older kernels ignore these events. - `outerloop status` shows a run's priced session cost as a lower bound when some legs are unpriced (a wake killed mid-turn leaves no usage event), for example `session-cost>=$4.2000 (1 unpriced)`, instead of hiding the priced part behind `unknown`; the status row and JSON gain the priced-leg count (#516). - The pre-PR panel accepts a composite change — a PR that carries several changes — when the report gives each component's isolated measurement and the combination's under the same protocol and the combination is what the kernel measured. A bundle without those isolated measurements stays blocked (#489). diff --git a/docs/design/trajectory.md b/docs/design/trajectory.md index 73e48f8c..62587f95 100644 --- a/docs/design/trajectory.md +++ b/docs/design/trajectory.md @@ -52,4 +52,9 @@ rollout JSONL are parsed host-side, bounded to 500 calls per session, and Hermes or unknown formats yield nothing. Each event carries the tool name, repo-relative files, a bounded shell command and, when the log has them, the exit code and outcome — never file contents or tool output. `outerloop trajectory` counts them -by tool and the PR body summarizes the top tools. Attestation is a later step. +by tool and the PR body summarizes the top tools. The PR body also runs a +report-only number check: numbers on report lines that mention the metric, +baseline or candidate are compared with every `measure` event's values and the +Experiments result cells, after rounding to the report's decimals, and the +unmatched ones are listed under "## Number check". Nothing is blocked and no +event is written for it. Attestation is a later step. diff --git a/src/outerloop/attempt.py b/src/outerloop/attempt.py index efc57156..35aa35e3 100644 --- a/src/outerloop/attempt.py +++ b/src/outerloop/attempt.py @@ -94,6 +94,7 @@ from outerloop.limits import bound_limits, capped_limits, clamp_bound_limits, effective_limits from outerloop.markers import has_marker, marker from outerloop.measure import DispatchedMeasurer, DispatchSettings, measured_image +from outerloop.number_check import measured_values as measured_from_events from outerloop.orchestrator import ( MAX_REPORT_BODY, AttemptResult, @@ -168,6 +169,14 @@ def tool_call_counts(run_dir: Path) -> dict[str, int]: return {} +def measure_values(run_dir: Path) -> list[float]: + """The run's recorded measure values; an unreadable chain reads as none.""" + try: + return measured_from_events(read_trajectory(run_dir)) + except OSError: + return [] + + def _pr_number(pr_url: str) -> int: tail = pr_url.rstrip("/").rsplit("/", 1)[-1] if not tail.isdigit(): @@ -4735,6 +4744,8 @@ def refuse( experiments=experiments_rows(run_dir), base_sha=base_sha, tool_calls=tool_call_counts(run_dir), + metric=bench.metric, + measured_values=measure_values(run_dir), ) body += f"\n\n{progress_link(config.target, github)}\n" if issue_number: diff --git a/src/outerloop/number_check.py b/src/outerloop/number_check.py new file mode 100644 index 00000000..876f51e1 --- /dev/null +++ b/src/outerloop/number_check.py @@ -0,0 +1,85 @@ +"""Report-only check that numbers in an author's report were measured. + +Nothing here records events or changes a gate or publish decision. +""" + +from __future__ import annotations + +import re +from collections.abc import Iterable + +# A number not embedded in a word or identifier, not part of a "#123" reference. +_NUMBER = re.compile(r"(? list[float]: + """Every measure event's baseline and candidate, in event order.""" + values: list[float] = [] + for event in events: + data = event.get("data") + if event.get("type") != "measure" or not isinstance(data, dict): + continue + for key in ("baseline", "candidate"): + value = data.get(key) + if isinstance(value, (int, float)) and not isinstance(value, bool): + values.append(float(value)) + return values + + +def unmatched_numbers( + report: str, + metric: str, + measured: Iterable[float], + experiment_results: Iterable[str], +) -> list[str]: + """Numbers in the report that look like metric values but match no measurement. + + A line counts when it mentions the metric name (case-insensitive), or + "baseline", or "candidate". On those lines every number is a candidate + except integers that are clearly not values: below 10 (step counts, list + indices), 1900-2099 (years), on a line mentioning a seed, followed by "%", + or part of a "#123" reference. A candidate matches when, rounded to the + decimals it was written with, it equals a measured value, the gap between + two measured values (a reported improvement), or a number in an + Experiments result cell; the sign is ignored. Returns unmatched + candidates in first-seen order. + """ + values = list(measured) + known = values + [abs(a - b) for a in values for b in values if a != b] + for cell in experiment_results: + known.extend(float(m.group(1)) for m in _NUMBER.finditer(cell)) + metric_name = metric.strip().lower() + unmatched: list[str] = [] + for line in report.splitlines(): + lower = line.lower() + if not ( + "baseline" in lower or "candidate" in lower or (metric_name and metric_name in lower) + ): + continue + for match in _NUMBER.finditer(line): + token = match.group(1) + if _ignored(line, match) or _matches(token, known): + continue + if token not in unmatched: + unmatched.append(token) + return unmatched + + +def _ignored(line: str, match: re.Match[str]) -> bool: + """Whether a number is clearly not a reported metric value.""" + token = match.group(1) + if line[match.end() : match.end() + 1] == "%": + return True + if "." in token: + return False + value = int(token) + if value < 10 or 1900 <= value <= 2099: + return True + return "seed" in line.lower() + + +def _matches(token: str, known: Iterable[float]) -> bool: + """Equality after rounding both sides to the report's decimals, ignoring sign.""" + token = token.lstrip("-") + decimals = len(token.split(".")[1]) if "." in token else 0 + return any(f"{abs(value):.{decimals}f}" == token for value in known) diff --git a/src/outerloop/orchestrator.py b/src/outerloop/orchestrator.py index 82d20c49..b0eb181d 100644 --- a/src/outerloop/orchestrator.py +++ b/src/outerloop/orchestrator.py @@ -29,6 +29,7 @@ from outerloop.containment import executable_roots, seatbelt_available, seatbelt_wrap from outerloop.image import apptainer_from_env +from outerloop.number_check import unmatched_numbers from outerloop.trajectory import file_refs from outerloop.trajectory import record as record_event @@ -2835,6 +2836,20 @@ def process_section(tool_calls: dict[str, int] | None) -> list[str]: return ["", "## Process", "", f"{total} tool {calls} from the run's trajectory: {top}."] +def number_check_section(numbers: list[str]) -> list[str]: + """Report-only list of report values that match no recorded measurement.""" + if not numbers: + return [] + return [ + "", + "## Number check", + "", + "These values in the report match no measurement recorded for this run: " + + ", ".join(numbers) + + ". Report-only; nothing was blocked.", + ] + + def pr_body( result: AttemptResult, config: RunConfig, @@ -2844,6 +2859,8 @@ def pr_body( *, base_sha: str = "", tool_calls: dict[str, int] | None = None, + metric: str = "", + measured_values: Sequence[float] | None = None, ) -> str: """The PR body for an improved run: the author's report, the experiments the run actually ran (from the launch ledger), the measured table, and @@ -2851,7 +2868,8 @@ def pr_body( Human surfaces render at the benchmark's conventional precision; full precision lives only in results/leader.json, and every - comparison runs on full floats. + comparison runs on full floats. When `measured_values` is given, a + report-only check lists report values that match no measurement. """ from outerloop.progress import fmt_metric @@ -2892,6 +2910,19 @@ def pr_body( if result.panel_transcript else [] ) + report_text = result.submit_report or (result.session.final_text if result.session else "") + number_lines = ( + number_check_section( + unmatched_numbers( + report_text, + metric, + measured_values, + [str(row.get("result") or "") for row in experiments or []], + ) + ) + if measured_values is not None + else [] + ) if result.submit_report: report_lines = [ "*Written by the author at submit, before the orchestrator measured; the " @@ -2925,6 +2956,7 @@ def pr_body( *report_lines, *experiments_section(experiments or []), *process_section(tool_calls), + *number_lines, "", "## Measured", "", @@ -2957,6 +2989,7 @@ def pr_body( redact(result.submit_report, redact_secrets)[:MAX_REPORT_BODY], *experiments_section(experiments or []), *process_section(tool_calls), + *number_lines, "", "## Self-reported values", "", diff --git a/tests/test_orchestrator.py b/tests/test_orchestrator.py index fbb44b58..f8e286c0 100644 --- a/tests/test_orchestrator.py +++ b/tests/test_orchestrator.py @@ -1807,6 +1807,47 @@ def test_pr_body_leads_with_the_report_and_lists_the_experiments() -> None: assert "Session prose" in plain and "## Experiments" not in plain +def test_unmatched_numbers_ignores_years_seeds_and_measurable_values() -> None: + from outerloop.number_check import unmatched_numbers + + report = ( + "Baseline 0.512 in 2024, candidate 0.501 from seed 8271 at step 4. " + "The sweep cell gave 3.28; the candidate curve peaked at 13.4." + ) + assert unmatched_numbers(report, "mean_tour_length", [0.5123, 0.5014], ['{"val": 3.28}']) == [ + "13.4" + ] + # a reported improvement is the gap between measurements, with either sign + gain = "Baseline 0.512, candidate 0.501: a gain of 0.011 (delta -0.011)." + assert unmatched_numbers(gain, "mean_tour_length", [0.5123, 0.5014], []) == [] + + +def test_pr_body_number_check_is_report_only() -> None: + from outerloop.orchestrator import AttemptResult + + result = AttemptResult( + outcome="improved", + baseline=0.512, + candidate=0.501, + submit_report="Baseline 0.512, candidate 0.501; the curve peaked at 13.4.", + ) + config = RunConfig(target="org/pilot", benchmark="tsp") + body = pr_body(result, config, (), metric="mean_tour_length", measured_values=[0.5123, 0.5014]) + assert "## Number check" in body + assert ( + "match no measurement recorded for this run: 13.4. " + "Report-only; nothing was blocked." in body + ) + matched = pr_body( + result, + config, + (), + metric="mean_tour_length", + measured_values=[0.5123, 0.5014, 13.4], + ) + assert "## Number check" not in matched + + def test_submit_without_report_is_accepted(tmp_path: Path) -> None: _write_syscall(tmp_path, {"launches": [], "submit": True}) result, harness, _ = run_climb( From c46a11a42a8239070917cfe7a16ede57e837e558 Mon Sep 17 00:00:00 2001 From: Mengye Ren Date: Sun, 25 Oct 2026 10:42:25 -0400 Subject: [PATCH 2/3] Number check reads the redacted, truncated report; caps its list; handles scientific notation, spaced percentages and zero gaps --- src/outerloop/number_check.py | 13 ++++++++----- src/outerloop/orchestrator.py | 14 +++++++++++--- tests/test_orchestrator.py | 14 ++++++++++++++ 3 files changed, 33 insertions(+), 8 deletions(-) diff --git a/src/outerloop/number_check.py b/src/outerloop/number_check.py index 876f51e1..54917b49 100644 --- a/src/outerloop/number_check.py +++ b/src/outerloop/number_check.py @@ -9,7 +9,7 @@ from collections.abc import Iterable # A number not embedded in a word or identifier, not part of a "#123" reference. -_NUMBER = re.compile(r"(? list[float]: @@ -45,7 +45,7 @@ def unmatched_numbers( candidates in first-seen order. """ values = list(measured) - known = values + [abs(a - b) for a in values for b in values if a != b] + known = values + [abs(a - b) for a in values for b in values] for cell in experiment_results: known.extend(float(m.group(1)) for m in _NUMBER.finditer(cell)) metric_name = metric.strip().lower() @@ -68,9 +68,9 @@ def unmatched_numbers( def _ignored(line: str, match: re.Match[str]) -> bool: """Whether a number is clearly not a reported metric value.""" token = match.group(1) - if line[match.end() : match.end() + 1] == "%": + if line[match.end() :].lstrip(" ")[:1] == "%": return True - if "." in token: + if "." in token or "e" in token.lower(): return False value = int(token) if value < 10 or 1900 <= value <= 2099: @@ -81,5 +81,8 @@ def _ignored(line: str, match: re.Match[str]) -> bool: def _matches(token: str, known: Iterable[float]) -> bool: """Equality after rounding both sides to the report's decimals, ignoring sign.""" token = token.lstrip("-") - decimals = len(token.split(".")[1]) if "." in token else 0 + mantissa, _, exponent = token.lower().partition("e") + decimals = len(mantissa.split(".")[1]) if "." in mantissa else 0 + if exponent: # scientific notation: round to the mantissa's digits + return any(float(f"{abs(value):.{decimals}e}") == float(token) for value in known) return any(f"{abs(value):.{decimals}f}" == token for value in known) diff --git a/src/outerloop/orchestrator.py b/src/outerloop/orchestrator.py index b0eb181d..e11001b6 100644 --- a/src/outerloop/orchestrator.py +++ b/src/outerloop/orchestrator.py @@ -2836,17 +2836,21 @@ def process_section(tool_calls: dict[str, int] | None) -> list[str]: return ["", "## Process", "", f"{total} tool {calls} from the run's trajectory: {top}."] +MAX_NUMBER_CHECK = 20 + + def number_check_section(numbers: list[str]) -> list[str]: """Report-only list of report values that match no recorded measurement.""" if not numbers: return [] + shown = ", ".join(numbers[:MAX_NUMBER_CHECK]) + more = f" and {len(numbers) - MAX_NUMBER_CHECK} more" if len(numbers) > MAX_NUMBER_CHECK else "" return [ "", "## Number check", "", "These values in the report match no measurement recorded for this run: " - + ", ".join(numbers) - + ". Report-only; nothing was blocked.", + f"{shown}{more}. Report-only; nothing was blocked.", ] @@ -2910,7 +2914,11 @@ def pr_body( if result.panel_transcript else [] ) - report_text = result.submit_report or (result.session.final_text if result.session else "") + # the same redacted, truncated text the body shows, never the raw report + report_text = redact( + result.submit_report or (result.session.final_text if result.session else ""), + redact_secrets, + )[:MAX_REPORT_BODY] number_lines = ( number_check_section( unmatched_numbers( diff --git a/tests/test_orchestrator.py b/tests/test_orchestrator.py index f8e286c0..2dfc5b84 100644 --- a/tests/test_orchestrator.py +++ b/tests/test_orchestrator.py @@ -1820,6 +1820,9 @@ def test_unmatched_numbers_ignores_years_seeds_and_measurable_values() -> None: # a reported improvement is the gap between measurements, with either sign gain = "Baseline 0.512, candidate 0.501: a gain of 0.011 (delta -0.011)." assert unmatched_numbers(gain, "mean_tour_length", [0.5123, 0.5014], []) == [] + # scientific notation, a spaced percentage, and a zero gap + sci = "Candidate loss 1.2e-3 vs baseline 1.20e-3; candidate 50 % faster; gap 0.0." + assert unmatched_numbers(sci, "loss", [0.0012, 0.0012], []) == [] def test_pr_body_number_check_is_report_only() -> None: @@ -1846,6 +1849,17 @@ def test_pr_body_number_check_is_report_only() -> None: measured_values=[0.5123, 0.5014, 13.4], ) assert "## Number check" not in matched + # the check reads the redacted text the body shows: a secret's digits never surface + leaky = AttemptResult( + outcome="improved", + baseline=0.512, + candidate=0.501, + submit_report="Baseline 0.512 with run-77814, candidate 0.501.", + ) + redacted = pr_body( + leaky, config, ("run-77814",), metric="mean_tour_length", measured_values=[0.5123, 0.5014] + ) + assert "77814" not in redacted def test_submit_without_report_is_accepted(tmp_path: Path) -> None: From e4b026024e809b9d8df81b386650efc34ae3e3f0 Mon Sep 17 00:00:00 2001 From: Mengye Ren Date: Sun, 25 Oct 2026 13:21:51 -0400 Subject: [PATCH 3/3] Number check skips only seeds and #references; percentages match the relative gap --- src/outerloop/number_check.py | 38 +++++++++++++++++------------------ tests/test_orchestrator.py | 23 +++++++++++++-------- 2 files changed, 33 insertions(+), 28 deletions(-) diff --git a/src/outerloop/number_check.py b/src/outerloop/number_check.py index 54917b49..ebbf89c6 100644 --- a/src/outerloop/number_check.py +++ b/src/outerloop/number_check.py @@ -10,6 +10,8 @@ # A number not embedded in a word or identifier, not part of a "#123" reference. _NUMBER = re.compile(r"(? list[float]: @@ -35,17 +37,18 @@ def unmatched_numbers( """Numbers in the report that look like metric values but match no measurement. A line counts when it mentions the metric name (case-insensitive), or - "baseline", or "candidate". On those lines every number is a candidate - except integers that are clearly not values: below 10 (step counts, list - indices), 1900-2099 (years), on a line mentioning a seed, followed by "%", - or part of a "#123" reference. A candidate matches when, rounded to the - decimals it was written with, it equals a measured value, the gap between - two measured values (a reported improvement), or a number in an - Experiments result cell; the sign is ignored. Returns unmatched - candidates in first-seen order. + "baseline", or "candidate". On those lines every number is checked except + the two that cannot be a measured value: a seed ("seed 8271", "seeds 3, + 4") and a "#123" reference. A number matches when, rounded to the decimals + it was written with and ignoring sign, it equals a measured value, the gap + between two measured values, or a number in an Experiments result cell; a + percentage matches the relative gap between two measured values. Returns + unmatched numbers in first-seen order. Report-only, so a rare false flag + is cheaper than hiding a claim. """ values = list(measured) known = values + [abs(a - b) for a in values for b in values] + percents = [abs(a - b) / abs(b) * 100 for a in values for b in values if b] for cell in experiment_results: known.extend(float(m.group(1)) for m in _NUMBER.finditer(cell)) metric_name = metric.strip().lower() @@ -58,24 +61,19 @@ def unmatched_numbers( continue for match in _NUMBER.finditer(line): token = match.group(1) - if _ignored(line, match) or _matches(token, known): + if _is_seed(line, match): + continue + percent = line[match.end() :].lstrip(" ")[:1] == "%" + if _matches(token, percents if percent else known): continue if token not in unmatched: unmatched.append(token) return unmatched -def _ignored(line: str, match: re.Match[str]) -> bool: - """Whether a number is clearly not a reported metric value.""" - token = match.group(1) - if line[match.end() :].lstrip(" ")[:1] == "%": - return True - if "." in token or "e" in token.lower(): - return False - value = int(token) - if value < 10 or 1900 <= value <= 2099: - return True - return "seed" in line.lower() +def _is_seed(line: str, match: re.Match[str]) -> bool: + """A number in a "seed 8271" / "seeds 3, 4" / "seed=8271" phrase.""" + return any(s.start(1) <= match.start(1) < s.end(1) for s in _SEEDS.finditer(line)) def _matches(token: str, known: Iterable[float]) -> bool: diff --git a/tests/test_orchestrator.py b/tests/test_orchestrator.py index 2dfc5b84..d3865a07 100644 --- a/tests/test_orchestrator.py +++ b/tests/test_orchestrator.py @@ -1807,21 +1807,28 @@ def test_pr_body_leads_with_the_report_and_lists_the_experiments() -> None: assert "Session prose" in plain and "## Experiments" not in plain -def test_unmatched_numbers_ignores_years_seeds_and_measurable_values() -> None: +def test_unmatched_numbers_skips_only_seeds_and_references() -> None: from outerloop.number_check import unmatched_numbers + # only the seed and the #12 reference are skipped; small numbers and years are checked report = ( - "Baseline 0.512 in 2024, candidate 0.501 from seed 8271 at step 4. " + "Baseline 0.512 (see #12) in 2024, candidate 0.501 from seed 8271 at step 4. " "The sweep cell gave 3.28; the candidate curve peaked at 13.4." ) assert unmatched_numbers(report, "mean_tour_length", [0.5123, 0.5014], ['{"val": 3.28}']) == [ - "13.4" + "2024", + "4", + "13.4", ] - # a reported improvement is the gap between measurements, with either sign - gain = "Baseline 0.512, candidate 0.501: a gain of 0.011 (delta -0.011)." - assert unmatched_numbers(gain, "mean_tour_length", [0.5123, 0.5014], []) == [] - # scientific notation, a spaced percentage, and a zero gap - sci = "Candidate loss 1.2e-3 vs baseline 1.20e-3; candidate 50 % faster; gap 0.0." + # an integer metric on a line naming seeds is still checked + steps = "Candidate steps_to_target with seeds 3, 4 and 5 was 1840; baseline 2176." + assert unmatched_numbers(steps, "steps_to_target", [2176, 1856], []) == ["1840"] + # a reported gain is the gap between measurements, with either sign; a + # percentage is the relative gap + gain = "Baseline 2176, candidate 1856: a gain of 320 (delta -320), 14.7 % fewer, not 30%." + assert unmatched_numbers(gain, "steps_to_target", [2176, 1856], []) == ["30"] + # scientific notation and a zero gap + sci = "Candidate loss 1.2e-3 vs baseline 1.20e-3; gap 0.0." assert unmatched_numbers(sci, "loss", [0.0012, 0.0012], []) == []