diff --git a/CHANGELOG.md b/CHANGELOG.md index 75e066d1..b5747e79 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ Versions follow [SemVer](https://semver.org). ## [Unreleased] +- The PR body gains a report-only Number check that lists values in the author's report matching no `measure` event or Experiments result when rounded to the report's decimals; nothing is blocked and no trajectory event is added. - Session end records each tool call from the retained Claude or Codex log as a `tool.call` trajectory event, once per call even when a session resumes. `outerloop trajectory` and a new Process section in the PR body show counts by tool. Older kernels ignore these events. - `outerloop status` shows a run's priced session cost as a lower bound when some legs are unpriced (a wake killed mid-turn leaves no usage event), for example `session-cost>=$4.2000 (1 unpriced)`, instead of hiding the priced part behind `unknown`; the status row and JSON gain the priced-leg count (#516). - The pre-PR panel accepts a composite change — a PR that carries several changes — when the report gives each component's isolated measurement and the combination's under the same protocol and the combination is what the kernel measured. A bundle without those isolated measurements stays blocked (#489). diff --git a/docs/design/trajectory.md b/docs/design/trajectory.md index 73e48f8c..62587f95 100644 --- a/docs/design/trajectory.md +++ b/docs/design/trajectory.md @@ -52,4 +52,9 @@ rollout JSONL are parsed host-side, bounded to 500 calls per session, and Hermes or unknown formats yield nothing. Each event carries the tool name, repo-relative files, a bounded shell command and, when the log has them, the exit code and outcome — never file contents or tool output. `outerloop trajectory` counts them -by tool and the PR body summarizes the top tools. Attestation is a later step. +by tool and the PR body summarizes the top tools. The PR body also runs a +report-only number check: numbers on report lines that mention the metric, +baseline or candidate are compared with every `measure` event's values and the +Experiments result cells, after rounding to the report's decimals, and the +unmatched ones are listed under "## Number check". Nothing is blocked and no +event is written for it. Attestation is a later step. diff --git a/src/outerloop/attempt.py b/src/outerloop/attempt.py index efc57156..35aa35e3 100644 --- a/src/outerloop/attempt.py +++ b/src/outerloop/attempt.py @@ -94,6 +94,7 @@ from outerloop.limits import bound_limits, capped_limits, clamp_bound_limits, effective_limits from outerloop.markers import has_marker, marker from outerloop.measure import DispatchedMeasurer, DispatchSettings, measured_image +from outerloop.number_check import measured_values as measured_from_events from outerloop.orchestrator import ( MAX_REPORT_BODY, AttemptResult, @@ -168,6 +169,14 @@ def tool_call_counts(run_dir: Path) -> dict[str, int]: return {} +def measure_values(run_dir: Path) -> list[float]: + """The run's recorded measure values; an unreadable chain reads as none.""" + try: + return measured_from_events(read_trajectory(run_dir)) + except OSError: + return [] + + def _pr_number(pr_url: str) -> int: tail = pr_url.rstrip("/").rsplit("/", 1)[-1] if not tail.isdigit(): @@ -4735,6 +4744,8 @@ def refuse( experiments=experiments_rows(run_dir), base_sha=base_sha, tool_calls=tool_call_counts(run_dir), + metric=bench.metric, + measured_values=measure_values(run_dir), ) body += f"\n\n{progress_link(config.target, github)}\n" if issue_number: diff --git a/src/outerloop/number_check.py b/src/outerloop/number_check.py new file mode 100644 index 00000000..ebbf89c6 --- /dev/null +++ b/src/outerloop/number_check.py @@ -0,0 +1,86 @@ +"""Report-only check that numbers in an author's report were measured. + +Nothing here records events or changes a gate or publish decision. +""" + +from __future__ import annotations + +import re +from collections.abc import Iterable + +# A number not embedded in a word or identifier, not part of a "#123" reference. +_NUMBER = re.compile(r"(? list[float]: + """Every measure event's baseline and candidate, in event order.""" + values: list[float] = [] + for event in events: + data = event.get("data") + if event.get("type") != "measure" or not isinstance(data, dict): + continue + for key in ("baseline", "candidate"): + value = data.get(key) + if isinstance(value, (int, float)) and not isinstance(value, bool): + values.append(float(value)) + return values + + +def unmatched_numbers( + report: str, + metric: str, + measured: Iterable[float], + experiment_results: Iterable[str], +) -> list[str]: + """Numbers in the report that look like metric values but match no measurement. + + A line counts when it mentions the metric name (case-insensitive), or + "baseline", or "candidate". On those lines every number is checked except + the two that cannot be a measured value: a seed ("seed 8271", "seeds 3, + 4") and a "#123" reference. A number matches when, rounded to the decimals + it was written with and ignoring sign, it equals a measured value, the gap + between two measured values, or a number in an Experiments result cell; a + percentage matches the relative gap between two measured values. Returns + unmatched numbers in first-seen order. Report-only, so a rare false flag + is cheaper than hiding a claim. + """ + values = list(measured) + known = values + [abs(a - b) for a in values for b in values] + percents = [abs(a - b) / abs(b) * 100 for a in values for b in values if b] + for cell in experiment_results: + known.extend(float(m.group(1)) for m in _NUMBER.finditer(cell)) + metric_name = metric.strip().lower() + unmatched: list[str] = [] + for line in report.splitlines(): + lower = line.lower() + if not ( + "baseline" in lower or "candidate" in lower or (metric_name and metric_name in lower) + ): + continue + for match in _NUMBER.finditer(line): + token = match.group(1) + if _is_seed(line, match): + continue + percent = line[match.end() :].lstrip(" ")[:1] == "%" + if _matches(token, percents if percent else known): + continue + if token not in unmatched: + unmatched.append(token) + return unmatched + + +def _is_seed(line: str, match: re.Match[str]) -> bool: + """A number in a "seed 8271" / "seeds 3, 4" / "seed=8271" phrase.""" + return any(s.start(1) <= match.start(1) < s.end(1) for s in _SEEDS.finditer(line)) + + +def _matches(token: str, known: Iterable[float]) -> bool: + """Equality after rounding both sides to the report's decimals, ignoring sign.""" + token = token.lstrip("-") + mantissa, _, exponent = token.lower().partition("e") + decimals = len(mantissa.split(".")[1]) if "." in mantissa else 0 + if exponent: # scientific notation: round to the mantissa's digits + return any(float(f"{abs(value):.{decimals}e}") == float(token) for value in known) + return any(f"{abs(value):.{decimals}f}" == token for value in known) diff --git a/src/outerloop/orchestrator.py b/src/outerloop/orchestrator.py index 82d20c49..e11001b6 100644 --- a/src/outerloop/orchestrator.py +++ b/src/outerloop/orchestrator.py @@ -29,6 +29,7 @@ from outerloop.containment import executable_roots, seatbelt_available, seatbelt_wrap from outerloop.image import apptainer_from_env +from outerloop.number_check import unmatched_numbers from outerloop.trajectory import file_refs from outerloop.trajectory import record as record_event @@ -2835,6 +2836,24 @@ def process_section(tool_calls: dict[str, int] | None) -> list[str]: return ["", "## Process", "", f"{total} tool {calls} from the run's trajectory: {top}."] +MAX_NUMBER_CHECK = 20 + + +def number_check_section(numbers: list[str]) -> list[str]: + """Report-only list of report values that match no recorded measurement.""" + if not numbers: + return [] + shown = ", ".join(numbers[:MAX_NUMBER_CHECK]) + more = f" and {len(numbers) - MAX_NUMBER_CHECK} more" if len(numbers) > MAX_NUMBER_CHECK else "" + return [ + "", + "## Number check", + "", + "These values in the report match no measurement recorded for this run: " + f"{shown}{more}. Report-only; nothing was blocked.", + ] + + def pr_body( result: AttemptResult, config: RunConfig, @@ -2844,6 +2863,8 @@ def pr_body( *, base_sha: str = "", tool_calls: dict[str, int] | None = None, + metric: str = "", + measured_values: Sequence[float] | None = None, ) -> str: """The PR body for an improved run: the author's report, the experiments the run actually ran (from the launch ledger), the measured table, and @@ -2851,7 +2872,8 @@ def pr_body( Human surfaces render at the benchmark's conventional precision; full precision lives only in results/leader.json, and every - comparison runs on full floats. + comparison runs on full floats. When `measured_values` is given, a + report-only check lists report values that match no measurement. """ from outerloop.progress import fmt_metric @@ -2892,6 +2914,23 @@ def pr_body( if result.panel_transcript else [] ) + # the same redacted, truncated text the body shows, never the raw report + report_text = redact( + result.submit_report or (result.session.final_text if result.session else ""), + redact_secrets, + )[:MAX_REPORT_BODY] + number_lines = ( + number_check_section( + unmatched_numbers( + report_text, + metric, + measured_values, + [str(row.get("result") or "") for row in experiments or []], + ) + ) + if measured_values is not None + else [] + ) if result.submit_report: report_lines = [ "*Written by the author at submit, before the orchestrator measured; the " @@ -2925,6 +2964,7 @@ def pr_body( *report_lines, *experiments_section(experiments or []), *process_section(tool_calls), + *number_lines, "", "## Measured", "", @@ -2957,6 +2997,7 @@ def pr_body( redact(result.submit_report, redact_secrets)[:MAX_REPORT_BODY], *experiments_section(experiments or []), *process_section(tool_calls), + *number_lines, "", "## Self-reported values", "", diff --git a/tests/test_orchestrator.py b/tests/test_orchestrator.py index fbb44b58..d3865a07 100644 --- a/tests/test_orchestrator.py +++ b/tests/test_orchestrator.py @@ -1807,6 +1807,68 @@ def test_pr_body_leads_with_the_report_and_lists_the_experiments() -> None: assert "Session prose" in plain and "## Experiments" not in plain +def test_unmatched_numbers_skips_only_seeds_and_references() -> None: + from outerloop.number_check import unmatched_numbers + + # only the seed and the #12 reference are skipped; small numbers and years are checked + report = ( + "Baseline 0.512 (see #12) in 2024, candidate 0.501 from seed 8271 at step 4. " + "The sweep cell gave 3.28; the candidate curve peaked at 13.4." + ) + assert unmatched_numbers(report, "mean_tour_length", [0.5123, 0.5014], ['{"val": 3.28}']) == [ + "2024", + "4", + "13.4", + ] + # an integer metric on a line naming seeds is still checked + steps = "Candidate steps_to_target with seeds 3, 4 and 5 was 1840; baseline 2176." + assert unmatched_numbers(steps, "steps_to_target", [2176, 1856], []) == ["1840"] + # a reported gain is the gap between measurements, with either sign; a + # percentage is the relative gap + gain = "Baseline 2176, candidate 1856: a gain of 320 (delta -320), 14.7 % fewer, not 30%." + assert unmatched_numbers(gain, "steps_to_target", [2176, 1856], []) == ["30"] + # scientific notation and a zero gap + sci = "Candidate loss 1.2e-3 vs baseline 1.20e-3; gap 0.0." + assert unmatched_numbers(sci, "loss", [0.0012, 0.0012], []) == [] + + +def test_pr_body_number_check_is_report_only() -> None: + from outerloop.orchestrator import AttemptResult + + result = AttemptResult( + outcome="improved", + baseline=0.512, + candidate=0.501, + submit_report="Baseline 0.512, candidate 0.501; the curve peaked at 13.4.", + ) + config = RunConfig(target="org/pilot", benchmark="tsp") + body = pr_body(result, config, (), metric="mean_tour_length", measured_values=[0.5123, 0.5014]) + assert "## Number check" in body + assert ( + "match no measurement recorded for this run: 13.4. " + "Report-only; nothing was blocked." in body + ) + matched = pr_body( + result, + config, + (), + metric="mean_tour_length", + measured_values=[0.5123, 0.5014, 13.4], + ) + assert "## Number check" not in matched + # the check reads the redacted text the body shows: a secret's digits never surface + leaky = AttemptResult( + outcome="improved", + baseline=0.512, + candidate=0.501, + submit_report="Baseline 0.512 with run-77814, candidate 0.501.", + ) + redacted = pr_body( + leaky, config, ("run-77814",), metric="mean_tour_length", measured_values=[0.5123, 0.5014] + ) + assert "77814" not in redacted + + def test_submit_without_report_is_accepted(tmp_path: Path) -> None: _write_syscall(tmp_path, {"launches": [], "submit": True}) result, harness, _ = run_climb(