Trajectory step 4: report-only number check in the PR body

CHANGELOG.md

@@ -6,6 +6,7 @@ Versions follow [SemVer](https://semver.org).
 
 ## [Unreleased]
 
+- The PR body gains a report-only Number check that lists values in the author's report matching no `measure` event or Experiments result when rounded to the report's decimals; nothing is blocked and no trajectory event is added.
 - Session end records each tool call from the retained Claude or Codex log as a `tool.call` trajectory event, once per call even when a session resumes. `outerloop trajectory` and a new Process section in the PR body show counts by tool. Older kernels ignore these events.
 - `outerloop status` shows a run's priced session cost as a lower bound when some legs are unpriced (a wake killed mid-turn leaves no usage event), for example `session-cost>=$4.2000 (1 unpriced)`, instead of hiding the priced part behind `unknown`; the status row and JSON gain the priced-leg count (#516).
 - The pre-PR panel accepts a composite change — a PR that carries several changes — when the report gives each component's isolated measurement and the combination's under the same protocol and the combination is what the kernel measured. A bundle without those isolated measurements stays blocked (#489).

docs/design/trajectory.md

@@ -52,4 +52,9 @@ rollout JSONL are parsed host-side, bounded to 500 calls per session, and Hermes
 or unknown formats yield nothing. Each event carries the tool name, repo-relative
 files, a bounded shell command and, when the log has them, the exit code and
 outcome — never file contents or tool output. `outerloop trajectory` counts them
-by tool and the PR body summarizes the top tools. Attestation is a later step.
+by tool and the PR body summarizes the top tools. The PR body also runs a
+report-only number check: numbers on report lines that mention the metric,
+baseline or candidate are compared with every `measure` event's values and the
+Experiments result cells, after rounding to the report's decimals, and the
+unmatched ones are listed under "## Number check". Nothing is blocked and no
+event is written for it. Attestation is a later step.

src/outerloop/attempt.py

@@ -94,6 +94,7 @@
 from outerloop.limits import bound_limits, capped_limits, clamp_bound_limits, effective_limits
 from outerloop.markers import has_marker, marker
 from outerloop.measure import DispatchedMeasurer, DispatchSettings, measured_image
+from outerloop.number_check import measured_values as measured_from_events
 from outerloop.orchestrator import (
     MAX_REPORT_BODY,
     AttemptResult,
@@ -168,6 +169,14 @@ def tool_call_counts(run_dir: Path) -> dict[str, int]:
         return {}
 
 
+def measure_values(run_dir: Path) -> list[float]:
+    """The run's recorded measure values; an unreadable chain reads as none."""
+    try:
+        return measured_from_events(read_trajectory(run_dir))
+    except OSError:
+        return []
+
+
 def _pr_number(pr_url: str) -> int:
     tail = pr_url.rstrip("/").rsplit("/", 1)[-1]
     if not tail.isdigit():
@@ -4735,6 +4744,8 @@ def refuse(
                 experiments=experiments_rows(run_dir),
                 base_sha=base_sha,
                 tool_calls=tool_call_counts(run_dir),
+                metric=bench.metric,
+                measured_values=measure_values(run_dir),
             )
             body += f"\n\n{progress_link(config.target, github)}\n"
             if issue_number:

src/outerloop/number_check.py

@@ -0,0 +1,86 @@
+"""Report-only check that numbers in an author's report were measured.
+
+Nothing here records events or changes a gate or publish decision.
+"""
+
+from __future__ import annotations
+
+import re
+from collections.abc import Iterable
+
+# A number not embedded in a word or identifier, not part of a "#123" reference.
+_NUMBER = re.compile(r"(?<![#\w.])(-?\d+(?:\.\d+)?(?:[eE][-+]?\d+)?)")
+# The numbers named as seeds: "seed 8271", "seed=8271", "seeds 3, 4 and 5".
+_SEEDS = re.compile(r"\bseeds?\b\s*[=:]?\s*(\d+(?:\s*(?:,|and)\s*\d+)*)", re.IGNORECASE)
+
+
+def measured_values(events: Iterable[dict]) -> list[float]:
+    """Every measure event's baseline and candidate, in event order."""
+    values: list[float] = []
+    for event in events:
+        data = event.get("data")
+        if event.get("type") != "measure" or not isinstance(data, dict):
+            continue
+        for key in ("baseline", "candidate"):
+            value = data.get(key)
+            if isinstance(value, (int, float)) and not isinstance(value, bool):
+                values.append(float(value))
+    return values
+
+
+def unmatched_numbers(
+    report: str,
+    metric: str,
+    measured: Iterable[float],
+    experiment_results: Iterable[str],
+) -> list[str]:
+    """Numbers in the report that look like metric values but match no measurement.
+
+    A line counts when it mentions the metric name (case-insensitive), or
+    "baseline", or "candidate". On those lines every number is checked except
+    the two that cannot be a measured value: a seed ("seed 8271", "seeds 3,
+    4") and a "#123" reference. A number matches when, rounded to the decimals
+    it was written with and ignoring sign, it equals a measured value, the gap
+    between two measured values, or a number in an Experiments result cell; a
+    percentage matches the relative gap between two measured values. Returns
+    unmatched numbers in first-seen order. Report-only, so a rare false flag
+    is cheaper than hiding a claim.
+    """
+    values = list(measured)
+    known = values + [abs(a - b) for a in values for b in values]
+    percents = [abs(a - b) / abs(b) * 100 for a in values for b in values if b]
+    for cell in experiment_results:
+        known.extend(float(m.group(1)) for m in _NUMBER.finditer(cell))
+    metric_name = metric.strip().lower()
+    unmatched: list[str] = []
+    for line in report.splitlines():
+        lower = line.lower()
+        if not (
+            "baseline" in lower or "candidate" in lower or (metric_name and metric_name in lower)
+        ):
+            continue
+        for match in _NUMBER.finditer(line):
+            token = match.group(1)
+            if _is_seed(line, match):
+                continue
+            percent = line[match.end() :].lstrip(" ")[:1] == "%"
+            if _matches(token, percents if percent else known):
+                continue
+            if token not in unmatched:
+                unmatched.append(token)
+    return unmatched
+
+
+def _is_seed(line: str, match: re.Match[str]) -> bool:
+    """A number in a "seed 8271" / "seeds 3, 4" / "seed=8271" phrase."""
+    return any(s.start(1) <= match.start(1) < s.end(1) for s in _SEEDS.finditer(line))
+
+
+def _matches(token: str, known: Iterable[float]) -> bool:
+    """Equality after rounding both sides to the report's decimals, ignoring sign."""
+    token = token.lstrip("-")
+    mantissa, _, exponent = token.lower().partition("e")
+    decimals = len(mantissa.split(".")[1]) if "." in mantissa else 0
+    if exponent:  # scientific notation: round to the mantissa's digits
+        return any(float(f"{abs(value):.{decimals}e}") == float(token) for value in known)
+    return any(f"{abs(value):.{decimals}f}" == token for value in known)

src/outerloop/orchestrator.py

@@ -29,6 +29,7 @@
 
 from outerloop.containment import executable_roots, seatbelt_available, seatbelt_wrap
 from outerloop.image import apptainer_from_env
+from outerloop.number_check import unmatched_numbers
 from outerloop.trajectory import file_refs
 from outerloop.trajectory import record as record_event
 
@@ -2835,6 +2836,24 @@ def process_section(tool_calls: dict[str, int] | None) -> list[str]:
     return ["", "## Process", "", f"{total} tool {calls} from the run's trajectory: {top}."]
 
 
+MAX_NUMBER_CHECK = 20
+
+
+def number_check_section(numbers: list[str]) -> list[str]:
+    """Report-only list of report values that match no recorded measurement."""
+    if not numbers:
+        return []
+    shown = ", ".join(numbers[:MAX_NUMBER_CHECK])
+    more = f" and {len(numbers) - MAX_NUMBER_CHECK} more" if len(numbers) > MAX_NUMBER_CHECK else ""
+    return [
+        "",
+        "## Number check",
+        "",
+        "These values in the report match no measurement recorded for this run: "
+        f"{shown}{more}. Report-only; nothing was blocked.",
+    ]
+
+
 def pr_body(
     result: AttemptResult,
     config: RunConfig,
@@ -2844,14 +2863,17 @@ def pr_body(
     *,
     base_sha: str = "",
     tool_calls: dict[str, int] | None = None,
+    metric: str = "",
+    measured_values: Sequence[float] | None = None,
 ) -> str:
     """The PR body for an improved run: the author's report, the experiments
     the run actually ran (from the launch ledger), the measured table, and
     the panel's transcript.
 
     Human surfaces render at the benchmark's conventional precision;
     full precision lives only in results/leader.json, and every
-    comparison runs on full floats.
+    comparison runs on full floats. When `measured_values` is given, a
+    report-only check lists report values that match no measurement.
     """
     from outerloop.progress import fmt_metric
 
@@ -2892,6 +2914,23 @@ def pr_body(
         if result.panel_transcript
         else []
     )
+    # the same redacted, truncated text the body shows, never the raw report
+    report_text = redact(
+        result.submit_report or (result.session.final_text if result.session else ""),
+        redact_secrets,
+    )[:MAX_REPORT_BODY]
+    number_lines = (
+        number_check_section(
+            unmatched_numbers(
+                report_text,
+                metric,
+                measured_values,
+                [str(row.get("result") or "") for row in experiments or []],
+            )
+        )
+        if measured_values is not None
+        else []
+    )
     if result.submit_report:
         report_lines = [
             "*Written by the author at submit, before the orchestrator measured; the "
@@ -2925,6 +2964,7 @@ def pr_body(
             *report_lines,
             *experiments_section(experiments or []),
             *process_section(tool_calls),
+            *number_lines,
             "",
             "## Measured",
             "",
@@ -2957,6 +2997,7 @@ def pr_body(
                 redact(result.submit_report, redact_secrets)[:MAX_REPORT_BODY],
                 *experiments_section(experiments or []),
                 *process_section(tool_calls),
+                *number_lines,
                 "",
                 "## Self-reported values",
                 "",

tests/test_orchestrator.py

@@ -1807,6 +1807,68 @@ def test_pr_body_leads_with_the_report_and_lists_the_experiments() -> None:
     assert "Session prose" in plain and "## Experiments" not in plain
 
 
+def test_unmatched_numbers_skips_only_seeds_and_references() -> None:
+    from outerloop.number_check import unmatched_numbers
+
+    # only the seed and the #12 reference are skipped; small numbers and years are checked
+    report = (
+        "Baseline 0.512 (see #12) in 2024, candidate 0.501 from seed 8271 at step 4. "
+        "The sweep cell gave 3.28; the candidate curve peaked at 13.4."
+    )
+    assert unmatched_numbers(report, "mean_tour_length", [0.5123, 0.5014], ['{"val": 3.28}']) == [
+        "2024",
+        "4",
+        "13.4",
+    ]
+    # an integer metric on a line naming seeds is still checked
+    steps = "Candidate steps_to_target with seeds 3, 4 and 5 was 1840; baseline 2176."
+    assert unmatched_numbers(steps, "steps_to_target", [2176, 1856], []) == ["1840"]
+    # a reported gain is the gap between measurements, with either sign; a
+    # percentage is the relative gap
+    gain = "Baseline 2176, candidate 1856: a gain of 320 (delta -320), 14.7 % fewer, not 30%."
+    assert unmatched_numbers(gain, "steps_to_target", [2176, 1856], []) == ["30"]
+    # scientific notation and a zero gap
+    sci = "Candidate loss 1.2e-3 vs baseline 1.20e-3; gap 0.0."
+    assert unmatched_numbers(sci, "loss", [0.0012, 0.0012], []) == []
+
+
+def test_pr_body_number_check_is_report_only() -> None:
+    from outerloop.orchestrator import AttemptResult
+
+    result = AttemptResult(
+        outcome="improved",
+        baseline=0.512,
+        candidate=0.501,
+        submit_report="Baseline 0.512, candidate 0.501; the curve peaked at 13.4.",
+    )
+    config = RunConfig(target="org/pilot", benchmark="tsp")
+    body = pr_body(result, config, (), metric="mean_tour_length", measured_values=[0.5123, 0.5014])
+    assert "## Number check" in body
+    assert (
+        "match no measurement recorded for this run: 13.4. "
+        "Report-only; nothing was blocked." in body
+    )
+    matched = pr_body(
+        result,
+        config,
+        (),
+        metric="mean_tour_length",
+        measured_values=[0.5123, 0.5014, 13.4],
+    )
+    assert "## Number check" not in matched
+    # the check reads the redacted text the body shows: a secret's digits never surface
+    leaky = AttemptResult(
+        outcome="improved",
+        baseline=0.512,
+        candidate=0.501,
+        submit_report="Baseline 0.512 with run-77814, candidate 0.501.",
+    )
+    redacted = pr_body(
+        leaky, config, ("run-77814",), metric="mean_tour_length", measured_values=[0.5123, 0.5014]
+    )
+    assert "77814" not in redacted
+
+
 def test_submit_without_report_is_accepted(tmp_path: Path) -> None:
     _write_syscall(tmp_path, {"launches": [], "submit": True})
     result, harness, _ = run_climb(