diff --git a/CHANGELOG.md b/CHANGELOG.md index d084abd4..43e83ece 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ Versions follow [SemVer](https://semver.org). - `outerloop trajectory ` now ends with the run's refusal count and its author models, for release canary comparisons; the agent-text inventory includes operator notes (#487, part 3). Session events record `role` (`author` or `judge`). Upgrading: session events written before this kernel have no `role` and count as author sessions. - On macOS without an image, sessions, evaluations and panel lenses run under a packaged Seatbelt profile (`sandbox-exec`, #292): they write only the workspace and a scratch directory, read only system and toolchain paths, and connect out only to TCP port 443, DNS and local ports. Linux and Apptainer runs are unchanged. - Kernel events now append to a per-run `trajectory.jsonl` with sequence numbers and a SHA-256 chain. `outerloop trajectory [--verify] [--json]` reads it without changing run state. Operational files and their formats are unchanged; older kernels ignore the new file. Upgrading: runs started before this kernel have no trajectory and read as "no trajectory" until their next recorded event. No backfill is attempted. +- `measure` trajectory events carry provenance: the ledger's ruler digest (the same value a pending submission computes), the container image digest when an image is configured, and refs to the evaluation result files the kernel read. Self-reported measures record the ruler alone. Pending ledger records now ignore fields this kernel does not know, so a later release can add one without breaking this release's ledger read. Upgrading: no action needed; older kernels ignore the trajectory, and pending records are unchanged. - Add a printable agent-text inventory and protocol wording lint, with clearer refusal recovery guidance (#487, parts 1–2). - `BENCHMARKS.md` shows `—` instead of "provenance unknown" in the main-commit column of a baseline row that nothing has improved on yet, as on a freshly seeded repository. - The README and the getting-started guide link to the quickstart repository. It runs two CPU benchmarks with the local forge and local compute, and needs no GitHub account or cluster (#563). The docs and the startup warning no longer say an uncontained session runs in a sandbox: on Linux without an image, it gets a scrubbed environment and its own home, but can read any file the account can. diff --git a/docs/design/trajectory.md b/docs/design/trajectory.md index 6cc43b7d..94f04790 100644 --- a/docs/design/trajectory.md +++ b/docs/design/trajectory.md @@ -38,4 +38,12 @@ The chain detects internal edits and missing lines, but without a trusted external anchor cannot detect a rewritten chain or removal of its tail. Verification checks the event chain, not the contents of referenced artifacts. -Tool calls, full measurement provenance and attestation are later steps. +`measure` events carry provenance. They record the ledger's ruler digest (the +measurement signature plus the non-solver tree entries, the value a pending +submission also computes), the container image digest when one is configured, +and refs to the evaluation result files the kernel read under the run +directory. Self-reported measures record the ruler alone: they have no +evaluation image or result files. A pending submission does not yet name the +trajectory head: kernels before this one reject unknown pending fields, so the +pending reader now ignores them, and a later release can add the field without +breaking this one. Tool calls and attestation are later steps. diff --git a/src/outerloop/attempt.py b/src/outerloop/attempt.py index 92a8196f..b96e58d8 100644 --- a/src/outerloop/attempt.py +++ b/src/outerloop/attempt.py @@ -89,10 +89,11 @@ measurement_pending, observe_target, queue_pending, + ruler_identity, ) from outerloop.limits import bound_limits, capped_limits, clamp_bound_limits, effective_limits from outerloop.markers import has_marker, marker -from outerloop.measure import DispatchedMeasurer, DispatchSettings +from outerloop.measure import DispatchedMeasurer, DispatchSettings, measured_image from outerloop.orchestrator import ( MAX_REPORT_BODY, AttemptResult, @@ -165,6 +166,16 @@ def _pr_number(pr_url: str) -> int: log = logging.getLogger(__name__) + +def _ruler_provenance(ws: Workspace, contract: Contract, bench: Benchmark, sha: str) -> str: + """Best-effort ledger ruler digest; a git read must not fail a wake.""" + try: + return ruler_identity(ws, contract, bench, sha)[0] + except Exception as exc: + log.warning("could not compute ruler identity: %s", exc) + return "" + + # Where a climb job reads its keys unless the CLI flags say otherwise. The # tick preflights the panel key (and compares it against the author key — # role separation) before claiming/submitting. @@ -3369,17 +3380,26 @@ def _resume_run( # more experiments, or concludes — instead of being decided by policy here. # Falls back to the plain-finish behavior (negative terminal / draft PR) # when the session cannot be resumed. + measure_data = { + "baseline": result.baseline, + "candidate": result.candidate, + "metric": bench.metric, + "credited": result.outcome == "improved", + "outcome": result.outcome, + "head_sha": candidate_sha, + "ruler": _ruler_provenance(ws, contract, bench, candidate_sha), + } + image_sha256 = measured_image(str(getattr(measurer, "image", "") or ""), result.eval_files) + if image_sha256: + measure_data["image_sha256"] = image_sha256 + if result.outcome == "improved" and result.note: + # resume_attempt carries the cached-baseline note here on a credit + measure_data["baseline_note"] = result.note record_event( run_dir, "measure", - { - "baseline": result.baseline, - "candidate": result.candidate, - "metric": bench.metric, - "credited": result.outcome == "improved", - "outcome": result.outcome, - "head_sha": candidate_sha, - }, + measure_data, + refs=file_refs(run_dir, *result.eval_files), ) submitted_park = bool(stage.get("submitted")) if stage.get("panel_skip"): @@ -5316,6 +5336,10 @@ def acknowledge(seq: int) -> None: parked: RunParked | None = None kept_ref = "" # the ONE candidate snapshot ref that must outlive a park + + def ruler_sha(sha: str) -> str: + return _ruler_provenance(ws, contract, _bench, sha) + try: # the last-known score orients the brief only; the gate re-measures # both sides after the session, so None (a first run) is fine. @@ -5365,6 +5389,7 @@ def acknowledge(seq: int) -> None: else None ), tree_of=lambda sha: ws.git("rev-parse", f"{sha}^{{tree}}").strip(), + ruler_sha=ruler_sha, ) except EndpointUnavailable as exc: # Reuse a jobless capacity park; its first wake starts the author. diff --git a/src/outerloop/dispatch.py b/src/outerloop/dispatch.py index 07b49330..0d1bfe0d 100644 --- a/src/outerloop/dispatch.py +++ b/src/outerloop/dispatch.py @@ -424,7 +424,15 @@ def write_eval_job( task_dir.mkdir(parents=True, exist_ok=True) # a resubmitted eval must never be read as its predecessor: every prior # artifact — including a leftover extracted tree — goes before submission - for stale in ("exit-code", "stdout", "stderr", "setup.log", "submitted", "artifacts.log"): + for stale in ( + "exit-code", + "stdout", + "stderr", + "setup.log", + "submitted", + "artifacts.log", + "image-sha256", + ): (task_dir / stale).unlink(missing_ok=True) for stale_node in task_dir.glob("node-[0-9]*"): # a multi-node eval's per-node results stale_node.unlink(missing_ok=True) diff --git a/src/outerloop/ledger_events.py b/src/outerloop/ledger_events.py index 0dbd5cce..1d40d934 100644 --- a/src/outerloop/ledger_events.py +++ b/src/outerloop/ledger_events.py @@ -50,6 +50,25 @@ def display_leader(github: Forge, target: str) -> dict[str, LeaderEntry]: return {} +def ruler_identity( + ws: Workspace, contract: Contract, bench: Benchmark, measured_sha: str +) -> tuple[str, str]: + """The ledger's ruler digest and measurement signature for a measured tree. + + Ruler identity includes harness contents, not just its command. Solver + edits do not change it; stewardship edits do, even with the same contract. + """ + signature = json.dumps(bench.measurement_signature(), separators=(",", ":")) + ruler_files = [] + for entry in ws.git("ls-tree", "-rz", measured_sha).split("\0"): + if entry: + _, path = entry.split("\t", 1) + if not steward_out_of_scope([path], contract): + ruler_files.append(entry) + ruler = hashlib.sha256((signature + "\0".join(sorted(ruler_files))).encode()).hexdigest() + return ruler, signature + + def measurement_pending( ws: Workspace, contract: Contract, @@ -66,16 +85,7 @@ def measurement_pending( kind: str, provenance: str = "measured", ) -> PendingSubmission: - signature = json.dumps(bench.measurement_signature(), separators=(",", ":")) - # Ruler identity includes harness contents, not just its command. Solver - # edits do not change it; stewardship edits do, even with the same contract. - ruler_files = [] - for entry in ws.git("ls-tree", "-rz", measured_sha).split("\0"): - if entry: - _, path = entry.split("\t", 1) - if not steward_out_of_scope([path], contract): - ruler_files.append(entry) - ruler = hashlib.sha256((signature + "\0".join(sorted(ruler_files))).encode()).hexdigest() + ruler, signature = ruler_identity(ws, contract, bench, measured_sha) return PendingSubmission( bench.name, bench.metric, diff --git a/src/outerloop/measure.py b/src/outerloop/measure.py index 91c52932..43d1ed95 100644 --- a/src/outerloop/measure.py +++ b/src/outerloop/measure.py @@ -17,9 +17,11 @@ from __future__ import annotations +import contextlib import hashlib import json import logging +import os import time from dataclasses import dataclass, field from pathlib import Path @@ -37,6 +39,37 @@ log = logging.getLogger(__name__) +# A container image is large and identical across measures in one process; +# (path, size, mtime) says when a re-hash is actually needed. +_image_digests: dict[tuple[str, int, int], str] = {} + + +def image_digest(image: str) -> str: + """SHA-256 of the container image file, or "" when none is used/readable.""" + if not image: + return "" + try: + stat = os.stat(image) + key = (image, stat.st_size, stat.st_mtime_ns) + if key not in _image_digests: + with open(image, "rb") as stream: + _image_digests[key] = hashlib.file_digest(stream, "sha256").hexdigest() + return _image_digests[key] + except OSError: + return "" + + +def measured_image(image: str, eval_files: tuple[Path, ...]) -> str: + """The image digest(s) the measures ran under: those recorded at dispatch, + comma-joined when baseline and candidate differ, else the image's digest now.""" + recorded = set() + for path in eval_files: + if path.name == "image-sha256": + with contextlib.suppress(OSError): + recorded.add(path.read_text().strip()) + recorded.discard("") + return ",".join(sorted(recorded)) if recorded else image_digest(image) + class MeasurementPending(Exception): """Raised when one or more measures have not completed. Carries the wake @@ -354,6 +387,22 @@ def _slot(self, m: Measure) -> str: def _ev(self, m: Measure) -> Path: return self.run_dir / f"eval-{self._slot(m)}" + def result_files(self, measures: list[Measure]) -> tuple[Path, ...]: + """The eval result files the kernel reads, when they exist. + + `read_eval_result` reads exit-code and stdout for the metric, and + stderr to explain a failure; provenance refs name the same files, plus + the image digest recorded at dispatch. + """ + files: list[Path] = [] + for m in measures: + ev = self._ev(m) + for name in ("exit-code", "stdout", "stderr", "image-sha256"): + path = ev / name + if path.exists(): + files.append(path) + return tuple(files) + def _job_name(self, m: Measure) -> str: # A LIVENESS HINT, not a durable key: the cluster is asked "is this # measure's job live" by name. Slurm caps name length, so this hash is @@ -505,6 +554,11 @@ def _dispatch(self, m: Measure) -> str: raise MeasurementPending( (), capacity_wait=True, note="retry state could not be saved; parked uncharged" ) from None + # recorded before submission: the image file may be replaced while the + # run parks, and a crash after submit must not lose the digest + digest = image_digest(self.image) + if digest: + (self._ev(m) / "image-sha256").write_text(digest) try: job_id = submit_batch( state_root(self.run_dir), diff --git a/src/outerloop/orchestrator.py b/src/outerloop/orchestrator.py index 32ca2792..f63b3414 100644 --- a/src/outerloop/orchestrator.py +++ b/src/outerloop/orchestrator.py @@ -29,6 +29,7 @@ from outerloop.containment import executable_roots, seatbelt_available, seatbelt_wrap from outerloop.image import apptainer_from_env +from outerloop.trajectory import file_refs from outerloop.trajectory import record as record_event if TYPE_CHECKING: @@ -575,6 +576,8 @@ class AttemptResult: # In-memory ending guard: rejected trees must never reach line snapshots. tree_rejected: bool = False provenance: str = "measured" + # the eval result files this decision read, for the measure event's refs + eval_files: tuple[Path, ...] = () def report(self, config: RunConfig, redact_secrets: tuple[str, ...] = ()) -> str: lines = [ @@ -932,6 +935,8 @@ class MeasureOK: # how the baseline number was obtained when it was NOT measured beside # this candidate (`baseline: cached`) — surfaces on the credited result baseline_note: str = "" + # the eval result files this decision read, for the measure event's refs + eval_files: tuple[Path, ...] = () def measure_and_decide( @@ -975,6 +980,11 @@ def measure_and_decide( write_baseline_cache, ) + def result_files(measures: Sequence[object]) -> tuple[Path, ...]: + """The measurer's result files, when its backend writes them.""" + fn = getattr(measurer, "result_files", None) + return tuple(fn(list(measures))) if callable(fn) else () + # Scope BEFORE measurement: an out-of-scope tree is never evaluated, # because the out-of-scope edit could be to the ruler itself. violations = out_of_scope(list(measured_paths), contract) @@ -1050,7 +1060,13 @@ def measure_and_decide( try: main = measurer.results(plan) except EvalError as exc: - return AttemptResult(outcome="eval-error", note=str(exc), run_seed=seed) + return AttemptResult( + outcome="eval-error", + note=str(exc), + run_seed=seed, + eval_files=result_files(plan), + ) + main_files = result_files(plan) baseline_note = "" if cached is not None: @@ -1084,6 +1100,7 @@ def measure_and_decide( baseline=baseline, candidate=candidate, run_seed=seed, + eval_files=main_files, ) # The contract's OWN significance floor, when the benchmark declares one. # Same-seed pairing removes pool noise but not training stochasticity — @@ -1109,13 +1126,19 @@ def measure_and_decide( baseline=baseline, candidate=candidate, run_seed=seed, + eval_files=main_files, ) # PHASE 2 — suite gate, only for a credited candidate whose diff touched # shared code. A second measure set (a second park for a dispatched # backend): an extra CPU wake, never a wasted GPU sibling eval. if not (siblings and shared_touched(measured_paths, contract)): - return MeasureOK(baseline=baseline, candidate=candidate, baseline_note=baseline_note) + return MeasureOK( + baseline=baseline, + candidate=candidate, + baseline_note=baseline_note, + eval_files=main_files, + ) # every seeded sibling runs its pair under the ONE suite_seed, read through # its own seed var (mirrors the in-job gate). @@ -1156,7 +1179,9 @@ def measure_and_decide( candidate=candidate, note=str(exc), run_seed=seed, + eval_files=main_files + result_files(sib_plan), ) + sib_files = main_files + result_files(sib_plan) # Exact decimal gain avoids rounding an inclusive unlock boundary down. main_base, main_cand = Fraction(repr(baseline)), Fraction(repr(candidate)) @@ -1198,6 +1223,7 @@ def measure_and_decide( run_seed=seed, suite=suite, suite_seed=suite_seed, + eval_files=sib_files, ) return MeasureOK( @@ -1206,6 +1232,7 @@ def measure_and_decide( suite=suite, suite_seed=suite_seed, # reached only on the suite path baseline_note=baseline_note, + eval_files=sib_files, ) @@ -1291,6 +1318,7 @@ def resume_attempt( suite_seed=outcome.suite_seed, candidate_sha=candidate_sha, note=outcome.baseline_note, + eval_files=outcome.eval_files, ) @@ -1378,6 +1406,9 @@ def attempt_once( sleeps_used: int = 0, gpu_hours_used: float = 0.0, tree_of: Callable[[str], str] | None = None, + # the ledger's ruler digest for a measured sha (git work stays with the + # caller, like tree_of); the measure event records it as provenance + ruler_sha: Callable[[str], str] | None = None, judged: tuple[str, str, AttemptResult] | None = None, on_replies: Callable[[tuple[dict, ...]], object] | None = None, on_stop: Callable[[SessionResult], AttemptResult] | None = None, @@ -1432,6 +1463,20 @@ def attempt_once( contract = load_contract(contract_text, config.target) bench = contract.benchmark(config.benchmark) spec = spec or author_spec() + + def _ruler(sha: str) -> str: + """Best-effort provenance: a git read must never fail the climb.""" + try: + return ruler_sha(sha) if ruler_sha is not None else "" + except Exception as exc: + log.warning("could not compute ruler identity: %s", exc) + return "" + + def _image_sha(eval_files: tuple[Path, ...]) -> str: + from outerloop.measure import measured_image + + return measured_image(str(getattr(measurer, "image", "") or ""), eval_files) + leg_start_seq = inbox_seq # the cursor before this leg delivers anything if not spec.execution.can_execute: raise ValueError("attempt_once runs an editing role; the spec must allow execution") @@ -1726,6 +1771,7 @@ def _ack(messages: list[Message]) -> None: suite: tuple[SuiteMeasurement, ...] = () suite_seed_ran = 0 baseline_note = "" + eval_files: tuple[Path, ...] = () measured: tuple[str, ...] = () refused_once = False # A session that stops without staging anything (an empty reply, a line of @@ -2394,6 +2440,7 @@ def _not_run_note(request: SyscallRequest | None) -> str: "credited": passed, "outcome": "improved" if passed else "no-improvement", "head_sha": candidate_sha, + "ruler": _ruler(candidate_sha), }, ) return AttemptResult( @@ -2455,17 +2502,26 @@ def _not_run_note(request: SyscallRequest | None) -> str: ) # a scope refusal returns before measuring; a failed eval was attempted if verdict_outcome != "scope-violation": + data = { + "baseline": outcome.baseline, + "candidate": outcome.candidate, + "metric": bench.metric, + "credited": verdict_outcome == "improved", + "outcome": verdict_outcome, + "head_sha": candidate_sha, + "ruler": _ruler(candidate_sha), + } + digest = _image_sha(outcome.eval_files) + if digest: + data["image_sha256"] = digest + # a cached baseline was not measured here: the digest is the candidate's + if isinstance(outcome, MeasureOK) and outcome.baseline_note: + data["baseline_note"] = outcome.baseline_note record_event( inbox_dir, "measure", - { - "baseline": outcome.baseline, - "candidate": outcome.candidate, - "metric": bench.metric, - "credited": verdict_outcome == "improved", - "outcome": verdict_outcome, - "head_sha": candidate_sha, - }, + data, + refs=file_refs(inbox_dir, *outcome.eval_files), ) except MeasurementPending as pending: # PARK 2 (dispatched candidate/suite, after the session): the wake @@ -2632,6 +2688,7 @@ def _not_run_note(request: SyscallRequest | None) -> str: suite = outcome.suite suite_seed_ran = outcome.suite_seed baseline_note = outcome.baseline_note # cached-baseline provenance, to the report + eval_files = outcome.eval_files if panel_runner is None: break @@ -2704,6 +2761,7 @@ def _not_run_note(request: SyscallRequest | None) -> str: panel_rounds=panel_reads, panel_blocking_open=panel_blocking_open, panel_degraded=panel_degraded, + eval_files=eval_files, ) diff --git a/src/outerloop/progress.py b/src/outerloop/progress.py index c78178ef..cd37bff5 100644 --- a/src/outerloop/progress.py +++ b/src/outerloop/progress.py @@ -6,7 +6,7 @@ import logging import math from collections.abc import Callable -from dataclasses import asdict, dataclass, replace +from dataclasses import asdict, dataclass, fields, replace from pathlib import Path from typing import TYPE_CHECKING @@ -132,7 +132,12 @@ def parse_pending(content: str) -> PendingSubmission | None: raw = json.loads(content) if raw is None: return None - pending = PendingSubmission(**raw) + if not isinstance(raw, dict): + raise ValueError("not a record") + # Fields a newer kernel adds are ignored, so a release can add one + # without breaking the ledger read of the release before it. + known = {f.name for f in fields(PendingSubmission)} + pending = PendingSubmission(**{k: v for k, v in raw.items() if k in known}) for field, value in asdict(pending).items(): if field in {"baseline", "candidate", "min_delta", "min_delta_rel"}: if type(value) not in (int, float) or not math.isfinite(value): diff --git a/tests/test_contract_switches.py b/tests/test_contract_switches.py index a7faf10c..7e790190 100644 --- a/tests/test_contract_switches.py +++ b/tests/test_contract_switches.py @@ -110,6 +110,7 @@ def test_self_report_skips_measurement_panel_and_compute( "credited": True, "outcome": "improved", "head_sha": result.candidate_sha, + "ruler": "", } ] assert evaluator.calls == [] and result.panel_rounds == 0 diff --git a/tests/test_measure.py b/tests/test_measure.py index d72b9b2f..ab4d77bf 100644 --- a/tests/test_measure.py +++ b/tests/test_measure.py @@ -89,6 +89,27 @@ def test_first_pass_dispatches_all_and_parks(tmp_path): assert exc.value.afterany() == "afterany:101:102" +def test_dispatch_records_the_image_it_ran_under(tmp_path): + import hashlib + + from outerloop.measure import measured_image + + image = tmp_path / "img.sif" + image.write_bytes(b"old image") + m = _measurer(tmp_path, []) + m.image = str(image) + with pytest.raises(MeasurementPending): + m.results(_measures()) + base, cand = _measures() + _land(m, base, 0.50) + _land(m, cand, 0.61) + image.write_bytes(b"new image, same path") # replaced while the run parked + files = m.result_files(_measures()) + assert {p.name for p in files} == {"exit-code", "stdout", "image-sha256"} + old = hashlib.sha256(b"old image").hexdigest() + assert measured_image(str(image), files) == old + + def test_resume_reads_completed_without_resubmitting(tmp_path): submitted: list = [] m = _measurer(tmp_path, submitted) diff --git a/tests/test_orchestrator.py b/tests/test_orchestrator.py index 567ee16e..fbb44b58 100644 --- a/tests/test_orchestrator.py +++ b/tests/test_orchestrator.py @@ -176,6 +176,41 @@ def test_improvement_end_to_end(tmp_path: Path) -> None: ) +def test_measure_event_records_ruler_image_and_result_refs(tmp_path: Path) -> None: + import hashlib + + from outerloop.trajectory import read + + evaluator = FakeEvaluator(values=[13.876, 13.10]) + measurer, snapshot = _wire(evaluator, tmp_path) + image = tmp_path / "image.sif" + image.write_bytes(b"container image bytes") + measurer.image = str(image) + directory = run_dir(tmp_path) + directory.mkdir(parents=True, exist_ok=True) + files = [] + for name, body in (("candidate-exit-code", b"0"), ("candidate-stdout", b'{"v": 13.1}')): + path = directory / name + path.write_bytes(body) + files.append(path) + + measurer.result_files = lambda measures: tuple(files) + run_climb( + tmp_path, + measurer=measurer, + snapshot=snapshot, + ruler_sha=lambda sha: f"ruler-{sha}", + ) + + event = next(e for e in read(directory) if e["type"] == "measure") + assert event["data"]["ruler"] == "ruler-cand1" + assert event["data"]["image_sha256"] == hashlib.sha256(image.read_bytes()).hexdigest() + assert event["refs"] == [ + {"path": path.name, "sha256": hashlib.sha256(path.read_bytes()).hexdigest()} + for path in files + ] + + def test_brief_carries_the_ledger_baseline_and_leaves_publishing_to_the_author( tmp_path: Path, ) -> None: diff --git a/tests/test_progress.py b/tests/test_progress.py index 8d228277..90c33e2e 100644 --- a/tests/test_progress.py +++ b/tests/test_progress.py @@ -128,6 +128,9 @@ def test_pending_roundtrip_and_rejection(): assert parse_pending(record_pending(p)[p.path]) == p assert parse_pending(reject(p)[p.path]) is None assert LEADER_FILE not in record_pending(p) and LEADER_FILE not in reject(p) + # a field from a newer kernel is ignored, not a ledger read failure + newer = json.loads(record_pending(p)[p.path]) | {"added_later": "x"} + assert parse_pending(json.dumps(newer)) == p @pytest.mark.parametrize("direction,better,worse", [("max", 3.0, 1.0), ("min", 1.0, 3.0)])