diff --git a/CHANGELOG.md b/CHANGELOG.md index b2cb9066..b8b959bb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ Versions follow [SemVer](https://semver.org). ## [Unreleased] +- `outerloop trajectory ` now ends with the run's refusal count and its author models, for release canary comparisons; the agent-text inventory includes operator notes (#487, part 3). Session events record `role` (`author` or `judge`). Upgrading: session events written before this kernel have no `role` and count as author sessions. - On macOS without an image, sessions, evaluations and panel lenses run under a packaged Seatbelt profile (`sandbox-exec`, #292): they write only the workspace and a scratch directory, read only system and toolchain paths, and connect out only to TCP port 443, DNS and local ports. Linux and Apptainer runs are unchanged. - Kernel events now append to a per-run `trajectory.jsonl` with sequence numbers and a SHA-256 chain. `outerloop trajectory [--verify] [--json]` reads it without changing run state. Operational files and their formats are unchanged; older kernels ignore the new file. Upgrading: runs started before this kernel have no trajectory and read as "no trajectory" until their next recorded event. No backfill is attempted. - Add a printable agent-text inventory and protocol wording lint, with clearer refusal recovery guidance (#487, parts 1–2). diff --git a/RELEASING.md b/RELEASING.md index dfe52e9e..2642faca 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -43,21 +43,24 @@ wording does not reach a parked run unless its deduplication key changes. run with an open PR, a run whose PR merged, and an ended run. A dev or rc pre-release still bumps the version (PyPI never accepts a version twice, so the next one is `.dev1`, `rc2`, ...) but leaves `[Unreleased]` in place until the final release. -3. `git tag vX.Y.Z && git push origin vX.Y.Z`. The `release` workflow builds +3. Pre-release canary runs report `refusals` per run by author model from + `outerloop trajectory `. If refusals per run rise against the previous + release, explain why in the release PR before shipping. +4. `git tag vX.Y.Z && git push origin vX.Y.Z`. The `release` workflow builds and publishes `outerloop-science` to PyPI through Trusted Publishing; the one-time PyPI setup is described at the top of `.github/workflows/release.yml`. -4. Once the `release` workflow run for the tag is green (`gh run watch` on +5. Once the `release` workflow run for the tag is green (`gh run watch` on it), `pip install outerloop-science==X.Y.Z` in a fresh venv, then `outerloop --help`. This comes before the GitHub release: publishing it is what Discord announces, so nothing is announced that did not install. -5. `gh release create vX.Y.Z --generate-notes`, with `--prerelease` for a dev +6. `gh release create vX.Y.Z --generate-notes`, with `--prerelease` for a dev or rc tag. For a final release, also move the `v0` tag that external callers of the reusable workflows track: `git tag -f v0 vX.Y.Z && git push -f origin v0`. Dev and rc tags never move it. This tag alias is the one force-push the repository allows; branches are never force-pushed. -6. Announce. Discord's `#announcements` gets the release from the GitHub +7. Announce. Discord's `#announcements` gets the release from the GitHub webhook on its own (docs/community.md). For a final release, also write the post for X (`@outerloop_sci`) and Bluesky (`@outerloop.science`): one or two sentences on what changed for the reader, the install line, the diff --git a/src/outerloop/agent_text.py b/src/outerloop/agent_text.py index 1e7e8ce1..c3af1dc0 100644 --- a/src/outerloop/agent_text.py +++ b/src/outerloop/agent_text.py @@ -50,6 +50,7 @@ def verb_parsers() -> dict[str, argparse.ArgumentParser]: ("orchestrator", "attempt_once", "keyword:problem", "refusal"), ("orchestrator", "attempt_once", "message", "refusal"), ("tick", "", "message", "refusal"), + ("cli", "message", "message", "note"), ) # Message keys that mark a refusal; any other kernel message with a literal text is a note. REFUSAL_KEYS = {"message-refused", "refusal", "capacity-refusal"} diff --git a/src/outerloop/cli.py b/src/outerloop/cli.py index 6780c8d1..9abea327 100644 --- a/src/outerloop/cli.py +++ b/src/outerloop/cli.py @@ -1319,7 +1319,7 @@ def main(argv: list[str] | None = None) -> int: return pr(args) if args.command == "trajectory": from outerloop.runstate import run_dir - from outerloop.trajectory import read, verify + from outerloop.trajectory import read, summarize, verify values = env_file_values(keys=("OUTERLOOP_ROOT",)) if not args.root else {} root = Path( @@ -1354,6 +1354,16 @@ def main(argv: list[str] | None = None) -> int: print("no trajectory") if args.verify: print("; ".join(problems) if problems else "chain intact") + if events: + summary = summarize(events) + verbs = ", ".join( + f"{verb} {count}" for verb, count in summary["refusals_by_verb"].items() + ) + authors = ", ".join(summary["authors"]) or "unknown" + print( + f"refusals: {summary['refusals']} of {summary['decisions']} decisions" + f"{f' ({verbs})' if verbs else ''}; author: {authors}" + ) return 1 if problems and problems != ["no trajectory"] else 0 if args.command == "status": from outerloop.status import collect_status, render_text diff --git a/src/outerloop/trajectory.py b/src/outerloop/trajectory.py index a2d3d1c5..bdad64d1 100644 --- a/src/outerloop/trajectory.py +++ b/src/outerloop/trajectory.py @@ -118,6 +118,34 @@ def read(run_dir: Path) -> list[dict]: ] +def summarize(events: Iterable[dict[str, Any]]) -> dict[str, Any]: + """Count recorded syscall decisions and collect session authors in first-seen order.""" + decisions = 0 + refusals = 0 + by_verb: dict[str, int] = {} + authors: list[str] = [] + for event in events: + data = event.get("data") + if not isinstance(data, dict): + data = {} + if event.get("type") == "syscall.decision": + decisions += 1 + if data.get("outcome") == "refused": + refusals += 1 + verb = data.get("verb") or "unknown" + by_verb[verb] = by_verb.get(verb, 0) + 1 + elif event.get("type") == "session.start" and data.get("role") != "judge": + author = f"{data.get('backend') or 'unknown'}/{data.get('model') or 'unknown'}" + if author not in authors: + authors.append(author) + return { + "decisions": decisions, + "refusals": refusals, + "refusals_by_verb": by_verb, + "authors": authors, + } + + def verify(run_dir: Path, run_id: str) -> list[str]: """Check exact line bytes and sequence continuity without changing files.""" try: @@ -168,6 +196,7 @@ def session_event( "model": model, "session_id": session_id, "stage": state.get("stage", {}).get("phase", "author"), + "role": "judge" if workspace.relative_to(directory).parts[0] == "panel" else "author", } refs = [] if result is not None: diff --git a/tests/test_session_evidence.py b/tests/test_session_evidence.py index 8b999d69..814b41b0 100644 --- a/tests/test_session_evidence.py +++ b/tests/test_session_evidence.py @@ -161,6 +161,7 @@ def test_evidence_redaction_hash_cap_resume_and_retention(tmp_path, monkeypatch, events = read(tmp_path) assert [e["type"] for e in events] == ["session.start", "session.end"] * 2 assert events[0]["data"]["stage"] == "author-sleep" + assert events[0]["data"]["role"] == "author" assert events[1]["data"]["session_id"] == "session-1" assert len(events[1]["refs"]) == 1 native_ref = events[1]["refs"][0] diff --git a/tests/test_trajectory.py b/tests/test_trajectory.py index 3fc8fe75..6168db1b 100644 --- a/tests/test_trajectory.py +++ b/tests/test_trajectory.py @@ -7,6 +7,60 @@ from outerloop import trajectory +def test_summarize() -> None: + start = {"type": "session.start", "data": {"backend": "claude", "model": "model-x"}} + events = [start, {"type": "wake", "data": {}}, start] + events.extend( + {"type": "syscall.decision", "data": {"verb": verb, "outcome": outcome}} + for verb, outcome in ( + ("submit", "accepted"), + ("submit", "refused"), + ("message", "refused"), + ("submit", "refused"), + ) + ) + events.append({"type": "session.start", "data": {"backend": "claude", "model": "model-y"}}) + judge = {"type": "session.start", "data": {"backend": "codex", "role": "judge"}} + events.extend([start, judge, {"type": "wake", "data": ["malformed"]}]) + assert trajectory.summarize(iter(events)) == { + "decisions": 4, + "refusals": 3, + "refusals_by_verb": {"submit": 2, "message": 1}, + "authors": ["claude/model-x", "claude/model-y"], + } + assert trajectory.summarize([]) == { + "decisions": 0, + "refusals": 0, + "refusals_by_verb": {}, + "authors": [], + } + + +def test_cli_summary(tmp_path: Path, capsys) -> None: + from outerloop.cli import main + + directory = tmp_path / "runs" / "sample" + trajectory.record(directory, "session.start", {"backend": "claude", "model": "model-x"}) + for verb, outcome in ( + ("submit", "accepted"), + ("submit", "refused"), + ("message", "refused"), + ("submit", "refused"), + ): + trajectory.record(directory, "syscall.decision", {"verb": verb, "outcome": outcome}) + args = ["trajectory", "sample", "--root", str(tmp_path)] + for flags in ([], ["--verify"]): + assert main(args + flags) == 0 + assert capsys.readouterr().out.splitlines()[-1] == ( + "refusals: 3 of 4 decisions (submit 2, message 1); author: claude/model-x" + ) + assert main(args + flags + ["--json"]) == 0 + expected = trajectory.read(directory) + assert json.loads(capsys.readouterr().out) == ( + {"events": expected, "problems": []} if flags else expected + ) + + def test_chain_detects_edits_and_gaps(tmp_path: Path) -> None: for n in range(3): trajectory.record(tmp_path, "measure", {"value": n}, at=n)