diff --git a/CHANGELOG.md b/CHANGELOG.md index b2fedede..75e066d1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ Versions follow [SemVer](https://semver.org). ## [Unreleased] +- Session end records each tool call from the retained Claude or Codex log as a `tool.call` trajectory event, once per call even when a session resumes. `outerloop trajectory` and a new Process section in the PR body show counts by tool. Older kernels ignore these events. - `outerloop status` shows a run's priced session cost as a lower bound when some legs are unpriced (a wake killed mid-turn leaves no usage event), for example `session-cost>=$4.2000 (1 unpriced)`, instead of hiding the priced part behind `unknown`; the status row and JSON gain the priced-leg count (#516). - The pre-PR panel accepts a composite change — a PR that carries several changes — when the report gives each component's isolated measurement and the combination's under the same protocol and the combination is what the kernel measured. A bundle without those isolated measurements stays blocked (#489). - Claude sessions can use a subscription token from `claude setup-token` (`sk-ant-oat...`). The harness sends it as `CLAUDE_CODE_OAUTH_TOKEN`, not `ANTHROPIC_API_KEY`, and `init` accepts it. A panel judge on a token runs `claude --safe-mode`, because `--bare` cannot read OAuth tokens. Both modes ignore the checkout's CLAUDE.md, hooks, plugins and MCP servers. The pinned Claude Code release (2.1.285) has `--safe-mode`. diff --git a/docs/design/trajectory.md b/docs/design/trajectory.md index 94f04790..73e48f8c 100644 --- a/docs/design/trajectory.md +++ b/docs/design/trajectory.md @@ -46,4 +46,10 @@ directory. Self-reported measures record the ruler alone: they have no evaluation image or result files. A pending submission does not yet name the trajectory head: kernels before this one reject unknown pending fields, so the pending reader now ignores them, and a later release can add the field without -breaking this one. Tool calls and attestation are later steps. +breaking this one. Session end also appends one `tool.call` event per tool call +found in the retained, redacted native log: Claude project transcripts and Codex +rollout JSONL are parsed host-side, bounded to 500 calls per session, and Hermes +or unknown formats yield nothing. Each event carries the tool name, repo-relative +files, a bounded shell command and, when the log has them, the exit code and +outcome — never file contents or tool output. `outerloop trajectory` counts them +by tool and the PR body summarizes the top tools. Attestation is a later step. diff --git a/src/outerloop/attempt.py b/src/outerloop/attempt.py index 41085212..efc57156 100644 --- a/src/outerloop/attempt.py +++ b/src/outerloop/attempt.py @@ -152,12 +152,22 @@ from outerloop.syscall import refresh_tool as syscall_refresh_tool from outerloop.syscall import write_siblings as syscall_write_siblings from outerloop.trajectory import file_refs +from outerloop.trajectory import read as read_trajectory from outerloop.trajectory import record as record_event +from outerloop.trajectory import summarize as summarize_trajectory from outerloop.verifier import MAX_CLAIM_CHARS REPLY_MARKER = marker("followup") +def tool_call_counts(run_dir: Path) -> dict[str, int]: + """The run's recorded tool calls by tool; an unreadable chain reads as none.""" + try: + return summarize_trajectory(read_trajectory(run_dir))["tool_calls_by_tool"] + except OSError: + return {} + + def _pr_number(pr_url: str) -> int: tail = pr_url.rstrip("/").rsplit("/", 1)[-1] if not tail.isdigit(): @@ -4724,6 +4734,7 @@ def refuse( display_digits=bench.display_digits, experiments=experiments_rows(run_dir), base_sha=base_sha, + tool_calls=tool_call_counts(run_dir), ) body += f"\n\n{progress_link(config.target, github)}\n" if issue_number: diff --git a/src/outerloop/cli.py b/src/outerloop/cli.py index 9abea327..9a369112 100644 --- a/src/outerloop/cli.py +++ b/src/outerloop/cli.py @@ -1360,9 +1360,16 @@ def main(argv: list[str] | None = None) -> int: f"{verb} {count}" for verb, count in summary["refusals_by_verb"].items() ) authors = ", ".join(summary["authors"]) or "unknown" + tools = ", ".join( + f"{tool} {count}" + for tool, count in sorted( + summary["tool_calls_by_tool"].items(), key=lambda item: (-item[1], item[0]) + ) + ) print( f"refusals: {summary['refusals']} of {summary['decisions']} decisions" - f"{f' ({verbs})' if verbs else ''}; author: {authors}" + f"{f' ({verbs})' if verbs else ''}; author: {authors}; " + f"tool calls: {summary['tool_calls']}{f' ({tools})' if tools else ''}" ) return 1 if problems and problems != ["no trajectory"] else 0 if args.command == "status": diff --git a/src/outerloop/orchestrator.py b/src/outerloop/orchestrator.py index 45c0c6cb..82d20c49 100644 --- a/src/outerloop/orchestrator.py +++ b/src/outerloop/orchestrator.py @@ -2821,6 +2821,20 @@ def experiments_section(rows: list[dict[str, Any]]) -> list[str]: return lines +MAX_PROCESS_TOOLS = 5 + + +def process_section(tool_calls: dict[str, int] | None) -> list[str]: + """Total and top tools from the run's trajectory; empty when there are none.""" + total = sum((tool_calls or {}).values()) + if not total: + return [] + ranked = sorted((tool_calls or {}).items(), key=lambda item: (-item[1], item[0])) + top = ", ".join(f"{name} {count}" for name, count in ranked[:MAX_PROCESS_TOOLS]) + calls = "call" if total == 1 else "calls" + return ["", "## Process", "", f"{total} tool {calls} from the run's trajectory: {top}."] + + def pr_body( result: AttemptResult, config: RunConfig, @@ -2829,6 +2843,7 @@ def pr_body( experiments: list[dict[str, Any]] | None = None, *, base_sha: str = "", + tool_calls: dict[str, int] | None = None, ) -> str: """The PR body for an improved run: the author's report, the experiments the run actually ran (from the launch ledger), the measured table, and @@ -2909,6 +2924,7 @@ def pr_body( "", *report_lines, *experiments_section(experiments or []), + *process_section(tool_calls), "", "## Measured", "", @@ -2940,6 +2956,7 @@ def pr_body( "", redact(result.submit_report, redact_secrets)[:MAX_REPORT_BODY], *experiments_section(experiments or []), + *process_section(tool_calls), "", "## Self-reported values", "", diff --git a/src/outerloop/session_evidence.py b/src/outerloop/session_evidence.py index 8471b93e..94e11e13 100644 --- a/src/outerloop/session_evidence.py +++ b/src/outerloop/session_evidence.py @@ -23,6 +23,7 @@ from pathlib import Path from typing import TYPE_CHECKING, Any +from outerloop.tool_calls import record_tool_calls from outerloop.trajectory import session_event if TYPE_CHECKING: @@ -320,6 +321,10 @@ def wrapped( ) except Exception: log.warning("could not retain session evidence", exc_info=True) + try: + record_tool_calls(workspace, backend, result.session_record_path) + except Exception: + log.warning("could not record session tool calls", exc_info=True) session_event( workspace, "session.end", backend, self.model, result.session_id, result=result ) diff --git a/src/outerloop/tool_calls.py b/src/outerloop/tool_calls.py new file mode 100644 index 00000000..051a0fac --- /dev/null +++ b/src/outerloop/tool_calls.py @@ -0,0 +1,296 @@ +"""Tool calls extracted host-side from the retained, redacted native session log. + +Pure parsers skip malformed lines and bound every field; the record hook is +best-effort, like the rest of the trajectory. +""" + +from __future__ import annotations + +import json +import re +from collections.abc import Iterator +from pathlib import Path +from typing import Any + +from outerloop.trajectory import read, record + +MAX_CALLS = 500 +MAX_FILES = 20 +MAX_PATH = 200 +MAX_COMMAND = 200 +MAX_TOOL = 100 +MAX_ID = 100 + +_AGENT_ID = re.compile(r"[A-Za-z0-9][A-Za-z0-9_-]{0,63}") +_EXIT_CODE = re.compile(r"(?:Process exited with code|Exit code)\s*:?\s*(-?\d+)", re.IGNORECASE) +_PATCH_PREFIXES = ( + "*** Add File: ", + "*** Update File: ", + "*** Delete File: ", + "*** Move to: ", +) + + +def _rows(text: str) -> Iterator[dict[str, Any]]: + """JSON objects from a JSONL log; malformed and non-object lines are skipped.""" + for line in text.splitlines(): + line = line.strip() + if not line: + continue + try: + row = json.loads(line) + except ValueError: + continue + if isinstance(row, dict): + yield row + + +def _text(value: Any) -> list[str]: + if isinstance(value, str): + return [value] + if isinstance(value, list): + return [ + part["text"] + for part in value + if isinstance(part, dict) and isinstance(part.get("text"), str) + ] + return [] + + +def _arguments(value: Any) -> dict[str, Any]: + if isinstance(value, dict): + return value + if isinstance(value, str): + try: + parsed = json.loads(value) + except ValueError: + return {} + if isinstance(parsed, dict): + return parsed + return {} + + +def _command(value: Any) -> str: + if isinstance(value, list): + value = " ".join(part for part in value if isinstance(part, str)) + if isinstance(value, str) and value.strip(): + return value[:MAX_COMMAND] + return "" + + +def _relative(value: str, cwd: str) -> str: + """Strip the session's working directory so files read as repo-relative; + a path outside it reads as "" and is dropped.""" + text = value.strip() + root = cwd.rstrip("/") + if root and text.startswith(root + "/"): + text = text[len(root) + 1 :] + if text.startswith(("/", "~")) or ".." in text.split("/"): + return "" + return text[:MAX_PATH] + + +def _files(args: dict[str, Any], cwd: str) -> list[str]: + found: list[str] = [] + for key in ("file_path", "path", "notebook_path"): + value = args.get(key) + if isinstance(value, str): + found.append(value) + edits = args.get("edits") + if isinstance(edits, list): + for edit in edits: + if isinstance(edit, dict) and isinstance(edit.get("file_path"), str): + found.append(edit["file_path"]) + files: list[str] = [] + for value in found: + relative = _relative(value, cwd) + if relative and relative not in files: + files.append(relative) + if len(files) >= MAX_FILES: + break + return files + + +def _exit_code(value: Any) -> int | None: + if not isinstance(value, dict): + return None + for key in ("exitCode", "exit_code", "returnCode", "return_code"): + code = value.get(key) + if isinstance(code, int) and not isinstance(code, bool): + return code + return None + + +def _apply_result(call: dict[str, Any], is_error: Any, text: str, extra: Any) -> None: + """Attach the exit code and outcome a completed call's log line carries.""" + code = _exit_code(extra) + if code is None and (match := _EXIT_CODE.search(text)): + code = int(match.group(1)) + if code is not None: + call["exit"] = code + call["ok"] = code == 0 + elif isinstance(is_error, bool): + call["ok"] = not is_error + + +def claude_calls(text: str) -> Iterator[dict[str, Any]]: + """Claude Code project transcripts: assistant `tool_use` parts matched to + user `tool_result` parts by id, with `toolUseResult` as the exit detail.""" + cwd = "" + calls: list[dict[str, Any]] = [] + by_id: dict[str, dict[str, Any]] = {} + for row in _rows(text): + if isinstance(row.get("cwd"), str): + cwd = row["cwd"] + message = row.get("message") + content = message.get("content") if isinstance(message, dict) else None + if not isinstance(content, list): + continue + for part in content: + if not isinstance(part, dict): + continue + if part.get("type") == "tool_use": + name = part.get("name") + if not isinstance(name, str) or not name.strip(): + continue + args = part.get("input") + args = args if isinstance(args, dict) else {} + call: dict[str, Any] = {"tool": name[:MAX_TOOL], "files": _files(args, cwd)} + if command := _command(args.get("command")): + call["command"] = command + call_id = part.get("id") + if isinstance(call_id, str): + call["id"] = call_id[:MAX_ID] + by_id[call_id] = call + calls.append(call) + elif part.get("type") == "tool_result": + result_id = part.get("tool_use_id") + pending = by_id.get(result_id) if isinstance(result_id, str) else None + if pending is not None: + _apply_result( + pending, + part.get("is_error"), + "\n".join(_text(part.get("content"))), + row.get("toolUseResult"), + ) + yield from calls[:MAX_CALLS] + + +def _patch_files(patch: str, cwd: str) -> list[str]: + files: list[str] = [] + for line in patch.splitlines(): + for prefix in _PATCH_PREFIXES: + if line.startswith(prefix): + relative = _relative(line[len(prefix) :], cwd) + if relative and relative not in files: + files.append(relative) + if len(files) >= MAX_FILES: + return files + return files + + +def codex_calls(text: str) -> Iterator[dict[str, Any]]: + """Codex rollout JSONL: `function_call` (shell) and `custom_tool_call` + (apply_patch) items matched to their outputs by call id.""" + cwd = "" + calls: list[dict[str, Any]] = [] + by_id: dict[str, dict[str, Any]] = {} + for row in _rows(text): + payload = row.get("payload") + if not isinstance(payload, dict): + continue + if row.get("type") == "session_meta": + if isinstance(payload.get("cwd"), str): + cwd = payload["cwd"] + continue + kind = payload.get("type") + if kind in ("function_call", "custom_tool_call"): + name = payload.get("name") + if not isinstance(name, str) or not name.strip(): + continue + args = _arguments(payload.get("arguments")) + patch = payload.get("input") if kind == "custom_tool_call" else None + call: dict[str, Any] = {"tool": name[:MAX_TOOL], "files": _files(args, cwd)} + if isinstance(patch, str): + call["files"] = _patch_files(patch, cwd) + elif isinstance(args.get("input"), str): + call["files"] = _patch_files(args["input"], cwd) + if command := _command(args.get("command") or args.get("cmd")): + call["command"] = command + elif name == "exec" and isinstance(patch, str) and (command := _command(patch)): + # newer Codex runs commands through a script tool whose input + # is the script itself, not JSON arguments + call["command"] = command + call_id = payload.get("call_id") + if isinstance(call_id, str): + call["id"] = call_id[:MAX_ID] + by_id[call_id] = call + calls.append(call) + elif kind in ("function_call_output", "custom_tool_call_output"): + result_id = payload.get("call_id") + pending = by_id.get(result_id) if isinstance(result_id, str) else None + if pending is not None: + output = "\n".join(_text(payload.get("output"))) + _apply_result(pending, None, output, None) + # the script tool reports success in its first words, not an exit code + if "ok" not in pending and output.startswith("Script completed"): + pending["ok"] = True + yield from calls[:MAX_CALLS] + + +def extract(backend: str, text: str) -> list[dict[str, Any]]: + """Tool calls from one retained native log. Unknown backends yield nothing.""" + parser = {"claude": claude_calls, "codex": codex_calls}.get(backend) + return list(parser(text)) if parser is not None else [] + + +def _read_retained(path: Path) -> str: + """The whole retained copy: session evidence already caps it at its + configured size (OUTERLOOP_NATIVE_LOG_MAX_BYTES).""" + try: + return path.read_bytes().decode("utf-8", errors="ignore") + except OSError: + return "" + + +def _actor(directory: Path, workspace: Path) -> str: + try: + state = json.loads((directory / "state.json").read_text()) + except (OSError, ValueError): + return "kernel" + if not isinstance(state, dict) or workspace.relative_to(directory).parts[:1] == ("panel",): + return "kernel" + agent = state.get("agent_id") + return f"author:{agent}" if isinstance(agent, str) and _AGENT_ID.fullmatch(agent) else "kernel" + + +def record_tool_calls(workspace: Path, backend: str, session_record_path: str) -> None: + """Append one `tool.call` event per call in the retained redacted log. + + The record names that copy; the raw home file is never read here. Callers + wrap this, and every step stays best-effort. + """ + if not session_record_path: + return + artifacts = json.loads(Path(session_record_path).read_text()).get("artifacts") + native = artifacts.get("native") if isinstance(artifacts, dict) else None + source = native.get("path") if isinstance(native, dict) else None + if not isinstance(source, str) or not source: + return + calls = extract(backend, _read_retained(Path(source))) + if not calls: + return + directory = next((p for p in workspace.parents if p.parent.name == "runs"), None) + if directory is None: + return + actor = _actor(directory, workspace) + # a resumed session's log repeats its earlier calls: record each id once + seen = { + event["data"].get("id") + for event in read(directory) + if event.get("type") == "tool.call" and isinstance(event.get("data"), dict) + } + for call in calls: + if call.get("id") and call["id"] in seen: + continue + record(directory, "tool.call", call, actor=actor) diff --git a/src/outerloop/trajectory.py b/src/outerloop/trajectory.py index bdad64d1..c4dedad9 100644 --- a/src/outerloop/trajectory.py +++ b/src/outerloop/trajectory.py @@ -119,10 +119,12 @@ def read(run_dir: Path) -> list[dict]: def summarize(events: Iterable[dict[str, Any]]) -> dict[str, Any]: - """Count recorded syscall decisions and collect session authors in first-seen order.""" + """Count recorded syscall decisions, tool calls and session authors in first-seen order.""" decisions = 0 refusals = 0 by_verb: dict[str, int] = {} + tool_calls = 0 + by_tool: dict[str, int] = {} authors: list[str] = [] for event in events: data = event.get("data") @@ -134,6 +136,12 @@ def summarize(events: Iterable[dict[str, Any]]) -> dict[str, Any]: refusals += 1 verb = data.get("verb") or "unknown" by_verb[verb] = by_verb.get(verb, 0) + 1 + elif event.get("type") == "tool.call": + tool_calls += 1 + tool = data.get("tool") + if not isinstance(tool, str) or not tool: + tool = "unknown" + by_tool[tool] = by_tool.get(tool, 0) + 1 elif event.get("type") == "session.start" and data.get("role") != "judge": author = f"{data.get('backend') or 'unknown'}/{data.get('model') or 'unknown'}" if author not in authors: @@ -142,6 +150,8 @@ def summarize(events: Iterable[dict[str, Any]]) -> dict[str, Any]: "decisions": decisions, "refusals": refusals, "refusals_by_verb": by_verb, + "tool_calls": tool_calls, + "tool_calls_by_tool": by_tool, "authors": authors, } diff --git a/tests/test_tool_calls.py b/tests/test_tool_calls.py new file mode 100644 index 00000000..2eb878f6 --- /dev/null +++ b/tests/test_tool_calls.py @@ -0,0 +1,281 @@ +"""Synthetic native logs only: no real session content or tool output.""" + +from __future__ import annotations + +import json +from pathlib import Path + +from outerloop import tool_calls, trajectory +from outerloop.harness import SessionResult +from outerloop.orchestrator import AttemptResult, RunConfig, pr_body +from outerloop.session_evidence import capture_session + + +def test_claude_calls() -> None: + log = "\n".join( + [ + json.dumps( + { + "type": "assistant", + "cwd": "/work", + "message": { + "content": [ + { + "type": "tool_use", + "id": "t1", + "name": "Bash", + "input": {"command": "uv run pytest -q"}, + } + ] + }, + } + ), + json.dumps( + { + "type": "user", + "cwd": "/work", + "message": { + "content": [ + { + "type": "tool_result", + "tool_use_id": "t1", + "content": "1 failed", + "is_error": True, + } + ] + }, + "toolUseResult": {"stdout": "1 failed", "exitCode": 1}, + } + ), + json.dumps( + { + "type": "assistant", + "cwd": "/work", + "message": { + "content": [ + { + "type": "tool_use", + "id": "t2", + "name": "Edit", + "input": {"file_path": "/work/src/bench.py"}, + } + ] + }, + } + ), + json.dumps( + { + "type": "user", + "message": { + "content": [ + { + "type": "tool_result", + "tool_use_id": "t2", + "content": "ok", + "is_error": False, + } + ] + }, + } + ), + '{"type": "assistant", malformed', + ] + ) + assert tool_calls.extract("claude", log) == [ + { + "tool": "Bash", + "files": [], + "command": "uv run pytest -q", + "id": "t1", + "exit": 1, + "ok": False, + }, + {"tool": "Edit", "files": ["src/bench.py"], "id": "t2", "ok": True}, + ] + # a path outside the session's working directory is dropped, not recorded + assert tool_calls._relative("/etc/hosts", "/work") == "" + assert tool_calls._relative("../elsewhere/x.py", "/work") == "" + + +def test_codex_calls() -> None: + log = "\n".join( + [ + json.dumps({"type": "session_meta", "payload": {"id": "s1", "cwd": "/work"}}), + json.dumps( + { + "type": "response_item", + "payload": { + "type": "function_call", + "name": "shell", + "arguments": json.dumps({"command": ["bash", "-lc", "uv run pytest"]}), + "call_id": "c1", + }, + } + ), + json.dumps( + { + "type": "response_item", + "payload": { + "type": "function_call_output", + "call_id": "c1", + "output": "Process exited with code 2\nOutput:\n", + }, + } + ), + json.dumps( + { + "type": "response_item", + "payload": { + "type": "custom_tool_call", + "name": "apply_patch", + "input": "*** Begin Patch\n*** Update File: src/bench.py\n+x\n" + "*** End Patch\n", + "call_id": "c2", + }, + } + ), + json.dumps( + { + "type": "response_item", + "payload": { + "type": "custom_tool_call_output", + "call_id": "c2", + "output": "Success. Updated the following files:\nM src/bench.py\n", + }, + } + ), + # newer Codex: a script tool whose input is the script itself + json.dumps( + { + "type": "response_item", + "payload": { + "type": "custom_tool_call", + "name": "exec", + "input": 'text(await shell("uv run pytest -q"))', + "call_id": "c3", + }, + } + ), + json.dumps( + { + "type": "response_item", + "payload": { + "type": "custom_tool_call_output", + "call_id": "c3", + "output": [{"type": "input_text", "text": "Script completed\nok"}], + }, + } + ), + "not json", + ] + ) + assert tool_calls.extract("codex", log) == [ + { + "tool": "shell", + "files": [], + "command": "bash -lc uv run pytest", + "id": "c1", + "exit": 2, + "ok": False, + }, + {"tool": "apply_patch", "files": ["src/bench.py"], "id": "c2"}, + { + "tool": "exec", + "files": [], + "command": 'text(await shell("uv run pytest -q"))', + "id": "c3", + "ok": True, + }, + ] + assert tool_calls.extract("hermes", log) == [] + assert tool_calls.extract("unknown", log) == [] + + +def test_session_end_appends_tool_call_events(tmp_path: Path) -> None: + run = tmp_path / "runs" / "synthetic" + ws = run / "ws" + ws.mkdir(parents=True) + (run / "state.json").write_text( + json.dumps({"agent_id": "agent-07", "stage": {"phase": "author"}}) + ) + native = run / "ws-home/.claude/projects/project/session-1.jsonl" + native.parent.mkdir(parents=True) + native.write_text( + json.dumps( + { + "type": "assistant", + "cwd": str(ws), + "message": { + "content": [ + { + "type": "tool_use", + "id": "t1", + "name": "Bash", + "input": {"command": "ls"}, + } + ] + }, + } + ) + + "\n" + + json.dumps( + { + "type": "user", + "message": { + "content": [ + { + "type": "tool_result", + "tool_use_id": "t1", + "content": "ok", + "is_error": False, + } + ] + }, + } + ) + + "\n" + ) + + class Adapter: + api_key = "synthetic-credential" + model = "test-model" + container_image = "" + + @capture_session("claude") + def run(self, brief: str, workspace: Path, resume_session_id: str | None = None): + capture = workspace.parent / "capture.json" + capture.write_text("{}") + return SessionResult("completed", False, 0.0, 1, "session-1", "done", str(capture)) + + Adapter().run("brief", ws) + events = trajectory.read(run) + assert [event["type"] for event in events] == ["session.start", "tool.call", "session.end"] + assert events[1]["actor"] == "author:agent-07" + call = {"tool": "Bash", "files": [], "command": "ls", "id": "t1", "ok": True} + assert events[1]["data"] == call + assert events[1]["refs"] == [] + # a resumed leg re-reads the whole session log; its earlier calls are not recorded again + Adapter().run("brief", ws) + assert [e["type"] for e in trajectory.read(run)].count("tool.call") == 1 + + +def test_pr_process_section_and_summary_line(tmp_path: Path, capsys) -> None: + from outerloop.cli import main + + result = AttemptResult(outcome="improved", baseline=1.0, candidate=0.9) + config = RunConfig(target="owner/repo", benchmark="bench") + body = pr_body(result, config, (), tool_calls={"Bash": 12, "Edit": 5}) + assert "## Process" in body + assert "17 tool calls from the run's trajectory: Bash 12, Edit 5." in body + assert "## Process" not in pr_body(result, config, ()) + + run = tmp_path / "runs" / "sample" + trajectory.record(run, "session.start", {"backend": "claude", "model": "model-x"}) + for tool in ("Bash", "Bash", "Edit"): + trajectory.record(run, "tool.call", {"tool": tool, "files": []}) + assert main(["trajectory", "sample", "--root", str(tmp_path)]) == 0 + assert ( + capsys.readouterr() + .out.splitlines()[-1] + .endswith("author: claude/model-x; tool calls: 3 (Bash 2, Edit 1)") + ) diff --git a/tests/test_trajectory.py b/tests/test_trajectory.py index 6168db1b..de571cd4 100644 --- a/tests/test_trajectory.py +++ b/tests/test_trajectory.py @@ -26,12 +26,16 @@ def test_summarize() -> None: "decisions": 4, "refusals": 3, "refusals_by_verb": {"submit": 2, "message": 1}, + "tool_calls": 0, + "tool_calls_by_tool": {}, "authors": ["claude/model-x", "claude/model-y"], } assert trajectory.summarize([]) == { "decisions": 0, "refusals": 0, "refusals_by_verb": {}, + "tool_calls": 0, + "tool_calls_by_tool": {}, "authors": [], } @@ -52,7 +56,8 @@ def test_cli_summary(tmp_path: Path, capsys) -> None: for flags in ([], ["--verify"]): assert main(args + flags) == 0 assert capsys.readouterr().out.splitlines()[-1] == ( - "refusals: 3 of 4 decisions (submit 2, message 1); author: claude/model-x" + "refusals: 3 of 4 decisions (submit 2, message 1); author: claude/model-x; " + "tool calls: 0" ) assert main(args + flags + ["--json"]) == 0 expected = trajectory.read(directory)