diff --git a/CHANGELOG.md b/CHANGELOG.md index 6dc5443d..c7d79b22 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ Versions follow [SemVer](https://semver.org). ## [Unreleased] +- Add a printable agent-text inventory and protocol wording lint, with clearer refusal recovery guidance (#487, parts 1–2). - `BENCHMARKS.md` shows `—` instead of "provenance unknown" in the main-commit column of a baseline row that nothing has improved on yet, as on a freshly seeded repository. - The README and the getting-started guide link to the quickstart repository. It runs two CPU benchmarks with the local forge and local compute, and needs no GitHub account or cluster (#563). The docs and the startup warning no longer say an uncontained session runs in a sandbox: it gets a scrubbed environment and its own home, but can read any file the account can. - The advisory review covers exactly the head its triggering event named (`REVIEW_HEAD_SHA`, #462). If the PR has moved off that head, nothing is reviewed or posted. Otherwise the diff is read between fixed commits, every envelope records the reviewed `head_sha`, and the review is attached to that commit. When GitHub refuses its inline comments, the fallback is a body-only review on the same commit, never an unpinned comment (#569). The summarizer skips clean when its lenses read different heads. For a fork PR (`REVIEW_CHECKOUT_BASE=1`) the brief says the tree is the base branch and the change is only in the diff. Upgrading: envelopes without `head_sha` (from an older reviewer ref) still post; callers that do not pass REVIEW_HEAD_SHA behave as before. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 25c721ae..f96e0410 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -61,6 +61,19 @@ uv run pytest `uv lock`; CI checks the lock. - Docs are plain prose. Short sentences, no metaphors, no internal jargon. +New agent-facing text must be reachable from `outerloop.agent_text.inventory()`. +Keep text with its renderer and add a representative rendering to the inventory; +review it with `python -m outerloop.agent_text`. The fast lint in +`tests/test_agent_text.py` checks protocol terms, syscall names, and refusal recovery +guidance. Keep refusals inline: the inventory reads explicit AST constructs listed in +`REFUSAL_CONSTRUCTS`, rendering f-string expressions as `{}`. Input constraints +use category `validation` and need no redundant next step. The syscall CLI mixes +constraints and state refusals in `ToolError`; `CLI_REFUSALS` identifies the latter +by function and call ordinal, so update it when adding or reordering those calls. Delivery reasons composed into a +refusal use category `reason`; the enclosing refusal carries recovery guidance. +Kernel refusals require a next step; any exception needs a specific reason beside +the lint rule. + ## Tests | Tier | Marker | Runs | diff --git a/src/outerloop/agent_text.py b/src/outerloop/agent_text.py new file mode 100644 index 00000000..1e7e8ce1 --- /dev/null +++ b/src/outerloop/agent_text.py @@ -0,0 +1,316 @@ +"""Rendered protocol text for review: ``python -m outerloop.agent_text``.""" + +from __future__ import annotations + +import argparse +import ast +from dataclasses import dataclass, replace +from itertools import product +from pathlib import Path + +from outerloop import syscall_cli +from outerloop.brief import BriefInputs, BudgetState, Task, build_brief, render +from outerloop.inbox import Message, base_moved_text, budgets_line, render_inbox +from outerloop.orchestrator import _render_author_inbox +from outerloop.syscall import tool_update_note + + +@dataclass(frozen=True) +class Entry: + source: str + text: str + + +def verb_parsers() -> dict[str, argparse.ArgumentParser]: + """Discover verbs from the same parser the installed tool uses.""" + return { + name: parser + for action in syscall_cli.build_parser()._actions + if isinstance(action, argparse._SubParsersAction) + for name, parser in action.choices.items() + } + + +# Each row names a source module, scope, construct, and default category. +# Scope is empty for module-wide calls. Operator CLI errors are deliberately +# excluded: only the installed syscall CLI and messages sent to agents belong here. +REFUSAL_CONSTRUCTS = ( + ("syscall_cli", "", "ToolError", "validation"), + ("syscall", "", "SyscallError", "validation"), + ("syscall", "read_request", "assign:problem", "validation"), + ("syscall", "budget_error", "return", "refusal"), + ("syscall", "_deliver_artifacts", "return", "refusal"), + ("attempt", "deliver_messages", "refuse", "reason"), + ("attempt", "", "message", "refusal"), + ("attempt", "_admit_working_tree", "ValueError", "refusal"), + ("attempt", "ScopeHistoryError", "super().__init__", "refusal"), + ("attempt", "publish", "refuse", "refusal"), + ("orchestrator", "attempt_once", "assign:problem", "refusal"), + ("orchestrator", "attempt_once", "assign:no_backend", "refusal"), + ("orchestrator", "attempt_once", "keyword:problem", "refusal"), + ("orchestrator", "attempt_once", "message", "refusal"), + ("tick", "", "message", "refusal"), +) +# Message keys that mark a refusal; any other kernel message with a literal text is a note. +REFUSAL_KEYS = {"message-refused", "refusal", "capacity-refusal"} + +# ToolError mixes input constraints with state/policy refusals. Within these +# functions, these call ordinals (in source order) are the state/policy checks. +# All other ToolError calls state an input constraint or an invalid input file. +CLI_REFUSALS = { + "_load_staged": {1}, + "_check_stop": {1}, + "cmd_stop": {1, 3, 4, 5, 6}, + "cmd_message": {3}, + "cmd_reports": {1, 2}, +} + + +def _literal_text(node: ast.AST) -> str: + """Render literal portions only; never evaluate source expressions.""" + if isinstance(node, ast.Constant) and isinstance(node.value, str): + return node.value + if isinstance(node, ast.JoinedStr): + return "".join( + "{}" if isinstance(value, ast.FormattedValue) else _literal_text(value) + for value in node.values + ) + return "" + + +def _strings(node: ast.AST) -> list[str]: + if isinstance(node, ast.IfExp): + return _strings(node.body) + _strings(node.orelse) + if isinstance(node, ast.Tuple): + return [text for value in node.elts for text in _strings(value)] + text = _literal_text(node) + return [text] if text else [] + + +def refusal_inventory() -> list[Entry]: + """Read inline refusals from explicit AST constructs, without running them.""" + entries = [] + trees = { + module: ast.parse(Path(__file__).with_name(f"{module}.py").read_text(encoding="utf-8")) + for module in dict.fromkeys(row[0] for row in REFUSAL_CONSTRUCTS) + } + for module, scope, construct, category in REFUSAL_CONSTRUCTS: + tree = trees[module] + root = next(n for n in tree.body if getattr(n, "name", None) == scope) if scope else tree + # Associate module-wide error calls with their containing function. + functions = { + id(node): function.name + for function in ast.walk(root) + if isinstance(function, (ast.FunctionDef, ast.AsyncFunctionDef)) + for node in ast.walk(function) + } + counts: dict[str, int] = {} + for node in sorted( + ast.walk(root), key=lambda n: (getattr(n, "lineno", 0), getattr(n, "col_offset", 0)) + ): + value: ast.AST | None = None + message_key: str | None = None + if isinstance(node, ast.Call) and ast.unparse(node.func) == construct: + index = 1 if scope == "deliver_messages" else 0 + if len(node.args) > index: + value = node.args[index] + elif ( + ( + isinstance(node, ast.Assign) + and any( + isinstance(target, ast.Name) and construct == f"assign:{target.id}" + for target in node.targets + ) + ) + or (isinstance(node, ast.keyword) and construct == f"keyword:{node.arg}") + or (isinstance(node, ast.Return) and construct == "return") + ): + value = node.value + elif ( + construct == "message" + and isinstance(node, ast.Call) + and isinstance(node.func, ast.Name) + and node.func.id == "Message" + and len(node.args) > 6 + and isinstance(node.args[6], ast.Dict) + ): + message_key = _literal_text(node.args[5]) + value = next( + ( + v + for k, v in zip(node.args[6].keys, node.args[6].values, strict=True) + if k is not None and _literal_text(k) == "text" + ), + None, + ) + if value is None: + continue + function = functions.get(id(node), scope) + for text in _strings(value): + counts[function] = counts.get(function, 0) + 1 + ordinal = counts[function] + kind = category + if message_key is not None and message_key.split(":")[0] not in REFUSAL_KEYS: + kind = "note" + if module == "syscall_cli" and ordinal in CLI_REFUSALS.get(function, set()): + kind = "refusal" + entries.append(Entry(f"{kind}:{module}:{function}:{construct}:{ordinal}", text)) + return entries + + +def inventory() -> list[Entry]: + """Collect real renderers with deterministic, public example data and source literals.""" + inputs = BriefInputs( + task=Task( + "Compare two search methods", "search", "score, higher is better", "Report results" + ), + contract_text="benchmarks:\n - name: search", + ruler="Score over a frozen evaluation set.", + budget=BudgetState(6.0, 4), + lessons="The previous method plateaued.", + recent_reports=("A previous experiment found no improvement.",), + report_archive=True, + syscalls=True, + launch_budget=4, + sleep_budget=6, + gpu_hour_budget=6.0, + eval_minutes_default=10, + line_ref="research/agent-01", + memory="Compare the two methods.", + line_divergence="1 file changed", + guidance="Prefer small experiments.", + ) + entries = [] + for name, sample in ( + ("gate", inputs), + ("self-report", replace(inputs, verification="self_report")), + ( + "channels-off", + replace( + inputs, + channels=dict.fromkeys( + ("messages", "siblings", "branches", "shared_reports"), False + ), + ), + ), + ( + "minimal", + BriefInputs(task=inputs.task, contract_text=inputs.contract_text, ruler=inputs.ruler), + ), + ): + entries.append(Entry(f"brief:{name}", render(build_brief(sample, "2026-01-01T00:00:00Z")))) + entries.append(Entry("verb:syscall --help", syscall_cli.build_parser().format_help())) + entries.extend( + Entry(f"verb:{name} --help", parser.format_help()) + for name, parser in verb_parsers().items() + ) + for messages, siblings, verification in product( + (True, False), (True, False), ("gate", "self_report") + ): + entries.append( + Entry( + f"note:tool-update:messages={messages}:siblings={siblings}:{verification}", + tool_update_note( + ".outerloop", messages=messages, siblings=siblings, verification=verification + ), + ) + ) + entries.extend(refusal_inventory()) + for messages, verification in product((True, False), ("gate", "self_report")): + entries.append( + Entry( + f"wake:author:messages={messages}:{verification}", + _render_author_inbox( + [], + budgets=budgets_line(launches=4, sleeps=6, gpu_hours=6), + redact_secrets=(), + messages_enabled=messages, + verification=verification, + ), + ) + ) + samples: list[tuple[str, str, dict]] = [ + ( + "launch-result", + "job", + { + "name": "experiment", + "exit_code": 0, + "elapsed": 2, + "why": "Compare methods", + "delivered": ["results/score.json"], + "skipped": ["missing artifact"], + "stdout_tail": "score: 1", + "stderr_tail": "", + }, + ), + ("launch-result", "job", {"name": "experiment", "slurm_state": "FAILED"}), + ("launch-result", "job", {"name": "experiment"}), + ("comment", "human", {"association": "MEMBER", "body": "Please compare methods."}), + ("comment", "human", {"context_only": True, "body": "Review context."}), + ("check-result", "ci", {"text": "Check completed", "url": "", "log_tail": "OK"}), + ( + "panel-verdict", + "panel", + { + "head": "abcdef0", + "findings": [ + { + "blocking": True, + "file": "solver.py", + "line": 1, + "summary": "Boundary error", + "detail": "Check empty input", + }, + {"blocking": False, "file": "solver.py", "summary": "Add an example"}, + ], + "transcript": "Review complete", + "text": "Read the findings.", + }, + ), + ( + "gate-verdict", + "kernel", + {"sealed_sha": "abcdef0", "base_sha": "1234567", "text": "Measurement complete"}, + ), + ("base-moved", "git", {"text": base_moved_text("a" * 40, "main")}), + ("base-moved", "git", {"text": base_moved_text("a" * 40, "unusual branch")}), + ("base-moved", "git", {"text": "Base changed. What landed:\nA change"}), + ("head-moved", "git", {"text": "The PR head changed."}), + ("note", "kernel", {"text": "Read the result.", "quoted_text": "Sample output"}), + ("agent-message", "agent", {"text": "Compare results."}), + ] + for index, (kind, source, payload) in enumerate(samples, 1): + message = Message( + index, + kind, + source, + "owner/repo#1", + 0, + f"sample:{index}", + payload, + origin="agent-02", + context_id="agent-01", + to="agent-01", + in_reply_to="missing" if index == 14 else "", + ) + entries.append( + Entry( + f"inbox:{kind}:{index}", + render_inbox( + [message], + budgets=budgets_line(launches=4, sleeps=6), + clock="Session time remaining: 10 minutes", + ), + ) + ) + return entries + + +def main() -> None: + for entry in inventory(): + print(f"=== {entry.source} ===\n{entry.text}\n") + + +if __name__ == "__main__": + main() diff --git a/src/outerloop/attempt.py b/src/outerloop/attempt.py index 77336110..7efad233 100644 --- a/src/outerloop/attempt.py +++ b/src/outerloop/attempt.py @@ -1172,7 +1172,10 @@ def refuse(index: int, reason: str) -> None: time.time(), f"message-refused:{record.run_id}:{entry['counter']}", { - "text": f"Message #{index} was not delivered: {reason}.", + "text": ( + f"Message #{index} was not delivered: {reason}. " + "Use this reason to revise your message or recipient." + ), "quoted_text": f"Recipient: {item.get('to')!r}", "context_only": True, }, @@ -1192,7 +1195,13 @@ def refuse(index: int, reason: str) -> None: if destination not in ("thread", "self", record.agent_id) and not record.channels.get( "messages", True ): - refuse(index, "inter-agent messages disabled by channels.messages") + refuse( + index, + ( + "inter-agent messages disabled by channels.messages; " + "use --to self or --to thread" + ), + ) continue reference = "" n = item.get("reply_to") @@ -1232,7 +1241,13 @@ def refuse(index: int, reason: str) -> None: refuse(index, "no live run on this target (absent or ended)") continue if recipient.run_id != record.run_id and not recipient.channels.get("messages", True): - refuse(index, "recipient has inter-agent messages disabled by channels.messages") + refuse( + index, + ( + "recipient has inter-agent messages disabled by channels.messages; " + "use another recipient" + ), + ) continue key = f"agent-msg:{record.run_id}:{entry['counter']}" entry["recipient"] = recipient.run_id @@ -1291,7 +1306,13 @@ def refuse(index: int, reason: str) -> None: ) ): item = entry["item"] - refuse(index, "inter-agent messages disabled by channels.messages") + refuse( + index, + ( + "inter-agent messages disabled by channels.messages; " + "use --to self or --to thread" + ), + ) append(run_dir, Message(**entry["message"])) elif ( recipient.run_id != record.run_id @@ -1306,7 +1327,13 @@ def refuse(index: int, reason: str) -> None: >= 4 ): item = entry["item"] - refuse(index, "recipient already has 4 undelivered messages from you") + refuse( + index, + ( + "recipient already has 4 undelivered messages from you; " + "wait for delivery before retrying" + ), + ) message = Message(**entry["message"]) append(run_dir, message) else: @@ -2439,7 +2466,10 @@ def _admit_working_tree(ws: Workspace, contract: Contract, base_branch: str, lin common = _scope_revision(ws, f"refs/remotes/origin/{base_branch}", "HEAD") violations = _out_of_scope_working_tree(ws, contract, common, line_ref) if violations: - raise ValueError(f"snapshot skipped: scope admission refused: {', '.join(violations)}") + raise ValueError( + f"snapshot skipped: scope admission refused: {', '.join(violations)}; " + "fix the out-of-scope changes before retrying" + ) def _push_line_snapshot( @@ -4255,7 +4285,9 @@ def refuse( github.comment(record.target, number, f"{REPLY_MARKER}\n{note}") if not result.measured_paths: - return refuse("Publish refused: no code change; metric noise.") + return refuse( + "Publish refused: no code change; metric noise. Submit a code change or use `stop`." + ) try: ws.fetch_origin() diff --git a/src/outerloop/orchestrator.py b/src/outerloop/orchestrator.py index a8d8abec..4491879c 100644 --- a/src/outerloop/orchestrator.py +++ b/src/outerloop/orchestrator.py @@ -1950,13 +1950,16 @@ def _not_run_note(request: SyscallRequest | None) -> str: ): request = dc_replace( request, - problem="self_report requires finite claimed_value and claimed_baseline", + problem=( + "self_report requires finite claimed_value and claimed_baseline; " + "use `submit --claimed-value --claimed-baseline `." + ), ) if request.withdraw and not request.problem: problem = ( on_withdraw(request.withdraw) if on_withdraw is not None - else "Withdrawal requires an open PR." + else "Withdrawal requires an open PR. Use `stop` without --withdraw instead." ) request = dc_replace(request, problem=problem) if request.problem: @@ -1976,7 +1979,10 @@ def _not_run_note(request: SyscallRequest | None) -> str: time.time(), f"refusal:{session.session_id}:{inbox_seq}", { - "text": "Your syscall request was REFUSED and nothing was launched.", + "text": ( + "Your syscall request was REFUSED and nothing was launched. " + "Fix the request before retrying." + ), "quoted_text": request.problem, }, origin=inbox_dir.name, @@ -2229,7 +2235,10 @@ def _not_run_note(request: SyscallRequest | None) -> str: time.time(), f"refusal:{session.session_id}:{inbox_seq}", { - "text": "Your syscall request was REFUSED and nothing was launched.", + "text": ( + "Your syscall request was REFUSED and nothing was launched. " + "Fix the request before retrying." + ), "quoted_text": problem, }, origin=inbox_dir.name, @@ -2392,7 +2401,8 @@ def _not_run_note(request: SyscallRequest | None) -> str: { "text": ( "Your sibling launch request was REFUSED " - "and nothing was launched." + "and nothing was launched. " + "Wait for capacity before retrying the launch." ), "quoted_text": str(exc), }, diff --git a/src/outerloop/syscall.py b/src/outerloop/syscall.py index cd2aa702..5d660976 100644 --- a/src/outerloop/syscall.py +++ b/src/outerloop/syscall.py @@ -894,12 +894,13 @@ def budget_error( if sleeps_used + 1 > sleep_budget: return ( f"sleep budget exhausted ({sleeps_used}/{sleep_budget} used): " - "conclude with what you have" + "use `stop` with what you have" ) if launches_used + len(request.launches) > launch_budget: return ( f"launch budget would be exceeded: {launches_used} used + " - f"{len(request.launches)} requested > {launch_budget} allowed" + f"{len(request.launches)} requested > {launch_budget} allowed; " + "use fewer launches or `stop`." ) if gpu_hour_budget is not None and (gpus > 0 or any(g > 0 for g in suite_gpus)): cost = gpu_hours_cost( @@ -1016,7 +1017,12 @@ def _deliver_artifacts( channel = workspace / chan results_root = channel / RESULTS_SUBDIR if channel.is_symlink() or results_root.is_symlink(): - return (), (f"artifacts not delivered: {chan} channel is a symlink (refused)",) + return (), ( + ( + f"artifacts not delivered: {chan} channel is a symlink (refused); " + "use real directories for the channel and its results path" + ), + ) # an earlier delivery under this name goes first, even when this job wrote # nothing, so a re-used name never shows stale results beside fresh ones diff --git a/src/outerloop/syscall_cli.py b/src/outerloop/syscall_cli.py index 1b4641bc..2ddfafe8 100644 --- a/src/outerloop/syscall_cli.py +++ b/src/outerloop/syscall_cli.py @@ -156,7 +156,9 @@ def cmd_stop(root: Path, args: argparse.Namespace) -> str: except (OSError, ValueError): open_pr = False if not open_pr: - raise ToolError("Withdrawal requires an open PR.") + raise ToolError( + "Withdrawal requires an open PR. Use `stop` without --withdraw instead." + ) if not args.withdraw.strip() or len(args.withdraw) > MAX_REPLY_CHARS: raise ToolError(f"withdrawal reason must contain 1 to {MAX_REPLY_CHARS} chars") staged = _load_staged(root) @@ -283,7 +285,9 @@ def cmd_message(root: Path, args: argparse.Namespace) -> str: if args.to not in ("thread", "self") and not _policy(root).get("channels", {}).get( "messages", True ): - raise ToolError("inter-agent messages disabled by channels.messages") + raise ToolError( + "inter-agent messages disabled by channels.messages; use --to self or --to thread." + ) if args.reply_to is not None and not any(m["seq"] == args.reply_to for m in _messages(root)): raise ToolError(f"unknown inbox message #{args.reply_to}") if (args.text is None) == (args.file is None): diff --git a/tests/test_agent_text.py b/tests/test_agent_text.py new file mode 100644 index 00000000..6c19108a --- /dev/null +++ b/tests/test_agent_text.py @@ -0,0 +1,177 @@ +"""A small protocol glossary over the text solvers actually receive.""" + +import re +from pathlib import Path + +import pytest + +from outerloop.agent_text import Entry, inventory, refusal_inventory, verb_parsers + +# These phrases name public actions or research data, not kernel state. +LEGITIMATE_TERMS = { + "stages a public message": "Preparing a message is part of the public syscall protocol.", + "stage a launch": "Preparing a request is part of the public syscall protocol.", + "stage a job": "Preparing a request is part of the public syscall protocol.", + "stage a submit": "Preparing a request is part of the public syscall protocol.", + "stage `submit": "Preparing a request is part of the public syscall protocol.", + "stage `stop": "Preparing a request is part of the public syscall protocol.", + "stage them again": "Preparing a request is part of the public syscall protocol.", + "stage the files": "Git staging is an agent action, not kernel state.", + "stage them": "Git staging during a merge is an agent action.", + "a sweep": "An experiment array is a parameter sweep, not a kernel cycle.", + "whole sweep": "This is the experiment array's concurrency limit.", + "these sweeps": "The author's experiment arrays, not a kernel cycle.", + "record one finding": "Record is a verb here, not an internal run record.", + "the records live on the research-log branch": "These are public benchmark results.", +} + +# These refusals need policy decisions, not invented promises about recovery. +REFUSAL_EXCEPTIONS = { + "attempt:publish:refuse:2": ("A changed measurement contract needs a new comparison policy."), + "attempt:publish:refuse:3": ("A changed measurement contract needs a new comparison policy."), + "attempt:publish:refuse:4": ( + "The cause may require operator intervention rather than a solver retry." + ), + "attempt:publish:refuse:5": ("Reopening a closed PR requires a separate policy decision."), + "attempt:publish:refuse:6": ( + "Recovery depends on which of the pinned and current bases moved." + ), + "attempt:publish:refuse:7": ("The solver cannot manage the forge's auto-merge setting."), + "attempt:publish:refuse:8": ( + "Recovery depends on whether the external head change should be retained." + ), + "attempt:publish:refuse:9": ( + "Missing history and forge outages require different recovery actions." + ), + "attempt:publish:refuse:10": ( + "An external head change needs ancestry inspection before another publish." + ), +} + +# Inline code that is deliberately not a syscall command. +CODE_DATA = { + ".outerloop/": "Channel directory.", + "research/agent-01": "Example branch.", + "experiment": "Example launch handle.", + "kernel -> you": "Inbox sender heading.", +} + + +def violations(entry: Entry) -> list[str]: + verbs = verb_parsers() + text = " ".join(entry.text.split()) + # The historical spelling must be named when upgrading an old session. + if entry.source.startswith("note:tool-update:"): + text = text.replace("The `end` verb is now `stop`.", "") + errors = [] + if re.search(r"`end\b|\bend\s+--|\bend\s+(?:verb|command)\b", text, re.I): + errors.append("obsolete end command") + glossary = text.lower() + for phrase in LEGITIMATE_TERMS: + glossary = glossary.replace(phrase, "") + if re.search(r"\b(?:legs?|leases?|ticks?|sweeps?|stages?|records?)\b", glossary): + errors.append("kernel vocabulary") + if entry.source.startswith("refusal:") and entry.source[8:] not in REFUSAL_EXCEPTIONS: + cues = "|".join( + ( + *verbs, + "retry", + "retrying", + "use", + "run", + "wait", + "fix", + "write", + "omit", + "provide", + "change", + ) + ) + if not re.search(rf"\b(?:{cues})\b", text, re.I): + errors.append("refusal has no next step") + # Commands are inline code or explicit syscall invocations; prose and + # fenced example payloads are not a command language. + text = re.sub(r"```.*?```", "", text) + for code in re.findall(r"(? 2 and words[1].endswith("/syscall"): + verb = words[2] + if verb == "": # Help notation, not a literal command. + continue + if verb not in verbs: + errors.append(f"unknown syscall: {verb}") + for verb in re.findall(r"(?:\bpython\s+\S*/syscall\s+|\bsyscall\s+)([a-z][a-z-]*)", text): + if verb not in verbs and verb not in {"tool", "request", "file", "keys", "only"}: + errors.append(f"unknown syscall: {verb}") + return errors + + +def test_inventory_protocol() -> None: + entries = inventory() + assert len({entry.source for entry in entries}) == len(entries) + assert all(entry.text for entry in entries) + assert {f"verb:{verb} --help" for verb in verb_parsers()} <= {e.source for e in entries} + assert {"refusal:" + key for key in REFUSAL_EXCEPTIONS} <= {e.source for e in entries} + failures = [f"{entry.source}: {error}" for entry in entries for error in violations(entry)] + assert not failures, "\n".join(failures) + texts = "\n".join(entry.text for entry in entries) + # kernel notes outside refusals are collected too, from every module that sends them + assert "Your session ended without a syscall" in texts + assert "The PR head moved to" in texts + # delivery reasons are composed into "…not delivered: {reason}." and carry no stop + reasons = [e for e in entries if e.source.startswith("reason:")] + assert reasons and not [e.source for e in reasons if e.text.endswith(".")] + + +@pytest.mark.parametrize( + "source,text", + [ + ("note:sample", "Use `end --withdraw reason`."), + ("note:sample", "Wait for the next tick."), + ("refusal:sample", "Invalid input."), + ("note:sample", "Use `teleport` or python .outerloop/syscall vanish."), + ], +) +def test_lint_detects_drift(source: str, text: str) -> None: + assert violations(Entry(source, text)) + + +@pytest.mark.parametrize("text", ["--minutes must be a positive integer", "at most 8 messages"]) +def test_validation_category_exempts_only_next_step(text: str) -> None: + assert not violations(Entry("validation:sample", text)) + assert violations(Entry("refusal:sample", text)) + assert violations(Entry("validation:sample", "Use `end`.")) + assert violations(Entry("validation:sample", "Wait for a tick.")) + assert violations(Entry("validation:sample", "Use `teleport`.")) + + +def test_collector_reads_new_calls_without_evaluating_expressions(monkeypatch) -> None: + original_read = Path.read_text + + def read_text(path, *args, **kwargs): + if path.name == "syscall_cli.py": + return """ +def cmd_launch(): + "An unrelated docstring." + raise ToolError(f"Wait for a tick {never_execute()!r:>10}.") + raise ToolError("new " "constraint") + raise ValueError("An unrelated exception.") + +def _load_staged(): + raise ToolError("An input-like sentence is still a state refusal here.") +""" + return original_read(path, *args, **kwargs) + + monkeypatch.setattr(Path, "read_text", read_text) + entries = [entry for entry in refusal_inventory() if ":syscall_cli:" in entry.source] + assert entries == [ + Entry("validation:syscall_cli:cmd_launch:ToolError:1", "Wait for a tick {}."), + Entry("validation:syscall_cli:cmd_launch:ToolError:2", "new constraint"), + Entry( + "refusal:syscall_cli:_load_staged:ToolError:1", + "An input-like sentence is still a state refusal here.", + ), + ]