diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/golden/__init__.py b/tests/golden/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/golden/agents/__init__.py b/tests/golden/agents/__init__.py new file mode 100644 index 0000000..9973de9 --- /dev/null +++ b/tests/golden/agents/__init__.py @@ -0,0 +1,20 @@ +"""Offline agent fixtures for export→pytest integration tests. + +These agents use a deterministic fake LLM and are designed to demonstrate +scenarios where frozen-replay-based pytest exports diverge when the fix +changes instrumented call inputs or order. They require no API keys or network. +""" + +from .crash_agent import run_agent as crash_run_agent +from .crash_agent import run_agent_fixed as crash_run_agent_fixed +from .fake_llm import FakeLLM +from .triage_agent import run_agent as triage_run_agent +from .triage_agent import run_agent_fixed as triage_run_agent_fixed + +__all__ = [ + "FakeLLM", + "crash_run_agent", + "crash_run_agent_fixed", + "triage_run_agent", + "triage_run_agent_fixed", +] diff --git a/tests/golden/agents/crash_agent.py b/tests/golden/agents/crash_agent.py new file mode 100644 index 0000000..b409a7f --- /dev/null +++ b/tests/golden/agents/crash_agent.py @@ -0,0 +1,80 @@ +"""Crash agent demonstrating extra-calls divergence. + +This agent processes angry customer messages: it parses an LLM response +(wrapped in a markdown fence) and escalates the ticket. The buggy version +crashes on JSON parsing because it doesn't strip the fence. + +The bug causes the agent to crash **before** calling the `escalate` tool: +- Recording the buggy run captures only the LLM call (no escalate) +- The fixed code successfully calls `escalate` +- Frozen replay fails with ReplayExhaustedError (unexpected tool call) +- A failure-trace export can never go green against the fixed code + +This demonstrates Finding #1 from the stepfork test report: frozen-replay +export works when the fix leaves instrumented call inputs and order unchanged, +but not when the fix enables a tool call that the buggy run never reached. + +Also tests redaction: the fake LLM output contains a synthetic secret pattern +(`sk-proj-...`) to verify that stepfork redacts it in the trace. +""" + +from __future__ import annotations + +import json +from typing import Any + +from stepfork import llm_request, trace_tool + + +def _fake_chat(messages: list[dict[str, str]]) -> dict[str, Any]: + """Return a markdown-fenced JSON response with a synthetic secret.""" + del messages # unused + return { + "content": ( + "```json\n" + '{"action": "escalate", ' + '"secret_seen": "sk-proj-ABCDEF1234567890abcdef1234567890"}' + "\n```" + ) + } + + +@trace_tool(name="escalate") +def escalate(reason: str) -> dict[str, Any]: + """Escalate a ticket. Returns the escalation ticket ID.""" + return { + "ticket": "ESC-1", + "reason": reason, + "api_key": "sk-proj-ZYXW9876543210zyxw9876543210ab", + } + + +def _decide(strip_fence: bool) -> dict[str, Any]: + msgs: list[dict[str, str]] = [{"role": "user", "content": "Customer is angry"}] + r = llm_request( + provider="fake", + model="stub-1", + input={"messages": msgs}, + call=lambda: _fake_chat(msgs), + ) + text = r["content"] + + if strip_fence: + text = text.strip().removeprefix("```json").removesuffix("```").strip() + + data = json.loads(text) # buggy version raises JSONDecodeError + res = escalate(data["action"]) + return {"action": data["action"], "ticket": res["ticket"]} + + +def run_agent() -> dict[str, Any]: + """Buggy implementation: crashes on markdown-fenced JSON.""" + return _decide(strip_fence=False) + + +def run_agent_fixed() -> dict[str, Any]: + """Corrected implementation: strips the markdown fence before parsing.""" + return _decide(strip_fence=True) + + +EXPECTED_OUTPUT: dict[str, Any] = {"action": "escalate", "ticket": "ESC-1"} diff --git a/tests/golden/agents/fake_llm.py b/tests/golden/agents/fake_llm.py new file mode 100644 index 0000000..dcd3be2 --- /dev/null +++ b/tests/golden/agents/fake_llm.py @@ -0,0 +1,41 @@ +"""Deterministic offline stub LLM for agent fixtures. + +No network, no API keys required. Returns predefined responses based on +message content patterns. +""" + +from __future__ import annotations + +import json +from typing import Any + + +class FakeLLM: + """Stateful fake LLM that returns deterministic responses.""" + + def __init__(self) -> None: + self.calls = 0 + + def chat(self, messages: list[dict[str, str]]) -> dict[str, Any]: + """Return a deterministic response based on message content.""" + self.calls += 1 + last = messages[-1]["content"] + + if last.startswith("TICKET:"): + return { + "role": "assistant", + "tool_call": { + "name": "lookup_customer", + "args": {"email": "ana@example.com"}, + }, + } + + if last.startswith("CUSTOMER:"): + cust = json.loads(last[len("CUSTOMER:") :]) + priority = "high" if cust.get("plan") == "enterprise" else "normal" + return { + "role": "assistant", + "content": json.dumps({"priority": priority, "refund_cents": 4999}), + } + + return {"role": "assistant", "content": "{}"} diff --git a/tests/golden/agents/triage_agent.py b/tests/golden/agents/triage_agent.py new file mode 100644 index 0000000..9f58ccc --- /dev/null +++ b/tests/golden/agents/triage_agent.py @@ -0,0 +1,85 @@ +"""Support-triage agent demonstrating changed-arguments divergence. + +This agent processes support tickets: it looks up customer info, gets an LLM +decision, then issues a refund. The buggy version divides cents by 100 before +refunding, so it refunds 49 cents instead of 4999. + +The bug changes the **argument** of the `issue_refund` tool call. This means: +- Recording the buggy run captures `issue_refund(amount_cents=49)` +- The fixed code calls `issue_refund(amount_cents=4999)` +- Frozen replay diverges at the tool call inputs (ReplayMismatchError) +- A failure-trace export can never go green against the fixed code + +This demonstrates Finding #1 from the stepfork test report: frozen-replay +export works when the fix leaves instrumented call inputs unchanged, but +not when the fix changes what arguments are passed to a later tool. +""" + +from __future__ import annotations + +import json +from typing import Any + +from stepfork import llm_request, trace_tool + +from .fake_llm import FakeLLM + +_llm = FakeLLM() + +DEFAULT_TICKET = "Charged twice for March, please refund $49.99" + + +@trace_tool(name="lookup_customer") +def lookup_customer(email: str) -> dict[str, Any]: + """Look up customer by email. Returns plan and refund limit.""" + return {"email": email, "plan": "enterprise", "refund_limit_cents": 10000} + + +@trace_tool(name="issue_refund") +def issue_refund(email: str, amount_cents: int) -> dict[str, Any]: + """Issue a refund to the customer.""" + return {"ok": True, "email": email, "amount_cents": amount_cents} + + +def _ask(messages: list[dict[str, str]]) -> dict[str, Any]: + result: dict[str, Any] = llm_request( + provider="fake", + model="stub-1", + input={"messages": messages}, + call=lambda: _llm.chat(messages), + ) + return result + + +def _run(ticket: str, buggy: bool) -> dict[str, Any]: + msgs: list[dict[str, str]] = [{"role": "user", "content": f"TICKET:{ticket}"}] + r1 = _ask(msgs) + call = r1["tool_call"] + cust = lookup_customer(**call["args"]) + + msgs.append({"role": "user", "content": "CUSTOMER:" + json.dumps(cust)}) + r2 = _ask(msgs) + decision = json.loads(r2["content"]) + + amount = decision["refund_cents"] + if buggy: + amount = amount // 100 # BUG: treats cents as dollars → refunds 49 cents + + refunded = None + if amount <= cust["refund_limit_cents"]: + refunded = issue_refund(cust["email"], amount)["amount_cents"] + + return {"priority": decision["priority"], "refunded_cents": refunded} + + +def run_agent(ticket: str = DEFAULT_TICKET) -> dict[str, Any]: + """Buggy implementation: divides cents by 100 before refunding.""" + return _run(ticket, buggy=True) + + +def run_agent_fixed(ticket: str = DEFAULT_TICKET) -> dict[str, Any]: + """Corrected implementation: uses the correct amount.""" + return _run(ticket, buggy=False) + + +EXPECTED_OUTPUT: dict[str, Any] = {"priority": "high", "refunded_cents": 4999} diff --git a/tests/integration/test_export_pytest_agent_fixtures.py b/tests/integration/test_export_pytest_agent_fixtures.py new file mode 100644 index 0000000..e61b185 --- /dev/null +++ b/tests/integration/test_export_pytest_agent_fixtures.py @@ -0,0 +1,284 @@ +"""End-to-end tests for export→pytest with agent fixtures. + +These tests exercise the frozen-replay-based pytest export behavior when +the fix changes instrumented call inputs or order. They demonstrate the +documented limitation: failure-trace exports diverge when the fix changes +what happens in subsequent tool calls. + +Test scenarios from the stepfork test report Finding #1: +1. triage_agent: fix changes tool arguments → ReplayMismatchError +2. crash_agent: fix enables tool call not in recording → ReplayExhaustedError + +Both agents use a deterministic offline fake LLM (no API keys, no network). +""" + +from __future__ import annotations + +import json +import subprocess +import sys +from pathlib import Path +from typing import Any + +import pytest + +from stepfork import record +from stepfork.export import export_pytest_test +from stepfork.inspect import inspect_bundle +from stepfork.replay import ReplayExhaustedError, ReplayMismatchError, ReplaySession +from stepfork.trace import RunStatus, Trace, validate_bundle + +REPO_ROOT = Path(__file__).resolve().parents[2] + + +def _triage_agent() -> Any: + if str(REPO_ROOT) not in sys.path: + sys.path.insert(0, str(REPO_ROOT)) + from tests.golden.agents import triage_agent + + return triage_agent + + +def _crash_agent() -> Any: + if str(REPO_ROOT) not in sys.path: + sys.path.insert(0, str(REPO_ROOT)) + from tests.golden.agents import crash_agent + + return crash_agent + + +def _run_pytest(test_file: Path) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [ + sys.executable, + "-m", + "pytest", + "-p", + "no:cacheprovider", + "-q", + str(test_file), + ], + cwd=REPO_ROOT, + text=True, + capture_output=True, + check=False, + ) + + +class TestTriageAgentChangedArgs: + """Tests for triage agent demonstrating changed-arguments divergence.""" + + def test_buggy_refunds_wrong_amount(self, tmp_path: Path) -> None: + """The buggy agent refunds 49 cents instead of 4999.""" + agent = _triage_agent() + result = agent.run_agent() + + assert result["refunded_cents"] == 49 + assert result["priority"] == "high" + + def test_fixed_refunds_correct_amount(self, tmp_path: Path) -> None: + """The fixed agent refunds 4999 cents.""" + agent = _triage_agent() + result = agent.run_agent_fixed() + + assert result["refunded_cents"] == 4999 + assert result["priority"] == "high" + + def test_recording_captures_buggy_tool_args(self, tmp_path: Path) -> None: + """Recording the buggy run captures the wrong amount in tool call.""" + agent = _triage_agent() + trace_path = tmp_path / "triage_failure.sftrace" + + with record("triage-agent", output=trace_path) as session: + result = agent.run_agent() + session.set_output(result) + + validation = validate_bundle(trace_path, strict=True, verify_integrity=True) + assert validation.valid, validation.issues + assert Trace.load(trace_path).status is RunStatus.COMPLETED + + inspection = inspect_bundle(trace_path) + assert inspection.agent_name == "triage-agent" + + def test_frozen_replay_diverges_on_changed_args(self, tmp_path: Path) -> None: + """Frozen replay of buggy trace diverges when run against fixed code. + + The fixed code calls issue_refund(amount_cents=4999) but the recording + captured issue_refund(amount_cents=49). This causes ReplayMismatchError. + """ + agent = _triage_agent() + trace_path = tmp_path / "triage_failure.sftrace" + + with record("triage-agent", output=trace_path) as session: + session.set_output(agent.run_agent()) + + with ( + pytest.raises(ReplayMismatchError, match="input diverged"), + ReplaySession.from_trace(trace_path, mode="frozen"), + ): + agent.run_agent_fixed() + + def test_exported_test_fails_on_buggy_code(self, tmp_path: Path) -> None: + """Exported test correctly fails when run against buggy code.""" + agent = _triage_agent() + trace_path = tmp_path / "triage_failure.sftrace" + + with record("triage-agent", output=trace_path) as session: + session.set_output(agent.run_agent()) + + expected = {"priority": "high", "refunded_cents": 4999} + test_file = export_pytest_test( + trace_path=trace_path, + output_path=tmp_path / "test_triage_buggy.py", + entrypoint="tests.golden.agents.triage_agent:run_agent", + expectation=expected, + has_expectation=True, + import_root=REPO_ROOT, + ) + + proc = _run_pytest(test_file) + + assert proc.returncode != 0, proc.stdout + assert "behavior mismatch" in proc.stdout + assert "refunded_cents" in proc.stdout + + def test_failure_trace_export_cannot_pass_on_fix(self, tmp_path: Path) -> None: + """A failure-trace export never goes green when fix changes tool args. + + This is the core demonstration: exporting from the buggy recording + creates a test that diverges when run against the fixed code because + the fixed code calls issue_refund with different arguments. + """ + agent = _triage_agent() + trace_path = tmp_path / "triage_failure.sftrace" + + with record("triage-agent", output=trace_path) as session: + session.set_output(agent.run_agent()) + + expected = {"priority": "high", "refunded_cents": 4999} + test_file = export_pytest_test( + trace_path=trace_path, + output_path=tmp_path / "test_triage_fixed.py", + entrypoint="tests.golden.agents.triage_agent:run_agent_fixed", + expectation=expected, + has_expectation=True, + import_root=REPO_ROOT, + ) + + proc = _run_pytest(test_file) + + assert proc.returncode != 0, proc.stdout + assert "diverged" in proc.stdout.lower() or "mismatch" in proc.stdout.lower() + + +class TestCrashAgentExtraCalls: + """Tests for crash agent demonstrating extra-calls divergence.""" + + def test_buggy_crashes_on_json_parse(self, tmp_path: Path) -> None: + """The buggy agent crashes trying to parse markdown-fenced JSON.""" + agent = _crash_agent() + + with pytest.raises(json.JSONDecodeError): + agent.run_agent() + + def test_fixed_parses_correctly(self, tmp_path: Path) -> None: + """The fixed agent strips the fence and parses correctly.""" + agent = _crash_agent() + result = agent.run_agent_fixed() + + assert result["action"] == "escalate" + assert result["ticket"] == "ESC-1" + + def test_recording_captures_crash(self, tmp_path: Path) -> None: + """Recording a crashed run still produces a valid trace bundle.""" + agent = _crash_agent() + trace_path = tmp_path / "crash_failure.sftrace" + + with ( + pytest.raises(json.JSONDecodeError), + record("crash-agent", output=trace_path), + ): + agent.run_agent() + + validation = validate_bundle(trace_path, strict=False, verify_integrity=True) + assert validation.valid, validation.issues + + trace = Trace.load(trace_path) + assert trace.status is RunStatus.FAILED + + def test_frozen_replay_diverges_on_extra_call(self, tmp_path: Path) -> None: + """Frozen replay of crash trace diverges when fix enables a new tool call. + + The fixed code calls escalate() which isn't in the recording, so + replay raises ReplayExhaustedError for the unexpected tool call. + """ + agent = _crash_agent() + trace_path = tmp_path / "crash_failure.sftrace" + + with ( + pytest.raises(json.JSONDecodeError), + record("crash-agent", output=trace_path), + ): + agent.run_agent() + + with ( + pytest.raises(ReplayExhaustedError, match="unexpected tool call"), + ReplaySession.from_trace(trace_path, mode="frozen"), + ): + agent.run_agent_fixed() + + def test_failure_trace_export_cannot_pass_on_fix(self, tmp_path: Path) -> None: + """A failure-trace export never goes green when fix makes extra calls. + + Exporting from the crash recording creates a test that diverges when + run against the fixed code because the fixed code calls escalate() + which was never reached in the buggy run. + """ + agent = _crash_agent() + trace_path = tmp_path / "crash_failure.sftrace" + + with ( + pytest.raises(json.JSONDecodeError), + record("crash-agent", output=trace_path), + ): + agent.run_agent() + + expected = {"action": "escalate", "ticket": "ESC-1"} + test_file = export_pytest_test( + trace_path=trace_path, + output_path=tmp_path / "test_crash_fixed.py", + entrypoint="tests.golden.agents.crash_agent:run_agent_fixed", + expectation=expected, + has_expectation=True, + import_root=REPO_ROOT, + ) + + proc = _run_pytest(test_file) + + assert proc.returncode != 0, proc.stdout + assert ( + "unexpected" in proc.stdout.lower() + or "exhausted" in proc.stdout.lower() + or "diverged" in proc.stdout.lower() + ) + + +class TestRedaction: + """Tests for secret redaction in agent traces.""" + + def test_crash_agent_redacts_secrets(self, tmp_path: Path) -> None: + """Secrets in LLM output and tool output are redacted in the trace.""" + agent = _crash_agent() + trace_path = tmp_path / "crash_redaction.sftrace" + + with ( + pytest.raises(json.JSONDecodeError), + record("crash-agent", output=trace_path), + ): + agent.run_agent() + + for path in trace_path.glob("*"): + if path.is_file(): + content = path.read_text(encoding="utf-8") + assert "sk-proj-ABCDEF" not in content + assert "sk-proj-ZYXW98" not in content