test: add agent fixtures for export→pytest integration tests

tests/golden/agents/__init__.py

@@ -0,0 +1,20 @@
+"""Offline agent fixtures for export→pytest integration tests.
+
+These agents use a deterministic fake LLM and are designed to demonstrate
+scenarios where frozen-replay-based pytest exports diverge when the fix
+changes instrumented call inputs or order. They require no API keys or network.
+"""
+
+from .crash_agent import run_agent as crash_run_agent
+from .crash_agent import run_agent_fixed as crash_run_agent_fixed
+from .fake_llm import FakeLLM
+from .triage_agent import run_agent as triage_run_agent
+from .triage_agent import run_agent_fixed as triage_run_agent_fixed
+
+__all__ = [
+    "FakeLLM",
+    "crash_run_agent",
+    "crash_run_agent_fixed",
+    "triage_run_agent",
+    "triage_run_agent_fixed",
+]

tests/golden/agents/crash_agent.py

@@ -0,0 +1,80 @@
+"""Crash agent demonstrating extra-calls divergence.
+
+This agent processes angry customer messages: it parses an LLM response
+(wrapped in a markdown fence) and escalates the ticket. The buggy version
+crashes on JSON parsing because it doesn't strip the fence.
+
+The bug causes the agent to crash **before** calling the `escalate` tool:
+- Recording the buggy run captures only the LLM call (no escalate)
+- The fixed code successfully calls `escalate`
+- Frozen replay fails with ReplayExhaustedError (unexpected tool call)
+- A failure-trace export can never go green against the fixed code
+
+This demonstrates Finding #1 from the stepfork test report: frozen-replay
+export works when the fix leaves instrumented call inputs and order unchanged,
+but not when the fix enables a tool call that the buggy run never reached.
+
+Also tests redaction: the fake LLM output contains a synthetic secret pattern
+(`sk-proj-...`) to verify that stepfork redacts it in the trace.
+"""
+
+from __future__ import annotations
+
+import json
+from typing import Any
+
+from stepfork import llm_request, trace_tool
+
+
+def _fake_chat(messages: list[dict[str, str]]) -> dict[str, Any]:
+    """Return a markdown-fenced JSON response with a synthetic secret."""
+    del messages  # unused
+    return {
+        "content": (
+            "```json\n"
+            '{"action": "escalate", '
+            '"secret_seen": "sk-proj-ABCDEF1234567890abcdef1234567890"}'
+            "\n```"
+        )
+    }
+
+
+@trace_tool(name="escalate")
+def escalate(reason: str) -> dict[str, Any]:
+    """Escalate a ticket. Returns the escalation ticket ID."""
+    return {
+        "ticket": "ESC-1",
+        "reason": reason,
+        "api_key": "sk-proj-ZYXW9876543210zyxw9876543210ab",
+    }
+
+
+def _decide(strip_fence: bool) -> dict[str, Any]:
+    msgs: list[dict[str, str]] = [{"role": "user", "content": "Customer is angry"}]
+    r = llm_request(
+        provider="fake",
+        model="stub-1",
+        input={"messages": msgs},
+        call=lambda: _fake_chat(msgs),
+    )
+    text = r["content"]
+
+    if strip_fence:
+        text = text.strip().removeprefix("```json").removesuffix("```").strip()
+
+    data = json.loads(text)  # buggy version raises JSONDecodeError
+    res = escalate(data["action"])
+    return {"action": data["action"], "ticket": res["ticket"]}
+
+
+def run_agent() -> dict[str, Any]:
+    """Buggy implementation: crashes on markdown-fenced JSON."""
+    return _decide(strip_fence=False)
+
+
+def run_agent_fixed() -> dict[str, Any]:
+    """Corrected implementation: strips the markdown fence before parsing."""
+    return _decide(strip_fence=True)
+
+
+EXPECTED_OUTPUT: dict[str, Any] = {"action": "escalate", "ticket": "ESC-1"}

tests/golden/agents/fake_llm.py

@@ -0,0 +1,41 @@
+"""Deterministic offline stub LLM for agent fixtures.
+
+No network, no API keys required. Returns predefined responses based on
+message content patterns.
+"""
+
+from __future__ import annotations
+
+import json
+from typing import Any
+
+
+class FakeLLM:
+    """Stateful fake LLM that returns deterministic responses."""
+
+    def __init__(self) -> None:
+        self.calls = 0
+
+    def chat(self, messages: list[dict[str, str]]) -> dict[str, Any]:
+        """Return a deterministic response based on message content."""
+        self.calls += 1
+        last = messages[-1]["content"]
+
+        if last.startswith("TICKET:"):
+            return {
+                "role": "assistant",
+                "tool_call": {
+                    "name": "lookup_customer",
+                    "args": {"email": "ana@example.com"},
+                },
+            }
+
+        if last.startswith("CUSTOMER:"):
+            cust = json.loads(last[len("CUSTOMER:") :])
+            priority = "high" if cust.get("plan") == "enterprise" else "normal"
+            return {
+                "role": "assistant",
+                "content": json.dumps({"priority": priority, "refund_cents": 4999}),
+            }
+
+        return {"role": "assistant", "content": "{}"}

tests/golden/agents/triage_agent.py

@@ -0,0 +1,85 @@
+"""Support-triage agent demonstrating changed-arguments divergence.
+
+This agent processes support tickets: it looks up customer info, gets an LLM
+decision, then issues a refund. The buggy version divides cents by 100 before
+refunding, so it refunds 49 cents instead of 4999.
+
+The bug changes the **argument** of the `issue_refund` tool call. This means:
+- Recording the buggy run captures `issue_refund(amount_cents=49)`
+- The fixed code calls `issue_refund(amount_cents=4999)`
+- Frozen replay diverges at the tool call inputs (ReplayMismatchError)
+- A failure-trace export can never go green against the fixed code
+
+This demonstrates Finding #1 from the stepfork test report: frozen-replay
+export works when the fix leaves instrumented call inputs unchanged, but
+not when the fix changes what arguments are passed to a later tool.
+"""
+
+from __future__ import annotations
+
+import json
+from typing import Any
+
+from stepfork import llm_request, trace_tool
+
+from .fake_llm import FakeLLM
+
+_llm = FakeLLM()
+
+DEFAULT_TICKET = "Charged twice for March, please refund $49.99"
+
+
+@trace_tool(name="lookup_customer")
+def lookup_customer(email: str) -> dict[str, Any]:
+    """Look up customer by email. Returns plan and refund limit."""
+    return {"email": email, "plan": "enterprise", "refund_limit_cents": 10000}
+
+
+@trace_tool(name="issue_refund")
+def issue_refund(email: str, amount_cents: int) -> dict[str, Any]:
+    """Issue a refund to the customer."""
+    return {"ok": True, "email": email, "amount_cents": amount_cents}
+
+
+def _ask(messages: list[dict[str, str]]) -> dict[str, Any]:
+    result: dict[str, Any] = llm_request(
+        provider="fake",
+        model="stub-1",
+        input={"messages": messages},
+        call=lambda: _llm.chat(messages),
+    )
+    return result
+
+
+def _run(ticket: str, buggy: bool) -> dict[str, Any]:
+    msgs: list[dict[str, str]] = [{"role": "user", "content": f"TICKET:{ticket}"}]
+    r1 = _ask(msgs)
+    call = r1["tool_call"]
+    cust = lookup_customer(**call["args"])
+
+    msgs.append({"role": "user", "content": "CUSTOMER:" + json.dumps(cust)})
+    r2 = _ask(msgs)
+    decision = json.loads(r2["content"])
+
+    amount = decision["refund_cents"]
+    if buggy:
+        amount = amount // 100  # BUG: treats cents as dollars → refunds 49 cents
+
+    refunded = None
+    if amount <= cust["refund_limit_cents"]:
+        refunded = issue_refund(cust["email"], amount)["amount_cents"]
+
+    return {"priority": decision["priority"], "refunded_cents": refunded}
+
+
+def run_agent(ticket: str = DEFAULT_TICKET) -> dict[str, Any]:
+    """Buggy implementation: divides cents by 100 before refunding."""
+    return _run(ticket, buggy=True)
+
+
+def run_agent_fixed(ticket: str = DEFAULT_TICKET) -> dict[str, Any]:
+    """Corrected implementation: uses the correct amount."""
+    return _run(ticket, buggy=False)
+
+
+EXPECTED_OUTPUT: dict[str, Any] = {"priority": "high", "refunded_cents": 4999}