Fix Responses trace capture and prepare 0.1.0 release

.github/workflows/ci.yml

@@ -19,11 +19,21 @@ jobs:
       - uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
+      - name: Install test dependencies
+        run: python -m pip install -e ".[dev,compat]"
+      - name: Run tests
+        env:
+          LITELLM_LOCAL_MODEL_COST_MAP: "True"
+        run: python -m pytest -q
       - run: python -m pip install build twine
       - run: python -m build
       - run: python -m twine check --strict dist/*
       - name: Test installed wheel
         run: |
-          python -m pip install dist/*.whl
-          python -c "import faultgate_zero; assert faultgate_zero.__version__ == '0.0.0'"
+          python -m pip install --force-reinstall --no-deps dist/*.whl
+          python -c "import faultgate_zero, importlib.metadata; assert faultgate_zero.__version__ == importlib.metadata.version('faultgate-zero')"
           python -m pip check
+      - name: Run tests against the wheel
+        env:
+          LITELLM_LOCAL_MODEL_COST_MAP: "True"
+        run: python -m pytest -q

.github/workflows/publish.yml

@@ -8,7 +8,24 @@ permissions:
   contents: read
 
 jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.11", "3.12", "3.13"]
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+      - run: python -m pip install -e ".[dev,compat]"
+      - name: Run tests
+        env:
+          LITELLM_LOCAL_MODEL_COST_MAP: "True"
+        run: python -m pytest -q
+
   build:
+    needs: test
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4

CHANGELOG.md

@@ -1,9 +1,13 @@
 # Changelog
 
-## 0.1.0 (unreleased)
+## 0.1.0
 
 First usable version.
 
+- Release review: detect Responses API failures returned without SDK exceptions; preserve received
+  text and function-call arguments when a Responses stream ends early.
+- CI and tagged releases run the test suite on Python 3.11, 3.12 and 3.13 before publishing.
+
 - pytest plugin (`pytest11` entry point): each test is one agent run, checked when it ends; `fail`,
   `warn` and `off` modes; per-test `@pytest.mark.faultgate(...)`; settings from marker, `--fgz-*`
   options, `FAULTGATE_ZERO*` environment variables and `[tool.faultgate_zero]`.

MANIFEST.in

@@ -0,0 +1,4 @@
+include CHANGELOG.md
+recursive-include docs *.md
+recursive-include examples *.py *.toml *.json
+recursive-include tests *.py

README.md

@@ -8,11 +8,17 @@ asserts miss: loops on empty results, repeated identical calls, tool errors the
 provider errors it hid behind a fallback message, answers cut at the token limit, and tool calls that
 break your policy (refund limits, identity checks, required steps, denied paths).
 
-> **Status:** not released yet. Install from source (below). The PyPI name `faultgate-zero` is
-> reserved for the first release, 0.1.0.
+> **Status:** 0.1.0 alpha. Trace-based checks for existing pytest tests; no dependency on the
+> `faultgate` package. See the [known limitations](docs/known-limitations.md).
 
 ## 60-second quickstart
 
+```bash
+pip install faultgate-zero
+```
+
+To try the included example from source:
+
 ```bash
 git clone https://github.com/Faultline-lab/faultgate-zero
 cd faultgate-zero
@@ -203,9 +209,9 @@ Tested here, with offline fakes that run the SDKs' own request and response code
 
 | Package | Tested version | What is captured |
 |---|---|---|
-| `openai` | 3.35 | `chat.completions.create` / `parse` / `stream`, `responses.create` / `parse` / `stream`; sync and async; streaming; `with_raw_response`. |
+| `openai` | 2.54.0 | `chat.completions.create` / `parse` / `stream`, `responses.create` / `parse` / `stream`; sync and async; streaming; `with_raw_response`. |
 | `anthropic` | 1.17 | `messages.create`, `messages.stream`, and the `beta` equivalents; sync and async; streaming; server tool results. |
-| `litellm` | 1.107 | `completion`, `acompletion` (a call LiteLLM makes through the OpenAI SDK underneath is counted once). |
+| `litellm` | 1.106.2 | `completion`, `acompletion` (a call LiteLLM makes through the OpenAI SDK underneath is counted once). |
 | `langchain-openai` / `langchain-anthropic` | 1.7 | `ChatOpenAI`, `ChatAnthropic` with bound tools. |
 | `opentelemetry-instrumentation-openai-v2` | 2.4b0 | Runs alongside it; both record. |
 | `pytest-xdist` | 3.8 | `-n` with merged reports. |

docs/known-limitations.md

@@ -1,6 +1,6 @@
 # Known limitations
 
-What faultgate-zero does not do, or does imperfectly, as of 0.1.0 (unreleased).
+What faultgate-zero does not do, or does imperfectly, as of 0.1.0.
 
 ## Capture
 

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "faultgate-zero"
-version = "0.0.0"
+version = "0.1.0"
 description = "Agent failure checks inside your existing pytest tests: loops, repeated calls, unrecovered errors, token cutoffs and policy breaks."
 readme = "README.md"
 license = "MIT"

src/faultgate_zero/__init__.py

@@ -1,5 +1,5 @@
 """faultgate-zero: agent failure checks that run inside your existing pytest tests."""
-__version__ = "0.0.0"
+__version__ = "0.1.0"
 
 from .patch import install, uninstall  # noqa: E402
 from .policy import Policy, PolicyError  # noqa: E402

src/faultgate_zero/adapters/openai_responses.py

@@ -66,6 +66,12 @@ def normalise(call: ModelCall) -> dict[str, Any]:
         usage = as_dict(d.get("usage"))
         attrs["gen_ai.usage.input_tokens"] = usage.get("input_tokens")
         attrs["gen_ai.usage.output_tokens"] = usage.get("output_tokens")
+    # The Responses API can report a provider failure in an HTTP 200 response,
+    # including a response.failed stream event, without the SDK raising.
+    if d.get("status") == "failed" or d.get("error"):
+        error = as_dict(d.get("error"))
+        attrs["error.type"] = str(error.get("code") or "response_failed")
+        attrs["error.message"] = str(error.get("message") or "Responses API returned status=failed")
     return drop_none(attrs)
 
 

src/faultgate_zero/builder.py

@@ -45,8 +45,8 @@ def build_trace(run: Run) -> Trace:
             if run.escaped(call.error):
                 attrs["faultgate.error_escaped"] = True
         chats.append(Span(span_id(run.id, f"chat-{i}"), root_id, f"chat {call.model}".strip(), call.start_ns,
-                          call.end_ns, drop_none(attrs), error=call.error is not None,
-                          status_message=_short(call.error)))
+                          call.end_ns, drop_none(attrs), error=call.error is not None or bool(attrs.get("error.type")),
+                          status_message=_short(call.error) or str(attrs.get("error.message", ""))))
 
     tools = [_tool_span(run, root_id, j, t) for j, t in enumerate(run.tool_calls)]
 

src/faultgate_zero/streams.py

@@ -380,11 +380,12 @@ def result(self) -> dict[str, Any]:
 
 
 class ResponsesEvents:
-    """OpenAI Responses stream events -> the final response (sent whole in the last event)."""
+    """Keep received output even when the caller stops before the final response event."""
 
     def __init__(self) -> None:
         self.final: Any = None
         self.partial: Any = None
+        self.items: dict[int, dict[str, Any]] = {}
 
     def feed(self, event: Any) -> None:
         e = as_dict(event)
@@ -393,6 +394,30 @@ def feed(self, event: Any) -> None:
             self.final = e.get("response")
         elif t in ("response.created", "response.in_progress") and e.get("response"):
             self.partial = e["response"]
+        elif t in ("response.output_item.added", "response.output_item.done"):
+            self.items[e.get("output_index", 0)] = dict(e.get("item") or {})
+        elif t == "response.content_part.added":
+            item = self.items.setdefault(e.get("output_index", 0), {"type": "message", "content": []})
+            content = item.setdefault("content", [])
+            index = e.get("content_index", 0)
+            while len(content) <= index:
+                content.append({})
+            content[index] = dict(e.get("part") or {})
+        elif t == "response.output_text.delta":
+            item = self.items.setdefault(e.get("output_index", 0), {"type": "message", "content": []})
+            content = item.setdefault("content", [])
+            index = e.get("content_index", 0)
+            while len(content) <= index:
+                content.append({"type": "output_text", "text": ""})
+            content[index]["text"] = content[index].get("text", "") + e.get("delta", "")
+        elif t in ("response.function_call_arguments.delta", "response.function_call_arguments.done"):
+            item = self.items.setdefault(e.get("output_index", 0), {"type": "function_call"})
+            item["arguments"] = (e.get("arguments", "") if t.endswith(".done")
+                                 else item.get("arguments", "") + e.get("delta", ""))
 
     def result(self) -> Any:
-        return self.final if self.final is not None else self.partial
+        if self.final is not None:
+            return self.final
+        if self.items:
+            return {**(self.partial or {}), "output": [self.items[i] for i in sorted(self.items)]}
+        return self.partial

tests/test_responses_api.py

@@ -1,6 +1,7 @@
 import json
 
 import openai
+import pytest
 
 import faultgate_zero as fz
 from fakes.openai_fakes import SSE, Script, client
@@ -79,3 +80,52 @@ def test_responses_stream_uses_the_completed_event():
         for _ in s:
             pass
     assert len(run.model_calls) == 1 and run.trace.answer == "streamed answer"
+
+
+@pytest.mark.parametrize("streamed", [False, True])
+@pytest.mark.parametrize("recovered", [False, True])
+def test_failed_response_is_checked_without_changing_sdk_behavior(streamed, recovered):
+    failed = {**response([], status="failed"),
+              "error": {"code": "server_error", "message": "Provider unavailable"}}
+    payload = SSE([("response.failed", {"type": "response.failed", "sequence_number": 0,
+                                        "response": failed})]) if streamed else failed
+    c = client(Script(payload, response([text("recovered")])))
+    with fz.run() as run:
+        result = c.responses.create(model="m", input="x", stream=streamed)
+        if streamed:
+            assert [event.type for event in result] == ["response.failed"]
+        else:
+            assert result.status == "failed"
+        if recovered:
+            assert c.responses.create(model="m", input="retry").output_text == "recovered"
+    assert run.capture_complete
+    assert run.trace.model_calls[0].error
+    assert bool(run.check().fired) is not recovered
+    if not recovered:
+        assert run.check().fired[0].check == "unrecovered_model_error"
+
+
+def test_responses_stream_keeps_tool_arguments_before_final_event():
+    events = [
+        ("response.output_item.added", {"type": "response.output_item.added", "sequence_number": 0,
+          "output_index": 0, "item": {**fcall("c1", "read_file", {}), "arguments": ""}}),
+        ("response.function_call_arguments.delta", {"type": "response.function_call_arguments.delta",
+          "sequence_number": 1, "output_index": 0, "item_id": "fc_c1", "delta": '{"path":"/etc/passwd"}'}),
+    ]
+    with fz.run() as run:
+        stream = client(Script(SSE(events))).responses.create(model="m", input="x", stream=True)
+        for event in stream:
+            if event.type == "response.function_call_arguments.delta":
+                break
+    [step] = run.steps
+    assert step.tool == "read_file" and json.loads(step.arguments) == {"path": "/etc/passwd"}
+    assert run.check().fired[0].check == "policy"
+
+
+def test_responses_stream_keeps_partial_text():
+    events = [("response.output_text.delta", {"type": "response.output_text.delta", "sequence_number": 0,
+               "item_id": "msg_1", "output_index": 0, "content_index": 0, "delta": "Hello", "logprobs": []})]
+    with fz.run() as run:
+        stream = client(Script(SSE(events))).responses.create(model="m", input="x", stream=True)
+        next(stream)
+    assert run.trace.answer == "Hello"