From 72b3efdeeae3b4dd34e10f97e229dbe3a7c433d5 Mon Sep 17 00:00:00 2001 From: samir Date: Thu, 22 Oct 2026 11:58:46 +0530 Subject: [PATCH] fix: close Responses capture gaps and gate 0.1.0 release on tests --- .github/workflows/ci.yml | 14 +++++- .github/workflows/publish.yml | 17 +++++++ CHANGELOG.md | 6 ++- MANIFEST.in | 4 ++ README.md | 14 ++++-- docs/known-limitations.md | 2 +- pyproject.toml | 2 +- src/faultgate_zero/__init__.py | 2 +- .../adapters/openai_responses.py | 6 +++ src/faultgate_zero/builder.py | 4 +- src/faultgate_zero/streams.py | 29 ++++++++++- tests/test_responses_api.py | 50 +++++++++++++++++++ 12 files changed, 136 insertions(+), 14 deletions(-) create mode 100644 MANIFEST.in diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 47d4327..68614da 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -19,11 +19,21 @@ jobs: - uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} + - name: Install test dependencies + run: python -m pip install -e ".[dev,compat]" + - name: Run tests + env: + LITELLM_LOCAL_MODEL_COST_MAP: "True" + run: python -m pytest -q - run: python -m pip install build twine - run: python -m build - run: python -m twine check --strict dist/* - name: Test installed wheel run: | - python -m pip install dist/*.whl - python -c "import faultgate_zero; assert faultgate_zero.__version__ == '0.0.0'" + python -m pip install --force-reinstall --no-deps dist/*.whl + python -c "import faultgate_zero, importlib.metadata; assert faultgate_zero.__version__ == importlib.metadata.version('faultgate-zero')" python -m pip check + - name: Run tests against the wheel + env: + LITELLM_LOCAL_MODEL_COST_MAP: "True" + run: python -m pytest -q diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index c4fea4c..2977882 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -8,7 +8,24 @@ permissions: contents: read jobs: + test: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.11", "3.12", "3.13"] + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + - run: python -m pip install -e ".[dev,compat]" + - name: Run tests + env: + LITELLM_LOCAL_MODEL_COST_MAP: "True" + run: python -m pytest -q + build: + needs: test runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 diff --git a/CHANGELOG.md b/CHANGELOG.md index 9f8237f..f5a6b37 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,9 +1,13 @@ # Changelog -## 0.1.0 (unreleased) +## 0.1.0 First usable version. +- Release review: detect Responses API failures returned without SDK exceptions; preserve received + text and function-call arguments when a Responses stream ends early. +- CI and tagged releases run the test suite on Python 3.11, 3.12 and 3.13 before publishing. + - pytest plugin (`pytest11` entry point): each test is one agent run, checked when it ends; `fail`, `warn` and `off` modes; per-test `@pytest.mark.faultgate(...)`; settings from marker, `--fgz-*` options, `FAULTGATE_ZERO*` environment variables and `[tool.faultgate_zero]`. diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..d081c62 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,4 @@ +include CHANGELOG.md +recursive-include docs *.md +recursive-include examples *.py *.toml *.json +recursive-include tests *.py diff --git a/README.md b/README.md index 2e1e165..55134e7 100644 --- a/README.md +++ b/README.md @@ -8,11 +8,17 @@ asserts miss: loops on empty results, repeated identical calls, tool errors the provider errors it hid behind a fallback message, answers cut at the token limit, and tool calls that break your policy (refund limits, identity checks, required steps, denied paths). -> **Status:** not released yet. Install from source (below). The PyPI name `faultgate-zero` is -> reserved for the first release, 0.1.0. +> **Status:** 0.1.0 alpha. Trace-based checks for existing pytest tests; no dependency on the +> `faultgate` package. See the [known limitations](docs/known-limitations.md). ## 60-second quickstart +```bash +pip install faultgate-zero +``` + +To try the included example from source: + ```bash git clone https://github.com/Faultline-lab/faultgate-zero cd faultgate-zero @@ -203,9 +209,9 @@ Tested here, with offline fakes that run the SDKs' own request and response code | Package | Tested version | What is captured | |---|---|---| -| `openai` | 3.35 | `chat.completions.create` / `parse` / `stream`, `responses.create` / `parse` / `stream`; sync and async; streaming; `with_raw_response`. | +| `openai` | 2.54.0 | `chat.completions.create` / `parse` / `stream`, `responses.create` / `parse` / `stream`; sync and async; streaming; `with_raw_response`. | | `anthropic` | 1.17 | `messages.create`, `messages.stream`, and the `beta` equivalents; sync and async; streaming; server tool results. | -| `litellm` | 1.107 | `completion`, `acompletion` (a call LiteLLM makes through the OpenAI SDK underneath is counted once). | +| `litellm` | 1.106.2 | `completion`, `acompletion` (a call LiteLLM makes through the OpenAI SDK underneath is counted once). | | `langchain-openai` / `langchain-anthropic` | 1.7 | `ChatOpenAI`, `ChatAnthropic` with bound tools. | | `opentelemetry-instrumentation-openai-v2` | 2.4b0 | Runs alongside it; both record. | | `pytest-xdist` | 3.8 | `-n` with merged reports. | diff --git a/docs/known-limitations.md b/docs/known-limitations.md index 1244deb..83403a5 100644 --- a/docs/known-limitations.md +++ b/docs/known-limitations.md @@ -1,6 +1,6 @@ # Known limitations -What faultgate-zero does not do, or does imperfectly, as of 0.1.0 (unreleased). +What faultgate-zero does not do, or does imperfectly, as of 0.1.0. ## Capture diff --git a/pyproject.toml b/pyproject.toml index 34dd28c..86c66a3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "faultgate-zero" -version = "0.0.0" +version = "0.1.0" description = "Agent failure checks inside your existing pytest tests: loops, repeated calls, unrecovered errors, token cutoffs and policy breaks." readme = "README.md" license = "MIT" diff --git a/src/faultgate_zero/__init__.py b/src/faultgate_zero/__init__.py index 65f83f2..65450d9 100644 --- a/src/faultgate_zero/__init__.py +++ b/src/faultgate_zero/__init__.py @@ -1,5 +1,5 @@ """faultgate-zero: agent failure checks that run inside your existing pytest tests.""" -__version__ = "0.0.0" +__version__ = "0.1.0" from .patch import install, uninstall # noqa: E402 from .policy import Policy, PolicyError # noqa: E402 diff --git a/src/faultgate_zero/adapters/openai_responses.py b/src/faultgate_zero/adapters/openai_responses.py index db63b37..ae3f8cb 100644 --- a/src/faultgate_zero/adapters/openai_responses.py +++ b/src/faultgate_zero/adapters/openai_responses.py @@ -66,6 +66,12 @@ def normalise(call: ModelCall) -> dict[str, Any]: usage = as_dict(d.get("usage")) attrs["gen_ai.usage.input_tokens"] = usage.get("input_tokens") attrs["gen_ai.usage.output_tokens"] = usage.get("output_tokens") + # The Responses API can report a provider failure in an HTTP 200 response, + # including a response.failed stream event, without the SDK raising. + if d.get("status") == "failed" or d.get("error"): + error = as_dict(d.get("error")) + attrs["error.type"] = str(error.get("code") or "response_failed") + attrs["error.message"] = str(error.get("message") or "Responses API returned status=failed") return drop_none(attrs) diff --git a/src/faultgate_zero/builder.py b/src/faultgate_zero/builder.py index afb87a4..01da90c 100644 --- a/src/faultgate_zero/builder.py +++ b/src/faultgate_zero/builder.py @@ -45,8 +45,8 @@ def build_trace(run: Run) -> Trace: if run.escaped(call.error): attrs["faultgate.error_escaped"] = True chats.append(Span(span_id(run.id, f"chat-{i}"), root_id, f"chat {call.model}".strip(), call.start_ns, - call.end_ns, drop_none(attrs), error=call.error is not None, - status_message=_short(call.error))) + call.end_ns, drop_none(attrs), error=call.error is not None or bool(attrs.get("error.type")), + status_message=_short(call.error) or str(attrs.get("error.message", "")))) tools = [_tool_span(run, root_id, j, t) for j, t in enumerate(run.tool_calls)] diff --git a/src/faultgate_zero/streams.py b/src/faultgate_zero/streams.py index 0e29f43..11f28e1 100644 --- a/src/faultgate_zero/streams.py +++ b/src/faultgate_zero/streams.py @@ -380,11 +380,12 @@ def result(self) -> dict[str, Any]: class ResponsesEvents: - """OpenAI Responses stream events -> the final response (sent whole in the last event).""" + """Keep received output even when the caller stops before the final response event.""" def __init__(self) -> None: self.final: Any = None self.partial: Any = None + self.items: dict[int, dict[str, Any]] = {} def feed(self, event: Any) -> None: e = as_dict(event) @@ -393,6 +394,30 @@ def feed(self, event: Any) -> None: self.final = e.get("response") elif t in ("response.created", "response.in_progress") and e.get("response"): self.partial = e["response"] + elif t in ("response.output_item.added", "response.output_item.done"): + self.items[e.get("output_index", 0)] = dict(e.get("item") or {}) + elif t == "response.content_part.added": + item = self.items.setdefault(e.get("output_index", 0), {"type": "message", "content": []}) + content = item.setdefault("content", []) + index = e.get("content_index", 0) + while len(content) <= index: + content.append({}) + content[index] = dict(e.get("part") or {}) + elif t == "response.output_text.delta": + item = self.items.setdefault(e.get("output_index", 0), {"type": "message", "content": []}) + content = item.setdefault("content", []) + index = e.get("content_index", 0) + while len(content) <= index: + content.append({"type": "output_text", "text": ""}) + content[index]["text"] = content[index].get("text", "") + e.get("delta", "") + elif t in ("response.function_call_arguments.delta", "response.function_call_arguments.done"): + item = self.items.setdefault(e.get("output_index", 0), {"type": "function_call"}) + item["arguments"] = (e.get("arguments", "") if t.endswith(".done") + else item.get("arguments", "") + e.get("delta", "")) def result(self) -> Any: - return self.final if self.final is not None else self.partial + if self.final is not None: + return self.final + if self.items: + return {**(self.partial or {}), "output": [self.items[i] for i in sorted(self.items)]} + return self.partial diff --git a/tests/test_responses_api.py b/tests/test_responses_api.py index 078f07f..65107e1 100644 --- a/tests/test_responses_api.py +++ b/tests/test_responses_api.py @@ -1,6 +1,7 @@ import json import openai +import pytest import faultgate_zero as fz from fakes.openai_fakes import SSE, Script, client @@ -79,3 +80,52 @@ def test_responses_stream_uses_the_completed_event(): for _ in s: pass assert len(run.model_calls) == 1 and run.trace.answer == "streamed answer" + + +@pytest.mark.parametrize("streamed", [False, True]) +@pytest.mark.parametrize("recovered", [False, True]) +def test_failed_response_is_checked_without_changing_sdk_behavior(streamed, recovered): + failed = {**response([], status="failed"), + "error": {"code": "server_error", "message": "Provider unavailable"}} + payload = SSE([("response.failed", {"type": "response.failed", "sequence_number": 0, + "response": failed})]) if streamed else failed + c = client(Script(payload, response([text("recovered")]))) + with fz.run() as run: + result = c.responses.create(model="m", input="x", stream=streamed) + if streamed: + assert [event.type for event in result] == ["response.failed"] + else: + assert result.status == "failed" + if recovered: + assert c.responses.create(model="m", input="retry").output_text == "recovered" + assert run.capture_complete + assert run.trace.model_calls[0].error + assert bool(run.check().fired) is not recovered + if not recovered: + assert run.check().fired[0].check == "unrecovered_model_error" + + +def test_responses_stream_keeps_tool_arguments_before_final_event(): + events = [ + ("response.output_item.added", {"type": "response.output_item.added", "sequence_number": 0, + "output_index": 0, "item": {**fcall("c1", "read_file", {}), "arguments": ""}}), + ("response.function_call_arguments.delta", {"type": "response.function_call_arguments.delta", + "sequence_number": 1, "output_index": 0, "item_id": "fc_c1", "delta": '{"path":"/etc/passwd"}'}), + ] + with fz.run() as run: + stream = client(Script(SSE(events))).responses.create(model="m", input="x", stream=True) + for event in stream: + if event.type == "response.function_call_arguments.delta": + break + [step] = run.steps + assert step.tool == "read_file" and json.loads(step.arguments) == {"path": "/etc/passwd"} + assert run.check().fired[0].check == "policy" + + +def test_responses_stream_keeps_partial_text(): + events = [("response.output_text.delta", {"type": "response.output_text.delta", "sequence_number": 0, + "item_id": "msg_1", "output_index": 0, "content_index": 0, "delta": "Hello", "logprobs": []})] + with fz.run() as run: + stream = client(Script(SSE(events))).responses.create(model="m", input="x", stream=True) + next(stream) + assert run.trace.answer == "Hello"