diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 68614da..1453efa 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -20,7 +20,7 @@ jobs: with: python-version: ${{ matrix.python-version }} - name: Install test dependencies - run: python -m pip install -e ".[dev,compat]" + run: python -m pip install -e ".[dev,compat,formats]" - name: Run tests env: LITELLM_LOCAL_MODEL_COST_MAP: "True" diff --git a/CHANGELOG.md b/CHANGELOG.md index f5a6b37..ce355e2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,17 @@ # Changelog +## 0.2.0 (unreleased) + +Adds features only; everything in 0.1.0 works as before. + +- Check plugins: the `faultgate_zero.checks` entry point group. A plugin object has `name`, `default` + and `run(trace, options) -> Verdict`; default plugins join `DEFAULT_CHECKS`. A name clash with a + built-in check is a usage error at startup; a plugin that raises gives a verdict that did not fire + (`check crashed: `), a `FaultgateZeroWarning` and a summary line. +- `faultgate_zero.formats.load(path, fmt="auto")`: OTLP through faultgate-zero's own loader, and + OpenInference, Langfuse, LangSmith and Braintrust through faultgate's loaders, behind the new + `faultgate-zero[formats]` extra. `formats.to_zero()` converts a faultgate trace. + ## 0.1.0 First usable version. diff --git a/README.md b/README.md index 55134e7..9dc87cc 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ asserts miss: loops on empty results, repeated identical calls, tool errors the provider errors it hid behind a fallback message, answers cut at the token limit, and tool calls that break your policy (refund limits, identity checks, required steps, denied paths). -> **Status:** 0.1.0 alpha. Trace-based checks for existing pytest tests; no dependency on the +> **Status:** 0.2.0 alpha. Trace-based checks for existing pytest tests; no dependency on the > `faultgate` package. See the [known limitations](docs/known-limitations.md). ## 60-second quickstart @@ -203,6 +203,31 @@ Paths on the command line are relative to where pytest runs; in `pyproject.toml` With pytest-xdist (`-n`), workers send their results to the controller and one merged report is written. +## Check plugins + +Other packages can add checks. A plugin registers an object in the `faultgate_zero.checks` entry point +group: + +```toml +[project.entry-points."faultgate_zero.checks"] +contract = "fault_contract.zero_check:CHECK" +``` + +The object has three members: `name` (str), `default` (bool: whether it joins the default checks) and +`run(trace, options) -> Verdict`. Plugins are discovered once, when faultgate-zero loads; installing one +is enough. A reason made of several parts joined by `; ` prints one part per line, as policy +reasons do. A plugin check works like a built-in one: `--fgz-checks`, `@pytest.mark.faultgate(skip=[...])`, +the modes and the reports. A plugin named like a built-in check, or two plugins with one name, stop +the session with a usage error. A plugin that raises gives a verdict that did not fire (`check crashed: +`), a `FaultgateZeroWarning` and a "check crashed" line in the summary: your test is +never failed by a bug in a plugin. + +## Reading other trace formats + +`faultgate_zero.formats.load(path, fmt="auto")` returns the traces in an export file. OTLP/JSON is read +by faultgate-zero itself. OpenInference, Langfuse, LangSmith and Braintrust exports are read by +faultgate's loaders: `pip install 'faultgate-zero[formats]'`. + ## Supported SDKs and frameworks Tested here, with offline fakes that run the SDKs' own request and response code: @@ -222,7 +247,7 @@ are not claimed until a test covers them. See [docs/known-limitations.md](docs/k ## Development ```bash -pip install -e ".[dev,compat]" +pip install -e ".[dev,compat,formats]" pytest # the repo's own suite; its pyproject turns the plugin off for itself ``` diff --git a/pyproject.toml b/pyproject.toml index 86c66a3..d9b4727 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "faultgate-zero" -version = "0.1.0" +version = "0.2.0" description = "Agent failure checks inside your existing pytest tests: loops, repeated calls, unrecovered errors, token cutoffs and policy breaks." readme = "README.md" license = "MIT" @@ -15,6 +15,8 @@ classifiers = ["Development Status :: 3 - Alpha", "Framework :: Pytest", "Progra dependencies = ["pytest>=8"] [project.optional-dependencies] +# every trace format through faultgate's loaders (formats.load); OTLP needs nothing extra +formats = ["faultgate>=0.4,<0.5"] dev = ["openai>=2", "anthropic>=0.40", "pytest-xdist>=3"] # framework compatibility tests (tests/test_compat.py); litellm: CTO licence exception, test-only compat = ["litellm>=1.70", "langchain-openai>=0.3", "langchain-anthropic>=0.3", diff --git a/src/faultgate_zero/__init__.py b/src/faultgate_zero/__init__.py index 65450d9..1974616 100644 --- a/src/faultgate_zero/__init__.py +++ b/src/faultgate_zero/__init__.py @@ -1,5 +1,5 @@ """faultgate-zero: agent failure checks that run inside your existing pytest tests.""" -__version__ = "0.1.0" +__version__ = "0.2.0" from .patch import install, uninstall # noqa: E402 from .policy import Policy, PolicyError # noqa: E402 diff --git a/src/faultgate_zero/checks.py b/src/faultgate_zero/checks.py index 510a7e7..a9ec7f9 100644 --- a/src/faultgate_zero/checks.py +++ b/src/faultgate_zero/checks.py @@ -6,9 +6,11 @@ from __future__ import annotations import re +import warnings from collections import defaultdict from dataclasses import dataclass -from typing import Callable +from importlib import metadata +from typing import Any, Callable, Iterable from .policy import Policy from .results import error_of, is_empty @@ -145,13 +147,68 @@ def policy_check(t: Trace, policy: Policy) -> Verdict: "context_overflow": context_overflow, "abstention": abstention, } -CHECK_NAMES = (*RULES, "policy") -DEFAULT_CHECKS = ("search_loop", "repeated_call", "unrecovered_tool_error", "unrecovered_model_error", - "context_overflow", "policy") +BUILTIN_DEFAULTS = ("search_loop", "repeated_call", "unrecovered_tool_error", "unrecovered_model_error", + "context_overflow", "policy") + +# Check plugins: other packages add checks through the ``faultgate_zero.checks`` entry point group. +# Each entry point loads an object with ``name: str``, ``default: bool`` and +# ``run(trace: Trace, options: Options) -> Verdict``. Discovery happens once, when this module loads. +PLUGIN_GROUP = "faultgate_zero.checks" +CRASHED = "check crashed: " + + +class PluginError(ValueError): + """A check plugin that cannot be used: its name clashes with a built-in check, or it is malformed.""" + + +def discover(eps: Iterable[Any] | None = None) -> tuple[dict[str, Any], list[str]]: + """Load check plugins from entry points. Returns (plugins by name, errors). Never raises.""" + if eps is None: + try: + eps = metadata.entry_points(group=PLUGIN_GROUP) + except Exception as exc: # a broken install must not stop faultgate-zero from loading + return {}, [f"cannot read {PLUGIN_GROUP} entry points: {type(exc).__name__}"] + found: dict[str, Any] = {} + errors: list[str] = [] + builtin = (*RULES, "policy") + for ep in eps: + where = f"check plugin {getattr(ep, 'name', '?')!r} ({getattr(ep, 'value', '?')})" + try: + obj = ep.load() + except Exception as exc: + errors.append(f"{where}: cannot load it: {type(exc).__name__}: {exc}") + continue + name = getattr(obj, "name", None) + if not isinstance(name, str) or not name or not callable(getattr(obj, "run", None)) or not isinstance(getattr(obj, "default", None), bool): + errors.append(f"{where}: needs name (str), default (bool) and run(trace, options)") + continue + if name in builtin: + errors.append(f"{where}: name {name!r} clashes with a built-in check") + continue + if name in found: + errors.append(f"{where}: name {name!r} is used by two plugins") + continue + found[name] = obj + return found, errors + + +PLUGINS, PLUGIN_ERRORS = discover() +CHECK_NAMES = (*RULES, "policy", *PLUGINS) +DEFAULT_CHECKS = (*BUILTIN_DEFAULTS, *(n for n, p in PLUGINS.items() if p.default)) + + +def plugin_errors() -> list[str]: + return list(PLUGIN_ERRORS) + + +def _raise_plugin_errors() -> None: + if PLUGIN_ERRORS: + raise PluginError(PLUGIN_ERRORS[0]) def resolve(names: str | list[str] | tuple[str, ...] | None) -> tuple[str, ...]: """Check names from a comma list or sequence; None means the default set. Unknown names are a usage error.""" + _raise_plugin_errors() if names is None: return DEFAULT_CHECKS wanted = [n.strip() for n in (names.split(",") if isinstance(names, str) else names) if n.strip()] @@ -161,8 +218,37 @@ def resolve(names: str | list[str] | tuple[str, ...] | None) -> tuple[str, ...]: return tuple(dict.fromkeys(wanted)) +def run_plugin(plugin: Any, t: Trace, options: Options) -> Verdict: + """Run one plugin check. A plugin that raises, or returns something else than a Verdict, gives a + verdict that did not fire and a warning: the user's test never fails because of a plugin bug.""" + from .recorder import FaultgateZeroWarning + + name = plugin.name + try: + v = plugin.run(t, options) + if not isinstance(v, Verdict): + raise TypeError(f"run() returned {type(v).__name__}, not Verdict") + except Exception as exc: + try: # under -W error the warning raises; it must not fail the test either + warnings.warn(f"faultgate-zero: check {name!r} crashed ({type(exc).__name__}); it did not fire", + FaultgateZeroWarning, stacklevel=2) + except Exception: + pass + return Verdict(name, False, f"{CRASHED}{type(exc).__name__}") + return v if v.check == name else Verdict(name, v.fired, v.reason, v.step) + + +def crashed(v: Verdict) -> bool: + return not v.fired and v.reason.startswith(CRASHED) + + def run_checks(t: Trace, names: tuple[str, ...], policy: Policy, options: Options) -> list[Verdict]: out = [] for n in names: - out.append(policy_check(t, policy) if n == "policy" else RULES[n](t, options)) + if n == "policy": + out.append(policy_check(t, policy)) + elif n in RULES: + out.append(RULES[n](t, options)) + else: + out.append(run_plugin(PLUGINS[n], t, options)) return out diff --git a/src/faultgate_zero/formats.py b/src/faultgate_zero/formats.py new file mode 100644 index 0000000..76e33e3 --- /dev/null +++ b/src/faultgate_zero/formats.py @@ -0,0 +1,139 @@ +"""Read any trace export as faultgate-zero traces: ``load(path, fmt="auto")``. + +OTLP/JSON goes through faultgate-zero's own loader. Every other format (OpenInference, Langfuse, +LangSmith, Braintrust) is read by faultgate's loaders, imported only when needed (install +``faultgate-zero[formats]``), then converted: both packages use the GenAI attribute names, so the +conversion copies spans as they are. +""" +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +from . import otlp +from .trace import Span, Trace + +FORMATS = ("auto", "otlp", "openinference", "langfuse", "langsmith", "braintrust") + + +class FormatError(ValueError): + """An export that cannot be read: unknown format, bad JSON, or a format that needs faultgate.""" + + +MESSAGE_ATTRS = ("gen_ai.input.messages", "gen_ai.output.messages") + + +def _part_from_openai(p: Any) -> dict[str, Any] | None: + if isinstance(p, str): + return {"type": "text", "content": p} + if isinstance(p, dict) and p.get("type") in ("text", "input_text", "output_text"): + return {"type": "text", "content": str(p.get("text", p.get("content", "")))} + return None + + +def normalize_messages(raw: Any) -> Any: + """Messages as GenAI ``parts``, which is what faultgate-zero reads. faultgate writes plain chat + messages (``{"role", "content"}``); their text becomes text parts. Tool calls are read the way + faultgate reads them, from parts and tool spans only, so a call is never counted twice. + Messages that already have parts are kept.""" + if isinstance(raw, str): + try: + raw = json.loads(raw) + except ValueError: + return raw + if not isinstance(raw, list): + return raw + out = [] + for m in raw: + if not isinstance(m, dict) or isinstance(m.get("parts"), list): + out.append(m) + continue + content = m.get("content") + parts: list[dict[str, Any]] = [] + if isinstance(content, list): + parts = [x for x in map(_part_from_openai, content) if x] + elif content not in (None, ""): + parts = [{"type": "text", "content": str(content)}] + out.append({**{k: v for k, v in m.items() if k != "content"}, "parts": parts}) + return out + + +def to_zero(t: Any) -> Trace: + """Convert a faultgate Trace to a faultgate-zero Trace: spans as they are, messages as parts.""" + spans = [] + for s in t.spans: + attrs = dict(s.attrs) + for k in MESSAGE_ATTRS: + if k in attrs: + attrs[k] = normalize_messages(attrs[k]) + spans.append(Span(span_id=s.span_id, parent_id=s.parent_id, name=s.name, start_ns=s.start_ns, + end_ns=s.end_ns, attrs=attrs, error=s.status_code == 2, status_message=s.status_message)) + root = t.root + attrs = root.attrs if root else {} + return Trace(trace_id=t.trace_id, scenario=str(attrs.get("faultgate.scenario") or t.key), + agent=t.agent, user_id=attrs.get("enduser.id"), spans=spans) + + +def _is_plain_otlp(doc: Any) -> bool: + if not (isinstance(doc, dict) and "resourceSpans" in doc): + return False + for rs in _dicts(doc.get("resourceSpans")): + for ss in _dicts(rs.get("scopeSpans")): + for s in _dicts(ss.get("spans")): + if any(a.get("key") == "openinference.span.kind" for a in _dicts(s.get("attributes"))): + return False + return True + + +def _dicts(v: Any) -> list[dict[str, Any]]: + return [x for x in v if isinstance(x, dict)] if isinstance(v, list) else [] + + +def detect(doc: Any) -> str: + if _is_plain_otlp(doc): + return "otlp" + return _faultgate().detect(doc) + + +def _faultgate() -> Any: + try: + from faultgate import formats as fg + except ImportError: + raise FormatError("this trace format needs faultgate's loaders: pip install 'faultgate-zero[formats]'") from None + return fg + + +def load(path: str | Path, fmt: str = "auto") -> list[Trace]: + """Every trace in one export file. Raises FormatError (a ValueError) on input it cannot read.""" + if fmt not in FORMATS: + raise FormatError(f"unknown format {fmt!r}; available: {', '.join(FORMATS)}") + p = Path(path) + try: + doc = json.loads(p.read_text(encoding="utf-8")) + except OSError as exc: + raise FormatError(f"{p}: cannot read it ({type(exc).__name__})") from None + except ValueError: + raise FormatError(f"{p}: not valid JSON") from None + if fmt == "auto": + try: + fmt = detect(doc) + except FormatError: + raise + except ValueError as exc: + raise FormatError(f"{p}: {exc}") from None + except (KeyError, TypeError, AttributeError): + raise FormatError(f"{p}: unrecognised trace format") from None + if fmt == "otlp": + try: + return otlp.from_dict(doc) + except (KeyError, TypeError, AttributeError, ValueError) as exc: + raise FormatError(f"{p}: not a valid OTLP/JSON export ({type(exc).__name__})") from None + try: + return [to_zero(t) for t in _faultgate().load(p, fmt)] + except FormatError: + raise + except ValueError as exc: + raise FormatError(f"{p}: {exc}") from None + except (KeyError, TypeError, AttributeError): + raise FormatError(f"{p}: not a valid {fmt} export") from None diff --git a/src/faultgate_zero/pytest_plugin.py b/src/faultgate_zero/pytest_plugin.py index b5174af..863b6c4 100644 --- a/src/faultgate_zero/pytest_plugin.py +++ b/src/faultgate_zero/pytest_plugin.py @@ -8,7 +8,7 @@ import pytest -from . import otlp, patch, recorder, runner +from . import checks, otlp, patch, recorder, runner from . import report as rpt from .config import Config, ConfigError, for_marker, resolve_session from .policy import PolicyError @@ -254,6 +254,11 @@ def pytest_terminal_summary(terminalreporter, exitstatus: int, config: pytest.Co tr.write_line(line) for name, n in s["fired"].items(): tr.write_line(f" {name}: {n}") + crashes = sum(1 for e in state.entries for v in e["verdicts"] + if not v["fired"] and str(v["reason"]).startswith(checks.CRASHED)) + if crashes: + tr.write_line(f"faultgate-zero: {crashes} check{'s' if crashes > 1 else ''} crashed and did not fire " + "(a bug in a check plugin; see the warnings)") for label, path in (("JSON report", cfg.json), ("Markdown summary", cfg.markdown), ("OTLP export", cfg.export_otlp)): if path: tr.write_line(f"faultgate-zero: {label} written to {path}") diff --git a/src/faultgate_zero/runner.py b/src/faultgate_zero/runner.py index 5c158e8..9d8dbf7 100644 --- a/src/faultgate_zero/runner.py +++ b/src/faultgate_zero/runner.py @@ -5,7 +5,7 @@ from typing import Callable from . import redact as _redact -from .checks import DEFAULT_CHECKS, Options, Verdict, resolve, run_checks +from .checks import DEFAULT_CHECKS, PLUGINS, Options, Verdict, resolve, run_checks from .policy import Policy from .trace import Trace @@ -69,7 +69,8 @@ def summary(self) -> str: lines = [f"faultgate-zero: {n} check{'s' if n > 1 else ''} fired on run {self.scenario!r} ({head})"] width = max(len(v.check) for v in self.fired) for v in self.fired: - for j, part in enumerate(v.reason.split("; ") if v.check == "policy" else [v.reason]): + multi = v.check == "policy" or v.check in PLUGINS # one part per line, joined by "; " + for j, part in enumerate(v.reason.split("; ") if multi else [v.reason]): lines.append(f" {(v.check if j == 0 else '').ljust(width)} {part}") return "\n".join(lines) diff --git a/tests/data/formats/braintrust.json b/tests/data/formats/braintrust.json new file mode 100644 index 0000000..c069b5b --- /dev/null +++ b/tests/data/formats/braintrust.json @@ -0,0 +1,19 @@ +[ + {"id": "e1", "span_id": "s-root", "root_span_id": "s-root", "span_parents": [], + "span_attributes": {"name": "support_agent", "type": "task"}, + "input": "Refund order o-7 for customer C-123", "output": "Refunded $120.00 to C-123.", + "metadata": {"scenario": "refund-o7", "enduser.id": "C-123"}, + "metrics": {"start": 1790848800.0, "end": 1790848804.5}}, + {"id": "e2", "span_id": "s-llm", "root_span_id": "s-root", "span_parents": ["s-root"], + "span_attributes": {"name": "chat", "type": "llm"}, + "metadata": {"finish_reason": "tool_calls"}, + "metrics": {"start": 1790848800.1, "end": 1790848801.0, "prompt_tokens": 812, "completion_tokens": 40}}, + {"id": "e3", "span_id": "s-t1", "root_span_id": "s-root", "span_parents": ["s-root"], + "span_attributes": {"name": "lookup_order", "type": "tool"}, + "input": {"order_id": "o-7"}, "output": {"total": 120}, + "metrics": {"start": 1790848801.1, "end": 1790848801.3}}, + {"id": "e4", "span_id": "s-t2", "root_span_id": "s-root", "span_parents": ["s-root"], + "span_attributes": {"name": "issue_refund", "type": "tool"}, + "input": {"customer_id": "C-123", "amount": 120}, "output": "ok", + "metrics": {"start": 1790848802.0, "end": 1790848802.4}} +] diff --git a/tests/data/formats/langfuse.json b/tests/data/formats/langfuse.json new file mode 100644 index 0000000..86c8f6f --- /dev/null +++ b/tests/data/formats/langfuse.json @@ -0,0 +1,17 @@ +{"data": [ + {"id": "t-lf-1", "name": "support_agent", "timestamp": "2026-10-01T10:00:00.000Z", "latency": 4.5, + "input": {"messages": [{"role": "user", "content": "Refund order o-7 for customer C-123"}]}, + "output": "Refunded $120.00 to C-123.", + "metadata": {"scenario": "refund-o7"}, "userId": "C-123", + "observations": [ + {"id": "o-gen", "traceId": "t-lf-1", "type": "GENERATION", "name": "chat", "parentObservationId": null, + "startTime": "2026-10-01T10:00:00.100Z", "endTime": "2026-10-01T10:00:01.000Z", + "usageDetails": {"input": 812, "output": 40}, "costDetails": {"total": 0.0021}, "level": "DEFAULT"}, + {"id": "o-t1", "traceId": "t-lf-1", "type": "TOOL", "name": "lookup_order", "parentObservationId": null, + "input": {"order_id": "o-7"}, "output": {"total": 120}, + "startTime": "2026-10-01T10:00:01.100Z", "endTime": "2026-10-01T10:00:01.300Z", "level": "DEFAULT"}, + {"id": "o-t2", "traceId": "t-lf-1", "type": "TOOL", "name": "issue_refund", "parentObservationId": null, + "input": {"customer_id": "C-123", "amount": 120}, "output": "ok", + "startTime": "2026-10-01T10:00:02.000Z", "endTime": "2026-10-01T10:00:02.400Z", "level": "DEFAULT"} + ]} +]} diff --git a/tests/data/formats/langsmith.json b/tests/data/formats/langsmith.json new file mode 100644 index 0000000..ac383e8 --- /dev/null +++ b/tests/data/formats/langsmith.json @@ -0,0 +1,16 @@ +[ + {"id": "r-root", "trace_id": "t-ls-1", "parent_run_id": null, "run_type": "chain", "name": "support_agent", + "inputs": {"input": "Refund order o-7 for customer C-123"}, "outputs": {"output": "Refunded $120.00 to C-123."}, + "start_time": "2026-10-01T10:00:00.000000", "end_time": "2026-10-01T10:00:04.500000", "error": null, + "extra": {"metadata": {"scenario": "refund-o7", "enduser.id": "C-123"}}}, + {"id": "r-llm", "trace_id": "t-ls-1", "parent_run_id": "r-root", "run_type": "llm", "name": "ChatOpenAI", + "inputs": {}, "outputs": {"generations": [[{"text": "", "generation_info": {"finish_reason": "tool_calls"}}]]}, + "start_time": "2026-10-01T10:00:00.100000", "end_time": "2026-10-01T10:00:01.000000", + "prompt_tokens": 812, "completion_tokens": 40, "total_cost": 0.0021}, + {"id": "r-t1", "trace_id": "t-ls-1", "parent_run_id": "r-root", "run_type": "tool", "name": "lookup_order", + "inputs": {"input": {"order_id": "o-7"}}, "outputs": {"output": {"total": 120}}, + "start_time": "2026-10-01T10:00:01.100000", "end_time": "2026-10-01T10:00:01.300000"}, + {"id": "r-t2", "trace_id": "t-ls-1", "parent_run_id": "r-root", "run_type": "tool", "name": "issue_refund", + "inputs": {"customer_id": "C-123", "amount": 120}, "outputs": {"output": "ok"}, + "start_time": "2026-10-01T10:00:02.000000", "end_time": "2026-10-01T10:00:02.400000"} +] diff --git a/tests/data/formats/openinference.json b/tests/data/formats/openinference.json new file mode 100644 index 0000000..a97f84e --- /dev/null +++ b/tests/data/formats/openinference.json @@ -0,0 +1,30 @@ +{"resourceSpans": [{"scopeSpans": [{"spans": [ + {"traceId": "0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f", "spanId": "1000000000000001", "name": "support_agent", + "startTimeUnixNano": "1790848800000000000", "endTimeUnixNano": "1790848804500000000", + "attributes": [ + {"key": "openinference.span.kind", "value": {"stringValue": "AGENT"}}, + {"key": "input.value", "value": {"stringValue": "Refund order o-7 for customer C-123"}}, + {"key": "output.value", "value": {"stringValue": "Refunded $120.00 to C-123."}}, + {"key": "metadata", "value": {"stringValue": "{\"scenario\": \"refund-o7\"}"}}, + {"key": "user.id", "value": {"stringValue": "C-123"}}]}, + {"traceId": "0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f", "spanId": "1000000000000002", "parentSpanId": "1000000000000001", + "name": "ChatCompletion", "startTimeUnixNano": "1790848800100000000", "endTimeUnixNano": "1790848801000000000", + "attributes": [ + {"key": "openinference.span.kind", "value": {"stringValue": "LLM"}}, + {"key": "llm.token_count.prompt", "value": {"intValue": "812"}}, + {"key": "llm.token_count.completion", "value": {"intValue": "40"}}]}, + {"traceId": "0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f", "spanId": "1000000000000003", "parentSpanId": "1000000000000001", + "name": "lookup_order", "startTimeUnixNano": "1790848801100000000", "endTimeUnixNano": "1790848801300000000", + "attributes": [ + {"key": "openinference.span.kind", "value": {"stringValue": "TOOL"}}, + {"key": "tool.name", "value": {"stringValue": "lookup_order"}}, + {"key": "input.value", "value": {"stringValue": "{\"order_id\": \"o-7\"}"}}, + {"key": "output.value", "value": {"stringValue": "{\"total\": 120}"}}]}, + {"traceId": "0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f", "spanId": "1000000000000004", "parentSpanId": "1000000000000001", + "name": "issue_refund", "startTimeUnixNano": "1790848802000000000", "endTimeUnixNano": "1790848802400000000", + "attributes": [ + {"key": "openinference.span.kind", "value": {"stringValue": "TOOL"}}, + {"key": "tool.name", "value": {"stringValue": "issue_refund"}}, + {"key": "tool.parameters", "value": {"stringValue": "{\"customer_id\": \"C-123\", \"amount\": 120}"}}, + {"key": "output.value", "value": {"stringValue": "ok"}}]} +]}]}]} diff --git a/tests/data/formats/otlp.json b/tests/data/formats/otlp.json new file mode 100644 index 0000000..572620e --- /dev/null +++ b/tests/data/formats/otlp.json @@ -0,0 +1,1539 @@ +{ + "resourceSpans": [ + { + "resource": { + "attributes": [ + { + "key": "service.name", + "value": { + "stringValue": "faultline-agent" + } + } + ] + }, + "scopeSpans": [ + { + "scope": { + "name": "faultgate.examples", + "version": "0.1.0" + }, + "spans": [ + { + "traceId": "fc2cdab304bd903931b3e698c585e8fb", + "spanId": "275532b54a183867", + "name": "invoke_agent", + "kind": 1, + "startTimeUnixNano": "1758000000000000000", + "endTimeUnixNano": "1758000012100000000", + "attributes": [ + { + "key": "gen_ai.operation.name", + "value": { + "stringValue": "invoke_agent" + } + }, + { + "key": "gen_ai.system", + "value": { + "stringValue": "openrouter" + } + }, + { + "key": "gen_ai.agent.name", + "value": { + "stringValue": "faultline-agent" + } + }, + { + "key": "gen_ai.request.model", + "value": { + "stringValue": "openai/gpt-5.6-luna" + } + }, + { + "key": "gen_ai.input.messages", + "value": { + "stringValue": "[{\"role\": \"user\", \"parts\": [{\"type\": \"text\", \"content\": \"Step 1: What is the internal codename of Zephyr Foundry? Step 2: The organization holding internal codename from Step 1 is affiliated with a partner organization. What is the external auditor of that partner organization? Step 3: The organization holding external auditor from Step 2 is affiliated with a partner organization. What is the internal codename of that partner organization? Provide the final answer for Step 3.\"}]}]" + } + }, + { + "key": "gen_ai.output.messages", + "value": { + "stringValue": "[{\"role\": \"assistant\", \"parts\": [{\"type\": \"text\", \"content\": \"\"}]}]" + } + }, + { + "key": "gen_ai.usage.input_tokens", + "value": { + "intValue": "19871" + } + }, + { + "key": "gen_ai.usage.output_tokens", + "value": { + "intValue": "1579" + } + }, + { + "key": "faultgate.termination", + "value": { + "stringValue": "Step cap reached" + } + } + ], + "status": { + "code": 2 + } + }, + { + "traceId": "fc2cdab304bd903931b3e698c585e8fb", + "spanId": "5dcd3e4007e456ea", + "parentSpanId": "275532b54a183867", + "name": "execute_tool search", + "kind": 1, + "startTimeUnixNano": "1758000001000000000", + "endTimeUnixNano": "1758000001800000000", + "attributes": [ + { + "key": "gen_ai.operation.name", + "value": { + "stringValue": "execute_tool" + } + }, + { + "key": "gen_ai.tool.name", + "value": { + "stringValue": "search" + } + }, + { + "key": "gen_ai.tool.call.arguments", + "value": { + "stringValue": "{\"query\": \"\\\"Zephyr Foundry\\\" internal codename\"}" + } + }, + { + "key": "gen_ai.tool.call.result", + "value": { + "stringValue": "{\"ok\": true, \"error\": null, \"n_candidates\": 0, \"candidates\": []}" + } + } + ], + "status": { + "code": 1 + } + }, + { + "traceId": "fc2cdab304bd903931b3e698c585e8fb", + "spanId": "3de338963f16a4f7", + "parentSpanId": "275532b54a183867", + "name": "execute_tool search", + "kind": 1, + "startTimeUnixNano": "1758000002000000000", + "endTimeUnixNano": "1758000002800000000", + "attributes": [ + { + "key": "gen_ai.operation.name", + "value": { + "stringValue": "execute_tool" + } + }, + { + "key": "gen_ai.tool.name", + "value": { + "stringValue": "search" + } + }, + { + "key": "gen_ai.tool.call.arguments", + "value": { + "stringValue": "{\"query\": \"Zephyr Foundry\"}" + } + }, + { + "key": "gen_ai.tool.call.result", + "value": { + "stringValue": "{\"ok\": true, \"error\": null, \"n_candidates\": 5, \"candidates\": [\"doc-0049\", \"doc-0184\", \"link-41b1d27ddb03\", \"link-52a7f49f34de\", \"link-618f865e3526\"]}" + } + } + ], + "status": { + "code": 1 + } + }, + { + "traceId": "fc2cdab304bd903931b3e698c585e8fb", + "spanId": "f57648a3c6d23ae1", + "parentSpanId": "275532b54a183867", + "name": "execute_tool search", + "kind": 1, + "startTimeUnixNano": "1758000003000000000", + "endTimeUnixNano": "1758000003800000000", + "attributes": [ + { + "key": "gen_ai.operation.name", + "value": { + "stringValue": "execute_tool" + } + }, + { + "key": "gen_ai.tool.name", + "value": { + "stringValue": "search" + } + }, + { + "key": "gen_ai.tool.call.arguments", + "value": { + "stringValue": "{\"query\": \"\\\"Zinc-4249\\\" affiliated partner organization\"}" + } + }, + { + "key": "gen_ai.tool.call.result", + "value": { + "stringValue": "{\"ok\": true, \"error\": null, \"n_candidates\": 0, \"candidates\": []}" + } + } + ], + "status": { + "code": 1 + } + }, + { + "traceId": "fc2cdab304bd903931b3e698c585e8fb", + "spanId": "60fd9797b4dc8e1d", + "parentSpanId": "275532b54a183867", + "name": "execute_tool search", + "kind": 1, + "startTimeUnixNano": "1758000004000000000", + "endTimeUnixNano": "1758000004800000000", + "attributes": [ + { + "key": "gen_ai.operation.name", + "value": { + "stringValue": "execute_tool" + } + }, + { + "key": "gen_ai.tool.name", + "value": { + "stringValue": "search" + } + }, + { + "key": "gen_ai.tool.call.arguments", + "value": { + "stringValue": "{\"query\": \"\\\"Zephyr Foundry\\\" \\\"affiliated\\\"\"}" + } + }, + { + "key": "gen_ai.tool.call.result", + "value": { + "stringValue": "{\"ok\": true, \"error\": null, \"n_candidates\": 0, \"candidates\": []}" + } + } + ], + "status": { + "code": 1 + } + }, + { + "traceId": "fc2cdab304bd903931b3e698c585e8fb", + "spanId": "2f4d34fc07bdaa7a", + "parentSpanId": "275532b54a183867", + "name": "execute_tool search", + "kind": 1, + "startTimeUnixNano": "1758000005000000000", + "endTimeUnixNano": "1758000005800000000", + "attributes": [ + { + "key": "gen_ai.operation.name", + "value": { + "stringValue": "execute_tool" + } + }, + { + "key": "gen_ai.tool.name", + "value": { + "stringValue": "search" + } + }, + { + "key": "gen_ai.tool.call.arguments", + "value": { + "stringValue": "{\"query\": \"\\\"Rune-4241\\\"\"}" + } + }, + { + "key": "gen_ai.tool.call.result", + "value": { + "stringValue": "{\"ok\": true, \"error\": null, \"n_candidates\": 0, \"candidates\": []}" + } + } + ], + "status": { + "code": 1 + } + }, + { + "traceId": "fc2cdab304bd903931b3e698c585e8fb", + "spanId": "31738957d1c2a2e6", + "parentSpanId": "275532b54a183867", + "name": "execute_tool search", + "kind": 1, + "startTimeUnixNano": "1758000006000000000", + "endTimeUnixNano": "1758000006800000000", + "attributes": [ + { + "key": "gen_ai.operation.name", + "value": { + "stringValue": "execute_tool" + } + }, + { + "key": "gen_ai.tool.name", + "value": { + "stringValue": "search" + } + }, + { + "key": "gen_ai.tool.call.arguments", + "value": { + "stringValue": "{\"query\": \"Rune-4241 external auditor organization\"}" + } + }, + { + "key": "gen_ai.tool.call.result", + "value": { + "stringValue": "{\"ok\": true, \"error\": null, \"n_candidates\": 0, \"candidates\": []}" + } + } + ], + "status": { + "code": 1 + } + }, + { + "traceId": "fc2cdab304bd903931b3e698c585e8fb", + "spanId": "8a5c7c65c24404ce", + "parentSpanId": "275532b54a183867", + "name": "execute_tool search", + "kind": 1, + "startTimeUnixNano": "1758000007000000000", + "endTimeUnixNano": "1758000007800000000", + "attributes": [ + { + "key": "gen_ai.operation.name", + "value": { + "stringValue": "execute_tool" + } + }, + { + "key": "gen_ai.tool.name", + "value": { + "stringValue": "search" + } + }, + { + "key": "gen_ai.tool.call.arguments", + "value": { + "stringValue": "{\"query\": \"\\\"officially affiliated with Zephyr Foundry\\\"\"}" + } + }, + { + "key": "gen_ai.tool.call.result", + "value": { + "stringValue": "{\"ok\": true, \"error\": null, \"n_candidates\": 0, \"candidates\": []}" + } + } + ], + "status": { + "code": 1 + } + }, + { + "traceId": "fc2cdab304bd903931b3e698c585e8fb", + "spanId": "f6328cf3b5901042", + "parentSpanId": "275532b54a183867", + "name": "execute_tool search", + "kind": 1, + "startTimeUnixNano": "1758000008000000000", + "endTimeUnixNano": "1758000008800000000", + "attributes": [ + { + "key": "gen_ai.operation.name", + "value": { + "stringValue": "execute_tool" + } + }, + { + "key": "gen_ai.tool.name", + "value": { + "stringValue": "search" + } + }, + { + "key": "gen_ai.tool.call.arguments", + "value": { + "stringValue": "{\"query\": \"Rune 4241\"}" + } + }, + { + "key": "gen_ai.tool.call.result", + "value": { + "stringValue": "{\"ok\": true, \"error\": null, \"n_candidates\": 0, \"candidates\": []}" + } + } + ], + "status": { + "code": 1 + } + }, + { + "traceId": "fc2cdab304bd903931b3e698c585e8fb", + "spanId": "5fa4a962c8924a92", + "parentSpanId": "275532b54a183867", + "name": "execute_tool search", + "kind": 1, + "startTimeUnixNano": "1758000009000000000", + "endTimeUnixNano": "1758000009800000000", + "attributes": [ + { + "key": "gen_ai.operation.name", + "value": { + "stringValue": "execute_tool" + } + }, + { + "key": "gen_ai.tool.name", + "value": { + "stringValue": "search" + } + }, + { + "key": "gen_ai.tool.call.arguments", + "value": { + "stringValue": "{\"query\": \"\\\"external auditor\\\" \\\"Rune-4241\\\"\"}" + } + }, + { + "key": "gen_ai.tool.call.result", + "value": { + "stringValue": "{\"ok\": true, \"error\": null, \"n_candidates\": 0, \"candidates\": []}" + } + } + ], + "status": { + "code": 1 + } + }, + { + "traceId": "fc2cdab304bd903931b3e698c585e8fb", + "spanId": "f337aff5b6f8c256", + "parentSpanId": "275532b54a183867", + "name": "execute_tool lookup", + "kind": 1, + "startTimeUnixNano": "1758000010000000000", + "endTimeUnixNano": "1758000010800000000", + "attributes": [ + { + "key": "gen_ai.operation.name", + "value": { + "stringValue": "execute_tool" + } + }, + { + "key": "gen_ai.tool.name", + "value": { + "stringValue": "lookup" + } + }, + { + "key": "gen_ai.tool.call.arguments", + "value": { + "stringValue": "{\"doc_id\": \"link-618f865e3526\"}" + } + }, + { + "key": "gen_ai.tool.call.result", + "value": { + "stringValue": "{\"ok\": true, \"error\": null, \"n_candidates\": 0, \"candidates\": []}" + } + } + ], + "status": { + "code": 1 + } + }, + { + "traceId": "fc2cdab304bd903931b3e698c585e8fb", + "spanId": "c9148fe0a9bdbdbf", + "parentSpanId": "275532b54a183867", + "name": "execute_tool search", + "kind": 1, + "startTimeUnixNano": "1758000011000000000", + "endTimeUnixNano": "1758000011800000000", + "attributes": [ + { + "key": "gen_ai.operation.name", + "value": { + "stringValue": "execute_tool" + } + }, + { + "key": "gen_ai.tool.name", + "value": { + "stringValue": "search" + } + }, + { + "key": "gen_ai.tool.call.arguments", + "value": { + "stringValue": "{\"query\": \"external auditor Rune\"}" + } + }, + { + "key": "gen_ai.tool.call.result", + "value": { + "stringValue": "{\"ok\": true, \"error\": null, \"n_candidates\": 5, \"candidates\": [\"link-0105f7c1b122\", \"link-025935fe9a65\", \"link-02da14669841\", \"link-04a01ded6ce6\", \"link-05bb197a3734\"]}" + } + } + ], + "status": { + "code": 1 + } + }, + { + "traceId": "fc2cdab304bd903931b3e698c585e8fb", + "spanId": "ed58c07adfc5f179", + "parentSpanId": "275532b54a183867", + "name": "execute_tool search", + "kind": 1, + "startTimeUnixNano": "1758000012000000000", + "endTimeUnixNano": "1758000012800000000", + "attributes": [ + { + "key": "gen_ai.operation.name", + "value": { + "stringValue": "execute_tool" + } + }, + { + "key": "gen_ai.tool.name", + "value": { + "stringValue": "search" + } + }, + { + "key": "gen_ai.tool.call.arguments", + "value": { + "stringValue": "{\"query\": \"\\\"4241\\\"\"}" + } + }, + { + "key": "gen_ai.tool.call.result", + "value": { + "stringValue": "{\"ok\": true, \"error\": null, \"n_candidates\": 0, \"candidates\": []}" + } + } + ], + "status": { + "code": 1 + } + }, + { + "traceId": "6d1f9d67a2d4f7d2f0d0793b00931995", + "spanId": "d8b43e84d7e5d958", + "name": "invoke_agent", + "kind": 1, + "startTimeUnixNano": "1758000000000000000", + "endTimeUnixNano": "1758000005100000000", + "attributes": [ + { + "key": "gen_ai.operation.name", + "value": { + "stringValue": "invoke_agent" + } + }, + { + "key": "gen_ai.system", + "value": { + "stringValue": "openrouter" + } + }, + { + "key": "gen_ai.agent.name", + "value": { + "stringValue": "faultline-agent" + } + }, + { + "key": "gen_ai.request.model", + "value": { + "stringValue": "openai/gpt-5.6-luna" + } + }, + { + "key": "gen_ai.input.messages", + "value": { + "stringValue": "[{\"role\": \"user\", \"parts\": [{\"type\": \"text\", \"content\": \"Which firm is the external auditor of Aster Labs?\"}]}]" + } + }, + { + "key": "gen_ai.output.messages", + "value": { + "stringValue": "[{\"role\": \"assistant\", \"parts\": [{\"type\": \"text\", \"content\": \"I couldn\\u2019t verify the external auditor for Aster Labs from the available documents\\u2014the searches returned no matching source.\"}]}]" + } + }, + { + "key": "gen_ai.usage.input_tokens", + "value": { + "intValue": "1675" + } + }, + { + "key": "gen_ai.usage.output_tokens", + "value": { + "intValue": "640" + } + }, + { + "key": "faultgate.termination", + "value": { + "stringValue": "answered" + } + } + ], + "status": { + "code": 1 + } + }, + { + "traceId": "6d1f9d67a2d4f7d2f0d0793b00931995", + "spanId": "ffb8cb377fbed2ee", + "parentSpanId": "d8b43e84d7e5d958", + "name": "execute_tool search", + "kind": 1, + "startTimeUnixNano": "1758000001000000000", + "endTimeUnixNano": "1758000001800000000", + "attributes": [ + { + "key": "gen_ai.operation.name", + "value": { + "stringValue": "execute_tool" + } + }, + { + "key": "gen_ai.tool.name", + "value": { + "stringValue": "search" + } + }, + { + "key": "gen_ai.tool.call.arguments", + "value": { + "stringValue": "{\"query\": \"Aster Labs external auditor firm\"}" + } + }, + { + "key": "gen_ai.tool.call.result", + "value": { + "stringValue": "{\"ok\": true, \"error\": null, \"n_candidates\": 0, \"candidates\": []}" + } + } + ], + "status": { + "code": 1 + } + }, + { + "traceId": "6d1f9d67a2d4f7d2f0d0793b00931995", + "spanId": "5c20cfc12754b8b4", + "parentSpanId": "d8b43e84d7e5d958", + "name": "execute_tool search", + "kind": 1, + "startTimeUnixNano": "1758000002000000000", + "endTimeUnixNano": "1758000002800000000", + "attributes": [ + { + "key": "gen_ai.operation.name", + "value": { + "stringValue": "execute_tool" + } + }, + { + "key": "gen_ai.tool.name", + "value": { + "stringValue": "search" + } + }, + { + "key": "gen_ai.tool.call.arguments", + "value": { + "stringValue": "{\"query\": \"\\\"Aster Labs\\\" auditor\"}" + } + }, + { + "key": "gen_ai.tool.call.result", + "value": { + "stringValue": "{\"ok\": true, \"error\": null, \"n_candidates\": 0, \"candidates\": []}" + } + } + ], + "status": { + "code": 1 + } + }, + { + "traceId": "6d1f9d67a2d4f7d2f0d0793b00931995", + "spanId": "515ad88812582245", + "parentSpanId": "d8b43e84d7e5d958", + "name": "execute_tool search", + "kind": 1, + "startTimeUnixNano": "1758000003000000000", + "endTimeUnixNano": "1758000003800000000", + "attributes": [ + { + "key": "gen_ai.operation.name", + "value": { + "stringValue": "execute_tool" + } + }, + { + "key": "gen_ai.tool.name", + "value": { + "stringValue": "search" + } + }, + { + "key": "gen_ai.tool.call.arguments", + "value": { + "stringValue": "{\"query\": \"Aster Labs annual report auditor external audit firm\"}" + } + }, + { + "key": "gen_ai.tool.call.result", + "value": { + "stringValue": "{\"ok\": true, \"error\": null, \"n_candidates\": 0, \"candidates\": []}" + } + } + ], + "status": { + "code": 1 + } + }, + { + "traceId": "6d1f9d67a2d4f7d2f0d0793b00931995", + "spanId": "4bd3b5545bdea306", + "parentSpanId": "d8b43e84d7e5d958", + "name": "execute_tool search", + "kind": 1, + "startTimeUnixNano": "1758000004000000000", + "endTimeUnixNano": "1758000004800000000", + "attributes": [ + { + "key": "gen_ai.operation.name", + "value": { + "stringValue": "execute_tool" + } + }, + { + "key": "gen_ai.tool.name", + "value": { + "stringValue": "search" + } + }, + { + "key": "gen_ai.tool.call.arguments", + "value": { + "stringValue": "{\"query\": \"Aster DM Healthcare external auditor firm Deloitte annual report\"}" + } + }, + { + "key": "gen_ai.tool.call.result", + "value": { + "stringValue": "{\"ok\": true, \"error\": null, \"n_candidates\": 0, \"candidates\": []}" + } + } + ], + "status": { + "code": 1 + } + }, + { + "traceId": "8bf3879339eba84583941e17ec56b3de", + "spanId": "fa44146e0c980d5c", + "name": "invoke_agent", + "kind": 1, + "startTimeUnixNano": "1758000000000000000", + "endTimeUnixNano": "1758000003100000000", + "attributes": [ + { + "key": "gen_ai.operation.name", + "value": { + "stringValue": "invoke_agent" + } + }, + { + "key": "gen_ai.system", + "value": { + "stringValue": "openrouter" + } + }, + { + "key": "gen_ai.agent.name", + "value": { + "stringValue": "faultline-agent" + } + }, + { + "key": "gen_ai.request.model", + "value": { + "stringValue": "openai/gpt-5.6-luna" + } + }, + { + "key": "gen_ai.input.messages", + "value": { + "stringValue": "[{\"role\": \"user\", \"parts\": [{\"type\": \"text\", \"content\": \"What is the internal codename of Girona Foundry?\"}]}]" + } + }, + { + "key": "gen_ai.output.messages", + "value": { + "stringValue": "[{\"role\": \"assistant\", \"parts\": [{\"type\": \"text\", \"content\": \"The internal codename of Girona Foundry is **Zinc-4524**.\"}]}]" + } + }, + { + "key": "gen_ai.usage.input_tokens", + "value": { + "intValue": "978" + } + }, + { + "key": "gen_ai.usage.output_tokens", + "value": { + "intValue": "324" + } + }, + { + "key": "faultgate.termination", + "value": { + "stringValue": "answered" + } + } + ], + "status": { + "code": 1 + } + }, + { + "traceId": "8bf3879339eba84583941e17ec56b3de", + "spanId": "8e823746b2003e24", + "parentSpanId": "fa44146e0c980d5c", + "name": "execute_tool search", + "kind": 1, + "startTimeUnixNano": "1758000001000000000", + "endTimeUnixNano": "1758000001800000000", + "attributes": [ + { + "key": "gen_ai.operation.name", + "value": { + "stringValue": "execute_tool" + } + }, + { + "key": "gen_ai.tool.name", + "value": { + "stringValue": "search" + } + }, + { + "key": "gen_ai.tool.call.arguments", + "value": { + "stringValue": "{\"query\": \"\\\"Girona Foundry\\\" internal codename\"}" + } + }, + { + "key": "gen_ai.tool.call.result", + "value": { + "stringValue": "{\"ok\": true, \"error\": null, \"n_candidates\": 0, \"candidates\": []}" + } + } + ], + "status": { + "code": 1 + } + }, + { + "traceId": "8bf3879339eba84583941e17ec56b3de", + "spanId": "7537fce2d8bfe949", + "parentSpanId": "fa44146e0c980d5c", + "name": "execute_tool search", + "kind": 1, + "startTimeUnixNano": "1758000002000000000", + "endTimeUnixNano": "1758000002800000000", + "attributes": [ + { + "key": "gen_ai.operation.name", + "value": { + "stringValue": "execute_tool" + } + }, + { + "key": "gen_ai.tool.name", + "value": { + "stringValue": "search" + } + }, + { + "key": "gen_ai.tool.call.arguments", + "value": { + "stringValue": "{\"query\": \"Girona Foundry\"}" + } + }, + { + "key": "gen_ai.tool.call.result", + "value": { + "stringValue": "{\"ok\": true, \"error\": null, \"n_candidates\": 5, \"candidates\": [\"doc-0104\", \"doc-0183\", \"link-08652480c27a\", \"link-120d73512e0f\", \"link-4f82dae92239\"]}" + } + } + ], + "status": { + "code": 1 + } + }, + { + "traceId": "3a9e507a0f6dd628395d642ce5db5a3e", + "spanId": "2426c8a9be08f906", + "name": "invoke_agent", + "kind": 1, + "startTimeUnixNano": "1758000000000000000", + "endTimeUnixNano": "1758000009100000000", + "attributes": [ + { + "key": "gen_ai.operation.name", + "value": { + "stringValue": "invoke_agent" + } + }, + { + "key": "gen_ai.system", + "value": { + "stringValue": "openrouter" + } + }, + { + "key": "gen_ai.agent.name", + "value": { + "stringValue": "faultline-agent" + } + }, + { + "key": "gen_ai.request.model", + "value": { + "stringValue": "openai/gpt-5.6-luna" + } + }, + { + "key": "gen_ai.input.messages", + "value": { + "stringValue": "[{\"role\": \"user\", \"parts\": [{\"type\": \"text\", \"content\": \"What is the internal codename of Aster Systems?\"}]}]" + } + }, + { + "key": "gen_ai.output.messages", + "value": { + "stringValue": "[{\"role\": \"assistant\", \"parts\": [{\"type\": \"text\", \"content\": \"The internal codename of Aster Systems is **Kelvin-4384**.\"}]}]" + } + }, + { + "key": "gen_ai.usage.input_tokens", + "value": { + "intValue": "8726" + } + }, + { + "key": "gen_ai.usage.output_tokens", + "value": { + "intValue": "1315" + } + }, + { + "key": "faultgate.termination", + "value": { + "stringValue": "answered" + } + } + ], + "status": { + "code": 1 + } + }, + { + "traceId": "3a9e507a0f6dd628395d642ce5db5a3e", + "spanId": "ba0da268c0756c3a", + "parentSpanId": "2426c8a9be08f906", + "name": "execute_tool search", + "kind": 1, + "startTimeUnixNano": "1758000001000000000", + "endTimeUnixNano": "1758000001800000000", + "attributes": [ + { + "key": "gen_ai.operation.name", + "value": { + "stringValue": "execute_tool" + } + }, + { + "key": "gen_ai.tool.name", + "value": { + "stringValue": "search" + } + }, + { + "key": "gen_ai.tool.call.arguments", + "value": { + "stringValue": "{\"query\": \"Aster Systems internal codename\"}" + } + }, + { + "key": "gen_ai.tool.call.result", + "value": { + "stringValue": "{\"ok\": true, \"error\": null, \"n_candidates\": 0, \"candidates\": []}" + } + } + ], + "status": { + "code": 1 + } + }, + { + "traceId": "3a9e507a0f6dd628395d642ce5db5a3e", + "spanId": "e8b742fe42466da5", + "parentSpanId": "2426c8a9be08f906", + "name": "execute_tool search", + "kind": 1, + "startTimeUnixNano": "1758000002000000000", + "endTimeUnixNano": "1758000002800000000", + "attributes": [ + { + "key": "gen_ai.operation.name", + "value": { + "stringValue": "execute_tool" + } + }, + { + "key": "gen_ai.tool.name", + "value": { + "stringValue": "search" + } + }, + { + "key": "gen_ai.tool.call.arguments", + "value": { + "stringValue": "{\"query\": \"\\\"Aster Systems\\\"\"}" + } + }, + { + "key": "gen_ai.tool.call.result", + "value": { + "stringValue": "{\"ok\": true, \"error\": null, \"n_candidates\": 0, \"candidates\": []}" + } + } + ], + "status": { + "code": 1 + } + }, + { + "traceId": "3a9e507a0f6dd628395d642ce5db5a3e", + "spanId": "568516d5ae320fd5", + "parentSpanId": "2426c8a9be08f906", + "name": "execute_tool search", + "kind": 1, + "startTimeUnixNano": "1758000003000000000", + "endTimeUnixNano": "1758000003800000000", + "attributes": [ + { + "key": "gen_ai.operation.name", + "value": { + "stringValue": "execute_tool" + } + }, + { + "key": "gen_ai.tool.name", + "value": { + "stringValue": "search" + } + }, + { + "key": "gen_ai.tool.call.arguments", + "value": { + "stringValue": "{\"query\": \"Aster system codename project\"}" + } + }, + { + "key": "gen_ai.tool.call.result", + "value": { + "stringValue": "{\"ok\": true, \"error\": null, \"n_candidates\": 0, \"candidates\": []}" + } + } + ], + "status": { + "code": 1 + } + }, + { + "traceId": "3a9e507a0f6dd628395d642ce5db5a3e", + "spanId": "eca1592e8ce19736", + "parentSpanId": "2426c8a9be08f906", + "name": "execute_tool search", + "kind": 1, + "startTimeUnixNano": "1758000004000000000", + "endTimeUnixNano": "1758000004800000000", + "attributes": [ + { + "key": "gen_ai.operation.name", + "value": { + "stringValue": "execute_tool" + } + }, + { + "key": "gen_ai.tool.name", + "value": { + "stringValue": "search" + } + }, + { + "key": "gen_ai.tool.call.arguments", + "value": { + "stringValue": "{\"query\": \"Aster internal name\"}" + } + }, + { + "key": "gen_ai.tool.call.result", + "value": { + "stringValue": "{\"ok\": true, \"error\": null, \"n_candidates\": 0, \"candidates\": []}" + } + } + ], + "status": { + "code": 1 + } + }, + { + "traceId": "3a9e507a0f6dd628395d642ce5db5a3e", + "spanId": "b82c1f5a3f8c475d", + "parentSpanId": "2426c8a9be08f906", + "name": "execute_tool search", + "kind": 1, + "startTimeUnixNano": "1758000005000000000", + "endTimeUnixNano": "1758000005800000000", + "attributes": [ + { + "key": "gen_ai.operation.name", + "value": { + "stringValue": "execute_tool" + } + }, + { + "key": "gen_ai.tool.name", + "value": { + "stringValue": "search" + } + }, + { + "key": "gen_ai.tool.call.arguments", + "value": { + "stringValue": "{\"query\": \"\\\"Aster\\\" \\\"codename\\\"\"}" + } + }, + { + "key": "gen_ai.tool.call.result", + "value": { + "stringValue": "{\"ok\": true, \"error\": null, \"n_candidates\": 0, \"candidates\": []}" + } + } + ], + "status": { + "code": 1 + } + }, + { + "traceId": "3a9e507a0f6dd628395d642ce5db5a3e", + "spanId": "8f0fc15cc44b8edf", + "parentSpanId": "2426c8a9be08f906", + "name": "execute_tool search", + "kind": 1, + "startTimeUnixNano": "1758000006000000000", + "endTimeUnixNano": "1758000006800000000", + "attributes": [ + { + "key": "gen_ai.operation.name", + "value": { + "stringValue": "execute_tool" + } + }, + { + "key": "gen_ai.tool.name", + "value": { + "stringValue": "search" + } + }, + { + "key": "gen_ai.tool.call.arguments", + "value": { + "stringValue": "{\"query\": \"internal codename\"}" + } + }, + { + "key": "gen_ai.tool.call.result", + "value": { + "stringValue": "{\"ok\": true, \"error\": null, \"n_candidates\": 5, \"candidates\": [\"doc-0000\", \"doc-0001\", \"doc-0002\", \"doc-0003\", \"doc-0004\"]}" + } + } + ], + "status": { + "code": 1 + } + }, + { + "traceId": "3a9e507a0f6dd628395d642ce5db5a3e", + "spanId": "c31264cf384dd5e7", + "parentSpanId": "2426c8a9be08f906", + "name": "execute_tool search", + "kind": 1, + "startTimeUnixNano": "1758000007000000000", + "endTimeUnixNano": "1758000007800000000", + "attributes": [ + { + "key": "gen_ai.operation.name", + "value": { + "stringValue": "execute_tool" + } + }, + { + "key": "gen_ai.tool.name", + "value": { + "stringValue": "search" + } + }, + { + "key": "gen_ai.tool.call.arguments", + "value": { + "stringValue": "{\"query\": \"Systems internal codename\"}" + } + }, + { + "key": "gen_ai.tool.call.result", + "value": { + "stringValue": "{\"ok\": true, \"error\": null, \"n_candidates\": 0, \"candidates\": []}" + } + } + ], + "status": { + "code": 1 + } + }, + { + "traceId": "3a9e507a0f6dd628395d642ce5db5a3e", + "spanId": "74e570ea9ac041f5", + "parentSpanId": "2426c8a9be08f906", + "name": "execute_tool search", + "kind": 1, + "startTimeUnixNano": "1758000008000000000", + "endTimeUnixNano": "1758000008800000000", + "attributes": [ + { + "key": "gen_ai.operation.name", + "value": { + "stringValue": "execute_tool" + } + }, + { + "key": "gen_ai.tool.name", + "value": { + "stringValue": "search" + } + }, + { + "key": "gen_ai.tool.call.arguments", + "value": { + "stringValue": "{\"query\": \"Aster\"}" + } + }, + { + "key": "gen_ai.tool.call.result", + "value": { + "stringValue": "{\"ok\": true, \"error\": null, \"n_candidates\": 5, \"candidates\": [\"doc-0001\", \"doc-0076\", \"doc-0078\", \"doc-0110\", \"doc-0145\"]}" + } + } + ], + "status": { + "code": 1 + } + }, + { + "traceId": "2ab3b6c1e8913b1951ea564e20404848", + "spanId": "beb931defc6b84cd", + "name": "invoke_agent", + "kind": 1, + "startTimeUnixNano": "1758000000000000000", + "endTimeUnixNano": "1758000006100000000", + "attributes": [ + { + "key": "gen_ai.operation.name", + "value": { + "stringValue": "invoke_agent" + } + }, + { + "key": "gen_ai.system", + "value": { + "stringValue": "openrouter" + } + }, + { + "key": "gen_ai.agent.name", + "value": { + "stringValue": "faultline-agent" + } + }, + { + "key": "gen_ai.request.model", + "value": { + "stringValue": "openai/gpt-5.6-luna" + } + }, + { + "key": "gen_ai.input.messages", + "value": { + "stringValue": "[{\"role\": \"user\", \"parts\": [{\"type\": \"text\", \"content\": \"In which district is Quill Systems headquartered?\"}]}]" + } + }, + { + "key": "gen_ai.output.messages", + "value": { + "stringValue": "[{\"role\": \"assistant\", \"parts\": [{\"type\": \"text\", \"content\": \"Quill Systems is headquartered in the Onyx district.\"}]}]" + } + }, + { + "key": "gen_ai.usage.input_tokens", + "value": { + "intValue": "3412" + } + }, + { + "key": "gen_ai.usage.output_tokens", + "value": { + "intValue": "853" + } + }, + { + "key": "faultgate.termination", + "value": { + "stringValue": "answered" + } + } + ], + "status": { + "code": 1 + } + }, + { + "traceId": "2ab3b6c1e8913b1951ea564e20404848", + "spanId": "e661907a6fd6aae7", + "parentSpanId": "beb931defc6b84cd", + "name": "execute_tool search", + "kind": 1, + "startTimeUnixNano": "1758000001000000000", + "endTimeUnixNano": "1758000001800000000", + "attributes": [ + { + "key": "gen_ai.operation.name", + "value": { + "stringValue": "execute_tool" + } + }, + { + "key": "gen_ai.tool.name", + "value": { + "stringValue": "search" + } + }, + { + "key": "gen_ai.tool.call.arguments", + "value": { + "stringValue": "{\"query\": \"Quill Systems headquartered district\"}" + } + }, + { + "key": "gen_ai.tool.call.result", + "value": { + "stringValue": "{\"ok\": true, \"error\": null, \"n_candidates\": 0, \"candidates\": []}" + } + } + ], + "status": { + "code": 1 + } + }, + { + "traceId": "2ab3b6c1e8913b1951ea564e20404848", + "spanId": "d08041f6ae1e478f", + "parentSpanId": "beb931defc6b84cd", + "name": "execute_tool search", + "kind": 1, + "startTimeUnixNano": "1758000002000000000", + "endTimeUnixNano": "1758000002800000000", + "attributes": [ + { + "key": "gen_ai.operation.name", + "value": { + "stringValue": "execute_tool" + } + }, + { + "key": "gen_ai.tool.name", + "value": { + "stringValue": "search" + } + }, + { + "key": "gen_ai.tool.call.arguments", + "value": { + "stringValue": "{\"query\": \"\\\"Quill Systems\\\" headquarters\"}" + } + }, + { + "key": "gen_ai.tool.call.result", + "value": { + "stringValue": "{\"ok\": true, \"error\": null, \"n_candidates\": 0, \"candidates\": []}" + } + } + ], + "status": { + "code": 1 + } + }, + { + "traceId": "2ab3b6c1e8913b1951ea564e20404848", + "spanId": "4de94d3bdbdcbad4", + "parentSpanId": "beb931defc6b84cd", + "name": "execute_tool search", + "kind": 1, + "startTimeUnixNano": "1758000003000000000", + "endTimeUnixNano": "1758000003800000000", + "attributes": [ + { + "key": "gen_ai.operation.name", + "value": { + "stringValue": "execute_tool" + } + }, + { + "key": "gen_ai.tool.name", + "value": { + "stringValue": "search" + } + }, + { + "key": "gen_ai.tool.call.arguments", + "value": { + "stringValue": "{\"query\": \"Quill Systems company location\"}" + } + }, + { + "key": "gen_ai.tool.call.result", + "value": { + "stringValue": "{\"ok\": true, \"error\": null, \"n_candidates\": 0, \"candidates\": []}" + } + } + ], + "status": { + "code": 1 + } + }, + { + "traceId": "2ab3b6c1e8913b1951ea564e20404848", + "spanId": "473ee5ec70f4f8fb", + "parentSpanId": "beb931defc6b84cd", + "name": "execute_tool search", + "kind": 1, + "startTimeUnixNano": "1758000004000000000", + "endTimeUnixNano": "1758000004800000000", + "attributes": [ + { + "key": "gen_ai.operation.name", + "value": { + "stringValue": "execute_tool" + } + }, + { + "key": "gen_ai.tool.name", + "value": { + "stringValue": "search" + } + }, + { + "key": "gen_ai.tool.call.arguments", + "value": { + "stringValue": "{\"query\": \"Quill Systems district\"}" + } + }, + { + "key": "gen_ai.tool.call.result", + "value": { + "stringValue": "{\"ok\": true, \"error\": null, \"n_candidates\": 0, \"candidates\": []}" + } + } + ], + "status": { + "code": 1 + } + }, + { + "traceId": "2ab3b6c1e8913b1951ea564e20404848", + "spanId": "02ccefd6d40c1bee", + "parentSpanId": "beb931defc6b84cd", + "name": "execute_tool search", + "kind": 1, + "startTimeUnixNano": "1758000005000000000", + "endTimeUnixNano": "1758000005800000000", + "attributes": [ + { + "key": "gen_ai.operation.name", + "value": { + "stringValue": "execute_tool" + } + }, + { + "key": "gen_ai.tool.name", + "value": { + "stringValue": "search" + } + }, + { + "key": "gen_ai.tool.call.arguments", + "value": { + "stringValue": "{\"query\": \"Quill\"}" + } + }, + { + "key": "gen_ai.tool.call.result", + "value": { + "stringValue": "{\"ok\": true, \"error\": null, \"n_candidates\": 5, \"candidates\": [\"doc-0072\", \"doc-0073\", \"doc-0082\", \"doc-0112\", \"doc-0116\"]}" + } + } + ], + "status": { + "code": 1 + } + } + ] + } + ] + } + ] +} \ No newline at end of file diff --git a/tests/test_check_plugins.py b/tests/test_check_plugins.py new file mode 100644 index 0000000..5dc68c9 --- /dev/null +++ b/tests/test_check_plugins.py @@ -0,0 +1,182 @@ +"""Check plugins: the ``faultgate_zero.checks`` entry point group (0.2).""" +import os +import warnings +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from faultgate_zero import checks +from faultgate_zero.checks import Options, Verdict, discover, run_plugin +from faultgate_zero.recorder import FaultgateZeroWarning +from faultgate_zero.trace import Trace + +TESTS = Path(__file__).parent + + +class EP: + """A stand-in for importlib.metadata.EntryPoint.""" + + def __init__(self, name, obj=None, error=None): + self.name, self.value, self._obj, self._error = name, f"fake:{name}", obj, error + + def load(self): + if self._error: + raise self._error + return self._obj + + +def plugin(name, default=True, run=None): + return SimpleNamespace(name=name, default=default, run=run or (lambda t, o: Verdict(name, False, "ok"))) + + +def test_a_plugin_is_discovered(): + found, errors = discover([EP("shape", plugin("shape"))]) + assert list(found) == ["shape"] and errors == [] + + +@pytest.mark.parametrize("name", ["policy", "search_loop", "abstention"]) +def test_a_name_clash_with_a_builtin_check_is_an_error(name): + found, errors = discover([EP("x", plugin(name))]) + assert found == {} and "clashes with a built-in check" in errors[0] and repr(name) in errors[0] + + +def test_two_plugins_with_one_name_and_malformed_plugins_are_errors(): + found, errors = discover([EP("a", plugin("dup")), EP("b", plugin("dup")), + EP("c", SimpleNamespace(name="nodefault", run=lambda t, o: None)), + EP("d", error=ImportError("no module"))]) + assert list(found) == ["dup"] + assert len(errors) == 3 + assert "used by two plugins" in errors[0] + assert "needs name (str), default (bool) and run(trace, options)" in errors[1] + assert "cannot load it: ImportError" in errors[2] + + +def _trace(): + return Trace("t1", "s", "agent", None, []) + + +def test_a_crashing_plugin_gives_a_verdict_that_did_not_fire_and_a_warning(): + def boom(t, o): + raise KeyError("secret@example.com") + + with pytest.warns(FaultgateZeroWarning, match="check 'shape' crashed"): + v = run_plugin(plugin("shape", run=boom), _trace(), Options()) + assert v == Verdict("shape", False, "check crashed: KeyError") + assert checks.crashed(v) + + +def test_a_crashing_plugin_under_warnings_as_errors_still_returns_a_verdict(): + def boom(t, o): + raise RuntimeError("boom") + + with warnings.catch_warnings(): + warnings.simplefilter("error") + v = run_plugin(plugin("shape", run=boom), _trace(), Options()) + assert v == Verdict("shape", False, "check crashed: RuntimeError") + + +def test_a_plugin_returning_the_wrong_type_counts_as_a_crash(): + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + v = run_plugin(plugin("shape", run=lambda t, o: "fired"), _trace(), Options()) + assert v.reason == "check crashed: TypeError" and not v.fired + + +def test_the_verdict_carries_the_plugin_name(): + v = run_plugin(plugin("shape", run=lambda t, o: Verdict("other", True, "why", 2)), _trace(), Options()) + assert v == Verdict("shape", True, "why", 2) + + +def test_builtin_defaults_come_first_and_are_unchanged(): + assert checks.BUILTIN_DEFAULTS == ("search_loop", "repeated_call", "unrecovered_tool_error", + "unrecovered_model_error", "context_overflow", "policy") + assert checks.DEFAULT_CHECKS[:len(checks.BUILTIN_DEFAULTS)] == checks.BUILTIN_DEFAULTS + + +# ---- end to end: a real entry point in a dist-info folder, in a pytest subprocess + +AGENTS = ''' +from fakes.agents import openai_agent +from fakes.openai_fakes import Script, client, completion + +def run_clean(): + s = Script(completion(tool_calls=[("c1", "search", {"q": "a"})]), completion("found a")) + return openai_agent(client(s), "find a", {"search": lambda q: [q]}) +''' + +PLUGIN = ''' +from faultgate_zero.checks import Verdict + +class Shape: + name = "shape" + default = True + def run(self, trace, options): + bad = [s for s in trace.steps if s.tool == "search"] + if bad: + return Verdict("shape", True, f"step {bad[0].index} search: expected number, got string; " + f"step {bad[0].index} search: missing required field", bad[0].index) + return Verdict("shape", False, "ok") + +class Crash: + name = "crash" + default = True + def run(self, trace, options): + raise RuntimeError("boom") + +class Clash: + name = "policy" + default = True + def run(self, trace, options): + return Verdict("policy", False, "") + +SHAPE, CRASH, CLASH = Shape(), Crash(), Clash() +''' + + +def _install(pytester, monkeypatch, entries): + pytester.makepyfile(agents=AGENTS, fakeplug=PLUGIN) + info = pytester.path / "fakeplug-0.1.dist-info" + info.mkdir() + (info / "METADATA").write_text("Metadata-Version: 2.1\nName: fakeplug\nVersion: 0.1\n", encoding="utf-8") + lines = "\n".join(f"{n} = fakeplug:{obj}" for n, obj in entries) + (info / "entry_points.txt").write_text(f"[faultgate_zero.checks]\n{lines}\n", encoding="utf-8") + monkeypatch.setenv("PYTHONPATH", os.pathsep.join([str(pytester.path), str(TESTS)])) + + +def test_an_installed_plugin_runs_and_can_be_skipped(pytester, monkeypatch): + _install(pytester, monkeypatch, [("shape", "SHAPE")]) + pytester.makepyfile(test_a=''' +import pytest +from agents import run_clean +def test_fires(): + assert run_clean() == "found a" +@pytest.mark.faultgate(skip=["shape"]) +def test_skipped(): + assert run_clean() == "found a" +''') + r = pytester.runpytest_subprocess() + r.assert_outcomes(passed=1, failed=1) + r.stdout.fnmatch_lines(["*shape step 1 search: expected number, got string", + "* step 1 search: missing required field", " shape: 1"]) + + +def test_a_crashing_plugin_never_fails_the_users_test(pytester, monkeypatch): + _install(pytester, monkeypatch, [("crash", "CRASH")]) + pytester.makepyfile(test_a=''' +from agents import run_clean +def test_ok(): + assert run_clean() == "found a" +''') + for flag in ("default", "error"): + r = pytester.runpytest_subprocess("-W", flag) + r.assert_outcomes(passed=1) + r.stdout.fnmatch_lines(["faultgate-zero: 1 check crashed and did not fire*"]) + + +def test_a_plugin_named_like_a_builtin_check_stops_the_session(pytester, monkeypatch): + _install(pytester, monkeypatch, [("clash", "CLASH")]) + pytester.makepyfile(test_a="def test_ok():\n assert True\n") + r = pytester.runpytest_subprocess() + assert r.ret == pytest.ExitCode.USAGE_ERROR + r.stderr.fnmatch_lines(["*check plugin 'clash' (fakeplug:CLASH): name 'policy' clashes with a built-in check*"]) diff --git a/tests/test_formats.py b/tests/test_formats.py new file mode 100644 index 0000000..7b7d413 --- /dev/null +++ b/tests/test_formats.py @@ -0,0 +1,106 @@ +"""formats.load: every trace format as faultgate-zero traces (0.2).""" +import builtins +import json +from pathlib import Path + +import pytest + +from faultgate_zero import formats, otlp + +DATA = Path(__file__).parent / "data" / "formats" + + +def test_otlp_loads_without_faultgate(monkeypatch): + real_import = builtins.__import__ + + def no_faultgate(name, *a, **k): + if name == "faultgate" or name.startswith("faultgate."): + raise ImportError(name) + return real_import(name, *a, **k) + + monkeypatch.setattr(builtins, "__import__", no_faultgate) + traces = formats.load(DATA / "otlp.json") + assert traces + assert [t.steps for t in traces] == [t.steps for t in otlp.load_otlp(DATA / "otlp.json")] + with pytest.raises(formats.FormatError, match=r"pip install 'faultgate-zero\[formats\]'"): + formats.load(DATA / "langfuse.json") + + +@pytest.mark.parametrize("fmt", ["openinference", "langfuse", "langsmith", "braintrust", "otlp"]) +def test_every_faultgate_example_format_loads(fmt): + pytest.importorskip("faultgate") + path = DATA / f"{fmt}.json" + auto = formats.load(path) + assert auto, fmt + assert [t.trace_id for t in formats.load(path, fmt)] == [t.trace_id for t in auto] + assert any(t.steps for t in auto), f"{fmt}: no tool steps read" + + +def test_converted_traces_keep_steps_and_results(): + fg = pytest.importorskip("faultgate.formats") + path = DATA / "langsmith.json" + zero = formats.load(path) + assert len(zero) == len(fg.load(path)) + steps = [s for t in zero for s in t.steps] + assert steps and all(s.tool for s in steps) + assert any(s.result_seen for s in steps) + + +def test_bad_input_is_a_format_error(tmp_path): + assert issubclass(formats.FormatError, ValueError) + with pytest.raises(formats.FormatError, match="unknown format 'csv'"): + formats.load(DATA / "otlp.json", "csv") + bad = tmp_path / "bad.json" + bad.write_text("{not json", encoding="utf-8") + with pytest.raises(formats.FormatError, match="not valid JSON"): + formats.load(bad) + with pytest.raises(formats.FormatError, match="cannot read it"): + formats.load(tmp_path / "missing.json") + pytest.importorskip("faultgate") + other = tmp_path / "other.json" + other.write_text(json.dumps({"hello": 1}), encoding="utf-8") + with pytest.raises(formats.FormatError, match="unrecognised trace format"): + formats.load(other) + + +@pytest.mark.parametrize("fmt", ["openinference", "langfuse", "langsmith", "braintrust"]) +def test_converted_traces_keep_prompt_and_answer(fmt): + pytest.importorskip("faultgate") + t = formats.load(DATA / f"{fmt}.json")[0] + assert t.prompt == "Refund order o-7 for customer C-123" + assert t.answer == "Refunded $120.00 to C-123." + + +def test_converted_answers_reach_the_checks(tmp_path): + pytest.importorskip("faultgate") + from faultgate_zero.checks import Options, abstention + + text = (DATA / "langsmith.json").read_text(encoding="utf-8") + assert "Refunded $120.00 to C-123." in text + f = tmp_path / "abstains.json" + f.write_text(text.replace("Refunded $120.00 to C-123.", "I could not find that information."), encoding="utf-8") + assert abstention(formats.load(f)[0], Options()).fired + + +def test_chat_message_text_becomes_parts(): + msgs = formats.normalize_messages(json.dumps([ + {"role": "user", "content": "hi"}, + {"role": "assistant", "content": None, "tool_calls": [{"id": "c1", "function": {"name": "lookup"}}]}, + {"role": "assistant", "content": [{"type": "text", "text": "done"}, {"type": "image_url"}]}, + {"role": "assistant", "parts": [{"type": "text", "content": "kept"}]}, + ])) + assert msgs[0] == {"role": "user", "parts": [{"type": "text", "content": "hi"}]} + assert msgs[1]["parts"] == [] # tool calls are read from parts and tool spans, as faultgate does + assert msgs[2]["parts"] == [{"type": "text", "content": "done"}] + assert msgs[3]["parts"] == [{"type": "text", "content": "kept"}] + + +@pytest.mark.parametrize("doc", [{"resourceSpans": [None]}, {"resourceSpans": [{"scopeSpans": [None]}]}, + {"resourceSpans": [{"scopeSpans": [{"spans": [None]}]}]}, + {"resourceSpans": [{"scopeSpans": [{"spans": [{"attributes": [None]}]}]}]}]) +def test_malformed_otlp_is_a_format_error_in_auto_mode(tmp_path, doc): + f = tmp_path / "bad.json" + f.write_text(json.dumps(doc), encoding="utf-8") + for fmt in ("auto", "otlp"): + with pytest.raises(formats.FormatError): + formats.load(f, fmt)