diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json index e261f52..39161fa 100644 --- a/.claude-plugin/plugin.json +++ b/.claude-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "ragops", - "version": "1.0.0", + "version": "2.0.0", "description": "Explainable regression tests and release gates for RAG systems and AI agents.", "author": { "name": "Thang Luu", diff --git a/.codex-plugin/plugin.json b/.codex-plugin/plugin.json index ed6de0a..a65ceb9 100644 --- a/.codex-plugin/plugin.json +++ b/.codex-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "ragops", - "version": "1.0.0", - "description": "Explainable regression tests and release gates for RAG systems and AI agents.", + "version": "2.0.0", + "description": "Offline, explainable release gates and evidence bundles for RAG systems and AI agents.", "author": { "name": "Thang Luu", "url": "https://github.com/thangldw" @@ -14,7 +14,7 @@ "interface": { "displayName": "RAGOps", "shortDescription": "Gate AI releases with evidence", - "longDescription": "Compare candidate AI behavior with an accepted baseline, explain failed gates, and produce reviewable release evidence without requiring a model provider.", + "longDescription": "Compare recorded candidate behavior with an accepted baseline, localize regressions by policy and case, and produce verifiable release evidence without requiring a model provider.", "developerName": "DUC THANG LUU", "category": "Productivity", "capabilities": ["Read"], diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..7585af2 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,42 @@ +name: CI + +on: + push: + branches: [main] + pull_request: + +permissions: + contents: read + +jobs: + test: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ["3.11", "3.12", "3.13"] + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: ${{ matrix.python-version }} + cache: pip + - name: Install test dependencies + run: python -m pip install --disable-pip-version-check -e '.[dev]' build + - name: Lint + run: ruff check . + - name: Test + env: + PYTEST_DISABLE_PLUGIN_AUTOLOAD: "1" + run: pytest -q + - name: Build and inspect distributions + run: | + python -m build + python -m zipfile --list dist/ragops-*.whl | grep 'ragops/web/index.html' + python -m zipfile --list dist/ragops-*.whl | grep 'ragops/api/main.py' + - name: Run credential-free release gate + run: | + ragops demo --profile engineer --output demo-output + ragops evidence verify --bundle demo-output/evidence diff --git a/.github/workflows/publish-pypi.yml b/.github/workflows/publish-pypi.yml index 4e20b3b..b1ed2c3 100644 --- a/.github/workflows/publish-pypi.yml +++ b/.github/workflows/publish-pypi.yml @@ -16,10 +16,10 @@ jobs: contents: read id-token: write steps: - - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: ref: ${{ inputs.tag }} - - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: "3.12" - name: Verify tag and package version @@ -53,6 +53,7 @@ jobs: --pattern 'ragops-*.whl' \ --pattern 'ragops-*.tar.gz' \ --pattern 'ragops-*.cdx.json' \ + --pattern 'ragops-chatgpt-plugin-*.zip' \ --pattern 'SHA256SUMS' \ --dir release-assets (cd release-assets && sha256sum --check SHA256SUMS) diff --git a/.kimi-plugin/plugin.json b/.kimi-plugin/plugin.json index 3c3108c..6430c6b 100644 --- a/.kimi-plugin/plugin.json +++ b/.kimi-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "ragops", - "version": "1.0.0", + "version": "2.0.0", "description": "Explainable regression tests and release gates for RAG systems and AI agents.", "author": { "name": "Thang Luu", diff --git a/CHANGELOG.md b/CHANGELOG.md index 640e9ec..3f77b62 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,28 @@ ### English -- Added an offline Phoenix recorded-score adapter example that preserves producer-owned score meaning, scale, and direction while enforcing complete case coverage. +- No unreleased changes. + +## [2.0.0] - 2026-08-23 + +### English + +- Added versioned contract migration, content-addressed evidence bundles, slice/distribution gates, evaluator calibration, trace-graph evaluation, dataset leakage controls, CI renderers, portable vendor adapters, and audited blind review. +- Packaged the optional FastAPI workbench with CSP-safe DOM rendering and added Python 3.11-3.13 CI. +- Upgraded the synthetic flagship demo to generate canonical JSON, Markdown, HTML, JUnit, SARIF, GitHub Summary, and verified evidence for executive, engineer, and auditor profiles. +- Added a deterministic ChatGPT skills-only plugin bundle and v2 release evidence. + +### Tiếng Việt + +- Bổ sung contract có version, evidence bundle content-addressed, slice/distribution gate, calibration, trace graph, chống dataset leakage, CI renderer, vendor adapter và blind review có audit. +- Đóng gói FastAPI workbench tùy chọn với CSP/DOM an toàn và CI Python 3.11-3.13. +- Nâng demo synthetic để xuất cùng một quyết định qua JSON, Markdown, HTML, JUnit, SARIF, GitHub Summary và evidence đã xác minh. + +### 日本語 + +- versioned contract、content-addressed evidence、slice/distribution gate、calibration、trace graph、dataset leakage check、CI renderer、vendor adapter、blind review audit を追加しました。 +- CSP-safe な任意 FastAPI workbench と Python 3.11-3.13 CI を同梱しました。 +- synthetic flagship demo は同一判定を JSON、Markdown、HTML、JUnit、SARIF、GitHub Summary、検証済み evidence に出力します。 ## [1.2.0] - 2026-08-02 @@ -46,5 +67,6 @@ - リポジトリ所有の GitHub Actions をすべて削除し、リリースゲートはローカル CLI と API で引き続き利用できます。 - 公開タグとパッケージ版を `v1.0.0` に統一しました。 +[2.0.0]: https://github.com/thangldw/ragops/releases/tag/v2.0.0 [1.2.0]: https://github.com/thangldw/ragops/releases/tag/v1.2.0 [1.0.0]: https://github.com/thangldw/ragops/releases/tag/v1.0.0 diff --git a/README.md b/README.md index b31560a..c98733f 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ [English](#english) · [Tiếng Việt](#tiếng-việt) · [日本語](#日本語) -Regression tests and explainable release gates for RAG systems and AI agents. +Offline, reproducible release gates for RAG systems and AI agents. ```mermaid %%{init: {"theme":"base","themeVariables":{"background":"#FFFFFF","fontFamily":"Arial, sans-serif","lineColor":"#667085","primaryTextColor":"#172B4D"}}}%% @@ -11,7 +11,7 @@ flowchart LR B["Accepted baseline
Mốc chuẩn / 基準"]:::blue C["Candidate traces
Ứng viên / 候補"]:::purple E["Evaluators & policy
Đánh giá / 評価"]:::green - G{"PASS / WARN / FAIL"}:::pink + G{"PASS / WARN / BLOCK"}:::pink S --> E B --> E C --> E --> G @@ -24,31 +24,34 @@ flowchart LR ## English -RAGOps compares recorded candidate behavior with an accepted baseline, applies versioned release policy and returns explainable `PASS`, `WARN` or `FAIL` evidence. The dependency-free core works offline; optional adapters expose API and UI surfaces. Runtime integrations use the local CLI or API; the repository-owned GitHub Actions workflow is limited to PyPI publishing. +RAGOps compares recorded candidate behavior with an accepted baseline, applies a versioned release policy, and emits one canonical `PASS`, `WARN`, or `BLOCK` decision across JSON, Markdown, HTML, JUnit, SARIF, and GitHub Summary. The dependency-free core performs no network calls. Optional adapters import vendor exports; the API/workbench ships in the wheel through the `api` extra. Requirements: Python 3.11+. -Current technical references: [architecture](docs/ARCHITECTURE.md), [operations](docs/OPERATIONS.md), [security](SECURITY.md) and [v1.2.0 release notes](docs/releases/v1.2.0.md). +Current technical references: [architecture](docs/ARCHITECTURE.md), [contracts](docs/CONTRACTS.md), [governance](docs/governance.md), [operations](docs/OPERATIONS.md), [security](SECURITY.md), and [v2.0.0 release notes](docs/releases/v2.0.0.md). The repository is also a skills-only plugin for ChatGPT, Codex, Claude Code and Cowork. It does not include a hosted MCP connector. See the [directory submission package](docs/submission/DIRECTORY_SUBMISSION.md), [privacy policy](PRIVACY.md), [terms](TERMS.md), and [support guidance](SUPPORT.md). ```bash -python -m venv .venv -.venv/bin/pip install -e '.[dev]' -.venv/bin/ragops inspect --scenario scenarios/japanese_troubleshooting/benchmark-v0.2.json -.venv/bin/ruff check . -PYTEST_DISABLE_PLUGIN_AUTOLOAD=1 .venv/bin/pytest -q +pip install ragops +ragops demo --profile engineer --output ./demo-output +ragops evidence verify --bundle ./demo-output/evidence +open ./demo-output/release-report.html ``` +The credential-free demo intentionally returns `BLOCK`: citation coverage `1.0 -> 0.5`, citation precision `1.0 -> 0.5`, and lexical groundedness `1.0 -> 0.6`. It is synthetic benchmark evidence, not production adoption evidence. + +Install `ragops[api]` and run `ragops serve` for the local authenticated API/workbench. Use `ragops adapter list` for Phoenix, Ragas, DeepEval, LangSmith, MLflow, Promptfoo, custom JSON, and installed entry-point adapters. + ## Tiếng Việt -RAGOps so sánh hành vi ứng viên đã ghi lại với baseline được chấp nhận, áp dụng policy phát hành có version và trả về bằng chứng giải thích được ở trạng thái `PASS`, `WARN` hoặc `FAIL`. Core không có dependency và chạy offline; adapter tùy chọn cung cấp API/UI. Tích hợp runtime dùng CLI hoặc API cục bộ; GitHub Actions thuộc repo chỉ dùng để phát hành PyPI. +RAGOps so sánh trace ứng viên với baseline được chấp nhận, áp dụng release policy có version và xuất cùng một quyết định `PASS`, `WARN` hoặc `BLOCK` sang JSON/Markdown/HTML/JUnit/SARIF. Core không dependency, chạy offline; API/workbench là extra tùy chọn. Demo synthetic cố ý tạo `BLOCK`, không phải bằng chứng production adoption. Yêu cầu: Python 3.11+. Dùng các lệnh ở phần English để cài đặt, kiểm tra scenario, lint và test. ## 日本語 -RAGOps は、記録済みの候補動作を承認済みベースラインと比較し、バージョン管理されたリリースポリシーを適用して、説明可能な `PASS`・`WARN`・`FAIL` の根拠を返します。依存関係のないコアはオフラインで動作し、任意のアダプターが API と UI を提供します。runtime integration はローカル CLI/API を使い、repository-owned GitHub Actions は PyPI 公開のみに使用します。 +RAGOps は記録済み候補を承認済みベースラインと比較し、バージョン管理された方針から一つの `PASS`・`WARN`・`BLOCK` 判定を JSON、Markdown、HTML、JUnit、SARIF に出力します。依存関係のない core はオフラインで動作し、API/workbench は任意の extra です。デモは synthetic evidence であり、本番導入実績ではありません。 必要環境は Python 3.11 以上です。セットアップ、lint、テストには English セクションのコマンドを使用してください。 diff --git a/apps/api/main.py b/apps/api/main.py index 514be41..b6103fc 100644 --- a/apps/api/main.py +++ b/apps/api/main.py @@ -1,378 +1,13 @@ -from __future__ import annotations - -import hmac -import os -from pathlib import Path - -from fastapi import Depends, FastAPI, Header, HTTPException, Request -from fastapi.responses import FileResponse, JSONResponse -from pydantic import BaseModel, ConfigDict - -from ragops import __version__ -from ragops.control_plane import ControlPlane -from ragops.drift import detect_evaluator_drift -from ragops.engine import compare, evaluate -from ragops.loader import ContractError, responses_from_data, scenario_from_dict -from ragops.models import ( - EvaluatorDriftMetricGate, - EvaluatorDriftPolicy, - SequentialPolicy, - StatisticalMetricGate, - StatisticalPolicy, +# ruff: noqa: F401,F403 +"""Source-tree compatibility shim for the packaged API.""" + +from ragops.api.main import * # noqa: F403 +from ragops.api.main import ( + _positive_env_int, + _validate_collection_limits, + _validate_replay_limits, + evaluator_drift_endpoint, + require_api_key, + sequential_compare_endpoint, + statistical_compare_endpoint, ) -from ragops.sequential import compare_replay_bundles_sequentially -from ragops.statistical import compare_replay_bundles, replay_bundle_from_dict -from ragops.store import ExperimentStore - -app = FastAPI(title="RAGOps API", version=__version__) - -DEFAULT_MAX_REQUEST_BYTES = 2 * 1024 * 1024 -DEFAULT_MAX_CASES = 1000 -DEFAULT_MAX_REPLAY_OBSERVATIONS = 100_000 - - -class EvaluateRequest(BaseModel): - scenario: dict - responses: list[dict] - - -class SavedEvaluateRequest(EvaluateRequest): - label: str = "" - - -class CompareRequest(BaseModel): - scenario: dict - baseline: list[dict] - candidate: list[dict] - - -class StrictRequest(BaseModel): - model_config = ConfigDict(extra="forbid") - - -class StatisticalCompareRequest(StrictRequest): - baseline: dict - candidate: dict - policy: dict - - -class EvaluatorDriftRequest(StrictRequest): - reference: dict - current: dict - policy: dict - - -class ReviewRequest(BaseModel): - status: str - reviewer: str - note: str = "" - - -def configured_store( - x_workspace_id: str | None = Header(default=None), - x_workspace_key: str | None = Header(default=None), -) -> ExperimentStore: - control_root = os.getenv("RAGOPS_CONTROL_PLANE") - if control_root: - if not x_workspace_id or not x_workspace_key: - raise HTTPException(status_code=401, detail="Workspace credentials are required") - try: - return ControlPlane(control_root).workspace_store(x_workspace_id, x_workspace_key) - except PermissionError as exc: - raise HTTPException(status_code=401, detail=str(exc)) from exc - path = os.getenv("RAGOPS_STORE") - if not path: - raise HTTPException(status_code=503, detail="RAGOPS_STORE is not configured") - return ExperimentStore(path) - - -def require_api_key(x_api_key: str | None = Header(default=None)) -> None: - expected = os.getenv("RAGOPS_API_KEY", "").strip() - insecure_dev_mode = os.getenv("RAGOPS_INSECURE_DEV_MODE", "").casefold() == "true" - if expected and insecure_dev_mode: - raise HTTPException( - status_code=503, - detail="RAGOPS_API_KEY and RAGOPS_INSECURE_DEV_MODE cannot both be enabled", - ) - if insecure_dev_mode: - return - if not expected: - raise HTTPException( - status_code=503, - detail=( - "API authentication is not configured; set RAGOPS_API_KEY or explicitly " - "enable RAGOPS_INSECURE_DEV_MODE=true for local development" - ), - ) - if x_api_key is None or not hmac.compare_digest(x_api_key, expected): - raise HTTPException(status_code=401, detail="Invalid API key") - - -@app.middleware("http") -async def enforce_request_size(request: Request, call_next): - if request.method in {"POST", "PUT", "PATCH"}: - maximum = _positive_env_int("RAGOPS_MAX_REQUEST_BYTES", DEFAULT_MAX_REQUEST_BYTES) - content_length = request.headers.get("content-length") - if content_length: - try: - declared = int(content_length) - except ValueError: - return JSONResponse(status_code=400, content={"detail": "Invalid Content-Length"}) - if declared > maximum: - return JSONResponse(status_code=413, content={"detail": "Request body is too large"}) - body = await request.body() - if len(body) > maximum: - return JSONResponse(status_code=413, content={"detail": "Request body is too large"}) - return await call_next(request) - - -def _positive_env_int(name: str, default: int) -> int: - raw = os.getenv(name) - if raw is None: - return default - try: - value = int(raw) - except ValueError as exc: - raise HTTPException(status_code=503, detail=f"{name} must be a positive integer") from exc - if value <= 0: - raise HTTPException(status_code=503, detail=f"{name} must be a positive integer") - return value - - -def _validate_collection_limits(scenario: dict, *response_sets: list[dict]) -> None: - maximum = _positive_env_int("RAGOPS_MAX_CASES", DEFAULT_MAX_CASES) - cases = scenario.get("cases", []) - if not isinstance(cases, list): - return - if len(cases) > maximum or any(len(responses) > maximum for responses in response_sets): - raise HTTPException( - status_code=413, - detail=f"Scenario and response collections are limited to {maximum} cases", - ) - - -def _validate_replay_limits(*bundles: dict) -> None: - maximum_cases = _positive_env_int("RAGOPS_MAX_CASES", DEFAULT_MAX_CASES) - maximum_observations = _positive_env_int( - "RAGOPS_MAX_REPLAY_OBSERVATIONS", DEFAULT_MAX_REPLAY_OBSERVATIONS - ) - for bundle in bundles: - records = bundle.get("records", []) - if not isinstance(records, list): - continue - case_ids = { - record.get("case_id") for record in records if isinstance(record, dict) - } - if len(records) > maximum_observations or len(case_ids) > maximum_cases: - raise HTTPException( - status_code=413, - detail=( - f"Replay bundles are limited to {maximum_cases} cases and " - f"{maximum_observations} observations" - ), - ) - - -@app.get("/health") -def health() -> dict[str, str]: - return {"status": "ok", "version": __version__} - - -@app.get("/", include_in_schema=False) -def dashboard() -> FileResponse: - return FileResponse(Path(__file__).parents[1] / "web" / "index.html") - - -@app.post("/v1/evaluate", dependencies=[Depends(require_api_key)]) -def evaluate_endpoint(payload: EvaluateRequest) -> dict: - try: - _validate_collection_limits(payload.scenario, payload.responses) - scenario = scenario_from_dict(payload.scenario) - responses = responses_from_data(payload.responses) - return evaluate(scenario, responses).to_dict() - except (ContractError, KeyError, TypeError, ValueError) as exc: - raise HTTPException(status_code=422, detail=str(exc)) from exc - - -@app.post("/v1/compare", dependencies=[Depends(require_api_key)]) -def compare_endpoint(payload: CompareRequest) -> dict: - try: - _validate_collection_limits(payload.scenario, payload.baseline, payload.candidate) - scenario = scenario_from_dict(payload.scenario) - baseline = responses_from_data(payload.baseline) - candidate = responses_from_data(payload.candidate) - return compare(scenario, baseline, candidate).to_dict() - except (ContractError, KeyError, TypeError, ValueError) as exc: - raise HTTPException(status_code=422, detail=str(exc)) from exc - - -@app.post("/v1/statistical/compare", dependencies=[Depends(require_api_key)]) -def statistical_compare_endpoint(payload: StatisticalCompareRequest) -> dict: - try: - _validate_replay_limits(payload.baseline, payload.candidate) - return compare_replay_bundles( - replay_bundle_from_dict(payload.baseline), - replay_bundle_from_dict(payload.candidate), - _statistical_policy(payload.policy), - ).to_dict() - except (ContractError, KeyError, TypeError, ValueError) as exc: - raise HTTPException(status_code=422, detail=str(exc)) from exc - - -@app.post("/v1/statistical/sequential", dependencies=[Depends(require_api_key)]) -def sequential_compare_endpoint(payload: StatisticalCompareRequest) -> dict: - try: - _validate_replay_limits(payload.baseline, payload.candidate) - return compare_replay_bundles_sequentially( - replay_bundle_from_dict(payload.baseline), - replay_bundle_from_dict(payload.candidate), - _sequential_policy(payload.policy), - ).to_dict() - except (ContractError, KeyError, TypeError, ValueError) as exc: - raise HTTPException(status_code=422, detail=str(exc)) from exc - - -@app.post("/v1/evaluator-drift", dependencies=[Depends(require_api_key)]) -def evaluator_drift_endpoint(payload: EvaluatorDriftRequest) -> dict: - try: - _validate_replay_limits(payload.reference, payload.current) - return detect_evaluator_drift( - replay_bundle_from_dict(payload.reference), - replay_bundle_from_dict(payload.current), - _drift_policy(payload.policy), - ).to_dict() - except (ContractError, KeyError, TypeError, ValueError) as exc: - raise HTTPException(status_code=422, detail=str(exc)) from exc - - -@app.get("/v1/runs", dependencies=[Depends(require_api_key)]) -def list_runs(limit: int = 20, store: ExperimentStore = Depends(configured_store)) -> list[dict]: - try: - return store.list_runs(limit=limit) - except ValueError as exc: - raise HTTPException(status_code=422, detail=str(exc)) from exc - - -@app.get("/v1/trends/{scenario_id}/{metric}", dependencies=[Depends(require_api_key)]) -def metric_trend( - scenario_id: str, - metric: str, - limit: int = 50, - store: ExperimentStore = Depends(configured_store), -) -> list[dict]: - try: - return store.metric_trend(scenario_id, metric, limit=limit) - except ValueError as exc: - raise HTTPException(status_code=422, detail=str(exc)) from exc - - -@app.post("/v1/runs/{run_id}/review", dependencies=[Depends(require_api_key)]) -def review_run( - run_id: str, - payload: ReviewRequest, - store: ExperimentStore = Depends(configured_store), -) -> dict[str, str]: - try: - store.review( - run_id, - status=payload.status, - reviewer=payload.reviewer, - note=payload.note, - ) - except KeyError as exc: - raise HTTPException(status_code=404, detail=str(exc)) from exc - except ValueError as exc: - raise HTTPException(status_code=422, detail=str(exc)) from exc - return {"run_id": run_id, "review_status": payload.status} - - -@app.post("/v1/runs/evaluate", dependencies=[Depends(require_api_key)]) -def evaluate_and_save( - payload: SavedEvaluateRequest, - store: ExperimentStore = Depends(configured_store), -) -> dict: - try: - _validate_collection_limits(payload.scenario, payload.responses) - scenario = scenario_from_dict(payload.scenario) - responses = responses_from_data(payload.responses) - report = evaluate(scenario, responses) - except (ContractError, KeyError, TypeError, ValueError) as exc: - raise HTTPException(status_code=422, detail=str(exc)) from exc - run_id = store.save(report, label=payload.label) - return {"run_id": run_id, "report": report.to_dict()} - - -def _statistical_metric_gates(raw: object) -> dict[str, StatisticalMetricGate]: - if not isinstance(raw, dict) or not raw: - raise ContractError("Statistical API policy metrics must be a non-empty object") - gates = {} - for name, value in raw.items(): - if not isinstance(name, str) or not name or not isinstance(value, dict): - raise ContractError("Statistical API metric gates must be named objects") - if set(value) - {"direction", "minimum", "maximum", "max_regression"}: - raise ContractError(f"Statistical API metric {name!r} has unknown fields") - gates[name] = StatisticalMetricGate( - direction=value["direction"], - minimum=value.get("minimum"), - maximum=value.get("maximum"), - max_regression=value["max_regression"], - ) - return gates - - -def _statistical_policy(raw: dict) -> StatisticalPolicy: - expected = {"confidence", "minimum_cases", "resamples", "seed", "metrics"} - if set(raw) != expected: - raise ContractError("Statistical API policy fields do not match the fixed policy contract") - return StatisticalPolicy( - confidence=raw["confidence"], - minimum_cases=raw["minimum_cases"], - resamples=raw["resamples"], - seed=raw["seed"], - metric_gates=_statistical_metric_gates(raw["metrics"]), - ) - - -def _sequential_policy(raw: dict) -> SequentialPolicy: - expected = { - "confidence", - "minimum_cases", - "minimum_repeats", - "maximum_repeats", - "look_every", - "resamples", - "seed", - "metrics", - } - if set(raw) != expected: - raise ContractError("Statistical API policy fields do not match the sequential contract") - return SequentialPolicy( - confidence=raw["confidence"], - minimum_cases=raw["minimum_cases"], - minimum_repeats=raw["minimum_repeats"], - maximum_repeats=raw["maximum_repeats"], - look_every=raw["look_every"], - resamples=raw["resamples"], - seed=raw["seed"], - metric_gates=_statistical_metric_gates(raw["metrics"]), - ) - - -def _drift_policy(raw: dict) -> EvaluatorDriftPolicy: - expected = {"confidence", "minimum_cases", "resamples", "seed", "metrics"} - if set(raw) != expected or not isinstance(raw["metrics"], dict): - raise ContractError("Evaluator drift API policy fields do not match the contract") - gates = {} - for name, value in raw["metrics"].items(): - if not isinstance(value, dict) or set(value) != {"max_absolute_change"}: - raise ContractError( - f"Evaluator drift API metric {name!r} must define max_absolute_change" - ) - gates[name] = EvaluatorDriftMetricGate(value["max_absolute_change"]) - return EvaluatorDriftPolicy( - confidence=raw["confidence"], - minimum_cases=raw["minimum_cases"], - resamples=raw["resamples"], - seed=raw["seed"], - metric_gates=gates, - ) diff --git a/apps/web/index.html b/apps/web/index.html index 0ef2ec8..8675a0a 100644 --- a/apps/web/index.html +++ b/apps/web/index.html @@ -3,10 +3,10 @@ + RAGOps Workbench - + +
Release confidence for AI systems

RAGOps

Evaluate a candidate locally, then inspect and review saved team runs from the same evidence contract.

@@ -15,13 +15,5 @@

Release decision

Waiting for a run

Failed gates

Team workflow

Experiment history

Set RAGOPS_STORE on the API to enable shared local history and review metadata.

-
- - + + diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 7ce9761..206c459 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -7,7 +7,7 @@ flowchart LR S["Scenario & policy
Kịch bản / 方針"]:::blue C["Dependency-free core
Core offline"]:::purple E["Evidence bundle
Bằng chứng / 証拠"]:::pink - G{"PASS / WARN / FAIL"}:::green + G{"PASS / WARN / BLOCK"}:::green T --> C S --> C --> E --> G classDef yellow fill:#FFF4A3,stroke:#C9A227,stroke-width:2px,color:#172B4D @@ -19,12 +19,12 @@ flowchart LR ## English -`src/ragops/` owns portable evaluation semantics, deterministic comparison and statistical gates. `scenarios/` and `schemas/` are versioned contracts. `apps/` contains optional API and GitHub adapters. Reusable workflows are read-only callers of the same core and publish bounded evidence artifacts. +`src/ragops/` owns portable contracts, deterministic evaluators, slice/distribution analysis, policy decisions, calibration, evidence bundles, adapters, and SQLite governance. `src/ragops/api/` and `src/ragops/web/` are optional packaged adapters; importing the core never imports FastAPI, Uvicorn, or vendor SDKs. `apps/api/main.py` is a source compatibility shim. CLI/API/UI and CI renderers consume the same canonical decision evidence. ## Tiếng Việt -`src/ragops/` quản lý evaluation portable, so sánh xác định và statistical gate. `scenarios/` cùng `schemas/` là contract có version. `apps/` chứa adapter API/GitHub tùy chọn. Reusable workflow chỉ đọc, gọi cùng core và xuất evidence artifact có giới hạn. +`src/ragops/` quản lý contract portable, evaluator xác định, policy, calibration, evidence bundle, adapter và SQLite governance. API/UI chỉ là adapter đóng gói tùy chọn; import core không import FastAPI/Uvicorn/vendor SDK. CLI/API/UI/CI dùng cùng canonical decision. ## 日本語 -`src/ragops/` がポータブル評価、決定的比較、統計ゲートを担当します。`scenarios/` と `schemas/` はバージョン管理された契約です。`apps/` は任意の API・GitHub アダプターです。再利用可能 workflow は同じコアを読み取り専用で呼び出し、上限付き証拠を出力します。 +`src/ragops/` は portable contract、決定的 evaluator、policy、calibration、evidence bundle、adapter、SQLite governance を担当します。API/UI は任意の packaged adapter で、core import は FastAPI、Uvicorn、vendor SDK を読み込みません。CLI/API/UI/CI は同じ canonical decision を使用します。 diff --git a/docs/CONTRACTS.md b/docs/CONTRACTS.md new file mode 100644 index 0000000..ba7dd24 --- /dev/null +++ b/docs/CONTRACTS.md @@ -0,0 +1,14 @@ +# Versioned contracts + +RAGOps keeps contract validation and migration in the dependency-free core. + +| Contract | Versions | Compatibility rule | +| --- | --- | --- | +| Scenario | 0.1, 0.2 | Explicit `contract migrate`; inputs are never overwritten | +| Trace | 0.3, 0.4 | Preserve unknown producer attributes during migration | +| Trace graph | 1.0 | Validate graph references and reject cycles | +| Dataset manifest | 1.0 | Provenance, review state, split membership, leakage checks | +| Evidence bundle | 1.0 | Canonical manifest, SHA-256 content addresses, no undeclared files | +| Release policy/decision | 2.0 / 1.0 | Stable gate IDs and canonical `PASS`, `WARN`, `BLOCK` | + +Use `ragops contract validate`, `ragops contract diff`, and `ragops contract migrate`. Schema files are packaged under `ragops/schemas` and available offline. diff --git a/docs/OPERATIONS.md b/docs/OPERATIONS.md index f6e3256..754b008 100644 --- a/docs/OPERATIONS.md +++ b/docs/OPERATIONS.md @@ -2,12 +2,12 @@ ## English -Install with `pip install -e '.[dev]'`, run `ruff check .`, then `PYTEST_DISABLE_PLUGIN_AUTOLOAD=1 pytest -q`. A release candidate must also exercise representative `inspect`, `evaluate`, `compare` and statistical commands. Build and validate from a clean commit, create the full semantic-version tag, promote the exact artifacts to a GitHub Release, then dispatch the release-only PyPI Trusted Publishing workflow. +Install with `pip install -e '.[dev]'`, run `ruff check .`, then `PYTEST_DISABLE_PLUGIN_AUTOLOAD=1 pytest -q`. Verify `ragops demo --profile engineer`, `ragops evidence verify`, the packaged API/UI, every schema instance, and a clean `--no-deps` wheel import. Build wheel/sdist/SBOM/checksums from a clean commit, promote those exact artifacts to a GitHub Release, then dispatch the tag-bound PyPI Trusted Publishing workflow. GitHub publication, PyPI publication, ChatGPT directory submission, and ChatGPT approval are separate states and must be reported separately. ## Tiếng Việt -Cài bằng `pip install -e '.[dev]'`, chạy `ruff check .`, sau đó `PYTEST_DISABLE_PLUGIN_AUTOLOAD=1 pytest -q`. Release candidate phải chạy thêm các lệnh `inspect`, `evaluate`, `compare` và statistical tiêu biểu. Build và kiểm tra từ commit sạch, tạo tag semantic version đầy đủ, đưa đúng artifact lên GitHub Release rồi chạy workflow Trusted Publishing dành riêng cho PyPI. +Cài bằng `pip install -e '.[dev]'`, chạy lint/test, demo/evidence verify, API/UI, schema và clean-wheel import. Build wheel/sdist/SBOM/checksum từ commit sạch, phát hành đúng artifact lên GitHub rồi mới chạy PyPI Trusted Publishing theo tag. Phải phân biệt GitHub release, PyPI release, ChatGPT submission và ChatGPT approval. ## 日本語 -`pip install -e '.[dev]'` で導入し、`ruff check .`、`PYTEST_DISABLE_PLUGIN_AUTOLOAD=1 pytest -q` を実行します。リリース候補では代表的な `inspect`、`evaluate`、`compare`、統計コマンドも確認します。クリーンな commit で build と検証を行い、完全な semantic-version tag と同一 artifact の GitHub Release を作成して、PyPI 専用 Trusted Publishing workflow を実行します。 +`pip install -e '.[dev]'` 後に lint/test、demo/evidence verify、API/UI、schema、clean-wheel import を確認します。clean commit から wheel/sdist/SBOM/checksum を作成し、同一 artifact を GitHub Release に昇格してから tag-bound PyPI workflow を実行します。GitHub、PyPI、ChatGPT submission、ChatGPT approval は別状態です。 diff --git a/docs/adapters.md b/docs/adapters.md new file mode 100644 index 0000000..d956382 --- /dev/null +++ b/docs/adapters.md @@ -0,0 +1,16 @@ +# Portable adapters + +RAGOps converts evaluator exports into a dependency-free envelope before evaluation. Built-in profiles support Phoenix, Ragas, DeepEval, LangSmith, MLflow, Promptfoo, and custom JSON; no vendor SDK is imported by the core. + +```bash +ragops adapter list +ragops adapter convert \ + --profile mlflow \ + --input mlflow-evaluation.json \ + --case-id case-001 \ + --output ragops-metrics.json +``` + +The output records the producer, adapter version, unmapped fields, and missing-case warnings. Unknown case IDs, duplicate metrics, and non-finite scores fail closed. + +Third-party packages can register an adapter through the `ragops.adapters` entry-point group. The entry point must expose an object with `name`, `version`, and `convert(payload, context)` members and return `ragops.adapter_sdk.AdapterOutput`. diff --git a/docs/examples/github-actions-ragops.yml b/docs/examples/github-actions-ragops.yml new file mode 100644 index 0000000..6720845 --- /dev/null +++ b/docs/examples/github-actions-ragops.yml @@ -0,0 +1,25 @@ +name: RAGOps + +on: + pull_request: + +permissions: + contents: read + +jobs: + release-gate: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.12" + - run: python -m pip install --disable-pip-version-check "ragops==2.0.0" + - name: Evaluate release candidate + run: ragops demo --profile engineer --output ragops-output + - name: Verify evidence bundle + run: ragops evidence verify --bundle ragops-output/evidence + - name: Write bounded summary + run: cat ragops-output/github-summary.md >> "$GITHUB_STEP_SUMMARY" diff --git a/docs/examples/github-pr-comment.yml b/docs/examples/github-pr-comment.yml index b024fe6..6b64a5f 100644 --- a/docs/examples/github-pr-comment.yml +++ b/docs/examples/github-pr-comment.yml @@ -17,7 +17,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout reviewed RAGOps publisher code - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: repository: thangldw/ragops ref: 15675a5b6abae546a936d43e9ac22bfa94dfa392 # v1.0.0 publisher diff --git a/docs/examples/gitlab-ci-ragops.yml b/docs/examples/gitlab-ci-ragops.yml index d2b6429..c31f73d 100644 --- a/docs/examples/gitlab-ci-ragops.yml +++ b/docs/examples/gitlab-ci-ragops.yml @@ -2,7 +2,7 @@ ragops_release_gate: stage: test image: python:3.12-slim variables: - RAGOPS_VERSION: "1.2.0" + RAGOPS_VERSION: "2.0.0" RAGOPS_SCENARIO: "scenarios/japanese_troubleshooting/benchmark-v0.2.json" RAGOPS_BASELINE: "scenarios/japanese_troubleshooting/benchmark-baseline.json" RAGOPS_CANDIDATE: "scenarios/japanese_troubleshooting/benchmark-regressed.json" @@ -39,6 +39,15 @@ ragops_release_gate: echo '```' } >ragops-report.md fi + decision=PASS + if [ "$gate_exit" -ne 0 ]; then decision=BLOCK; fi + ragops evidence create \ + --bundle ragops-evidence \ + --decision "$decision" \ + --artifact report=ragops-report.md \ + --artifact command_log=ragops-command.log \ + --limitation "Consumer-provided recorded benchmark; production representativeness is not inferred" + ragops evidence verify --bundle ragops-evidence cat ragops-report.md exit "$gate_exit" artifacts: @@ -47,5 +56,6 @@ ragops_release_gate: paths: - ragops-report.md - ragops-command.log + - ragops-evidence/ rules: - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' diff --git a/docs/governance.md b/docs/governance.md new file mode 100644 index 0000000..b533d96 --- /dev/null +++ b/docs/governance.md @@ -0,0 +1,16 @@ +# Governance and review + +`ragops governance` stores artifact digests, lifecycle state, append-only reviews, waivers, and audit events in local SQLite. The valid lifecycle is `draft -> reviewed -> accepted -> superseded`; acceptance requires an approval review. + +Reviewer-specific queues hide candidate/provider metadata until that reviewer votes. This reduces anchoring bias but is not a substitute for access control: protect the database with normal filesystem controls. Verify the referenced evidence bundle before accepting an artifact. + +```bash +ragops governance register --store .ragops/governance.db \ + --artifact-id release-2026-08-23 --kind release-decision \ + --digest "$SHA256" --metadata public.json --blinded-metadata candidate.json --actor ci +ragops governance transition --store .ragops/governance.db \ + --artifact-id release-2026-08-23 --state reviewed --actor owner +ragops governance queue --store .ragops/governance.db --reviewer alice +ragops governance review --store .ragops/governance.db \ + --artifact-id release-2026-08-23 --reviewer alice --verdict approve +``` diff --git a/docs/releases/v2.0.0.md b/docs/releases/v2.0.0.md new file mode 100644 index 0000000..8ea3d45 --- /dev/null +++ b/docs/releases/v2.0.0.md @@ -0,0 +1,15 @@ +# RAGOps v2.0.0 + +RAGOps 2.0 strengthens the offline release contract without changing the existing 1.x readers or CLI paths. It adds verifiable evidence bundles, slice/distribution policy gates, calibration, trace graphs, dataset leakage controls, CI-native outputs, portable vendor import profiles, blind review governance, and a packaged optional API/workbench. + +The credential-free flagship demo intentionally blocks a synthetic regression: citation coverage `1.0 -> 0.5`, citation precision `1.0 -> 0.5`, and lexical groundedness `1.0 -> 0.6`. This is reproducible synthetic evidence, not production adoption evidence. + +Release artifacts include wheel, sdist, CycloneDX SBOM, SHA-256 manifest, local validation evidence, and a deterministic ChatGPT skills-only plugin ZIP. Publication to GitHub, PyPI, and the ChatGPT directory are independently verified states. + +## Compatibility + +- Python 3.11 or newer. +- Dependency-free core remains offline. +- Existing scenario 0.1/0.2 and trace 0.3 readers remain supported. +- FastAPI/Uvicorn remain optional through `ragops[api]`. +- No hosted MCP connector is included. diff --git a/docs/submission/DIRECTORY_SUBMISSION.md b/docs/submission/DIRECTORY_SUBMISSION.md index 270be98..680a431 100644 --- a/docs/submission/DIRECTORY_SUBMISSION.md +++ b/docs/submission/DIRECTORY_SUBMISSION.md @@ -1,10 +1,12 @@ # RAGOps directory submission -Prepared: 2026-08-02 +Prepared: 2026-08-23 Published on the OpenAI Plugin Directory: https://chatgpt.com/plugins/plugins_6a6ed9e25c60819194c48f4233ae507e -Published on PyPI: https://pypi.org/project/ragops/1.2.0/ +PyPI release target: https://pypi.org/project/ragops/2.0.0/ + +Submission artifact: `dist/ragops-chatgpt-plugin-2.0.0.zip` ## Submission choice @@ -17,7 +19,7 @@ Published on PyPI: https://pypi.org/project/ragops/1.2.0/ - Name: RAGOps - Category: Developer Tools / Productivity - Short description: Gate RAG and agent releases with evidence. -- Long description: RAGOps compares recorded candidate behavior with an accepted baseline, applies versioned evaluation and regression policies, and returns explainable PASS, WARN, or BLOCK evidence. Use it to inspect benchmark coverage, catch citation or retrieval regressions, and create review-ready release reports without requiring an AI provider. +- Long description: RAGOps compares recorded candidate behavior with an accepted baseline, applies versioned policy, localizes regressions by case and policy path, and returns verifiable PASS, WARN, or BLOCK evidence. Use it to catch citation, groundedness, retrieval, trajectory, or evaluator regressions without requiring an AI provider. - Developer: DUC THANG LUU - Website: https://github.com/thangldw/ragops - Support: https://github.com/thangldw/ragops/blob/main/SUPPORT.md @@ -59,14 +61,15 @@ Published on PyPI: https://pypi.org/project/ragops/1.2.0/ No account or credential is required. Use Python 3.11+ and run the bundled wrapper at `skills/evaluate-ai-release/scripts/run_ragops.py`. The OpenAI ZIP stores review fixtures under `skills/evaluate-ai-release/references/fixtures/japanese_troubleshooting`; the GitHub plugin uses the canonical copies under `scenarios/japanese_troubleshooting`. A gate block may return exit code 2; this is an expected evaluated outcome. -## Initial release notes +## v2.0 update notes -Initial public plugin submission. RAGOps packages deterministic RAG and AI-agent evaluation workflows, reproducible CLI execution, release-safety guardrails, and review-ready listing metadata. This version does not include a hosted MCP connector or interactive UI. +RAGOps 2.0 adds content-addressed evidence bundles, slice-aware policy, calibration, trace graphs, dataset leakage checks, CI-native output, portable import profiles, and audited blind review. The plugin remains skills-only and does not include a hosted MCP connector. The optional local API/workbench is distributed through the Python package, not as a remote ChatGPT connector. ## Final portal checks - Verify the publisher identity as Thang Luu or the intended business name. - Upload the repository/plugin package without `.venv`, caches, generated customer data, or credentials. +- Verify the plugin ZIP checksum from `dist/SHA256SUMS` and run its vendored wrapper before upload. - Confirm all public legal and support links resolve from the default branch. - Select only countries where English-language support can be provided. - For OpenAI, include five positive and three negative tests exactly as above. diff --git a/docs/superpowers/plans/2026-08-23-ragops-2-implementation.md b/docs/superpowers/plans/2026-08-23-ragops-2-implementation.md new file mode 100644 index 0000000..eb403fa --- /dev/null +++ b/docs/superpowers/plans/2026-08-23-ragops-2-implementation.md @@ -0,0 +1,470 @@ +# RAGOps 2.0 Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Ship RAGOps 2.0 as a backward-compatible, verifiable offline release-governance toolkit, then publish the verified artifacts to GitHub, PyPI, and the ChatGPT plugin workflow. + +**Architecture:** Extend the dependency-free core with versioned contracts, evidence, policy, calibration, datasets, trace graphs, explanations, adapters, and governance. Package FastAPI/workbench as an optional application surface, keep source compatibility shims, and make every reporter consume the same canonical decision evidence. + +**Tech Stack:** Python 3.11+ standard library core, optional FastAPI/Uvicorn, SQLite, JSON Schema 2020-12 artifacts, Pytest, Ruff, Hatchling, GitHub Actions, OIDC Trusted Publishing. + +**Spec:** `docs/superpowers/specs/2026-08-23-ragops-production-flagship-adoption-design.md` + +## Global Constraints + +- Python remains `>=3.11`; `ragops` retains zero required runtime dependencies. +- Existing CLI commands, flags, exit codes, and 1.x artifact readers remain supported. +- Core performs no implicit network calls and imports no optional packages. +- Canonical decisions are `PASS`, `WARN`, and `BLOCK`; legacy `passed` remains derived. +- Inputs and accepted evidence are immutable; migration always writes a separate output. +- Synthetic evidence is labeled synthetic and is never described as production adoption. +- Every production behavior follows witnessed RED, minimal GREEN, then regression verification. + +--- + +### Task 1: Contract registry, validation, migration, and packaged schemas + +**Files:** Create `src/ragops/contracts.py`, `tests/test_contracts_v2.py`; modify `src/ragops/cli.py`, `pyproject.toml`, `tests/test_schema_files.py`. + +**Interfaces:** Produces `ContractDescriptor`, `ValidationResult`, `registered_contracts()`, `validate_contract(data, kind="auto")`, `diff_contract(kind, from_version, to_version)`, `migrate_contract(data, kind, from_version, to_version, question="")`, and `ragops contract validate|diff|migrate`. + +- [ ] **Step 1: Write failing migration and fail-closed tests** + +```python +def test_migrate_scenario_01_to_02_is_explicit_and_valid() -> None: + source = scenario_01_fixture() + migrated = migrate_contract(source, "scenario", "0.1", "0.2") + assert source["schema_version"] == "0.1" + assert migrated["thresholds"]["citation_precision"] == 0.0 + assert migrated["cases"][0]["severity"] == "medium" + assert validate_contract(migrated, "scenario").version == "0.2" + +def test_unknown_contract_version_fails_closed() -> None: + with pytest.raises(ContractError, match="Unsupported scenario schema"): + validate_contract({**scenario_01_fixture(), "schema_version": "9.9"}, "scenario") +``` + +- [ ] **Step 2: Run RED** + +Run: `PYTEST_DISABLE_PLUGIN_AUTOLOAD=1 .venv/bin/pytest tests/test_contracts_v2.py -q` + +Expected: import failure because `ragops.contracts` does not exist. + +- [ ] **Step 3: Implement frozen descriptors, literal registry, pure migrations, and nested CLI commands** + +Scenario migration adds the conservative reader defaults. Trace migration adds required 0.4 fields and records `metadata.migrated_from="0.3"`. CLI rejects resolved input/output equality. + +- [ ] **Step 4: Package root schemas inside the wheel and verify contents** + +Add Hatch `force-include` from `schemas/` to `ragops/schemas/`; test for `ragops/schemas/scenario-0.2.schema.json` in the wheel. + +- [ ] **Step 5: Verify and commit** + +```bash +.venv/bin/ruff check src/ragops/contracts.py src/ragops/cli.py tests/test_contracts_v2.py +PYTEST_DISABLE_PLUGIN_AUTOLOAD=1 .venv/bin/pytest tests/test_contracts_v2.py tests/test_schema_files.py -q +git add src/ragops/contracts.py src/ragops/cli.py tests/test_contracts_v2.py tests/test_schema_files.py pyproject.toml +git commit -m "feat: add versioned contract tooling" +``` + +### Task 2: Verifiable evidence bundles + +**Files:** Create `src/ragops/evidence.py`, `schemas/evidence-bundle-1.0.schema.json`, `tests/test_evidence_bundle.py`; modify `src/ragops/cli.py`, `src/ragops/__init__.py`. + +**Interfaces:** Produces `ArtifactRecord`, `EvidenceManifest`, `create_evidence_bundle(...)`, `load_evidence_manifest(path)`, `verify_evidence_bundle(path)`, and `ragops evidence create|verify`. + +- [ ] **Step 1: Write failing reproducibility, tamper, traversal, symlink, missing-file, and decision tests** + +```python +def test_evidence_bundle_is_reproducible_and_detects_tampering(tmp_path: Path) -> None: + source = tmp_path / "report.json" + source.write_text('{"passed":false}\n') + first = create_evidence_bundle(tmp_path / "one", {"report": source}, "BLOCK", + ("synthetic benchmark",), "2026-08-23T00:00:00Z", {}) + second = create_evidence_bundle(tmp_path / "two", {"report": source}, "BLOCK", + ("synthetic benchmark",), "2026-08-23T00:00:00Z", {}) + assert first.canonical_bytes() == second.canonical_bytes() + (tmp_path / "one/artifacts/report.json").write_text("tampered") + with pytest.raises(ContractError, match="digest mismatch"): + verify_evidence_bundle(tmp_path / "one") +``` + +- [ ] **Step 2: Run RED, implement bounded canonical bundles, then run GREEN** + +Copy regular files under `artifacts/`, canonicalize JSON with sorted keys, and verify path, size, digest, schema, decision, and undeclared regular files. + +Run: `.venv/bin/ruff check src/ragops/evidence.py tests/test_evidence_bundle.py && PYTEST_DISABLE_PLUGIN_AUTOLOAD=1 .venv/bin/pytest tests/test_evidence_bundle.py -q` + +- [ ] **Step 3: Add CLI and commit** + +```bash +git add src/ragops/evidence.py src/ragops/cli.py src/ragops/__init__.py schemas/evidence-bundle-1.0.schema.json tests/test_evidence_bundle.py +git commit -m "feat: add verifiable evidence bundles" +``` + +### Task 3: Slice and distribution evidence + +**Files:** Create `src/ragops/slices.py`, `src/ragops/distribution.py`, `tests/test_slice_distribution.py`; modify `src/ragops/models.py`. + +**Interfaces:** Produces `SliceSelector`, `SliceSummary`, `summarize_slices(...)`, `quantile(...)`, `paired_effect(...)`, `holm_adjust(...)`, and `minimum_paired_sample(...)`. + +- [ ] **Step 1: Write failing selector and hand-derived math tests** + +```python +def test_quantile_and_holm_are_hand_checkable() -> None: + assert quantile((0.0, 10.0, 20.0, 30.0), 0.5) == 15.0 + assert holm_adjust({"a": 0.01, "b": 0.04, "c": 0.03}) == { + "a": 0.03, "b": 0.06, "c": 0.06, + } +``` + +Add exact case-ID/count/metric assertions for language, severity, category, and tag selectors; reject non-finite inputs and invalid probabilities. + +- [ ] **Step 2: Run RED, implement deterministic primitives, then run GREEN** + +Run: `.venv/bin/ruff check src/ragops/slices.py src/ragops/distribution.py tests/test_slice_distribution.py && PYTEST_DISABLE_PLUGIN_AUTOLOAD=1 .venv/bin/pytest tests/test_slice_distribution.py -q` + +- [ ] **Step 3: Commit** + +```bash +git add src/ragops/slices.py src/ragops/distribution.py src/ragops/models.py tests/test_slice_distribution.py +git commit -m "feat: add slice and distribution evidence" +``` + +### Task 4: Policy 2.0, waivers, and canonical decisions + +**Files:** Create `src/ragops/policy_v2.py`, `schemas/release-policy-2.0.schema.json`, `schemas/release-decision-1.0.schema.json`, `tests/test_policy_v2.py`; modify `src/ragops/models.py`, `src/ragops/cli.py`. + +**Interfaces:** Produces `ReleasePolicyV2`, `PolicyGate`, `PolicyWaiver`, `GateEvidence`, `ReleaseDecision`, `load_release_policy_v2(path)`, `apply_release_policy(...)`, and `ragops gate-v2`. + +- [ ] **Step 1: Write failing worst-slice, insufficient-evidence, and waiver tests** + +```python +def test_active_waiver_can_only_change_block_to_warn() -> None: + decision = apply_release_policy(policy_with_waiver("2030-01-01T00:00:00Z"), baseline(), + candidate(), scenario(), now="2026-08-23T00:00:00Z") + assert decision.decision == "WARN" + assert decision.gates[0].waived is True +``` + +Add a separate expired-waiver test expecting `BLOCK`, plus malformed timestamp, duplicate gate ID, unknown section, and empty selector tests. + +- [ ] **Step 2: Run RED, implement strict TOML policy and gate evaluation, then run GREEN** + +Run: `.venv/bin/ruff check src/ragops/policy_v2.py tests/test_policy_v2.py && PYTEST_DISABLE_PLUGIN_AUTOLOAD=1 .venv/bin/pytest tests/test_policy_v2.py -q` + +- [ ] **Step 3: Add schemas/CLI and commit** + +```bash +git add src/ragops/policy_v2.py src/ragops/models.py src/ragops/cli.py schemas/release-policy-2.0.schema.json schemas/release-decision-1.0.schema.json tests/test_policy_v2.py +git commit -m "feat: add slice-aware release policy v2" +``` + +### Task 5: Evaluator calibration + +**Files:** Create `src/ragops/calibration.py`, `schemas/calibration-set-1.0.schema.json`, `schemas/calibration-report-1.0.schema.json`, `tests/test_calibration.py`; modify `src/ragops/cli.py`. + +**Interfaces:** Produces `CalibrationRecord`, `CalibrationPolicy`, `CalibrationReport`, `load_calibration_set(path)`, `calibrate_evaluator(...)`, and `ragops calibrate`. + +- [ ] **Step 1: Write failing coverage, MAE, threshold accuracy, false-pass, false-block, kappa, and insufficient-overlap tests** + +```python +def test_calibration_reports_false_passes_and_reviewer_agreement() -> None: + report = calibrate_evaluator(calibration_fixture(), calibration_policy()) + assert report.coverage == 1.0 + assert report.false_pass_rate == 0.25 + assert report.reviewer_agreement is not None +``` + +- [ ] **Step 2: Run RED, implement calibration math/gates, then run GREEN** + +Run: `.venv/bin/ruff check src/ragops/calibration.py tests/test_calibration.py && PYTEST_DISABLE_PLUGIN_AUTOLOAD=1 .venv/bin/pytest tests/test_calibration.py -q` + +- [ ] **Step 3: Add CLI/schemas and commit** + +```bash +git add src/ragops/calibration.py src/ragops/cli.py schemas/calibration-set-1.0.schema.json schemas/calibration-report-1.0.schema.json tests/test_calibration.py +git commit -m "feat: add evaluator calibration gates" +``` + +### Task 6: Portable agent trace graph and OpenTelemetry adapter + +**Files:** Create `src/ragops/trace_graph.py`, `src/ragops/adapters/otel_genai.py`, `schemas/trace-graph-1.0.schema.json`, `tests/test_trace_graph.py`; modify `src/ragops/cli.py`. + +**Interfaces:** Produces `TraceSpan`, `TraceGraph`, `TraceExpectation`, `TraceGraphReport`, `trace_graph_from_dict(...)`, `evaluate_trace_graph(...)`, `otel_spans_to_trace_graph(...)`, and `ragops trace evaluate|convert-otel`. + +- [ ] **Step 1: Write failing graph-integrity and trajectory tests** + +```python +def test_trace_graph_localizes_wrong_tool_arguments() -> None: + report = evaluate_trace_graph(tool_trace_fixture(), trace_expectation_fixture()) + assert report.metrics["tool_name_precision"] == 1.0 + assert report.metrics["tool_argument_exactness"] == 0.0 + assert report.failed_span_ids == ("tool-1",) +``` + +Add cycles, missing parents, duplicate IDs, negative duration, tool order, recovery, excess-step, retrieval recall, and raw unknown `gen_ai.*` preservation tests. + +- [ ] **Step 2: Run RED, implement graph/adapter, then run GREEN** + +Run: `.venv/bin/ruff check src/ragops/trace_graph.py src/ragops/adapters/otel_genai.py tests/test_trace_graph.py && PYTEST_DISABLE_PLUGIN_AUTOLOAD=1 .venv/bin/pytest tests/test_trace_graph.py -q` + +- [ ] **Step 3: Add CLI/schema and commit** + +```bash +git add src/ragops/trace_graph.py src/ragops/adapters/otel_genai.py src/ragops/cli.py schemas/trace-graph-1.0.schema.json tests/test_trace_graph.py +git commit -m "feat: evaluate portable agent trace graphs" +``` + +### Task 7: Benchmark manifests and leakage checks + +**Files:** Create `src/ragops/datasets.py`, `schemas/dataset-manifest-1.0.schema.json`, `tests/test_datasets.py`; modify `src/ragops/cli.py`. + +**Interfaces:** Produces `DatasetManifest`, `DatasetIssue`, `DatasetDiff`, `create_dataset_manifest(...)`, `validate_dataset_manifest(...)`, `diff_dataset_manifests(...)`, and `ragops dataset create|validate|diff`. + +- [ ] **Step 1: Write failing duplicate, leakage, provenance, coverage, and deterministic-diff tests** + +```python +def test_dataset_validation_quarantines_cross_split_leakage() -> None: + issues = validate_dataset_manifest(manifest_with_cross_split_duplicate()) + assert {item.code for item in issues} >= {"normalized_content_leakage", "review_required"} +``` + +- [ ] **Step 2: Run RED, implement normalized hashing/manifests/diff, then run GREEN** + +Run: `.venv/bin/ruff check src/ragops/datasets.py tests/test_datasets.py && PYTEST_DISABLE_PLUGIN_AUTOLOAD=1 .venv/bin/pytest tests/test_datasets.py -q` + +- [ ] **Step 3: Add CLI/schema and commit** + +```bash +git add src/ragops/datasets.py src/ragops/cli.py schemas/dataset-manifest-1.0.schema.json tests/test_datasets.py +git commit -m "feat: govern benchmark dataset lifecycle" +``` + +### Task 8: Regression explanation and CI renderers + +**Files:** Create `src/ragops/explain.py`, `src/ragops/ci.py`, `tests/test_explain_and_ci.py`; modify `src/ragops/reporters.py`, `src/ragops/cli.py`. + +**Interfaces:** Produces `explain_decision(...)`, `remediation_for_gate(...)`, `junit_report(...)`, `github_summary(...)`, `sarif_report(...)`, `ragops explain`, and CI output formats. + +- [ ] **Step 1: Write failing causal-path, counterfactual, JUnit, summary, SARIF, and escaping tests** + +```python +def test_explanation_maps_gate_to_policy_case_and_remediation() -> None: + item = explain_decision(blocked_decision_fixture()).items[0] + assert item.policy_path == "metrics.citation_coverage.max_regression" + assert item.case_ids == ("q2",) + assert item.required_change == 0.5 +``` + +- [ ] **Step 2: Run RED, implement bounded renderers, then run GREEN** + +Use ElementTree for XML and dictionaries for SARIF. Strip control characters and bound Markdown fields. + +Run: `.venv/bin/ruff check src/ragops/explain.py src/ragops/ci.py tests/test_explain_and_ci.py && PYTEST_DISABLE_PLUGIN_AUTOLOAD=1 .venv/bin/pytest tests/test_explain_and_ci.py tests/test_store_and_html.py -q` + +- [ ] **Step 3: Add CLI and commit** + +```bash +git add src/ragops/explain.py src/ragops/ci.py src/ragops/reporters.py src/ragops/cli.py tests/test_explain_and_ci.py +git commit -m "feat: explain regressions and emit CI evidence" +``` + +### Task 9: Adapter SDK and offline import profiles + +**Files:** Create `src/ragops/adapter_sdk.py`, `src/ragops/adapters/importers.py`, `tests/test_adapter_sdk.py`, `docs/ADAPTERS.md`; modify `src/ragops/cli.py`. + +**Interfaces:** Produces `Adapter`, `AdapterContext`, `AdapterOutput`, `discover_adapters()`, `convert_export(...)`, entry-point group `ragops.adapters`, and `ragops adapter list|convert`. Built-ins are `phoenix`, `ragas`, `deepeval`, `langsmith`, `mlflow`, `promptfoo`, and `custom`. + +- [ ] **Step 1: Write failing producer, complete-coverage, provenance, unmapped-field, duplicate, non-finite, and safe-discovery tests** + +```python +@pytest.mark.parametrize("producer", ["phoenix", "ragas", "deepeval", "langsmith", "mlflow", "promptfoo"]) +def test_builtin_imports_preserve_provenance(producer: str) -> None: + output = convert_export(producer, producer_fixture(producer), AdapterContext(("q1",))) + assert output.provenance["producer"] == producer + assert output.records[0]["case_id"] == "q1" +``` + +- [ ] **Step 2: Run RED, implement strict converters/discovery, then run GREEN** + +Run: `.venv/bin/ruff check src/ragops/adapter_sdk.py src/ragops/adapters/importers.py tests/test_adapter_sdk.py && PYTEST_DISABLE_PLUGIN_AUTOLOAD=1 .venv/bin/pytest tests/test_adapter_sdk.py tests/test_phoenix_recorded_scores.py tests/test_provider_metric_adapters.py -q` + +- [ ] **Step 3: Document and commit** + +```bash +git add src/ragops/adapter_sdk.py src/ragops/adapters/importers.py src/ragops/cli.py tests/test_adapter_sdk.py docs/ADAPTERS.md +git commit -m "feat: add vendor-neutral adapter SDK" +``` + +### Task 10: Local governance and blind review queue + +**Files:** Create `src/ragops/governance.py`, `tests/test_governance.py`; modify `src/ragops/cli.py`. + +**Interfaces:** Produces `GovernanceStore.register_artifact|submit_review|record_review|promote|supersede|create_waiver|list_queue|audit_events` and `ragops governance` subcommands. + +- [ ] **Step 1: Write failing immutability, blind-review, quorum, transition, expiry, and append-only audit tests** + +```python +def test_invalid_promotion_transition_fails_closed(tmp_path: Path) -> None: + store = GovernanceStore(tmp_path / "governance.db") + with pytest.raises(ValueError, match="draft -> accepted"): + store.promote("digest", actor="alice", target="accepted", reason="skip review") +``` + +- [ ] **Step 2: Run RED, implement transactional SQLite governance, then run GREEN** + +Use foreign keys, `BEGIN IMMEDIATE`, immutable artifact rows, ISO-8601 timestamps, and deterministic JSON. + +Run: `.venv/bin/ruff check src/ragops/governance.py tests/test_governance.py && PYTEST_DISABLE_PLUGIN_AUTOLOAD=1 .venv/bin/pytest tests/test_governance.py tests/test_control_plane.py tests/test_team_workflow.py -q` + +- [ ] **Step 3: Add CLI and commit** + +```bash +git add src/ragops/governance.py src/ragops/cli.py tests/test_governance.py +git commit -m "feat: add local release governance" +``` + +### Task 11: Packaged API and CSP-safe workbench + +**Files:** Create `src/ragops/api/__init__.py`, `src/ragops/api/main.py`, `src/ragops/web/__init__.py`, `src/ragops/web/index.html`, `tests/test_packaged_api.py`; modify `apps/api/main.py`, `apps/web/index.html`, `src/ragops/cli.py`, `pyproject.toml`, `tests/test_workbench.py`, `tests/test_api_contract.py`. + +**Interfaces:** Produces wheel-importable `ragops.api.main:app`, `ragops serve`, packaged web asset, and preserves `apps.api.main:app`. + +- [ ] **Step 1: Write failing wheel-content, API import, CSP, and DOM-safety tests** + +```python +def test_packaged_workbench_has_no_untrusted_inner_html() -> None: + html = files("ragops.web").joinpath("index.html").read_text() + assert ".innerHTML" not in html + assert "Content-Security-Policy" in html +``` + +Add a clean-wheel test for schema/API/web contents and a `--no-deps` test proving `import ragops` works without FastAPI. + +- [ ] **Step 2: Run RED, package API/assets, add compatibility shim, and replace unsafe DOM writes** + +Dynamic values use `textContent`/element creation. No event-handler attributes or remote assets are allowed. `serve` imports Uvicorn lazily. + +- [ ] **Step 3: Run API/workbench tests and commit** + +```bash +.venv/bin/ruff check src/ragops/api apps/api src/ragops/cli.py tests/test_packaged_api.py +PYTEST_DISABLE_PLUGIN_AUTOLOAD=1 .venv/bin/pytest tests/test_packaged_api.py tests/test_api_contract.py tests/test_workbench.py -q +git add src/ragops/api src/ragops/web apps/api/main.py apps/web/index.html src/ragops/cli.py pyproject.toml tests/test_packaged_api.py tests/test_workbench.py tests/test_api_contract.py +git commit -m "feat: package secure API workbench" +``` + +### Task 12: Flagship demo, docs, and consumer CI + +**Files:** Modify `src/ragops/demo.py`, `src/ragops/cli.py`, `src/ragops/reporters.py`, `tests/test_demo.py`, `README.md`, `docs/ARCHITECTURE.md`, `docs/OPERATIONS.md`, `docs/examples/github-pr-comment.yml`, `docs/examples/gitlab-ci-ragops.yml`; create `tests/test_flagship_demo.py`, `docs/CONTRACTS.md`, `docs/GOVERNANCE.md`, `.github/workflows/ci.yml`. + +**Interfaces:** Produces `ragops demo --profile executive|engineer|auditor` with evidence bundle, JSON, Markdown, safe HTML, JUnit, SARIF, and GitHub summary. + +- [ ] **Step 1: Write failing cross-artifact/profile consistency tests** + +```python +@pytest.mark.parametrize("profile", ["executive", "engineer", "auditor"]) +def test_profiles_share_one_block_decision(tmp_path: Path, profile: str) -> None: + summary = write_demo(tmp_path / profile, scenario_id="customer-support", profile=profile) + assert summary["candidate_decision"] == "BLOCK" + assert summary["evidence_classification"] == "synthetic" + assert verify_evidence_bundle(tmp_path / profile / "evidence").decision == "BLOCK" +``` + +- [ ] **Step 2: Run RED, implement all formats/profiles, then run GREEN** + +Verify stable failed-gate IDs across formats and absence of production-adoption claims. + +Run: `.venv/bin/ruff check src/ragops/demo.py src/ragops/reporters.py tests/test_flagship_demo.py && PYTEST_DISABLE_PLUGIN_AUTOLOAD=1 .venv/bin/pytest tests/test_demo.py tests/test_flagship_demo.py tests/test_adoption_recipes.py -q` + +- [ ] **Step 3: Add pinned Python 3.11/3.12/3.13 CI and adoption docs, then commit** + +```bash +git add src/ragops/demo.py src/ragops/cli.py src/ragops/reporters.py tests/test_demo.py tests/test_flagship_demo.py README.md docs .github/workflows/ci.yml +git commit -m "feat: deliver flagship release-gate workflow" +``` + +### Task 13: Version 2.0 release candidate and plugin package + +**Files:** Modify `pyproject.toml`, `src/ragops/__init__.py`, `CHANGELOG.md`, `scripts/local_release.py`, release/version/plugin tests, `.codex-plugin/plugin.json`, `.claude-plugin/plugin.json`, `.kimi-plugin/plugin.json`, `skills/evaluate-ai-release/`, `docs/submission/DIRECTORY_SUBMISSION.md`; create `docs/releases/v2.0.0.md`. + +**Interfaces:** Produces version-consistent 2.0.0 source, wheel, sdist, SBOM, checksums, local release evidence, and skills-only plugin submission bundle. + +- [ ] **Step 1: Write failing parity and clean-wheel tests** + +```python +def test_public_release_metadata_matches_200() -> None: + assert project_version() == runtime_version() == "2.0.0" + assert plugin_versions() == {"2.0.0"} + +def test_clean_wheel_contains_optional_surfaces(built_wheel: Path) -> None: + names = wheel_names(built_wheel) + assert "ragops/web/index.html" in names + assert "ragops/api/main.py" in names + assert "ragops/schemas/evidence-bundle-1.0.schema.json" in names +``` + +- [ ] **Step 2: Run RED, update release metadata/evidence, then run targeted GREEN** + +Run: `PYTEST_DISABLE_PLUGIN_AUTOLOAD=1 .venv/bin/pytest tests/test_release_version.py tests/test_local_release.py tests/test_plugin_release.py -q` + +- [ ] **Step 3: Run full verification and exact artifact build** + +```bash +.venv/bin/ruff check . +PYTEST_DISABLE_PLUGIN_AUTOLOAD=1 .venv/bin/pytest -q +.venv/bin/ragops demo --profile auditor --output /tmp/ragops-v2-demo +.venv/bin/ragops evidence verify --bundle /tmp/ragops-v2-demo/evidence +.venv/bin/pip install -q build cyclonedx-bom==7.3.0 +.venv/bin/python scripts/local_release.py verify --tag v2.0.0 +git diff --check +``` + +- [ ] **Step 4: Commit release candidate** + +```bash +git add pyproject.toml src/ragops/__init__.py CHANGELOG.md docs/releases/v2.0.0.md scripts/local_release.py tests .codex-plugin .claude-plugin .kimi-plugin skills docs/submission +git commit -m "release: prepare RagOps 2.0.0" +``` + +### Task 14: Review and external publication + +**Files:** Evidence only: `dist/*`, GitHub Release `v2.0.0`, PyPI `ragops`, and ChatGPT plugin submission state. + +**Interfaces:** Consumes exact clean-commit artifacts from Task 13; produces publication URLs and submission state. + +- [ ] **Step 1: Review `origin/main...HEAD` against every spec criterion and fix critical/important findings** + +Inspect path handling, HTML, SQLite, signatures, contract compatibility, and release scripts. Run the full suite after fixes. + +- [ ] **Step 2: Verify provenance immediately before release** + +```bash +git status --short +git rev-parse HEAD +.venv/bin/python scripts/local_release.py verify --tag v2.0.0 +sha256sum dist/ragops-2.0.0-py3-none-any.whl dist/ragops-2.0.0.tar.gz +``` + +- [ ] **Step 3: Push branch and land through repository policy without force operations** + +If branch protection requires review, create a PR against `main`, wait for required checks, and merge through GitHub. + +- [ ] **Step 4: Create GitHub Release from exact verified artifacts** + +Run: `.venv/bin/python scripts/local_release.py publish-github --tag v2.0.0 --yes` + +Verify one wheel, one sdist, SBOM, `SHA256SUMS`, and `LOCAL_RELEASE_EVIDENCE.json` on the release. + +- [ ] **Step 5: Publish through PyPI Trusted Publishing and verify public install** + +Dispatch the tag-bound workflow, wait for success, install `ragops==2.0.0` into a clean temporary venv, then run `ragops --version` and the credential-free demo. + +- [ ] **Step 6: Validate and submit the ChatGPT plugin update** + +Use official OpenAI guidance and authenticated browser state. Validate manifest, skills, policy URLs, icons, version, and directory test prompts. Record `submitted`, `in review`, or `approved` accurately. + +- [ ] **Step 7: Report final evidence** + +Report commit/tag, test count, Ruff, clean-wheel evidence, GitHub URL, PyPI URL/install result, ChatGPT submission state, and synthetic limitations. diff --git a/docs/superpowers/specs/2026-08-23-ragops-production-flagship-adoption-design.md b/docs/superpowers/specs/2026-08-23-ragops-production-flagship-adoption-design.md new file mode 100644 index 0000000..2990316 --- /dev/null +++ b/docs/superpowers/specs/2026-08-23-ragops-production-flagship-adoption-design.md @@ -0,0 +1,296 @@ +# RAGOps Production, Flagship, and Adoption Design + +**Status:** Approved for autonomous implementation + +**Date:** 2026-08-23 + +**Target release:** 2.0.0 + +## Baseline + +RAGOps 1.2.0 has a dependency-free Python core, deterministic and statistical gates, +portable recorded traces, baseline manifests, evaluator-drift checks, an API/workbench in +the source tree, adapter examples, and a skills-only plugin. The audited baseline passes +Ruff and 202 tests. Its five-minute demo correctly blocks a candidate whose citation +coverage and precision fall from 1.0 to 0.5 and groundedness falls from 1.0 to 0.6. + +The upgrade must not turn synthetic results into production-adoption claims. It must make +the evidence contract stronger, the regression easier to localize, the installable package +complete, and production-derived evidence easier to import and govern. + +## Goals + +1. Make release evidence content-addressed, reproducible, verifiable, and explainable. +2. Preserve all supported 1.x CLI behavior and artifact readers while adding explicit + contract migration and compatibility inspection. +3. Gate worst slices and distribution tails rather than relying only on global means. +4. evaluate agents at trace, span, retrieval, and tool-call scopes without binding the + core to a tracing or model vendor. +5. Calibrate automated evaluators against human labels and make evaluator quality a + release input. +6. Govern benchmark promotion, baseline acceptance, review, and temporary waivers through + append-only local evidence. +7. Ship the optional API and safe workbench in the wheel and provide CI-native reports. +8. Provide a differentiated five-minute flagship demo and practical import adapters. +9. Release the exact verified artifacts to GitHub and PyPI, then update the skills-only + ChatGPT plugin submission package. + +## Non-goals + +- Hosted trace ingestion, model proxying, billing, prompt playgrounds, or user management. +- A proprietary LLM judge or red-team attack generator. +- Automatic claims of semantic correctness, compliance, security, or production fitness. +- Silent migration, mutation of accepted artifacts, or network calls from the core. + +## Compatibility policy + +- Python remains `>=3.11` and `ragops` retains zero required runtime dependencies. +- Existing commands, flags, exit codes, scenario 0.1/0.2, trace 0.3/0.4, report 0.2, + replay 0.1, and baseline-manifest 0.1 remain readable. +- New contracts use stable names with explicit versions. Readers fail closed on unknown + versions and unknown fields where the schema is closed. +- Migrations are explicit commands that write a new file; inputs are never modified. +- Canonical decision values are `PASS`, `WARN`, and `BLOCK`. Existing `passed` booleans + remain in legacy reports and are derived from the canonical decision. +- Deprecations appear in machine-readable `contract diff` output and release notes. No 1.x + contract is removed in 2.0.0. + +## Architecture + +### Dependency-free core + +`src/ragops/` owns contracts, hashing, policies, deterministic/statistical evaluation, +calibration, dataset governance, trace-graph evaluation, evidence creation, explanation, +and report rendering. It uses only the Python standard library. + +Core modules have narrow responsibilities: + +- `contracts.py`: registry, validation dispatch, compatibility diff, and migrations. +- `evidence.py`: canonical JSON, artifact digests, evidence-bundle creation/verification. +- `policy_v2.py`: structured TOML policy parsing, global/slice/tail gates, and waivers. +- `slices.py`: case selection and slice aggregation. +- `distribution.py`: quantiles, paired effects, multiple-comparison correction, and + deterministic minimum-sample estimates. +- `calibration.py`: human-label loading, agreement, automated-evaluator accuracy, and + calibration gates. +- `datasets.py`: manifests, normalized deduplication, split leakage checks, coverage, and + promotion evidence. +- `trace_graph.py`: portable trace graph and component/trajectory metrics. +- `explain.py`: causal gate path and counterfactual remediation. +- `adapter_sdk.py`: local adapter registry and normalized conversion contract. +- `governance.py`: baseline registry, review queue, state transitions, and waiver audit. +- `ci.py`: JUnit, GitHub step-summary, and SARIF rendering. + +Existing modules remain import-compatible. New code may call existing evaluators and +statistics but must not duplicate their public semantics. + +### Optional adapters and applications + +- `ragops.adapters.*` converts exported vendor data into portable contracts. Built-ins + cover OpenTelemetry GenAI JSON, Phoenix annotations, Ragas, DeepEval, LangSmith, MLflow, + Promptfoo, and a documented custom profile. +- Third parties register adapters with the `ragops.adapters` Python entry-point group. +- `src/ragops/api/` contains the optional FastAPI application. +- `src/ragops/web/` contains the packaged workbench asset. +- `apps/api/main.py` remains a compatibility shim importing the packaged application. +- `ragops serve` imports FastAPI/Uvicorn lazily and gives an actionable error when the + `api` extra is absent. + +The core never imports FastAPI, Uvicorn, provider SDKs, or adapter packages. + +## Data flow + +1. `contract validate` identifies and validates each input contract. +2. Optional adapters normalize exported traces, annotations, or evaluator scores. +3. Dataset and baseline manifests bind approved inputs to SHA-256 digests. +4. Deterministic, statistical, slice, trace-graph, and calibration evaluators produce + typed result objects. +5. Policy v2 evaluates global, slice, tail, calibration, and provenance gates. +6. Explanation maps every failed gate to policy path, metric evidence, affected cases, + provenance dimensions, and a counterfactual remediation value when calculable. +7. Evidence bundling copies the bounded artifacts, records sizes and digests, writes a + canonical manifest, and optionally signs that manifest using the existing SSH lane. +8. Reporters render JSON, Markdown, safe self-contained HTML, JUnit, step summary, or SARIF + from the same evidence object. +9. Governance can accept or reject the candidate, but never rewrites evaluation evidence. + +## Contracts + +### Contract registry + +Every registered contract declares `kind`, `version`, schema filename, validator, and +available migrations. CLI commands are: + +```text +ragops contract validate --kind auto --input FILE +ragops contract diff --kind scenario --from 0.1 --to 0.2 +ragops contract migrate --kind scenario --from 0.1 --to 0.2 --input IN --output OUT +``` + +Initial migration coverage is scenario 0.1 to 0.2 and trace 0.3 to 0.4. The scenario +migration supplies explicit conservative defaults already used by the 1.x reader: +`citation_precision=0.0`, `category=unspecified`, `severity=medium`, and `language=und`. +The trace migration requires a caller-provided or empty question, records the migration in +metadata, and never invents semantic provenance. + +Schemas are included inside the wheel while the repository-level `schemas/` paths remain +available for existing consumers. + +### Evidence bundle 1.0 + +An evidence bundle is a directory with `manifest.json` plus copied artifacts. The manifest +contains: + +- schema version, RAGOps version, creation timestamp, and canonical decision; +- scenario, policy, input, baseline, candidate, evaluator, report, and environment artifact + records where available; +- original logical name, media type, byte length, SHA-256, and relative bundle path; +- evaluator identities/configuration digests and relevant application provenance; +- limitations including synthetic/production-derived classification; +- optional source commit and CI run metadata. + +Verification rejects path traversal, symlinks, missing/extra declared files, size changes, +digest changes, malformed decisions, and unknown schema versions. Deterministic mode accepts +an explicit creation time so fixtures are byte-for-byte reproducible. + +### Policy 2.0 + +Policy remains structured TOML rather than a general-purpose expression language. It has: + +- `[release]`: decision defaults and minimum case counts; +- `[metrics.]`: direction, absolute threshold, maximum regression, and severity; +- `[[slices]]`: name plus exact-match selectors for category, severity, language, or tags; +- `[tails.]`: p50/p95/p99 thresholds; +- `[calibration.]`: minimum accuracy/agreement and maximum error; +- `[[waivers]]`: exact gate ID, owner, reason, issued-at, expires-at, and optional case/slice. + +Expired or malformed waivers fail closed. Active waivers change `BLOCK` to `WARN`, never to +`PASS`, and remain visible in evidence. + +## Evaluation additions + +### Slice and tail gates + +Case results are grouped by declared slices. Reports include selected case IDs, count, +means, tails, and failed gates for every slice. A slice below its minimum count is +`insufficient_evidence`; policy decides whether that is `WARN` or `BLOCK`. + +Quantiles use a documented deterministic linear interpolation method. Latency and cost +support p50, p95, and p99. Paired comparisons include mean delta, median delta, standardized +paired effect, and deterministic bootstrap bounds. Holm correction controls family-wise +error across simultaneously gated metrics. Minimum sample estimates use a normal +approximation and are advisory evidence, not a pass condition unless policy declares it. + +### Evaluator calibration + +Calibration input binds case IDs to one or more human reviewer labels and automated scores. +The report includes coverage, MAE, threshold accuracy, false-pass rate, false-block rate, +and pairwise Cohen kappa for binary reviewer labels. Multi-reviewer agreement is the mean +of defined reviewer-pair kappas and reports insufficient evidence when no pair overlaps. +Calibration policies can gate accuracy, MAE, false-pass rate, and agreement. + +### Agent trace graph + +Trace graph 1.0 represents a trace as ordered spans with stable IDs, parent IDs, kind, +operation, input/output, timing, status, attributes, and optional tool call details. It +preserves original vendor attributes and records the adapter version. + +Deterministic metrics include tool-name precision/recall, tool-argument exactness, expected +ordering, tool success rate, retrieval document recall, error recovery, and excess-step +ratio. Component failures retain span IDs for localization. The OpenTelemetry adapter maps +known `gen_ai.*` attributes but preserves unrecognized and development-status attributes. + +## Benchmark lifecycle + +Dataset manifest 1.0 records dataset ID/version, case digests, splits, slice counts, source +classification, owners, and review state. Validation provides: + +- duplicate case-ID and normalized-content detection; +- exact and normalized-content leakage across declared splits; +- minimum per-slice coverage; +- missing provenance and unreviewed production-derived case quarantine; +- a deterministic diff between dataset versions. + +Promotion is `draft -> reviewed -> accepted -> superseded`. Every transition records actor, +timestamp, reason, previous state, and manifest digest. Invalid transitions fail closed. + +## Explainability and reports + +`ragops explain` consumes a comparison/evidence report and returns: + +- canonical decision and unwaived/waived gates; +- policy path and observed/allowed values; +- affected slice and case IDs; +- relevant provenance changes; +- minimum threshold or metric change required to clear numeric gates. + +The flagship HTML report renders summary, regression path, slice heatmap, case evidence, +provenance, limitations, and remediation without JavaScript injection sinks. All dynamic +content uses text nodes. A restrictive CSP blocks remote resources and inline event +handlers. Executive, engineer, and auditor demo profiles render different summaries from +the same evidence. + +## Governance and adoption + +The SQLite governance store contains immutable artifact records, baseline states, review +queue items, reviewer decisions, waiver records, and audit events. The CLI exposes list, +submit, review, promote, supersede, and audit operations. Pairwise reviews hide candidate +labels until a decision is recorded. Multi-reviewer agreement is reported, while state +transitions require the configured minimum approvals. + +Adapters operate on explicit local files. They never fetch remote traces, upload data, or +read credentials. Import output records producer, producer schema/version when present, +adapter name/version, unmapped fields, and conversion warnings. + +## Packaging, security, and CI + +- The wheel includes schemas, packaged API, and workbench assets. +- A clean-wheel smoke test runs CLI demo and imports the API only when the `api` extra is + installed. +- The workbench never stores API keys outside the active tab, interpolates untrusted HTML, + or loads remote assets. API request limits and fail-closed authentication remain. +- JUnit represents failed gates as test failures and warnings as skipped tests. SARIF uses + stable rule IDs and artifact-relative locations. GitHub summaries use bounded Markdown. +- Consumer GitHub/GitLab templates call a pinned RAGOps version, verify the evidence bundle, + publish bounded artifacts, and expose the canonical decision as a required check. +- Repository CI validates Python 3.11, 3.12, and 3.13 on pushes and pull requests. Publishing + remains a separate tag-bound Trusted Publishing workflow. + +## Flagship demo + +`ragops demo --profile executive|engineer|auditor` remains credential-free and under five +minutes. It generates the same BLOCK regression, an evidence bundle, JSON/Markdown/HTML, +JUnit, SARIF, and step-summary outputs. The report explicitly marks the data synthetic. + +The engineer view localizes citation and groundedness failures to cases and policy paths. +The auditor view exposes hashes, versions, provenance, active waivers, and limitations. The +executive view states decision, business risk, and remediation without overstating adoption. + +## Release process + +1. Run Ruff, the full test suite, supported-Python matrix where available, schema instance + validation, demos, API/workbench smoke tests, and clean-wheel installation. +2. Build wheel/sdist, SBOM, checksums, and local release evidence from a clean commit. +3. Verify version parity across package, changelog, plugin manifests, skills, examples, and + release notes. +4. Create annotated `v2.0.0`, push the branch/tag, and promote the exact local artifacts to + a GitHub Release. +5. Publish those verified artifacts to PyPI through OIDC Trusted Publishing and verify a + clean install from PyPI. +6. Update the ChatGPT skills-only plugin manifest and submission evidence, validate the + plugin package, submit/update it through the authenticated OpenAI directory workflow, + and record any account-only review state without claiming approval prematurely. + +## Verification criteria + +- All existing tests remain green and every production behavior added here has a witnessed + failing test before implementation. +- Legacy CLI smoke fixtures are byte-compatible unless the output is explicitly versioned. +- Contract migration round trips validate and never overwrite inputs. +- Evidence tampering, traversal, symlinks, expiration, leakage, invalid transitions, and + untrusted HTML each have negative tests. +- The built wheel contains API, workbench, and schemas and passes isolated installation. +- The dependency-free core imports without any optional dependency installed. +- Demo artifacts agree on decision, gate IDs, hashes, limitations, and version. +- Release claims distinguish synthetic benchmark evidence, production-derived evidence, + PyPI publication, GitHub publication, ChatGPT submission, and ChatGPT approval. diff --git a/pyproject.toml b/pyproject.toml index f67258c..4f01a44 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "ragops" -version = "1.2.0" +version = "2.0.0" description = "Regression tests and explainable release gates for RAG and AI agents" readme = "README.md" requires-python = ">=3.11" @@ -33,6 +33,7 @@ Changelog = "https://github.com/thangldw/ragops/blob/main/CHANGELOG.md" [project.optional-dependencies] api = ["fastapi>=0.115", "uvicorn>=0.30"] dev = ["pytest>=8.0", "ruff>=0.6", "fastapi>=0.115", "jsonschema>=4.23"] +release = ["build>=1.2", "cyclonedx-bom>=7.3", "twine>=5.1"] [project.scripts] ragops = "ragops.cli:main" @@ -40,6 +41,12 @@ ragops = "ragops.cli:main" [tool.hatch.build.targets.wheel] packages = ["src/ragops"] +[tool.hatch.build.targets.wheel.force-include] +"schemas" = "ragops/schemas" +"src/ragops/web/index.html" = "ragops/web/index.html" +"src/ragops/web/app.js" = "ragops/web/app.js" +"src/ragops/web/styles.css" = "ragops/web/styles.css" + [tool.pytest.ini_options] pythonpath = ["src", "."] testpaths = ["tests"] diff --git a/schemas/calibration-report-1.0.schema.json b/schemas/calibration-report-1.0.schema.json new file mode 100644 index 0000000..584e990 --- /dev/null +++ b/schemas/calibration-report-1.0.schema.json @@ -0,0 +1,23 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://ragops.dev/schemas/calibration-report-1.0.schema.json", + "title": "RAGOps evaluator calibration report 1.0", + "type": "object", + "additionalProperties": false, + "required": ["schema_version", "evaluator", "decision", "case_count", "labeled_case_count", "coverage", "accuracy", "mae", "false_pass_rate", "false_block_rate", "reviewer_agreement", "reviewer_pair_count", "failed_gates"], + "properties": { + "schema_version": {"const": "1.0"}, + "evaluator": {"type": "string", "minLength": 1}, + "decision": {"enum": ["PASS", "BLOCK"]}, + "case_count": {"type": "integer", "minimum": 1}, + "labeled_case_count": {"type": "integer", "minimum": 0}, + "coverage": {"type": "number", "minimum": 0, "maximum": 1}, + "accuracy": {"type": "number", "minimum": 0, "maximum": 1}, + "mae": {"type": "number", "minimum": 0, "maximum": 1}, + "false_pass_rate": {"type": "number", "minimum": 0, "maximum": 1}, + "false_block_rate": {"type": "number", "minimum": 0, "maximum": 1}, + "reviewer_agreement": {"type": ["number", "null"]}, + "reviewer_pair_count": {"type": "integer", "minimum": 0}, + "failed_gates": {"type": "array", "items": {"type": "string"}} + } +} diff --git a/schemas/calibration-set-1.0.schema.json b/schemas/calibration-set-1.0.schema.json new file mode 100644 index 0000000..8ba59a6 --- /dev/null +++ b/schemas/calibration-set-1.0.schema.json @@ -0,0 +1,14 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://ragops.dev/schemas/calibration-set-1.0.schema.json", + "title": "RAGOps evaluator calibration set 1.0", + "type": "object", + "additionalProperties": false, + "required": ["schema_version", "evaluator", "policy", "records"], + "properties": { + "schema_version": {"const": "1.0"}, + "evaluator": {"type": "string", "minLength": 1}, + "policy": {"type": "object"}, + "records": {"type": "array", "minItems": 1, "items": {"type": "object"}} + } +} diff --git a/schemas/dataset-manifest-1.0.schema.json b/schemas/dataset-manifest-1.0.schema.json new file mode 100644 index 0000000..b78ca32 --- /dev/null +++ b/schemas/dataset-manifest-1.0.schema.json @@ -0,0 +1,18 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://ragops.dev/schemas/dataset-manifest-1.0.schema.json", + "title": "RAGOps dataset manifest 1.0", + "type": "object", + "additionalProperties": false, + "required": ["schema_version", "dataset_id", "version", "source_classification", "owners", "review_state", "cases", "splits"], + "properties": { + "schema_version": {"const": "1.0"}, + "dataset_id": {"type": "string", "minLength": 1}, + "version": {"type": "string", "minLength": 1}, + "source_classification": {"enum": ["synthetic", "public", "production-derived"]}, + "owners": {"type": "array", "minItems": 1, "items": {"type": "string", "minLength": 1}}, + "review_state": {"enum": ["draft", "reviewed", "accepted", "superseded"]}, + "cases": {"type": "array", "minItems": 1, "items": {"type": "object"}}, + "splits": {"type": "object", "minProperties": 1} + } +} diff --git a/schemas/evidence-bundle-1.0.schema.json b/schemas/evidence-bundle-1.0.schema.json new file mode 100644 index 0000000..2a50e53 --- /dev/null +++ b/schemas/evidence-bundle-1.0.schema.json @@ -0,0 +1,32 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://ragops.dev/schemas/evidence-bundle-1.0.schema.json", + "title": "RAGOps evidence bundle 1.0", + "type": "object", + "additionalProperties": false, + "required": ["schema_version", "ragops_version", "created_at", "decision", "limitations", "artifacts", "metadata"], + "properties": { + "schema_version": {"const": "1.0"}, + "ragops_version": {"type": "string", "minLength": 1}, + "created_at": {"type": "string", "format": "date-time"}, + "decision": {"enum": ["PASS", "WARN", "BLOCK"]}, + "limitations": {"type": "array", "minItems": 1, "items": {"type": "string", "minLength": 1}}, + "artifacts": { + "type": "array", + "minItems": 1, + "items": { + "type": "object", + "additionalProperties": false, + "required": ["logical_name", "path", "media_type", "bytes", "sha256"], + "properties": { + "logical_name": {"type": "string", "pattern": "^[A-Za-z0-9][A-Za-z0-9_.-]*$"}, + "path": {"type": "string", "pattern": "^artifacts/[^/]+$"}, + "media_type": {"type": "string", "minLength": 1}, + "bytes": {"type": "integer", "minimum": 0}, + "sha256": {"type": "string", "pattern": "^[0-9a-f]{64}$"} + } + } + }, + "metadata": {"type": "object"} + } +} diff --git a/schemas/release-decision-1.0.schema.json b/schemas/release-decision-1.0.schema.json new file mode 100644 index 0000000..8655546 --- /dev/null +++ b/schemas/release-decision-1.0.schema.json @@ -0,0 +1,15 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://ragops.dev/schemas/release-decision-1.0.schema.json", + "title": "RAGOps release decision 1.0", + "type": "object", + "additionalProperties": false, + "required": ["schema_version", "scenario_id", "decision", "passed", "gates"], + "properties": { + "schema_version": {"const": "1.0"}, + "scenario_id": {"type": "string", "minLength": 1}, + "decision": {"enum": ["PASS", "WARN", "BLOCK"]}, + "passed": {"type": "boolean"}, + "gates": {"type": "array", "items": {"type": "object"}} + } +} diff --git a/schemas/release-policy-2.0.schema.json b/schemas/release-policy-2.0.schema.json new file mode 100644 index 0000000..3524023 --- /dev/null +++ b/schemas/release-policy-2.0.schema.json @@ -0,0 +1,16 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://ragops.dev/schemas/release-policy-2.0.schema.json", + "title": "RAGOps release policy 2.0 normalized JSON form", + "type": "object", + "additionalProperties": false, + "required": ["schema_version", "release", "metrics", "slices", "tails", "waivers"], + "properties": { + "schema_version": {"const": "2.0"}, + "release": {"type": "object"}, + "metrics": {"type": "array"}, + "slices": {"type": "array"}, + "tails": {"type": "array"}, + "waivers": {"type": "array"} + } +} diff --git a/schemas/trace-graph-1.0.schema.json b/schemas/trace-graph-1.0.schema.json new file mode 100644 index 0000000..7bb2d64 --- /dev/null +++ b/schemas/trace-graph-1.0.schema.json @@ -0,0 +1,14 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://ragops.dev/schemas/trace-graph-1.0.schema.json", + "title": "RAGOps portable trace graph 1.0", + "type": "object", + "additionalProperties": false, + "required": ["schema_version", "trace_id", "spans", "metadata"], + "properties": { + "schema_version": {"const": "1.0"}, + "trace_id": {"type": "string", "minLength": 1}, + "spans": {"type": "array", "minItems": 1, "items": {"type": "object"}}, + "metadata": {"type": "object"} + } +} diff --git a/scripts/local_release.py b/scripts/local_release.py index 84dcee9..9fcf335 100644 --- a/scripts/local_release.py +++ b/scripts/local_release.py @@ -16,6 +16,7 @@ import sys import tempfile import tomllib +import zipfile from datetime import UTC, datetime from pathlib import Path @@ -23,6 +24,12 @@ DIST = ROOT / "dist" +def environment_tool(name: str) -> str: + suffix = ".exe" if os.name == "nt" else "" + candidate = Path(sys.executable).parent / f"{name}{suffix}" + return str(candidate) if candidate.is_file() else name + + def run( *args: str, env: dict[str, str] | None = None, @@ -89,6 +96,17 @@ def verify_checksums(directory: Path) -> None: def verify_clean_install(wheel: Path, expected_version: str) -> None: + with zipfile.ZipFile(wheel) as archive: + names = set(archive.namelist()) + required = { + "ragops/api/main.py", + "ragops/web/index.html", + "ragops/web/app.js", + "ragops/schemas/evidence-bundle-1.0.schema.json", + } + missing = sorted(required - names) + if missing: + raise SystemExit(f"wheel is missing packaged surfaces: {missing}") with tempfile.TemporaryDirectory(prefix="ragops-release-") as temp: venv = Path(temp) / "venv" run(sys.executable, "-m", "venv", str(venv)) @@ -99,18 +117,63 @@ def verify_clean_install(wheel: Path, expected_version: str) -> None: "-c", f"import ragops; assert ragops.__version__ == {expected_version!r}", ) + demo = Path(temp) / "demo" + run(str(python), "-m", "ragops.cli", "demo", "--output", str(demo)) + run(str(python), "-m", "ragops.cli", "evidence", "verify", "--bundle", str(demo / "evidence")) + + +def build_plugin_bundle(expected_version: str) -> Path: + manifest = json.loads((ROOT / ".codex-plugin/plugin.json").read_text(encoding="utf-8")) + if manifest.get("version") != expected_version: + raise SystemExit("ChatGPT plugin manifest version does not match package version") + DIST.mkdir(parents=True, exist_ok=True) + output = DIST / f"ragops-chatgpt-plugin-{expected_version}.zip" + sources: dict[str, Path] = { + ".codex-plugin/plugin.json": ROOT / ".codex-plugin/plugin.json", + "assets/plugin-icon.png": ROOT / "assets/plugin-icon.png", + } + skill_root = ROOT / "skills" / "evaluate-ai-release" + for path in skill_root.rglob("*"): + if path.is_file() and "__pycache__" not in path.parts and path.suffix != ".pyc": + sources[path.relative_to(ROOT).as_posix()] = path + for path in (ROOT / "src" / "ragops").rglob("*"): + if path.is_file() and "__pycache__" not in path.parts and path.suffix != ".pyc": + relative = path.relative_to(ROOT / "src" / "ragops") + sources[f"skills/evaluate-ai-release/scripts/vendor/ragops/{relative.as_posix()}"] = path + for path in (ROOT / "schemas").glob("*.json"): + sources[f"skills/evaluate-ai-release/scripts/vendor/ragops/schemas/{path.name}"] = path + fixture_root = ROOT / "scenarios" / "japanese_troubleshooting" + fixture_names = ( + "benchmark-baseline.json", + "benchmark-regressed.json", + "benchmark-v0.2.json", + "sample_responses.json", + "scenario.json", + ) + for name in fixture_names: + sources[f"skills/evaluate-ai-release/references/fixtures/japanese_troubleshooting/{name}"] = fixture_root / name + with zipfile.ZipFile(output, "w", compression=zipfile.ZIP_DEFLATED, compresslevel=9) as archive: + for archive_name, source in sorted(sources.items()): + info = zipfile.ZipInfo(archive_name, date_time=(1980, 1, 1, 0, 0, 0)) + info.compress_type = zipfile.ZIP_DEFLATED + mode = 0o755 if archive_name.endswith("/run_ragops.py") else 0o644 + info.external_attr = mode << 16 + archive.writestr(info, source.read_bytes()) + return output def verify(tag: str) -> None: assert_tag(tag) - run("ruff", "check", ".") + run(environment_tool("ruff"), "check", ".") env = os.environ.copy() env["PYTEST_DISABLE_PLUGIN_AUTOLOAD"] = "1" run(sys.executable, "-m", "pytest", "-q", env=env) - run("ragops", "evaluate", "--scenario", "scenarios/japanese_troubleshooting/scenario.json", - "--responses", "scenarios/japanese_troubleshooting/sample_responses.json") + run(sys.executable, "-m", "ragops.cli", "evaluate", "--scenario", + "scenarios/japanese_troubleshooting/scenario.json", "--responses", + "scenarios/japanese_troubleshooting/sample_responses.json") blocked = subprocess.run( - ["ragops", "compare", "--scenario", "scenarios/japanese_troubleshooting/benchmark-v0.2.json", + [sys.executable, "-m", "ragops.cli", "compare", "--scenario", + "scenarios/japanese_troubleshooting/benchmark-v0.2.json", "--baseline", "scenarios/japanese_troubleshooting/benchmark-baseline.json", "--candidate", "scenarios/japanese_troubleshooting/benchmark-regressed.json"], cwd=ROOT ) @@ -121,13 +184,15 @@ def verify(tag: str) -> None: built = artifacts() wheel = next(path for path in built if path.suffix == ".whl") verify_clean_install(wheel, version()) + plugin = build_plugin_bundle(version()) sbom = DIST / f"ragops-{version()}.cdx.json" - if shutil.which("cyclonedx-py"): - run("cyclonedx-py", "environment", sys.executable, "--output-reproducible", + cyclonedx = environment_tool("cyclonedx-py") + if Path(cyclonedx).is_file() or shutil.which(cyclonedx): + run(cyclonedx, "environment", sys.executable, "--output-reproducible", "--output-file", str(sbom)) else: raise SystemExit("cyclonedx-py is required: python -m pip install cyclonedx-bom==7.3.0") - manifest = checksums([*built, sbom]) + manifest = checksums([*built, sbom, plugin]) evidence = { "schema": "ragops-local-release-evidence-0.1", "tag": tag, @@ -135,7 +200,7 @@ def verify(tag: str) -> None: "created_at": datetime.now(UTC).isoformat(), "validation": "pass", "trusted_publishing": True, - "artifacts": [path.name for path in [*built, sbom, manifest]], + "artifacts": [path.name for path in [*built, sbom, plugin, manifest]], } (DIST / "LOCAL_RELEASE_EVIDENCE.json").write_text( json.dumps(evidence, indent=2) + "\n", encoding="utf-8" @@ -146,7 +211,8 @@ def github(tag: str, yes: bool) -> None: assert_tag(tag) if not yes: raise SystemExit("publishing requires --yes") - paths = [*artifacts(), DIST / f"ragops-{version()}.cdx.json", DIST / "SHA256SUMS", + paths = [*artifacts(), DIST / f"ragops-{version()}.cdx.json", + DIST / f"ragops-chatgpt-plugin-{version()}.zip", DIST / "SHA256SUMS", DIST / "LOCAL_RELEASE_EVIDENCE.json"] if any(not path.is_file() for path in paths): raise SystemExit("run verify first; release artifacts are incomplete") diff --git a/skills/evaluate-ai-release/SKILL.md b/skills/evaluate-ai-release/SKILL.md index 300d5ab..989d59b 100644 --- a/skills/evaluate-ai-release/SKILL.md +++ b/skills/evaluate-ai-release/SKILL.md @@ -14,7 +14,7 @@ Use RAGOps as a deterministic evaluator. Treat its result as release evidence, n 3. Inspect inputs for secrets or personal data before running an evaluation. Keep generated reports private unless the user approves sharing. 4. Run the narrowest applicable command through `python3 scripts/run_ragops.py ...` from this skill directory. 5. Interpret exit code `0` as PASS and exit code `2` as an evaluated BLOCK, not an execution failure. Treat other nonzero codes as operational errors. -6. Report the decision, failed gates, material metric deltas, provenance limitations, and the smallest next action. +6. Verify generated evidence bundles before using them, then report the decision, failed gates, material metric deltas, provenance limitations, and the smallest next action. ## Guardrails @@ -23,6 +23,7 @@ Use RAGOps as a deterministic evaluator. Treat its result as release evidence, n - Do not run provider-backed evaluation unless the user explicitly supplies and authorizes the provider configuration. - Do not publish, merge, tag, or deploy based only on this skill's output. - Preserve input artifacts and command parameters in the final evidence so another reviewer can reproduce the result. +- Distinguish synthetic, production-derived, and live production evidence; never infer adoption from the bundled demo. ## Output diff --git a/skills/evaluate-ai-release/references/commands.md b/skills/evaluate-ai-release/references/commands.md index 51ac962..6bcede7 100644 --- a/skills/evaluate-ai-release/references/commands.md +++ b/skills/evaluate-ai-release/references/commands.md @@ -3,7 +3,9 @@ - Inspect a scenario: `python3 scripts/run_ragops.py inspect --scenario PATH` - Evaluate candidate responses: `python3 scripts/run_ragops.py evaluate --scenario PATH --responses PATH` - Compare candidate and baseline: run `python3 scripts/run_ragops.py compare --help` and provide the exact required paths. -- Generate a credential-free example: `python3 scripts/run_ragops.py demo --output OUTPUT_DIR` +- Generate a credential-free evidence bundle: `python3 scripts/run_ragops.py demo --profile engineer --output OUTPUT_DIR` +- Verify an evidence bundle: `python3 scripts/run_ragops.py evidence verify --bundle OUTPUT_DIR/evidence` +- Explain a canonical decision: `python3 scripts/run_ragops.py explain --input OUTPUT_DIR/decision.json` Run `python3 scripts/run_ragops.py COMMAND --help` before using an unfamiliar option. Exit code `2` from an evaluation means the release gate deliberately blocked the candidate. diff --git a/src/ragops/__init__.py b/src/ragops/__init__.py index b291433..369a22b 100644 --- a/src/ragops/__init__.py +++ b/src/ragops/__init__.py @@ -2,6 +2,7 @@ from ragops.drift import detect_evaluator_drift from ragops.engine import compare, evaluate +from ragops.evidence import create_evidence_bundle, verify_evidence_bundle from ragops.loader import load_responses, load_scenario, responses_from_data, scenario_from_dict from ragops.provenance import diagnose_provenance from ragops.sequential import compare_replay_bundles_sequentially @@ -11,7 +12,7 @@ replay_bundle_from_dict, ) -__version__ = "1.2.0" +__version__ = "2.0.0" __all__ = [ "compare", @@ -19,6 +20,7 @@ "detect_evaluator_drift", "diagnose_provenance", "evaluate", + "create_evidence_bundle", "compare_replay_bundles", "load_replay_bundle", "load_responses", @@ -26,5 +28,6 @@ "responses_from_data", "replay_bundle_from_dict", "scenario_from_dict", + "verify_evidence_bundle", "__version__", ] diff --git a/src/ragops/adapter_sdk.py b/src/ragops/adapter_sdk.py new file mode 100644 index 0000000..e23a288 --- /dev/null +++ b/src/ragops/adapter_sdk.py @@ -0,0 +1,68 @@ +from __future__ import annotations + +from dataclasses import dataclass +from importlib import metadata +from typing import Any, Protocol + + +@dataclass(frozen=True) +class AdapterContext: + case_ids: tuple[str, ...] = () + + +@dataclass(frozen=True) +class AdapterOutput: + provider: str + records: tuple[dict[str, Any], ...] + provenance: dict[str, Any] + warnings: tuple[str, ...] = () + + def to_dict(self) -> dict[str, Any]: + return { + "schema_version": "0.1", + "provider": self.provider, + "records": [dict(item) for item in self.records], + "provenance": self.provenance, + "warnings": list(self.warnings), + } + + +class Adapter(Protocol): + name: str + version: str + + def convert(self, payload: Any, context: AdapterContext) -> AdapterOutput: ... + + +@dataclass(frozen=True) +class AdapterDescriptor: + name: str + version: str + source: str + adapter: Adapter | None = None + + def to_dict(self) -> dict[str, str]: + return {"name": self.name, "version": self.version, "source": self.source} + + +BUILTIN_ADAPTERS = ("custom", "deepeval", "langsmith", "mlflow", "phoenix", "promptfoo", "ragas") + + +def discover_adapters(*, include_entry_points: bool = True) -> tuple[AdapterDescriptor, ...]: + discovered = [AdapterDescriptor(name, "1.0", "builtin") for name in BUILTIN_ADAPTERS] + if include_entry_points: + for entry_point in metadata.entry_points(group="ragops.adapters"): + loaded = entry_point.load() + adapter = loaded() if isinstance(loaded, type) else loaded + discovered.append( + AdapterDescriptor( + name=str(adapter.name), + version=str(adapter.version), + source=f"entry-point:{entry_point.name}", + adapter=adapter, + ) + ) + by_name: dict[str, AdapterDescriptor] = {} + for item in discovered: + by_name.setdefault(item.name, item) + return tuple(by_name[name] for name in sorted(by_name)) diff --git a/src/ragops/adapters/external_metrics.py b/src/ragops/adapters/external_metrics.py index 7deac03..2e1182c 100644 --- a/src/ragops/adapters/external_metrics.py +++ b/src/ragops/adapters/external_metrics.py @@ -10,7 +10,9 @@ from ragops.models import EvalCase, RecordedResponse, Scenario from ragops.plugins import PluginResult -ALLOWED_PROVIDERS = {"ragas", "deepeval", "langfuse", "custom"} +ALLOWED_PROVIDERS = { + "custom", "deepeval", "langfuse", "langsmith", "mlflow", "phoenix", "promptfoo", "ragas" +} @dataclass(frozen=True) @@ -64,12 +66,12 @@ def load_external_metric_evaluator( def external_metric_envelope_from_dict(data: Any) -> ExternalMetricEnvelope: if not isinstance(data, dict) or data.get("schema_version") != "0.1": raise ContractError("External metrics must use schema version 0.1") - unknown_fields = set(data) - {"schema_version", "provider", "records"} + unknown_fields = set(data) - {"schema_version", "provider", "records", "provenance", "warnings"} if unknown_fields: raise ContractError(f"External metrics have unknown fields: {sorted(unknown_fields)}") provider = data.get("provider") if provider not in ALLOWED_PROVIDERS: - raise ContractError("External metric provider must be ragas, deepeval, langfuse, or custom") + raise ContractError(f"External metric provider is unsupported: {provider!r}") records = data.get("records") if not isinstance(records, list) or not records: raise ContractError("External metrics need at least one record") diff --git a/src/ragops/adapters/import_profiles.py b/src/ragops/adapters/import_profiles.py new file mode 100644 index 0000000..2863cbb --- /dev/null +++ b/src/ragops/adapters/import_profiles.py @@ -0,0 +1,146 @@ +from __future__ import annotations + +import math +from collections.abc import Iterable +from typing import Any + +from ragops.adapter_sdk import BUILTIN_ADAPTERS, AdapterContext, AdapterOutput, discover_adapters +from ragops.loader import ContractError + + +def convert_import(profile: str, payload: Any, context: AdapterContext) -> AdapterOutput: + if profile not in BUILTIN_ADAPTERS: + descriptor = next((item for item in discover_adapters() if item.name == profile), None) + if descriptor is None or descriptor.adapter is None: + raise ContractError(f"Unknown adapter profile: {profile}") + return descriptor.adapter.convert(payload, context) + rows = _rows(profile, payload) + by_case: dict[str, dict[str, float]] = {} + unmapped: set[str] = set() + for index, row in enumerate(rows): + if not isinstance(row, dict): + raise ContractError(f"{profile} record {index} must be an object") + case_id = _case_id(profile, row, index) + if context.case_ids and case_id not in context.case_ids: + raise ContractError(f"{profile} record {index} has unknown case ID {case_id!r}") + metrics, ignored = _metrics(profile, row, index) + unmapped.update(ignored) + target = by_case.setdefault(case_id, {}) + conflicts = set(target) & set(metrics) + if conflicts: + raise ContractError(f"{profile} has duplicate metrics for case {case_id!r}: {sorted(conflicts)}") + target.update(metrics) + warnings = [] + if context.case_ids: + missing = [case_id for case_id in context.case_ids if case_id not in by_case] + if missing: + warnings.append(f"missing case IDs: {', '.join(missing)}") + records = tuple( + {"case_id": case_id, "metrics": dict(sorted(metrics.items()))} + for case_id, metrics in sorted(by_case.items()) + ) + return AdapterOutput( + provider=profile, + records=records, + provenance={ + "producer": profile, + "adapter_version": "1.0", + "unmapped_fields": sorted(unmapped), + }, + warnings=tuple(warnings), + ) + + +def _rows(profile: str, payload: Any) -> list[Any]: + if isinstance(payload, list): + return payload + if not isinstance(payload, dict): + raise ContractError(f"{profile} import must be a JSON object or array") + keys = { + "ragas": ("records", "data"), + "deepeval": ("test_results", "results"), + "phoenix": ("evaluations", "records"), + "langsmith": ("runs", "results"), + "mlflow": ("results", "evaluations"), + "promptfoo": ("results",), + "custom": ("records",), + }[profile] + for key in keys: + value = payload.get(key) + if isinstance(value, list): + return value + raise ContractError(f"{profile} import has no supported record collection: {', '.join(keys)}") + + +def _case_id(profile: str, row: dict[str, Any], index: int) -> str: + candidates: Iterable[Any] + if profile == "promptfoo": + variables = row.get("vars") + candidates = ( + variables.get("case_id") if isinstance(variables, dict) else None, + row.get("case_id"), + row.get("id"), + ) + else: + candidates = ( + row.get("case_id"), row.get("reference_example_id"), row.get("request_id"), + row.get("span_id"), row.get("name") if profile == "deepeval" else None, row.get("id"), + ) + case_id = next((item for item in candidates if isinstance(item, str) and item), None) + if case_id is None: + raise ContractError(f"{profile} record {index} has no case ID") + return case_id + + +def _metrics(profile: str, row: dict[str, Any], index: int) -> tuple[dict[str, float], set[str]]: + identifier_fields = { + "case_id", "reference_example_id", "request_id", "span_id", "id", "vars" + } + if profile == "deepeval": + identifier_fields.add("name") + raw = row.get("metrics") + if isinstance(raw, list): + metrics = { + str(item.get("name")): _score(item.get("score"), profile, index) + for item in raw + if isinstance(item, dict) and isinstance(item.get("name"), str) + } + ignored = set(row) - identifier_fields - {"metrics"} + return _nonempty(metrics, profile, index), ignored + if profile == "phoenix" and isinstance(row.get("name"), str) and "score" in row: + metrics = {row["name"]: _score(row["score"], profile, index)} + ignored = set(row) - identifier_fields - {"name", "score"} + return metrics, ignored + if profile == "langsmith" and isinstance(row.get("feedback"), list): + metrics = { + str(item.get("key")): _score(item.get("score"), profile, index) + for item in row["feedback"] + if isinstance(item, dict) and isinstance(item.get("key"), str) + } + ignored = set(row) - identifier_fields - {"feedback"} + return _nonempty(metrics, profile, index), ignored + for container in ("metrics", "scores"): + raw = row.get(container) + if isinstance(raw, dict): + metrics = {name: _score(value, profile, index) for name, value in raw.items()} + ignored = set(row) - identifier_fields - {container} + return _nonempty(metrics, profile, index), ignored + metrics = { + name: _score(value, profile, index) + for name, value in row.items() + if name not in identifier_fields and isinstance(value, (int, float)) and not isinstance(value, bool) + } + ignored = set(row) - identifier_fields - set(metrics) + return _nonempty(metrics, profile, index), ignored + + +def _score(value: object, profile: str, index: int) -> float: + if isinstance(value, bool) or not isinstance(value, (int, float)) or not math.isfinite(value): + raise ContractError(f"{profile} record {index} metrics must be finite numeric values") + return float(value) + + +def _nonempty(metrics: dict[str, float], profile: str, index: int) -> dict[str, float]: + if not metrics: + raise ContractError(f"{profile} record {index} has no metrics") + return metrics diff --git a/src/ragops/adapters/otel_genai.py b/src/ragops/adapters/otel_genai.py new file mode 100644 index 0000000..42754ae --- /dev/null +++ b/src/ragops/adapters/otel_genai.py @@ -0,0 +1,112 @@ +from __future__ import annotations + +from typing import Any, Iterable, Mapping + +from ragops.loader import ContractError +from ragops.trace_graph import TraceGraph, trace_graph_from_dict + + +def otel_spans_to_trace_graph(spans: Iterable[Mapping[str, Any]]) -> TraceGraph: + items = list(spans) + if not items: + raise ContractError("OpenTelemetry export contains no spans") + trace_ids = {item.get("trace_id") for item in items} + if len(trace_ids) != 1 or not next(iter(trace_ids)): + raise ContractError("OpenTelemetry spans must share one non-empty trace_id") + normalized = [] + for index, item in enumerate(items): + if not isinstance(item, Mapping): + raise ContractError(f"OpenTelemetry span {index} must be an object") + attributes = item.get("attributes", {}) + if not isinstance(attributes, Mapping): + raise ContractError(f"OpenTelemetry span {index} attributes must be an object") + attributes = dict(attributes) + operation_name = attributes.get("gen_ai.operation.name", "") + kind = _kind(operation_name) + operation = ( + attributes.get("gen_ai.tool.name") + or attributes.get("gen_ai.agent.name") + or item.get("name") + ) + if not isinstance(operation, str) or not operation: + raise ContractError(f"OpenTelemetry span {index} operation is missing") + start = _nanoseconds(item.get("start_time_unix_nano"), index, "start") + end = _nanoseconds(item.get("end_time_unix_nano"), index, "end") + normalized.append( + { + "id": item.get("span_id"), + "parent_id": item.get("parent_span_id") or None, + "kind": kind, + "operation": operation, + "started_at_ms": start, + "ended_at_ms": end, + "status": _status(item.get("status")), + "input": _input(attributes, kind), + "output": _output(attributes, kind), + "attributes": attributes, + } + ) + return trace_graph_from_dict( + { + "schema_version": "1.0", + "trace_id": next(iter(trace_ids)), + "spans": normalized, + "metadata": { + "adapter": "otel-genai", + "adapter_version": "1.0", + "semantic_convention_stability": "development", + }, + } + ) + + +def _kind(operation: object) -> str: + if operation == "execute_tool": + return "tool" + if operation == "retrieval": + return "retrieval" + if operation == "invoke_agent": + return "agent" + if operation == "invoke_workflow": + return "workflow" + return "llm" if operation in {"chat", "text_completion", "generate_content"} else "other" + + +def _input(attributes: dict[str, Any], kind: str) -> dict[str, Any]: + result = {} + if "gen_ai.input.messages" in attributes: + result["messages"] = attributes["gen_ai.input.messages"] + if kind == "tool" and "gen_ai.tool.call.arguments" in attributes: + result["arguments"] = attributes["gen_ai.tool.call.arguments"] + if kind == "retrieval" and "gen_ai.retrieval.query" in attributes: + result["query"] = attributes["gen_ai.retrieval.query"] + return result + + +def _output(attributes: dict[str, Any], kind: str) -> dict[str, Any]: + result = {} + if "gen_ai.output.messages" in attributes: + result["messages"] = attributes["gen_ai.output.messages"] + if kind == "tool" and "gen_ai.tool.call.result" in attributes: + result["result"] = attributes["gen_ai.tool.call.result"] + documents = attributes.get("gen_ai.retrieval.documents") + if kind == "retrieval" and isinstance(documents, list): + result["document_ids"] = documents + return result + + +def _status(value: object) -> str: + if isinstance(value, Mapping): + value = value.get("code") + normalized = str(value or "").casefold() + if "error" in normalized: + return "error" + if "ok" in normalized: + return "ok" + return "unset" + + +def _nanoseconds(value: object, index: int, label: str) -> float: + if isinstance(value, bool) or not isinstance(value, (int, float)) or value < 0: + raise ContractError(f"OpenTelemetry span {index} {label} timestamp is invalid") + return float(value) / 1_000_000 diff --git a/src/ragops/api/__init__.py b/src/ragops/api/__init__.py new file mode 100644 index 0000000..e68f159 --- /dev/null +++ b/src/ragops/api/__init__.py @@ -0,0 +1 @@ +"""Optional FastAPI application shipped with the ``api`` extra.""" diff --git a/src/ragops/api/main.py b/src/ragops/api/main.py new file mode 100644 index 0000000..e738b70 --- /dev/null +++ b/src/ragops/api/main.py @@ -0,0 +1,403 @@ +from __future__ import annotations + +import hmac +import os +from pathlib import Path + +from fastapi import Depends, FastAPI, Header, HTTPException, Request +from fastapi.responses import FileResponse, JSONResponse +from pydantic import BaseModel, ConfigDict + +from ragops import __version__ +from ragops.control_plane import ControlPlane +from ragops.drift import detect_evaluator_drift +from ragops.engine import compare, evaluate +from ragops.loader import ContractError, responses_from_data, scenario_from_dict +from ragops.models import ( + EvaluatorDriftMetricGate, + EvaluatorDriftPolicy, + SequentialPolicy, + StatisticalMetricGate, + StatisticalPolicy, +) +from ragops.sequential import compare_replay_bundles_sequentially +from ragops.statistical import compare_replay_bundles, replay_bundle_from_dict +from ragops.store import ExperimentStore + +app = FastAPI(title="RAGOps API", version=__version__) + +WEB_ROOT = Path(__file__).parents[1] / "web" +SECURITY_HEADERS = { + "Content-Security-Policy": ( + "default-src 'self'; connect-src 'self'; img-src 'self'; style-src 'self'; " + "script-src 'self'; base-uri 'none'; form-action 'none'; frame-ancestors 'none'" + ), + "X-Content-Type-Options": "nosniff", + "Referrer-Policy": "no-referrer", + "X-Frame-Options": "DENY", +} + +DEFAULT_MAX_REQUEST_BYTES = 2 * 1024 * 1024 +DEFAULT_MAX_CASES = 1000 +DEFAULT_MAX_REPLAY_OBSERVATIONS = 100_000 + + +class EvaluateRequest(BaseModel): + scenario: dict + responses: list[dict] + + +class SavedEvaluateRequest(EvaluateRequest): + label: str = "" + + +class CompareRequest(BaseModel): + scenario: dict + baseline: list[dict] + candidate: list[dict] + + +class StrictRequest(BaseModel): + model_config = ConfigDict(extra="forbid") + + +class StatisticalCompareRequest(StrictRequest): + baseline: dict + candidate: dict + policy: dict + + +class EvaluatorDriftRequest(StrictRequest): + reference: dict + current: dict + policy: dict + + +class ReviewRequest(BaseModel): + status: str + reviewer: str + note: str = "" + + +def configured_store( + x_workspace_id: str | None = Header(default=None), + x_workspace_key: str | None = Header(default=None), +) -> ExperimentStore: + control_root = os.getenv("RAGOPS_CONTROL_PLANE") + if control_root: + if not x_workspace_id or not x_workspace_key: + raise HTTPException(status_code=401, detail="Workspace credentials are required") + try: + return ControlPlane(control_root).workspace_store(x_workspace_id, x_workspace_key) + except PermissionError as exc: + raise HTTPException(status_code=401, detail=str(exc)) from exc + path = os.getenv("RAGOPS_STORE") + if not path: + raise HTTPException(status_code=503, detail="RAGOPS_STORE is not configured") + return ExperimentStore(path) + + +def require_api_key(x_api_key: str | None = Header(default=None)) -> None: + expected = os.getenv("RAGOPS_API_KEY", "").strip() + insecure_dev_mode = os.getenv("RAGOPS_INSECURE_DEV_MODE", "").casefold() == "true" + if expected and insecure_dev_mode: + raise HTTPException( + status_code=503, + detail="RAGOPS_API_KEY and RAGOPS_INSECURE_DEV_MODE cannot both be enabled", + ) + if insecure_dev_mode: + return + if not expected: + raise HTTPException( + status_code=503, + detail=( + "API authentication is not configured; set RAGOPS_API_KEY or explicitly " + "enable RAGOPS_INSECURE_DEV_MODE=true for local development" + ), + ) + if x_api_key is None or not hmac.compare_digest(x_api_key, expected): + raise HTTPException(status_code=401, detail="Invalid API key") + + +@app.middleware("http") +async def enforce_request_size(request: Request, call_next): + response = None + if request.method in {"POST", "PUT", "PATCH"}: + maximum = _positive_env_int("RAGOPS_MAX_REQUEST_BYTES", DEFAULT_MAX_REQUEST_BYTES) + content_length = request.headers.get("content-length") + if content_length: + try: + declared = int(content_length) + except ValueError: + response = JSONResponse(status_code=400, content={"detail": "Invalid Content-Length"}) + else: + if declared > maximum: + response = JSONResponse(status_code=413, content={"detail": "Request body is too large"}) + if response is None: + body = await request.body() + if len(body) > maximum: + response = JSONResponse(status_code=413, content={"detail": "Request body is too large"}) + if response is None: + response = await call_next(request) + for name, value in SECURITY_HEADERS.items(): + response.headers[name] = value + return response + + +def _positive_env_int(name: str, default: int) -> int: + raw = os.getenv(name) + if raw is None: + return default + try: + value = int(raw) + except ValueError as exc: + raise HTTPException(status_code=503, detail=f"{name} must be a positive integer") from exc + if value <= 0: + raise HTTPException(status_code=503, detail=f"{name} must be a positive integer") + return value + + +def _validate_collection_limits(scenario: dict, *response_sets: list[dict]) -> None: + maximum = _positive_env_int("RAGOPS_MAX_CASES", DEFAULT_MAX_CASES) + cases = scenario.get("cases", []) + if not isinstance(cases, list): + return + if len(cases) > maximum or any(len(responses) > maximum for responses in response_sets): + raise HTTPException( + status_code=413, + detail=f"Scenario and response collections are limited to {maximum} cases", + ) + + +def _validate_replay_limits(*bundles: dict) -> None: + maximum_cases = _positive_env_int("RAGOPS_MAX_CASES", DEFAULT_MAX_CASES) + maximum_observations = _positive_env_int( + "RAGOPS_MAX_REPLAY_OBSERVATIONS", DEFAULT_MAX_REPLAY_OBSERVATIONS + ) + for bundle in bundles: + records = bundle.get("records", []) + if not isinstance(records, list): + continue + case_ids = { + record.get("case_id") for record in records if isinstance(record, dict) + } + if len(records) > maximum_observations or len(case_ids) > maximum_cases: + raise HTTPException( + status_code=413, + detail=( + f"Replay bundles are limited to {maximum_cases} cases and " + f"{maximum_observations} observations" + ), + ) + + +@app.get("/health") +def health() -> dict[str, str]: + return {"status": "ok", "version": __version__} + + +@app.get("/", include_in_schema=False) +def dashboard() -> FileResponse: + return FileResponse(WEB_ROOT / "index.html") + + +@app.get("/assets/{asset_name}", include_in_schema=False) +def workbench_asset(asset_name: str) -> FileResponse: + if asset_name not in {"app.js", "styles.css"}: + raise HTTPException(status_code=404, detail="Unknown workbench asset") + return FileResponse(WEB_ROOT / asset_name) + + +@app.post("/v1/evaluate", dependencies=[Depends(require_api_key)]) +def evaluate_endpoint(payload: EvaluateRequest) -> dict: + try: + _validate_collection_limits(payload.scenario, payload.responses) + scenario = scenario_from_dict(payload.scenario) + responses = responses_from_data(payload.responses) + return evaluate(scenario, responses).to_dict() + except (ContractError, KeyError, TypeError, ValueError) as exc: + raise HTTPException(status_code=422, detail=str(exc)) from exc + + +@app.post("/v1/compare", dependencies=[Depends(require_api_key)]) +def compare_endpoint(payload: CompareRequest) -> dict: + try: + _validate_collection_limits(payload.scenario, payload.baseline, payload.candidate) + scenario = scenario_from_dict(payload.scenario) + baseline = responses_from_data(payload.baseline) + candidate = responses_from_data(payload.candidate) + return compare(scenario, baseline, candidate).to_dict() + except (ContractError, KeyError, TypeError, ValueError) as exc: + raise HTTPException(status_code=422, detail=str(exc)) from exc + + +@app.post("/v1/statistical/compare", dependencies=[Depends(require_api_key)]) +def statistical_compare_endpoint(payload: StatisticalCompareRequest) -> dict: + try: + _validate_replay_limits(payload.baseline, payload.candidate) + return compare_replay_bundles( + replay_bundle_from_dict(payload.baseline), + replay_bundle_from_dict(payload.candidate), + _statistical_policy(payload.policy), + ).to_dict() + except (ContractError, KeyError, TypeError, ValueError) as exc: + raise HTTPException(status_code=422, detail=str(exc)) from exc + + +@app.post("/v1/statistical/sequential", dependencies=[Depends(require_api_key)]) +def sequential_compare_endpoint(payload: StatisticalCompareRequest) -> dict: + try: + _validate_replay_limits(payload.baseline, payload.candidate) + return compare_replay_bundles_sequentially( + replay_bundle_from_dict(payload.baseline), + replay_bundle_from_dict(payload.candidate), + _sequential_policy(payload.policy), + ).to_dict() + except (ContractError, KeyError, TypeError, ValueError) as exc: + raise HTTPException(status_code=422, detail=str(exc)) from exc + + +@app.post("/v1/evaluator-drift", dependencies=[Depends(require_api_key)]) +def evaluator_drift_endpoint(payload: EvaluatorDriftRequest) -> dict: + try: + _validate_replay_limits(payload.reference, payload.current) + return detect_evaluator_drift( + replay_bundle_from_dict(payload.reference), + replay_bundle_from_dict(payload.current), + _drift_policy(payload.policy), + ).to_dict() + except (ContractError, KeyError, TypeError, ValueError) as exc: + raise HTTPException(status_code=422, detail=str(exc)) from exc + + +@app.get("/v1/runs", dependencies=[Depends(require_api_key)]) +def list_runs(limit: int = 20, store: ExperimentStore = Depends(configured_store)) -> list[dict]: + try: + return store.list_runs(limit=limit) + except ValueError as exc: + raise HTTPException(status_code=422, detail=str(exc)) from exc + + +@app.get("/v1/trends/{scenario_id}/{metric}", dependencies=[Depends(require_api_key)]) +def metric_trend( + scenario_id: str, + metric: str, + limit: int = 50, + store: ExperimentStore = Depends(configured_store), +) -> list[dict]: + try: + return store.metric_trend(scenario_id, metric, limit=limit) + except ValueError as exc: + raise HTTPException(status_code=422, detail=str(exc)) from exc + + +@app.post("/v1/runs/{run_id}/review", dependencies=[Depends(require_api_key)]) +def review_run( + run_id: str, + payload: ReviewRequest, + store: ExperimentStore = Depends(configured_store), +) -> dict[str, str]: + try: + store.review( + run_id, + status=payload.status, + reviewer=payload.reviewer, + note=payload.note, + ) + except KeyError as exc: + raise HTTPException(status_code=404, detail=str(exc)) from exc + except ValueError as exc: + raise HTTPException(status_code=422, detail=str(exc)) from exc + return {"run_id": run_id, "review_status": payload.status} + + +@app.post("/v1/runs/evaluate", dependencies=[Depends(require_api_key)]) +def evaluate_and_save( + payload: SavedEvaluateRequest, + store: ExperimentStore = Depends(configured_store), +) -> dict: + try: + _validate_collection_limits(payload.scenario, payload.responses) + scenario = scenario_from_dict(payload.scenario) + responses = responses_from_data(payload.responses) + report = evaluate(scenario, responses) + except (ContractError, KeyError, TypeError, ValueError) as exc: + raise HTTPException(status_code=422, detail=str(exc)) from exc + run_id = store.save(report, label=payload.label) + return {"run_id": run_id, "report": report.to_dict()} + + +def _statistical_metric_gates(raw: object) -> dict[str, StatisticalMetricGate]: + if not isinstance(raw, dict) or not raw: + raise ContractError("Statistical API policy metrics must be a non-empty object") + gates = {} + for name, value in raw.items(): + if not isinstance(name, str) or not name or not isinstance(value, dict): + raise ContractError("Statistical API metric gates must be named objects") + if set(value) - {"direction", "minimum", "maximum", "max_regression"}: + raise ContractError(f"Statistical API metric {name!r} has unknown fields") + gates[name] = StatisticalMetricGate( + direction=value["direction"], + minimum=value.get("minimum"), + maximum=value.get("maximum"), + max_regression=value["max_regression"], + ) + return gates + + +def _statistical_policy(raw: dict) -> StatisticalPolicy: + expected = {"confidence", "minimum_cases", "resamples", "seed", "metrics"} + if set(raw) != expected: + raise ContractError("Statistical API policy fields do not match the fixed policy contract") + return StatisticalPolicy( + confidence=raw["confidence"], + minimum_cases=raw["minimum_cases"], + resamples=raw["resamples"], + seed=raw["seed"], + metric_gates=_statistical_metric_gates(raw["metrics"]), + ) + + +def _sequential_policy(raw: dict) -> SequentialPolicy: + expected = { + "confidence", + "minimum_cases", + "minimum_repeats", + "maximum_repeats", + "look_every", + "resamples", + "seed", + "metrics", + } + if set(raw) != expected: + raise ContractError("Statistical API policy fields do not match the sequential contract") + return SequentialPolicy( + confidence=raw["confidence"], + minimum_cases=raw["minimum_cases"], + minimum_repeats=raw["minimum_repeats"], + maximum_repeats=raw["maximum_repeats"], + look_every=raw["look_every"], + resamples=raw["resamples"], + seed=raw["seed"], + metric_gates=_statistical_metric_gates(raw["metrics"]), + ) + + +def _drift_policy(raw: dict) -> EvaluatorDriftPolicy: + expected = {"confidence", "minimum_cases", "resamples", "seed", "metrics"} + if set(raw) != expected or not isinstance(raw["metrics"], dict): + raise ContractError("Evaluator drift API policy fields do not match the contract") + gates = {} + for name, value in raw["metrics"].items(): + if not isinstance(value, dict) or set(value) != {"max_absolute_change"}: + raise ContractError( + f"Evaluator drift API metric {name!r} must define max_absolute_change" + ) + gates[name] = EvaluatorDriftMetricGate(value["max_absolute_change"]) + return EvaluatorDriftPolicy( + confidence=raw["confidence"], + minimum_cases=raw["minimum_cases"], + resamples=raw["resamples"], + seed=raw["seed"], + metric_gates=gates, + ) diff --git a/src/ragops/calibration.py b/src/ragops/calibration.py new file mode 100644 index 0000000..3886611 --- /dev/null +++ b/src/ragops/calibration.py @@ -0,0 +1,256 @@ +from __future__ import annotations + +import itertools +import json +import math +from dataclasses import asdict, dataclass +from pathlib import Path +from statistics import fmean +from typing import Any + +from ragops.loader import ContractError + + +@dataclass(frozen=True) +class HumanLabel: + reviewer: str + label: int + + +@dataclass(frozen=True) +class CalibrationRecord: + case_id: str + automated_score: float + human_labels: tuple[HumanLabel, ...] + + +@dataclass(frozen=True) +class CalibrationPolicy: + threshold: float + minimum_coverage: float + minimum_accuracy: float + maximum_mae: float + maximum_false_pass_rate: float + minimum_agreement: float + + +@dataclass(frozen=True) +class CalibrationSet: + schema_version: str + evaluator: str + records: tuple[CalibrationRecord, ...] + policy: CalibrationPolicy + + +@dataclass(frozen=True) +class CalibrationReport: + schema_version: str + evaluator: str + decision: str + case_count: int + labeled_case_count: int + coverage: float + accuracy: float + mae: float + false_pass_rate: float + false_block_rate: float + reviewer_agreement: float | None + reviewer_pair_count: int + failed_gates: tuple[str, ...] + + def to_dict(self) -> dict[str, Any]: + return asdict(self) + + +def load_calibration_set(path: str | Path) -> CalibrationSet: + try: + data = json.loads( + Path(path).read_text(encoding="utf-8"), + parse_constant=lambda value: (_ for _ in ()).throw( + ValueError(f"non-finite JSON number {value}") + ), + ) + except (OSError, json.JSONDecodeError, ValueError) as exc: + raise ContractError(f"Cannot load calibration set from {path}: {exc}") from exc + if not isinstance(data, dict) or set(data) != { + "schema_version", + "evaluator", + "policy", + "records", + }: + raise ContractError("Calibration set fields do not match schema 1.0") + if data["schema_version"] != "1.0": + raise ContractError(f"Unsupported calibration set schema: {data['schema_version']}") + evaluator = data["evaluator"] + if not isinstance(evaluator, str) or not evaluator: + raise ContractError("Calibration evaluator must be a non-empty string") + raw_policy = data["policy"] + policy_fields = { + "threshold", + "minimum_coverage", + "minimum_accuracy", + "maximum_mae", + "maximum_false_pass_rate", + "minimum_agreement", + } + if not isinstance(raw_policy, dict) or set(raw_policy) != policy_fields: + raise ContractError("Calibration policy fields do not match schema 1.0") + policy = CalibrationPolicy(**raw_policy) + raw_records = data["records"] + if not isinstance(raw_records, list) or not raw_records: + raise ContractError("Calibration records must be a non-empty array") + records = [] + for index, raw_record in enumerate(raw_records): + if not isinstance(raw_record, dict) or set(raw_record) != { + "case_id", + "automated_score", + "human_labels", + }: + raise ContractError(f"Calibration record {index} fields do not match schema 1.0") + raw_labels = raw_record["human_labels"] + if not isinstance(raw_labels, list): + raise ContractError(f"Calibration record {index} human_labels must be an array") + labels = [] + for label_index, raw_label in enumerate(raw_labels): + if not isinstance(raw_label, dict) or set(raw_label) != {"reviewer", "label"}: + raise ContractError( + f"Calibration record {index} label {label_index} fields are invalid" + ) + labels.append(HumanLabel(raw_label["reviewer"], raw_label["label"])) + records.append( + CalibrationRecord( + raw_record["case_id"], raw_record["automated_score"], tuple(labels) + ) + ) + calibration_set = CalibrationSet("1.0", evaluator, tuple(records), policy) + _validate_inputs(calibration_set.evaluator, calibration_set.records, calibration_set.policy) + return calibration_set + + +def calibrate_evaluator( + evaluator: str, + records: tuple[CalibrationRecord, ...], + policy: CalibrationPolicy, +) -> CalibrationReport: + _validate_inputs(evaluator, records, policy) + labeled = tuple(record for record in records if record.human_labels) + coverage = len(labeled) / len(records) + human_scores = { + record.case_id: fmean(label.label for label in record.human_labels) for record in labeled + } + human_binary = { + case_id: int(score >= policy.threshold) for case_id, score in human_scores.items() + } + automated_binary = { + record.case_id: int(record.automated_score >= policy.threshold) for record in labeled + } + accuracy = ( + sum(automated_binary[item.case_id] == human_binary[item.case_id] for item in labeled) + / len(labeled) + if labeled + else 0.0 + ) + mae = ( + fmean(abs(item.automated_score - human_scores[item.case_id]) for item in labeled) + if labeled + else 1.0 + ) + false_passes = sum( + automated_binary[item.case_id] == 1 and human_binary[item.case_id] == 0 + for item in labeled + ) + false_blocks = sum( + automated_binary[item.case_id] == 0 and human_binary[item.case_id] == 1 + for item in labeled + ) + denominator = len(labeled) if labeled else 1 + agreement, pair_count = _reviewer_agreement(records) + failed = [] + if coverage < policy.minimum_coverage: + failed.append("coverage") + if accuracy < policy.minimum_accuracy: + failed.append("accuracy") + if mae > policy.maximum_mae: + failed.append("mae") + if false_passes / denominator > policy.maximum_false_pass_rate: + failed.append("false_pass_rate") + if agreement is None: + if policy.minimum_agreement > 0: + failed.append("reviewer_agreement_insufficient") + elif agreement < policy.minimum_agreement: + failed.append("reviewer_agreement") + return CalibrationReport( + schema_version="1.0", + evaluator=evaluator, + decision="PASS" if not failed else "BLOCK", + case_count=len(records), + labeled_case_count=len(labeled), + coverage=coverage, + accuracy=accuracy, + mae=mae, + false_pass_rate=false_passes / denominator, + false_block_rate=false_blocks / denominator, + reviewer_agreement=agreement, + reviewer_pair_count=pair_count, + failed_gates=tuple(failed), + ) + + +def _reviewer_agreement( + records: tuple[CalibrationRecord, ...], +) -> tuple[float | None, int]: + by_reviewer: dict[str, dict[str, int]] = {} + for record in records: + for label in record.human_labels: + by_reviewer.setdefault(label.reviewer, {})[record.case_id] = label.label + kappas = [] + for first, second in itertools.combinations(sorted(by_reviewer), 2): + shared = sorted(set(by_reviewer[first]) & set(by_reviewer[second])) + if len(shared) < 2: + continue + first_values = [by_reviewer[first][case_id] for case_id in shared] + second_values = [by_reviewer[second][case_id] for case_id in shared] + observed = sum(a == b for a, b in zip(first_values, second_values)) / len(shared) + first_yes = sum(first_values) / len(shared) + second_yes = sum(second_values) / len(shared) + expected = first_yes * second_yes + (1 - first_yes) * (1 - second_yes) + kappa = 1.0 if expected == 1.0 and observed == 1.0 else (observed - expected) / (1 - expected) + kappas.append(kappa) + return (fmean(kappas), len(kappas)) if kappas else (None, 0) + + +def _validate_inputs( + evaluator: str, + records: tuple[CalibrationRecord, ...], + policy: CalibrationPolicy, +) -> None: + if not isinstance(evaluator, str) or not evaluator: + raise ContractError("Calibration evaluator must be a non-empty string") + if not records: + raise ContractError("Calibration requires at least one record") + case_ids = [record.case_id for record in records] + if any(not isinstance(case_id, str) or not case_id for case_id in case_ids): + raise ContractError("Calibration case IDs must be non-empty strings") + if len(set(case_ids)) != len(case_ids): + raise ContractError("Calibration case IDs must be unique") + for record in records: + _ratio(record.automated_score, f"Calibration score for {record.case_id}") + reviewers = [label.reviewer for label in record.human_labels] + if len(set(reviewers)) != len(reviewers): + raise ContractError(f"Calibration case {record.case_id} has duplicate reviewer labels") + for label in record.human_labels: + if not isinstance(label.reviewer, str) or not label.reviewer: + raise ContractError("Calibration reviewer must be a non-empty string") + if isinstance(label.label, bool) or label.label not in {0, 1}: + raise ContractError("Calibration human label must be 0 or 1") + for field_name, value in asdict(policy).items(): + _ratio(value, f"Calibration policy {field_name}") + + +def _ratio(value: object, label: str) -> float: + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise ContractError(f"{label} must be a number between 0 and 1") + number = float(value) + if not math.isfinite(number) or not 0 <= number <= 1: + raise ContractError(f"{label} must be between 0 and 1") + return number diff --git a/src/ragops/ci.py b/src/ragops/ci.py new file mode 100644 index 0000000..d0cad5d --- /dev/null +++ b/src/ragops/ci.py @@ -0,0 +1,116 @@ +from __future__ import annotations + +import json +import xml.etree.ElementTree as ET +from typing import Any + +from ragops.policy_v2 import GateEvidence, ReleaseDecision + + +def junit_xml(decision: ReleaseDecision) -> str: + failures = sum(not gate.passed and not gate.waived and gate.severity == "block" for gate in decision.gates) + skipped = sum(not gate.passed and (gate.waived or gate.severity == "warn") for gate in decision.gates) + suite = ET.Element( + "testsuite", + { + "name": f"ragops: {decision.scenario_id}", + "tests": str(len(decision.gates)), + "failures": str(failures), + "skipped": str(skipped), + }, + ) + for gate in decision.gates: + case = ET.SubElement(suite, "testcase", {"name": gate.id, "classname": "ragops.release_gate"}) + message = _message(gate) + if not gate.passed and gate.waived: + ET.SubElement(case, "skipped", {"message": f"waived: {message}"}) + elif not gate.passed and gate.severity == "warn": + ET.SubElement(case, "skipped", {"message": f"warning: {message}"}) + elif not gate.passed: + ET.SubElement(case, "failure", {"message": message, "type": gate.policy_path}).text = message + ET.indent(suite) + return ET.tostring(suite, encoding="unicode", xml_declaration=True) + "\n" + + +def github_summary(decision: ReleaseDecision, *, max_gates: int = 50) -> str: + max_gates = max(1, min(max_gates, 100)) + lines = [ + f"## RAGOps release gate: {_markdown(decision.decision)}", + "", + f"Scenario: `{_markdown(decision.scenario_id)}`", + "", + "| Gate | Scope | Observed | Threshold | Result |", + "| --- | --- | ---: | ---: | --- |", + ] + for gate in decision.gates[:max_gates]: + result = "PASS" if gate.passed else "WAIVED" if gate.waived else gate.severity.upper() + lines.append( + f"| `{_markdown(gate.id)}` | `{_markdown(gate.scope)}` | {gate.observed:.6g} | " + f"{gate.comparator} {gate.threshold:.6g} | {result} |" + ) + omitted = len(decision.gates) - max_gates + if omitted > 0: + lines.extend(["", f"{omitted} additional gates omitted."]) + return "\n".join(lines) + "\n" + + +def sarif_report(decision: ReleaseDecision) -> dict[str, Any]: + failed = tuple(gate for gate in decision.gates if not gate.passed) + rules = [ + { + "id": gate.id, + "name": _sarif_name(gate.id), + "shortDescription": {"text": gate.policy_path}, + "properties": {"severity": gate.severity, "scope": gate.scope}, + } + for gate in failed + ] + results = [ + { + "ruleId": gate.id, + "level": "note" if gate.waived else "error" if gate.severity == "block" else "warning", + "message": {"text": _message(gate)}, + "properties": { + "policyPath": gate.policy_path, + "caseIds": list(gate.case_ids), + "waived": gate.waived, + }, + } + for gate in failed + ] + return { + "$schema": "https://json.schemastore.org/sarif-2.1.0.json", + "version": "2.1.0", + "runs": [ + { + "tool": {"driver": {"name": "RAGOps", "informationUri": "https://github.com/thangldw/ragops", "rules": rules}}, + "results": results, + } + ], + } + + +def render_ci(decision: ReleaseDecision, output_format: str) -> str: + if output_format == "junit": + return junit_xml(decision) + if output_format == "github": + return github_summary(decision) + if output_format == "sarif": + return json.dumps(sarif_report(decision), ensure_ascii=False, indent=2) + "\n" + raise ValueError(f"Unsupported CI format: {output_format}") + + +def _message(gate: GateEvidence) -> str: + return ( + f"{gate.policy_path}: observed {gate.observed:.6g} " + f"must be {gate.comparator} {gate.threshold:.6g}; cases={','.join(gate.case_ids) or 'none'}" + ) + + +def _markdown(value: str) -> str: + return value.replace("\\", "\\\\").replace("|", "\\|").replace("`", "\\`").replace("\n", " ") + + +def _sarif_name(value: str) -> str: + result = "".join(character if character.isalnum() else "_" for character in value) + return result[:128] or "ragops_gate" diff --git a/src/ragops/cli.py b/src/ragops/cli.py index b13b271..c9553a0 100644 --- a/src/ragops/cli.py +++ b/src/ragops/cli.py @@ -2,13 +2,17 @@ import argparse import json +from datetime import UTC, datetime from pathlib import Path from ragops import __version__ +from ragops.adapter_sdk import AdapterContext, discover_adapters from ragops.adapters.external_metrics import ( load_external_metric_evaluator, validate_external_metric_pair, ) +from ragops.adapters.import_profiles import convert_import +from ragops.adapters.otel_genai import otel_spans_to_trace_graph from ragops.adapters.repeated_runs import ( CommandMetricAdapter, collect_repeated_runs, @@ -24,6 +28,8 @@ write_baseline_manifest, ) from ragops.benchmarks import scenario_summary +from ragops.calibration import calibrate_evaluator, load_calibration_set +from ragops.ci import render_ci from ragops.config import ( load_evaluation_policy, load_evaluator_drift_policy, @@ -31,10 +37,21 @@ load_sequential_policy, load_statistical_policy, ) +from ragops.contracts import diff_contract, migrate_contract, validate_contract from ragops.control_plane import ControlPlane +from ragops.datasets import ( + create_dataset_manifest, + diff_dataset_manifests, + load_dataset_manifest, + validate_dataset_manifest, + write_dataset_manifest, +) from ragops.demo import DEFAULT_DEMO_SCENARIO, DEMO_BUNDLES, write_demo from ragops.drift import detect_evaluator_drift from ragops.engine import compare, evaluate +from ragops.evidence import create_evidence_bundle, verify_evidence_bundle +from ragops.explain import explain_decision +from ragops.governance import GovernanceStore from ragops.loader import ContractError, load_responses, load_scenario from ragops.pilot import ( PilotContractError, @@ -53,6 +70,7 @@ RetrievalRecallEvaluator, SourceFreshnessEvaluator, ) +from ragops.policy_v2 import apply_release_policy, load_release_decision, load_release_policy_v2 from ragops.provenance import diagnose_provenance from ragops.reporters import ( comparison_html, @@ -65,6 +83,7 @@ from ragops.sequential import compare_replay_bundles_sequentially from ragops.statistical import compare_replay_bundles, load_replay_bundle from ragops.store import ExperimentStore +from ragops.trace_graph import evaluate_trace_graph, load_trace_expectation, load_trace_graph from ragops.traces import load_trace_jsonl @@ -72,6 +91,117 @@ def build_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser(prog="ragops") parser.add_argument("--version", action="version", version=f"%(prog)s {__version__}") commands = parser.add_subparsers(dest="command", required=True) + contract_parser = commands.add_parser("contract", help="Validate and migrate contracts") + contract_commands = contract_parser.add_subparsers(dest="contract_command", required=True) + contract_validate = contract_commands.add_parser("validate", help="Validate a JSON contract") + contract_validate.add_argument("--kind", choices=("auto", "scenario", "trace"), default="auto") + contract_validate.add_argument("--input", required=True) + contract_diff = contract_commands.add_parser("diff", help="Compare contract versions") + contract_diff.add_argument("--kind", choices=("scenario", "trace"), required=True) + contract_diff.add_argument("--from", dest="from_version", required=True) + contract_diff.add_argument("--to", dest="to_version", required=True) + contract_migrate = contract_commands.add_parser("migrate", help="Migrate a JSON contract") + contract_migrate.add_argument("--kind", choices=("scenario", "trace"), required=True) + contract_migrate.add_argument("--from", dest="from_version", required=True) + contract_migrate.add_argument("--to", dest="to_version", required=True) + contract_migrate.add_argument("--input", required=True) + contract_migrate.add_argument("--output", required=True) + contract_migrate.add_argument("--question", default="") + evidence_parser = commands.add_parser("evidence", help="Create and verify evidence bundles") + evidence_commands = evidence_parser.add_subparsers(dest="evidence_command", required=True) + evidence_create = evidence_commands.add_parser("create", help="Create an evidence bundle") + evidence_create.add_argument("--bundle", required=True) + evidence_create.add_argument("--decision", choices=("PASS", "WARN", "BLOCK"), required=True) + evidence_create.add_argument("--artifact", action="append", required=True, metavar="NAME=PATH") + evidence_create.add_argument("--limitation", action="append", required=True) + evidence_create.add_argument("--created-at") + evidence_create.add_argument("--metadata-json", default="{}") + evidence_verify = evidence_commands.add_parser("verify", help="Verify an evidence bundle") + evidence_verify.add_argument("--bundle", required=True) + gate_v2 = commands.add_parser("gate-v2", help="Apply slice-aware release policy v2") + gate_v2.add_argument("--scenario", required=True) + gate_v2.add_argument("--baseline", required=True) + gate_v2.add_argument("--candidate", required=True) + gate_v2.add_argument("--policy", required=True) + gate_v2.add_argument("--now") + gate_v2.add_argument("--output") + calibrate = commands.add_parser("calibrate", help="Calibrate an evaluator against human labels") + calibrate.add_argument("--input", required=True) + calibrate.add_argument("--output") + trace_parser = commands.add_parser("trace", help="Convert and evaluate agent trace graphs") + trace_commands = trace_parser.add_subparsers(dest="trace_command", required=True) + trace_evaluate = trace_commands.add_parser("evaluate", help="Evaluate a trace graph") + trace_evaluate.add_argument("--trace", required=True) + trace_evaluate.add_argument("--expectation", required=True) + trace_evaluate.add_argument("--output") + trace_convert = trace_commands.add_parser("convert-otel", help="Convert OpenTelemetry JSON") + trace_convert.add_argument("--input", required=True) + trace_convert.add_argument("--output", required=True) + dataset_parser = commands.add_parser("dataset", help="Manage benchmark dataset manifests") + dataset_commands = dataset_parser.add_subparsers(dest="dataset_command", required=True) + dataset_create = dataset_commands.add_parser("create", help="Create a dataset manifest") + dataset_create.add_argument("--scenario", required=True) + dataset_create.add_argument("--dataset-id", required=True) + dataset_create.add_argument("--version", required=True) + dataset_create.add_argument( + "--source-classification", choices=("synthetic", "public", "production-derived"), required=True + ) + dataset_create.add_argument("--owner", action="append", required=True) + dataset_create.add_argument("--split", action="append", required=True, metavar="NAME=ID,ID") + dataset_create.add_argument("--reviewed", action="store_true") + dataset_create.add_argument("--output", required=True) + dataset_validate = dataset_commands.add_parser("validate", help="Validate a dataset manifest") + dataset_validate.add_argument("--manifest", required=True) + dataset_validate.add_argument("--minimum-slice", action="append", default=[], metavar="SELECTOR=COUNT") + dataset_diff = dataset_commands.add_parser("diff", help="Diff dataset manifests") + dataset_diff.add_argument("--before", required=True) + dataset_diff.add_argument("--after", required=True) + explain_parser = commands.add_parser("explain", help="Explain failed release gates") + explain_parser.add_argument("--input", required=True) + explain_parser.add_argument("--output") + ci_render = commands.add_parser("ci-render", help="Render release decisions for CI systems") + ci_render.add_argument("--input", required=True) + ci_render.add_argument("--format", choices=("junit", "sarif", "github"), required=True) + ci_render.add_argument("--output", required=True) + adapter_parser = commands.add_parser("adapter", help="Discover and run portable import adapters") + adapter_commands = adapter_parser.add_subparsers(dest="adapter_command", required=True) + adapter_commands.add_parser("list", help="List built-in and installed adapters") + adapter_convert = adapter_commands.add_parser("convert", help="Convert vendor JSON to a portable envelope") + adapter_convert.add_argument("--profile", required=True) + adapter_convert.add_argument("--input", required=True) + adapter_convert.add_argument("--case-id", action="append", default=[]) + adapter_convert.add_argument("--output", required=True) + governance = commands.add_parser("governance", help="Manage governed artifacts and blind reviews") + governance_commands = governance.add_subparsers(dest="governance_command", required=True) + governance_register = governance_commands.add_parser("register") + governance_register.add_argument("--store", required=True) + governance_register.add_argument("--artifact-id", required=True) + governance_register.add_argument("--kind", required=True) + governance_register.add_argument("--digest", required=True) + governance_register.add_argument("--metadata", required=True) + governance_register.add_argument("--blinded-metadata") + governance_register.add_argument("--actor", required=True) + governance_transition = governance_commands.add_parser("transition") + governance_transition.add_argument("--store", required=True) + governance_transition.add_argument("--artifact-id", required=True) + governance_transition.add_argument("--state", choices=("reviewed", "accepted", "superseded"), required=True) + governance_transition.add_argument("--actor", required=True) + governance_review = governance_commands.add_parser("review") + governance_review.add_argument("--store", required=True) + governance_review.add_argument("--artifact-id", required=True) + governance_review.add_argument("--reviewer", required=True) + governance_review.add_argument("--verdict", choices=("approve", "block"), required=True) + governance_review.add_argument("--note", default="") + governance_queue = governance_commands.add_parser("queue") + governance_queue.add_argument("--store", required=True) + governance_queue.add_argument("--reviewer", required=True) + governance_audit = governance_commands.add_parser("audit") + governance_audit.add_argument("--store", required=True) + governance_audit.add_argument("--artifact-id") + serve = commands.add_parser("serve", help="Run the optional local API and workbench") + serve.add_argument("--host", default="127.0.0.1") + serve.add_argument("--port", type=int, default=8000) + serve.add_argument("--reload", action="store_true") demo_parser = commands.add_parser("demo", help="Generate a credential-free release-gate demo") demo_parser.add_argument("--output", default="ragops-demo") demo_parser.add_argument( @@ -80,6 +210,9 @@ def build_parser() -> argparse.ArgumentParser: default=DEFAULT_DEMO_SCENARIO, help="Credential-free workflow scenario to generate", ) + demo_parser.add_argument( + "--profile", choices=("executive", "engineer", "auditor"), default="engineer" + ) demo_parser.add_argument( "--force", action="store_true", @@ -280,9 +413,261 @@ def build_parser() -> argparse.ArgumentParser: def main() -> int: args = build_parser().parse_args() + if args.command == "contract": + try: + if args.contract_command == "diff": + payload = diff_contract(args.kind, args.from_version, args.to_version).to_dict() + else: + source = Path(args.input) + data = json.loads(source.read_text(encoding="utf-8")) + if args.contract_command == "validate": + payload = validate_contract(data, args.kind).to_dict() + else: + output = Path(args.output) + if source.resolve() == output.resolve(): + raise SystemExit("contract input and output must differ") + migrated = migrate_contract( + data, + args.kind, + args.from_version, + args.to_version, + question=args.question, + ) + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text( + json.dumps(migrated, ensure_ascii=False, indent=2) + "\n", + encoding="utf-8", + ) + payload = { + "kind": args.kind, + "from_version": args.from_version, + "to_version": args.to_version, + "output": str(output), + } + except (ContractError, OSError, json.JSONDecodeError) as exc: + raise SystemExit(f"contract error: {exc}") from exc + print(json.dumps(payload, ensure_ascii=False, indent=2)) + return 0 + if args.command == "evidence": + try: + if args.evidence_command == "verify": + manifest = verify_evidence_bundle(args.bundle) + payload = {"verified": True, **manifest.to_dict()} + else: + artifact_paths = {} + for value in args.artifact: + if "=" not in value: + raise ContractError("Evidence artifacts must use NAME=PATH") + name, raw_path = value.split("=", 1) + if name in artifact_paths: + raise ContractError(f"Duplicate evidence artifact name: {name}") + artifact_paths[name] = raw_path + created_at = args.created_at or datetime.now(UTC).isoformat().replace("+00:00", "Z") + metadata = json.loads(args.metadata_json) + manifest = create_evidence_bundle( + args.bundle, + artifact_paths, + args.decision, + tuple(args.limitation), + created_at, + metadata, + ) + payload = manifest.to_dict() + except (ContractError, FileExistsError, OSError, json.JSONDecodeError) as exc: + raise SystemExit(f"evidence error: {exc}") from exc + print(json.dumps(payload, ensure_ascii=False, indent=2)) + return 0 + if args.command == "gate-v2": + try: + scenario = load_scenario(args.scenario) + report = apply_release_policy( + load_release_policy_v2(args.policy), + evaluate(scenario, load_responses(args.baseline)), + evaluate(scenario, load_responses(args.candidate)), + scenario, + now=args.now or datetime.now(UTC).isoformat().replace("+00:00", "Z"), + ) + except (ContractError, OSError, ValueError) as exc: + raise SystemExit(f"release policy error: {exc}") from exc + rendered = json.dumps(report.to_dict(), ensure_ascii=False, indent=2) + "\n" + if args.output: + output = Path(args.output) + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text(rendered, encoding="utf-8") + else: + print(rendered, end="") + return 2 if report.decision == "BLOCK" else 0 + if args.command == "calibrate": + try: + calibration_set = load_calibration_set(args.input) + report = calibrate_evaluator( + calibration_set.evaluator, + calibration_set.records, + calibration_set.policy, + ) + except (ContractError, OSError, ValueError) as exc: + raise SystemExit(f"calibration error: {exc}") from exc + rendered = json.dumps(report.to_dict(), ensure_ascii=False, indent=2) + "\n" + if args.output: + output = Path(args.output) + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text(rendered, encoding="utf-8") + print(rendered, end="") + return 0 if report.decision == "PASS" else 2 + if args.command == "trace": + try: + if args.trace_command == "convert-otel": + raw_spans = json.loads(Path(args.input).read_text(encoding="utf-8")) + graph = otel_spans_to_trace_graph(raw_spans) + output = Path(args.output) + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text( + json.dumps(graph.to_dict(), ensure_ascii=False, indent=2) + "\n", + encoding="utf-8", + ) + payload = {"converted": True, "trace_id": graph.trace_id, "output": str(output)} + exit_code = 0 + else: + report = evaluate_trace_graph( + load_trace_graph(args.trace), load_trace_expectation(args.expectation) + ) + payload = report.to_dict() + rendered = json.dumps(payload, ensure_ascii=False, indent=2) + "\n" + if args.output: + output = Path(args.output) + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text(rendered, encoding="utf-8") + exit_code = 0 if report.decision == "PASS" else 2 + except (ContractError, OSError, json.JSONDecodeError, ValueError) as exc: + raise SystemExit(f"trace error: {exc}") from exc + print(json.dumps(payload, ensure_ascii=False, indent=2)) + return exit_code + if args.command == "dataset": + try: + if args.dataset_command == "create": + scenario = load_scenario(args.scenario) + splits = _named_csv_values(args.split, "dataset split") + cases = tuple( + { + "id": item.id, + "question": item.question, + "evidence": list(item.evidence), + "required_citation_ids": list(item.required_citation_ids), + "category": item.category, + "severity": item.severity, + "language": item.language, + "tags": list(item.tags), + "source": f"scenario:{scenario.id}", + "reviewed": args.reviewed, + } + for item in scenario.cases + ) + manifest = create_dataset_manifest( + args.dataset_id, + args.version, + cases, + splits=splits, + source_classification=args.source_classification, + owners=tuple(args.owner), + ) + write_dataset_manifest(args.output, manifest) + payload = manifest.to_dict() + exit_code = 0 + elif args.dataset_command == "validate": + minimum_slices = _named_int_values(args.minimum_slice, "dataset minimum slice") + manifest = load_dataset_manifest(args.manifest) + issues = validate_dataset_manifest(manifest, minimum_slice_counts=minimum_slices) + payload = {"valid": not issues, "issues": [item.to_dict() for item in issues]} + exit_code = 0 if not issues else 2 + else: + difference = diff_dataset_manifests( + load_dataset_manifest(args.before), load_dataset_manifest(args.after) + ) + payload = difference.to_dict() + exit_code = 0 + except (ContractError, OSError, ValueError) as exc: + raise SystemExit(f"dataset error: {exc}") from exc + print(json.dumps(payload, ensure_ascii=False, indent=2)) + return exit_code + if args.command in {"explain", "ci-render"}: + try: + decision = load_release_decision(args.input) + if args.command == "explain": + rendered = json.dumps(explain_decision(decision).to_dict(), ensure_ascii=False, indent=2) + "\n" + else: + rendered = render_ci(decision, args.format) + if args.output: + output = Path(args.output) + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text(rendered, encoding="utf-8") + else: + print(rendered, end="") + except (ContractError, OSError, ValueError) as exc: + raise SystemExit(f"release explanation error: {exc}") from exc + return 0 + if args.command == "adapter": + try: + if args.adapter_command == "list": + payload = {"adapters": [item.to_dict() for item in discover_adapters()]} + else: + raw = json.loads(Path(args.input).read_text(encoding="utf-8")) + converted = convert_import( + args.profile, raw, AdapterContext(case_ids=tuple(args.case_id)) + ) + payload = converted.to_dict() + output = Path(args.output) + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text( + json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8" + ) + except (ContractError, OSError, json.JSONDecodeError, ValueError) as exc: + raise SystemExit(f"adapter error: {exc}") from exc + print(json.dumps(payload, ensure_ascii=False, indent=2)) + return 0 + if args.command == "governance": + try: + governance_store = GovernanceStore(args.store) + if args.governance_command == "register": + public = json.loads(Path(args.metadata).read_text(encoding="utf-8")) + blinded = ( + json.loads(Path(args.blinded_metadata).read_text(encoding="utf-8")) + if args.blinded_metadata else {} + ) + governance_store.register_artifact( + args.artifact_id, kind=args.kind, digest=args.digest, + public_metadata=public, blinded_metadata=blinded, actor=args.actor, + ) + payload = governance_store.get_artifact(args.artifact_id) + elif args.governance_command == "transition": + governance_store.transition(args.artifact_id, args.state, actor=args.actor) + payload = governance_store.get_artifact(args.artifact_id) + elif args.governance_command == "review": + governance_store.record_review( + args.artifact_id, reviewer=args.reviewer, verdict=args.verdict, note=args.note + ) + payload = {"recorded": True} + elif args.governance_command == "queue": + payload = {"artifacts": governance_store.review_queue(args.reviewer)} + else: + payload = {"events": governance_store.audit_events(args.artifact_id)} + except (KeyError, ValueError, OSError, json.JSONDecodeError) as exc: + raise SystemExit(f"governance error: {exc}") from exc + print(json.dumps(payload, ensure_ascii=False, indent=2)) + return 0 + if args.command == "serve": + if not 1 <= args.port <= 65535: + raise SystemExit("serve error: port must be between 1 and 65535") + try: + import uvicorn + except ImportError as exc: + raise SystemExit("serve requires the API extra: pip install 'ragops[api]'") from exc + uvicorn.run("ragops.api.main:app", host=args.host, port=args.port, reload=args.reload) + return 0 if args.command == "demo": try: - summary = write_demo(args.output, force=args.force, scenario_id=args.scenario) + summary = write_demo( + args.output, force=args.force, scenario_id=args.scenario, profile=args.profile + ) except (OSError, ValueError) as exc: raise SystemExit(f"demo output error: {exc}") from exc print(json.dumps(summary, ensure_ascii=False, indent=2)) @@ -644,6 +1029,35 @@ def stop_when(current): return 0 if report.passed else 2 +def _named_csv_values(values: list[str], label: str) -> dict[str, tuple[str, ...]]: + result = {} + for value in values: + if "=" not in value: + raise ContractError(f"{label} must use NAME=VALUE,VALUE") + name, raw_items = value.split("=", 1) + items = tuple(item for item in raw_items.split(",") if item) + if not name or not items or name in result: + raise ContractError(f"{label} names must be non-empty and unique") + result[name] = items + return result + + +def _named_int_values(values: list[str], label: str) -> dict[str, int]: + result = {} + for value in values: + if "=" not in value: + raise ContractError(f"{label} must use SELECTOR=COUNT") + selector, raw_count = value.rsplit("=", 1) + try: + count = int(raw_count) + except ValueError as exc: + raise ContractError(f"{label} count must be an integer") from exc + if not selector or selector in result or count <= 0: + raise ContractError(f"{label} selectors must be non-empty, unique, and positive") + result[selector] = count + return result + + def _evaluators_from_names( names: list[str], *, diff --git a/src/ragops/contracts.py b/src/ragops/contracts.py new file mode 100644 index 0000000..95e0627 --- /dev/null +++ b/src/ragops/contracts.py @@ -0,0 +1,236 @@ +from __future__ import annotations + +import copy +import json +import math +import re +from dataclasses import asdict, dataclass +from importlib.resources import files +from pathlib import Path +from typing import Any + +from ragops.loader import ContractError + + +@dataclass(frozen=True) +class ContractDescriptor: + kind: str + version: str + schema_file: str + + def to_dict(self) -> dict[str, str]: + return asdict(self) + + +@dataclass(frozen=True) +class ValidationResult: + kind: str + version: str + schema_file: str + valid: bool = True + + def to_dict(self) -> dict[str, str | bool]: + return asdict(self) + + +@dataclass(frozen=True) +class ContractDiff: + kind: str + from_version: str + to_version: str + compatibility: str + added_required_fields: tuple[str, ...] + removed_required_fields: tuple[str, ...] + + def to_dict(self) -> dict[str, Any]: + return asdict(self) + + +_CONTRACTS = ( + ContractDescriptor("scenario", "0.1", "scenario-0.1.schema.json"), + ContractDescriptor("scenario", "0.2", "scenario-0.2.schema.json"), + ContractDescriptor("trace", "0.3", "trace-0.3.schema.json"), + ContractDescriptor("trace", "0.4", "trace-0.4.schema.json"), +) + + +def registered_contracts() -> tuple[ContractDescriptor, ...]: + return _CONTRACTS + + +def validate_contract(data: object, kind: str = "auto") -> ValidationResult: + if not isinstance(data, dict): + raise ContractError("Contract input must be a JSON object") + resolved_kind = _detect_kind(data) if kind == "auto" else kind + if resolved_kind not in {"scenario", "trace"}: + raise ContractError(f"Unsupported contract kind: {resolved_kind}") + version = data.get("schema_version") + if resolved_kind == "trace" and version is None: + version = "0.3" + if not isinstance(version, str): + raise ContractError(f"{resolved_kind.capitalize()} schema_version must be a string") + descriptor = _descriptor(resolved_kind, version) + schema = _schema_document(descriptor.schema_file) + _validate_schema(data, schema, path=resolved_kind) + return ValidationResult(descriptor.kind, descriptor.version, descriptor.schema_file) + + +def diff_contract(kind: str, from_version: str, to_version: str) -> ContractDiff: + source = _schema_document(_descriptor(kind, from_version).schema_file) + target = _schema_document(_descriptor(kind, to_version).schema_file) + source_required = _required_paths(source) + target_required = _required_paths(target) + added = tuple(sorted(target_required - source_required)) + removed = tuple(sorted(source_required - target_required)) + compatibility = "migration-required" if added or removed else "compatible" + return ContractDiff(kind, from_version, to_version, compatibility, added, removed) + + +def migrate_contract( + data: object, + kind: str, + from_version: str, + to_version: str, + *, + question: str = "", +) -> dict[str, Any]: + if not isinstance(data, dict): + raise ContractError("Contract input must be a JSON object") + validate_contract(data, kind) + actual_version = data.get("schema_version", "0.3" if kind == "trace" else None) + if actual_version != from_version: + raise ContractError( + f"Declared migration source {from_version!r} does not match input {actual_version!r}" + ) + migrated = copy.deepcopy(data) + if (kind, from_version, to_version) == ("scenario", "0.1", "0.2"): + migrated["schema_version"] = "0.2" + migrated["thresholds"].setdefault("citation_precision", 0.0) + for case in migrated["cases"]: + case.setdefault("category", "unspecified") + case.setdefault("severity", "medium") + case.setdefault("language", "und") + elif (kind, from_version, to_version) == ("trace", "0.3", "0.4"): + migrated["schema_version"] = "0.4" + migrated.setdefault("input", {"question": question}) + metadata = migrated.setdefault("metadata", {}) + if not isinstance(metadata, dict): + raise ContractError("Trace metadata must be an object") + metadata["migrated_from"] = "0.3" + else: + raise ContractError( + f"Unsupported {kind} migration: {from_version} -> {to_version}" + ) + validate_contract(migrated, kind) + return migrated + + +def _detect_kind(data: dict[str, Any]) -> str: + candidates = [] + if "thresholds" in data and "cases" in data: + candidates.append("scenario") + if "case_id" in data and "output" in data: + candidates.append("trace") + if len(candidates) != 1: + raise ContractError(f"Cannot identify contract kind; candidates={candidates}") + return candidates[0] + + +def _descriptor(kind: str, version: str) -> ContractDescriptor: + for descriptor in _CONTRACTS: + if descriptor.kind == kind and descriptor.version == version: + return descriptor + raise ContractError(f"Unsupported {kind} schema: {version}") + + +def _schema_document(filename: str) -> dict[str, Any]: + repository_schema = Path(__file__).resolve().parents[2] / "schemas" / filename + try: + if repository_schema.is_file(): + return json.loads(repository_schema.read_text(encoding="utf-8")) + resource = files("ragops").joinpath("schemas", filename) + return json.loads(resource.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise ContractError(f"Cannot load contract schema {filename}: {exc}") from exc + + +def _required_paths(schema: dict[str, Any], prefix: str = "") -> set[str]: + paths: set[str] = set() + required = schema.get("required", []) + properties = schema.get("properties", {}) + if isinstance(required, list): + for name in required: + path = f"{prefix}.{name}" if prefix else str(name) + paths.add(path) + if isinstance(properties, dict): + for name, child in properties.items(): + if not isinstance(child, dict): + continue + path = f"{prefix}.{name}" if prefix else name + paths.update(_required_paths(child, path)) + items = child.get("items") + if isinstance(items, dict): + paths.update(_required_paths(items, f"{path}[]")) + return paths + + +def _validate_schema(value: Any, schema: dict[str, Any], *, path: str) -> None: + if "const" in schema and value != schema["const"]: + raise ContractError(f"{path} must equal {schema['const']!r}") + if "enum" in schema and value not in schema["enum"]: + raise ContractError(f"{path} must be one of {schema['enum']}") + expected = schema.get("type") + if expected is not None and not _matches_type(value, expected): + raise ContractError(f"{path} must have JSON type {expected}") + if isinstance(value, dict): + properties = schema.get("properties", {}) + required = schema.get("required", []) + missing = [name for name in required if name not in value] + if missing: + raise ContractError(f"{path} missing required fields: {missing}") + if schema.get("additionalProperties") is False: + unknown = sorted(set(value) - set(properties)) + if unknown: + raise ContractError(f"{path} has unknown fields: {unknown}") + for name, item in value.items(): + child = properties.get(name) + if isinstance(child, dict): + _validate_schema(item, child, path=f"{path}.{name}") + if isinstance(value, list): + minimum = schema.get("minItems") + if isinstance(minimum, int) and len(value) < minimum: + raise ContractError(f"{path} must contain at least {minimum} items") + if schema.get("uniqueItems") and len({json.dumps(item, sort_keys=True) for item in value}) != len(value): + raise ContractError(f"{path} items must be unique") + child = schema.get("items") + if isinstance(child, dict): + for index, item in enumerate(value): + _validate_schema(item, child, path=f"{path}[{index}]") + if isinstance(value, str): + minimum_length = schema.get("minLength") + if isinstance(minimum_length, int) and len(value) < minimum_length: + raise ContractError(f"{path} must have length at least {minimum_length}") + pattern = schema.get("pattern") + if isinstance(pattern, str) and re.fullmatch(pattern, value) is None: + raise ContractError(f"{path} does not match required pattern") + if isinstance(value, (int, float)) and not isinstance(value, bool): + if not math.isfinite(float(value)): + raise ContractError(f"{path} must be finite") + if "minimum" in schema and value < schema["minimum"]: + raise ContractError(f"{path} must be at least {schema['minimum']}") + if "maximum" in schema and value > schema["maximum"]: + raise ContractError(f"{path} must be at most {schema['maximum']}") + + +def _matches_type(value: Any, expected: str | list[str]) -> bool: + options = [expected] if isinstance(expected, str) else expected + checks = { + "object": lambda item: isinstance(item, dict), + "array": lambda item: isinstance(item, list), + "string": lambda item: isinstance(item, str), + "number": lambda item: isinstance(item, (int, float)) and not isinstance(item, bool), + "integer": lambda item: isinstance(item, int) and not isinstance(item, bool), + "boolean": lambda item: isinstance(item, bool), + "null": lambda item: item is None, + } + return any(checks[option](value) for option in options if option in checks) diff --git a/src/ragops/datasets.py b/src/ragops/datasets.py new file mode 100644 index 0000000..6fcdd24 --- /dev/null +++ b/src/ragops/datasets.py @@ -0,0 +1,363 @@ +from __future__ import annotations + +import hashlib +import json +import re +import unicodedata +from dataclasses import asdict, dataclass +from pathlib import Path +from typing import Any, Iterable, Mapping + +from ragops.loader import ContractError + + +@dataclass(frozen=True) +class DatasetCaseRecord: + id: str + digest: str + normalized_content_digest: str + category: str + severity: str + language: str + tags: tuple[str, ...] + source: str + reviewed: bool + + +@dataclass(frozen=True) +class DatasetManifest: + schema_version: str + dataset_id: str + version: str + source_classification: str + owners: tuple[str, ...] + review_state: str + cases: tuple[DatasetCaseRecord, ...] + splits: dict[str, tuple[str, ...]] + + def to_dict(self) -> dict[str, Any]: + return { + "schema_version": self.schema_version, + "dataset_id": self.dataset_id, + "version": self.version, + "source_classification": self.source_classification, + "owners": list(self.owners), + "review_state": self.review_state, + "cases": [asdict(item) | {"tags": list(item.tags)} for item in self.cases], + "splits": {name: list(case_ids) for name, case_ids in sorted(self.splits.items())}, + } + + +@dataclass(frozen=True) +class DatasetIssue: + code: str + message: str + case_ids: tuple[str, ...] = () + observed: int | None = None + required: int | None = None + + def to_dict(self) -> dict[str, Any]: + return asdict(self) + + +@dataclass(frozen=True) +class DatasetDiff: + dataset_id: str + from_version: str + to_version: str + added_case_ids: tuple[str, ...] + removed_case_ids: tuple[str, ...] + changed_case_ids: tuple[str, ...] + + def to_dict(self) -> dict[str, Any]: + return asdict(self) + + +def create_dataset_manifest( + dataset_id: str, + version: str, + cases: Iterable[Mapping[str, Any]], + *, + splits: Mapping[str, tuple[str, ...]], + source_classification: str, + owners: tuple[str, ...], + review_state: str = "draft", +) -> DatasetManifest: + if not isinstance(dataset_id, str) or not dataset_id: + raise ContractError("Dataset ID must be a non-empty string") + if not isinstance(version, str) or not version: + raise ContractError("Dataset version must be a non-empty string") + raw_cases = tuple(cases) + if not raw_cases: + raise ContractError("Dataset manifest requires at least one case") + records = tuple(_case_record(item, index) for index, item in enumerate(raw_cases)) + identifiers = [item.id for item in records] + if len(set(identifiers)) != len(identifiers): + raise ContractError("Dataset case IDs must be unique") + normalized_splits = _splits(splits, set(identifiers)) + manifest = DatasetManifest( + schema_version="1.0", + dataset_id=dataset_id, + version=version, + source_classification=source_classification, + owners=tuple(owners), + review_state=review_state, + cases=records, + splits=normalized_splits, + ) + _validate_manifest_contract(manifest) + return manifest + + +def validate_dataset_manifest( + manifest: DatasetManifest, + *, + minimum_slice_counts: Mapping[str, int] | None = None, +) -> tuple[DatasetIssue, ...]: + _validate_manifest_contract(manifest) + issues = [] + split_by_case: dict[str, set[str]] = {} + for split_name, case_ids in manifest.splits.items(): + for case_id in case_ids: + split_by_case.setdefault(case_id, set()).add(split_name) + for case_id, split_names in sorted(split_by_case.items()): + if len(split_names) > 1: + issues.append( + DatasetIssue( + "split_membership_leakage", + f"Case {case_id!r} appears in multiple splits: {sorted(split_names)}", + (case_id,), + ) + ) + grouped: dict[str, list[DatasetCaseRecord]] = {} + for case in manifest.cases: + grouped.setdefault(case.normalized_content_digest, []).append(case) + for matching in grouped.values(): + if len(matching) < 2: + continue + case_ids = tuple(sorted(item.id for item in matching)) + split_names = { + split + for case_id in case_ids + for split in split_by_case.get(case_id, set()) + } + code = "normalized_content_leakage" if len(split_names) > 1 else "duplicate_content" + issues.append( + DatasetIssue(code, f"Normalized duplicate cases detected: {list(case_ids)}", case_ids) + ) + missing_source = tuple(sorted(item.id for item in manifest.cases if not item.source)) + if missing_source: + issues.append( + DatasetIssue("missing_provenance", "Dataset cases are missing source provenance", missing_source) + ) + if manifest.source_classification == "production-derived": + unreviewed = tuple(sorted(item.id for item in manifest.cases if not item.reviewed)) + if unreviewed: + issues.append( + DatasetIssue( + "review_required", + "Production-derived cases must remain quarantined until reviewed", + unreviewed, + ) + ) + for selector, required in sorted((minimum_slice_counts or {}).items()): + if isinstance(required, bool) or not isinstance(required, int) or required <= 0: + raise ContractError(f"Dataset slice {selector!r} minimum must be positive") + selected = tuple(item.id for item in manifest.cases if _slice_match(item, selector)) + if len(selected) < required: + issues.append( + DatasetIssue( + "slice_coverage", + f"Dataset slice {selector!r} has {len(selected)} cases; requires {required}", + selected, + len(selected), + required, + ) + ) + return tuple(sorted(issues, key=lambda item: (item.code, item.case_ids))) + + +def diff_dataset_manifests(before: DatasetManifest, after: DatasetManifest) -> DatasetDiff: + if before.dataset_id != after.dataset_id: + raise ContractError("Dataset diff requires matching dataset IDs") + before_cases = {item.id: item.digest for item in before.cases} + after_cases = {item.id: item.digest for item in after.cases} + shared = set(before_cases) & set(after_cases) + return DatasetDiff( + dataset_id=before.dataset_id, + from_version=before.version, + to_version=after.version, + added_case_ids=tuple(sorted(set(after_cases) - set(before_cases))), + removed_case_ids=tuple(sorted(set(before_cases) - set(after_cases))), + changed_case_ids=tuple( + sorted(case_id for case_id in shared if before_cases[case_id] != after_cases[case_id]) + ), + ) + + +def write_dataset_manifest(path: str | Path, manifest: DatasetManifest) -> None: + _validate_manifest_contract(manifest) + output = Path(path) + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text( + json.dumps(manifest.to_dict(), ensure_ascii=False, sort_keys=True, indent=2) + "\n", + encoding="utf-8", + ) + + +def load_dataset_manifest(path: str | Path) -> DatasetManifest: + try: + data = json.loads(Path(path).read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise ContractError(f"Cannot load dataset manifest from {path}: {exc}") from exc + if not isinstance(data, dict) or set(data) != { + "schema_version", + "dataset_id", + "version", + "source_classification", + "owners", + "review_state", + "cases", + "splits", + }: + raise ContractError("Dataset manifest fields do not match schema 1.0") + if data["schema_version"] != "1.0": + raise ContractError(f"Unsupported dataset manifest schema: {data['schema_version']}") + raw_cases = data["cases"] + if not isinstance(raw_cases, list): + raise ContractError("Dataset manifest cases must be an array") + records = [] + expected_case_fields = { + "id", + "digest", + "normalized_content_digest", + "category", + "severity", + "language", + "tags", + "source", + "reviewed", + } + for index, item in enumerate(raw_cases): + if not isinstance(item, dict) or set(item) != expected_case_fields: + raise ContractError(f"Dataset case record {index} fields are invalid") + records.append( + DatasetCaseRecord( + id=item["id"], + digest=item["digest"], + normalized_content_digest=item["normalized_content_digest"], + category=item["category"], + severity=item["severity"], + language=item["language"], + tags=tuple(item["tags"]), + source=item["source"], + reviewed=item["reviewed"], + ) + ) + raw_splits = data["splits"] + if not isinstance(raw_splits, dict): + raise ContractError("Dataset splits must be an object") + manifest = DatasetManifest( + "1.0", + data["dataset_id"], + data["version"], + data["source_classification"], + tuple(data["owners"]), + data["review_state"], + tuple(records), + {name: tuple(case_ids) for name, case_ids in raw_splits.items()}, + ) + _validate_manifest_contract(manifest) + return manifest + + +def _case_record(value: Mapping[str, Any], index: int) -> DatasetCaseRecord: + required = {"id", "question", "evidence", "required_citation_ids"} + missing = sorted(required - set(value)) + if missing: + raise ContractError(f"Dataset case {index} missing fields: {missing}") + try: + canonical = json.dumps(dict(value), ensure_ascii=False, sort_keys=True, separators=(",", ":"), allow_nan=False) + except (TypeError, ValueError) as exc: + raise ContractError(f"Dataset case {index} must contain finite JSON values") from exc + normalized_payload = { + "question": _normalize_text(value["question"]), + "evidence": sorted(_normalize_text(item) for item in value["evidence"]), + } + normalized = json.dumps(normalized_payload, ensure_ascii=False, sort_keys=True, separators=(",", ":")) + tags = value.get("tags", []) + if not isinstance(tags, (list, tuple)) or any(not isinstance(item, str) for item in tags): + raise ContractError(f"Dataset case {index} tags must be a string array") + reviewed = value.get("reviewed", False) + if not isinstance(reviewed, bool): + raise ContractError(f"Dataset case {index} reviewed must be boolean") + return DatasetCaseRecord( + id=value["id"], + digest=hashlib.sha256(canonical.encode("utf-8")).hexdigest(), + normalized_content_digest=hashlib.sha256(normalized.encode("utf-8")).hexdigest(), + category=value.get("category", "unspecified"), + severity=value.get("severity", "medium"), + language=value.get("language", "und"), + tags=tuple(tags), + source=value.get("source", ""), + reviewed=reviewed, + ) + + +def _splits(value: Mapping[str, tuple[str, ...]], case_ids: set[str]) -> dict[str, tuple[str, ...]]: + if not isinstance(value, Mapping) or not value: + raise ContractError("Dataset splits must be a non-empty mapping") + result = {} + for name, identifiers in value.items(): + if not isinstance(name, str) or not name: + raise ContractError("Dataset split names must be non-empty strings") + if not isinstance(identifiers, (list, tuple)) or not identifiers: + raise ContractError(f"Dataset split {name!r} must contain case IDs") + if any(not isinstance(item, str) or not item for item in identifiers): + raise ContractError(f"Dataset split {name!r} case IDs must be non-empty strings") + if len(set(identifiers)) != len(identifiers): + raise ContractError(f"Dataset split {name!r} case IDs must be unique") + unknown = sorted(set(identifiers) - case_ids) + if unknown: + raise ContractError(f"Dataset split {name!r} has unknown case IDs: {unknown}") + result[name] = tuple(identifiers) + return dict(sorted(result.items())) + + +def _validate_manifest_contract(manifest: DatasetManifest) -> None: + if manifest.schema_version != "1.0": + raise ContractError(f"Unsupported dataset manifest schema: {manifest.schema_version}") + if not manifest.dataset_id or not manifest.version: + raise ContractError("Dataset ID and version must be non-empty") + if manifest.source_classification not in {"synthetic", "public", "production-derived"}: + raise ContractError("Dataset source classification is invalid") + if not manifest.owners or any(not owner for owner in manifest.owners): + raise ContractError("Dataset owners must be non-empty strings") + if manifest.review_state not in {"draft", "reviewed", "accepted", "superseded"}: + raise ContractError("Dataset review state is invalid") + identifiers = [item.id for item in manifest.cases] + if not identifiers or len(set(identifiers)) != len(identifiers): + raise ContractError("Dataset case IDs must be unique") + digest_pattern = re.compile(r"^[0-9a-f]{64}$") + for item in manifest.cases: + if not item.id or digest_pattern.fullmatch(item.digest) is None or digest_pattern.fullmatch(item.normalized_content_digest) is None: + raise ContractError(f"Dataset case record {item.id!r} is invalid") + _splits(manifest.splits, set(identifiers)) + + +def _slice_match(item: DatasetCaseRecord, selector: str) -> bool: + if "=" not in selector: + field, expected = "severity", selector + else: + field, expected = selector.split("=", 1) + if field == "tag": + return expected in item.tags + if field not in {"category", "severity", "language"} or not expected: + raise ContractError(f"Invalid dataset slice selector: {selector!r}") + return getattr(item, field) == expected + + +def _normalize_text(value: object) -> str: + if not isinstance(value, str): + raise ContractError("Dataset question and evidence must be strings") + return " ".join(unicodedata.normalize("NFKC", value).casefold().split()) diff --git a/src/ragops/demo.py b/src/ragops/demo.py index 9b98c21..75340c6 100644 --- a/src/ragops/demo.py +++ b/src/ragops/demo.py @@ -5,9 +5,21 @@ from pathlib import Path from uuid import uuid4 +from ragops.ci import github_summary, junit_xml, sarif_report from ragops.engine import compare +from ragops.evidence import create_evidence_bundle +from ragops.explain import explain_decision from ragops.loader import responses_from_data, scenario_from_dict -from ragops.reporters import comparison_html, comparison_markdown +from ragops.models import ComparisonReport, RegressionPolicy +from ragops.policy_v2 import GateEvidence, ReleaseDecision +from ragops.reporters import comparison_html, comparison_markdown, demo_profile_markdown + +DEMO_CREATED_AT = "2026-08-23T00:00:00Z" +DEMO_LIMITATIONS = ( + "Synthetic benchmark evidence; this is not production adoption evidence.", + "Deterministic lexical evaluators do not replace domain-expert or production monitoring review.", +) +DEMO_PROFILES = ("executive", "engineer", "auditor") DEMO_SCENARIO = { "schema_version": "0.2", @@ -238,9 +250,12 @@ def write_demo( *, force: bool = False, scenario_id: str = DEFAULT_DEMO_SCENARIO, + profile: str = "engineer", ) -> dict[str, object]: """Write a credential-free demo bundle and return its release summary.""" + if profile not in DEMO_PROFILES: + raise ValueError(f"unknown demo profile {profile!r}; choose one of: {', '.join(DEMO_PROFILES)}") try: scenario_data, baseline_data, candidate_data = DEMO_BUNDLES[scenario_id] except KeyError as exc: @@ -259,6 +274,9 @@ def write_demo( raise NotADirectoryError(f"demo output is not a directory: {destination}") else: destination.mkdir(parents=True, exist_ok=False) + evidence_path = destination / "evidence" + if force and (evidence_path.exists() or evidence_path.is_symlink()): + _remove_demo_evidence(evidence_path) scenario = scenario_from_dict(scenario_data) report = compare( scenario, @@ -271,7 +289,15 @@ def write_demo( "candidate": destination / "candidate.json", "markdown_report": destination / "release-report.md", "html_report": destination / "release-report.html", + "comparison": destination / "comparison.json", + "decision": destination / "decision.json", + "explanation": destination / "explanation.json", + "junit": destination / "junit.xml", + "sarif": destination / "release.sarif", + "github_summary": destination / "github-summary.md", + "profile_summary": destination / "profile-summary.md", } + decision = _release_decision(report) _write_demo_file( files["scenario"], json.dumps(scenario_data, ensure_ascii=False, indent=2) + "\n", @@ -289,14 +315,146 @@ def write_demo( ) _write_demo_file(files["markdown_report"], comparison_markdown(report), force=force) _write_demo_file(files["html_report"], comparison_html(report), force=force) + _write_demo_file( + files["comparison"], + json.dumps(report.to_dict(), ensure_ascii=False, indent=2) + "\n", + force=force, + ) + _write_demo_file( + files["decision"], + json.dumps(decision.to_dict(), ensure_ascii=False, indent=2) + "\n", + force=force, + ) + _write_demo_file( + files["explanation"], + json.dumps(explain_decision(decision).to_dict(), ensure_ascii=False, indent=2) + "\n", + force=force, + ) + _write_demo_file(files["junit"], junit_xml(decision), force=force) + _write_demo_file( + files["sarif"], + json.dumps(sarif_report(decision), ensure_ascii=False, indent=2) + "\n", + force=force, + ) + _write_demo_file(files["github_summary"], github_summary(decision), force=force) + manifest = create_evidence_bundle( + evidence_path, + { + "baseline": files["baseline"], + "candidate": files["candidate"], + "comparison": files["comparison"], + "decision": files["decision"], + "explanation": files["explanation"], + "github_summary": files["github_summary"], + "html": files["html_report"], + "junit": files["junit"], + "markdown": files["markdown_report"], + "sarif": files["sarif"], + "scenario": files["scenario"], + }, + decision.decision, + DEMO_LIMITATIONS, + DEMO_CREATED_AT, + { + "evidence_classification": "synthetic", + "profile": profile, + "scenario_id": scenario_id, + }, + ) + _write_demo_file( + files["profile_summary"], + demo_profile_markdown(profile, report, manifest.to_dict()), + force=force, + ) return { "demo_completed": True, "scenario_id": scenario_id, "candidate_decision": "BLOCK" if not report.passed else "PASS", + "evidence_classification": "synthetic", + "profile": profile, "failed_gates": list(report.failed_gates), "output_dir": str(destination), - "files": {name: str(path) for name, path in files.items()}, + "files": {**{name: str(path) for name, path in files.items()}, "evidence": str(evidence_path)}, + } + + +def _release_decision(report: ComparisonReport) -> ReleaseDecision: + policy = RegressionPolicy() + definitions = { + "candidate_release_gate": ( + "candidate_release_gate", 1.0 if report.candidate_passed else 0.0, 1.0, ">=", + "candidate.release_gate", + ), + "citation_coverage_regression": ( + "citation_coverage", report.deltas["citation_coverage"], + -policy.max_citation_coverage_drop, ">=", "regression.max_citation_coverage_drop", + ), + "citation_precision_regression": ( + "citation_precision", report.deltas["citation_precision"], + -policy.max_citation_precision_drop, ">=", "regression.max_citation_precision_drop", + ), + "groundedness_regression": ( + "lexical_groundedness", report.deltas["lexical_groundedness"], + -policy.max_groundedness_drop, ">=", "regression.max_groundedness_drop", + ), + "latency_regression": ( + "avg_latency_ms", report.deltas["avg_latency_ms"], + policy.max_latency_increase_ms, "<=", "regression.max_latency_increase_ms", + ), + "cost_regression": ( + "avg_cost_usd", report.deltas["avg_cost_usd"], + policy.max_cost_increase_usd, "<=", "regression.max_cost_increase_usd", + ), + "new_critical_findings": ( + "critical_findings", report.deltas["critical_findings"], 0.0, "<=", + "regression.new_critical_findings", + ), } + affected = _affected_case_ids(report) + gates = tuple( + GateEvidence( + id=gate_id, + metric=definitions[gate_id][0], + scope="global", + observed=float(definitions[gate_id][1]), + threshold=float(definitions[gate_id][2]), + comparator=definitions[gate_id][3], + passed=False, + severity="block", + policy_path=definitions[gate_id][4], + case_ids=affected, + ) + for gate_id in report.failed_gates + ) + release = "PASS" if report.passed else "BLOCK" + return ReleaseDecision("1.0", report.scenario_id, release, report.passed, gates) + + +def _affected_case_ids(report: ComparisonReport) -> tuple[str, ...]: + affected = [] + for baseline, candidate in zip(report.baseline.cases, report.candidate.cases, strict=True): + if ( + candidate.citation_coverage < baseline.citation_coverage + or candidate.citation_precision < baseline.citation_precision + or candidate.lexical_groundedness < baseline.lexical_groundedness + or len(candidate.findings) > len(baseline.findings) + ): + affected.append(candidate.case_id) + return tuple(affected) + + +def _remove_demo_evidence(path: Path) -> None: + if path.is_symlink() or not path.is_dir(): + raise FileExistsError(f"refusing non-directory demo evidence target: {path}") + entries = sorted(path.rglob("*"), key=lambda item: len(item.parts), reverse=True) + if any(item.is_symlink() for item in entries): + raise FileExistsError(f"refusing symlink inside demo evidence: {path}") + for item in entries: + if item.is_file(): + item.unlink() + elif item.is_dir(): + item.rmdir() + path.rmdir() def _write_demo_file(path: Path, content: str, *, force: bool) -> None: diff --git a/src/ragops/distribution.py b/src/ragops/distribution.py new file mode 100644 index 0000000..fe7cdf1 --- /dev/null +++ b/src/ragops/distribution.py @@ -0,0 +1,103 @@ +from __future__ import annotations + +import math +from dataclasses import dataclass +from statistics import NormalDist, fmean, median, stdev +from typing import Mapping, Sequence + + +@dataclass(frozen=True) +class PairedEffect: + count: int + mean_delta: float + median_delta: float + standardized_effect: float | None + + +def quantile(values: Sequence[float], probability: float) -> float: + data = _finite_values(values, "quantile values") + if isinstance(probability, bool) or not isinstance(probability, (int, float)): + raise ValueError("quantile probability must be numeric") + probability = float(probability) + if not math.isfinite(probability) or not 0.0 <= probability <= 1.0: + raise ValueError("quantile probability must be between 0 and 1") + ordered = sorted(data) + if len(ordered) == 1: + return ordered[0] + position = (len(ordered) - 1) * probability + lower = math.floor(position) + upper = math.ceil(position) + if lower == upper: + return ordered[lower] + fraction = position - lower + return ordered[lower] + (ordered[upper] - ordered[lower]) * fraction + + +def paired_effect(baseline: Sequence[float], candidate: Sequence[float]) -> PairedEffect: + baseline_values = _finite_values(baseline, "baseline") + candidate_values = _finite_values(candidate, "candidate") + if len(baseline_values) != len(candidate_values): + raise ValueError("Paired samples must have equal lengths") + deltas = tuple(after - before for before, after in zip(baseline_values, candidate_values)) + mean_delta = fmean(deltas) + standard_deviation = stdev(deltas) if len(deltas) > 1 else 0.0 + standardized = mean_delta / standard_deviation if standard_deviation > 0 else None + return PairedEffect(len(deltas), mean_delta, median(deltas), standardized) + + +def holm_adjust(p_values: Mapping[str, float]) -> dict[str, float]: + if not p_values: + raise ValueError("Holm adjustment requires at least one p-value") + validated = {} + for name, value in p_values.items(): + if not isinstance(name, str) or not name: + raise ValueError("Holm p-value names must be non-empty strings") + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise ValueError(f"Holm p-value {name!r} must be numeric") + number = float(value) + if not math.isfinite(number) or not 0.0 <= number <= 1.0: + raise ValueError(f"Holm p-value {name!r} must be between 0 and 1") + validated[name] = number + ordered = sorted(validated.items(), key=lambda item: (item[1], item[0])) + adjusted: dict[str, float] = {} + previous = 0.0 + count = len(ordered) + for index, (name, value) in enumerate(ordered): + correction = min(1.0, (count - index) * value) + previous = max(previous, correction) + adjusted[name] = previous + return {name: adjusted[name] for name in p_values} + + +def minimum_paired_sample(effect: float, alpha: float = 0.05, power: float = 0.8) -> int: + effect = _positive_probability_input(effect, "effect", bounded=False) + alpha = _positive_probability_input(alpha, "alpha") + power = _positive_probability_input(power, "power") + z_alpha = NormalDist().inv_cdf(1.0 - alpha / 2.0) + z_power = NormalDist().inv_cdf(power) + return max(2, math.ceil(((z_alpha + z_power) / effect) ** 2)) + + +def _finite_values(values: Sequence[float], label: str) -> tuple[float, ...]: + if not values: + raise ValueError(f"{label} must not be empty") + result = [] + for value in values: + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise ValueError(f"{label} must be numeric") + number = float(value) + if not math.isfinite(number): + raise ValueError(f"{label} must be finite") + result.append(number) + return tuple(result) + + +def _positive_probability_input(value: float, label: str, *, bounded: bool = True) -> float: + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise ValueError(f"{label} must be numeric") + number = float(value) + maximum_ok = number < 1.0 if bounded else True + if not math.isfinite(number) or number <= 0.0 or not maximum_ok: + suffix = " between 0 and 1" if bounded else " positive" + raise ValueError(f"{label} must be{suffix}") + return number diff --git a/src/ragops/evidence.py b/src/ragops/evidence.py new file mode 100644 index 0000000..f2ca56c --- /dev/null +++ b/src/ragops/evidence.py @@ -0,0 +1,298 @@ +from __future__ import annotations + +import hashlib +import json +import mimetypes +import re +import shutil +import tempfile +from dataclasses import asdict, dataclass +from datetime import datetime +from pathlib import Path, PurePosixPath +from typing import Any, Mapping + +from ragops.loader import ContractError + +DEFAULT_MAX_ARTIFACT_BYTES = 50 * 1024 * 1024 +_DECISIONS = {"PASS", "WARN", "BLOCK"} +_LOGICAL_NAME = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.-]*$") + + +@dataclass(frozen=True) +class ArtifactRecord: + logical_name: str + path: str + media_type: str + bytes: int + sha256: str + + def to_dict(self) -> dict[str, Any]: + return asdict(self) + + +@dataclass(frozen=True) +class EvidenceManifest: + schema_version: str + ragops_version: str + created_at: str + decision: str + limitations: tuple[str, ...] + artifacts: tuple[ArtifactRecord, ...] + metadata: dict[str, Any] + + def to_dict(self) -> dict[str, Any]: + return { + "schema_version": self.schema_version, + "ragops_version": self.ragops_version, + "created_at": self.created_at, + "decision": self.decision, + "limitations": list(self.limitations), + "artifacts": [artifact.to_dict() for artifact in self.artifacts], + "metadata": self.metadata, + } + + def canonical_bytes(self) -> bytes: + return _canonical_json(self.to_dict()) + + +def create_evidence_bundle( + output: str | Path, + artifacts: Mapping[str, str | Path], + decision: str, + limitations: tuple[str, ...], + created_at: str, + metadata: Mapping[str, Any], + *, + max_artifact_bytes: int = DEFAULT_MAX_ARTIFACT_BYTES, +) -> EvidenceManifest: + _validate_decision(decision) + _validate_timestamp(created_at) + if not artifacts: + raise ContractError("Evidence bundle requires at least one artifact") + if not limitations or any(not isinstance(item, str) or not item.strip() for item in limitations): + raise ContractError("Evidence bundle limitations must be non-empty strings") + if isinstance(max_artifact_bytes, bool) or max_artifact_bytes <= 0: + raise ContractError("Evidence artifact byte limit must be positive") + normalized_metadata = _json_object(metadata, "Evidence metadata") + target = Path(output) + if target.exists() or target.is_symlink(): + raise FileExistsError(f"Evidence bundle already exists: {target}") + target.parent.mkdir(parents=True, exist_ok=True) + temporary = Path(tempfile.mkdtemp(prefix=f".{target.name}-", dir=target.parent)) + try: + artifact_dir = temporary / "artifacts" + artifact_dir.mkdir() + records = [] + destinations: set[str] = set() + for logical_name, raw_source in sorted(artifacts.items()): + if not isinstance(logical_name, str) or _LOGICAL_NAME.fullmatch(logical_name) is None: + raise ContractError(f"Invalid evidence artifact logical name: {logical_name!r}") + source = Path(raw_source) + if source.is_symlink() or not source.is_file(): + raise ContractError(f"Evidence artifact must be a regular non-symlink file: {source}") + size = source.stat().st_size + if size > max_artifact_bytes: + raise ContractError( + f"Evidence artifact {logical_name!r} exceeds {max_artifact_bytes} bytes" + ) + suffix = "".join(source.suffixes[-2:]) if source.name.endswith(".tar.gz") else source.suffix + filename = f"{logical_name}{suffix}" + if filename in destinations: + raise ContractError(f"Duplicate evidence artifact destination: {filename}") + destinations.add(filename) + destination = artifact_dir / filename + shutil.copyfile(source, destination) + relative = destination.relative_to(temporary).as_posix() + records.append( + ArtifactRecord( + logical_name=logical_name, + path=relative, + media_type=mimetypes.guess_type(source.name)[0] or "application/octet-stream", + bytes=size, + sha256=_digest(destination), + ) + ) + manifest = EvidenceManifest( + schema_version="1.0", + ragops_version=_ragops_version(), + created_at=created_at, + decision=decision, + limitations=tuple(limitations), + artifacts=tuple(records), + metadata=normalized_metadata, + ) + (temporary / "manifest.json").write_bytes(manifest.canonical_bytes() + b"\n") + temporary.rename(target) + except Exception: + shutil.rmtree(temporary, ignore_errors=True) + raise + return manifest + + +def load_evidence_manifest(path: str | Path) -> EvidenceManifest: + manifest_path = Path(path) + if manifest_path.is_dir(): + manifest_path = manifest_path / "manifest.json" + if manifest_path.is_symlink() or not manifest_path.is_file(): + raise ContractError(f"Evidence manifest must be a regular non-symlink file: {manifest_path}") + try: + data = json.loads(manifest_path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise ContractError(f"Cannot load evidence manifest: {exc}") from exc + if not isinstance(data, dict): + raise ContractError("Evidence manifest must be a JSON object") + expected = { + "schema_version", + "ragops_version", + "created_at", + "decision", + "limitations", + "artifacts", + "metadata", + } + if set(data) != expected: + raise ContractError("Evidence manifest fields do not match schema 1.0") + if data["schema_version"] != "1.0": + raise ContractError(f"Unsupported evidence bundle schema: {data['schema_version']}") + _validate_decision(data["decision"]) + _validate_timestamp(data["created_at"]) + if not isinstance(data["ragops_version"], str) or not data["ragops_version"]: + raise ContractError("Evidence ragops_version must be a non-empty string") + limitations = data["limitations"] + if not isinstance(limitations, list) or not limitations or any( + not isinstance(item, str) or not item.strip() for item in limitations + ): + raise ContractError("Evidence limitations must be non-empty strings") + raw_artifacts = data["artifacts"] + if not isinstance(raw_artifacts, list) or not raw_artifacts: + raise ContractError("Evidence manifest requires at least one artifact") + records = tuple(_artifact_record(item) for item in raw_artifacts) + if len({item.logical_name for item in records}) != len(records): + raise ContractError("Evidence artifact logical names must be unique") + if len({item.path for item in records}) != len(records): + raise ContractError("Evidence artifact paths must be unique") + return EvidenceManifest( + schema_version="1.0", + ragops_version=data["ragops_version"], + created_at=data["created_at"], + decision=data["decision"], + limitations=tuple(limitations), + artifacts=records, + metadata=_json_object(data["metadata"], "Evidence metadata"), + ) + + +def verify_evidence_bundle(path: str | Path) -> EvidenceManifest: + root = Path(path) + if root.is_symlink() or not root.is_dir(): + raise ContractError(f"Evidence bundle must be a regular non-symlink directory: {root}") + manifest = load_evidence_manifest(root / "manifest.json") + declared = {"manifest.json"} + for record in manifest.artifacts: + relative = _safe_artifact_path(record.path) + artifact = root.joinpath(*relative.parts) + declared.add(relative.as_posix()) + if artifact.is_symlink() or not artifact.is_file(): + raise ContractError(f"Evidence artifact is missing or not regular: {record.path}") + if artifact.stat().st_size != record.bytes: + raise ContractError(f"Evidence artifact size mismatch: {record.logical_name}") + if _digest(artifact) != record.sha256: + raise ContractError(f"Evidence artifact digest mismatch: {record.logical_name}") + actual = set() + for item in root.rglob("*"): + if item.is_symlink(): + raise ContractError(f"Evidence bundle contains symlink: {item.relative_to(root)}") + if item.is_file(): + actual.add(item.relative_to(root).as_posix()) + extra = sorted(actual - declared) + if extra: + raise ContractError(f"Evidence bundle contains undeclared files: {extra}") + missing = sorted(declared - actual) + if missing: + raise ContractError(f"Evidence bundle is missing declared files: {missing}") + return manifest + + +def _artifact_record(value: object) -> ArtifactRecord: + if not isinstance(value, dict): + raise ContractError("Evidence artifact record must be an object") + expected = {"logical_name", "path", "media_type", "bytes", "sha256"} + if set(value) != expected: + raise ContractError("Evidence artifact fields do not match schema 1.0") + logical_name = value["logical_name"] + if not isinstance(logical_name, str) or _LOGICAL_NAME.fullmatch(logical_name) is None: + raise ContractError("Evidence artifact logical name is invalid") + if not isinstance(value["path"], str): + raise ContractError("Evidence artifact path must be a string") + _safe_artifact_path(value["path"]) + if not isinstance(value["media_type"], str) or not value["media_type"]: + raise ContractError("Evidence artifact media_type must be a non-empty string") + size = value["bytes"] + if isinstance(size, bool) or not isinstance(size, int) or size < 0: + raise ContractError("Evidence artifact bytes must be a non-negative integer") + digest = value["sha256"] + if not isinstance(digest, str) or re.fullmatch(r"[0-9a-f]{64}", digest) is None: + raise ContractError("Evidence artifact sha256 must be lowercase hexadecimal") + return ArtifactRecord(logical_name, value["path"], value["media_type"], size, digest) + + +def _safe_artifact_path(value: str) -> PurePosixPath: + path = PurePosixPath(value) + if path.is_absolute() or not path.parts or path.parts[0] != "artifacts" or ".." in path.parts: + raise ContractError(f"Invalid evidence artifact path: {value!r}") + if len(path.parts) != 2 or path.parts[1] in {"", "."}: + raise ContractError(f"Invalid evidence artifact path: {value!r}") + return path + + +def _validate_decision(value: object) -> None: + if value not in _DECISIONS: + raise ContractError("Evidence decision must be PASS, WARN, or BLOCK") + + +def _validate_timestamp(value: object) -> None: + if not isinstance(value, str): + raise ContractError("Evidence created_at must be an ISO-8601 timestamp") + try: + parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) + except ValueError as exc: + raise ContractError("Evidence created_at must be an ISO-8601 timestamp") from exc + if parsed.tzinfo is None: + raise ContractError("Evidence created_at must include a timezone") + + +def _json_object(value: Mapping[str, Any] | object, label: str) -> dict[str, Any]: + if not isinstance(value, Mapping): + raise ContractError(f"{label} must be an object") + try: + encoded = _canonical_json(dict(value)) + decoded = json.loads(encoded) + except (TypeError, ValueError) as exc: + raise ContractError(f"{label} must contain finite JSON values") from exc + if not isinstance(decoded, dict): + raise ContractError(f"{label} must be an object") + return decoded + + +def _canonical_json(value: object) -> bytes: + return json.dumps( + value, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + allow_nan=False, + ).encode("utf-8") + + +def _digest(path: Path) -> str: + hasher = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + hasher.update(chunk) + return hasher.hexdigest() + + +def _ragops_version() -> str: + from ragops import __version__ + + return __version__ diff --git a/src/ragops/explain.py b/src/ragops/explain.py new file mode 100644 index 0000000..c476bcf --- /dev/null +++ b/src/ragops/explain.py @@ -0,0 +1,73 @@ +from __future__ import annotations + +from dataclasses import asdict, dataclass +from typing import Any + +from ragops.policy_v2 import ReleaseDecision + + +@dataclass(frozen=True) +class GateExplanation: + gate_id: str + policy_path: str + scope: str + case_ids: tuple[str, ...] + observed: float + threshold: float + comparator: str + required_change: float + waived: bool + waiver: dict[str, str | None] | None + + def to_dict(self) -> dict[str, Any]: + return asdict(self) + + +@dataclass(frozen=True) +class DecisionExplanation: + schema_version: str + scenario_id: str + decision: str + items: tuple[GateExplanation, ...] + + def to_dict(self) -> dict[str, Any]: + return { + "schema_version": self.schema_version, + "scenario_id": self.scenario_id, + "decision": self.decision, + "items": [item.to_dict() for item in self.items], + } + + +def explain_decision(decision: ReleaseDecision) -> DecisionExplanation: + items = [] + for gate in decision.gates: + if gate.passed: + continue + required_change = ( + max(0.0, gate.threshold - gate.observed) + if gate.comparator == ">=" + else max(0.0, gate.observed - gate.threshold) + ) + waiver = None + if gate.waived: + waiver = { + "owner": gate.waiver_owner, + "reason": gate.waiver_reason, + "expires_at": gate.waiver_expires_at, + } + items.append( + GateExplanation( + gate_id=gate.id, + policy_path=gate.policy_path, + scope=gate.scope, + case_ids=gate.case_ids, + observed=gate.observed, + threshold=gate.threshold, + comparator=gate.comparator, + required_change=required_change, + waived=gate.waived, + waiver=waiver, + ) + ) + return DecisionExplanation("1.0", decision.scenario_id, decision.decision, tuple(items)) diff --git a/src/ragops/governance.py b/src/ragops/governance.py new file mode 100644 index 0000000..847eec5 --- /dev/null +++ b/src/ragops/governance.py @@ -0,0 +1,314 @@ +from __future__ import annotations + +import json +import re +import sqlite3 +from collections.abc import Mapping +from datetime import datetime +from pathlib import Path +from typing import Any +from uuid import uuid4 + +SHA256 = re.compile(r"^[0-9a-f]{64}$") +TRANSITIONS = { + "draft": {"reviewed"}, + "reviewed": {"accepted"}, + "accepted": {"superseded"}, + "superseded": set(), +} + + +class GovernanceStore: + def __init__(self, path: str | Path) -> None: + self.path = str(path) + Path(path).parent.mkdir(parents=True, exist_ok=True) + self._initialize() + + def register_artifact( + self, + artifact_id: str, + *, + kind: str, + digest: str, + public_metadata: dict[str, Any], + blinded_metadata: dict[str, Any], + actor: str, + ) -> None: + if not all((artifact_id.strip(), kind.strip(), actor.strip())): + raise ValueError("artifact ID, kind, and actor are required") + if not SHA256.fullmatch(digest): + raise ValueError("artifact digest must be a lowercase SHA-256") + public_json = _metadata_json(public_metadata) + blinded_json = _metadata_json(blinded_metadata) + created_at = _now() + try: + with self._connect() as connection: + connection.execute( + """ + INSERT INTO artifacts + (id, kind, digest, state, public_metadata, blinded_metadata, created_at, updated_at) + VALUES (?, ?, ?, 'draft', ?, ?, ?, ?) + """, + ( + artifact_id, kind, digest, public_json, blinded_json, + created_at, created_at, + ), + ) + self._audit(connection, artifact_id, actor, "artifact.registered", {"state": "draft"}) + except sqlite3.IntegrityError as exc: + raise ValueError(f"artifact already exists: {artifact_id}") from exc + + def transition(self, artifact_id: str, state: str, *, actor: str) -> None: + if not actor.strip(): + raise ValueError("actor is required") + with self._connect() as connection: + row = connection.execute( + "SELECT state FROM artifacts WHERE id = ?", (artifact_id,) + ).fetchone() + if row is None: + raise KeyError(f"Unknown artifact: {artifact_id}") + current = row[0] + if state not in TRANSITIONS[current]: + raise ValueError(f"invalid governance transition: {current} -> {state}") + if state == "accepted": + approval = connection.execute( + "SELECT 1 FROM reviews WHERE artifact_id = ? AND verdict = 'approve' LIMIT 1", + (artifact_id,), + ).fetchone() + if approval is None: + raise ValueError("accepted state requires at least one approval review") + connection.execute( + "UPDATE artifacts SET state = ?, updated_at = ? WHERE id = ?", + (state, _now(), artifact_id), + ) + self._audit( + connection, artifact_id, actor, "artifact.transitioned", {"from": current, "to": state} + ) + + def record_review( + self, + artifact_id: str, + *, + reviewer: str, + verdict: str, + note: str = "", + ) -> None: + if not reviewer.strip(): + raise ValueError("reviewer is required") + if verdict not in {"approve", "block"}: + raise ValueError("review verdict must be approve or block") + try: + with self._connect() as connection: + state = connection.execute( + "SELECT state FROM artifacts WHERE id = ?", (artifact_id,) + ).fetchone() + if state is None: + raise KeyError(f"Unknown artifact: {artifact_id}") + if state[0] != "reviewed": + raise ValueError("reviews can only be recorded in reviewed state") + connection.execute( + """ + INSERT INTO reviews (artifact_id, reviewer, verdict, note, created_at) + VALUES (?, ?, ?, ?, ?) + """, + (artifact_id, reviewer, verdict, note, _now()), + ) + self._audit( + connection, + artifact_id, + reviewer, + "review.recorded", + {"verdict": verdict, "note": note}, + ) + except sqlite3.IntegrityError as exc: + raise ValueError(f"reviewer {reviewer!r} already reviewed artifact {artifact_id!r}") from exc + + def review_queue(self, reviewer: str) -> list[dict[str, Any]]: + if not reviewer.strip(): + raise ValueError("reviewer is required") + with self._connect() as connection: + rows = connection.execute( + """ + SELECT a.id, a.kind, a.digest, a.state, a.public_metadata, a.blinded_metadata, + EXISTS(SELECT 1 FROM reviews r WHERE r.artifact_id = a.id AND r.reviewer = ?) + FROM artifacts a WHERE a.state = 'reviewed' ORDER BY a.created_at, a.id + """, + (reviewer,), + ).fetchall() + result = [] + for artifact_id, kind, digest, state, public_json, blinded_json, reviewed in rows: + metadata = json.loads(public_json) + if reviewed: + metadata.update(json.loads(blinded_json)) + result.append( + { + "id": artifact_id, + "kind": kind, + "digest": digest, + "state": state, + "metadata": metadata, + "blinded": not bool(reviewed), + } + ) + return result + + def get_artifact(self, artifact_id: str) -> dict[str, Any]: + with self._connect() as connection: + row = connection.execute( + "SELECT id, kind, digest, state, public_metadata, blinded_metadata FROM artifacts WHERE id = ?", + (artifact_id,), + ).fetchone() + if row is None: + raise KeyError(f"Unknown artifact: {artifact_id}") + return { + "id": row[0], "kind": row[1], "digest": row[2], "state": row[3], + "public_metadata": json.loads(row[4]), "blinded_metadata": json.loads(row[5]), + } + + def create_waiver( + self, + *, + gate_id: str, + owner: str, + reason: str, + issued_at: str, + expires_at: str, + actor: str, + ) -> str: + if not all(value.strip() for value in (gate_id, owner, reason, actor)): + raise ValueError("waiver gate, owner, reason, and actor are required") + issued = _timestamp(issued_at) + expires = _timestamp(expires_at) + if expires <= issued: + raise ValueError("waiver must expire after issuance") + waiver_id = uuid4().hex + with self._connect() as connection: + connection.execute( + """ + INSERT INTO waivers (id, gate_id, owner, reason, issued_at, expires_at, created_by) + VALUES (?, ?, ?, ?, ?, ?, ?) + """, + (waiver_id, gate_id, owner, reason, issued_at, expires_at, actor), + ) + self._audit( + connection, None, actor, "waiver.created", {"waiver_id": waiver_id, "gate_id": gate_id} + ) + return waiver_id + + def list_waivers(self) -> list[dict[str, str]]: + with self._connect() as connection: + rows = connection.execute( + "SELECT id, gate_id, owner, reason, issued_at, expires_at, created_by FROM waivers ORDER BY issued_at" + ).fetchall() + keys = ("id", "gate_id", "owner", "reason", "issued_at", "expires_at", "created_by") + return [dict(zip(keys, row, strict=True)) for row in rows] + + def audit_events(self, artifact_id: str | None = None) -> list[dict[str, Any]]: + with self._connect() as connection: + if artifact_id is None: + rows = connection.execute( + "SELECT created_at, artifact_id, actor, action, detail FROM audit_log ORDER BY id" + ).fetchall() + else: + rows = connection.execute( + """ + SELECT created_at, artifact_id, actor, action, detail FROM audit_log + WHERE artifact_id = ? ORDER BY id + """, + (artifact_id,), + ).fetchall() + return [ + {"created_at": row[0], "artifact_id": row[1], "actor": row[2], "action": row[3], "detail": json.loads(row[4])} + for row in rows + ] + + def _connect(self) -> sqlite3.Connection: + connection = sqlite3.connect(self.path) + connection.execute("PRAGMA foreign_keys = ON") + return connection + + def _audit( + self, + connection: sqlite3.Connection, + artifact_id: str | None, + actor: str, + action: str, + detail: dict[str, Any], + ) -> None: + connection.execute( + "INSERT INTO audit_log (created_at, artifact_id, actor, action, detail) VALUES (?, ?, ?, ?, ?)", + (_now(), artifact_id, actor, action, _json(detail)), + ) + + def _initialize(self) -> None: + with self._connect() as connection: + connection.executescript( + """ + CREATE TABLE IF NOT EXISTS artifacts ( + id TEXT PRIMARY KEY, + kind TEXT NOT NULL, + digest TEXT NOT NULL, + state TEXT NOT NULL CHECK(state IN ('draft', 'reviewed', 'accepted', 'superseded')), + public_metadata TEXT NOT NULL, + blinded_metadata TEXT NOT NULL, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL + ); + CREATE TABLE IF NOT EXISTS reviews ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + artifact_id TEXT NOT NULL REFERENCES artifacts(id), + reviewer TEXT NOT NULL, + verdict TEXT NOT NULL CHECK(verdict IN ('approve', 'block')), + note TEXT NOT NULL, + created_at TEXT NOT NULL, + UNIQUE(artifact_id, reviewer) + ); + CREATE TABLE IF NOT EXISTS waivers ( + id TEXT PRIMARY KEY, + gate_id TEXT NOT NULL, + owner TEXT NOT NULL, + reason TEXT NOT NULL, + issued_at TEXT NOT NULL, + expires_at TEXT NOT NULL, + created_by TEXT NOT NULL + ); + CREATE TABLE IF NOT EXISTS audit_log ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + created_at TEXT NOT NULL, + artifact_id TEXT, + actor TEXT NOT NULL, + action TEXT NOT NULL, + detail TEXT NOT NULL + ); + """ + ) + + +def _json(value: dict[str, Any]) -> str: + return json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":")) + + +def _metadata_json(value: object) -> str: + if not isinstance(value, Mapping): + raise ValueError("artifact metadata must be a JSON object") + try: + encoded = json.dumps( + dict(value), ensure_ascii=False, sort_keys=True, separators=(",", ":"), allow_nan=False + ) + except (TypeError, ValueError) as exc: + raise ValueError("artifact metadata must contain finite JSON values") from exc + return encoded + + +def _now() -> str: + return datetime.now().astimezone().isoformat() + + +def _timestamp(value: str) -> datetime: + try: + parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) + except ValueError as exc: + raise ValueError("waiver timestamps must use ISO-8601") from exc + if parsed.tzinfo is None: + raise ValueError("waiver timestamps must include a timezone") + return parsed diff --git a/src/ragops/policy_v2.py b/src/ragops/policy_v2.py new file mode 100644 index 0000000..62f008b --- /dev/null +++ b/src/ragops/policy_v2.py @@ -0,0 +1,621 @@ +from __future__ import annotations + +import json +import math +import tomllib +from dataclasses import asdict, dataclass, replace +from datetime import datetime +from pathlib import Path +from typing import Any + +from ragops.distribution import quantile +from ragops.loader import ContractError +from ragops.models import CaseResult, EvaluationReport, Scenario +from ragops.slices import SliceSelector, summarize_slices + + +@dataclass(frozen=True) +class PolicyGate: + metric: str + direction: str + minimum: float | None = None + maximum: float | None = None + max_regression: float | None = None + severity: str = "block" + + +@dataclass(frozen=True) +class SlicePolicy: + selector: SliceSelector + minimum_cases: int = 1 + + +@dataclass(frozen=True) +class TailGate: + metric: str + percentile: float + minimum: float | None = None + maximum: float | None = None + severity: str = "block" + + +@dataclass(frozen=True) +class PolicyWaiver: + gate_id: str + owner: str + reason: str + issued_at: str + expires_at: str + scope: str | None = None + + +@dataclass(frozen=True) +class ReleasePolicyV2: + minimum_cases: int + insufficient_evidence: str + metrics: tuple[PolicyGate, ...] + slices: tuple[SlicePolicy, ...] + tails: tuple[TailGate, ...] + waivers: tuple[PolicyWaiver, ...] + + +@dataclass(frozen=True) +class GateEvidence: + id: str + metric: str + scope: str + observed: float + threshold: float + comparator: str + passed: bool + severity: str + policy_path: str + case_ids: tuple[str, ...] = () + waived: bool = False + waiver_owner: str | None = None + waiver_reason: str | None = None + waiver_expires_at: str | None = None + + def to_dict(self) -> dict[str, Any]: + return asdict(self) + + +@dataclass(frozen=True) +class ReleaseDecision: + schema_version: str + scenario_id: str + decision: str + passed: bool + gates: tuple[GateEvidence, ...] + + def to_dict(self) -> dict[str, Any]: + return { + "schema_version": self.schema_version, + "scenario_id": self.scenario_id, + "decision": self.decision, + "passed": self.passed, + "gates": [gate.to_dict() for gate in self.gates], + } + + +def load_release_decision(path: str | Path) -> ReleaseDecision: + try: + raw = json.loads(Path(path).read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise ContractError(f"Cannot load release decision from {path}: {exc}") from exc + if not isinstance(raw, dict): + raise ContractError("Release decision must be a JSON object") + expected = {"schema_version", "scenario_id", "decision", "passed", "gates"} + if set(raw) != expected or not isinstance(raw["gates"], list): + raise ContractError(f"Release decision fields must be {sorted(expected)}") + gates = tuple(_gate_evidence_from_dict(item, index) for index, item in enumerate(raw["gates"])) + decision = raw["decision"] + if decision not in {"PASS", "WARN", "BLOCK"}: + raise ContractError("Release decision must be PASS, WARN, or BLOCK") + if not isinstance(raw["passed"], bool) or raw["passed"] != (decision == "PASS"): + raise ContractError("Release decision passed flag must match decision") + if ( + raw["schema_version"] != "1.0" + or not isinstance(raw["scenario_id"], str) + or not raw["scenario_id"] + ): + raise ContractError("Unsupported release decision contract") + unwaived_failures = [gate for gate in gates if not gate.passed and not gate.waived] + if any(gate.severity == "block" for gate in unwaived_failures): + expected_decision = "BLOCK" + elif unwaived_failures or any(gate.waived for gate in gates): + expected_decision = "WARN" + else: + expected_decision = "PASS" + if decision != expected_decision: + raise ContractError( + f"Release decision {decision} does not match gate evidence {expected_decision}" + ) + return ReleaseDecision("1.0", raw["scenario_id"], decision, raw["passed"], gates) + + +def _gate_evidence_from_dict(value: object, index: int) -> GateEvidence: + if not isinstance(value, dict): + raise ContractError(f"Release decision gate {index} must be an object") + required = { + "id", "metric", "scope", "observed", "threshold", "comparator", "passed", + "severity", "policy_path", "case_ids", "waived", "waiver_owner", "waiver_reason", + "waiver_expires_at", + } + if set(value) != required: + raise ContractError(f"Release decision gate {index} fields must be {sorted(required)}") + if value["comparator"] not in {">=", "<="} or value["severity"] not in {"warn", "block"}: + raise ContractError(f"Release decision gate {index} has invalid comparator or severity") + string_fields = ("id", "metric", "scope", "policy_path") + if any(not isinstance(value[field], str) or not value[field] for field in string_fields): + raise ContractError(f"Release decision gate {index} identifiers must be non-empty strings") + if not isinstance(value["passed"], bool) or not isinstance(value["waived"], bool): + raise ContractError(f"Release decision gate {index} passed and waived must be boolean") + if not isinstance(value["case_ids"], list) or not all( + isinstance(item, str) for item in value["case_ids"] + ): + raise ContractError(f"Release decision gate {index} case_ids must be strings") + observed = _finite(value["observed"], f"gate {index} observed") + threshold = _finite(value["threshold"], f"gate {index} threshold") + expected_passed = observed >= threshold if value["comparator"] == ">=" else observed <= threshold + if value["passed"] != expected_passed: + raise ContractError(f"Release decision gate {index} passed flag contradicts evidence") + waiver_fields = ("waiver_owner", "waiver_reason", "waiver_expires_at") + if value["waived"]: + if value["passed"] or any( + not isinstance(value[field], str) or not value[field] for field in waiver_fields + ): + raise ContractError(f"Release decision gate {index} has invalid waiver evidence") + _timestamp(value["waiver_expires_at"], f"gate {index} waiver expiry") + elif any(value[field] is not None for field in waiver_fields): + raise ContractError(f"Release decision gate {index} has undeclared waiver fields") + return GateEvidence( + id=value["id"], + metric=value["metric"], + scope=value["scope"], + observed=observed, + threshold=threshold, + comparator=value["comparator"], + passed=value["passed"], + severity=value["severity"], + policy_path=value["policy_path"], + case_ids=tuple(value["case_ids"]), + waived=value["waived"], + waiver_owner=value["waiver_owner"], + waiver_reason=value["waiver_reason"], + waiver_expires_at=value["waiver_expires_at"], + ) + + +def load_release_policy_v2(path: str | Path) -> ReleasePolicyV2: + try: + data = tomllib.loads(Path(path).read_text(encoding="utf-8")) + except (OSError, tomllib.TOMLDecodeError) as exc: + raise ContractError(f"Cannot load release policy v2 from {path}: {exc}") from exc + if not isinstance(data, dict): + raise ContractError("Release policy v2 must be a TOML object") + allowed_sections = {"release", "metrics", "slices", "tails", "waivers"} + unknown_sections = sorted(set(data) - allowed_sections) + if unknown_sections: + raise ContractError(f"Release policy v2 has unknown sections: {unknown_sections}") + release = _exact_mapping( + data.get("release"), {"minimum_cases", "insufficient_evidence"}, "release" + ) + raw_metrics = data.get("metrics", {}) + if not isinstance(raw_metrics, dict): + raise ContractError("Release policy metrics must be a table") + metrics = [] + for name, raw_gate in raw_metrics.items(): + gate = _subset_mapping( + raw_gate, + {"direction", "minimum", "maximum", "max_regression", "severity"}, + f"metrics.{name}", + ) + metrics.append( + PolicyGate( + metric=name, + direction=gate.get("direction"), + minimum=gate.get("minimum"), + maximum=gate.get("maximum"), + max_regression=gate.get("max_regression"), + severity=gate.get("severity", "block"), + ) + ) + raw_slices = data.get("slices", []) + if not isinstance(raw_slices, list): + raise ContractError("Release policy slices must use array-of-table syntax") + slices = [] + for index, raw_slice in enumerate(raw_slices): + item = _subset_mapping( + raw_slice, + {"name", "category", "severity", "language", "tags", "minimum_cases"}, + f"slices[{index}]", + ) + slices.append( + SlicePolicy( + selector=SliceSelector( + name=item.get("name"), + category=item.get("category"), + severity=item.get("severity"), + language=item.get("language"), + tags=tuple(item.get("tags", [])), + ), + minimum_cases=item.get("minimum_cases", release["minimum_cases"]), + ) + ) + raw_tails = data.get("tails", {}) + if not isinstance(raw_tails, dict): + raise ContractError("Release policy tails must be a table") + tails = [] + for metric, raw_tail in raw_tails.items(): + item = _subset_mapping( + raw_tail, {"percentile", "minimum", "maximum", "severity"}, f"tails.{metric}" + ) + tails.append( + TailGate( + metric=metric, + percentile=item.get("percentile"), + minimum=item.get("minimum"), + maximum=item.get("maximum"), + severity=item.get("severity", "block"), + ) + ) + raw_waivers = data.get("waivers", []) + if not isinstance(raw_waivers, list): + raise ContractError("Release policy waivers must use array-of-table syntax") + waivers = [] + for index, raw_waiver in enumerate(raw_waivers): + item = _subset_mapping( + raw_waiver, + {"gate_id", "owner", "reason", "issued_at", "expires_at", "scope"}, + f"waivers[{index}]", + ) + required = {"gate_id", "owner", "reason", "issued_at", "expires_at"} + missing = sorted(required - set(item)) + if missing: + raise ContractError(f"waivers[{index}] missing fields: {missing}") + waivers.append( + PolicyWaiver( + gate_id=item["gate_id"], + owner=item["owner"], + reason=item["reason"], + issued_at=item["issued_at"], + expires_at=item["expires_at"], + scope=item.get("scope"), + ) + ) + policy = ReleasePolicyV2( + minimum_cases=release["minimum_cases"], + insufficient_evidence=release["insufficient_evidence"], + metrics=tuple(metrics), + slices=tuple(slices), + tails=tuple(tails), + waivers=tuple(waivers), + ) + _validate_policy(policy) + return policy + + +def apply_release_policy( + policy: ReleasePolicyV2, + baseline: EvaluationReport, + candidate: EvaluationReport, + scenario: Scenario, + *, + now: str, +) -> ReleaseDecision: + _validate_policy(policy) + now_value = _timestamp(now, "policy evaluation time") + if baseline.scenario_id != scenario.id or candidate.scenario_id != scenario.id: + raise ContractError("Release policy scenario and reports must match") + if len(candidate.cases) < policy.minimum_cases: + raise ContractError( + f"Release policy requires {policy.minimum_cases} cases; candidate has {len(candidate.cases)}" + ) + gates: list[GateEvidence] = [] + all_case_ids = tuple(item.case_id for item in candidate.cases) + for metric_policy in policy.metrics: + gates.extend( + _metric_gates( + metric_policy, + baseline.metrics, + candidate.metrics, + scope="global", + case_ids=all_case_ids, + policy_prefix=f"metrics.{metric_policy.metric}", + ) + ) + if policy.slices: + selectors = tuple(item.selector for item in policy.slices) + baseline_slices = {item.name: item for item in summarize_slices(scenario, baseline, selectors)} + candidate_slices = { + item.name: item for item in summarize_slices(scenario, candidate, selectors) + } + for slice_policy in policy.slices: + name = slice_policy.selector.name + before = baseline_slices[name] + after = candidate_slices[name] + if after.count < slice_policy.minimum_cases: + severity = policy.insufficient_evidence + gates.append( + GateEvidence( + id=f"slice:{name}:minimum_cases", + metric="case_count", + scope=name, + observed=float(after.count), + threshold=float(slice_policy.minimum_cases), + comparator=">=", + passed=False, + severity=severity, + policy_path=f"slices.{name}.minimum_cases", + case_ids=after.case_ids, + ) + ) + continue + for metric_policy in policy.metrics: + gates.extend( + _metric_gates( + metric_policy, + before.metrics, + after.metrics, + scope=name, + case_ids=after.case_ids, + policy_prefix=f"slices.{name}.metrics.{metric_policy.metric}", + ) + ) + for tail in policy.tails: + values = tuple(_case_metric(item, tail.metric) for item in candidate.cases) + observed = quantile(values, tail.percentile) + suffix = f"p{round(tail.percentile * 100)}" + if tail.minimum is not None: + gates.append( + _absolute_gate( + f"tail:{tail.metric}:{suffix}:minimum", + tail.metric, + "global", + observed, + tail.minimum, + ">=", + tail.severity, + f"tails.{tail.metric}.minimum", + all_case_ids, + ) + ) + if tail.maximum is not None: + gates.append( + _absolute_gate( + f"tail:{tail.metric}:{suffix}:maximum", + tail.metric, + "global", + observed, + tail.maximum, + "<=", + tail.severity, + f"tails.{tail.metric}.maximum", + all_case_ids, + ) + ) + gated = tuple(_apply_waiver(item, policy.waivers, now_value) for item in gates) + unwaived_failures = [item for item in gated if not item.passed and not item.waived] + if any(item.severity == "block" for item in unwaived_failures): + decision = "BLOCK" + elif unwaived_failures or any(item.waived for item in gated): + decision = "WARN" + else: + decision = "PASS" + return ReleaseDecision("1.0", scenario.id, decision, decision == "PASS", gated) + + +def _metric_gates( + policy: PolicyGate, + baseline: dict[str, float], + candidate: dict[str, float], + *, + scope: str, + case_ids: tuple[str, ...], + policy_prefix: str, +) -> list[GateEvidence]: + if policy.metric not in baseline or policy.metric not in candidate: + raise ContractError(f"Release metric {policy.metric!r} is unavailable") + observed = candidate[policy.metric] + result = [] + gate_prefix = "global" if scope == "global" else f"slice:{scope}" + if policy.minimum is not None: + result.append( + _absolute_gate( + f"{gate_prefix}:{policy.metric}:minimum", + policy.metric, + scope, + observed, + policy.minimum, + ">=", + policy.severity, + f"{policy_prefix}.minimum", + case_ids, + ) + ) + if policy.maximum is not None: + result.append( + _absolute_gate( + f"{gate_prefix}:{policy.metric}:maximum", + policy.metric, + scope, + observed, + policy.maximum, + "<=", + policy.severity, + f"{policy_prefix}.maximum", + case_ids, + ) + ) + if policy.max_regression is not None: + delta = observed - baseline[policy.metric] + threshold = -policy.max_regression if policy.direction == "higher" else policy.max_regression + comparator = ">=" if policy.direction == "higher" else "<=" + result.append( + _absolute_gate( + f"{gate_prefix}:{policy.metric}:regression", + policy.metric, + scope, + delta, + threshold, + comparator, + policy.severity, + f"{policy_prefix}.max_regression", + case_ids, + ) + ) + return result + + +def _absolute_gate( + gate_id: str, + metric: str, + scope: str, + observed: float, + threshold: float, + comparator: str, + severity: str, + policy_path: str, + case_ids: tuple[str, ...], +) -> GateEvidence: + passed = observed >= threshold if comparator == ">=" else observed <= threshold + return GateEvidence( + id=gate_id, + metric=metric, + scope=scope, + observed=float(observed), + threshold=float(threshold), + comparator=comparator, + passed=passed, + severity=severity, + policy_path=policy_path, + case_ids=case_ids, + ) + + +def _apply_waiver( + gate: GateEvidence, waivers: tuple[PolicyWaiver, ...], now: datetime +) -> GateEvidence: + if gate.passed: + return gate + for waiver in waivers: + if waiver.gate_id != gate.id or (waiver.scope is not None and waiver.scope != gate.scope): + continue + issued = _timestamp(waiver.issued_at, f"waiver {waiver.gate_id} issued_at") + expires = _timestamp(waiver.expires_at, f"waiver {waiver.gate_id} expires_at") + if issued <= now < expires: + return replace( + gate, + waived=True, + waiver_owner=waiver.owner, + waiver_reason=waiver.reason, + waiver_expires_at=waiver.expires_at, + ) + return gate + + +def _validate_policy(policy: ReleasePolicyV2) -> None: + if isinstance(policy.minimum_cases, bool) or not isinstance(policy.minimum_cases, int) or policy.minimum_cases <= 0: + raise ContractError("Release minimum_cases must be a positive integer") + if policy.insufficient_evidence not in {"warn", "block"}: + raise ContractError("insufficient_evidence must be warn or block") + metric_names = [item.metric for item in policy.metrics] + if any(not name for name in metric_names) or len(set(metric_names)) != len(metric_names): + raise ContractError("Release metric names must be non-empty and unique") + for gate in policy.metrics: + if gate.direction not in {"higher", "lower"}: + raise ContractError(f"Release metric {gate.metric!r} direction must be higher or lower") + if (gate.minimum is None) == (gate.maximum is None): + raise ContractError( + f"Release metric {gate.metric!r} must define exactly one absolute threshold" + ) + _finite(gate.minimum if gate.minimum is not None else gate.maximum, gate.metric) + if gate.max_regression is not None and _finite(gate.max_regression, gate.metric) < 0: + raise ContractError(f"Release metric {gate.metric!r} max_regression must be non-negative") + _severity(gate.severity) + slice_names = [item.selector.name for item in policy.slices] + if any(not name for name in slice_names) or len(set(slice_names)) != len(slice_names): + raise ContractError("Release slice names must be non-empty and unique") + for item in policy.slices: + selector = item.selector + if not any((selector.category, selector.severity, selector.language, selector.tags)): + raise ContractError(f"Release slice {selector.name!r} must select at least one field") + if isinstance(item.minimum_cases, bool) or not isinstance(item.minimum_cases, int) or item.minimum_cases <= 0: + raise ContractError(f"Release slice {selector.name!r} minimum_cases must be positive") + for tail in policy.tails: + if not tail.metric: + raise ContractError("Release tail metric must be non-empty") + if not 0 < _finite(tail.percentile, tail.metric) <= 1: + raise ContractError(f"Release tail {tail.metric!r} percentile must be in (0, 1]") + if (tail.minimum is None) == (tail.maximum is None): + raise ContractError(f"Release tail {tail.metric!r} must define one threshold") + _finite(tail.minimum if tail.minimum is not None else tail.maximum, tail.metric) + _severity(tail.severity) + waiver_keys = [(item.gate_id, item.scope) for item in policy.waivers] + if len(set(waiver_keys)) != len(waiver_keys): + raise ContractError("Release waivers must have unique gate_id and scope") + for waiver in policy.waivers: + if not all((waiver.gate_id, waiver.owner, waiver.reason)): + raise ContractError("Release waiver gate_id, owner, and reason must be non-empty") + issued = _timestamp(waiver.issued_at, f"waiver {waiver.gate_id} issued_at") + expires = _timestamp(waiver.expires_at, f"waiver {waiver.gate_id} expires_at") + if expires <= issued: + raise ContractError(f"Release waiver {waiver.gate_id!r} must expire after issuance") + + +def _case_metric(result: CaseResult, name: str) -> float: + standard = { + "citation_coverage": result.citation_coverage, + "citation_precision": result.citation_precision, + "lexical_groundedness": result.lexical_groundedness, + "latency_ms": float(result.latency_ms), + "cost_usd": result.cost_usd, + "finding_count": float(len(result.findings)), + } + if name in standard: + return standard[name] + if name in result.custom_metrics: + return result.custom_metrics[name] + raise ContractError(f"Release tail metric {name!r} is unavailable") + + +def _timestamp(value: object, label: str) -> datetime: + if not isinstance(value, str): + raise ContractError(f"{label} must be an ISO-8601 timestamp") + try: + parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) + except ValueError as exc: + raise ContractError(f"{label} must be an ISO-8601 timestamp") from exc + if parsed.tzinfo is None: + raise ContractError(f"{label} must include a timezone") + return parsed + + +def _finite(value: object, label: str) -> float: + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise ContractError(f"{label} threshold must be numeric") + result = float(value) + if not math.isfinite(result): + raise ContractError(f"{label} threshold must be finite") + return result + + +def _severity(value: str) -> None: + if value not in {"warn", "block"}: + raise ContractError("Release gate severity must be warn or block") + + +def _exact_mapping(value: object, expected: set[str], label: str) -> dict[str, Any]: + if not isinstance(value, dict) or set(value) != expected: + raise ContractError(f"Release policy {label} fields must be {sorted(expected)}") + return value + + +def _subset_mapping(value: object, allowed: set[str], label: str) -> dict[str, Any]: + if not isinstance(value, dict): + raise ContractError(f"Release policy {label} must be a table") + unknown = sorted(set(value) - allowed) + if unknown: + raise ContractError(f"Release policy {label} has unknown fields: {unknown}") + return value diff --git a/src/ragops/reporters.py b/src/ragops/reporters.py index 970f952..8a6c3ae 100644 --- a/src/ragops/reporters.py +++ b/src/ragops/reporters.py @@ -1,6 +1,7 @@ from __future__ import annotations import html +from typing import Any from ragops.models import ( ComparisonReport, @@ -101,6 +102,60 @@ def comparison_html(report: ComparisonReport) -> str: """ +def demo_profile_markdown( + profile: str, + report: ComparisonReport, + evidence_manifest: dict[str, Any], +) -> str: + decision = "PASS" if report.passed else "BLOCK" + common = [ + f"# RAGOps {profile} demo view: {decision}", + "", + "Evidence classification: `synthetic`.", + "This credential-free benchmark does not demonstrate live deployment or customer usage.", + "", + ] + if profile == "executive": + common.extend( + [ + "## Release risk", + "", + "The candidate loses required citations and grounded answers while latency and cost remain acceptable.", + "", + "## Required action", + "", + "Hold release, repair the affected retrieval/answer path, then rerun the same evidence contract.", + ] + ) + elif profile == "engineer": + common.extend( + [ + "## Metric deltas", + "", + "| Metric | Baseline | Candidate | Delta |", + "| --- | ---: | ---: | ---: |", + ] + ) + for name, delta in report.deltas.items(): + common.append( + f"| {name} | {_format(report.baseline.metrics[name])} | " + f"{_format(report.candidate.metrics[name])} | {_signed(delta)} |" + ) + common.extend(["", "## Blocking gates", ""]) + common.extend(f"- `{gate}`" for gate in report.failed_gates) + elif profile == "auditor": + common.extend(["## Evidence manifest", ""]) + common.extend( + f"- `{item['logical_name']}`: `{item['sha256']}`" + for item in evidence_manifest["artifacts"] + ) + common.extend(["", "## Declared limitations", ""]) + common.extend(f"- {item}" for item in evidence_manifest["limitations"]) + else: + raise ValueError(f"unknown demo profile: {profile}") + return "\n".join(common) + "\n" + + def statistical_comparison_markdown(report: StatisticalComparisonReport) -> str: status = "PASS" if report.passed else "BLOCK" confidence = f"{report.confidence:.1%}" diff --git a/src/ragops/slices.py b/src/ragops/slices.py new file mode 100644 index 0000000..b7f86f6 --- /dev/null +++ b/src/ragops/slices.py @@ -0,0 +1,88 @@ +from __future__ import annotations + +from dataclasses import asdict, dataclass +from statistics import fmean +from typing import Any + +from ragops.loader import ContractError +from ragops.models import CaseResult, EvalCase, EvaluationReport, Scenario + + +@dataclass(frozen=True) +class SliceSelector: + name: str + category: str | None = None + severity: str | None = None + language: str | None = None + tags: tuple[str, ...] = () + + def matches(self, case: EvalCase) -> bool: + return ( + (self.category is None or case.category == self.category) + and (self.severity is None or case.severity == self.severity) + and (self.language is None or case.language == self.language) + and all(tag in case.tags for tag in self.tags) + ) + + +@dataclass(frozen=True) +class SliceSummary: + name: str + count: int + case_ids: tuple[str, ...] + metrics: dict[str, float] + + def to_dict(self) -> dict[str, Any]: + return asdict(self) + + +def summarize_slices( + scenario: Scenario, + report: EvaluationReport, + selectors: tuple[SliceSelector, ...], +) -> tuple[SliceSummary, ...]: + if report.scenario_id != scenario.id: + raise ContractError( + f"Slice report scenario mismatch: {report.scenario_id!r} != {scenario.id!r}" + ) + result_by_id = {result.case_id: result for result in report.cases} + expected_ids = {case.id for case in scenario.cases} + if set(result_by_id) != expected_ids: + raise ContractError("Slice aggregation requires complete scenario case coverage") + names = [selector.name for selector in selectors] + if any(not name for name in names) or len(set(names)) != len(names): + raise ContractError("Slice selector names must be non-empty and unique") + summaries = [] + for selector in selectors: + matched_cases = tuple(case for case in scenario.cases if selector.matches(case)) + matched_results = tuple(result_by_id[case.id] for case in matched_cases) + metrics = _mean_metrics(matched_results) if matched_results else {} + summaries.append( + SliceSummary( + name=selector.name, + count=len(matched_cases), + case_ids=tuple(case.id for case in matched_cases), + metrics=metrics, + ) + ) + return tuple(summaries) + + +def _mean_metrics(results: tuple[CaseResult, ...]) -> dict[str, float]: + standard = { + "citation_coverage": lambda item: item.citation_coverage, + "citation_precision": lambda item: item.citation_precision, + "lexical_groundedness": lambda item: item.lexical_groundedness, + "latency_ms": lambda item: float(item.latency_ms), + "cost_usd": lambda item: item.cost_usd, + "finding_count": lambda item: float(len(item.findings)), + } + metrics = {name: fmean(accessor(item) for item in results) for name, accessor in standard.items()} + custom_names = set.intersection(*(set(item.custom_metrics) for item in results)) + metrics.update( + { + name: fmean(item.custom_metrics[name] for item in results) + for name in sorted(custom_names) + } + ) + return metrics diff --git a/src/ragops/trace_graph.py b/src/ragops/trace_graph.py new file mode 100644 index 0000000..0c98862 --- /dev/null +++ b/src/ragops/trace_graph.py @@ -0,0 +1,323 @@ +from __future__ import annotations + +import json +import math +from collections import Counter +from dataclasses import asdict, dataclass +from pathlib import Path +from typing import Any, Mapping + +from ragops.loader import ContractError + + +@dataclass(frozen=True) +class TraceSpan: + id: str + parent_id: str | None + kind: str + operation: str + started_at_ms: float + ended_at_ms: float + status: str + input: dict[str, Any] + output: dict[str, Any] + attributes: dict[str, Any] + + +@dataclass(frozen=True) +class TraceGraph: + schema_version: str + trace_id: str + spans: tuple[TraceSpan, ...] + metadata: dict[str, Any] + + def to_dict(self) -> dict[str, Any]: + return asdict(self) + + +@dataclass(frozen=True) +class ExpectedTool: + name: str + arguments: dict[str, Any] + + +@dataclass(frozen=True) +class TraceExpectation: + expected_tools: tuple[ExpectedTool, ...] = () + expected_document_ids: tuple[str, ...] = () + maximum_steps: int = 1 + + +@dataclass(frozen=True) +class TraceGraphReport: + schema_version: str + trace_id: str + decision: str + metrics: dict[str, float] + failed_gates: tuple[str, ...] + failed_span_ids: tuple[str, ...] + + def to_dict(self) -> dict[str, Any]: + return asdict(self) + + +def load_trace_graph(path: str | Path) -> TraceGraph: + return trace_graph_from_dict(_read_json(path)) + + +def load_trace_expectation(path: str | Path) -> TraceExpectation: + data = _read_json(path) + if not isinstance(data, dict) or set(data) != { + "expected_tools", + "expected_document_ids", + "maximum_steps", + }: + raise ContractError("Trace expectation fields do not match the fixed contract") + raw_tools = data["expected_tools"] + if not isinstance(raw_tools, list): + raise ContractError("Trace expected_tools must be an array") + tools = [] + for index, item in enumerate(raw_tools): + if not isinstance(item, dict) or set(item) != {"name", "arguments"}: + raise ContractError(f"Trace expected tool {index} fields are invalid") + if not isinstance(item["arguments"], dict): + raise ContractError(f"Trace expected tool {index} arguments must be an object") + tools.append(ExpectedTool(item["name"], _json_object(item["arguments"], "tool arguments"))) + document_ids = data["expected_document_ids"] + if not isinstance(document_ids, list) or any(not isinstance(item, str) for item in document_ids): + raise ContractError("Trace expected_document_ids must be a string array") + expectation = TraceExpectation(tuple(tools), tuple(document_ids), data["maximum_steps"]) + _validate_expectation(expectation) + return expectation + + +def trace_graph_from_dict(data: object) -> TraceGraph: + if not isinstance(data, dict) or set(data) != {"schema_version", "trace_id", "spans", "metadata"}: + raise ContractError("Trace graph fields do not match schema 1.0") + if data["schema_version"] != "1.0": + raise ContractError(f"Unsupported trace graph schema: {data['schema_version']}") + trace_id = data["trace_id"] + if not isinstance(trace_id, str) or not trace_id: + raise ContractError("Trace graph trace_id must be a non-empty string") + raw_spans = data["spans"] + if not isinstance(raw_spans, list) or not raw_spans: + raise ContractError("Trace graph spans must be a non-empty array") + spans = tuple(_span_from_dict(item, index) for index, item in enumerate(raw_spans)) + identifiers = [span.id for span in spans] + if len(set(identifiers)) != len(identifiers): + raise ContractError("Trace span IDs must be unique") + identifier_set = set(identifiers) + for span in spans: + if span.parent_id is not None and span.parent_id not in identifier_set: + raise ContractError(f"Trace span {span.id!r} references missing parent {span.parent_id!r}") + _reject_cycles(spans) + roots = [span.id for span in spans if span.parent_id is None] + if len(roots) != 1: + raise ContractError(f"Trace graph requires exactly one root span; roots={roots}") + return TraceGraph("1.0", trace_id, spans, _json_object(data["metadata"], "trace metadata")) + + +def evaluate_trace_graph(graph: TraceGraph, expectation: TraceExpectation) -> TraceGraphReport: + _validate_expectation(expectation) + tool_spans = tuple(span for span in graph.spans if span.kind == "tool") + expected_names = tuple(item.name for item in expectation.expected_tools) + actual_names = tuple(item.operation for item in tool_spans) + expected_counts = Counter(expected_names) + actual_counts = Counter(actual_names) + matched_tools = sum(min(count, expected_counts[name]) for name, count in actual_counts.items()) + precision = matched_tools / len(actual_names) if actual_names else (1.0 if not expected_names else 0.0) + recall = matched_tools / len(expected_names) if expected_names else 1.0 + argument_matches = 0 + wrong_argument_ids = [] + used_actual: set[int] = set() + for expected in expectation.expected_tools: + matching_index = next( + ( + index + for index, span in enumerate(tool_spans) + if index not in used_actual and span.operation == expected.name + ), + None, + ) + if matching_index is None: + continue + used_actual.add(matching_index) + span = tool_spans[matching_index] + arguments = span.input.get("arguments", {}) + if arguments == expected.arguments: + argument_matches += 1 + else: + wrong_argument_ids.append(span.id) + argument_exactness = ( + argument_matches / len(expectation.expected_tools) if expectation.expected_tools else 1.0 + ) + ordering = 1.0 if actual_names == expected_names else 0.0 + success_rate = ( + sum(span.status == "ok" for span in tool_spans) / len(tool_spans) + if tool_spans + else (1.0 if not expectation.expected_tools else 0.0) + ) + retrieved = { + document_id + for span in graph.spans + if span.kind == "retrieval" + for document_id in span.output.get("document_ids", []) + if isinstance(document_id, str) + } + expected_documents = set(expectation.expected_document_ids) + retrieval_recall = ( + len(retrieved & expected_documents) / len(expected_documents) if expected_documents else 1.0 + ) + ordered_spans = sorted(graph.spans, key=lambda span: (span.started_at_ms, span.id)) + failed_spans = [span for span in ordered_spans if span.status == "error"] + recovered = [] + for failed in failed_spans: + recovered.append( + any( + later.started_at_ms > failed.started_at_ms + and later.operation == failed.operation + and later.status == "ok" + for later in ordered_spans + ) + ) + error_recovery = sum(recovered) / len(recovered) if recovered else 1.0 + excess = max(0, len(graph.spans) - expectation.maximum_steps) / expectation.maximum_steps + metrics = { + "tool_name_precision": precision, + "tool_name_recall": recall, + "tool_argument_exactness": argument_exactness, + "tool_ordering": ordering, + "tool_success_rate": success_rate, + "retrieval_document_recall": retrieval_recall, + "error_recovery": error_recovery, + "excess_step_ratio": float(excess), + } + gate_ids = { + "tool_name_precision": "tool_names", + "tool_name_recall": "tool_names", + "tool_argument_exactness": "tool_arguments", + "tool_ordering": "tool_order", + "tool_success_rate": "tool_success", + "retrieval_document_recall": "retrieval_documents", + "error_recovery": "error_recovery", + "excess_step_ratio": "excess_steps", + } + failed_gates = tuple( + dict.fromkeys( + gate_ids[name] + for name, value in metrics.items() + if (name == "excess_step_ratio" and value > 0.0) + or (name != "excess_step_ratio" and value < 1.0) + ) + ) + failed_span_ids = tuple( + dict.fromkeys([*wrong_argument_ids, *(span.id for span in failed_spans)]) + ) + return TraceGraphReport( + schema_version="1.0", + trace_id=graph.trace_id, + decision="PASS" if not failed_gates else "BLOCK", + metrics=metrics, + failed_gates=failed_gates, + failed_span_ids=failed_span_ids, + ) + + +def _span_from_dict(value: object, index: int) -> TraceSpan: + expected = { + "id", + "parent_id", + "kind", + "operation", + "started_at_ms", + "ended_at_ms", + "status", + "input", + "output", + "attributes", + } + if not isinstance(value, dict) or set(value) != expected: + raise ContractError(f"Trace span {index} fields do not match schema 1.0") + span_id = value["id"] + if not isinstance(span_id, str) or not span_id: + raise ContractError(f"Trace span {index} id must be non-empty") + parent_id = value["parent_id"] + if parent_id is not None and (not isinstance(parent_id, str) or not parent_id): + raise ContractError(f"Trace span {span_id!r} parent_id is invalid") + if value["kind"] not in {"agent", "llm", "tool", "retrieval", "workflow", "other"}: + raise ContractError(f"Trace span {span_id!r} kind is invalid") + if not isinstance(value["operation"], str) or not value["operation"]: + raise ContractError(f"Trace span {span_id!r} operation must be non-empty") + started = _finite(value["started_at_ms"], f"Trace span {span_id!r} started_at_ms") + ended = _finite(value["ended_at_ms"], f"Trace span {span_id!r} ended_at_ms") + if ended < started: + raise ContractError(f"Trace span {span_id!r} cannot end before it starts") + if value["status"] not in {"ok", "error", "unset"}: + raise ContractError(f"Trace span {span_id!r} status is invalid") + return TraceSpan( + id=span_id, + parent_id=parent_id, + kind=value["kind"], + operation=value["operation"], + started_at_ms=started, + ended_at_ms=ended, + status=value["status"], + input=_json_object(value["input"], f"Trace span {span_id!r} input"), + output=_json_object(value["output"], f"Trace span {span_id!r} output"), + attributes=_json_object(value["attributes"], f"Trace span {span_id!r} attributes"), + ) + + +def _reject_cycles(spans: tuple[TraceSpan, ...]) -> None: + parents = {span.id: span.parent_id for span in spans} + for identifier in parents: + seen = set() + current: str | None = identifier + while current is not None: + if current in seen: + raise ContractError(f"Trace graph contains a parent cycle at {current!r}") + seen.add(current) + current = parents[current] + + +def _validate_expectation(expectation: TraceExpectation) -> None: + if ( + isinstance(expectation.maximum_steps, bool) + or not isinstance(expectation.maximum_steps, int) + or expectation.maximum_steps <= 0 + ): + raise ContractError("Trace maximum_steps must be a positive integer") + if any(not item.name for item in expectation.expected_tools): + raise ContractError("Trace expected tool names must be non-empty") + if len(set(expectation.expected_document_ids)) != len(expectation.expected_document_ids): + raise ContractError("Trace expected document IDs must be unique") + + +def _read_json(path: str | Path) -> Any: + try: + return json.loads( + Path(path).read_text(encoding="utf-8"), + parse_constant=lambda value: (_ for _ in ()).throw( + ValueError(f"non-finite JSON number {value}") + ), + ) + except (OSError, json.JSONDecodeError, ValueError) as exc: + raise ContractError(f"Cannot load trace JSON from {path}: {exc}") from exc + + +def _json_object(value: object, label: str) -> dict[str, Any]: + if not isinstance(value, Mapping): + raise ContractError(f"{label} must be an object") + try: + return json.loads(json.dumps(dict(value), allow_nan=False)) + except (TypeError, ValueError) as exc: + raise ContractError(f"{label} must contain finite JSON values") from exc + + +def _finite(value: object, label: str) -> float: + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise ContractError(f"{label} must be numeric") + number = float(value) + if not math.isfinite(number): + raise ContractError(f"{label} must be finite") + return number diff --git a/src/ragops/web/__init__.py b/src/ragops/web/__init__.py new file mode 100644 index 0000000..37e1d94 --- /dev/null +++ b/src/ragops/web/__init__.py @@ -0,0 +1 @@ +"""Packaged, self-contained RAGOps workbench assets.""" diff --git a/src/ragops/web/app.js b/src/ragops/web/app.js new file mode 100644 index 0000000..5fd5dbc --- /dev/null +++ b/src/ragops/web/app.js @@ -0,0 +1,14 @@ +const byId=(id)=>document.getElementById(id); +const scenario=byId('scenario'),responses=byId('responses'),apiKey=byId('api-key'); +scenario.value='{"schema_version":"0.2","id":"demo","name":"Demo","thresholds":{"citation_coverage":1,"citation_precision":1,"lexical_groundedness":0.5,"max_latency_ms":3000,"max_cost_usd":0.03},"cases":[{"id":"q1","question":"What is the policy?","evidence":["The policy requires human approval."],"required_citation_ids":["policy-1"],"category":"policy","severity":"high","language":"en"}]}'; +responses.value='[{"case_id":"q1","answer":"The policy requires human approval.","citation_ids":["policy-1"],"latency_ms":900,"cost_usd":0.01}]'; +const authHeaders=()=>apiKey.value?{'x-api-key':apiKey.value}:{}; +const clear=(node)=>node.replaceChildren(); +const element=(tag,text,className)=>{const node=document.createElement(tag);node.textContent=String(text);if(className)node.className=className;return node}; + +function renderMetrics(metrics){const container=byId('metrics');clear(container);Object.entries(metrics).forEach(([key,value])=>{const row=element('div','', 'metric');row.append(element('span',key),element('strong',Number(value).toFixed(4)));container.append(row)})} +function renderGates(gates){const container=byId('gates');clear(container);(gates.length?gates:['None']).forEach((gate)=>container.append(element('li',gate)))} +function renderRuns(runs){const container=byId('runs');clear(container);if(!runs.length){container.append(element('p','No runs saved.'));return}runs.forEach((run)=>{const row=element('div','', 'run');row.append(element('b',run.label||run.id.slice(0,8)),element('span',run.scenario_id),element('span',run.review_status,'badge'),element('span',run.passed?'PASS':'FAIL'));container.append(row)})} + +byId('run').addEventListener('click',async()=>{const status=byId('status'),errorMessage=byId('evaluation-error');errorMessage.hidden=true;try{const response=await fetch('/v1/evaluate',{method:'POST',headers:{'content-type':'application/json',...authHeaders()},body:JSON.stringify({scenario:JSON.parse(scenario.value),responses:JSON.parse(responses.value)})});const data=await response.json();if(!response.ok)throw Error(data.detail||'Evaluation failed');status.textContent=data.passed?'PASS':'FAIL';status.className='status '+(data.passed?'pass':'fail');renderMetrics(data.metrics);renderGates(data.failed_gates)}catch(error){status.textContent='NOT EVALUATED';status.className='status';errorMessage.textContent=error.message;errorMessage.hidden=false;clear(byId('metrics'));renderGates(['Not evaluated'])}}); +byId('load').addEventListener('click',async()=>{const message=byId('history-message'),limit=byId('limit').value;try{const response=await fetch(`/v1/runs?limit=${encodeURIComponent(limit)}`,{headers:authHeaders()});const data=await response.json();if(!response.ok)throw Error(data.detail||'History failed');message.textContent=`${data.length} saved runs`;message.className='lede';renderRuns(data)}catch(error){message.textContent=error.message;message.className='lede error';clear(byId('runs'))}}); diff --git a/src/ragops/web/index.html b/src/ragops/web/index.html new file mode 100644 index 0000000..8675a0a --- /dev/null +++ b/src/ragops/web/index.html @@ -0,0 +1,19 @@ + + + + + + + RAGOps Workbench + + + +
+
Release confidence for AI systems

RAGOps

Evaluate a candidate locally, then inspect and review saved team runs from the same evidence contract.

+
+

Evaluation input

+

Release decision

Waiting for a run

Failed gates

  • Not evaluated
+
Team workflow

Experiment history

Set RAGOPS_STORE on the API to enable shared local history and review metadata.

+
+
+ diff --git a/src/ragops/web/styles.css b/src/ragops/web/styles.css new file mode 100644 index 0000000..8f492b8 --- /dev/null +++ b/src/ragops/web/styles.css @@ -0,0 +1 @@ +:root{color-scheme:dark;--bg:#08110f;--ink:#eef3ed;--muted:#98aaa3;--panel:#111e1a;--line:#2a4038;--mint:#75e8ba;--red:#ff8580;--sans:system-ui,-apple-system,BlinkMacSystemFont,"Segoe UI",sans-serif;--mono:ui-monospace,"SFMono-Regular",Menlo,Monaco,Consolas,monospace}*{box-sizing:border-box}body{margin:0;background:var(--bg);color:var(--ink);font:15px/1.5 var(--sans)}main{max-width:1180px;margin:auto;padding:48px 24px}header{display:flex;justify-content:space-between;gap:24px;align-items:end;border-bottom:1px solid var(--line);padding-bottom:28px}.eyebrow{color:var(--mint);font-weight:800;letter-spacing:.12em;text-transform:uppercase;font-size:11px}h1{font-size:clamp(40px,6vw,72px);line-height:1;margin:8px 0}.lede{max-width:620px;color:var(--muted);font-size:17px}.grid{display:grid;grid-template-columns:1fr 1fr;gap:18px;margin-top:30px}.panel{background:var(--panel);border:1px solid var(--line);border-radius:16px;padding:22px}.wide{grid-column:1/-1}label{display:block;margin:12px 0 6px;color:var(--muted)}textarea,input{width:100%;background:#08120f;color:var(--ink);border:1px solid var(--line);border-radius:9px;padding:11px;font:12px var(--mono)}textarea{min-height:190px}button{margin-top:14px;background:var(--mint);border:0;border-radius:9px;padding:10px 16px;font-weight:800;color:#06120e;cursor:pointer}.secondary{background:transparent;color:var(--ink);border:1px solid var(--line)}.toolbar{display:flex;gap:10px;align-items:end}.toolbar>div{flex:1}.status{font-size:30px;font-weight:900}.pass{color:var(--mint)}.fail,.error{color:var(--red)}.metric,.run{display:grid;grid-template-columns:1.5fr repeat(3,1fr);gap:10px;padding:10px 0;border-bottom:1px solid var(--line)}.metric strong{text-align:right}.runs{margin-top:18px}.run{grid-template-columns:1.5fr 1fr 1fr 1fr;color:var(--muted)}.run b{color:var(--ink)}.badge{width:max-content;border-radius:99px;padding:2px 8px;background:#20342d;font-size:11px}code{color:#a5c7ff}@media(max-width:760px){.grid{grid-template-columns:1fr}.wide{grid-column:auto}header{display:block}.toolbar{display:grid}.metric{grid-template-columns:1.5fr 1fr}.run{grid-template-columns:1fr 1fr}} diff --git a/tests/test_adapter_sdk.py b/tests/test_adapter_sdk.py new file mode 100644 index 0000000..9d6cc0a --- /dev/null +++ b/tests/test_adapter_sdk.py @@ -0,0 +1,85 @@ +from __future__ import annotations + +import json + +import pytest + +from ragops.adapter_sdk import AdapterContext, discover_adapters +from ragops.adapters.import_profiles import convert_import +from ragops.cli import main +from ragops.loader import ContractError + + +@pytest.mark.parametrize( + ("profile", "payload"), + [ + ("ragas", {"records": [{"case_id": "q1", "faithfulness": 0.91}]}), + ("deepeval", {"test_results": [{"name": "q1", "metrics": [{"name": "faithfulness", "score": 0.92}]}]}), + ("phoenix", {"evaluations": [{"span_id": "q1", "name": "groundedness", "score": 0.93}]}), + ("langsmith", {"runs": [{"reference_example_id": "q1", "feedback": [{"key": "correctness", "score": 0.94}]}]}), + ("mlflow", {"results": [{"request_id": "q1", "metrics": {"relevance": 0.95}}]}), + ("promptfoo", {"results": [{"vars": {"case_id": "q1"}, "scores": {"faithfulness": 0.96}}]}), + ], +) +def test_builtin_profiles_normalize_without_vendor_sdks(profile, payload): + output = convert_import(profile, payload, AdapterContext(case_ids=("q1",))) + + assert output.records[0]["case_id"] == "q1" + assert output.records[0]["metrics"] + assert output.provenance["producer"] == profile + assert output.provenance["adapter_version"] == "1.0" + + +def test_import_preserves_unmapped_fields_and_reports_missing_cases(): + output = convert_import( + "ragas", + {"records": [{"case_id": "q1", "faithfulness": 0.9, "note": "manual"}]}, + AdapterContext(case_ids=("q1", "q2")), + ) + + assert output.provenance["unmapped_fields"] == ["note"] + assert output.warnings == ("missing case IDs: q2",) + + +def test_unknown_case_and_non_finite_score_fail_closed(): + with pytest.raises(ContractError, match="unknown case ID"): + convert_import( + "ragas", + {"records": [{"case_id": "q9", "faithfulness": 0.9}]}, + AdapterContext(case_ids=("q1",)), + ) + with pytest.raises(ContractError, match="finite numeric"): + convert_import( + "mlflow", + {"results": [{"request_id": "q1", "metrics": {"score": float("nan")}}]}, + AdapterContext(), + ) + + +def test_adapter_discovery_includes_builtins_without_loading_vendors(): + names = [item.name for item in discover_adapters(include_entry_points=False)] + + assert names == ["custom", "deepeval", "langsmith", "mlflow", "phoenix", "promptfoo", "ragas"] + + +def test_adapter_convert_cli_writes_portable_envelope(tmp_path, monkeypatch): + source = tmp_path / "source.json" + output = tmp_path / "portable.json" + source.write_text( + json.dumps({"results": [{"request_id": "q1", "metrics": {"relevance": 0.95}}]}), + encoding="utf-8", + ) + monkeypatch.setattr( + "sys.argv", + [ + "ragops", "adapter", "convert", "--profile", "mlflow", "--input", str(source), + "--case-id", "q1", "--output", str(output), + ], + ) + + assert main() == 0 + payload = json.loads(output.read_text(encoding="utf-8")) + assert payload["schema_version"] == "0.1" + assert payload["provider"] == "mlflow" + assert payload["records"] == [{"case_id": "q1", "metrics": {"relevance": 0.95}}] + assert payload["provenance"]["producer"] == "mlflow" diff --git a/tests/test_adoption_recipes.py b/tests/test_adoption_recipes.py index ea85cad..c9400b5 100644 --- a/tests/test_adoption_recipes.py +++ b/tests/test_adoption_recipes.py @@ -5,7 +5,7 @@ def test_gitlab_recipe_preserves_evidence_and_exit_code() -> None: recipe = Path("docs/examples/gitlab-ci-ragops.yml").read_text(encoding="utf-8") - assert 'RAGOPS_VERSION: "1.2.0"' in recipe + assert 'RAGOPS_VERSION: "2.0.0"' in recipe assert 'if: \'$CI_PIPELINE_SOURCE == "merge_request_event"\'' in recipe assert "gate_exit=$?" in recipe assert 'exit "$gate_exit"' in recipe @@ -30,3 +30,16 @@ def test_downstream_github_publisher_is_copyable_and_least_privilege() -> None: assert re.search(r"ref: [0-9a-f]{40}", recipe) assert re.search(r"actions/checkout@[0-9a-f]{40}", recipe) assert "python .ragops-publisher/apps/github_pr_comment.py" in recipe + + +def test_github_caller_is_read_only_and_verifies_evidence() -> None: + recipe = Path("docs/examples/github-actions-ragops.yml").read_text(encoding="utf-8") + + assert "pull_request:" in recipe + assert "contents: read" in recipe + assert "id-token: write" not in recipe + assert "persist-credentials: false" in recipe + assert re.search(r"actions/checkout@[0-9a-f]{40}", recipe) + assert '"ragops==2.0.0"' in recipe + assert "ragops evidence verify" in recipe + assert '>> "$GITHUB_STEP_SUMMARY"' in recipe diff --git a/tests/test_calibration.py b/tests/test_calibration.py new file mode 100644 index 0000000..4edc81f --- /dev/null +++ b/tests/test_calibration.py @@ -0,0 +1,134 @@ +import json +import math +from pathlib import Path + +import pytest + +from ragops.calibration import ( + CalibrationPolicy, + CalibrationRecord, + HumanLabel, + calibrate_evaluator, + load_calibration_set, +) +from ragops.cli import main +from ragops.loader import ContractError + + +def _records() -> tuple[CalibrationRecord, ...]: + return ( + CalibrationRecord("q1", 0.9, (HumanLabel("alice", 1), HumanLabel("bob", 1))), + CalibrationRecord("q2", 0.1, (HumanLabel("alice", 1), HumanLabel("bob", 1))), + CalibrationRecord("q3", 0.9, (HumanLabel("alice", 0), HumanLabel("bob", 0))), + CalibrationRecord("q4", 0.1, (HumanLabel("alice", 0), HumanLabel("bob", 0))), + ) + + +def _policy() -> CalibrationPolicy: + return CalibrationPolicy( + threshold=0.5, + minimum_coverage=1.0, + minimum_accuracy=0.5, + maximum_mae=0.5, + maximum_false_pass_rate=0.25, + minimum_agreement=0.5, + ) + + +def test_calibration_reports_false_decisions_and_reviewer_agreement() -> None: + report = calibrate_evaluator("groundedness-v1", _records(), _policy()) + + assert report.coverage == 1.0 + assert report.accuracy == 0.5 + assert report.mae == pytest.approx(0.5) + assert report.false_pass_rate == 0.25 + assert report.false_block_rate == 0.25 + assert report.reviewer_agreement == 1.0 + assert report.decision == "PASS" + + +def test_calibration_blocks_evaluator_misaligned_with_reviewers() -> None: + records = ( + CalibrationRecord("q1", 0.9, (HumanLabel("alice", 1), HumanLabel("bob", 1))), + CalibrationRecord("q2", 0.9, (HumanLabel("alice", 1), HumanLabel("bob", 0))), + CalibrationRecord("q3", 0.1, (HumanLabel("alice", 0), HumanLabel("bob", 1))), + CalibrationRecord("q4", 0.1, (HumanLabel("alice", 0), HumanLabel("bob", 0))), + ) + + report = calibrate_evaluator("judge", records, _policy()) + + assert report.reviewer_agreement == 0.0 + assert "reviewer_agreement" in report.failed_gates + assert report.decision == "BLOCK" + + +def test_calibration_reports_insufficient_reviewer_overlap() -> None: + records = ( + CalibrationRecord("q1", 0.9, (HumanLabel("alice", 1),)), + CalibrationRecord("q2", 0.1, (HumanLabel("bob", 0),)), + ) + + report = calibrate_evaluator("judge", records, _policy()) + + assert report.reviewer_agreement is None + assert "reviewer_agreement_insufficient" in report.failed_gates + assert report.decision == "BLOCK" + + +def test_calibration_rejects_duplicate_reviewer_label_for_case() -> None: + records = ( + CalibrationRecord("q1", 0.9, (HumanLabel("alice", 1), HumanLabel("alice", 0))), + ) + + with pytest.raises(ContractError, match="duplicate reviewer"): + calibrate_evaluator("judge", records, _policy()) + + +@pytest.mark.parametrize("score", [math.nan, math.inf, -0.1, 1.1]) +def test_calibration_rejects_invalid_automated_score(score: float) -> None: + with pytest.raises(ContractError, match="between 0 and 1"): + calibrate_evaluator( + "judge", (CalibrationRecord("q1", score, (HumanLabel("alice", 1),)),), _policy() + ) + + +def test_load_calibration_set_and_cli(tmp_path: Path, monkeypatch, capsys) -> None: + source = tmp_path / "calibration.json" + output = tmp_path / "report.json" + source.write_text( + json.dumps( + { + "schema_version": "1.0", + "evaluator": "groundedness-v1", + "policy": { + "threshold": 0.5, + "minimum_coverage": 1.0, + "minimum_accuracy": 0.5, + "maximum_mae": 0.5, + "maximum_false_pass_rate": 0.25, + "minimum_agreement": 0.5, + }, + "records": [ + { + "case_id": record.case_id, + "automated_score": record.automated_score, + "human_labels": [ + {"reviewer": label.reviewer, "label": label.label} + for label in record.human_labels + ], + } + for record in _records() + ], + } + ), + encoding="utf-8", + ) + + loaded = load_calibration_set(source) + assert loaded.evaluator == "groundedness-v1" + monkeypatch.setattr( + "sys.argv", ["ragops", "calibrate", "--input", str(source), "--output", str(output)] + ) + assert main() == 0 + assert json.loads(output.read_text(encoding="utf-8"))["decision"] == "PASS" + assert json.loads(capsys.readouterr().out)["decision"] == "PASS" diff --git a/tests/test_contracts_v2.py b/tests/test_contracts_v2.py new file mode 100644 index 0000000..059550c --- /dev/null +++ b/tests/test_contracts_v2.py @@ -0,0 +1,137 @@ +import json +from pathlib import Path + +import pytest + +from ragops.cli import main +from ragops.contracts import diff_contract, migrate_contract, validate_contract +from ragops.loader import ContractError + + +def scenario_01_fixture() -> dict: + return { + "schema_version": "0.1", + "id": "support", + "name": "Support", + "thresholds": { + "citation_coverage": 1.0, + "lexical_groundedness": 0.5, + "max_latency_ms": 1000, + "max_cost_usd": 0.01, + }, + "cases": [ + { + "id": "q1", + "question": "What is the policy?", + "evidence": ["Human approval is required."], + "required_citation_ids": ["policy-1"], + } + ], + } + + +def trace_03_fixture() -> dict: + return { + "case_id": "q1", + "output": {"answer": "Human approval is required.", "citation_ids": ["policy-1"]}, + "latency_ms": 10, + "usage": {"cost_usd": 0.001}, + } + + +def test_migrate_scenario_01_to_02_is_explicit_and_valid() -> None: + source = scenario_01_fixture() + + migrated = migrate_contract(source, "scenario", "0.1", "0.2") + + assert source["schema_version"] == "0.1" + assert migrated["schema_version"] == "0.2" + assert migrated["thresholds"]["citation_precision"] == 0.0 + assert migrated["cases"][0]["category"] == "unspecified" + assert migrated["cases"][0]["severity"] == "medium" + assert migrated["cases"][0]["language"] == "und" + result = validate_contract(migrated, "scenario") + assert result.kind == "scenario" + assert result.version == "0.2" + + +def test_migrate_trace_03_to_04_preserves_payload_and_records_migration() -> None: + migrated = migrate_contract(trace_03_fixture(), "trace", "0.3", "0.4", question="Policy?") + + assert migrated["schema_version"] == "0.4" + assert migrated["input"] == {"question": "Policy?"} + assert migrated["output"]["answer"] == "Human approval is required." + assert migrated["metadata"]["migrated_from"] == "0.3" + assert validate_contract(migrated, "trace").version == "0.4" + + +def test_unknown_contract_version_fails_closed() -> None: + with pytest.raises(ContractError, match="Unsupported scenario schema"): + validate_contract({**scenario_01_fixture(), "schema_version": "9.9"}, "scenario") + + +def test_contract_diff_reports_required_field_additions() -> None: + difference = diff_contract("scenario", "0.1", "0.2").to_dict() + + assert difference["compatibility"] == "migration-required" + assert "thresholds.citation_precision" in difference["added_required_fields"] + assert "cases[].severity" in difference["added_required_fields"] + + +def test_contract_migrate_cli_never_overwrites_input( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + source = tmp_path / "scenario.json" + target = tmp_path / "scenario-v02.json" + source.write_text(json.dumps(scenario_01_fixture()), encoding="utf-8") + monkeypatch.setattr( + "sys.argv", + [ + "ragops", + "contract", + "migrate", + "--kind", + "scenario", + "--from", + "0.1", + "--to", + "0.2", + "--input", + str(source), + "--output", + str(target), + ], + ) + + assert main() == 0 + assert json.loads(source.read_text(encoding="utf-8"))["schema_version"] == "0.1" + assert json.loads(target.read_text(encoding="utf-8"))["schema_version"] == "0.2" + assert json.loads(capsys.readouterr().out)["output"] == str(target) + + +def test_contract_migrate_cli_rejects_same_input_and_output( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + source = tmp_path / "scenario.json" + source.write_text(json.dumps(scenario_01_fixture()), encoding="utf-8") + monkeypatch.setattr( + "sys.argv", + [ + "ragops", + "contract", + "migrate", + "--kind", + "scenario", + "--from", + "0.1", + "--to", + "0.2", + "--input", + str(source), + "--output", + str(source), + ], + ) + + with pytest.raises(SystemExit, match="must differ"): + main() diff --git a/tests/test_datasets.py b/tests/test_datasets.py new file mode 100644 index 0000000..11e5fb7 --- /dev/null +++ b/tests/test_datasets.py @@ -0,0 +1,171 @@ +import json +from pathlib import Path + +import pytest + +from ragops.cli import main +from ragops.datasets import ( + create_dataset_manifest, + diff_dataset_manifests, + load_dataset_manifest, + validate_dataset_manifest, + write_dataset_manifest, +) +from ragops.loader import ContractError + + +def _case(case_id: str, question: str, *, reviewed: bool = True) -> dict: + return { + "id": case_id, + "question": question, + "evidence": ["Human approval is required."], + "required_citation_ids": ["policy-1"], + "category": "policy", + "severity": "critical", + "language": "en", + "tags": ["approval"], + "source": "support-export-2026-08", + "reviewed": reviewed, + } + + +def test_dataset_validation_quarantines_cross_split_leakage() -> None: + manifest = create_dataset_manifest( + "support", + "1.0", + (_case("a", "Reset password"), _case("b", " RESET PASSWORD ")), + splits={"train": ("a",), "eval": ("b",)}, + source_classification="production-derived", + owners=("eval-team",), + ) + + issues = validate_dataset_manifest(manifest) + + assert {item.code for item in issues} >= {"normalized_content_leakage"} + leakage = next(item for item in issues if item.code == "normalized_content_leakage") + assert leakage.case_ids == ("a", "b") + + +def test_production_case_requires_review_before_promotion() -> None: + manifest = create_dataset_manifest( + "support", + "1.0", + (_case("a", "Reset password", reviewed=False),), + splits={"eval": ("a",)}, + source_classification="production-derived", + owners=("eval-team",), + ) + + assert "review_required" in {item.code for item in validate_dataset_manifest(manifest)} + + +def test_dataset_validation_enforces_minimum_slice_coverage() -> None: + manifest = create_dataset_manifest( + "support", + "1.0", + (_case("a", "Reset password"),), + splits={"eval": ("a",)}, + source_classification="synthetic", + owners=("eval-team",), + ) + + issues = validate_dataset_manifest(manifest, minimum_slice_counts={"severity=critical": 2}) + + issue = next(item for item in issues if item.code == "slice_coverage") + assert issue.observed == 1 + assert issue.required == 2 + + +def test_dataset_diff_is_deterministic() -> None: + before = create_dataset_manifest( + "support", + "1.0", + (_case("a", "Reset password"), _case("b", "Billing")), + splits={"eval": ("a", "b")}, + source_classification="synthetic", + owners=("eval-team",), + ) + after = create_dataset_manifest( + "support", + "1.1", + (_case("a", "Reset password safely"), _case("c", "Shipping")), + splits={"eval": ("a", "c")}, + source_classification="synthetic", + owners=("eval-team",), + ) + + difference = diff_dataset_manifests(before, after) + + assert difference.added_case_ids == ("c",) + assert difference.removed_case_ids == ("b",) + assert difference.changed_case_ids == ("a",) + + +def test_dataset_creation_rejects_duplicate_case_ids() -> None: + with pytest.raises(ContractError, match="case IDs must be unique"): + create_dataset_manifest( + "support", + "1.0", + (_case("a", "One"), _case("a", "Two")), + splits={"eval": ("a",)}, + source_classification="synthetic", + owners=("eval-team",), + ) + + +def test_dataset_manifest_round_trip_and_cli_validation( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + manifest = create_dataset_manifest( + "support", + "1.0", + (_case("a", "Reset password"),), + splits={"eval": ("a",)}, + source_classification="synthetic", + owners=("eval-team",), + ) + path = tmp_path / "dataset.json" + write_dataset_manifest(path, manifest) + + assert load_dataset_manifest(path) == manifest + monkeypatch.setattr("sys.argv", ["ragops", "dataset", "validate", "--manifest", str(path)]) + assert main() == 0 + assert json.loads(capsys.readouterr().out)["valid"] is True + + +def test_dataset_cli_creates_and_diffs_manifests( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + output = tmp_path / "dataset.json" + monkeypatch.setattr( + "sys.argv", + [ + "ragops", + "dataset", + "create", + "--scenario", + "scenarios/japanese_troubleshooting/scenario.json", + "--dataset-id", + "support", + "--version", + "1.0", + "--source-classification", + "synthetic", + "--owner", + "eval-team", + "--split", + "eval=error-e42,escalation-e42", + "--reviewed", + "--output", + str(output), + ], + ) + assert main() == 0 + assert json.loads(capsys.readouterr().out)["dataset_id"] == "support" + + monkeypatch.setattr( + "sys.argv", + ["ragops", "dataset", "diff", "--before", str(output), "--after", str(output)], + ) + assert main() == 0 + assert json.loads(capsys.readouterr().out)["changed_case_ids"] == [] diff --git a/tests/test_evidence_bundle.py b/tests/test_evidence_bundle.py new file mode 100644 index 0000000..41985c8 --- /dev/null +++ b/tests/test_evidence_bundle.py @@ -0,0 +1,155 @@ +import json +import os +from pathlib import Path + +import pytest + +from ragops.cli import main +from ragops.evidence import create_evidence_bundle, verify_evidence_bundle +from ragops.loader import ContractError + +CREATED_AT = "2026-08-23T00:00:00Z" + + +def test_evidence_bundle_is_reproducible_and_detects_tampering(tmp_path: Path) -> None: + source = tmp_path / "report.json" + source.write_text('{"passed":false}\n', encoding="utf-8") + + first = create_evidence_bundle( + tmp_path / "one", + {"report": source}, + "BLOCK", + ("synthetic benchmark",), + CREATED_AT, + {"commit": "abc123"}, + ) + second = create_evidence_bundle( + tmp_path / "two", + {"report": source}, + "BLOCK", + ("synthetic benchmark",), + CREATED_AT, + {"commit": "abc123"}, + ) + + assert first.canonical_bytes() == second.canonical_bytes() + bundled = tmp_path / "one" / first.artifacts[0].path + bundled.write_bytes(b"X" * first.artifacts[0].bytes) + with pytest.raises(ContractError, match="digest mismatch"): + verify_evidence_bundle(tmp_path / "one") + + +def test_evidence_bundle_verifies_complete_artifacts(tmp_path: Path) -> None: + report = tmp_path / "release-report.md" + report.write_text("# BLOCK\n", encoding="utf-8") + create_evidence_bundle( + tmp_path / "evidence", + {"release_report": report}, + "BLOCK", + ("synthetic", "not production adoption"), + CREATED_AT, + {}, + ) + + verified = verify_evidence_bundle(tmp_path / "evidence") + + assert verified.decision == "BLOCK" + assert verified.artifacts[0].logical_name == "release_report" + assert verified.artifacts[0].bytes == len(b"# BLOCK\n") + + +def test_evidence_verification_rejects_manifest_path_traversal(tmp_path: Path) -> None: + report = tmp_path / "report.json" + report.write_text("{}", encoding="utf-8") + create_evidence_bundle( + tmp_path / "evidence", {"report": report}, "PASS", ("synthetic",), CREATED_AT, {} + ) + path = tmp_path / "evidence" / "manifest.json" + payload = json.loads(path.read_text(encoding="utf-8")) + payload["artifacts"][0]["path"] = "../report.json" + path.write_text(json.dumps(payload), encoding="utf-8") + + with pytest.raises(ContractError, match="artifact path"): + verify_evidence_bundle(tmp_path / "evidence") + + +def test_evidence_verification_rejects_undeclared_file(tmp_path: Path) -> None: + report = tmp_path / "report.json" + report.write_text("{}", encoding="utf-8") + create_evidence_bundle( + tmp_path / "evidence", {"report": report}, "PASS", ("synthetic",), CREATED_AT, {} + ) + (tmp_path / "evidence" / "artifacts" / "extra.txt").write_text("extra", encoding="utf-8") + + with pytest.raises(ContractError, match="undeclared files"): + verify_evidence_bundle(tmp_path / "evidence") + + +def test_evidence_creation_rejects_invalid_decision(tmp_path: Path) -> None: + report = tmp_path / "report.json" + report.write_text("{}", encoding="utf-8") + + with pytest.raises(ContractError, match="PASS, WARN, or BLOCK"): + create_evidence_bundle( + tmp_path / "evidence", {"report": report}, "FAIL", ("synthetic",), CREATED_AT, {} + ) + + +@pytest.mark.skipif(not hasattr(os, "symlink"), reason="symlinks are unavailable") +def test_evidence_creation_rejects_symlinked_artifact(tmp_path: Path) -> None: + source = tmp_path / "source.json" + source.write_text("{}", encoding="utf-8") + linked = tmp_path / "linked.json" + linked.symlink_to(source) + + with pytest.raises(ContractError, match="regular non-symlink"): + create_evidence_bundle( + tmp_path / "evidence", {"report": linked}, "PASS", ("synthetic",), CREATED_AT, {} + ) + + +def test_evidence_verification_rejects_unknown_schema_version(tmp_path: Path) -> None: + report = tmp_path / "report.json" + report.write_text("{}", encoding="utf-8") + create_evidence_bundle( + tmp_path / "evidence", {"report": report}, "PASS", ("synthetic",), CREATED_AT, {} + ) + path = tmp_path / "evidence" / "manifest.json" + payload = json.loads(path.read_text(encoding="utf-8")) + payload["schema_version"] = "9.9" + path.write_text(json.dumps(payload), encoding="utf-8") + + with pytest.raises(ContractError, match="Unsupported evidence bundle schema"): + verify_evidence_bundle(tmp_path / "evidence") + + +def test_evidence_cli_creates_and_verifies_bundle( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + report = tmp_path / "report.json" + report.write_text("{}", encoding="utf-8") + bundle = tmp_path / "bundle" + monkeypatch.setattr( + "sys.argv", + [ + "ragops", + "evidence", + "create", + "--bundle", + str(bundle), + "--decision", + "PASS", + "--artifact", + f"report={report}", + "--limitation", + "synthetic", + "--created-at", + CREATED_AT, + ], + ) + assert main() == 0 + assert json.loads(capsys.readouterr().out)["decision"] == "PASS" + + monkeypatch.setattr("sys.argv", ["ragops", "evidence", "verify", "--bundle", str(bundle)]) + assert main() == 0 + assert json.loads(capsys.readouterr().out)["verified"] is True diff --git a/tests/test_explain_ci.py b/tests/test_explain_ci.py new file mode 100644 index 0000000..8286df8 --- /dev/null +++ b/tests/test_explain_ci.py @@ -0,0 +1,131 @@ +from __future__ import annotations + +import json +import xml.etree.ElementTree as ET + +import pytest + +from ragops.ci import github_summary, junit_xml, sarif_report +from ragops.cli import main +from ragops.explain import explain_decision +from ragops.loader import ContractError +from ragops.policy_v2 import GateEvidence, ReleaseDecision, load_release_decision + + +def _decision() -> ReleaseDecision: + return ReleaseDecision( + schema_version="1.0", + scenario_id="support", + decision="BLOCK", + passed=False, + gates=( + GateEvidence( + id="global:citation_coverage:regression", + metric="citation_coverage", + scope="global", + observed=-0.5, + threshold=0.0, + comparator=">=", + passed=False, + severity="block", + policy_path="metrics.citation_coverage.max_regression", + case_ids=("q2",), + ), + GateEvidence( + id="global:latency_ms:maximum", + metric="latency_ms", + scope="global", + observed=80.0, + threshold=100.0, + comparator="<=", + passed=True, + severity="block", + policy_path="metrics.latency_ms.maximum", + case_ids=("q1", "q2"), + ), + GateEvidence( + id="slice:ja:groundedness:minimum", + metric="groundedness", + scope="ja", + observed=0.6, + threshold=0.8, + comparator=">=", + passed=False, + severity="block", + policy_path="slices.ja.metrics.groundedness.minimum", + case_ids=("q2",), + waived=True, + waiver_owner="ml-platform", + waiver_reason="migration window", + waiver_expires_at="2026-09-01T00:00:00Z", + ), + ), + ) + + +def test_explanation_points_to_policy_cases_and_required_change(): + explanation = explain_decision(_decision()) + + assert explanation.decision == "BLOCK" + failed = explanation.items[0] + assert failed.policy_path == "metrics.citation_coverage.max_regression" + assert failed.case_ids == ("q2",) + assert failed.required_change == 0.5 + assert explanation.items[1].waived is True + + +def test_junit_marks_blocking_failure_and_waiver_as_skipped(): + root = ET.fromstring(junit_xml(_decision())) + + assert root.attrib == {"name": "ragops: support", "tests": "3", "failures": "1", "skipped": "1"} + assert len(root.findall(".//failure")) == 1 + assert len(root.findall(".//skipped")) == 1 + + +def test_sarif_and_github_summary_are_bounded_and_escape_untrusted_text(): + decision = _decision() + sarif = sarif_report(decision) + summary = github_summary(decision, max_gates=2) + + assert sarif["version"] == "2.1.0" + assert sarif["runs"][0]["results"][0]["ruleId"] == "global:citation_coverage:regression" + assert "## RAGOps release gate: BLOCK" in summary + assert len(summary.splitlines()) < 30 + + +def test_explain_cli_renders_all_ci_formats(tmp_path, monkeypatch, capsys): + source = tmp_path / "decision.json" + source.write_text(json.dumps(_decision().to_dict()), encoding="utf-8") + + monkeypatch.setattr("sys.argv", ["ragops", "explain", "--input", str(source)]) + assert main() == 0 + assert json.loads(capsys.readouterr().out)["decision"] == "BLOCK" + + for output_format, suffix in (("junit", "xml"), ("sarif", "json"), ("github", "md")): + output = tmp_path / f"report.{suffix}" + monkeypatch.setattr( + "sys.argv", + [ + "ragops", + "ci-render", + "--input", + str(source), + "--format", + output_format, + "--output", + str(output), + ], + ) + assert main() == 0 + assert output.stat().st_size > 20 + + +@pytest.mark.parametrize(("field", "value"), [("passed", "false"), ("waived", 0), ("id", 42)]) +def test_release_decision_loader_rejects_type_coercion(tmp_path, field, value): + payload = _decision().to_dict() + payload["gates"][0][field] = value + source = tmp_path / "decision.json" + source.write_text(json.dumps(payload), encoding="utf-8") + + with pytest.raises(ContractError): + load_release_decision(source) diff --git a/tests/test_flagship_demo.py b/tests/test_flagship_demo.py new file mode 100644 index 0000000..9a44a95 --- /dev/null +++ b/tests/test_flagship_demo.py @@ -0,0 +1,67 @@ +from __future__ import annotations + +import json +import xml.etree.ElementTree as ET +from pathlib import Path + +import pytest + +from ragops.cli import main +from ragops.demo import write_demo +from ragops.evidence import verify_evidence_bundle + + +@pytest.mark.parametrize("profile", ["executive", "engineer", "auditor"]) +def test_profiles_share_one_synthetic_block_decision(tmp_path: Path, profile: str) -> None: + output = tmp_path / profile + summary = write_demo(output, scenario_id="japanese-troubleshooting", profile=profile) + + assert summary["candidate_decision"] == "BLOCK" + assert summary["evidence_classification"] == "synthetic" + assert summary["profile"] == profile + assert verify_evidence_bundle(output / "evidence").decision == "BLOCK" + profile_text = (output / "profile-summary.md").read_text(encoding="utf-8") + assert "synthetic" in profile_text.casefold() + assert "demonstrates production adoption" not in profile_text.casefold() + + +def test_demo_formats_share_stable_gate_ids_and_metrics(tmp_path: Path) -> None: + output = tmp_path / "demo" + summary = write_demo(output, profile="engineer") + decision = json.loads((output / "decision.json").read_text(encoding="utf-8")) + comparison = json.loads((output / "comparison.json").read_text(encoding="utf-8")) + sarif = json.loads((output / "release.sarif").read_text(encoding="utf-8")) + junit = ET.parse(output / "junit.xml").getroot() + github = (output / "github-summary.md").read_text(encoding="utf-8") + + gate_ids = [gate["id"] for gate in decision["gates"]] + assert gate_ids == summary["failed_gates"] + assert [item["ruleId"] for item in sarif["runs"][0]["results"]] == gate_ids + assert [item.attrib["name"] for item in junit.findall("testcase")] == gate_ids + assert all(gate_id in github for gate_id in gate_ids) + assert comparison["baseline"]["metrics"]["citation_coverage"] == 1.0 + assert comparison["candidate"]["metrics"]["citation_coverage"] == 0.5 + assert comparison["candidate"]["metrics"]["citation_precision"] == 0.5 + assert comparison["candidate"]["metrics"]["lexical_groundedness"] == 0.6 + + +def test_demo_evidence_declares_limitations_and_verifies_after_force(tmp_path: Path) -> None: + output = tmp_path / "demo" + write_demo(output) + write_demo(output, force=True) + + manifest = verify_evidence_bundle(output / "evidence") + logical_names = {item.logical_name for item in manifest.artifacts} + assert {"scenario", "baseline", "candidate", "comparison", "decision", "html", "junit", "sarif"} <= logical_names + assert manifest.metadata["evidence_classification"] == "synthetic" + assert any("not production adoption" in item.casefold() for item in manifest.limitations) + + +def test_demo_cli_accepts_profile(monkeypatch, capsys, tmp_path: Path) -> None: + output = tmp_path / "auditor" + monkeypatch.setattr( + "sys.argv", ["ragops", "demo", "--profile", "auditor", "--output", str(output)] + ) + + assert main() == 0 + assert json.loads(capsys.readouterr().out)["profile"] == "auditor" diff --git a/tests/test_github_workflows.py b/tests/test_github_workflows.py index 18d606a..fbc18bb 100644 --- a/tests/test_github_workflows.py +++ b/tests/test_github_workflows.py @@ -3,12 +3,21 @@ WORKFLOW_ROOT = Path(".github/workflows") -def test_repository_only_has_release_publisher() -> None: - assert sorted(path.name for path in WORKFLOW_ROOT.iterdir()) == ["publish-pypi.yml"] +def test_repository_has_consumer_ci_and_separate_release_publisher() -> None: + assert sorted(path.name for path in WORKFLOW_ROOT.iterdir()) == ["ci.yml", "publish-pypi.yml"] workflow = (WORKFLOW_ROOT / "publish-pypi.yml").read_text(encoding="utf-8") assert "id-token: write" in workflow assert "environment: pypi" in workflow assert "pull_request" not in workflow + ci = (WORKFLOW_ROOT / "ci.yml").read_text(encoding="utf-8") + assert "pull_request:" in ci + assert 'python-version: ${{ matrix.python-version }}' in ci + assert '["3.11", "3.12", "3.13"]' in ci + assert "ruff check ." in ci + assert "pytest -q" in ci + assert "ragops demo" in ci + assert "evidence verify" in ci + assert "id-token: write" not in ci def test_current_operations_are_linked() -> None: @@ -16,7 +25,7 @@ def test_current_operations_are_linked() -> None: for guide in ( "docs/ARCHITECTURE.md", "docs/OPERATIONS.md", - "docs/releases/v1.2.0.md", + "docs/releases/v2.0.0.md", ): assert Path(guide).is_file() assert guide in readme diff --git a/tests/test_governance.py b/tests/test_governance.py new file mode 100644 index 0000000..8a6675d --- /dev/null +++ b/tests/test_governance.py @@ -0,0 +1,133 @@ +from __future__ import annotations + +import hashlib +import json + +import pytest + +from ragops.cli import main +from ragops.governance import GovernanceStore + + +def _digest(value: str = "artifact") -> str: + return hashlib.sha256(value.encode()).hexdigest() + + +def test_artifact_state_machine_and_audit_are_transactional(tmp_path): + store = GovernanceStore(tmp_path / "governance.db") + store.register_artifact( + "eval-1", + kind="release-decision", + digest=_digest(), + public_metadata={"scenario_id": "support"}, + blinded_metadata={"candidate": "model-b"}, + actor="ci", + ) + + with pytest.raises(ValueError, match="draft -> accepted"): + store.transition("eval-1", "accepted", actor="owner") + + store.transition("eval-1", "reviewed", actor="owner") + store.record_review("eval-1", reviewer="alice", verdict="approve", note="evidence holds") + store.transition("eval-1", "accepted", actor="owner") + store.transition("eval-1", "superseded", actor="ci") + + assert store.get_artifact("eval-1")["state"] == "superseded" + assert [item["action"] for item in store.audit_events("eval-1")] == [ + "artifact.registered", + "artifact.transitioned", + "review.recorded", + "artifact.transitioned", + "artifact.transitioned", + ] + + +def test_blind_queue_reveals_candidate_only_after_reviewer_votes(tmp_path): + store = GovernanceStore(tmp_path / "governance.db") + store.register_artifact( + "eval-1", + kind="release-decision", + digest=_digest(), + public_metadata={"scenario_id": "support"}, + blinded_metadata={"candidate": "model-b", "provider": "vendor-x"}, + actor="ci", + ) + store.transition("eval-1", "reviewed", actor="owner") + + before = store.review_queue("alice")[0] + assert before["metadata"] == {"scenario_id": "support"} + assert before["blinded"] is True + + store.record_review("eval-1", reviewer="alice", verdict="block", note="weak slice") + + after = store.review_queue("alice")[0] + assert after["metadata"]["candidate"] == "model-b" + assert after["blinded"] is False + assert store.review_queue("bob")[0]["blinded"] is True + + +def test_reviews_are_append_only_and_acceptance_requires_approval(tmp_path): + store = GovernanceStore(tmp_path / "governance.db") + store.register_artifact( + "eval-1", kind="decision", digest=_digest(), public_metadata={}, blinded_metadata={}, actor="ci" + ) + store.transition("eval-1", "reviewed", actor="owner") + store.record_review("eval-1", reviewer="alice", verdict="block") + + with pytest.raises(ValueError, match="already reviewed"): + store.record_review("eval-1", reviewer="alice", verdict="approve") + with pytest.raises(ValueError, match="approval review"): + store.transition("eval-1", "accepted", actor="owner") + + +def test_waiver_is_audited_and_must_expire_after_issue(tmp_path): + store = GovernanceStore(tmp_path / "governance.db") + + waiver_id = store.create_waiver( + gate_id="slice:ja:coverage", + owner="ml-platform", + reason="migration", + issued_at="2026-08-23T00:00:00Z", + expires_at="2026-09-01T00:00:00Z", + actor="release-manager", + ) + + assert store.list_waivers()[0]["id"] == waiver_id + assert store.audit_events()[0]["action"] == "waiver.created" + with pytest.raises(ValueError, match="expire after"): + store.create_waiver( + gate_id="bad", owner="x", reason="x", + issued_at="2026-09-01T00:00:00Z", expires_at="2026-08-23T00:00:00Z", actor="x", + ) + + +def test_governance_cli_registers_and_reads_artifact(tmp_path, monkeypatch, capsys): + database = tmp_path / "governance.db" + metadata = tmp_path / "metadata.json" + metadata.write_text(json.dumps({"scenario_id": "support"}), encoding="utf-8") + monkeypatch.setattr( + "sys.argv", + [ + "ragops", "governance", "register", "--store", str(database), "--artifact-id", "eval-1", + "--kind", "decision", "--digest", _digest(), "--metadata", str(metadata), "--actor", "ci", + ], + ) + + assert main() == 0 + assert json.loads(capsys.readouterr().out)["state"] == "draft" + + monkeypatch.setattr( + "sys.argv", ["ragops", "governance", "audit", "--store", str(database), "--artifact-id", "eval-1"] + ) + assert main() == 0 + assert json.loads(capsys.readouterr().out)["events"][0]["action"] == "artifact.registered" + + +def test_artifact_metadata_must_be_json_objects(tmp_path): + store = GovernanceStore(tmp_path / "governance.db") + + with pytest.raises(ValueError, match="metadata must be a JSON object"): + store.register_artifact( + "eval-1", kind="decision", digest=_digest(), public_metadata=[], + blinded_metadata={}, actor="ci", + ) diff --git a/tests/test_local_release.py b/tests/test_local_release.py index 2b01316..879ba27 100644 --- a/tests/test_local_release.py +++ b/tests/test_local_release.py @@ -8,7 +8,7 @@ def test_tag_must_match_package_version() -> None: - assert local_release.assert_tag(local_release.milestone_tag()) == "v1.2.0" + assert local_release.assert_tag(local_release.milestone_tag()) == "v2.0.0" def test_checksum_manifest_is_deterministic(tmp_path, monkeypatch) -> None: @@ -53,3 +53,15 @@ def test_pypi_requires_explicit_token(monkeypatch) -> None: assert "PYPI_API_TOKEN" in str(error) else: raise AssertionError("publish must fail closed without a token") + + +def test_environment_tool_prefers_active_python_scripts_directory(tmp_path, monkeypatch) -> None: + scripts = tmp_path / "bin" + scripts.mkdir() + python = scripts / "python" + python.touch() + ruff = scripts / "ruff" + ruff.touch() + monkeypatch.setattr(local_release.sys, "executable", str(python)) + + assert local_release.environment_tool("ruff") == str(ruff) diff --git a/tests/test_packaged_api.py b/tests/test_packaged_api.py new file mode 100644 index 0000000..a46a08b --- /dev/null +++ b/tests/test_packaged_api.py @@ -0,0 +1,58 @@ +from __future__ import annotations + +from importlib.resources import files +from pathlib import Path + +import pytest + +pytest.importorskip("fastapi") + +from apps.api.main import app as compatibility_app +from ragops.api.main import SECURITY_HEADERS, app, dashboard, workbench_asset +from ragops.cli import build_parser + + +def test_api_compatibility_shim_uses_packaged_app(): + assert compatibility_app is app + + +def test_packaged_workbench_has_csp_and_no_untrusted_html_writes(): + web = files("ragops.web") + page = web.joinpath("index.html").read_text(encoding="utf-8") + script = web.joinpath("app.js").read_text(encoding="utf-8") + + assert "Content-Security-Policy" in page + assert "default-src 'self'" in page + assert "innerHTML" not in page + script + assert "outerHTML" not in page + script + assert "document.write" not in page + script + assert "http://" not in page + script + assert "https://" not in page + script + assert ".textContent" in script + assert "createElement" in script + + +def test_dashboard_serves_packaged_assets_with_security_headers(): + page = dashboard() + script = workbench_asset("app.js") + + assert Path(page.path).read_text(encoding="utf-8").startswith("") + assert "createElement" in Path(script.path).read_text(encoding="utf-8") + assert SECURITY_HEADERS["Content-Security-Policy"].startswith("default-src 'self'") + assert SECURITY_HEADERS["X-Content-Type-Options"] == "nosniff" + + +def test_serve_command_is_lazy_and_configurable(): + args = build_parser().parse_args(["serve", "--host", "127.0.0.1", "--port", "9010"]) + + assert args.command == "serve" + assert args.host == "127.0.0.1" + assert args.port == 9010 + + +def test_hatch_configuration_packages_api_web_and_schemas(): + project = Path("pyproject.toml").read_text(encoding="utf-8") + + assert '"schemas" = "ragops/schemas"' in project + assert Path("src/ragops/api/main.py").is_file() + assert Path("src/ragops/web/index.html").is_file() diff --git a/tests/test_plugin_release.py b/tests/test_plugin_release.py new file mode 100644 index 0000000..d66edd5 --- /dev/null +++ b/tests/test_plugin_release.py @@ -0,0 +1,52 @@ +from __future__ import annotations + +import importlib.util +import json +import subprocess +import sys +import zipfile +from pathlib import Path + +SPEC = importlib.util.spec_from_file_location("local_release", Path("scripts/local_release.py")) +assert SPEC and SPEC.loader +local_release = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(local_release) + + +def test_chatgpt_plugin_bundle_is_deterministic_and_self_contained(tmp_path, monkeypatch): + monkeypatch.setattr(local_release, "DIST", tmp_path) + + first = local_release.build_plugin_bundle("2.0.0") + first_bytes = first.read_bytes() + second = local_release.build_plugin_bundle("2.0.0") + + assert second.read_bytes() == first_bytes + with zipfile.ZipFile(first) as archive: + names = archive.namelist() + manifest = json.loads(archive.read(".codex-plugin/plugin.json")) + assert manifest["version"] == "2.0.0" + assert "skills/evaluate-ai-release/SKILL.md" in names + assert "skills/evaluate-ai-release/scripts/run_ragops.py" in names + assert "skills/evaluate-ai-release/scripts/vendor/ragops/cli.py" in names + assert "skills/evaluate-ai-release/scripts/vendor/ragops/schemas/evidence-bundle-1.0.schema.json" in names + assert "skills/evaluate-ai-release/references/fixtures/japanese_troubleshooting/scenario.json" in names + assert not any("__pycache__" in name or name.endswith(".pyc") for name in names) + + +def test_extracted_chatgpt_bundle_runs_without_install(tmp_path, monkeypatch): + monkeypatch.setattr(local_release, "DIST", tmp_path) + bundle = local_release.build_plugin_bundle("2.0.0") + extracted = tmp_path / "plugin" + with zipfile.ZipFile(bundle) as archive: + archive.extractall(extracted) + runner = extracted / "skills/evaluate-ai-release/scripts/run_ragops.py" + + result = subprocess.run( + [sys.executable, str(runner), "--version"], + cwd=extracted, + check=True, + capture_output=True, + text=True, + ) + + assert result.stdout.strip() == "ragops 2.0.0" diff --git a/tests/test_policy_v2.py b/tests/test_policy_v2.py new file mode 100644 index 0000000..2ec016c --- /dev/null +++ b/tests/test_policy_v2.py @@ -0,0 +1,251 @@ +from pathlib import Path + +import pytest + +from ragops.cli import main +from ragops.loader import ContractError, scenario_from_dict +from ragops.models import CaseResult, EvaluationReport +from ragops.policy_v2 import ( + PolicyGate, + PolicyWaiver, + ReleasePolicyV2, + SlicePolicy, + TailGate, + apply_release_policy, + load_release_policy_v2, +) +from ragops.slices import SliceSelector + +NOW = "2026-08-23T00:00:00Z" + + +def _scenario(): + return scenario_from_dict( + { + "schema_version": "0.2", + "id": "release-v2", + "name": "Release v2", + "thresholds": { + "citation_coverage": 0, + "citation_precision": 0, + "lexical_groundedness": 0, + "max_latency_ms": 5000, + "max_cost_usd": 1, + }, + "cases": [ + { + "id": "critical", + "question": "Critical?", + "evidence": ["Critical evidence"], + "required_citation_ids": ["c1"], + "category": "policy", + "severity": "critical", + "language": "ja", + }, + { + "id": "routine", + "question": "Routine?", + "evidence": ["Routine evidence"], + "required_citation_ids": ["c2"], + "category": "faq", + "severity": "low", + "language": "en", + }, + ], + } + ) + + +def _report(critical_coverage: float, routine_coverage: float, latencies=(100, 200)): + average = (critical_coverage + routine_coverage) / 2 + cases = ( + CaseResult("critical", critical_coverage, 1.0, 1.0, latencies[0], 0.001), + CaseResult("routine", routine_coverage, 1.0, 1.0, latencies[1], 0.001), + ) + return EvaluationReport( + report_version="0.1", + scenario_id="release-v2", + passed=True, + metrics={ + "citation_coverage": average, + "citation_precision": 1.0, + "lexical_groundedness": 1.0, + "avg_latency_ms": sum(latencies) / 2, + "avg_cost_usd": 0.001, + "critical_findings": 0.0, + }, + failed_gates=(), + cases=cases, + ) + + +def _policy(*, waiver: PolicyWaiver | None = None) -> ReleasePolicyV2: + return ReleasePolicyV2( + minimum_cases=1, + insufficient_evidence="block", + metrics=( + PolicyGate( + metric="citation_coverage", + direction="higher", + minimum=0.9, + severity="block", + ), + ), + slices=(), + tails=(), + waivers=(waiver,) if waiver else (), + ) + + +def test_active_waiver_can_only_change_block_to_warn() -> None: + waiver = PolicyWaiver( + gate_id="global:citation_coverage:minimum", + owner="release-owner", + reason="bounded rollout", + issued_at="2026-08-20T00:00:00Z", + expires_at="2030-01-01T00:00:00Z", + ) + + decision = apply_release_policy( + _policy(waiver=waiver), _report(1.0, 1.0), _report(0.5, 1.0), _scenario(), now=NOW + ) + + assert decision.decision == "WARN" + assert decision.passed is False + assert decision.gates[0].waived is True + assert decision.gates[0].waiver_owner == "release-owner" + + +def test_expired_waiver_leaves_gate_blocked() -> None: + waiver = PolicyWaiver( + gate_id="global:citation_coverage:minimum", + owner="release-owner", + reason="expired", + issued_at="2026-01-01T00:00:00Z", + expires_at="2026-08-01T00:00:00Z", + ) + + decision = apply_release_policy( + _policy(waiver=waiver), _report(1.0, 1.0), _report(0.5, 1.0), _scenario(), now=NOW + ) + + assert decision.decision == "BLOCK" + assert decision.gates[0].waived is False + + +def test_worst_slice_blocks_even_when_global_metric_passes() -> None: + policy = ReleasePolicyV2( + minimum_cases=1, + insufficient_evidence="block", + metrics=(PolicyGate("citation_coverage", "higher", minimum=0.7),), + slices=( + SlicePolicy( + selector=SliceSelector("critical", severity="critical"), minimum_cases=1 + ), + ), + tails=(), + waivers=(), + ) + + decision = apply_release_policy( + policy, _report(1.0, 1.0), _report(0.5, 1.0), _scenario(), now=NOW + ) + + assert decision.decision == "BLOCK" + assert any(gate.id == "slice:critical:citation_coverage:minimum" for gate in decision.gates) + failed = next(gate for gate in decision.gates if not gate.passed and gate.scope == "critical") + assert failed.case_ids == ("critical",) + + +def test_insufficient_slice_evidence_fails_closed() -> None: + policy = ReleasePolicyV2( + minimum_cases=1, + insufficient_evidence="block", + metrics=(PolicyGate("citation_coverage", "higher", minimum=0.4),), + slices=( + SlicePolicy( + selector=SliceSelector("critical", severity="critical"), minimum_cases=2 + ), + ), + tails=(), + waivers=(), + ) + + decision = apply_release_policy( + policy, _report(1.0, 1.0), _report(1.0, 1.0), _scenario(), now=NOW + ) + + gate = next(item for item in decision.gates if item.id == "slice:critical:minimum_cases") + assert gate.passed is False + assert decision.decision == "BLOCK" + + +def test_tail_gate_blocks_latency_p95() -> None: + policy = ReleasePolicyV2( + minimum_cases=1, + insufficient_evidence="warn", + metrics=(), + slices=(), + tails=(TailGate("latency_ms", 0.95, maximum=500.0),), + waivers=(), + ) + + decision = apply_release_policy( + policy, _report(1.0, 1.0), _report(1.0, 1.0, latencies=(100, 1000)), _scenario(), now=NOW + ) + + gate = next(item for item in decision.gates if item.id == "tail:latency_ms:p95:maximum") + assert gate.observed == pytest.approx(955.0) + assert decision.decision == "BLOCK" + + +def test_policy_loader_rejects_unknown_section(tmp_path: Path) -> None: + path = tmp_path / "policy.toml" + path.write_text("[release]\nminimum_cases=1\ninsufficient_evidence='block'\n[unknown]\nx=1\n") + + with pytest.raises(ContractError, match="unknown sections"): + load_release_policy_v2(path) + + +def test_gate_v2_cli_returns_block_exit_code( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + policy = tmp_path / "policy.toml" + policy.write_text( + """ +[release] +minimum_cases = 1 +insufficient_evidence = "block" + +[metrics.citation_coverage] +direction = "higher" +minimum = 1.0 +max_regression = 0.0 +severity = "block" +""".strip() + + "\n", + encoding="utf-8", + ) + scenario = Path("scenarios/japanese_troubleshooting/benchmark-v0.2.json") + baseline = Path("scenarios/japanese_troubleshooting/benchmark-baseline.json") + candidate = Path("scenarios/japanese_troubleshooting/benchmark-regressed.json") + monkeypatch.setattr( + "sys.argv", + [ + "ragops", + "gate-v2", + "--scenario", + str(scenario), + "--baseline", + str(baseline), + "--candidate", + str(candidate), + "--policy", + str(policy), + "--now", + NOW, + ], + ) + + assert main() == 2 + assert '"decision": "BLOCK"' in capsys.readouterr().out diff --git a/tests/test_public_api.py b/tests/test_public_api.py index 7c26a7f..5c1d8ea 100644 --- a/tests/test_public_api.py +++ b/tests/test_public_api.py @@ -3,7 +3,7 @@ def test_stable_version() -> None: - assert __version__ == "1.2.0" + assert __version__ == "2.0.0" def test_cli_exposes_version(capsys) -> None: diff --git a/tests/test_release_version.py b/tests/test_release_version.py index 66ff036..7403292 100644 --- a/tests/test_release_version.py +++ b/tests/test_release_version.py @@ -1,3 +1,4 @@ +import json import tomllib from pathlib import Path @@ -9,6 +10,19 @@ def test_package_and_release_metadata_versions_match() -> None: version = metadata["project"]["version"] changelog = Path("CHANGELOG.md").read_text(encoding="utf-8") - assert version == __version__ + assert version == __version__ == "2.0.0" assert f"## [{version}]" in changelog - assert Path("docs/releases/v1.2.0.md").is_file() + assert Path("docs/releases/v2.0.0.md").is_file() + + +def test_plugin_manifests_match_package_version() -> None: + versions = { + json.loads(Path(path).read_text(encoding="utf-8"))["version"] + for path in ( + ".codex-plugin/plugin.json", + ".claude-plugin/plugin.json", + ".kimi-plugin/plugin.json", + ) + } + + assert versions == {"2.0.0"} diff --git a/tests/test_schema_files.py b/tests/test_schema_files.py index 4df9b95..0ad222c 100644 --- a/tests/test_schema_files.py +++ b/tests/test_schema_files.py @@ -1,4 +1,5 @@ import json +import tomllib from pathlib import Path @@ -10,3 +11,10 @@ def test_all_published_schemas_are_valid_json() -> None: document = json.loads(schema.read_text(encoding="utf-8")) assert document["$schema"] == "https://json-schema.org/draft/2020-12/schema" assert document["$id"] + + +def test_wheel_configuration_packages_public_schemas() -> None: + project = tomllib.loads(Path("pyproject.toml").read_text(encoding="utf-8")) + + force_include = project["tool"]["hatch"]["build"]["targets"]["wheel"]["force-include"] + assert force_include["schemas"] == "ragops/schemas" diff --git a/tests/test_slice_distribution.py b/tests/test_slice_distribution.py new file mode 100644 index 0000000..3372f7c --- /dev/null +++ b/tests/test_slice_distribution.py @@ -0,0 +1,125 @@ +import math + +import pytest + +from ragops.distribution import holm_adjust, minimum_paired_sample, paired_effect, quantile +from ragops.loader import scenario_from_dict +from ragops.models import CaseResult, EvaluationReport +from ragops.slices import SliceSelector, summarize_slices + + +def _scenario(): + return scenario_from_dict( + { + "schema_version": "0.2", + "id": "slices", + "name": "Slice checks", + "thresholds": { + "citation_coverage": 0, + "citation_precision": 0, + "lexical_groundedness": 0, + "max_latency_ms": 1000, + "max_cost_usd": 1, + }, + "cases": [ + { + "id": "ja-critical", + "question": "Q1", + "evidence": ["E1"], + "required_citation_ids": ["e1"], + "category": "policy", + "severity": "critical", + "language": "ja", + "tags": ["production"], + }, + { + "id": "en-low", + "question": "Q2", + "evidence": ["E2"], + "required_citation_ids": ["e2"], + "category": "faq", + "severity": "low", + "language": "en", + "tags": ["synthetic"], + }, + ], + } + ) + + +def _report() -> EvaluationReport: + return EvaluationReport( + report_version="0.1", + scenario_id="slices", + passed=False, + metrics={}, + failed_gates=("citation_coverage",), + cases=( + CaseResult("ja-critical", 0.0, 0.5, 0.4, 900, 0.02), + CaseResult("en-low", 1.0, 1.0, 1.0, 100, 0.001), + ), + ) + + +def test_slice_summary_exposes_worst_language_and_severity_slice() -> None: + summaries = summarize_slices( + _scenario(), + _report(), + (SliceSelector("ja-critical", language="ja", severity="critical"),), + ) + + assert summaries[0].case_ids == ("ja-critical",) + assert summaries[0].count == 1 + assert summaries[0].metrics["citation_coverage"] == 0.0 + assert summaries[0].metrics["latency_ms"] == 900.0 + + +def test_slice_selector_combines_category_and_tag() -> None: + summaries = summarize_slices( + _scenario(), + _report(), + (SliceSelector("production-policy", category="policy", tags=("production",)),), + ) + + assert summaries[0].case_ids == ("ja-critical",) + + +def test_empty_slice_has_no_fabricated_metrics() -> None: + summary = summarize_slices( + _scenario(), _report(), (SliceSelector("missing", language="vi"),) + )[0] + + assert summary.count == 0 + assert summary.metrics == {} + + +def test_quantile_and_holm_are_hand_checkable() -> None: + assert quantile((0.0, 10.0, 20.0, 30.0), 0.5) == 15.0 + assert holm_adjust({"a": 0.01, "b": 0.04, "c": 0.03}) == { + "a": pytest.approx(0.03), + "b": pytest.approx(0.06), + "c": pytest.approx(0.06), + } + + +def test_paired_effect_uses_candidate_minus_baseline() -> None: + result = paired_effect((1.0, 2.0, 4.0), (2.0, 4.0, 5.0)) + + assert result.mean_delta == pytest.approx(4 / 3) + assert result.median_delta == 1.0 + assert result.standardized_effect == pytest.approx(2.309401076758503) + + +def test_minimum_sample_estimate_is_deterministic() -> None: + assert minimum_paired_sample(effect=0.5, alpha=0.05, power=0.8) == 32 + + +@pytest.mark.parametrize("values", [(), (math.nan,), (math.inf,)]) +def test_quantile_rejects_empty_or_nonfinite_values(values: tuple[float, ...]) -> None: + with pytest.raises(ValueError): + quantile(values, 0.5) + + +def test_distribution_rejects_invalid_probability() -> None: + with pytest.raises(ValueError, match="probability"): + quantile((1.0,), 1.1) diff --git a/tests/test_trace_graph.py b/tests/test_trace_graph.py new file mode 100644 index 0000000..3ee0907 --- /dev/null +++ b/tests/test_trace_graph.py @@ -0,0 +1,170 @@ +import json +from pathlib import Path + +import pytest + +from ragops.adapters.otel_genai import otel_spans_to_trace_graph +from ragops.cli import main +from ragops.loader import ContractError +from ragops.trace_graph import ( + ExpectedTool, + TraceExpectation, + evaluate_trace_graph, + trace_graph_from_dict, +) + + +def _tool_trace(arguments: dict | None = None) -> dict: + return { + "schema_version": "1.0", + "trace_id": "trace-1", + "metadata": {"producer": "fixture"}, + "spans": [ + { + "id": "root", + "parent_id": None, + "kind": "agent", + "operation": "support_agent", + "started_at_ms": 0, + "ended_at_ms": 30, + "status": "ok", + "input": {"question": "Order?"}, + "output": {"answer": "Found"}, + "attributes": {}, + }, + { + "id": "tool-1", + "parent_id": "root", + "kind": "tool", + "operation": "lookup_order", + "started_at_ms": 5, + "ended_at_ms": 10, + "status": "ok", + "input": {"arguments": arguments or {"order_id": "WRONG"}}, + "output": {"order": "ORD-1"}, + "attributes": {}, + }, + { + "id": "retrieval-1", + "parent_id": "root", + "kind": "retrieval", + "operation": "policy_search", + "started_at_ms": 11, + "ended_at_ms": 15, + "status": "ok", + "input": {"query": "refund"}, + "output": {"document_ids": ["policy-1"]}, + "attributes": {}, + }, + ], + } + + +def _expectation() -> TraceExpectation: + return TraceExpectation( + expected_tools=(ExpectedTool("lookup_order", {"order_id": "ORD-1"}),), + expected_document_ids=("policy-1",), + maximum_steps=3, + ) + + +def test_trace_graph_localizes_wrong_tool_arguments() -> None: + report = evaluate_trace_graph(trace_graph_from_dict(_tool_trace()), _expectation()) + + assert report.metrics["tool_name_precision"] == 1.0 + assert report.metrics["tool_argument_exactness"] == 0.0 + assert report.metrics["retrieval_document_recall"] == 1.0 + assert report.failed_span_ids == ("tool-1",) + assert "tool_arguments" in report.failed_gates + + +def test_trace_graph_passes_expected_tool_and_trajectory() -> None: + graph = trace_graph_from_dict(_tool_trace({"order_id": "ORD-1"})) + + report = evaluate_trace_graph(graph, _expectation()) + + assert report.decision == "PASS" + assert report.metrics["tool_argument_exactness"] == 1.0 + assert report.metrics["tool_ordering"] == 1.0 + assert report.metrics["excess_step_ratio"] == 0.0 + + +@pytest.mark.parametrize( + ("mutation", "message"), + [ + (lambda data: data["spans"].append(dict(data["spans"][0])), "unique"), + (lambda data: data["spans"][1].update(parent_id="missing"), "parent"), + (lambda data: data["spans"][0].update(parent_id="tool-1"), "cycle"), + (lambda data: data["spans"][1].update(ended_at_ms=4), "end before"), + ], +) +def test_trace_graph_rejects_invalid_structure(mutation, message: str) -> None: + data = _tool_trace() + mutation(data) + + with pytest.raises(ContractError, match=message): + trace_graph_from_dict(data) + + +def test_trace_graph_scores_error_recovery() -> None: + data = _tool_trace({"order_id": "ORD-1"}) + failed = dict(data["spans"][1]) + failed.update(id="tool-failed", started_at_ms=3, ended_at_ms=4, status="error") + data["spans"].insert(1, failed) + + report = evaluate_trace_graph(trace_graph_from_dict(data), _expectation()) + + assert report.metrics["error_recovery"] == 1.0 + assert report.metrics["tool_success_rate"] == 0.5 + + +def test_otel_adapter_preserves_unknown_genai_attributes() -> None: + graph = otel_spans_to_trace_graph( + [ + { + "trace_id": "otel-trace", + "span_id": "root", + "parent_span_id": None, + "name": "invoke support agent", + "start_time_unix_nano": 1_000_000, + "end_time_unix_nano": 2_000_000, + "status": {"code": "STATUS_CODE_OK"}, + "attributes": { + "gen_ai.operation.name": "invoke_agent", + "gen_ai.agent.name": "support", + "gen_ai.experimental.future": "preserve-me", + }, + } + ] + ) + + assert graph.spans[0].kind == "agent" + assert graph.spans[0].attributes["gen_ai.experimental.future"] == "preserve-me" + assert graph.metadata["adapter"] == "otel-genai" + + +def test_trace_cli_converts_and_evaluates( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + trace = tmp_path / "trace.json" + expected = tmp_path / "expected.json" + trace.write_text(json.dumps(_tool_trace({"order_id": "ORD-1"})), encoding="utf-8") + expected.write_text( + json.dumps( + { + "expected_tools": [ + {"name": "lookup_order", "arguments": {"order_id": "ORD-1"}} + ], + "expected_document_ids": ["policy-1"], + "maximum_steps": 3, + } + ), + encoding="utf-8", + ) + monkeypatch.setattr( + "sys.argv", + ["ragops", "trace", "evaluate", "--trace", str(trace), "--expectation", str(expected)], + ) + + assert main() == 0 + assert json.loads(capsys.readouterr().out)["decision"] == "PASS" diff --git a/tests/test_workbench.py b/tests/test_workbench.py index 2022599..58d3b72 100644 --- a/tests/test_workbench.py +++ b/tests/test_workbench.py @@ -3,16 +3,17 @@ def test_workbench_exposes_evaluation_and_team_history() -> None: page = Path("apps/web/index.html").read_text(encoding="utf-8") + script = Path("src/ragops/web/app.js").read_text(encoding="utf-8") assert 'id="scenario"' in page assert 'id="responses"' in page assert 'id="api-key" type="password"' in page - assert "'x-api-key':apiKey.value" in page - assert "localStorage" not in page + assert "'x-api-key':apiKey.value" in script + assert "localStorage" not in page + script assert 'id="runs"' in page - assert "fetch('/v1/evaluate'" in page - assert "fetch(`/v1/runs?limit=" in page + assert "fetch('/v1/evaluate'" in script + assert "fetch(`/v1/runs?limit=" in script assert "RAGOPS_STORE" in page - assert "NOT EVALUATED" in page - assert "metrics.innerHTML=''" in page - assert "gates.innerHTML='
  • Not evaluated
    • '" in page + assert "NOT EVALUATED" in script + assert "innerHTML" not in page + script + assert "replaceChildren" in script