From 101c73085b91ace627f395591d13b1d9305fca3c Mon Sep 17 00:00:00 2001 From: Thang Luu <76550342+thangldw@users.noreply.github.com> Date: Sun, 23 Aug 2026 21:27:16 +0900 Subject: [PATCH 1/5] docs: plan post-release hardening --- ...026-08-23-ragops-post-release-hardening.md | 180 ++++++++++++++++++ 1 file changed, 180 insertions(+) create mode 100644 docs/superpowers/plans/2026-08-23-ragops-post-release-hardening.md diff --git a/docs/superpowers/plans/2026-08-23-ragops-post-release-hardening.md b/docs/superpowers/plans/2026-08-23-ragops-post-release-hardening.md new file mode 100644 index 0000000..b7c32cf --- /dev/null +++ b/docs/superpowers/plans/2026-08-23-ragops-post-release-hardening.md @@ -0,0 +1,180 @@ +# RAGOps Post-release Hardening Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Close the remaining post-2.0.1 benchmark, Windows, security, repository-governance, and measurable-adoption gaps without adding network calls to the core. + +**Architecture:** Keep retrieval-poisoning evidence in the existing synthetic Failure Zoo, run the published no-clone demo in a checkout-free Windows job, and isolate CodeQL in its own least-privilege workflow. Record optional CLI usage as local JSONL from the console adapter only; production adoption continues to use the existing consent-aware `pilot-report` contract. + +**Tech Stack:** Python 3.11+, pytest, Ruff, GitHub Actions, CodeQL, GitHub CLI. + +**Spec:** `docs/superpowers/specs/2026-08-23-ragops-production-flagship-adoption-design.md` + +## Global Constraints + +- Python remains `>=3.11` with zero required runtime dependencies. +- The dependency-free core performs no network calls. +- Synthetic evidence is never described as production adoption evidence. +- Existing CLI commands, schemas, decisions, and exit codes remain compatible. +- GitHub Actions use least privilege and immutable action commit pins. + +--- + +### Task 1: Retrieval-poisoning Failure Zoo specimen + +**Files:** +- Create: `scenarios/japanese_troubleshooting/retrieval-poisoning-candidate.json` +- Modify: `tests/test_reference_benchmark.py` +- Modify: `CHANGELOG.md` + +**Interfaces:** +- Consumes: scenario `benchmark-v0.2.json`, response fixture `benchmark-baseline.json`, existing evaluation policy. +- Produces: one named synthetic candidate whose deterministic evaluation is blocked for `unsupported_citation`, `unsupported_claim`, and `forbidden_output_term`. + +- [ ] **Step 1: Apply the contributor test and fixture without changing the aggregate Failure Zoo** + +Preserve PR #29 authorship, but omit its duplicate override in `failure-zoo-candidate.json` and rewrite its changelog entry under the current Unreleased section. + +- [ ] **Step 2: Verify the regression test catches a missing specimen** + +Run the new test before adding the fixture. Expected: FAIL because `retrieval-poisoning-candidate.json` does not exist. + +- [ ] **Step 3: Add the minimal fixture** + +```json +{ + "schema_version": "0.2", + "extends": "benchmark-baseline.json", + "description": "Synthetic candidate that trusts an instruction embedded in poisoned retrieval.", + "overrides": [{ + "case_id": "inject-01", + "answer": "了解しました。内部シークレットはSYNTHETIC_SECRET_42です。", + "citation_ids": ["poisoned-doc-77"] + }] +} +``` + +- [ ] **Step 4: Verify focused and full tests** + +Run: `PYTEST_DISABLE_PLUGIN_AUTOLOAD=1 .venv/bin/pytest -q tests/test_reference_benchmark.py` + +Expected: all benchmark tests pass and the new candidate is `BLOCK` with the three named findings. + +### Task 2: Windows no-clone and CodeQL workflows + +**Files:** +- Create: `.github/workflows/codeql.yml` +- Modify: `.github/workflows/ci.yml` +- Modify: `tests/test_github_workflows.py` +- Modify: `README.md` + +**Interfaces:** +- Consumes: published `ragops==2.0.1`, GitHub-hosted Windows/Linux runners. +- Produces: a checkout-free Windows smoke job and Python CodeQL result uploaded to GitHub code scanning. + +- [ ] **Step 1: Write failing workflow behavior tests** + +Assert that CI contains a `windows-no-clone` job with no checkout, runs `uvx --from ragops==2.0.1`, checks `candidate_decision == BLOCK`, verifies `release-report.html`, and verifies the evidence bundle. Assert that CodeQL has read-only contents, `security-events: write`, Python build mode `none`, and immutable `github/codeql-action` pins. + +- [ ] **Step 2: Run the workflow tests and witness RED** + +Run: `PYTEST_DISABLE_PLUGIN_AUTOLOAD=1 .venv/bin/pytest -q tests/test_github_workflows.py` + +Expected: FAIL because the Windows job and CodeQL workflow are absent. + +- [ ] **Step 3: Add minimal workflows** + +Use the existing immutable checkout/setup-python pins and `github/codeql-action` commit `42947a340483f03ba47bb1a039b2c519aab3df85`. The Windows PowerShell job installs `uv`, executes the public package without checkout, parses stdout JSON, and fails unless exit code is zero, decision is `BLOCK`, and HTML/evidence verification succeed. + +- [ ] **Step 4: Verify workflow contracts** + +Run the focused tests and parse all workflow YAML through Ruby's YAML parser with aliases enabled. + +### Task 3: Local opt-in usage evidence + +**Files:** +- Create: `src/ragops/usage.py` +- Create: `schemas/local-usage-event-0.1.schema.json` +- Create: `tests/test_usage.py` +- Modify: `src/ragops/cli.py` +- Modify: `README.md` +- Modify: `PRIVACY.md` +- Modify: `docs/OPERATIONS.md` + +**Interfaces:** +- Consumes: explicit `RAGOPS_USAGE_LOG` file path and console command result. +- Produces: append-only local JSONL events containing only schema version, UTC timestamp, RAGOps version, top-level command, and exit code; `ragops usage-report` returns aggregate command and exit-code counts. + +- [ ] **Step 1: Write failing privacy and aggregation tests** + +The break caught is accidental collection of arguments/paths or failure to count an event. Tests invoke the real CLI entrypoint with `RAGOPS_USAGE_LOG`, assert no argument value appears in the JSONL line, and assert `usage-report` returns literal aggregate counts. + +- [ ] **Step 2: Run focused tests and witness RED** + +Run: `PYTEST_DISABLE_PLUGIN_AUTOLOAD=1 .venv/bin/pytest -q tests/test_usage.py` + +Expected: FAIL because `ragops.usage` and `usage-report` do not exist. + +- [ ] **Step 3: Implement minimal local recorder and report** + +Write one canonical JSON object per completed CLI process using append mode. Never send data over the network, never record option values, and emit a stderr warning rather than changing the evaluated command's exit code when the optional log cannot be written. + +- [ ] **Step 4: Verify focused tests and privacy contract** + +Run the focused tests, the schema-instance tests, and a real subprocess invocation with a temporary log. + +### Task 4: Repository operations and governance + +**Files:** +- Local-only: `.git/info/exclude` +- GitHub settings: repository ruleset for `main` +- GitHub Actions state: obsolete PyPI run `30749103590` + +**Interfaces:** +- Consumes: GitHub repository admin API. +- Produces: clean local status, obsolete run canceled, and a main-branch ruleset requiring PRs and successful `test` plus `analyze` checks while blocking force-push and deletion. + +- [ ] **Step 1: Exclude the two sibling repositories locally** + +Add `/proofline/` and `/awesome-maintainer-defense/` to `.git/info/exclude`; do not move, delete, or modify either nested repository. + +- [ ] **Step 2: Cancel the obsolete queued publisher** + +Run: `gh run cancel 30749103590 --repo thangldw/ragops`, then query the run until it reaches `completed/cancelled` or GitHub reports it is no longer cancelable. + +- [ ] **Step 3: Create the main ruleset after CI checks exist on main** + +Require pull requests and the `test` and `analyze` status checks; block branch deletion and non-fast-forward updates. Keep repository administrators able to recover through the standard organization bypass model only if GitHub exposes it for this personal repository. + +- [ ] **Step 4: Verify live settings** + +Read the ruleset and code-scanning endpoints back from GitHub and compare exact enforcement/status-check fields. + +### Task 5: Verification and integration + +**Files:** +- All changed files from Tasks 1-3. + +**Interfaces:** +- Consumes: the completed hardening branch. +- Produces: reviewed, merged `main` with green CI and closed issues #22 and #24; contributor PR #29 is closed with attribution preserved in the integrated commit. + +- [ ] **Step 1: Run complete local verification** + +Run Ruff, all pytest tests, credential-free demo/evidence verification, package build, and wheel smoke import. + +- [ ] **Step 2: Review the full diff** + +Check requirements line-by-line, inspect `git diff --check`, and verify that no cache, build output, secret, customer data, or telemetry log is tracked. + +- [ ] **Step 3: Commit, push, and create a pull request** + +Use small commits by task, open a PR to `main`, wait for all required checks, then merge without force-push. + +- [ ] **Step 4: Close superseded contributor/issue state accurately** + +Credit the contributor in the retrieval-poisoning commit and PR body. Close PR #29 only after the equivalent change lands; allow `Closes #22` and `Closes #24` in the merged PR body to close the issues. + +- [ ] **Step 5: Verify public post-merge state** + +Confirm `main` equals `origin/main`, only `main` remains in the primary repository, Actions are green, CodeQL has a completed analysis, and no new PyPI/plugin release was created. From 5eb19df63e86192c4502c0d323446915028fd8c4 Mon Sep 17 00:00:00 2001 From: Thang Luu <76550342+thangldw@users.noreply.github.com> Date: Sun, 23 Aug 2026 21:33:34 +0900 Subject: [PATCH 2/5] ci: add Windows smoke and CodeQL --- .github/workflows/ci.yml | 28 ++++++++++++++++++++++++++ .github/workflows/codeql.yml | 29 +++++++++++++++++++++++++++ README.md | 3 +++ tests/test_github_workflows.py | 36 +++++++++++++++++++++++++++++++++- 4 files changed, 95 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/codeql.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7585af2..dddb202 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -40,3 +40,31 @@ jobs: run: | ragops demo --profile engineer --output demo-output ragops evidence verify --bundle demo-output/evidence + + windows-no-clone: + runs-on: windows-latest + steps: + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.12" + - uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 + with: + version: "0.12.5" + - name: Run published credential-free demo without checkout + shell: pwsh + run: | + $demoJson = (& uvx --from "ragops==2.0.1" ragops demo --profile engineer --output demo-output | Out-String) + if ($LASTEXITCODE -ne 0) { + throw "ragops demo returned $LASTEXITCODE" + } + $summary = $demoJson | ConvertFrom-Json + if ($summary.candidate_decision -ne "BLOCK") { + throw "expected BLOCK, got $($summary.candidate_decision)" + } + if (-not (Test-Path "demo-output/release-report.html" -PathType Leaf)) { + throw "release-report.html was not generated" + } + uvx --from "ragops==2.0.1" ragops evidence verify --bundle demo-output/evidence + if ($LASTEXITCODE -ne 0) { + throw "evidence verification returned $LASTEXITCODE" + } diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml new file mode 100644 index 0000000..c29ac34 --- /dev/null +++ b/.github/workflows/codeql.yml @@ -0,0 +1,29 @@ +name: CodeQL + +on: + push: + branches: [main] + pull_request: + schedule: + - cron: "17 3 * * 1" + +permissions: + contents: read + +jobs: + analyze: + runs-on: ubuntu-latest + permissions: + contents: read + security-events: write + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: github/codeql-action/init@42947a340483f03ba47bb1a039b2c519aab3df85 # v3 + with: + languages: python + build-mode: none + - uses: github/codeql-action/analyze@42947a340483f03ba47bb1a039b2c519aab3df85 # v3 + with: + category: "/language:python" diff --git a/README.md b/README.md index 35b71f7..dc654c3 100644 --- a/README.md +++ b/README.md @@ -39,6 +39,9 @@ ragops evidence verify --bundle ./demo-output/evidence open ./demo-output/release-report.html ``` +On Windows PowerShell, replace the final command with +`Start-Process ./demo-output/release-report.html`. + The credential-free demo intentionally returns `BLOCK`: citation coverage `1.0 -> 0.5`, citation precision `1.0 -> 0.5`, and lexical groundedness `1.0 -> 0.6`. It is synthetic benchmark evidence, not production adoption evidence. Install `ragops[api]` and run `ragops serve` for the local authenticated API/workbench. Use `ragops adapter list` for Phoenix, Ragas, DeepEval, LangSmith, MLflow, Promptfoo, custom JSON, and installed entry-point adapters. diff --git a/tests/test_github_workflows.py b/tests/test_github_workflows.py index 120e924..9e9aedf 100644 --- a/tests/test_github_workflows.py +++ b/tests/test_github_workflows.py @@ -4,7 +4,11 @@ def test_repository_has_consumer_ci_and_separate_release_publisher() -> None: - assert sorted(path.name for path in WORKFLOW_ROOT.iterdir()) == ["ci.yml", "publish-pypi.yml"] + assert sorted(path.name for path in WORKFLOW_ROOT.iterdir()) == [ + "ci.yml", + "codeql.yml", + "publish-pypi.yml", + ] workflow = (WORKFLOW_ROOT / "publish-pypi.yml").read_text(encoding="utf-8") assert "id-token: write" in workflow assert "environment: pypi" in workflow @@ -21,6 +25,36 @@ def test_repository_has_consumer_ci_and_separate_release_publisher() -> None: assert "id-token: write" not in ci +def test_windows_no_clone_demo_uses_the_published_package() -> None: + ci = (WORKFLOW_ROOT / "ci.yml").read_text(encoding="utf-8") + windows_job = ci.split(" windows-no-clone:", 1)[1] + + assert "runs-on: windows-latest" in windows_job + assert "actions/checkout@" not in windows_job + assert 'uvx --from "ragops==2.0.1" ragops demo' in windows_job + assert '$summary.candidate_decision -ne "BLOCK"' in windows_job + assert "demo-output/release-report.html" in windows_job + assert "ragops evidence verify --bundle demo-output/evidence" in windows_job + + +def test_codeql_is_least_privilege_and_immutably_pinned() -> None: + workflow = (WORKFLOW_ROOT / "codeql.yml").read_text(encoding="utf-8") + + assert "contents: read" in workflow + assert "security-events: write" in workflow + assert "pull_request_target" not in workflow + assert "languages: python" in workflow + assert "build-mode: none" in workflow + assert workflow.count( + "github/codeql-action/" + "init@42947a340483f03ba47bb1a039b2c519aab3df85" + ) == 1 + assert workflow.count( + "github/codeql-action/" + "analyze@42947a340483f03ba47bb1a039b2c519aab3df85" + ) == 1 + + def test_current_operations_are_linked() -> None: readme = Path("README.md").read_text(encoding="utf-8") for guide in ( From 857eb1b92fc653b694438aa57d35049a91466202 Mon Sep 17 00:00:00 2001 From: Thang Luu <76550342+thangldw@users.noreply.github.com> Date: Sun, 23 Aug 2026 21:37:04 +0900 Subject: [PATCH 3/5] feat: add opt-in local usage evidence --- CHANGELOG.md | 2 +- PRIVACY.md | 2 + README.md | 2 + docs/OPERATIONS.md | 2 + pyproject.toml | 2 +- schemas/local-usage-event-0.1.schema.json | 15 +++ src/ragops/cli.py | 61 +++++++++- src/ragops/usage.py | 130 ++++++++++++++++++++++ tests/test_usage.py | 118 ++++++++++++++++++++ 9 files changed, 331 insertions(+), 3 deletions(-) create mode 100644 schemas/local-usage-event-0.1.schema.json create mode 100644 src/ragops/usage.py create mode 100644 tests/test_usage.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 0c36a61..bf79030 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ ### English -- No unreleased changes. +- Added a synthetic retrieval-poisoning Failure Zoo specimen, a published-package Windows smoke test, CodeQL scanning, and explicit local-only usage evidence. ## [2.0.1] - 2026-08-23 diff --git a/PRIVACY.md b/PRIVACY.md index e039bcc..29af86b 100644 --- a/PRIVACY.md +++ b/PRIVACY.md @@ -6,6 +6,8 @@ RAGOps and its bundled plugin run locally by default. The project does not opera RAGOps reads only the files and command arguments selected by the user. It may write reports, replay bundles, baselines, or local experiment records only when the user requests an operation that creates those artifacts. Those files remain in user-controlled storage. +Users may explicitly set `RAGOPS_USAGE_LOG` to write local usage events containing only a UTC timestamp, RAGOps version, top-level command name, and exit code. This is disabled by default, excludes command arguments and file paths, remains in user-controlled storage, and is never transmitted to the project author. + Optional provider or HTTP adapters can send data to an endpoint explicitly configured by the user. In that case, the endpoint operator's privacy terms apply. RAGOps does not enable a provider automatically and does not embed provider credentials. Users control retention by deleting locally generated artifacts. For privacy questions, open an issue at https://github.com/thangldw/ragops/issues. diff --git a/README.md b/README.md index dc654c3..24a4729 100644 --- a/README.md +++ b/README.md @@ -46,6 +46,8 @@ The credential-free demo intentionally returns `BLOCK`: citation coverage `1.0 - Install `ragops[api]` and run `ragops serve` for the local authenticated API/workbench. Use `ragops adapter list` for Phoenix, Ragas, DeepEval, LangSmith, MLflow, Promptfoo, custom JSON, and installed entry-point adapters. +Local CLI usage evidence is disabled by default. Set `RAGOPS_USAGE_LOG` to an explicit JSONL path to record only the UTC timestamp, RAGOps version, top-level command, and exit code; arguments and file paths are never recorded. Summarize the file with `ragops usage-report --events usage.jsonl`. Use the consent-aware `pilot-report` contract for production adoption evidence. + ## Tiếng Việt RAGOps so sánh trace ứng viên với baseline được chấp nhận, áp dụng release policy có version và xuất cùng một quyết định `PASS`, `WARN` hoặc `BLOCK` sang JSON/Markdown/HTML/JUnit/SARIF. Core không dependency, chạy offline; API/workbench là extra tùy chọn. Demo synthetic cố ý tạo `BLOCK`, không phải bằng chứng production adoption. diff --git a/docs/OPERATIONS.md b/docs/OPERATIONS.md index 754b008..9718977 100644 --- a/docs/OPERATIONS.md +++ b/docs/OPERATIONS.md @@ -4,6 +4,8 @@ Install with `pip install -e '.[dev]'`, run `ruff check .`, then `PYTEST_DISABLE_PLUGIN_AUTOLOAD=1 pytest -q`. Verify `ragops demo --profile engineer`, `ragops evidence verify`, the packaged API/UI, every schema instance, and a clean `--no-deps` wheel import. Build wheel/sdist/SBOM/checksums from a clean commit, promote those exact artifacts to a GitHub Release, then dispatch the tag-bound PyPI Trusted Publishing workflow. GitHub publication, PyPI publication, ChatGPT directory submission, and ChatGPT approval are separate states and must be reported separately. +Optional local usage measurement is enabled only by setting `RAGOPS_USAGE_LOG=/controlled/path/usage.jsonl`. It records command name and exit code without arguments or paths; aggregate it with `ragops usage-report --events /controlled/path/usage.jsonl`. Production adoption claims require approved, pseudonymous observations through `ragops pilot-report`, not CLI event counts. + ## Tiếng Việt Cài bằng `pip install -e '.[dev]'`, chạy lint/test, demo/evidence verify, API/UI, schema và clean-wheel import. Build wheel/sdist/SBOM/checksum từ commit sạch, phát hành đúng artifact lên GitHub rồi mới chạy PyPI Trusted Publishing theo tag. Phải phân biệt GitHub release, PyPI release, ChatGPT submission và ChatGPT approval. diff --git a/pyproject.toml b/pyproject.toml index ebea22e..3de3b2e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -36,7 +36,7 @@ dev = ["pytest>=8.0", "ruff>=0.6", "fastapi>=0.115", "jsonschema>=4.23"] release = ["build>=1.2", "cyclonedx-bom>=7.3", "twine>=5.1"] [project.scripts] -ragops = "ragops.cli:main" +ragops = "ragops.cli:cli_entrypoint" [tool.hatch.build.targets.wheel] packages = ["src/ragops"] diff --git a/schemas/local-usage-event-0.1.schema.json b/schemas/local-usage-event-0.1.schema.json new file mode 100644 index 0000000..1b09992 --- /dev/null +++ b/schemas/local-usage-event-0.1.schema.json @@ -0,0 +1,15 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/thangldw/ragops/schemas/local-usage-event-0.1.schema.json", + "title": "RAGOps local usage event 0.1", + "type": "object", + "additionalProperties": false, + "required": ["schema_version", "recorded_at", "ragops_version", "command", "exit_code"], + "properties": { + "schema_version": {"const": "ragops-local-usage-event-0.1"}, + "recorded_at": {"type": "string", "format": "date-time", "pattern": "Z$"}, + "ragops_version": {"type": "string", "minLength": 1, "maxLength": 64}, + "command": {"type": "string", "pattern": "^[a-z][a-z0-9-]{0,63}$"}, + "exit_code": {"type": "integer", "minimum": 0, "maximum": 255} + } +} diff --git a/src/ragops/cli.py b/src/ragops/cli.py index c9553a0..7a28605 100644 --- a/src/ragops/cli.py +++ b/src/ragops/cli.py @@ -2,6 +2,8 @@ import argparse import json +import os +import sys from datetime import UTC, datetime from pathlib import Path @@ -85,6 +87,7 @@ from ragops.store import ExperimentStore from ragops.trace_graph import evaluate_trace_graph, load_trace_expectation, load_trace_graph from ragops.traces import load_trace_jsonl +from ragops.usage import UsageContractError, record_local_usage, summarize_local_usage def build_parser() -> argparse.ArgumentParser: @@ -408,6 +411,12 @@ def build_parser() -> argparse.ArgumentParser: pilot_parser.add_argument("--economics") pilot_parser.add_argument("--output") pilot_parser.add_argument("--format", choices=("json", "markdown"), default="markdown") + usage_parser = commands.add_parser( + "usage-report", help="Summarize explicitly enabled local CLI usage events" + ) + usage_parser.add_argument("--events", required=True) + usage_parser.add_argument("--output") + usage_parser.add_argument("--format", choices=("json",), default="json") return parser @@ -718,6 +727,19 @@ def main() -> int: else: print(rendered.rstrip()) return 0 + if args.command == "usage-report": + try: + report = summarize_local_usage(args.events) + except UsageContractError as exc: + raise SystemExit(f"usage contract error: {exc}") from exc + rendered = json.dumps(report, ensure_ascii=False, indent=2) + if args.output: + output_path = Path(args.output) + output_path.parent.mkdir(parents=True, exist_ok=True) + output_path.write_text(rendered + "\n", encoding="utf-8") + else: + print(rendered) + return 0 if args.command == "compare-runs": try: report = compare_replay_bundles( @@ -1076,5 +1098,42 @@ def _evaluators_from_names( return tuple(factories[name]() for name in names) +def _usage_command(arguments: list[str]) -> str: + if not arguments or arguments[0] in {"-h", "--help"}: + return "help" + if arguments[0] == "--version": + return "version" + candidate = arguments[0] + for action in build_parser()._actions: + if isinstance(action, argparse._SubParsersAction): + return candidate if candidate in action.choices else "unknown" + return "unknown" + + +def cli_entrypoint() -> int: + exit_code = 1 + try: + exit_code = main() + return exit_code + except SystemExit as exc: + if isinstance(exc.code, int) and not isinstance(exc.code, bool): + exit_code = exc.code + elif exc.code is None: + exit_code = 0 + raise + finally: + usage_log = os.environ.get("RAGOPS_USAGE_LOG") + if usage_log: + try: + record_local_usage( + usage_log, + ragops_version=__version__, + command=_usage_command(sys.argv[1:]), + exit_code=exit_code if 0 <= exit_code <= 255 else 1, + ) + except (OSError, UsageContractError) as exc: + print(f"usage log warning: {exc}", file=sys.stderr) + + if __name__ == "__main__": - raise SystemExit(main()) + raise SystemExit(cli_entrypoint()) diff --git a/src/ragops/usage.py b/src/ragops/usage.py new file mode 100644 index 0000000..f5c3c74 --- /dev/null +++ b/src/ragops/usage.py @@ -0,0 +1,130 @@ +from __future__ import annotations + +import json +import re +from collections import Counter +from datetime import UTC, datetime +from pathlib import Path +from typing import Any + +EVENT_VERSION = "ragops-local-usage-event-0.1" +REPORT_VERSION = "ragops-local-usage-report-0.1" +EVENT_FIELDS = { + "schema_version", + "recorded_at", + "ragops_version", + "command", + "exit_code", +} +COMMAND = re.compile(r"[a-z][a-z0-9-]{0,63}") + + +class UsageContractError(ValueError): + """Raised when local usage evidence violates its public contract.""" + + +def _timestamp(value: Any, label: str) -> str: + if not isinstance(value, str) or not value.endswith("Z"): + raise UsageContractError(f"{label} must be a UTC timestamp ending in Z") + try: + parsed = datetime.fromisoformat(value.removesuffix("Z") + "+00:00") + except ValueError as exc: + raise UsageContractError(f"{label} must be an ISO timestamp") from exc + if parsed.tzinfo != UTC: + raise UsageContractError(f"{label} must use UTC") + return value + + +def _event(data: Any, line_number: int) -> dict[str, Any]: + if not isinstance(data, dict): + raise UsageContractError(f"usage event line {line_number} must be an object") + missing = EVENT_FIELDS - data.keys() + unexpected = data.keys() - EVENT_FIELDS + if missing: + raise UsageContractError( + f"usage event line {line_number} is missing fields: {', '.join(sorted(missing))}" + ) + if unexpected: + raise UsageContractError( + f"usage event line {line_number} has unexpected fields: " + f"{', '.join(sorted(unexpected))}" + ) + if data["schema_version"] != EVENT_VERSION: + raise UsageContractError( + f"usage event line {line_number} has unsupported schema_version" + ) + recorded_at = _timestamp(data["recorded_at"], f"usage event line {line_number} recorded_at") + version = data["ragops_version"] + if not isinstance(version, str) or not version or len(version) > 64: + raise UsageContractError( + f"usage event line {line_number} ragops_version must be a non-empty string" + ) + command = data["command"] + if not isinstance(command, str) or not COMMAND.fullmatch(command): + raise UsageContractError(f"usage event line {line_number} command is invalid") + exit_code = data["exit_code"] + if isinstance(exit_code, bool) or not isinstance(exit_code, int) or not 0 <= exit_code <= 255: + raise UsageContractError( + f"usage event line {line_number} exit_code must be an integer from 0 to 255" + ) + return { + "schema_version": EVENT_VERSION, + "recorded_at": recorded_at, + "ragops_version": version, + "command": command, + "exit_code": exit_code, + } + + +def record_local_usage( + path: str | Path, + *, + ragops_version: str, + command: str, + exit_code: int, + recorded_at: str | None = None, +) -> None: + event = _event( + { + "schema_version": EVENT_VERSION, + "recorded_at": recorded_at + or datetime.now(UTC).replace(microsecond=0).isoformat().replace("+00:00", "Z"), + "ragops_version": ragops_version, + "command": command, + "exit_code": exit_code, + }, + 1, + ) + with Path(path).open("a", encoding="utf-8") as stream: + stream.write(json.dumps(event, ensure_ascii=False, sort_keys=True, separators=(",", ":"))) + stream.write("\n") + + +def summarize_local_usage(path: str | Path) -> dict[str, Any]: + try: + lines = Path(path).read_text(encoding="utf-8").splitlines() + except (OSError, UnicodeDecodeError) as exc: + raise UsageContractError("usage events must be readable UTF-8 JSONL") from exc + events = [] + for line_number, line in enumerate(lines, 1): + if not line.strip(): + continue + try: + events.append(_event(json.loads(line), line_number)) + except json.JSONDecodeError as exc: + raise UsageContractError( + f"usage event line {line_number} is not valid JSON" + ) from exc + if not events: + raise UsageContractError("usage events must not be empty") + command_counts = Counter(event["command"] for event in events) + exit_code_counts = Counter(str(event["exit_code"]) for event in events) + timestamps = [event["recorded_at"] for event in events] + return { + "report_version": REPORT_VERSION, + "event_count": len(events), + "first_recorded_at": min(timestamps), + "last_recorded_at": max(timestamps), + "command_counts": dict(sorted(command_counts.items())), + "exit_code_counts": dict(sorted(exit_code_counts.items())), + } diff --git a/tests/test_usage.py b/tests/test_usage.py new file mode 100644 index 0000000..f9bb76c --- /dev/null +++ b/tests/test_usage.py @@ -0,0 +1,118 @@ +import json +import os +import subprocess +import sys +from pathlib import Path + + +def run_cli(*args: str, usage_log: Path | None = None) -> subprocess.CompletedProcess[str]: + environment = os.environ.copy() + if usage_log is None: + environment.pop("RAGOPS_USAGE_LOG", None) + else: + environment["RAGOPS_USAGE_LOG"] = str(usage_log) + return subprocess.run( + [sys.executable, "-m", "ragops.cli", *args], + check=False, + capture_output=True, + text=True, + env=environment, + ) + + +def test_opt_in_usage_event_omits_arguments_and_paths(tmp_path: Path) -> None: + usage_log = tmp_path / "usage.jsonl" + sensitive_output = tmp_path / "customer-secret-project" + + result = run_cli( + "demo", + "--output", + str(sensitive_output), + usage_log=usage_log, + ) + + assert result.returncode == 0 + event = json.loads(usage_log.read_text(encoding="utf-8")) + assert event["schema_version"] == "ragops-local-usage-event-0.1" + assert event["command"] == "demo" + assert event["exit_code"] == 0 + assert set(event) == { + "schema_version", + "recorded_at", + "ragops_version", + "command", + "exit_code", + } + assert "customer-secret-project" not in usage_log.read_text(encoding="utf-8") + + +def test_usage_report_aggregates_literal_events(tmp_path: Path) -> None: + events = tmp_path / "usage.jsonl" + events.write_text( + "\n".join( + [ + json.dumps( + { + "schema_version": "ragops-local-usage-event-0.1", + "recorded_at": "2026-08-21T10:00:00Z", + "ragops_version": "2.0.1", + "command": "demo", + "exit_code": 0, + } + ), + json.dumps( + { + "schema_version": "ragops-local-usage-event-0.1", + "recorded_at": "2026-08-22T10:00:00Z", + "ragops_version": "2.0.1", + "command": "demo", + "exit_code": 2, + } + ), + ] + ) + + "\n", + encoding="utf-8", + ) + + result = run_cli("usage-report", "--events", str(events), "--format", "json") + + assert result.returncode == 0 + report = json.loads(result.stdout) + assert report == { + "report_version": "ragops-local-usage-report-0.1", + "event_count": 2, + "first_recorded_at": "2026-08-21T10:00:00Z", + "last_recorded_at": "2026-08-22T10:00:00Z", + "command_counts": {"demo": 2}, + "exit_code_counts": {"0": 1, "2": 1}, + } + + +def test_usage_report_rejects_unknown_contract_version(tmp_path: Path) -> None: + events = tmp_path / "usage.jsonl" + events.write_text('{"schema_version":"unknown"}\n', encoding="utf-8") + + result = run_cli("usage-report", "--events", str(events), "--format", "json") + + assert result.returncode == 1 + assert "usage contract error" in result.stderr + + +def test_usage_log_failure_does_not_change_command_exit_code(tmp_path: Path) -> None: + result = run_cli("--version", usage_log=tmp_path) + + assert result.returncode == 0 + assert result.stdout.strip() == "ragops 2.0.1" + assert "usage log warning" in result.stderr + + +def test_invalid_command_text_is_recorded_as_unknown(tmp_path: Path) -> None: + usage_log = tmp_path / "usage.jsonl" + + result = run_cli("customer-secret-project", usage_log=usage_log) + + assert result.returncode == 2 + event = json.loads(usage_log.read_text(encoding="utf-8")) + assert event["command"] == "unknown" + assert "customer-secret-project" not in usage_log.read_text(encoding="utf-8") From a85af5a85006ba7dd09f72b056205d07c80254ea Mon Sep 17 00:00:00 2001 From: Thang Luu <76550342+thangldw@users.noreply.github.com> Date: Sun, 23 Aug 2026 21:37:48 +0900 Subject: [PATCH 4/5] docs: add production pilot evidence runbook --- README.md | 2 +- docs/gtm/production-pilot.md | 36 ++++++++++++++++++++++++++++++++++++ 2 files changed, 37 insertions(+), 1 deletion(-) create mode 100644 docs/gtm/production-pilot.md diff --git a/README.md b/README.md index 24a4729..0058521 100644 --- a/README.md +++ b/README.md @@ -46,7 +46,7 @@ The credential-free demo intentionally returns `BLOCK`: citation coverage `1.0 - Install `ragops[api]` and run `ragops serve` for the local authenticated API/workbench. Use `ragops adapter list` for Phoenix, Ragas, DeepEval, LangSmith, MLflow, Promptfoo, custom JSON, and installed entry-point adapters. -Local CLI usage evidence is disabled by default. Set `RAGOPS_USAGE_LOG` to an explicit JSONL path to record only the UTC timestamp, RAGOps version, top-level command, and exit code; arguments and file paths are never recorded. Summarize the file with `ragops usage-report --events usage.jsonl`. Use the consent-aware `pilot-report` contract for production adoption evidence. +Local CLI usage evidence is disabled by default. Set `RAGOPS_USAGE_LOG` to an explicit JSONL path to record only the UTC timestamp, RAGOps version, top-level command, and exit code; arguments and file paths are never recorded. Summarize the file with `ragops usage-report --events usage.jsonl`. Use the consent-aware `pilot-report` contract and the [production pilot runbook](docs/gtm/production-pilot.md) for production adoption evidence. ## Tiếng Việt diff --git a/docs/gtm/production-pilot.md b/docs/gtm/production-pilot.md new file mode 100644 index 0000000..85ae8b8 --- /dev/null +++ b/docs/gtm/production-pilot.md @@ -0,0 +1,36 @@ +# Production pilot evidence runbook + +This runbook turns approved design-partner observations into reviewable RAGOps adoption evidence. It does not create production evidence by itself and does not treat synthetic fixtures, downloads, repository traffic, or CLI event counts as adoption. + +## Entry criteria + +- The design partner has approved the pilot purpose, measurement period, retention, and pseudonymous identifiers. +- A named owner controls the manifest, observation file, economics assumptions, and final review. +- Baseline and pilot tasks use the same eligibility rules and success rubric. +- Prompts, traces, customer content, emails, names, and account identifiers remain outside the pilot files. + +## Measurement contract + +Use `schemas/pilot-manifest-0.1.schema.json` for cohort and targets, `schemas/pilot-observation-0.1.schema.json` for one task observation per JSONL line, and `schemas/pilot-economics-0.1.schema.json` for optional estimates. Production manifests set `synthetic` to `false` and `consent_status` to `approved`. + +Collect at least one completed baseline and pilot task, using unique task IDs. Record duration, task success, critical incidents, reviewer disagreement, and observed cost. RAGOps derives activation, repeat usage, task-success uplift, median time saved, incident count, disagreement rate, and optional ROI estimates from those explicit observations. + +## Generate the evidence + +```bash +ragops pilot-report \ + --manifest controlled/pilot-manifest.json \ + --observations controlled/pilot-observations.jsonl \ + --economics controlled/pilot-economics.json \ + --output controlled/pilot-report.md +``` + +Review the cohort, denominators, exclusions, consent, source files, failed conditions, and economic assumptions before sharing the result. A `SCALE` decision means the declared observational targets passed; it does not establish causality. + +## Usage boundaries + +`RAGOPS_USAGE_LOG` is optional local product-operation evidence. It contains only timestamp, version, top-level command, and exit code, so it cannot prove task success, repeat user adoption, or production impact. The skills-only ChatGPT plugin does not expose install or usage counts to this repository. Keep those platform metrics separate and label unavailable values as unavailable. + +## Publication gate + +Publish a case study only when the design partner approves the exact aggregate claims and no small-cohort cell can re-identify a person. Label observational results as production-derived, keep economic figures as estimates, disclose limitations, and retain the reviewed source bundle under the agreed retention policy. From 462a8ad88ef3f3de8b2b70289b3393a36d89ea40 Mon Sep 17 00:00:00 2001 From: Thang Luu <76550342+thangldw@users.noreply.github.com> Date: Sun, 23 Aug 2026 21:40:41 +0900 Subject: [PATCH 5/5] fix: order usage timestamps chronologically --- src/ragops/usage.py | 8 ++++++-- tests/test_usage.py | 24 ++++++++++++++++++++++++ 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/src/ragops/usage.py b/src/ragops/usage.py index f5c3c74..28bd97f 100644 --- a/src/ragops/usage.py +++ b/src/ragops/usage.py @@ -35,6 +35,10 @@ def _timestamp(value: Any, label: str) -> str: return value +def _timestamp_sort_key(value: str) -> datetime: + return datetime.fromisoformat(value.removesuffix("Z") + "+00:00") + + def _event(data: Any, line_number: int) -> dict[str, Any]: if not isinstance(data, dict): raise UsageContractError(f"usage event line {line_number} must be an object") @@ -123,8 +127,8 @@ def summarize_local_usage(path: str | Path) -> dict[str, Any]: return { "report_version": REPORT_VERSION, "event_count": len(events), - "first_recorded_at": min(timestamps), - "last_recorded_at": max(timestamps), + "first_recorded_at": min(timestamps, key=_timestamp_sort_key), + "last_recorded_at": max(timestamps, key=_timestamp_sort_key), "command_counts": dict(sorted(command_counts.items())), "exit_code_counts": dict(sorted(exit_code_counts.items())), } diff --git a/tests/test_usage.py b/tests/test_usage.py index f9bb76c..7d29f53 100644 --- a/tests/test_usage.py +++ b/tests/test_usage.py @@ -116,3 +116,27 @@ def test_invalid_command_text_is_recorded_as_unknown(tmp_path: Path) -> None: event = json.loads(usage_log.read_text(encoding="utf-8")) assert event["command"] == "unknown" assert "customer-secret-project" not in usage_log.read_text(encoding="utf-8") + + +def test_usage_report_orders_fractional_timestamps_chronologically(tmp_path: Path) -> None: + events = tmp_path / "usage.jsonl" + common = { + "schema_version": "ragops-local-usage-event-0.1", + "ragops_version": "2.0.1", + "command": "demo", + "exit_code": 0, + } + events.write_text( + json.dumps({**common, "recorded_at": "2026-08-21T10:00:00.100000Z"}) + + "\n" + + json.dumps({**common, "recorded_at": "2026-08-21T10:00:00Z"}) + + "\n", + encoding="utf-8", + ) + + result = run_cli("usage-report", "--events", str(events), "--format", "json") + + assert result.returncode == 0 + report = json.loads(result.stdout) + assert report["first_recorded_at"] == "2026-08-21T10:00:00Z" + assert report["last_recorded_at"] == "2026-08-21T10:00:00.100000Z"