diff --git a/README.md b/README.md index 82452c33..301d517b 100644 --- a/README.md +++ b/README.md @@ -71,6 +71,7 @@ Options: - `--baseline, -b PATH`: Compare against a persistent baseline trace file (see [Baseline workflow](#baseline-workflow)). - `--update-baseline`: Overwrite the persistent baseline with the candidate after a clean diff. - `--baseline-config PATH`: The `agentdiff.toml` the **baseline** was recorded with. When gate values differ from this run's config, the report flags the change (Goodhart guard — see [Gate governance](https://agentdiff.lostmartian.in/docs/configuration#gate-governance-goodhart-guard)). +- `--stale-days N`: Warn via `--explain` when the baseline file is older than N days (default: config `stale_baseline_days` or 30 — advisory only). - `--config PATH`: Load defaults from an `agentdiff.toml` (auto-discovered if not given). #### Config-as-code (`agentdiff.toml`) diff --git a/src/agentdiff/cli.py b/src/agentdiff/cli.py index 2b51b20d..48f784e6 100644 --- a/src/agentdiff/cli.py +++ b/src/agentdiff/cli.py @@ -19,6 +19,7 @@ from agentdiff.reporters.markdown import generate_markdown from agentdiff.reporters.pr import generate_pr_markdown from agentdiff.reporters.terminal import print_report +from agentdiff.staleness import check_baseline_staleness app = typer.Typer( help="AgentDiff CLI - Compare multi-turn agent execution trajectories." @@ -186,6 +187,11 @@ def diff( "--baseline-config", help="Path to the agentdiff.toml the BASELINE was recorded with. When gate values differ from this run's config, the report flags the change (Goodhart guard).", ), + stale_days: int | None = typer.Option( + None, + "--stale-days", + help="Warn (with --explain) when the baseline file is older than this many days (default: config or 30).", + ), ): """Compares baseline and candidate agent trajectories.""" cfg = load_config(config) @@ -199,6 +205,11 @@ def diff( sys.exit(2) threshold_changes = diff_gate_thresholds(baseline_cfg, cfg) gate_provenance = provenance_line(cfg, config_source) + stale_days = ( + stale_days + if stale_days is not None + else getattr(cfg.cli, "stale_baseline_days", 30) + ) values = _resolve_cli( cfg, adapter=adapter, @@ -246,6 +257,11 @@ def diff( else: actual_baseline = baseline_path + # F7 — surface baseline age so stale baselines are re-recorded deliberately + staleness = check_baseline_staleness( + actual_baseline, stale_after_days=stale_days + ) + baseline = load_trace(actual_baseline, adapter) except (json.JSONDecodeError, ValueError, FileNotFoundError) as e: typer.echo(f"Error loading or parsing trace: {e}", err=True) @@ -318,6 +334,8 @@ def diff( culprit = locate_culprit(report) if culprit: typer.echo("\n" + culprit.render()) + if staleness.is_stale: + typer.echo(f"\n! {staleness.render()}") if threshold_changes: typer.echo("\nGate thresholds changed vs baseline config:") for change in threshold_changes: diff --git a/src/agentdiff/config.py b/src/agentdiff/config.py index d363b5d8..5a5bb93c 100644 --- a/src/agentdiff/config.py +++ b/src/agentdiff/config.py @@ -69,6 +69,7 @@ class CliConfig: max_cost_delta: float = 10.0 max_wasted_effort: float = 0.1 max_recovery_ratio: float | None = None + stale_baseline_days: int = 30 @dataclass diff --git a/src/agentdiff/staleness.py b/src/agentdiff/staleness.py new file mode 100644 index 00000000..2f08fcce --- /dev/null +++ b/src/agentdiff/staleness.py @@ -0,0 +1,61 @@ +"""F7 — stale-baseline detection (regressional Goodhart guard). + +Baselines age silently: a golden trace from three sprints ago still anchors +every diff, but the product it represented no longer exists. This module +makes that age visible so teams re-record baselines deliberately instead of +losing trust in a gate that "just got noisy". +""" + +from __future__ import annotations + +import os +import time +from dataclasses import dataclass + +DEFAULT_STALE_DAYS = 30 + + +@dataclass(frozen=True) +class Staleness: + """How old a baseline file is, and whether that crosses the stale line.""" + + age_days: float + stale_after_days: int + missing: bool = False + + @property + def is_stale(self) -> bool: + return not self.missing and self.age_days > self.stale_after_days + + def render(self) -> str: + if self.missing: + return "" + age = f"{self.age_days:.0f} days" if self.age_days >= 1 else "less than a day" + line = f"Baseline is {age} old (last modified {time.strftime('%Y-%m-%d', time.localtime(self._mtime))})." + if self.is_stale: + line += ( + f" Stale threshold is {self.stale_after_days} days - consider re-recording" + " it if the agent's expected behavior has legitimately changed" + " (agentdiff record ... --update-baseline)." + ) + return line + + _mtime: float = 0.0 + + +def check_baseline_staleness( + baseline_path: str | os.PathLike, + stale_after_days: int = DEFAULT_STALE_DAYS, +) -> Staleness: + """Inspects the baseline file's modification time. + + Missing files return ``missing=True`` with ``is_stale=False`` - a missing + baseline is a load error, not a staleness problem. + """ + try: + mtime = os.path.getmtime(baseline_path) + except OSError: + return Staleness(age_days=0.0, stale_after_days=stale_after_days, missing=True) + + age_days = max(0.0, (time.time() - mtime) / 86400) + return Staleness(age_days=age_days, stale_after_days=stale_after_days, _mtime=mtime) diff --git a/tests/test_governance.py b/tests/test_governance.py index 810cb2b7..ff3f65a9 100644 --- a/tests/test_governance.py +++ b/tests/test_governance.py @@ -271,3 +271,90 @@ def test_explicit_config_path_shown_as_source(self, tmp_path): assert result.exit_code == 0 assert "max_divergence=0.45" in result.output assert "agentdiff.toml" in result.output # path may wrap; filename stays whole + + +class TestStaleBaseline: + """F7 — baselines age silently; --explain surfaces the age.""" + + def test_old_baseline_warns_in_explain(self, tmp_path, monkeypatch): + import os + import time + + from agentdiff.cli import app as cli_app + + base = tmp_path / "base.json" + cand = tmp_path / "cand.json" + base.write_text( + json.dumps(record_run("json:loads", task_input={"s": "{}"}).model_dump()) + ) + cand.write_text( + json.dumps(record_run("json:loads", task_input={"s": "{}"}).model_dump()) + ) + old = time.time() - 47 * 86400 + os.utime(base, (old, old)) + + result = runner.invoke( + cli_app, ["diff", str(base), str(cand), "--explain", "--stale-days", "30"] + ) + assert result.exit_code == 0 + assert "Baseline is 47 days old" in result.output + assert "consider re-recording" in result.output + + def test_fresh_baseline_no_warning(self, tmp_path): + from agentdiff.cli import app as cli_app + + base = tmp_path / "base.json" + cand = tmp_path / "cand.json" + base.write_text( + json.dumps(record_run("json:loads", task_input={"s": "{}"}).model_dump()) + ) + cand.write_text( + json.dumps(record_run("json:loads", task_input={"s": "{}"}).model_dump()) + ) + + result = runner.invoke(cli_app, ["diff", str(base), str(cand), "--explain"]) + assert result.exit_code == 0 + assert "consider re-recording" not in result.output + + def test_stale_days_config_respected(self, tmp_path): + import os + import time + + from agentdiff.cli import app as cli_app + + base = tmp_path / "base.json" + cand = tmp_path / "cand.json" + base.write_text( + json.dumps(record_run("json:loads", task_input={"s": "{}"}).model_dump()) + ) + cand.write_text( + json.dumps(record_run("json:loads", task_input={"s": "{}"}).model_dump()) + ) + old = time.time() - 5 * 86400 + os.utime(base, (old, old)) + cfg = tmp_path / "agentdiff.toml" + cfg.write_text("[cli]\nstale_baseline_days = 3\n") + + result = runner.invoke( + cli_app, + ["diff", str(base), str(cand), "--explain", "--config", str(cfg)], + ) + assert result.exit_code == 0 + assert "Baseline is 5 days old" in result.output + + def test_staleness_missing_file_not_stale(self): + from agentdiff.staleness import check_baseline_staleness + + s = check_baseline_staleness("/nonexistent/baseline.json") + assert s.missing is True + assert s.is_stale is False + assert s.render() == "" + + def test_staleness_render_fresh(self, tmp_path): + from agentdiff.staleness import check_baseline_staleness + + f = tmp_path / "fresh.json" + f.write_text("{}") + s = check_baseline_staleness(f) + assert s.is_stale is False + assert "less than a day old" in s.render() diff --git a/website/docs/03-Guides/05-Configuration.md b/website/docs/03-Guides/05-Configuration.md index 2ce411e5..e691b8c7 100644 --- a/website/docs/03-Guides/05-Configuration.md +++ b/website/docs/03-Guides/05-Configuration.md @@ -71,6 +71,26 @@ Every report also self-describes its rules: a one-line gate provenance appears in the terminal summary, the JSON (`gate_provenance` field), and the PR comment footer - so a diff always answers "what rules judged me?". +### Stale baselines + +Thresholds aren't the only thing that drifts - baselines age too. A golden +trace from three sprints ago still anchors every diff, but the behavior it +represents may be long gone. With `--explain`, AgentDiff warns when the +baseline file is older than `stale_baseline_days` (default 30): + +```text +! Baseline is 47 days old (last modified 2026-07-10). Stale threshold is + 30 days - consider re-recording it if the agent's expected behavior has + legitimately changed (agentdiff record ... --update-baseline). +``` + +Tune or disable per project: + +```toml +[cli] +stale_baseline_days = 14 # any positive number; advisory only, never blocks +``` + ## From the SDK You can load config programmatically with `load_config()`, which returns an