Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,7 @@ Options:
- `--baseline, -b PATH`: Compare against a persistent baseline trace file (see [Baseline workflow](#baseline-workflow)).
- `--update-baseline`: Overwrite the persistent baseline with the candidate after a clean diff.
- `--baseline-config PATH`: The `agentdiff.toml` the **baseline** was recorded with. When gate values differ from this run's config, the report flags the change (Goodhart guard — see [Gate governance](https://agentdiff.lostmartian.in/docs/configuration#gate-governance-goodhart-guard)).
- `--stale-days N`: Warn via `--explain` when the baseline file is older than N days (default: config `stale_baseline_days` or 30 — advisory only).
- `--config PATH`: Load defaults from an `agentdiff.toml` (auto-discovered if not given).

#### Config-as-code (`agentdiff.toml`)
Expand Down
18 changes: 18 additions & 0 deletions src/agentdiff/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
from agentdiff.reporters.markdown import generate_markdown
from agentdiff.reporters.pr import generate_pr_markdown
from agentdiff.reporters.terminal import print_report
from agentdiff.staleness import check_baseline_staleness

app = typer.Typer(
help="AgentDiff CLI - Compare multi-turn agent execution trajectories."
Expand Down Expand Up @@ -186,6 +187,11 @@ def diff(
"--baseline-config",
help="Path to the agentdiff.toml the BASELINE was recorded with. When gate values differ from this run's config, the report flags the change (Goodhart guard).",
),
stale_days: int | None = typer.Option(
None,
"--stale-days",
help="Warn (with --explain) when the baseline file is older than this many days (default: config or 30).",
),
):
"""Compares baseline and candidate agent trajectories."""
cfg = load_config(config)
Expand All @@ -199,6 +205,11 @@ def diff(
sys.exit(2)
threshold_changes = diff_gate_thresholds(baseline_cfg, cfg)
gate_provenance = provenance_line(cfg, config_source)
stale_days = (
stale_days
if stale_days is not None
else getattr(cfg.cli, "stale_baseline_days", 30)
)
values = _resolve_cli(
cfg,
adapter=adapter,
Expand Down Expand Up @@ -246,6 +257,11 @@ def diff(
else:
actual_baseline = baseline_path

# F7 — surface baseline age so stale baselines are re-recorded deliberately
staleness = check_baseline_staleness(
actual_baseline, stale_after_days=stale_days
)

baseline = load_trace(actual_baseline, adapter)
except (json.JSONDecodeError, ValueError, FileNotFoundError) as e:
typer.echo(f"Error loading or parsing trace: {e}", err=True)
Expand Down Expand Up @@ -318,6 +334,8 @@ def diff(
culprit = locate_culprit(report)
if culprit:
typer.echo("\n" + culprit.render())
if staleness.is_stale:
typer.echo(f"\n! {staleness.render()}")
if threshold_changes:
typer.echo("\nGate thresholds changed vs baseline config:")
for change in threshold_changes:
Expand Down
1 change: 1 addition & 0 deletions src/agentdiff/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,7 @@ class CliConfig:
max_cost_delta: float = 10.0
max_wasted_effort: float = 0.1
max_recovery_ratio: float | None = None
stale_baseline_days: int = 30


@dataclass
Expand Down
61 changes: 61 additions & 0 deletions src/agentdiff/staleness.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
"""F7 — stale-baseline detection (regressional Goodhart guard).

Baselines age silently: a golden trace from three sprints ago still anchors
every diff, but the product it represented no longer exists. This module
makes that age visible so teams re-record baselines deliberately instead of
losing trust in a gate that "just got noisy".
"""

from __future__ import annotations

import os
import time
from dataclasses import dataclass

DEFAULT_STALE_DAYS = 30


@dataclass(frozen=True)
class Staleness:
"""How old a baseline file is, and whether that crosses the stale line."""

age_days: float
stale_after_days: int
missing: bool = False

@property
def is_stale(self) -> bool:
return not self.missing and self.age_days > self.stale_after_days

def render(self) -> str:
if self.missing:
return ""
age = f"{self.age_days:.0f} days" if self.age_days >= 1 else "less than a day"
line = f"Baseline is {age} old (last modified {time.strftime('%Y-%m-%d', time.localtime(self._mtime))})."
if self.is_stale:
line += (
f" Stale threshold is {self.stale_after_days} days - consider re-recording"
" it if the agent's expected behavior has legitimately changed"
" (agentdiff record ... --update-baseline)."
)
return line

_mtime: float = 0.0


def check_baseline_staleness(
baseline_path: str | os.PathLike,
stale_after_days: int = DEFAULT_STALE_DAYS,
) -> Staleness:
"""Inspects the baseline file's modification time.

Missing files return ``missing=True`` with ``is_stale=False`` - a missing
baseline is a load error, not a staleness problem.
"""
try:
mtime = os.path.getmtime(baseline_path)
except OSError:
return Staleness(age_days=0.0, stale_after_days=stale_after_days, missing=True)

age_days = max(0.0, (time.time() - mtime) / 86400)
return Staleness(age_days=age_days, stale_after_days=stale_after_days, _mtime=mtime)
87 changes: 87 additions & 0 deletions tests/test_governance.py
Original file line number Diff line number Diff line change
Expand Up @@ -271,3 +271,90 @@ def test_explicit_config_path_shown_as_source(self, tmp_path):
assert result.exit_code == 0
assert "max_divergence=0.45" in result.output
assert "agentdiff.toml" in result.output # path may wrap; filename stays whole


class TestStaleBaseline:
"""F7 — baselines age silently; --explain surfaces the age."""

def test_old_baseline_warns_in_explain(self, tmp_path, monkeypatch):
import os
import time

from agentdiff.cli import app as cli_app

base = tmp_path / "base.json"
cand = tmp_path / "cand.json"
base.write_text(
json.dumps(record_run("json:loads", task_input={"s": "{}"}).model_dump())
)
cand.write_text(
json.dumps(record_run("json:loads", task_input={"s": "{}"}).model_dump())
)
old = time.time() - 47 * 86400
os.utime(base, (old, old))

result = runner.invoke(
cli_app, ["diff", str(base), str(cand), "--explain", "--stale-days", "30"]
)
assert result.exit_code == 0
assert "Baseline is 47 days old" in result.output
assert "consider re-recording" in result.output

def test_fresh_baseline_no_warning(self, tmp_path):
from agentdiff.cli import app as cli_app

base = tmp_path / "base.json"
cand = tmp_path / "cand.json"
base.write_text(
json.dumps(record_run("json:loads", task_input={"s": "{}"}).model_dump())
)
cand.write_text(
json.dumps(record_run("json:loads", task_input={"s": "{}"}).model_dump())
)

result = runner.invoke(cli_app, ["diff", str(base), str(cand), "--explain"])
assert result.exit_code == 0
assert "consider re-recording" not in result.output

def test_stale_days_config_respected(self, tmp_path):
import os
import time

from agentdiff.cli import app as cli_app

base = tmp_path / "base.json"
cand = tmp_path / "cand.json"
base.write_text(
json.dumps(record_run("json:loads", task_input={"s": "{}"}).model_dump())
)
cand.write_text(
json.dumps(record_run("json:loads", task_input={"s": "{}"}).model_dump())
)
old = time.time() - 5 * 86400
os.utime(base, (old, old))
cfg = tmp_path / "agentdiff.toml"
cfg.write_text("[cli]\nstale_baseline_days = 3\n")

result = runner.invoke(
cli_app,
["diff", str(base), str(cand), "--explain", "--config", str(cfg)],
)
assert result.exit_code == 0
assert "Baseline is 5 days old" in result.output

def test_staleness_missing_file_not_stale(self):
from agentdiff.staleness import check_baseline_staleness

s = check_baseline_staleness("/nonexistent/baseline.json")
assert s.missing is True
assert s.is_stale is False
assert s.render() == ""

def test_staleness_render_fresh(self, tmp_path):
from agentdiff.staleness import check_baseline_staleness

f = tmp_path / "fresh.json"
f.write_text("{}")
s = check_baseline_staleness(f)
assert s.is_stale is False
assert "less than a day old" in s.render()
20 changes: 20 additions & 0 deletions website/docs/03-Guides/05-Configuration.md
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,26 @@ Every report also self-describes its rules: a one-line gate provenance
appears in the terminal summary, the JSON (`gate_provenance` field), and
the PR comment footer - so a diff always answers "what rules judged me?".

### Stale baselines

Thresholds aren't the only thing that drifts - baselines age too. A golden
trace from three sprints ago still anchors every diff, but the behavior it
represents may be long gone. With `--explain`, AgentDiff warns when the
baseline file is older than `stale_baseline_days` (default 30):

```text
! Baseline is 47 days old (last modified 2026-07-10). Stale threshold is
30 days - consider re-recording it if the agent's expected behavior has
legitimately changed (agentdiff record ... --update-baseline).
```

Tune or disable per project:

```toml
[cli]
stale_baseline_days = 14 # any positive number; advisory only, never blocks
```

## From the SDK

You can load config programmatically with `load_config()`, which returns an
Expand Down
Loading