diff --git a/CHANGELOG.md b/CHANGELOG.md index 3bee29dd..113e25be 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,7 @@ Versions follow [SemVer](https://semver.org). ### Added +- The climb board: `CLIMB.md` (numbers + attempts table), `climb/.json` (the data), and `climb.html` (a chart of the JSON) on the target's `research-log` branch, kernel-published as runs end. Idempotent: rows merge by run id, files are written only on change. - SWEEPS: `launch --array K` fans one command out to K jobs, each with `SWEEP_INDEX=0..K-1` in its environment, delivered back as K results (artifacts under `results///`) in one wake. It counts as one diff --git a/docs/roadmap.md b/docs/roadmap.md index 8f8a17d1..b6fd4b24 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -160,6 +160,13 @@ open-ended sweep space. - [ ] Task selection reads lessons/ + recent runs/ (the loop's research memory) - [ ] Periodic distillation pass: raw reports → bounded lessons/.md - [ ] Weekly digest aggregates per-run reports; cost ledger; leaderboard history +- [x] Climb board on the research-log branch: `climb/.json` + + `CLIMB.md` + `climb.html`; a Pages site can serve the same files at the + public flip +- [ ] Local-inference roles (big item, 2026-08-29): open-weight author/judge + models served on cluster GPUs (vLLM/SGLang behind the hermes provider + seam) — stability against API model retirement, and pinned weights make + the research program reproducible ## Cross-cluster (dev plan, 2026-08-29) diff --git a/src/autoresearch/attempt.py b/src/autoresearch/attempt.py index f0e82e50..1e25a01a 100644 --- a/src/autoresearch/attempt.py +++ b/src/autoresearch/attempt.py @@ -279,9 +279,12 @@ def _clear_stage(record: RunRecord) -> RunRecord: especially `wake_attempts` ride into `in-review`, where in-review follow-up servicing reuses `wake_attempts` as its OWN retry cap — so a run that woke once would reach review with a shrunk follow-up budget.""" + # the run's spend survives the wipe: terminal reporting (the climb + # board) reads it after the transition + kept = {k: record.stage[k] for k in ("gpu_hours_used",) if record.stage and k in record.stage} return dc_replace( record, - stage={}, + stage=kept, experiment_job_id="", deadline=0.0, terminal_seen=0.0, diff --git a/src/autoresearch/climbboard.py b/src/autoresearch/climbboard.py new file mode 100644 index 00000000..e2ee20da --- /dev/null +++ b/src/autoresearch/climbboard.py @@ -0,0 +1,385 @@ +"""The climb board: what every attempt on a benchmark tried and what came +of it, published to the target's `research-log` branch as data plus two +views whenever runs end. + +- `climb/data/.json` — one row per terminal attempt, dedup by run + id (its own directory: benchmark names are contract-controlled, and a + benchmark named `index` must not collide with `climb/index.json`); + the contract everything else derives from (and what an external dashboard + reads after the public flip). +- `CLIMB.md` — the numbers and the attempts table, rendered by GitHub. +- `climb.html` — a self-contained page that charts the JSON next to it + (open it from a clone; a Pages site can serve it as-is later). + +Publishing is idempotent by construction: rows merge by run id and files +are written only when their content changes, so the tick can call this +every pass without commit spam. Like the research-log ledger, the board is +advisory — a failure never stops the tick. +""" + +from __future__ import annotations + +import json +import logging +import re +from dataclasses import asdict, dataclass +from pathlib import Path +from typing import Any + +from autoresearch.runstate import ENDED, list_runs, run_dir + +log = logging.getLogger("autoresearch.climbboard") + +BOARD_BRANCH = "research-log" +MAX_HYPOTHESIS_CHARS = 160 +MAX_ROWS_PER_BENCHMARK = 2000 + + +@dataclass(frozen=True) +class ClimbRow: + run_id: str + agent: str + ended: str # ISO date + outcome: str + baseline: float | None + candidate: float | None + gpu_hours: float + hypothesis: str + pr_url: str + + +_NUM = re.compile(r"^(Baseline|Candidate): ([-+0-9.e]+)", re.M) +_HYP = re.compile(r"Hypothesis[:*\s]+(.+)", re.I) + + +def _report_fields(text: str) -> tuple[float | None, float | None, str]: + """(baseline, candidate, hypothesis one-liner) out of a run report.""" + baseline = candidate = None + for key, raw in _NUM.findall(text): + try: + value = float(raw) + except ValueError: + continue + if key == "Baseline": + baseline = value + else: + candidate = value + hyp = "" + m = _HYP.search(text) + if m: + hyp = re.sub(r"[`*_]|\s+", lambda g: " " if g.group().isspace() else "", m.group(1)) + hyp = hyp.strip().rstrip("-").strip()[:MAX_HYPOTHESIS_CHARS] + return baseline, candidate, hyp + + +def collect_rows(root: Path, target: str) -> dict[str, list[ClimbRow]]: + """Terminal attempts of `target` with a report, grouped by benchmark.""" + from datetime import UTC, datetime + + out: dict[str, list[ClimbRow]] = {} + for record in list_runs(root): + # only ENDED runs: an in-review run's outcome is not known yet (its + # PR may be rejected), and a published row is never rewritten + if record.target != target or record.state != ENDED: + continue + try: + report = (run_dir(root, record.run_id) / "report.md").read_text() + except OSError: + continue + baseline, candidate, hyp = _report_fields(report) + stage = record.stage or {} + ended = datetime.fromtimestamp(record.updated or record.created, tz=UTC) + outcome = record.ending or "ended" + out.setdefault(record.benchmark or "benchmark", []).append( + ClimbRow( + run_id=record.run_id, + agent=record.agent_id, + ended=ended.strftime("%Y-%m-%d %H:%M:%S"), + outcome=outcome, + baseline=baseline, + candidate=candidate, + gpu_hours=round(float(stage.get("gpu_hours_used") or 0.0), 2), # type: ignore[arg-type] + hypothesis=hyp, + pr_url=record.pr_url, + ) + ) + return out + + +def merge_rows(existing_json: str | None, fresh: list[ClimbRow]) -> list[dict[str, Any]]: + """Existing board rows plus any new ones, one per run id, oldest first. + A run already on the board keeps its published row (reports are final at + terminal state; the board never rewrites history).""" + rows: list[dict[str, Any]] = [] + seen: set[str] = set() + if existing_json: + try: + for item in json.loads(existing_json): + if isinstance(item, dict) and item.get("run_id") not in seen: + rows.append(item) + seen.add(str(item.get("run_id"))) + except ValueError: + log.warning("unreadable board JSON; rebuilding from local records") + for row in fresh: + if row.run_id not in seen: + rows.append(asdict(row)) + seen.add(row.run_id) + rows.sort(key=lambda r: str(r.get("ended", ""))) + # the board is a bounded VIEW (the contents API caps file sizes); the + # full history stays in reports/ on this same branch, and the trim is + # said out loud in CLIMB.md, never silent + return rows[-MAX_ROWS_PER_BENCHMARK:] + + +def _fmt(value: Any) -> str: + if value is None or value == "": + return "—" + if isinstance(value, float) and value == int(value): + return str(int(value)) + return str(value) + + +def render_md( + target: str, boards: dict[str, list[dict[str, Any]]], directions: dict[str, str] +) -> str: + """CLIMB.md: per benchmark, the headline numbers and the attempts table + (newest first). Plain markdown; the chart lives in climb.html.""" + lines = [ + "", + f"# Climb — {target}", + "", + "Written by the kernel when runs end. Data: `climb/data/.json`;", + "chart: open `climb.html` from a clone of this branch.", + ] + for benchmark in sorted(boards): + rows = boards[benchmark] + direction = directions.get(benchmark, "min") + pick = max if direction == "max" else min + measured = [r for r in rows if isinstance(r.get("candidate"), int | float)] + improved = [r for r in rows if r.get("outcome") in ("merged", "improved")] + baselines = [r["baseline"] for r in rows if isinstance(r.get("baseline"), int | float)] + best = pick((r["candidate"] for r in measured), default=None) + gpu = sum(float(r.get("gpu_hours") or 0.0) for r in rows) + lines += [ + "", + f"## {benchmark}", + "", + f"Attempts: **{len(rows)}** ({len(improved)} improved) · best candidate: " + f"**{_fmt(best)}** ({direction}) · latest baseline: " + f"**{_fmt(baselines[-1] if baselines else None)}** · GPU-hours: **{gpu:.1f}**", + ] + if len(rows) >= MAX_ROWS_PER_BENCHMARK: + lines += [ + "", + f"Only the newest {MAX_ROWS_PER_BENCHMARK} attempts are on the board; " + "archived reports stay in `reports/` on this branch.", + ] + lines += [ + "", + "| ended (UTC) | agent | hypothesis | outcome | candidate | GPU-h |", + "| --- | --- | --- | --- | --- | --- |", + ] + for r in reversed(rows): + outcome = str(r.get("outcome", "")) + if r.get("pr_url"): + outcome = f"[{outcome}]({r['pr_url']})" + hyp = str(r.get("hypothesis") or "").replace("|", "\\|") + lines.append( + f"| {r.get('ended', '')} | {r.get('agent', '')} | {hyp} | {outcome} " + f"| {_fmt(r.get('candidate'))} | {_fmt(r.get('gpu_hours'))} |" + ) + return "\n".join(lines) + "\n" + + +def render_html( + target: str, boards: dict[str, list[dict[str, Any]]], directions: dict[str, str] +) -> str: + """One self-contained page: the data is EMBEDDED (a browser blocks + fetch() from a file:// page, and the direct-from-clone view must work), + only chart.js arrives from its CDN — nothing bulky is committed to the + target's branch. Light and dark follow the viewer's system theme.""" + # "<" is escaped INSIDE the JSON (still valid JSON): a hypothesis line is + # agent-written text, and a literal in it would close the inline + # script and run whatever follows in the published page + payload = json.dumps({"boards": boards, "directions": directions}).replace("<", "\\u003c") + return ( + "\n\n" + "\n" + f"Climb — {target}\n" + "\n" + "
\n" + f"

{target} · climb

\n" + "

Written by the kernel when runs end. Full reports live in " + "reports/ on this branch.

\n" + "
\n
\n" + ) + + +def contract_directions(contract: Any) -> dict[str, str]: + """{benchmark: direction} out of a loaded contract (None -> {}).""" + if contract is None: + return {} + return {b.name: b.direction for b in contract.benchmarks} + + +def _read_index(github: Any, target: str) -> dict[str, str] | None: + """climb/index.json: {benchmark: direction} for every benchmark ever + published — the board's own memory, so a benchmark whose local records + were cleaned up (or that left the contract) keeps its place in the + views. {} means the board has no index yet (a 404); None means the read + FAILED — the caller must not rewrite the index it could not see, or a + transient outage would shrink it.""" + try: + raw = github.get_file(target, "climb/index.json", BOARD_BRANCH) + except Exception as exc: + status = getattr(exc, "status", None) + if status == 404: + return {} + log.warning("climb index unreadable (%s); not rewriting it", exc) + return None + try: + data = json.loads(raw) + except ValueError: + log.warning("climb index malformed; not rewriting it") + return None # same stance as an outage: never rewrite what we cannot see + return {str(k): str(v) for k, v in data.items()} if isinstance(data, dict) else None + + +def service_climb_board( + root: Path, github: Any, target: str, directions: dict[str, str] | None = None +) -> int: + """Publish the board for `target`. Returns how many files changed. + + Every file is compared against the branch and written only when + different — which is also the retry: a view (or index) whose upload + failed last pass differs again next pass. A benchmark whose fresh JSON + could not be uploaded is rendered from its last PUBLISHED rows, so the + views never point at data that is not on the branch.""" + local = collect_rows(root, target) + index = _read_index(github, target) + names = set(local) | set(index or {}) + if not names: + return 0 + directions = {**(index or {}), **(directions or {})} + changed = 0 + boards: dict[str, list[dict[str, Any]]] = {} + for benchmark in sorted(names): + path = f"climb/data/{benchmark}.json" + try: + existing: str | None = github.get_file(target, path, BOARD_BRANCH) + if existing is not None: + parsed = json.loads(existing) + if not isinstance(parsed, list): + raise ValueError("board data is not a list") + except (ValueError, TypeError) as exc: + # readable but not a row list: same stance as an outage — never + # let a fresh merge overwrite what we cannot interpret + log.warning("board JSON malformed for %s (%s); skipped", benchmark, exc) + continue + except Exception as exc: + if getattr(exc, "status", None) != 404: + # an outage is not an empty board: overwriting would replace + # this benchmark's published history — sit the pass out (its + # index entry survives) + log.warning("board JSON unreadable for %s (%s); skipped", benchmark, exc) + continue + existing = None + rows = merge_rows(existing, local.get(benchmark, [])) + if not rows: + continue + text = json.dumps(rows, indent=1) + "\n" + if text == existing: + boards[benchmark] = rows + continue + if github.ensure_branch(target, BOARD_BRANCH) and github.put_file( + target, path, text, BOARD_BRANCH, f"climb board: {benchmark}" + ): + changed += 1 + boards[benchmark] = rows + elif existing: + # the branch still holds the previous rows: render those + boards[benchmark] = merge_rows(existing, []) + if not boards: + return changed + # the index keeps every benchmark it knows — a transient failed read of + # one benchmark's JSON must not orphan its history; the views simply + # render without it until a later pass reads it again. An index that + # could not be READ at all (None) is never rewritten this pass. + wanted = {b: directions.get(b, "min") for b in sorted(set(boards) | set(index or {}))} + views: list[tuple[str, str]] = [ + ("CLIMB.md", render_md(target, boards, wanted)), + ("climb.html", render_html(target, boards, wanted)), + ] + if index is not None: + views.insert(0, ("climb/index.json", json.dumps(wanted, indent=1) + "\n")) + for path, content in views: + if ( + content != github.get_file_content(target, path, BOARD_BRANCH) + and github.ensure_branch(target, BOARD_BRANCH) + and github.put_file(target, path, content, BOARD_BRANCH, "climb board") + ): + changed += 1 + return changed diff --git a/src/autoresearch/tick.py b/src/autoresearch/tick.py index 10e59d68..64add949 100644 --- a/src/autoresearch/tick.py +++ b/src/autoresearch/tick.py @@ -1451,6 +1451,12 @@ def tick( service_research_log(root, github, spec, now) except Exception as exc: # the ledger is advisory; the tick continues log.warning("research-log service failed: %s", exc) + try: + from autoresearch.climbboard import contract_directions, service_climb_board + + service_climb_board(root, github, spec.target, contract_directions(contract)) + except Exception as exc: # the board is advisory; the tick continues + log.warning("climb board service failed: %s", exc) intake_job = ( service_intake( root, github, compute, spec, now, contract, limits, dry_run=followup_dry_run diff --git a/tests/test_climbboard.py b/tests/test_climbboard.py new file mode 100644 index 00000000..5e648f7d --- /dev/null +++ b/tests/test_climbboard.py @@ -0,0 +1,424 @@ +"""The climb board: rows out of terminal records, idempotent merge, and the +three published views.""" + +import json +from dataclasses import replace as dc_replace +from pathlib import Path + +from autoresearch.climbboard import ( + ClimbRow, + collect_rows, + contract_directions, + merge_rows, + render_html, + render_md, + service_climb_board, +) +from autoresearch.runstate import RunRecord, run_dir, save_record + +REPORT = """# Run report — org/repo / speedrun +Outcome: **no-improvement** +Baseline: 9472.0 +Candidate: 38146.0 + +## Agent's report +- **Hypothesis:** the warmdown starts too early for the crossing. +- **Change:** moved it. +""" + + +def _terminal_run(root: Path, run_id: str, *, state="ended", ending="negative-result") -> None: + record = RunRecord( + run_id=run_id, + target="org/repo", + task_title="t", + state=state, + ending=ending, + benchmark="speedrun", + agent_id="agent-03", + pr_url="https://github.com/org/repo/pull/9" if ending == "merged" else "", + created=1_787_900_000.0, + updated=1_787_950_000.0, + stage={"gpu_hours_used": 12.5}, + ) + save_record(root, record, 1_787_950_000.0) + (run_dir(root, run_id) / "report.md").write_text(REPORT) + + +def test_collect_rows_reads_terminal_records_and_reports(tmp_path: Path) -> None: + _terminal_run(tmp_path, "speedrun-1") + _terminal_run(tmp_path, "speedrun-2", ending="merged") + # non-terminal runs stay off the board: an in-review run's outcome is not + # known yet (its PR may be rejected), and a waiting run is mid-flight + _terminal_run(tmp_path, "speedrun-3", state="in-review", ending="") + save_record( + tmp_path, + RunRecord(run_id="w", target="org/repo", task_title="t", state="waiting"), + 1.0, + ) + boards = collect_rows(tmp_path, "org/repo") + rows = boards["speedrun"] + assert [r.run_id for r in rows] == ["speedrun-1", "speedrun-2"] + row = rows[0] + assert row.baseline == 9472.0 and row.candidate == 38146.0 + assert row.hypothesis.startswith("the warmdown starts too early") + assert row.gpu_hours == 12.5 and row.agent == "agent-03" + assert rows[1].outcome == "merged" and rows[1].pr_url.endswith("/pull/9") + + +def test_merge_is_idempotent_and_keeps_published_history(tmp_path: Path) -> None: + row = ClimbRow( + run_id="r1", + agent="agent-01", + ended="2026-08-29 10:00", + outcome="negative-result", + baseline=9472.0, + candidate=None, + gpu_hours=4.0, + hypothesis="h", + pr_url="", + ) + first = merge_rows(None, [row]) + assert [r["run_id"] for r in first] == ["r1"] + # a published row wins over a re-collected one (history is never rewritten), + # merging again changes nothing, and unreadable JSON falls back to fresh rows + edited = json.dumps([{**first[0], "hypothesis": "published wording"}]) + again = merge_rows(edited, [row]) + assert again[0]["hypothesis"] == "published wording" + assert merge_rows(json.dumps(again), [row]) == again + assert [r["run_id"] for r in merge_rows("not json", [row])] == ["r1"] + + +def test_views_render_the_rows_and_respect_direction(tmp_path: Path) -> None: + _terminal_run(tmp_path, "speedrun-1") + boards = {b: merge_rows(None, rows) for b, rows in collect_rows(tmp_path, "org/repo").items()} + md = render_md("org/repo", boards, {"speedrun": "min"}) + assert "## speedrun" in md and "| 2026-" in md and "38146" in md + assert "Attempts: **1**" in md + # direction decides which candidate is "best" (contracts support max) + two = {"acc": [{"run_id": "a", "candidate": 0.3}, {"run_id": "b", "candidate": 0.7}]} + assert "best candidate: **0.7** (max)" in render_md("org/repo", two, {"acc": "max"}) + assert "best candidate: **0.3** (min)" in render_md("org/repo", two, {"acc": "min"}) + # the chart data is EMBEDDED (fetch() is blocked on a file:// page) and + # carries the direction + html = render_html("org/repo", boards, {"speedrun": "min"}) + assert "fetch(" not in html and '"boards"' in html and "38146" in html + # agent-written text cannot break out of the inline script: "<" is + # escaped inside the JSON, so a payload stays data + hostile = {"b": [{"run_id": "x", "hypothesis": ""}]} + page = render_html("org/repo", hostile, {"b": "min"}) + assert page.count("") == 2 # chart.js include + our own script, nothing injected + assert "\\u003c/script" in page + + +def test_row_cap_is_loud_not_silent() -> None: + import json as _json + + from autoresearch.climbboard import MAX_ROWS_PER_BENCHMARK + + rows = [{"run_id": f"r{i}", "ended": f"2026-01-01 {i:02d}:00"} for i in range(40)] + many = [ + {"run_id": f"m{i}", "ended": f"2026-{1 + i // 10000:02d}-01 00:{i % 60:02d}"} + for i in range(MAX_ROWS_PER_BENCHMARK + 50) + ] + capped = merge_rows(_json.dumps(many), []) + assert len(capped) <= MAX_ROWS_PER_BENCHMARK + md = render_md("org/repo", {"b": capped}, {"b": "min"}) + assert f"Only the newest {MAX_ROWS_PER_BENCHMARK} attempts" in md + small = render_md("org/repo", {"b": rows}, {"b": "min"}) + assert "Only the newest" not in small + + +def test_a_benchmark_named_index_cannot_collide_with_the_index(tmp_path: Path) -> None: + """Contract names allow `index`; its data lives under climb/data/, so the + direction index is never overwritten by a benchmark's rows.""" + gh = _BoardGitHub() + record = RunRecord( + run_id="idx-1", + target="org/repo", + task_title="t", + state="ended", + ending="negative-result", + benchmark="index", + created=1.0, + updated=2.0, + ) + save_record(tmp_path, record, 2.0) + (run_dir(tmp_path, "idx-1") / "report.md").write_text(REPORT) + service_climb_board(tmp_path, gh, "org/repo", {"index": "min"}) + assert json.loads(gh.files["climb/data/index.json"])[0]["run_id"] == "idx-1" + assert json.loads(gh.files["climb/index.json"]) == {"index": "min"} + + +def test_terminal_transition_keeps_the_spend_for_the_board(tmp_path: Path) -> None: + """_clear_stage wipes the waiting bookkeeping but keeps gpu_hours_used — + the production transition the board reads after (review #193 r6: rows + published zero GPU-hours because the collector ran on a wiped stage).""" + from autoresearch.attempt import _clear_stage + + record = RunRecord( + run_id="r", + target="org/repo", + task_title="t", + state="waiting", + benchmark="speedrun", + stage={"phase": "candidate", "afterany": "afterany:1", "gpu_hours_used": 36.0}, + wake_attempts=2, + deadline=9.0, + ) + cleared = _clear_stage(record) + assert cleared.stage == {"gpu_hours_used": 36.0} + assert cleared.wake_attempts == 0 and cleared.deadline == 0.0 + save_record(tmp_path, dc_replace(cleared, state="ended", ending="negative-result"), 2.0) + (run_dir(tmp_path, "r") / "report.md").write_text(REPORT) + rows = collect_rows(tmp_path, "org/repo")["speedrun"] + assert rows[0].gpu_hours == 36.0 + + +def test_contract_directions_maps_benchmarks() -> None: + from autoresearch.contract import load_contract + + contract = load_contract( + """ +benchmarks: + - name: tsp + command: uv run x --json + metric: m + direction: min + - name: acc + command: uv run y --json + metric: a + direction: max +budgets: {gpu_hours_per_run: 1, runs_per_week: 1} +scope: {allowed: [src/]} +roadmap: docs/roadmap.md +""", + "org/repo", + ) + assert contract_directions(contract) == {"tsp": "min", "acc": "max"} + assert contract_directions(None) == {} + + +class _GitHubError(Exception): + def __init__(self, status: int) -> None: + super().__init__(f"{status}") + self.status = status + + +class _BoardGitHub: + """get/put fake over an in-memory branch, with the client's semantics: + get_file raises (404 when absent), get_file_content is tolerant.""" + + def __init__(self) -> None: + self.files: dict[str, str] = {} + self.puts: list[str] = [] + self.index_outage = False + + def get_file(self, repo, path, ref): + if self.index_outage: + raise _GitHubError(500) + if path not in self.files: + raise _GitHubError(404) + return self.files[path] + + def get_file_content(self, repo, path, ref): + return self.files.get(path) + + def ensure_branch(self, repo, branch): + return True + + def put_file(self, repo, path, content, branch, message): + self.files[path] = content + self.puts.append(path) + return "created" + + +def test_service_publishes_once_per_change(tmp_path: Path) -> None: + _terminal_run(tmp_path, "speedrun-1") + gh = _BoardGitHub() + assert service_climb_board(tmp_path, gh, "org/repo") == 4 + assert sorted(gh.puts) == [ + "CLIMB.md", + "climb.html", + "climb/data/speedrun.json", + "climb/index.json", + ] + # unchanged state: nothing is written again (no commit spam) + gh.puts.clear() + assert service_climb_board(tmp_path, gh, "org/repo") == 0 + assert gh.puts == [] + # a new terminal run publishes exactly the changed files + _terminal_run(tmp_path, "speedrun-2", ending="merged") + assert service_climb_board(tmp_path, gh, "org/repo") >= 2 + assert json.loads(gh.files["climb/data/speedrun.json"])[-1]["run_id"] == "speedrun-2" + # no runs and no index at all: a quiet no-op + assert service_climb_board(tmp_path / "empty", gh, "org/repo") == 0 + + +def test_board_remembers_benchmarks_whose_local_records_are_gone(tmp_path: Path) -> None: + """The index on the branch is the board's memory: a benchmark published + long ago (records reaped since) keeps its place in every view when a + different benchmark publishes.""" + gh = _BoardGitHub() + gh.files["climb/index.json"] = json.dumps({"old-bench": "max"}) + gh.files["climb/data/old-bench.json"] = json.dumps( + [{"run_id": "old-1", "ended": "2026-01-01 00:00", "candidate": 0.5, "outcome": "improved"}] + ) + _terminal_run(tmp_path, "speedrun-1") + assert service_climb_board(tmp_path, gh, "org/repo", {"speedrun": "min"}) >= 3 + md = gh.files["CLIMB.md"] + assert "## old-bench" in md and "## speedrun" in md + assert "best candidate: **0.5** (max)" in md + assert json.loads(gh.files["climb/index.json"]) == {"old-bench": "max", "speedrun": "min"} + + +def test_transient_json_read_failure_never_orphans_an_indexed_benchmark(tmp_path: Path) -> None: + """get_file_content returns None on a transient API failure too: an + indexed benchmark whose JSON could not be read this pass stays in the + index (its history is still on the branch), and the views render it + again once a later read succeeds.""" + gh = _BoardGitHub() + gh.files["climb/index.json"] = json.dumps({"old-bench": "max"}) + old_rows = json.dumps( + [{"run_id": "old-1", "ended": "2026-01-01 00:00", "candidate": 0.5, "outcome": "improved"}] + ) + gh.files["climb/data/old-bench.json"] = old_rows + real_get = gh.get_file + blackout = {"climb/data/old-bench.json"} + + def flaky_get(repo, path, ref): + if path in blackout: + raise _GitHubError(500) + return real_get(repo, path, ref) + + gh.get_file = flaky_get # type: ignore[method-assign] + _terminal_run(tmp_path, "speedrun-1") + service_climb_board(tmp_path, gh, "org/repo", {"speedrun": "min"}) + assert "## old-bench" not in gh.files["CLIMB.md"] # skipped this pass... + assert json.loads(gh.files["climb/index.json"]) == {"old-bench": "max", "speedrun": "min"} + blackout.clear() # ...and back in the views once the read works again + service_climb_board(tmp_path, gh, "org/repo", {"speedrun": "min"}) + assert "## old-bench" in gh.files["CLIMB.md"] + + +def test_unreadable_board_json_is_never_overwritten(tmp_path: Path) -> None: + """An outage reading one benchmark's JSON must not let a fresh merge + replace its published history: that benchmark sits the pass out and its + index entry survives.""" + gh = _BoardGitHub() + gh.files["climb/index.json"] = json.dumps({"speedrun": "min"}) + gh.files["climb/data/speedrun.json"] = json.dumps( + [{"run_id": "old-1", "ended": "2026-01-01 00:00", "candidate": 9472}] + ) + _terminal_run(tmp_path, "speedrun-1") + real_get = gh.get_file + blackout = {"climb/data/speedrun.json"} + + def flaky(repo, path, ref): + if path in blackout: + raise _GitHubError(500) + return real_get(repo, path, ref) + + gh.get_file = flaky # type: ignore[method-assign] + service_climb_board(tmp_path, gh, "org/repo", {"speedrun": "min"}) + assert json.loads(gh.files["climb/data/speedrun.json"])[0]["run_id"] == "old-1" # untouched + assert json.loads(gh.files["climb/index.json"]) == {"speedrun": "min"} + blackout.clear() + service_climb_board(tmp_path, gh, "org/repo", {"speedrun": "min"}) + ids = [r["run_id"] for r in json.loads(gh.files["climb/data/speedrun.json"])] + assert ids == ["old-1", "speedrun-1"] # history + the fresh row, after the outage + + +def test_non_list_board_json_is_never_overwritten(tmp_path: Path) -> None: + """A data file that decodes to something other than a row list is as + opaque as an outage: the benchmark sits the pass out untouched.""" + gh = _BoardGitHub() + gh.files["climb/index.json"] = json.dumps({"speedrun": "min"}) + gh.files["climb/data/speedrun.json"] = json.dumps({"rows": "elsewhere"}) + _terminal_run(tmp_path, "speedrun-1") + service_climb_board(tmp_path, gh, "org/repo", {"speedrun": "min"}) + assert json.loads(gh.files["climb/data/speedrun.json"]) == {"rows": "elsewhere"} + + +def test_same_minute_attempts_order_by_end_time(tmp_path: Path) -> None: + """`ended` carries seconds: it is the merge sort key, and two attempts + in one minute must not fall back to directory order.""" + _terminal_run(tmp_path, "speedrun-b") # updated=1_787_950_000 + record = RunRecord( + run_id="speedrun-a", + target="org/repo", + task_title="t", + state="ended", + ending="negative-result", + benchmark="speedrun", + created=1_787_900_000.0, + updated=1_787_950_030.0, # same minute, 30 s later + ) + save_record(tmp_path, record, record.updated) + (run_dir(tmp_path, "speedrun-a") / "report.md").write_text(REPORT) + rows = merge_rows(None, collect_rows(tmp_path, "org/repo")["speedrun"]) + assert [r["run_id"] for r in rows] == ["speedrun-b", "speedrun-a"] + assert rows[0]["ended"].count(":") == 2 # seconds present + + +def test_failed_index_read_never_rewrites_the_index(tmp_path: Path) -> None: + """A transient failure reading the index itself must not shrink it: the + pass publishes what it can but leaves climb/index.json untouched.""" + gh = _BoardGitHub() + gh.files["climb/index.json"] = "not json {" + _terminal_run(tmp_path, "speedrun-1") + service_climb_board(tmp_path, gh, "org/repo", {"speedrun": "min"}) + assert gh.files["climb/index.json"] == "not json {" # malformed: untouched + gh.files["climb/index.json"] = json.dumps({"old-bench": "max"}) + gh.index_outage = True + service_climb_board(tmp_path, gh, "org/repo", {"speedrun": "min"}) + assert json.loads(gh.files["climb/index.json"]) == {"old-bench": "max"} # untouched + gh.index_outage = False # outage over: the union is restored + service_climb_board(tmp_path, gh, "org/repo", {"speedrun": "min"}) + assert json.loads(gh.files["climb/index.json"]) == {"old-bench": "max", "speedrun": "min"} + + +def test_failed_view_upload_is_retried_next_pass(tmp_path: Path) -> None: + _terminal_run(tmp_path, "speedrun-1") + + gh = _BoardGitHub() + fail = {"CLIMB.md"} + real_put = gh.put_file + + def flaky(repo, path, content, branch, message): + if path in fail: + return "" + return real_put(repo, path, content, branch, message) + + gh.put_file = flaky # type: ignore[method-assign] + service_climb_board(tmp_path, gh, "org/repo") + assert "CLIMB.md" not in gh.files + fail.clear() # the outage ends; the next pass sees the view differ and retries + assert service_climb_board(tmp_path, gh, "org/repo") == 1 + assert "## speedrun" in gh.files["CLIMB.md"] + + +def test_failed_json_upload_renders_last_published_rows(tmp_path: Path) -> None: + """Views never point at data that is not on the branch: when a fresh + JSON cannot be uploaded, the benchmark renders from its last published + rows.""" + gh = _BoardGitHub() + published = [ + {"run_id": "old-1", "ended": "2026-01-01 00:00", "candidate": 9472, "outcome": "improved"} + ] + gh.files["climb/index.json"] = json.dumps({"speedrun": "min"}) + gh.files["climb/data/speedrun.json"] = json.dumps(published) + _terminal_run(tmp_path, "speedrun-1") # fresh row that will fail to upload + fail = {"climb/data/speedrun.json"} + real_put = gh.put_file + + def flaky(repo, path, content, branch, message): + if path in fail: + return "" + return real_put(repo, path, content, branch, message) + + gh.put_file = flaky # type: ignore[method-assign] + service_climb_board(tmp_path, gh, "org/repo") + md = gh.files["CLIMB.md"] + assert "old-1" not in md # run ids never render; check by content: + assert "9472" in md and "38146" not in md # published row only, not the failed fresh one