From 0d38f9c8a2ed4539fea020bfa722709fc3ad0509 Mon Sep 17 00:00:00 2001 From: Mengye Ren Date: Sat, 29 Aug 2026 17:33:31 -0400 Subject: [PATCH 01/14] The climb board: data, table, and chart of every attempt, kernel-published MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CLIMB.md (numbers + attempts table, rendered by GitHub), climb/.json (the data contract), and climb.html (a chart.js page over the JSON) land on the target's research-log branch as runs end — published by a tick service like the ledger, not a GitHub Action: Pages is public on a private repo's plan, an Action would rebuild kernel state, and Actions minutes stay spent on CI. Rows merge by run id and files are written only on change, so the service is idempotent and history survives a state-root cleanup. At the public flip a Pages site serves the same files unchanged. Co-Authored-By: Claude Fable 5 --- CHANGELOG.md | 1 + docs/roadmap.md | 7 + src/autoresearch/climbboard.py | 239 +++++++++++++++++++++++++++++++++ src/autoresearch/tick.py | 6 + tests/test_climbboard.py | 131 ++++++++++++++++++ 5 files changed, 384 insertions(+) create mode 100644 src/autoresearch/climbboard.py create mode 100644 tests/test_climbboard.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 3bee29dd..113e25be 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,7 @@ Versions follow [SemVer](https://semver.org). ### Added +- The climb board: `CLIMB.md` (numbers + attempts table), `climb/.json` (the data), and `climb.html` (a chart of the JSON) on the target's `research-log` branch, kernel-published as runs end. Idempotent: rows merge by run id, files are written only on change. - SWEEPS: `launch --array K` fans one command out to K jobs, each with `SWEEP_INDEX=0..K-1` in its environment, delivered back as K results (artifacts under `results///`) in one wake. It counts as one diff --git a/docs/roadmap.md b/docs/roadmap.md index 8f8a17d1..b6fd4b24 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -160,6 +160,13 @@ open-ended sweep space. - [ ] Task selection reads lessons/ + recent runs/ (the loop's research memory) - [ ] Periodic distillation pass: raw reports → bounded lessons/.md - [ ] Weekly digest aggregates per-run reports; cost ledger; leaderboard history +- [x] Climb board on the research-log branch: `climb/.json` + + `CLIMB.md` + `climb.html`; a Pages site can serve the same files at the + public flip +- [ ] Local-inference roles (big item, 2026-08-29): open-weight author/judge + models served on cluster GPUs (vLLM/SGLang behind the hermes provider + seam) — stability against API model retirement, and pinned weights make + the research program reproducible ## Cross-cluster (dev plan, 2026-08-29) diff --git a/src/autoresearch/climbboard.py b/src/autoresearch/climbboard.py new file mode 100644 index 00000000..476ed559 --- /dev/null +++ b/src/autoresearch/climbboard.py @@ -0,0 +1,239 @@ +"""The climb board: what every attempt on a benchmark tried and what came +of it, published to the target's `research-log` branch as data plus two +views whenever runs end. + +- `climb/.json` — one row per terminal attempt, dedup by run id; + the contract everything else derives from (and what an external dashboard + reads after the public flip). +- `CLIMB.md` — the numbers and the attempts table, rendered by GitHub. +- `climb.html` — a self-contained page that charts the JSON next to it + (open it from a clone; a Pages site can serve it as-is later). + +Publishing is idempotent by construction: rows merge by run id and files +are written only when their content changes, so the tick can call this +every pass without commit spam. Like the research-log ledger, the board is +advisory — a failure never stops the tick. +""" + +from __future__ import annotations + +import json +import logging +import re +from dataclasses import asdict, dataclass +from pathlib import Path +from typing import Any + +from autoresearch.runstate import ENDED, IN_REVIEW, list_runs, run_dir + +log = logging.getLogger("autoresearch.climbboard") + +BOARD_BRANCH = "research-log" +MAX_HYPOTHESIS_CHARS = 160 +MAX_ROWS_PER_BENCHMARK = 2000 + + +@dataclass(frozen=True) +class ClimbRow: + run_id: str + agent: str + ended: str # ISO date + outcome: str + baseline: float | None + candidate: float | None + gpu_hours: float + hypothesis: str + pr_url: str + + +_NUM = re.compile(r"^(Baseline|Candidate): ([-+0-9.e]+)", re.M) +_HYP = re.compile(r"Hypothesis[:*\s]+(.+)", re.I) + + +def _report_fields(text: str) -> tuple[float | None, float | None, str]: + """(baseline, candidate, hypothesis one-liner) out of a run report.""" + baseline = candidate = None + for key, raw in _NUM.findall(text): + try: + value = float(raw) + except ValueError: + continue + if key == "Baseline": + baseline = value + else: + candidate = value + hyp = "" + m = _HYP.search(text) + if m: + hyp = re.sub(r"[`*_]|\s+", lambda g: " " if g.group().isspace() else "", m.group(1)) + hyp = hyp.strip().rstrip("-").strip()[:MAX_HYPOTHESIS_CHARS] + return baseline, candidate, hyp + + +def collect_rows(root: Path, target: str) -> dict[str, list[ClimbRow]]: + """Terminal attempts of `target` with a report, grouped by benchmark.""" + from datetime import UTC, datetime + + out: dict[str, list[ClimbRow]] = {} + for record in list_runs(root): + if record.target != target or record.state not in (ENDED, IN_REVIEW): + continue + try: + report = (run_dir(root, record.run_id) / "report.md").read_text() + except OSError: + continue + baseline, candidate, hyp = _report_fields(report) + stage = record.stage or {} + ended = datetime.fromtimestamp(record.updated or record.created, tz=UTC) + outcome = record.ending or ("improved" if record.state == IN_REVIEW else "ended") + out.setdefault(record.benchmark or "benchmark", []).append( + ClimbRow( + run_id=record.run_id, + agent=record.agent_id, + ended=ended.strftime("%Y-%m-%d %H:%M"), + outcome=outcome, + baseline=baseline, + candidate=candidate, + gpu_hours=round(float(stage.get("gpu_hours_used") or 0.0), 2), # type: ignore[arg-type] + hypothesis=hyp, + pr_url=record.pr_url, + ) + ) + return out + + +def merge_rows(existing_json: str | None, fresh: list[ClimbRow]) -> list[dict[str, Any]]: + """Existing board rows plus any new ones, one per run id, oldest first. + A run already on the board keeps its published row (reports are final at + terminal state; the board never rewrites history).""" + rows: list[dict[str, Any]] = [] + seen: set[str] = set() + if existing_json: + try: + for item in json.loads(existing_json): + if isinstance(item, dict) and item.get("run_id") not in seen: + rows.append(item) + seen.add(str(item.get("run_id"))) + except ValueError: + log.warning("unreadable board JSON; rebuilding from local records") + for row in fresh: + if row.run_id not in seen: + rows.append(asdict(row)) + seen.add(row.run_id) + rows.sort(key=lambda r: str(r.get("ended", ""))) + return rows[-MAX_ROWS_PER_BENCHMARK:] + + +def _fmt(value: Any) -> str: + if value is None or value == "": + return "—" + if isinstance(value, float) and value == int(value): + return str(int(value)) + return str(value) + + +def render_md(target: str, boards: dict[str, list[dict[str, Any]]]) -> str: + """CLIMB.md: per benchmark, the headline numbers and the attempts table + (newest first). Plain markdown; the chart lives in climb.html.""" + lines = [ + "", + f"# Climb — {target}", + "", + "Written by the kernel when runs end. Data: `climb/.json`;", + "chart: open `climb.html` from a clone of this branch.", + ] + for benchmark in sorted(boards): + rows = boards[benchmark] + measured = [r for r in rows if isinstance(r.get("candidate"), int | float)] + improved = [r for r in rows if r.get("outcome") == "improved"] + baselines = [r["baseline"] for r in rows if isinstance(r.get("baseline"), int | float)] + best = min((r["candidate"] for r in measured), default=None) + gpu = sum(float(r.get("gpu_hours") or 0.0) for r in rows) + lines += [ + "", + f"## {benchmark}", + "", + f"Attempts: **{len(rows)}** ({len(improved)} improved) · best candidate: " + f"**{_fmt(best)}** · latest baseline: **{_fmt(baselines[-1] if baselines else None)}**" + f" · GPU-hours: **{gpu:.1f}**", + "", + "| ended (UTC) | agent | hypothesis | outcome | candidate | GPU-h |", + "| --- | --- | --- | --- | --- | --- |", + ] + for r in reversed(rows): + outcome = str(r.get("outcome", "")) + if r.get("pr_url"): + outcome = f"[{outcome}]({r['pr_url']})" + hyp = str(r.get("hypothesis") or "").replace("|", "\\|") + lines.append( + f"| {r.get('ended', '')} | {r.get('agent', '')} | {hyp} | {outcome} " + f"| {_fmt(r.get('candidate'))} | {_fmt(r.get('gpu_hours'))} |" + ) + return "\n".join(lines) + "\n" + + +def render_html(target: str, benchmarks: list[str]) -> str: + """One self-contained page charting every benchmark's JSON (same dir). + Chart.js arrives from its CDN — nothing bulky is committed to the + target's branch, and the page works anywhere with network.""" + benches = json.dumps(sorted(benchmarks)) + return ( + "\n\n" + f"Climb — {target}\n" + "\n" + "\n" + f"

Climb — {target}

\n
\n\n" + ) + + +def service_climb_board(root: Path, github: Any, target: str) -> int: + """Publish the board for `target`. Returns how many files changed.""" + boards_local = collect_rows(root, target) + if not boards_local: + return 0 + changed = 0 + merged: dict[str, list[dict[str, Any]]] = {} + for benchmark, fresh in sorted(boards_local.items()): + path = f"climb/{benchmark}.json" + existing = github.get_file_content(target, path, BOARD_BRANCH) + rows = merge_rows(existing, fresh) + merged[benchmark] = rows + text = json.dumps(rows, indent=1) + "\n" + if text != existing: + if not github.ensure_branch(target, BOARD_BRANCH): + return changed + if github.put_file(target, path, text, BOARD_BRANCH, f"climb board: {benchmark}"): + changed += 1 + if not changed: + return 0 + md = render_md(target, merged) + if md != github.get_file_content(target, "CLIMB.md", BOARD_BRANCH): + changed += bool(github.put_file(target, "CLIMB.md", md, BOARD_BRANCH, "climb board")) + html = render_html(target, list(merged)) + if html != github.get_file_content(target, "climb.html", BOARD_BRANCH): + changed += bool(github.put_file(target, "climb.html", html, BOARD_BRANCH, "climb chart")) + return changed diff --git a/src/autoresearch/tick.py b/src/autoresearch/tick.py index 10e59d68..9ae70e78 100644 --- a/src/autoresearch/tick.py +++ b/src/autoresearch/tick.py @@ -1451,6 +1451,12 @@ def tick( service_research_log(root, github, spec, now) except Exception as exc: # the ledger is advisory; the tick continues log.warning("research-log service failed: %s", exc) + try: + from autoresearch.climbboard import service_climb_board + + service_climb_board(root, github, spec.target) + except Exception as exc: # the board is advisory; the tick continues + log.warning("climb board service failed: %s", exc) intake_job = ( service_intake( root, github, compute, spec, now, contract, limits, dry_run=followup_dry_run diff --git a/tests/test_climbboard.py b/tests/test_climbboard.py new file mode 100644 index 00000000..b52ffa80 --- /dev/null +++ b/tests/test_climbboard.py @@ -0,0 +1,131 @@ +"""The climb board: rows out of terminal records, idempotent merge, and the +three published views.""" + +import json +from pathlib import Path + +from autoresearch.climbboard import ( + ClimbRow, + collect_rows, + merge_rows, + render_html, + render_md, + service_climb_board, +) +from autoresearch.runstate import RunRecord, run_dir, save_record + +REPORT = """# Run report — org/repo / speedrun +Outcome: **no-improvement** +Baseline: 9472.0 +Candidate: 38146.0 + +## Agent's report +- **Hypothesis:** the warmdown starts too early for the crossing. +- **Change:** moved it. +""" + + +def _terminal_run(root: Path, run_id: str, *, state="ended", ending="negative-result") -> None: + record = RunRecord( + run_id=run_id, + target="org/repo", + task_title="t", + state=state, + ending=ending, + benchmark="speedrun", + agent_id="agent-03", + pr_url="https://github.com/org/repo/pull/9" if state == "in-review" else "", + created=1_787_900_000.0, + updated=1_787_950_000.0, + stage={"gpu_hours_used": 12.5}, + ) + save_record(root, record, 1_787_950_000.0) + (run_dir(root, run_id) / "report.md").write_text(REPORT) + + +def test_collect_rows_reads_terminal_records_and_reports(tmp_path: Path) -> None: + _terminal_run(tmp_path, "speedrun-1") + _terminal_run(tmp_path, "speedrun-2", state="in-review", ending="") + # non-terminal and reportless runs stay off the board + save_record( + tmp_path, + RunRecord(run_id="w", target="org/repo", task_title="t", state="waiting"), + 1.0, + ) + boards = collect_rows(tmp_path, "org/repo") + rows = boards["speedrun"] + assert [r.run_id for r in rows] == ["speedrun-1", "speedrun-2"] + row = rows[0] + assert row.baseline == 9472.0 and row.candidate == 38146.0 + assert row.hypothesis.startswith("the warmdown starts too early") + assert row.gpu_hours == 12.5 and row.agent == "agent-03" + assert rows[1].outcome == "improved" and rows[1].pr_url.endswith("/pull/9") + + +def test_merge_is_idempotent_and_keeps_published_history(tmp_path: Path) -> None: + row = ClimbRow( + run_id="r1", + agent="agent-01", + ended="2026-08-29 10:00", + outcome="negative-result", + baseline=9472.0, + candidate=None, + gpu_hours=4.0, + hypothesis="h", + pr_url="", + ) + first = merge_rows(None, [row]) + assert [r["run_id"] for r in first] == ["r1"] + # a published row wins over a re-collected one (history is never rewritten), + # merging again changes nothing, and unreadable JSON falls back to fresh rows + edited = json.dumps([{**first[0], "hypothesis": "published wording"}]) + again = merge_rows(edited, [row]) + assert again[0]["hypothesis"] == "published wording" + assert merge_rows(json.dumps(again), [row]) == again + assert [r["run_id"] for r in merge_rows("not json", [row])] == ["r1"] + + +def test_views_render_the_rows(tmp_path: Path) -> None: + _terminal_run(tmp_path, "speedrun-1") + boards = {b: merge_rows(None, rows) for b, rows in collect_rows(tmp_path, "org/repo").items()} + md = render_md("org/repo", boards) + assert "## speedrun" in md and "| 2026-" in md and "38146" in md + assert "Attempts: **1**" in md + html = render_html("org/repo", list(boards)) + assert "climb/${b}.json" in html and '"speedrun"' in html + + +class _BoardGitHub: + """get/put fake over an in-memory branch.""" + + def __init__(self) -> None: + self.files: dict[str, str] = {} + self.puts: list[str] = [] + + def get_file_content(self, repo, path, ref): + return self.files.get(path) + + def ensure_branch(self, repo, branch): + return True + + def put_file(self, repo, path, content, branch, message): + self.files[path] = content + self.puts.append(path) + return "created" + + +def test_service_publishes_once_per_change(tmp_path: Path) -> None: + _terminal_run(tmp_path, "speedrun-1") + gh = _BoardGitHub() + assert service_climb_board(tmp_path, gh, "org/repo") == 3 + assert sorted(gh.puts) == ["CLIMB.md", "climb.html", "climb/speedrun.json"] + # unchanged state: nothing is written again (no commit spam) + gh.puts.clear() + assert service_climb_board(tmp_path, gh, "org/repo") == 0 + assert gh.puts == [] + # a new terminal run publishes exactly the changed files + _terminal_run(tmp_path, "speedrun-2", state="in-review", ending="") + assert service_climb_board(tmp_path, gh, "org/repo") >= 2 + assert json.loads(gh.files["climb/speedrun.json"])[-1]["run_id"] == "speedrun-2" + # no runs for the target at all: a quiet no-op + assert service_climb_board(tmp_path / "empty", gh, "org/repo") == 0 From 7fdd0cb5fca5b5e9e844a70412432548213072b1 Mon Sep 17 00:00:00 2001 From: Mengye Ren Date: Sat, 29 Aug 2026 17:42:21 -0400 Subject: [PATCH 02/14] Climb board review round 1: memory, direction, retries, embedded chart data MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The board keeps its own memory (climb/index.json: benchmark -> direction), so a benchmark whose local records were cleaned up keeps its place in the views. Both views honor direction: max. Every file is compared against the branch each pass, which is also the retry for a failed upload. A benchmark whose fresh JSON could not be uploaded renders from its last published rows, never data that is not on the branch. The chart data is embedded in climb.html — a browser blocks fetch() on a file:// page, so the direct-from-clone view needs it inline. Co-Authored-By: Claude Fable 5 --- src/autoresearch/climbboard.py | 152 +++++++++++++++++++++------------ tests/test_climbboard.py | 83 ++++++++++++++++-- 2 files changed, 175 insertions(+), 60 deletions(-) diff --git a/src/autoresearch/climbboard.py b/src/autoresearch/climbboard.py index 476ed559..e2ed2e08 100644 --- a/src/autoresearch/climbboard.py +++ b/src/autoresearch/climbboard.py @@ -132,7 +132,9 @@ def _fmt(value: Any) -> str: return str(value) -def render_md(target: str, boards: dict[str, list[dict[str, Any]]]) -> str: +def render_md( + target: str, boards: dict[str, list[dict[str, Any]]], directions: dict[str, str] +) -> str: """CLIMB.md: per benchmark, the headline numbers and the attempts table (newest first). Plain markdown; the chart lives in climb.html.""" lines = [ @@ -144,18 +146,20 @@ def render_md(target: str, boards: dict[str, list[dict[str, Any]]]) -> str: ] for benchmark in sorted(boards): rows = boards[benchmark] + direction = directions.get(benchmark, "min") + pick = max if direction == "max" else min measured = [r for r in rows if isinstance(r.get("candidate"), int | float)] improved = [r for r in rows if r.get("outcome") == "improved"] baselines = [r["baseline"] for r in rows if isinstance(r.get("baseline"), int | float)] - best = min((r["candidate"] for r in measured), default=None) + best = pick((r["candidate"] for r in measured), default=None) gpu = sum(float(r.get("gpu_hours") or 0.0) for r in rows) lines += [ "", f"## {benchmark}", "", f"Attempts: **{len(rows)}** ({len(improved)} improved) · best candidate: " - f"**{_fmt(best)}** · latest baseline: **{_fmt(baselines[-1] if baselines else None)}**" - f" · GPU-hours: **{gpu:.1f}**", + f"**{_fmt(best)}** ({direction}) · latest baseline: " + f"**{_fmt(baselines[-1] if baselines else None)}** · GPU-hours: **{gpu:.1f}**", "", "| ended (UTC) | agent | hypothesis | outcome | candidate | GPU-h |", "| --- | --- | --- | --- | --- | --- |", @@ -172,11 +176,14 @@ def render_md(target: str, boards: dict[str, list[dict[str, Any]]]) -> str: return "\n".join(lines) + "\n" -def render_html(target: str, benchmarks: list[str]) -> str: - """One self-contained page charting every benchmark's JSON (same dir). - Chart.js arrives from its CDN — nothing bulky is committed to the - target's branch, and the page works anywhere with network.""" - benches = json.dumps(sorted(benchmarks)) +def render_html( + target: str, boards: dict[str, list[dict[str, Any]]], directions: dict[str, str] +) -> str: + """One self-contained page: the data is EMBEDDED (a browser blocks + fetch() from a file:// page, and the direct-from-clone view must work), + only chart.js arrives from its CDN — nothing bulky is committed to the + target's branch.""" + payload = json.dumps({"boards": boards, "directions": directions}) return ( "\n\n" f"Climb — {target}\n" @@ -184,56 +191,95 @@ def render_html(target: str, benchmarks: list[str]) -> str: "\n" f"

Climb — {target}

\n
\n\n" + f"const data = {payload};\n" + "for (const b of Object.keys(data.boards).sort()) {\n" + " const rows = data.boards[b];\n" + " const pick = data.directions[b] === 'max' ? Math.max : Math.min;\n" + " const measured = rows.filter(r => typeof r.candidate === 'number');\n" + " let best;\n" + " const bestSoFar = measured.map(r =>\n" + " best = best === undefined ? r.candidate : pick(best, r.candidate));\n" + " const h = document.createElement('h2'); h.textContent = b;\n" + " const c = document.createElement('canvas');\n" + " document.getElementById('charts').append(h, c);\n" + " new Chart(c, {data: {labels: measured.map(r => r.ended),\n" + " datasets: [\n" + " {type: 'scatter', label: 'candidate', data: measured.map(r => r.candidate),\n" + " pointBackgroundColor: measured.map(r =>\n" + " r.outcome === 'improved' ? '#2a7' : '#999')},\n" + " {type: 'line', label: 'best so far', data: bestSoFar, stepped: true,\n" + " borderColor: '#26c', pointRadius: 0},\n" + " {type: 'line', label: 'baseline', data: measured.map(r => r.baseline),\n" + " borderColor: '#c33', borderDash: [6, 4], pointRadius: 0}]},\n" + " options: {plugins: {tooltip: {callbacks: {afterLabel:\n" + " (i) => measured[i.dataIndex].hypothesis}}}}});\n" + "}\n\n" ) -def service_climb_board(root: Path, github: Any, target: str) -> int: - """Publish the board for `target`. Returns how many files changed.""" - boards_local = collect_rows(root, target) - if not boards_local: +def _read_index(github: Any, target: str) -> dict[str, str]: + """climb/index.json: {benchmark: direction} for every benchmark ever + published — the board's own memory, so a benchmark whose local records + were cleaned up (or that left the contract) keeps its place in the + views.""" + raw = github.get_file_content(target, "climb/index.json", BOARD_BRANCH) + if not raw: + return {} + try: + data = json.loads(raw) + return {str(k): str(v) for k, v in data.items()} if isinstance(data, dict) else {} + except ValueError: + return {} + + +def service_climb_board( + root: Path, github: Any, target: str, directions: dict[str, str] | None = None +) -> int: + """Publish the board for `target`. Returns how many files changed. + + Every file is compared against the branch and written only when + different — which is also the retry: a view (or index) whose upload + failed last pass differs again next pass. A benchmark whose fresh JSON + could not be uploaded is rendered from its last PUBLISHED rows, so the + views never point at data that is not on the branch.""" + local = collect_rows(root, target) + index = _read_index(github, target) + names = set(local) | set(index) + if not names: return 0 + directions = {**index, **(directions or {})} changed = 0 - merged: dict[str, list[dict[str, Any]]] = {} - for benchmark, fresh in sorted(boards_local.items()): + boards: dict[str, list[dict[str, Any]]] = {} + for benchmark in sorted(names): path = f"climb/{benchmark}.json" existing = github.get_file_content(target, path, BOARD_BRANCH) - rows = merge_rows(existing, fresh) - merged[benchmark] = rows + rows = merge_rows(existing, local.get(benchmark, [])) + if not rows: + continue text = json.dumps(rows, indent=1) + "\n" - if text != existing: - if not github.ensure_branch(target, BOARD_BRANCH): - return changed - if github.put_file(target, path, text, BOARD_BRANCH, f"climb board: {benchmark}"): - changed += 1 - if not changed: - return 0 - md = render_md(target, merged) - if md != github.get_file_content(target, "CLIMB.md", BOARD_BRANCH): - changed += bool(github.put_file(target, "CLIMB.md", md, BOARD_BRANCH, "climb board")) - html = render_html(target, list(merged)) - if html != github.get_file_content(target, "climb.html", BOARD_BRANCH): - changed += bool(github.put_file(target, "climb.html", html, BOARD_BRANCH, "climb chart")) + if text == existing: + boards[benchmark] = rows + continue + if github.ensure_branch(target, BOARD_BRANCH) and github.put_file( + target, path, text, BOARD_BRANCH, f"climb board: {benchmark}" + ): + changed += 1 + boards[benchmark] = rows + elif existing: + # the branch still holds the previous rows: render those + boards[benchmark] = merge_rows(existing, []) + if not boards: + return changed + wanted = {b: directions.get(b, "min") for b in boards} + for path, content in ( + ("climb/index.json", json.dumps(wanted, indent=1) + "\n"), + ("CLIMB.md", render_md(target, boards, wanted)), + ("climb.html", render_html(target, boards, wanted)), + ): + if ( + content != github.get_file_content(target, path, BOARD_BRANCH) + and github.ensure_branch(target, BOARD_BRANCH) + and github.put_file(target, path, content, BOARD_BRANCH, "climb board") + ): + changed += 1 return changed diff --git a/tests/test_climbboard.py b/tests/test_climbboard.py index b52ffa80..cbfe79ec 100644 --- a/tests/test_climbboard.py +++ b/tests/test_climbboard.py @@ -85,14 +85,20 @@ def test_merge_is_idempotent_and_keeps_published_history(tmp_path: Path) -> None assert [r["run_id"] for r in merge_rows("not json", [row])] == ["r1"] -def test_views_render_the_rows(tmp_path: Path) -> None: +def test_views_render_the_rows_and_respect_direction(tmp_path: Path) -> None: _terminal_run(tmp_path, "speedrun-1") boards = {b: merge_rows(None, rows) for b, rows in collect_rows(tmp_path, "org/repo").items()} - md = render_md("org/repo", boards) + md = render_md("org/repo", boards, {"speedrun": "min"}) assert "## speedrun" in md and "| 2026-" in md and "38146" in md assert "Attempts: **1**" in md - html = render_html("org/repo", list(boards)) - assert "climb/${b}.json" in html and '"speedrun"' in html + # direction decides which candidate is "best" (contracts support max) + two = {"acc": [{"run_id": "a", "candidate": 0.3}, {"run_id": "b", "candidate": 0.7}]} + assert "best candidate: **0.7** (max)" in render_md("org/repo", two, {"acc": "max"}) + assert "best candidate: **0.3** (min)" in render_md("org/repo", two, {"acc": "min"}) + # the chart data is EMBEDDED (fetch() is blocked on a file:// page) and + # carries the direction + html = render_html("org/repo", boards, {"speedrun": "min"}) + assert "fetch(" not in html and '"boards"' in html and "38146" in html class _BoardGitHub: @@ -117,8 +123,8 @@ def put_file(self, repo, path, content, branch, message): def test_service_publishes_once_per_change(tmp_path: Path) -> None: _terminal_run(tmp_path, "speedrun-1") gh = _BoardGitHub() - assert service_climb_board(tmp_path, gh, "org/repo") == 3 - assert sorted(gh.puts) == ["CLIMB.md", "climb.html", "climb/speedrun.json"] + assert service_climb_board(tmp_path, gh, "org/repo") == 4 + assert sorted(gh.puts) == ["CLIMB.md", "climb.html", "climb/index.json", "climb/speedrun.json"] # unchanged state: nothing is written again (no commit spam) gh.puts.clear() assert service_climb_board(tmp_path, gh, "org/repo") == 0 @@ -127,5 +133,68 @@ def test_service_publishes_once_per_change(tmp_path: Path) -> None: _terminal_run(tmp_path, "speedrun-2", state="in-review", ending="") assert service_climb_board(tmp_path, gh, "org/repo") >= 2 assert json.loads(gh.files["climb/speedrun.json"])[-1]["run_id"] == "speedrun-2" - # no runs for the target at all: a quiet no-op + # no runs and no index at all: a quiet no-op assert service_climb_board(tmp_path / "empty", gh, "org/repo") == 0 + + +def test_board_remembers_benchmarks_whose_local_records_are_gone(tmp_path: Path) -> None: + """The index on the branch is the board's memory: a benchmark published + long ago (records reaped since) keeps its place in every view when a + different benchmark publishes.""" + gh = _BoardGitHub() + gh.files["climb/index.json"] = json.dumps({"old-bench": "max"}) + gh.files["climb/old-bench.json"] = json.dumps( + [{"run_id": "old-1", "ended": "2026-01-01 00:00", "candidate": 0.5, "outcome": "improved"}] + ) + _terminal_run(tmp_path, "speedrun-1") + assert service_climb_board(tmp_path, gh, "org/repo", {"speedrun": "min"}) >= 3 + md = gh.files["CLIMB.md"] + assert "## old-bench" in md and "## speedrun" in md + assert "best candidate: **0.5** (max)" in md + assert json.loads(gh.files["climb/index.json"]) == {"old-bench": "max", "speedrun": "min"} + + +def test_failed_view_upload_is_retried_next_pass(tmp_path: Path) -> None: + _terminal_run(tmp_path, "speedrun-1") + + gh = _BoardGitHub() + fail = {"CLIMB.md"} + real_put = gh.put_file + + def flaky(repo, path, content, branch, message): + if path in fail: + return "" + return real_put(repo, path, content, branch, message) + + gh.put_file = flaky # type: ignore[method-assign] + service_climb_board(tmp_path, gh, "org/repo") + assert "CLIMB.md" not in gh.files + fail.clear() # the outage ends; the next pass sees the view differ and retries + assert service_climb_board(tmp_path, gh, "org/repo") == 1 + assert "## speedrun" in gh.files["CLIMB.md"] + + +def test_failed_json_upload_renders_last_published_rows(tmp_path: Path) -> None: + """Views never point at data that is not on the branch: when a fresh + JSON cannot be uploaded, the benchmark renders from its last published + rows.""" + gh = _BoardGitHub() + published = [ + {"run_id": "old-1", "ended": "2026-01-01 00:00", "candidate": 9472, "outcome": "improved"} + ] + gh.files["climb/index.json"] = json.dumps({"speedrun": "min"}) + gh.files["climb/speedrun.json"] = json.dumps(published) + _terminal_run(tmp_path, "speedrun-1") # fresh row that will fail to upload + fail = {"climb/speedrun.json"} + real_put = gh.put_file + + def flaky(repo, path, content, branch, message): + if path in fail: + return "" + return real_put(repo, path, content, branch, message) + + gh.put_file = flaky # type: ignore[method-assign] + service_climb_board(tmp_path, gh, "org/repo") + md = gh.files["CLIMB.md"] + assert "old-1" not in md # run ids never render; check by content: + assert "9472" in md and "38146" not in md # published row only, not the failed fresh one From 9aed2bf1f64116760e00064470bdb30c5d234763 Mon Sep 17 00:00:00 2001 From: Mengye Ren Date: Sat, 29 Aug 2026 17:47:44 -0400 Subject: [PATCH 03/14] Climb board round 2: directions reach the board; embedded data cannot escape its script The tick passes the contract's benchmark directions (it had loaded the contract lines earlier and passed nothing). "<" is escaped inside the embedded JSON, so an agent-written hypothesis carrying stays data instead of running in the published page. Co-Authored-By: Claude Fable 5 --- src/autoresearch/climbboard.py | 12 +++++++++++- src/autoresearch/tick.py | 4 ++-- tests/test_climbboard.py | 31 +++++++++++++++++++++++++++++++ 3 files changed, 44 insertions(+), 3 deletions(-) diff --git a/src/autoresearch/climbboard.py b/src/autoresearch/climbboard.py index e2ed2e08..19456af9 100644 --- a/src/autoresearch/climbboard.py +++ b/src/autoresearch/climbboard.py @@ -183,7 +183,10 @@ def render_html( fetch() from a file:// page, and the direct-from-clone view must work), only chart.js arrives from its CDN — nothing bulky is committed to the target's branch.""" - payload = json.dumps({"boards": boards, "directions": directions}) + # "<" is escaped INSIDE the JSON (still valid JSON): a hypothesis line is + # agent-written text, and a literal in it would close the inline + # script and run whatever follows in the published page + payload = json.dumps({"boards": boards, "directions": directions}).replace("<", "\\u003c") return ( "\n\n" f"Climb — {target}\n" @@ -217,6 +220,13 @@ def render_html( ) +def contract_directions(contract: Any) -> dict[str, str]: + """{benchmark: direction} out of a loaded contract (None -> {}).""" + if contract is None: + return {} + return {b.name: b.direction for b in contract.benchmarks} + + def _read_index(github: Any, target: str) -> dict[str, str]: """climb/index.json: {benchmark: direction} for every benchmark ever published — the board's own memory, so a benchmark whose local records diff --git a/src/autoresearch/tick.py b/src/autoresearch/tick.py index 9ae70e78..64add949 100644 --- a/src/autoresearch/tick.py +++ b/src/autoresearch/tick.py @@ -1452,9 +1452,9 @@ def tick( except Exception as exc: # the ledger is advisory; the tick continues log.warning("research-log service failed: %s", exc) try: - from autoresearch.climbboard import service_climb_board + from autoresearch.climbboard import contract_directions, service_climb_board - service_climb_board(root, github, spec.target) + service_climb_board(root, github, spec.target, contract_directions(contract)) except Exception as exc: # the board is advisory; the tick continues log.warning("climb board service failed: %s", exc) intake_job = ( diff --git a/tests/test_climbboard.py b/tests/test_climbboard.py index cbfe79ec..02866197 100644 --- a/tests/test_climbboard.py +++ b/tests/test_climbboard.py @@ -7,6 +7,7 @@ from autoresearch.climbboard import ( ClimbRow, collect_rows, + contract_directions, merge_rows, render_html, render_md, @@ -99,6 +100,36 @@ def test_views_render_the_rows_and_respect_direction(tmp_path: Path) -> None: # carries the direction html = render_html("org/repo", boards, {"speedrun": "min"}) assert "fetch(" not in html and '"boards"' in html and "38146" in html + # agent-written text cannot break out of the inline script: "<" is + # escaped inside the JSON, so a payload stays data + hostile = {"b": [{"run_id": "x", "hypothesis": ""}]} + page = render_html("org/repo", hostile, {"b": "min"}) + assert page.count("") == 2 # chart.js include + our own script, nothing injected + assert "\\u003c/script" in page + + +def test_contract_directions_maps_benchmarks() -> None: + from autoresearch.contract import load_contract + + contract = load_contract( + """ +benchmarks: + - name: tsp + command: uv run x --json + metric: m + direction: min + - name: acc + command: uv run y --json + metric: a + direction: max +budgets: {gpu_hours_per_run: 1, runs_per_week: 1} +scope: {allowed: [src/]} +roadmap: docs/roadmap.md +""", + "org/repo", + ) + assert contract_directions(contract) == {"tsp": "min", "acc": "max"} + assert contract_directions(None) == {} class _BoardGitHub: From 489ddaaea76e0cbdd0b64aa2f7833367857c7138 Mon Sep 17 00:00:00 2001 From: Mengye Ren Date: Sat, 29 Aug 2026 17:52:01 -0400 Subject: [PATCH 04/14] A transient read failure never orphans an indexed benchmark Co-Authored-By: Claude Fable 5 --- src/autoresearch/climbboard.py | 5 ++++- tests/test_climbboard.py | 27 +++++++++++++++++++++++++++ 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/src/autoresearch/climbboard.py b/src/autoresearch/climbboard.py index 19456af9..f6cdc30a 100644 --- a/src/autoresearch/climbboard.py +++ b/src/autoresearch/climbboard.py @@ -280,7 +280,10 @@ def service_climb_board( boards[benchmark] = merge_rows(existing, []) if not boards: return changed - wanted = {b: directions.get(b, "min") for b in boards} + # the index keeps every benchmark it knows — a transient failed read of + # one benchmark's JSON must not orphan its history; the views simply + # render without it until a later pass reads it again + wanted = {b: directions.get(b, "min") for b in sorted(set(boards) | set(index))} for path, content in ( ("climb/index.json", json.dumps(wanted, indent=1) + "\n"), ("CLIMB.md", render_md(target, boards, wanted)), diff --git a/tests/test_climbboard.py b/tests/test_climbboard.py index 02866197..0ba7c375 100644 --- a/tests/test_climbboard.py +++ b/tests/test_climbboard.py @@ -185,6 +185,33 @@ def test_board_remembers_benchmarks_whose_local_records_are_gone(tmp_path: Path) assert json.loads(gh.files["climb/index.json"]) == {"old-bench": "max", "speedrun": "min"} +def test_transient_json_read_failure_never_orphans_an_indexed_benchmark(tmp_path: Path) -> None: + """get_file_content returns None on a transient API failure too: an + indexed benchmark whose JSON could not be read this pass stays in the + index (its history is still on the branch), and the views render it + again once a later read succeeds.""" + gh = _BoardGitHub() + gh.files["climb/index.json"] = json.dumps({"old-bench": "max"}) + old_rows = json.dumps( + [{"run_id": "old-1", "ended": "2026-01-01 00:00", "candidate": 0.5, "outcome": "improved"}] + ) + gh.files["climb/old-bench.json"] = old_rows + real_get = gh.get_file_content + blackout = {"climb/old-bench.json"} + + def flaky_get(repo, path, ref): + return None if path in blackout else real_get(repo, path, ref) + + gh.get_file_content = flaky_get # type: ignore[method-assign] + _terminal_run(tmp_path, "speedrun-1") + service_climb_board(tmp_path, gh, "org/repo", {"speedrun": "min"}) + assert "## old-bench" not in gh.files["CLIMB.md"] # skipped this pass... + assert json.loads(gh.files["climb/index.json"]) == {"old-bench": "max", "speedrun": "min"} + blackout.clear() # ...and back in the views once the read works again + service_climb_board(tmp_path, gh, "org/repo", {"speedrun": "min"}) + assert "## old-bench" in gh.files["CLIMB.md"] + + def test_failed_view_upload_is_retried_next_pass(tmp_path: Path) -> None: _terminal_run(tmp_path, "speedrun-1") From a3211fadc5f43f3b54a9d5e459ea17ac98e09c1a Mon Sep 17 00:00:00 2001 From: Mengye Ren Date: Sat, 29 Aug 2026 17:57:23 -0400 Subject: [PATCH 05/14] A failed index read never rewrites the index MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The index is read with the raising client call so a 404 (no board yet) and an outage are distinct: the first starts fresh, the second skips the index write for the pass — a transient failure must not shrink the board's memory. Co-Authored-By: Claude Fable 5 --- src/autoresearch/climbboard.py | 34 ++++++++++++++++++++++------------ tests/test_climbboard.py | 31 ++++++++++++++++++++++++++++++- 2 files changed, 52 insertions(+), 13 deletions(-) diff --git a/src/autoresearch/climbboard.py b/src/autoresearch/climbboard.py index f6cdc30a..f47543c7 100644 --- a/src/autoresearch/climbboard.py +++ b/src/autoresearch/climbboard.py @@ -227,14 +227,21 @@ def contract_directions(contract: Any) -> dict[str, str]: return {b.name: b.direction for b in contract.benchmarks} -def _read_index(github: Any, target: str) -> dict[str, str]: +def _read_index(github: Any, target: str) -> dict[str, str] | None: """climb/index.json: {benchmark: direction} for every benchmark ever published — the board's own memory, so a benchmark whose local records were cleaned up (or that left the contract) keeps its place in the - views.""" - raw = github.get_file_content(target, "climb/index.json", BOARD_BRANCH) - if not raw: - return {} + views. {} means the board has no index yet (a 404); None means the read + FAILED — the caller must not rewrite the index it could not see, or a + transient outage would shrink it.""" + try: + raw = github.get_file(target, "climb/index.json", BOARD_BRANCH) + except Exception as exc: + status = getattr(exc, "status", None) + if status == 404: + return {} + log.warning("climb index unreadable (%s); not rewriting it", exc) + return None try: data = json.loads(raw) return {str(k): str(v) for k, v in data.items()} if isinstance(data, dict) else {} @@ -254,10 +261,10 @@ def service_climb_board( views never point at data that is not on the branch.""" local = collect_rows(root, target) index = _read_index(github, target) - names = set(local) | set(index) + names = set(local) | set(index or {}) if not names: return 0 - directions = {**index, **(directions or {})} + directions = {**(index or {}), **(directions or {})} changed = 0 boards: dict[str, list[dict[str, Any]]] = {} for benchmark in sorted(names): @@ -282,13 +289,16 @@ def service_climb_board( return changed # the index keeps every benchmark it knows — a transient failed read of # one benchmark's JSON must not orphan its history; the views simply - # render without it until a later pass reads it again - wanted = {b: directions.get(b, "min") for b in sorted(set(boards) | set(index))} - for path, content in ( - ("climb/index.json", json.dumps(wanted, indent=1) + "\n"), + # render without it until a later pass reads it again. An index that + # could not be READ at all (None) is never rewritten this pass. + wanted = {b: directions.get(b, "min") for b in sorted(set(boards) | set(index or {}))} + views: list[tuple[str, str]] = [ ("CLIMB.md", render_md(target, boards, wanted)), ("climb.html", render_html(target, boards, wanted)), - ): + ] + if index is not None: + views.insert(0, ("climb/index.json", json.dumps(wanted, indent=1) + "\n")) + for path, content in views: if ( content != github.get_file_content(target, path, BOARD_BRANCH) and github.ensure_branch(target, BOARD_BRANCH) diff --git a/tests/test_climbboard.py b/tests/test_climbboard.py index 0ba7c375..7d93ef12 100644 --- a/tests/test_climbboard.py +++ b/tests/test_climbboard.py @@ -132,12 +132,27 @@ def test_contract_directions_maps_benchmarks() -> None: assert contract_directions(None) == {} +class _GitHubError(Exception): + def __init__(self, status: int) -> None: + super().__init__(f"{status}") + self.status = status + + class _BoardGitHub: - """get/put fake over an in-memory branch.""" + """get/put fake over an in-memory branch, with the client's semantics: + get_file raises (404 when absent), get_file_content is tolerant.""" def __init__(self) -> None: self.files: dict[str, str] = {} self.puts: list[str] = [] + self.index_outage = False + + def get_file(self, repo, path, ref): + if self.index_outage: + raise _GitHubError(500) + if path not in self.files: + raise _GitHubError(404) + return self.files[path] def get_file_content(self, repo, path, ref): return self.files.get(path) @@ -212,6 +227,20 @@ def flaky_get(repo, path, ref): assert "## old-bench" in gh.files["CLIMB.md"] +def test_failed_index_read_never_rewrites_the_index(tmp_path: Path) -> None: + """A transient failure reading the index itself must not shrink it: the + pass publishes what it can but leaves climb/index.json untouched.""" + gh = _BoardGitHub() + gh.files["climb/index.json"] = json.dumps({"old-bench": "max"}) + _terminal_run(tmp_path, "speedrun-1") + gh.index_outage = True + service_climb_board(tmp_path, gh, "org/repo", {"speedrun": "min"}) + assert json.loads(gh.files["climb/index.json"]) == {"old-bench": "max"} # untouched + gh.index_outage = False # outage over: the union is restored + service_climb_board(tmp_path, gh, "org/repo", {"speedrun": "min"}) + assert json.loads(gh.files["climb/index.json"]) == {"old-bench": "max", "speedrun": "min"} + + def test_failed_view_upload_is_retried_next_pass(tmp_path: Path) -> None: _terminal_run(tmp_path, "speedrun-1") From 191eb014939384443b890d6bdcd9a79ddb779bbc Mon Sep 17 00:00:00 2001 From: Mengye Ren Date: Sat, 29 Aug 2026 20:24:46 -0400 Subject: [PATCH 06/14] The row cap is loud, not silent Co-Authored-By: Claude Fable 5 --- src/autoresearch/climbboard.py | 11 +++++++++++ tests/test_climbboard.py | 18 ++++++++++++++++++ 2 files changed, 29 insertions(+) diff --git a/src/autoresearch/climbboard.py b/src/autoresearch/climbboard.py index f47543c7..b70240ab 100644 --- a/src/autoresearch/climbboard.py +++ b/src/autoresearch/climbboard.py @@ -121,6 +121,9 @@ def merge_rows(existing_json: str | None, fresh: list[ClimbRow]) -> list[dict[st rows.append(asdict(row)) seen.add(row.run_id) rows.sort(key=lambda r: str(r.get("ended", ""))) + # the board is a bounded VIEW (the contents API caps file sizes); the + # full history stays in reports/ on this same branch, and the trim is + # said out loud in CLIMB.md, never silent return rows[-MAX_ROWS_PER_BENCHMARK:] @@ -160,6 +163,14 @@ def render_md( f"Attempts: **{len(rows)}** ({len(improved)} improved) · best candidate: " f"**{_fmt(best)}** ({direction}) · latest baseline: " f"**{_fmt(baselines[-1] if baselines else None)}** · GPU-hours: **{gpu:.1f}**", + ] + if len(rows) >= MAX_ROWS_PER_BENCHMARK: + lines += [ + "", + f"Only the newest {MAX_ROWS_PER_BENCHMARK} attempts are on the board; " + "every report stays in `reports/` on this branch.", + ] + lines += [ "", "| ended (UTC) | agent | hypothesis | outcome | candidate | GPU-h |", "| --- | --- | --- | --- | --- | --- |", diff --git a/tests/test_climbboard.py b/tests/test_climbboard.py index 7d93ef12..ea22c600 100644 --- a/tests/test_climbboard.py +++ b/tests/test_climbboard.py @@ -108,6 +108,24 @@ def test_views_render_the_rows_and_respect_direction(tmp_path: Path) -> None: assert "\\u003c/script" in page +def test_row_cap_is_loud_not_silent() -> None: + import json as _json + + from autoresearch.climbboard import MAX_ROWS_PER_BENCHMARK + + rows = [{"run_id": f"r{i}", "ended": f"2026-01-01 {i:02d}:00"} for i in range(40)] + many = [ + {"run_id": f"m{i}", "ended": f"2026-{1 + i // 10000:02d}-01 00:{i % 60:02d}"} + for i in range(MAX_ROWS_PER_BENCHMARK + 50) + ] + capped = merge_rows(_json.dumps(many), []) + assert len(capped) <= MAX_ROWS_PER_BENCHMARK + md = render_md("org/repo", {"b": capped}, {"b": "min"}) + assert f"Only the newest {MAX_ROWS_PER_BENCHMARK} attempts" in md + small = render_md("org/repo", {"b": rows}, {"b": "min"}) + assert "Only the newest" not in small + + def test_contract_directions_maps_benchmarks() -> None: from autoresearch.contract import load_contract From 0b3ac590c82f5c78afe2e9b26d530aa1f04f194d Mon Sep 17 00:00:00 2001 From: Mengye Ren Date: Sat, 29 Aug 2026 20:31:42 -0400 Subject: [PATCH 07/14] Climb board round 3: ended runs only; an unreadable JSON is never overwritten MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A run in review is not published until its PR's fate is known (a merged run lands with its final ending; the views count merged as a win). The per-benchmark JSON is read with the raising client call, like the index: a 404 starts fresh, an outage sits the pass out — published history is never replaced by a blind merge. Co-Authored-By: Claude Fable 5 --- src/autoresearch/climbboard.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/autoresearch/climbboard.py b/src/autoresearch/climbboard.py index b70240ab..ef084d10 100644 --- a/src/autoresearch/climbboard.py +++ b/src/autoresearch/climbboard.py @@ -152,7 +152,7 @@ def render_md( direction = directions.get(benchmark, "min") pick = max if direction == "max" else min measured = [r for r in rows if isinstance(r.get("candidate"), int | float)] - improved = [r for r in rows if r.get("outcome") == "improved"] + improved = [r for r in rows if r.get("outcome") in ("merged", "improved")] baselines = [r["baseline"] for r in rows if isinstance(r.get("baseline"), int | float)] best = pick((r["candidate"] for r in measured), default=None) gpu = sum(float(r.get("gpu_hours") or 0.0) for r in rows) @@ -209,6 +209,7 @@ def render_html( "for (const b of Object.keys(data.boards).sort()) {\n" " const rows = data.boards[b];\n" " const pick = data.directions[b] === 'max' ? Math.max : Math.min;\n" + " const won = new Set(['merged', 'improved']);\n" " const measured = rows.filter(r => typeof r.candidate === 'number');\n" " let best;\n" " const bestSoFar = measured.map(r =>\n" @@ -220,7 +221,7 @@ def render_html( " datasets: [\n" " {type: 'scatter', label: 'candidate', data: measured.map(r => r.candidate),\n" " pointBackgroundColor: measured.map(r =>\n" - " r.outcome === 'improved' ? '#2a7' : '#999')},\n" + " won.has(r.outcome) ? '#2a7' : '#999')},\n" " {type: 'line', label: 'best so far', data: bestSoFar, stepped: true,\n" " borderColor: '#26c', pointRadius: 0},\n" " {type: 'line', label: 'baseline', data: measured.map(r => r.baseline),\n" From f46fefc30d0b47e7350b42211d0639896d11532f Mon Sep 17 00:00:00 2001 From: Mengye Ren Date: Sat, 29 Aug 2026 20:38:05 -0400 Subject: [PATCH 08/14] Climb board round 4: a benchmark may be named index; the spend survives terminal transitions Data files live under climb/data/ (benchmark names are contract- controlled and 'index' is legal), and _clear_stage keeps gpu_hours_used when it wipes the waiting bookkeeping, so the board reads real spend after the transition instead of zero. Co-Authored-By: Claude Fable 5 --- src/autoresearch/attempt.py | 5 ++- src/autoresearch/climbboard.py | 8 +++-- tests/test_climbboard.py | 66 ++++++++++++++++++++++++++++++---- 3 files changed, 68 insertions(+), 11 deletions(-) diff --git a/src/autoresearch/attempt.py b/src/autoresearch/attempt.py index f0e82e50..1e25a01a 100644 --- a/src/autoresearch/attempt.py +++ b/src/autoresearch/attempt.py @@ -279,9 +279,12 @@ def _clear_stage(record: RunRecord) -> RunRecord: especially `wake_attempts` ride into `in-review`, where in-review follow-up servicing reuses `wake_attempts` as its OWN retry cap — so a run that woke once would reach review with a shrunk follow-up budget.""" + # the run's spend survives the wipe: terminal reporting (the climb + # board) reads it after the transition + kept = {k: record.stage[k] for k in ("gpu_hours_used",) if record.stage and k in record.stage} return dc_replace( record, - stage={}, + stage=kept, experiment_job_id="", deadline=0.0, terminal_seen=0.0, diff --git a/src/autoresearch/climbboard.py b/src/autoresearch/climbboard.py index ef084d10..5bf5ac63 100644 --- a/src/autoresearch/climbboard.py +++ b/src/autoresearch/climbboard.py @@ -2,7 +2,9 @@ of it, published to the target's `research-log` branch as data plus two views whenever runs end. -- `climb/.json` — one row per terminal attempt, dedup by run id; +- `climb/data/.json` — one row per terminal attempt, dedup by run + id (its own directory: benchmark names are contract-controlled, and a + benchmark named `index` must not collide with `climb/index.json`); the contract everything else derives from (and what an external dashboard reads after the public flip). - `CLIMB.md` — the numbers and the attempts table, rendered by GitHub. @@ -144,7 +146,7 @@ def render_md( "", f"# Climb — {target}", "", - "Written by the kernel when runs end. Data: `climb/.json`;", + "Written by the kernel when runs end. Data: `climb/data/.json`;", "chart: open `climb.html` from a clone of this branch.", ] for benchmark in sorted(boards): @@ -280,7 +282,7 @@ def service_climb_board( changed = 0 boards: dict[str, list[dict[str, Any]]] = {} for benchmark in sorted(names): - path = f"climb/{benchmark}.json" + path = f"climb/data/{benchmark}.json" existing = github.get_file_content(target, path, BOARD_BRANCH) rows = merge_rows(existing, local.get(benchmark, [])) if not rows: diff --git a/tests/test_climbboard.py b/tests/test_climbboard.py index ea22c600..e78a06f9 100644 --- a/tests/test_climbboard.py +++ b/tests/test_climbboard.py @@ -2,6 +2,7 @@ three published views.""" import json +from dataclasses import replace as dc_replace from pathlib import Path from autoresearch.climbboard import ( @@ -126,6 +127,52 @@ def test_row_cap_is_loud_not_silent() -> None: assert "Only the newest" not in small +def test_a_benchmark_named_index_cannot_collide_with_the_index(tmp_path: Path) -> None: + """Contract names allow `index`; its data lives under climb/data/, so the + direction index is never overwritten by a benchmark's rows.""" + gh = _BoardGitHub() + record = RunRecord( + run_id="idx-1", + target="org/repo", + task_title="t", + state="ended", + ending="negative-result", + benchmark="index", + created=1.0, + updated=2.0, + ) + save_record(tmp_path, record, 2.0) + (run_dir(tmp_path, "idx-1") / "report.md").write_text(REPORT) + service_climb_board(tmp_path, gh, "org/repo", {"index": "min"}) + assert json.loads(gh.files["climb/data/index.json"])[0]["run_id"] == "idx-1" + assert json.loads(gh.files["climb/index.json"]) == {"index": "min"} + + +def test_terminal_transition_keeps_the_spend_for_the_board(tmp_path: Path) -> None: + """_clear_stage wipes the waiting bookkeeping but keeps gpu_hours_used — + the production transition the board reads after (review #193 r6: rows + published zero GPU-hours because the collector ran on a wiped stage).""" + from autoresearch.attempt import _clear_stage + + record = RunRecord( + run_id="r", + target="org/repo", + task_title="t", + state="waiting", + benchmark="speedrun", + stage={"phase": "candidate", "afterany": "afterany:1", "gpu_hours_used": 36.0}, + wake_attempts=2, + deadline=9.0, + ) + cleared = _clear_stage(record) + assert cleared.stage == {"gpu_hours_used": 36.0} + assert cleared.wake_attempts == 0 and cleared.deadline == 0.0 + save_record(tmp_path, dc_replace(cleared, state="ended", ending="negative-result"), 2.0) + (run_dir(tmp_path, "r") / "report.md").write_text(REPORT) + rows = collect_rows(tmp_path, "org/repo")["speedrun"] + assert rows[0].gpu_hours == 36.0 + + def test_contract_directions_maps_benchmarks() -> None: from autoresearch.contract import load_contract @@ -188,7 +235,12 @@ def test_service_publishes_once_per_change(tmp_path: Path) -> None: _terminal_run(tmp_path, "speedrun-1") gh = _BoardGitHub() assert service_climb_board(tmp_path, gh, "org/repo") == 4 - assert sorted(gh.puts) == ["CLIMB.md", "climb.html", "climb/index.json", "climb/speedrun.json"] + assert sorted(gh.puts) == [ + "CLIMB.md", + "climb.html", + "climb/data/speedrun.json", + "climb/index.json", + ] # unchanged state: nothing is written again (no commit spam) gh.puts.clear() assert service_climb_board(tmp_path, gh, "org/repo") == 0 @@ -196,7 +248,7 @@ def test_service_publishes_once_per_change(tmp_path: Path) -> None: # a new terminal run publishes exactly the changed files _terminal_run(tmp_path, "speedrun-2", state="in-review", ending="") assert service_climb_board(tmp_path, gh, "org/repo") >= 2 - assert json.loads(gh.files["climb/speedrun.json"])[-1]["run_id"] == "speedrun-2" + assert json.loads(gh.files["climb/data/speedrun.json"])[-1]["run_id"] == "speedrun-2" # no runs and no index at all: a quiet no-op assert service_climb_board(tmp_path / "empty", gh, "org/repo") == 0 @@ -207,7 +259,7 @@ def test_board_remembers_benchmarks_whose_local_records_are_gone(tmp_path: Path) different benchmark publishes.""" gh = _BoardGitHub() gh.files["climb/index.json"] = json.dumps({"old-bench": "max"}) - gh.files["climb/old-bench.json"] = json.dumps( + gh.files["climb/data/old-bench.json"] = json.dumps( [{"run_id": "old-1", "ended": "2026-01-01 00:00", "candidate": 0.5, "outcome": "improved"}] ) _terminal_run(tmp_path, "speedrun-1") @@ -228,9 +280,9 @@ def test_transient_json_read_failure_never_orphans_an_indexed_benchmark(tmp_path old_rows = json.dumps( [{"run_id": "old-1", "ended": "2026-01-01 00:00", "candidate": 0.5, "outcome": "improved"}] ) - gh.files["climb/old-bench.json"] = old_rows + gh.files["climb/data/old-bench.json"] = old_rows real_get = gh.get_file_content - blackout = {"climb/old-bench.json"} + blackout = {"climb/data/old-bench.json"} def flaky_get(repo, path, ref): return None if path in blackout else real_get(repo, path, ref) @@ -288,9 +340,9 @@ def test_failed_json_upload_renders_last_published_rows(tmp_path: Path) -> None: {"run_id": "old-1", "ended": "2026-01-01 00:00", "candidate": 9472, "outcome": "improved"} ] gh.files["climb/index.json"] = json.dumps({"speedrun": "min"}) - gh.files["climb/speedrun.json"] = json.dumps(published) + gh.files["climb/data/speedrun.json"] = json.dumps(published) _terminal_run(tmp_path, "speedrun-1") # fresh row that will fail to upload - fail = {"climb/speedrun.json"} + fail = {"climb/data/speedrun.json"} real_put = gh.put_file def flaky(repo, path, content, branch, message): From b7f9b83cca3a33c7833684c61b822abff12053ab Mon Sep 17 00:00:00 2001 From: Mengye Ren Date: Sat, 29 Aug 2026 20:48:58 -0400 Subject: [PATCH 09/14] climb.html: a considered page, both themes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Stat chips, a card with a faint grid, and light/dark from the viewer's system theme — the page people will actually watch deserves more than default chart.js on white. Content and behavior unchanged. Co-Authored-By: Claude Fable 5 --- src/autoresearch/climbboard.py | 71 +++++++++++++++++++++++++++------- 1 file changed, 56 insertions(+), 15 deletions(-) diff --git a/src/autoresearch/climbboard.py b/src/autoresearch/climbboard.py index 5bf5ac63..72a2aee8 100644 --- a/src/autoresearch/climbboard.py +++ b/src/autoresearch/climbboard.py @@ -195,42 +195,83 @@ def render_html( """One self-contained page: the data is EMBEDDED (a browser blocks fetch() from a file:// page, and the direct-from-clone view must work), only chart.js arrives from its CDN — nothing bulky is committed to the - target's branch.""" + target's branch. Light and dark follow the viewer's system theme.""" # "<" is escaped INSIDE the JSON (still valid JSON): a hypothesis line is # agent-written text, and a literal in it would close the inline # script and run whatever follows in the published page payload = json.dumps({"boards": boards, "directions": directions}).replace("<", "\\u003c") return ( "\n\n" + "\n" f"Climb — {target}\n" "\n" - "\n" - f"

Climb — {target}

\n
\n\n" + " borderColor: css('--base'), borderDash: [6, 4], borderWidth: 1.5,\n" + " pointRadius: 0}]},\n" + " options: {color: css('--muted'),\n" + " scales: {x: {ticks: {color: css('--muted'), maxTicksLimit: 8},\n" + " grid: {color: css('--line')}},\n" + " y: {ticks: {color: css('--muted')}, grid: {color: css('--line')}}},\n" + " plugins: {legend: {labels: {color: css('--muted')}},\n" + " tooltip: {callbacks: {afterLabel:\n" + " (i) => measured[i.dataIndex].hypothesis}}}}});\n" + "}\n\n" ) From 0514c118dee793eb28c89aebb6241adb4ffa018c Mon Sep 17 00:00:00 2001 From: Mengye Ren Date: Sat, 29 Aug 2026 21:12:57 -0400 Subject: [PATCH 10/14] The cap note promises only what reports/ holds Co-Authored-By: Claude Fable 5 --- src/autoresearch/climbboard.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/autoresearch/climbboard.py b/src/autoresearch/climbboard.py index 72a2aee8..543f54ac 100644 --- a/src/autoresearch/climbboard.py +++ b/src/autoresearch/climbboard.py @@ -170,7 +170,7 @@ def render_md( lines += [ "", f"Only the newest {MAX_ROWS_PER_BENCHMARK} attempts are on the board; " - "every report stays in `reports/` on this branch.", + "archived reports stay in `reports/` on this branch.", ] lines += [ "", From 7eb420a5340d4ddbdf88a898bf35c49a542fefe9 Mon Sep 17 00:00:00 2001 From: Mengye Ren Date: Sat, 29 Aug 2026 21:19:27 -0400 Subject: [PATCH 11/14] The round-6 fixes, actually: ended runs only; an unreadable JSON is never overwritten MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Commit 0b3ac59 said this and did not do it: the patch script died on an anchor mismatch before writing the module, the follow-up applied only the cosmetic remainder, and the commit landed under the original message unverified. This commit carries the real changes — collect publishes only ENDED runs (a merged run lands with its final ending), and the per-benchmark JSON is read with the raising client call so an outage sits the pass out instead of blindly overwriting history — plus the tests that pin both. Co-Authored-By: Claude Fable 5 --- src/autoresearch/climbboard.py | 19 +++++++++++--- tests/test_climbboard.py | 48 ++++++++++++++++++++++++++++------ 2 files changed, 55 insertions(+), 12 deletions(-) diff --git a/src/autoresearch/climbboard.py b/src/autoresearch/climbboard.py index 543f54ac..42f3f092 100644 --- a/src/autoresearch/climbboard.py +++ b/src/autoresearch/climbboard.py @@ -26,7 +26,7 @@ from pathlib import Path from typing import Any -from autoresearch.runstate import ENDED, IN_REVIEW, list_runs, run_dir +from autoresearch.runstate import ENDED, list_runs, run_dir log = logging.getLogger("autoresearch.climbboard") @@ -78,7 +78,9 @@ def collect_rows(root: Path, target: str) -> dict[str, list[ClimbRow]]: out: dict[str, list[ClimbRow]] = {} for record in list_runs(root): - if record.target != target or record.state not in (ENDED, IN_REVIEW): + # only ENDED runs: an in-review run's outcome is not known yet (its + # PR may be rejected), and a published row is never rewritten + if record.target != target or record.state != ENDED: continue try: report = (run_dir(root, record.run_id) / "report.md").read_text() @@ -87,7 +89,7 @@ def collect_rows(root: Path, target: str) -> dict[str, list[ClimbRow]]: baseline, candidate, hyp = _report_fields(report) stage = record.stage or {} ended = datetime.fromtimestamp(record.updated or record.created, tz=UTC) - outcome = record.ending or ("improved" if record.state == IN_REVIEW else "ended") + outcome = record.ending or "ended" out.setdefault(record.benchmark or "benchmark", []).append( ClimbRow( run_id=record.run_id, @@ -324,7 +326,16 @@ def service_climb_board( boards: dict[str, list[dict[str, Any]]] = {} for benchmark in sorted(names): path = f"climb/data/{benchmark}.json" - existing = github.get_file_content(target, path, BOARD_BRANCH) + try: + existing: str | None = github.get_file(target, path, BOARD_BRANCH) + except Exception as exc: + if getattr(exc, "status", None) != 404: + # an outage is not an empty board: overwriting would replace + # this benchmark's published history — sit the pass out (its + # index entry survives) + log.warning("board JSON unreadable for %s (%s); skipped", benchmark, exc) + continue + existing = None rows = merge_rows(existing, local.get(benchmark, [])) if not rows: continue diff --git a/tests/test_climbboard.py b/tests/test_climbboard.py index e78a06f9..c5580f6e 100644 --- a/tests/test_climbboard.py +++ b/tests/test_climbboard.py @@ -36,7 +36,7 @@ def _terminal_run(root: Path, run_id: str, *, state="ended", ending="negative-re ending=ending, benchmark="speedrun", agent_id="agent-03", - pr_url="https://github.com/org/repo/pull/9" if state == "in-review" else "", + pr_url="https://github.com/org/repo/pull/9" if ending == "merged" else "", created=1_787_900_000.0, updated=1_787_950_000.0, stage={"gpu_hours_used": 12.5}, @@ -47,8 +47,10 @@ def _terminal_run(root: Path, run_id: str, *, state="ended", ending="negative-re def test_collect_rows_reads_terminal_records_and_reports(tmp_path: Path) -> None: _terminal_run(tmp_path, "speedrun-1") - _terminal_run(tmp_path, "speedrun-2", state="in-review", ending="") - # non-terminal and reportless runs stay off the board + _terminal_run(tmp_path, "speedrun-2", ending="merged") + # non-terminal runs stay off the board: an in-review run's outcome is not + # known yet (its PR may be rejected), and a waiting run is mid-flight + _terminal_run(tmp_path, "speedrun-3", state="in-review", ending="") save_record( tmp_path, RunRecord(run_id="w", target="org/repo", task_title="t", state="waiting"), @@ -61,7 +63,7 @@ def test_collect_rows_reads_terminal_records_and_reports(tmp_path: Path) -> None assert row.baseline == 9472.0 and row.candidate == 38146.0 assert row.hypothesis.startswith("the warmdown starts too early") assert row.gpu_hours == 12.5 and row.agent == "agent-03" - assert rows[1].outcome == "improved" and rows[1].pr_url.endswith("/pull/9") + assert rows[1].outcome == "merged" and rows[1].pr_url.endswith("/pull/9") def test_merge_is_idempotent_and_keeps_published_history(tmp_path: Path) -> None: @@ -246,7 +248,7 @@ def test_service_publishes_once_per_change(tmp_path: Path) -> None: assert service_climb_board(tmp_path, gh, "org/repo") == 0 assert gh.puts == [] # a new terminal run publishes exactly the changed files - _terminal_run(tmp_path, "speedrun-2", state="in-review", ending="") + _terminal_run(tmp_path, "speedrun-2", ending="merged") assert service_climb_board(tmp_path, gh, "org/repo") >= 2 assert json.loads(gh.files["climb/data/speedrun.json"])[-1]["run_id"] == "speedrun-2" # no runs and no index at all: a quiet no-op @@ -281,13 +283,15 @@ def test_transient_json_read_failure_never_orphans_an_indexed_benchmark(tmp_path [{"run_id": "old-1", "ended": "2026-01-01 00:00", "candidate": 0.5, "outcome": "improved"}] ) gh.files["climb/data/old-bench.json"] = old_rows - real_get = gh.get_file_content + real_get = gh.get_file blackout = {"climb/data/old-bench.json"} def flaky_get(repo, path, ref): - return None if path in blackout else real_get(repo, path, ref) + if path in blackout: + raise _GitHubError(500) + return real_get(repo, path, ref) - gh.get_file_content = flaky_get # type: ignore[method-assign] + gh.get_file = flaky_get # type: ignore[method-assign] _terminal_run(tmp_path, "speedrun-1") service_climb_board(tmp_path, gh, "org/repo", {"speedrun": "min"}) assert "## old-bench" not in gh.files["CLIMB.md"] # skipped this pass... @@ -297,6 +301,34 @@ def flaky_get(repo, path, ref): assert "## old-bench" in gh.files["CLIMB.md"] +def test_unreadable_board_json_is_never_overwritten(tmp_path: Path) -> None: + """An outage reading one benchmark's JSON must not let a fresh merge + replace its published history: that benchmark sits the pass out and its + index entry survives.""" + gh = _BoardGitHub() + gh.files["climb/index.json"] = json.dumps({"speedrun": "min"}) + gh.files["climb/data/speedrun.json"] = json.dumps( + [{"run_id": "old-1", "ended": "2026-01-01 00:00", "candidate": 9472}] + ) + _terminal_run(tmp_path, "speedrun-1") + real_get = gh.get_file + blackout = {"climb/data/speedrun.json"} + + def flaky(repo, path, ref): + if path in blackout: + raise _GitHubError(500) + return real_get(repo, path, ref) + + gh.get_file = flaky # type: ignore[method-assign] + service_climb_board(tmp_path, gh, "org/repo", {"speedrun": "min"}) + assert json.loads(gh.files["climb/data/speedrun.json"])[0]["run_id"] == "old-1" # untouched + assert json.loads(gh.files["climb/index.json"]) == {"speedrun": "min"} + blackout.clear() + service_climb_board(tmp_path, gh, "org/repo", {"speedrun": "min"}) + ids = [r["run_id"] for r in json.loads(gh.files["climb/data/speedrun.json"])] + assert ids == ["old-1", "speedrun-1"] # history + the fresh row, after the outage + + def test_failed_index_read_never_rewrites_the_index(tmp_path: Path) -> None: """A transient failure reading the index itself must not shrink it: the pass publishes what it can but leaves climb/index.json untouched.""" From 66f154201f45ed0c1453f3a2c4234399687e5e4e Mon Sep 17 00:00:00 2001 From: Mengye Ren Date: Sat, 29 Aug 2026 21:21:38 -0400 Subject: [PATCH 12/14] The chart draws: one line chart, candidates as points A scatter dataset ignores category labels (it wants x/y pairs on a linear axis), so the mixed chart rendered nothing. All three datasets now share the category axis; candidates are a point-only line. Co-Authored-By: Claude Fable 5 --- src/autoresearch/climbboard.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/autoresearch/climbboard.py b/src/autoresearch/climbboard.py index 42f3f092..bdaae87d 100644 --- a/src/autoresearch/climbboard.py +++ b/src/autoresearch/climbboard.py @@ -256,14 +256,15 @@ def render_html( " chip('GPU-hours', gpu.toFixed(1)); el.append(chips);\n" " const card = document.createElement('div'); card.className = 'card';\n" " const c = document.createElement('canvas'); card.append(c); el.append(card);\n" - " new Chart(c, {data: {labels: measured.map(r => r.ended),\n" + " new Chart(c, {type: 'line', data: {labels: measured.map(r => r.ended),\n" " datasets: [\n" - " {type: 'scatter', label: 'candidate', data: measured.map(r => r.candidate),\n" - " pointRadius: 4, pointBackgroundColor: measured.map(r =>\n" + " {label: 'candidate', data: measured.map(r => r.candidate),\n" + " showLine: false, pointRadius: 4,\n" + " pointBackgroundColor: measured.map(r =>\n" " won.has(r.outcome) ? css('--win') : css('--lose'))},\n" - " {type: 'line', label: 'best so far', data: bestSoFar, stepped: true,\n" + " {label: 'best so far', data: bestSoFar, stepped: true,\n" " borderColor: css('--accent'), borderWidth: 2, pointRadius: 0},\n" - " {type: 'line', label: 'baseline', data: measured.map(r => r.baseline),\n" + " {label: 'baseline', data: measured.map(r => r.baseline),\n" " borderColor: css('--base'), borderDash: [6, 4], borderWidth: 1.5,\n" " pointRadius: 0}]},\n" " options: {color: css('--muted'),\n" From 636efb15b8ee9ff1c3d01b06061bbd43ce9c16bc Mon Sep 17 00:00:00 2001 From: Mengye Ren Date: Sat, 29 Aug 2026 21:23:52 -0400 Subject: [PATCH 13/14] A malformed index is treated like an outage: never rewritten Co-Authored-By: Claude Fable 5 --- src/autoresearch/climbboard.py | 5 +++-- tests/test_climbboard.py | 5 ++++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/src/autoresearch/climbboard.py b/src/autoresearch/climbboard.py index bdaae87d..c185af00 100644 --- a/src/autoresearch/climbboard.py +++ b/src/autoresearch/climbboard.py @@ -302,9 +302,10 @@ def _read_index(github: Any, target: str) -> dict[str, str] | None: return None try: data = json.loads(raw) - return {str(k): str(v) for k, v in data.items()} if isinstance(data, dict) else {} except ValueError: - return {} + log.warning("climb index malformed; not rewriting it") + return None # same stance as an outage: never rewrite what we cannot see + return {str(k): str(v) for k, v in data.items()} if isinstance(data, dict) else None def service_climb_board( diff --git a/tests/test_climbboard.py b/tests/test_climbboard.py index c5580f6e..c03ea67a 100644 --- a/tests/test_climbboard.py +++ b/tests/test_climbboard.py @@ -333,8 +333,11 @@ def test_failed_index_read_never_rewrites_the_index(tmp_path: Path) -> None: """A transient failure reading the index itself must not shrink it: the pass publishes what it can but leaves climb/index.json untouched.""" gh = _BoardGitHub() - gh.files["climb/index.json"] = json.dumps({"old-bench": "max"}) + gh.files["climb/index.json"] = "not json {" _terminal_run(tmp_path, "speedrun-1") + service_climb_board(tmp_path, gh, "org/repo", {"speedrun": "min"}) + assert gh.files["climb/index.json"] == "not json {" # malformed: untouched + gh.files["climb/index.json"] = json.dumps({"old-bench": "max"}) gh.index_outage = True service_climb_board(tmp_path, gh, "org/repo", {"speedrun": "min"}) assert json.loads(gh.files["climb/index.json"]) == {"old-bench": "max"} # untouched From 08adb5c3a76f26334349d6a8f753ef4df478a16b Mon Sep 17 00:00:00 2001 From: Mengye Ren Date: Sat, 29 Aug 2026 21:29:12 -0400 Subject: [PATCH 14/14] Round tail: non-list board data is opaque; ended carries seconds Co-Authored-By: Claude Fable 5 --- src/autoresearch/climbboard.py | 11 ++++++++++- tests/test_climbboard.py | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+), 1 deletion(-) diff --git a/src/autoresearch/climbboard.py b/src/autoresearch/climbboard.py index c185af00..e2ee20da 100644 --- a/src/autoresearch/climbboard.py +++ b/src/autoresearch/climbboard.py @@ -94,7 +94,7 @@ def collect_rows(root: Path, target: str) -> dict[str, list[ClimbRow]]: ClimbRow( run_id=record.run_id, agent=record.agent_id, - ended=ended.strftime("%Y-%m-%d %H:%M"), + ended=ended.strftime("%Y-%m-%d %H:%M:%S"), outcome=outcome, baseline=baseline, candidate=candidate, @@ -330,6 +330,15 @@ def service_climb_board( path = f"climb/data/{benchmark}.json" try: existing: str | None = github.get_file(target, path, BOARD_BRANCH) + if existing is not None: + parsed = json.loads(existing) + if not isinstance(parsed, list): + raise ValueError("board data is not a list") + except (ValueError, TypeError) as exc: + # readable but not a row list: same stance as an outage — never + # let a fresh merge overwrite what we cannot interpret + log.warning("board JSON malformed for %s (%s); skipped", benchmark, exc) + continue except Exception as exc: if getattr(exc, "status", None) != 404: # an outage is not an empty board: overwriting would replace diff --git a/tests/test_climbboard.py b/tests/test_climbboard.py index c03ea67a..5e648f7d 100644 --- a/tests/test_climbboard.py +++ b/tests/test_climbboard.py @@ -329,6 +329,38 @@ def flaky(repo, path, ref): assert ids == ["old-1", "speedrun-1"] # history + the fresh row, after the outage +def test_non_list_board_json_is_never_overwritten(tmp_path: Path) -> None: + """A data file that decodes to something other than a row list is as + opaque as an outage: the benchmark sits the pass out untouched.""" + gh = _BoardGitHub() + gh.files["climb/index.json"] = json.dumps({"speedrun": "min"}) + gh.files["climb/data/speedrun.json"] = json.dumps({"rows": "elsewhere"}) + _terminal_run(tmp_path, "speedrun-1") + service_climb_board(tmp_path, gh, "org/repo", {"speedrun": "min"}) + assert json.loads(gh.files["climb/data/speedrun.json"]) == {"rows": "elsewhere"} + + +def test_same_minute_attempts_order_by_end_time(tmp_path: Path) -> None: + """`ended` carries seconds: it is the merge sort key, and two attempts + in one minute must not fall back to directory order.""" + _terminal_run(tmp_path, "speedrun-b") # updated=1_787_950_000 + record = RunRecord( + run_id="speedrun-a", + target="org/repo", + task_title="t", + state="ended", + ending="negative-result", + benchmark="speedrun", + created=1_787_900_000.0, + updated=1_787_950_030.0, # same minute, 30 s later + ) + save_record(tmp_path, record, record.updated) + (run_dir(tmp_path, "speedrun-a") / "report.md").write_text(REPORT) + rows = merge_rows(None, collect_rows(tmp_path, "org/repo")["speedrun"]) + assert [r["run_id"] for r in rows] == ["speedrun-b", "speedrun-a"] + assert rows[0]["ended"].count(":") == 2 # seconds present + + def test_failed_index_read_never_rewrites_the_index(tmp_path: Path) -> None: """A transient failure reading the index itself must not shrink it: the pass publishes what it can but leaves climb/index.json untouched."""