From 6f40309c8e0422997998ebdd9bf8c59cd1e944b3 Mon Sep 17 00:00:00 2001 From: Mengye Ren Date: Sat, 24 Oct 2026 22:29:54 -0400 Subject: [PATCH 1/3] Status shows a priced lower bound when some legs are unpriced (#516) --- CHANGELOG.md | 1 + docs/session-capture.md | 8 ++++--- src/outerloop/session_evidence.py | 2 ++ src/outerloop/status.py | 13 +++++++++-- tests/test_session_evidence.py | 1 + tests/test_status.py | 36 +++++++++++++++++++++++++++++++ 6 files changed, 56 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 43e83ece..2e6bd0fb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ Versions follow [SemVer](https://semver.org). ## [Unreleased] +- `outerloop status` shows a run's priced session cost as a lower bound when some legs are unpriced (a wake killed mid-turn leaves no usage event), for example `session-cost>=$4.2000 (1 unpriced)`, instead of hiding the priced part behind `unknown`; the status row and JSON gain the priced-leg count (#516). - Claude sessions can use a subscription token from `claude setup-token` (`sk-ant-oat...`). The harness sends it as `CLAUDE_CODE_OAUTH_TOKEN`, not `ANTHROPIC_API_KEY`, and `init` accepts it. A panel judge on a token runs `claude --safe-mode`, because `--bare` cannot read OAuth tokens. Both modes ignore the checkout's CLAUDE.md, hooks, plugins and MCP servers. The pinned Claude Code release (2.1.285) has `--safe-mode`. - A session now hears once when a live sibling states a new or changed hypothesis (#360). The note names the sibling, quotes its hypothesis, and points to `siblings` for its launches and score. It is sent only when `channels.siblings` is on. Upgrading: the note uses the existing `note` kind, so an older kernel shows it as ordinary inbox text. - A skills-only plugin for Claude Code and Codex sets up Outerloop, writes and validates the contract, and operates runs from the agent (#562). Install it with `/plugin marketplace add outerloop-science/outerloop` or `codex plugin marketplace add outerloop-science/outerloop`. Its setup skill offers a Claude subscription token from `claude setup-token` as the author credential. diff --git a/docs/session-capture.md b/docs/session-capture.md index d70192ff..dde8750c 100644 --- a/docs/session-capture.md +++ b/docs/session-capture.md @@ -67,9 +67,11 @@ persisted index, never from files the author could touch: my-run target=owner/repo ... session-cost=$1.2345 unpriced-sessions=0 ``` -A run shows `unknown` cost while any captured session is unpriced, and -counts those sessions separately. Totals carry a `verified` flag, true when -every indexed session ran inside the session container. An uncontained +Cost is `unknown` only when nothing is priced. When some legs are priced and +others are unpriced (a wake killed mid-turn leaves no usage event), status +shows the priced subtotal as a lower bound and counts the unpriced legs, for +example `session-cost>=$1.2345 (1 unpriced)`. Totals carry a `verified` flag, +true when every indexed session ran inside the session container. An uncontained session labels the totals `unverified`, and in local mode the author shares the operator's filesystem, so persisted totals are informational there. diff --git a/src/outerloop/session_evidence.py b/src/outerloop/session_evidence.py index a072100a..50897d71 100644 --- a/src/outerloop/session_evidence.py +++ b/src/outerloop/session_evidence.py @@ -576,6 +576,8 @@ def cumulative(entry: Mapping[str, Any]) -> bool: return { "session_cost_usd": known if costs and not unknown else None, "known_session_cost_usd": known, + # A leg with no usage event is counted as unpriced, never as zero. + "priced_sessions": len(costs) - unknown, "unpriced_sessions": unknown, "captured_sessions": len(sessions), "verified": bool(sessions) and all(entry["verified"] for entry in sessions.values()), diff --git a/src/outerloop/status.py b/src/outerloop/status.py index 1a5a7713..508445f2 100644 --- a/src/outerloop/status.py +++ b/src/outerloop/status.py @@ -105,10 +105,19 @@ def render_text(status: dict[str, Any]) -> str: if run.get("verification") == "self_report": line += " verification=self_report (self-reported)" cost = run.get("session_cost_usd") - line += f" session-cost={'$' + format(cost, '.4f') if cost is not None else 'unknown'}" + known = run.get("known_session_cost_usd") + unpriced = run.get("unpriced_sessions", 0) + if cost is not None: + line += " session-cost=$" + format(cost, ".4f") + elif known is not None and unpriced and run.get("priced_sessions", 0): + # Some legs have no usage event (a wake killed mid-turn), so only + # the priced part is known: show it as a lower bound. + line += " session-cost>=$" + format(known, ".4f") + f" ({unpriced} unpriced)" + else: + line += " session-cost=unknown" if not run.get("verified", False): line += " (unverified)" - line += f" unpriced-sessions={run.get('unpriced_sessions', 0)}" + line += f" unpriced-sessions={unpriced}" if wait := run["endpoint_wait"]: line += ( f" waiting for endpoint {wait.get('endpoint', '?')} " diff --git a/tests/test_session_evidence.py b/tests/test_session_evidence.py index 814b41b0..0c387aea 100644 --- a/tests/test_session_evidence.py +++ b/tests/test_session_evidence.py @@ -647,6 +647,7 @@ def test_totals_ignore_forged_and_replaced_sidecars(tmp_path, contained): expected = { "session_cost_usd": 0.42, "known_session_cost_usd": 0.42, + "priced_sessions": 1, "unpriced_sessions": 0, "captured_sessions": 1, "verified": contained, diff --git a/tests/test_status.py b/tests/test_status.py index 8f264635..7eb02e27 100644 --- a/tests/test_status.py +++ b/tests/test_status.py @@ -8,6 +8,7 @@ from outerloop.endpoint_wait import recovered, unavailable from outerloop.endpoints import EndpointUnavailable from outerloop.runstate import RunRecord, save_record +from outerloop.status import render_text def test_status_wait_and_outage_read_only(tmp_path, capsys, monkeypatch): @@ -75,6 +76,7 @@ def forbidden(*args, **kwargs): "author_overridden": True, "session_cost_usd": None, "known_session_cost_usd": 0.0, + "priced_sessions": 0, "unpriced_sessions": 0, "captured_sessions": 0, "verified": False, @@ -94,6 +96,40 @@ def forbidden(*args, **kwargs): assert payload["runs"][0]["endpoint_wait"] is None +@pytest.mark.parametrize( + ("cost", "known", "priced", "unpriced", "expected"), + [ + # a wake killed mid-turn is unpriced; show the priced part as a floor + (None, 4.2, 2, 1, "session-cost>=$4.2000 (1 unpriced)"), + (4.2, 4.2, 3, 0, "session-cost=$4.2000"), + # nothing was priced: a floor of zero would hide that + (None, 0.0, 0, 1, "session-cost=unknown"), + ], +) +def test_status_cost_line(cost, known, priced, unpriced, expected): + run = { + "run_id": "one", + "target": "owner/repo", + "agent": "agent-01", + "state": "parked", + "phase": "", + "author_backend": "claude", + "author_model": None, + "author_overridden": False, + "gpu_hours_used": 0.0, + "gpu_hours_budget": None, + "session_cost_usd": cost, + "known_session_cost_usd": known, + "priced_sessions": priced, + "unpriced_sessions": unpriced, + "verified": True, + "endpoint_wait": None, + } + line = render_text({"runs": [run], "outages": []}) + assert expected in line + assert f"unpriced-sessions={unpriced}" in line + + @pytest.mark.parametrize("exists", [True, False]) def test_status_empty_root(tmp_path, capsys, exists): root = tmp_path / "state" From 2780ce658c3f7546b21bbbe19c0f95b4ea3c8676 Mon Sep 17 00:00:00 2001 From: Mengye Ren Date: Sat, 24 Oct 2026 22:48:17 -0400 Subject: [PATCH 2/3] Test priced_sessions on a mixed priced/unpriced run --- tests/test_session_evidence.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_session_evidence.py b/tests/test_session_evidence.py index 0c387aea..f9da6bd5 100644 --- a/tests/test_session_evidence.py +++ b/tests/test_session_evidence.py @@ -244,6 +244,8 @@ def test_legacy_and_unknown_totals(tmp_path): assert totals["session_cost_usd"] is None assert totals["known_session_cost_usd"] == 0.42 assert totals["unpriced_sessions"] == 1 + # a priced and an unpriced leg together: status can show the lower bound + assert totals["priced_sessions"] == 1 def test_hermes_wrapper_keeps_reported_usage_and_full_messages(tmp_path, monkeypatch, capsys): From 85517d22933426e473992fabfa8b3de8db9f26da Mon Sep 17 00:00:00 2001 From: Mengye Ren Date: Sat, 24 Oct 2026 23:00:43 -0400 Subject: [PATCH 3/3] A cumulative session's last priced total is a lower bound when its newest leg is unpriced --- src/outerloop/session_evidence.py | 16 ++++++++++++++-- tests/test_session_evidence.py | 17 +++++++++++++++++ 2 files changed, 31 insertions(+), 2 deletions(-) diff --git a/src/outerloop/session_evidence.py b/src/outerloop/session_evidence.py index 50897d71..8471b93e 100644 --- a/src/outerloop/session_evidence.py +++ b/src/outerloop/session_evidence.py @@ -561,9 +561,13 @@ def cumulative(entry: Mapping[str, Any]) -> bool: ) latest: dict[tuple[str, str], str] = {} + last_priced: dict[tuple[str, str], float] = {} for invocation, entry in sessions.items(): if cumulative(entry): - latest[(entry["backend"], entry["session_id"])] = invocation + key = (entry["backend"], entry["session_id"]) + latest[key] = invocation + if (cost := number(entry["cost_usd"])) is not None: + last_priced[key] = cost costs = tuple( number(entry["cost_usd"]) for invocation, entry in sessions.items() @@ -573,11 +577,19 @@ def cumulative(entry: Mapping[str, Any]) -> bool: ) known = sum(cost for cost in costs if cost is not None) unknown = sum(cost is None for cost in costs) + # A cumulative session whose newest leg is unpriced (killed mid-turn) still + # has its last priced total as a lower bound. + floors = [ + last_priced[key] + for key, invocation in latest.items() + if number(sessions[invocation]["cost_usd"]) is None and key in last_priced + ] + known += sum(floors) return { "session_cost_usd": known if costs and not unknown else None, "known_session_cost_usd": known, # A leg with no usage event is counted as unpriced, never as zero. - "priced_sessions": len(costs) - unknown, + "priced_sessions": len(costs) - unknown + len(floors), "unpriced_sessions": unknown, "captured_sessions": len(sessions), "verified": bool(sessions) and all(entry["verified"] for entry in sessions.values()), diff --git a/tests/test_session_evidence.py b/tests/test_session_evidence.py index f9da6bd5..027534d3 100644 --- a/tests/test_session_evidence.py +++ b/tests/test_session_evidence.py @@ -248,6 +248,23 @@ def test_legacy_and_unknown_totals(tmp_path): assert totals["priced_sessions"] == 1 +def test_cumulative_session_keeps_its_last_priced_total_as_a_lower_bound(tmp_path): + from outerloop.session_evidence import index_session + + # a Codex session's first leg is priced; its resumed wake was killed mid-turn + for invocation, cost in (("first", 0.2), ("wake", None)): + index_session( + tmp_path, + invocation, + f"{invocation}.json", + {"cost_usd": cost, "verified": True, "backend": "codex", "session_id": "s"}, + ) + totals = session_totals(tmp_path) + assert totals["session_cost_usd"] is None + assert totals["known_session_cost_usd"] == pytest.approx(0.2) + assert totals["priced_sessions"] == 1 and totals["unpriced_sessions"] == 1 + + def test_hermes_wrapper_keeps_reported_usage_and_full_messages(tmp_path, monkeypatch, capsys): import sys from types import ModuleType