diff --git a/CHANGELOG.md b/CHANGELOG.md index 9f0e0e6..35211ab 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,18 @@ All notable changes to blockrun-llm will be documented in this file. ## Unreleased +### Fixed +- **A skip marker never hides a real charge.** A response that names a skipped + settlement (`X-Settlement-Skipped` / `X-Payment-Waived`) but also proves the + payment settled (a `PAYMENT-RESPONSE` that did not fail and names a + transaction or a positive `amount` / `extra.chargedAmount`, or an + `X-Payment-Receipt` hash) is booked as charged, and the conflict is written + to stderr. The gateways never send both today (blockrun.ai's free-fallback + receipt has an empty transaction and no amount), so today's skips still book + $0. Ports @blockrun/llm 3.24.9 (TS #92). +- A marker value that says the opposite (`false`, `0`, `no`, `none`) is no + longer read as a skip. + ## 1.21.2 — 2026-10-27 ### Fixed diff --git a/blockrun_llm/tx_log.py b/blockrun_llm/tx_log.py index f9deff5..4a430c1 100644 --- a/blockrun_llm/tx_log.py +++ b/blockrun_llm/tx_log.py @@ -33,6 +33,7 @@ import base64 import json import os +import sys import time from datetime import datetime from pathlib import Path @@ -89,6 +90,8 @@ def read_settlement_header(headers: Any) -> str | None: # real spend and let a session spend limit be exceeded. _SETTLEMENT_SKIPPED_HEADER = "X-Settlement-Skipped" _PAYMENT_WAIVED_HEADER = "X-Payment-Waived" +# Values that say settlement was NOT skipped: never a marker. +_NOT_A_SKIP = frozenset({"false", "0", "no", "none"}) def _header_value(headers: Any, name: str) -> str | None: @@ -111,16 +114,71 @@ def settlement_skipped(headers: Any) -> bool: """True when the gateway served this paid response without settling it. ``X-Settlement-Skipped`` with any non-empty value, or ``X-Payment-Waived`` - present at all; lookup is case-insensitive. Never raises: a header object - that misbehaves reads as "not skipped", which books the quote (the + present at all; lookup is case-insensitive. A value that says the opposite + (``false``, ``0``, ``no``, ``none``) is not a marker. Nor is a marker on a + response that proves the payment settled anyway (a receipt with a + transaction or a positive charge, or an ``X-Payment-Receipt`` hash): that + call is booked, and the conflict written to stderr. Never raises: a header + object that misbehaves reads as "not skipped", which books the quote (the behaviour before this marker existed). """ if headers is None: return False skipped = _header_value(headers, _SETTLEMENT_SKIPPED_HEADER) - if skipped is not None and skipped.strip(): - return True - return _header_value(headers, _PAYMENT_WAIVED_HEADER) is not None + waived = _header_value(headers, _PAYMENT_WAIVED_HEADER) + reason: str | None = None + if skipped is not None and skipped.strip() and skipped.strip().lower() not in _NOT_A_SKIP: + reason = skipped.strip() + elif waived is not None and waived.strip().lower() not in _NOT_A_SKIP: + reason = waived.strip() or "waived" + if reason is None: + return False + # Settlement evidence beats a skip marker: money that moved is booked. + evidence = _settlement_evidence(headers) + if evidence is not None: + sys.stderr.write( + f"[blockrun_llm] settlement: the gateway named a skipped settlement ({reason!r}) " + f"on a response that also carries {evidence}; booked as charged\n" + ) + return False + return True + + +def _positive_atomic(value: Any) -> bool: + try: + return int(str(value)) > 0 + except (TypeError, ValueError): + return False + + +def _settlement_evidence(headers: Any) -> str | None: + """Proof the payment settled anyway, or None. + + A ``PAYMENT-RESPONSE`` that did not fail and names a transaction or a + positive charge (``amount`` / ``extra.chargedAmount``), or an + ``X-Payment-Receipt`` transaction hash. A free-fallback receipt + (``{"success": true, "transaction": ""}``, no amount) is no evidence. + """ + try: + header = _header_value(headers, "PAYMENT-RESPONSE") or _header_value( + headers, "X-PAYMENT-RESPONSE" + ) + settlement = decode_settlement_header(header) + if settlement is not None and settlement.get("success") is not False: + if settlement.get("tx_hash"): + return "a settlement receipt with a transaction" + raw = settlement.get("raw") or {} + extra = raw.get("extra") if isinstance(raw.get("extra"), dict) else {} + if _positive_atomic(settlement.get("amount_micro_usdc")) or _positive_atomic( + extra.get("chargedAmount") + ): + return "a settlement receipt with a charge" + receipt = _header_value(headers, "X-Payment-Receipt") + if receipt is not None and receipt.strip(): + return "an X-Payment-Receipt transaction" + except Exception: + return None + return None def booked_cost_usd(headers: Any, cost_usd: float) -> float: diff --git a/tests/unit/test_settlement_skipped.py b/tests/unit/test_settlement_skipped.py index 9985183..9eb0b7c 100644 --- a/tests/unit/test_settlement_skipped.py +++ b/tests/unit/test_settlement_skipped.py @@ -22,6 +22,8 @@ from __future__ import annotations +import base64 +import json import warnings from typing import Any from unittest import mock @@ -36,6 +38,12 @@ from ..helpers import TEST_PRIVATE_KEY, build_chat_response, build_payment_required_response MESSAGES = [{"role": "user", "content": "hi"}] + + +def _b64(value: Any) -> str: + return base64.b64encode(json.dumps(value).encode()).decode() + + QUOTE_USD = 1.0 # build_payment_required_response / the Solana fake payload: 1 USDC # What sol.blockrun.ai sends on an empty pm search (the full marker set). @@ -95,6 +103,73 @@ def test_never_raises(self) -> None: assert settlement_skipped(None) is False assert settlement_skipped(object()) is False + @pytest.mark.parametrize( + "headers", + [ + # A value that says it was NOT skipped. + {"X-Settlement-Skipped": "false"}, + {"X-Settlement-Skipped": "0", "X-Payment-Waived": "none"}, + # Settlement evidence beats the marker. + { + "X-Settlement-Skipped": "free-fallback", + "PAYMENT-RESPONSE": _b64({"success": True, "transaction": "0xpaid"}), + }, + { + "X-Payment-Waived": "empty-result", + "PAYMENT-RESPONSE": _b64({"success": True, "signature": "5sig"}), + }, + { + "X-Payment-Waived": "empty-result", + "PAYMENT-RESPONSE": _b64( + {"success": True, "transaction": "", "extra": {"chargedAmount": "4000"}} + ), + }, + { + "X-Settlement-Skipped": "unexecuted", + "PAYMENT-RESPONSE": _b64({"success": True, "transaction": "", "amount": "4000"}), + }, + {"X-Settlement-Skipped": "unexecuted", "X-Payment-Receipt": "0xpaid"}, + ], + ) + def test_evidence_or_negative_value_books( + self, headers: dict[str, str], capsys: pytest.CaptureFixture + ) -> None: + assert settlement_skipped(headers) is False + assert settlement_skipped(httpx.Headers(headers)) is False + assert booked_cost_usd(headers, 0.25) == 0.25 + + def test_conflict_is_written_to_stderr(self, capsys: pytest.CaptureFixture) -> None: + headers = { + "X-Settlement-Skipped": "free-fallback", + "PAYMENT-RESPONSE": _b64({"success": True, "transaction": "0xpaid"}), + } + assert settlement_skipped(headers) is False + err = capsys.readouterr().err + assert "skipped settlement ('free-fallback')" in err + assert "booked as charged" in err + + @pytest.mark.parametrize( + "headers", + [ + # blockrun.ai's free fallback: a receipt with an empty transaction and no amount. + { + "X-Settlement-Skipped": "free-fallback", + "PAYMENT-RESPONSE": _b64({"success": True, "transaction": ""}), + }, + # A failed receipt is no evidence. + { + "X-Settlement-Skipped": "unexecuted", + "PAYMENT-RESPONSE": _b64({"success": False, "transaction": "0xfailed"}), + }, + ], + ) + def test_receipt_without_evidence_stays_skipped( + self, headers: dict[str, str], capsys: pytest.CaptureFixture + ) -> None: + assert settlement_skipped(headers) is True + assert booked_cost_usd(headers, 0.25) == 0.0 + assert capsys.readouterr().err == "" + # --------------------------------------------------------------------------- # Shared plumbing