From 3e5590bb54df6748e2b93dbd0ee32c964d783d4d Mon Sep 17 00:00:00 2001 From: Abhishek Gahlot Date: Mon, 19 Oct 2026 02:28:06 +0530 Subject: [PATCH 01/21] refactor: share provider accounting and response guards between agents --- deepgym/src/deepgym/agents/computer.py | 152 ++++------------------ deepgym/src/deepgym/agents/providers.py | 160 ++++++++++++++++++++++++ 2 files changed, 186 insertions(+), 126 deletions(-) create mode 100644 deepgym/src/deepgym/agents/providers.py diff --git a/deepgym/src/deepgym/agents/computer.py b/deepgym/src/deepgym/agents/computer.py index 523204d..74fedf6 100644 --- a/deepgym/src/deepgym/agents/computer.py +++ b/deepgym/src/deepgym/agents/computer.py @@ -21,6 +21,15 @@ TokenPrices, Usage, ) +from deepgym.agents.providers import ( + ANTHROPIC_MAX_TOKENS, + SDK_RETRIES, + account, + anthropic_message, + api_error, + openai_incomplete, + validate_budget, +) from deepgym.exceptions import ActionError, DeepGymError from deepgym.models import Action, Observation from deepgym.session import Session @@ -48,97 +57,6 @@ def _step(session: Session, action: Action, tracker: BudgetTracker) -> Observati return observation -def _snapshot(stream: Any) -> Any: - """The accumulated message, or None when no message_start arrived (the SDK asserts then).""" - try: - return stream.current_message_snapshot - except AssertionError: - return None - - -def _incomplete(tracker: BudgetTracker, snapshot: Any, prices: TokenPrices | None) -> None: - """Keep the usage an unfinished stream reported, with the total bill marked unresolved.""" - data = snapshot.model_dump(exclude_none=True).get('usage') if snapshot is not None else None - usage = _anthropic_usage(data) if isinstance(data, dict) else Usage() - unresolved = 'incomplete streamed response: partial usage, billing unresolved' - tracker.add_usage(usage, Cost(provenance=unresolved)) - - -def _account( - tracker: BudgetTracker, - data: dict[str, Any] | None, - prices: TokenPrices | None, - provider: Literal['anthropic', 'openai'], -) -> None: - try: - if not isinstance(data, dict): - raise ValueError('provider did not return token usage') - usage = _anthropic_usage(data) if provider == 'anthropic' else _openai_usage(data) - except (KeyError, TypeError, ValueError): - tracker.add_usage( - Usage(), Cost(provenance=f'{provider} usage unavailable; billing unknown') - ) - raise - tracker.add_usage( - usage, - prices.estimate(usage) - if prices - else Cost( - provenance='provider token usage; prices not supplied', - ), - ) - tracker.check() - - -def _validate_budget(budget: Budget, prices: TokenPrices | None) -> None: - if budget.max_cost_usd is not None and prices is None: - raise ValueError('A USD budget requires explicit TokenPrices with source provenance') - - -def _anthropic_usage(data: dict[str, Any]) -> Usage: - return Usage( - input_tokens=data['input_tokens'], - output_tokens=data['output_tokens'], - cache_read_tokens=data.get('cache_read_input_tokens') or 0, - cache_write_tokens=data.get('cache_creation_input_tokens') or 0, - ) - - -def _openai_usage(data: dict[str, Any]) -> Usage: - details = data.get('input_tokens_details') or {} - cached = details.get('cached_tokens') or 0 - written = details.get('cache_write_tokens') or 0 - return Usage( - input_tokens=data['input_tokens'] - cached - written, - output_tokens=data['output_tokens'], - cache_read_tokens=cached, - cache_write_tokens=written, - ) - - -# The SDK retries connection drops, 429s and 5xx with backoff; one blip shouldn't end an -# hours-long episode. A retried request may be billed twice; usage counts only answers. -_SDK_RETRIES = 4 -ANTHROPIC_MAX_TOKENS = ( - 128_000 # Opus 5.5's maximum output (the API states it); streamed, as the SDK requires -) - - -def _api_error(exc: Exception) -> str: - """Describe a provider API error by class, status and the provider's own validation message. - - Only the structured error body is used, never the exception text, which may echo request - content. - """ - status = getattr(exc, 'status_code', None) - body = getattr(exc, 'body', None) - error = body.get('error', body) if isinstance(body, dict) else None - detail = error.get('message') if isinstance(error, dict) else None - status_text = f' {status}' if status else '' - detail_text = f': {str(detail)[:300]}' if detail else '' - return f'{type(exc).__name__}{status_text}{detail_text}' - - class AnthropicComputerAgent: """Drive Session with Anthropic's computer toolset; import the SDK only on run. @@ -171,7 +89,7 @@ def run( ImportError: If the optional anthropic extra is absent. ValueError: If required prices or the initial screenshot are missing. """ - _validate_budget(budget, self.prices) + validate_budget(budget, self.prices) screenshot = _image(initial.screenshot) import anthropic @@ -188,7 +106,7 @@ def run( ], } ] - client = anthropic.Anthropic(max_retries=_SDK_RETRIES) + client = anthropic.Anthropic(max_retries=SDK_RETRIES) try: while True: if session.done: @@ -204,7 +122,10 @@ def run( if self.effort else {} ) - with client.messages.stream( + response = anthropic_message( + client, + tracker, + self.prices, model=self.model, max_tokens=self.max_tokens, cache_control={'type': 'ephemeral'}, @@ -218,26 +139,10 @@ def run( ], messages=messages, timeout=tracker.request_timeout, - ) as stream: - stopped, failure = False, None - try: # consume every event; only a received message_stop completes a response - for event in stream: - stopped = stopped or event.type == 'message_stop' - # Anything raised while reading (httpx or httpx2 transport errors, by SDK - # version, or an error event) ends this response. - except Exception as exc: # noqa: BLE001 - failure = type(exc).__name__ - snapshot = _snapshot(stream) - if failure or not stopped or snapshot is None: - _incomplete(tracker, snapshot, self.prices) - error = ( - f'Response stream failed: {failure}' - if failure - else 'Response stream ended before completion' - ) - return tracker.result(session, 'error', error=error) - response = snapshot.model_dump(exclude_none=True) - _account(tracker, response.get('usage'), self.prices, 'anthropic') + ) + if isinstance(response, str): + return tracker.result(session, 'error', error=response) + account(tracker, response.get('usage'), self.prices, 'anthropic') content = response['content'] messages.append({'role': 'assistant', 'content': content}) text = ' '.join(part['text'] for part in content if part['type'] == 'text') @@ -302,7 +207,7 @@ def run( except anthropic.APIError as exc: tracker.add_usage(Usage(), Cost(provenance='Anthropic failed request; billing unknown')) reason = 'time_budget' if tracker.remaining_seconds <= 0 else 'error' - return tracker.result(session, reason, error=f'Anthropic {_api_error(exc)}') + return tracker.result(session, reason, error=f'Anthropic {api_error(exc)}') except (ValueError, KeyError, TypeError) as exc: return tracker.result(session, 'error', error=f'Invalid Anthropic response: {exc}') finally: @@ -338,7 +243,7 @@ def run( ImportError: If the optional openai extra is absent. ValueError: If required prices or the initial screenshot are missing. """ - _validate_budget(budget, self.prices) + validate_budget(budget, self.prices) screenshot = _image(initial.screenshot) import openai @@ -353,7 +258,7 @@ def run( } ] previous: str | None = None - client = openai.OpenAI(max_retries=_SDK_RETRIES) + client = openai.OpenAI(max_retries=SDK_RETRIES) try: while True: if session.done: @@ -368,7 +273,7 @@ def run( **({'reasoning': {'effort': self.effort}} if self.effort else {}), timeout=tracker.request_timeout, ).model_dump(exclude_none=True) - _account(tracker, response.get('usage'), self.prices, 'openai') + account(tracker, response.get('usage'), self.prices, 'openai') previous = response['id'] output = response['output'] parts = [ @@ -388,13 +293,8 @@ def run( 'safety_check', error=('Provider safety checks require approval; no actions executed'), ) - if response.get('status') != 'completed': - # Never act on, or blindly continue, a partial response; keep its reason. - details = response.get('incomplete_details') or {} - reason = details.get('reason') or response.get('status') - return tracker.result( - session, 'error', error=f'Response did not complete: {reason}' - ) + if error := openai_incomplete(response): + return tracker.result(session, 'error', error=error) if not calls: return tracker.result(session, 'completed', text) items = [] @@ -441,7 +341,7 @@ def run( except openai.APIError as exc: tracker.add_usage(Usage(), Cost(provenance='OpenAI failed request; billing unknown')) reason = 'time_budget' if tracker.remaining_seconds <= 0 else 'error' - return tracker.result(session, reason, error=f'OpenAI {_api_error(exc)}') + return tracker.result(session, reason, error=f'OpenAI {api_error(exc)}') except (ValueError, KeyError, TypeError) as exc: return tracker.result(session, 'error', error=f'Invalid OpenAI response: {exc}') finally: diff --git a/deepgym/src/deepgym/agents/providers.py b/deepgym/src/deepgym/agents/providers.py new file mode 100644 index 0000000..d9523b4 --- /dev/null +++ b/deepgym/src/deepgym/agents/providers.py @@ -0,0 +1,160 @@ +"""Provider request plumbing shared by the host-side agents: accounting and response guards. + +USD enforcement uses returned usage and caller-supplied prices, so one response can +overshoot the cap. Only a complete response is ever acted on: an Anthropic stream must +reach message_stop, an OpenAI response must have status completed. +""" + +from __future__ import annotations + +from typing import Any, Literal + +from deepgym.agents.base import Budget, BudgetTracker, Cost, TokenPrices, Usage + +# The SDK retries connection drops, 429s and 5xx with backoff; one blip shouldn't end an +# hours-long episode. A retried request may be billed twice; usage counts only answers. +SDK_RETRIES = 4 +ANTHROPIC_MAX_TOKENS = ( + 128_000 # Opus 5.5's maximum output (the API states it); streamed, as the SDK requires +) + + +def _snapshot(stream: Any) -> Any: + """The accumulated message, or None when no message_start arrived (the SDK asserts then).""" + try: + return stream.current_message_snapshot + except AssertionError: + return None + + +def _incomplete(tracker: BudgetTracker, snapshot: Any, prices: TokenPrices | None) -> None: + """Keep the usage an unfinished stream reported, with the total bill marked unresolved.""" + data = snapshot.model_dump(exclude_none=True).get('usage') if snapshot is not None else None + usage = _anthropic_usage(data) if isinstance(data, dict) else Usage() + unresolved = 'incomplete streamed response: partial usage, billing unresolved' + tracker.add_usage(usage, Cost(provenance=unresolved)) + + +def anthropic_message( + client: Any, tracker: BudgetTracker, prices: TokenPrices | None, **request: Any +) -> dict[str, Any] | str: + """Stream one Anthropic response; return it whole, or why it never completed. + + Every event is consumed and only a received message_stop completes a response. An + unfinished one is never acted on: the usage it reported is kept, its billing unresolved. + + Args: + client: The Anthropic client. + tracker: The episode's accounting. + prices: Caller rates, if any. + **request: ``messages.stream`` keyword arguments. + + Raises: + anthropic.APIError: If the request itself fails. + """ + with client.messages.stream(**request) as stream: + stopped, failure = False, None + try: + for event in stream: + stopped = stopped or event.type == 'message_stop' + # Anything raised while reading (httpx or httpx2 transport errors, by SDK + # version, or an error event) ends this response. + except Exception as exc: # noqa: BLE001 + failure = type(exc).__name__ + snapshot = _snapshot(stream) + if failure or not stopped or snapshot is None: + _incomplete(tracker, snapshot, prices) + if failure: + return f'Response stream failed: {failure}' + return 'Response stream ended before completion' + return snapshot.model_dump(exclude_none=True) + + +def openai_incomplete(response: dict[str, Any]) -> str | None: + """Why an OpenAI response did not complete, or None if it did. + + Never act on, or blindly continue, a partial response; keep its reason. + """ + if response.get('status') == 'completed': + return None + details = response.get('incomplete_details') or {} + reason = details.get('reason') or response.get('status') + return f'Response did not complete: {reason}' + + +def account( + tracker: BudgetTracker, + data: dict[str, Any] | None, + prices: TokenPrices | None, + provider: Literal['anthropic', 'openai'], +) -> None: + """Add a response's usage and cost, then stop if a bound is spent. + + Raises: + ValueError: If the provider returned no usage (billing is then unknown). + BudgetExceeded: If time or spend is exhausted. + """ + try: + if not isinstance(data, dict): + raise ValueError('provider did not return token usage') + usage = _anthropic_usage(data) if provider == 'anthropic' else _openai_usage(data) + except (KeyError, TypeError, ValueError): + tracker.add_usage( + Usage(), Cost(provenance=f'{provider} usage unavailable; billing unknown') + ) + raise + tracker.add_usage( + usage, + prices.estimate(usage) + if prices + else Cost( + provenance='provider token usage; prices not supplied', + ), + ) + tracker.check() + + +def validate_budget(budget: Budget, prices: TokenPrices | None) -> None: + """Refuse a USD budget without explicit prices to measure it. + + Raises: + ValueError: If *budget* has a USD limit and *prices* is None. + """ + if budget.max_cost_usd is not None and prices is None: + raise ValueError('A USD budget requires explicit TokenPrices with source provenance') + + +def _anthropic_usage(data: dict[str, Any]) -> Usage: + return Usage( + input_tokens=data['input_tokens'], + output_tokens=data['output_tokens'], + cache_read_tokens=data.get('cache_read_input_tokens') or 0, + cache_write_tokens=data.get('cache_creation_input_tokens') or 0, + ) + + +def _openai_usage(data: dict[str, Any]) -> Usage: + details = data.get('input_tokens_details') or {} + cached = details.get('cached_tokens') or 0 + written = details.get('cache_write_tokens') or 0 + return Usage( + input_tokens=data['input_tokens'] - cached - written, + output_tokens=data['output_tokens'], + cache_read_tokens=cached, + cache_write_tokens=written, + ) + + +def api_error(exc: Exception) -> str: + """Describe a provider API error by class, status and the provider's own validation message. + + Only the structured error body is used, never the exception text, which may echo request + content. + """ + status = getattr(exc, 'status_code', None) + body = getattr(exc, 'body', None) + error = body.get('error', body) if isinstance(body, dict) else None + detail = error.get('message') if isinstance(error, dict) else None + status_text = f' {status}' if status else '' + detail_text = f': {str(detail)[:300]}' if detail else '' + return f'{type(exc).__name__}{status_text}{detail_text}' From d2943387dc184abfd766c268aca6b028827c0210 Mon Sep 17 00:00:00 2001 From: Abhishek Gahlot Date: Mon, 19 Oct 2026 02:33:02 +0530 Subject: [PATCH 02/21] feat: declared http api tools as offline session tool actions --- deepgym/src/deepgym/_http_request.py | 72 ++++++ deepgym/src/deepgym/models.py | 30 ++- deepgym/src/deepgym/qa/agents.py | 25 +- deepgym/src/deepgym/session.py | 69 ++++- deepgym/src/deepgym/tools.py | 264 +++++++++++++++++++ deepgym/src/deepgym/traces/capture.py | 6 + deepgym/src/deepgym/traces/export.py | 6 +- deepgym/tests/conftest.py | 11 +- deepgym/tests/golden/traces/rl.jsonl | 4 +- deepgym/tests/golden/traces/summary.jsonl | 6 +- deepgym/tests/golden/traces/traces.jsonl | 6 +- deepgym/tests/http_app.py | 125 +++++++++ deepgym/tests/test_tools.py | 301 ++++++++++++++++++++++ deepgym/tests/test_traces_export.py | 12 +- 14 files changed, 913 insertions(+), 24 deletions(-) create mode 100644 deepgym/src/deepgym/_http_request.py create mode 100644 deepgym/src/deepgym/tools.py create mode 100644 deepgym/tests/http_app.py create mode 100644 deepgym/tests/test_tools.py diff --git a/deepgym/src/deepgym/_http_request.py b/deepgym/src/deepgym/_http_request.py new file mode 100644 index 0000000..122bbd9 --- /dev/null +++ b/deepgym/src/deepgym/_http_request.py @@ -0,0 +1,72 @@ +"""Send one HTTP request described by a JSON file and print what happened as one JSON line. + +The session runs this inside the sandbox (``python3 -I``) as the agent user, in the offline +network namespace, for an environment's declared API tool (see :mod:`deepgym.tools`). The +host built and checked the request; this only sends it. Nothing here comes from the model +but that request's data: no proxy is used and redirects come back as the response, never +followed, so the request reaches only its own URL. Stdlib only, as the image's python3 runs it. +""" + +import base64 +import http.client +import json +import sys +import time +import urllib.error +import urllib.request + + +class _Unfollowed(urllib.request.HTTPRedirectHandler): + """Hand a 3xx back as the response instead of following its Location.""" + + def redirect_request(self, *args: object, **kwargs: object) -> None: + return None + + +def main(path: str) -> int: + """Send the request in *path*; print its record; return 0 if a response came back.""" + with open(path, encoding='utf-8') as file: + spec = json.load(file) + body = spec['body'] + request = urllib.request.Request( + spec['url'], + data=None if body is None else body.encode('utf-8'), + headers=spec['headers'], + method=spec['method'], + ) + opener = urllib.request.build_opener(urllib.request.ProxyHandler({}), _Unfollowed) + record: dict[str, object] = {'request': spec['record']} + started = time.monotonic() + try: + try: + response = opener.open(request) + except urllib.error.HTTPError as error: # 4xx, 5xx and unfollowed 3xx are answers too + response = error + with response: + raw = response.read() + except (OSError, http.client.HTTPException) as exc: + reason = getattr(exc, 'reason', exc) + record['error'] = f'No response: {type(reason).__name__}: {reason}' + # Only a refused connection proves nothing was sent; any other loss may follow a write. + record['uncertain'] = spec['mutation'] and not isinstance(reason, ConnectionRefusedError) + status = 1 + else: + answer: dict[str, object] = { + 'status': response.status, + 'reason': response.reason, + 'headers': [list(header) for header in response.headers.items()], + } + try: + answer['body'] = raw.decode('utf-8') + except UnicodeDecodeError: + answer['body_base64'] = base64.b64encode(raw).decode('ascii') + record['response'] = answer + status = 0 + record['elapsed_ms'] = (time.monotonic() - started) * 1000 + # UTF-8 bytes whatever the sandbox locale, on one line: JSON escapes every newline. + sys.stdout.buffer.write(json.dumps(record, ensure_ascii=False).encode('utf-8') + b'\n') + return status + + +if __name__ == '__main__': + sys.exit(main(sys.argv[1])) diff --git a/deepgym/src/deepgym/models.py b/deepgym/src/deepgym/models.py index b64c63e..50b944c 100644 --- a/deepgym/src/deepgym/models.py +++ b/deepgym/src/deepgym/models.py @@ -13,6 +13,7 @@ from pydantic import BaseModel, ConfigDict, Field, JsonValue, field_validator, model_validator from deepgym.exceptions import InvalidScoreError +from deepgym.tools import HttpTool logger = logging.getLogger(__name__) @@ -411,7 +412,7 @@ class Observation(BaseModel): 'screenshot', 'wait', ] -ActionType = Literal[ShellActionType, GuiActionType, 'done', 'cli'] +ActionType = Literal[ShellActionType, GuiActionType, 'done', 'cli', 'tool'] GUI_ACTIONS: frozenset[GuiActionType] = frozenset(get_args(GuiActionType)) @@ -425,6 +426,7 @@ class Action(BaseModel): Shell actions (code, bash, write_file) run in the session work directory. GUI actions drive the desktop; coordinates are screenshot pixels. + Tool actions call one of the environment's declared API tools (see :mod:`deepgym.tools`). """ action_type: ActionType = 'code' @@ -455,6 +457,12 @@ class Action(BaseModel): amount: int = Field(default=1, ge=1, le=60) """Scroll clicks, or seconds for wait.""" + tool: str | None = None + """Name of the declared tool a tool action calls.""" + + arguments: dict[str, JsonValue] = Field(default_factory=dict) + """A tool action's arguments; the session checks them against the tool's routes.""" + @model_validator(mode='after') def check_arguments(self) -> Action: """Reject actions missing the arguments their type needs.""" @@ -467,6 +475,8 @@ def check_arguments(self) -> Action: raise ValueError(f'scroll content must be one of {sorted(_SCROLL_DIRECTIONS)}') if kind in _TEXT_ACTIONS and not self.content: raise ValueError(f'{kind} needs content') + if kind == 'tool' and not self.tool: + raise ValueError('tool needs the name of a declared tool') if kind == 'write_file': rel = PurePosixPath(self.path or '') if not rel.parts or rel.is_absolute() or '..' in rel.parts: @@ -559,6 +569,12 @@ class MultiTurnEnvironment(BaseModel): allowed_actions: frozenset[ActionType] = frozenset({'code', 'bash', 'write_file', 'done'}) """Action types the agent may use. Anything else is rejected.""" + tools: tuple[HttpTool, ...] = () + """API tools that tool actions call, run in the sandbox's offline namespace (see + :mod:`deepgym.tools`). Trusted environment configuration only; never accept from an agent. + Declaring any needs ``tool`` and ``done`` in allowed_actions: a tool-calling agent ends + the episode only by an explicit ``done``.""" + desktop: bool = False """Start a desktop (Xvfb + xfce) and return screenshots. Needs the Daytona backend.""" @@ -593,6 +609,18 @@ def check_desktop_actions(self) -> MultiTurnEnvironment: raise ValueError('cli_provider_egress requires bash or cli in allowed_actions') return self + @model_validator(mode='after') + def check_tools(self) -> MultiTurnEnvironment: + """Require unique tool names, and tools exactly when tool actions are allowed.""" + names = [tool.name for tool in self.tools] + if len(set(names)) != len(names): + raise ValueError(f'tool names must be unique, got {names}') + if self.tools and not {'tool', 'done'} <= self.allowed_actions: + raise ValueError('tools need tool and done in allowed_actions') + if 'tool' in self.allowed_actions and not self.tools: + raise ValueError('tool actions need declared tools') + return self + class VerdictTotals(BaseModel): """Totals over only the runs that got a verdict (``RunResult.has_verdict``). diff --git a/deepgym/src/deepgym/qa/agents.py b/deepgym/src/deepgym/qa/agents.py index 1b524ce..34386e2 100644 --- a/deepgym/src/deepgym/qa/agents.py +++ b/deepgym/src/deepgym/qa/agents.py @@ -3,12 +3,13 @@ A verifier must give them nothing. One that pays for being busy, or for random clicks and commands, rewards a policy for activity instead of the outcome. Their shell actions are harmless (``ls``, ``print``, notes files), because a local -session runs them on the host. +session runs them on the host; their tool calls only read (GET on a declared route). """ from __future__ import annotations import random +import re from deepgym.models import Action, ActionType, MultiTurnEnvironment @@ -29,7 +30,8 @@ def do_everything(env: MultiTurnEnvironment) -> list[Action]: """Every allowed action type in turn, with fixed arguments, for the whole step budget.""" kinds = _kinds(env) return [ - _action(kinds[step % len(kinds)], step, None) for step in range(_control_steps(env, kinds)) + _action(env, kinds[step % len(kinds)], step, None) + for step in range(_control_steps(env, kinds)) ] @@ -37,7 +39,9 @@ def random_agent(env: MultiTurnEnvironment) -> list[Action]: """Seeded random actions of the allowed types, for the whole step budget.""" rng = random.Random(_SEED) kinds = _kinds(env) - return [_action(rng.choice(kinds), step, rng) for step in range(_control_steps(env, kinds))] + return [ + _action(env, rng.choice(kinds), step, rng) for step in range(_control_steps(env, kinds)) + ] def _control_steps(env: MultiTurnEnvironment, kinds: list[ActionType]) -> int: @@ -51,12 +55,23 @@ def _kinds(env: MultiTurnEnvironment) -> list[ActionType]: return sorted(env.allowed_actions - {'done'}) or ['done'] -def _action(kind: ActionType, step: int, rng: random.Random | None) -> Action: +def _action( + env: MultiTurnEnvironment, kind: ActionType, step: int, rng: random.Random | None +) -> Action: """One harmless action of *kind*: fixed arguments without *rng*, random ones with it.""" def pick(options: tuple[str, ...]) -> str: return rng.choice(options) if rng else options[step % len(options)] + def read() -> Action: + """A GET on a declared route, each placeholder filled with 1; done if none reads.""" + reads = [(t.name, r.path) for t in env.tools for r in t.routes if r.method == 'GET'] + if not reads: + return Action(action_type='done') + name, path = rng.choice(reads) if rng else reads[step % len(reads)] + arguments = {'method': 'GET', 'path': re.sub(r'\{\w+\}', '1', path)} + return Action(action_type=kind, tool=name, arguments=arguments) + def point() -> tuple[int, int]: if rng is None: return 10 + step, 10 + step @@ -81,5 +96,7 @@ def point() -> tuple[int, int]: return Action(action_type=kind, x=x, y=y, to_x=to_x, to_y=to_y) case 'click' | 'double_click' | 'right_click' | 'move': return Action(action_type=kind, x=x, y=y) + case 'tool': + return read() case _: # screenshot, wait, done return Action(action_type=kind) diff --git a/deepgym/src/deepgym/session.py b/deepgym/src/deepgym/session.py index 08bb508..fef4a71 100644 --- a/deepgym/src/deepgym/session.py +++ b/deepgym/src/deepgym/session.py @@ -81,6 +81,7 @@ SandboxTimeoutError, sealed_path, ) +from deepgym.tools import REQUEST_FILE, HttpTool, lost_response, run_command from deepgym.verifier_template import guard_crashes if TYPE_CHECKING: @@ -964,6 +965,11 @@ def trajectory(self) -> Trajectory: """Everything that happened so far.""" return self._trajectory + @property + def tools(self) -> tuple[HttpTool, ...]: + """The API tools the environment declares, for tool-calling agents.""" + return self._env.tools + @property def step_verdict(self) -> dict[str, object] | None: """The step verifier output of the last step (score, passed, details, ...). @@ -1040,15 +1046,20 @@ def step(self, action: Action, *, timeout: float | None = None) -> StepResult: verifier that times out after reading the agent's output gives the step a judged 0 (``solution_timeout``) instead, and the episode goes on. + A tool action's arguments are checked against its declared tool before it counts + as a step. Only an explicit ``done`` ends an episode of tool actions: their output + is the app's, so a DONE line in it never does. + ``timeout`` bounds agent execution and the following screenshot by one remaining wall-time deadline. Desktop overrides require SandboxBackend. Verifier time is separate; provider transport can take its own cleanup grace after a deadline. Raises: DeepGymError: If the session is not running. - ActionError: If the action is not allowed here, the desktop rejects it, or the - agent's own files are in the way of a write (the episode goes on, unless - they blocked the step's output file). + ActionError: If the action is not allowed here, names no declared tool or has + arguments its tool rejects, the desktop rejects it, or the agent's own files + are in the way of a write (the episode goes on, unless they blocked the + step's output file). VerifierError: If the step verifier fails, gives no verdict, or times out before it reads the agent's output. SandboxError: If the sandbox fails to apply the action, take the screenshot or @@ -1073,11 +1084,14 @@ def step(self, action: Action, *, timeout: float | None = None) -> StepResult: and not isinstance(self._backend, SandboxBackend) ): raise ActionError('Desktop deadline overrides require SandboxBackend') + request = self._tool_request(action) if action.action_type == 'tool' else None self._step += 1 self._step_verdict = None try: - output, exit_code = self._apply(action, timeout=_remaining(deadline)) + output, exit_code = self._apply( + action, timeout=_remaining(deadline), request=request + ) except ActionError: raise except BaseException as exc: @@ -1085,7 +1099,7 @@ def step(self, action: Action, *, timeout: float | None = None) -> StepResult: raise done = ( action.action_type == 'done' - or _signals_done(output) + or (action.action_type != 'tool' and _signals_done(output)) or (self._env.max_steps is not None and self._step >= self._env.max_steps) ) observation: Observation | None = None @@ -1276,10 +1290,17 @@ def __exit__(self, *args: object) -> None: 'Could not delete sandbox %s; it may still exist', self.sandbox_id, exc_info=True ) - def _apply(self, action: Action, *, timeout: float | None = None) -> tuple[str, int | None]: + def _apply( + self, + action: Action, + *, + timeout: float | None = None, + request: Mapping[str, JsonValue] | None = None, + ) -> tuple[str, int | None]: """Execute *action*; return its text output and exit code (None if not a command). - Its time, however it ends, counts as the agent's (see :meth:`_ran_out_the_clock`). + A tool action sends *request*, from :meth:`_tool_request`. Its time, however it + ends, counts as the agent's (see :meth:`_ran_out_the_clock`). """ bounded = timeout is not None timeout = ( @@ -1300,6 +1321,9 @@ def _apply(self, action: Action, *, timeout: float | None = None) -> tuple[str, case 'write_file': self._backend.write_file(str(action.path), action.content.encode('utf-8')) return f'wrote {action.path}', None + case 'tool': + assert request is not None + return self._call_tool(request, timeout) case 'wait': time.sleep(min(action.amount, timeout)) return '', None @@ -1315,6 +1339,37 @@ def _apply(self, action: Action, *, timeout: float | None = None) -> tuple[str, finally: self._acting_s += time.monotonic() - started + def _tool_request(self, action: Action) -> dict[str, JsonValue]: + """Return the request *action* sends with its declared tool (see HttpTool.prepare). + + Raises: + ActionError: If no declared tool has its name, or the tool rejects its arguments. + """ + tool = next((tool for tool in self._env.tools if tool.name == action.tool), None) + if tool is None: + names = sorted(tool.name for tool in self._env.tools) + raise ActionError(f'No tool named {action.tool!r}. Declared: {names}') + try: + return tool.prepare(action.arguments) + except ValueError as exc: + raise ActionError(f'Tool {tool.name!r} rejected the call: {exc}') from exc + + def _call_tool( + self, request: Mapping[str, JsonValue], timeout: float + ) -> tuple[str, int | None]: + """Send *request* from the sandbox; return the runner's record of it and its exit code. + + The request is data in a file the constant runner command reads. It is never + retried: a runner stopped by the timeout reports no response, and a mutation it + may have sent is recorded as uncertain. + """ + self._backend.write_file(REQUEST_FILE, json.dumps(request).encode('utf-8')) + try: + exit_code, output = self._backend.exec(run_command(self._backend.python), timeout) + except TimeoutError: + return lost_response(request, f'the request was stopped after {timeout}s'), None + return output, exit_code + def _run_python(self, filename: str, code: str, timeout: float | None = None) -> ExecResult: self._backend.write_file(filename, code.encode('utf-8')) return self._backend.exec( diff --git a/deepgym/src/deepgym/tools.py b/deepgym/src/deepgym/tools.py new file mode 100644 index 0000000..6d08bbe --- /dev/null +++ b/deepgym/src/deepgym/tools.py @@ -0,0 +1,264 @@ +"""HTTP API tools an environment declares for tool-calling agents, and how a session runs them. + +A tool is trusted environment configuration, never model output: its name, base URL, routes +and headers come from the environment's author. A model supplies only a call's arguments +(method, path, query, body). :meth:`HttpTool.prepare` checks them against the declared +routes, and the session writes the request to a file in the work directory. The command that +sends it is a constant (:func:`run_command`), so nothing the model writes reaches a shell; it +runs as the agent user in the sandbox's offline network namespace, where only the episode's +own services answer, and it follows no redirect (see :mod:`deepgym._http_request`). +""" + +from __future__ import annotations + +import ipaddress +import json +import re +import shlex +from collections.abc import Mapping +from pathlib import Path +from typing import Literal +from urllib.parse import unquote, urlencode, urlsplit + +from pydantic import BaseModel, ConfigDict, Field, JsonValue, field_validator + +HttpMethod = Literal['GET', 'POST', 'PUT', 'PATCH', 'DELETE'] +QueryScalar = str | int | float | bool + +DONE_TOOL = 'done' +"""Reserved tool name: agents offer it so the model ends the episode with a ``done`` action.""" + +REQUEST_FILE = '_tool_request.json' +"""Where in the work directory the session writes a call's request for the runner.""" + +_RUNNER = Path(__file__).with_name('_http_request.py').read_text(encoding='utf-8') +_BODY_METHODS = frozenset({'POST', 'PUT', 'PATCH'}) +# RFC 3986 path characters; a model percent-encodes anything else, so nothing it sends can +# start a query, a fragment or another authority. +_SEGMENT = re.compile(r"[A-Za-z0-9\-._~!$&'()*+,;=:@]+") +_PATH = re.compile(r"/[A-Za-z0-9\-._~!$&'()*+,;=:@%/]*") +_PLACEHOLDER = re.compile(r'\{\w+\}') +_REDACTED = '[redacted]' + + +class HttpRoute(BaseModel): + """One endpoint a tool may call: a method and a path template. + + ``{name}`` in the template stands for exactly one non-empty path segment; every other + segment must match literally. An endpoint no route names (an admin page, a script + runner) is unreachable. + """ + + model_config = ConfigDict(frozen=True, extra='forbid') + method: HttpMethod + path: str + + @field_validator('path') + @classmethod + def check_path(cls, value: str) -> str: + """Accept '/' or '/'-separated literal segments and ``{name}`` placeholders.""" + segments = value.split('/') + literal = [segment for segment in segments[1:] if not _PLACEHOLDER.fullmatch(segment)] + if value != '/' and ( + segments[0] + or not all(_SEGMENT.fullmatch(part) and part not in ('.', '..') for part in literal) + ): + raise ValueError( + f'route path {value!r} must start with "/" and hold only non-empty literal ' + 'segments (no dot segments or percent signs) and {name} placeholders' + ) + return value + + def matches(self, path: str) -> bool: + """Whether request *path* fits this template, segment by segment.""" + want, got = self.path.split('/'), path.split('/') + return len(want) == len(got) and all( + expected == actual or (bool(actual) and bool(_PLACEHOLDER.fullmatch(expected))) + for expected, actual in zip(want, got) + ) + + +class _Call(BaseModel): + """A tool call's arguments, as the model sent them.""" + + model_config = ConfigDict(extra='forbid', strict=True) + method: HttpMethod + path: str + query: dict[str, QueryScalar | list[QueryScalar]] = Field(default_factory=dict) + body: dict[str, JsonValue] | list[JsonValue] | None = None + + +class HttpTool(BaseModel): + """An API tool: HTTP requests to declared routes under one loopback base URL. + + The model sees *name*, *description* (the place for the API docs a task needs) and the + routes. *headers* go with every request (an API key for the local app, say) and are + recorded with their values redacted. Responses come back whole: status, every header + (pagination links included) and the full body. + + Example:: + + HttpTool( + name='bugzilla', + description='Bugzilla 5.0 REST API: search bugs, read comments, add a comment.', + base_url='http://127.0.0.1:8080', + routes=( + HttpRoute(method='GET', path='/rest/bug'), + HttpRoute(method='GET', path='/rest/bug/{id}/comment'), + HttpRoute(method='POST', path='/rest/bug/{id}/comment'), + ), + headers={'X-BUGZILLA-API-KEY': api_key}, + ) + """ + + model_config = ConfigDict(frozen=True, extra='forbid') + name: str = Field(pattern=r'^[A-Za-z0-9_-]{1,64}$') + description: str = Field(min_length=1) + base_url: str + routes: tuple[HttpRoute, ...] = Field(min_length=1) + headers: dict[str, str] = Field(default_factory=dict) + + @field_validator('name') + @classmethod + def check_name(cls, value: str) -> str: + """Keep the done tool's name for the done tool.""" + if value == DONE_TOOL: + raise ValueError(f'{DONE_TOOL!r} is reserved for the tool that ends the episode') + return value + + @field_validator('base_url') + @classmethod + def check_base_url(cls, value: str) -> str: + """Accept an http(s) URL on a loopback host, without credentials, query or fragment. + + Returns: + The URL without a trailing slash, so a call's path appends to it. + """ + parts = urlsplit(value) + try: + host = parts.hostname or '' + loopback = host == 'localhost' or ipaddress.ip_address(host).is_loopback + parts.port # raises ValueError on a malformed port + except ValueError: + loopback = False + if ( + parts.scheme not in ('http', 'https') + or not loopback + or '@' in parts.netloc + or '?' in value + or '#' in value + or (parts.path and not _PATH.fullmatch(parts.path)) + ): + raise ValueError( + f'base_url {value!r} must be an http(s) URL on a loopback host ' + '(localhost, 127.0.0.0/8, ::1) with no credentials, query or fragment' + ) + return value.rstrip('/') + + @property + def input_schema(self) -> dict[str, JsonValue]: + """The JSON schema of a call's arguments, as tool-calling APIs take it.""" + scalar: dict[str, JsonValue] = {'type': ['string', 'number', 'boolean']} + return { + 'type': 'object', + 'properties': { + 'method': {'type': 'string', 'enum': sorted({r.method for r in self.routes})}, + 'path': { + 'type': 'string', + 'description': ( + f'Path under {self.base_url}, fitting one of these routes: ' + f'{self._listing}. {{name}} stands for one path segment; ' + 'percent-encode reserved characters in it. Query parameters go in ' + '"query", not in the path.' + ), + }, + 'query': { + 'type': 'object', + 'description': 'Query parameters; a list value repeats its parameter.', + 'additionalProperties': {'anyOf': [scalar, {'type': 'array', 'items': scalar}]}, + }, + 'body': { + 'type': ['object', 'array'], + 'description': 'JSON request body, for POST, PUT and PATCH only.', + }, + }, + 'required': ['method', 'path'], + 'additionalProperties': False, + } + + @property + def _listing(self) -> str: + return ', '.join(f'{route.method} {route.path}' for route in self.routes) + + def prepare(self, arguments: Mapping[str, JsonValue]) -> dict[str, JsonValue]: + """Check a call's *arguments*; return the request the runner sends, with its record. + + Args: + arguments: The model's arguments: method, path, optional query and body. + + Returns: + The request (method, url, headers, body text), whether it is a mutation (any + method but GET), and its record: the same with header values redacted. + + Raises: + ValueError: If an argument is unknown or of the wrong type, the path has a + character to percent-encode or a dot segment, no declared route fits the + method and path, or the method takes no body and one was given. + """ + call = _Call.model_validate(arguments) + decoded = unquote(call.path) + if ( + not _PATH.fullmatch(call.path) + or '\\' in decoded + or any(part in ('.', '..') for part in decoded.split('/')) + ): + raise ValueError( + f'path {call.path!r} must start with "/", percent-encode reserved ' + 'characters and have no dot segments' + ) + if not any(r.method == call.method and r.matches(call.path) for r in self.routes): + raise ValueError( + f'{call.method} {call.path} fits no route of this tool: {self._listing}' + ) + if call.body is not None and call.method not in _BODY_METHODS: + raise ValueError(f'{call.method} takes no body') + pairs = [ + (key, item if isinstance(item, str) else json.dumps(item)) + for key, value in call.query.items() + for item in (value if isinstance(value, list) else [value]) + ] + url = self.base_url + call.path + (f'?{urlencode(pairs)}' if pairs else '') + headers = dict(self.headers) + record: dict[str, JsonValue] = {'method': call.method, 'url': url} + if call.body is not None: + headers['Content-Type'] = 'application/json' + record['body'] = call.body + record['headers'] = { + name: _REDACTED if name in self.headers else value for name, value in headers.items() + } + return { + 'method': call.method, + 'url': url, + 'headers': headers, + 'body': None if call.body is None else json.dumps(call.body, ensure_ascii=False), + 'mutation': call.method != 'GET', + 'record': record, + } + + +def run_command(python: str) -> str: + """The constant command that sends the request in :data:`REQUEST_FILE` with *python*.""" + return f'{python} -I -c {shlex.quote(_RUNNER)} {REQUEST_FILE}' + + +def lost_response(request: Mapping[str, JsonValue], reason: str) -> str: + """Record a call whose runner was stopped before it reported, as the runner would. + + A mutation is uncertain: it may have been applied before the response was lost. + """ + record = { + 'request': request['record'], + 'error': f'No response: {reason}', + 'uncertain': request['mutation'], + } + return json.dumps(record, ensure_ascii=False) diff --git a/deepgym/src/deepgym/traces/capture.py b/deepgym/src/deepgym/traces/capture.py index d2ed28f..b4c1bac 100644 --- a/deepgym/src/deepgym/traces/capture.py +++ b/deepgym/src/deepgym/traces/capture.py @@ -33,6 +33,7 @@ is_verdict, ) from deepgym.session import Session, StepResult +from deepgym.tools import HttpTool from deepgym.traces.schema import ( TRACE_SCHEMA_VERSION, Provenance, @@ -102,6 +103,11 @@ def done(self) -> bool: """Whether the episode has ended.""" return self._session.done + @property + def tools(self) -> tuple[HttpTool, ...]: + """The API tools the environment declares, for tool-calling agents.""" + return self._session.tools + @property def trajectory(self) -> Trajectory: """Return a copy of the session trajectory for an agent's result.""" diff --git a/deepgym/src/deepgym/traces/export.py b/deepgym/src/deepgym/traces/export.py index 228e295..8466b91 100644 --- a/deepgym/src/deepgym/traces/export.py +++ b/deepgym/src/deepgym/traces/export.py @@ -16,7 +16,8 @@ ``message_index``: where in ``messages`` the reply that produced it is, shared by the actions of one turn. A trace with a turn that has no model messages has no whole conversation to train on and is skipped. In RL Parquet, ``action.events`` is a - list of JSON strings, preserving arbitrary nested CLI event shapes. + list of JSON strings, preserving arbitrary nested CLI event shapes, and a tool action's + ``action.arguments`` is one JSON string. Every row carries ``has_verdict``, and its ``score``/``reward`` and ``passed`` are null without one, so a failed sandbox or verifier never reads as a scored 0. ``only_valid`` @@ -196,6 +197,7 @@ def _write_parquet(records: Sequence[Row], out: Path, rows: RowShape) -> None: 'action': { **step['action'], 'events': [json_text(event) for event in step['action']['events']], + 'arguments': json_text(step['action']['arguments']), }, } for step in record['steps'] @@ -243,6 +245,8 @@ def _schema(rows: RowShape) -> pa.Schema: ('path', pa.string()), ('amount', pa.int64()), ('events', pa.list_(pa.string())), + ('tool', pa.string()), + ('arguments', pa.string()), ]) # fmt: skip step = pa.struct([ ('message_index', pa.int64()), diff --git a/deepgym/tests/conftest.py b/deepgym/tests/conftest.py index 1d989a6..9f2e337 100644 --- a/deepgym/tests/conftest.py +++ b/deepgym/tests/conftest.py @@ -5,7 +5,7 @@ import subprocess import sys import tempfile -from collections.abc import Mapping +from collections.abc import Iterator, Mapping from pathlib import Path import pytest @@ -14,6 +14,7 @@ from deepgym.core import DeepGym from deepgym.models import Environment from deepgym.sandboxes import ExecResult +from tests.http_app import App # --------------------------------------------------------------------------- # Paths @@ -170,6 +171,14 @@ def delete(self) -> None: shutil.rmtree(self.root, ignore_errors=True) +@pytest.fixture +def app() -> Iterator[App]: + """A running loopback HTTP app for API tool tests, stopped afterwards.""" + served = App() + yield served + served.close() + + @pytest.fixture() def sandbox_dg(monkeypatch: pytest.MonkeyPatch) -> DeepGym: """A sandbox-mode client whose benchmark runners run in a RunnerSandbox.""" diff --git a/deepgym/tests/golden/traces/rl.jsonl b/deepgym/tests/golden/traces/rl.jsonl index 9f709bd..7065ade 100644 --- a/deepgym/tests/golden/traces/rl.jsonl +++ b/deepgym/tests/golden/traces/rl.jsonl @@ -1,2 +1,2 @@ -{"trace_id":"trace0001","data_source":"write_notes","agent":"react-shell","model":"Qwen/Qwen3-8B","messages":[{"role":"system","content":"You are a shell agent."},{"role":"user","content":"Write hello to notes/out.txt, then finish.\n$ "},{"role":"assistant","content":"mkdir -p notes && echo hello > notes/out.txt","token_ids":[11,12,13],"logprobs":[-0.25,-0.5,-0.125]},{"role":"tool","content":"exit 0"},{"role":"assistant","content":"write notes/extra.txt"},{"role":"user","content":"write_file is not allowed."},{"role":"assistant","content":"done"}],"initial_observation":"$ ","initial_screenshot":"iVBORw0KGgogc2FtcGxlIHNjcmVlbnNob3Q=","reward":1.0,"passed":true,"step_rewards":[0.5,null,0.5],"steps":[{"message_index":2,"action":{"action_type":"bash","content":"mkdir -p notes && echo hello > notes/out.txt","events":[],"x":null,"y":null,"to_x":null,"to_y":null,"path":null,"amount":1},"observation":"","screenshot":null,"done":false,"exit_code":0,"error":null,"reward":0.5},{"message_index":4,"action":{"action_type":"write_file","content":"x","events":[],"x":null,"y":null,"to_x":null,"to_y":null,"path":"notes/extra.txt","amount":1},"observation":null,"screenshot":null,"done":null,"exit_code":null,"error":"ActionError: Action 'write_file' not allowed here.","reward":null},{"message_index":6,"action":{"action_type":"done","content":"","events":[],"x":null,"y":null,"to_x":null,"to_y":null,"path":null,"amount":1},"observation":"","screenshot":null,"done":true,"exit_code":null,"error":null,"reward":0.5}],"has_verdict":true} -{"trace_id":"trace0003","data_source":"write_notes","agent":"react-shell","model":"Qwen/Qwen3-8B","messages":[{"role":"system","content":"You are a shell agent."},{"role":"user","content":"Write hello to notes/out.txt, then finish.\n$ "},{"role":"assistant","content":"mkdir -p notes && echo hello > notes/out.txt","token_ids":[11,12,13],"logprobs":[-0.25,-0.5,-0.125]},{"role":"tool","content":"exit 0"},{"role":"assistant","content":"write notes/extra.txt"},{"role":"user","content":"write_file is not allowed."},{"role":"assistant","content":"done"}],"initial_observation":"$ ","initial_screenshot":"iVBORw0KGgogc2FtcGxlIHNjcmVlbnNob3Q=","reward":null,"passed":null,"step_rewards":[0.5,null,0.5],"steps":[{"message_index":2,"action":{"action_type":"bash","content":"mkdir -p notes && echo hello > notes/out.txt","events":[],"x":null,"y":null,"to_x":null,"to_y":null,"path":null,"amount":1},"observation":"","screenshot":null,"done":false,"exit_code":0,"error":null,"reward":0.5},{"message_index":4,"action":{"action_type":"write_file","content":"x","events":[],"x":null,"y":null,"to_x":null,"to_y":null,"path":"notes/extra.txt","amount":1},"observation":null,"screenshot":null,"done":null,"exit_code":null,"error":"ActionError: Action 'write_file' not allowed here.","reward":null},{"message_index":6,"action":{"action_type":"done","content":"","events":[],"x":null,"y":null,"to_x":null,"to_y":null,"path":null,"amount":1},"observation":"","screenshot":null,"done":true,"exit_code":null,"error":null,"reward":0.5}],"has_verdict":false} +{"trace_id":"trace0001","data_source":"write_notes","agent":"react-shell","model":"Qwen/Qwen3-8B","messages":[{"role":"system","content":"You are a shell agent."},{"role":"user","content":"Write hello to notes/out.txt, then finish.\n$ "},{"role":"assistant","content":"mkdir -p notes && echo hello > notes/out.txt","token_ids":[11,12,13],"logprobs":[-0.25,-0.5,-0.125]},{"role":"tool","content":"exit 0"},{"role":"assistant","content":"write notes/extra.txt"},{"role":"user","content":"write_file is not allowed."},{"role":"assistant","content":"done"}],"initial_observation":"$ ","initial_screenshot":"iVBORw0KGgogc2FtcGxlIHNjcmVlbnNob3Q=","reward":1.0,"passed":true,"step_rewards":[0.5,null,0.5],"steps":[{"message_index":2,"action":{"action_type":"bash","content":"mkdir -p notes && echo hello > notes/out.txt","events":[],"x":null,"y":null,"to_x":null,"to_y":null,"path":null,"amount":1,"tool":null,"arguments":{}},"observation":"","screenshot":null,"done":false,"exit_code":0,"error":null,"reward":0.5},{"message_index":4,"action":{"action_type":"write_file","content":"x","events":[],"x":null,"y":null,"to_x":null,"to_y":null,"path":"notes/extra.txt","amount":1,"tool":null,"arguments":{}},"observation":null,"screenshot":null,"done":null,"exit_code":null,"error":"ActionError: Action 'write_file' not allowed here.","reward":null},{"message_index":6,"action":{"action_type":"done","content":"","events":[],"x":null,"y":null,"to_x":null,"to_y":null,"path":null,"amount":1,"tool":null,"arguments":{}},"observation":"","screenshot":null,"done":true,"exit_code":null,"error":null,"reward":0.5}],"has_verdict":true} +{"trace_id":"trace0003","data_source":"write_notes","agent":"react-shell","model":"Qwen/Qwen3-8B","messages":[{"role":"system","content":"You are a shell agent."},{"role":"user","content":"Write hello to notes/out.txt, then finish.\n$ "},{"role":"assistant","content":"mkdir -p notes && echo hello > notes/out.txt","token_ids":[11,12,13],"logprobs":[-0.25,-0.5,-0.125]},{"role":"tool","content":"exit 0"},{"role":"assistant","content":"write notes/extra.txt"},{"role":"user","content":"write_file is not allowed."},{"role":"assistant","content":"done"}],"initial_observation":"$ ","initial_screenshot":"iVBORw0KGgogc2FtcGxlIHNjcmVlbnNob3Q=","reward":null,"passed":null,"step_rewards":[0.5,null,0.5],"steps":[{"message_index":2,"action":{"action_type":"bash","content":"mkdir -p notes && echo hello > notes/out.txt","events":[],"x":null,"y":null,"to_x":null,"to_y":null,"path":null,"amount":1,"tool":null,"arguments":{}},"observation":"","screenshot":null,"done":false,"exit_code":0,"error":null,"reward":0.5},{"message_index":4,"action":{"action_type":"write_file","content":"x","events":[],"x":null,"y":null,"to_x":null,"to_y":null,"path":"notes/extra.txt","amount":1,"tool":null,"arguments":{}},"observation":null,"screenshot":null,"done":null,"exit_code":null,"error":"ActionError: Action 'write_file' not allowed here.","reward":null},{"message_index":6,"action":{"action_type":"done","content":"","events":[],"x":null,"y":null,"to_x":null,"to_y":null,"path":null,"amount":1,"tool":null,"arguments":{}},"observation":"","screenshot":null,"done":true,"exit_code":null,"error":null,"reward":0.5}],"has_verdict":false} diff --git a/deepgym/tests/golden/traces/summary.jsonl b/deepgym/tests/golden/traces/summary.jsonl index 45e7a74..c0fe4c5 100644 --- a/deepgym/tests/golden/traces/summary.jsonl +++ b/deepgym/tests/golden/traces/summary.jsonl @@ -1,3 +1,3 @@ -{"trace_id": "trace0001", "env_name": "write_notes", "env_variant": 2, "agent": "react-shell", "model": "Qwen/Qwen3-8B", "provider": "local", "status": "complete", "has_verdict": true, "score": 1.0, "passed": true, "steps": 3, "started_at": "2026-10-01T12:00:00+00:00", "trace": {"schema_version": "1.1", "trace_id": "trace0001", "task": "Write hello to notes/out.txt, then finish.", "provenance": {"env": {"name": "write_notes", "variant": 2, "seed": 7, "environment_sha256": "7f142979bcdf5748ac3baa872a5db2a58d701da9bb29b7451d747e8a3777b6f9", "task_sha256": "1bbf96220c257b1281f09efdf074dc9cb437d3f4401d77c008308bfd72fccc28", "verifier_sha256": "5d6ee51bb39cafa8308963062208847333565120aa1e94980872375cd011d158", "step_verifier_sha256": "e4a424cca79d7c48e6fd4bc2827e14523db51284f401d9c0321d43117878860b", "reference_sha256": "a8bad614127cbaaaaa67e14952ae0fc6c0dce71bdff0822d038556b7dfbe3431", "image": "python:3.12-slim@sha256:abababababababababababababababababababababababababababababababab", "image_digest": "sha256:abababababababababababababababababababababababababababababababab", "digest_unavailable": null, "snapshot": null}, "agent": {"name": "react-shell", "version": "1.2.0", "model": "Qwen/Qwen3-8B", "sampling": {"temperature": 0.7, "top_p": 0.95}}, "provider": "local", "deepgym_version": "0.4.1", "deepgym_commit": null, "deepgym_dirty": null, "started_at": "2026-10-01T12:00:00Z"}, "initial_observation": {"content": "$ ", "step": 0, "done": false, "screenshot": "iVBORw0KGgogc2FtcGxlIHNjcmVlbnNob3Q=", "metadata": null}, "steps": [{"index": 1, "action": {"action_type": "bash", "content": "mkdir -p notes && echo hello > notes/out.txt", "events": [], "x": null, "y": null, "to_x": null, "to_y": null, "path": null, "amount": 1}, "messages": [{"role": "system", "content": "You are a shell agent.", "token_ids": null, "logprobs": null}, {"role": "user", "content": "Write hello to notes/out.txt, then finish.\n$ ", "token_ids": null, "logprobs": null}, {"role": "assistant", "content": "mkdir -p notes && echo hello > notes/out.txt", "token_ids": [11, 12, 13], "logprobs": [-0.25, -0.5, -0.125]}], "observation": {"content": "", "step": 1, "done": false, "screenshot": null, "metadata": {"exit_code": 0}}, "applied": true, "reward": 0.5, "step_verifier": {"score": 0.5}, "error": null, "timestamp": "2026-10-01T12:00:01Z", "duration_ms": 40.0, "exit_code": 0}, {"index": 2, "action": {"action_type": "write_file", "content": "x", "events": [], "x": null, "y": null, "to_x": null, "to_y": null, "path": "notes/extra.txt", "amount": 1}, "messages": [{"role": "tool", "content": "exit 0", "token_ids": null, "logprobs": null}, {"role": "assistant", "content": "write notes/extra.txt", "token_ids": null, "logprobs": null}], "observation": null, "applied": false, "reward": null, "step_verifier": null, "error": "ActionError: Action 'write_file' not allowed here.", "timestamp": "2026-10-01T12:00:02Z", "duration_ms": 1.0, "exit_code": null}, {"index": 3, "action": {"action_type": "done", "content": "", "events": [], "x": null, "y": null, "to_x": null, "to_y": null, "path": null, "amount": 1}, "messages": [{"role": "user", "content": "write_file is not allowed.", "token_ids": null, "logprobs": null}, {"role": "assistant", "content": "done", "token_ids": null, "logprobs": null}], "observation": {"content": "", "step": 2, "done": true, "screenshot": null, "metadata": null}, "applied": true, "reward": 0.5, "step_verifier": {"score": 0.5}, "error": null, "timestamp": "2026-10-01T12:00:03Z", "duration_ms": 30.0, "exit_code": null}], "result": {"score": 1.0, "passed": true, "output": "notes/out.txt says hello", "stderr": "", "exit_code": 0, "execution_time_ms": 12.5, "sandbox_id": "local-session", "reward_components": null, "metrics": null, "seed": null, "truncated": false, "error_type": null, "cases": null}, "prior_results": [], "error": null, "finished_at": "2026-10-01T12:00:04Z", "turns": [[1], [2], [3]], "status": "complete", "has_verdict": true}} -{"trace_id": "trace0002", "env_name": "write_notes", "env_variant": 2, "agent": "react-shell", "model": "Qwen/Qwen3-8B", "provider": "local", "status": "incomplete", "has_verdict": false, "score": null, "passed": null, "steps": 1, "started_at": "2026-10-01T12:00:00+00:00", "trace": {"schema_version": "1.1", "trace_id": "trace0002", "task": "Write hello to notes/out.txt, then finish.", "provenance": {"env": {"name": "write_notes", "variant": 2, "seed": 7, "environment_sha256": "7f142979bcdf5748ac3baa872a5db2a58d701da9bb29b7451d747e8a3777b6f9", "task_sha256": "1bbf96220c257b1281f09efdf074dc9cb437d3f4401d77c008308bfd72fccc28", "verifier_sha256": "5d6ee51bb39cafa8308963062208847333565120aa1e94980872375cd011d158", "step_verifier_sha256": "e4a424cca79d7c48e6fd4bc2827e14523db51284f401d9c0321d43117878860b", "reference_sha256": "a8bad614127cbaaaaa67e14952ae0fc6c0dce71bdff0822d038556b7dfbe3431", "image": "python:3.12-slim@sha256:abababababababababababababababababababababababababababababababab", "image_digest": "sha256:abababababababababababababababababababababababababababababababab", "digest_unavailable": null, "snapshot": null}, "agent": {"name": "react-shell", "version": "1.2.0", "model": "Qwen/Qwen3-8B", "sampling": {"temperature": 0.7, "top_p": 0.95}}, "provider": "local", "deepgym_version": "0.4.1", "deepgym_commit": null, "deepgym_dirty": null, "started_at": "2026-10-01T12:00:00Z"}, "initial_observation": {"content": "$ ", "step": 0, "done": false, "screenshot": "iVBORw0KGgogc2FtcGxlIHNjcmVlbnNob3Q=", "metadata": null}, "steps": [{"index": 1, "action": {"action_type": "bash", "content": "mkdir -p notes && echo hello > notes/out.txt", "events": [], "x": null, "y": null, "to_x": null, "to_y": null, "path": null, "amount": 1}, "messages": [], "observation": {"content": "", "step": 1, "done": false, "screenshot": null, "metadata": {"exit_code": 0}}, "applied": true, "reward": 0.5, "step_verifier": {"score": 0.5}, "error": null, "timestamp": "2026-10-01T12:00:01Z", "duration_ms": 40.0, "exit_code": 0}], "result": null, "prior_results": [], "error": "SandboxError: sandbox vanished", "finished_at": null, "turns": [[1]], "status": "incomplete", "has_verdict": false}} -{"trace_id": "trace0003", "env_name": "write_notes", "env_variant": 2, "agent": "react-shell", "model": "Qwen/Qwen3-8B", "provider": "local", "status": "complete", "has_verdict": false, "score": null, "passed": null, "steps": 3, "started_at": "2026-10-01T12:00:00+00:00", "trace": {"schema_version": "1.1", "trace_id": "trace0003", "task": "Write hello to notes/out.txt, then finish.", "provenance": {"env": {"name": "write_notes", "variant": 2, "seed": 7, "environment_sha256": "7f142979bcdf5748ac3baa872a5db2a58d701da9bb29b7451d747e8a3777b6f9", "task_sha256": "1bbf96220c257b1281f09efdf074dc9cb437d3f4401d77c008308bfd72fccc28", "verifier_sha256": "5d6ee51bb39cafa8308963062208847333565120aa1e94980872375cd011d158", "step_verifier_sha256": "e4a424cca79d7c48e6fd4bc2827e14523db51284f401d9c0321d43117878860b", "reference_sha256": "a8bad614127cbaaaaa67e14952ae0fc6c0dce71bdff0822d038556b7dfbe3431", "image": "python:3.12-slim@sha256:abababababababababababababababababababababababababababababababab", "image_digest": "sha256:abababababababababababababababababababababababababababababababab", "digest_unavailable": null, "snapshot": null}, "agent": {"name": "react-shell", "version": "1.2.0", "model": "Qwen/Qwen3-8B", "sampling": {"temperature": 0.7, "top_p": 0.95}}, "provider": "local", "deepgym_version": "0.4.1", "deepgym_commit": null, "deepgym_dirty": null, "started_at": "2026-10-01T12:00:00Z"}, "initial_observation": {"content": "$ ", "step": 0, "done": false, "screenshot": "iVBORw0KGgogc2FtcGxlIHNjcmVlbnNob3Q=", "metadata": null}, "steps": [{"index": 1, "action": {"action_type": "bash", "content": "mkdir -p notes && echo hello > notes/out.txt", "events": [], "x": null, "y": null, "to_x": null, "to_y": null, "path": null, "amount": 1}, "messages": [{"role": "system", "content": "You are a shell agent.", "token_ids": null, "logprobs": null}, {"role": "user", "content": "Write hello to notes/out.txt, then finish.\n$ ", "token_ids": null, "logprobs": null}, {"role": "assistant", "content": "mkdir -p notes && echo hello > notes/out.txt", "token_ids": [11, 12, 13], "logprobs": [-0.25, -0.5, -0.125]}], "observation": {"content": "", "step": 1, "done": false, "screenshot": null, "metadata": {"exit_code": 0}}, "applied": true, "reward": 0.5, "step_verifier": {"score": 0.5}, "error": null, "timestamp": "2026-10-01T12:00:01Z", "duration_ms": 40.0, "exit_code": 0}, {"index": 2, "action": {"action_type": "write_file", "content": "x", "events": [], "x": null, "y": null, "to_x": null, "to_y": null, "path": "notes/extra.txt", "amount": 1}, "messages": [{"role": "tool", "content": "exit 0", "token_ids": null, "logprobs": null}, {"role": "assistant", "content": "write notes/extra.txt", "token_ids": null, "logprobs": null}], "observation": null, "applied": false, "reward": null, "step_verifier": null, "error": "ActionError: Action 'write_file' not allowed here.", "timestamp": "2026-10-01T12:00:02Z", "duration_ms": 1.0, "exit_code": null}, {"index": 3, "action": {"action_type": "done", "content": "", "events": [], "x": null, "y": null, "to_x": null, "to_y": null, "path": null, "amount": 1}, "messages": [{"role": "user", "content": "write_file is not allowed.", "token_ids": null, "logprobs": null}, {"role": "assistant", "content": "done", "token_ids": null, "logprobs": null}], "observation": {"content": "", "step": 2, "done": true, "screenshot": null, "metadata": null}, "applied": true, "reward": 0.5, "step_verifier": {"score": 0.5}, "error": null, "timestamp": "2026-10-01T12:00:03Z", "duration_ms": 30.0, "exit_code": null}], "result": {"score": 0.0, "passed": false, "output": "notes/out.txt says hello", "stderr": "", "exit_code": 2, "execution_time_ms": 12.5, "sandbox_id": "local-session", "reward_components": null, "metrics": null, "seed": null, "truncated": false, "error_type": "sandbox_error", "cases": null}, "prior_results": [], "error": null, "finished_at": "2026-10-01T12:00:04Z", "turns": [[1], [2], [3]], "status": "complete", "has_verdict": false}} +{"trace_id": "trace0001", "env_name": "write_notes", "env_variant": 2, "agent": "react-shell", "model": "Qwen/Qwen3-8B", "provider": "local", "status": "complete", "has_verdict": true, "score": 1.0, "passed": true, "steps": 3, "started_at": "2026-10-01T12:00:00+00:00", "trace": {"schema_version": "1.1", "trace_id": "trace0001", "task": "Write hello to notes/out.txt, then finish.", "provenance": {"env": {"name": "write_notes", "variant": 2, "seed": 7, "environment_sha256": "7f142979bcdf5748ac3baa872a5db2a58d701da9bb29b7451d747e8a3777b6f9", "task_sha256": "1bbf96220c257b1281f09efdf074dc9cb437d3f4401d77c008308bfd72fccc28", "verifier_sha256": "5d6ee51bb39cafa8308963062208847333565120aa1e94980872375cd011d158", "step_verifier_sha256": "e4a424cca79d7c48e6fd4bc2827e14523db51284f401d9c0321d43117878860b", "reference_sha256": "a8bad614127cbaaaaa67e14952ae0fc6c0dce71bdff0822d038556b7dfbe3431", "image": "python:3.12-slim@sha256:abababababababababababababababababababababababababababababababab", "image_digest": "sha256:abababababababababababababababababababababababababababababababab", "digest_unavailable": null, "snapshot": null}, "agent": {"name": "react-shell", "version": "1.2.0", "model": "Qwen/Qwen3-8B", "sampling": {"temperature": 0.7, "top_p": 0.95}}, "provider": "local", "deepgym_version": "0.4.1", "deepgym_commit": null, "deepgym_dirty": null, "started_at": "2026-10-01T12:00:00Z"}, "initial_observation": {"content": "$ ", "step": 0, "done": false, "screenshot": "iVBORw0KGgogc2FtcGxlIHNjcmVlbnNob3Q=", "metadata": null}, "steps": [{"index": 1, "action": {"action_type": "bash", "content": "mkdir -p notes && echo hello > notes/out.txt", "events": [], "x": null, "y": null, "to_x": null, "to_y": null, "path": null, "amount": 1, "tool": null, "arguments": {}}, "messages": [{"role": "system", "content": "You are a shell agent.", "token_ids": null, "logprobs": null}, {"role": "user", "content": "Write hello to notes/out.txt, then finish.\n$ ", "token_ids": null, "logprobs": null}, {"role": "assistant", "content": "mkdir -p notes && echo hello > notes/out.txt", "token_ids": [11, 12, 13], "logprobs": [-0.25, -0.5, -0.125]}], "observation": {"content": "", "step": 1, "done": false, "screenshot": null, "metadata": {"exit_code": 0}}, "applied": true, "reward": 0.5, "step_verifier": {"score": 0.5}, "error": null, "timestamp": "2026-10-01T12:00:01Z", "duration_ms": 40.0, "exit_code": 0}, {"index": 2, "action": {"action_type": "write_file", "content": "x", "events": [], "x": null, "y": null, "to_x": null, "to_y": null, "path": "notes/extra.txt", "amount": 1, "tool": null, "arguments": {}}, "messages": [{"role": "tool", "content": "exit 0", "token_ids": null, "logprobs": null}, {"role": "assistant", "content": "write notes/extra.txt", "token_ids": null, "logprobs": null}], "observation": null, "applied": false, "reward": null, "step_verifier": null, "error": "ActionError: Action 'write_file' not allowed here.", "timestamp": "2026-10-01T12:00:02Z", "duration_ms": 1.0, "exit_code": null}, {"index": 3, "action": {"action_type": "done", "content": "", "events": [], "x": null, "y": null, "to_x": null, "to_y": null, "path": null, "amount": 1, "tool": null, "arguments": {}}, "messages": [{"role": "user", "content": "write_file is not allowed.", "token_ids": null, "logprobs": null}, {"role": "assistant", "content": "done", "token_ids": null, "logprobs": null}], "observation": {"content": "", "step": 2, "done": true, "screenshot": null, "metadata": null}, "applied": true, "reward": 0.5, "step_verifier": {"score": 0.5}, "error": null, "timestamp": "2026-10-01T12:00:03Z", "duration_ms": 30.0, "exit_code": null}], "result": {"score": 1.0, "passed": true, "output": "notes/out.txt says hello", "stderr": "", "exit_code": 0, "execution_time_ms": 12.5, "sandbox_id": "local-session", "reward_components": null, "metrics": null, "seed": null, "truncated": false, "error_type": null, "cases": null}, "prior_results": [], "error": null, "finished_at": "2026-10-01T12:00:04Z", "turns": [[1], [2], [3]], "status": "complete", "has_verdict": true}} +{"trace_id": "trace0002", "env_name": "write_notes", "env_variant": 2, "agent": "react-shell", "model": "Qwen/Qwen3-8B", "provider": "local", "status": "incomplete", "has_verdict": false, "score": null, "passed": null, "steps": 1, "started_at": "2026-10-01T12:00:00+00:00", "trace": {"schema_version": "1.1", "trace_id": "trace0002", "task": "Write hello to notes/out.txt, then finish.", "provenance": {"env": {"name": "write_notes", "variant": 2, "seed": 7, "environment_sha256": "7f142979bcdf5748ac3baa872a5db2a58d701da9bb29b7451d747e8a3777b6f9", "task_sha256": "1bbf96220c257b1281f09efdf074dc9cb437d3f4401d77c008308bfd72fccc28", "verifier_sha256": "5d6ee51bb39cafa8308963062208847333565120aa1e94980872375cd011d158", "step_verifier_sha256": "e4a424cca79d7c48e6fd4bc2827e14523db51284f401d9c0321d43117878860b", "reference_sha256": "a8bad614127cbaaaaa67e14952ae0fc6c0dce71bdff0822d038556b7dfbe3431", "image": "python:3.12-slim@sha256:abababababababababababababababababababababababababababababababab", "image_digest": "sha256:abababababababababababababababababababababababababababababababab", "digest_unavailable": null, "snapshot": null}, "agent": {"name": "react-shell", "version": "1.2.0", "model": "Qwen/Qwen3-8B", "sampling": {"temperature": 0.7, "top_p": 0.95}}, "provider": "local", "deepgym_version": "0.4.1", "deepgym_commit": null, "deepgym_dirty": null, "started_at": "2026-10-01T12:00:00Z"}, "initial_observation": {"content": "$ ", "step": 0, "done": false, "screenshot": "iVBORw0KGgogc2FtcGxlIHNjcmVlbnNob3Q=", "metadata": null}, "steps": [{"index": 1, "action": {"action_type": "bash", "content": "mkdir -p notes && echo hello > notes/out.txt", "events": [], "x": null, "y": null, "to_x": null, "to_y": null, "path": null, "amount": 1, "tool": null, "arguments": {}}, "messages": [], "observation": {"content": "", "step": 1, "done": false, "screenshot": null, "metadata": {"exit_code": 0}}, "applied": true, "reward": 0.5, "step_verifier": {"score": 0.5}, "error": null, "timestamp": "2026-10-01T12:00:01Z", "duration_ms": 40.0, "exit_code": 0}], "result": null, "prior_results": [], "error": "SandboxError: sandbox vanished", "finished_at": null, "turns": [[1]], "status": "incomplete", "has_verdict": false}} +{"trace_id": "trace0003", "env_name": "write_notes", "env_variant": 2, "agent": "react-shell", "model": "Qwen/Qwen3-8B", "provider": "local", "status": "complete", "has_verdict": false, "score": null, "passed": null, "steps": 3, "started_at": "2026-10-01T12:00:00+00:00", "trace": {"schema_version": "1.1", "trace_id": "trace0003", "task": "Write hello to notes/out.txt, then finish.", "provenance": {"env": {"name": "write_notes", "variant": 2, "seed": 7, "environment_sha256": "7f142979bcdf5748ac3baa872a5db2a58d701da9bb29b7451d747e8a3777b6f9", "task_sha256": "1bbf96220c257b1281f09efdf074dc9cb437d3f4401d77c008308bfd72fccc28", "verifier_sha256": "5d6ee51bb39cafa8308963062208847333565120aa1e94980872375cd011d158", "step_verifier_sha256": "e4a424cca79d7c48e6fd4bc2827e14523db51284f401d9c0321d43117878860b", "reference_sha256": "a8bad614127cbaaaaa67e14952ae0fc6c0dce71bdff0822d038556b7dfbe3431", "image": "python:3.12-slim@sha256:abababababababababababababababababababababababababababababababab", "image_digest": "sha256:abababababababababababababababababababababababababababababababab", "digest_unavailable": null, "snapshot": null}, "agent": {"name": "react-shell", "version": "1.2.0", "model": "Qwen/Qwen3-8B", "sampling": {"temperature": 0.7, "top_p": 0.95}}, "provider": "local", "deepgym_version": "0.4.1", "deepgym_commit": null, "deepgym_dirty": null, "started_at": "2026-10-01T12:00:00Z"}, "initial_observation": {"content": "$ ", "step": 0, "done": false, "screenshot": "iVBORw0KGgogc2FtcGxlIHNjcmVlbnNob3Q=", "metadata": null}, "steps": [{"index": 1, "action": {"action_type": "bash", "content": "mkdir -p notes && echo hello > notes/out.txt", "events": [], "x": null, "y": null, "to_x": null, "to_y": null, "path": null, "amount": 1, "tool": null, "arguments": {}}, "messages": [{"role": "system", "content": "You are a shell agent.", "token_ids": null, "logprobs": null}, {"role": "user", "content": "Write hello to notes/out.txt, then finish.\n$ ", "token_ids": null, "logprobs": null}, {"role": "assistant", "content": "mkdir -p notes && echo hello > notes/out.txt", "token_ids": [11, 12, 13], "logprobs": [-0.25, -0.5, -0.125]}], "observation": {"content": "", "step": 1, "done": false, "screenshot": null, "metadata": {"exit_code": 0}}, "applied": true, "reward": 0.5, "step_verifier": {"score": 0.5}, "error": null, "timestamp": "2026-10-01T12:00:01Z", "duration_ms": 40.0, "exit_code": 0}, {"index": 2, "action": {"action_type": "write_file", "content": "x", "events": [], "x": null, "y": null, "to_x": null, "to_y": null, "path": "notes/extra.txt", "amount": 1, "tool": null, "arguments": {}}, "messages": [{"role": "tool", "content": "exit 0", "token_ids": null, "logprobs": null}, {"role": "assistant", "content": "write notes/extra.txt", "token_ids": null, "logprobs": null}], "observation": null, "applied": false, "reward": null, "step_verifier": null, "error": "ActionError: Action 'write_file' not allowed here.", "timestamp": "2026-10-01T12:00:02Z", "duration_ms": 1.0, "exit_code": null}, {"index": 3, "action": {"action_type": "done", "content": "", "events": [], "x": null, "y": null, "to_x": null, "to_y": null, "path": null, "amount": 1, "tool": null, "arguments": {}}, "messages": [{"role": "user", "content": "write_file is not allowed.", "token_ids": null, "logprobs": null}, {"role": "assistant", "content": "done", "token_ids": null, "logprobs": null}], "observation": {"content": "", "step": 2, "done": true, "screenshot": null, "metadata": null}, "applied": true, "reward": 0.5, "step_verifier": {"score": 0.5}, "error": null, "timestamp": "2026-10-01T12:00:03Z", "duration_ms": 30.0, "exit_code": null}], "result": {"score": 0.0, "passed": false, "output": "notes/out.txt says hello", "stderr": "", "exit_code": 2, "execution_time_ms": 12.5, "sandbox_id": "local-session", "reward_components": null, "metrics": null, "seed": null, "truncated": false, "error_type": "sandbox_error", "cases": null}, "prior_results": [], "error": null, "finished_at": "2026-10-01T12:00:04Z", "turns": [[1], [2], [3]], "status": "complete", "has_verdict": false}} diff --git a/deepgym/tests/golden/traces/traces.jsonl b/deepgym/tests/golden/traces/traces.jsonl index b9a6e2d..399d0e7 100644 --- a/deepgym/tests/golden/traces/traces.jsonl +++ b/deepgym/tests/golden/traces/traces.jsonl @@ -1,3 +1,3 @@ -{"schema_version":"1.1","trace_id":"trace0001","task":"Write hello to notes/out.txt, then finish.","provenance":{"env":{"name":"write_notes","variant":2,"seed":7,"environment_sha256":"7f142979bcdf5748ac3baa872a5db2a58d701da9bb29b7451d747e8a3777b6f9","task_sha256":"1bbf96220c257b1281f09efdf074dc9cb437d3f4401d77c008308bfd72fccc28","verifier_sha256":"5d6ee51bb39cafa8308963062208847333565120aa1e94980872375cd011d158","step_verifier_sha256":"e4a424cca79d7c48e6fd4bc2827e14523db51284f401d9c0321d43117878860b","reference_sha256":"a8bad614127cbaaaaa67e14952ae0fc6c0dce71bdff0822d038556b7dfbe3431","image":"python:3.12-slim@sha256:abababababababababababababababababababababababababababababababab","image_digest":"sha256:abababababababababababababababababababababababababababababababab","digest_unavailable":null,"snapshot":null},"agent":{"name":"react-shell","version":"1.2.0","model":"Qwen/Qwen3-8B","sampling":{"temperature":0.7,"top_p":0.95}},"provider":"local","deepgym_version":"0.4.1","deepgym_commit":null,"deepgym_dirty":null,"started_at":"2026-10-01T12:00:00Z"},"initial_observation":{"content":"$ ","step":0,"done":false,"screenshot":"iVBORw0KGgogc2FtcGxlIHNjcmVlbnNob3Q=","metadata":null},"steps":[{"index":1,"action":{"action_type":"bash","content":"mkdir -p notes && echo hello > notes/out.txt","events":[],"x":null,"y":null,"to_x":null,"to_y":null,"path":null,"amount":1},"messages":[{"role":"system","content":"You are a shell agent.","token_ids":null,"logprobs":null},{"role":"user","content":"Write hello to notes/out.txt, then finish.\n$ ","token_ids":null,"logprobs":null},{"role":"assistant","content":"mkdir -p notes && echo hello > notes/out.txt","token_ids":[11,12,13],"logprobs":[-0.25,-0.5,-0.125]}],"observation":{"content":"","step":1,"done":false,"screenshot":null,"metadata":{"exit_code":0}},"applied":true,"reward":0.5,"step_verifier":{"score":0.5},"error":null,"timestamp":"2026-10-01T12:00:01Z","duration_ms":40.0,"exit_code":0},{"index":2,"action":{"action_type":"write_file","content":"x","events":[],"x":null,"y":null,"to_x":null,"to_y":null,"path":"notes/extra.txt","amount":1},"messages":[{"role":"tool","content":"exit 0","token_ids":null,"logprobs":null},{"role":"assistant","content":"write notes/extra.txt","token_ids":null,"logprobs":null}],"observation":null,"applied":false,"reward":null,"step_verifier":null,"error":"ActionError: Action 'write_file' not allowed here.","timestamp":"2026-10-01T12:00:02Z","duration_ms":1.0,"exit_code":null},{"index":3,"action":{"action_type":"done","content":"","events":[],"x":null,"y":null,"to_x":null,"to_y":null,"path":null,"amount":1},"messages":[{"role":"user","content":"write_file is not allowed.","token_ids":null,"logprobs":null},{"role":"assistant","content":"done","token_ids":null,"logprobs":null}],"observation":{"content":"","step":2,"done":true,"screenshot":null,"metadata":null},"applied":true,"reward":0.5,"step_verifier":{"score":0.5},"error":null,"timestamp":"2026-10-01T12:00:03Z","duration_ms":30.0,"exit_code":null}],"result":{"score":1.0,"passed":true,"output":"notes/out.txt says hello","stderr":"","exit_code":0,"execution_time_ms":12.5,"sandbox_id":"local-session","reward_components":null,"metrics":null,"seed":null,"truncated":false,"error_type":null,"cases":null},"prior_results":[],"error":null,"finished_at":"2026-10-01T12:00:04Z","turns":[[1],[2],[3]],"status":"complete","has_verdict":true} -{"schema_version":"1.1","trace_id":"trace0002","task":"Write hello to notes/out.txt, then finish.","provenance":{"env":{"name":"write_notes","variant":2,"seed":7,"environment_sha256":"7f142979bcdf5748ac3baa872a5db2a58d701da9bb29b7451d747e8a3777b6f9","task_sha256":"1bbf96220c257b1281f09efdf074dc9cb437d3f4401d77c008308bfd72fccc28","verifier_sha256":"5d6ee51bb39cafa8308963062208847333565120aa1e94980872375cd011d158","step_verifier_sha256":"e4a424cca79d7c48e6fd4bc2827e14523db51284f401d9c0321d43117878860b","reference_sha256":"a8bad614127cbaaaaa67e14952ae0fc6c0dce71bdff0822d038556b7dfbe3431","image":"python:3.12-slim@sha256:abababababababababababababababababababababababababababababababab","image_digest":"sha256:abababababababababababababababababababababababababababababababab","digest_unavailable":null,"snapshot":null},"agent":{"name":"react-shell","version":"1.2.0","model":"Qwen/Qwen3-8B","sampling":{"temperature":0.7,"top_p":0.95}},"provider":"local","deepgym_version":"0.4.1","deepgym_commit":null,"deepgym_dirty":null,"started_at":"2026-10-01T12:00:00Z"},"initial_observation":{"content":"$ ","step":0,"done":false,"screenshot":"iVBORw0KGgogc2FtcGxlIHNjcmVlbnNob3Q=","metadata":null},"steps":[{"index":1,"action":{"action_type":"bash","content":"mkdir -p notes && echo hello > notes/out.txt","events":[],"x":null,"y":null,"to_x":null,"to_y":null,"path":null,"amount":1},"messages":[],"observation":{"content":"","step":1,"done":false,"screenshot":null,"metadata":{"exit_code":0}},"applied":true,"reward":0.5,"step_verifier":{"score":0.5},"error":null,"timestamp":"2026-10-01T12:00:01Z","duration_ms":40.0,"exit_code":0}],"result":null,"prior_results":[],"error":"SandboxError: sandbox vanished","finished_at":null,"turns":[[1]],"status":"incomplete","has_verdict":false} -{"schema_version":"1.1","trace_id":"trace0003","task":"Write hello to notes/out.txt, then finish.","provenance":{"env":{"name":"write_notes","variant":2,"seed":7,"environment_sha256":"7f142979bcdf5748ac3baa872a5db2a58d701da9bb29b7451d747e8a3777b6f9","task_sha256":"1bbf96220c257b1281f09efdf074dc9cb437d3f4401d77c008308bfd72fccc28","verifier_sha256":"5d6ee51bb39cafa8308963062208847333565120aa1e94980872375cd011d158","step_verifier_sha256":"e4a424cca79d7c48e6fd4bc2827e14523db51284f401d9c0321d43117878860b","reference_sha256":"a8bad614127cbaaaaa67e14952ae0fc6c0dce71bdff0822d038556b7dfbe3431","image":"python:3.12-slim@sha256:abababababababababababababababababababababababababababababababab","image_digest":"sha256:abababababababababababababababababababababababababababababababab","digest_unavailable":null,"snapshot":null},"agent":{"name":"react-shell","version":"1.2.0","model":"Qwen/Qwen3-8B","sampling":{"temperature":0.7,"top_p":0.95}},"provider":"local","deepgym_version":"0.4.1","deepgym_commit":null,"deepgym_dirty":null,"started_at":"2026-10-01T12:00:00Z"},"initial_observation":{"content":"$ ","step":0,"done":false,"screenshot":"iVBORw0KGgogc2FtcGxlIHNjcmVlbnNob3Q=","metadata":null},"steps":[{"index":1,"action":{"action_type":"bash","content":"mkdir -p notes && echo hello > notes/out.txt","events":[],"x":null,"y":null,"to_x":null,"to_y":null,"path":null,"amount":1},"messages":[{"role":"system","content":"You are a shell agent.","token_ids":null,"logprobs":null},{"role":"user","content":"Write hello to notes/out.txt, then finish.\n$ ","token_ids":null,"logprobs":null},{"role":"assistant","content":"mkdir -p notes && echo hello > notes/out.txt","token_ids":[11,12,13],"logprobs":[-0.25,-0.5,-0.125]}],"observation":{"content":"","step":1,"done":false,"screenshot":null,"metadata":{"exit_code":0}},"applied":true,"reward":0.5,"step_verifier":{"score":0.5},"error":null,"timestamp":"2026-10-01T12:00:01Z","duration_ms":40.0,"exit_code":0},{"index":2,"action":{"action_type":"write_file","content":"x","events":[],"x":null,"y":null,"to_x":null,"to_y":null,"path":"notes/extra.txt","amount":1},"messages":[{"role":"tool","content":"exit 0","token_ids":null,"logprobs":null},{"role":"assistant","content":"write notes/extra.txt","token_ids":null,"logprobs":null}],"observation":null,"applied":false,"reward":null,"step_verifier":null,"error":"ActionError: Action 'write_file' not allowed here.","timestamp":"2026-10-01T12:00:02Z","duration_ms":1.0,"exit_code":null},{"index":3,"action":{"action_type":"done","content":"","events":[],"x":null,"y":null,"to_x":null,"to_y":null,"path":null,"amount":1},"messages":[{"role":"user","content":"write_file is not allowed.","token_ids":null,"logprobs":null},{"role":"assistant","content":"done","token_ids":null,"logprobs":null}],"observation":{"content":"","step":2,"done":true,"screenshot":null,"metadata":null},"applied":true,"reward":0.5,"step_verifier":{"score":0.5},"error":null,"timestamp":"2026-10-01T12:00:03Z","duration_ms":30.0,"exit_code":null}],"result":{"score":0.0,"passed":false,"output":"notes/out.txt says hello","stderr":"","exit_code":2,"execution_time_ms":12.5,"sandbox_id":"local-session","reward_components":null,"metrics":null,"seed":null,"truncated":false,"error_type":"sandbox_error","cases":null},"prior_results":[],"error":null,"finished_at":"2026-10-01T12:00:04Z","turns":[[1],[2],[3]],"status":"complete","has_verdict":false} +{"schema_version":"1.1","trace_id":"trace0001","task":"Write hello to notes/out.txt, then finish.","provenance":{"env":{"name":"write_notes","variant":2,"seed":7,"environment_sha256":"7f142979bcdf5748ac3baa872a5db2a58d701da9bb29b7451d747e8a3777b6f9","task_sha256":"1bbf96220c257b1281f09efdf074dc9cb437d3f4401d77c008308bfd72fccc28","verifier_sha256":"5d6ee51bb39cafa8308963062208847333565120aa1e94980872375cd011d158","step_verifier_sha256":"e4a424cca79d7c48e6fd4bc2827e14523db51284f401d9c0321d43117878860b","reference_sha256":"a8bad614127cbaaaaa67e14952ae0fc6c0dce71bdff0822d038556b7dfbe3431","image":"python:3.12-slim@sha256:abababababababababababababababababababababababababababababababab","image_digest":"sha256:abababababababababababababababababababababababababababababababab","digest_unavailable":null,"snapshot":null},"agent":{"name":"react-shell","version":"1.2.0","model":"Qwen/Qwen3-8B","sampling":{"temperature":0.7,"top_p":0.95}},"provider":"local","deepgym_version":"0.4.1","deepgym_commit":null,"deepgym_dirty":null,"started_at":"2026-10-01T12:00:00Z"},"initial_observation":{"content":"$ ","step":0,"done":false,"screenshot":"iVBORw0KGgogc2FtcGxlIHNjcmVlbnNob3Q=","metadata":null},"steps":[{"index":1,"action":{"action_type":"bash","content":"mkdir -p notes && echo hello > notes/out.txt","events":[],"x":null,"y":null,"to_x":null,"to_y":null,"path":null,"amount":1,"tool":null,"arguments":{}},"messages":[{"role":"system","content":"You are a shell agent.","token_ids":null,"logprobs":null},{"role":"user","content":"Write hello to notes/out.txt, then finish.\n$ ","token_ids":null,"logprobs":null},{"role":"assistant","content":"mkdir -p notes && echo hello > notes/out.txt","token_ids":[11,12,13],"logprobs":[-0.25,-0.5,-0.125]}],"observation":{"content":"","step":1,"done":false,"screenshot":null,"metadata":{"exit_code":0}},"applied":true,"reward":0.5,"step_verifier":{"score":0.5},"error":null,"timestamp":"2026-10-01T12:00:01Z","duration_ms":40.0,"exit_code":0},{"index":2,"action":{"action_type":"write_file","content":"x","events":[],"x":null,"y":null,"to_x":null,"to_y":null,"path":"notes/extra.txt","amount":1,"tool":null,"arguments":{}},"messages":[{"role":"tool","content":"exit 0","token_ids":null,"logprobs":null},{"role":"assistant","content":"write notes/extra.txt","token_ids":null,"logprobs":null}],"observation":null,"applied":false,"reward":null,"step_verifier":null,"error":"ActionError: Action 'write_file' not allowed here.","timestamp":"2026-10-01T12:00:02Z","duration_ms":1.0,"exit_code":null},{"index":3,"action":{"action_type":"done","content":"","events":[],"x":null,"y":null,"to_x":null,"to_y":null,"path":null,"amount":1,"tool":null,"arguments":{}},"messages":[{"role":"user","content":"write_file is not allowed.","token_ids":null,"logprobs":null},{"role":"assistant","content":"done","token_ids":null,"logprobs":null}],"observation":{"content":"","step":2,"done":true,"screenshot":null,"metadata":null},"applied":true,"reward":0.5,"step_verifier":{"score":0.5},"error":null,"timestamp":"2026-10-01T12:00:03Z","duration_ms":30.0,"exit_code":null}],"result":{"score":1.0,"passed":true,"output":"notes/out.txt says hello","stderr":"","exit_code":0,"execution_time_ms":12.5,"sandbox_id":"local-session","reward_components":null,"metrics":null,"seed":null,"truncated":false,"error_type":null,"cases":null},"prior_results":[],"error":null,"finished_at":"2026-10-01T12:00:04Z","turns":[[1],[2],[3]],"status":"complete","has_verdict":true} +{"schema_version":"1.1","trace_id":"trace0002","task":"Write hello to notes/out.txt, then finish.","provenance":{"env":{"name":"write_notes","variant":2,"seed":7,"environment_sha256":"7f142979bcdf5748ac3baa872a5db2a58d701da9bb29b7451d747e8a3777b6f9","task_sha256":"1bbf96220c257b1281f09efdf074dc9cb437d3f4401d77c008308bfd72fccc28","verifier_sha256":"5d6ee51bb39cafa8308963062208847333565120aa1e94980872375cd011d158","step_verifier_sha256":"e4a424cca79d7c48e6fd4bc2827e14523db51284f401d9c0321d43117878860b","reference_sha256":"a8bad614127cbaaaaa67e14952ae0fc6c0dce71bdff0822d038556b7dfbe3431","image":"python:3.12-slim@sha256:abababababababababababababababababababababababababababababababab","image_digest":"sha256:abababababababababababababababababababababababababababababababab","digest_unavailable":null,"snapshot":null},"agent":{"name":"react-shell","version":"1.2.0","model":"Qwen/Qwen3-8B","sampling":{"temperature":0.7,"top_p":0.95}},"provider":"local","deepgym_version":"0.4.1","deepgym_commit":null,"deepgym_dirty":null,"started_at":"2026-10-01T12:00:00Z"},"initial_observation":{"content":"$ ","step":0,"done":false,"screenshot":"iVBORw0KGgogc2FtcGxlIHNjcmVlbnNob3Q=","metadata":null},"steps":[{"index":1,"action":{"action_type":"bash","content":"mkdir -p notes && echo hello > notes/out.txt","events":[],"x":null,"y":null,"to_x":null,"to_y":null,"path":null,"amount":1,"tool":null,"arguments":{}},"messages":[],"observation":{"content":"","step":1,"done":false,"screenshot":null,"metadata":{"exit_code":0}},"applied":true,"reward":0.5,"step_verifier":{"score":0.5},"error":null,"timestamp":"2026-10-01T12:00:01Z","duration_ms":40.0,"exit_code":0}],"result":null,"prior_results":[],"error":"SandboxError: sandbox vanished","finished_at":null,"turns":[[1]],"status":"incomplete","has_verdict":false} +{"schema_version":"1.1","trace_id":"trace0003","task":"Write hello to notes/out.txt, then finish.","provenance":{"env":{"name":"write_notes","variant":2,"seed":7,"environment_sha256":"7f142979bcdf5748ac3baa872a5db2a58d701da9bb29b7451d747e8a3777b6f9","task_sha256":"1bbf96220c257b1281f09efdf074dc9cb437d3f4401d77c008308bfd72fccc28","verifier_sha256":"5d6ee51bb39cafa8308963062208847333565120aa1e94980872375cd011d158","step_verifier_sha256":"e4a424cca79d7c48e6fd4bc2827e14523db51284f401d9c0321d43117878860b","reference_sha256":"a8bad614127cbaaaaa67e14952ae0fc6c0dce71bdff0822d038556b7dfbe3431","image":"python:3.12-slim@sha256:abababababababababababababababababababababababababababababababab","image_digest":"sha256:abababababababababababababababababababababababababababababababab","digest_unavailable":null,"snapshot":null},"agent":{"name":"react-shell","version":"1.2.0","model":"Qwen/Qwen3-8B","sampling":{"temperature":0.7,"top_p":0.95}},"provider":"local","deepgym_version":"0.4.1","deepgym_commit":null,"deepgym_dirty":null,"started_at":"2026-10-01T12:00:00Z"},"initial_observation":{"content":"$ ","step":0,"done":false,"screenshot":"iVBORw0KGgogc2FtcGxlIHNjcmVlbnNob3Q=","metadata":null},"steps":[{"index":1,"action":{"action_type":"bash","content":"mkdir -p notes && echo hello > notes/out.txt","events":[],"x":null,"y":null,"to_x":null,"to_y":null,"path":null,"amount":1,"tool":null,"arguments":{}},"messages":[{"role":"system","content":"You are a shell agent.","token_ids":null,"logprobs":null},{"role":"user","content":"Write hello to notes/out.txt, then finish.\n$ ","token_ids":null,"logprobs":null},{"role":"assistant","content":"mkdir -p notes && echo hello > notes/out.txt","token_ids":[11,12,13],"logprobs":[-0.25,-0.5,-0.125]}],"observation":{"content":"","step":1,"done":false,"screenshot":null,"metadata":{"exit_code":0}},"applied":true,"reward":0.5,"step_verifier":{"score":0.5},"error":null,"timestamp":"2026-10-01T12:00:01Z","duration_ms":40.0,"exit_code":0},{"index":2,"action":{"action_type":"write_file","content":"x","events":[],"x":null,"y":null,"to_x":null,"to_y":null,"path":"notes/extra.txt","amount":1,"tool":null,"arguments":{}},"messages":[{"role":"tool","content":"exit 0","token_ids":null,"logprobs":null},{"role":"assistant","content":"write notes/extra.txt","token_ids":null,"logprobs":null}],"observation":null,"applied":false,"reward":null,"step_verifier":null,"error":"ActionError: Action 'write_file' not allowed here.","timestamp":"2026-10-01T12:00:02Z","duration_ms":1.0,"exit_code":null},{"index":3,"action":{"action_type":"done","content":"","events":[],"x":null,"y":null,"to_x":null,"to_y":null,"path":null,"amount":1,"tool":null,"arguments":{}},"messages":[{"role":"user","content":"write_file is not allowed.","token_ids":null,"logprobs":null},{"role":"assistant","content":"done","token_ids":null,"logprobs":null}],"observation":{"content":"","step":2,"done":true,"screenshot":null,"metadata":null},"applied":true,"reward":0.5,"step_verifier":{"score":0.5},"error":null,"timestamp":"2026-10-01T12:00:03Z","duration_ms":30.0,"exit_code":null}],"result":{"score":0.0,"passed":false,"output":"notes/out.txt says hello","stderr":"","exit_code":2,"execution_time_ms":12.5,"sandbox_id":"local-session","reward_components":null,"metrics":null,"seed":null,"truncated":false,"error_type":"sandbox_error","cases":null},"prior_results":[],"error":null,"finished_at":"2026-10-01T12:00:04Z","turns":[[1],[2],[3]],"status":"complete","has_verdict":false} diff --git a/deepgym/tests/http_app.py b/deepgym/tests/http_app.py new file mode 100644 index 0000000..e9797b1 --- /dev/null +++ b/deepgym/tests/http_app.py @@ -0,0 +1,125 @@ +"""A loopback HTTP app for API tool tests: real sockets, no sandbox, no provider call.""" + +import json +import threading +import time +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + +from deepgym.models import MultiTurnEnvironment +from deepgym.session import LocalBackend, Session +from deepgym.tools import HttpRoute, HttpTool + +API_KEY = 'local-app-key-not-a-secret' +BIG = 'x' * 300_000 +"""A field long enough that any clipping of a response body would show.""" + +ROUTES = ( + HttpRoute(method='GET', path='/rest/bug/{id}'), + HttpRoute(method='POST', path='/rest/bug/{id}/comment'), + HttpRoute(method='GET', path='/rest/done'), + HttpRoute(method='GET', path='/rest/redirect'), + HttpRoute(method='POST', path='/rest/lost'), + HttpRoute(method='GET', path='/rest/lost'), + HttpRoute(method='POST', path='/rest/slow'), + HttpRoute(method='GET', path='/rest/binary'), +) + + +class App: + """A loopback server that records every request and answers a few fixed routes.""" + + def __init__(self, redirect_to: str = 'http://127.0.0.1:9/') -> None: + self.requests: list[dict] = [] + self.redirect_to = redirect_to + app = self + + class Handler(BaseHTTPRequestHandler): + def log_message(self, *args: object) -> None: + pass + + def do_GET(self) -> None: + self._handle() + + def do_POST(self) -> None: + self._handle() + + def _handle(self) -> None: + length = int(self.headers.get('Content-Length') or 0) + body = self.rfile.read(length).decode('utf-8') + app.requests.append( + { + 'method': self.command, + 'path': self.path, + 'headers': {name.lower(): value for name, value in self.headers.items()}, + 'body': body, + } + ) + route = self.path.split('?')[0] + if route.endswith('/lost'): + self.close_connection = True # the request arrived; the response never does + return + if route.endswith('/slow'): + time.sleep(3) + if route.endswith('/redirect'): + self._send(302, b'', {'Location': app.redirect_to}) + elif route.endswith('/done'): + self._send(200, b'DONE\n{"done": true}\n', {'Content-Type': 'text/plain'}) + elif route.endswith('/binary'): + self._send(200, b'\xff\x00\xfe', {}) + elif route.endswith('/comment'): + self._send(201, json.dumps({'id': 7}).encode(), {}) + elif route.startswith('/rest/bug/'): + bug = {'id': route.rsplit('/', 1)[1], 'summary': BIG} + link = {'Link': f'<{app.url}/rest/bug?page=2>; rel="next"'} + self._send(200, json.dumps({'bugs': [bug]}).encode(), link) + else: + self._send(404, b'{"error": "not found"}', {}) + + def _send(self, status: int, body: bytes, headers: dict[str, str]) -> None: + self.send_response(status) + for name, value in headers.items(): + self.send_header(name, value) + self.send_header('Content-Length', str(len(body))) + self.end_headers() + self.wfile.write(body) + + self.server = ThreadingHTTPServer(('127.0.0.1', 0), Handler) + self.server.daemon_threads = True + threading.Thread(target=self.server.serve_forever, daemon=True).start() + + @property + def url(self) -> str: + """The app's base URL.""" + return f'http://127.0.0.1:{self.server.server_address[1]}' + + def close(self) -> None: + """Stop serving.""" + self.server.shutdown() + self.server.server_close() + + +def tool(base_url: str) -> HttpTool: + """The app's API tool, sending the app key with every request.""" + return HttpTool( + name='tracker', + description='Bug tracker REST API.', + base_url=base_url, + routes=ROUTES, + headers={'X-API-KEY': API_KEY}, + ) + + +def app_env(base_url: str, **settings: object) -> MultiTurnEnvironment: + """An environment whose only tools call the app, with no step limit.""" + return MultiTurnEnvironment( + task='Triage bug 1.', + tools=(tool(base_url),), + allowed_actions=frozenset({'tool', 'done'}), + max_steps=None, + **settings, + ) + + +def app_session(base_url: str, **settings: object) -> Session: + """An unstarted local session of :func:`app_env`.""" + return Session(app_env(base_url, **settings), LocalBackend()) diff --git a/deepgym/tests/test_tools.py b/deepgym/tests/test_tools.py new file mode 100644 index 0000000..7d098db --- /dev/null +++ b/deepgym/tests/test_tools.py @@ -0,0 +1,301 @@ +"""Tests for declared HTTP API tools run as Session tool actions, against a loopback app.""" + +import json +from pathlib import Path + +import pytest +from pydantic import ValidationError + +from deepgym.exceptions import ActionError +from deepgym.models import Action, MultiTurnEnvironment +from deepgym.qa.agents import do_everything, random_agent +from deepgym.sandboxes import ExecResult +from deepgym.session import Session +from deepgym.tools import HttpRoute, HttpTool, run_command +from tests.http_app import API_KEY, BIG, App, app_env, app_session, tool + + +def call(path: str, method: str = 'GET', **arguments: object) -> Action: + return Action( + action_type='tool', tool='tracker', arguments={'method': method, 'path': path, **arguments} + ) + + +def record(session: Session, action: Action) -> dict: + return json.loads(session.step(action).observation.content) + + +def test_call_returns_the_full_request_and_response(app: App) -> None: + with app_session(app.url) as session: + session.start() + + step = session.step(call('/rest/bug/1', query={'include': ['id', 'summary'], 'n': 2})) + + result = json.loads(step.observation.content) + assert step.observation.metadata == {'exit_code': 0} + assert result['request'] == { + 'method': 'GET', + 'url': f'{app.url}/rest/bug/1?include=id&include=summary&n=2', + 'headers': {'X-API-KEY': '[redacted]'}, + } + response = result['response'] + assert response['status'] == 200 + assert json.loads(response['body'])['bugs'][0]['summary'] == BIG # never clipped + assert ['Link', f'<{app.url}/rest/bug?page=2>; rel="next"'] in response['headers'] + assert result['elapsed_ms'] >= 0 + assert API_KEY not in step.observation.content + (seen,) = app.requests + assert seen['headers']['x-api-key'] == API_KEY + assert seen['path'] == '/rest/bug/1?include=id&include=summary&n=2' + assert [a.action_type for _, a in session.trajectory.steps] == ['tool'] + + +def test_write_sends_its_json_body(app: App) -> None: + with app_session(app.url) as session: + session.start() + + result = record(session, call('/rest/bug/1/comment', 'POST', body={'comment': 'ok'})) + + assert result['response']['status'] == 201 + assert result['request']['body'] == {'comment': 'ok'} + assert result['request']['headers']['Content-Type'] == 'application/json' + assert json.loads(app.requests[0]['body']) == {'comment': 'ok'} + + +def test_binary_body_comes_back_whole_as_base64(app: App) -> None: + with app_session(app.url) as session: + session.start() + assert record(session, call('/rest/binary'))['response']['body_base64'] == '/wD+' + + +@pytest.mark.parametrize( + 'arguments', + [ + {'method': 'GET', 'path': 'http://evil.example/rest/bug/1'}, + {'method': 'GET', 'path': '//evil.example/rest/bug/1'}, + {'method': 'GET', 'path': '/rest/bug/1?x=1'}, + {'method': 'GET', 'path': '/rest/bug/1#frag'}, + {'method': 'GET', 'path': '/rest/bug/../../admin'}, + {'method': 'GET', 'path': '/rest/bug/%2e%2e'}, + {'method': 'GET', 'path': '/rest/bug/a%2F..%2F..%2Fadmin'}, + {'method': 'GET', 'path': '/rest/bug/a%5Cb'}, + {'method': 'GET', 'path': '/rest/bug/1 2'}, + {'method': 'GET', 'path': '/admin/run_script'}, + {'method': 'GET', 'path': '/rest/bug/1/'}, + {'method': 'DELETE', 'path': '/rest/bug/1'}, + {'method': 'GET', 'path': '/rest/bug/1', 'body': {'a': 1}}, + {'method': 'POST', 'path': '/rest/bug/1/comment', 'body': 'text'}, + {'method': 'GET', 'path': '/rest/bug/1', 'headers': {'Host': 'evil.example'}}, + {'method': 'GET', 'path': '/rest/bug/1', 'url': 'http://evil.example/'}, + {'method': 'GET', 'path': '/rest/bug/1', 'query': {'q': {'nested': 1}}}, + {'method': 'GET'}, + ], +) +def test_off_origin_or_undeclared_calls_are_rejected_unsent(app: App, arguments: dict) -> None: + with app_session(app.url) as session: + session.start() + + with pytest.raises(ActionError, match='rejected the call'): + session.step(Action(action_type='tool', tool='tracker', arguments=arguments)) + + assert app.requests == [] + assert not session.done + assert session.trajectory.steps == [] + + +def test_unknown_tool_is_rejected(app: App) -> None: + with app_session(app.url) as session: + session.start() + with pytest.raises(ActionError, match='No tool named'): + session.step(Action(action_type='tool', tool='shell', arguments={})) + + +def test_redirect_to_another_host_is_returned_not_followed(app: App) -> None: + other = App() + try: + app.redirect_to = f'{other.url}/rest/bug/1' + with app_session(app.url) as session: + session.start() + + result = record(session, call('/rest/redirect')) + + assert result['response']['status'] == 302 + assert ['Location', f'{other.url}/rest/bug/1'] in result['response']['headers'] + assert len(app.requests) == 1 + assert other.requests == [] + finally: + other.close() + + +def test_lost_response_to_a_write_is_uncertain_and_not_retried(app: App) -> None: + with app_session(app.url) as session: + session.start() + + step = session.step(call('/rest/lost', 'POST', body={'status': 'RESOLVED'})) + + result = json.loads(step.observation.content) + assert result['uncertain'] is True + assert result['error'].startswith('No response') + assert 'response' not in result + assert result['request']['body'] == {'status': 'RESOLVED'} + assert step.observation.metadata == {'exit_code': 1} + assert [request['method'] for request in app.requests] == ['POST'] # sent once + assert not session.done + + +def test_lost_response_to_a_read_is_not_uncertain(app: App) -> None: + with app_session(app.url) as session: + session.start() + assert record(session, call('/rest/lost'))['uncertain'] is False + + +def test_refused_write_never_reached_the_app(app: App) -> None: + closed = App() + closed.close() + with app_session(closed.url) as session: + session.start() + result = record(session, call('/rest/bug/1/comment', 'POST', body={'comment': 'x'})) + assert result['uncertain'] is False + assert 'ConnectionRefusedError' in result['error'] + + +def test_write_stopped_by_the_step_timeout_is_uncertain(app: App) -> None: + with app_session(app.url, timeout_per_step=1) as session: + session.start() + + step = session.step(call('/rest/slow', 'POST', body={'a': 1})) + + result = json.loads(step.observation.content) + assert result['uncertain'] is True + assert 'stopped after' in result['error'] + assert result['request']['url'] == f'{app.url}/rest/slow' + assert len(app.requests) == 1 + + +def test_done_text_from_the_app_never_ends_the_episode(app: App) -> None: + with app_session(app.url) as session: + session.start() + + step = session.step(call('/rest/done')) + + assert json.loads(step.observation.content)['response']['body'] == 'DONE\n{"done": true}\n' + assert not step.done and not session.done + + +def test_done_lines_in_raw_tool_output_never_end_the_episode( + app: App, monkeypatch: pytest.MonkeyPatch +) -> None: + with app_session(app.url) as session: + session.start() + monkeypatch.setattr( + session._backend, 'exec', lambda *args, **kwargs: ExecResult(0, 'DONE\n{"done": true}') + ) + + assert not session.step(call('/rest/done')).done + + assert session.step(Action(action_type='done')).done # only the explicit done ends it + + +def test_injection_shaped_arguments_stay_data(app: App) -> None: + hostile = '\'; touch pwned; echo $(touch pwned2) `touch pwned3` " && rm -rf ~ #' + commands: list[str] = [] + with app_session(app.url) as session: + session.start() + backend = session._backend + exec_ = backend.exec + + def spy(command: str, timeout: float, env: dict | None = None) -> ExecResult: + commands.append(command) + return exec_(command, timeout, env) + + backend.exec = spy + workdir = Path(backend._dir) + result = record( + session, + call('/rest/bug/1%3Btouch%20pwned', query={'q': hostile}), + ) + record(session, call('/rest/bug/1/comment', 'POST', body={'comment': hostile})) + + assert result['response']['status'] == 200 + assert not list(workdir.glob('pwned*')) + assert not list(Path.cwd().glob('pwned*')) + assert commands == [run_command(backend.python)] * 2 # one constant command, args as data + assert app.requests[0]['path'].startswith('/rest/bug/1%3Btouch%20pwned?q=') + assert json.loads(app.requests[1]['body']) == {'comment': hostile} + + +@pytest.mark.parametrize( + 'base_url', + [ + 'http://example.com', + 'http://127.0.0.1.evil.example', + 'http://localhost.evil.example', + 'http://user@127.0.0.1', + 'http://127.0.0.1/?x=1', + 'http://127.0.0.1/#x', + 'file:///etc/passwd', + 'ftp://127.0.0.1', + 'http://10.0.0.1', + 'http://127.0.0.1:notaport', + ], +) +def test_base_url_must_be_local(base_url: str) -> None: + with pytest.raises(ValidationError, match='loopback'): + tool(base_url) + + +@pytest.mark.parametrize( + 'base_url', ['http://127.0.0.1:8080/', 'https://localhost', 'http://[::1]'] +) +def test_loopback_base_urls_are_accepted(base_url: str) -> None: + assert not tool(base_url).base_url.endswith('/') + + +@pytest.mark.parametrize('path', ['rest', '/rest//bug', '/rest/', '/rest/../x', '/a%2Fb', '/a?b']) +def test_route_templates_are_literal_segments_and_placeholders(path: str) -> None: + with pytest.raises(ValidationError): + HttpRoute(method='GET', path=path) + + +def test_environment_tool_declarations_are_checked() -> None: + with pytest.raises(ValidationError, match='tool and done'): + MultiTurnEnvironment(task='t', tools=(tool('http://127.0.0.1'),)) + with pytest.raises(ValidationError, match='need declared tools'): + MultiTurnEnvironment(task='t', allowed_actions=frozenset({'tool', 'done'})) + local = tool('http://127.0.0.1') + with pytest.raises(ValidationError, match='unique'): + MultiTurnEnvironment( + task='t', tools=(local, local), allowed_actions=frozenset({'tool', 'done'}) + ) + with pytest.raises(ValidationError, match='reserved'): + HttpTool(name='done', description='d', base_url=local.base_url, routes=local.routes) + with pytest.raises(ValidationError, match='tool needs'): + Action(action_type='tool') + + +def test_input_schema_lists_the_routes_and_methods() -> None: + schema = tool('http://127.0.0.1:8080').input_schema + assert schema['properties']['method']['enum'] == ['GET', 'POST'] + assert 'POST /rest/bug/{id}/comment' in schema['properties']['path']['description'] + assert schema['required'] == ['method', 'path'] and schema['additionalProperties'] is False + + +def test_environment_fingerprint_covers_tools_deterministically() -> None: + from deepgym.traces.schema import environment_sha256 + + assert environment_sha256(app_env('http://127.0.0.1')) == environment_sha256( + app_env('http://127.0.0.1') + ) + assert environment_sha256(app_env('http://127.0.0.1')) != environment_sha256( + app_env('http://127.0.0.2') + ) + + +def test_qa_controls_only_read_declared_routes() -> None: + env = app_env('http://127.0.0.1') + for actions in (do_everything(env), random_agent(env)): + assert actions + assert {action.action_type for action in actions} == {'tool'} + assert {action.arguments['method'] for action in actions} == {'GET'} + for action in actions: + env.tools[0].prepare(action.arguments) # each one is a valid call diff --git a/deepgym/tests/test_traces_export.py b/deepgym/tests/test_traces_export.py index 96da6aa..293898e 100644 --- a/deepgym/tests/test_traces_export.py +++ b/deepgym/tests/test_traces_export.py @@ -106,15 +106,23 @@ def _trace_records(data: dict) -> list[tuple]: def _rl_records(row: dict) -> list[tuple]: - """(turn, action, observation, reward) of each step of an RL row; a turn is its reply.""" + """(turn, action, observation, reward) of each step of an RL row; a turn is its reply. + + Parquet holds a tool action's arguments as one JSON string. + """ replies = sorted({step['message_index'] for step in row['steps']}) return [ - (replies.index(step['message_index']) + 1, Action.model_validate(step['action']), + (replies.index(step['message_index']) + 1, Action.model_validate(_decoded(step['action'])), step['observation'], step['reward']) for step in row['steps'] ] # fmt: skip +def _decoded(action: dict) -> dict: + arguments = action['arguments'] + return {**action, 'arguments': json.loads(arguments) if isinstance(arguments, str) else arguments} + + def _atif_records(document: dict) -> list[tuple]: """(turn, action, observation, reward) of each tool call; a turn is one agent step.""" records = [] From 15f917b63530f483daa7653b8bac7f211adc037f Mon Sep 17 00:00:00 2001 From: Abhishek Gahlot Date: Mon, 19 Oct 2026 02:45:16 +0530 Subject: [PATCH 03/21] fix: keep tool credentials root-only and send exactly the validated request --- deepgym/src/deepgym/_http_request.py | 98 +++++++------- deepgym/src/deepgym/models.py | 15 ++- deepgym/src/deepgym/qa/agents.py | 4 +- deepgym/src/deepgym/session.py | 93 ++++++++++---- deepgym/src/deepgym/tools.py | 184 +++++++++++++++++++-------- deepgym/tests/http_app.py | 4 + deepgym/tests/test_tools.py | 181 ++++++++++++++++++++++++-- deepgym/tests/test_traces_export.py | 5 +- 8 files changed, 439 insertions(+), 145 deletions(-) diff --git a/deepgym/src/deepgym/_http_request.py b/deepgym/src/deepgym/_http_request.py index 122bbd9..a9e0972 100644 --- a/deepgym/src/deepgym/_http_request.py +++ b/deepgym/src/deepgym/_http_request.py @@ -1,67 +1,67 @@ -"""Send one HTTP request described by a JSON file and print what happened as one JSON line. +"""Send one HTTP request, read as JSON from stdin, and print what happened as one JSON line. The session runs this inside the sandbox (``python3 -I``) as the agent user, in the offline -network namespace, for an environment's declared API tool (see :mod:`deepgym.tools`). The -host built and checked the request; this only sends it. Nothing here comes from the model -but that request's data: no proxy is used and redirects come back as the response, never -followed, so the request reaches only its own URL. Stdlib only, as the image's python3 runs it. +network namespace, for an environment's declared API tool (see :mod:`deepgym.tools`). Root +pipes the request in from a file only root can read, so the app credentials it carries +never sit where the agent user could read them. The host built and checked every part of +the request; this only sends it with http.client, which parses no URL, uses no proxy and +follows no redirect: the request line carries exactly the target the host built, to the +host and port it named. Stdlib only, as the image's python3 runs it. """ import base64 import http.client import json +import ssl import sys import time -import urllib.error -import urllib.request -class _Unfollowed(urllib.request.HTTPRedirectHandler): - """Hand a 3xx back as the response instead of following its Location.""" - - def redirect_request(self, *args: object, **kwargs: object) -> None: - return None - - -def main(path: str) -> int: - """Send the request in *path*; print its record; return 0 if a response came back.""" - with open(path, encoding='utf-8') as file: - spec = json.load(file) - body = spec['body'] - request = urllib.request.Request( - spec['url'], - data=None if body is None else body.encode('utf-8'), - headers=spec['headers'], - method=spec['method'], - ) - opener = urllib.request.build_opener(urllib.request.ProxyHandler({}), _Unfollowed) +def main() -> int: + """Send the request on stdin; print its record; return 0 if a response came back.""" + spec = json.load(sys.stdin) + if spec['scheme'] == 'https': + context = ssl.create_default_context() + connection = http.client.HTTPSConnection(spec['host'], spec['port'], context=context) + else: + connection = http.client.HTTPConnection(spec['host'], spec['port']) record: dict[str, object] = {'request': spec['record']} + status = 1 started = time.monotonic() try: - try: - response = opener.open(request) - except urllib.error.HTTPError as error: # 4xx, 5xx and unfollowed 3xx are answers too - response = error - with response: - raw = response.read() - except (OSError, http.client.HTTPException) as exc: - reason = getattr(exc, 'reason', exc) - record['error'] = f'No response: {type(reason).__name__}: {reason}' - # Only a refused connection proves nothing was sent; any other loss may follow a write. - record['uncertain'] = spec['mutation'] and not isinstance(reason, ConnectionRefusedError) - status = 1 + connection.connect() + except OSError as exc: # nothing was sent + record['error'] = f'No connection: {type(exc).__name__}: {exc}' + record['uncertain'] = False else: - answer: dict[str, object] = { - 'status': response.status, - 'reason': response.reason, - 'headers': [list(header) for header in response.headers.items()], - } + body = spec['body'] try: - answer['body'] = raw.decode('utf-8') - except UnicodeDecodeError: - answer['body_base64'] = base64.b64encode(raw).decode('ascii') - record['response'] = answer - status = 0 + connection.request( + spec['method'], + spec['target'], + None if body is None else body.encode('utf-8'), + spec['headers'], + ) + response = connection.getresponse() + raw = response.read() + except (OSError, http.client.HTTPException) as exc: + # The request may have reached the app: a write is then neither known done nor not. + record['error'] = f'No response: {type(exc).__name__}: {exc}' + record['uncertain'] = spec['mutation'] + else: + answer: dict[str, object] = { + 'status': response.status, + 'reason': response.reason, + 'headers': [list(header) for header in response.getheaders()], + } + try: + answer['body'] = raw.decode('utf-8') + except UnicodeDecodeError: + answer['body_base64'] = base64.b64encode(raw).decode('ascii') + record['response'] = answer + status = 0 + finally: + connection.close() record['elapsed_ms'] = (time.monotonic() - started) * 1000 # UTF-8 bytes whatever the sandbox locale, on one line: JSON escapes every newline. sys.stdout.buffer.write(json.dumps(record, ensure_ascii=False).encode('utf-8') + b'\n') @@ -69,4 +69,4 @@ def main(path: str) -> int: if __name__ == '__main__': - sys.exit(main(sys.argv[1])) + sys.exit(main()) diff --git a/deepgym/src/deepgym/models.py b/deepgym/src/deepgym/models.py index 50b944c..e654036 100644 --- a/deepgym/src/deepgym/models.py +++ b/deepgym/src/deepgym/models.py @@ -572,8 +572,9 @@ class MultiTurnEnvironment(BaseModel): tools: tuple[HttpTool, ...] = () """API tools that tool actions call, run in the sandbox's offline namespace (see :mod:`deepgym.tools`). Trusted environment configuration only; never accept from an agent. - Declaring any needs ``tool`` and ``done`` in allowed_actions: a tool-calling agent ends - the episode only by an explicit ``done``.""" + Declaring any needs allowed_actions to be exactly ``tool`` and ``done``: a tool-calling + agent ends the episode only by an explicit ``done``, and no shell, file or GUI action + runs as the agent user beside the tool runner, which holds the tools' credentials.""" desktop: bool = False """Start a desktop (Xvfb + xfce) and return screenshots. Needs the Daytona backend.""" @@ -611,12 +612,16 @@ def check_desktop_actions(self) -> MultiTurnEnvironment: @model_validator(mode='after') def check_tools(self) -> MultiTurnEnvironment: - """Require unique tool names, and tools exactly when tool actions are allowed.""" + """Require unique tool names, tools exactly when tool actions are allowed, and + no other action beside them but done.""" names = [tool.name for tool in self.tools] if len(set(names)) != len(names): raise ValueError(f'tool names must be unique, got {names}') - if self.tools and not {'tool', 'done'} <= self.allowed_actions: - raise ValueError('tools need tool and done in allowed_actions') + if self.tools and self.allowed_actions != {'tool', 'done'}: + raise ValueError( + 'tools need allowed_actions to be exactly tool and done: no other action may ' + 'run beside the tool runner and its credentials' + ) if 'tool' in self.allowed_actions and not self.tools: raise ValueError('tool actions need declared tools') return self diff --git a/deepgym/src/deepgym/qa/agents.py b/deepgym/src/deepgym/qa/agents.py index 34386e2..1f4b187 100644 --- a/deepgym/src/deepgym/qa/agents.py +++ b/deepgym/src/deepgym/qa/agents.py @@ -9,9 +9,9 @@ from __future__ import annotations import random -import re from deepgym.models import Action, ActionType, MultiTurnEnvironment +from deepgym.tools import PLACEHOLDER _SCREEN = (1024, 768) """Daytona's default desktop, in screenshot pixels.""" @@ -69,7 +69,7 @@ def read() -> Action: if not reads: return Action(action_type='done') name, path = rng.choice(reads) if rng else reads[step % len(reads)] - arguments = {'method': 'GET', 'path': re.sub(r'\{\w+\}', '1', path)} + arguments = {'method': 'GET', 'path': PLACEHOLDER.sub('1', path)} return Action(action_type=kind, tool=name, arguments=arguments) def point() -> tuple[int, int]: diff --git a/deepgym/src/deepgym/session.py b/deepgym/src/deepgym/session.py index fef4a71..a4ede8e 100644 --- a/deepgym/src/deepgym/session.py +++ b/deepgym/src/deepgym/session.py @@ -81,7 +81,7 @@ SandboxTimeoutError, sealed_path, ) -from deepgym.tools import REQUEST_FILE, HttpTool, lost_response, run_command +from deepgym.tools import HttpTool, lost_response, run_command from deepgym.verifier_template import guard_crashes if TYPE_CHECKING: @@ -178,6 +178,11 @@ def exec(self, command: str, timeout: int, env: dict[str, str] | None = None) -> """Run a bash command in the work directory.""" ... + def exec_input(self, command: str, data: str, timeout: float) -> ExecResult: + """Run a bash command as exec does, with *data* on its stdin; where the backend + isolates the agent, *data* is never in a file the session user can read.""" + ... + def write_file(self, path: str, content: bytes) -> None: """Write a file relative to the work directory. @@ -235,10 +240,19 @@ def exec(self, command: str, timeout: int, env: dict[str, str] | None = None) -> Raises: TimeoutError: If the command exceeds *timeout* seconds. """ - exit_code, stdout, stderr = self._run(command, timeout, env) - return ExecResult(exit_code, stdout + ('\n' + stderr if stderr else '')) + return _joined(self._run(command, timeout, env)) + + def exec_input(self, command: str, data: str, timeout: float) -> ExecResult: + """Run *command* as :meth:`exec` does, with *data* on its stdin. + + Raises: + TimeoutError: If the command exceeds *timeout* seconds. + """ + return _joined(self._run(command, timeout, None, data)) - def _run(self, command: str, timeout: int, env: dict[str, str] | None) -> VerifierOutput: + def _run( + self, command: str, timeout: float, env: dict[str, str] | None, data: str | None = None + ) -> VerifierOutput: """Run *command* with bash in the temp directory, keeping stdout and stderr apart. Raises: @@ -248,6 +262,7 @@ def _run(self, command: str, timeout: int, env: dict[str, str] | None) -> Verifi # Own process group, so background children die with the command or the session. proc = subprocess.Popen( ['bash', '-c', command], + stdin=None if data is None else subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, @@ -258,7 +273,7 @@ def _run(self, command: str, timeout: int, env: dict[str, str] | None) -> Verifi ) self._groups.add(proc.pid) try: - stdout, stderr = proc.communicate(timeout=timeout) + stdout, stderr = proc.communicate(data, timeout=timeout) except subprocess.TimeoutExpired as exc: _kill_group(proc.pid) stdout, _ = proc.communicate() @@ -596,6 +611,29 @@ def exec(self, command: str, timeout: int, env: dict[str, str] | None = None) -> timeout=timeout, ) + def exec_input(self, command: str, data: str, timeout: float) -> ExecResult: + """Run *command* as the session user, offline, with *data* on its stdin. + + Root uploads *data* to its private staging directory and opens it for the command, + so the session user gets the bytes but never a path to them, and deletes it after (a + command the timeout stops can leave it there, still root-only). The command gets + only :data:`~deepgym.sandboxes.TRUSTED_PATH`, never the image's environment. + + Raises: + TimeoutError: If the command exceeds *timeout* seconds. + """ + staged = f'{_STAGING_DIR}/{uuid4().hex}' + self._sandbox.upload(data.encode('utf-8'), staged) + return self._sandbox.exec( + trusted( + f'{self._offline}{self._as_agent} {command} < {staged}; ' + f'status=$?; rm -f {staged}; exit $status' + ), + cwd=self._workdir, + env=self._clear, + timeout=timeout, + ) + def exec_root( self, command: str, timeout: int, *, offline: bool, env: Mapping[str, str] ) -> ExecResult: @@ -1084,14 +1122,12 @@ def step(self, action: Action, *, timeout: float | None = None) -> StepResult: and not isinstance(self._backend, SandboxBackend) ): raise ActionError('Desktop deadline overrides require SandboxBackend') - request = self._tool_request(action) if action.action_type == 'tool' else None + call = self._tool_call(action) if action.action_type == 'tool' else None self._step += 1 self._step_verdict = None try: - output, exit_code = self._apply( - action, timeout=_remaining(deadline), request=request - ) + output, exit_code = self._apply(action, timeout=_remaining(deadline), call=call) except ActionError: raise except BaseException as exc: @@ -1295,12 +1331,12 @@ def _apply( action: Action, *, timeout: float | None = None, - request: Mapping[str, JsonValue] | None = None, + call: tuple[HttpTool, dict[str, JsonValue]] | None = None, ) -> tuple[str, int | None]: """Execute *action*; return its text output and exit code (None if not a command). - A tool action sends *request*, from :meth:`_tool_request`. Its time, however it - ends, counts as the agent's (see :meth:`_ran_out_the_clock`). + A tool action sends *call*, from :meth:`_tool_call`. Its time, however it ends, + counts as the agent's (see :meth:`_ran_out_the_clock`). """ bounded = timeout is not None timeout = ( @@ -1322,8 +1358,8 @@ def _apply( self._backend.write_file(str(action.path), action.content.encode('utf-8')) return f'wrote {action.path}', None case 'tool': - assert request is not None - return self._call_tool(request, timeout) + assert call is not None + return self._send(*call, timeout) case 'wait': time.sleep(min(action.amount, timeout)) return '', None @@ -1339,8 +1375,8 @@ def _apply( finally: self._acting_s += time.monotonic() - started - def _tool_request(self, action: Action) -> dict[str, JsonValue]: - """Return the request *action* sends with its declared tool (see HttpTool.prepare). + def _tool_call(self, action: Action) -> tuple[HttpTool, dict[str, JsonValue]]: + """Return *action*'s declared tool and the request it sends (see HttpTool.prepare). Raises: ActionError: If no declared tool has its name, or the tool rejects its arguments. @@ -1350,25 +1386,26 @@ def _tool_request(self, action: Action) -> dict[str, JsonValue]: names = sorted(tool.name for tool in self._env.tools) raise ActionError(f'No tool named {action.tool!r}. Declared: {names}') try: - return tool.prepare(action.arguments) + return tool, tool.prepare(action.arguments) except ValueError as exc: raise ActionError(f'Tool {tool.name!r} rejected the call: {exc}') from exc - def _call_tool( - self, request: Mapping[str, JsonValue], timeout: float + def _send( + self, tool: HttpTool, request: Mapping[str, JsonValue], timeout: float ) -> tuple[str, int | None]: """Send *request* from the sandbox; return the runner's record of it and its exit code. - The request is data in a file the constant runner command reads. It is never - retried: a runner stopped by the timeout reports no response, and a mutation it - may have sent is recorded as uncertain. + The request reaches the constant runner command as data on its stdin, never through + a file or argument the agent user could read or change, and the tool's header values + are redacted from what comes back. It is never retried: a runner the timeout stops + reports no response, and a mutation it may have sent is recorded as uncertain. """ - self._backend.write_file(REQUEST_FILE, json.dumps(request).encode('utf-8')) + command = run_command(self._backend.python) try: - exit_code, output = self._backend.exec(run_command(self._backend.python), timeout) + exit_code, output = self._backend.exec_input(command, json.dumps(request), timeout) except TimeoutError: return lost_response(request, f'the request was stopped after {timeout}s'), None - return output, exit_code + return tool.redact(output), exit_code def _run_python(self, filename: str, code: str, timeout: float | None = None) -> ExecResult: self._backend.write_file(filename, code.encode('utf-8')) @@ -1609,6 +1646,12 @@ def failure_result(exc: BaseException, sandbox_id: str, *, exhausted: bool = Fal return result.model_copy(update={'error_type': 'policy_sandbox_failure'}) +def _joined(output: VerifierOutput) -> ExecResult: + """Return a command's *output* as one text: stdout, then stderr.""" + exit_code, stdout, stderr = output + return ExecResult(exit_code, stdout + ('\n' + stderr if stderr else '')) + + def _kill_group(group: int) -> None: """SIGKILL a process group; it may already be gone.""" # ponytail: a group id could be reused within one session's lifetime; accepted, low risk. diff --git a/deepgym/src/deepgym/tools.py b/deepgym/src/deepgym/tools.py index 6d08bbe..2164fcf 100644 --- a/deepgym/src/deepgym/tools.py +++ b/deepgym/src/deepgym/tools.py @@ -3,10 +3,12 @@ A tool is trusted environment configuration, never model output: its name, base URL, routes and headers come from the environment's author. A model supplies only a call's arguments (method, path, query, body). :meth:`HttpTool.prepare` checks them against the declared -routes, and the session writes the request to a file in the work directory. The command that -sends it is a constant (:func:`run_command`), so nothing the model writes reaches a shell; it +routes and builds the request from validated parts: host, port and the exact request +target. The command that sends it is a constant (:func:`run_command`) reading the request +on stdin, so nothing the model writes reaches a shell or a file it could read or change; it runs as the agent user in the sandbox's offline network namespace, where only the episode's -own services answer, and it follows no redirect (see :mod:`deepgym._http_request`). +own services answer, with http.client, which parses no URL and follows no redirect (see +:mod:`deepgym._http_request`). """ from __future__ import annotations @@ -15,7 +17,8 @@ import json import re import shlex -from collections.abc import Mapping +import unicodedata +from collections.abc import Iterator, Mapping from pathlib import Path from typing import Literal from urllib.parse import unquote, urlencode, urlsplit @@ -28,25 +31,60 @@ DONE_TOOL = 'done' """Reserved tool name: agents offer it so the model ends the episode with a ``done`` action.""" -REQUEST_FILE = '_tool_request.json' -"""Where in the work directory the session writes a call's request for the runner.""" +REDACTED = '[redacted]' +"""What stands in for a tool header's value in every record and result.""" _RUNNER = Path(__file__).with_name('_http_request.py').read_text(encoding='utf-8') _BODY_METHODS = frozenset({'POST', 'PUT', 'PATCH'}) # RFC 3986 path characters; a model percent-encodes anything else, so nothing it sends can -# start a query, a fragment or another authority. +# start a query, a fragment or another authority, and no space or control character is sent. _SEGMENT = re.compile(r"[A-Za-z0-9\-._~!$&'()*+,;=:@]+") _PATH = re.compile(r"/[A-Za-z0-9\-._~!$&'()*+,;=:@%/]*") -_PLACEHOLDER = re.compile(r'\{\w+\}') -_REDACTED = '[redacted]' +_STRAY_PERCENT = re.compile(r'%(?![0-9A-Fa-f]{2})') +PLACEHOLDER = re.compile(r'\{\w+(:path)?\}') +"""A route template's ``{name}`` or ``{name:path}`` segment.""" +# '.', '..', '...', '. .': segments servers resolve as dot segments, however spelled. +_DOTS = re.compile(r'[. ]*\.[. ]*') +_TOKEN = re.compile(r"[!#$%&'*+\-.^_`|~0-9A-Za-z]+") +# Names frameworks (Rack, Laravel, Symfony, Express) read to replace the request's method. +_METHOD_OVERRIDES = frozenset( + {'_method', 'x-http-method-override', 'x-http-method', 'x-method-override'} +) +_RESERVED_HEADERS = _METHOD_OVERRIDES | {'host', 'content-length', 'transfer-encoding'} +_PORTS = {'http': 80, 'https': 443} + + +def _readings(text: str) -> Iterator[str]: + """*text* and each further percent-decoding of it until none changes, NFKC-normalized: + every way an app that decodes once, twice, ... or folds compatibility forms may read it. + Each decoding is shorter than the last, so this ends.""" + while True: + yield unicodedata.normalize('NFKC', text) + decoded = unquote(text) + if decoded == text: + return + text = decoded + + +def _unsafe(path: str) -> bool: + """Whether a reading of *path* has a dot segment (``..;x`` too), a backslash or a + control character.""" + return any( + '\\' in form + or any(char < ' ' or char == '\x7f' for char in form) + or any(_DOTS.fullmatch(part.split(';', 1)[0]) for part in form.split('/')) + for form in _readings(path) + ) class HttpRoute(BaseModel): """One endpoint a tool may call: a method and a path template. - ``{name}`` in the template stands for exactly one non-empty path segment; every other - segment must match literally. An endpoint no route names (an admin page, a script - runner) is unreachable. + ``{name}`` in the template stands for exactly one non-empty path segment, which may not + hold an encoded ``/`` either, since an app that decodes before routing would read more + segments; ``{name:path}`` allows one (GitLab's ``group%2Fproject`` ids and file paths). + Every other segment must match literally. An endpoint no route names (an admin page, a + script runner) is unreachable. """ model_config = ConfigDict(frozen=True, extra='forbid') @@ -56,12 +94,12 @@ class HttpRoute(BaseModel): @field_validator('path') @classmethod def check_path(cls, value: str) -> str: - """Accept '/' or '/'-separated literal segments and ``{name}`` placeholders.""" + """Accept '/' or '/'-separated literal segments and placeholders.""" segments = value.split('/') - literal = [segment for segment in segments[1:] if not _PLACEHOLDER.fullmatch(segment)] + literal = [segment for segment in segments[1:] if not PLACEHOLDER.fullmatch(segment)] if value != '/' and ( segments[0] - or not all(_SEGMENT.fullmatch(part) and part not in ('.', '..') for part in literal) + or not all(_SEGMENT.fullmatch(part) and not _DOTS.fullmatch(part) for part in literal) ): raise ValueError( f'route path {value!r} must start with "/" and hold only non-empty literal ' @@ -72,10 +110,16 @@ def check_path(cls, value: str) -> str: def matches(self, path: str) -> bool: """Whether request *path* fits this template, segment by segment.""" want, got = self.path.split('/'), path.split('/') - return len(want) == len(got) and all( - expected == actual or (bool(actual) and bool(_PLACEHOLDER.fullmatch(expected))) - for expected, actual in zip(want, got) - ) + return len(want) == len(got) and all(map(_fits, want, got)) + + +def _fits(template: str, segment: str) -> bool: + """Whether request path *segment* fits one *template* segment.""" + if not PLACEHOLDER.fullmatch(template): + return template == segment + if template.endswith(':path}'): + return bool(segment) + return bool(segment) and not any('/' in form for form in _readings(segment)) class _Call(BaseModel): @@ -92,9 +136,10 @@ class HttpTool(BaseModel): """An API tool: HTTP requests to declared routes under one loopback base URL. The model sees *name*, *description* (the place for the API docs a task needs) and the - routes. *headers* go with every request (an API key for the local app, say) and are - recorded with their values redacted. Responses come back whole: status, every header - (pagination links included) and the full body. + routes. *headers* go with every request and are treated as credentials (an API key for + the local app, say): their values appear in no record, result, trace or error, and + never on disk where the agent user could read them. Responses come back whole: status, + every header (pagination links included) and the full body. Example:: @@ -132,28 +177,49 @@ def check_base_url(cls, value: str) -> str: """Accept an http(s) URL on a loopback host, without credentials, query or fragment. Returns: - The URL without a trailing slash, so a call's path appends to it. + The URL rebuilt from its checked parts: scheme, host, explicit port and path + without a trailing slash, so a call's path appends to it. """ parts = urlsplit(value) try: host = parts.hostname or '' loopback = host == 'localhost' or ipaddress.ip_address(host).is_loopback - parts.port # raises ValueError on a malformed port + port = parts.port or _PORTS.get(parts.scheme) except ValueError: loopback = False + path = parts.path.rstrip('/') if ( - parts.scheme not in ('http', 'https') + parts.scheme not in _PORTS or not loopback - or '@' in parts.netloc - or '?' in value - or '#' in value - or (parts.path and not _PATH.fullmatch(parts.path)) + or not value.isascii() + or not value.isprintable() + or any(mark in value for mark in '@?#\\') + or (path and (not _PATH.fullmatch(path) or _unsafe(path))) ): raise ValueError( f'base_url {value!r} must be an http(s) URL on a loopback host ' '(localhost, 127.0.0.0/8, ::1) with no credentials, query or fragment' ) - return value.rstrip('/') + netloc = f'[{host}]' if ':' in host else host + return f'{parts.scheme}://{netloc}:{port}{path}' + + @field_validator('headers') + @classmethod + def check_headers(cls, value: dict[str, str]) -> dict[str, str]: + """Accept token names, none that reroutes or reframes a request, with plain values.""" + for name, text in value.items(): + if ( + not _TOKEN.fullmatch(name) + or name.lower() in _RESERVED_HEADERS + or not text + or any((char < ' ' and char != '\t') or char == '\x7f' for char in text) + ): + raise ValueError( + f'header {name!r} must be a token name other than ' + f'{sorted(_RESERVED_HEADERS)}, with a non-empty value and no control ' + 'characters' + ) + return value @property def input_schema(self) -> dict[str, JsonValue]: @@ -197,24 +263,22 @@ def prepare(self, arguments: Mapping[str, JsonValue]) -> dict[str, JsonValue]: arguments: The model's arguments: method, path, optional query and body. Returns: - The request (method, url, headers, body text), whether it is a mutation (any - method but GET), and its record: the same with header values redacted. + The request from validated parts (scheme, host, port, the exact request target, + method, headers, body text), whether it is a mutation (any method but GET), and + its record: the URL it sends to, with header values redacted. Raises: - ValueError: If an argument is unknown or of the wrong type, the path has a - character to percent-encode or a dot segment, no declared route fits the - method and path, or the method takes no body and one was given. + ValueError: If an argument is unknown or of the wrong type; the path has a + character to percent-encode, a stray percent sign, or a reading with a dot + segment, backslash or control character; no declared route fits the method + and path; a body comes with a method that takes none, is not plain JSON, or + it or the query names a method override. """ call = _Call.model_validate(arguments) - decoded = unquote(call.path) - if ( - not _PATH.fullmatch(call.path) - or '\\' in decoded - or any(part in ('.', '..') for part in decoded.split('/')) - ): + if not _PATH.fullmatch(call.path) or _STRAY_PERCENT.search(call.path) or _unsafe(call.path): raise ValueError( - f'path {call.path!r} must start with "/", percent-encode reserved ' - 'characters and have no dot segments' + f'path {call.path!r} must start with "/", percent-encode reserved characters ' + 'and have no dot segments, backslashes or control characters, encoded or not' ) if not any(r.method == call.method and r.matches(call.path) for r in self.routes): raise ValueError( @@ -222,33 +286,53 @@ def prepare(self, arguments: Mapping[str, JsonValue]) -> dict[str, JsonValue]: ) if call.body is not None and call.method not in _BODY_METHODS: raise ValueError(f'{call.method} takes no body') + keys = [*call.query, *(call.body if isinstance(call.body, dict) else ())] + if overrides := [key for key in keys if key.lower() in _METHOD_OVERRIDES]: + raise ValueError(f'{overrides} would override the request method') pairs = [ (key, item if isinstance(item, str) else json.dumps(item)) for key, value in call.query.items() for item in (value if isinstance(value, list) else [value]) ] - url = self.base_url + call.path + (f'?{urlencode(pairs)}' if pairs else '') + base = urlsplit(self.base_url) + target = base.path + call.path + (f'?{urlencode(pairs)}' if pairs else '') headers = dict(self.headers) - record: dict[str, JsonValue] = {'method': call.method, 'url': url} + record: dict[str, JsonValue] = { + 'method': call.method, + 'url': f'{base.scheme}://{base.netloc}{target}', + } + body = None if call.body is not None: + body = json.dumps(call.body, ensure_ascii=False, allow_nan=False) headers['Content-Type'] = 'application/json' record['body'] = call.body record['headers'] = { - name: _REDACTED if name in self.headers else value for name, value in headers.items() + name: REDACTED if name in self.headers else value for name, value in headers.items() } return { + 'scheme': base.scheme, + 'host': base.hostname, + 'port': base.port, + 'target': target, 'method': call.method, - 'url': url, 'headers': headers, - 'body': None if call.body is None else json.dumps(call.body, ensure_ascii=False), + 'body': body, 'mutation': call.method != 'GET', 'record': record, } + def redact(self, text: str) -> str: + """*text* with each header value, as is and JSON-escaped, replaced by REDACTED, in + case the app echoes one back.""" + for value in self.headers.values(): + for form in {value, json.dumps(value)[1:-1]}: + text = text.replace(form, REDACTED) + return text + def run_command(python: str) -> str: - """The constant command that sends the request in :data:`REQUEST_FILE` with *python*.""" - return f'{python} -I -c {shlex.quote(_RUNNER)} {REQUEST_FILE}' + """The constant command that sends the request it reads on stdin, with *python*.""" + return f'{python} -I -c {shlex.quote(_RUNNER)}' def lost_response(request: Mapping[str, JsonValue], reason: str) -> str: diff --git a/deepgym/tests/http_app.py b/deepgym/tests/http_app.py index e9797b1..5e619ee 100644 --- a/deepgym/tests/http_app.py +++ b/deepgym/tests/http_app.py @@ -22,6 +22,8 @@ HttpRoute(method='GET', path='/rest/lost'), HttpRoute(method='POST', path='/rest/slow'), HttpRoute(method='GET', path='/rest/binary'), + HttpRoute(method='GET', path='/rest/echo'), + HttpRoute(method='GET', path='/rest/file/{name:path}'), ) @@ -64,6 +66,8 @@ def _handle(self) -> None: self._send(302, b'', {'Location': app.redirect_to}) elif route.endswith('/done'): self._send(200, b'DONE\n{"done": true}\n', {'Content-Type': 'text/plain'}) + elif route.endswith('/echo'): # an app that reflects request headers + self._send(200, json.dumps(dict(self.headers.items())).encode(), {}) elif route.endswith('/binary'): self._send(200, b'\xff\x00\xfe', {}) elif route.endswith('/comment'): diff --git a/deepgym/tests/test_tools.py b/deepgym/tests/test_tools.py index 7d098db..e13bb68 100644 --- a/deepgym/tests/test_tools.py +++ b/deepgym/tests/test_tools.py @@ -1,6 +1,7 @@ """Tests for declared HTTP API tools run as Session tool actions, against a loopback app.""" import json +import shlex from pathlib import Path import pytest @@ -10,9 +11,9 @@ from deepgym.models import Action, MultiTurnEnvironment from deepgym.qa.agents import do_everything, random_agent from deepgym.sandboxes import ExecResult -from deepgym.session import Session +from deepgym.session import SandboxBackend, Session from deepgym.tools import HttpRoute, HttpTool, run_command -from tests.http_app import API_KEY, BIG, App, app_env, app_session, tool +from tests.http_app import API_KEY, BIG, ROUTES, App, app_env, app_session, tool def call(path: str, method: str = 'GET', **arguments: object) -> Action: @@ -89,6 +90,19 @@ def test_binary_body_comes_back_whole_as_base64(app: App) -> None: {'method': 'GET', 'path': '/rest/bug/1', 'url': 'http://evil.example/'}, {'method': 'GET', 'path': '/rest/bug/1', 'query': {'q': {'nested': 1}}}, {'method': 'GET'}, + {'method': 'GET', 'path': '/rest/bug/..;'}, + {'method': 'GET', 'path': '/rest/bug/%2e%2e;x'}, + {'method': 'GET', 'path': '/rest/bug/%252e%252e'}, + {'method': 'GET', 'path': '/rest/bug/%EF%BC%8E%EF%BC%8E'}, + {'method': 'GET', 'path': '/rest/bug/...'}, + {'method': 'GET', 'path': '/rest/bug/%00'}, + {'method': 'GET', 'path': '/rest/bug/1%0d%0aX-Injected:%201'}, + {'method': 'GET', 'path': '/rest/bug/%zz'}, + {'method': 'GET', 'path': '/rest/bug/%2'}, + {'method': 'GET', 'path': '/rest/bug/\u00e9'}, + {'method': 'GET', 'path': '/rest/bug/1\t'}, + {'method': 'GET', 'path': '/rest/bug/%EF%BC%BC'}, + {'method': 'POST', 'path': '/rest/bug/1/comment', 'body': {'n': float('nan')}}, ], ) def test_off_origin_or_undeclared_calls_are_rejected_unsent(app: App, arguments: dict) -> None: @@ -198,32 +212,171 @@ def test_done_lines_in_raw_tool_output_never_end_the_episode( def test_injection_shaped_arguments_stay_data(app: App) -> None: hostile = '\'; touch pwned; echo $(touch pwned2) `touch pwned3` " && rm -rf ~ #' - commands: list[str] = [] + sent: list[tuple[str, str]] = [] with app_session(app.url) as session: session.start() backend = session._backend - exec_ = backend.exec + exec_input = backend.exec_input - def spy(command: str, timeout: float, env: dict | None = None) -> ExecResult: - commands.append(command) - return exec_(command, timeout, env) + def spy(command: str, data: str, timeout: float) -> ExecResult: + sent.append((command, data)) + return exec_input(command, data, timeout) - backend.exec = spy + backend.exec_input = spy workdir = Path(backend._dir) - result = record( - session, - call('/rest/bug/1%3Btouch%20pwned', query={'q': hostile}), - ) + result = record(session, call('/rest/bug/1%3Btouch%20pwned', query={'q': hostile})) record(session, call('/rest/bug/1/comment', 'POST', body={'comment': hostile})) assert result['response']['status'] == 200 assert not list(workdir.glob('pwned*')) assert not list(Path.cwd().glob('pwned*')) - assert commands == [run_command(backend.python)] * 2 # one constant command, args as data + # One constant command; the arguments travel only as JSON data on its stdin. + assert [command for command, _ in sent] == [run_command(backend.python)] * 2 + assert all(hostile not in command for command, _ in sent) + assert json.loads(sent[1][1])['body'] == json.dumps({'comment': hostile}, ensure_ascii=False) assert app.requests[0]['path'].startswith('/rest/bug/1%3Btouch%20pwned?q=') assert json.loads(app.requests[1]['body']) == {'comment': hostile} +def test_credentials_stay_out_of_files_results_and_errors(app: App) -> None: + with app_session(app.url) as session: + session.start() + workdir = Path(session._backend._dir) + + echoed = session.step(call('/rest/echo')).observation.content + with pytest.raises(ActionError) as rejected: + session.step(call('/rest/bug/1', body={'x': 1})) + files = [path.read_bytes() for path in workdir.rglob('*') if path.is_file()] + + assert app.requests[0]['headers']['x-api-key'] == API_KEY # the app did get it + assert API_KEY not in echoed and '[redacted]' in json.loads(echoed)['response']['body'] + assert API_KEY not in str(rejected.value) + assert not any(API_KEY.encode() in content for content in files) + + +def test_sandbox_stdin_is_staged_root_only_and_never_on_a_command_line() -> None: + from tests.test_sandboxes import _CONTAINED_AGENT_PREFIX, _USERS, FakeSandbox + + box = FakeSandbox(network_blocked=False, answers=[_USERS]) + uploads: dict[str, bytes] = {} + box.upload = lambda content, path: uploads.update({path: content}) # type: ignore[method-assign] + backend = SandboxBackend(box) + + backend.exec_input(run_command(backend.python), f'{{"key": "{API_KEY}"}}', timeout=5) + + command = box.commands[-1] + (staged,) = [path for path, content in uploads.items() if API_KEY.encode() in content] + assert staged.startswith('/opt/deepgym-staging/') + assert command.startswith('/usr/bin/env -i -- PATH=') + inner = shlex.split(command)[-1] # root's sh opens the root-only file, then drops to the agent + assert inner.startswith(_CONTAINED_AGENT_PREFIX) + assert f'< {staged}; status=$?; rm -f {staged}; exit $status' in inner + assert API_KEY not in command + + +@pytest.mark.parametrize('method', ['get', 'Get', 'GET ', 'HEAD', 'OPTIONS', 'TRACE', 'CONNECT']) +def test_method_must_be_a_declared_one_exactly(app: App, method: str) -> None: + with app_session(app.url) as session: + session.start() + with pytest.raises(ActionError): + session.step(call('/rest/bug/1', method)) + assert app.requests == [] + + +@pytest.mark.parametrize( + 'arguments', + [ + {'query': {'_method': 'DELETE'}}, + {'query': {'_METHOD': 'DELETE'}}, + {'query': {'X-HTTP-Method-Override': 'DELETE'}}, + {'query': {'x-method-override': 'PUT'}}, + {'body': {'_method': 'DELETE', 'comment': 'x'}}, + {'body': {'X-Http-Method': 'DELETE'}}, + ], +) +def test_method_overrides_are_rejected(app: App, arguments: dict) -> None: + method = 'POST' if 'body' in arguments else 'GET' + path = '/rest/bug/1/comment' if 'body' in arguments else '/rest/bug/1' + with app_session(app.url) as session: + session.start() + with pytest.raises(ActionError, match='override the request method'): + session.step(call(path, method, **arguments)) + assert app.requests == [] + + +@pytest.mark.parametrize( + 'headers', + [ + {'X-HTTP-Method-Override': 'DELETE'}, + {'x-http-method': 'DELETE'}, + {'Host': 'evil.example'}, + {'Content-Length': '0'}, + {'Transfer-Encoding': 'chunked'}, + {'X-Key': 'a\r\nX-Injected: 1'}, + {'X-Key': ''}, + {'Bad Name': 'x'}, + ], +) +def test_tool_headers_cannot_reroute_or_reframe_requests(headers: dict) -> None: + with pytest.raises(ValidationError, match='header'): + HttpTool( + name='t', description='d', base_url='http://127.0.0.1', routes=ROUTES, headers=headers + ) + + +@pytest.mark.parametrize( + 'path', + [ + '/rest/bug/1@evil.example', + '/rest/bug/a;b=c', + "/rest/bug/!$&'()*+,=:~_-.x", + '/rest/bug/%41%42', + '/rest/file/src%2Fmain.c', + ], +) +def test_the_app_receives_exactly_the_recorded_target(app: App, path: str) -> None: + with app_session(app.url) as session: + session.start() + result = record(session, call(path, query={'q': 'a b/é&x=1'})) + + (seen,) = app.requests + assert result['request']['url'] == f'{app.url}{seen["path"]}' # byte for byte + assert seen['path'].startswith(f'{path}?q=') + + +def test_an_encoded_slash_needs_a_path_placeholder(app: App) -> None: + with app_session(app.url) as session: + session.start() + for path in ('/rest/bug/a%2Fcomment', '/rest/bug/a%252Fb', '/rest/bug/%EF%BC%8F'): + with pytest.raises(ActionError, match='fits no route'): + session.step(call(path)) + assert record(session, call('/rest/file/a%2Fb'))['response']['status'] == 404 + assert [request['path'] for request in app.requests] == ['/rest/file/a%2Fb'] + + +@pytest.mark.parametrize( + ('base_url', 'normal'), + [ + ('http://LOCALHOST:8080/api/', 'http://localhost:8080/api'), + ('http://localhost', 'http://localhost:80'), + ('https://127.0.0.2', 'https://127.0.0.2:443'), + ('http://[::1]:9000', 'http://[::1]:9000'), + ], +) +def test_base_url_is_rebuilt_from_its_checked_parts(base_url: str, normal: str) -> None: + assert tool(base_url).base_url == normal + + +def test_tools_allow_no_other_agent_action() -> None: + for extra in ('bash', 'code', 'write_file'): + with pytest.raises(ValidationError, match='exactly tool and done'): + MultiTurnEnvironment( + task='t', + tools=(tool('http://127.0.0.1'),), + allowed_actions=frozenset({'tool', 'done', extra}), + ) + + @pytest.mark.parametrize( 'base_url', [ @@ -237,6 +390,8 @@ def spy(command: str, timeout: float, env: dict | None = None) -> ExecResult: 'ftp://127.0.0.1', 'http://10.0.0.1', 'http://127.0.0.1:notaport', + 'http://local\thost', + 'http://127.0.0.1\\@evil.example', ], ) def test_base_url_must_be_local(base_url: str) -> None: diff --git a/deepgym/tests/test_traces_export.py b/deepgym/tests/test_traces_export.py index 293898e..1a15e94 100644 --- a/deepgym/tests/test_traces_export.py +++ b/deepgym/tests/test_traces_export.py @@ -120,7 +120,10 @@ def _rl_records(row: dict) -> list[tuple]: def _decoded(action: dict) -> dict: arguments = action['arguments'] - return {**action, 'arguments': json.loads(arguments) if isinstance(arguments, str) else arguments} + return { + **action, + 'arguments': json.loads(arguments) if isinstance(arguments, str) else arguments, + } def _atif_records(document: dict) -> list[tuple]: From 7e8f6c6557755e84ebb77fbee6dda9567739503d Mon Sep 17 00:00:00 2001 From: Abhishek Gahlot Date: Mon, 19 Oct 2026 02:45:46 +0530 Subject: [PATCH 04/21] feat: anthropic and openai agents that act only through declared api tools --- deepgym/src/deepgym/agents/__init__.py | 3 + deepgym/src/deepgym/agents/api.py | 396 +++++++++++++++++++++ deepgym/src/deepgym/agents/base.py | 5 + deepgym/tests/agents/test_api.py | 469 +++++++++++++++++++++++++ 4 files changed, 873 insertions(+) create mode 100644 deepgym/src/deepgym/agents/api.py create mode 100644 deepgym/tests/agents/test_api.py diff --git a/deepgym/src/deepgym/agents/__init__.py b/deepgym/src/deepgym/agents/__init__.py index 1cd0e3c..0ad8875 100644 --- a/deepgym/src/deepgym/agents/__init__.py +++ b/deepgym/src/deepgym/agents/__init__.py @@ -1,5 +1,6 @@ """Run provider agents over DeepGym sessions; SDKs are optional and imported on run.""" +from deepgym.agents.api import AnthropicToolAgent, OpenAIToolAgent from deepgym.agents.base import Agent, AgentResult, Budget, Cost, TokenPrices, Usage from deepgym.agents.cli import ClaudeCodeAgent, CodexAgent from deepgym.agents.computer import AnthropicComputerAgent, OpenAIComputerAgent @@ -9,11 +10,13 @@ 'Agent', 'AgentResult', 'AnthropicComputerAgent', + 'AnthropicToolAgent', 'Budget', 'ClaudeCodeAgent', 'CodexAgent', 'Cost', 'OpenAIComputerAgent', + 'OpenAIToolAgent', 'TokenPrices', 'Usage', 'SUPPORTED_CLIS', diff --git a/deepgym/src/deepgym/agents/api.py b/deepgym/src/deepgym/agents/api.py new file mode 100644 index 0000000..77d46ae --- /dev/null +++ b/deepgym/src/deepgym/agents/api.py @@ -0,0 +1,396 @@ +"""Host-side provider loops that act only through an environment's declared API tools. + +The model gets the environment's tools (HTTP calls to the app under test, sent from inside +the sandbox's offline namespace; see :mod:`deepgym.tools`) and a ``done`` tool, nothing +else: no hosted or server-side tool (web search, code execution, browsing) is ever +requested, so the model cannot read a live public copy of the data it is graded on. +Provider keys stay in this process; the sandbox only ever sees the app's requests. + +Each tool call is one Session ``tool`` action. Its observation, which the model gets back +whole, is the runner's record of the call: the full request, the full response (status, +headers, body), timing, and whether a write whose response was lost may have happened. +A call is never retried. Only the done tool ends an episode; a reply without tool calls +gets a reminder instead. Streaming, accounting and incomplete-response handling are the +computer agents' (see :mod:`deepgym.agents.providers`). Every model message goes to +``AgentResult.messages``; under a TracedSession, each step also carries the messages that +produced it. +""" + +from __future__ import annotations + +import json +from collections.abc import Mapping, Sequence +from typing import Any, Literal + +from deepgym.agents.base import ( + AgentResult, + Budget, + BudgetExceeded, + BudgetTracker, + Cost, + TerminationReason, + TokenPrices, + Usage, +) +from deepgym.agents.providers import ( + ANTHROPIC_MAX_TOKENS, + SDK_RETRIES, + account, + anthropic_message, + api_error, + openai_incomplete, + validate_budget, +) +from deepgym.exceptions import ActionError, DeepGymError +from deepgym.models import Action, Observation +from deepgym.session import Session +from deepgym.tools import DONE_TOOL +from deepgym.traces.capture import TracedSession +from deepgym.traces.schema import TraceMessage, json_text + +_SYSTEM = ( + "Complete the user task using only the provided tools, which call the application's HTTP " + 'API. Each tool result is a JSON record of the request and the full response. A result ' + 'with "uncertain": true means a write may or may not have been applied: check the ' + "application's state before repeating it. Call done once the task is finished, or when " + 'it cannot be finished.' +) +_NO_CALL = 'No tool was called. Continue with the tools, or call done when the task is finished.' +_SKIPPED = 'Not executed: an earlier call in this batch failed' +_DONE_DESCRIPTION = 'End the episode. Call it once the task is finished, or when it cannot be.' +_DONE_SCHEMA = {'type': 'object', 'properties': {}, 'additionalProperties': False} + + +def _specs(session: Session) -> list[tuple[str, str, dict[str, Any]]]: + """Name, description and input schema of each tool the model gets: the declared ones + and done. + + Raises: + ValueError: If the environment declares no tools. + """ + if not session.tools: + raise ValueError('API tool agents need an environment that declares tools') + declared = [(tool.name, tool.description, tool.input_schema) for tool in session.tools] + return [*declared, (DONE_TOOL, _DONE_DESCRIPTION, _DONE_SCHEMA)] + + +def _prompt(task: str, initial: Observation) -> str: + return f'{task}\n\n{initial.content}' if initial.content else task + + +class _Episode: + """One run's session, accounting and model messages. + + Every message goes to the result. Under a TracedSession the messages not yet on a step + also go with the next action, so each step carries the messages that produced it. + """ + + def __init__(self, session: Session, budget: Budget, prompt: str) -> None: + self.session = session + self.tracker = BudgetTracker(budget) + self.messages = [ + TraceMessage(role='system', content=_SYSTEM), + TraceMessage(role='user', content=prompt), + ] + self._sent = 0 + + def say(self, role: Literal['user', 'assistant', 'tool'], content: str | Mapping) -> None: + """Record a message: text as it is, a provider payload as its full JSON.""" + text = content if isinstance(content, str) else json_text(content) + self.messages.append(TraceMessage(role=role, content=text)) + + def act(self, action: Action) -> Observation: + """Apply *action* within budget, with the messages since the last recorded step.""" + self.tracker.check() + if self.session.done: + raise BudgetExceeded('session_done') + new, self._sent = self.messages[self._sent :], len(self.messages) + timeout = self.tracker.request_timeout + if isinstance(self.session, TracedSession): + result = self.session.step(action, new, timeout=timeout) + else: + result = self.session.step(action, timeout=timeout) + self.tracker.check() + return result.observation + + def call(self, name: str, arguments: object) -> tuple[str, bool]: + """Run one tool call; return its full result and whether it failed. + + A call fails when the session rejects it (unknown tool, arguments its routes don't + allow) or no response came back. + """ + try: + if not isinstance(arguments, dict): + raise ValueError('arguments must be a JSON object') + action = Action(action_type='tool', tool=name, arguments=arguments) + except ValueError as exc: + return f'Tool call rejected: {exc}', True + try: + observation = self.act(action) + except ActionError as exc: + return f'Tool call rejected: {exc}', True + return observation.content, (observation.metadata or {}).get('exit_code') != 0 + + def execute(self, calls: Sequence[tuple[str, object]]) -> list[tuple[str, bool]] | None: + """Run a reply's calls in order; None once its done call ended the episode. + + After a call that failed, the rest are not executed, since they may rest on it: + each gets an error result instead. + """ + outcomes: list[tuple[str, bool]] = [] + for name, arguments in calls: + if outcomes and outcomes[-1][1]: + outcomes.append((_SKIPPED, True)) + elif name == DONE_TOOL: + self.act(Action(action_type='done')) + return None + else: + outcomes.append(self.call(name, arguments)) + return outcomes + + def result( + self, reason: TerminationReason, final_text: str = '', error: str | None = None + ) -> AgentResult: + """Snapshot accounting, the trajectory and every message, without grading.""" + result = self.tracker.result(self.session, reason, final_text, error) + return result.model_copy(update={'messages': tuple(self.messages)}) + + +class AnthropicToolAgent: + """Drive Session through its declared API tools with Anthropic; import the SDK on run. + + Args: + model: Provider model with client tool use. + prices: Explicit model-specific rates, required for a USD budget. + effort: Adaptive-thinking effort; None sends no thinking or effort settings. + max_tokens: Output limit per response; defaults to Opus 5.5's maximum so the + harness never caps the model below what it can produce. + """ + + def __init__( + self, + model: str, + prices: TokenPrices | None = None, + effort: Literal['low', 'medium', 'high', 'xhigh', 'max'] | None = 'max', + max_tokens: int = ANTHROPIC_MAX_TOKENS, + ) -> None: + self.model = model + self.prices = prices + self.effort = effort + self.max_tokens = max_tokens + + def run( + self, session: Session, initial: Observation, *, task: str, budget: Budget + ) -> AgentResult: + """Run the task until the model calls done, and return its trajectory without grading. + + Raises: + ImportError: If the optional anthropic extra is absent. + ValueError: If required prices are missing or the environment declares no tools. + """ + validate_budget(budget, self.prices) + tools = [ + {'name': name, 'description': description, 'input_schema': schema} + for name, description, schema in _specs(session) + ] + import anthropic + + prompt = _prompt(task, initial) + episode = _Episode(session, budget, prompt) + tracker = episode.tracker + messages: list[dict[str, Any]] = [{'role': 'user', 'content': prompt}] + thinking = ( + { + 'thinking': {'type': 'adaptive', 'display': 'summarized'}, + 'output_config': {'effort': self.effort}, + } + if self.effort + else {} + ) + client = anthropic.Anthropic(max_retries=SDK_RETRIES) + try: + while True: + if session.done: + return episode.result('session_done') + tracker.before_turn() + response = anthropic_message( + client, + tracker, + self.prices, + model=self.model, + max_tokens=self.max_tokens, + cache_control={'type': 'ephemeral'}, + **thinking, + system=_SYSTEM, + tools=tools, + messages=messages, + timeout=tracker.request_timeout, + ) + if isinstance(response, str): + return episode.result('error', error=response) + episode.say('assistant', response) + account(tracker, response.get('usage'), self.prices, 'anthropic') + content = response['content'] + messages.append({'role': 'assistant', 'content': content}) + text = ' '.join(part['text'] for part in content if part['type'] == 'text') + stop = response.get('stop_reason') + if stop == 'refusal': + return episode.result('refusal', text) + calls = [part for part in content if part['type'] == 'tool_use'] + if not calls: + if stop in {'max_tokens', 'pause_turn'}: + note = 'Continue from the output limit.' + elif stop in {'end_turn', 'stop_sequence'}: + note = _NO_CALL + else: + return episode.result('error', error='Response did not complete') + messages.append({'role': 'user', 'content': note}) + episode.say('user', note) + continue + outcomes = episode.execute([(call['name'], call.get('input')) for call in calls]) + if outcomes is None: + return episode.result('completed', text) + results = [ + { + 'type': 'tool_result', + 'tool_use_id': call['id'], + 'content': output, + **({'is_error': True} if failed else {}), + } + for call, (output, failed) in zip(calls, outcomes) + ] + for result in results: + episode.say('tool', result) + messages.append({'role': 'user', 'content': results}) + except BudgetExceeded as exc: + return episode.result(exc.reason) + except DeepGymError as exc: + reason = 'time_budget' if tracker.remaining_seconds <= 0 else 'error' + return episode.result(reason, error=f'Session step failed: {type(exc).__name__}') + except anthropic.APIError as exc: + tracker.add_usage(Usage(), Cost(provenance='Anthropic failed request; billing unknown')) + reason = 'time_budget' if tracker.remaining_seconds <= 0 else 'error' + return episode.result(reason, error=f'Anthropic {api_error(exc)}') + except (ValueError, KeyError, TypeError) as exc: + return episode.result('error', error=f'Invalid Anthropic response: {exc}') + finally: + client.close() + + +def _arguments(text: str) -> object: + """A function call's arguments, or its raw text when that isn't JSON (then rejected).""" + try: + return json.loads(text) + except ValueError: + return text + + +class OpenAIToolAgent: + """Drive Session through its declared API tools with OpenAI Responses function calls. + + Args: + model: Provider model with function calling. + prices: Explicit model-specific rates, required for a USD budget. + effort: Reasoning effort; None sends no reasoning setting. No output limit is sent, + so the model may use its full output allowance. + """ + + def __init__( + self, + model: str, + prices: TokenPrices | None = None, + effort: Literal['none', 'minimal', 'low', 'medium', 'high', 'xhigh', 'max'] | None = 'max', + ) -> None: + self.model = model + self.prices = prices + self.effort = effort + + def run( + self, session: Session, initial: Observation, *, task: str, budget: Budget + ) -> AgentResult: + """Run the task until the model calls done, and return its trajectory without grading. + + Raises: + ImportError: If the optional openai extra is absent. + ValueError: If required prices are missing or the environment declares no tools. + """ + validate_budget(budget, self.prices) + tools = [ + { + 'type': 'function', + 'name': name, + 'description': description, + 'parameters': schema, + 'strict': False, # strict mode can't express a free-form JSON body + } + for name, description, schema in _specs(session) + ] + import openai + + prompt = _prompt(task, initial) + episode = _Episode(session, budget, prompt) + tracker = episode.tracker + items: list[dict[str, Any]] = [ + {'role': 'user', 'content': [{'type': 'input_text', 'text': prompt}]} + ] + previous: str | None = None + client = openai.OpenAI(max_retries=SDK_RETRIES) + try: + while True: + if session.done: + return episode.result('session_done') + tracker.before_turn() + response = client.responses.create( + model=self.model, + instructions=_SYSTEM, + tools=tools, + input=items, + previous_response_id=previous, + **({'reasoning': {'effort': self.effort}} if self.effort else {}), + timeout=tracker.request_timeout, + ).model_dump(exclude_none=True) + episode.say('assistant', response) + account(tracker, response.get('usage'), self.prices, 'openai') + previous = response['id'] + output = response['output'] + parts = [ + part + for item in output + if item['type'] == 'message' + for part in item.get('content', []) + ] + refusal = ' '.join(part['refusal'] for part in parts if part['type'] == 'refusal') + if refusal: + return episode.result('refusal', refusal) + text = ' '.join(part['text'] for part in parts if part['type'] == 'output_text') + if error := openai_incomplete(response): + return episode.result('error', error=error) + calls = [item for item in output if item['type'] == 'function_call'] + if not calls: + items = [{'role': 'user', 'content': _NO_CALL}] + episode.say('user', _NO_CALL) + continue + outcomes = episode.execute( + [(call['name'], _arguments(call['arguments'])) for call in calls] + ) + if outcomes is None: + return episode.result('completed', text) + items = [ + {'type': 'function_call_output', 'call_id': call['call_id'], 'output': result} + for call, (result, _) in zip(calls, outcomes) + ] + for item in items: + episode.say('tool', item) + except BudgetExceeded as exc: + return episode.result(exc.reason) + except DeepGymError as exc: + reason = 'time_budget' if tracker.remaining_seconds <= 0 else 'error' + return episode.result(reason, error=f'Session step failed: {type(exc).__name__}') + except openai.APIError as exc: + tracker.add_usage(Usage(), Cost(provenance='OpenAI failed request; billing unknown')) + reason = 'time_budget' if tracker.remaining_seconds <= 0 else 'error' + return episode.result(reason, error=f'OpenAI {api_error(exc)}') + except (ValueError, KeyError, TypeError) as exc: + return episode.result('error', error=f'Invalid OpenAI response: {exc}') + finally: + client.close() diff --git a/deepgym/src/deepgym/agents/base.py b/deepgym/src/deepgym/agents/base.py index 574e965..f0779dc 100644 --- a/deepgym/src/deepgym/agents/base.py +++ b/deepgym/src/deepgym/agents/base.py @@ -11,6 +11,7 @@ from deepgym.exceptions import DeepGymError from deepgym.models import Observation, Trajectory from deepgym.session import Session +from deepgym.traces.schema import TraceMessage TerminationReason = Literal[ 'completed', @@ -95,6 +96,10 @@ class AgentResult(BaseModel): error: str | None = None cli_version: str | None = None """Installed CLI version observed by the sandbox supervisor via --version.""" + messages: tuple[TraceMessage, ...] = () + """Every model message as exchanged with the provider, for agents that record them (the + API tool agents): the system prompt, the task, each full response with its usage, and + each tool result. Complete however the run ended; empty for other agents.""" @runtime_checkable diff --git a/deepgym/tests/agents/test_api.py b/deepgym/tests/agents/test_api.py new file mode 100644 index 0000000..3a3f680 --- /dev/null +++ b/deepgym/tests/agents/test_api.py @@ -0,0 +1,469 @@ +"""API tool agents: real SDK requests over mock transports, tool calls against a loopback app. + +No provider is called and no sandbox is created: the SDKs talk to in-memory transports and +each tool call runs through a local Session to a real HTTP server on 127.0.0.1. +""" + +import importlib +import json +from datetime import datetime, timezone +from pathlib import Path + +import pytest + +from deepgym.agents import AnthropicToolAgent, Budget, OpenAIToolAgent, TokenPrices +from deepgym.agents.api import _NO_CALL, _SKIPPED +from deepgym.agents.providers import ANTHROPIC_MAX_TOKENS, SDK_RETRIES +from deepgym.models import MultiTurnEnvironment +from deepgym.session import LocalBackend, Session +from deepgym.traces import AgentMeta, EnvFingerprint, Provenance, TracedSession, load_trace +from tests.http_app import API_KEY, BIG, App, app_env, app_session + +UNLIMITED = Budget(max_turns=None, max_seconds=None) +NO_TIMEOUT = {'connect': None, 'read': None, 'write': None, 'pool': None} + + +def _transport(sdk: str): + """The HTTP package the installed SDK uses (httpx, or httpx2 in newer releases).""" + base = importlib.import_module(f'{sdk}._base_client') + return getattr(base, 'httpx2', None) or base.httpx + + +def _install(monkeypatch, sdk: str, respond) -> list: + """Route the SDK's client through a mock transport; return the requests it sends.""" + module = pytest.importorskip(sdk) + httpx = _transport(sdk) + sent: list = [] + + def handler(request): + sent.append({'body': json.loads(request.content), 'timeout': request.extensions['timeout']}) + return respond(httpx, len(sent) - 1) + + name = 'Anthropic' if sdk == 'anthropic' else 'OpenAI' + real = getattr(module, name) + + def client(**kwargs): + assert kwargs == {'max_retries': SDK_RETRIES} + transport = httpx.MockTransport(handler) + return real(api_key='test', http_client=httpx.Client(transport=transport), max_retries=0) + + monkeypatch.setattr(module, name, client) + return sent + + +# -- Anthropic: streamed messages ------------------------------------------------------------ + +START = { + 'type': 'message_start', + 'message': { + 'id': 'msg', + 'type': 'message', + 'role': 'assistant', + 'model': 'test', + 'content': [], + 'stop_reason': None, + 'stop_sequence': None, + 'usage': {'input_tokens': 100, 'output_tokens': 1}, + }, +} + + +def text(value: str) -> dict: + return {'type': 'text', 'text': value} + + +def use(name: str, arguments: dict, ident: str = 'call-1') -> dict: + return {'type': 'tool_use', 'id': ident, 'name': name, 'input': arguments} + + +def stream(*blocks: dict, stop: str = 'tool_use', finished: bool = True) -> bytes: + """An Anthropic SSE body with *blocks*, ending at message_stop when *finished*.""" + events: list[dict] = [START] + for index, block in enumerate(blocks): + if block['type'] == 'text': + start = {'type': 'text', 'text': ''} + delta = {'type': 'text_delta', 'text': block['text']} + else: + start = {**block, 'input': {}} + delta = {'type': 'input_json_delta', 'partial_json': json.dumps(block['input'])} + events += [ + {'type': 'content_block_start', 'index': index, 'content_block': start}, + {'type': 'content_block_delta', 'index': index, 'delta': delta}, + {'type': 'content_block_stop', 'index': index}, + ] + events.append( + {'type': 'message_delta', 'delta': {'stop_reason': stop}, 'usage': {'output_tokens': 5}} + ) + if finished: + events.append({'type': 'message_stop'}) + return b''.join(f'event: {e["type"]}\ndata: {json.dumps(e)}\n\n'.encode() for e in events) + + +def anthropic(monkeypatch, *bodies: bytes) -> list: + def respond(httpx, index): + headers = {'content-type': 'text/event-stream'} + return httpx.Response(200, headers=headers, content=bodies[index]) + + return _install(monkeypatch, 'anthropic', respond) + + +# -- OpenAI: Responses API ------------------------------------------------------------------- + + +def function_call(name: str, arguments: dict | str, ident: str = 'call-1') -> dict: + raw = arguments if isinstance(arguments, str) else json.dumps(arguments) + return { + 'type': 'function_call', + 'id': f'fc-{ident}', + 'call_id': ident, + 'name': name, + 'arguments': raw, + 'status': 'completed', + } + + +def message(value: str) -> dict: + return { + 'type': 'message', + 'id': 'msg-1', + 'role': 'assistant', + 'status': 'completed', + 'content': [{'type': 'output_text', 'text': value, 'annotations': []}], + } + + +def response(*output: dict, status: str = 'completed', **extra: object) -> dict: + return { + 'id': 'resp-1', + 'object': 'response', + 'created_at': 0, + 'model': 'test', + 'status': status, + 'output': list(output), + 'usage': { + 'input_tokens': 100, + 'output_tokens': 10, + 'total_tokens': 110, + 'input_tokens_details': {'cached_tokens': 20}, + 'output_tokens_details': {'reasoning_tokens': 4}, + }, + **extra, + } + + +def openai(monkeypatch, *bodies: dict) -> list: + def respond(httpx, index): + return httpx.Response(200, json={**bodies[index], 'id': f'resp-{index + 1}'}) + + return _install(monkeypatch, 'openai', respond) + + +GET_BUG = {'method': 'GET', 'path': '/rest/bug/1'} + + +def started(app: App) -> tuple[Session, object]: + session = app_session(app.url) + return session, session.start() + + +def test_anthropic_tool_call_returns_its_full_result_then_done_completes(app, monkeypatch): + sent = anthropic( + monkeypatch, + stream(use('tracker', GET_BUG)), + stream(text('Triaged.'), use('done', {}, 'call-2')), + ) + session, initial = started(app) + with session: + result = AnthropicToolAgent('test').run(session, initial, task='Triage', budget=UNLIMITED) + + assert result.termination_reason == 'completed' + assert result.final_text == 'Triaged.' + assert [action.action_type for _, action in result.trajectory.steps] == ['tool', 'done'] + assert session.done + observation = result.trajectory.steps[1][0] # what the tool call produced + (tool_result,) = sent[1]['body']['messages'][2]['content'] + assert tool_result == { + 'type': 'tool_result', + 'tool_use_id': 'call-1', + 'content': observation.content, + } + assert json.loads(json.loads(observation.content)['response']['body'])['bugs'][0][ + 'summary' + ] == BIG # fmt: skip + assert [request['method'] for request in app.requests] == ['GET'] + assert result.usage.input_tokens == 200 and result.turns == 2 + + +def test_openai_tool_call_returns_its_full_result_then_done_completes(app, monkeypatch): + sent = openai( + monkeypatch, + response(function_call('tracker', GET_BUG)), + response(message('Triaged.'), function_call('done', {}, 'call-2')), + ) + session, initial = started(app) + with session: + result = OpenAIToolAgent('test').run(session, initial, task='Triage', budget=UNLIMITED) + + assert result.termination_reason == 'completed' + assert result.final_text == 'Triaged.' + assert [action.action_type for _, action in result.trajectory.steps] == ['tool', 'done'] + observation = result.trajectory.steps[1][0] + assert sent[1]['body']['previous_response_id'] == 'resp-1' + assert sent[1]['body']['input'] == [ + {'type': 'function_call_output', 'call_id': 'call-1', 'output': observation.content} + ] + assert BIG in observation.content + assert result.usage.input_tokens == 160 and result.usage.cache_read_tokens == 40 + + +def test_requests_carry_only_declared_tools_and_done(app, monkeypatch): + claude = anthropic(monkeypatch, stream(use('done', {}))) + gpt = openai(monkeypatch, response(function_call('done', {}))) + for agent in (AnthropicToolAgent('test'), OpenAIToolAgent('test')): + session, initial = started(app) + with session: + assert agent.run(session, initial, task='T', budget=UNLIMITED).termination_reason == ( + 'completed' + ) + + claude_body, gpt_body = claude[0]['body'], gpt[0]['body'] + assert [tool['name'] for tool in claude_body['tools']] == ['tracker', 'done'] + assert all( + set(tool) == {'name', 'description', 'input_schema'} for tool in claude_body['tools'] + ) + assert set(claude_body) <= { + 'model', 'max_tokens', 'cache_control', 'thinking', 'output_config', 'system', 'tools', + 'messages', 'stream', + } # fmt: skip + assert [(tool['type'], tool['name']) for tool in gpt_body['tools']] == [ + ('function', 'tracker'), + ('function', 'done'), + ] + assert set(gpt_body) <= { + 'model', 'instructions', 'tools', 'input', 'reasoning', 'previous_response_id', + } # fmt: skip + for body in (claude_body, gpt_body): + text_of = json.dumps(body) + for hosted in ('web_search', 'web_fetch', 'code_execution', 'computer', 'mcp', 'bash'): + assert hosted not in text_of + assert API_KEY not in text_of # the app's credential never goes to the provider + + +def test_defaults_are_uncapped(app, monkeypatch): + claude = anthropic(monkeypatch, stream(use('done', {}))) + gpt = openai(monkeypatch, response(function_call('done', {}))) + for agent in (AnthropicToolAgent('test'), OpenAIToolAgent('test')): + session, initial = started(app) + with session: + agent.run(session, initial, task='T', budget=UNLIMITED) + + claude_body = claude[0]['body'] + assert claude_body['max_tokens'] == ANTHROPIC_MAX_TOKENS + assert claude_body['thinking'] == {'type': 'adaptive', 'display': 'summarized'} + assert claude_body['output_config'] == {'effort': 'max'} + assert gpt[0]['body']['reasoning'] == {'effort': 'max'} + assert 'max_output_tokens' not in gpt[0]['body'] + assert claude[0]['timeout'] == NO_TIMEOUT and gpt[0]['timeout'] == NO_TIMEOUT + + +def test_injection_shaped_calls_are_rejected_and_the_rest_of_the_batch_skipped(app, monkeypatch): + hostile = {'method': 'GET', 'path': '/rest/bug/1;touch pwned', 'query': {'q': '$(id)'}} + sent = anthropic( + monkeypatch, + stream(use('tracker', hostile), use('tracker', GET_BUG, 'call-2'), use('sh', {}, 'c3')), + stream(use('done', {})), + ) + session, initial = started(app) + with session: + workdir = Path(session._backend._dir) + result = AnthropicToolAgent('test').run(session, initial, task='T', budget=UNLIMITED) + assert not list(workdir.glob('pwned*')) + + results = sent[1]['body']['messages'][2]['content'] + assert results[0]['is_error'] and results[0]['content'].startswith('Tool call rejected') + assert [r['content'] for r in results[1:]] == [_SKIPPED, _SKIPPED] + assert all(r['is_error'] for r in results) + assert app.requests == [] + assert result.termination_reason == 'completed' + assert [action.action_type for _, action in result.trajectory.steps] == ['done'] + + +def test_openai_non_json_arguments_are_rejected(app, monkeypatch): + sent = openai( + monkeypatch, + response(function_call('tracker', '"; rm -rf / #')), + response(function_call('done', {})), + ) + session, initial = started(app) + with session: + OpenAIToolAgent('test').run(session, initial, task='T', budget=UNLIMITED) + + (output,) = sent[1]['body']['input'] + assert output['output'].startswith('Tool call rejected: arguments must be a JSON object') + assert app.requests == [] + + +@pytest.mark.parametrize('failure', ['eof', 'disconnect', 'eof_after_delta']) +def test_anthropic_unfinished_stream_never_acts(app, monkeypatch, failure): + anthropic_sdk = pytest.importorskip('anthropic') + httpx = _transport('anthropic') + body = stream(use('tracker', GET_BUG), finished=False) + if failure != 'eof_after_delta': + body = body.rsplit(b'event: message_delta', 1)[0] + + class Cut(httpx.SyncByteStream): + def __iter__(self): + yield body + if failure == 'disconnect': + raise httpx.RemoteProtocolError('peer closed connection') + + real = anthropic_sdk.Anthropic + + def client(**kwargs): + def handler(request): + return httpx.Response(200, headers={'content-type': 'text/event-stream'}, stream=Cut()) + + transport = httpx.MockTransport(handler) + return real(api_key='test', http_client=httpx.Client(transport=transport), max_retries=0) + + monkeypatch.setattr(anthropic_sdk, 'Anthropic', client) + session, initial = started(app) + with session: + result = AnthropicToolAgent('test').run(session, initial, task='T', budget=UNLIMITED) + + assert result.termination_reason == 'error' + assert not result.trajectory.steps and app.requests == [] # the received call never ran + assert result.usage.input_tokens == 100 # the partial usage is kept + assert result.cost.usd is None and 'billing unresolved' in result.cost.provenance + + +@pytest.mark.parametrize('details', [{'reason': 'max_output_tokens'}, None]) +def test_openai_incomplete_response_never_acts(app, monkeypatch, details): + body = response(function_call('tracker', GET_BUG), status='incomplete') + if details: + body['incomplete_details'] = details + sent = openai(monkeypatch, body) + session, initial = started(app) + with session: + result = OpenAIToolAgent('test').run(session, initial, task='T', budget=UNLIMITED) + + assert result.termination_reason == 'error' + assert ( + result.error == f'Response did not complete: {(details or {}).get("reason", "incomplete")}' + ) + assert len(sent) == 1 and not result.trajectory.steps and app.requests == [] + assert result.usage.output_tokens == 10 + + +def test_reply_without_tool_calls_is_reminded_not_completed(app, monkeypatch): + sent = anthropic( + monkeypatch, stream(text('All done, I think.'), stop='end_turn'), stream(use('done', {})) + ) + session, initial = started(app) + with session: + result = AnthropicToolAgent('test').run(session, initial, task='T', budget=UNLIMITED) + + assert sent[1]['body']['messages'][-1] == {'role': 'user', 'content': _NO_CALL} + assert result.termination_reason == 'completed' and result.turns == 2 + + +def test_done_text_in_a_tool_result_does_not_end_the_episode(app, monkeypatch): + sent = openai( + monkeypatch, + response(function_call('tracker', {'method': 'GET', 'path': '/rest/done'})), + response(function_call('done', {})), + ) + session, initial = started(app) + with session: + result = OpenAIToolAgent('test').run(session, initial, task='T', budget=UNLIMITED) + + assert 'DONE' in sent[1]['body']['input'][0]['output'] + assert len(sent) == 2 + assert [action.action_type for _, action in result.trajectory.steps] == ['tool', 'done'] + + +def test_lost_write_is_recorded_uncertain_and_never_retried(app, monkeypatch): + write = {'method': 'POST', 'path': '/rest/lost', 'body': {'status': 'RESOLVED'}} + sent = openai( + monkeypatch, response(function_call('tracker', write)), response(function_call('done', {})) + ) + session, initial = started(app) + with session: + result = OpenAIToolAgent('test').run(session, initial, task='T', budget=UNLIMITED) + + record = json.loads(sent[1]['body']['input'][0]['output']) + assert record['uncertain'] is True and record['request']['body'] == write['body'] + assert [request['method'] for request in app.requests] == ['POST'] + assert result.termination_reason == 'completed' + + +def test_traced_steps_and_result_carry_every_message(app, monkeypatch, tmp_path): + anthropic( + monkeypatch, + stream(use('tracker', GET_BUG)), + stream(text('Triaged.'), use('done', {}, 'call-2')), + ) + env = app_env(app.url) + provenance = Provenance( + env=EnvFingerprint.of(env), + agent=AgentMeta(name='api-tool', model='test'), + provider='local', + deepgym_version='test', + started_at=datetime.now(timezone.utc), + ) + with TracedSession(Session(env, LocalBackend()), env, provenance, tmp_path / 't.jsonl') as tr: + initial = tr.start() + result = AnthropicToolAgent('test').run(tr, initial, task=env.task, budget=UNLIMITED) + tr.score() + + assert API_KEY not in (tmp_path / 't.jsonl').read_text() + assert all(API_KEY not in message.content for message in result.messages) + trace = load_trace(tmp_path / 't.jsonl') + first, done = trace.steps + assert [m.role for m in first.messages] == ['system', 'user', 'assistant'] + assert json.loads(first.messages[-1].content)['usage']['input_tokens'] == 100 + assert first.action.tool == 'tracker' and first.action.arguments == GET_BUG + assert BIG in first.observation.content + assert [m.role for m in done.messages] == ['tool', 'assistant'] + assert done.action.action_type == 'done' + assert result.messages == (*first.messages, *done.messages) + assert json.loads(done.messages[0].content)['content'] == first.observation.content + + +def test_cost_cap_stops_before_running_returned_calls_and_keeps_the_reply(app, monkeypatch): + prices = TokenPrices( + input_per_million=100, + output_per_million=100, + cache_read_per_million=100, + cache_write_per_million=100, + source='test rates', + ) + anthropic(monkeypatch, stream(use('tracker', GET_BUG))) + session, initial = started(app) + with session: + result = AnthropicToolAgent('test', prices).run( + session, initial, task='T', budget=Budget(max_cost_usd=0.001) + ) + + assert result.termination_reason == 'cost_budget' + assert app.requests == [] and not result.trajectory.steps + assert json.loads(result.messages[-1].content)['content'][0]['name'] == 'tracker' + + +def test_turn_budget_is_the_callers(app, monkeypatch): + sent = anthropic(monkeypatch, stream(use('tracker', GET_BUG))) + session, initial = started(app) + with session: + result = AnthropicToolAgent('test').run( + session, initial, task='T', budget=Budget(max_turns=1, max_seconds=None) + ) + + assert result.termination_reason == 'turn_budget' and len(sent) == 1 + assert [action.action_type for _, action in result.trajectory.steps] == ['tool'] + + +@pytest.mark.parametrize('agent', [AnthropicToolAgent, OpenAIToolAgent]) +def test_environment_without_tools_is_refused(agent): + session = Session(MultiTurnEnvironment(task='T'), LocalBackend()) + with session, pytest.raises(ValueError, match='declares tools'): + agent('test').run(session, session.start(), task='T', budget=UNLIMITED) From 3ea530bcf554df6fc764149b43da1421f292d33b Mon Sep 17 00:00:00 2001 From: Abhishek Gahlot Date: Mon, 19 Oct 2026 02:46:15 +0530 Subject: [PATCH 05/21] docs: api tool agents and declaring environment tools --- deepgym/docs/agents.md | 55 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 54 insertions(+), 1 deletion(-) diff --git a/deepgym/docs/agents.md b/deepgym/docs/agents.md index 4f85eac..a216bca 100644 --- a/deepgym/docs/agents.md +++ b/deepgym/docs/agents.md @@ -1,7 +1,7 @@ # Agent harnesses `deepgym.agents` supplies `AnthropicComputerAgent`, `OpenAIComputerAgent`, -`ClaudeCodeAgent`, and `CodexAgent`. Each implements +`AnthropicToolAgent`, `OpenAIToolAgent`, `ClaudeCodeAgent`, and `CodexAgent`. Each implements `run(session, initial, *, task, budget) -> AgentResult`. Start the Session first; the caller owns grading and cleanup. `DeepGym.run_episode(env, agent, budget=...)` does that lifecycle and returns `(AgentResult, RunResult)`. Its existing callable @@ -54,6 +54,59 @@ structured provider codes use `provider_error`; message-only or unknown-code err remain `error`, regardless of their prose. Evidence and fixture limits are recorded in `tests/agents/fixtures/README.md`. +## API tool agents + +`AnthropicToolAgent` and `OpenAIToolAgent` give the model only the environment's +declared API tools plus a `done` tool. No hosted or server-side tool (web search, code +execution, browsing) is requested, and provider keys stay on the host. An environment +declares its tools as trusted configuration: + +```python +from deepgym.models import MultiTurnEnvironment +from deepgym.tools import HttpRoute, HttpTool + +env = MultiTurnEnvironment( + task='your task', + snapshot='your-app-snapshot', + root_service_start='your trusted service start command', + tools=( + HttpTool( + name='bugzilla', + description='Bugzilla 5.0 REST API. Put the API docs the task needs here.', + base_url='http://127.0.0.1:8080', + routes=( + HttpRoute(method='GET', path='/rest/bug'), + HttpRoute(method='GET', path='/rest/bug/{id}/comment'), + HttpRoute(method='POST', path='/rest/bug/{id}/comment'), + ), + headers={'X-BUGZILLA-API-KEY': app_key}, + ), + ), + allowed_actions=frozenset({'tool', 'done'}), + max_steps=None, +) +``` + +`base_url` must be on a loopback host. A call names a method and a path that fits a +declared route; `{name}` matches one path segment and `{name:path}` also takes an +encoded `/` (GitLab project paths and file paths). The model supplies only the method, +path, query and JSON body. Each call is a Session `tool` action: the host checks it, +then a constant runner sends it with http.client from inside the sandbox, as the agent +user in the offline network namespace. Its request arrives on stdin from a root-only +file. Redirects are returned, never followed. +Header values are treated as credentials and redacted from every record and result. +An environment with tools allows exactly `tool` and `done`. + +The observation, which the model gets back whole, records the full request, the full +response (status, every header, the body) and timing. A write whose response was lost +is marked `"uncertain": true` and is never retried. Only the `done` tool ends the +episode; app output that reads `DONE` does not. A reply without tool calls gets a +reminder. Defaults are uncapped: maximum effort, Anthropic's model-maximum +`max_tokens`, no OpenAI output limit, and only the turn, time and USD limits the +caller's `Budget` sets. `AgentResult.messages` holds every model message, full +responses with their usage included; under `dg.trace`, each step also carries the +messages that produced it. + ## CLI isolation and its exposure CLIs must already be installed in the sandbox image; the library installs nothing. From c7e68dcdf3f3c5222c7e2d4e58c40efd35ccea02 Mon Sep 17 00:00:00 2001 From: Abhishek Gahlot Date: Mon, 19 Oct 2026 02:56:07 +0530 Subject: [PATCH 06/21] fix: mask tool credentials everywhere and redact their echoed forms --- deepgym/src/deepgym/tools.py | 153 ++++++++++++++++++++++++++++------- deepgym/tests/http_app.py | 45 ++++++++++- deepgym/tests/test_tools.py | 116 +++++++++++++++++++++++++- 3 files changed, 281 insertions(+), 33 deletions(-) diff --git a/deepgym/src/deepgym/tools.py b/deepgym/src/deepgym/tools.py index 2164fcf..2ba2c64 100644 --- a/deepgym/src/deepgym/tools.py +++ b/deepgym/src/deepgym/tools.py @@ -13,6 +13,8 @@ from __future__ import annotations +import base64 +import html import ipaddress import json import re @@ -21,9 +23,9 @@ from collections.abc import Iterator, Mapping from pathlib import Path from typing import Literal -from urllib.parse import unquote, urlencode, urlsplit +from urllib.parse import quote, quote_plus, unquote, urlencode, urlsplit -from pydantic import BaseModel, ConfigDict, Field, JsonValue, field_validator +from pydantic import BaseModel, ConfigDict, Field, JsonValue, SecretStr, field_validator HttpMethod = Literal['GET', 'POST', 'PUT', 'PATCH', 'DELETE'] QueryScalar = str | int | float | bool @@ -50,17 +52,29 @@ _METHOD_OVERRIDES = frozenset( {'_method', 'x-http-method-override', 'x-http-method', 'x-method-override'} ) -_RESERVED_HEADERS = _METHOD_OVERRIDES | {'host', 'content-length', 'transfer-encoding'} +_RESERVED_HEADERS = _METHOD_OVERRIDES | { + 'host', 'content-length', 'transfer-encoding', 'content-type', +} # fmt: skip +# Response headers carrying credentials the app issues (a session cookie, say). +_ISSUED_CREDENTIALS = frozenset({'set-cookie', 'set-cookie2'}) +_URL_SAFE = str.maketrans('+/', '-_') +# A base64 run shorter than one 4-character group also turns up in unrelated text. +_MIN_BASE64 = 4 _PORTS = {'http': 80, 'https': 443} def _readings(text: str) -> Iterator[str]: """*text* and each further percent-decoding of it until none changes, NFKC-normalized: every way an app that decodes once, twice, ... or folds compatibility forms may read it. - Each decoding is shorter than the last, so this ends.""" + Each decoding is shorter than the last, so this ends. + + Raises: + UnicodeDecodeError: If a decoding is not UTF-8, such as an overlong ``%C0%AE`` + that some servers read as ``.``. + """ while True: yield unicodedata.normalize('NFKC', text) - decoded = unquote(text) + decoded = unquote(text, errors='strict') if decoded == text: return text = decoded @@ -68,13 +82,69 @@ def _readings(text: str) -> Iterator[str]: def _unsafe(path: str) -> bool: """Whether a reading of *path* has a dot segment (``..;x`` too), a backslash or a - control character.""" - return any( - '\\' in form - or any(char < ' ' or char == '\x7f' for char in form) - or any(_DOTS.fullmatch(part.split(';', 1)[0]) for part in form.split('/')) - for form in _readings(path) - ) + control character, or one is not UTF-8.""" + try: + return any( + '\\' in form + or any(char < ' ' or char == '\x7f' for char in form) + or any(_DOTS.fullmatch(part.split(';', 1)[0]) for part in form.split('/')) + for form in _readings(path) + ) + except UnicodeDecodeError: + return True + + +def _needles(name: str, value: str) -> set[str]: + """A credential header's value and the parts an app may echo on their own: the + credentials after an Authorization scheme, a Basic pair and its password, each Cookie + value.""" + needles = {value} + if name.lower() in ('authorization', 'proxy-authorization'): + scheme, _, credentials = value.partition(' ') + needles.add(credentials.strip()) + if scheme.lower() == 'basic': + try: + pair = base64.b64decode(credentials.strip(), validate=True).decode('utf-8') + except ValueError: # binascii.Error and UnicodeDecodeError are ValueErrors + pair = '' + needles |= {pair, pair.partition(':')[2]} + if name.lower() == 'cookie': + needles |= {part.partition('=')[2].strip() for part in value.split(';')} + return {needle for needle in needles if needle} + + +def _forms(needle: str) -> set[str]: + """The forms an app may echo *needle* in: as is, JSON- and HTML-escaped, percent- and + form-encoded (matched in any letter case, so either hex case).""" + return { + needle, + json.dumps(needle)[1:-1], + json.dumps(needle, ensure_ascii=False)[1:-1], + html.escape(needle), + quote(needle, safe=''), + quote_plus(needle, safe=''), + } + + +def _base64_forms(needle: str) -> set[str]: + """The base64 characters *needle*'s bytes alone determine, at each of the three byte + alignments it can have inside a longer encoding, in the standard and URL-safe alphabets. + Matched as they are: base64 is case-sensitive.""" + raw = needle.encode('utf-8') + forms = set() + for shift in range(3): + encoded = base64.b64encode(bytes(shift) + raw).decode('ascii') + inner = encoded[(8 * shift + 5) // 6 : 8 * (shift + len(raw)) // 6] + forms |= {inner, inner.translate(_URL_SAFE)} + return {form for form in forms if len(form) >= _MIN_BASE64} + + +def _replaced(text: str, forms: set[str], flags: int) -> str: + """*text* with every one of *forms* replaced by REDACTED, longest first.""" + if not forms: + return text + pattern = '|'.join(map(re.escape, sorted(forms, key=len, reverse=True))) + return re.sub(pattern, REDACTED, text, flags=flags) class HttpRoute(BaseModel): @@ -136,10 +206,11 @@ class HttpTool(BaseModel): """An API tool: HTTP requests to declared routes under one loopback base URL. The model sees *name*, *description* (the place for the API docs a task needs) and the - routes. *headers* go with every request and are treated as credentials (an API key for - the local app, say): their values appear in no record, result, trace or error, and - never on disk where the agent user could read them. Responses come back whole: status, - every header (pagination links included) and the full body. + routes. *headers* go with every request and are credentials (an API key or login for + the local app, say), kept as secrets: masked in repr, dumps, fingerprints and + validation errors, redacted from every record and result (see :meth:`redact`), and + never on disk where the agent user could read them. Responses come back whole (status, + every header, pagination links included, and the full body) but for those redactions. Example:: @@ -156,12 +227,12 @@ class HttpTool(BaseModel): ) """ - model_config = ConfigDict(frozen=True, extra='forbid') + model_config = ConfigDict(frozen=True, extra='forbid', hide_input_in_errors=True) name: str = Field(pattern=r'^[A-Za-z0-9_-]{1,64}$') description: str = Field(min_length=1) base_url: str routes: tuple[HttpRoute, ...] = Field(min_length=1) - headers: dict[str, str] = Field(default_factory=dict) + headers: dict[str, SecretStr] = Field(default_factory=dict) @field_validator('name') @classmethod @@ -205,9 +276,13 @@ def check_base_url(cls, value: str) -> str: @field_validator('headers') @classmethod - def check_headers(cls, value: dict[str, str]) -> dict[str, str]: - """Accept token names, none that reroutes or reframes a request, with plain values.""" - for name, text in value.items(): + def check_headers(cls, value: dict[str, SecretStr]) -> dict[str, SecretStr]: + """Accept distinct token names, none that reroutes or reframes a request, with + plain values.""" + if len({name.lower() for name in value}) != len(value): + raise ValueError('header names must differ in more than letter case') + for name, secret in value.items(): + text = secret.get_secret_value() if ( not _TOKEN.fullmatch(name) or name.lower() in _RESERVED_HEADERS @@ -296,7 +371,7 @@ def prepare(self, arguments: Mapping[str, JsonValue]) -> dict[str, JsonValue]: ] base = urlsplit(self.base_url) target = base.path + call.path + (f'?{urlencode(pairs)}' if pairs else '') - headers = dict(self.headers) + headers = {name: secret.get_secret_value() for name, secret in self.headers.items()} record: dict[str, JsonValue] = { 'method': call.method, 'url': f'{base.scheme}://{base.netloc}{target}', @@ -322,12 +397,34 @@ def prepare(self, arguments: Mapping[str, JsonValue]) -> dict[str, JsonValue]: } def redact(self, text: str) -> str: - """*text* with each header value, as is and JSON-escaped, replaced by REDACTED, in - case the app echoes one back.""" - for value in self.headers.values(): - for form in {value, json.dumps(value)[1:-1]}: - text = text.replace(form, REDACTED) - return text + """Return a runner's record (or any text) with the credentials an app may echo + redacted, where the session records it. + + A response header named like a tool header, or carrying a credential the app issued + (Set-Cookie), loses its whole value. Then every form of every header value and of + its echoable parts (see :func:`_needles`) is replaced wherever it appears (body, + headers, a redirect's Location, an error): escaped or encoded in any letter case + (:func:`_forms`), and in base64 (:func:`_base64_forms`). + """ + try: + record = json.loads(text) + except ValueError: + record = None + if isinstance(record, dict) and isinstance(record.get('response'), dict): + response = record['response'] + hidden = {name.lower() for name in self.headers} | _ISSUED_CREDENTIALS + response['headers'] = [ + [name, REDACTED if str(name).lower() in hidden else value] + for name, value in response.get('headers', []) + ] + text = json.dumps(record, ensure_ascii=False) + needles = { + needle + for name, secret in self.headers.items() + for needle in _needles(name, secret.get_secret_value()) + } + text = _replaced(text, {form for n in needles for form in _forms(n)}, re.IGNORECASE) + return _replaced(text, {form for n in needles for form in _base64_forms(n)}, 0) def run_command(python: str) -> str: diff --git a/deepgym/tests/http_app.py b/deepgym/tests/http_app.py index 5e619ee..35a15a4 100644 --- a/deepgym/tests/http_app.py +++ b/deepgym/tests/http_app.py @@ -1,9 +1,12 @@ """A loopback HTTP app for API tool tests: real sockets, no sandbox, no provider call.""" +import base64 +import html import json import threading import time from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from urllib.parse import quote, quote_plus from deepgym.models import MultiTurnEnvironment from deepgym.session import LocalBackend, Session @@ -27,6 +30,43 @@ ) +def _echo(headers) -> dict: + """Every request header value back, in the forms an app may echo it in.""" + received = dict(headers.items()) + authorization = received.get('Authorization', '') + pair = ( + base64.b64decode(authorization.split(' ', 1)[1]).decode() + if authorization.startswith('Basic ') + else '' + ) + return { + 'raw': received, + 'quoted': {name: quote(value, safe='') for name, value in received.items()}, + 'lower_hex': {name: quote(value, safe='').lower() for name, value in received.items()}, + 'form': {name: quote_plus(value, safe='') for name, value in received.items()}, + 'html': f'

{"".join(html.escape(value) for value in received.values())}

', + 'upper': {name: value.upper() for name, value in received.items()}, + 'basic': {'pair': pair, 'password': pair.partition(':')[2]}, + 'base64': { + name: base64.b64encode(f'{prefix}{value}!'.encode()).decode() + for name, value in received.items() + for prefix in ('', 'x', 'xy') # each byte alignment inside a longer encoding + }, + } + + +def _echoed(headers) -> dict[str, str]: + """Response headers that hand a credential back: an issued cookie, an echoed key, a + redirect carrying it.""" + key = headers.get('X-API-KEY', '') + return { + 'Set-Cookie': 'session=issued-session-id; Path=/; HttpOnly', + 'x-api-key': key, + 'X-Seen': key, + 'Location': f'/next?key={quote(key, safe="")}', + } + + class App: """A loopback server that records every request and answers a few fixed routes.""" @@ -50,6 +90,7 @@ def _handle(self) -> None: body = self.rfile.read(length).decode('utf-8') app.requests.append( { + 'line': self.requestline, 'method': self.command, 'path': self.path, 'headers': {name.lower(): value for name, value in self.headers.items()}, @@ -66,8 +107,8 @@ def _handle(self) -> None: self._send(302, b'', {'Location': app.redirect_to}) elif route.endswith('/done'): self._send(200, b'DONE\n{"done": true}\n', {'Content-Type': 'text/plain'}) - elif route.endswith('/echo'): # an app that reflects request headers - self._send(200, json.dumps(dict(self.headers.items())).encode(), {}) + elif route.endswith('/echo'): # an app that reflects its credentials + self._send(200, json.dumps(_echo(self.headers)).encode(), _echoed(self.headers)) elif route.endswith('/binary'): self._send(200, b'\xff\x00\xfe', {}) elif route.endswith('/comment'): diff --git a/deepgym/tests/test_tools.py b/deepgym/tests/test_tools.py index e13bb68..b641c78 100644 --- a/deepgym/tests/test_tools.py +++ b/deepgym/tests/test_tools.py @@ -1,18 +1,21 @@ """Tests for declared HTTP API tools run as Session tool actions, against a loopback app.""" +import base64 +import html import json import shlex from pathlib import Path +from urllib.parse import quote, quote_plus import pytest -from pydantic import ValidationError +from pydantic import SecretStr, ValidationError from deepgym.exceptions import ActionError from deepgym.models import Action, MultiTurnEnvironment from deepgym.qa.agents import do_everything, random_agent from deepgym.sandboxes import ExecResult -from deepgym.session import SandboxBackend, Session -from deepgym.tools import HttpRoute, HttpTool, run_command +from deepgym.session import LocalBackend, SandboxBackend, Session +from deepgym.tools import REDACTED, HttpRoute, HttpTool, run_command from tests.http_app import API_KEY, BIG, ROUTES, App, app_env, app_session, tool @@ -274,6 +277,111 @@ def test_sandbox_stdin_is_staged_root_only_and_never_on_a_command_line() -> None assert API_KEY not in command +SECRET = 'k3y/with+special&"chars<>' +PASSWORD = 's3cret-pass:word' + + +def test_echoed_credentials_are_redacted_in_every_form(app: App) -> None: + basic = base64.b64encode(f'admin:{PASSWORD}'.encode()).decode() + env = app_env(app.url).model_copy( + update={ + 'tools': ( + tool(app.url).model_copy( + update={ + 'headers': { + 'X-API-KEY': SecretStr(SECRET), + 'Authorization': SecretStr(f'Basic {basic}'), + } + } + ), + ) + } + ) + with Session(env, LocalBackend()) as session: + session.start() + + content = session.step(call('/rest/echo')).observation.content + + seen = app.requests[0]['headers'] + assert seen['x-api-key'] == SECRET # the app did get both credentials + assert seen['authorization'] == f'Basic {basic}' + for secret in (SECRET, f'Basic {basic}', basic, f'admin:{PASSWORD}', PASSWORD): + for form in ( + secret, + secret.upper(), + json.dumps(secret)[1:-1], + html.escape(secret), + quote(secret, safe=''), + quote(secret, safe='').lower(), + quote_plus(secret, safe=''), + ): + assert form not in content, form + response = json.loads(content)['response'] + headers = dict(response['headers']) + assert headers['Set-Cookie'] == headers['x-api-key'] == headers['X-Seen'] == REDACTED + assert headers['Location'] == f'/next?key={REDACTED}' + assert 'issued-session-id' not in content + for prefix in (b'', b'x', b'xy'): # the 12 middle characters depend on the key alone + encoded = base64.b64encode(prefix + SECRET.encode() + b'!').decode() + assert encoded[4:16] not in content + assert json.loads(response['body'])['basic'] == {'pair': REDACTED, 'password': REDACTED} + + +def test_credentials_are_masked_in_dumps_fingerprints_and_errors() -> None: + from deepgym.traces.schema import environment_sha256 + + one, other = ( + app_env('http://127.0.0.1').model_copy( + update={'tools': (tool('http://127.0.0.1').model_copy(update={'headers': headers}),)} + ) + for headers in ({'X-API-KEY': SecretStr(SECRET)}, {'X-API-KEY': SecretStr('other')}) + ) + assert SECRET not in repr(one) and SECRET not in one.model_dump_json() + # A fingerprint hash of a secret could be brute-forced; the secret is not in it. + assert environment_sha256(one) == environment_sha256(other) + for headers in ({'Host': SECRET}, {'X-Key': f'{SECRET}\n'}, {'X-Key': SECRET, 'x-key': SECRET}): + with pytest.raises(ValidationError) as invalid: + HttpTool( + name='t', + description='d', + base_url='http://127.0.0.1', + routes=ROUTES, + headers=headers, + ) + assert SECRET not in str(invalid.value) + + +@pytest.mark.parametrize('path', ['/rest/bug/%C0%AE%C0%AE', '/rest/bug/%C0%AF', '/rest/bug/%FF']) +def test_escapes_that_are_not_utf8_are_rejected(app: App, path: str) -> None: + with app_session(app.url) as session: + session.start() + with pytest.raises(ActionError, match='no dot segments'): + session.step(call(path)) + assert app.requests == [] + + +@pytest.mark.parametrize( + ('path', 'method', 'body'), + [ + ('/rest/bug/%C3%A9', 'GET', None), + ('/rest/bug/a;b', 'GET', None), + ('/rest/bug/1/comment', 'POST', {'comment': 'x'}), + ], +) +def test_the_request_line_is_the_validated_target( + app: App, path: str, method: str, body: dict | None +) -> None: + with app_session(app.url) as session: + session.start() + arguments = {'query': {'q': 'a b'}} | ({'body': body} if body else {}) + result = record(session, call(path, method, **arguments)) + + (seen,) = app.requests + target = result['request']['url'].removeprefix(app.url) + assert target == f'{path}?q=a+b' + assert seen['line'] == f'{method} {target} HTTP/1.1' # exactly what was validated + + @pytest.mark.parametrize('method', ['get', 'Get', 'GET ', 'HEAD', 'OPTIONS', 'TRACE', 'CONNECT']) def test_method_must_be_a_declared_one_exactly(app: App, method: str) -> None: with app_session(app.url) as session: @@ -315,6 +423,8 @@ def test_method_overrides_are_rejected(app: App, arguments: dict) -> None: {'X-Key': 'a\r\nX-Injected: 1'}, {'X-Key': ''}, {'Bad Name': 'x'}, + {'Content-Type': 'text/plain'}, + {'X-Key': 'a', 'x-key': 'b'}, ], ) def test_tool_headers_cannot_reroute_or_reframe_requests(headers: dict) -> None: From 954af94d9fdbf2f1118b36f72353235e40ae7680 Mon Sep 17 00:00:00 2001 From: Abhishek Gahlot Date: Mon, 19 Oct 2026 02:56:16 +0530 Subject: [PATCH 07/21] docs: how api tool credentials are masked and redacted --- deepgym/docs/agents.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/deepgym/docs/agents.md b/deepgym/docs/agents.md index a216bca..3d08987 100644 --- a/deepgym/docs/agents.md +++ b/deepgym/docs/agents.md @@ -94,8 +94,12 @@ path, query and JSON body. Each call is a Session `tool` action: the host checks then a constant runner sends it with http.client from inside the sandbox, as the agent user in the offline network namespace. Its request arrives on stdin from a root-only file. Redirects are returned, never followed. -Header values are treated as credentials and redacted from every record and result. -An environment with tools allows exactly `tool` and `done`. +Header values are credentials: masked in dumps, environment fingerprints and validation +errors, and redacted from every record and result, including forms an app may echo +(escaped, percent-encoded, base64, a Basic pair or password, any letter case) and the +cookies it sets. Pattern redaction cannot catch every transformation, so declare only +routes that never return credentials. An environment with tools allows exactly `tool` +and `done`. The observation, which the model gets back whole, records the full request, the full response (status, every header, the body) and timing. A write whose response was lost From df335ae80860588987d4c34ebe3c1fa9dd315d2a Mon Sep 17 00:00:00 2001 From: Abhishek Gahlot Date: Mon, 19 Oct 2026 09:13:07 +0530 Subject: [PATCH 08/21] fix: settle tool writes of unknown outcome before a passing verdict --- deepgym/src/deepgym/api/schemas.py | 2 + deepgym/src/deepgym/models.py | 10 +++++ deepgym/src/deepgym/session.py | 70 +++++++++++++++++++++++++++++- deepgym/tests/test_api_schemas.py | 2 +- deepgym/tests/test_tools.py | 58 ++++++++++++++++++++++++- 5 files changed, 138 insertions(+), 4 deletions(-) diff --git a/deepgym/src/deepgym/api/schemas.py b/deepgym/src/deepgym/api/schemas.py index 679dad0..6d83236 100644 --- a/deepgym/src/deepgym/api/schemas.py +++ b/deepgym/src/deepgym/api/schemas.py @@ -217,6 +217,8 @@ def reject_runtime( raise ValueError('image and snapshot cannot be set over the API') if not isinstance(v, str) and v.root_service_start: raise ValueError('root_service_start cannot be set over the API') + if not isinstance(v, str) and v.root_tool_settle: + raise ValueError('root_tool_settle cannot be set over the API') if not isinstance(v, str) and v.cli_provider_egress: raise ValueError('cli_provider_egress cannot be set over the API') if not isinstance(v, str) and v.volumes: diff --git a/deepgym/src/deepgym/models.py b/deepgym/src/deepgym/models.py index e654036..77ddb00 100644 --- a/deepgym/src/deepgym/models.py +++ b/deepgym/src/deepgym/models.py @@ -576,6 +576,16 @@ class MultiTurnEnvironment(BaseModel): agent ends the episode only by an explicit ``done``, and no shell, file or GUI action runs as the agent user beside the tool runner, which holds the tools' credentials.""" + root_tool_settle: str = '' + """Trusted reconciliation of tool writes whose outcome is unknown: a response lost, cut + short or stopped by the step timeout, after which the app may still apply the write. + Runs before the final verifier, only when such a write exists, as root in the offline + namespace (as the session user on LocalBackend, which isolates nothing), with + setup_timeout. It must exit 0 only once the app has applied or abandoned every request + it received: stop its server, wait for its queue to drain, or check the write in its + database. Without it, or if it fails, a score above 0 gets no verdict; a 0 stands. + Never accept this from an agent.""" + desktop: bool = False """Start a desktop (Xvfb + xfce) and return screenshots. Needs the Daytona backend.""" diff --git a/deepgym/src/deepgym/session.py b/deepgym/src/deepgym/session.py index a4ede8e..5268e4a 100644 --- a/deepgym/src/deepgym/session.py +++ b/deepgym/src/deepgym/session.py @@ -990,6 +990,8 @@ def __init__(self, env: MultiTurnEnvironment, backend: SessionBackend) -> None: # The session's start, and the seconds spent applying the agent's actions since. self._opened = time.monotonic() self._acting_s = 0.0 + # Request records of tool writes whose outcome is unknown, until settled (see score()). + self._unsettled: list[JsonValue] = [] # One sandbox, one actor at a time: concurrent step/score/close calls queue up. self._lock = threading.Lock() @@ -1257,6 +1259,11 @@ def score(self) -> RunResult: after reading the agent's output (``solution_timeout``), and a provider lifetime the agent's own actions ran out (see :meth:`_ran_out_the_clock`). + A tool write whose outcome is unknown (its response lost, cut short or stopped) may + still land after grading, so the environment's ``root_tool_settle`` must settle it + first; until then a score above 0 has no verdict (``unsettled_write``), and a 0 + stands. + Raises: VerifierError: If, before it read the agent's output, the verifier timed out or its output is not valid protocol JSON or isn't backed by its exit code; or its @@ -1281,6 +1288,7 @@ def score(self) -> RunResult: start = time.perf_counter() acted = bool(self._trajectory.steps) try: + unsettled = self._settle() (exit_code, stdout, stderr), consulted = self._verify(self._env.final_verifier_code) except SandboxDiedError as exc: if not acted: @@ -1304,6 +1312,8 @@ def score(self) -> RunResult: if consulted and not result.has_verdict and result.error_type != 'sandbox_error': detail = VerifierError(f'{result.error_type}: {result.output}') return _failed_after_reading(detail, elapsed_ms, self.sandbox_id, stderr) + if unsettled and result.has_verdict and result.score > 0: + return _unsettled_result(result, unsettled, self._unsettled) return result def close(self) -> None: @@ -1398,15 +1408,45 @@ def _send( The request reaches the constant runner command as data on its stdin, never through a file or argument the agent user could read or change, and the tool's header values are redacted from what comes back. It is never retried: a runner the timeout stops - reports no response, and a mutation it may have sent is recorded as uncertain. + reports no response, and a mutation it may have sent is recorded as uncertain and + kept unsettled for :meth:`score`, since stopping the runner never stops the app. """ command = run_command(self._backend.python) try: exit_code, output = self._backend.exec_input(command, json.dumps(request), timeout) except TimeoutError: - return lost_response(request, f'the request was stopped after {timeout}s'), None + exit_code = None + output = lost_response(request, f'the request was stopped after {timeout}s') + if request['mutation'] and exit_code != 0 and not _never_sent(output): + self._unsettled.append(request['record']) return tool.redact(output), exit_code + def _settle(self) -> str | None: + """Settle unsettled tool writes with the environment's ``root_tool_settle``; return + why they stay unsettled, or None once none are. + + Only the environment's own reconciliation settles them: the app may apply a write + whose runner was stopped, and a model's later reads prove nothing. + """ + if not self._unsettled: + return None + command, timeout = self._env.root_tool_settle, self._env.setup_timeout + if not command: + return 'the environment declares no root_tool_settle' + try: + if isinstance(self._backend, SandboxBackend): + settled = self._backend.exec_root( + command, timeout, offline=True, env={'PATH': TRUSTED_PATH} + ) + else: + settled = self._backend.exec(command, timeout) + except TimeoutError: + return f'root_tool_settle timed out after {timeout}s' + if settled.exit_code != 0: + return f'root_tool_settle exited {settled.exit_code}' + self._unsettled.clear() + return None + def _run_python(self, filename: str, code: str, timeout: float | None = None) -> ExecResult: self._backend.write_file(filename, code.encode('utf-8')) return self._backend.exec( @@ -1711,6 +1751,32 @@ def _failed_after_reading( return build_run_result(verdict, elapsed_ms, sandbox_id, stderr=stderr, exit_code=1) +def _unsettled_result(result: RunResult, why: str, writes: Sequence[JsonValue]) -> RunResult: + """Return *result*, a score above 0 over tool *writes* the app may still apply, with + no verdict: the write may have earned it, or may yet undo it. A 0 keeps its verdict, + or a policy could dodge it by sending a write that never answers.""" + listed = json.dumps(writes, ensure_ascii=False) + return RunResult( + score=0.0, + passed=False, + output=f'Not scored, tool writes unsettled ({why}): {listed}. ' + f'The verifier said: {result.output}', + stderr=result.stderr, + exit_code=result.exit_code, + execution_time_ms=result.execution_time_ms, + sandbox_id=result.sandbox_id, + error_type='unsettled_write', + ) + + +def _never_sent(output: str) -> bool: + """Whether a tool runner's record shows its request never reached the app.""" + try: + return json.loads(output).get('uncertain') is False + except (ValueError, AttributeError): + return False + + def _reported(executed: ExecutionResult) -> VerifierOutput: """Return *executed*, a judged 0, as the verifier would have printed it.""" return VerifierOutput(executed.exit_code, executed.verifier_result.model_dump_json(), '') diff --git a/deepgym/tests/test_api_schemas.py b/deepgym/tests/test_api_schemas.py index ddab303..10b1c46 100644 --- a/deepgym/tests/test_api_schemas.py +++ b/deepgym/tests/test_api_schemas.py @@ -9,7 +9,7 @@ @pytest.mark.parametrize('environment_type', ['multi-turn', 'computer-use']) -@pytest.mark.parametrize('field', ['image', 'snapshot', 'root_service_start']) +@pytest.mark.parametrize('field', ['image', 'snapshot', 'root_service_start', 'root_tool_settle']) def test_inline_session_rejects_privileged_configuration( environment_type: str, field: str, diff --git a/deepgym/tests/test_tools.py b/deepgym/tests/test_tools.py index b641c78..2d06fb3 100644 --- a/deepgym/tests/test_tools.py +++ b/deepgym/tests/test_tools.py @@ -11,7 +11,7 @@ from pydantic import SecretStr, ValidationError from deepgym.exceptions import ActionError -from deepgym.models import Action, MultiTurnEnvironment +from deepgym.models import Action, MultiTurnEnvironment, RunResult from deepgym.qa.agents import do_everything, random_agent from deepgym.sandboxes import ExecResult from deepgym.session import LocalBackend, SandboxBackend, Session @@ -189,6 +189,62 @@ def test_write_stopped_by_the_step_timeout_is_uncertain(app: App) -> None: assert len(app.requests) == 1 +def verifier(score: float) -> str: + return f'import json\nprint(json.dumps({{"score": {score}, "passed": {score == 1}}}))\n' + + +def scored(session: Session, *actions: Action) -> RunResult: + session.start() + for action in actions: + session.step(action) + session.step(Action(action_type='done')) + return session.score() + + +@pytest.mark.parametrize( + ('path', 'settings'), [('/rest/lost', {}), ('/rest/slow', {'timeout_per_step': 2})] +) +def test_a_pass_over_an_unsettled_write_gets_no_verdict( + app: App, path: str, settings: dict +) -> None: + write = call(path, 'POST', body={'status': 'RESOLVED'}) + with app_session(app.url, final_verifier_code=verifier(1.0), **settings) as session: + result = scored(session, write) + + assert not result.has_verdict and result.error_type == 'unsettled_write' + assert result.score == 0 and not result.passed + assert f'{app.url}{path}' in result.output and 'no root_tool_settle' in result.output + + +def test_a_zero_over_an_unsettled_write_stands(app: App) -> None: + write = call('/rest/lost', 'POST', body={'status': 'RESOLVED'}) + with app_session(app.url, final_verifier_code=verifier(0.0)) as session: + result = scored(session, write) + assert result.has_verdict and result.score == 0 + + +@pytest.mark.parametrize(('settle', 'graded'), [('true', True), ('exit 3', False)]) +def test_settled_writes_are_graded(app: App, settle: str, graded: bool) -> None: + write = call('/rest/lost', 'POST', body={'status': 'RESOLVED'}) + with app_session( + app.url, final_verifier_code=verifier(1.0), root_tool_settle=settle + ) as session: + result = scored(session, write) + assert result.has_verdict is graded and result.passed is graded + assert graded or 'root_tool_settle exited 3' in result.output + + +def test_reads_and_writes_that_never_arrived_leave_nothing_to_settle(app: App) -> None: + closed = App() + closed.close() + for base_url, action in ( + (app.url, call('/rest/lost')), + (closed.url, call('/rest/bug/1/comment', 'POST', body={'comment': 'x'})), + ): + with app_session(base_url, final_verifier_code=verifier(1.0)) as session: + assert scored(session, action).passed + + def test_done_text_from_the_app_never_ends_the_episode(app: App) -> None: with app_session(app.url) as session: session.start() From 80f5b1a2a54b257236320c292c658fc89a0d81c1 Mon Sep 17 00:00:00 2001 From: Abhishek Gahlot Date: Mon, 19 Oct 2026 09:13:14 +0530 Subject: [PATCH 09/21] fix: reject inline api tools in session requests --- deepgym/src/deepgym/api/schemas.py | 2 ++ deepgym/src/deepgym/tools.py | 4 +++- deepgym/tests/test_api_schemas.py | 20 ++++++++++++++++++++ 3 files changed, 25 insertions(+), 1 deletion(-) diff --git a/deepgym/src/deepgym/api/schemas.py b/deepgym/src/deepgym/api/schemas.py index 6d83236..faa58be 100644 --- a/deepgym/src/deepgym/api/schemas.py +++ b/deepgym/src/deepgym/api/schemas.py @@ -223,6 +223,8 @@ def reject_runtime( raise ValueError('cli_provider_egress cannot be set over the API') if not isinstance(v, str) and v.volumes: raise ValueError('volumes cannot be set over the API') + if not isinstance(v, str) and v.tools: + raise ValueError('tools cannot be set over the API') return v diff --git a/deepgym/src/deepgym/tools.py b/deepgym/src/deepgym/tools.py index 2ba2c64..1796e2a 100644 --- a/deepgym/src/deepgym/tools.py +++ b/deepgym/src/deepgym/tools.py @@ -154,7 +154,9 @@ class HttpRoute(BaseModel): hold an encoded ``/`` either, since an app that decodes before routing would read more segments; ``{name:path}`` allows one (GitLab's ``group%2Fproject`` ids and file paths). Every other segment must match literally. An endpoint no route names (an admin page, a - script runner) is unreachable. + script runner) is unreachable. A route checks only method and path, so one that + dispatches on its body (GraphQL, JSON-RPC) allows every operation the tool's + credentials permit. """ model_config = ConfigDict(frozen=True, extra='forbid') diff --git a/deepgym/tests/test_api_schemas.py b/deepgym/tests/test_api_schemas.py index 10b1c46..d18acd0 100644 --- a/deepgym/tests/test_api_schemas.py +++ b/deepgym/tests/test_api_schemas.py @@ -68,3 +68,23 @@ def test_inline_api_cannot_mount_volumes(as_model: bool) -> None: environment = MultiTurnEnvironment(**environment) with pytest.raises(ValidationError, match='volumes cannot be set over the API'): CreateSessionRequest(environment=environment) + + +@pytest.mark.parametrize('as_model', [False, True]) +def test_inline_api_cannot_declare_tools(as_model: bool) -> None: + environment = { + 'task': 'task', + 'allowed_actions': ['tool', 'done'], + 'tools': [ + { + 'name': 'jenkins', + 'description': 'Jenkins', + 'base_url': 'http://127.0.0.1:8080', + 'routes': [{'method': 'POST', 'path': '/scriptText'}], + } + ], + } + if as_model: + environment = MultiTurnEnvironment(**environment) + with pytest.raises(ValidationError, match='tools cannot be set over the API'): + CreateSessionRequest(environment=environment) From 424e81c46ea31e7ea2b5f6f42144ea7d16e05947 Mon Sep 17 00:00:00 2001 From: Abhishek Gahlot Date: Mon, 19 Oct 2026 09:13:32 +0530 Subject: [PATCH 10/21] fix: redact tool records field by field so they stay valid json --- deepgym/src/deepgym/tools.py | 38 ++++++++++++++++++++++++++---------- deepgym/tests/test_tools.py | 20 +++++++++++++++++++ 2 files changed, 48 insertions(+), 10 deletions(-) diff --git a/deepgym/src/deepgym/tools.py b/deepgym/src/deepgym/tools.py index 1796e2a..9511101 100644 --- a/deepgym/src/deepgym/tools.py +++ b/deepgym/src/deepgym/tools.py @@ -406,27 +406,45 @@ def redact(self, text: str) -> str: (Set-Cookie), loses its whole value. Then every form of every header value and of its echoable parts (see :func:`_needles`) is replaced wherever it appears (body, headers, a redirect's Location, an error): escaped or encoded in any letter case - (:func:`_forms`), and in base64 (:func:`_base64_forms`). + (:func:`_forms`), and in base64 (:func:`_base64_forms`). In a JSON record only its + string values are searched, so it stays JSON with its numbers and booleans (a + credential ``200`` leaves status 200 alone); its keys are the runner's or the + model's, never the app's. """ + needles = { + needle + for name, secret in self.headers.items() + for needle in _needles(name, secret.get_secret_value()) + } + forms = {form for needle in needles for form in _forms(needle)} + encoded = {form for needle in needles for form in _base64_forms(needle)} + + def hide(text: str) -> str: + return _replaced(_replaced(text, forms, re.IGNORECASE), encoded, 0) + + def scrub(value: JsonValue) -> JsonValue: + if isinstance(value, str): + return hide(value) + if isinstance(value, list): + return [scrub(item) for item in value] + if isinstance(value, dict): + return {key: scrub(item) for key, item in value.items()} + return value + try: record = json.loads(text) except ValueError: record = None - if isinstance(record, dict) and isinstance(record.get('response'), dict): + if not isinstance(record, dict): + return hide(text) + if isinstance(record.get('response'), dict): response = record['response'] hidden = {name.lower() for name in self.headers} | _ISSUED_CREDENTIALS response['headers'] = [ [name, REDACTED if str(name).lower() in hidden else value] for name, value in response.get('headers', []) ] - text = json.dumps(record, ensure_ascii=False) - needles = { - needle - for name, secret in self.headers.items() - for needle in _needles(name, secret.get_secret_value()) - } - text = _replaced(text, {form for n in needles for form in _forms(n)}, re.IGNORECASE) - return _replaced(text, {form for n in needles for form in _base64_forms(n)}, 0) + return json.dumps(scrub(record), ensure_ascii=False) def run_command(python: str) -> str: diff --git a/deepgym/tests/test_tools.py b/deepgym/tests/test_tools.py index 2d06fb3..ccd1d7c 100644 --- a/deepgym/tests/test_tools.py +++ b/deepgym/tests/test_tools.py @@ -383,6 +383,26 @@ def test_echoed_credentials_are_redacted_in_every_form(app: App) -> None: assert json.loads(response['body'])['basic'] == {'pair': REDACTED, 'password': REDACTED} +def test_redaction_keeps_the_record_json_and_its_protocol_types(app: App) -> None: + env = app_env(app.url).model_copy( + update={ + 'tools': ( + tool(app.url).model_copy(update={'headers': {'X-API-KEY': SecretStr('200')}}), + ) + } + ) + with Session(env, LocalBackend()) as session: + session.start() + step = session.step(call('/rest/echo')) + + assert step.observation.metadata == {'exit_code': 0} + result = json.loads(step.observation.content) # still a record, the credential gone + response = result['response'] + assert response['status'] == 200 and isinstance(result['elapsed_ms'], float) + assert dict(response['headers'])['X-Seen'] == REDACTED + assert json.loads(response['body'])['raw']['X-API-KEY'] == REDACTED + + def test_credentials_are_masked_in_dumps_fingerprints_and_errors() -> None: from deepgym.traces.schema import environment_sha256 From eb28f2bb892aaa3a1f0664413276a7fa39bff9d4 Mon Sep 17 00:00:00 2001 From: Abhishek Gahlot Date: Mon, 19 Oct 2026 09:13:39 +0530 Subject: [PATCH 11/21] fix: keep partial http responses and provider payloads, marked incomplete --- deepgym/src/deepgym/_http_request.py | 38 +++++++++++++++++-------- deepgym/src/deepgym/agents/api.py | 10 ++++--- deepgym/src/deepgym/agents/computer.py | 5 ++-- deepgym/src/deepgym/agents/providers.py | 36 ++++++++++++++++------- deepgym/tests/agents/test_api.py | 9 ++++++ deepgym/tests/http_app.py | 8 ++++++ deepgym/tests/test_tools.py | 15 ++++++++++ 7 files changed, 93 insertions(+), 28 deletions(-) diff --git a/deepgym/src/deepgym/_http_request.py b/deepgym/src/deepgym/_http_request.py index a9e0972..495c6fc 100644 --- a/deepgym/src/deepgym/_http_request.py +++ b/deepgym/src/deepgym/_http_request.py @@ -17,8 +17,26 @@ import time +def _answer(response: http.client.HTTPResponse, raw: bytes) -> dict: + """The record of *response*: status, every header, and *raw* as its body.""" + answer: dict[str, object] = { + 'status': response.status, + 'reason': response.reason, + 'headers': [list(header) for header in response.getheaders()], + } + try: + answer['body'] = raw.decode('utf-8') + except UnicodeDecodeError: + answer['body_base64'] = base64.b64encode(raw).decode('ascii') + return answer + + def main() -> int: - """Send the request on stdin; print its record; return 0 if a response came back.""" + """Send the request on stdin; print its record; return 0 if a whole response came back. + + A response cut short keeps what arrived (status, headers, the partial body), marked + ``incomplete``. + """ spec = json.load(sys.stdin) if spec['scheme'] == 'https': context = ssl.create_default_context() @@ -35,6 +53,7 @@ def main() -> int: record['uncertain'] = False else: body = spec['body'] + response = None try: connection.request( spec['method'], @@ -46,19 +65,14 @@ def main() -> int: raw = response.read() except (OSError, http.client.HTTPException) as exc: # The request may have reached the app: a write is then neither known done nor not. - record['error'] = f'No response: {type(exc).__name__}: {exc}' + kind = 'No' if response is None else 'Incomplete' + record['error'] = f'{kind} response: {type(exc).__name__}: {exc}' record['uncertain'] = spec['mutation'] + if response is not None: + partial = exc.partial if isinstance(exc, http.client.IncompleteRead) else b'' + record['response'] = {**_answer(response, partial), 'incomplete': True} else: - answer: dict[str, object] = { - 'status': response.status, - 'reason': response.reason, - 'headers': [list(header) for header in response.getheaders()], - } - try: - answer['body'] = raw.decode('utf-8') - except UnicodeDecodeError: - answer['body_base64'] = base64.b64encode(raw).decode('ascii') - record['response'] = answer + record['response'] = _answer(response, raw) status = 0 finally: connection.close() diff --git a/deepgym/src/deepgym/agents/api.py b/deepgym/src/deepgym/agents/api.py index 77d46ae..41faaca 100644 --- a/deepgym/src/deepgym/agents/api.py +++ b/deepgym/src/deepgym/agents/api.py @@ -12,8 +12,9 @@ A call is never retried. Only the done tool ends an episode; a reply without tool calls gets a reminder instead. Streaming, accounting and incomplete-response handling are the computer agents' (see :mod:`deepgym.agents.providers`). Every model message goes to -``AgentResult.messages``; under a TracedSession, each step also carries the messages that -produced it. +``AgentResult.messages``, a response that never completed too (what arrived of it, marked +``status: incomplete``; its tool calls never run); under a TracedSession, each step also +carries the messages that produced it. """ from __future__ import annotations @@ -36,6 +37,7 @@ ANTHROPIC_MAX_TOKENS, SDK_RETRIES, account, + anthropic_incomplete, anthropic_message, api_error, openai_incomplete, @@ -226,9 +228,9 @@ def run( messages=messages, timeout=tracker.request_timeout, ) - if isinstance(response, str): - return episode.result('error', error=response) episode.say('assistant', response) + if error := anthropic_incomplete(response): + return episode.result('error', error=error) account(tracker, response.get('usage'), self.prices, 'anthropic') content = response['content'] messages.append({'role': 'assistant', 'content': content}) diff --git a/deepgym/src/deepgym/agents/computer.py b/deepgym/src/deepgym/agents/computer.py index 74fedf6..4367449 100644 --- a/deepgym/src/deepgym/agents/computer.py +++ b/deepgym/src/deepgym/agents/computer.py @@ -25,6 +25,7 @@ ANTHROPIC_MAX_TOKENS, SDK_RETRIES, account, + anthropic_incomplete, anthropic_message, api_error, openai_incomplete, @@ -140,8 +141,8 @@ def run( messages=messages, timeout=tracker.request_timeout, ) - if isinstance(response, str): - return tracker.result(session, 'error', error=response) + if error := anthropic_incomplete(response): + return tracker.result(session, 'error', error=error) account(tracker, response.get('usage'), self.prices, 'anthropic') content = response['content'] messages.append({'role': 'assistant', 'content': content}) diff --git a/deepgym/src/deepgym/agents/providers.py b/deepgym/src/deepgym/agents/providers.py index d9523b4..9762cab 100644 --- a/deepgym/src/deepgym/agents/providers.py +++ b/deepgym/src/deepgym/agents/providers.py @@ -27,9 +27,11 @@ def _snapshot(stream: Any) -> Any: return None -def _incomplete(tracker: BudgetTracker, snapshot: Any, prices: TokenPrices | None) -> None: +def _incomplete( + tracker: BudgetTracker, message: dict[str, Any], prices: TokenPrices | None +) -> None: """Keep the usage an unfinished stream reported, with the total bill marked unresolved.""" - data = snapshot.model_dump(exclude_none=True).get('usage') if snapshot is not None else None + data = message.get('usage') usage = _anthropic_usage(data) if isinstance(data, dict) else Usage() unresolved = 'incomplete streamed response: partial usage, billing unresolved' tracker.add_usage(usage, Cost(provenance=unresolved)) @@ -37,11 +39,14 @@ def _incomplete(tracker: BudgetTracker, snapshot: Any, prices: TokenPrices | Non def anthropic_message( client: Any, tracker: BudgetTracker, prices: TokenPrices | None, **request: Any -) -> dict[str, Any] | str: - """Stream one Anthropic response; return it whole, or why it never completed. +) -> dict[str, Any]: + """Stream one Anthropic response; return it whole, or what arrived of it marked incomplete. Every event is consumed and only a received message_stop completes a response. An - unfinished one is never acted on: the usage it reported is kept, its billing unresolved. + unfinished one is never acted on (see :func:`anthropic_incomplete`): the usage it + reported is kept, its billing unresolved, and it comes back as the partial message + (empty if none started) with ``status`` ``incomplete`` and the reason in + ``incomplete_details``, as OpenAI marks its own. Args: client: The Anthropic client. @@ -62,12 +67,23 @@ def anthropic_message( except Exception as exc: # noqa: BLE001 failure = type(exc).__name__ snapshot = _snapshot(stream) + message = snapshot.model_dump(exclude_none=True) if snapshot is not None else {} if failure or not stopped or snapshot is None: - _incomplete(tracker, snapshot, prices) - if failure: - return f'Response stream failed: {failure}' - return 'Response stream ended before completion' - return snapshot.model_dump(exclude_none=True) + _incomplete(tracker, message, prices) + reason = ( + f'Response stream failed: {failure}' + if failure + else 'Response stream ended before completion' + ) + return {**message, 'status': 'incomplete', 'incomplete_details': {'reason': reason}} + return message + + +def anthropic_incomplete(message: dict[str, Any]) -> str | None: + """Why :func:`anthropic_message` returned *message* incomplete, or None if it completed.""" + if message.get('status') != 'incomplete': + return None + return message['incomplete_details']['reason'] def openai_incomplete(response: dict[str, Any]) -> str | None: diff --git a/deepgym/tests/agents/test_api.py b/deepgym/tests/agents/test_api.py index 3a3f680..ff4e5f4 100644 --- a/deepgym/tests/agents/test_api.py +++ b/deepgym/tests/agents/test_api.py @@ -335,6 +335,13 @@ def handler(request): assert not result.trajectory.steps and app.requests == [] # the received call never ran assert result.usage.input_tokens == 100 # the partial usage is kept assert result.cost.usd is None and 'billing unresolved' in result.cost.provenance + partial = result.messages[-1] # what arrived is kept, marked incomplete + assert partial.role == 'assistant' + payload = json.loads(partial.content) + assert payload['status'] == 'incomplete' + assert payload['incomplete_details'] == {'reason': result.error} + assert payload['usage']['input_tokens'] == 100 + assert payload['content'][0]['name'] == 'tracker' @pytest.mark.parametrize('details', [{'reason': 'max_output_tokens'}, None]) @@ -353,6 +360,8 @@ def test_openai_incomplete_response_never_acts(app, monkeypatch, details): ) assert len(sent) == 1 and not result.trajectory.steps and app.requests == [] assert result.usage.output_tokens == 10 + payload = json.loads(result.messages[-1].content) # what arrived is kept, marked incomplete + assert payload['status'] == 'incomplete' and payload['output'][0]['name'] == 'tracker' def test_reply_without_tool_calls_is_reminded_not_completed(app, monkeypatch): diff --git a/deepgym/tests/http_app.py b/deepgym/tests/http_app.py index 35a15a4..60acfd6 100644 --- a/deepgym/tests/http_app.py +++ b/deepgym/tests/http_app.py @@ -27,6 +27,7 @@ HttpRoute(method='GET', path='/rest/binary'), HttpRoute(method='GET', path='/rest/echo'), HttpRoute(method='GET', path='/rest/file/{name:path}'), + HttpRoute(method='GET', path='/rest/partial'), ) @@ -103,6 +104,13 @@ def _handle(self) -> None: return if route.endswith('/slow'): time.sleep(3) + if route.endswith('/partial'): # the status, headers and part of the body + self.send_response(200) + self.send_header('Content-Length', '100') + self.end_headers() + self.wfile.write(b'{"id": 7, "sta') + self.close_connection = True + return if route.endswith('/redirect'): self._send(302, b'', {'Location': app.redirect_to}) elif route.endswith('/done'): diff --git a/deepgym/tests/test_tools.py b/deepgym/tests/test_tools.py index ccd1d7c..4847c56 100644 --- a/deepgym/tests/test_tools.py +++ b/deepgym/tests/test_tools.py @@ -189,6 +189,21 @@ def test_write_stopped_by_the_step_timeout_is_uncertain(app: App) -> None: assert len(app.requests) == 1 +def test_incomplete_response_keeps_what_arrived(app: App) -> None: + with app_session(app.url) as session: + session.start() + step = session.step(call('/rest/partial')) + + result = json.loads(step.observation.content) + assert step.observation.metadata == {'exit_code': 1} # no complete response + assert result['error'].startswith('Incomplete response: IncompleteRead') + assert result['uncertain'] is False + response = result['response'] + assert response['incomplete'] is True + assert response['status'] == 200 and ['Content-Length', '100'] in response['headers'] + assert response['body'] == '{"id": 7, "sta' + + def verifier(score: float) -> str: return f'import json\nprint(json.dumps({{"score": {score}, "passed": {score == 1}}}))\n' From 160f7d451c468cd47e124331a1beaa98ff121957 Mon Sep 17 00:00:00 2001 From: Abhishek Gahlot Date: Mon, 19 Oct 2026 09:13:39 +0530 Subject: [PATCH 12/21] docs: unsettled tool writes, partial responses and body-dispatched routes --- deepgym/docs/agents.md | 29 ++++++++++++++++++++++------- 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/deepgym/docs/agents.md b/deepgym/docs/agents.md index 3d08987..6f3af94 100644 --- a/deepgym/docs/agents.md +++ b/deepgym/docs/agents.md @@ -99,17 +99,32 @@ errors, and redacted from every record and result, including forms an app may ec (escaped, percent-encoded, base64, a Basic pair or password, any letter case) and the cookies it sets. Pattern redaction cannot catch every transformation, so declare only routes that never return credentials. An environment with tools allows exactly `tool` -and `done`. +and `done`. Tools are trusted configuration, so an inline environment sent to the +sessions API cannot declare any; only registered environments can. + +A route allows exactly a method and a path; the body is checked only as JSON, not +against the app's operations. A declared GraphQL endpoint (`POST /graphql`) or any +other endpoint that dispatches on its body (a JSON-RPC method, a batch call) therefore +allows every operation the tool's credentials permit. Give such a tool an app account +whose role allows only what the task needs. The observation, which the model gets back whole, records the full request, the full -response (status, every header, the body) and timing. A write whose response was lost -is marked `"uncertain": true` and is never retried. Only the `done` tool ends the -episode; app output that reads `DONE` does not. A reply without tool calls gets a -reminder. Defaults are uncapped: maximum effort, Anthropic's model-maximum +response (status, every header, the body) and timing. A response cut short keeps what +arrived, marked `"incomplete": true`, and counts as a failed call. A write whose response +was lost, cut short or stopped by the step timeout is marked `"uncertain": true` and is +never retried. Stopping the request does not stop the app, which may still apply the +write after the episode, so the session keeps such writes unsettled. Before the final +verifier, the environment's trusted `root_tool_settle` command (root, offline namespace, +`setup_timeout`) must settle them: exit 0 only once the app has applied or abandoned +every request it received, for example by stopping its server or waiting for its queue +to drain. Without it, or if it fails, a score above 0 gets no verdict +(`error_type='unsettled_write'`) and a 0 stands. Only the `done` tool ends the episode; +app output that reads `DONE` does not. A reply without tool calls gets a reminder. Defaults are uncapped: maximum effort, Anthropic's model-maximum `max_tokens`, no OpenAI output limit, and only the turn, time and USD limits the caller's `Budget` sets. `AgentResult.messages` holds every model message, full -responses with their usage included; under `dg.trace`, each step also carries the -messages that produced it. +responses with their usage included. A provider response that never completed is kept +too, as what arrived marked `"status": "incomplete"`, and its tool calls never run. Under +`dg.trace`, each step also carries the messages that produced it. ## CLI isolation and its exposure From 4cf8816664cdb1fa51d96e57f1dbc02daf1a3175 Mon Sep 17 00:00:00 2001 From: Abhishek Gahlot Date: Mon, 19 Oct 2026 09:37:09 +0530 Subject: [PATCH 13/21] fix: hold every tool write unsettled until settled, and grade nothing before --- deepgym/src/deepgym/models.py | 10 +++--- deepgym/src/deepgym/session.py | 57 +++++++++++++++++++--------------- deepgym/tests/test_tools.py | 36 ++++++++++++--------- 3 files changed, 58 insertions(+), 45 deletions(-) diff --git a/deepgym/src/deepgym/models.py b/deepgym/src/deepgym/models.py index 77ddb00..6869412 100644 --- a/deepgym/src/deepgym/models.py +++ b/deepgym/src/deepgym/models.py @@ -577,13 +577,13 @@ class MultiTurnEnvironment(BaseModel): runs as the agent user beside the tool runner, which holds the tools' credentials.""" root_tool_settle: str = '' - """Trusted reconciliation of tool writes whose outcome is unknown: a response lost, cut - short or stopped by the step timeout, after which the app may still apply the write. - Runs before the final verifier, only when such a write exists, as root in the offline + """Trusted reconciliation of the tool writes an episode sent: the app may still apply + one after answering it (a queued job) or after its runner was stopped. Runs before the + final verifier whenever a write may have reached the app, as root in the offline namespace (as the session user on LocalBackend, which isolates nothing), with setup_timeout. It must exit 0 only once the app has applied or abandoned every request - it received: stop its server, wait for its queue to drain, or check the write in its - database. Without it, or if it fails, a score above 0 gets no verdict; a 0 stands. + it received: stop its server, wait for its queue to drain, or check the writes in its + database. Without it, or if it fails, the episode gets no verdict, whatever its score. Never accept this from an agent.""" desktop: bool = False diff --git a/deepgym/src/deepgym/session.py b/deepgym/src/deepgym/session.py index 5268e4a..9148367 100644 --- a/deepgym/src/deepgym/session.py +++ b/deepgym/src/deepgym/session.py @@ -990,7 +990,7 @@ def __init__(self, env: MultiTurnEnvironment, backend: SessionBackend) -> None: # The session's start, and the seconds spent applying the agent's actions since. self._opened = time.monotonic() self._acting_s = 0.0 - # Request records of tool writes whose outcome is unknown, until settled (see score()). + # Request records of the tool writes sent, until settled (see score()). self._unsettled: list[JsonValue] = [] # One sandbox, one actor at a time: concurrent step/score/close calls queue up. self._lock = threading.Lock() @@ -1259,10 +1259,10 @@ def score(self) -> RunResult: after reading the agent's output (``solution_timeout``), and a provider lifetime the agent's own actions ran out (see :meth:`_ran_out_the_clock`). - A tool write whose outcome is unknown (its response lost, cut short or stopped) may - still land after grading, so the environment's ``root_tool_settle`` must settle it - first; until then a score above 0 has no verdict (``unsettled_write``), and a 0 - stands. + Every tool write sent may still change the app after its response (a queued job, a + write whose runner was stopped), so the environment's ``root_tool_settle`` settles + them first. If it can't (none declared, or it fails), the episode gets no verdict + (``unsettled_write``), whatever the verifier would say, and the verifier doesn't run. Raises: VerifierError: If, before it read the agent's output, the verifier timed out or @@ -1285,10 +1285,11 @@ def score(self) -> RunResult: sandbox_id=self.sandbox_id, error_type='no_verifier', ) + if unsettled := self._settle(): + return _unsettled_result(unsettled, self._unsettled, self.sandbox_id) start = time.perf_counter() acted = bool(self._trajectory.steps) try: - unsettled = self._settle() (exit_code, stdout, stderr), consulted = self._verify(self._env.final_verifier_code) except SandboxDiedError as exc: if not acted: @@ -1312,8 +1313,6 @@ def score(self) -> RunResult: if consulted and not result.has_verdict and result.error_type != 'sandbox_error': detail = VerifierError(f'{result.error_type}: {result.output}') return _failed_after_reading(detail, elapsed_ms, self.sandbox_id, stderr) - if unsettled and result.has_verdict and result.score > 0: - return _unsettled_result(result, unsettled, self._unsettled) return result def close(self) -> None: @@ -1408,25 +1407,31 @@ def _send( The request reaches the constant runner command as data on its stdin, never through a file or argument the agent user could read or change, and the tool's header values are redacted from what comes back. It is never retried: a runner the timeout stops - reports no response, and a mutation it may have sent is recorded as uncertain and - kept unsettled for :meth:`score`, since stopping the runner never stops the app. + reports no response, and a mutation it may have sent is recorded as uncertain. + + A mutation counts as unsettled from before it is sent until :meth:`score` settles + it, since neither a response nor a stopped runner shows the app is done with it; + only a runner that proves it never connected clears it. """ + if request['mutation']: + self._unsettled.append(request['record']) command = run_command(self._backend.python) try: exit_code, output = self._backend.exec_input(command, json.dumps(request), timeout) except TimeoutError: exit_code = None output = lost_response(request, f'the request was stopped after {timeout}s') - if request['mutation'] and exit_code != 0 and not _never_sent(output): - self._unsettled.append(request['record']) + if request['mutation'] and _never_sent(output): + self._unsettled.pop() return tool.redact(output), exit_code def _settle(self) -> str | None: - """Settle unsettled tool writes with the environment's ``root_tool_settle``; return + """Settle the tool writes sent with the environment's ``root_tool_settle``; return why they stay unsettled, or None once none are. - Only the environment's own reconciliation settles them: the app may apply a write - whose runner was stopped, and a model's later reads prove nothing. + Only the environment's own reconciliation settles them: an app may still apply a + write it answered (queued it, say) or whose runner was stopped, and a model's later + reads prove nothing. """ if not self._unsettled: return None @@ -1442,6 +1447,8 @@ def _settle(self) -> str | None: settled = self._backend.exec(command, timeout) except TimeoutError: return f'root_tool_settle timed out after {timeout}s' + except DeepGymError as exc: # the sandbox failed under it: nothing is proven + return f'root_tool_settle failed: {type(exc).__name__}' if settled.exit_code != 0: return f'root_tool_settle exited {settled.exit_code}' self._unsettled.clear() @@ -1751,20 +1758,20 @@ def _failed_after_reading( return build_run_result(verdict, elapsed_ms, sandbox_id, stderr=stderr, exit_code=1) -def _unsettled_result(result: RunResult, why: str, writes: Sequence[JsonValue]) -> RunResult: - """Return *result*, a score above 0 over tool *writes* the app may still apply, with - no verdict: the write may have earned it, or may yet undo it. A 0 keeps its verdict, - or a policy could dodge it by sending a write that never answers.""" +def _unsettled_result(why: str, writes: Sequence[JsonValue], sandbox_id: str) -> RunResult: + """Return the result of an episode whose tool *writes* the app may still apply: no + verdict, whatever the score would be, since any write could still raise or lower it. A + policy that learns to leave writes unsettled shows in the masked rate (see + :mod:`deepgym.integrations.reward`).""" listed = json.dumps(writes, ensure_ascii=False) return RunResult( score=0.0, passed=False, - output=f'Not scored, tool writes unsettled ({why}): {listed}. ' - f'The verifier said: {result.output}', - stderr=result.stderr, - exit_code=result.exit_code, - execution_time_ms=result.execution_time_ms, - sandbox_id=result.sandbox_id, + output=f'Not graded, tool writes unsettled ({why}): {listed}', + stderr='', + exit_code=1, + execution_time_ms=0.0, + sandbox_id=sandbox_id, error_type='unsettled_write', ) diff --git a/deepgym/tests/test_tools.py b/deepgym/tests/test_tools.py index 4847c56..4997a45 100644 --- a/deepgym/tests/test_tools.py +++ b/deepgym/tests/test_tools.py @@ -216,14 +216,27 @@ def scored(session: Session, *actions: Action) -> RunResult: return session.score() +SETTLED_VERIFIER = ( + 'import json, pathlib\n' + 'ok = pathlib.Path("settled").exists()\n' + 'print(json.dumps({"score": float(ok), "passed": ok}))\n' +) + + +@pytest.mark.parametrize('score', [1.0, 0.0]) @pytest.mark.parametrize( - ('path', 'settings'), [('/rest/lost', {}), ('/rest/slow', {'timeout_per_step': 2})] + ('path', 'settings'), + [ + ('/rest/bug/1/comment', {}), # answered 201: the app may still be applying it + ('/rest/lost', {}), + ('/rest/slow', {'timeout_per_step': 2}), + ], ) -def test_a_pass_over_an_unsettled_write_gets_no_verdict( - app: App, path: str, settings: dict +def test_a_write_never_settled_gets_no_verdict_whatever_its_score( + app: App, path: str, settings: dict, score: float ) -> None: write = call(path, 'POST', body={'status': 'RESOLVED'}) - with app_session(app.url, final_verifier_code=verifier(1.0), **settings) as session: + with app_session(app.url, final_verifier_code=verifier(score), **settings) as session: result = scored(session, write) assert not result.has_verdict and result.error_type == 'unsettled_write' @@ -231,18 +244,11 @@ def test_a_pass_over_an_unsettled_write_gets_no_verdict( assert f'{app.url}{path}' in result.output and 'no root_tool_settle' in result.output -def test_a_zero_over_an_unsettled_write_stands(app: App) -> None: - write = call('/rest/lost', 'POST', body={'status': 'RESOLVED'}) - with app_session(app.url, final_verifier_code=verifier(0.0)) as session: - result = scored(session, write) - assert result.has_verdict and result.score == 0 - - -@pytest.mark.parametrize(('settle', 'graded'), [('true', True), ('exit 3', False)]) -def test_settled_writes_are_graded(app: App, settle: str, graded: bool) -> None: - write = call('/rest/lost', 'POST', body={'status': 'RESOLVED'}) +@pytest.mark.parametrize(('settle', 'graded'), [('touch settled', True), ('exit 3', False)]) +def test_writes_are_settled_before_grading(app: App, settle: str, graded: bool) -> None: + write = call('/rest/bug/1/comment', 'POST', body={'comment': 'x'}) with app_session( - app.url, final_verifier_code=verifier(1.0), root_tool_settle=settle + app.url, final_verifier_code=SETTLED_VERIFIER, root_tool_settle=settle ) as session: result = scored(session, write) assert result.has_verdict is graded and result.passed is graded From fa7248eaf87eb017d45610c9aaa389cea6938d96 Mon Sep 17 00:00:00 2001 From: Abhishek Gahlot Date: Mon, 19 Oct 2026 09:37:32 +0530 Subject: [PATCH 14/21] fix: refuse step verifiers beside tools that write --- deepgym/src/deepgym/models.py | 11 +++++++++-- deepgym/tests/test_tools.py | 14 ++++++++++++++ 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/deepgym/src/deepgym/models.py b/deepgym/src/deepgym/models.py index 6869412..b9aafe5 100644 --- a/deepgym/src/deepgym/models.py +++ b/deepgym/src/deepgym/models.py @@ -622,8 +622,8 @@ def check_desktop_actions(self) -> MultiTurnEnvironment: @model_validator(mode='after') def check_tools(self) -> MultiTurnEnvironment: - """Require unique tool names, tools exactly when tool actions are allowed, and - no other action beside them but done.""" + """Require unique tool names, tools exactly when tool actions are allowed, no other + action beside them but done, and no step verifier beside tools that write.""" names = [tool.name for tool in self.tools] if len(set(names)) != len(names): raise ValueError(f'tool names must be unique, got {names}') @@ -634,6 +634,13 @@ def check_tools(self) -> MultiTurnEnvironment: ) if 'tool' in self.allowed_actions and not self.tools: raise ValueError('tool actions need declared tools') + if self.step_verifier_code and any( + route.method != 'GET' for tool in self.tools for route in tool.routes + ): + raise ValueError( + 'a step verifier cannot run beside tools that write: their writes settle ' + 'only once the episode ends (see root_tool_settle)' + ) return self diff --git a/deepgym/tests/test_tools.py b/deepgym/tests/test_tools.py index 4997a45..9897766 100644 --- a/deepgym/tests/test_tools.py +++ b/deepgym/tests/test_tools.py @@ -255,6 +255,20 @@ def test_writes_are_settled_before_grading(app: App, settle: str, graded: bool) assert graded or 'root_tool_settle exited 3' in result.output +def test_step_verifiers_cannot_run_beside_write_tools() -> None: + with pytest.raises(ValidationError, match='step verifier'): + app_env('http://127.0.0.1', step_verifier_code='print(1)') + reads = tool('http://127.0.0.1').model_copy( + update={'routes': tuple(r for r in ROUTES if r.method == 'GET')} + ) + MultiTurnEnvironment( # a tool that only reads leaves nothing to settle + task='t', + tools=(reads,), + allowed_actions=frozenset({'tool', 'done'}), + step_verifier_code='print(1)', + ) + + def test_reads_and_writes_that_never_arrived_leave_nothing_to_settle(app: App) -> None: closed = App() closed.close() From 4e485ff141db4e5e9d5f3d8342d5e4b4495ecd19 Mon Sep 17 00:00:00 2001 From: Abhishek Gahlot Date: Mon, 19 Oct 2026 09:37:47 +0530 Subject: [PATCH 15/21] fix: never run anthropic tool calls cut short of tool_use, and keep their raw input --- deepgym/src/deepgym/agents/api.py | 3 ++ deepgym/src/deepgym/agents/computer.py | 3 ++ deepgym/src/deepgym/agents/providers.py | 25 +++++++++++++-- deepgym/tests/agents/test_api.py | 41 +++++++++++++++++++++++++ deepgym/tests/agents/test_computer.py | 36 +++++++++++++++++++++- 5 files changed, 105 insertions(+), 3 deletions(-) diff --git a/deepgym/src/deepgym/agents/api.py b/deepgym/src/deepgym/agents/api.py index 41faaca..86ec89b 100644 --- a/deepgym/src/deepgym/agents/api.py +++ b/deepgym/src/deepgym/agents/api.py @@ -39,6 +39,7 @@ account, anthropic_incomplete, anthropic_message, + anthropic_unfinished_calls, api_error, openai_incomplete, validate_budget, @@ -238,6 +239,8 @@ def run( stop = response.get('stop_reason') if stop == 'refusal': return episode.result('refusal', text) + if error := anthropic_unfinished_calls(response): + return episode.result('error', text, error=error) calls = [part for part in content if part['type'] == 'tool_use'] if not calls: if stop in {'max_tokens', 'pause_turn'}: diff --git a/deepgym/src/deepgym/agents/computer.py b/deepgym/src/deepgym/agents/computer.py index 4367449..2838219 100644 --- a/deepgym/src/deepgym/agents/computer.py +++ b/deepgym/src/deepgym/agents/computer.py @@ -27,6 +27,7 @@ account, anthropic_incomplete, anthropic_message, + anthropic_unfinished_calls, api_error, openai_incomplete, validate_budget, @@ -149,6 +150,8 @@ def run( text = ' '.join(part['text'] for part in content if part['type'] == 'text') if response.get('stop_reason') == 'refusal': return tracker.result(session, 'refusal', text) + if error := anthropic_unfinished_calls(response): + return tracker.result(session, 'error', text, error=error) calls = [part for part in content if part['type'] == 'tool_use'] if not calls: if response.get('stop_reason') in {'max_tokens', 'pause_turn'}: diff --git a/deepgym/src/deepgym/agents/providers.py b/deepgym/src/deepgym/agents/providers.py index 9762cab..7532d01 100644 --- a/deepgym/src/deepgym/agents/providers.py +++ b/deepgym/src/deepgym/agents/providers.py @@ -2,7 +2,8 @@ USD enforcement uses returned usage and caller-supplied prices, so one response can overshoot the cap. Only a complete response is ever acted on: an Anthropic stream must -reach message_stop, an OpenAI response must have status completed. +reach message_stop, and its tool calls run only if it stopped for them; an OpenAI response +must have status completed. """ from __future__ import annotations @@ -46,7 +47,9 @@ def anthropic_message( unfinished one is never acted on (see :func:`anthropic_incomplete`): the usage it reported is kept, its billing unresolved, and it comes back as the partial message (empty if none started) with ``status`` ``incomplete`` and the reason in - ``incomplete_details``, as OpenAI marks its own. + ``incomplete_details``, as OpenAI marks its own. A message that didn't stop for tool + use also carries ``input_json``: each tool call's input exactly as it streamed, by + content index, since the SDK's parse of an unfinished one drops what doesn't parse. Args: client: The Anthropic client. @@ -59,15 +62,21 @@ def anthropic_message( """ with client.messages.stream(**request) as stream: stopped, failure = False, None + fragments: dict[str, str] = {} try: for event in stream: stopped = stopped or event.type == 'message_stop' + if event.type == 'content_block_delta' and event.delta.type == 'input_json_delta': + index = str(event.index) + fragments[index] = fragments.get(index, '') + event.delta.partial_json # Anything raised while reading (httpx or httpx2 transport errors, by SDK # version, or an error event) ends this response. except Exception as exc: # noqa: BLE001 failure = type(exc).__name__ snapshot = _snapshot(stream) message = snapshot.model_dump(exclude_none=True) if snapshot is not None else {} + if fragments and message.get('stop_reason') != 'tool_use': + message['input_json'] = fragments if failure or not stopped or snapshot is None: _incomplete(tracker, message, prices) reason = ( @@ -86,6 +95,18 @@ def anthropic_incomplete(message: dict[str, Any]) -> str | None: return message['incomplete_details']['reason'] +def anthropic_unfinished_calls(message: dict[str, Any]) -> str | None: + """Why a complete *message*'s tool calls must not run, or None if they may. + + Its tool calls are whole only if it stopped for tool use; one cut at max_tokens may + hold a call whose input was parsed from a fragment. + """ + stop = message.get('stop_reason') + if stop == 'tool_use' or not any(part['type'] == 'tool_use' for part in message['content']): + return None + return f'Response stopped ({stop}) before its tool calls were complete' + + def openai_incomplete(response: dict[str, Any]) -> str | None: """Why an OpenAI response did not complete, or None if it did. diff --git a/deepgym/tests/agents/test_api.py b/deepgym/tests/agents/test_api.py index ff4e5f4..a658ff3 100644 --- a/deepgym/tests/agents/test_api.py +++ b/deepgym/tests/agents/test_api.py @@ -344,6 +344,47 @@ def handler(request): assert payload['content'][0]['name'] == 'tracker' +@pytest.mark.parametrize('finished', [True, False]) +def test_anthropic_tool_calls_cut_short_never_run(app, monkeypatch, finished): + raw = '{"method": "POST", "path": "/rest/bug/1/comment", "body": {"comment": "unfini' + events = [ + START, + {'type': 'content_block_start', 'index': 0, 'content_block': use('tracker', {})}, + { + 'type': 'content_block_delta', + 'index': 0, + 'delta': {'type': 'input_json_delta', 'partial_json': raw}, + }, + ] + if finished: # a whole message, stopped at the output limit mid-call + events += [ + {'type': 'content_block_stop', 'index': 0}, + { + 'type': 'message_delta', + 'delta': {'stop_reason': 'max_tokens'}, + 'usage': {'output_tokens': 5}, + }, + {'type': 'message_stop'}, + ] + anthropic( + monkeypatch, + b''.join(f'event: {e["type"]}\ndata: {json.dumps(e)}\n\n'.encode() for e in events), + ) + session, initial = started(app) + with session: + result = AnthropicToolAgent('test').run(session, initial, task='T', budget=UNLIMITED) + + assert result.termination_reason == 'error' + assert app.requests == [] and not result.trajectory.steps # the cut call never ran + payload = json.loads(result.messages[-1].content) + assert payload['input_json'] == {'0': raw} # every fragment received, kept + if finished: + assert payload['stop_reason'] == 'max_tokens' and 'max_tokens' in result.error + assert payload['usage']['output_tokens'] == 5 and result.usage.output_tokens == 5 + else: + assert payload['status'] == 'incomplete' + + @pytest.mark.parametrize('details', [{'reason': 'max_output_tokens'}, None]) def test_openai_incomplete_response_never_acts(app, monkeypatch, details): body = response(function_call('tracker', GET_BUG), status='incomplete') diff --git a/deepgym/tests/agents/test_computer.py b/deepgym/tests/agents/test_computer.py index 5eccd4e..b8afc01 100644 --- a/deepgym/tests/agents/test_computer.py +++ b/deepgym/tests/agents/test_computer.py @@ -178,10 +178,11 @@ def constructor(**kwargs): def anthropic_response(content, **kwargs): + calls = any(part.get('type') == 'tool_use' for part in content) return { 'usage': {'input_tokens': 100, 'output_tokens': 10}, 'content': content, - 'stop_reason': 'end_turn', + 'stop_reason': 'tool_use' if calls else 'end_turn', **kwargs, } @@ -877,6 +878,39 @@ def client(**kwargs): assert result.cost.usd is None and 'billing unresolved' in result.cost.provenance +def test_anthropic_actions_cut_at_max_tokens_never_run(monkeypatch): + anthropic = pytest.importorskip('anthropic') + httpx = _sdk_transport() + limit = {'type': 'message_delta', 'delta': {'stop_reason': 'max_tokens'}, 'usage': OUT} + text = ( + {'type': 'content_block_start', 'index': 0, 'content_block': {'type': 'text', 'text': ''}}, + {'type': 'content_block_stop', 'index': 0}, + {'type': 'message_delta', 'delta': {'stop_reason': 'end_turn'}, 'usage': OUT}, + {'type': 'message_stop'}, + ) + bodies = iter([_sse(START, *CLICK_EVENTS, limit, {'type': 'message_stop'}), _sse(START, *text)]) + + def handler(request): + return httpx.Response( + 200, headers={'content-type': 'text/event-stream'}, content=next(bodies) + ) + + real = anthropic.Anthropic + + def client(**kwargs): + transport = httpx.MockTransport(handler) + return real(api_key='test', http_client=httpx.Client(transport=transport), max_retries=0) + + monkeypatch.setattr(anthropic, 'Anthropic', client) + session = FakeSession() + result = AnthropicComputerAgent('test').run( + session, session.observation, task='Task', budget=Budget() + ) + assert result.termination_reason == 'error' and 'max_tokens' in result.error + assert not result.trajectory.steps # the click it was cut in never ran + assert result.usage.output_tokens == 5 # the response is still accounted + + def test_anthropic_completed_stream_acts(monkeypatch): anthropic = pytest.importorskip('anthropic') httpx = _sdk_transport() From 77ef45711ec97ee2424f45e8944ef3f48dc18738 Mon Sep 17 00:00:00 2001 From: Abhishek Gahlot Date: Mon, 19 Oct 2026 09:38:08 +0530 Subject: [PATCH 16/21] fix: keep every body byte a tool runner received before a reset --- deepgym/src/deepgym/_http_request.py | 30 ++++++++++++++++++++++------ deepgym/tests/http_app.py | 10 +++++++++- deepgym/tests/test_tools.py | 9 +++++---- 3 files changed, 38 insertions(+), 11 deletions(-) diff --git a/deepgym/src/deepgym/_http_request.py b/deepgym/src/deepgym/_http_request.py index 495c6fc..dfc6d31 100644 --- a/deepgym/src/deepgym/_http_request.py +++ b/deepgym/src/deepgym/_http_request.py @@ -16,6 +16,9 @@ import sys import time +_READ = 1 << 16 +"""Bytes asked for per read; reads repeat until the body ends, so this bounds nothing.""" + def _answer(response: http.client.HTTPResponse, raw: bytes) -> dict: """The record of *response*: status, every header, and *raw* as its body.""" @@ -31,11 +34,26 @@ def _answer(response: http.client.HTTPResponse, raw: bytes) -> dict: return answer +def _read(response: http.client.HTTPResponse, received: bytearray) -> None: + """Read *response*'s body into *received* as it arrives, so a cut keeps every byte before it. + + Raises: + http.client.IncompleteRead: If the connection ended before the declared length. + OSError: If the connection failed mid-body (a reset, say). + """ + chunk = response.read1(_READ) + while chunk: + received += chunk + chunk = response.read1(_READ) + if response.length: # read1 ends quietly at a close; the length shows what never came + raise http.client.IncompleteRead(bytes(received), response.length) + + def main() -> int: """Send the request on stdin; print its record; return 0 if a whole response came back. - A response cut short keeps what arrived (status, headers, the partial body), marked - ``incomplete``. + A response cut short keeps what arrived (status, headers, every body byte received), + marked ``incomplete``. """ spec = json.load(sys.stdin) if spec['scheme'] == 'https': @@ -54,6 +72,7 @@ def main() -> int: else: body = spec['body'] response = None + received = bytearray() try: connection.request( spec['method'], @@ -62,17 +81,16 @@ def main() -> int: spec['headers'], ) response = connection.getresponse() - raw = response.read() + _read(response, received) except (OSError, http.client.HTTPException) as exc: # The request may have reached the app: a write is then neither known done nor not. kind = 'No' if response is None else 'Incomplete' record['error'] = f'{kind} response: {type(exc).__name__}: {exc}' record['uncertain'] = spec['mutation'] if response is not None: - partial = exc.partial if isinstance(exc, http.client.IncompleteRead) else b'' - record['response'] = {**_answer(response, partial), 'incomplete': True} + record['response'] = {**_answer(response, bytes(received)), 'incomplete': True} else: - record['response'] = _answer(response, raw) + record['response'] = _answer(response, bytes(received)) status = 0 finally: connection.close() diff --git a/deepgym/tests/http_app.py b/deepgym/tests/http_app.py index 60acfd6..5bd2b76 100644 --- a/deepgym/tests/http_app.py +++ b/deepgym/tests/http_app.py @@ -3,6 +3,8 @@ import base64 import html import json +import socket +import struct import threading import time from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer @@ -28,6 +30,7 @@ HttpRoute(method='GET', path='/rest/echo'), HttpRoute(method='GET', path='/rest/file/{name:path}'), HttpRoute(method='GET', path='/rest/partial'), + HttpRoute(method='GET', path='/rest/reset'), ) @@ -104,12 +107,17 @@ def _handle(self) -> None: return if route.endswith('/slow'): time.sleep(3) - if route.endswith('/partial'): # the status, headers and part of the body + if route.endswith(('/partial', '/reset')): # the status, headers, part of the body self.send_response(200) self.send_header('Content-Length', '100') self.end_headers() self.wfile.write(b'{"id": 7, "sta') self.close_connection = True + if route.endswith('/reset'): # then a reset, not a close + time.sleep(0.2) # the client reads what was sent first + linger = struct.pack('ii', 1, 0) + self.connection.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, linger) + self.connection.close() return if route.endswith('/redirect'): self._send(302, b'', {'Location': app.redirect_to}) diff --git a/deepgym/tests/test_tools.py b/deepgym/tests/test_tools.py index 9897766..ca106db 100644 --- a/deepgym/tests/test_tools.py +++ b/deepgym/tests/test_tools.py @@ -189,19 +189,20 @@ def test_write_stopped_by_the_step_timeout_is_uncertain(app: App) -> None: assert len(app.requests) == 1 -def test_incomplete_response_keeps_what_arrived(app: App) -> None: +@pytest.mark.parametrize('path', ['/rest/partial', '/rest/reset']) +def test_incomplete_response_keeps_what_arrived(app: App, path: str) -> None: with app_session(app.url) as session: session.start() - step = session.step(call('/rest/partial')) + step = session.step(call(path)) result = json.loads(step.observation.content) assert step.observation.metadata == {'exit_code': 1} # no complete response - assert result['error'].startswith('Incomplete response: IncompleteRead') + assert result['error'].startswith('Incomplete response: ') assert result['uncertain'] is False response = result['response'] assert response['incomplete'] is True assert response['status'] == 200 and ['Content-Length', '100'] in response['headers'] - assert response['body'] == '{"id": 7, "sta' + assert response['body'] == '{"id": 7, "sta' # every byte received def verifier(score: float) -> str: From 9d3dd06626faf9045854139b171b6a787af7bbef Mon Sep 17 00:00:00 2001 From: Abhishek Gahlot Date: Mon, 19 Oct 2026 09:38:34 +0530 Subject: [PATCH 17/21] fix: redact every tool's credentials from every record, binary bodies in their bytes --- deepgym/src/deepgym/session.py | 24 ++++--- deepgym/src/deepgym/tools.py | 110 ++++++++++++++++++--------------- deepgym/tests/http_app.py | 5 ++ deepgym/tests/test_tools.py | 31 ++++++++++ 4 files changed, 108 insertions(+), 62 deletions(-) diff --git a/deepgym/src/deepgym/session.py b/deepgym/src/deepgym/session.py index 9148367..ac2fa0e 100644 --- a/deepgym/src/deepgym/session.py +++ b/deepgym/src/deepgym/session.py @@ -81,7 +81,7 @@ SandboxTimeoutError, sealed_path, ) -from deepgym.tools import HttpTool, lost_response, run_command +from deepgym.tools import HttpTool, lost_response, redact, run_command from deepgym.verifier_template import guard_crashes if TYPE_CHECKING: @@ -1340,7 +1340,7 @@ def _apply( action: Action, *, timeout: float | None = None, - call: tuple[HttpTool, dict[str, JsonValue]] | None = None, + call: dict[str, JsonValue] | None = None, ) -> tuple[str, int | None]: """Execute *action*; return its text output and exit code (None if not a command). @@ -1368,7 +1368,7 @@ def _apply( return f'wrote {action.path}', None case 'tool': assert call is not None - return self._send(*call, timeout) + return self._send(call, timeout) case 'wait': time.sleep(min(action.amount, timeout)) return '', None @@ -1384,8 +1384,8 @@ def _apply( finally: self._acting_s += time.monotonic() - started - def _tool_call(self, action: Action) -> tuple[HttpTool, dict[str, JsonValue]]: - """Return *action*'s declared tool and the request it sends (see HttpTool.prepare). + def _tool_call(self, action: Action) -> dict[str, JsonValue]: + """Return the request *action* sends with its declared tool (see HttpTool.prepare). Raises: ActionError: If no declared tool has its name, or the tool rejects its arguments. @@ -1395,19 +1395,17 @@ def _tool_call(self, action: Action) -> tuple[HttpTool, dict[str, JsonValue]]: names = sorted(tool.name for tool in self._env.tools) raise ActionError(f'No tool named {action.tool!r}. Declared: {names}') try: - return tool, tool.prepare(action.arguments) + return tool.prepare(action.arguments) except ValueError as exc: raise ActionError(f'Tool {tool.name!r} rejected the call: {exc}') from exc - def _send( - self, tool: HttpTool, request: Mapping[str, JsonValue], timeout: float - ) -> tuple[str, int | None]: + def _send(self, request: Mapping[str, JsonValue], timeout: float) -> tuple[str, int | None]: """Send *request* from the sandbox; return the runner's record of it and its exit code. The request reaches the constant runner command as data on its stdin, never through - a file or argument the agent user could read or change, and the tool's header values - are redacted from what comes back. It is never retried: a runner the timeout stops - reports no response, and a mutation it may have sent is recorded as uncertain. + a file or argument the agent user could read or change, and every tool's header + values are redacted from what comes back. It is never retried: a runner the timeout + stops reports no response, and a mutation it may have sent is recorded as uncertain. A mutation counts as unsettled from before it is sent until :meth:`score` settles it, since neither a response nor a stopped runner shows the app is done with it; @@ -1423,7 +1421,7 @@ def _send( output = lost_response(request, f'the request was stopped after {timeout}s') if request['mutation'] and _never_sent(output): self._unsettled.pop() - return tool.redact(output), exit_code + return redact(output, self._env.tools), exit_code def _settle(self) -> str | None: """Settle the tool writes sent with the environment's ``root_tool_settle``; return diff --git a/deepgym/src/deepgym/tools.py b/deepgym/src/deepgym/tools.py index 9511101..3629f0c 100644 --- a/deepgym/src/deepgym/tools.py +++ b/deepgym/src/deepgym/tools.py @@ -20,9 +20,9 @@ import re import shlex import unicodedata -from collections.abc import Iterator, Mapping +from collections.abc import Iterable, Iterator, Mapping from pathlib import Path -from typing import Literal +from typing import AnyStr, Literal from urllib.parse import quote, quote_plus, unquote, urlencode, urlsplit from pydantic import BaseModel, ConfigDict, Field, JsonValue, SecretStr, field_validator @@ -139,10 +139,14 @@ def _base64_forms(needle: str) -> set[str]: return {form for form in forms if len(form) >= _MIN_BASE64} -def _replaced(text: str, forms: set[str], flags: int) -> str: - """*text* with every one of *forms* replaced by REDACTED, longest first.""" +def _replaced(text: AnyStr, forms: set[str], flags: int) -> AnyStr: + """*text* with every one of *forms* (as UTF-8, in bytes) replaced by REDACTED, longest + first.""" if not forms: return text + if isinstance(text, bytes): + encoded = sorted((form.encode('utf-8') for form in forms), key=len, reverse=True) + return re.sub(b'|'.join(map(re.escape, encoded)), REDACTED.encode(), text, flags=flags) pattern = '|'.join(map(re.escape, sorted(forms, key=len, reverse=True))) return re.sub(pattern, REDACTED, text, flags=flags) @@ -210,7 +214,7 @@ class HttpTool(BaseModel): The model sees *name*, *description* (the place for the API docs a task needs) and the routes. *headers* go with every request and are credentials (an API key or login for the local app, say), kept as secrets: masked in repr, dumps, fingerprints and - validation errors, redacted from every record and result (see :meth:`redact`), and + validation errors, redacted from every record and result (see :func:`redact`), and never on disk where the agent user could read them. Responses come back whole (status, every header, pagination links included, and the full body) but for those redactions. @@ -398,53 +402,61 @@ def prepare(self, arguments: Mapping[str, JsonValue]) -> dict[str, JsonValue]: 'record': record, } - def redact(self, text: str) -> str: - """Return a runner's record (or any text) with the credentials an app may echo - redacted, where the session records it. - - A response header named like a tool header, or carrying a credential the app issued - (Set-Cookie), loses its whole value. Then every form of every header value and of - its echoable parts (see :func:`_needles`) is replaced wherever it appears (body, - headers, a redirect's Location, an error): escaped or encoded in any letter case - (:func:`_forms`), and in base64 (:func:`_base64_forms`). In a JSON record only its - string values are searched, so it stays JSON with its numbers and booleans (a - credential ``200`` leaves status 200 alone); its keys are the runner's or the - model's, never the app's. - """ - needles = { - needle - for name, secret in self.headers.items() - for needle in _needles(name, secret.get_secret_value()) - } - forms = {form for needle in needles for form in _forms(needle)} - encoded = {form for needle in needles for form in _base64_forms(needle)} - def hide(text: str) -> str: - return _replaced(_replaced(text, forms, re.IGNORECASE), encoded, 0) +def redact(text: str, tools: Iterable[HttpTool]) -> str: + """Return a runner's record (or any text) with the credentials an app may echo + redacted, where the session records it: every credential of every one of *tools*, + whichever tool's call the record is of. - def scrub(value: JsonValue) -> JsonValue: - if isinstance(value, str): - return hide(value) - if isinstance(value, list): - return [scrub(item) for item in value] - if isinstance(value, dict): - return {key: scrub(item) for key, item in value.items()} - return value + A response header named like a tool header, or carrying a credential the app issued + (Set-Cookie), loses its whole value. Then every form of every header value and of its + echoable parts (see :func:`_needles`) is replaced wherever it appears (body, headers, a + redirect's Location, an error): escaped or encoded in any letter case (:func:`_forms`), + and in base64 (:func:`_base64_forms`). In a JSON record only its string values are + searched, so it stays JSON with its numbers and booleans (a credential ``200`` leaves + status 200 alone); its keys are the runner's or the model's, never the app's. A body + that isn't UTF-8 is searched in its bytes, then base64-encoded again. + """ + headers = [ + (name, secret.get_secret_value()) for tool in tools for name, secret in tool.headers.items() + ] + needles = {needle for name, value in headers for needle in _needles(name, value)} + forms = {form for needle in needles for form in _forms(needle)} + encoded = {form for needle in needles for form in _base64_forms(needle)} + + def hide(data: AnyStr) -> AnyStr: + return _replaced(_replaced(data, forms, re.IGNORECASE), encoded, 0) + + def scrub(value: JsonValue) -> JsonValue: + if isinstance(value, str): + return hide(value) + if isinstance(value, list): + return [scrub(item) for item in value] + if isinstance(value, dict): + return {key: scrub(item) for key, item in value.items()} + return value - try: - record = json.loads(text) - except ValueError: - record = None - if not isinstance(record, dict): - return hide(text) - if isinstance(record.get('response'), dict): - response = record['response'] - hidden = {name.lower() for name in self.headers} | _ISSUED_CREDENTIALS - response['headers'] = [ - [name, REDACTED if str(name).lower() in hidden else value] - for name, value in response.get('headers', []) - ] - return json.dumps(scrub(record), ensure_ascii=False) + try: + record = json.loads(text) + except ValueError: + record = None + if not isinstance(record, dict): + return hide(text) + response = record.get('response') + binary = None + if isinstance(response, dict): + hidden = {name.lower() for name, _ in headers} | _ISSUED_CREDENTIALS + response['headers'] = [ + [name, REDACTED if str(name).lower() in hidden else value] + for name, value in response.get('headers', []) + ] + if isinstance(response.get('body_base64'), str): + raw = hide(base64.b64decode(response['body_base64'])) + binary = base64.b64encode(raw).decode('ascii') + scrubbed = scrub(record) + if binary is not None: # its masked bytes, not a search of their encoding + scrubbed['response']['body_base64'] = binary + return json.dumps(scrubbed, ensure_ascii=False) def run_command(python: str) -> str: diff --git a/deepgym/tests/http_app.py b/deepgym/tests/http_app.py index 5bd2b76..d4263d7 100644 --- a/deepgym/tests/http_app.py +++ b/deepgym/tests/http_app.py @@ -31,6 +31,7 @@ HttpRoute(method='GET', path='/rest/file/{name:path}'), HttpRoute(method='GET', path='/rest/partial'), HttpRoute(method='GET', path='/rest/reset'), + HttpRoute(method='GET', path='/rest/binary-key'), ) @@ -119,6 +120,10 @@ def _handle(self) -> None: self.connection.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, linger) self.connection.close() return + if route.endswith('/binary-key'): # the key inside a body that isn't UTF-8 + key = self.headers.get('X-API-KEY', '').encode() + self._send(200, b'\xff' + key + b'\x80', {}) + return if route.endswith('/redirect'): self._send(302, b'', {'Location': app.redirect_to}) elif route.endswith('/done'): diff --git a/deepgym/tests/test_tools.py b/deepgym/tests/test_tools.py index ca106db..a1d9ce1 100644 --- a/deepgym/tests/test_tools.py +++ b/deepgym/tests/test_tools.py @@ -439,6 +439,37 @@ def test_redaction_keeps_the_record_json_and_its_protocol_types(app: App) -> Non assert json.loads(response['body'])['raw']['X-API-KEY'] == REDACTED +def test_every_tools_credentials_are_redacted_from_every_record(app: App) -> None: + app.redirect_to = f'http://127.0.0.1:9/?k={SECRET}' # the app hands back the other key + admin = tool(app.url).model_copy( + update={'name': 'admin', 'headers': {'X-Admin-Key': SecretStr(SECRET)}} + ) + env = app_env(app.url).model_copy(update={'tools': (tool(app.url), admin)}) + with Session(env, LocalBackend()) as session: + session.start() + content = session.step(call('/rest/redirect')).observation.content + + assert json.dumps(SECRET)[1:-1] not in content and quote(SECRET, safe='') not in content + headers = dict(json.loads(content)['response']['headers']) + assert headers['Location'] == f'http://127.0.0.1:9/?k={REDACTED}' + + +def test_credentials_in_a_binary_body_are_redacted_in_its_bytes(app: App) -> None: + env = app_env(app.url).model_copy( + update={ + 'tools': ( + tool(app.url).model_copy(update={'headers': {'X-API-KEY': SecretStr('200')}}), + ) + } + ) + with Session(env, LocalBackend()) as session: + session.start() + result = record(session, call('/rest/binary-key')) + + body = base64.b64decode(result['response']['body_base64'], validate=True) + assert body == b'\xff' + REDACTED.encode() + b'\x80' + + def test_credentials_are_masked_in_dumps_fingerprints_and_errors() -> None: from deepgym.traces.schema import environment_sha256 From 5cd1a3875314ef3dc70d8660de035e5aa281f5af Mon Sep 17 00:00:00 2001 From: Abhishek Gahlot Date: Mon, 19 Oct 2026 09:38:34 +0530 Subject: [PATCH 18/21] fix: load tools, allowed actions and root_tool_settle from env metadata --- deepgym/src/deepgym/registry.py | 6 +++++- deepgym/tests/test_registry.py | 33 ++++++++++++++++++++++++++++++++- 2 files changed, 37 insertions(+), 2 deletions(-) diff --git a/deepgym/src/deepgym/registry.py b/deepgym/src/deepgym/registry.py index d39e9f0..b3cfd8c 100644 --- a/deepgym/src/deepgym/registry.py +++ b/deepgym/src/deepgym/registry.py @@ -332,8 +332,12 @@ def _read_reference_actions(env_dir: Path, variant: int = 0) -> list[Action] | N 'timeout_per_step', 'image', 'snapshot', + 'allowed_actions', + 'tools', + 'root_tool_settle', ) -"""metadata.json keys a multi-turn or computer-use env takes as they are.""" +"""metadata.json keys a multi-turn or computer-use env takes as they are. The folder is +trusted configuration, so its tools' credentials and root commands are too.""" def _load_computer_use_env(env_dir: Path, metadata: dict, variant: int) -> ComputerUseEnvironment: diff --git a/deepgym/tests/test_registry.py b/deepgym/tests/test_registry.py index bac91f0..6a6f88b 100644 --- a/deepgym/tests/test_registry.py +++ b/deepgym/tests/test_registry.py @@ -1,11 +1,12 @@ """Tests for deepgym.registry -- environment registry loading.""" +import json from pathlib import Path import pytest from deepgym import registry as registry_module -from deepgym.models import Environment +from deepgym.models import Environment, MultiTurnEnvironment from deepgym.registry import ( _read_registry, list_environments, @@ -98,6 +99,36 @@ def test_metadata_image_reaches_the_environment(self, tmp_path: Path) -> None: assert env.image == 'python:3.11-slim' + def test_multi_turn_metadata_declares_tools_and_their_settling(self, tmp_path: Path) -> None: + tools = [ + { + 'name': 'tracker', + 'description': 'Bug tracker REST API.', + 'base_url': 'http://127.0.0.1:8080', + 'routes': [{'method': 'POST', 'path': '/rest/bug/{id}/comment'}], + 'headers': {'X-API-KEY': 'app-key'}, + } + ] + metadata = { + 'type': 'multi-turn', + 'allowed_actions': ['tool', 'done'], + 'tools': tools, + 'root_tool_settle': 'service tracker stop', + } + (tmp_path / 'task.md').write_text('Triage bug 1.\n', encoding='utf-8') + (tmp_path / 'verifier.py').write_text('print(1.0)\n', encoding='utf-8') + (tmp_path / 'metadata.json').write_text(json.dumps(metadata), encoding='utf-8') + + env = registry_module._load_env_from_dir(tmp_path) + + assert isinstance(env, MultiTurnEnvironment) + assert env.allowed_actions == {'tool', 'done'} + assert env.root_tool_settle == 'service tracker stop' + (declared,) = env.tools + assert declared.name == 'tracker' + assert declared.headers['X-API-KEY'].get_secret_value() == 'app-key' + assert 'metadata.json' not in env.files # the key never reaches the work directory + def test_load_humaneval_task_id_with_slash_alias( self, tmp_path: Path, From c4d522c1d53ba4bc016285610c314cfe49dac291 Mon Sep 17 00:00:00 2001 From: Abhishek Gahlot Date: Mon, 19 Oct 2026 09:38:34 +0530 Subject: [PATCH 19/21] docs: settling every tool write, cut tool calls and redaction across tools --- deepgym/docs/agents.md | 51 +++++++++++++++++++++++++----------------- 1 file changed, 31 insertions(+), 20 deletions(-) diff --git a/deepgym/docs/agents.md b/deepgym/docs/agents.md index 6f3af94..59953dd 100644 --- a/deepgym/docs/agents.md +++ b/deepgym/docs/agents.md @@ -95,12 +95,14 @@ then a constant runner sends it with http.client from inside the sandbox, as the user in the offline network namespace. Its request arrives on stdin from a root-only file. Redirects are returned, never followed. Header values are credentials: masked in dumps, environment fingerprints and validation -errors, and redacted from every record and result, including forms an app may echo -(escaped, percent-encoded, base64, a Basic pair or password, any letter case) and the -cookies it sets. Pattern redaction cannot catch every transformation, so declare only -routes that never return credentials. An environment with tools allows exactly `tool` -and `done`. Tools are trusted configuration, so an inline environment sent to the -sessions API cannot declare any; only registered environments can. +errors, and every tool's are redacted from every call's record and result, including +forms an app may echo (escaped, percent-encoded, base64, a Basic pair or password, any +letter case) and the cookies it sets. A body that isn't UTF-8 comes back base64-encoded, +with its bytes redacted. Pattern redaction cannot catch every transformation, so declare +only routes that never return credentials. An environment with tools allows exactly +`tool` and `done`. Tools are trusted configuration, so an inline environment sent to the +sessions API cannot declare any; a registered environment can, in code or in its +folder's `metadata.json` (`tools`, `allowed_actions`, `root_tool_settle`). A route allows exactly a method and a path; the body is checked only as JSON, not against the app's operations. A declared GraphQL endpoint (`POST /graphql`) or any @@ -109,21 +111,30 @@ allows every operation the tool's credentials permit. Give such a tool an app ac whose role allows only what the task needs. The observation, which the model gets back whole, records the full request, the full -response (status, every header, the body) and timing. A response cut short keeps what -arrived, marked `"incomplete": true`, and counts as a failed call. A write whose response -was lost, cut short or stopped by the step timeout is marked `"uncertain": true` and is -never retried. Stopping the request does not stop the app, which may still apply the -write after the episode, so the session keeps such writes unsettled. Before the final -verifier, the environment's trusted `root_tool_settle` command (root, offline namespace, -`setup_timeout`) must settle them: exit 0 only once the app has applied or abandoned -every request it received, for example by stopping its server or waiting for its queue -to drain. Without it, or if it fails, a score above 0 gets no verdict -(`error_type='unsettled_write'`) and a 0 stands. Only the `done` tool ends the episode; -app output that reads `DONE` does not. A reply without tool calls gets a reminder. Defaults are uncapped: maximum effort, Anthropic's model-maximum -`max_tokens`, no OpenAI output limit, and only the turn, time and USD limits the -caller's `Budget` sets. `AgentResult.messages` holds every model message, full +response (status, every header, the body) and timing. A response cut short keeps every +byte that arrived, marked `"incomplete": true`, and counts as a failed call. A write +whose response was lost, cut short or stopped by the step timeout is marked +`"uncertain": true` and is never retried. + +An app may still change after it answers a write (a queued job, say) or after its runner +was stopped, so the session holds every write it sends as unsettled; only a call that +never connected is cleared. Before the final verifier runs, the environment's trusted +`root_tool_settle` command (root, offline namespace, `setup_timeout`) must settle them: +exit 0 only once the app has applied or abandoned every request it received, for example +by stopping its server or waiting for its queue to drain. Without it, or if it fails, +the episode gets no verdict (`error_type='unsettled_write'`) whatever its score, and +counts toward the masked rate. Writes leave no settled state between steps, so an +environment whose tools have any route other than `GET` cannot have a step verifier. + +Only the `done` tool ends the episode; app output that reads `DONE` does not. A reply +without tool calls gets a reminder. Defaults are uncapped: maximum effort, Anthropic's +model-maximum `max_tokens`, no OpenAI output limit, and only the turn, time and USD +limits the caller's `Budget` sets. `AgentResult.messages` holds every model message, full responses with their usage included. A provider response that never completed is kept -too, as what arrived marked `"status": "incomplete"`, and its tool calls never run. Under +too, as what arrived marked `"status": "incomplete"`, and its tool calls never run. Nor +do the calls of an Anthropic response that stopped for any reason but tool use +(`max_tokens` mid-call, say): the run ends with an error, and the response is kept with +its `stop_reason`, usage and `input_json`, each call's input exactly as it streamed. Under `dg.trace`, each step also carries the messages that produced it. ## CLI isolation and its exposure From 9e429f7fc55b7c1b643e713c79dea909316383f3 Mon Sep 17 00:00:00 2001 From: Abhishek Gahlot Date: Mon, 19 Oct 2026 09:47:40 +0530 Subject: [PATCH 20/21] fix: keep the text of what ended an anthropic stream, an unparsable delta included --- deepgym/src/deepgym/agents/providers.py | 29 ++++++++++-------- deepgym/tests/agents/test_api.py | 40 ++++++++++++++++++++++++- 2 files changed, 56 insertions(+), 13 deletions(-) diff --git a/deepgym/src/deepgym/agents/providers.py b/deepgym/src/deepgym/agents/providers.py index 7532d01..032ffe0 100644 --- a/deepgym/src/deepgym/agents/providers.py +++ b/deepgym/src/deepgym/agents/providers.py @@ -47,9 +47,10 @@ def anthropic_message( unfinished one is never acted on (see :func:`anthropic_incomplete`): the usage it reported is kept, its billing unresolved, and it comes back as the partial message (empty if none started) with ``status`` ``incomplete`` and the reason in - ``incomplete_details``, as OpenAI marks its own. A message that didn't stop for tool - use also carries ``input_json``: each tool call's input exactly as it streamed, by - content index, since the SDK's parse of an unfinished one drops what doesn't parse. + ``incomplete_details``, as OpenAI marks its own, with the text of what ended the + stream as ``detail``. A message that didn't stop for tool use also carries + ``input_json``: each tool call's input exactly as it streamed, by content index, since + the SDK's parse of an unfinished one drops what doesn't parse. Args: client: The Anthropic client. @@ -61,7 +62,8 @@ def anthropic_message( anthropic.APIError: If the request itself fails. """ with client.messages.stream(**request) as stream: - stopped, failure = False, None + stopped = False + failure: Exception | None = None fragments: dict[str, str] = {} try: for event in stream: @@ -72,19 +74,22 @@ def anthropic_message( # Anything raised while reading (httpx or httpx2 transport errors, by SDK # version, or an error event) ends this response. except Exception as exc: # noqa: BLE001 - failure = type(exc).__name__ + failure = exc snapshot = _snapshot(stream) message = snapshot.model_dump(exclude_none=True) if snapshot is not None else {} if fragments and message.get('stop_reason') != 'tool_use': message['input_json'] = fragments - if failure or not stopped or snapshot is None: + if failure is not None or not stopped or snapshot is None: _incomplete(tracker, message, prices) - reason = ( - f'Response stream failed: {failure}' - if failure - else 'Response stream ended before completion' - ) - return {**message, 'status': 'incomplete', 'incomplete_details': {'reason': reason}} + details = {'reason': 'Response stream ended before completion'} + if failure is not None: + # Its text too: the SDK's error for a delta it can't parse quotes that delta, + # which never reaches the snapshot or the fragments above. + details = { + 'reason': f'Response stream failed: {type(failure).__name__}', + 'detail': str(failure), + } + return {**message, 'status': 'incomplete', 'incomplete_details': details} return message diff --git a/deepgym/tests/agents/test_api.py b/deepgym/tests/agents/test_api.py index a658ff3..b4bdd52 100644 --- a/deepgym/tests/agents/test_api.py +++ b/deepgym/tests/agents/test_api.py @@ -339,7 +339,7 @@ def handler(request): assert partial.role == 'assistant' payload = json.loads(partial.content) assert payload['status'] == 'incomplete' - assert payload['incomplete_details'] == {'reason': result.error} + assert payload['incomplete_details']['reason'] == result.error assert payload['usage']['input_tokens'] == 100 assert payload['content'][0]['name'] == 'tracker' @@ -385,6 +385,44 @@ def test_anthropic_tool_calls_cut_short_never_run(app, monkeypatch, finished): assert payload['status'] == 'incomplete' +def test_anthropic_delta_the_sdk_cannot_parse_is_kept_and_never_runs(app, monkeypatch): + first = '{"method": "POST", "path": "/rest/bug/1/comment", ' + events = [ + START, + {'type': 'content_block_start', 'index': 0, 'content_block': use('tracker', {})}, + { + 'type': 'content_block_delta', + 'index': 0, + 'delta': {'type': 'input_json_delta', 'partial_json': first}, + }, + { + 'type': 'content_block_delta', + 'index': 0, + 'delta': {'type': 'input_json_delta', 'partial_json': 'BROKEN_LAST_DELTA_MARKER}'}, + }, + {'type': 'content_block_stop', 'index': 0}, + { + 'type': 'message_delta', + 'delta': {'stop_reason': 'tool_use'}, + 'usage': {'output_tokens': 5}, + }, + {'type': 'message_stop'}, + ] + anthropic( + monkeypatch, + b''.join(f'event: {e["type"]}\ndata: {json.dumps(e)}\n\n'.encode() for e in events), + ) + session, initial = started(app) + with session: + result = AnthropicToolAgent('test').run(session, initial, task='T', budget=UNLIMITED) + + assert result.termination_reason == 'error' + assert app.requests == [] and not result.trajectory.steps # nothing it held ran + payload = json.loads(result.messages[-1].content) + assert payload['status'] == 'incomplete' + assert 'BROKEN_LAST_DELTA_MARKER' in payload['incomplete_details']['detail'] + + @pytest.mark.parametrize('details', [{'reason': 'max_output_tokens'}, None]) def test_openai_incomplete_response_never_acts(app, monkeypatch, details): body = response(function_call('tracker', GET_BUG), status='incomplete') From 05cb73447cba666d42b5484a1e95fe9e742af4b7 Mon Sep 17 00:00:00 2001 From: Abhishek Gahlot Date: Mon, 19 Oct 2026 09:47:40 +0530 Subject: [PATCH 21/21] fix: require ascii tool header credentials, basic pairs included --- deepgym/docs/agents.md | 5 +++-- deepgym/src/deepgym/tools.py | 27 +++++++++++++++++++++++---- deepgym/tests/test_tools.py | 21 +++++++++++++++++++++ 3 files changed, 47 insertions(+), 6 deletions(-) diff --git a/deepgym/docs/agents.md b/deepgym/docs/agents.md index 59953dd..abd1822 100644 --- a/deepgym/docs/agents.md +++ b/deepgym/docs/agents.md @@ -98,8 +98,9 @@ Header values are credentials: masked in dumps, environment fingerprints and val errors, and every tool's are redacted from every call's record and result, including forms an app may echo (escaped, percent-encoded, base64, a Basic pair or password, any letter case) and the cookies it sets. A body that isn't UTF-8 comes back base64-encoded, -with its bytes redacted. Pattern redaction cannot catch every transformation, so declare -only routes that never return credentials. An environment with tools allows exactly +with its bytes redacted. Header values must be ASCII, a Basic pair inside one too, so the +bytes an app sees match what redaction looks for. Pattern redaction cannot catch every +transformation, so declare only routes that never return credentials. An environment with tools allows exactly `tool` and `done`. Tools are trusted configuration, so an inline environment sent to the sessions API cannot declare any; a registered environment can, in code or in its folder's `metadata.json` (`tools`, `allowed_actions`, `root_tool_settle`). diff --git a/deepgym/src/deepgym/tools.py b/deepgym/src/deepgym/tools.py index 3629f0c..29d1279 100644 --- a/deepgym/src/deepgym/tools.py +++ b/deepgym/src/deepgym/tools.py @@ -57,6 +57,7 @@ } # fmt: skip # Response headers carrying credentials the app issues (a session cookie, say). _ISSUED_CREDENTIALS = frozenset({'set-cookie', 'set-cookie2'}) +_AUTHORIZATION = frozenset({'authorization', 'proxy-authorization'}) _URL_SAFE = str.maketrans('+/', '-_') # A base64 run shorter than one 4-character group also turns up in unrelated text. _MIN_BASE64 = 4 @@ -99,7 +100,7 @@ def _needles(name: str, value: str) -> set[str]: credentials after an Authorization scheme, a Basic pair and its password, each Cookie value.""" needles = {value} - if name.lower() in ('authorization', 'proxy-authorization'): + if name.lower() in _AUTHORIZATION: scheme, _, credentials = value.partition(' ') needles.add(credentials.strip()) if scheme.lower() == 'basic': @@ -113,6 +114,23 @@ def _needles(name: str, value: str) -> set[str]: return {needle for needle in needles if needle} +def _ascii(name: str, value: str) -> bool: + """Whether credential header *value* is ASCII, a Basic pair inside it too. + + http.client sends a header as Latin-1, and an app decodes a Basic pair as it likes, so + the bytes of a non-ASCII credential differ from the UTF-8 that redaction looks for. + """ + if not value.isascii(): + return False + scheme, _, credentials = value.partition(' ') + if name.lower() not in _AUTHORIZATION or scheme.lower() != 'basic': + return True + try: + return base64.b64decode(credentials.strip(), validate=True).isascii() + except ValueError: # binascii.Error + return False + + def _forms(needle: str) -> set[str]: """The forms an app may echo *needle* in: as is, JSON- and HTML-escaped, percent- and form-encoded (matched in any letter case, so either hex case).""" @@ -284,7 +302,7 @@ def check_base_url(cls, value: str) -> str: @classmethod def check_headers(cls, value: dict[str, SecretStr]) -> dict[str, SecretStr]: """Accept distinct token names, none that reroutes or reframes a request, with - plain values.""" + plain ASCII values (see :func:`_ascii`).""" if len({name.lower() for name in value}) != len(value): raise ValueError('header names must differ in more than letter case') for name, secret in value.items(): @@ -293,12 +311,13 @@ def check_headers(cls, value: dict[str, SecretStr]) -> dict[str, SecretStr]: not _TOKEN.fullmatch(name) or name.lower() in _RESERVED_HEADERS or not text + or not _ascii(name, text) or any((char < ' ' and char != '\t') or char == '\x7f' for char in text) ): raise ValueError( f'header {name!r} must be a token name other than ' - f'{sorted(_RESERVED_HEADERS)}, with a non-empty value and no control ' - 'characters' + f'{sorted(_RESERVED_HEADERS)}, with a non-empty ASCII value (a Basic pair ' + 'too) and no control characters' ) return value diff --git a/deepgym/tests/test_tools.py b/deepgym/tests/test_tools.py index a1d9ce1..e46f0d7 100644 --- a/deepgym/tests/test_tools.py +++ b/deepgym/tests/test_tools.py @@ -577,6 +577,27 @@ def test_tool_headers_cannot_reroute_or_reframe_requests(headers: dict) -> None: ) +@pytest.mark.parametrize( + 'value', + [ + 'café', # http.client sends it as Latin-1, bytes no UTF-8 search finds + 'Basic ' + base64.b64encode('admin:café'.encode('latin-1')).decode(), + 'Basic ' + base64.b64encode('admin:café'.encode()).decode(), + ], +) +def test_header_credentials_must_be_ascii(value: str) -> None: + name = 'Authorization' if value.startswith('Basic') else 'X-Key' + with pytest.raises(ValidationError, match='ASCII') as invalid: + HttpTool( + name='t', + description='d', + base_url='http://127.0.0.1', + routes=ROUTES, + headers={name: value}, + ) + assert 'caf' not in str(invalid.value) and value not in str(invalid.value) + + @pytest.mark.parametrize( 'path', [