From 5bb8e8cc1bbca1f0b035660d3d82fe0d520c8b58 Mon Sep 17 00:00:00 2001 From: Abhishek Gahlot Date: Sun, 18 Oct 2026 05:23:04 +0530 Subject: [PATCH] feat: agent harnesses for claude code, codex and computer use --- deepgym/README.md | 3 + deepgym/docs/agents.md | 214 +++++ deepgym/pyproject.toml | 3 + deepgym/src/deepgym/_cli_worker.py | 271 ++++++ deepgym/src/deepgym/_pid_namespace.py | 152 ++++ deepgym/src/deepgym/agents/__init__.py | 21 + deepgym/src/deepgym/agents/actions.py | 190 +++++ deepgym/src/deepgym/agents/base.py | 192 +++++ deepgym/src/deepgym/agents/cli.py | 195 +++++ deepgym/src/deepgym/agents/cli_events.py | 297 +++++++ deepgym/src/deepgym/agents/computer.py | 401 +++++++++ deepgym/src/deepgym/agents/versions.py | 48 ++ deepgym/src/deepgym/api/schemas.py | 6 +- deepgym/src/deepgym/cli_runtime.py | 255 ++++++ deepgym/src/deepgym/core.py | 40 +- deepgym/src/deepgym/models.py | 23 +- .../src/deepgym/sandboxes/daytona/desktop.py | 17 +- .../src/deepgym/sandboxes/daytona/sandbox.py | 15 +- deepgym/src/deepgym/session.py | 239 +++++- deepgym/src/deepgym/traces/atif.py | 5 +- deepgym/src/deepgym/traces/capture.py | 97 ++- deepgym/src/deepgym/traces/export.py | 21 +- deepgym/tests/agents/fixtures/README.md | 55 ++ deepgym/tests/agents/fixtures/claude.jsonl | 5 + .../fixtures/claude/2.1.302/stream.jsonl | 5 + deepgym/tests/agents/fixtures/codex.jsonl | 6 + .../codex/0.162.1/retry-then-success.jsonl | 5 + .../fixtures/codex/0.162.1/stream.jsonl | 6 + .../0.162.1/terminal-provider-error.jsonl | 4 + deepgym/tests/agents/test_base.py | 79 ++ deepgym/tests/agents/test_cli.py | 801 ++++++++++++++++++ deepgym/tests/agents/test_cli_compat.py | 91 ++ deepgym/tests/agents/test_cli_versions.py | 199 +++++ deepgym/tests/agents/test_computer.py | 749 ++++++++++++++++ deepgym/tests/agents/test_episode.py | 66 ++ deepgym/tests/agents/test_live.py | 114 +++ deepgym/tests/golden/traces/rl.jsonl | 4 +- deepgym/tests/golden/traces/summary.jsonl | 6 +- deepgym/tests/golden/traces/traces.jsonl | 6 +- deepgym/tests/test_api_schemas.py | 61 ++ deepgym/tests/test_cli_runtime.py | 557 ++++++++++++ deepgym/tests/test_pid_namespace.py | 193 +++++ deepgym/tests/test_sandbox_images.py | 58 +- deepgym/tests/test_sandboxes.py | 13 +- deepgym/tests/test_traces_agents.py | 243 ++++++ 45 files changed, 5972 insertions(+), 59 deletions(-) create mode 100644 deepgym/docs/agents.md create mode 100644 deepgym/src/deepgym/_cli_worker.py create mode 100644 deepgym/src/deepgym/_pid_namespace.py create mode 100644 deepgym/src/deepgym/agents/__init__.py create mode 100644 deepgym/src/deepgym/agents/actions.py create mode 100644 deepgym/src/deepgym/agents/base.py create mode 100644 deepgym/src/deepgym/agents/cli.py create mode 100644 deepgym/src/deepgym/agents/cli_events.py create mode 100644 deepgym/src/deepgym/agents/computer.py create mode 100644 deepgym/src/deepgym/agents/versions.py create mode 100644 deepgym/src/deepgym/cli_runtime.py create mode 100644 deepgym/tests/agents/fixtures/README.md create mode 100644 deepgym/tests/agents/fixtures/claude.jsonl create mode 100644 deepgym/tests/agents/fixtures/claude/2.1.302/stream.jsonl create mode 100644 deepgym/tests/agents/fixtures/codex.jsonl create mode 100644 deepgym/tests/agents/fixtures/codex/0.162.1/retry-then-success.jsonl create mode 100644 deepgym/tests/agents/fixtures/codex/0.162.1/stream.jsonl create mode 100644 deepgym/tests/agents/fixtures/codex/0.162.1/terminal-provider-error.jsonl create mode 100644 deepgym/tests/agents/test_base.py create mode 100644 deepgym/tests/agents/test_cli.py create mode 100644 deepgym/tests/agents/test_cli_compat.py create mode 100644 deepgym/tests/agents/test_cli_versions.py create mode 100644 deepgym/tests/agents/test_computer.py create mode 100644 deepgym/tests/agents/test_episode.py create mode 100644 deepgym/tests/agents/test_live.py create mode 100644 deepgym/tests/test_api_schemas.py create mode 100644 deepgym/tests/test_cli_runtime.py create mode 100644 deepgym/tests/test_pid_namespace.py create mode 100644 deepgym/tests/test_traces_agents.py diff --git a/deepgym/README.md b/deepgym/README.md index 3e9746b1..a18aa926 100644 --- a/deepgym/README.md +++ b/deepgym/README.md @@ -258,6 +258,9 @@ Agents act with `code`, `bash`, `write_file` and `done`; an optional step verifi each step. `dg.run_episode(env, agent)` runs a whole episode in one call. More in [sessions][sessions]. +For Claude Code, Codex, and provider computer-use loops, see the +[agent harnesses and budget/isolation limits](docs/agents.md). + ### Drive a desktop Computer-use tasks boot a locked-down desktop on Daytona and score what the agent changed, diff --git a/deepgym/docs/agents.md b/deepgym/docs/agents.md new file mode 100644 index 00000000..4f85eac5 --- /dev/null +++ b/deepgym/docs/agents.md @@ -0,0 +1,214 @@ +# Agent harnesses + +`deepgym.agents` supplies `AnthropicComputerAgent`, `OpenAIComputerAgent`, +`ClaudeCodeAgent`, and `CodexAgent`. Each implements +`run(session, initial, *, task, budget) -> AgentResult`. Start the Session first; +the caller owns grading and cleanup. `DeepGym.run_episode(env, agent, budget=...)` +does that lifecycle and returns `(AgentResult, RunResult)`. Its existing callable +interface still returns `(Trajectory, RunResult)`. + +Computer agents need the `anthropic` or `openai` optional extra, respectively. +Imports stay lazy. Choose a model supporting Anthropic's +`computer_toolset_20260801` or OpenAI's Responses `computer` tool. Provider keys stay +in the host process for these two agents. Unsupported mouse buttons, modifiers, +curved drag paths, and other unrepresentable actions produce explicit tool errors. +Pending OpenAI safety checks stop the run without executing their actions. + +```python +from deepgym import DeepGym +from deepgym.agents import AnthropicComputerAgent, Budget + +# env is an already-defined ComputerUseEnvironment. +result, grade = DeepGym(mode='daytona').run_episode( + env, + AnthropicComputerAgent(model='your-computer-capable-model'), + budget=Budget(max_turns=30, max_seconds=300), +) +``` + +`Budget` always supplies finite positive turns and wall time. For computer agents, +a turn is one provider request. For Claude Code it is a unique assistant message; +for Codex it is a completed activity item (message, reasoning, or tool), because +Codex's JSON `turn` covers the entire invocation. These units are deliberately +different and are not suitable for cross-provider model-call comparisons. + +`TokenPrices` requires all four token-category rates and a dated source. No price +is guessed from a mutable model name. Costs distinguish caller-rate estimates, +CLI-reported totals, and unavailable accounting. USD caps stop work when observed +usage crosses the threshold; one in-flight response and polling latency can +overshoot it. Computer agents let the SDK retry connection drops, 429s and 5xx responses up to 4 times with backoff; a retried request may be billed twice. Failed requests may have unknown +billing. Wall limits include provider time and actions; transport timeouts and +cleanup introduce bounded overhead rather than a precise real-time interrupt. + +Codex rejects every finite `max_cost_usd` before launch: usage arrives only at the +end of its task. Claude accepts a dollar cap only with explicit `TokenPrices` and +stops from per-message usage; missing usage fails closed. Its native dollar-budget +flag is an additional control. Turn caps are enforced by the library; the adapter +does not rely on undocumented CLI flags. Interrupted accounting may be incomplete. + +Codex standalone `error` frames do not terminate a run: an older local exec exporter +forwards retry notifications without their retry flag. The exact pinned release's +retry emission is **not confirmed**. Terminal events/process exit decide failure, +with the existing wall deadline as the bound. Terminal failures with recognized +structured provider codes use `provider_error`; message-only or unknown-code errors +remain `error`, regardless of their prose. Evidence and fixture limits are recorded +in `tests/agents/fixtures/README.md`. + +## CLI isolation and its exposure + +CLIs must already be installed in the sandbox image; the library installs nothing. +Supported releases are pinned in `deepgym.agents.SUPPORTED_CLIS`. Each run checks +the installed version inside the sandbox before launching the task; unsupported +versions are refused. The observed version is retained in result and trace +provenance. Their flags must match the adapters. The environment +must allow `bash` or `cli`, and its backend must retain a separate network +namespace. LocalBackend and provider-only network blocking without that namespace +are refused. GUI-only environments cannot gain shell access via an agent adapter. + +```python +import os + +from deepgym.agents import CodexAgent +from deepgym.models import MultiTurnEnvironment + +env = MultiTurnEnvironment( + task='your task', + snapshot=os.environ['DEEPGYM_CLI_SNAPSHOT'], + cli_provider_egress=True, + allowed_actions=frozenset({'cli'}), +) +agent = CodexAgent(model='your-codex-model', allow_model_credentials=True) +# OPENAI_API_KEY is read on the host; the CLI receives its CODEX_API_KEY equivalent. +``` + +`MultiTurnEnvironment.cli_provider_egress=True` is a separate, required opt-in for +CLI transport. By default Daytona blocks all sandbox network traffic; the provider +tunnel cannot reach its upstream API through that outer block. This flag removes +Daytona's sandbox-wide network block so the trusted proxy can connect outward. +**Trusted root processes outside the offline namespace then have unrestricted +egress**, not provider-only or HTTPS-route-limited access. The agent still runs in +its loopback-only network namespace and PID containment; root service startup and +the verifier remain inside the offline network namespace. Missing opt-in is +rejected before CLI launch, and inline API environment specs reject the override. +Use it only on trusted CLI snapshots. + +`allow_model_credentials=True` explicitly permits the model and its commands to +read the real provider key. A root-owned CONNECT proxy outside the namespace and +a loopback bridge inside it permit only the selected provider hostname on port +443. DNS results must be public addresses; connections use the validated numeric +address. Daytona's privileged loopback toolbox remains outside the agent's +namespace. This is destination filtering, not HTTPS-route filtering or account +spend enforcement: arbitrary calls to that provider can bypass CLI accounting. +The CLI and its children inherit matching `NO_PROXY`/`no_proxy` values for +`localhost` (including its trailing-dot form), `127.0.0.1`, and compressed/expanded +`::1` with or without URL brackets. Local app traffic therefore bypasses the provider +proxy inside the isolated namespace. Other 127/8 addresses and IPv4-mapped IPv6 +forms are not explicitly bypassed; remote destinations retain the HTTP(S) proxy. +TLS is opaque to the tunnel, so shared CDN addresses may also serve other hosts; +the CONNECT allowlist is not an application-layer endpoint guarantee. +Exact credential strings are redacted from capture; transformed secrets and task +contents can still appear in traces, which should be treated as sensitive. + +The CLI runs as `dgagent` with capabilities and privilege escalation disabled. +CLI-capable sessions create a persistent PID namespace before any agent execution, +including setup and desktop processes. Each CLI invocation runs as PID 1 inside a +nested PID namespace, created before dropping privileges. Teardown kills the +persistent namespace's pinned init through a pidfd and waits for kernel destruction +before verifying that no live agent-UID process remains. The UID scan never serves +as the termination mechanism. Root services and verifiers stay outside containment. +CLI finalization also covers zero-dollar budgets and expired preflight deadlines, +before returning a result for grading; failed cleanup leaves the session ungradable. +Ordinary GUI scoring does not finalize CLI containment or stop the desktop. +Linux pidfds and util-linux PID/mount namespace support are required; unavailable +containment fails closed. This avoids relying on unverified cgroup delegation. +Root captures stdout outside agent-writable directories. One `cli` Session action +contains its nested events; events are never replayed through `Session.step`. +All processes with that episode's agent UID, including detached children, stop +before grading. This also stops an agent-owned browser/desktop: CLI grading must +inspect persisted state or separately owned services, not a live agent browser. +The verifier retains its separate unprivileged identity. + +`MultiTurnEnvironment.root_service_start` starts trusted image services as root +inside the offline namespace before files, desktop, and ordinary setup. It uses +`setup_timeout`; the command must fail if readiness fails. Never populate this +field from model output. It supports snapshots whose database/web services need +root startup, without making the agent or verifier root. +Inline API specifications reject nonempty `root_service_start`; trusted registry +environments can retain it. + +## Upgrading the CLIs + +`deepgym.agents.SUPPORTED_CLIS` is the sole source of supported versions and npm +package names. Phase 1 supports one exact release per CLI. Image builders can +generate the installation command without maintaining their own pins: + +```python +from deepgym.agents import SUPPORTED_CLIS, cli_install_command + +print(cli_install_command()) # npm install -g, with both packages pinned +print({name: release.version for name, release in SUPPORTED_CLIS.items()}) +``` + +To upgrade, change the version in that mapping, record sanitized stream fixtures +under `tests/agents/fixtures///stream.jsonl` with capture +provenance, run `python -m pytest tests/agents/test_cli_compat.py tests/agents/test_cli.py -q`, +then rebuild sandbox images using `cli_install_command()`. The compatibility test +checks the installed CLI's version and every flag supplied by the adapter against +its help; it skips a CLI absent locally and never invokes a model. Initial stream +fixtures remain schema-derived, as documented in their README; recording model +runs requires separate authorization. Versioned parser fixtures catch missing +upgrade coverage but do not turn schema samples into live compatibility evidence. + +## Offline verification and live smoke + +The offline suite covers mappings, provider loops, event streams, budgets, +namespace and identity invariants, capture/ATIF/Parquet preservation, and startup +ordering. Stream fixtures are schema-derived samples, not claimed live recordings. +No performance improvement or live sandbox isolation proof is claimed by those +tests. The opt-in GUI test is pinned to `img-itop-3.3.0-v1`: it creates a uniquely named +organization through the GUI and verifies that organization through iTop's API. + +With `DAYTONA_API_KEY`, `ANTHROPIC_API_KEY`, and `ITOP_ADMIN_PASSWORD` exported, run +from the package directory: + +```bash +DEEPGYM_LIVE_AGENTS=1 DEEPGYM_AGENT_MODEL=claude-opus-5-5 python -m pytest tests/agents/test_live.py::test_anthropic_itop_snapshot -q -s +``` + +The CLI smoke is independently gated by `DEEPGYM_LIVE_CLI_AGENTS=1`. Set +`DEEPGYM_CLI_SNAPSHOT` to your trusted snapshot with the supported CLIs already +installed. It creates a unique file through the model's tools, then verifies its +exact contents after process teardown; no iTop credentials are needed. The test +explicitly enables `cli_provider_egress` and model-readable credentials. With +`DAYTONA_API_KEY` and the corresponding provider key exported, run either test: + +```bash +DEEPGYM_LIVE_CLI_AGENTS=1 DEEPGYM_CLI_SNAPSHOT="$YOUR_CLI_SNAPSHOT" DEEPGYM_CLAUDE_MODEL="$YOUR_CLAUDE_MODEL" python -m pytest 'tests/agents/test_live.py::test_cli_snapshot_roundtrip[anthropic]' -q -s +DEEPGYM_LIVE_CLI_AGENTS=1 DEEPGYM_CLI_SNAPSHOT="$YOUR_CLI_SNAPSHOT" DEEPGYM_CODEX_MODEL="$YOUR_CODEX_MODEL" python -m pytest 'tests/agents/test_live.py::test_cli_snapshot_roundtrip[openai]' -q -s +``` + +Each case skips unless its own snapshot, model, provider key, and Daytona key are +present. These commands create paid model calls and Daytona sandboxes when enabled; +the CLI roundtrips have not been run during this change. + +Phase 2 replaces the transport at `Session.run_cli` with the hosted key-injecting +relay, revocable episode credentials, inference-route restrictions, and live dollar +metering. Phase 1 does not pretend to provide those guarantees. + +Implementation status: all four adapters, startup and transport hooks, optional +extras, and offline checks are present. The live smoke remains unrun; no sandbox, +provider request, or performance benchmark was used to validate this change. + +Validation on 2026-10-18: `ruff check src/` passed; `ruff format --check src/` +reported 173 files already formatted. The loopback-proxy pytest run with four workers +passed 230 tests and skipped 3, including CLI environment bypass checks, namespace +invariants, and both local CLI help checks. All three live tests were skipped: + +```bash +python -m pytest tests/agents tests/test_cli_runtime.py tests/test_pid_namespace.py -n 4 -q +``` + +A broader check found `test_computer_use_envs.py::test_computer_use_suite_is_separate` +failing because its expected list omits 182 registered environments. The same failure +was reproduced from clean starting commit `62d39f532a8944b20d3264fc09d2c44ab25c5472`; +that unrelated test and registry were left unchanged. diff --git a/deepgym/pyproject.toml b/deepgym/pyproject.toml index 4c236e05..2ff0f6e7 100644 --- a/deepgym/pyproject.toml +++ b/deepgym/pyproject.toml @@ -39,6 +39,8 @@ e2b = ["e2b>=2.55"] hf = ["datasets>=2.16", "huggingface_hub>=0.20"] lm_eval = ["lm_eval>=0.4.9"] traces = ["pyarrow>=14"] +anthropic = ["anthropic>=0.68"] +openai = ["openai>=2"] dev = ["pytest", "pytest-asyncio", "pytest-xdist", "ruff", "fastapi>=0.100", "uvicorn[standard]>=0.20", "pyarrow>=14"] all = ["daytona>=0.223", "modal>=1.6", "e2b>=2.55", "fastapi>=0.100", "uvicorn[standard]>=0.20", "datasets>=2.16", "huggingface_hub>=0.20", "pyarrow>=14", "pytest", "pytest-asyncio", "ruff"] @@ -86,3 +88,4 @@ inline-quotes = "single" asyncio_mode = "auto" testpaths = ["tests"] pythonpath = ["src"] +markers = ["live_agents: explicitly opted-in provider and sandbox integration test"] diff --git a/deepgym/src/deepgym/_cli_worker.py b/deepgym/src/deepgym/_cli_worker.py new file mode 100644 index 00000000..02606a3a --- /dev/null +++ b/deepgym/src/deepgym/_cli_worker.py @@ -0,0 +1,271 @@ +"""Run the sandbox-side CLI supervisor and fixed-destination CONNECT transport. + +This file is copied into a root-only directory and runs with the standard library. +It never imports the agent's work directory or executes a model-produced root command. +""" + +import ipaddress +import json +import os +import selectors +import signal +import socket +import socketserver +import subprocess +import sys +import threading +import time +from pathlib import Path + +_LOOPBACK_NO_PROXY = 'localhost,localhost.,127.0.0.1,::1,[::1],0:0:0:0:0:0:0:1,[0:0:0:0:0:0:0:1]' + + +def public_addresses(host: str) -> list[tuple[int, tuple]]: + """Resolve an API name once; refuse any non-public answer before connecting. + + Args: + host: The fixed provider API hostname. + + Returns: + Numeric socket addresses, without a second DNS lookup. + + Raises: + ValueError: If DNS returns a private, loopback, or otherwise reserved address. + """ + addresses = socket.getaddrinfo(host, 443, type=socket.SOCK_STREAM) + resolved = [ipaddress.ip_address(row[4][0]) for row in addresses] + if not addresses or any(not address.is_global or address.is_multicast for address in resolved): + raise ValueError('Provider API resolved to a non-public address') + return [(row[0], row[4]) for row in addresses] + + +def _copy(left: socket.socket, right: socket.socket) -> None: + with selectors.DefaultSelector() as selector: + selector.register(left, selectors.EVENT_READ, right) + selector.register(right, selectors.EVENT_READ, left) + while True: + ready = selector.select() + for key, _ in ready: + data = key.fileobj.recv(65536) + if not data: + return + key.data.sendall(data) + + +def _proxy_handler(host: str) -> type[socketserver.BaseRequestHandler]: + class Connect(socketserver.BaseRequestHandler): + def handle(self) -> None: + self.request.settimeout(15) + header = b'' + try: + while b'\r\n\r\n' not in header and len(header) <= 8192: + part = self.request.recv(1) + if not part: + return + header += part + first = header.split(b'\r\n', 1)[0].decode('ascii') + if b'\r\n\r\n' not in header or first not in ( + f'CONNECT {host}:443 HTTP/1.1', + f'CONNECT {host}:443 HTTP/1.0', + ): + self.request.sendall(b'HTTP/1.1 403 Forbidden\r\n\r\n') + return + addresses = public_addresses(host) + family, address = addresses[0] + with socket.socket(family, socket.SOCK_STREAM) as upstream: + upstream.settimeout(30) + upstream.connect(address) + self.request.sendall(b'HTTP/1.1 200 Connection Established\r\n\r\n') + _copy(self.request, upstream) + except (OSError, ValueError, UnicodeError): + return + + return Connect + + +def _bridge(directory: Path) -> None: + class Forward(socketserver.BaseRequestHandler): + def handle(self) -> None: + try: + with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as upstream: + upstream.connect(str(directory / 'proxy.sock')) + _copy(self.request, upstream) + except OSError: + return + + with socketserver.ThreadingTCPServer(('127.0.0.1', 0), Forward) as server: + server.daemon_threads = True + (directory / 'port').write_text(str(server.server_address[1])) + server.serve_forever() + + +def _supervise(directory: Path) -> None: + config = json.loads((directory / 'request.json').read_text()) + host = {'anthropic': 'api.anthropic.com', 'openai': 'api.openai.com'}[config['provider']] + started = time.monotonic() + deadline = started + config['timeout'] + bridge = None + process = None + server = socketserver.ThreadingUnixStreamServer( + str(directory / 'proxy.sock'), _proxy_handler(host) + ) + server.daemon_threads = True + threading.Thread(target=server.serve_forever, daemon=True).start() + status = {'exit_code': -1, 'stop_reason': None} + try: + descriptor = os.pidfd_open(os.getpid()) + try: + signal.pidfd_send_signal(descriptor, 0) + finally: + os.close(descriptor) + status['stop_reason'] = _stop_reason(directory, deadline) + if status['stop_reason'] is not None: + return + if config.get('supported_version') is not None: + _check_version(config, status, deadline) + if status.get('version_error'): + return + bridge = subprocess.Popen( + [*config['namespace'], sys.executable, '-I', __file__, 'bridge', str(directory)], + stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + start_new_session=True, + ) + while not (directory / 'port').exists(): + status['stop_reason'] = _stop_reason(directory, deadline) + if status['stop_reason'] is not None: + return + if bridge.poll() is not None or time.monotonic() - started > 10: + raise RuntimeError('Could not start the isolated provider proxy') + time.sleep(0.01) + proxy = f'http://127.0.0.1:{(directory / "port").read_text()}' + environment = { + **config['env'], + 'HTTP_PROXY': proxy, + 'HTTPS_PROXY': proxy, + 'http_proxy': proxy, + 'https_proxy': proxy, + 'NO_PROXY': _LOOPBACK_NO_PROXY, + 'no_proxy': _LOOPBACK_NO_PROXY, + } + with (directory / 'output').open('wb') as output: + status['stop_reason'] = _stop_reason(directory, deadline) + if status['stop_reason'] is not None: + return + process = subprocess.Popen( + [*config['prefix'], *config['argv']], + cwd=config['cwd'], + env=environment, + stdin=subprocess.DEVNULL, + stdout=output, + stderr=subprocess.STDOUT, + start_new_session=True, + ) + while process.poll() is None: + status['stop_reason'] = _stop_reason(directory, deadline) + if status['stop_reason'] is not None: + break + if output.tell() > 32 * 1024 * 1024: + status['stop_reason'] = 'output_limit' + break + time.sleep(0.02) + except Exception as exc: + status['error'] = str(exc) + finally: + if process is not None: + if process.poll() is None: + os.killpg(process.pid, signal.SIGKILL) + status['exit_code'] = process.wait(timeout=10) + if bridge is not None: + if bridge.poll() is None: + os.killpg(bridge.pid, signal.SIGKILL) + bridge.wait(timeout=10) + server.shutdown() + server.server_close() + temporary = directory / 'status.tmp' + temporary.write_text(json.dumps(status)) + temporary.replace(directory / 'status.json') + + +def parse_cli_version(output: str) -> str | None: + """Extract one unambiguous stable x.y.z release from a CLI version banner. + + Args: + output: stdout from claude --version or codex --version. + + Returns: + The release, or None for missing, malformed, or ambiguous banners. + """ + versions = [ + word + for word in output.split() + if len(word.split('.')) == 3 + and all(part.isascii() and part.isdecimal() for part in word.split('.')) + ] + return versions[0] if len(versions) == 1 else None + + +def _check_version(config: dict, status: dict, deadline: float) -> None: + command = config['argv'][0] + remaining = deadline - time.monotonic() + if remaining <= 0: + status['stop_reason'] = 'time_budget' + status['version_error'] = 'Wall budget exhausted before CLI version check' + return + failure = None + try: + observed = subprocess.run( + [*config['prefix'], command, '--version'], + cwd=config['cwd'], + env=config['version_env'], + stdin=subprocess.DEVNULL, + capture_output=True, + text=True, + timeout=min(10, remaining), + check=False, + ) + version = parse_cli_version(observed.stdout) + if observed.returncode in (126, 127): + failure = 'executable or interpreter is missing or not executable on the agent PATH' + elif observed.returncode != 0: + failure = f'version command exited with code {observed.returncode}' + except OSError: + version = None + failure = 'executable or interpreter could not be started on the agent PATH' + except subprocess.TimeoutExpired: + version = None + failure = 'version command timed out' + if time.monotonic() >= deadline: + status['stop_reason'] = 'time_budget' + status['cli_version'] = version + expected = config['supported_version'] + if failure: + status['version_error'] = ( + f'Cannot start {command}: {failure}. Rebuild the sandbox image using ' + 'deepgym.agents.cli_install_command() and ensure its root-owned Node ' + 'interpreter is executable on the same unprivileged PATH (for example ' + '/usr/local/bin/node); do not load interactive nvm profiles.' + ) + elif version != expected: + status['version_error'] = ( + f'Unsupported {command} version {version or "unknown"}; supported: {expected}. ' + 'Install the pinned release using deepgym.agents.cli_install_command(), ' + 'or bump SUPPORTED_CLIS, record versioned fixtures, run ' + 'tests/agents/test_cli_compat.py, and rebuild sandbox images.' + ) + + +def _stop_reason(directory: Path, deadline: float) -> str | None: + marker = directory / 'stop' + if marker.exists(): + return marker.read_text() + return 'time_budget' if time.monotonic() >= deadline else None + + +if __name__ == '__main__': + operation, target = sys.argv[1:] + if operation == 'bridge': + _bridge(Path(target)) + else: + _supervise(Path(target)) diff --git a/deepgym/src/deepgym/_pid_namespace.py b/deepgym/src/deepgym/_pid_namespace.py new file mode 100644 index 00000000..20ab7163 --- /dev/null +++ b/deepgym/src/deepgym/_pid_namespace.py @@ -0,0 +1,152 @@ +"""Keep every session agent process in one kernel-owned lifetime (sandbox-only).""" + +import json +import os +import select +import signal +import subprocess +import sys +import time +from pathlib import Path + + +def keep(directory: Path) -> None: + """Act as the trusted PID 1, reaping orphaned setup and agent children.""" + if os.getpid() != 1: + raise RuntimeError('Agent keeper is not PID 1') + + def reap(signum: int, frame: object) -> None: + try: + while os.waitpid(-1, os.WNOHANG)[0]: + pass + except ChildProcessError: + pass + + signal.signal(signal.SIGCHLD, reap) + (directory / 'ready').touch() + while True: + signal.pause() + + +def create(directory: Path, uid: int) -> None: + """Create and pin PID/mount namespaces before any agent process can start. + + Args: + directory: Root-only control directory created by SandboxBackend. + uid: Exclusive, non-root agent UID. + + Raises: + RuntimeError: If the namespace keeper cannot be established. + subprocess.SubprocessError: If kernel containment is unavailable. + """ + if uid <= 0: + raise ValueError('Refusing to contain root as an agent') + descriptor = os.pidfd_open(os.getpid()) + try: + signal.pidfd_send_signal(descriptor, 0) + finally: + os.close(descriptor) + wrapper = subprocess.Popen( + [ + 'unshare', + '--pid', + '--fork', + '--mount-proc', + '--kill-child=KILL', + '--', + sys.executable, + '-I', + __file__, + 'keep', + str(directory), + ], + stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + start_new_session=True, + ) + try: + deadline = time.monotonic() + 10 + while not (directory / 'ready').exists(): + if wrapper.poll() is not None or time.monotonic() >= deadline: + raise RuntimeError('Could not create the agent PID namespace') + time.sleep(0.01) + children = Path(f'/proc/{wrapper.pid}/task/{wrapper.pid}/children').read_text().split() + if len(children) != 1: + raise RuntimeError('Cannot identify the agent namespace init') + init = int(children[0]) + for namespace in ('pid', 'mnt'): + target = directory / namespace + target.touch(mode=0o600) + subprocess.run( + ['mount', '--bind', f'/proc/{init}/ns/{namespace}', str(target)], + check=True, + timeout=10, + ) + (directory / 'identity.json').write_text(json.dumps({'pid': init, 'uid': uid})) + except BaseException: + wrapper.kill() + wrapper.wait(timeout=10) + raise + + +def verify_empty(uid: int) -> None: + """Refuse live agent survivors after the kernel has destroyed their PID namespace. + + This scan is a postcondition, never the mechanism used to terminate processes. + All unprivileged creation paths must enter the now-dead namespace first. + """ + if uid <= 0: + raise ValueError('Refusing to inspect root as an agent') + for entry in Path('/proc').iterdir(): + if not entry.name.isdecimal(): + continue + try: + fields = dict( + line.split(':', 1) + for line in (entry / 'status').read_text().splitlines() + if ':' in line + ) + except (FileNotFoundError, ProcessLookupError): + continue + if int(fields['Uid'].split()[0]) == uid and not fields['State'].strip().startswith('Z'): + raise RuntimeError('Agent process survived PID namespace teardown; grading is unsafe') + + +def stop(directory: Path) -> None: + """Kill the pinned namespace init from its ancestor, then verify kernel teardown. + + Raises: + RuntimeError: If the init identity or namespace death cannot be established. + OSError: If pidfds or namespace introspection are unavailable. + """ + identity = json.loads((directory / 'identity.json').read_text()) + if identity['uid'] <= 0 or identity['pid'] <= 1: + raise ValueError('Refusing root UID or host init as an agent namespace') + if (directory / 'stopped').exists(): + verify_empty(identity['uid']) + return + descriptor = os.pidfd_open(identity['pid']) + try: + actual = Path(f'/proc/{identity["pid"]}/ns/pid').stat().st_ino + if actual != (directory / 'pid').stat().st_ino: + raise RuntimeError('Agent namespace init identity changed; refusing PID reuse') + signal.pidfd_send_signal(descriptor, signal.SIGKILL) + if not select.select([descriptor], [], [], 10)[0]: + raise RuntimeError('Agent PID namespace did not terminate') + finally: + os.close(descriptor) + verify_empty(identity['uid']) + (directory / 'stopped').touch() + + +if __name__ == '__main__': + operation, path, *arguments = sys.argv[1:] + if operation == 'keep': + keep(Path(path)) + elif operation == 'create': + create(Path(path), int(arguments[0])) + elif operation == 'stop': + stop(Path(path)) + else: + raise ValueError('Unknown namespace operation') diff --git a/deepgym/src/deepgym/agents/__init__.py b/deepgym/src/deepgym/agents/__init__.py new file mode 100644 index 00000000..1cd0e3c3 --- /dev/null +++ b/deepgym/src/deepgym/agents/__init__.py @@ -0,0 +1,21 @@ +"""Run provider agents over DeepGym sessions; SDKs are optional and imported on run.""" + +from deepgym.agents.base import Agent, AgentResult, Budget, Cost, TokenPrices, Usage +from deepgym.agents.cli import ClaudeCodeAgent, CodexAgent +from deepgym.agents.computer import AnthropicComputerAgent, OpenAIComputerAgent +from deepgym.agents.versions import SUPPORTED_CLIS, cli_install_command + +__all__ = [ + 'Agent', + 'AgentResult', + 'AnthropicComputerAgent', + 'Budget', + 'ClaudeCodeAgent', + 'CodexAgent', + 'Cost', + 'OpenAIComputerAgent', + 'TokenPrices', + 'Usage', + 'SUPPORTED_CLIS', + 'cli_install_command', +] diff --git a/deepgym/src/deepgym/agents/actions.py b/deepgym/src/deepgym/agents/actions.py new file mode 100644 index 00000000..a470dd01 --- /dev/null +++ b/deepgym/src/deepgym/agents/actions.py @@ -0,0 +1,190 @@ +"""Translate provider computer actions without silently dropping semantics.""" + +import math +from collections.abc import Mapping, Sequence +from typing import Any + +from deepgym.models import Action + +_KEY_ALIASES = { + 'return': 'enter', + 'kp_enter': 'enter', + 'esc': 'escape', + 'back_space': 'backspace', + 'del': 'delete', + 'page_down': 'pagedown', + 'page_up': 'pageup', + 'pgdn': 'pagedown', + 'pgup': 'pageup', + 'arrowdown': 'down', + 'arrowup': 'up', + 'arrowleft': 'left', + 'arrowright': 'right', + 'control': 'ctrl', + 'super': 'cmd', + 'meta': 'cmd', + 'win': 'cmd', + 'option': 'alt', + 'spacebar': 'space', + ' ': 'space', +} + + +def _key(value: str) -> str: + return _KEY_ALIASES.get(value.lower(), value.lower() if len(value) > 1 else value) + + +def _integer(value: object, label: str) -> int: + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise ValueError(f'{label} must be an integer') + if not math.isfinite(value) or int(value) != value: + raise ValueError(f'{label} must be a finite integer') + return int(value) + + +def _point(value: object) -> dict[str, int]: + if not isinstance(value, (list, tuple)) or len(value) != 2: + raise ValueError('a coordinate with exactly two integers is required') + return {'x': _integer(value[0], 'x'), 'y': _integer(value[1], 'y')} + + +def _text(value: object) -> str: + if not isinstance(value, str) or not value: + raise ValueError('nonempty text is required') + return value + + +def anthropic_actions(name: str, data: Mapping[str, Any]) -> list[Action]: + """Map an Anthropic tool call; reject actions the Session cannot represent. + + Args: + name: Computer tool name, or ``computer`` for legacy input.action calls. + data: Provider tool input. + + Returns: + Actions preserving the supported operation. + + Raises: + ValueError: For unsupported operations, modifiers, or malformed arguments. + """ + if name == 'computer': + name = str(data.get('action', '')) + pointer = { + 'left_click', + 'right_click', + 'double_click', + 'mouse_move', + 'scroll', + 'left_click_drag', + } + if name in pointer and data.get('text'): + raise ValueError('mouse modifier keys are unsupported by Session') + if name == 'screenshot': + return [Action(action_type='screenshot')] + if name in pointer: + point = _point(data.get('coordinate')) + if name == 'scroll': + return [ + Action( + action_type='scroll', + content=_text(data.get('scroll_direction')), + amount=_integer(data.get('scroll_amount', 3), 'scroll_amount'), + **point, + ) + ] + if name == 'left_click_drag': + start = _point(data.get('start_coordinate')) + return [Action(action_type='drag', **start, to_x=point['x'], to_y=point['y'])] + kind = { + 'left_click': 'click', + 'right_click': 'right_click', + 'double_click': 'double_click', + 'mouse_move': 'move', + }[name] + return [Action(action_type=kind, **point)] + if name == 'type': + return [Action(action_type='type', content=_text(data.get('text')))] + if name == 'key': + keys = _text(data.get('text')).split('+') + if any(not key.strip() for key in keys): + raise ValueError('keypress contains an empty key') + repeat = _integer(data.get('repeat', 1), 'repeat') + if not 1 <= repeat <= 50: + raise ValueError('key repeat must be between 1 and 50') + return [ + Action(action_type='key', content='+'.join(_key(key.strip()) for key in keys)) + for _ in range(repeat) + ] + if name == 'wait': + seconds = data.get('duration', 1) + if isinstance(seconds, bool) or not isinstance(seconds, (int, float)): + raise ValueError('wait duration must be numeric') + if not math.isfinite(seconds) or not 0 < seconds <= 60: + raise ValueError('wait duration must be between 0 and 60 seconds') + return [Action(action_type='wait', amount=math.ceil(seconds))] + raise ValueError(f'Anthropic computer action {name!r} is unsupported by Session') + + +def openai_actions(data: Mapping[str, Any]) -> list[Action]: + """Map a Responses computer action; reject unrepresentable buttons and paths. + + Args: + data: Provider action serialized with model_dump. + + Returns: + Actions for Session.step. Pixel scroll distances become wheel clicks at 100px/click. + + Raises: + ValueError: For unsupported actions, modifiers, malformed inputs, or curved drags. + """ + name = data.get('type') + if data.get('keys') and name != 'keypress': + raise ValueError('mouse modifier keys are unsupported by Session') + if name in {'screenshot', 'wait'}: + return [Action(action_type=name, amount=2 if name == 'wait' else 1)] + if name == 'type': + return [Action(action_type='type', content=_text(data.get('text')))] + if name == 'keypress': + keys = data.get('keys') + if not isinstance(keys, (list, tuple)) or not keys: + raise ValueError('keypress needs a nonempty keys list') + return [Action(action_type='key', content='+'.join(_key(_text(key)) for key in keys))] + if name == 'drag': + path = data.get('path') + if not isinstance(path, Sequence) or len(path) != 2: + raise ValueError('Session drag requires exactly two points; curved paths unsupported') + if not all(isinstance(point, Mapping) for point in path): + raise ValueError('drag points must contain x and y') + start = _point([path[0].get('x'), path[0].get('y')]) + end = _point([path[1].get('x'), path[1].get('y')]) + return [Action(action_type='drag', **start, to_x=end['x'], to_y=end['y'])] + if name in {'click', 'double_click', 'move', 'scroll'}: + point = _point([data.get('x'), data.get('y')]) + if name == 'click': + button = data.get('button', 'left') + if button not in {'left', 'right'}: + raise ValueError(f'Session cannot represent mouse button {button!r}') + return [Action(action_type='click' if button == 'left' else 'right_click', **point)] + if name == 'scroll': + horizontal = _integer(data.get('scroll_x', 0), 'scroll_x') + vertical = _integer(data.get('scroll_y', 0), 'scroll_y') + if horizontal and vertical: + raise ValueError('Session cannot represent simultaneous diagonal scrolling') + if not horizontal and not vertical: + raise ValueError('scroll requires a nonzero distance') + distance = vertical or horizontal + direction = ( + ('down' if distance > 0 else 'up') + if vertical + else ('right' if distance > 0 else 'left') + ) + return [ + Action( + action_type='scroll', + content=direction, + amount=max(1, round(abs(distance) / 100)), + **point, + ) + ] + return [Action(action_type=name, **point)] + raise ValueError(f'OpenAI computer action {name!r} is unsupported by Session') diff --git a/deepgym/src/deepgym/agents/base.py b/deepgym/src/deepgym/agents/base.py new file mode 100644 index 00000000..168e7e3d --- /dev/null +++ b/deepgym/src/deepgym/agents/base.py @@ -0,0 +1,192 @@ +"""Agent contracts and shared, monotonic episode accounting.""" + +from __future__ import annotations + +import time +from typing import Literal, Protocol, runtime_checkable + +from pydantic import BaseModel, ConfigDict, Field + +from deepgym.exceptions import DeepGymError +from deepgym.models import Observation, Trajectory +from deepgym.session import Session + +TerminationReason = Literal[ + 'completed', + 'turn_budget', + 'time_budget', + 'cost_budget', + 'session_done', + 'refusal', + 'safety_check', + 'error', + 'provider_error', +] + + +class Budget(BaseModel): + """Bound episode work; USD limits stop new calls after reported spend reaches the cap. + + An in-flight response can exceed the USD limit. Provider billing and model-readable + CLI credentials cannot be hard-capped by this local accounting mechanism. + """ + + model_config = ConfigDict(frozen=True, allow_inf_nan=False) + max_turns: int = Field(default=30, ge=1) + max_seconds: float = Field(default=300, gt=0) + max_cost_usd: float | None = Field(default=None, ge=0) + + +class Usage(BaseModel): + """Record disjoint token categories; output includes reasoning tokens.""" + + model_config = ConfigDict(frozen=True) + input_tokens: int = Field(default=0, ge=0) + output_tokens: int = Field(default=0, ge=0) + cache_read_tokens: int = Field(default=0, ge=0) + cache_write_tokens: int = Field(default=0, ge=0) + + +class Cost(BaseModel): + """Report USD with its measurement or estimation provenance; None means unknown.""" + + model_config = ConfigDict(frozen=True, allow_inf_nan=False) + usd: float | None = Field(default=None, ge=0) + provenance: str = Field(default='unavailable', min_length=1) + + +class TokenPrices(BaseModel): + """Supply model-specific USD rates per million tokens and their dated source. + + Rates are caller-supplied, never silently inferred from a mutable model alias. + Cache prices are required so cached tokens cannot become silently free. + """ + + model_config = ConfigDict(frozen=True, allow_inf_nan=False) + input_per_million: float = Field(ge=0) + output_per_million: float = Field(ge=0) + cache_read_per_million: float = Field(ge=0) + cache_write_per_million: float = Field(ge=0) + source: str = Field(min_length=1) + + def estimate(self, usage: Usage) -> Cost: + """Calculate cost from provider usage and caller-supplied rates.""" + total = ( + usage.input_tokens * self.input_per_million + + usage.output_tokens * self.output_per_million + + usage.cache_read_tokens * self.cache_read_per_million + + usage.cache_write_tokens * self.cache_write_per_million + ) / 1_000_000 + return Cost(usd=total, provenance=f'provider token usage * caller rates: {self.source}') + + +class AgentResult(BaseModel): + """Return the observed trajectory and accounting, without grading or closing it.""" + + model_config = ConfigDict(frozen=True, allow_inf_nan=False) + trajectory: Trajectory + usage: Usage = Field(default_factory=Usage) + cost: Cost = Field(default_factory=Cost) + termination_reason: TerminationReason + turns: int = Field(default=0, ge=0) + elapsed_seconds: float = Field(default=0, ge=0) + final_text: str = '' + error: str | None = None + cli_version: str | None = None + """Installed CLI version observed by the sandbox supervisor via --version.""" + + +@runtime_checkable +class Agent(Protocol): + """Drive an already-started session; its caller owns scoring and cleanup.""" + + def run( + self, session: Session, initial: Observation, *, task: str, budget: Budget + ) -> AgentResult: + """Execute the task within budget and return observed work.""" + ... + + +class BudgetExceeded(DeepGymError): # noqa: N818 - control-flow signal, like StopIteration + """Signal that no further provider calls or actions are permitted.""" + + def __init__(self, reason: TerminationReason) -> None: + super().__init__(reason) + self.reason = reason + + +class BudgetTracker: + """Share cumulative accounting across every turn, including failed calls. + + Check before calls and actions, and after responses. SDK request timeouts use + remaining wall time; Session propagates remaining deadlines to backend operations. + SDK transport timeouts alone do not guarantee an absolute response deadline. + """ + + def __init__(self, budget: Budget) -> None: + self.budget = budget + self.started = time.monotonic() + self.turns = 0 + self.usage = Usage() + self.cost = Cost(usd=0, provenance='no provider calls') + + @property + def remaining_seconds(self) -> float: + """Return remaining wall time, including retries and tool execution.""" + return max(0, self.budget.max_seconds - (time.monotonic() - self.started)) + + @property + def reason(self) -> TerminationReason | None: + """Return an exhausted time or spend bound, without checking turn admission.""" + if self.remaining_seconds <= 0: + return 'time_budget' + limit = self.budget.max_cost_usd + if limit is not None and (self.cost.usd is None or self.cost.usd >= limit): + return 'cost_budget' + return None + + def check(self) -> None: + """Raise BudgetExceeded when time or spend disallows further work.""" + if reason := self.reason: + raise BudgetExceeded(reason) + + def before_turn(self) -> None: + """Admit one provider call, raising BudgetExceeded at an exhausted bound.""" + self.check() + if self.turns >= self.budget.max_turns: + raise BudgetExceeded('turn_budget') + self.turns += 1 + + def add_usage(self, usage: Usage, cost: Cost) -> None: + """Accumulate disjoint response usage and propagate unknown costs.""" + self.usage = Usage( + **{ + name: getattr(self.usage, name) + value + for name, value in usage.model_dump().items() + } + ) + total = None if self.cost.usd is None or cost.usd is None else self.cost.usd + cost.usd + sources = list(dict.fromkeys((self.cost.provenance, cost.provenance))) + self.cost = Cost( + usd=total, + provenance='; '.join(source for source in sources if source != 'no provider calls'), + ) + + def result( + self, + session: Session, + reason: TerminationReason, + final_text: str = '', + error: str | None = None, + ) -> AgentResult: + """Snapshot accounting and the session's real actions without grading.""" + return AgentResult( + trajectory=session.trajectory.model_copy(deep=True), + usage=self.usage, + cost=self.cost, + termination_reason=reason, + turns=self.turns, + elapsed_seconds=time.monotonic() - self.started, + final_text=final_text, + error=error, + ) diff --git a/deepgym/src/deepgym/agents/cli.py b/deepgym/src/deepgym/agents/cli.py new file mode 100644 index 00000000..14c134b3 --- /dev/null +++ b/deepgym/src/deepgym/agents/cli.py @@ -0,0 +1,195 @@ +"""Run installed, headless CLIs as unprivileged sandbox processes. + +Phase 1 requires explicit consent to model-readable credentials. The fixed provider +public-address tunnel passes opaque TLS, not verified HTTP routes; a shared CDN can +serve other SNI/Host names. Keys can be used outside CLI accounting. These adapters +never grade, replay a parsed tool call, or run the CLI on the host. +""" + +from __future__ import annotations + +import os +from typing import ClassVar, Literal + +from pydantic import BaseModel, ConfigDict, Field, SecretStr + +from deepgym.agents.base import AgentResult, Budget, BudgetTracker, Cost, TokenPrices +from deepgym.agents.cli_events import CliEvents +from deepgym.agents.versions import SUPPORTED_CLIS +from deepgym.cli_runtime import CliExecutionError, CliRequest +from deepgym.exceptions import ActionError, DeepGymError +from deepgym.models import Observation +from deepgym.session import Session + + +class _CliAgent(BaseModel): + model_config = ConfigDict(frozen=True, extra='forbid') + model: str = Field(min_length=1) + api_key: SecretStr | None = Field(default=None, exclude=True, repr=False) + allow_model_credentials: bool = False + prices: TokenPrices | None = None + provider: ClassVar[Literal['anthropic', 'openai']] + + def run( + self, + session: Session, + initial: Observation, + *, + task: str, + budget: Budget, + ) -> AgentResult: + """Run once and record a single Session action containing nested CLI events. + + Args: + session: Started sandbox Session with supervised CLI support. + initial: Observation returned by Session.start. + task: Task supplied as one literal CLI argument. + budget: Wall deadline, activity cap, and optional USD bound. + + Returns: + Trajectory and reported usage/cost, including interruption or errors. + + Raises: + ValueError: If credentials are missing, exposure was not allowed, or a + finite USD budget cannot be enforced by this CLI. + ActionError: If the environment disallows CLI actions or provider egress. + DeepGymError: If containment cannot be finalized; the session is ungradable. + """ + try: + result = self._run(session, task, budget) + finally: + session.finalize_cli() + return result.model_copy(update={'trajectory': session.trajectory.model_copy(deep=True)}) + + def _run(self, session: Session, task: str, budget: Budget) -> AgentResult: + tracker = BudgetTracker(budget) + if self.provider == 'openai' and budget.max_cost_usd is not None: + raise ValueError( + 'Codex CLI reports usage only after the entire task: a finite USD budget ' + 'cannot be enforced in phase 1; use a capped relay or omit max_cost_usd' + ) + if session.done: + return tracker.result(session, 'session_done') + if budget.max_cost_usd == 0: + return tracker.result(session, 'cost_budget') + if not self.allow_model_credentials: + raise ValueError( + 'CLI agents expose the provider API key to model-generated commands; ' + 'set allow_model_credentials=True to opt in' + ) + if budget.max_cost_usd is not None and self.prices is None: + raise ValueError( + 'A finite CLI USD budget requires explicit TokenPrices to monitor ' + 'each assistant message; the native CLI budget flag is secondary' + ) + source = 'ANTHROPIC_API_KEY' if self.provider == 'anthropic' else 'OPENAI_API_KEY' + key = self.api_key.get_secret_value() if self.api_key else os.environ.get(source) + if not key: + raise ValueError(f'{source} or api_key is required') + if tracker.remaining_seconds <= 0: + return tracker.result(session, 'time_budget') + events = CliEvents(self.provider, tracker, self.prices) + credential = source if self.provider == 'anthropic' else 'CODEX_API_KEY' + request = CliRequest( + argv=self._argv(task, budget), + env={credential: key}, + provider=self.provider, + timeout=tracker.remaining_seconds, + allow_model_credentials=True, + supported_version=SUPPORTED_CLIS[self.provider].version, + ) + try: + execution = session.run_cli(request, events.consume) + except ActionError: + raise + except DeepGymError as exc: + tracker.cost = tracker.cost.model_copy( + update={ + 'provenance': tracker.cost.provenance + + '; CLI failure: usage may be incomplete', + } + ) + reason = 'time_budget' if tracker.remaining_seconds <= 0 else 'error' + result = tracker.result(session, reason, events.final_text, type(exc).__name__) + if isinstance(exc, CliExecutionError): + result = result.model_copy(update={'cli_version': exc.cli_version}) + return result + if execution.version_error: + tracker.cost = Cost(usd=0, provenance='CLI version rejected before inference') + reason = ( + 'time_budget' + if execution.stop_reason == 'time_budget' or tracker.remaining_seconds <= 0 + else 'error' + ) + result = tracker.result(session, reason, error=execution.version_error) + else: + reason = events.finish(execution.exit_code, execution.stop_reason) + result = tracker.result(session, reason, events.final_text, events.error) + return result.model_copy(update={'cli_version': execution.cli_version}) + + def _argv(self, task: str, budget: Budget) -> tuple[str, ...]: + if self.provider == 'anthropic': + args = ( + SUPPORTED_CLIS[self.provider].command, + '--print', + '--verbose', + '--output-format', + 'stream-json', + '--bare', + '--setting-sources', + '', + '--strict-mcp-config', + '--no-session-persistence', + '--dangerously-skip-permissions', + '--model', + self.model, + ) + if budget.max_cost_usd is not None: + args += ('--max-budget-usd', str(budget.max_cost_usd)) + return (*args, '--', task) + return ( + SUPPORTED_CLIS[self.provider].command, + 'exec', + '--json', + '--ephemeral', + '--ignore-user-config', + '--skip-git-repo-check', + '--color', + 'never', + '--dangerously-bypass-approvals-and-sandbox', + '--model', + self.model, + '--', + task, + ) + + +class ClaudeCodeAgent(_CliAgent): + """Drive a sandbox with an installed Claude Code CLI. + + Opt in with ``allow_model_credentials=True``: the agent can read the real API + key and make additional provider requests. The sandbox namespace and provider + allowlist remain enforced, but this is not credential isolation or a hard bill + cap. Claude's native dollar-budget flag complements host event policing. + ``Budget.max_turns`` counts observed assistant messages; polling can overshoot. + CLI-reported USD takes precedence over optional caller-rate estimates. + A finite USD budget requires ``prices`` to stop on per-message reported usage; + missing usage fails closed. In-flight calls can still exceed the dollar cap. + """ + + provider: ClassVar[Literal['anthropic']] = 'anthropic' + + +class CodexAgent(_CliAgent): + """Drive a sandbox with an installed Codex CLI using explicit API credentials. + + Opt in with ``allow_model_credentials=True`` to expose the real provider key + to model-generated commands. ``Budget.max_turns`` counts completed activity + items, including messages, reasoning and tools; it is not a model-call count. + The host stops at observed boundaries with polling latency. Codex reports token + usage only after the whole task; finite USD limits therefore fail closed before + launching. Optional ``prices`` estimate cost from reported usage with provenance. + Interrupted runs can have unavailable usage/cost. Capped relay support is phase 2. + """ + + provider: ClassVar[Literal['openai']] = 'openai' diff --git a/deepgym/src/deepgym/agents/cli_events.py b/deepgym/src/deepgym/agents/cli_events.py new file mode 100644 index 00000000..c5e8d02d --- /dev/null +++ b/deepgym/src/deepgym/agents/cli_events.py @@ -0,0 +1,297 @@ +"""Interpret CLI JSON events as observations, never executable Session actions. + +Claude usage is per assistant message until its cumulative result arrives. Codex +reports usage only at ``turn.completed`` (the entire user task, not one model call). +Unknown event types remain in the Session's raw nested events for forward fidelity. +""" + +from __future__ import annotations + +from typing import Literal + +from pydantic import JsonValue + +from deepgym.agents.base import BudgetTracker, Cost, TerminationReason, TokenPrices, Usage + +_PROVIDER_CODES = frozenset( + { + 'usageLimitExceeded', + 'rateLimitExceeded', + 'serverOverloaded', + 'internalServerError', + 'unauthorized', + } +) +_PROVIDER_TRANSPORT_CODES = frozenset( + { + 'httpConnectionFailed', + 'responseStreamConnectionFailed', + 'responseStreamDisconnected', + 'responseTooManyFailedAttempts', + } +) + + +def _codex_error_reason(detail: dict[str, JsonValue]) -> Literal['error', 'provider_error']: + """Classify only explicit provider codes; message-only exec errors stay generic.""" + code = detail.get('codexErrorInfo') + if isinstance(code, str) and code in _PROVIDER_CODES: + return 'provider_error' + if isinstance(code, dict) and len(code) == 1: + name, metadata = next(iter(code.items())) + if name in _PROVIDER_TRANSPORT_CODES and isinstance(metadata, dict): + status = metadata.get('httpStatusCode') + if status is None or ( + type(status) is int and (status in (401, 403, 408, 429) or 500 <= status <= 599) + ): + return 'provider_error' + return 'error' + + +def _object(value: JsonValue, label: str) -> dict[str, JsonValue]: + if not isinstance(value, dict): + raise ValueError(f'{label} must be a JSON object') + return value + + +def _tokens(data: dict[str, JsonValue], key: str, *, optional: bool = False) -> int: + value = data.get(key, 0 if optional else None) + if not isinstance(value, int) or isinstance(value, bool) or value < 0: + raise ValueError(f'{key} must be a non-negative integer') + return value + + +def _usage(data: dict[str, JsonValue], provider: str) -> Usage: + cached = _tokens( + data, + 'cache_read_input_tokens' if provider == 'anthropic' else 'cached_input_tokens', + optional=True, + ) + input_tokens = _tokens(data, 'input_tokens') + if provider == 'openai': + input_tokens -= cached + if input_tokens < 0: + raise ValueError('cached_input_tokens exceeds input_tokens') + return Usage( + input_tokens=input_tokens, + output_tokens=_tokens(data, 'output_tokens'), + cache_read_tokens=cached, + cache_write_tokens=_tokens(data, 'cache_creation_input_tokens', optional=True), + ) + + +class CliEvents: + """Track a CLI stream while leaving all side effects to the CLI process. + + Turns are unique Claude assistant messages or completed Codex activity items. + Neither JSON stream exposes every request before it executes; caps are enforced + at observed event boundaries with supervisor polling latency. A cancelled run's + token usage can be incomplete. Cost provenance states that limitation. + + Args: + provider: CLI event dialect. + tracker: Episode accounting and monotonic deadline. + prices: Optional caller-supplied rates for token estimates. + """ + + def __init__( + self, + provider: Literal['anthropic', 'openai'], + tracker: BudgetTracker, + prices: TokenPrices | None = None, + ) -> None: + self.provider = provider + self.tracker = tracker + self.prices = prices + self.completed = False + self.final_text = '' + self.error: str | None = None + self._error_reason: Literal['error', 'provider_error'] = 'error' + self.stop_reason: TerminationReason | None = None + self._seen: set[str] = set() + self._messages: dict[str, Usage | None] = {} + self._unreported_messages: set[str] = set() + self._reported_usage = False + self.tracker.cost = Cost(provenance='CLI usage/cost not yet reported') + + def consume(self, event: dict[str, JsonValue]) -> str | None: + """Consume one event and request supervisor termination when a bound fires. + + Args: + event: One decoded stdout JSON object. + + Returns: + A termination reason, or None to keep reading. + """ + first_stop = self.stop_reason + try: + if not isinstance(event.get('type'), str): + raise ValueError('CLI event requires a string type') + if self.provider == 'anthropic': + self._claude(event) + else: + self._codex(event) + except (ValueError, TypeError) as exc: + self.error = f'Invalid {self.provider} CLI event: {exc}' + self._error_reason = 'error' + if first_stop is not None: + self.stop_reason = first_stop + return first_stop + if self.error: + self.stop_reason = self._error_reason + elif self.tracker.remaining_seconds <= 0: + self.stop_reason = 'time_budget' + elif ( + self.tracker.budget.max_cost_usd is not None + and self.tracker.cost.usd is not None + and self.tracker.cost.usd >= self.tracker.budget.max_cost_usd + ): + self.stop_reason = 'cost_budget' + elif not self.completed and self.tracker.turns >= self.tracker.budget.max_turns: + self.stop_reason = 'turn_budget' + return self.stop_reason + + def finish(self, exit_code: int, stop_reason: str | None) -> TerminationReason: + """Validate terminal evidence and label incomplete accounting on interruption. + + Args: + exit_code: Supervised process exit code. + stop_reason: Supervisor deadline or event-bound stop reason. + + Returns: + The observed episode termination reason. + """ + reason = self.stop_reason or stop_reason + if reason not in ('turn_budget', 'time_budget', 'cost_budget', 'provider_error', 'error'): + if self.error is None and exit_code != 0: + self.error = f'{self.provider} CLI exited with code {exit_code}' + if self.error is None and not self.completed: + self.error = f'{self.provider} CLI exited without a terminal completion event' + if self.error is None and (not self._reported_usage or self._unreported_messages): + self.error = f'{self.provider} CLI completed without token usage' + reason = self._error_reason if self.error else 'completed' + if reason != 'completed': + self.tracker.cost = self.tracker.cost.model_copy( + update={ + 'provenance': self.tracker.cost.provenance + + '; incomplete observed CLI subtotal: additional usage may be unreported', + } + ) + return reason + + def _record_usage(self, usage: Usage, *, cumulative: bool = False) -> None: + if cumulative: + if any( + value < getattr(self.tracker.usage, key) + for key, value in usage.model_dump().items() + ): + raise ValueError('cumulative result usage is below observed assistant usage') + self.tracker.usage = usage + self._unreported_messages.clear() + else: + self.tracker.usage = Usage( + **{ + key: getattr(self.tracker.usage, key) + value + for key, value in usage.model_dump().items() + } + ) + self._reported_usage = True + self.tracker.cost = ( + self.prices.estimate(self.tracker.usage) + if self.prices is not None + else Cost(provenance='CLI reported token usage; USD unavailable without caller rates') + ) + if self._unreported_messages: + self.tracker.cost = Cost( + provenance='Some Claude assistant messages omitted token usage' + ) + + def _claude(self, event: dict[str, JsonValue]) -> None: + kind = event['type'] + if kind == 'assistant': + message = _object(event.get('message'), 'assistant message') + identity = message.get('id') + if not isinstance(identity, str) or not identity: + raise ValueError('assistant message requires an id') + if identity not in self._messages: + self.tracker.turns += 1 + if message.get('usage') is None: + self._messages.setdefault(identity, None) + self._unreported_messages.add(identity) + self.tracker.cost = Cost(provenance='Claude assistant message omitted token usage') + if self.tracker.budget.max_cost_usd is not None: + raise ValueError('assistant usage is required for a finite USD budget') + else: + usage = _usage(_object(message['usage'], 'assistant usage'), self.provider) + previous = self._messages.get(identity) or Usage() + delta = { + key: value - getattr(previous, key) for key, value in usage.model_dump().items() + } + if any(value < 0 for value in delta.values()): + raise ValueError('assistant message usage counters regressed') + self._messages[identity] = usage + self._unreported_messages.discard(identity) + self._record_usage(Usage(**delta)) + content = message.get('content') + if not isinstance(content, list): + raise ValueError('assistant content must be an array') + texts = [ + block['text'] + for block in content + if isinstance(block, dict) + and block.get('type') == 'text' + and isinstance(block.get('text'), str) + ] + if texts: + self.final_text = '\n'.join(texts) + elif kind == 'result': + subtype = event.get('subtype') + if subtype == 'error_max_turns': + self.stop_reason = 'turn_budget' + elif subtype == 'error_max_budget_usd': + self.stop_reason = 'cost_budget' + elif event.get('is_error') is True or subtype != 'success': + self.error = str(event.get('errors') or event.get('result') or subtype) + else: + self.completed = True + if isinstance(event.get('result'), str): + self.final_text = event['result'] + if event.get('usage') is not None: + self._record_usage( + _usage(_object(event['usage'], 'result usage'), self.provider), + cumulative=True, + ) + if event.get('total_cost_usd') is not None: + self.tracker.cost = Cost( + usd=event['total_cost_usd'], + provenance=( + 'Claude Code result.total_cost_usd (CLI-reported, not billing verified)' + ), + ) + + def _codex(self, event: dict[str, JsonValue]) -> None: + kind = event['type'] + if kind == 'item.completed': + item = _object(event.get('item'), 'completed item') + identity = item.get('id') + if not isinstance(identity, str) or not identity: + raise ValueError('completed item requires an id') + if identity in self._seen: + return + self._seen.add(identity) + self.tracker.turns += 1 + if item.get('type') == 'agent_message': + if not isinstance(item.get('text'), str): + raise ValueError('agent_message requires text') + self.final_text = item['text'] + elif kind == 'turn.completed': + self._record_usage(_usage(_object(event.get('usage'), 'turn usage'), self.provider)) + self.completed = True + elif kind == 'error': + # exec can strip willRetry; only a terminal event/exit proves failure. + _object(event.get('error', event), 'CLI error') + elif kind == 'turn.failed': + detail = event.get('error', event) + detail = _object(detail, 'CLI error') + self.error = str(detail.get('message') or 'CLI turn failed') + self._error_reason = _codex_error_reason(detail) diff --git a/deepgym/src/deepgym/agents/computer.py b/deepgym/src/deepgym/agents/computer.py new file mode 100644 index 00000000..e849db76 --- /dev/null +++ b/deepgym/src/deepgym/agents/computer.py @@ -0,0 +1,401 @@ +"""Host-side provider loops: credentials never enter the desktop sandbox. + +USD enforcement uses returned usage and caller-supplied prices, so one response +can overshoot the cap. SDK retries are disabled and request timeouts use remaining +wall time. Session propagates remaining deadlines through actions and screenshots; +SDK transport timeouts do not guarantee an absolute response deadline. These agents +do not score, close, or acknowledge provider safety checks automatically. +""" + +from __future__ import annotations + +from typing import Any, Literal + +from deepgym.agents.actions import anthropic_actions, openai_actions +from deepgym.agents.base import ( + AgentResult, + Budget, + BudgetExceeded, + BudgetTracker, + Cost, + TokenPrices, + Usage, +) +from deepgym.exceptions import ActionError, DeepGymError +from deepgym.models import Action, Observation +from deepgym.session import Session + +_SYSTEM = ( + 'Operate the desktop using screenshots and computer actions to complete the user task. ' + 'When finished, stop calling tools and explain the result. If unable to finish, say so.' +) + + +def _image(screenshot: str | None) -> str: + if not screenshot: + raise ValueError('Computer agents require a desktop observation with a screenshot') + return screenshot + + +def _step(session: Session, action: Action, tracker: BudgetTracker) -> Observation: + tracker.check() + if session.done: + raise BudgetExceeded('session_done') + if action.action_type == 'wait' and action.amount > tracker.remaining_seconds: + raise BudgetExceeded('time_budget') + observation = session.step(action, timeout=tracker.remaining_seconds).observation + tracker.check() + return observation + + +def _account( + tracker: BudgetTracker, + data: dict[str, Any] | None, + prices: TokenPrices | None, + provider: Literal['anthropic', 'openai'], +) -> None: + try: + if not isinstance(data, dict): + raise ValueError('provider did not return token usage') + usage = _anthropic_usage(data) if provider == 'anthropic' else _openai_usage(data) + except (KeyError, TypeError, ValueError): + tracker.add_usage( + Usage(), Cost(provenance=f'{provider} usage unavailable; billing unknown') + ) + raise + tracker.add_usage( + usage, + prices.estimate(usage) + if prices + else Cost( + provenance='provider token usage; prices not supplied', + ), + ) + tracker.check() + + +def _validate_budget(budget: Budget, prices: TokenPrices | None) -> None: + if budget.max_cost_usd is not None and prices is None: + raise ValueError('A USD budget requires explicit TokenPrices with source provenance') + + +def _anthropic_usage(data: dict[str, Any]) -> Usage: + return Usage( + input_tokens=data['input_tokens'], + output_tokens=data['output_tokens'], + cache_read_tokens=data.get('cache_read_input_tokens') or 0, + cache_write_tokens=data.get('cache_creation_input_tokens') or 0, + ) + + +def _openai_usage(data: dict[str, Any]) -> Usage: + details = data.get('input_tokens_details') or {} + cached = details.get('cached_tokens') or 0 + written = details.get('cache_write_tokens') or 0 + return Usage( + input_tokens=data['input_tokens'] - cached - written, + output_tokens=data['output_tokens'], + cache_read_tokens=cached, + cache_write_tokens=written, + ) + + +# The SDK retries connection drops, 429s and 5xx with backoff; one blip shouldn't end an +# hours-long episode. A retried request may be billed twice; usage counts only answers. +_SDK_RETRIES = 4 + + +def _api_error(exc: Exception) -> str: + """Describe a provider API error by class, status and the provider's own validation message. + + Only the structured error body is used, never the exception text, which may echo request + content. + """ + status = getattr(exc, 'status_code', None) + body = getattr(exc, 'body', None) + error = body.get('error', body) if isinstance(body, dict) else None + detail = error.get('message') if isinstance(error, dict) else None + status_text = f' {status}' if status else '' + detail_text = f': {str(detail)[:300]}' if detail else '' + return f'{type(exc).__name__}{status_text}{detail_text}' + + +class AnthropicComputerAgent: + """Drive Session with Anthropic's computer toolset; import the SDK only on run. + + Args: + model: Provider model supporting computer_toolset_20260801. + prices: Explicit model-specific rates, required for a USD budget. + effort: Adaptive-thinking effort; None sends no thinking or effort settings. + """ + + def __init__( + self, + model: str, + prices: TokenPrices | None = None, + effort: Literal['low', 'medium', 'high'] | None = 'medium', + ) -> None: + self.model = model + self.prices = prices + self.effort = effort + + def run( + self, session: Session, initial: Observation, *, task: str, budget: Budget + ) -> AgentResult: + """Run the task and return its trajectory without grading. + + Raises: + ImportError: If the optional anthropic extra is absent. + ValueError: If required prices or the initial screenshot are missing. + """ + _validate_budget(budget, self.prices) + screenshot = _image(initial.screenshot) + import anthropic + + tracker = BudgetTracker(budget) + messages: list[dict[str, Any]] = [ + { + 'role': 'user', + 'content': [ + {'type': 'text', 'text': task}, + { + 'type': 'image', + 'source': {'type': 'base64', 'media_type': 'image/png', 'data': screenshot}, + }, + ], + } + ] + client = anthropic.Anthropic(max_retries=_SDK_RETRIES) + try: + while True: + if session.done: + return tracker.result(session, 'session_done') + tracker.before_turn() + # Cache the growing prefix: without it every turn re-bills all earlier + # screenshots. + thinking = ( + { + 'thinking': {'type': 'adaptive', 'display': 'summarized'}, + 'output_config': {'effort': self.effort}, + } + if self.effort + else {} + ) + response = client.messages.create( + model=self.model, + max_tokens=16000, + cache_control={'type': 'ephemeral'}, + **thinking, + system=_SYSTEM, + tools=[ + { + 'type': 'computer_toolset_20260801', + 'configs': {'zoom': {'enabled': False}}, + } + ], + messages=messages, + timeout=tracker.remaining_seconds, + ).model_dump(exclude_none=True) + _account(tracker, response.get('usage'), self.prices, 'anthropic') + content = response['content'] + messages.append({'role': 'assistant', 'content': content}) + text = ' '.join(part['text'] for part in content if part['type'] == 'text') + if response.get('stop_reason') == 'refusal': + return tracker.result(session, 'refusal', text) + calls = [part for part in content if part['type'] == 'tool_use'] + if not calls: + if response.get('stop_reason') in {'max_tokens', 'pause_turn'}: + messages.append( + {'role': 'user', 'content': 'Continue from the output limit.'} + ) + continue + if response.get('stop_reason') not in {'end_turn', 'stop_sequence'}: + return tracker.result(session, 'error', error='Response did not complete') + return tracker.result(session, 'completed', text) + results = [] + observation: Observation | None = None + failed = False + for call in calls: + result: dict[str, Any] = { + 'type': 'tool_result', + 'tool_use_id': call['id'], + 'toolset_name': call.get('toolset_name') or 'computer', + 'content': 'OK', + } + try: + if failed: + raise ValueError('Not executed: an earlier action in this batch failed') + for action in anthropic_actions(call['name'], call.get('input') or {}): + observation = _step(session, action, tracker) + except (ValueError, ActionError) as exc: + failed = True + result.update(is_error=True, content=f'Action rejected: {exc}') + results.append(result) + if session.done: + return tracker.result(session, 'session_done') + if observation is None or not observation.screenshot: + observation = _step(session, Action(action_type='screenshot'), tracker) + image = { + 'type': 'image', + 'source': { + 'type': 'base64', + 'media_type': 'image/png', + 'data': _image(observation.screenshot), + }, + } + # The API accepts only text in an error result, so the screenshot goes on the + # last successful result, or after the results when every action failed. + ok = [result for result in results if not result.get('is_error')] + if ok: + ok[-1]['content'] = [{'type': 'text', 'text': ok[-1]['content']}, image] + messages.append({'role': 'user', 'content': results}) + else: + messages.append({'role': 'user', 'content': [*results, image]}) + except BudgetExceeded as exc: + return tracker.result(session, exc.reason) + except DeepGymError as exc: + reason = 'time_budget' if tracker.remaining_seconds <= 0 else 'error' + return tracker.result( + session, reason, error=f'Session step failed: {type(exc).__name__}' + ) + except anthropic.APIError as exc: + tracker.add_usage(Usage(), Cost(provenance='Anthropic failed request; billing unknown')) + reason = 'time_budget' if tracker.remaining_seconds <= 0 else 'error' + return tracker.result(session, reason, error=f'Anthropic {_api_error(exc)}') + except (ValueError, KeyError, TypeError) as exc: + return tracker.result(session, 'error', error=f'Invalid Anthropic response: {exc}') + finally: + client.close() + + +class OpenAIComputerAgent: + """Drive Session with the OpenAI Responses computer tool, importing the SDK lazily. + + Args: + model: Provider model supporting the Responses computer tool. + prices: Explicit model-specific rates, required for a USD budget. + """ + + def __init__(self, model: str, prices: TokenPrices | None = None) -> None: + self.model = model + self.prices = prices + + def run( + self, session: Session, initial: Observation, *, task: str, budget: Budget + ) -> AgentResult: + """Run the task, stopping before any call with pending safety checks. + + Raises: + ImportError: If the optional openai extra is absent. + ValueError: If required prices or the initial screenshot are missing. + """ + _validate_budget(budget, self.prices) + screenshot = _image(initial.screenshot) + import openai + + tracker = BudgetTracker(budget) + items: list[dict[str, Any]] = [ + { + 'role': 'user', + 'content': [ + {'type': 'input_text', 'text': task}, + {'type': 'input_image', 'image_url': f'data:image/png;base64,{screenshot}'}, + ], + } + ] + previous: str | None = None + client = openai.OpenAI(max_retries=_SDK_RETRIES) + try: + while True: + if session.done: + return tracker.result(session, 'session_done') + tracker.before_turn() + response = client.responses.create( + model=self.model, + instructions=_SYSTEM, + tools=[{'type': 'computer'}], + input=items, + previous_response_id=previous, + max_output_tokens=4096, + timeout=tracker.remaining_seconds, + ).model_dump(exclude_none=True) + _account(tracker, response.get('usage'), self.prices, 'openai') + previous = response['id'] + output = response['output'] + parts = [ + part + for item in output + if item['type'] == 'message' + for part in item.get('content', []) + ] + refusal = ' '.join(part['refusal'] for part in parts if part['type'] == 'refusal') + if refusal: + return tracker.result(session, 'refusal', refusal) + text = ' '.join(part['text'] for part in parts if part['type'] == 'output_text') + calls = [item for item in output if item['type'] == 'computer_call'] + if any(call.get('pending_safety_checks') for call in calls): + return tracker.result( + session, + 'safety_check', + error=('Provider safety checks require approval; no actions executed'), + ) + if not calls: + if response.get('status') == 'incomplete': + items = [{'role': 'user', 'content': 'Continue from the output limit.'}] + continue + if response.get('status') != 'completed': + return tracker.result(session, 'error', error='Response did not complete') + return tracker.result(session, 'completed', text) + if response.get('status') != 'completed': + return tracker.result( + session, 'error', error='Computer response did not complete' + ) + items = [] + notes = [] + for call in calls: + actions = call.get('actions') or ( + [call['action']] if call.get('action') else [] + ) + if not actions: + raise ValueError('computer_call contains no actions') + observation: Observation | None = None + try: + for data in actions: + for action in openai_actions(data): + observation = _step(session, action, tracker) + except (ValueError, ActionError) as exc: + notes.append( + f'Action rejected: {exc}. Remaining actions in this call skipped.' + ) + if session.done: + return tracker.result(session, 'session_done') + if observation is None or not observation.screenshot: + observation = _step(session, Action(action_type='screenshot'), tracker) + screenshot = _image(observation.screenshot) + items.append( + { + 'type': 'computer_call_output', + 'call_id': call['call_id'], + 'output': { + 'type': 'computer_screenshot', + 'image_url': f'data:image/png;base64,{screenshot}', + }, + } + ) + if notes: + items.append({'role': 'user', 'content': ' '.join(notes)}) + except BudgetExceeded as exc: + return tracker.result(session, exc.reason) + except DeepGymError as exc: + reason = 'time_budget' if tracker.remaining_seconds <= 0 else 'error' + return tracker.result( + session, reason, error=f'Session step failed: {type(exc).__name__}' + ) + except openai.APIError as exc: + tracker.add_usage(Usage(), Cost(provenance='OpenAI failed request; billing unknown')) + reason = 'time_budget' if tracker.remaining_seconds <= 0 else 'error' + return tracker.result(session, reason, error=f'OpenAI {_api_error(exc)}') + except (ValueError, KeyError, TypeError) as exc: + return tracker.result(session, 'error', error=f'Invalid OpenAI response: {exc}') + finally: + client.close() diff --git a/deepgym/src/deepgym/agents/versions.py b/deepgym/src/deepgym/agents/versions.py new file mode 100644 index 00000000..8d31f62c --- /dev/null +++ b/deepgym/src/deepgym/agents/versions.py @@ -0,0 +1,48 @@ +"""Pin supported CLI releases in one place for adapters and image builders.""" + +import shlex +from collections.abc import Mapping +from types import MappingProxyType +from typing import Literal + +from pydantic import BaseModel, ConfigDict + + +class CliVersion(BaseModel): + """Describe one tested release; the supported range is this exact version.""" + + model_config = ConfigDict(frozen=True) + command: str + package: str + version: str + + +SUPPORTED_CLIS: Mapping[Literal['anthropic', 'openai'], CliVersion] = MappingProxyType( + { + 'anthropic': CliVersion( + command='claude', package='@anthropic-ai/claude-code', version='2.1.302' + ), + 'openai': CliVersion(command='codex', package='@openai/codex', version='0.162.1'), + } +) + + +def cli_install_command() -> str: + """Return the pinned npm installation command for sandbox image builders. + + It installs under /usr/local, so the binaries land on the standard PATH the sandbox + agent gets even when the image's Node comes from nvm or another custom prefix. + + Returns: + A shell command installing both supported CLI releases; nothing is executed. + """ + return shlex.join( + [ + 'npm', + 'install', + '-g', + '--prefix', + '/usr/local', + *(f'{release.package}@{release.version}' for release in SUPPORTED_CLIS.values()), + ] + ) diff --git a/deepgym/src/deepgym/api/schemas.py b/deepgym/src/deepgym/api/schemas.py index e49f9f87..e7524cab 100644 --- a/deepgym/src/deepgym/api/schemas.py +++ b/deepgym/src/deepgym/api/schemas.py @@ -212,9 +212,13 @@ class CreateSessionRequest(BaseModel): def reject_runtime( cls, v: str | MultiTurnEnvironment | ComputerUseEnvironment ) -> str | MultiTurnEnvironment | ComputerUseEnvironment: - """Refuse a caller-chosen image or snapshot: root in the sandbox runs its binaries.""" + """Refuse inline runtime overrides and root commands; registry specs stay trusted.""" if not isinstance(v, str) and (v.image is not None or v.snapshot is not None): raise ValueError('image and snapshot cannot be set over the API') + if not isinstance(v, str) and v.root_service_start: + raise ValueError('root_service_start cannot be set over the API') + if not isinstance(v, str) and v.cli_provider_egress: + raise ValueError('cli_provider_egress cannot be set over the API') return v diff --git a/deepgym/src/deepgym/cli_runtime.py b/deepgym/src/deepgym/cli_runtime.py new file mode 100644 index 00000000..5d9e35ff --- /dev/null +++ b/deepgym/src/deepgym/cli_runtime.py @@ -0,0 +1,255 @@ +"""Supervise opt-in CLI agents inside an isolated sandbox network namespace. + +Phase 1 exposes the real provider API key to the model and its subprocesses. The +CONNECT proxy limits destination host and port, not HTTPS routes, account spending, +or request contents. Provider-reported usage is untrusted and delayed; the later +key-injecting relay can replace this transport without changing Session.run_cli. +""" + +from __future__ import annotations + +import base64 +import codecs +import json +import shlex +import time +from collections.abc import Callable +from pathlib import Path +from typing import TYPE_CHECKING, Literal +from uuid import uuid4 + +from pydantic import BaseModel, ConfigDict, Field, JsonValue, model_validator + +from deepgym.exceptions import SandboxError + +if TYPE_CHECKING: + from deepgym.session import SandboxBackend + +EventObserver = Callable[[dict[str, JsonValue]], str | None] +_WORKER = Path(__file__).with_name('_cli_worker.py') + + +class CliRequest(BaseModel): + """Describe one explicitly authorized, unprivileged CLI process. + + Only the selected provider key is accepted in env. The opt-in acknowledges that + the model can read and reuse it. argv must contain no credentials. + """ + + model_config = ConfigDict(frozen=True) + argv: tuple[str, ...] = Field(min_length=1) + env: dict[str, str] = Field(repr=False) + provider: Literal['anthropic', 'openai'] + timeout: float = Field(gt=0, allow_inf_nan=False) + allow_model_credentials: bool = False + supported_version: str | None = None + """Exact supported release for agent adapters; None for direct runtime callers.""" + + @model_validator(mode='after') + def check_credentials(self) -> CliRequest: + """Accept exactly the chosen provider key; never arbitrary startup variables.""" + name = {'anthropic': 'ANTHROPIC_API_KEY', 'openai': 'CODEX_API_KEY'}[self.provider] + if set(self.env) != {name} or not self.env[name]: + raise ValueError(f'CLI env must contain only a non-empty {name}') + if any('\x00' in arg for arg in self.argv): + raise ValueError('CLI argv cannot contain NUL') + return self + + +class CliExecution(BaseModel): + """Return protected capture of a CLI process after all agent processes stop.""" + + model_config = ConfigDict(frozen=True) + events: tuple[dict[str, JsonValue], ...] + output: str + exit_code: int + stop_reason: str | None = None + cli_version: str | None = None + version_error: str | None = None + + +class CliExecutionError(SandboxError): + """Preserve an observed CLI version when later supervision fails.""" + + def __init__(self, message: str, cli_version: str | None) -> None: + super().__init__(message) + self.cli_version = cli_version + + +def execute_cli( + backend: SandboxBackend, request: CliRequest, on_event: EventObserver +) -> CliExecution: + """Run a sandbox CLI, poll protected output, and stop on observer or deadline limits. + + Args: + backend: An isolated SandboxBackend with its exclusive agent UID. + request: The process and explicit credential-exposure opt-in. + on_event: Return a stop reason when a parsed JSON stdout event exceeds a budget. + + Returns: + Captured output and nested events; no parsed command is executed again. + + Raises: + SandboxError: If opt-in, namespace support, supervision, or cleanup is unavailable. + """ + if not request.allow_model_credentials: + raise SandboxError( + 'CLI agents require allow_model_credentials=True: the key is model-readable' + ) + if not backend._offline or not backend._agent_namespace: + raise SandboxError('CLI egress needs a retained isolated namespace and provider egress') + deadline = time.monotonic() + request.timeout + directory = f'/opt/deepgym-staging/cli-{uuid4().hex}' + worker = f'{directory}/worker.py' + config = { + 'argv': request.argv, + 'env': {**backend._agent_env, **request.env}, + 'provider': request.provider, + 'timeout': request.timeout, + 'namespace': shlex.split(backend._offline), + 'prefix': [ + *shlex.split(backend._offline + backend._agent_namespace), + '/usr/bin/unshare', + '--pid', + '--fork', + '--mount-proc', + '--kill-child=KILL', + '--', + *shlex.split(backend._agent_drop), + ], + 'uid': backend._agent_uid, + 'cwd': backend.workdir, + 'supported_version': request.supported_version, + 'version_env': backend._agent_env, + } + events: list[dict[str, JsonValue]] = [] + chunks: list[str] = [] + pending = '' + decoder = codecs.getincrementaldecoder('utf-8')(errors='replace') + offset = 0 + reason = None + status = None + launch_attempted = False + cleanup_deadline = deadline + 20 + try: + _checked(backend, f'mkdir -m 700 {directory}') + backend._sandbox.upload(_WORKER.read_bytes(), worker) + config['timeout'] = deadline - time.monotonic() + if config['timeout'] <= 0: + return CliExecution(events=(), output='', exit_code=-1, stop_reason='time_budget') + backend._sandbox.upload(json.dumps(config).encode(), f'{directory}/request.json') + command = f'python3 -I {worker} supervise {directory}' + launch_attempted = True + _checked(backend, f'setsid {command} /dev/null 2>&1 &') + while status is None: + if reason is None and time.monotonic() >= deadline: + reason = 'time_budget' + _request_stop(backend, directory, reason) + packet = _poll(backend, directory, offset) + raw = base64.b64decode(packet['data']) + offset += len(raw) + text = decoder.decode(raw, final=packet['status'] is not None) + chunks.append(text) + pending += text + status = packet['status'] + lines = pending.split('\n') + pending = lines.pop() + if status is not None and pending: + lines.append(pending) + pending = '' + for line in lines: + try: + event = json.loads(line) + except ValueError: + continue + if isinstance(event, dict): + event = _redact_event(event, request) + events.append(event) + observed_reason = on_event(event) + if reason is None and observed_reason: + reason = observed_reason + _request_stop(backend, directory, reason) + if time.monotonic() > cleanup_deadline: + raise SandboxError('CLI supervisor did not finish before its cleanup deadline') + if status is None: + time.sleep(0.1) + if status.get('error'): + raise CliExecutionError( + f'CLI supervision failed: {status["error"]}', status.get('cli_version') + ) + return CliExecution( + events=tuple(events), + output=_redact_text(''.join(chunks), request), + exit_code=status['exit_code'], + stop_reason=reason or status['stop_reason'], + cli_version=status.get('cli_version'), + version_error=status.get('version_error'), + ) + finally: + failures = [] + try: + # Setup descendants also need teardown when staging exhausts the deadline. + backend.stop_agent() + except Exception as exc: + failures.append(f'Agent containment cleanup failed: {type(exc).__name__}') + if launch_attempted and status is None: + try: + _request_stop(backend, directory, 'cancelled') + for _ in range(100): + if _poll(backend, directory, offset)['status'] is not None: + break + time.sleep(0.1) + else: + raise SandboxError('CLI supervisor cleanup was not acknowledged') + except Exception as exc: + failures.append(f'CLI transport cleanup failed: {type(exc).__name__}') + try: + _checked(backend, f'rm -rf -- {directory}') + except Exception as exc: + failures.append(f'CLI credential staging cleanup failed: {type(exc).__name__}') + if failures: + raise CliExecutionError('; '.join(failures), (status or {}).get('cli_version')) + + +def _checked(backend: SandboxBackend, command: str) -> str: + result = backend.exec_trusted(command, 30) + if result.exit_code: + raise SandboxError(f'CLI supervisor operation failed: {result.output[-300:]}') + return result.output + + +def _redact_text(text: str, request: CliRequest) -> str: + for value in request.env.values(): + text = text.replace(value, '[REDACTED]') + return text + + +def _redact_event(event: JsonValue, request: CliRequest) -> JsonValue: + if isinstance(event, str): + return _redact_text(event, request) + if isinstance(event, list): + return [_redact_event(item, request) for item in event] + if isinstance(event, dict): + return { + _redact_text(key, request): _redact_event(value, request) + for key, value in event.items() + } + return event + + +def _request_stop(backend: SandboxBackend, directory: str, reason: str) -> None: + script = f'from pathlib import Path; Path({directory + "/stop"!r}).write_text({reason!r})' + _checked(backend, f'python3 -I -c {shlex.quote(script)}') + + +def _poll(backend: SandboxBackend, directory: str, offset: int) -> dict: + script = ( + 'import base64,json; from pathlib import Path; ' + f'p=Path({directory!r}); f=p/"output"; s=p/"status.json"; ' + 'status=json.loads(s.read_text()) if s.exists() else None; ' + f'h=f.open("rb") if f.exists() else None; ' + f'h.seek({offset}) if h else None; d=h.read() if h else b""; ' + 'print(json.dumps({"data":base64.b64encode(d).decode(),' + '"status":status}))' + ) + return json.loads(_checked(backend, f'python3 -I -c {shlex.quote(script)}')) diff --git a/deepgym/src/deepgym/core.py b/deepgym/src/deepgym/core.py index 92a3796d..07c5b24a 100644 --- a/deepgym/src/deepgym/core.py +++ b/deepgym/src/deepgym/core.py @@ -8,7 +8,7 @@ from collections.abc import Callable, Sequence from concurrent.futures import ThreadPoolExecutor, as_completed from datetime import datetime, timezone -from typing import Literal +from typing import TYPE_CHECKING, Literal, overload from deepgym import __version__ from deepgym.benchmark_envs import _infer_environment_name, build_run_requests, route_single_run @@ -34,6 +34,9 @@ logger = logging.getLogger(__name__) +if TYPE_CHECKING: + from deepgym.agents.base import Agent, AgentResult, Budget + class DeepGym: """Provide the main client for running RL environments in sandboxed execution. @@ -206,18 +209,47 @@ def session(self, env: MultiTurnEnvironment) -> Session: desktop = DaytonaDesktop(box, env.timeout_per_step, env.setup_timeout) return Session(env, SandboxBackend(box, desktop=desktop if env.desktop else None)) + @overload def run_episode( self, env: MultiTurnEnvironment, agent: Callable[[Observation], Action] - ) -> tuple[Trajectory, RunResult]: + ) -> tuple[Trajectory, RunResult]: ... + + @overload + def run_episode( + self, env: MultiTurnEnvironment, agent: Agent, *, budget: Budget | None = None + ) -> tuple[AgentResult, RunResult]: ... + + def run_episode( + self, + env: MultiTurnEnvironment, + agent: Callable[[Observation], Action] | Agent, + *, + budget: Budget | None = None, + ) -> tuple[Trajectory | AgentResult, RunResult]: """Run one full episode of *env* driven by *agent* and score it. Args: env: The multi-turn or computer-use environment. - agent: Callable mapping an observation to the next action. + agent: An agent harness, or a callable mapping observations to actions. + budget: Harness limits; omitted uses Budget defaults. Returns: - Tuple of (trajectory, final_result). + Tuple of (AgentResult, final_result) for a harness, or + (trajectory, final_result) for a legacy callable. The grade describes + final state independently of the harness termination reason. + + Raises: + ValueError: If a budget is supplied to a legacy callable. """ + from deepgym.agents.base import Agent, Budget + + if isinstance(agent, Agent): + with self.session(env) as session: + initial = session.start() + result = agent.run(session, initial, task=env.task, budget=budget or Budget()) + return result, session.score() + if budget is not None: + raise ValueError('budget requires an Agent harness with a run method') return run_episode(self.session(env), agent) def trace( diff --git a/deepgym/src/deepgym/models.py b/deepgym/src/deepgym/models.py index 62b4d213..c4030fde 100644 --- a/deepgym/src/deepgym/models.py +++ b/deepgym/src/deepgym/models.py @@ -10,7 +10,7 @@ from pathlib import Path, PurePosixPath from typing import Any, Literal, get_args -from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator +from pydantic import BaseModel, ConfigDict, Field, JsonValue, field_validator, model_validator from deepgym.exceptions import InvalidScoreError @@ -411,7 +411,7 @@ class Observation(BaseModel): 'screenshot', 'wait', ] -ActionType = Literal[ShellActionType, GuiActionType, 'done'] +ActionType = Literal[ShellActionType, GuiActionType, 'done', 'cli'] GUI_ACTIONS: frozenset[GuiActionType] = frozenset(get_args(GuiActionType)) @@ -434,6 +434,9 @@ class Action(BaseModel): """Code, shell command, file contents, text to type, key chord ('ctrl+s'), or scroll direction (up/down/left/right).""" + events: tuple[dict[str, JsonValue], ...] = () + """Nested CLI stdout events, recorded once after execution; never replayed as actions.""" + x: int | None = Field(default=None, ge=0) """Pointer x for click/move/scroll, or drag start.""" @@ -507,6 +510,18 @@ class MultiTurnEnvironment(BaseModel): setup_script: str = '' """Bash run in the work directory after files are written (start servers, open apps).""" + root_service_start: str = '' + """Trusted image service startup, as root in the offline namespace before desktop/setup. + Uses setup_timeout. Only SandboxBackend supports it; never accept this from an agent. + The command must start the service itself and exit non-zero if readiness fails. + """ + + cli_provider_egress: bool = Field(default=False, strict=True) + """Opt in to sandbox-level egress for the trusted CLI provider proxy (Daytona). + Agent commands, root service startup and verification remain in the offline namespace. + Other trusted root processes outside that namespace also gain network access. + """ + setup_code: str = '' """Python run after setup_script. Its output is the first observation.""" @@ -555,9 +570,11 @@ class MultiTurnEnvironment(BaseModel): @model_validator(mode='after') def check_desktop_actions(self) -> MultiTurnEnvironment: - """Reject GUI actions on environments without a desktop.""" + """Reject GUI actions without a desktop and CLI egress without shell actions.""" if not self.desktop and self.allowed_actions & GUI_ACTIONS: raise ValueError('GUI actions need desktop=True') + if self.cli_provider_egress and not self.allowed_actions & {'bash', 'cli'}: + raise ValueError('cli_provider_egress requires bash or cli in allowed_actions') return self diff --git a/deepgym/src/deepgym/sandboxes/daytona/desktop.py b/deepgym/src/deepgym/sandboxes/daytona/desktop.py index b6841221..db3d5023 100644 --- a/deepgym/src/deepgym/sandboxes/daytona/desktop.py +++ b/deepgym/src/deepgym/sandboxes/daytona/desktop.py @@ -280,7 +280,7 @@ def start( ) # fmt: skip _confirm('check', as_root(check, 60)) - def act(self, action: Action) -> None: + def act(self, action: Action, *, timeout: float | None = None) -> None: """Apply a GUI action through Daytona computer use. Raises: @@ -288,7 +288,7 @@ def act(self, action: Action) -> None: SandboxError: If the Daytona computer-use call fails for any other reason. """ try: - self._dispatch(action) + self._dispatch(action, timeout=timeout) except Exception as exc: # ponytail: Daytona reports bad input as a generic daemon 500, so match its wording. message = str(exc).lower() @@ -296,10 +296,12 @@ def act(self, action: Action) -> None: raise ActionError(f'{action.action_type} rejected: {exc}') from exc raise SandboxError(f'{action.action_type} failed: {exc}') from exc - def _dispatch(self, action: Action) -> None: + def _dispatch(self, action: Action, *, timeout: float | None = None) -> None: mouse = self._box.computer_use.mouse keyboard = self._box.computer_use.keyboard - deadline = self._action_timeout + deadline = ( + min(timeout, self._action_timeout) if timeout is not None else self._action_timeout + ) match action.action_type: case 'click': mouse.click(action.x, action.y, request_timeout=deadline) @@ -320,7 +322,7 @@ def _dispatch(self, action: Action) -> None: case 'key': keyboard.hotkey(action.content, request_timeout=deadline) - def screenshot(self) -> str: + def screenshot(self, *, timeout: float | None = None) -> str: """Return a full-resolution PNG so pixel coordinates match click targets. Raises: @@ -328,9 +330,12 @@ def screenshot(self) -> str: """ from daytona import ScreenshotOptions + deadline = ( + min(timeout, self._action_timeout) if timeout is not None else self._action_timeout + ) try: shot = self._box.computer_use.screenshot.take_compressed( - ScreenshotOptions(fmt='png', scale=1.0), request_timeout=self._action_timeout + ScreenshotOptions(fmt='png', scale=1.0), request_timeout=deadline ) except Exception as exc: raise SandboxError(f'Failed to take screenshot: {exc}') from exc diff --git a/deepgym/src/deepgym/sandboxes/daytona/sandbox.py b/deepgym/src/deepgym/sandboxes/daytona/sandbox.py index 75708f70..84ad998e 100644 --- a/deepgym/src/deepgym/sandboxes/daytona/sandbox.py +++ b/deepgym/src/deepgym/sandboxes/daytona/sandbox.py @@ -9,6 +9,7 @@ from __future__ import annotations import logging +import math import posixpath import re import shlex @@ -77,15 +78,16 @@ def create_params( def _session_params( env: MultiTurnEnvironment, ) -> CreateSandboxFromSnapshotParams | CreateSandboxFromImageParams: - """Return params for an ephemeral, network-blocked session sandbox: ``env.image`` + """Return params for an ephemeral session sandbox, network-blocked by default: ``env.image`` unless a snapshot is set, else ``env.snapshot`` or Daytona's default snapshot (the - one with a desktop). + one with a desktop). Only explicit CLI provider egress unblocks the trusted root + proxy; SandboxBackend still isolates agent, service and verifier networking. Raises: SandboxError: If the daytona SDK is missing. """ sdk = _sdk() - options = {'ephemeral': True, 'network_block_all': True} + options = {'ephemeral': True, 'network_block_all': not env.cli_provider_egress} if env.image and not env.snapshot: return sdk.CreateSandboxFromImageParams(image=sdk.Image.base(env.image), **options) return sdk.CreateSandboxFromSnapshotParams(snapshot=env.snapshot, **options) @@ -172,7 +174,8 @@ def __init__(self, daytona: Daytona, env: Environment | MultiTurnEnvironment) -> A single run gets :func:`create_params`. A live session (a multi-turn *env*) gets its image or Daytona's default snapshot, which has the desktop, with the - network blocked and ``setup_timeout`` to boot; deleting it waits until Daytona + network blocked unless CLI provider egress is requested, and ``setup_timeout`` + to boot; deleting it waits until Daytona confirms it is gone, because the HTTP API frees a session slot on close. Raises: @@ -240,7 +243,7 @@ def exec( *, cwd: str | None = None, env: Mapping[str, str] | None = None, - timeout: int | None = None, + timeout: float | None = None, ) -> ExecResult: """Run *command* with sh as root. @@ -260,7 +263,7 @@ def exec( f'{self._sudo}/bin/sh -c {shlex.quote(command)}', cwd=cwd, env=dict(env) if env else None, - timeout=timeout + _WAIT_GRACE_S if timeout else None, + timeout=math.ceil(timeout + _WAIT_GRACE_S) if timeout else None, ) except Exception as exc: if is_timeout(exc): diff --git a/deepgym/src/deepgym/session.py b/deepgym/src/deepgym/session.py index b84b199c..37156984 100644 --- a/deepgym/src/deepgym/session.py +++ b/deepgym/src/deepgym/session.py @@ -14,6 +14,7 @@ import errno import json import logging +import math import os import posixpath import shlex @@ -29,6 +30,9 @@ from typing import TYPE_CHECKING, NamedTuple, Protocol from uuid import uuid4 +from pydantic import JsonValue + +from deepgym.cli_runtime import CliExecution, CliRequest, EventObserver, execute_cli from deepgym.exceptions import ( ActionError, DeepGymError, @@ -475,6 +479,10 @@ def __init__( if sealed.exit_code != 0 or sealed.output.strip() != sealed_path.SEALED: raise SandboxError(f'Refusing the sandbox: {sealed.output[-500:]}') self._as_agent = _drop_to(setpriv, agent_uid, agent_gid) + self._agent_drop = self._as_agent + self._agent_namespace = '' + self._nsenter = nsenter + self._agent_uid = int(agent_uid) self._as_verifier = _drop_to( setpriv, verifier_uid, verifier_gid, allow_sudo=solver == '1' ) @@ -506,6 +514,8 @@ def __init__( **locale, } self._verifier_base = {'DEEPGYM_AGENT_HOME': _AGENT_HOME, **locale} + if self._offline: + self._make_agent_namespace() except BaseException: self.close() raise @@ -515,6 +525,36 @@ def workdir(self) -> str: """The work directory, as it really is in the sandbox (links resolved).""" return self._workdir + def _make_agent_namespace(self) -> None: + directory = f'{_STAGING_DIR}/agent-processes' + self._agent_scope = directory + control = f'{directory}/control.py' + _checked('Agent containment directory', self.exec_trusted(f'mkdir -m 700 {directory}', 30)) + source = Path(__file__).with_name('_pid_namespace.py').read_bytes() + self._sandbox.upload(source, control) + _checked( + 'Agent PID namespace', + self.exec_trusted(f'python3 -I {control} create {directory} {self._agent_uid}', 30), + ) + self._agent_namespace = ( + f'{shlex.quote(self._nsenter)} --pid={directory}/pid --mount={directory}/mnt -- ' + ) + self._as_agent = self._agent_namespace + self._agent_drop + + def stop_agent(self) -> None: + """Destroy all setup and CLI descendants before grading; refuse uncertain cleanup. + + Root services and the verifier never enter this PID namespace. + + Raises: + SandboxError: If kernel teardown or the no-survivor check fails. + """ + directory = self._agent_scope + _checked( + 'Agent PID namespace teardown', + self.exec_trusted(f'python3 -I {directory}/control.py stop {directory}', 30), + ) + def _make_offline_namespace(self) -> None: """Make a loopback-only network namespace and bind-mount it at ``_NETNS``. @@ -577,6 +617,21 @@ def exec_root( timeout=timeout, ) + def run_cli(self, request: CliRequest, on_event: EventObserver) -> CliExecution: + """Run the CLI with fixed-provider egress and kill every agent process on exit. + + Args: + request: CLI command and explicit model-readable credential opt-in. + on_event: Observer returning a stop reason to terminate the CLI. + + Returns: + Protected captured events and process termination status. + + Raises: + SandboxError: If supervision or isolated provider transport is unavailable. + """ + return execute_cli(self, request, on_event) + def exec_trusted(self, command: str, timeout: int) -> ExecResult: """Run a harness *command* as root with an empty environment and only trusted folders on PATH (see :func:`trusted`), in the work directory. @@ -745,21 +800,26 @@ def start_desktop(self) -> None: ) self._agent_env['DISPLAY'] = _DISPLAY - def act(self, action: Action) -> None: + def act(self, action: Action, *, timeout: float | None = None) -> None: """Apply a GUI action. Raises: DeepGymError: If this provider has no desktop. """ - self._need_desktop('GUI actions').act(action) + if timeout is None: + self._need_desktop('GUI actions').act(action) + else: + self._need_desktop('GUI actions').act(action, timeout=timeout) - def screenshot(self) -> str: + def screenshot(self, *, timeout: float | None = None) -> str: """Return the screen as a base64 PNG. Raises: DeepGymError: If this provider has no desktop. """ - return self._need_desktop('Screenshots').screenshot() + if timeout is None: + return self._need_desktop('Screenshots').screenshot() + return self._need_desktop('Screenshots').screenshot(timeout=timeout) def booted_image(self) -> SandboxImage: """Return the image the sandbox booted and its digest, as its provider reports them.""" @@ -884,6 +944,7 @@ def __init__(self, env: MultiTurnEnvironment, backend: SessionBackend) -> None: self._obs: Observation | None = None self._step = 0 self._done = False + self._cli_finalized = False self._step_verdict: dict[str, object] | None = None # The episode's result, once a failed step fixed it (see step()). self._failed_result: RunResult | None = None @@ -939,6 +1000,18 @@ def start(self) -> Observation: if self._obs is not None: raise DeepGymError('Session already started; open a new session per episode') env = self._env + if env.root_service_start: + if not isinstance(self._backend, SandboxBackend): + raise SandboxError('root_service_start requires an isolated SandboxBackend') + _checked( + 'Root service startup', + self._backend.exec_root( + env.root_service_start, + env.setup_timeout, + offline=True, + env={'PATH': TRUSTED_PATH}, + ), + ) for path, content in env.files.items(): self._backend.write_file(path, content.encode('utf-8')) if env.desktop: @@ -955,7 +1028,7 @@ def start(self) -> Observation: self._obs = self._observe(output, done=False) return self._obs - def step(self, action: Action) -> StepResult: + def step(self, action: Action, *, timeout: float | None = None) -> StepResult: """Apply one action and return the new observation, step reward, and done flag. The new observation (with its screenshot) is taken before the step verifier runs, @@ -967,6 +1040,10 @@ def step(self, action: Action) -> StepResult: verifier that times out after reading the agent's output gives the step a judged 0 (``solution_timeout``) instead, and the episode goes on. + ``timeout`` bounds agent execution and the following screenshot by one remaining + wall-time deadline. Desktop overrides require SandboxBackend. Verifier time is + separate; provider transport can take its own cleanup grace after a deadline. + Raises: DeepGymError: If the session is not running. ActionError: If the action is not allowed here, the desktop rejects it, or the @@ -977,6 +1054,9 @@ def step(self, action: Action) -> StepResult: SandboxError: If the sandbox fails to apply the action, take the screenshot or run the step verifier. """ + if timeout is not None and (not math.isfinite(timeout) or timeout <= 0): + raise ValueError('Step timeout must be finite and positive') + deadline = time.monotonic() + timeout if timeout is not None else None with self._lock: if self._obs is None or self._done: raise DeepGymError('Session is not running; call start() or open a new session') @@ -985,11 +1065,19 @@ def step(self, action: Action) -> StepResult: f'Action {action.action_type!r} not allowed here. ' f'Allowed: {sorted(self._env.allowed_actions)}' ) + if action.action_type == 'cli': + raise ActionError('CLI actions can only be executed through Session.run_cli') + if ( + timeout is not None + and self._env.desktop + and not isinstance(self._backend, SandboxBackend) + ): + raise ActionError('Desktop deadline overrides require SandboxBackend') self._step += 1 self._step_verdict = None try: - output, exit_code = self._apply(action) + output, exit_code = self._apply(action, timeout=_remaining(deadline)) except ActionError: raise except BaseException as exc: @@ -1002,7 +1090,9 @@ def step(self, action: Action) -> StepResult: ) observation: Observation | None = None try: - observation = self._observe(output, done=done, exit_code=exit_code) + observation = self._observe( + output, done=done, exit_code=exit_code, timeout=_remaining(deadline) + ) reward = self._step_reward(output) if self._env.step_verifier_code else 0.0 except BaseException as exc: # Here an ActionError means the agent's own files blocked the step's output. @@ -1011,6 +1101,102 @@ def step(self, action: Action) -> StepResult: self._record(action, observation, reward=reward, done=done) return StepResult(observation, reward, done) + def run_cli(self, request: CliRequest, on_event: EventObserver) -> CliExecution: + """Record one real CLI action with nested events, without replaying its commands. + + CLI agents require shell access and SandboxBackend. Completion is explicit: + stdout DONE lines do not stop supervision. All dgagent processes, including its + desktop, are killed before returning, so the final observation is text-only. + + Args: + request: CLI process and opt-in for model-readable provider credentials. + on_event: Return a stop reason when parsed events exceed the caller's budget. + + Returns: + Captured stdout, events, exit code, and any budget stop reason. + + Raises: + DeepGymError: If the session is not running. + ActionError: If the environment disallows shell execution or CLI provider egress. + SandboxError: If the backend cannot isolate or supervise the CLI safely. + """ + with self._lock: + if self._obs is None or self._done: + raise DeepGymError('Session is not running; call start() or open a new session') + if not {'bash', 'cli'} & self._env.allowed_actions: + raise ActionError('CLI agents require bash or cli in allowed_actions') + if not self._env.cli_provider_egress: + raise ActionError( + 'CLI provider egress is disabled; set cli_provider_egress=True on the ' + 'environment before creating the session' + ) + if not isinstance(self._backend, SandboxBackend): + raise SandboxError('CLI agents require an isolated SandboxBackend') + action = Action(action_type='cli', content=shlex.join(request.argv)) + self._step += 1 + self._step_verdict = None + started = time.monotonic() + events: list[dict[str, JsonValue]] = [] + + def observe(event: dict[str, JsonValue]) -> str | None: + events.append(event) + return on_event(event) + + try: + result = self._backend.run_cli(request, observe) + self._cli_finalized = True + action = action.model_copy(update={'events': result.events}) + observation = Observation( + content=result.output, + step=self._step, + done=True, + metadata={ + 'exit_code': result.exit_code, + 'stop_reason': result.stop_reason, + 'cli_version': result.cli_version, + 'version_source': 'sandbox CLI --version', + 'version_error': result.version_error, + }, + ) + reward = self._step_reward(result.output) if self._env.step_verifier_code else 0.0 + self._record(action, observation, reward=reward, done=True) + return result + except BaseException as exc: + action = action.model_copy(update={'events': tuple(events)}) + self._end(action, None, exc) + raise + finally: + self._acting_s += time.monotonic() - started + + def finalize_cli(self) -> None: + """Stop the CLI episode's agent population even when no CLI was launched. + + The shared backend teardown includes setup descendants. Successful runtime + execution already performed it; no extra action or reward is recorded here. + GUI agents never call this method, so their desktops survive normal scoring. + Already-ended episodes retain their terminal observation, including unknown state. + + Raises: + SandboxError: If containment is unavailable or cannot be stopped. + The stored failure has no verdict and prevents subsequent verification. + """ + with self._lock: + if self._cli_finalized: + return + was_done = self._done + self._done = True + try: + if not isinstance(self._backend, SandboxBackend): + raise SandboxError('CLI finalization requires an isolated SandboxBackend') + self._backend.stop_agent() + except BaseException as exc: + self._failed_result = error_result(exc, self.sandbox_id) + raise + self._cli_finalized = True + if not was_done and self._obs is not None: + self._obs = self._obs.model_copy(update={'done': True}) + self._trajectory.final_observation = self._obs + def score(self) -> RunResult: """Run the final verifier against the current state. @@ -1090,18 +1276,23 @@ def __exit__(self, *args: object) -> None: 'Could not delete sandbox %s; it may still exist', self.sandbox_id, exc_info=True ) - def _apply(self, action: Action) -> tuple[str, int | None]: + def _apply(self, action: Action, *, timeout: float | None = None) -> tuple[str, int | None]: """Execute *action*; return its text output and exit code (None if not a command). Its time, however it ends, counts as the agent's (see :meth:`_ran_out_the_clock`). """ - timeout = self._env.timeout_per_step + bounded = timeout is not None + timeout = ( + min(timeout, self._env.timeout_per_step) + if timeout is not None + else self._env.timeout_per_step + ) started = time.monotonic() try: try: match action.action_type: case 'code': - exit_code, output = self._run_python('_step.py', action.content) + exit_code, output = self._run_python('_step.py', action.content, timeout) return output, exit_code case 'bash': exit_code, output = self._backend.exec(action.content, timeout) @@ -1116,12 +1307,15 @@ def _apply(self, action: Action) -> tuple[str, int | None]: return '', None except TimeoutError: return f'[timed out after {timeout}s]', None - self._backend.act(action) + if bounded and isinstance(self._backend, SandboxBackend): + self._backend.act(action, timeout=timeout) + else: + self._backend.act(action) return '', None finally: self._acting_s += time.monotonic() - started - def _run_python(self, filename: str, code: str, timeout: int | None = None) -> ExecResult: + def _run_python(self, filename: str, code: str, timeout: float | None = None) -> ExecResult: self._backend.write_file(filename, code.encode('utf-8')) return self._backend.exec( f'{self._backend.python} {filename}', timeout or self._env.timeout_per_step @@ -1163,12 +1357,20 @@ def _record( if done: self._trajectory.final_observation = observation - def _observe(self, output: str, *, done: bool, exit_code: int | None = None) -> Observation: + def _observe( + self, output: str, *, done: bool, exit_code: int | None = None, timeout: float | None = None + ) -> Observation: + screenshot = None + if self._env.desktop: + if timeout is not None and isinstance(self._backend, SandboxBackend): + screenshot = self._backend.screenshot(timeout=timeout) + else: + screenshot = self._backend.screenshot() return Observation( content=output, step=self._step, done=done, - screenshot=self._backend.screenshot() if self._env.desktop else None, + screenshot=screenshot, metadata=None if exit_code is None else {'exit_code': exit_code}, ) @@ -1297,6 +1499,15 @@ def run_episode( return session.trajectory, session.score() +def _remaining(deadline: float | None) -> float | None: + if deadline is None: + return None + remaining = deadline - time.monotonic() + if remaining <= 0: + raise TimeoutError('Step exhausted the agent wall-time budget') + return remaining + + def ran_out_the_clock(exc: BaseException, acting_s: float, elapsed_s: float) -> bool: """Whether *exc* is the provider retiring the sandbox at the end of its lifetime after the agent's own actions took at least half of the *elapsed_s* seconds (*acting_s*): then diff --git a/deepgym/src/deepgym/traces/atif.py b/deepgym/src/deepgym/traces/atif.py index c8a4407c..cce610de 100644 --- a/deepgym/src/deepgym/traces/atif.py +++ b/deepgym/src/deepgym/traces/atif.py @@ -20,6 +20,8 @@ fields), in order, each with its own observation result: what the action produced, or the error if it produced nothing. Exit code, done flag, ``applied``, error, step reward and step verifier output go in that result's ``extra``. +* CLI launches remain one tool call; their raw nested events are preserved in the + observation result's ``extra.cli_events``, never translated into executable actions. * Assistant token ids and logprobs become ``metrics.completion_token_ids`` and ``metrics.logprobs``; token data on other roles has no ATIF field and is dropped. * Screenshots are PNG files under ``images//`` next to the document, referenced @@ -141,7 +143,7 @@ def _turn( 'tool_call_id': call_id, 'function_name': step.action.action_type, 'arguments': step.action.model_dump( - mode='json', exclude={'action_type'}, exclude_defaults=True + mode='json', exclude={'action_type', 'events'}, exclude_defaults=True ), }) # fmt: skip observed = ( @@ -156,6 +158,7 @@ def _turn( 'error': step.error, 'reward': step.reward, 'step_verifier': step.step_verifier, + 'cli_events': list(step.action.events) if step.action.action_type == 'cli' else None, }))) # fmt: skip action = _compact({ 'timestamp': steps[0].timestamp.isoformat(), diff --git a/deepgym/src/deepgym/traces/capture.py b/deepgym/src/deepgym/traces/capture.py index 72b909be..d2ed28f6 100644 --- a/deepgym/src/deepgym/traces/capture.py +++ b/deepgym/src/deepgym/traces/capture.py @@ -13,6 +13,7 @@ import json import logging import os +import shlex import threading import time from collections.abc import Callable, Sequence @@ -21,8 +22,16 @@ from typing import TextIO, TypeVar from uuid import uuid4 +from deepgym.cli_runtime import CliExecution, CliExecutionError, CliRequest, EventObserver from deepgym.exceptions import DeepGymError, SandboxError -from deepgym.models import Action, MultiTurnEnvironment, Observation, RunResult, is_verdict +from deepgym.models import ( + Action, + MultiTurnEnvironment, + Observation, + RunResult, + Trajectory, + is_verdict, +) from deepgym.session import Session, StepResult from deepgym.traces.schema import ( TRACE_SCHEMA_VERSION, @@ -93,6 +102,12 @@ def done(self) -> bool: """Whether the episode has ended.""" return self._session.done + @property + def trajectory(self) -> Trajectory: + """Return a copy of the session trajectory for an agent's result.""" + with self._lock: + return self._session.trajectory.model_copy(deep=True) + @property def trace(self) -> Trace: """A copy of everything recorded so far.""" @@ -127,7 +142,13 @@ def start(self) -> Observation: self._write({'event': 'reset', 'observation': self._initial}) return observation - def step(self, action: Action, messages: Sequence[TraceMessage] = ()) -> StepResult: + def step( + self, + action: Action, + messages: Sequence[TraceMessage] = (), + *, + timeout: float | None = None, + ) -> StepResult: """Apply *action* and record it with the model *messages* that produced it. A step that raises is recorded with its error (and its observation, when the @@ -137,6 +158,7 @@ def step(self, action: Action, messages: Sequence[TraceMessage] = ()) -> StepRes action: The agent's action. messages: Model messages new at this step, ending with the assistant reply that produced *action* (see :attr:`TraceStep.messages`). + timeout: Optional action deadline forwarded to the session. Returns: The session's step result. @@ -146,6 +168,41 @@ def step(self, action: Action, messages: Sequence[TraceMessage] = ()) -> StepRes started, ended or already scored: a verdict must match the state the trace ends in). Nothing is recorded then. """ + return self._capture(action, messages, lambda: self._session.step(action, timeout=timeout)) + + def run_cli(self, request: CliRequest, on_event: EventObserver) -> CliExecution: + """Run and capture one CLI launch, with its events nested under that action. + + Args: + request: The sandbox CLI and explicit credential-exposure opt-in. + on_event: The agent's budget observer; never replays parsed commands. + + Returns: + The supervised process result after its descendants have stopped. + + Raises: + DeepGymError: If the trace cannot accept actions or CLI execution fails. + """ + return self._capture( + Action(action_type='cli', content=shlex.join(request.argv)), + (), + lambda: self._session.run_cli(request, on_event), + ) + + def finalize_cli(self) -> None: + """Finalize CLI containment without inventing an action in the trace. + + Raises: + DeepGymError: If containment cleanup fails; the trace records the error. + """ + with self._lock: + self._check_open() + self._recorded(self._session.finalize_cli) + + def _capture( + self, action: Action, messages: Sequence[TraceMessage], call: Callable[[], _T] + ) -> _T: + """Record the one real Session action produced by *call*, even on failure.""" with self._lock: self._check_open() if self._results: @@ -154,7 +211,6 @@ def step(self, action: Action, messages: Sequence[TraceMessage] = ()) -> StepRes raise DeepGymError('The episode is not running; call start() or open a new trace') trajectory = self._session.trajectory applied_before = len(trajectory.steps) - scored = bool(self._env.step_verifier_code) record = { 'index': len(self._steps) + 1, 'action': action.model_copy(deep=True), @@ -163,12 +219,15 @@ def step(self, action: Action, messages: Sequence[TraceMessage] = ()) -> StepRes } start = time.perf_counter() try: - result = self._session.step(action) + result = call() except BaseException as exc: - # The session records a failed step, with no step reward, whenever its action - # may have taken effect (anything but a rejected action), and ends the episode. + if isinstance(exc, CliExecutionError): + self._cli_version(exc.cli_version) + # Only the session knows whether a failed operation changed the sandbox. applied = len(trajectory.steps) > applied_before observation = trajectory.final_observation if applied else None + if applied: + record['action'] = trajectory.steps[-1][1].model_copy(deep=True) self._add_step( TraceStep( **record, @@ -179,16 +238,36 @@ def step(self, action: Action, messages: Sequence[TraceMessage] = ()) -> StepRes ) ) raise + record['action'] = trajectory.steps[-1][1].model_copy(deep=True) + if isinstance(result, CliExecution) and result.cli_version is not None: + self._cli_version(result.cli_version) + observation = ( + result.observation + if isinstance(result, StepResult) + else trajectory.final_observation + ) + assert observation is not None self._add_step( TraceStep( **record, - **(self._step_score(result.reward) if scored else {}), - observation=result.observation.model_copy(deep=True), + **( + self._step_score(trajectory.step_rewards[-1]) + if self._env.step_verifier_code + else {} + ), + observation=observation.model_copy(deep=True), duration_ms=_elapsed_ms(start), ) ) return result + def _cli_version(self, version: str | None) -> None: + if version is not None: + self._provenance = self._provenance.model_copy( + update={'agent': self._provenance.agent.model_copy(update={'version': version})} + ) + self._write({'event': 'agent_version', 'version': version}) + def score(self) -> RunResult: """Run the final verifier and record its result. @@ -337,6 +416,8 @@ def load_trace(path: Path) -> Trace: fields['initial_observation'] = event['observation'] case 'step': steps.append(event['step']) + case 'agent_version': + fields['provenance']['agent']['version'] = event['version'] case 'result': results.append(event['result']) case 'end': diff --git a/deepgym/src/deepgym/traces/export.py b/deepgym/src/deepgym/traces/export.py index eee01e71..228e295f 100644 --- a/deepgym/src/deepgym/traces/export.py +++ b/deepgym/src/deepgym/traces/export.py @@ -15,7 +15,8 @@ screenshot, done flag, exit code, or the error if nothing), its step reward, and ``message_index``: where in ``messages`` the reply that produced it is, shared by the actions of one turn. A trace with a turn that has no model messages has no whole - conversation to train on and is skipped. + conversation to train on and is skipped. In RL Parquet, ``action.events`` is a + list of JSON strings, preserving arbitrary nested CLI event shapes. Every row carries ``has_verdict``, and its ``score``/``reward`` and ``passed`` are null without one, so a failed sandbox or verifier never reads as a scored 0. ``only_valid`` @@ -185,6 +186,23 @@ def _write_parquet(records: Sequence[Row], out: Path, rows: RowShape) -> None: import pyarrow.parquet as pq except ImportError as exc: raise DeepGymError("Parquet export needs pyarrow: pip install 'deepgym[traces]'") from exc + if rows == 'rl': + records = [ + { + **record, + 'steps': [ + { + **step, + 'action': { + **step['action'], + 'events': [json_text(event) for event in step['action']['events']], + }, + } + for step in record['steps'] + ], + } + for record in records + ] out.parent.mkdir(parents=True, exist_ok=True) pq.write_table(pa.Table.from_pylist(list(records), schema=_schema(rows)), out) @@ -224,6 +242,7 @@ def _schema(rows: RowShape) -> pa.Schema: ('to_y', pa.int64()), ('path', pa.string()), ('amount', pa.int64()), + ('events', pa.list_(pa.string())), ]) # fmt: skip step = pa.struct([ ('message_index', pa.int64()), diff --git a/deepgym/tests/agents/fixtures/README.md b/deepgym/tests/agents/fixtures/README.md new file mode 100644 index 00000000..54d3b3aa --- /dev/null +++ b/deepgym/tests/agents/fixtures/README.md @@ -0,0 +1,55 @@ +Version compatibility streams live at `//stream.jsonl`. Tests +resolve the command and version through `deepgym.agents.SUPPORTED_CLIS`; a version +bump fails until its fixture exists. The directory names identify compatibility +targets, not evidence that the CLI produced these initial samples. The separate +`claude.jsonl` and `codex.jsonl` schema samples remain fixed inputs for detailed +accounting and malformed-event unit tests, so new recordings need not reproduce +their particular token counts, identifiers, or text. + +The initial JSONL samples are hand-authored from the documented CLI event shapes +and the HUD reference parser schemas. They are **schema-derived fixtures**, not +captured model runs. No SDK or provider API is used to replay them. Authentic +recordings remain required before claiming end-to-end compatibility for a release. + +- `claude/*/stream.jsonl`: stream-json assistant/tool/result lifecycle, cumulative usage. +- `codex/*/stream.jsonl`: exec --json command/message/task lifecycle, cached input usage. + +For an upgrade, record a harmless completed task from the pinned CLI in an +authorized environment, including a tool call, final response, and token usage. +Sanitize credentials, user paths, and identifiers without changing event shape or +usage fields. Add a provenance note beside the fixture with the command, capture +date, and sanitizations. Never relabel a schema-derived sample as a recording. +The compatibility parser check accepts any completed task with nonempty final +text and reported output-token usage. Run +`python -m pytest tests/agents/test_cli_compat.py tests/agents/test_cli.py -q`. + +Commands in these samples must only be preserved as nested observations. Tests +fail if an adapter tries to execute them again through `Session.step`. + +## Codex error evidence (2026-10-18) + +The installed npm package at `/opt/homebrew/lib/node_modules/@openai/codex/package.json` +matches the pinned release in `SUPPORTED_CLIS`; its `codex --version` agrees. +`codex exec --json --help` documents JSONL output but does not define error/retry fields. +Exact pinned-release retry emission is **not confirmed**; no model call was made. + +Local source `/Users/abhishek/Desktop/typesafe/research/codex-jev` imports upstream +`7498521d288b9b3b96ffba4eedf089d8d6e06a84` (an older snapshot, not release proof): + +- `codex-rs/app-server/src/bespoke_event_handling.rs:1054-1070` emits intermediate + stream errors with `will_retry: true`. +- `codex-rs/exec/src/event_processor_with_jsonl_output.rs:447-457` converts these + notifications to message-only `error` events and keeps running, dropping retry + and structured error metadata. Lines 525-557 handle terminal turn outcomes. +- `codex-rs/exec/src/exec_events.rs:90-94` defines `ThreadErrorEvent` with only + `message`; `exec/tests/event_processor_with_json_output.rs:1691-1740` also keeps + running after an error notification with `will_retry: false`. +- `codex-rs/app-server-protocol/src/protocol/v2/shared.rs:70-120` defines the + structured provider codes used only when a terminal error actually retains them. + +The versioned `retry-then-success.jsonl` is a source-derived regression scenario, +not a captured run. `terminal-provider-error.jsonl` tests retained structured fields; +their presence in the pinned `exec --json` output is **not confirmed**. Message-only +terminal failures stay generic `error`. Standalone errors cannot prove terminality, +so the parser waits for a terminal event or process exit, bounded by the existing +wall-time budget. Neither success nor provider attribution is inferred from prose. diff --git a/deepgym/tests/agents/fixtures/claude.jsonl b/deepgym/tests/agents/fixtures/claude.jsonl new file mode 100644 index 00000000..fcb12074 --- /dev/null +++ b/deepgym/tests/agents/fixtures/claude.jsonl @@ -0,0 +1,5 @@ +{"type":"system","subtype":"init","session_id":"sample-session"} +{"type":"assistant","message":{"id":"message-one","content":[{"type":"tool_use","id":"tool-one","name":"Bash","input":{"command":"printf fixture-only"}}],"usage":{"input_tokens":20,"output_tokens":8,"cache_read_input_tokens":4,"cache_creation_input_tokens":2}}} +{"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"tool-one","content":"fixture-only","is_error":false}]}} +{"type":"assistant","message":{"id":"message-two","content":[{"type":"text","text":"The requested change is complete."}],"usage":{"input_tokens":10,"output_tokens":5,"cache_read_input_tokens":6,"cache_creation_input_tokens":0}}} +{"type":"result","subtype":"success","is_error":false,"num_turns":2,"result":"The requested change is complete.","usage":{"input_tokens":30,"output_tokens":13,"cache_read_input_tokens":10,"cache_creation_input_tokens":2},"total_cost_usd":0.0012} diff --git a/deepgym/tests/agents/fixtures/claude/2.1.302/stream.jsonl b/deepgym/tests/agents/fixtures/claude/2.1.302/stream.jsonl new file mode 100644 index 00000000..fcb12074 --- /dev/null +++ b/deepgym/tests/agents/fixtures/claude/2.1.302/stream.jsonl @@ -0,0 +1,5 @@ +{"type":"system","subtype":"init","session_id":"sample-session"} +{"type":"assistant","message":{"id":"message-one","content":[{"type":"tool_use","id":"tool-one","name":"Bash","input":{"command":"printf fixture-only"}}],"usage":{"input_tokens":20,"output_tokens":8,"cache_read_input_tokens":4,"cache_creation_input_tokens":2}}} +{"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"tool-one","content":"fixture-only","is_error":false}]}} +{"type":"assistant","message":{"id":"message-two","content":[{"type":"text","text":"The requested change is complete."}],"usage":{"input_tokens":10,"output_tokens":5,"cache_read_input_tokens":6,"cache_creation_input_tokens":0}}} +{"type":"result","subtype":"success","is_error":false,"num_turns":2,"result":"The requested change is complete.","usage":{"input_tokens":30,"output_tokens":13,"cache_read_input_tokens":10,"cache_creation_input_tokens":2},"total_cost_usd":0.0012} diff --git a/deepgym/tests/agents/fixtures/codex.jsonl b/deepgym/tests/agents/fixtures/codex.jsonl new file mode 100644 index 00000000..d55ea89a --- /dev/null +++ b/deepgym/tests/agents/fixtures/codex.jsonl @@ -0,0 +1,6 @@ +{"type":"thread.started","thread_id":"sample-thread"} +{"type":"turn.started"} +{"type":"item.started","item":{"id":"command-one","type":"command_execution","command":"printf fixture-only","aggregated_output":"","status":"in_progress"}} +{"type":"item.completed","item":{"id":"command-one","type":"command_execution","command":"printf fixture-only","aggregated_output":"fixture-only","exit_code":0,"status":"completed"}} +{"type":"item.completed","item":{"id":"message-one","type":"agent_message","text":"The requested change is complete."}} +{"type":"turn.completed","usage":{"input_tokens":40,"cached_input_tokens":10,"output_tokens":12}} diff --git a/deepgym/tests/agents/fixtures/codex/0.162.1/retry-then-success.jsonl b/deepgym/tests/agents/fixtures/codex/0.162.1/retry-then-success.jsonl new file mode 100644 index 00000000..5eb38b2a --- /dev/null +++ b/deepgym/tests/agents/fixtures/codex/0.162.1/retry-then-success.jsonl @@ -0,0 +1,5 @@ +{"type":"thread.started","thread_id":"retry-fixture"} +{"type":"turn.started"} +{"type":"error","message":"An opaque intermediate diagnostic"} +{"type":"item.completed","item":{"id":"reply","type":"agent_message","text":"Completed after recovery."}} +{"type":"turn.completed","usage":{"input_tokens":20,"cached_input_tokens":0,"output_tokens":5}} diff --git a/deepgym/tests/agents/fixtures/codex/0.162.1/stream.jsonl b/deepgym/tests/agents/fixtures/codex/0.162.1/stream.jsonl new file mode 100644 index 00000000..d55ea89a --- /dev/null +++ b/deepgym/tests/agents/fixtures/codex/0.162.1/stream.jsonl @@ -0,0 +1,6 @@ +{"type":"thread.started","thread_id":"sample-thread"} +{"type":"turn.started"} +{"type":"item.started","item":{"id":"command-one","type":"command_execution","command":"printf fixture-only","aggregated_output":"","status":"in_progress"}} +{"type":"item.completed","item":{"id":"command-one","type":"command_execution","command":"printf fixture-only","aggregated_output":"fixture-only","exit_code":0,"status":"completed"}} +{"type":"item.completed","item":{"id":"message-one","type":"agent_message","text":"The requested change is complete."}} +{"type":"turn.completed","usage":{"input_tokens":40,"cached_input_tokens":10,"output_tokens":12}} diff --git a/deepgym/tests/agents/fixtures/codex/0.162.1/terminal-provider-error.jsonl b/deepgym/tests/agents/fixtures/codex/0.162.1/terminal-provider-error.jsonl new file mode 100644 index 00000000..6e82e4f1 --- /dev/null +++ b/deepgym/tests/agents/fixtures/codex/0.162.1/terminal-provider-error.jsonl @@ -0,0 +1,4 @@ +{"type":"thread.started","thread_id":"provider-failure-fixture"} +{"type":"turn.started"} +{"type":"error","willRetry":false,"error":{"message":"Opaque diagnostic","codexErrorInfo":{"responseTooManyFailedAttempts":{"httpStatusCode":503}}}} +{"type":"turn.failed","error":{"message":"Opaque terminal diagnostic","codexErrorInfo":{"responseTooManyFailedAttempts":{"httpStatusCode":503}}}} diff --git a/deepgym/tests/agents/test_base.py b/deepgym/tests/agents/test_base.py new file mode 100644 index 00000000..d00103ef --- /dev/null +++ b/deepgym/tests/agents/test_base.py @@ -0,0 +1,79 @@ +"""Offline checks for shared episode accounting.""" + +import pytest +from pydantic import ValidationError + +from deepgym.agents.base import Budget, BudgetExceeded, BudgetTracker, Cost, TokenPrices, Usage + + +@pytest.mark.parametrize( + 'field,value', + [ + ('max_turns', 0), + ('max_seconds', 0), + ('max_seconds', float('nan')), + ('max_cost_usd', -1), + ('max_cost_usd', float('inf')), + ], +) +def test_invalid_budget(field, value): + with pytest.raises(ValidationError): + Budget(**{field: value}) + + +def test_turn_budget(): + tracker = BudgetTracker(Budget(max_turns=1)) + tracker.before_turn() + with pytest.raises(BudgetExceeded, match='turn_budget'): + tracker.before_turn() + assert tracker.turns == 1 + + +def test_time_counts_all_elapsed_time(monkeypatch): + clock = [100.0] + monkeypatch.setattr('deepgym.agents.base.time.monotonic', lambda: clock[0]) + tracker = BudgetTracker(Budget(max_seconds=5)) + clock[0] += 5 + with pytest.raises(BudgetExceeded, match='time_budget'): + tracker.before_turn() + + +def test_disjoint_usage_and_price_provenance(): + prices = TokenPrices( + input_per_million=2, + output_per_million=10, + cache_read_per_million=0.5, + cache_write_per_million=2.5, + source='test rate card 2026-10-18', + ) + usage = Usage( + input_tokens=1_000, output_tokens=100, cache_read_tokens=200, cache_write_tokens=400 + ) + tracker = BudgetTracker(Budget(max_cost_usd=0.006)) + tracker.add_usage(usage, prices.estimate(usage)) + tracker.add_usage(usage, prices.estimate(usage)) + assert tracker.usage.input_tokens == 2000 + assert tracker.cost.usd == pytest.approx(0.0082) + assert 'test rate card 2026-10-18' in tracker.cost.provenance + assert tracker.reason == 'cost_budget' + + +def test_unknown_cost_stays_unknown_and_fails_closed(): + tracker = BudgetTracker(Budget(max_cost_usd=1)) + tracker.add_usage(Usage(), Cost(provenance='response missing usage')) + tracker.add_usage(Usage(input_tokens=10), Cost(usd=0.1, provenance='estimate')) + assert tracker.cost.usd is None + with pytest.raises(BudgetExceeded, match='cost_budget'): + tracker.check() + + +def test_zero_budget_blocks_first_call(): + tracker = BudgetTracker(Budget(max_cost_usd=0)) + with pytest.raises(BudgetExceeded, match='cost_budget'): + tracker.before_turn() + assert tracker.turns == 0 + + +def test_value_models_are_frozen(): + with pytest.raises(ValidationError): + Budget().max_turns = 2 diff --git a/deepgym/tests/agents/test_cli.py b/deepgym/tests/agents/test_cli.py new file mode 100644 index 00000000..ed73f7de --- /dev/null +++ b/deepgym/tests/agents/test_cli.py @@ -0,0 +1,801 @@ +"""Replay schema-derived CLI samples offline, without provider or sandbox calls.""" + +from __future__ import annotations + +import json +from collections.abc import Callable +from pathlib import Path +from typing import cast + +import pytest +from pydantic import JsonValue, SecretStr + +from deepgym import DeepGym +from deepgym.agents.base import Budget, BudgetTracker, TokenPrices +from deepgym.agents.cli import ClaudeCodeAgent, CodexAgent +from deepgym.agents.cli_events import CliEvents +from deepgym.agents.versions import SUPPORTED_CLIS +from deepgym.cli_runtime import CliExecution, CliRequest +from deepgym.exceptions import ActionError, SandboxError +from deepgym.models import Action, MultiTurnEnvironment, Observation, Trajectory +from deepgym.sandboxes import ExecResult +from deepgym.session import SandboxBackend, Session, VerifierOutput + +FIXTURES = Path(__file__).parent / 'fixtures' +PRICES = TokenPrices( + input_per_million=2, + output_per_million=4, + cache_read_per_million=1, + cache_write_per_million=3, + source='offline test rates, not real model prices', +) + + +def stream(name: str) -> list[dict[str, JsonValue]]: + return [json.loads(line) for line in (FIXTURES / f'{name}.jsonl').read_text().splitlines()] + + +def parser(provider: str, **budget: float | int | None) -> CliEvents: + return CliEvents(provider, BudgetTracker(Budget(**budget)), PRICES) + + +def test_claude_aggregate_usage_replaces_message_sum() -> None: + events = parser('anthropic') + for event in stream('claude'): + assert events.consume(event) is None + assert events.finish(0, None) == 'completed' + assert events.tracker.turns == 2 + assert events.tracker.usage.model_dump() == { + 'input_tokens': 30, + 'output_tokens': 13, + 'cache_read_tokens': 10, + 'cache_write_tokens': 2, + } + assert events.tracker.cost.usd == 0.0012 + assert 'CLI-reported' in events.tracker.cost.provenance + + +def test_codex_usage_subtracts_cached_input_and_estimates_cost() -> None: + events = parser('openai') + for event in stream('codex'): + assert events.consume(event) is None + assert events.finish(0, None) == 'completed' + assert events.tracker.turns == 2 + assert events.tracker.usage.input_tokens == 30 + assert events.tracker.usage.cache_read_tokens == 10 + assert events.tracker.cost.usd == pytest.approx(0.000118) + assert PRICES.source in events.tracker.cost.provenance + + +@pytest.mark.parametrize('provider,name', [('anthropic', 'claude'), ('openai', 'codex')]) +def test_activity_budget_stops_stream_before_next_item(provider: str, name: str) -> None: + events = parser(provider, max_turns=1) + consumed = 0 + for event in stream(name): + consumed += 1 + if events.consume(event): + break + assert consumed < len(stream(name)) + assert events.finish(-9, 'turn_budget') == 'turn_budget' + assert events.tracker.turns == 1 + assert 'incomplete' in events.tracker.cost.provenance + + +def test_duplicate_assistant_message_does_not_double_count() -> None: + events = parser('anthropic') + event = stream('claude')[1] + events.consume(event) + events.consume(event) + assert events.tracker.turns == 1 + assert events.tracker.usage.input_tokens == 20 + + +def test_same_message_growing_usage_crosses_live_cost_cap() -> None: + events = parser('anthropic', max_cost_usd=0.0001) + event = stream('claude')[1] + assert events.consume(event) is None + event['message']['usage']['output_tokens'] = 100 + assert events.consume(event) == 'cost_budget' + assert events.tracker.turns == 1 + assert events.tracker.usage.output_tokens == 100 + assert events.tracker.usage.input_tokens == 20 + + +def test_same_message_usage_regression_fails_closed() -> None: + events = parser('anthropic') + event = stream('claude')[1] + assert events.consume(event) is None + event['message']['usage']['output_tokens'] = 1 + assert events.consume(event) == 'error' + assert 'regressed' in events.error + + +def test_cumulative_result_cannot_erase_observed_usage() -> None: + events = parser('anthropic') + for event in stream('claude')[:-1]: + assert events.consume(event) is None + result = stream('claude')[-1] + result['usage']['input_tokens'] = 1 + assert events.consume(result) == 'error' + assert events.tracker.usage.input_tokens == 30 + + +def test_duplicate_codex_item_does_not_double_count() -> None: + events = parser('openai') + event = stream('codex')[3] + events.consume(event) + events.consume(event) + assert events.tracker.turns == 1 + + +def test_usage_after_activity_stop_is_accounted_without_changing_reason() -> None: + events = parser('openai', max_turns=1) + assert events.consume(stream('codex')[3]) == 'turn_budget' + assert events.consume(stream('codex')[-1]) == 'turn_budget' + assert events.tracker.usage.input_tokens == 30 + assert events.tracker.usage.output_tokens == 12 + assert events.tracker.cost.usd == pytest.approx(0.000118) + assert events.finish(-9, 'turn_budget') == 'turn_budget' + + +def test_claude_estimated_spend_stops_on_observed_usage() -> None: + events = parser('anthropic', max_cost_usd=0.000001) + assert events.consume(stream('claude')[1]) == 'cost_budget' + assert events.finish(-9, None) == 'cost_budget' + + +def test_deadline_is_checked_while_events_arrive() -> None: + events = parser('openai') + events.tracker.started -= 301 + assert events.consume({'type': 'thread.started'}) == 'time_budget' + + +@pytest.mark.parametrize('provider', ['anthropic', 'openai']) +def test_zero_exit_requires_terminal_event(provider: str) -> None: + events = parser(provider) + assert events.finish(0, None) == 'error' + assert 'without a terminal' in events.error + + +@pytest.mark.parametrize('provider', ['anthropic', 'openai']) +def test_nonzero_exit_is_error(provider: str) -> None: + events = parser(provider) + assert events.finish(127, None) == 'error' + assert '127' in events.error + + +@pytest.mark.parametrize('exit_code', [0, 1]) +def test_crash_or_nonterminal_exit_labels_observed_cost_incomplete(exit_code: int) -> None: + events = parser('anthropic') + assert events.consume(stream('claude')[1]) is None + observed_usage = events.tracker.usage + observed_cost = events.tracker.cost + + assert events.finish(exit_code, None) == 'error' + assert events.tracker.usage == observed_usage + assert events.tracker.cost.usd == observed_cost.usd + assert observed_cost.provenance in events.tracker.cost.provenance + assert 'incomplete observed CLI subtotal' in events.tracker.cost.provenance + assert 'additional usage may be unreported' in events.tracker.cost.provenance + + +def test_successful_terminal_accounting_is_not_labeled_incomplete() -> None: + events = parser('anthropic') + for event in stream('claude'): + assert events.consume(event) is None + assert events.finish(0, None) == 'completed' + assert 'incomplete' not in events.tracker.cost.provenance + + +@pytest.mark.parametrize( + 'subtype,reason', + [ + ('error_max_turns', 'turn_budget'), + ('error_max_budget_usd', 'cost_budget'), + ('error_during_execution', 'error'), + ], +) +def test_claude_terminal_errors(subtype: str, reason: str) -> None: + events = parser('anthropic') + events.consume({'type': 'result', 'subtype': subtype, 'is_error': True}) + assert events.finish(1, None) == reason + + +@pytest.mark.parametrize( + 'event,reason', + [ + ({'type': 'turn.failed', 'error': {'message': 'provider refused'}}, 'error'), + ({'type': 'error', 'message': 'provider refused'}, None), + ], +) +def test_codex_error_events(event: dict[str, JsonValue], reason: str | None) -> None: + events = parser('openai') + assert events.consume(event) == reason + if reason: + assert events.error == 'provider refused' + else: + assert events.error is None + assert events.finish(1, None) == 'error' + + +@pytest.mark.parametrize('retry_flag', [True, False, None]) +def test_codex_standalone_error_waits_for_terminal_evidence(retry_flag: bool | None) -> None: + events = parser('openai') + error = {'type': 'error', 'message': 'No prose classification'} + if retry_flag is not None: + error['willRetry'] = retry_flag + assert events.consume(error) is None + for event in stream('codex'): + assert events.consume(event) is None + assert events.finish(0, None) == 'completed' + + +@pytest.mark.parametrize( + 'filename,reason', + [('retry-then-success', 'completed'), ('terminal-provider-error', 'provider_error')], +) +def test_codex_error_scenario_fixtures(filename: str, reason: str) -> None: + release = SUPPORTED_CLIS['openai'] + path = FIXTURES / release.command / release.version / f'{filename}.jsonl' + events = parser('openai') + for event in map(json.loads, path.read_text().splitlines()): + observed = events.consume(event) + assert observed == ('provider_error' if event['type'] == 'turn.failed' else None) + assert events.finish(0 if reason == 'completed' else 1, None) == reason + if reason == 'provider_error': + assert 'incomplete observed CLI subtotal' in events.tracker.cost.provenance + + +@pytest.mark.parametrize( + 'code,reason', + [ + ('rateLimitExceeded', 'provider_error'), + ('unauthorized', 'provider_error'), + ('serverOverloaded', 'provider_error'), + ({'responseStreamDisconnected': {}}, 'provider_error'), + ({'httpConnectionFailed': {'httpStatusCode': 503}}, 'provider_error'), + ({'httpConnectionFailed': {'httpStatusCode': 400}}, 'error'), + ({'httpConnectionFailed': {'httpStatusCode': '503'}}, 'error'), + ('contextWindowExceeded', 'error'), + ('sandboxError', 'error'), + ('unknownFutureCode', 'error'), + (None, 'error'), + ], +) +def test_codex_provider_classification_uses_only_structured_fields(code, reason: str) -> None: + events = parser('openai') + event = { + 'type': 'turn.failed', + 'error': {'message': 'HTTP 503 rate limit unauthorized', 'codexErrorInfo': code}, + } + assert events.consume(event) == reason + assert events.finish(1, None) == reason + + +@pytest.mark.parametrize('provider_failure', [True, False]) +def test_first_terminal_error_reason_survives_later_events(provider_failure: bool) -> None: + events = parser('openai') + reason = 'provider_error' if provider_failure else 'error' + code = 'rateLimitExceeded' if provider_failure else 'sandboxError' + assert ( + events.consume( + {'type': 'turn.failed', 'error': {'message': 'first', 'codexErrorInfo': code}} + ) + == reason + ) + assert events.consume({}) == reason + assert ( + events.consume( + { + 'type': 'turn.failed', + 'error': {'message': 'later', 'codexErrorInfo': 'serverOverloaded'}, + } + ) + == reason + ) + assert events.finish(1, None) == reason + + +@pytest.mark.parametrize( + 'event', + [ + {}, + {'type': 'item.completed', 'item': []}, + {'type': 'item.completed', 'item': {'id': 'a', 'type': 'agent_message', 'text': 2}}, + {'type': 'turn.completed', 'usage': {'input_tokens': -1, 'output_tokens': 1}}, + {'type': 'turn.completed', 'usage': {'input_tokens': True, 'output_tokens': 1}}, + { + 'type': 'turn.completed', + 'usage': {'input_tokens': 1, 'output_tokens': 1, 'cached_input_tokens': 2}, + }, + ], +) +def test_malformed_events_fail_closed(event: dict[str, JsonValue]) -> None: + events = parser('openai') + assert events.consume(event) == 'error' + assert 'Invalid openai CLI event' in events.error + + +def test_unknown_event_is_preserved_by_caller_without_inventing_actions() -> None: + events = parser('openai') + assert events.consume({'type': 'future.event', 'payload': {'anything': True}}) is None + assert events.tracker.turns == 0 + + +class RecordingSession: + """Model the supervised Session boundary, not Daytona or a provider SDK.""" + + def __init__(self, name: str) -> None: + self.name = name + self.done = False + self.trajectory = Trajectory() + self.requests: list[CliRequest] = [] + self.finalizations = 0 + + def finalize_cli(self) -> None: + self.finalizations += 1 + self.done = True + self.trajectory.final_observation = Observation(content='', step=0, done=True) + + def run_cli( + self, + request: CliRequest, + consume: Callable[[dict[str, JsonValue]], str | None], + ) -> CliExecution: + self.requests.append(request) + events = [] + reason = None + for event in stream(self.name): + events.append(event) + if reason := consume(event): + break + self.trajectory.steps.append( + ( + Observation(content='', step=0), + Action(action_type='cli', content='installed CLI', events=events), + ) + ) + return CliExecution( + events=tuple(events), + output='', + exit_code=0, + stop_reason=reason, + cli_version=request.supported_version, + ) + + def step(self, action: Action) -> None: + pytest.fail(f'Parsed CLI events must never be replayed: {action}') + + +@pytest.mark.parametrize('agent_type,name', [(ClaudeCodeAgent, 'claude'), (CodexAgent, 'codex')]) +def test_run_once_records_nested_events_without_replay(agent_type: type, name: str) -> None: + session = RecordingSession(name) + agent = agent_type( + model='fixture-model', + api_key='secret-fixture', + allow_model_credentials=True, + prices=PRICES, + ) + result = agent.run( + cast(Session, session), + Observation(content='', step=0), + task='literal $(task)', + budget=Budget(), + ) + assert result.termination_reason == 'completed' + assert result.cli_version == SUPPORTED_CLIS[agent.provider].version + assert len(result.trajectory.steps) == len(session.requests) == 1 + assert result.trajectory.steps[0][1].events == tuple(stream(name)) + assert session.requests[0].argv[-1] == 'literal $(task)' + assert 'secret-fixture' not in repr(agent) + assert 'secret-fixture' not in result.model_dump_json() + assert not any('secret-fixture' in arg for arg in session.requests[0].argv) + assert session.finalizations == 1 + assert result.trajectory.final_observation.done + + +@pytest.mark.parametrize('agent_type', [ClaudeCodeAgent, CodexAgent]) +def test_credentials_require_explicit_opt_in(agent_type: type) -> None: + session = RecordingSession('claude') + with pytest.raises(ValueError, match='allow_model_credentials=True'): + agent_type(model='m', api_key='k').run( + cast(Session, session), + Observation(content='', step=0), + task='task', + budget=Budget(), + ) + assert not session.requests + assert session.finalizations == 1 + + +@pytest.mark.parametrize('cap', [0, 1]) +def test_codex_finite_usd_budget_rejected_before_launch(cap: float) -> None: + session = RecordingSession('codex') + agent = CodexAgent(model='m', api_key='k', allow_model_credentials=True, prices=PRICES) + with pytest.raises(ValueError, match='finite USD budget'): + agent.run( + cast(Session, session), + Observation(content='', step=0), + task='task', + budget=Budget(max_cost_usd=cap), + ) + assert not session.requests + + +def test_claude_native_dollar_budget() -> None: + session = RecordingSession('claude') + agent = ClaudeCodeAgent(model='m', api_key='k', allow_model_credentials=True, prices=PRICES) + agent.run( + cast(Session, session), + Observation(content='', step=0), + task='task', + budget=Budget( + max_turns=8, + max_seconds=22, + max_cost_usd=3, + ), + ) + request = session.requests[0] + assert request.argv[request.argv.index('--max-budget-usd') + 1] == '3.0' + assert 0 < request.timeout <= 22 + + +def test_claude_finite_budget_requires_prices_before_launch() -> None: + session = RecordingSession('claude') + agent = ClaudeCodeAgent(model='m', api_key='k', allow_model_credentials=True) + with pytest.raises(ValueError, match='TokenPrices'): + agent.run( + cast(Session, session), + Observation(content='', step=0), + task='task', + budget=Budget(max_cost_usd=1), + ) + assert not session.requests + + +def test_claude_missing_per_message_usage_fails_closed() -> None: + events = parser('anthropic', max_cost_usd=1) + assert ( + events.consume( + { + 'type': 'assistant', + 'message': {'id': 'missing-usage', 'content': []}, + } + ) + == 'error' + ) + assert 'assistant usage' in events.error + + +def test_claude_without_usd_budget_accepts_final_only_usage() -> None: + events = parser('anthropic') + event = stream('claude')[1] + del event['message']['usage'] + assert events.consume(event) is None + assert events.tracker.cost.usd is None + assert events.consume(stream('claude')[-1]) is None + assert events.finish(0, None) == 'completed' + assert events.tracker.usage.input_tokens == 30 + assert events.tracker.cost.usd == 0.0012 + + +def test_claude_missing_message_and_final_usage_is_error() -> None: + events = parser('anthropic') + event = stream('claude')[1] + del event['message']['usage'] + assert events.consume(event) is None + result = stream('claude')[-1] + del result['usage'] + assert events.consume(result) is None + assert events.finish(0, None) == 'error' + assert 'without token usage' in events.error + + +def test_claude_zero_dollar_budget_does_not_launch() -> None: + session = RecordingSession('claude') + result = ClaudeCodeAgent(model='m').run( + cast(Session, session), + Observation(content='', step=0), + task='task', + budget=Budget(max_cost_usd=0), + ) + assert result.termination_reason == 'cost_budget' + assert not session.requests + + +def test_done_session_does_not_launch() -> None: + session = RecordingSession('claude') + session.done = True + result = ClaudeCodeAgent(model='m').run( + session, + Observation(content='', step=0), + task='task', + budget=Budget(), + ) + assert result.termination_reason == 'session_done' + assert session.finalizations == 1 + + +def test_exhausted_wall_budget_does_not_launch() -> None: + session = RecordingSession('claude') + agent = ClaudeCodeAgent(model='m', api_key='k', allow_model_credentials=True) + result = agent.run( + cast(Session, session), + Observation(content='', step=0), + task='task', + budget=Budget(max_seconds=1e-100), + ) + assert result.termination_reason == 'time_budget' + assert not session.requests + + +@pytest.mark.parametrize('expired,reason', [(False, 'error'), (True, 'time_budget')]) +def test_failure_retains_partial_accounting_and_safe_error( + monkeypatch: pytest.MonkeyPatch, + expired: bool, + reason: str, +) -> None: + now = [0.0] + monkeypatch.setattr('deepgym.agents.base.time.monotonic', lambda: now[0]) + + class FailingSession(RecordingSession): + def run_cli( + self, + request: CliRequest, + consume: Callable[[dict[str, JsonValue]], str | None], + ) -> CliExecution: + event = stream('claude')[1] + consume(event) + self.trajectory.steps.append( + ( + Observation(content='', step=0), + Action(action_type='cli', content='installed CLI', events=(event,)), + ) + ) + if expired: + now[0] = 301.0 + raise SandboxError('provider failure includes secret-fixture') + + session = FailingSession('claude') + agent = ClaudeCodeAgent( + model='m', + api_key='secret-fixture', + allow_model_credentials=True, + prices=PRICES, + ) + result = agent.run( + cast(Session, session), + Observation(content='', step=0), + task='task', + budget=Budget(), + ) + assert result.termination_reason == reason + assert result.error == 'SandboxError' + assert result.usage.input_tokens == 20 + assert result.cost.usd > 0 + assert 'incomplete' in result.cost.provenance + assert len(result.trajectory.steps) == 1 + assert 'secret-fixture' not in result.model_dump_json() + + +class SetupWriterBackend(SandboxBackend): + """Track setup population and grading order without creating processes or a sandbox.""" + + def __init__(self, *, cleanup_fails: bool = False) -> None: + self.sandbox_id = 'setup-writer-fixture' + self.events: list[str] = [] + self.writer_alive = False + self.cleanup_fails = cleanup_fails + + def exec( + self, + command: str, + timeout: int, + env: dict[str, str] | None = None, + ) -> ExecResult: + self.events.append('setup') + self.writer_alive = True + return ExecResult(0, 'setup writer started') + + def stop_agent(self) -> None: + self.events.append('stop') + if self.cleanup_fails: + raise SandboxError('fixture cleanup failed') + self.writer_alive = False + + def verify( + self, + code: str, + timeout: int, + env: dict[str, str] | None = None, + ) -> VerifierOutput: + self.events.append('verify') + assert not self.writer_alive, 'grading ran while setup writer was alive' + return VerifierOutput(0, '{"score":1.0,"passed":true}', '') + + def run_cli( + self, + request: CliRequest, + consume: Callable[[dict[str, JsonValue]], str | None], + ) -> CliExecution: + pytest.fail('A preflight return must not launch the CLI') + + def close(self) -> None: + self.events.append('close') + + +@pytest.mark.parametrize('agent_type', [ClaudeCodeAgent, CodexAgent]) +def test_cli_agent_requires_explicit_environment_egress(agent_type) -> None: + backend = SetupWriterBackend() + session = Session(MultiTurnEnvironment(task='task', setup_script='start writer'), backend) + initial = session.start() + agent = agent_type(model='m', api_key='fixture-key', allow_model_credentials=True) + with pytest.raises(ActionError, match='set cli_provider_egress=True.*before creating'): + agent.run(session, initial, task='task', budget=Budget()) + assert backend.events == ['setup', 'stop'] + assert session.trajectory.steps == [] + + +@pytest.mark.parametrize('agent_type,name', [(ClaudeCodeAgent, 'claude'), (CodexAgent, 'codex')]) +def test_finalization_preserves_unknown_state_after_partial_cli_failure( + agent_type: type[ClaudeCodeAgent] | type[CodexAgent], name: str +) -> None: + partial = next( + event for event in stream(name) if event['type'] in ('assistant', 'item.completed') + ) + + class MutatingBackend(SetupWriterBackend): + def run_cli( + self, request: CliRequest, consume: Callable[[dict[str, JsonValue]], str | None] + ) -> CliExecution: + self.events.append('mutated') + consume(partial) + raise SandboxError('transport failed after possible side effects') + + backend = MutatingBackend() + session = Session( + MultiTurnEnvironment(task='task', setup_script='start writer', cli_provider_egress=True), + backend, + ) + initial = session.start() + agent = agent_type(model='m', api_key='fixture-key', allow_model_credentials=True) + result = agent.run(session, initial, task='task', budget=Budget()) + + assert result.termination_reason == 'error' + assert session.done + assert result.trajectory.final_observation is None + assert session.trajectory.final_observation is None + assert len(result.trajectory.steps) == 1 + before, action = result.trajectory.steps[0] + assert before.content == initial.content + assert action.action_type == 'cli' + assert action.events == (partial,) + assert backend.events == ['setup', 'mutated', 'stop'] + assert not backend.writer_alive + + +@pytest.mark.parametrize('reason', ['cost_budget', 'time_budget']) +@pytest.mark.parametrize('cleanup_fails', [False, True]) +def test_no_launch_episode_cleans_setup_before_grading( + monkeypatch: pytest.MonkeyPatch, + reason: str, + cleanup_fails: bool, +) -> None: + now = [0.0] + monkeypatch.setattr('deepgym.agents.base.time.monotonic', lambda: now[0]) + environment = MultiTurnEnvironment( + task='fixture task', + setup_script='start fixture writer', + final_verifier_code='return 1.0', + ) + backend = SetupWriterBackend(cleanup_fails=cleanup_fails) + session = Session(environment, backend) + client = DeepGym(mode='local') + monkeypatch.setattr(client, 'session', lambda env: session) + agent = ClaudeCodeAgent(model='m', api_key='fixture-key', allow_model_credentials=True) + budget = Budget(max_cost_usd=0) if reason == 'cost_budget' else Budget(max_seconds=1) + + if reason == 'time_budget': + original = SecretStr.get_secret_value + + def expire_preflight(key: SecretStr) -> str: + assert backend.writer_alive + now[0] = 2.0 + return original(key) + + monkeypatch.setattr(SecretStr, 'get_secret_value', expire_preflight) + + if cleanup_fails: + with pytest.raises(SandboxError, match='fixture cleanup failed'): + client.run_episode(environment, agent, budget=budget) + assert backend.events == ['setup', 'stop', 'close'] + assert backend.writer_alive + assert session.failed_result is not None + assert not session.score().has_verdict + assert 'verify' not in backend.events + else: + result, grade = client.run_episode(environment, agent, budget=budget) + assert result.termination_reason == reason + assert result.trajectory.final_observation.done + assert result.trajectory.steps == [] + assert grade.has_verdict and grade.score == 1.0 + assert backend.events == ['setup', 'stop', 'verify', 'close'] + assert not backend.writer_alive + assert session.done + assert session.trajectory.steps == [] + + +def test_already_done_session_still_cleans_setup_population() -> None: + environment = MultiTurnEnvironment(task='fixture task', setup_script='start fixture writer') + backend = SetupWriterBackend() + session = Session(environment, backend) + initial = session.start() + session.step(Action(action_type='done')) + assert backend.writer_alive + + result = ClaudeCodeAgent(model='m').run(session, initial, task='task', budget=Budget()) + session.finalize_cli() + + assert result.termination_reason == 'session_done' + assert result.trajectory.final_observation.done + assert len(result.trajectory.steps) == 1 + assert result.trajectory.steps[0][1].action_type == 'done' + assert backend.events == ['setup', 'stop'] + assert not backend.writer_alive + + +def test_gui_scoring_does_not_finalize_cli_containment(monkeypatch: pytest.MonkeyPatch) -> None: + backend = SetupWriterBackend() + monkeypatch.setattr(backend, 'start_desktop', lambda: None) + monkeypatch.setattr(backend, 'screenshot', lambda: 'fixture-png') + monkeypatch.setattr(backend, 'stop_agent', lambda: pytest.fail('GUI desktop was stopped')) + monkeypatch.setattr( + backend, + 'verify', + lambda *args: VerifierOutput(0, '{"score":1.0,"passed":true}', ''), + ) + session = Session( + MultiTurnEnvironment( + task='GUI task', + desktop=True, + allowed_actions={'screenshot'}, + setup_script='start fixture desktop', + final_verifier_code='return 1.0', + ), + backend, + ) + session.start() + assert session.score().has_verdict + assert backend.writer_alive + assert backend.events == ['setup'] + + +@pytest.mark.parametrize('preflight', ['credentials', 'prices', 'codex_budget']) +def test_preflight_exception_cleans_setup_population( + monkeypatch: pytest.MonkeyPatch, + preflight: str, +) -> None: + environment = MultiTurnEnvironment( + task='fixture task', + setup_script='start fixture writer', + final_verifier_code='return 1.0', + ) + backend = SetupWriterBackend() + session = Session(environment, backend) + client = DeepGym(mode='local') + monkeypatch.setattr(client, 'session', lambda env: session) + agent_type = CodexAgent if preflight == 'codex_budget' else ClaudeCodeAgent + agent = agent_type( + model='m', + api_key='fixture-key', + allow_model_credentials=preflight != 'credentials', + ) + budget = Budget() if preflight == 'credentials' else Budget(max_cost_usd=1) + + with pytest.raises(ValueError): + client.run_episode(environment, agent, budget=budget) + + assert backend.events == ['setup', 'stop', 'close'] + assert not backend.writer_alive + assert session.done + assert session.trajectory.steps == [] diff --git a/deepgym/tests/agents/test_cli_compat.py b/deepgym/tests/agents/test_cli_compat.py new file mode 100644 index 00000000..5d5c94ac --- /dev/null +++ b/deepgym/tests/agents/test_cli_compat.py @@ -0,0 +1,91 @@ +"""Check pinned CLI flags locally and replay version-specific stream samples.""" + +import json +import os +import re +import shlex +import shutil +import subprocess +from pathlib import Path + +import pytest + +from deepgym._cli_worker import parse_cli_version +from deepgym.agents.base import Budget, BudgetTracker +from deepgym.agents.cli import ClaudeCodeAgent, CodexAgent +from deepgym.agents.cli_events import CliEvents +from deepgym.agents.versions import SUPPORTED_CLIS, cli_install_command + + +def test_install_command_uses_supported_releases() -> None: + assert shlex.split(cli_install_command()) == [ + 'npm', + 'install', + '-g', + '--prefix', + '/usr/local', + *(f'{release.package}@{release.version}' for release in SUPPORTED_CLIS.values()), + ] + + +@pytest.mark.parametrize('provider', SUPPORTED_CLIS) +def test_installed_cli_supports_adapter_flags(provider: str, tmp_path: Path) -> None: + release = SUPPORTED_CLIS[provider] + executable = shutil.which(release.command) + if executable is None: + pytest.skip(f'{release.command} is not installed locally') + + config = tmp_path / release.command + config.mkdir() + env = dict(os.environ) + env['CLAUDE_CONFIG_DIR' if provider == 'anthropic' else 'CODEX_HOME'] = str(config) + version = subprocess.run( + [executable, '--version'], + capture_output=True, + text=True, + check=True, + timeout=20, + env=env, + ) + assert parse_cli_version(version.stdout) == release.version, ( + f'Unsupported local {release.command}: {version.stdout.strip()}. ' + f'Install the pinned CLIs with `{cli_install_command()}`, or bump SUPPORTED_CLIS, ' + 'record new stream fixtures, run the compat test, and rebuild images.' + ) + agent = (ClaudeCodeAgent if provider == 'anthropic' else CodexAgent)(model='compat-only') + budget = Budget(max_cost_usd=1 if provider == 'anthropic' else None) + argv = agent._argv('compat-only', budget) + subcommand = list(argv[1:2]) if provider == 'openai' else [] + help_result = subprocess.run( + [executable, *subcommand, '--help'], + capture_output=True, + text=True, + check=True, + timeout=20, + env=env, + ) + flags = {arg for arg in argv[1 : argv.index('--')] if arg.startswith('--')} + missing = sorted( + flag + for flag in flags + if not re.search(rf'(? None: + release = SUPPORTED_CLIS[provider] + fixture = ( + Path(__file__).parent / 'fixtures' / release.command / release.version / 'stream.jsonl' + ) + assert fixture.is_file(), ( + f'Missing compatibility fixture: {fixture}. Record the new release stream before ' + 'accepting the version bump; see fixtures/README.md for provenance requirements.' + ) + parser = CliEvents(provider, BudgetTracker(Budget()), None) + for line in fixture.read_text().splitlines(): + assert parser.consume(json.loads(line)) is None + assert parser.finish(0, None) == 'completed' + assert parser.final_text + assert parser.tracker.usage.output_tokens > 0 diff --git a/deepgym/tests/agents/test_cli_versions.py b/deepgym/tests/agents/test_cli_versions.py new file mode 100644 index 00000000..e8121956 --- /dev/null +++ b/deepgym/tests/agents/test_cli_versions.py @@ -0,0 +1,199 @@ +"""Check version admission without running a sandbox or an inference command.""" + +import json +import subprocess +from types import SimpleNamespace +from typing import cast + +import pytest + +from deepgym import _cli_worker +from deepgym.agents import SUPPORTED_CLIS, Budget, ClaudeCodeAgent +from deepgym.cli_runtime import CliExecution, CliExecutionError +from deepgym.models import Observation +from deepgym.session import Session +from tests.agents.test_cli import RecordingSession + + +@pytest.mark.parametrize('provider', SUPPORTED_CLIS) +def test_version_probe_is_unprivileged_keyless_and_bounded(monkeypatch, provider: str) -> None: + release = SUPPORTED_CLIS[provider] + captured = {} + + def run(argv, **kwargs): + captured.update(argv=argv, **kwargs) + return SimpleNamespace(stdout=f'{release.command} {release.version}', returncode=0) + + monkeypatch.setattr(_cli_worker.subprocess, 'run', run) + monkeypatch.setattr(_cli_worker.time, 'monotonic', lambda: 100) + status = {} + config = { + 'prefix': ['nsenter', '--net=isolated', '--', 'setpriv', '--reuid=997', '--'], + 'argv': [release.command, '--model', 'unused'], + 'cwd': '/workspace', + 'version_env': {'PATH': '/usr/bin'}, + 'env': {'ANTHROPIC_API_KEY': 'must-not-reach-probe'}, + 'supported_version': release.version, + } + _cli_worker._check_version(config, status, 102) + assert captured['argv'] == [*config['prefix'], release.command, '--version'] + assert captured['env'] == config['version_env'] + assert captured['timeout'] == 2 + assert status == {'cli_version': release.version} + + +@pytest.mark.parametrize('output', ['unversioned', '1.2.3 4.5.6', '1.2.3-beta', 'v1.2.3']) +def test_malformed_or_ambiguous_versions_fail_closed(output: str) -> None: + assert _cli_worker.parse_cli_version(output) is None + + +@pytest.mark.parametrize('failure', [FileNotFoundError(), subprocess.TimeoutExpired('cli', 1)]) +def test_unavailable_version_has_launch_diagnostic(monkeypatch, failure: Exception) -> None: + def run(*args, **kwargs): + raise failure + + monkeypatch.setattr(_cli_worker.subprocess, 'run', run) + monkeypatch.setattr(_cli_worker.time, 'monotonic', lambda: 0) + release = SUPPORTED_CLIS['anthropic'] + status = {} + _cli_worker._check_version( + { + 'prefix': [], + 'argv': [release.command], + 'cwd': '/', + 'version_env': {}, + 'supported_version': release.version, + }, + status, + 10, + ) + assert status['cli_version'] is None + assert 'Cannot start' in status['version_error'] + assert 'Unsupported' not in status['version_error'] + assert 'cli_install_command()' in status['version_error'] + assert '/usr/local/bin/node' in status['version_error'] + + +def test_supervisor_rejects_wrong_version_before_egress_or_inference(monkeypatch, tmp_path) -> None: + release = SUPPORTED_CLIS['anthropic'] + config = { + 'provider': 'anthropic', + 'timeout': 30, + 'uid': 997, + 'prefix': ['setpriv'], + 'argv': [release.command], + 'cwd': str(tmp_path), + 'version_env': {}, + 'supported_version': release.version, + } + (tmp_path / 'request.json').write_text(json.dumps(config)) + server = SimpleNamespace( + serve_forever=lambda: None, shutdown=lambda: None, server_close=lambda: None + ) + monkeypatch.setattr(_cli_worker.socketserver, 'ThreadingUnixStreamServer', lambda *args: server) + monkeypatch.setattr( + _cli_worker.threading, 'Thread', lambda **kwargs: SimpleNamespace(start=lambda: None) + ) + monkeypatch.setattr(_cli_worker.os, 'pidfd_open', lambda pid: 123, raising=False) + monkeypatch.setattr(_cli_worker.signal, 'pidfd_send_signal', lambda *args: None, raising=False) + monkeypatch.setattr(_cli_worker.os, 'close', lambda fd: None) + monkeypatch.setattr( + _cli_worker.subprocess, + 'run', + lambda *args, **kwargs: SimpleNamespace(stdout='9.9.9', returncode=0), + ) + monkeypatch.setattr( + _cli_worker.subprocess, + 'Popen', + lambda *args, **kwargs: pytest.fail('Version mismatch must not launch bridge/inference'), + ) + _cli_worker._supervise(tmp_path) + status = json.loads((tmp_path / 'status.json').read_text()) + assert status['cli_version'] == '9.9.9' + assert 'Unsupported' in status['version_error'] + + +@pytest.mark.parametrize('exit_code', [126, 127]) +def test_missing_binary_or_node_is_not_reported_as_version_mismatch(monkeypatch, exit_code) -> None: + monkeypatch.setattr( + _cli_worker.subprocess, + 'run', + lambda *args, **kwargs: SimpleNamespace(stdout='', returncode=exit_code), + ) + monkeypatch.setattr(_cli_worker.time, 'monotonic', lambda: 0) + status = {} + release = SUPPORTED_CLIS['anthropic'] + _cli_worker._check_version( + { + 'prefix': [], + 'argv': [release.command], + 'cwd': '/', + 'version_env': {}, + 'supported_version': release.version, + }, + status, + 10, + ) + assert 'executable or interpreter' in status['version_error'] + assert 'agent PATH' in status['version_error'] + assert 'Unsupported' not in status['version_error'] + + +def test_rejected_version_reaches_result_with_zero_spend() -> None: + class UnsupportedSession(RecordingSession): + def run_cli(self, request, consume): + return CliExecution( + events=(), + output='', + exit_code=-1, + cli_version='9.9.9', + version_error='Unsupported version; bump SUPPORTED_CLIS and rebuild images', + ) + + result = ClaudeCodeAgent(model='m', api_key='fixture', allow_model_credentials=True).run( + cast(Session, UnsupportedSession('claude')), + Observation(content='', step=0), + task='task', + budget=Budget(), + ) + assert result.termination_reason == 'error' + assert result.cli_version == '9.9.9' + assert 'rebuild images' in result.error + assert result.cost.usd == 0 + + +def test_later_supervision_failure_keeps_observed_version() -> None: + version = SUPPORTED_CLIS['anthropic'].version + + class FailedSession(RecordingSession): + def run_cli(self, request, consume): + raise CliExecutionError('supervision failed', version) + + result = ClaudeCodeAgent(model='m', api_key='fixture', allow_model_credentials=True).run( + cast(Session, FailedSession('claude')), + Observation(content='', step=0), + task='task', + budget=Budget(), + ) + assert result.termination_reason == 'error' + assert result.cli_version == version + + +def test_version_probe_deadline_preserves_budget_reason() -> None: + class ExpiredSession(RecordingSession): + def run_cli(self, request, consume): + return CliExecution( + events=(), + output='', + exit_code=-1, + stop_reason='time_budget', + version_error='Wall budget exhausted before CLI version check', + ) + + result = ClaudeCodeAgent(model='m', api_key='fixture', allow_model_credentials=True).run( + cast(Session, ExpiredSession('claude')), + Observation(content='', step=0), + task='task', + budget=Budget(), + ) + assert result.termination_reason == 'time_budget' diff --git a/deepgym/tests/agents/test_computer.py b/deepgym/tests/agents/test_computer.py new file mode 100644 index 00000000..22dd9bc0 --- /dev/null +++ b/deepgym/tests/agents/test_computer.py @@ -0,0 +1,749 @@ +"""Test actual provider loops with in-memory SDK responses and Session stepping.""" + +import sys +from types import SimpleNamespace + +import pytest + +from deepgym.agents.actions import anthropic_actions, openai_actions +from deepgym.agents.base import Budget, TokenPrices +from deepgym.agents.computer import AnthropicComputerAgent, OpenAIComputerAgent +from deepgym.models import Action, Observation, Trajectory +from deepgym.session import StepResult + + +@pytest.mark.parametrize( + 'name,data,kind', + [ + ('screenshot', {}, 'screenshot'), + ('left_click', {'coordinate': [2, 3]}, 'click'), + ('right_click', {'coordinate': [2, 3]}, 'right_click'), + ('double_click', {'coordinate': [2, 3]}, 'double_click'), + ('mouse_move', {'coordinate': [2, 3]}, 'move'), + ('left_click_drag', {'start_coordinate': [1, 1], 'coordinate': [2, 3]}, 'drag'), + ('scroll', {'coordinate': [2, 3], 'scroll_direction': 'down'}, 'scroll'), + ('type', {'text': 'hello'}, 'type'), + ('key', {'text': 'Control+Return'}, 'key'), + ('wait', {'duration': 1.5}, 'wait'), + ], +) +def test_anthropic_mapping(name, data, kind): + assert anthropic_actions(name, data)[0].action_type == kind + + +@pytest.mark.parametrize( + 'name,data', + [ + ('middle_click', {'coordinate': [1, 2]}), + ('left_mouse_down', {}), + ('left_mouse_up', {}), + ('cursor_position', {}), + ('hold_key', {'text': 'a'}), + ('zoom', {}), + ('triple_click', {'coordinate': [1, 2]}), + ('left_click', {}), + ('left_click', {'coordinate': [1, 2], 'text': 'shift'}), + ('left_click', {'coordinate': [1.5, 2]}), + ('left_click', {'coordinate': [True, 2]}), + ('scroll', {'coordinate': [1, 2], 'scroll_direction': 'down', 'scroll_amount': 100}), + ('key', {'text': 'a', 'repeat': 0}), + ('key', {'text': 'ctrl++a'}), + ('wait', {'duration': float('nan')}), + ('unknown', {}), + ], +) +def test_anthropic_rejects_without_changing_meaning(name, data): + with pytest.raises(ValueError): + anthropic_actions(name, data) + + +@pytest.mark.parametrize( + 'data,kind', + [ + ({'type': 'click', 'x': 2, 'y': 3, 'button': 'left'}, 'click'), + ({'type': 'click', 'x': 2, 'y': 3, 'button': 'right'}, 'right_click'), + ({'type': 'double_click', 'x': 2, 'y': 3}, 'double_click'), + ({'type': 'move', 'x': 2, 'y': 3}, 'move'), + ({'type': 'drag', 'path': [{'x': 1, 'y': 2}, {'x': 3, 'y': 4}]}, 'drag'), + ({'type': 'scroll', 'x': 2, 'y': 3, 'scroll_y': -200}, 'scroll'), + ({'type': 'scroll', 'x': 2, 'y': 3, 'scroll_x': 200}, 'scroll'), + ({'type': 'type', 'text': 'hello'}, 'type'), + ({'type': 'keypress', 'keys': ['CTRL', 'A']}, 'key'), + ({'type': 'wait'}, 'wait'), + ({'type': 'screenshot'}, 'screenshot'), + ], +) +def test_openai_mapping(data, kind): + assert openai_actions(data)[0].action_type == kind + + +@pytest.mark.parametrize( + 'data', + [ + {'type': 'click', 'x': 1, 'y': 2, 'button': 'middle'}, + {'type': 'click', 'x': 1, 'y': 2, 'keys': ['shift']}, + {'type': 'scroll', 'x': 1, 'y': 2, 'scroll_x': 10, 'scroll_y': 20}, + {'type': 'drag', 'path': [{'x': 1, 'y': 2}] * 3}, + {'type': 'keypress', 'keys': []}, + {'type': 'unknown'}, + ], +) +def test_openai_rejects_unsupported(data): + with pytest.raises(ValueError): + openai_actions(data) + + +class FakeSession: + def __init__(self, max_steps: int | None = None): + self.max_steps = max_steps + self.done = False + self.trajectory = Trajectory() + self.observation = Observation(content='', step=0, screenshot='aW1hZ2U=') + + def step(self, action: Action, *, timeout=None): + assert timeout > 0 + self.trajectory.steps.append((self.observation, action)) + step = self.observation.step + 1 + self.done = self.max_steps is not None and step >= self.max_steps + self.observation = self.observation.model_copy( + update={ + 'step': step, + 'done': self.done, + 'screenshot': f'image-{step}', + } + ) + self.trajectory.step_rewards.append(0) + self.trajectory.final_observation = self.observation + return StepResult(self.observation, 0, self.done) + + +class FakeClient: + def __init__(self, responses): + self.responses = self.messages = self + self.results = iter(responses) + self.requests = [] + self.closed = False + + def create(self, **kwargs): + self.requests.append(kwargs) + response = next(self.results) + if isinstance(response, Exception): + raise response + return SimpleNamespace(model_dump=lambda **_: response) + + def close(self): + self.closed = True + + +@pytest.fixture +def sdk(monkeypatch): + def install(provider, responses): + client = FakeClient(responses) + + def constructor(**kwargs): + assert kwargs['max_retries'] == 4 + return client + + module = SimpleNamespace( + APIError=RuntimeError, + **{'Anthropic' if provider == 'anthropic' else 'OpenAI': constructor}, + ) + monkeypatch.setitem(sys.modules, provider, module) + return client + + return install + + +def anthropic_response(content, **kwargs): + return { + 'usage': {'input_tokens': 100, 'output_tokens': 10}, + 'content': content, + 'stop_reason': 'end_turn', + **kwargs, + } + + +def openai_response(output, **kwargs): + return { + 'id': 'response-1', + 'usage': { + 'input_tokens': 100, + 'output_tokens': 10, + 'input_tokens_details': {'cached_tokens': 20}, + }, + 'output': output, + 'status': 'completed', + **kwargs, + } + + +def test_anthropic_real_loop_tracks_actions_and_usage(sdk): + client = sdk( + 'anthropic', + [ + anthropic_response( + [ + { + 'type': 'tool_use', + 'id': 'call-1', + 'name': 'left_click', + 'input': {'coordinate': [10, 20]}, + } + ] + ), + anthropic_response([{'type': 'text', 'text': 'Finished'}]), + ], + ) + session = FakeSession() + result = AnthropicComputerAgent('test-model').run( + session, session.observation, task='Do it', budget=Budget() + ) + assert result.termination_reason == 'completed' + assert result.final_text == 'Finished' + assert result.turns == 2 + assert result.usage.input_tokens == 200 + assert result.cost.usd is None + assert [action.action_type for _, action in result.trajectory.steps] == ['click'] + assert client.closed + assert 0 < client.requests[0]['timeout'] <= 300 + + +def test_anthropic_bad_action_skips_rest_of_batch_and_reports_error(sdk): + client = sdk( + 'anthropic', + [ + anthropic_response( + [ + {'type': 'tool_use', 'id': 'bad', 'name': 'hold_key', 'input': {'text': 'a'}}, + { + 'type': 'tool_use', + 'id': 'next', + 'name': 'type', + 'input': {'text': 'must not type'}, + }, + ] + ), + anthropic_response([]), + ], + ) + session = FakeSession() + result = AnthropicComputerAgent('test').run( + session, session.observation, task='Task', budget=Budget() + ) + assert [a.action_type for _, a in result.trajectory.steps] == ['screenshot'] + content = client.requests[1]['messages'][-2]['content'] + results = [item for item in content if item['type'] == 'tool_result'] + assert all(item['is_error'] and isinstance(item['content'], str) for item in results) + assert content[-1]['type'] == 'image' + + +def test_openai_real_loop_preserves_response_chain(sdk): + client = sdk( + 'openai', + [ + openai_response( + [ + { + 'type': 'computer_call', + 'call_id': 'call-1', + 'actions': [ + {'type': 'click', 'x': 5, 'y': 6, 'button': 'left'}, + {'type': 'type', 'text': 'hello'}, + ], + } + ] + ), + openai_response( + [{'type': 'message', 'content': [{'type': 'output_text', 'text': 'Finished'}]}] + ), + ], + ) + session = FakeSession() + result = OpenAIComputerAgent('test').run( + session, session.observation, task='Do it', budget=Budget() + ) + assert result.termination_reason == 'completed' + assert result.usage.input_tokens == 160 + assert result.usage.cache_read_tokens == 40 + assert [a.action_type for _, a in result.trajectory.steps] == ['click', 'type'] + assert client.requests[1]['previous_response_id'] == 'response-1' + assert client.requests[1]['input'][0]['call_id'] == 'call-1' + + +def test_openai_safety_checks_block_all_actions(sdk): + sdk( + 'openai', + [ + openai_response( + [ + { + 'type': 'computer_call', + 'call_id': 'call-1', + 'pending_safety_checks': [ + {'id': 'check-1', 'code': 'malicious_instructions'} + ], + 'action': {'type': 'click', 'x': 5, 'y': 6}, + } + ] + ) + ], + ) + session = FakeSession() + result = OpenAIComputerAgent('test').run( + session, session.observation, task='Do it', budget=Budget() + ) + assert result.termination_reason == 'safety_check' + assert result.trajectory.steps == [] + + +@pytest.mark.parametrize( + 'agent,provider,response', + [ + (AnthropicComputerAgent, 'anthropic', anthropic_response([], stop_reason='max_tokens')), + (OpenAIComputerAgent, 'openai', openai_response([], status='incomplete')), + ], +) +def test_incomplete_responses_consume_turn_budget(sdk, agent, provider, response): + client = sdk(provider, [response]) + session = FakeSession() + result = agent('test').run( + session, session.observation, task='Task', budget=Budget(max_turns=1) + ) + assert result.termination_reason == 'turn_budget' + assert len(client.requests) == 1 + + +@pytest.mark.parametrize('agent', [AnthropicComputerAgent, OpenAIComputerAgent]) +def test_unknown_prices_reject_usd_budget_before_sdk_import(agent): + session = FakeSession() + with pytest.raises(ValueError, match='TokenPrices'): + agent('unknown').run( + session, session.observation, task='Task', budget=Budget(max_cost_usd=1) + ) + + +def test_cost_cap_stops_before_applying_returned_actions(sdk): + prices = TokenPrices( + input_per_million=100, + output_per_million=100, + cache_read_per_million=100, + cache_write_per_million=100, + source='test rates', + ) + sdk( + 'openai', + [ + openai_response( + [ + { + 'type': 'computer_call', + 'call_id': 'call', + 'action': {'type': 'click', 'x': 1, 'y': 2}, + } + ] + ) + ], + ) + session = FakeSession() + result = OpenAIComputerAgent('test', prices).run( + session, session.observation, task='Task', budget=Budget(max_cost_usd=0.001) + ) + assert result.termination_reason == 'cost_budget' + assert result.cost.usd == pytest.approx(0.011) + assert not result.trajectory.steps + + +def test_api_failure_cost_unknown_and_no_retry(sdk): + client = sdk('anthropic', [RuntimeError('secret must not appear in result')]) + session = FakeSession() + result = AnthropicComputerAgent('test').run( + session, session.observation, task='Task', budget=Budget() + ) + assert result.termination_reason == 'error' + assert result.cost.usd is None + assert 'secret' not in result.error + assert len(client.requests) == 1 + + +@pytest.mark.parametrize( + 'agent,provider,response', + [ + (AnthropicComputerAgent, 'anthropic', anthropic_response([], usage=None)), + (OpenAIComputerAgent, 'openai', openai_response([], usage={'output_tokens': 1})), + ], +) +def test_missing_provider_usage_never_reports_zero_cost(sdk, agent, provider, response): + sdk(provider, [response]) + session = FakeSession() + result = agent('test').run(session, session.observation, task='Task', budget=Budget()) + assert result.termination_reason == 'error' + assert result.cost.usd is None + assert 'billing unknown' in result.cost.provenance + + +def test_failed_openai_response_cannot_execute_actions(sdk): + sdk( + 'openai', + [ + openai_response( + [ + { + 'type': 'computer_call', + 'call_id': 'call', + 'action': {'type': 'click', 'x': 1, 'y': 2}, + } + ], + status='failed', + ) + ], + ) + session = FakeSession() + result = OpenAIComputerAgent('test').run( + session, session.observation, task='Task', budget=Budget() + ) + assert result.termination_reason == 'error' + assert not result.trajectory.steps + + +def test_deadline_after_response_prevents_tool_execution(sdk, monkeypatch): + clock = [0.0] + monkeypatch.setattr('deepgym.agents.base.time.monotonic', lambda: clock[0]) + client = sdk( + 'openai', + [ + openai_response( + [ + { + 'type': 'computer_call', + 'call_id': 'call', + 'action': {'type': 'click', 'x': 1, 'y': 2}, + } + ] + ) + ], + ) + create = client.create + + def delayed(**kwargs): + clock[0] = 10 + return create(**kwargs) + + client.create = delayed + session = FakeSession() + result = OpenAIComputerAgent('test').run( + session, session.observation, task='Task', budget=Budget(max_seconds=5) + ) + assert result.termination_reason == 'time_budget' + assert not result.trajectory.steps + assert result.usage.input_tokens == 80 + + +@pytest.mark.parametrize( + 'agent,provider,response', + [ + ( + AnthropicComputerAgent, + 'anthropic', + anthropic_response( + [ + { + 'type': 'tool_use', + 'id': 'call', + 'name': 'left_click', + 'input': {'coordinate': [1, 2]}, + } + ] + ), + ), + ( + OpenAIComputerAgent, + 'openai', + openai_response( + [ + { + 'type': 'computer_call', + 'call_id': 'call', + 'action': {'type': 'click', 'x': 1, 'y': 2}, + } + ] + ), + ), + ], +) +@pytest.mark.parametrize('expired', [False, True]) +def test_session_failure_preserves_accounting(sdk, monkeypatch, agent, provider, response, expired): + from deepgym.exceptions import SandboxError, TimeoutError + + clock = [0.0] + monkeypatch.setattr('deepgym.agents.base.time.monotonic', lambda: clock[0]) + client = sdk(provider, [response]) + session = FakeSession() + + def failed_step(action, *, timeout): + session.trajectory.steps.append((session.observation, action)) + session.done = True + if expired: + clock[0] = 10 + raise TimeoutError('sensitive backend context') + raise SandboxError('sensitive backend context') + + session.step = failed_step + prices = TokenPrices( + input_per_million=100, + output_per_million=100, + cache_read_per_million=100, + cache_write_per_million=100, + source='test rates', + ) + result = agent('test', prices).run( + session, session.observation, task='Task', budget=Budget(max_seconds=5) + ) + assert result.termination_reason == ('time_budget' if expired else 'error') + assert result.turns == 1 + assert result.cost.usd == pytest.approx(0.011) + assert result.usage.output_tokens == 10 + assert len(result.trajectory.steps) == 1 + assert result.error == f'Session step failed: {"TimeoutError" if expired else "SandboxError"}' + assert client.closed + + +@pytest.mark.parametrize( + 'agent,provider,response', + [ + ( + AnthropicComputerAgent, + 'anthropic', + anthropic_response( + [ + { + 'type': 'tool_use', + 'id': 'call', + 'name': 'left_click', + 'input': {'coordinate': [1, 2]}, + } + ] + ), + ), + ( + OpenAIComputerAgent, + 'openai', + openai_response( + [ + { + 'type': 'computer_call', + 'call_id': 'call', + 'action': {'type': 'click', 'x': 1, 'y': 2}, + } + ] + ), + ), + ], +) +def test_two_step_session_allows_two_click_turns(sdk, agent, provider, response): + client = sdk(provider, [response, response]) + session = FakeSession(max_steps=2) + result = agent('test').run(session, session.observation, task='Task', budget=Budget()) + assert result.termination_reason == 'session_done' + assert len(client.requests) == 2 + assert result.turns == 2 + assert [action.action_type for _, action in result.trajectory.steps] == ['click', 'click'] + assert len(result.trajectory.step_rewards) == 2 + assert result.trajectory.final_observation.step == 2 + if provider == 'anthropic': + results = client.requests[1]['messages'][2]['content'] + assert results[-1]['content'][-1]['source']['data'] == 'image-1' + else: + assert client.requests[1]['input'][0]['output']['image_url'].endswith('image-1') + + +@pytest.mark.parametrize( + 'agent,provider,response,completed', + [ + ( + AnthropicComputerAgent, + 'anthropic', + anthropic_response( + [ + { + 'type': 'tool_use', + 'id': 'first', + 'name': 'left_click', + 'input': {'coordinate': [1, 2]}, + }, + {'type': 'tool_use', 'id': 'bad', 'name': 'hold_key', 'input': {'text': 'a'}}, + { + 'type': 'tool_use', + 'id': 'skip', + 'name': 'type', + 'input': {'text': 'must not type'}, + }, + ] + ), + anthropic_response([]), + ), + ( + OpenAIComputerAgent, + 'openai', + openai_response( + [ + { + 'type': 'computer_call', + 'call_id': 'call', + 'actions': [ + {'type': 'click', 'x': 1, 'y': 2}, + {'type': 'unsupported'}, + {'type': 'type', 'text': 'must not type'}, + ], + } + ] + ), + openai_response([]), + ), + ], +) +def test_rejected_later_action_reuses_last_successful_observation( + sdk, + agent, + provider, + response, + completed, +): + client = sdk(provider, [response, completed]) + session = FakeSession() + result = agent('test').run(session, session.observation, task='Task', budget=Budget()) + assert result.termination_reason == 'completed' + assert [action.action_type for _, action in result.trajectory.steps] == ['click'] + if provider == 'anthropic': + results = client.requests[1]['messages'][2]['content'] + assert results[0]['content'][-1]['source']['data'] == 'image-1' + assert results[1]['is_error'] and results[2]['is_error'] + assert isinstance(results[1]['content'], str) and isinstance(results[2]['content'], str) + else: + assert client.requests[1]['input'][0]['output']['image_url'].endswith('image-1') + assert 'Action rejected' in client.requests[1]['input'][1]['content'] + + +@pytest.mark.parametrize( + 'agent,provider,response,completed', + [ + ( + AnthropicComputerAgent, + 'anthropic', + anthropic_response( + [ + { + 'type': 'tool_use', + 'id': 'call', + 'name': 'left_click', + 'input': {'coordinate': [1, 2]}, + } + ] + ), + anthropic_response([]), + ), + ( + OpenAIComputerAgent, + 'openai', + openai_response( + [ + { + 'type': 'computer_call', + 'call_id': 'call', + 'action': {'type': 'click', 'x': 1, 'y': 2}, + } + ] + ), + openai_response([]), + ), + ], +) +def test_missing_action_screenshot_gets_one_fresh_observation( + sdk, + agent, + provider, + response, + completed, +): + client = sdk(provider, [response, completed]) + session = FakeSession() + original_step = session.step + + def missing_first_screenshot(action, *, timeout): + result = original_step(action, timeout=timeout) + if result.observation.step == 1: + return StepResult(result.observation.model_copy(update={'screenshot': None}), 0, False) + return result + + session.step = missing_first_screenshot + result = agent('test').run(session, session.observation, task='Task', budget=Budget()) + assert result.termination_reason == 'completed' + assert [action.action_type for _, action in result.trajectory.steps] == ['click', 'screenshot'] + if provider == 'anthropic': + results = client.requests[1]['messages'][2]['content'] + assert results[-1]['content'][-1]['source']['data'] == 'image-2' + else: + assert client.requests[1]['input'][0]['output']['image_url'].endswith('image-2') + + +def test_openai_rejected_batch_gets_one_fresh_observation(sdk): + client = sdk( + 'openai', + [ + openai_response( + [ + { + 'type': 'computer_call', + 'call_id': 'call', + 'actions': [ + {'type': 'unsupported'}, + {'type': 'type', 'text': 'must not type'}, + ], + } + ] + ), + openai_response([]), + ], + ) + session = FakeSession() + result = OpenAIComputerAgent('test').run( + session, session.observation, task='Task', budget=Budget() + ) + assert result.termination_reason == 'completed' + assert [action.action_type for _, action in result.trajectory.steps] == ['screenshot'] + assert client.requests[1]['input'][0]['output']['image_url'].endswith('image-1') + + +def test_screenshot_never_attached_to_an_error_result(sdk): + failing = { + 'type': 'tool_use', + 'id': 'bad', + 'name': 'middle_click', + 'input': {'coordinate': [1, 1]}, + } + client = sdk( + 'anthropic', + [anthropic_response([failing], stop_reason='tool_use'), anthropic_response([])], + ) + session = FakeSession() + AnthropicComputerAgent('test').run(session, session.observation, task='Task', budget=Budget()) + follow_up = client.requests[1]['messages'][2]['content'] + result = follow_up[0] + assert result['type'] == 'tool_result' and result['is_error'] is True + assert isinstance(result['content'], str) + assert follow_up[-1]['type'] == 'image' + + +@pytest.mark.parametrize('effort', ['high', None]) +def test_anthropic_requests_cache_the_prefix_and_set_thinking(sdk, effort): + client = sdk('anthropic', [anthropic_response([])]) + session = FakeSession() + AnthropicComputerAgent('test', effort=effort).run( + session, session.observation, task='Task', budget=Budget() + ) + request = client.requests[0] + assert request['cache_control'] == {'type': 'ephemeral'} + if effort: + assert request['thinking']['type'] == 'adaptive' + assert request['output_config'] == {'effort': 'high'} + else: + assert 'thinking' not in request and 'output_config' not in request diff --git a/deepgym/tests/agents/test_episode.py b/deepgym/tests/agents/test_episode.py new file mode 100644 index 00000000..cfd26921 --- /dev/null +++ b/deepgym/tests/agents/test_episode.py @@ -0,0 +1,66 @@ +"""Exercise the public runner without optional SDKs or remote sandboxes.""" + +import os +import subprocess +import sys +from pathlib import Path + +import pytest + +from deepgym import Action, DeepGym, MultiTurnEnvironment +from deepgym.agents.base import AgentResult, Budget +from deepgym.models import Observation, Trajectory +from deepgym.session import Session + + +class DoneAgent: + def run( + self, session: Session, initial: Observation, *, task: str, budget: Budget + ) -> AgentResult: + assert initial.step == 0 + assert task == 'finish' + assert budget.max_turns == 2 + session.step(Action(action_type='done')) + return AgentResult(trajectory=session.trajectory, termination_reason='completed') + + +def test_public_import_needs_no_provider_sdk() -> None: + source = str(Path(__file__).resolve().parents[2] / 'src') + checked = subprocess.run( + [ + sys.executable, + '-c', + 'import sys; import deepgym.agents; ' + 'assert "anthropic" not in sys.modules; assert "openai" not in sys.modules', + ], + env={**os.environ, 'PYTHONPATH': source, 'PYTHONDONTWRITEBYTECODE': '1'}, + capture_output=True, + text=True, + timeout=10, + ) + assert checked.returncode == 0, checked.stderr + + +def test_harness_runner_returns_agent_result_and_grade() -> None: + result, grade = DeepGym(mode='local').run_episode( + MultiTurnEnvironment(task='finish'), DoneAgent(), budget=Budget(max_turns=2) + ) + assert isinstance(result, AgentResult) + assert result.trajectory.final_observation.done + assert grade.score == 0 + + +def test_legacy_callable_keeps_trajectory_return() -> None: + result, _ = DeepGym(mode='local').run_episode( + MultiTurnEnvironment(task='finish'), lambda obs: Action(action_type='done') + ) + assert isinstance(result, Trajectory) + + +def test_legacy_callable_cannot_silently_ignore_budget() -> None: + with pytest.raises(ValueError, match='budget requires'): + DeepGym(mode='local').run_episode( + MultiTurnEnvironment(task='finish'), + lambda obs: Action(action_type='done'), + budget=Budget(), + ) diff --git a/deepgym/tests/agents/test_live.py b/deepgym/tests/agents/test_live.py new file mode 100644 index 00000000..02e030bb --- /dev/null +++ b/deepgym/tests/agents/test_live.py @@ -0,0 +1,114 @@ +"""Opt-in GUI and CLI roundtrips; default CI needs no credentials or provider SDKs.""" + +import json +import os +from uuid import uuid4 + +import pytest + +_GUI_REQUIRED = ( + 'DAYTONA_API_KEY', + 'ANTHROPIC_API_KEY', + 'ITOP_ADMIN_PASSWORD', + 'DEEPGYM_AGENT_MODEL', +) +pytestmark = pytest.mark.live_agents + + +@pytest.mark.skipif( + os.getenv('DEEPGYM_LIVE_AGENTS') != '1' or not all(os.getenv(key) for key in _GUI_REQUIRED), + reason='set DEEPGYM_LIVE_AGENTS=1 and Daytona/Anthropic keys, iTop password and model', +) +def test_anthropic_itop_snapshot() -> None: + """Create a unique organization through the GUI and verify the live iTop database.""" + from deepgym import DeepGym + from deepgym.agents import AnthropicComputerAgent, Budget + from deepgym.computer_use import ComputerUseEnvironment + + organization = f'DeepGym smoke {uuid4().hex[:12]}' + password = os.environ['ITOP_ADMIN_PASSWORD'] + verifier = f""" +import json +import urllib.parse +import urllib.request +query = {{'operation': 'core/get', 'class': 'Organization', + 'key': "SELECT Organization WHERE name = '{organization}'", 'output_fields': 'name'}} +body = urllib.parse.urlencode({{'auth_user': 'admin', 'auth_pwd': {json.dumps(password)}, + 'json_data': json.dumps(query)}}).encode() +with urllib.request.urlopen('http://127.0.0.1:8000/webservices/rest.php?version=1.3', + body, timeout=60) as response: + result = json.load(response) +found = result.get('code') == 0 and len(result.get('objects') or {{}}) == 1 +print(json.dumps({{'score': float(found), 'passed': found}})) +""" + env = ComputerUseEnvironment( + name='itop-agent-smoke', + snapshot='img-itop-3.3.0-v1', + task=f'Sign in to iTop as admin with password {password}. ' + f'Create a new Organization named {organization}. Then stop.', + root_service_start='bash /opt/probe/start.sh', + setup_script=( + 'setsid nohup chromium --no-first-run --no-default-browser-check ' + '--password-store=basic --disable-gpu --no-sandbox --test-type ' + '--disable-session-crashed-bubble --disable-features=Translate ' + '--user-data-dir="$HOME/.chrome" --start-maximized ' + 'http://localhost:8000/pages/UI.php >/tmp/chrome.log 2>&1 /dev/null' + ), + final_verifier_code=verifier, + max_steps=240, + timeout_per_step=120, + setup_timeout=600, + ) + agent = AnthropicComputerAgent(model=os.environ['DEEPGYM_AGENT_MODEL']) + result, grade = DeepGym(mode='daytona').run_episode( + env, agent, budget=Budget(max_turns=30, max_seconds=300) + ) + assert result.termination_reason == 'completed', result.termination_reason + assert result.usage.output_tokens > 0 + assert grade.passed and grade.score == 1.0, grade.output + + +@pytest.mark.parametrize('provider', ['anthropic', 'openai']) +def test_cli_snapshot_roundtrip(provider: str) -> None: + """Require a model/tool file write and verify it after supervised CLI teardown.""" + key = 'ANTHROPIC_API_KEY' if provider == 'anthropic' else 'OPENAI_API_KEY' + model = 'DEEPGYM_CLAUDE_MODEL' if provider == 'anthropic' else 'DEEPGYM_CODEX_MODEL' + required = ('DAYTONA_API_KEY', 'DEEPGYM_CLI_SNAPSHOT', key, model) + if os.getenv('DEEPGYM_LIVE_CLI_AGENTS') != '1' or not all(os.getenv(name) for name in required): + pytest.skip(f'set DEEPGYM_LIVE_CLI_AGENTS=1 and {", ".join(required)}') + + from deepgym import DeepGym + from deepgym.agents import Budget, ClaudeCodeAgent, CodexAgent + from deepgym.models import MultiTurnEnvironment + + filename = f'cli-smoke-{uuid4().hex}.txt' + content = f'deepgym-roundtrip-{uuid4().hex}' + verifier = f""" +import json +from pathlib import Path +path = Path({filename!r}) +passed = path.is_file() and path.read_text() == {content!r} +print(json.dumps({{'score': float(passed), 'passed': passed}})) +""" + env = MultiTurnEnvironment( + name=f'{provider}-cli-smoke', + snapshot=os.environ['DEEPGYM_CLI_SNAPSHOT'], + cli_provider_egress=True, + allowed_actions=frozenset({'cli'}), + task=f'Create {filename} in your current working directory, containing exactly ' + f'{content!r} with no newline. Use your file-writing or shell tool and make the ' + 'file readable by other users (mode 0644). Then stop.', + final_verifier_code=verifier, + max_steps=1, + timeout_per_step=120, + ) + agent_type = ClaudeCodeAgent if provider == 'anthropic' else CodexAgent + agent = agent_type(model=os.environ[model], allow_model_credentials=True) + result, grade = DeepGym(mode='daytona').run_episode( + env, agent, budget=Budget(max_turns=20, max_seconds=120) + ) + assert result.termination_reason == 'completed', result.error or result.termination_reason + assert result.usage.output_tokens > 0 + assert [action.action_type for _, action in result.trajectory.steps] == ['cli'] + assert grade.passed and grade.score == 1.0, grade.output diff --git a/deepgym/tests/golden/traces/rl.jsonl b/deepgym/tests/golden/traces/rl.jsonl index dfb0371c..9f709bd0 100644 --- a/deepgym/tests/golden/traces/rl.jsonl +++ b/deepgym/tests/golden/traces/rl.jsonl @@ -1,2 +1,2 @@ -{"trace_id":"trace0001","data_source":"write_notes","agent":"react-shell","model":"Qwen/Qwen3-8B","messages":[{"role":"system","content":"You are a shell agent."},{"role":"user","content":"Write hello to notes/out.txt, then finish.\n$ "},{"role":"assistant","content":"mkdir -p notes && echo hello > notes/out.txt","token_ids":[11,12,13],"logprobs":[-0.25,-0.5,-0.125]},{"role":"tool","content":"exit 0"},{"role":"assistant","content":"write notes/extra.txt"},{"role":"user","content":"write_file is not allowed."},{"role":"assistant","content":"done"}],"initial_observation":"$ ","initial_screenshot":"iVBORw0KGgogc2FtcGxlIHNjcmVlbnNob3Q=","reward":1.0,"passed":true,"step_rewards":[0.5,null,0.5],"steps":[{"message_index":2,"action":{"action_type":"bash","content":"mkdir -p notes && echo hello > notes/out.txt","x":null,"y":null,"to_x":null,"to_y":null,"path":null,"amount":1},"observation":"","screenshot":null,"done":false,"exit_code":0,"error":null,"reward":0.5},{"message_index":4,"action":{"action_type":"write_file","content":"x","x":null,"y":null,"to_x":null,"to_y":null,"path":"notes/extra.txt","amount":1},"observation":null,"screenshot":null,"done":null,"exit_code":null,"error":"ActionError: Action 'write_file' not allowed here.","reward":null},{"message_index":6,"action":{"action_type":"done","content":"","x":null,"y":null,"to_x":null,"to_y":null,"path":null,"amount":1},"observation":"","screenshot":null,"done":true,"exit_code":null,"error":null,"reward":0.5}],"has_verdict":true} -{"trace_id":"trace0003","data_source":"write_notes","agent":"react-shell","model":"Qwen/Qwen3-8B","messages":[{"role":"system","content":"You are a shell agent."},{"role":"user","content":"Write hello to notes/out.txt, then finish.\n$ "},{"role":"assistant","content":"mkdir -p notes && echo hello > notes/out.txt","token_ids":[11,12,13],"logprobs":[-0.25,-0.5,-0.125]},{"role":"tool","content":"exit 0"},{"role":"assistant","content":"write notes/extra.txt"},{"role":"user","content":"write_file is not allowed."},{"role":"assistant","content":"done"}],"initial_observation":"$ ","initial_screenshot":"iVBORw0KGgogc2FtcGxlIHNjcmVlbnNob3Q=","reward":null,"passed":null,"step_rewards":[0.5,null,0.5],"steps":[{"message_index":2,"action":{"action_type":"bash","content":"mkdir -p notes && echo hello > notes/out.txt","x":null,"y":null,"to_x":null,"to_y":null,"path":null,"amount":1},"observation":"","screenshot":null,"done":false,"exit_code":0,"error":null,"reward":0.5},{"message_index":4,"action":{"action_type":"write_file","content":"x","x":null,"y":null,"to_x":null,"to_y":null,"path":"notes/extra.txt","amount":1},"observation":null,"screenshot":null,"done":null,"exit_code":null,"error":"ActionError: Action 'write_file' not allowed here.","reward":null},{"message_index":6,"action":{"action_type":"done","content":"","x":null,"y":null,"to_x":null,"to_y":null,"path":null,"amount":1},"observation":"","screenshot":null,"done":true,"exit_code":null,"error":null,"reward":0.5}],"has_verdict":false} +{"trace_id":"trace0001","data_source":"write_notes","agent":"react-shell","model":"Qwen/Qwen3-8B","messages":[{"role":"system","content":"You are a shell agent."},{"role":"user","content":"Write hello to notes/out.txt, then finish.\n$ "},{"role":"assistant","content":"mkdir -p notes && echo hello > notes/out.txt","token_ids":[11,12,13],"logprobs":[-0.25,-0.5,-0.125]},{"role":"tool","content":"exit 0"},{"role":"assistant","content":"write notes/extra.txt"},{"role":"user","content":"write_file is not allowed."},{"role":"assistant","content":"done"}],"initial_observation":"$ ","initial_screenshot":"iVBORw0KGgogc2FtcGxlIHNjcmVlbnNob3Q=","reward":1.0,"passed":true,"step_rewards":[0.5,null,0.5],"steps":[{"message_index":2,"action":{"action_type":"bash","content":"mkdir -p notes && echo hello > notes/out.txt","events":[],"x":null,"y":null,"to_x":null,"to_y":null,"path":null,"amount":1},"observation":"","screenshot":null,"done":false,"exit_code":0,"error":null,"reward":0.5},{"message_index":4,"action":{"action_type":"write_file","content":"x","events":[],"x":null,"y":null,"to_x":null,"to_y":null,"path":"notes/extra.txt","amount":1},"observation":null,"screenshot":null,"done":null,"exit_code":null,"error":"ActionError: Action 'write_file' not allowed here.","reward":null},{"message_index":6,"action":{"action_type":"done","content":"","events":[],"x":null,"y":null,"to_x":null,"to_y":null,"path":null,"amount":1},"observation":"","screenshot":null,"done":true,"exit_code":null,"error":null,"reward":0.5}],"has_verdict":true} +{"trace_id":"trace0003","data_source":"write_notes","agent":"react-shell","model":"Qwen/Qwen3-8B","messages":[{"role":"system","content":"You are a shell agent."},{"role":"user","content":"Write hello to notes/out.txt, then finish.\n$ "},{"role":"assistant","content":"mkdir -p notes && echo hello > notes/out.txt","token_ids":[11,12,13],"logprobs":[-0.25,-0.5,-0.125]},{"role":"tool","content":"exit 0"},{"role":"assistant","content":"write notes/extra.txt"},{"role":"user","content":"write_file is not allowed."},{"role":"assistant","content":"done"}],"initial_observation":"$ ","initial_screenshot":"iVBORw0KGgogc2FtcGxlIHNjcmVlbnNob3Q=","reward":null,"passed":null,"step_rewards":[0.5,null,0.5],"steps":[{"message_index":2,"action":{"action_type":"bash","content":"mkdir -p notes && echo hello > notes/out.txt","events":[],"x":null,"y":null,"to_x":null,"to_y":null,"path":null,"amount":1},"observation":"","screenshot":null,"done":false,"exit_code":0,"error":null,"reward":0.5},{"message_index":4,"action":{"action_type":"write_file","content":"x","events":[],"x":null,"y":null,"to_x":null,"to_y":null,"path":"notes/extra.txt","amount":1},"observation":null,"screenshot":null,"done":null,"exit_code":null,"error":"ActionError: Action 'write_file' not allowed here.","reward":null},{"message_index":6,"action":{"action_type":"done","content":"","events":[],"x":null,"y":null,"to_x":null,"to_y":null,"path":null,"amount":1},"observation":"","screenshot":null,"done":true,"exit_code":null,"error":null,"reward":0.5}],"has_verdict":false} diff --git a/deepgym/tests/golden/traces/summary.jsonl b/deepgym/tests/golden/traces/summary.jsonl index bab6780b..45e7a741 100644 --- a/deepgym/tests/golden/traces/summary.jsonl +++ b/deepgym/tests/golden/traces/summary.jsonl @@ -1,3 +1,3 @@ -{"trace_id": "trace0001", "env_name": "write_notes", "env_variant": 2, "agent": "react-shell", "model": "Qwen/Qwen3-8B", "provider": "local", "status": "complete", "has_verdict": true, "score": 1.0, "passed": true, "steps": 3, "started_at": "2026-10-01T12:00:00+00:00", "trace": {"schema_version": "1.1", "trace_id": "trace0001", "task": "Write hello to notes/out.txt, then finish.", "provenance": {"env": {"name": "write_notes", "variant": 2, "seed": 7, "environment_sha256": "7f142979bcdf5748ac3baa872a5db2a58d701da9bb29b7451d747e8a3777b6f9", "task_sha256": "1bbf96220c257b1281f09efdf074dc9cb437d3f4401d77c008308bfd72fccc28", "verifier_sha256": "5d6ee51bb39cafa8308963062208847333565120aa1e94980872375cd011d158", "step_verifier_sha256": "e4a424cca79d7c48e6fd4bc2827e14523db51284f401d9c0321d43117878860b", "reference_sha256": "a8bad614127cbaaaaa67e14952ae0fc6c0dce71bdff0822d038556b7dfbe3431", "image": "python:3.12-slim@sha256:abababababababababababababababababababababababababababababababab", "image_digest": "sha256:abababababababababababababababababababababababababababababababab", "digest_unavailable": null, "snapshot": null}, "agent": {"name": "react-shell", "version": "1.2.0", "model": "Qwen/Qwen3-8B", "sampling": {"temperature": 0.7, "top_p": 0.95}}, "provider": "local", "deepgym_version": "0.4.1", "deepgym_commit": null, "deepgym_dirty": null, "started_at": "2026-10-01T12:00:00Z"}, "initial_observation": {"content": "$ ", "step": 0, "done": false, "screenshot": "iVBORw0KGgogc2FtcGxlIHNjcmVlbnNob3Q=", "metadata": null}, "steps": [{"index": 1, "action": {"action_type": "bash", "content": "mkdir -p notes && echo hello > notes/out.txt", "x": null, "y": null, "to_x": null, "to_y": null, "path": null, "amount": 1}, "messages": [{"role": "system", "content": "You are a shell agent.", "token_ids": null, "logprobs": null}, {"role": "user", "content": "Write hello to notes/out.txt, then finish.\n$ ", "token_ids": null, "logprobs": null}, {"role": "assistant", "content": "mkdir -p notes && echo hello > notes/out.txt", "token_ids": [11, 12, 13], "logprobs": [-0.25, -0.5, -0.125]}], "observation": {"content": "", "step": 1, "done": false, "screenshot": null, "metadata": {"exit_code": 0}}, "applied": true, "reward": 0.5, "step_verifier": {"score": 0.5}, "error": null, "timestamp": "2026-10-01T12:00:01Z", "duration_ms": 40.0, "exit_code": 0}, {"index": 2, "action": {"action_type": "write_file", "content": "x", "x": null, "y": null, "to_x": null, "to_y": null, "path": "notes/extra.txt", "amount": 1}, "messages": [{"role": "tool", "content": "exit 0", "token_ids": null, "logprobs": null}, {"role": "assistant", "content": "write notes/extra.txt", "token_ids": null, "logprobs": null}], "observation": null, "applied": false, "reward": null, "step_verifier": null, "error": "ActionError: Action 'write_file' not allowed here.", "timestamp": "2026-10-01T12:00:02Z", "duration_ms": 1.0, "exit_code": null}, {"index": 3, "action": {"action_type": "done", "content": "", "x": null, "y": null, "to_x": null, "to_y": null, "path": null, "amount": 1}, "messages": [{"role": "user", "content": "write_file is not allowed.", "token_ids": null, "logprobs": null}, {"role": "assistant", "content": "done", "token_ids": null, "logprobs": null}], "observation": {"content": "", "step": 2, "done": true, "screenshot": null, "metadata": null}, "applied": true, "reward": 0.5, "step_verifier": {"score": 0.5}, "error": null, "timestamp": "2026-10-01T12:00:03Z", "duration_ms": 30.0, "exit_code": null}], "result": {"score": 1.0, "passed": true, "output": "notes/out.txt says hello", "stderr": "", "exit_code": 0, "execution_time_ms": 12.5, "sandbox_id": "local-session", "reward_components": null, "metrics": null, "seed": null, "truncated": false, "error_type": null, "cases": null}, "prior_results": [], "error": null, "finished_at": "2026-10-01T12:00:04Z", "turns": [[1], [2], [3]], "status": "complete", "has_verdict": true}} -{"trace_id": "trace0002", "env_name": "write_notes", "env_variant": 2, "agent": "react-shell", "model": "Qwen/Qwen3-8B", "provider": "local", "status": "incomplete", "has_verdict": false, "score": null, "passed": null, "steps": 1, "started_at": "2026-10-01T12:00:00+00:00", "trace": {"schema_version": "1.1", "trace_id": "trace0002", "task": "Write hello to notes/out.txt, then finish.", "provenance": {"env": {"name": "write_notes", "variant": 2, "seed": 7, "environment_sha256": "7f142979bcdf5748ac3baa872a5db2a58d701da9bb29b7451d747e8a3777b6f9", "task_sha256": "1bbf96220c257b1281f09efdf074dc9cb437d3f4401d77c008308bfd72fccc28", "verifier_sha256": "5d6ee51bb39cafa8308963062208847333565120aa1e94980872375cd011d158", "step_verifier_sha256": "e4a424cca79d7c48e6fd4bc2827e14523db51284f401d9c0321d43117878860b", "reference_sha256": "a8bad614127cbaaaaa67e14952ae0fc6c0dce71bdff0822d038556b7dfbe3431", "image": "python:3.12-slim@sha256:abababababababababababababababababababababababababababababababab", "image_digest": "sha256:abababababababababababababababababababababababababababababababab", "digest_unavailable": null, "snapshot": null}, "agent": {"name": "react-shell", "version": "1.2.0", "model": "Qwen/Qwen3-8B", "sampling": {"temperature": 0.7, "top_p": 0.95}}, "provider": "local", "deepgym_version": "0.4.1", "deepgym_commit": null, "deepgym_dirty": null, "started_at": "2026-10-01T12:00:00Z"}, "initial_observation": {"content": "$ ", "step": 0, "done": false, "screenshot": "iVBORw0KGgogc2FtcGxlIHNjcmVlbnNob3Q=", "metadata": null}, "steps": [{"index": 1, "action": {"action_type": "bash", "content": "mkdir -p notes && echo hello > notes/out.txt", "x": null, "y": null, "to_x": null, "to_y": null, "path": null, "amount": 1}, "messages": [], "observation": {"content": "", "step": 1, "done": false, "screenshot": null, "metadata": {"exit_code": 0}}, "applied": true, "reward": 0.5, "step_verifier": {"score": 0.5}, "error": null, "timestamp": "2026-10-01T12:00:01Z", "duration_ms": 40.0, "exit_code": 0}], "result": null, "prior_results": [], "error": "SandboxError: sandbox vanished", "finished_at": null, "turns": [[1]], "status": "incomplete", "has_verdict": false}} -{"trace_id": "trace0003", "env_name": "write_notes", "env_variant": 2, "agent": "react-shell", "model": "Qwen/Qwen3-8B", "provider": "local", "status": "complete", "has_verdict": false, "score": null, "passed": null, "steps": 3, "started_at": "2026-10-01T12:00:00+00:00", "trace": {"schema_version": "1.1", "trace_id": "trace0003", "task": "Write hello to notes/out.txt, then finish.", "provenance": {"env": {"name": "write_notes", "variant": 2, "seed": 7, "environment_sha256": "7f142979bcdf5748ac3baa872a5db2a58d701da9bb29b7451d747e8a3777b6f9", "task_sha256": "1bbf96220c257b1281f09efdf074dc9cb437d3f4401d77c008308bfd72fccc28", "verifier_sha256": "5d6ee51bb39cafa8308963062208847333565120aa1e94980872375cd011d158", "step_verifier_sha256": "e4a424cca79d7c48e6fd4bc2827e14523db51284f401d9c0321d43117878860b", "reference_sha256": "a8bad614127cbaaaaa67e14952ae0fc6c0dce71bdff0822d038556b7dfbe3431", "image": "python:3.12-slim@sha256:abababababababababababababababababababababababababababababababab", "image_digest": "sha256:abababababababababababababababababababababababababababababababab", "digest_unavailable": null, "snapshot": null}, "agent": {"name": "react-shell", "version": "1.2.0", "model": "Qwen/Qwen3-8B", "sampling": {"temperature": 0.7, "top_p": 0.95}}, "provider": "local", "deepgym_version": "0.4.1", "deepgym_commit": null, "deepgym_dirty": null, "started_at": "2026-10-01T12:00:00Z"}, "initial_observation": {"content": "$ ", "step": 0, "done": false, "screenshot": "iVBORw0KGgogc2FtcGxlIHNjcmVlbnNob3Q=", "metadata": null}, "steps": [{"index": 1, "action": {"action_type": "bash", "content": "mkdir -p notes && echo hello > notes/out.txt", "x": null, "y": null, "to_x": null, "to_y": null, "path": null, "amount": 1}, "messages": [{"role": "system", "content": "You are a shell agent.", "token_ids": null, "logprobs": null}, {"role": "user", "content": "Write hello to notes/out.txt, then finish.\n$ ", "token_ids": null, "logprobs": null}, {"role": "assistant", "content": "mkdir -p notes && echo hello > notes/out.txt", "token_ids": [11, 12, 13], "logprobs": [-0.25, -0.5, -0.125]}], "observation": {"content": "", "step": 1, "done": false, "screenshot": null, "metadata": {"exit_code": 0}}, "applied": true, "reward": 0.5, "step_verifier": {"score": 0.5}, "error": null, "timestamp": "2026-10-01T12:00:01Z", "duration_ms": 40.0, "exit_code": 0}, {"index": 2, "action": {"action_type": "write_file", "content": "x", "x": null, "y": null, "to_x": null, "to_y": null, "path": "notes/extra.txt", "amount": 1}, "messages": [{"role": "tool", "content": "exit 0", "token_ids": null, "logprobs": null}, {"role": "assistant", "content": "write notes/extra.txt", "token_ids": null, "logprobs": null}], "observation": null, "applied": false, "reward": null, "step_verifier": null, "error": "ActionError: Action 'write_file' not allowed here.", "timestamp": "2026-10-01T12:00:02Z", "duration_ms": 1.0, "exit_code": null}, {"index": 3, "action": {"action_type": "done", "content": "", "x": null, "y": null, "to_x": null, "to_y": null, "path": null, "amount": 1}, "messages": [{"role": "user", "content": "write_file is not allowed.", "token_ids": null, "logprobs": null}, {"role": "assistant", "content": "done", "token_ids": null, "logprobs": null}], "observation": {"content": "", "step": 2, "done": true, "screenshot": null, "metadata": null}, "applied": true, "reward": 0.5, "step_verifier": {"score": 0.5}, "error": null, "timestamp": "2026-10-01T12:00:03Z", "duration_ms": 30.0, "exit_code": null}], "result": {"score": 0.0, "passed": false, "output": "notes/out.txt says hello", "stderr": "", "exit_code": 2, "execution_time_ms": 12.5, "sandbox_id": "local-session", "reward_components": null, "metrics": null, "seed": null, "truncated": false, "error_type": "sandbox_error", "cases": null}, "prior_results": [], "error": null, "finished_at": "2026-10-01T12:00:04Z", "turns": [[1], [2], [3]], "status": "complete", "has_verdict": false}} +{"trace_id": "trace0001", "env_name": "write_notes", "env_variant": 2, "agent": "react-shell", "model": "Qwen/Qwen3-8B", "provider": "local", "status": "complete", "has_verdict": true, "score": 1.0, "passed": true, "steps": 3, "started_at": "2026-10-01T12:00:00+00:00", "trace": {"schema_version": "1.1", "trace_id": "trace0001", "task": "Write hello to notes/out.txt, then finish.", "provenance": {"env": {"name": "write_notes", "variant": 2, "seed": 7, "environment_sha256": "7f142979bcdf5748ac3baa872a5db2a58d701da9bb29b7451d747e8a3777b6f9", "task_sha256": "1bbf96220c257b1281f09efdf074dc9cb437d3f4401d77c008308bfd72fccc28", "verifier_sha256": "5d6ee51bb39cafa8308963062208847333565120aa1e94980872375cd011d158", "step_verifier_sha256": "e4a424cca79d7c48e6fd4bc2827e14523db51284f401d9c0321d43117878860b", "reference_sha256": "a8bad614127cbaaaaa67e14952ae0fc6c0dce71bdff0822d038556b7dfbe3431", "image": "python:3.12-slim@sha256:abababababababababababababababababababababababababababababababab", "image_digest": "sha256:abababababababababababababababababababababababababababababababab", "digest_unavailable": null, "snapshot": null}, "agent": {"name": "react-shell", "version": "1.2.0", "model": "Qwen/Qwen3-8B", "sampling": {"temperature": 0.7, "top_p": 0.95}}, "provider": "local", "deepgym_version": "0.4.1", "deepgym_commit": null, "deepgym_dirty": null, "started_at": "2026-10-01T12:00:00Z"}, "initial_observation": {"content": "$ ", "step": 0, "done": false, "screenshot": "iVBORw0KGgogc2FtcGxlIHNjcmVlbnNob3Q=", "metadata": null}, "steps": [{"index": 1, "action": {"action_type": "bash", "content": "mkdir -p notes && echo hello > notes/out.txt", "events": [], "x": null, "y": null, "to_x": null, "to_y": null, "path": null, "amount": 1}, "messages": [{"role": "system", "content": "You are a shell agent.", "token_ids": null, "logprobs": null}, {"role": "user", "content": "Write hello to notes/out.txt, then finish.\n$ ", "token_ids": null, "logprobs": null}, {"role": "assistant", "content": "mkdir -p notes && echo hello > notes/out.txt", "token_ids": [11, 12, 13], "logprobs": [-0.25, -0.5, -0.125]}], "observation": {"content": "", "step": 1, "done": false, "screenshot": null, "metadata": {"exit_code": 0}}, "applied": true, "reward": 0.5, "step_verifier": {"score": 0.5}, "error": null, "timestamp": "2026-10-01T12:00:01Z", "duration_ms": 40.0, "exit_code": 0}, {"index": 2, "action": {"action_type": "write_file", "content": "x", "events": [], "x": null, "y": null, "to_x": null, "to_y": null, "path": "notes/extra.txt", "amount": 1}, "messages": [{"role": "tool", "content": "exit 0", "token_ids": null, "logprobs": null}, {"role": "assistant", "content": "write notes/extra.txt", "token_ids": null, "logprobs": null}], "observation": null, "applied": false, "reward": null, "step_verifier": null, "error": "ActionError: Action 'write_file' not allowed here.", "timestamp": "2026-10-01T12:00:02Z", "duration_ms": 1.0, "exit_code": null}, {"index": 3, "action": {"action_type": "done", "content": "", "events": [], "x": null, "y": null, "to_x": null, "to_y": null, "path": null, "amount": 1}, "messages": [{"role": "user", "content": "write_file is not allowed.", "token_ids": null, "logprobs": null}, {"role": "assistant", "content": "done", "token_ids": null, "logprobs": null}], "observation": {"content": "", "step": 2, "done": true, "screenshot": null, "metadata": null}, "applied": true, "reward": 0.5, "step_verifier": {"score": 0.5}, "error": null, "timestamp": "2026-10-01T12:00:03Z", "duration_ms": 30.0, "exit_code": null}], "result": {"score": 1.0, "passed": true, "output": "notes/out.txt says hello", "stderr": "", "exit_code": 0, "execution_time_ms": 12.5, "sandbox_id": "local-session", "reward_components": null, "metrics": null, "seed": null, "truncated": false, "error_type": null, "cases": null}, "prior_results": [], "error": null, "finished_at": "2026-10-01T12:00:04Z", "turns": [[1], [2], [3]], "status": "complete", "has_verdict": true}} +{"trace_id": "trace0002", "env_name": "write_notes", "env_variant": 2, "agent": "react-shell", "model": "Qwen/Qwen3-8B", "provider": "local", "status": "incomplete", "has_verdict": false, "score": null, "passed": null, "steps": 1, "started_at": "2026-10-01T12:00:00+00:00", "trace": {"schema_version": "1.1", "trace_id": "trace0002", "task": "Write hello to notes/out.txt, then finish.", "provenance": {"env": {"name": "write_notes", "variant": 2, "seed": 7, "environment_sha256": "7f142979bcdf5748ac3baa872a5db2a58d701da9bb29b7451d747e8a3777b6f9", "task_sha256": "1bbf96220c257b1281f09efdf074dc9cb437d3f4401d77c008308bfd72fccc28", "verifier_sha256": "5d6ee51bb39cafa8308963062208847333565120aa1e94980872375cd011d158", "step_verifier_sha256": "e4a424cca79d7c48e6fd4bc2827e14523db51284f401d9c0321d43117878860b", "reference_sha256": "a8bad614127cbaaaaa67e14952ae0fc6c0dce71bdff0822d038556b7dfbe3431", "image": "python:3.12-slim@sha256:abababababababababababababababababababababababababababababababab", "image_digest": "sha256:abababababababababababababababababababababababababababababababab", "digest_unavailable": null, "snapshot": null}, "agent": {"name": "react-shell", "version": "1.2.0", "model": "Qwen/Qwen3-8B", "sampling": {"temperature": 0.7, "top_p": 0.95}}, "provider": "local", "deepgym_version": "0.4.1", "deepgym_commit": null, "deepgym_dirty": null, "started_at": "2026-10-01T12:00:00Z"}, "initial_observation": {"content": "$ ", "step": 0, "done": false, "screenshot": "iVBORw0KGgogc2FtcGxlIHNjcmVlbnNob3Q=", "metadata": null}, "steps": [{"index": 1, "action": {"action_type": "bash", "content": "mkdir -p notes && echo hello > notes/out.txt", "events": [], "x": null, "y": null, "to_x": null, "to_y": null, "path": null, "amount": 1}, "messages": [], "observation": {"content": "", "step": 1, "done": false, "screenshot": null, "metadata": {"exit_code": 0}}, "applied": true, "reward": 0.5, "step_verifier": {"score": 0.5}, "error": null, "timestamp": "2026-10-01T12:00:01Z", "duration_ms": 40.0, "exit_code": 0}], "result": null, "prior_results": [], "error": "SandboxError: sandbox vanished", "finished_at": null, "turns": [[1]], "status": "incomplete", "has_verdict": false}} +{"trace_id": "trace0003", "env_name": "write_notes", "env_variant": 2, "agent": "react-shell", "model": "Qwen/Qwen3-8B", "provider": "local", "status": "complete", "has_verdict": false, "score": null, "passed": null, "steps": 3, "started_at": "2026-10-01T12:00:00+00:00", "trace": {"schema_version": "1.1", "trace_id": "trace0003", "task": "Write hello to notes/out.txt, then finish.", "provenance": {"env": {"name": "write_notes", "variant": 2, "seed": 7, "environment_sha256": "7f142979bcdf5748ac3baa872a5db2a58d701da9bb29b7451d747e8a3777b6f9", "task_sha256": "1bbf96220c257b1281f09efdf074dc9cb437d3f4401d77c008308bfd72fccc28", "verifier_sha256": "5d6ee51bb39cafa8308963062208847333565120aa1e94980872375cd011d158", "step_verifier_sha256": "e4a424cca79d7c48e6fd4bc2827e14523db51284f401d9c0321d43117878860b", "reference_sha256": "a8bad614127cbaaaaa67e14952ae0fc6c0dce71bdff0822d038556b7dfbe3431", "image": "python:3.12-slim@sha256:abababababababababababababababababababababababababababababababab", "image_digest": "sha256:abababababababababababababababababababababababababababababababab", "digest_unavailable": null, "snapshot": null}, "agent": {"name": "react-shell", "version": "1.2.0", "model": "Qwen/Qwen3-8B", "sampling": {"temperature": 0.7, "top_p": 0.95}}, "provider": "local", "deepgym_version": "0.4.1", "deepgym_commit": null, "deepgym_dirty": null, "started_at": "2026-10-01T12:00:00Z"}, "initial_observation": {"content": "$ ", "step": 0, "done": false, "screenshot": "iVBORw0KGgogc2FtcGxlIHNjcmVlbnNob3Q=", "metadata": null}, "steps": [{"index": 1, "action": {"action_type": "bash", "content": "mkdir -p notes && echo hello > notes/out.txt", "events": [], "x": null, "y": null, "to_x": null, "to_y": null, "path": null, "amount": 1}, "messages": [{"role": "system", "content": "You are a shell agent.", "token_ids": null, "logprobs": null}, {"role": "user", "content": "Write hello to notes/out.txt, then finish.\n$ ", "token_ids": null, "logprobs": null}, {"role": "assistant", "content": "mkdir -p notes && echo hello > notes/out.txt", "token_ids": [11, 12, 13], "logprobs": [-0.25, -0.5, -0.125]}], "observation": {"content": "", "step": 1, "done": false, "screenshot": null, "metadata": {"exit_code": 0}}, "applied": true, "reward": 0.5, "step_verifier": {"score": 0.5}, "error": null, "timestamp": "2026-10-01T12:00:01Z", "duration_ms": 40.0, "exit_code": 0}, {"index": 2, "action": {"action_type": "write_file", "content": "x", "events": [], "x": null, "y": null, "to_x": null, "to_y": null, "path": "notes/extra.txt", "amount": 1}, "messages": [{"role": "tool", "content": "exit 0", "token_ids": null, "logprobs": null}, {"role": "assistant", "content": "write notes/extra.txt", "token_ids": null, "logprobs": null}], "observation": null, "applied": false, "reward": null, "step_verifier": null, "error": "ActionError: Action 'write_file' not allowed here.", "timestamp": "2026-10-01T12:00:02Z", "duration_ms": 1.0, "exit_code": null}, {"index": 3, "action": {"action_type": "done", "content": "", "events": [], "x": null, "y": null, "to_x": null, "to_y": null, "path": null, "amount": 1}, "messages": [{"role": "user", "content": "write_file is not allowed.", "token_ids": null, "logprobs": null}, {"role": "assistant", "content": "done", "token_ids": null, "logprobs": null}], "observation": {"content": "", "step": 2, "done": true, "screenshot": null, "metadata": null}, "applied": true, "reward": 0.5, "step_verifier": {"score": 0.5}, "error": null, "timestamp": "2026-10-01T12:00:03Z", "duration_ms": 30.0, "exit_code": null}], "result": {"score": 0.0, "passed": false, "output": "notes/out.txt says hello", "stderr": "", "exit_code": 2, "execution_time_ms": 12.5, "sandbox_id": "local-session", "reward_components": null, "metrics": null, "seed": null, "truncated": false, "error_type": "sandbox_error", "cases": null}, "prior_results": [], "error": null, "finished_at": "2026-10-01T12:00:04Z", "turns": [[1], [2], [3]], "status": "complete", "has_verdict": false}} diff --git a/deepgym/tests/golden/traces/traces.jsonl b/deepgym/tests/golden/traces/traces.jsonl index b5dbeaad..b9a6e2da 100644 --- a/deepgym/tests/golden/traces/traces.jsonl +++ b/deepgym/tests/golden/traces/traces.jsonl @@ -1,3 +1,3 @@ -{"schema_version":"1.1","trace_id":"trace0001","task":"Write hello to notes/out.txt, then finish.","provenance":{"env":{"name":"write_notes","variant":2,"seed":7,"environment_sha256":"7f142979bcdf5748ac3baa872a5db2a58d701da9bb29b7451d747e8a3777b6f9","task_sha256":"1bbf96220c257b1281f09efdf074dc9cb437d3f4401d77c008308bfd72fccc28","verifier_sha256":"5d6ee51bb39cafa8308963062208847333565120aa1e94980872375cd011d158","step_verifier_sha256":"e4a424cca79d7c48e6fd4bc2827e14523db51284f401d9c0321d43117878860b","reference_sha256":"a8bad614127cbaaaaa67e14952ae0fc6c0dce71bdff0822d038556b7dfbe3431","image":"python:3.12-slim@sha256:abababababababababababababababababababababababababababababababab","image_digest":"sha256:abababababababababababababababababababababababababababababababab","digest_unavailable":null,"snapshot":null},"agent":{"name":"react-shell","version":"1.2.0","model":"Qwen/Qwen3-8B","sampling":{"temperature":0.7,"top_p":0.95}},"provider":"local","deepgym_version":"0.4.1","deepgym_commit":null,"deepgym_dirty":null,"started_at":"2026-10-01T12:00:00Z"},"initial_observation":{"content":"$ ","step":0,"done":false,"screenshot":"iVBORw0KGgogc2FtcGxlIHNjcmVlbnNob3Q=","metadata":null},"steps":[{"index":1,"action":{"action_type":"bash","content":"mkdir -p notes && echo hello > notes/out.txt","x":null,"y":null,"to_x":null,"to_y":null,"path":null,"amount":1},"messages":[{"role":"system","content":"You are a shell agent.","token_ids":null,"logprobs":null},{"role":"user","content":"Write hello to notes/out.txt, then finish.\n$ ","token_ids":null,"logprobs":null},{"role":"assistant","content":"mkdir -p notes && echo hello > notes/out.txt","token_ids":[11,12,13],"logprobs":[-0.25,-0.5,-0.125]}],"observation":{"content":"","step":1,"done":false,"screenshot":null,"metadata":{"exit_code":0}},"applied":true,"reward":0.5,"step_verifier":{"score":0.5},"error":null,"timestamp":"2026-10-01T12:00:01Z","duration_ms":40.0,"exit_code":0},{"index":2,"action":{"action_type":"write_file","content":"x","x":null,"y":null,"to_x":null,"to_y":null,"path":"notes/extra.txt","amount":1},"messages":[{"role":"tool","content":"exit 0","token_ids":null,"logprobs":null},{"role":"assistant","content":"write notes/extra.txt","token_ids":null,"logprobs":null}],"observation":null,"applied":false,"reward":null,"step_verifier":null,"error":"ActionError: Action 'write_file' not allowed here.","timestamp":"2026-10-01T12:00:02Z","duration_ms":1.0,"exit_code":null},{"index":3,"action":{"action_type":"done","content":"","x":null,"y":null,"to_x":null,"to_y":null,"path":null,"amount":1},"messages":[{"role":"user","content":"write_file is not allowed.","token_ids":null,"logprobs":null},{"role":"assistant","content":"done","token_ids":null,"logprobs":null}],"observation":{"content":"","step":2,"done":true,"screenshot":null,"metadata":null},"applied":true,"reward":0.5,"step_verifier":{"score":0.5},"error":null,"timestamp":"2026-10-01T12:00:03Z","duration_ms":30.0,"exit_code":null}],"result":{"score":1.0,"passed":true,"output":"notes/out.txt says hello","stderr":"","exit_code":0,"execution_time_ms":12.5,"sandbox_id":"local-session","reward_components":null,"metrics":null,"seed":null,"truncated":false,"error_type":null,"cases":null},"prior_results":[],"error":null,"finished_at":"2026-10-01T12:00:04Z","turns":[[1],[2],[3]],"status":"complete","has_verdict":true} -{"schema_version":"1.1","trace_id":"trace0002","task":"Write hello to notes/out.txt, then finish.","provenance":{"env":{"name":"write_notes","variant":2,"seed":7,"environment_sha256":"7f142979bcdf5748ac3baa872a5db2a58d701da9bb29b7451d747e8a3777b6f9","task_sha256":"1bbf96220c257b1281f09efdf074dc9cb437d3f4401d77c008308bfd72fccc28","verifier_sha256":"5d6ee51bb39cafa8308963062208847333565120aa1e94980872375cd011d158","step_verifier_sha256":"e4a424cca79d7c48e6fd4bc2827e14523db51284f401d9c0321d43117878860b","reference_sha256":"a8bad614127cbaaaaa67e14952ae0fc6c0dce71bdff0822d038556b7dfbe3431","image":"python:3.12-slim@sha256:abababababababababababababababababababababababababababababababab","image_digest":"sha256:abababababababababababababababababababababababababababababababab","digest_unavailable":null,"snapshot":null},"agent":{"name":"react-shell","version":"1.2.0","model":"Qwen/Qwen3-8B","sampling":{"temperature":0.7,"top_p":0.95}},"provider":"local","deepgym_version":"0.4.1","deepgym_commit":null,"deepgym_dirty":null,"started_at":"2026-10-01T12:00:00Z"},"initial_observation":{"content":"$ ","step":0,"done":false,"screenshot":"iVBORw0KGgogc2FtcGxlIHNjcmVlbnNob3Q=","metadata":null},"steps":[{"index":1,"action":{"action_type":"bash","content":"mkdir -p notes && echo hello > notes/out.txt","x":null,"y":null,"to_x":null,"to_y":null,"path":null,"amount":1},"messages":[],"observation":{"content":"","step":1,"done":false,"screenshot":null,"metadata":{"exit_code":0}},"applied":true,"reward":0.5,"step_verifier":{"score":0.5},"error":null,"timestamp":"2026-10-01T12:00:01Z","duration_ms":40.0,"exit_code":0}],"result":null,"prior_results":[],"error":"SandboxError: sandbox vanished","finished_at":null,"turns":[[1]],"status":"incomplete","has_verdict":false} -{"schema_version":"1.1","trace_id":"trace0003","task":"Write hello to notes/out.txt, then finish.","provenance":{"env":{"name":"write_notes","variant":2,"seed":7,"environment_sha256":"7f142979bcdf5748ac3baa872a5db2a58d701da9bb29b7451d747e8a3777b6f9","task_sha256":"1bbf96220c257b1281f09efdf074dc9cb437d3f4401d77c008308bfd72fccc28","verifier_sha256":"5d6ee51bb39cafa8308963062208847333565120aa1e94980872375cd011d158","step_verifier_sha256":"e4a424cca79d7c48e6fd4bc2827e14523db51284f401d9c0321d43117878860b","reference_sha256":"a8bad614127cbaaaaa67e14952ae0fc6c0dce71bdff0822d038556b7dfbe3431","image":"python:3.12-slim@sha256:abababababababababababababababababababababababababababababababab","image_digest":"sha256:abababababababababababababababababababababababababababababababab","digest_unavailable":null,"snapshot":null},"agent":{"name":"react-shell","version":"1.2.0","model":"Qwen/Qwen3-8B","sampling":{"temperature":0.7,"top_p":0.95}},"provider":"local","deepgym_version":"0.4.1","deepgym_commit":null,"deepgym_dirty":null,"started_at":"2026-10-01T12:00:00Z"},"initial_observation":{"content":"$ ","step":0,"done":false,"screenshot":"iVBORw0KGgogc2FtcGxlIHNjcmVlbnNob3Q=","metadata":null},"steps":[{"index":1,"action":{"action_type":"bash","content":"mkdir -p notes && echo hello > notes/out.txt","x":null,"y":null,"to_x":null,"to_y":null,"path":null,"amount":1},"messages":[{"role":"system","content":"You are a shell agent.","token_ids":null,"logprobs":null},{"role":"user","content":"Write hello to notes/out.txt, then finish.\n$ ","token_ids":null,"logprobs":null},{"role":"assistant","content":"mkdir -p notes && echo hello > notes/out.txt","token_ids":[11,12,13],"logprobs":[-0.25,-0.5,-0.125]}],"observation":{"content":"","step":1,"done":false,"screenshot":null,"metadata":{"exit_code":0}},"applied":true,"reward":0.5,"step_verifier":{"score":0.5},"error":null,"timestamp":"2026-10-01T12:00:01Z","duration_ms":40.0,"exit_code":0},{"index":2,"action":{"action_type":"write_file","content":"x","x":null,"y":null,"to_x":null,"to_y":null,"path":"notes/extra.txt","amount":1},"messages":[{"role":"tool","content":"exit 0","token_ids":null,"logprobs":null},{"role":"assistant","content":"write notes/extra.txt","token_ids":null,"logprobs":null}],"observation":null,"applied":false,"reward":null,"step_verifier":null,"error":"ActionError: Action 'write_file' not allowed here.","timestamp":"2026-10-01T12:00:02Z","duration_ms":1.0,"exit_code":null},{"index":3,"action":{"action_type":"done","content":"","x":null,"y":null,"to_x":null,"to_y":null,"path":null,"amount":1},"messages":[{"role":"user","content":"write_file is not allowed.","token_ids":null,"logprobs":null},{"role":"assistant","content":"done","token_ids":null,"logprobs":null}],"observation":{"content":"","step":2,"done":true,"screenshot":null,"metadata":null},"applied":true,"reward":0.5,"step_verifier":{"score":0.5},"error":null,"timestamp":"2026-10-01T12:00:03Z","duration_ms":30.0,"exit_code":null}],"result":{"score":0.0,"passed":false,"output":"notes/out.txt says hello","stderr":"","exit_code":2,"execution_time_ms":12.5,"sandbox_id":"local-session","reward_components":null,"metrics":null,"seed":null,"truncated":false,"error_type":"sandbox_error","cases":null},"prior_results":[],"error":null,"finished_at":"2026-10-01T12:00:04Z","turns":[[1],[2],[3]],"status":"complete","has_verdict":false} +{"schema_version":"1.1","trace_id":"trace0001","task":"Write hello to notes/out.txt, then finish.","provenance":{"env":{"name":"write_notes","variant":2,"seed":7,"environment_sha256":"7f142979bcdf5748ac3baa872a5db2a58d701da9bb29b7451d747e8a3777b6f9","task_sha256":"1bbf96220c257b1281f09efdf074dc9cb437d3f4401d77c008308bfd72fccc28","verifier_sha256":"5d6ee51bb39cafa8308963062208847333565120aa1e94980872375cd011d158","step_verifier_sha256":"e4a424cca79d7c48e6fd4bc2827e14523db51284f401d9c0321d43117878860b","reference_sha256":"a8bad614127cbaaaaa67e14952ae0fc6c0dce71bdff0822d038556b7dfbe3431","image":"python:3.12-slim@sha256:abababababababababababababababababababababababababababababababab","image_digest":"sha256:abababababababababababababababababababababababababababababababab","digest_unavailable":null,"snapshot":null},"agent":{"name":"react-shell","version":"1.2.0","model":"Qwen/Qwen3-8B","sampling":{"temperature":0.7,"top_p":0.95}},"provider":"local","deepgym_version":"0.4.1","deepgym_commit":null,"deepgym_dirty":null,"started_at":"2026-10-01T12:00:00Z"},"initial_observation":{"content":"$ ","step":0,"done":false,"screenshot":"iVBORw0KGgogc2FtcGxlIHNjcmVlbnNob3Q=","metadata":null},"steps":[{"index":1,"action":{"action_type":"bash","content":"mkdir -p notes && echo hello > notes/out.txt","events":[],"x":null,"y":null,"to_x":null,"to_y":null,"path":null,"amount":1},"messages":[{"role":"system","content":"You are a shell agent.","token_ids":null,"logprobs":null},{"role":"user","content":"Write hello to notes/out.txt, then finish.\n$ ","token_ids":null,"logprobs":null},{"role":"assistant","content":"mkdir -p notes && echo hello > notes/out.txt","token_ids":[11,12,13],"logprobs":[-0.25,-0.5,-0.125]}],"observation":{"content":"","step":1,"done":false,"screenshot":null,"metadata":{"exit_code":0}},"applied":true,"reward":0.5,"step_verifier":{"score":0.5},"error":null,"timestamp":"2026-10-01T12:00:01Z","duration_ms":40.0,"exit_code":0},{"index":2,"action":{"action_type":"write_file","content":"x","events":[],"x":null,"y":null,"to_x":null,"to_y":null,"path":"notes/extra.txt","amount":1},"messages":[{"role":"tool","content":"exit 0","token_ids":null,"logprobs":null},{"role":"assistant","content":"write notes/extra.txt","token_ids":null,"logprobs":null}],"observation":null,"applied":false,"reward":null,"step_verifier":null,"error":"ActionError: Action 'write_file' not allowed here.","timestamp":"2026-10-01T12:00:02Z","duration_ms":1.0,"exit_code":null},{"index":3,"action":{"action_type":"done","content":"","events":[],"x":null,"y":null,"to_x":null,"to_y":null,"path":null,"amount":1},"messages":[{"role":"user","content":"write_file is not allowed.","token_ids":null,"logprobs":null},{"role":"assistant","content":"done","token_ids":null,"logprobs":null}],"observation":{"content":"","step":2,"done":true,"screenshot":null,"metadata":null},"applied":true,"reward":0.5,"step_verifier":{"score":0.5},"error":null,"timestamp":"2026-10-01T12:00:03Z","duration_ms":30.0,"exit_code":null}],"result":{"score":1.0,"passed":true,"output":"notes/out.txt says hello","stderr":"","exit_code":0,"execution_time_ms":12.5,"sandbox_id":"local-session","reward_components":null,"metrics":null,"seed":null,"truncated":false,"error_type":null,"cases":null},"prior_results":[],"error":null,"finished_at":"2026-10-01T12:00:04Z","turns":[[1],[2],[3]],"status":"complete","has_verdict":true} +{"schema_version":"1.1","trace_id":"trace0002","task":"Write hello to notes/out.txt, then finish.","provenance":{"env":{"name":"write_notes","variant":2,"seed":7,"environment_sha256":"7f142979bcdf5748ac3baa872a5db2a58d701da9bb29b7451d747e8a3777b6f9","task_sha256":"1bbf96220c257b1281f09efdf074dc9cb437d3f4401d77c008308bfd72fccc28","verifier_sha256":"5d6ee51bb39cafa8308963062208847333565120aa1e94980872375cd011d158","step_verifier_sha256":"e4a424cca79d7c48e6fd4bc2827e14523db51284f401d9c0321d43117878860b","reference_sha256":"a8bad614127cbaaaaa67e14952ae0fc6c0dce71bdff0822d038556b7dfbe3431","image":"python:3.12-slim@sha256:abababababababababababababababababababababababababababababababab","image_digest":"sha256:abababababababababababababababababababababababababababababababab","digest_unavailable":null,"snapshot":null},"agent":{"name":"react-shell","version":"1.2.0","model":"Qwen/Qwen3-8B","sampling":{"temperature":0.7,"top_p":0.95}},"provider":"local","deepgym_version":"0.4.1","deepgym_commit":null,"deepgym_dirty":null,"started_at":"2026-10-01T12:00:00Z"},"initial_observation":{"content":"$ ","step":0,"done":false,"screenshot":"iVBORw0KGgogc2FtcGxlIHNjcmVlbnNob3Q=","metadata":null},"steps":[{"index":1,"action":{"action_type":"bash","content":"mkdir -p notes && echo hello > notes/out.txt","events":[],"x":null,"y":null,"to_x":null,"to_y":null,"path":null,"amount":1},"messages":[],"observation":{"content":"","step":1,"done":false,"screenshot":null,"metadata":{"exit_code":0}},"applied":true,"reward":0.5,"step_verifier":{"score":0.5},"error":null,"timestamp":"2026-10-01T12:00:01Z","duration_ms":40.0,"exit_code":0}],"result":null,"prior_results":[],"error":"SandboxError: sandbox vanished","finished_at":null,"turns":[[1]],"status":"incomplete","has_verdict":false} +{"schema_version":"1.1","trace_id":"trace0003","task":"Write hello to notes/out.txt, then finish.","provenance":{"env":{"name":"write_notes","variant":2,"seed":7,"environment_sha256":"7f142979bcdf5748ac3baa872a5db2a58d701da9bb29b7451d747e8a3777b6f9","task_sha256":"1bbf96220c257b1281f09efdf074dc9cb437d3f4401d77c008308bfd72fccc28","verifier_sha256":"5d6ee51bb39cafa8308963062208847333565120aa1e94980872375cd011d158","step_verifier_sha256":"e4a424cca79d7c48e6fd4bc2827e14523db51284f401d9c0321d43117878860b","reference_sha256":"a8bad614127cbaaaaa67e14952ae0fc6c0dce71bdff0822d038556b7dfbe3431","image":"python:3.12-slim@sha256:abababababababababababababababababababababababababababababababab","image_digest":"sha256:abababababababababababababababababababababababababababababababab","digest_unavailable":null,"snapshot":null},"agent":{"name":"react-shell","version":"1.2.0","model":"Qwen/Qwen3-8B","sampling":{"temperature":0.7,"top_p":0.95}},"provider":"local","deepgym_version":"0.4.1","deepgym_commit":null,"deepgym_dirty":null,"started_at":"2026-10-01T12:00:00Z"},"initial_observation":{"content":"$ ","step":0,"done":false,"screenshot":"iVBORw0KGgogc2FtcGxlIHNjcmVlbnNob3Q=","metadata":null},"steps":[{"index":1,"action":{"action_type":"bash","content":"mkdir -p notes && echo hello > notes/out.txt","events":[],"x":null,"y":null,"to_x":null,"to_y":null,"path":null,"amount":1},"messages":[{"role":"system","content":"You are a shell agent.","token_ids":null,"logprobs":null},{"role":"user","content":"Write hello to notes/out.txt, then finish.\n$ ","token_ids":null,"logprobs":null},{"role":"assistant","content":"mkdir -p notes && echo hello > notes/out.txt","token_ids":[11,12,13],"logprobs":[-0.25,-0.5,-0.125]}],"observation":{"content":"","step":1,"done":false,"screenshot":null,"metadata":{"exit_code":0}},"applied":true,"reward":0.5,"step_verifier":{"score":0.5},"error":null,"timestamp":"2026-10-01T12:00:01Z","duration_ms":40.0,"exit_code":0},{"index":2,"action":{"action_type":"write_file","content":"x","events":[],"x":null,"y":null,"to_x":null,"to_y":null,"path":"notes/extra.txt","amount":1},"messages":[{"role":"tool","content":"exit 0","token_ids":null,"logprobs":null},{"role":"assistant","content":"write notes/extra.txt","token_ids":null,"logprobs":null}],"observation":null,"applied":false,"reward":null,"step_verifier":null,"error":"ActionError: Action 'write_file' not allowed here.","timestamp":"2026-10-01T12:00:02Z","duration_ms":1.0,"exit_code":null},{"index":3,"action":{"action_type":"done","content":"","events":[],"x":null,"y":null,"to_x":null,"to_y":null,"path":null,"amount":1},"messages":[{"role":"user","content":"write_file is not allowed.","token_ids":null,"logprobs":null},{"role":"assistant","content":"done","token_ids":null,"logprobs":null}],"observation":{"content":"","step":2,"done":true,"screenshot":null,"metadata":null},"applied":true,"reward":0.5,"step_verifier":{"score":0.5},"error":null,"timestamp":"2026-10-01T12:00:03Z","duration_ms":30.0,"exit_code":null}],"result":{"score":0.0,"passed":false,"output":"notes/out.txt says hello","stderr":"","exit_code":2,"execution_time_ms":12.5,"sandbox_id":"local-session","reward_components":null,"metrics":null,"seed":null,"truncated":false,"error_type":"sandbox_error","cases":null},"prior_results":[],"error":null,"finished_at":"2026-10-01T12:00:04Z","turns":[[1],[2],[3]],"status":"complete","has_verdict":false} diff --git a/deepgym/tests/test_api_schemas.py b/deepgym/tests/test_api_schemas.py new file mode 100644 index 00000000..05cba126 --- /dev/null +++ b/deepgym/tests/test_api_schemas.py @@ -0,0 +1,61 @@ +"""Reject privileged inline session configuration before creating a backend.""" + +import pytest +from pydantic import ValidationError + +from deepgym.api.schemas import CreateSessionRequest +from deepgym.computer_use import ComputerUseEnvironment +from deepgym.models import MultiTurnEnvironment + + +@pytest.mark.parametrize('environment_type', ['multi-turn', 'computer-use']) +@pytest.mark.parametrize('field', ['image', 'snapshot', 'root_service_start']) +def test_inline_session_rejects_privileged_configuration( + environment_type: str, + field: str, +) -> None: + with pytest.raises(ValidationError, match=field): + CreateSessionRequest.model_validate( + { + 'environment': {'task': 't', 'type': environment_type, field: 'untrusted'}, + } + ) + + +@pytest.mark.parametrize('model', [MultiTurnEnvironment, ComputerUseEnvironment]) +def test_inline_environment_instances_cannot_bypass_root_start_rejection( + model: type[MultiTurnEnvironment], +) -> None: + environment = model(task='t', root_service_start='start-trusted-service') + with pytest.raises(ValidationError, match='root_service_start'): + CreateSessionRequest(environment=environment) + + +def test_empty_inline_root_start_remains_allowed() -> None: + request = CreateSessionRequest.model_validate( + { + 'environment': {'task': 't', 'root_service_start': ''}, + } + ) + assert isinstance(request.environment, MultiTurnEnvironment) + assert request.environment.root_service_start == '' + + +def test_trusted_registry_reference_remains_allowed() -> None: + request = CreateSessionRequest(environment='trusted-vendor-snapshot', variant=2) + assert request.environment == 'trusted-vendor-snapshot' + assert request.variant == 2 + + +def test_trusted_environment_can_still_specify_root_start() -> None: + environment = MultiTurnEnvironment(task='t', root_service_start='start-trusted-service') + assert environment.root_service_start == 'start-trusted-service' + + +@pytest.mark.parametrize('as_model', [False, True]) +def test_inline_api_cannot_open_trusted_root_egress(as_model: bool) -> None: + environment = {'task': 'task', 'cli_provider_egress': True} + if as_model: + environment = MultiTurnEnvironment(**environment) + with pytest.raises(ValidationError, match='cli_provider_egress cannot be set over the API'): + CreateSessionRequest(environment=environment) diff --git a/deepgym/tests/test_cli_runtime.py b/deepgym/tests/test_cli_runtime.py new file mode 100644 index 00000000..11823d00 --- /dev/null +++ b/deepgym/tests/test_cli_runtime.py @@ -0,0 +1,557 @@ +"""Offline checks for CLI supervision, retained isolation and root service ordering.""" + +import base64 +import json +import socket +from types import SimpleNamespace +from urllib.request import proxy_bypass_environment + +import pytest +from pydantic import ValidationError + +from deepgym import _cli_worker, cli_runtime +from deepgym.cli_runtime import CliExecution, CliRequest +from deepgym.exceptions import ActionError, SandboxError +from deepgym.models import Action, MultiTurnEnvironment +from deepgym.sandboxes import ExecResult +from deepgym.session import LocalBackend, SandboxBackend, Session +from tests.test_sandboxes import _USERS, FakeSandbox + + +def request(**changes: object) -> CliRequest: + return CliRequest( + **{ + 'argv': ('claude', '-p', 'task'), + 'env': {'ANTHROPIC_API_KEY': 'test-key'}, + 'provider': 'anthropic', + 'timeout': 5, + 'allow_model_credentials': True, + **changes, + } + ) + + +def test_cli_refuses_unacknowledged_key_exposure() -> None: + backend = SandboxBackend(FakeSandbox(False, [_USERS])) + with pytest.raises(SandboxError, match='model-readable'): + backend.run_cli(request(allow_model_credentials=False), lambda event: None) + + +def test_cli_refuses_to_remove_namespace_for_provider_egress() -> None: + backend = SandboxBackend(FakeSandbox(True, [_USERS])) + with pytest.raises(SandboxError, match='retained isolated namespace'): + backend.run_cli(request(), lambda event: None) + + +@pytest.mark.parametrize( + 'env', + [ + {'ANTHROPIC_API_KEY': 'key', 'LD_PRELOAD': '/tmp/evil.so'}, + {'OPENAI_API_KEY': 'key'}, + {'ANTHROPIC_API_KEY': ''}, + ], +) +def test_only_selected_provider_credential_is_injected(env: dict[str, str]) -> None: + with pytest.raises(ValidationError, match='only a non-empty'): + request(env=env) + + +@pytest.mark.parametrize( + 'address', ['127.0.0.1', '169.254.169.254', '10.0.0.1', '::1', '224.0.0.1'] +) +def test_provider_dns_cannot_open_toolbox_or_metadata(monkeypatch, address: str) -> None: + monkeypatch.setattr( + socket, + 'getaddrinfo', + lambda *args, **kwargs: [(socket.AF_INET, socket.SOCK_STREAM, 6, '', (address, 443))], + ) + with pytest.raises(ValueError, match='non-public'): + _cli_worker.public_addresses('api.anthropic.com') + + +def test_provider_connect_uses_the_validated_numeric_address(monkeypatch) -> None: + monkeypatch.setattr( + socket, + 'getaddrinfo', + lambda *args, **kwargs: [(socket.AF_INET, socket.SOCK_STREAM, 6, '', ('1.1.1.1', 443))], + ) + assert _cli_worker.public_addresses('api.anthropic.com') == [(socket.AF_INET, ('1.1.1.1', 443))] + + +@pytest.mark.parametrize('target', ['127.0.0.1:3000', 'api.openai.com:443', 'api.anthropic.com:80']) +def test_connect_rejects_every_other_destination(target: str) -> None: + class Connection: + def __init__(self) -> None: + self.incoming = bytearray(f'CONNECT {target} HTTP/1.1\r\n\r\n'.encode()) + self.sent = b'' + + def settimeout(self, timeout: float) -> None: + pass + + def recv(self, amount: int) -> bytes: + result = bytes(self.incoming[:amount]) + del self.incoming[:amount] + return result + + def sendall(self, data: bytes) -> None: + self.sent += data + + connection = Connection() + _cli_worker._proxy_handler('api.anthropic.com')(connection, None, None) + assert connection.sent.startswith(b'HTTP/1.1 403') + + +def test_cli_supervisor_retains_namespace_and_collects_events_once(monkeypatch) -> None: + box = FakeSandbox(False, [_USERS]) + uploads = {} + monkeypatch.setattr(box, 'upload', lambda content, path: uploads.update({path: content})) + backend = SandboxBackend(box) + packet = { + 'data': base64.b64encode(b'{"type":"assistant"}\nDONE\n').decode(), + 'status': {'exit_code': 0, 'stop_reason': None}, + } + monkeypatch.setattr(cli_runtime, '_poll', lambda *args: packet) + observed = [] + result = backend.run_cli(request(), lambda event: observed.append(event)) + config = json.loads( + next(value for key, value in uploads.items() if key.endswith('request.json')) + ) + assert config['namespace'] == ['/usr/bin/nsenter', '--net=/opt/deepgym-netns', '--'] + assert '--reuid=997' in config['prefix'] + assert '--no-new-privs' in config['prefix'] + assert config['prefix'].index('/usr/bin/unshare') < config['prefix'].index('--reuid=997') + assert ['--pid', '--fork', '--mount-proc'] == config['prefix'][ + config['prefix'].index('/usr/bin/unshare') + 1 : config['prefix'].index('/usr/bin/unshare') + + 4 + ] + assert '--pid=/opt/deepgym-staging/agent-processes/pid' in config['prefix'] + assert config['env']['ANTHROPIC_API_KEY'] == 'test-key' + assert observed == [{'type': 'assistant'}] + assert result.events == ({'type': 'assistant'},) + assert result.stop_reason is None + assert 'DONE' in result.output + assert all('test-key' not in command for command in box.commands) + assert any('rm -rf' in command for command in box.commands) + assert any('control.py stop' in command for command in box.commands) + + +def test_callback_budget_stop_reaches_root_supervisor(monkeypatch) -> None: + backend = SandboxBackend(FakeSandbox(False, [_USERS])) + packets = iter( + [ + {'data': base64.b64encode(b'{"type":"assistant"}\n').decode(), 'status': None}, + {'data': '', 'status': {'exit_code': -9, 'stop_reason': 'turn_budget'}}, + ] + ) + monkeypatch.setattr(cli_runtime, '_poll', lambda *args: next(packets)) + monkeypatch.setattr(cli_runtime.time, 'sleep', lambda seconds: None) + result = backend.run_cli(request(), lambda event: 'turn_budget') + assert result.stop_reason == 'turn_budget' + assert any('/stop' in command for command in backend._sandbox.commands) + + +def test_launch_then_rpc_failure_independently_destroys_containment(monkeypatch) -> None: + backend = SandboxBackend(FakeSandbox(False, [_USERS])) + operations = [] + original = backend.exec_trusted + + def execute(command: str, timeout: int) -> ExecResult: + if 'supervise' in command and 'setsid' in command: + operations.append('launch-took-effect') + raise SandboxError('launch RPC response lost') + if 'control.py stop' in command: + operations.append('kernel-teardown') + if '/stop' in command and 'write_text' in command: + operations.append('stop-marker') + if 'rm -rf' in command: + operations.append('remove-credentials') + return original(command, timeout) + + monkeypatch.setattr(backend, 'exec_trusted', execute) + monkeypatch.setattr( + cli_runtime, + '_poll', + lambda *args: {'data': '', 'status': {'exit_code': -9, 'stop_reason': 'cancelled'}}, + ) + with pytest.raises(SandboxError, match='launch RPC response lost'): + backend.run_cli(request(), lambda event: None) + assert operations == [ + 'launch-took-effect', + 'kernel-teardown', + 'stop-marker', + 'remove-credentials', + ] + + +@pytest.mark.parametrize('failed_file', ['worker.py', 'request.json']) +def test_upload_failure_still_removes_staged_credentials(monkeypatch, failed_file: str) -> None: + box = FakeSandbox(False, [_USERS]) + backend = SandboxBackend(box) + uploaded = [] + + def upload(content: bytes, path: str) -> None: + uploaded.append(path) + if path.endswith(failed_file): + raise SandboxError('upload took effect, response lost') + + monkeypatch.setattr(box, 'upload', upload) + with pytest.raises(SandboxError, match='upload took effect'): + backend.run_cli(request(), lambda event: None) + assert uploaded + assert any('rm -rf -- /opt/deepgym-staging/cli-' in command for command in box.commands) + assert not any('supervise' in command for command in box.commands) + + +def test_containment_cleanup_failure_refuses_grading(monkeypatch) -> None: + backend = SandboxBackend(FakeSandbox(False, [_USERS])) + monkeypatch.setattr( + cli_runtime, + '_poll', + lambda *args: {'data': '', 'status': {'exit_code': 0, 'stop_reason': None}}, + ) + + def stop() -> None: + raise SandboxError('namespace survived') + + monkeypatch.setattr(backend, 'stop_agent', stop) + monkeypatch.setattr(backend, 'verify', lambda *args, **kwargs: pytest.fail('unsafe grading')) + session = Session( + MultiTurnEnvironment(task='task', final_verifier_code='print(1)', cli_provider_egress=True), + backend, + ) + session.start() + with pytest.raises(SandboxError, match='containment cleanup failed'): + session.run_cli(request(), lambda event: None) + assert session.failed_result is not None + assert session.score() == session.failed_result + assert session.score().score == 0 and not session.score().passed + assert any('rm -rf' in command for command in backend._sandbox.commands) + + +def test_setup_and_desktop_share_containment_but_root_services_do_not() -> None: + box = FakeSandbox(False, [_USERS]) + backend = SandboxBackend(box) + assert '--pid=/opt/deepgym-staging/agent-processes/pid' in backend._as_agent + backend.exec('setup-command', 10) + assert '--pid=/opt/deepgym-staging/agent-processes/pid' in box.commands[-1] + backend.exec_root('trusted-service', 10, offline=True, env={}) + assert '--pid=' not in box.commands[-1] + + +def test_staging_timeout_stops_setup_descendants_before_step_grading(monkeypatch) -> None: + backend = SandboxBackend(FakeSandbox(False, [_USERS])) + now = [0.0] + operations = [] + monkeypatch.setattr(cli_runtime.time, 'monotonic', lambda: now[0]) + + def upload(content: bytes, path: str) -> None: + if path.endswith('worker.py'): + now[0] = 2.0 + + def grade(output: str) -> float: + assert operations == ['kernel-teardown'] + operations.append('step-grade') + return 0.0 + + monkeypatch.setattr(backend._sandbox, 'upload', upload) + monkeypatch.setattr(backend, 'stop_agent', lambda: operations.append('kernel-teardown')) + session = Session( + MultiTurnEnvironment(task='task', step_verifier_code='print(0)', cli_provider_egress=True), + backend, + ) + monkeypatch.setattr(session, '_step_reward', grade) + session.start() + result = session.run_cli(request(timeout=1), lambda event: None) + assert result.stop_reason == 'time_budget' + assert operations == ['kernel-teardown', 'step-grade'] + assert not any('supervise' in command for command in backend._sandbox.commands) + + +def test_local_backend_refuses_root_service_before_running_anything() -> None: + backend = LocalBackend() + try: + session = Session(MultiTurnEnvironment(task='task', root_service_start='true'), backend) + with pytest.raises(SandboxError, match='isolated SandboxBackend'): + session.start() + finally: + backend.close() + + +def test_root_service_runs_before_desktop_and_setup(monkeypatch) -> None: + backend = SandboxBackend(FakeSandbox(False, [_USERS])) + calls = [] + monkeypatch.setattr( + backend, + 'exec_root', + lambda command, timeout, **kwargs: calls.append(('root', kwargs)) or ExecResult(0, 'ready'), + ) + monkeypatch.setattr(backend, 'start_desktop', lambda: calls.append(('desktop', {}))) + monkeypatch.setattr( + backend, 'exec', lambda *args: calls.append(('setup', {})) or ExecResult(0, '') + ) + monkeypatch.setattr(backend, 'screenshot', lambda: 'png') + session = Session( + MultiTurnEnvironment( + task='task', + root_service_start='bash /opt/probe/start.sh', + desktop=True, + setup_script='browser', + ), + backend, + ) + session.start() + assert [name for name, _ in calls] == ['root', 'desktop', 'setup'] + assert calls[0][1]['offline'] is True + assert set(calls[0][1]['env']) == {'PATH'} + + +def test_cli_is_one_action_and_never_replays_commands(monkeypatch) -> None: + backend = SandboxBackend(FakeSandbox(False, [_USERS])) + event = {'type': 'tool', 'command': 'touch already-ran'} + execution = CliExecution(events=(event,), output='DONE\n', exit_code=0) + monkeypatch.setattr(backend, 'run_cli', lambda request, observer: execution) + session = Session(MultiTurnEnvironment(task='task', cli_provider_egress=True), backend) + session.start() + count = len(backend._sandbox.commands) + assert session.run_cli(request(), lambda event: None) == execution + assert session.done + assert len(session.trajectory.steps) == 1 + action = session.trajectory.steps[0][1] + assert action.action_type == 'cli' + assert action.events == (event,) + assert len(backend._sandbox.commands) == count + assert 'test-key' not in action.model_dump_json() + + +def test_cli_action_cannot_be_replayed_through_step() -> None: + backend = LocalBackend() + try: + session = Session(MultiTurnEnvironment(task='task', allowed_actions={'cli'}), backend) + session.start() + with pytest.raises(ActionError, match='Session.run_cli'): + session.step(Action(action_type='cli', content='touch unsafe')) + finally: + backend.close() + + +def test_events_and_output_redact_known_key_before_observer(monkeypatch) -> None: + backend = SandboxBackend(FakeSandbox(False, [_USERS])) + raw = b'{"type":"assistant","nested":[{"text":"test-key"}],"tokens":12}\nkey=test-key\n' + packet = { + 'data': base64.b64encode(raw).decode(), + 'status': {'exit_code': 0, 'stop_reason': None}, + } + monkeypatch.setattr(cli_runtime, '_poll', lambda *args: packet) + observed = [] + result = backend.run_cli(request(), lambda event: observed.append(event)) + assert 'test-key' not in result.model_dump_json() + assert observed[0]['nested'] == [{'text': '[REDACTED]'}] + assert observed[0]['tokens'] == 12 + + +def test_cli_failure_keeps_already_observed_events(monkeypatch) -> None: + backend = SandboxBackend(FakeSandbox(False, [_USERS])) + event = {'type': 'assistant', 'text': 'partial'} + + def fail(request, observer): + observer(event) + raise SandboxError('transport lost') + + monkeypatch.setattr(backend, 'run_cli', fail) + session = Session(MultiTurnEnvironment(task='task', cli_provider_egress=True), backend) + session.start() + with pytest.raises(SandboxError, match='transport lost'): + session.run_cli(request(), lambda event: None) + assert session.trajectory.steps[0][1].events == (event,) + assert session.done + + +def test_step_budget_restricts_command_timeout(monkeypatch) -> None: + backend = SandboxBackend(FakeSandbox(False, [_USERS])) + timeouts = [] + monkeypatch.setattr( + backend, 'exec', lambda command, timeout: timeouts.append(timeout) or ExecResult(0, '') + ) + session = Session(MultiTurnEnvironment(task='task', timeout_per_step=30), backend) + session.start() + session.step(Action(action_type='bash', content='true'), timeout=0.25) + session.step(Action(action_type='bash', content='true'), timeout=100) + assert 0 < timeouts[0] <= 0.25 + assert timeouts[1] == 30 + + +@pytest.mark.parametrize('timeout', [0, -1, float('nan'), float('inf')]) +def test_step_rejects_invalid_budget_timeout(timeout: float) -> None: + backend = SandboxBackend(FakeSandbox(False, [_USERS])) + session = Session(MultiTurnEnvironment(task='task'), backend) + session.start() + with pytest.raises(ValueError, match='finite and positive'): + session.step(Action(action_type='bash', content='true'), timeout=timeout) + + +def test_observer_accounts_for_trailing_events_after_budget_stop(monkeypatch) -> None: + backend = SandboxBackend(FakeSandbox(False, [_USERS])) + raw = b'{"type":"tool"}\n{"type":"result","usage":{"tokens":123}}\n' + packet = { + 'data': base64.b64encode(raw).decode(), + 'status': {'exit_code': 0, 'stop_reason': None}, + } + monkeypatch.setattr(cli_runtime, '_poll', lambda *args: packet) + observed = [] + + def observer(event: dict) -> str: + observed.append(event) + return 'turn_budget' + + result = backend.run_cli(request(), observer) + assert observed[-1]['usage'] == {'tokens': 123} + assert result.stop_reason == 'turn_budget' + assert sum('/stop' in command for command in backend._sandbox.commands) == 1 + + +def test_gui_action_and_screenshot_share_remaining_deadline(monkeypatch) -> None: + backend = SandboxBackend(FakeSandbox(False, [_USERS])) + monkeypatch.setattr(backend, 'start_desktop', lambda: None) + now = [100.0] + timeouts = [] + + def act(action: Action, *, timeout: float) -> None: + timeouts.append(('action', timeout)) + now[0] += 0.6 + + def screenshot(*, timeout: float | None = None) -> str: + if timeout is not None: + timeouts.append(('screenshot', timeout)) + return 'png' + + monkeypatch.setattr(backend, 'act', act) + monkeypatch.setattr(backend, 'screenshot', screenshot) + monkeypatch.setattr('deepgym.session.time.monotonic', lambda: now[0]) + session = Session( + MultiTurnEnvironment( + task='task', + desktop=True, + allowed_actions={'click'}, + timeout_per_step=120, + ), + backend, + ) + session.start() + session.step(Action(action_type='click', x=1, y=2), timeout=1.0) + assert timeouts[0] == ('action', 1.0) + assert timeouts[1][0] == 'screenshot' + assert timeouts[1][1] == pytest.approx(0.4) + + +@pytest.mark.parametrize('bridge_ready', [False, True]) +@pytest.mark.parametrize('stop_reason', ['time_budget', 'turn_budget']) +def test_worker_does_not_launch_cli_after_setup_exhausts_budget( + monkeypatch, tmp_path, bridge_ready: bool, stop_reason: str +) -> None: + config = { + 'provider': 'anthropic', + 'timeout': 1, + 'uid': 997, + 'namespace': ['nsenter'], + 'prefix': ['setpriv'], + 'argv': ['claude'], + 'env': {}, + 'cwd': str(tmp_path), + } + (tmp_path / 'request.json').write_text(json.dumps(config)) + now = [0.0] + calls = [] + + def popen(argv, **kwargs): + calls.append(argv) + if bridge_ready: + (tmp_path / 'port').write_text('12345') + if stop_reason == 'time_budget': + now[0] = 2.0 + else: + (tmp_path / 'stop').write_text(stop_reason) + return SimpleNamespace(pid=999, poll=lambda: None, wait=lambda **kwargs: -9) + + server = SimpleNamespace( + serve_forever=lambda: None, shutdown=lambda: None, server_close=lambda: None + ) + monkeypatch.setattr(_cli_worker.socketserver, 'ThreadingUnixStreamServer', lambda *args: server) + monkeypatch.setattr( + _cli_worker.threading, 'Thread', lambda **kwargs: SimpleNamespace(start=lambda: None) + ) + monkeypatch.setattr(_cli_worker.time, 'monotonic', lambda: now[0]) + monkeypatch.setattr(_cli_worker.subprocess, 'Popen', popen) + monkeypatch.setattr(_cli_worker.os, 'pidfd_open', lambda pid: 123, raising=False) + monkeypatch.setattr(_cli_worker.signal, 'pidfd_send_signal', lambda *args: None, raising=False) + monkeypatch.setattr(_cli_worker.os, 'close', lambda fd: None) + monkeypatch.setattr(_cli_worker.os, 'killpg', lambda *args: None) + _cli_worker._supervise(tmp_path) + assert len(calls) == 1 + assert 'bridge' in calls[0] + status = json.loads((tmp_path / 'status.json').read_text()) + assert status['stop_reason'] == stop_reason + assert status['exit_code'] == -1 + + +@pytest.mark.parametrize('provider', ['anthropic', 'openai']) +def test_worker_passes_loopback_only_bypass_to_cli_and_child_environment( + monkeypatch, tmp_path, provider: str +) -> None: + config = { + 'provider': provider, + 'timeout': 10, + 'namespace': ['nsenter'], + 'prefix': ['setpriv'], + 'argv': ['fixture-cli'], + 'env': {'NO_PROXY': '*', 'no_proxy': '*'}, + 'cwd': str(tmp_path), + } + (tmp_path / 'request.json').write_text(json.dumps(config)) + captured = {} + + def popen(argv, **kwargs): + if 'bridge' in argv: + (tmp_path / 'port').write_text('12345') + else: + captured.update(kwargs['env']) + return SimpleNamespace(pid=999, poll=lambda: 0, wait=lambda **kwargs: 0) + + server = SimpleNamespace( + serve_forever=lambda: None, shutdown=lambda: None, server_close=lambda: None + ) + monkeypatch.setattr(_cli_worker.socketserver, 'ThreadingUnixStreamServer', lambda *args: server) + monkeypatch.setattr( + _cli_worker.threading, 'Thread', lambda **kwargs: SimpleNamespace(start=lambda: None) + ) + monkeypatch.setattr(_cli_worker.subprocess, 'Popen', popen) + monkeypatch.setattr(_cli_worker.os, 'pidfd_open', lambda pid: 123, raising=False) + monkeypatch.setattr(_cli_worker.signal, 'pidfd_send_signal', lambda *args: None, raising=False) + monkeypatch.setattr(_cli_worker.os, 'close', lambda fd: None) + _cli_worker._supervise(tmp_path) + + assert captured['NO_PROXY'] == captured['no_proxy'] + bypass = {'no': captured['no_proxy']} + for host in ( + 'localhost:8000', + 'localhost.:8000', + '127.0.0.1:8000', + '::1', + '[::1]:8000', + '0:0:0:0:0:0:0:1', + '[0:0:0:0:0:0:0:1]:8000', + ): + assert proxy_bypass_environment(host, bypass), host + for host in ( + 'api.anthropic.com:443', + 'api.openai.com:443', + 'example.com:80', + 'localhost.example.com:8000', + '10.0.0.1:8000', + '169.254.169.254:80', + '[2001:4860:4860::8888]:443', + '127.0.0.2:8000', + '[::ffff:127.0.0.1]:8000', + ): + assert not proxy_bypass_environment(host, bypass), host + for variable in ('HTTP_PROXY', 'HTTPS_PROXY', 'http_proxy', 'https_proxy'): + assert captured[variable] == 'http://127.0.0.1:12345' diff --git a/deepgym/tests/test_pid_namespace.py b/deepgym/tests/test_pid_namespace.py new file mode 100644 index 00000000..07d03c0d --- /dev/null +++ b/deepgym/tests/test_pid_namespace.py @@ -0,0 +1,193 @@ +"""Offline kernel-model regressions for session-wide process containment.""" + +import json +import signal +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from deepgym import _pid_namespace + + +class KernelModel: + """Model PID 1 teardown separately from the worker's /proc inspection.""" + + def __init__(self, root: Path) -> None: + self.directory = root / 'control' + self.directory.mkdir() + self.proc = root / 'proc' + self.proc.mkdir() + self.init = 101 + self.uid = 997 + self.members: set[int] = set() + self.events: list[str] = [] + self.signals: list[tuple[int, int]] = [] + self.closed: list[int] = [] + self.dead = False + self.death_confirmed = True + self.fork_at_kill = False + self.add(self.init, 0, contained=True) + self.add(14, self.uid, contained=True) + self.add(900, 998, contained=False) + namespace = self.proc / str(self.init) / 'ns' + namespace.mkdir() + (self.directory / 'pid').touch() + (namespace / 'pid').hardlink_to(self.directory / 'pid') + self.identity(self.uid) + + def identity(self, uid: int) -> None: + (self.directory / 'identity.json').write_text(json.dumps({'pid': self.init, 'uid': uid})) + + def add(self, pid: int, uid: int, *, contained: bool) -> None: + process = self.proc / str(pid) + process.mkdir(exist_ok=True) + (process / 'status').write_text(f'Uid:\t{uid}\t{uid}\t{uid}\t{uid}\nState:\tS\n') + if contained: + self.members.add(pid) + + def fork_then_exit(self) -> None: + self.add(15, self.uid, contained=True) + (self.proc / '14' / 'status').unlink() + self.members.discard(14) + self.events.append('fork_then_exit') + + def path(self, path: str) -> object: + if path == '/proc': + + def entries() -> list[Path]: + self.events.append('enumerate') + return list(self.proc.iterdir()) + + return SimpleNamespace(iterdir=entries) + if path.startswith('/proc/'): + return self.proc / path.removeprefix('/proc/') + return Path(path) + + def open_pidfd(self, pid: int) -> int: + self.events.append('open_pidfd') + if pid != self.init or not (self.proc / str(pid) / 'status').exists(): + raise ProcessLookupError(pid) + return pid + 10000 + + def send_signal(self, descriptor: int, sig: int) -> None: + self.signals.append((descriptor, sig)) + assert descriptor == self.init + 10000 + assert sig == signal.SIGKILL + if self.fork_at_kill: + self.fork_then_exit() + self.events.append('kernel_teardown') + for pid in self.members: + (self.proc / str(pid) / 'status').unlink(missing_ok=True) + self.members.clear() + self.dead = True + + def wait(self, readers: list[int], *args: object) -> tuple[list[int], list, list]: + self.events.append('wait') + return (readers if self.dead and self.death_confirmed else [], [], []) + + +@pytest.fixture +def kernel(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> KernelModel: + model = KernelModel(tmp_path) + monkeypatch.setattr(_pid_namespace, 'Path', model.path) + monkeypatch.setattr( + _pid_namespace, + 'os', + SimpleNamespace(pidfd_open=model.open_pidfd, close=model.closed.append), + ) + monkeypatch.setattr( + _pid_namespace, + 'signal', + SimpleNamespace(SIGKILL=signal.SIGKILL, pidfd_send_signal=model.send_signal), + ) + monkeypatch.setattr(_pid_namespace, 'select', SimpleNamespace(select=model.wait)) + return model + + +def test_kernel_teardown_covers_setup_child_fork_after_enumeration(kernel: KernelModel) -> None: + # The old scan can miss Q: its snapshot contains P, which exits before status is read. + snapshot = list(kernel.proc.iterdir()) + kernel.fork_then_exit() + observed = [] + for entry in snapshot: + try: + if f'Uid:\t{kernel.uid}\t' in (entry / 'status').read_text(): + observed.append(entry.name) + except FileNotFoundError: + pass + assert observed == [] + assert (kernel.proc / '15' / 'status').exists() + + # Fork again at SIGKILL delivery: namespace destruction must include the new child. + (kernel.proc / '15' / 'status').unlink() + kernel.members.discard(15) + kernel.add(14, kernel.uid, contained=True) + kernel.fork_at_kill = True + kernel.events.clear() + _pid_namespace.stop(kernel.directory) + + assert not (kernel.proc / '15' / 'status').exists() + assert (kernel.proc / '900' / 'status').exists() + assert kernel.signals == [(kernel.init + 10000, signal.SIGKILL)] + assert kernel.events.index('fork_then_exit') < kernel.events.index('kernel_teardown') + assert kernel.events.index('kernel_teardown') < kernel.events.index('enumerate') + assert (kernel.directory / 'stopped').exists() + + +def test_live_agent_uid_outside_containment_fails_closed(kernel: KernelModel) -> None: + kernel.add(902, kernel.uid, contained=False) + with pytest.raises(RuntimeError, match='survived.*grading is unsafe'): + _pid_namespace.stop(kernel.directory) + assert not (kernel.directory / 'stopped').exists() + assert (kernel.proc / '902' / 'status').exists() + + +def test_reused_init_pid_is_not_signalled(kernel: KernelModel) -> None: + actual = kernel.proc / str(kernel.init) / 'ns' / 'pid' + actual.unlink() + actual.touch() + with pytest.raises(RuntimeError, match='identity changed.*PID reuse'): + _pid_namespace.stop(kernel.directory) + assert kernel.signals == [] + assert kernel.closed == [kernel.init + 10000] + assert not (kernel.directory / 'stopped').exists() + + +def test_missing_init_fails_closed(kernel: KernelModel) -> None: + (kernel.proc / str(kernel.init) / 'status').unlink() + with pytest.raises(ProcessLookupError): + _pid_namespace.stop(kernel.directory) + assert kernel.signals == [] + assert not (kernel.directory / 'stopped').exists() + + +def test_unconfirmed_namespace_death_fails_closed(kernel: KernelModel) -> None: + kernel.death_confirmed = False + with pytest.raises(RuntimeError, match='did not terminate'): + _pid_namespace.stop(kernel.directory) + assert 'enumerate' not in kernel.events + assert kernel.closed == [kernel.init + 10000] + assert not (kernel.directory / 'stopped').exists() + + +def test_repeated_stop_rechecks_survivors(kernel: KernelModel) -> None: + _pid_namespace.stop(kernel.directory) + kernel.add(902, kernel.uid, contained=False) + with pytest.raises(RuntimeError, match='survived'): + _pid_namespace.stop(kernel.directory) + assert len(kernel.signals) == 1 + + +@pytest.mark.parametrize('operation', ['create', 'verify_empty', 'stop']) +def test_root_uid_is_rejected_before_any_signal(kernel: KernelModel, operation: str) -> None: + kernel.identity(0) + with pytest.raises(ValueError, match='root'): + if operation == 'create': + _pid_namespace.create(kernel.directory, 0) + elif operation == 'verify_empty': + _pid_namespace.verify_empty(0) + else: + _pid_namespace.stop(kernel.directory) + assert kernel.signals == [] + assert 'open_pidfd' not in kernel.events diff --git a/deepgym/tests/test_sandbox_images.py b/deepgym/tests/test_sandbox_images.py index dced2bdc..5da6d15d 100644 --- a/deepgym/tests/test_sandbox_images.py +++ b/deepgym/tests/test_sandbox_images.py @@ -4,12 +4,14 @@ test_sandboxes_live.py. """ +import contextlib + import pytest from deepgym import DeepGym -from deepgym.exceptions import SandboxError +from deepgym.exceptions import SandboxError, VerifierError from deepgym.models import Environment, MultiTurnEnvironment -from tests.test_sandboxes import FakeSandbox +from tests.test_sandboxes import FakeSandbox, _daytona_client _ENV = Environment(task='t', verifier_code='return 1.0', image='python:3.11-slim') @@ -79,3 +81,55 @@ def list(self, query: object) -> list[object]: assert params.image.dockerfile() == 'FROM python:3.11-slim\n' assert params.network_block_all is True assert params.labels # so a create that fails after allocating can be found and deleted + + +@pytest.mark.parametrize('runtime', [{}, {'snapshot': 'cli-snapshot'}, {'image': 'debian:12'}]) +@pytest.mark.parametrize('egress', [False, True]) +def test_daytona_session_network_requires_explicit_cli_opt_in(runtime, egress: bool) -> None: + pytest.importorskip('daytona') + from deepgym.sandboxes.daytona.sandbox import DaytonaSandbox, _session_params + + environment = MultiTurnEnvironment(task='task', cli_provider_egress=egress, **runtime) + params = _session_params(environment) + assert params.network_block_all is (not egress) + assert params.ephemeral + # Provider-level egress must never remove the namespace shielding the toolbox. + assert DaytonaSandbox.network_blocked is False + + +def test_cli_provider_egress_is_default_off_and_not_for_gui_only() -> None: + from pydantic import ValidationError + + from deepgym.computer_use import ComputerUseEnvironment + + assert MultiTurnEnvironment(task='task').cli_provider_egress is False + with pytest.raises(ValidationError, match='requires bash or cli'): + ComputerUseEnvironment(task='task', cli_provider_egress=True) + + +def test_cli_egress_keeps_agent_root_services_and_verifier_offline(monkeypatch) -> None: + client, daytona = _daytona_client(monkeypatch) + environment = MultiTurnEnvironment( + task='task', + cli_provider_egress=True, + root_service_start='root-service-marker', + setup_script='agent-setup-marker', + final_verifier_code='return 0.0', + ) + with client.session(environment) as session: + session.start() + with contextlib.suppress(VerifierError): + session.score() + box = daytona.boxes[0] + assert box.params.network_block_all is False + root = next(command for command in box.events if 'root-service-marker' in command) + agent = next(command for command in box.events if 'agent-setup-marker' in command) + verifier = next( + command + for command in box.events + if '/opt/deepgym-verifier/' in command and 'python3' in command + ) + for command in (root, agent, verifier): + assert '/usr/bin/nsenter --net=/opt/deepgym-netns --' in command + assert '--reuid=997' in agent + assert '--reuid=996' in verifier diff --git a/deepgym/tests/test_sandboxes.py b/deepgym/tests/test_sandboxes.py index 3a0a7ae9..ef883660 100644 --- a/deepgym/tests/test_sandboxes.py +++ b/deepgym/tests/test_sandboxes.py @@ -56,6 +56,11 @@ # Fake sandboxes pass session setup's check that only root can change TRUSTED_PATH. _SEALED_NEEDLE = 'root would run what others can change' _SEALED = (_SEALED_NEEDLE, sealed_path.SEALED) +_CONTAINED_AGENT_PREFIX = ( + '/usr/bin/nsenter --net=/opt/deepgym-netns -- ' + '/usr/bin/nsenter --pid=/opt/deepgym-staging/agent-processes/pid ' + '--mount=/opt/deepgym-staging/agent-processes/mnt -- /usr/bin/setpriv' +) class FakeSandbox: @@ -567,9 +572,7 @@ def test_session_runs_as_agent_and_joins_netns_only_when_needed() -> None: open_ = FakeSandbox(network_blocked=False, answers=[_USERS]) SandboxBackend(open_).exec('ls', timeout=5) assert any('mount --bind /proc/$pid/ns/net /opt/deepgym-netns' in c for c in open_.commands) - assert open_.commands[-1].startswith( - '/usr/bin/nsenter --net=/opt/deepgym-netns -- /usr/bin/setpriv' - ) + assert open_.commands[-1].startswith(_CONTAINED_AGENT_PREFIX) def test_session_verifier_runs_as_its_own_user_from_a_private_file() -> None: @@ -1452,7 +1455,7 @@ def test_daytona_session_runs_agent_and_verifier_as_separate_users( box = client.boxes[0] step = next(c for c in box.events if 'step-marker' in c) assert step.startswith('/usr/bin/sudo -n -E /bin/sh -c') - assert '/usr/bin/nsenter --net=/opt/deepgym-netns -- /usr/bin/setpriv --reuid=997' in step + assert f'{_CONTAINED_AGENT_PREFIX} --reuid=997' in step assert not any('SECRET' in c for c in box.events), 'verifier code must not be in a command' score = next(c for c in box.events if '/opt/deepgym-verifier/' in c and 'python3' in c) assert '--reuid=996' in score @@ -1646,6 +1649,6 @@ def test_lockdown_writes_the_chromium_policy_to_every_dir_and_inspects_no_binary assert 'is not owned by root' in prepare and 'the agent can still write' in prepare # Discovery runs as the agent (so it skips what only root can reach); root makes the pin. assert 'for name in chromium chromium-browser google-chrome' in prepare - assert 'launch=$(/usr/bin/nsenter --net=/opt/deepgym-netns -- /usr/bin/setpriv' in prepare + assert f'launch=$({_CONTAINED_AGENT_PREFIX}' in prepare for inspection in ('177ELF', 'head -c', 'grep -qF /etc/chromium/policies', 'LIBDIR'): assert inspection not in prepare diff --git a/deepgym/tests/test_traces_agents.py b/deepgym/tests/test_traces_agents.py new file mode 100644 index 00000000..ed23ae1b --- /dev/null +++ b/deepgym/tests/test_traces_agents.py @@ -0,0 +1,243 @@ +"""Offline checks for capturing supervised CLI launches and agent session access.""" + +import json +from datetime import datetime, timezone +from pathlib import Path + +import pytest + +from deepgym.agents import Budget, ClaudeCodeAgent +from deepgym.agents.versions import SUPPORTED_CLIS +from deepgym.cli_runtime import CliExecution, CliExecutionError, CliRequest, EventObserver +from deepgym.exceptions import DeepGymError, SandboxError +from deepgym.models import Action, MultiTurnEnvironment +from deepgym.sandboxes import ExecResult +from deepgym.session import SandboxBackend, Session +from deepgym.traces import ( + AgentMeta, + EnvFingerprint, + Provenance, + TracedSession, + TraceMessage, + export_traces, + load_trace, +) +from deepgym.traces.atif import to_atif + + +class RecordedBackend(SandboxBackend): + """Return recorded output without launching a process, sandbox, or provider call.""" + + sandbox_id = 'recorded' + + def __init__(self) -> None: + self.executions = 0 + self.shell_calls: list[tuple[str, float]] = [] + self.failure: Exception | None = None + self.result = CliExecution( + events=( + { + 'type': 'item.completed', + 'item': { + 'id': 'cmd-1', + 'type': 'command_execution', + 'command': 'touch already_executed', + 'aggregated_output': '', + }, + }, + {'type': 'turn.completed', 'usage': {'input_tokens': 12, 'output_tokens': 3}}, + ), + output='recorded CLI stdout', + exit_code=0, + cli_version=SUPPORTED_CLIS['openai'].version, + ) + + def run_cli(self, request: CliRequest, on_event: EventObserver) -> CliExecution: + self.executions += 1 + for event in self.result.events: + on_event(event) + if self.failure: + raise self.failure + return self.result + + def exec(self, command: str, timeout: float, env: dict[str, str] | None = None) -> ExecResult: + self.shell_calls.append((command, timeout)) + return ExecResult(0, 'shell output') + + def close(self) -> None: + pass + + +def make_trace(path: Path, backend: RecordedBackend) -> TracedSession: + env = MultiTurnEnvironment(task='Complete the fixture task', cli_provider_egress=True) + provenance = Provenance( + env=EnvFingerprint.of(env), + agent=AgentMeta(name='codex'), + provider='recorded', + deepgym_version='test', + started_at=datetime.now(timezone.utc), + ) + return TracedSession(Session(env, backend), env, provenance, path) + + +def request() -> CliRequest: + return CliRequest( + argv=('codex', 'exec', '--json', 'Complete the fixture task'), + env={'CODEX_API_KEY': 'fixture-key-not-a-secret'}, + provider='openai', + timeout=30, + allow_model_credentials=True, + ) + + +def test_cli_capture_round_trips_nested_events_without_replaying(tmp_path: Path) -> None: + backend = RecordedBackend() + received = [] + with make_trace(tmp_path / 'episode.jsonl', backend) as traced: + traced.start() + result = traced.run_cli(request(), lambda event: received.append(event)) + assert result == backend.result + assert traced.done + assert len(traced.trajectory.steps) == 1 + assert len(traced.trace.steps) == 1 + assert traced.trace.provenance.agent.version == SUPPORTED_CLIS['openai'].version + assert received == list(result.events) + assert backend.executions == 1 + assert backend.shell_calls == [] + # Neither the returned process result nor an agent's trajectory can alter the trace. + result.events[0]['type'] = 'mutated' + traced.trajectory.steps.clear() + assert traced.trace.steps[0].action.events[0]['type'] == 'item.completed' + assert len(traced.trajectory.steps) == 1 + + restored = load_trace(traced.path) + assert restored.provenance.agent.version == SUPPORTED_CLIS['openai'].version + action = restored.steps[0].action + assert action.action_type == 'cli' + assert action.events[0]['item']['command'] == 'touch already_executed' + assert 'fixture-key-not-a-secret' not in traced.path.read_text() + document, _ = to_atif(restored) + agent_steps = [step for step in document['steps'] if step['source'] == 'agent'] + assert len(agent_steps) == 1 + step = agent_steps[0] + assert len(step['tool_calls']) == 1 + assert step['tool_calls'][0]['function_name'] == 'cli' + assert 'events' not in step['tool_calls'][0]['arguments'] + assert step['observation']['results'][0]['extra']['cli_events'] == list(action.events) + + +@pytest.mark.parametrize('cleanup_fails', [False, True]) +def test_no_launch_cli_finalization_is_forwarded_without_a_trace_action( + monkeypatch, tmp_path: Path, cleanup_fails: bool +) -> None: + backend = RecordedBackend() + stops = [] + + def stop() -> None: + stops.append('stop') + if cleanup_fails: + raise SandboxError('setup population still active') + + monkeypatch.setattr(backend, 'stop_agent', stop) + with make_trace(tmp_path / 'no-launch.jsonl', backend) as traced: + initial = traced.start() + agent = ClaudeCodeAgent(model='m') + if cleanup_fails: + with pytest.raises(SandboxError, match='setup population'): + agent.run(traced, initial, task='task', budget=Budget(max_cost_usd=0)) + assert not traced.score().has_verdict + else: + result = agent.run(traced, initial, task='task', budget=Budget(max_cost_usd=0)) + assert result.termination_reason == 'cost_budget' + assert result.trajectory.final_observation.done + traced.finalize_cli() + assert traced.done + assert traced.trace.steps == () + assert backend.executions == 0 + assert stops == ['stop'] + if cleanup_fails: + restored = load_trace(traced.path) + assert 'setup population' in restored.error + assert not restored.has_verdict + + +def test_cli_failure_preserves_observed_version_in_persisted_trace(tmp_path: Path) -> None: + backend = RecordedBackend() + backend.failure = CliExecutionError('supervision failed', SUPPORTED_CLIS['openai'].version) + with make_trace(tmp_path / 'failed-version.jsonl', backend) as traced: + traced.start() + with pytest.raises(CliExecutionError): + traced.run_cli(request(), lambda event: None) + restored = load_trace(traced.path) + assert restored.provenance.agent.version == SUPPORTED_CLIS['openai'].version + + +def test_cli_failure_records_the_single_applied_action(tmp_path: Path) -> None: + backend = RecordedBackend() + backend.failure = SandboxError('fixture supervision failure') + with make_trace(tmp_path / 'failed.jsonl', backend) as traced: + traced.start() + with pytest.raises(SandboxError, match='fixture supervision failure'): + traced.run_cli(request(), lambda _: None) + assert traced.done + restored = load_trace(traced.path) + assert len(restored.steps) == 1 + assert restored.steps[0].applied + assert restored.steps[0].action.action_type == 'cli' + assert restored.steps[0].action.events == backend.result.events + assert restored.steps[0].error == 'SandboxError: fixture supervision failure' + + +def test_cli_capture_rejects_unstarted_and_scored_sessions(tmp_path: Path) -> None: + backend = RecordedBackend() + with make_trace(tmp_path / 'blocked.jsonl', backend) as traced: + with pytest.raises(DeepGymError, match='not running'): + traced.run_cli(request(), lambda _: None) + traced.start() + traced.score() + with pytest.raises(DeepGymError, match='already scored'): + traced.run_cli(request(), lambda _: None) + assert backend.executions == 0 + + +def test_traced_step_forwards_deadline_and_copies_trajectory(tmp_path: Path) -> None: + backend = RecordedBackend() + with make_trace(tmp_path / 'step.jsonl', backend) as traced: + traced.start() + result = traced.step(Action(action_type='bash', content='echo fixture'), timeout=0.25) + assert not result.done + command, timeout = backend.shell_calls[0] + assert command == 'echo fixture' + assert 0 < timeout <= 0.25 + assert traced.trace.steps[0].observation.content == 'shell output' + trajectory = traced.trajectory + trajectory.steps.clear() + assert len(traced.trajectory.steps) == 1 + + +def test_cli_events_survive_parquet_exports(tmp_path: Path) -> None: + parquet = pytest.importorskip('pyarrow.parquet') + backend = RecordedBackend() + with make_trace(tmp_path / 'episode.jsonl', backend) as traced: + traced.start() + traced.run_cli(request(), lambda _: None) + original = traced.trace + export_traces([original], 'parquet', tmp_path / 'trace.parquet') + (summary,) = parquet.read_table(tmp_path / 'trace.parquet').to_pylist() + assert json.loads(summary['trace'])['steps'][0]['action']['events'] == list( + backend.result.events + ) + + # Explicitly supplied transcript messages make this trace eligible for RL export. + step = original.steps[0].model_copy( + update={ + 'messages': (TraceMessage(role='assistant', content='Run the CLI'),), + } + ) + annotated = original.model_copy(update={'steps': (step,)}) + report = export_traces([annotated], 'parquet', tmp_path / 'rl.parquet', rows='rl') + assert report.written == 1 + (row,) = parquet.read_table(tmp_path / 'rl.parquet').to_pylist() + events = [json.loads(event) for event in row['steps'][0]['action']['events']] + assert events == list(backend.result.events) + assert annotated.steps[0].action.events == backend.result.events