From 05f93c0829780fe6654a9184d6bc4d9a2b301778 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Wed, 11 Nov 2026 14:41:52 -0800 Subject: [PATCH 1/4] feat(computer): a boot stops when its sandbox does and says what it last wrote A computer's boot asks the sandbox running it whether its container, or the launcher booting its disk, still runs. Within seconds of it exiting, the boot fails with ComputerStopped, naming the exit status and the sandbox's last lines: the guest init's agentenv-init: and the launcher's launch: lines, then the last few. A boot that times out ends its error with those lines too. Sandbox gains optional exit_status() and output_tail(), defaulting to nothing, which local Docker answers from its container and Modal from its sandbox's process. Co-Authored-By: Claude Opus 5.5 --- src/agent_env/env/envs/computer/utils/boot.py | 87 ++++++++-- .../env_providers/env_computer_provider.py | 2 +- .../providers/sandbox_providers/AGENTS.md | 1 + .../local_docker/provider.py | 27 ++++ .../sandbox_providers/modal_sandbox.py | 27 ++++ .../sandbox_providers/modal_vm_sandbox.py | 12 ++ .../providers/sandbox_providers/sandbox.py | 10 ++ .../computer/container_stopped_test.py | 53 ++++++ .../disk_boot_failure_modal_vm_test.py | 40 +++++ tst/integration/computer/ghost_disk.py | 46 ++++++ tst/integration/computer/launcher_test.py | 46 +++++- tst/unit/env/envs/test_computer_boot_stops.py | 151 ++++++++++++++++++ tst/unit/env/envs/test_computer_disk.py | 2 +- tst/unit/env/envs/test_computer_env.py | 4 +- .../sandbox_providers/sandbox_exit_test.py | 102 ++++++++++++ 15 files changed, 594 insertions(+), 16 deletions(-) create mode 100644 tst/integration/computer/container_stopped_test.py create mode 100644 tst/integration/computer/disk_boot_failure_modal_vm_test.py create mode 100644 tst/integration/computer/ghost_disk.py create mode 100644 tst/unit/env/envs/test_computer_boot_stops.py create mode 100644 tst/unit/providers/sandbox_providers/sandbox_exit_test.py diff --git a/src/agent_env/env/envs/computer/utils/boot.py b/src/agent_env/env/envs/computer/utils/boot.py index 4192b5f94..37f0188a5 100644 --- a/src/agent_env/env/envs/computer/utils/boot.py +++ b/src/agent_env/env/envs/computer/utils/boot.py @@ -8,7 +8,9 @@ import contextlib import json import logging +import re import secrets +import signal from collections.abc import Iterable from dataclasses import dataclass from typing import TYPE_CHECKING, Any, Optional @@ -44,12 +46,28 @@ _POLL_S = 0.5 #: How long one read of the driver's card may take. _CARD_TIMEOUT_S = 5 +#: How often a boot asks whether the computer's sandbox still runs it. +_EXIT_POLL_S = 2.0 +#: How many of the last lines the computer's sandbox wrote a boot reads, and how long that may take. +_OUTPUT_LINES, _OUTPUT_TIMEOUT_S = 200, 20 +#: How many of them a boot's error names: the last that say why, and the last of all. +_REASON_LINES, _LAST_LINES = 3, 5 +#: How the lines a disk's guest init and its launcher say why with begin (``computers/disk/linux``). +_REASONS = ("agentenv-init:", "launch:", "launch reset:") +_ESCAPE = re.compile(r"\x1b\[[0-9;?]*[A-Za-z]") #: How a setting a record keeps shows a secret's or a generated one's value, which it never holds. WITHHELD = "" class ComputerFailed(RuntimeError): """The computer's ``computer-ready`` status says it failed for good, a check that can't pass again.""" + + +class ComputerStopped(ComputerFailed): + """The computer's sandbox stopped running it before it was ready: its container, or the launcher booting its disk, + exited.""" + + #: How many random bytes a computer's control token carries: 256 bits, past guessing. CONTROL_TOKEN_BYTES = 32 #: The size of a disk's VM when a deploy sets none: what the desktop was measured booting and passing the probe in; for @@ -201,7 +219,7 @@ async def boot(sandbox_provider: SandboxProvider, *, image: str, port: int, envi **shm_kwargs(sandbox_provider, shm_mb)) try: url = sandbox.tunnel_urls[port] - return Booted(sandbox, url, await ready(url, ready_s, token, answering=held)) + return Booted(sandbox, url, await ready(url, ready_s, token, answering=held, sandbox=sandbox)) except BaseException: await discard(sandbox) raise @@ -226,16 +244,19 @@ def ready_timeout(apps: list[ComputerAppEnv], platform: str) -> float: if (launch := app.launch(platform)) and launch.at == "boot" and launch.ready)]) -async def ready(url: str, timeout: float, token: Optional[str] = None, *, answering: bool = False) -> dict: +async def ready(url: str, timeout: float, token: Optional[str] = None, *, answering: bool = False, + sandbox: Optional[Sandbox] = None) -> dict: """The card of the driver at ``url``, read once its ``computer-ready`` status, asked with its control ``token``, says it's ready, so each app it fronts is on it as the app's own server serves it, or with ``answering``, once it answers at all: a computer whose boot apps' launch is held can't be ready before they launch, as a page driver's - screen is its browser's page. Raises ComputerFailed as soon as the status says it failed, and TimeoutError, naming - what it last saw, when it isn't ready within ``timeout`` seconds.""" - last, failed = "nothing answered", None + screen is its browser's page. Raises ComputerFailed as soon as the status says it failed, ComputerStopped within + seconds of ``sandbox``, the sandbox running the computer, no longer running it, and TimeoutError, naming what it + last saw, when it isn't ready within ``timeout`` seconds; each error ends with what ``sandbox`` last wrote.""" + last, failed, stopped, asked = "nothing answered", None, None, None + loop = asyncio.get_running_loop() try: async with asyncio.timeout(timeout): - while failed is None: + while failed is None and stopped is None: try: card = await protocol_v1.get_card(url, timeout=_CARD_TIMEOUT_S) status = await protocol_v1.invoke_extension(url, card, COMPUTER_READY, None, _STATUS_TIMEOUT_S, @@ -248,11 +269,59 @@ async def ready(url: str, timeout: float, token: Optional[str] = None, *, answer for check in status.get("checks", []) if not check.get("ok")) except Exception as e: last = f"{type(e).__name__}: {e}" - if failed is None: + if failed is None and sandbox is not None and (asked is None or loop.time() - asked >= _EXIT_POLL_S): + asked = loop.time() + stopped = await _exit_status(sandbox) + if failed is None and stopped is None: await asyncio.sleep(_POLL_S) except TimeoutError: - raise TimeoutError(f"the computer at {url} wasn't ready within {timeout:.0f} s ({last})") from None - raise ComputerFailed(f"the computer at {url} failed: {failed}") + raise TimeoutError(f"the computer at {url} wasn't ready within {timeout:.0f} s ({last})" + f"{await _last_written(sandbox)}") from None + if stopped is not None: + raise ComputerStopped(f"the computer at {url} stopped before it was ready, exiting with {_described(stopped)}" + f"{await _last_written(sandbox)}") + raise ComputerFailed(f"the computer at {url} failed: {failed}{await _last_written(sandbox)}") + + +async def _exit_status(sandbox: Sandbox) -> Optional[int]: + """``sandbox.exit_status()``, None when asking fails, which the next ask may not.""" + try: + return await sandbox.exit_status() + except Exception as e: + logger.debug("Couldn't ask whether sandbox %s still runs its computer: %s", sandbox.sandbox_id, e) + return None + + +async def _last_written(sandbox: Optional[Sandbox]) -> str: + """What an error adds of the lines ``sandbox`` last wrote (``said_why``); nothing when there are none, it can't + read them, or reading them takes past ``_OUTPUT_TIMEOUT_S``.""" + if sandbox is None: + return "" + try: + async with asyncio.timeout(_OUTPUT_TIMEOUT_S): + lines = said_why(await sandbox.output_tail(_OUTPUT_LINES)) + except Exception as e: + logger.info("Couldn't read what sandbox %s last wrote: %s", sandbox.sandbox_id, e) + return "" + return "; its sandbox last wrote:\n" + "\n".join(f" {line}" for line in lines) if lines else "" + + +def said_why(lines: list[str]) -> list[str]: + """Of ``lines``, the last a sandbox wrote, each without its terminal escapes and surrounding space, in their order: + the last few a disk's guest init and its launcher say why with, and the last few that aren't blank.""" + cleaned = [text for line in lines if (text := _ESCAPE.sub("", line).strip())] + reasons = [index for index, line in enumerate(cleaned) if line.startswith(_REASONS)][-_REASON_LINES:] + return [cleaned[index] for index in sorted({*reasons, *range(max(0, len(cleaned) - _LAST_LINES), len(cleaned))})] + + +def _described(status: int) -> str: + """``status``, a process's exit status, naming the signal that ended it when one did, as a shell's 128 plus its + number, or Python's minus it, says.""" + number = status - 128 if status > 128 else -status + with contextlib.suppress(ValueError): + if number > 0: + return f"{status} ({signal.Signals(number).name})" + return str(status) async def listed(url: str, apps: list[str], timeout: float) -> dict: diff --git a/src/agent_env/providers/env_providers/env_computer_provider.py b/src/agent_env/providers/env_providers/env_computer_provider.py index 1322c06ba..929fdff5d 100644 --- a/src/agent_env/providers/env_providers/env_computer_provider.py +++ b/src/agent_env/providers/env_providers/env_computer_provider.py @@ -236,7 +236,7 @@ async def reboot(self, env: ComputerEnv, deployed: DeployedSandboxEnv) -> None: except NotImplementedError as e: # a reset the env supports, which this provider can't carry out raise RuntimeError(f"'{env.id}' can't be reset on {type(sandbox_provider).__name__}: {e}") from None card = await ready(deployed.environment_url, ready_timeout(env.composed_apps(), platform_of(env.computer.os)), - deployed.control_token, answering=bool(params.site_routing)) + deployed.control_token, answering=bool(params.site_routing), sandbox=sandbox) _require_name(env, card) valid_for_s = (record.get("sites") or {}).get("valid_for_s", 0) if routed := await _finish_boot(env, deployed, sandbox, params, valid_for_s, values.get("resolution")): diff --git a/src/agent_env/providers/sandbox_providers/AGENTS.md b/src/agent_env/providers/sandbox_providers/AGENTS.md index 4b45b4fe3..4727eaff5 100644 --- a/src/agent_env/providers/sandbox_providers/AGENTS.md +++ b/src/agent_env/providers/sandbox_providers/AGENTS.md @@ -102,6 +102,7 @@ sandboxes off this machine, runs containers' images by name, has no private netw | `platform` | the image platform it runs; `linux/amd64` by default, set another when the provider creates it | | `exec(*argv)` | returns an object with async `stdout.read()` and `stderr.read()` (bytes) and async `wait()` (exit code). If the platform takes one command string, build it with `shlex.join(argv)` | | `terminate()` | abstract; idempotent: terminating a sandbox that is already gone is not an error | +| `exit_status()`, `output_tail(lines)` | optional: the exit status of the process `create_container` or a disk's `create_vm` made the sandbox run, once it has exited, else `None`; and the last `lines` lines it wrote, a disk's console among them. A computer's boot stops waiting within seconds of that process exiting and puts its last lines in its error. The defaults, `None` and `[]`, leave a boot waiting out its readiness timeout. Local Docker reads its container's; `modal` and `modal_vm` their sandbox's process, its output only once it has exited | Everything else is built on `exec`. `exec_script` runs `sudo bash -c