diff --git a/src/agent_env/env/envs/computer/utils/boot.py b/src/agent_env/env/envs/computer/utils/boot.py index 4192b5f94..37f0188a5 100644 --- a/src/agent_env/env/envs/computer/utils/boot.py +++ b/src/agent_env/env/envs/computer/utils/boot.py @@ -8,7 +8,9 @@ import contextlib import json import logging +import re import secrets +import signal from collections.abc import Iterable from dataclasses import dataclass from typing import TYPE_CHECKING, Any, Optional @@ -44,12 +46,28 @@ _POLL_S = 0.5 #: How long one read of the driver's card may take. _CARD_TIMEOUT_S = 5 +#: How often a boot asks whether the computer's sandbox still runs it. +_EXIT_POLL_S = 2.0 +#: How many of the last lines the computer's sandbox wrote a boot reads, and how long that may take. +_OUTPUT_LINES, _OUTPUT_TIMEOUT_S = 200, 20 +#: How many of them a boot's error names: the last that say why, and the last of all. +_REASON_LINES, _LAST_LINES = 3, 5 +#: How the lines a disk's guest init and its launcher say why with begin (``computers/disk/linux``). +_REASONS = ("agentenv-init:", "launch:", "launch reset:") +_ESCAPE = re.compile(r"\x1b\[[0-9;?]*[A-Za-z]") #: How a setting a record keeps shows a secret's or a generated one's value, which it never holds. WITHHELD = "" class ComputerFailed(RuntimeError): """The computer's ``computer-ready`` status says it failed for good, a check that can't pass again.""" + + +class ComputerStopped(ComputerFailed): + """The computer's sandbox stopped running it before it was ready: its container, or the launcher booting its disk, + exited.""" + + #: How many random bytes a computer's control token carries: 256 bits, past guessing. CONTROL_TOKEN_BYTES = 32 #: The size of a disk's VM when a deploy sets none: what the desktop was measured booting and passing the probe in; for @@ -201,7 +219,7 @@ async def boot(sandbox_provider: SandboxProvider, *, image: str, port: int, envi **shm_kwargs(sandbox_provider, shm_mb)) try: url = sandbox.tunnel_urls[port] - return Booted(sandbox, url, await ready(url, ready_s, token, answering=held)) + return Booted(sandbox, url, await ready(url, ready_s, token, answering=held, sandbox=sandbox)) except BaseException: await discard(sandbox) raise @@ -226,16 +244,19 @@ def ready_timeout(apps: list[ComputerAppEnv], platform: str) -> float: if (launch := app.launch(platform)) and launch.at == "boot" and launch.ready)]) -async def ready(url: str, timeout: float, token: Optional[str] = None, *, answering: bool = False) -> dict: +async def ready(url: str, timeout: float, token: Optional[str] = None, *, answering: bool = False, + sandbox: Optional[Sandbox] = None) -> dict: """The card of the driver at ``url``, read once its ``computer-ready`` status, asked with its control ``token``, says it's ready, so each app it fronts is on it as the app's own server serves it, or with ``answering``, once it answers at all: a computer whose boot apps' launch is held can't be ready before they launch, as a page driver's - screen is its browser's page. Raises ComputerFailed as soon as the status says it failed, and TimeoutError, naming - what it last saw, when it isn't ready within ``timeout`` seconds.""" - last, failed = "nothing answered", None + screen is its browser's page. Raises ComputerFailed as soon as the status says it failed, ComputerStopped within + seconds of ``sandbox``, the sandbox running the computer, no longer running it, and TimeoutError, naming what it + last saw, when it isn't ready within ``timeout`` seconds; each error ends with what ``sandbox`` last wrote.""" + last, failed, stopped, asked = "nothing answered", None, None, None + loop = asyncio.get_running_loop() try: async with asyncio.timeout(timeout): - while failed is None: + while failed is None and stopped is None: try: card = await protocol_v1.get_card(url, timeout=_CARD_TIMEOUT_S) status = await protocol_v1.invoke_extension(url, card, COMPUTER_READY, None, _STATUS_TIMEOUT_S, @@ -248,11 +269,59 @@ async def ready(url: str, timeout: float, token: Optional[str] = None, *, answer for check in status.get("checks", []) if not check.get("ok")) except Exception as e: last = f"{type(e).__name__}: {e}" - if failed is None: + if failed is None and sandbox is not None and (asked is None or loop.time() - asked >= _EXIT_POLL_S): + asked = loop.time() + stopped = await _exit_status(sandbox) + if failed is None and stopped is None: await asyncio.sleep(_POLL_S) except TimeoutError: - raise TimeoutError(f"the computer at {url} wasn't ready within {timeout:.0f} s ({last})") from None - raise ComputerFailed(f"the computer at {url} failed: {failed}") + raise TimeoutError(f"the computer at {url} wasn't ready within {timeout:.0f} s ({last})" + f"{await _last_written(sandbox)}") from None + if stopped is not None: + raise ComputerStopped(f"the computer at {url} stopped before it was ready, exiting with {_described(stopped)}" + f"{await _last_written(sandbox)}") + raise ComputerFailed(f"the computer at {url} failed: {failed}{await _last_written(sandbox)}") + + +async def _exit_status(sandbox: Sandbox) -> Optional[int]: + """``sandbox.exit_status()``, None when asking fails, which the next ask may not.""" + try: + return await sandbox.exit_status() + except Exception as e: + logger.debug("Couldn't ask whether sandbox %s still runs its computer: %s", sandbox.sandbox_id, e) + return None + + +async def _last_written(sandbox: Optional[Sandbox]) -> str: + """What an error adds of the lines ``sandbox`` last wrote (``said_why``); nothing when there are none, it can't + read them, or reading them takes past ``_OUTPUT_TIMEOUT_S``.""" + if sandbox is None: + return "" + try: + async with asyncio.timeout(_OUTPUT_TIMEOUT_S): + lines = said_why(await sandbox.output_tail(_OUTPUT_LINES)) + except Exception as e: + logger.info("Couldn't read what sandbox %s last wrote: %s", sandbox.sandbox_id, e) + return "" + return "; its sandbox last wrote:\n" + "\n".join(f" {line}" for line in lines) if lines else "" + + +def said_why(lines: list[str]) -> list[str]: + """Of ``lines``, the last a sandbox wrote, each without its terminal escapes and surrounding space, in their order: + the last few a disk's guest init and its launcher say why with, and the last few that aren't blank.""" + cleaned = [text for line in lines if (text := _ESCAPE.sub("", line).strip())] + reasons = [index for index, line in enumerate(cleaned) if line.startswith(_REASONS)][-_REASON_LINES:] + return [cleaned[index] for index in sorted({*reasons, *range(max(0, len(cleaned) - _LAST_LINES), len(cleaned))})] + + +def _described(status: int) -> str: + """``status``, a process's exit status, naming the signal that ended it when one did, as a shell's 128 plus its + number, or Python's minus it, says.""" + number = status - 128 if status > 128 else -status + with contextlib.suppress(ValueError): + if number > 0: + return f"{status} ({signal.Signals(number).name})" + return str(status) async def listed(url: str, apps: list[str], timeout: float) -> dict: diff --git a/src/agent_env/providers/env_providers/env_computer_provider.py b/src/agent_env/providers/env_providers/env_computer_provider.py index 1322c06ba..929fdff5d 100644 --- a/src/agent_env/providers/env_providers/env_computer_provider.py +++ b/src/agent_env/providers/env_providers/env_computer_provider.py @@ -236,7 +236,7 @@ async def reboot(self, env: ComputerEnv, deployed: DeployedSandboxEnv) -> None: except NotImplementedError as e: # a reset the env supports, which this provider can't carry out raise RuntimeError(f"'{env.id}' can't be reset on {type(sandbox_provider).__name__}: {e}") from None card = await ready(deployed.environment_url, ready_timeout(env.composed_apps(), platform_of(env.computer.os)), - deployed.control_token, answering=bool(params.site_routing)) + deployed.control_token, answering=bool(params.site_routing), sandbox=sandbox) _require_name(env, card) valid_for_s = (record.get("sites") or {}).get("valid_for_s", 0) if routed := await _finish_boot(env, deployed, sandbox, params, valid_for_s, values.get("resolution")): diff --git a/src/agent_env/providers/sandbox_providers/AGENTS.md b/src/agent_env/providers/sandbox_providers/AGENTS.md index 4b45b4fe3..4727eaff5 100644 --- a/src/agent_env/providers/sandbox_providers/AGENTS.md +++ b/src/agent_env/providers/sandbox_providers/AGENTS.md @@ -102,6 +102,7 @@ sandboxes off this machine, runs containers' images by name, has no private netw | `platform` | the image platform it runs; `linux/amd64` by default, set another when the provider creates it | | `exec(*argv)` | returns an object with async `stdout.read()` and `stderr.read()` (bytes) and async `wait()` (exit code). If the platform takes one command string, build it with `shlex.join(argv)` | | `terminate()` | abstract; idempotent: terminating a sandbox that is already gone is not an error | +| `exit_status()`, `output_tail(lines)` | optional: the exit status of the process `create_container` or a disk's `create_vm` made the sandbox run, once it has exited, else `None`; and the last `lines` lines it wrote, a disk's console among them. A computer's boot stops waiting within seconds of that process exiting and puts its last lines in its error. The defaults, `None` and `[]`, leave a boot waiting out its readiness timeout. Local Docker reads its container's; `modal` and `modal_vm` their sandbox's process, its output only once it has exited | Everything else is built on `exec`. `exec_script` runs `sudo bash -c