diff --git a/.github/dependabot.yml b/.github/dependabot.yml index b8f5dfe73..5fd12fcab 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -57,7 +57,7 @@ updates: - "/src/agent_env/agents/agents/pydantic-ai" - "/src/agent_env/examples/mail-mcp/envs/mail" - "/src/agent_env/computers/artifacts/ubuntu-24.04" - - "/src/agent_env/computers/artifacts/ubuntu-24.04-headless" + - "/src/agent_env/computers/artifacts/ubuntu-24.04-browser" - "/src/agent_env/computers/disk/linux/boot" - "/src/agent_env/computers/disk/linux/tools" - "/src/agent_env/computers/disk/linux/launcher" diff --git a/.github/scripts/infra_build.py b/.github/scripts/infra_build.py index f90e4eb5a..ebb10b0c7 100644 --- a/.github/scripts/infra_build.py +++ b/.github/scripts/infra_build.py @@ -57,7 +57,7 @@ "agentenv-pydantic-ai": "src/agent_env/agents/agents/pydantic-ai/Dockerfile", "agentenv-mail-mcp": "src/agent_env/examples/mail-mcp/envs/mail/Dockerfile", "agentenv-ubuntu-24.04": "src/agent_env/computers/artifacts/ubuntu-24.04/Dockerfile", - "agentenv-ubuntu-24.04-headless": "src/agent_env/computers/artifacts/ubuntu-24.04-headless/Dockerfile", + "agentenv-ubuntu-24.04-browser": "src/agent_env/computers/artifacts/ubuntu-24.04-browser/Dockerfile", "agentenv-disk-tools": "src/agent_env/computers/disk/linux/tools/Dockerfile", "agentenv-qemu-launcher": "src/agent_env/computers/disk/linux/launcher/Dockerfile", "agentenv-ubuntu-24.04-disk": "src/agent_env/computers/disk/linux/boot/Dockerfile", @@ -125,7 +125,7 @@ " s=$(supervisorctl -c /etc/supervisor/supervisord.conf status);" " echo \"$s\" | grep -q ' RUNNING ' && ! echo \"$s\" | grep -qv ' RUNNING ' && exit 0;" " done; supervisorctl -c /etc/supervisor/supervisord.conf status; exit 1"], - "agentenv-ubuntu-24.04-headless": [ + "agentenv-ubuntu-24.04-browser": [ "docker", "run", "--rm", "--entrypoint", "sh", "{image}", "-c", "start-computer & for _ in $(seq 60); do sleep 1;" " s=$(supervisorctl -c /etc/supervisor/supervisord.conf status);" diff --git a/AGENTS.md b/AGENTS.md index 159af67fc..5af2559e5 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -121,7 +121,7 @@ and deserialized through a registry. | Package | Contents | |---|---| -| `env/` | `Env` base class: a subclass declares `type`, implements `from_dict` and `async deploy(...) -> DeployedEnv`, and may override `reset`, `close` and `load_file_artifact_universe`; teardown closes each env a run deployed before terminating its sandboxes by id. Built-ins in `env/envs/`: `MCPServerEnv` (`mcp_server`), `MultiEnv` (`multi`, MCP servers plus websites), `ServiceDBEnv` (`service_db`, PostgreSQL), `WebsiteEnv` (`website`, one site: a server image, `backend_image` in its env.toml, serving the pages and API, ready when its card's `website/v1` `ready` answers a `WebsiteReadiness` that says so, as `agentenv_protocol.website` (`website_readiness`, `env_readiness`) defines a website env's health for the deploy and `DeployedEnv.readiness()` alike, or, for a card without `ready`, when its `health` path answers 200, which is deprecated; its env.toml's `ready_timeout_s`, how long it may take from its server's container starting to ready, is that container's health check `start_period` and what the deploy waits, or its card's `ready` timeout if longer, and its `shm_mb` is the container's `shm_size`, each left to the gateway and Docker without them; the gateway container's `env/gateway/constants.py` mirrors the URI and op, a test keeping them equal; a separate frontend image is deprecated, warned of at put and optional in a bundle, and a stored `backend_docker_image_artifact` reads as the server image; a live one, `live = true`, is a real site whose server serves none of its pages, its `live`, `domains` and `allow_hosts` read at put from its server's `@website(...)` (`utils/card_naming.py` `WebsiteSource`) unless given, its `settings`' secrets only as references the gateway resolves as it writes the site's container (`WEBSITE_SETTINGS_ENV`); the gateway gives it no site route, a computer's site table leaves it out, a computer whose network is an allowlist is let reach its names (`MultiEnv.live_hosts`), a browser alone's forward port takes them straight to the real site (`site_proxy.py` `direct`), as the headless base's stand-in proxy takes any host, refusing nothing, each proxy logging each such connection as a JSON line, and a run that deploys one is marked live, `reproducible: false`), `GatewayEnv` (`gateway_server`), `ComputerDriverEnv` (`computer_driver`, `env/envs/computer/driver_env.py`: a computer driver's files and what its class declares for each platform it supports, read at put in a process of its own, `agentenv_protocol.computer._declare`, where the driver's own dependencies needn't be installed, with the `port` it serves on and its install by OS, `[install.]`, or for one distribution `[install."/"]`, which replaces its OS type's there, as its `known_failures` do (`env/envs/computer/utils/install.py`): files, apt `packages` with extra `sources`, a `python` requirements file for a venv, `downloads` by URL and sha256, `run` scripts, the services the base's init runs, and `ports`; it deploys only as a computer env's driver), `ComputerEnv` (`computer`, in `env/envs/computer/`: a base computer and a driver, matched at put by gate 1 before anything is stored, then composed by `composer.py` into an image for each form and arch on a builder of the put's sandbox provider, by default the env's (`utils/builder.py`; `--sandbox` on `env build`, `bundle register` and `run`): this machine's Docker for `local`, else a VM of the provider, which fetches the put's stored files by signed URL, takes the rest inline and pushes to the image store itself, so it builds only with stores a remote VM reaches; each arch goes to the first link that runs it natively, else one that emulates it (a VM registers QEMU; `--no-emulate` leaves it `skipped` instead, for a later put), and its pair records the provider that built it (`builder`); a run or register that reuses a computer env composes what it has left (`pending`), a run only the forms its sandbox's deploys boot (`booted_forms`: on each link, the first form it boots natively, by the deploy's own rule, `boot.first_native`; a compose of a disk adds its container), and a deploy composes first what it boots that no compose started, and a container it runs natively that was built but not probed, as one whose probe a setting that didn't resolve stopped, booting it only once ready (`ComputerEnv.compose_unstarted`); tagged `agentenv-computers:` in the image store by a key of the image's Dockerfile, build context and lock so any env with the same inputs reuses it; every layer's packages, requirements and downloads are resolved at put in a container of the base for each arch (`utils/packages.py`, `utils/resolve_in_base.py`), stored in the object store by sha256 under `agentenv-packages/`, and listed in a lock stored by a digest of its inputs, which any env with the same inputs reuses, and builds install only the lock's files, every install step `RUN --network=none`; each pair's status goes into its metadata, `composed`, the arches built are written as its computer, a `computer` artifact over one index, and `agent-env env build` resumes what a put left; gate 3 (`utils/probing.py`) then boots each native build on the put's sandbox provider and probes it at `full`, making it `ready` or `failed`, and a build no provider runs natively stays `unverified`. Its deploy, through the `computer` env provider (`providers/env_providers/env_computer_provider.py`), boots its first `ready` build on a sandbox provider that runs its arch natively (`native_platforms()`), with no gateway in front and the driver's card as the env card, its `/dev/shm` the most its apps' Linux installs ask for (`shm_mb`, in MiB, which its composed computer and the deploy record keep; `--shm-size` on a container, `AGENTENV_SHM_MB` read by a Linux disk's init, passed only to a provider whose `SHARED_MEMORY` says it gives that much, else the sandbox's default, which the deploy logs), then checks its `display`; `deploy_env` `env_params` take a `form`, `computer_settings` (deploy-phase values of the base's settings; a secret one, here or on the env, only as a bare `secret:`/`env:` reference) and `driver_settings` (the driver's `ComputerSettings`, such as its default coordinate space); a `network` that allows nothing runs it on local Docker's internal network behind a relay (`providers/sandbox_providers/local_docker/relay/`); `reset` and `close` have the driver end its recordings first (`computer-recording/v1` `end_all`, each uploading what's left and its manifest under the grant it started with), and `reset` then boots a fresh container in its place (`reset_container`), keeping the computer's address but not an agent's MCP session to the old driver; its `load_file_artifact_universe` writes files under the session user's home through the driver's `data/add`, `Downloads` by default, each file's bytes through a read grant when the store's grants reach it and its network allows everything, else inline (`env/envs/computer/utils/inside.py`)), `ComputerAppEnv` (`computer_app`, `env/envs/computer/app_env.py`: an app on a computer, a `ComputerAppEnvironment` class its env.toml's `module` names, read at put by the same `_declare`, which declares each OS with its own decorator (`@linux(...)`, or `@install(os_type, spec)` for any OS type; `@linux(distribution=...)` or `@install("/", spec)` for one distribution, which replaces its OS type's there) holding its install, its launch and, optionally, a skill and settings and build settings that add to the class's there, and install steps and ready checks for every OS or those their `os=` names; everything that reads its settings, launch or steps asks for the computer's OS type; deployed alone it is a one-app computer env, a `ComputerEnv` of the computer and driver its env.toml names, the composer installing it under `/opt/agentenv/apps//` with its services as supervisord programs beside its own server (`agentenv_protocol.computer.serve_app`, in a venv with the protocol's `app` extra, on `.agentenv/server.sock` in its folder), which adds its launcher, launches a `boot` app once per boot and says through `computer-app/v1` whether it's ready, as the driver's `computer-ready` asks, its card named after it, and `env_params` `app_settings` handing it values of its `Settings` in `AGENTENV_APP_SETTINGS`, a secret one only as a bare reference, over its env.toml's `[settings]` and a computer env's `settings` beside it in `apps`, each by top-level field (`params.app_settings`; checked at put, a required one left to the deploy; the deploy record keeps the merged values as `app_settings`, secrets withheld); its env.toml's `build_settings`, values of its `BuildSettings` checked at put, are written to `.agentenv/build_settings.json` in its folder, so they're part of the image's key, its install steps read them, and a deploy can't change them). An env type whose services take environment data defines `load_environment_artifact` and `load_environment_universe_artifact`; `load_artifact` refuses an environment artifact aimed at a type that doesn't, at save and before it loads. An env built on the protocol SDK can serve agents a skill (`@agent_skill`, `skill/v1`), which `add_skills` `env_ids` installs under its environment name (behind a gateway, each server's under that server's name); a skill with files beside its SKILL.md goes as a bundle stored once per content under `agentenv-env-skills/` in the object store. `env/gateway/gateway.py` is the env-side gateway server, container-only code. Its site route, `/website//`, never forwards a site's data plane or card (`/agentenv/…`, `/.well-known/agent-env.json`), which agentenv reaches through `/svc//`; a site with no frontend image is served by its server image there. Its CDP route, `/browser//cdp//` (`env/gateway/browser_proxy.py`), reaches the website browser behind the `/svc` child ``, found from that child's card (it, or an app among its children, offering `browser/v1` `cdp` with `/cdp/` in its `computer-app/v1` `agent_routes`): it relays the browser's discovery and WebSocket both ways, each leg pinged every 20 s, and every CDP address the browser hands out, in `json/version`, `json/list` and its `cdp` answer through `/svc//`, comes back at the gateway, everything before `/cdp//` becoming the address the caller reached the gateway at and `/browser/`; each connection is a `browser_connection` trajectory event, without its token. While its clock is armed it serves agents `get_time` (`env/gateway/get_time.py`), which answers in UTC and, given an IANA `time_zone`, also in that zone; executors' working `hours` (`env/gateway/triggers.py`) take a fixed `utc_offset` or an IANA `time_zone`. | +| `env/` | `Env` base class: a subclass declares `type`, implements `from_dict` and `async deploy(...) -> DeployedEnv`, and may override `reset`, `close` and `load_file_artifact_universe`; teardown closes each env a run deployed before terminating its sandboxes by id. Built-ins in `env/envs/`: `MCPServerEnv` (`mcp_server`), `MultiEnv` (`multi`, MCP servers plus websites), `ServiceDBEnv` (`service_db`, PostgreSQL), `WebsiteEnv` (`website`, one site: a server image, `backend_image` in its env.toml, serving the pages and API, ready when its card's `website/v1` `ready` answers a `WebsiteReadiness` that says so, as `agentenv_protocol.website` (`website_readiness`, `env_readiness`) defines a website env's health for the deploy and `DeployedEnv.readiness()` alike, or, for a card without `ready`, when its `health` path answers 200, which is deprecated; its env.toml's `ready_timeout_s`, how long it may take from its server's container starting to ready, is that container's health check `start_period` and what the deploy waits, or its card's `ready` timeout if longer, and its `shm_mb` is the container's `shm_size`, each left to the gateway and Docker without them; the gateway container's `env/gateway/constants.py` mirrors the URI and op, a test keeping them equal; a separate frontend image is deprecated, warned of at put and optional in a bundle, and a stored `backend_docker_image_artifact` reads as the server image; a live one, `live = true`, is a real site whose server serves none of its pages, its `live`, `domains` and `allow_hosts` read at put from its server's `@website(...)` (`utils/card_naming.py` `WebsiteSource`) unless given, its `settings`' secrets only as references the gateway resolves as it writes the site's container (`WEBSITE_SETTINGS_ENV`); the gateway gives it no site route, a computer's site table leaves it out, a computer whose network is an allowlist is let reach its names (`MultiEnv.live_hosts`), a browser alone's forward port takes them straight to the real site (`site_proxy.py` `direct`), as the browser base's stand-in proxy takes any host, refusing nothing, each proxy logging each such connection as a JSON line, and a run that deploys one is marked live, `reproducible: false`), `GatewayEnv` (`gateway_server`), `ComputerDriverEnv` (`computer_driver`, `env/envs/computer/driver_env.py`: a computer driver's files and what its class declares for each platform it supports, read at put in a process of its own, `agentenv_protocol.computer._declare`, where the driver's own dependencies needn't be installed, with the `port` it serves on and its install by OS, `[install.]`, or for one distribution `[install."/"]`, which replaces its OS type's there, as its `known_failures` do (`env/envs/computer/utils/install.py`): files, apt `packages` with extra `sources`, a `python` requirements file for a venv, `downloads` by URL and sha256, `run` scripts, the services the base's init runs, and `ports`; it deploys only as a computer env's driver), `ComputerEnv` (`computer`, in `env/envs/computer/`: a base computer and a driver, matched at put by gate 1 before anything is stored, then composed by `composer.py` into an image for each form and arch on a builder of the put's sandbox provider, by default the env's (`utils/builder.py`; `--sandbox` on `env build`, `bundle register` and `run`): this machine's Docker for `local`, else a VM of the provider, which fetches the put's stored files by signed URL, takes the rest inline and pushes to the image store itself, so it builds only with stores a remote VM reaches; each arch goes to the first link that runs it natively, else one that emulates it (a VM registers QEMU; `--no-emulate` leaves it `skipped` instead, for a later put), and its pair records the provider that built it (`builder`); a run or register that reuses a computer env composes what it has left (`pending`), a run only the forms its sandbox's deploys boot (`booted_forms`: on each link, the first form it boots natively, by the deploy's own rule, `boot.first_native`; a compose of a disk adds its container), and a deploy composes first what it boots that no compose started, and a container it runs natively that was built but not probed, as one whose probe a setting that didn't resolve stopped, booting it only once ready (`ComputerEnv.compose_unstarted`); tagged `agentenv-computers:` in the image store by a key of the image's Dockerfile, build context and lock so any env with the same inputs reuses it; every layer's packages, requirements and downloads are resolved at put in a container of the base for each arch (`utils/packages.py`, `utils/resolve_in_base.py`), stored in the object store by sha256 under `agentenv-packages/`, and listed in a lock stored by a digest of its inputs, which any env with the same inputs reuses, and builds install only the lock's files, every install step `RUN --network=none`; each pair's status goes into its metadata, `composed`, the arches built are written as its computer, a `computer` artifact over one index, and `agent-env env build` resumes what a put left; gate 3 (`utils/probing.py`) then boots each native build on the put's sandbox provider and probes it at `full`, making it `ready` or `failed`, and a build no provider runs natively stays `unverified`. Its deploy, through the `computer` env provider (`providers/env_providers/env_computer_provider.py`), boots its first `ready` build on a sandbox provider that runs its arch natively (`native_platforms()`), with no gateway in front and the driver's card as the env card, its `/dev/shm` the most its apps' Linux installs ask for (`shm_mb`, in MiB, which its composed computer and the deploy record keep; `--shm-size` on a container, `AGENTENV_SHM_MB` read by a Linux disk's init, passed only to a provider whose `SHARED_MEMORY` says it gives that much, else the sandbox's default, which the deploy logs), then checks its `display`; `deploy_env` `env_params` take a `form`, `computer_settings` (deploy-phase values of the base's settings; a secret one, here or on the env, only as a bare `secret:`/`env:` reference) and `driver_settings` (the driver's `ComputerSettings`, such as its default coordinate space); a `network` that allows nothing runs it on local Docker's internal network behind a relay (`providers/sandbox_providers/local_docker/relay/`); `reset` and `close` have the driver end its recordings first (`computer-recording/v1` `end_all`, each uploading what's left and its manifest under the grant it started with), and `reset` then boots a fresh container in its place (`reset_container`), keeping the computer's address but not an agent's MCP session to the old driver; its `load_file_artifact_universe` writes files under the session user's home through the driver's `data/add`, `Downloads` by default, each file's bytes through a read grant when the store's grants reach it and its network allows everything, else inline (`env/envs/computer/utils/inside.py`)), `ComputerAppEnv` (`computer_app`, `env/envs/computer/app_env.py`: an app on a computer, a `ComputerAppEnvironment` class its env.toml's `module` names, read at put by the same `_declare`, which declares each OS with its own decorator (`@linux(...)`, or `@install(os_type, spec)` for any OS type; `@linux(distribution=...)` or `@install("/", spec)` for one distribution, which replaces its OS type's there) holding its install, its launch and, optionally, a skill and settings and build settings that add to the class's there, and install steps and ready checks for every OS or those their `os=` names; everything that reads its settings, launch or steps asks for the computer's OS type; deployed alone it is a one-app computer env, a `ComputerEnv` of the computer and driver its env.toml names, the composer installing it under `/opt/agentenv/apps//` with its services as supervisord programs beside its own server (`agentenv_protocol.computer.serve_app`, in a venv with the protocol's `app` extra, on `.agentenv/server.sock` in its folder), which adds its launcher, launches a `boot` app once per boot and says through `computer-app/v1` whether it's ready, as the driver's `computer-ready` asks, its card named after it, and `env_params` `app_settings` handing it values of its `Settings` in `AGENTENV_APP_SETTINGS`, a secret one only as a bare reference, over its env.toml's `[settings]` and a computer env's `settings` beside it in `apps`, each by top-level field (`params.app_settings`; checked at put, a required one left to the deploy; the deploy record keeps the merged values as `app_settings`, secrets withheld); its env.toml's `build_settings`, values of its `BuildSettings` checked at put, are written to `.agentenv/build_settings.json` in its folder, so they're part of the image's key, its install steps read them, and a deploy can't change them). An env type whose services take environment data defines `load_environment_artifact` and `load_environment_universe_artifact`; `load_artifact` refuses an environment artifact aimed at a type that doesn't, at save and before it loads. An env built on the protocol SDK can serve agents a skill (`@agent_skill`, `skill/v1`), which `add_skills` `env_ids` installs under its environment name (behind a gateway, each server's under that server's name); a skill with files beside its SKILL.md goes as a bundle stored once per content under `agentenv-env-skills/` in the object store. `env/gateway/gateway.py` is the env-side gateway server, container-only code. Its site route, `/website//`, never forwards a site's data plane or card (`/agentenv/…`, `/.well-known/agent-env.json`), which agentenv reaches through `/svc//`; a site with no frontend image is served by its server image there. Its CDP route, `/browser//cdp//` (`env/gateway/browser_proxy.py`), reaches the website browser behind the `/svc` child ``, found from that child's card (it, or an app among its children, offering `browser/v1` `cdp` with `/cdp/` in its `computer-app/v1` `agent_routes`): it relays the browser's discovery and WebSocket both ways, each leg pinged every 20 s, and every CDP address the browser hands out, in `json/version`, `json/list` and its `cdp` answer through `/svc//`, comes back at the gateway, everything before `/cdp//` becoming the address the caller reached the gateway at and `/browser/`; each connection is a `browser_connection` trajectory event, without its token. While its clock is armed it serves agents `get_time` (`env/gateway/get_time.py`), which answers in UTC and, given an IANA `time_zone`, also in that zone; executors' working `hours` (`env/gateway/triggers.py`) take a fixed `utc_offset` or an IANA `time_zone`. | | `artifact/` | Immutable versioned artifacts; every `put()` creates a new version. The document goes to the document store, binary payloads to the object store. Every artifact but an environment universe, whose `metadata` key holds its legacy metadata file ids, carries a `metadata` dict outside its versioned fields, changed by compare-and-swap (`merge_metadata`). Built-ins: `CliArtifact` (`cli`), `ComputerArtifact` (`computer`, a base computer: its architectures, OS, session user, settings and container form, whose image `put` reads from its registry into `metadata.validated_images`), `FileArtifact` (`file`), `FileArtifactUniverse` (`file_artifact_universe`), `DockerImageArtifact` (`docker_image`), `EnvironmentArtifact` (`environment`: its environment's files by path, stored as the one wrapped `file_artifact_ref` every environment artifact had when it has one file at its filename, else as `file_artifact_refs`; `get_file_artifacts()` reads either, and a load hands each file to the env's `data/add` with its path, which a pre-v1 load, taking one file, refuses), `EnvironmentUniverseArtifact` (`environment_universe`), `SkillArtifact` (`skill`); `[artifacts] type_aliases` maps another stored spelling of a type onto one of them. A `DockerImageArtifact` holds a tar.gz of its image, or, written by `put_ref` (`--image-ref` on `env mcp-server put` and `a2a-agent put`), only a registry reference with its tag pinned to a digest, which sandboxes pull, or a published image's tag with its `image_digest` and its build context, which sandboxes pull by the digest and build from the context when they can't (the infra images, and a bundle agent built from a shipped agent's sources, `Plan.published`), or, written by `put_context`, only its build context, which a VM sandbox builds before running it (`VmSandbox.load_docker_images`) and Modal builds before creating its container (`ModalSandboxProvider.prepare_image`, outside a fallback chain's deadline); anything else that runs images by name (a container on the local provider, a lone server on any provider but Modal) refuses such an image. A build context (`utils/docker/context.py`) is what `docker build` would send, `.dockerignore` applied, written so equal files give equal bytes and an equal `source_digest`. | | `task/`, `task_step/` | A `Task` holds its `TaskStep`s inline; `Task.run()` executes them as a DAG. `depends_on` (None means all prior steps) gates a step, independent steps run concurrently, `fail_task_on_error` makes a failure fatal or tolerated, `retry_config` rolls a failed span back through the step journal and re-dispatches it. Built-in steps live in `task_step/task_steps/` (`deploy_env`, `deploy_agent`, `prompt_agent`, the verifiers under `verifiers/`, among them `start_computer_recording` and `end_computer_recording`, and more). On an env whose card serves computer-exec/v1 (a computer), `run_code` with `env_id` and `run_container_unit_tests_verifier` with `env_id` run inside it as its session user through the driver: files staged with `data/add` and read with `data/get`, commands run with `run`; `env_outcome_verifier` hands a `verify` that takes an `env_url` keyword the env's address. A computer's framework-only routes (everything but MCP and the card) take the per-deploy control token its provider boots it with (`DeployedEnv.control_token`, `AGENTENV_CONTROL_TOKEN`); framework calls send `deployed.control_headers`, `Task.run` runs every step under `protocol_v1.authorized` with each deployed env's (a `verify` included), and a `run_code` script inside the computer gets `input["env"]["headers"]`. `teardown_run` and the `teardown_sandboxes` step share one core (`task/teardown.py` `tear_down`): the recordings the run left open end with fresh grants, each env still up closes (`Env.close`; a computer env ends any recording still open), what stayed open is registered from its namespace, then every sandbox is terminated by id. `task/episode/` runs a task with some of its agents played from outside, as a training loop does (`EpisodeSpec`). | | `store/` | Four store ABCs with local and cloud implementations: `DocumentStore` (SQLite, MongoDB, DynamoDB, Firestore with MongoDB compatibility), `ObjectStore` (filesystem, S3, Cloud Storage), `ImageStore` (local OCI registry, ECR), `SecretStore` (env vars or file, AWS Secrets Manager, Google Cloud Secret Manager). `VersionedEntityStore` implements the shared versioned get/put logic, `QueryBuilder` is the immutable chained query API, `store/base.py` holds the error types. `store/routing.py` is the local namespaces, `@local` and `@hf` (`is_local_id`), each with a documents file of its own: while the CLI or a `build_app` server runs, a local id and what a local task's run records go to local stores beside the configured ones, and that run's writes to the configured stores are refused. A new backend must pass the conformance kits in `tst/store/`. | diff --git a/packages/agentenv-protocol/src/agentenv_protocol/website_browser/__init__.py b/packages/agentenv-protocol/src/agentenv_protocol/website_browser/__init__.py index bc0e84800..d40a684be 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/website_browser/__init__.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/website_browser/__init__.py @@ -2,9 +2,9 @@ its build installs and starts; this class adds what agentenv owns, the same in both: its flags after the subclass's command, the unattended-use policies, a CDP connection over the browser's debugging pipe, the settings every browser takes, a fresh cache and the deploy's prompt answers at every launch, another window for a launch while it runs, its -skill and ``browser/v1`` on its card. It runs alone, headless, on a computer with no display, a browser computer's, -whose screen is its page; which site proxy it goes through comes from a variable agentenv sets, never from its -settings.""" +skill and ``browser/v1`` on its card. It runs alone on a computer with no display, a browser computer's, whose screen +is its page, headful on an X server that computer gives it alone, else headless; which site proxy it goes through comes +from a variable agentenv sets, never from its settings.""" from .control import (ACTIVE_CHECK_TIMEOUT_S, AGENT, AGENTENV, CAPTURE_TIMEOUT_S, DOWNLOADS, EXIT, FAILED, HASH_CHUNK_BYTES, LIVE_START_TAB_TIMEOUT_S, LOADED, PAGE_INPUT, PAGES, PIN_TIMEOUT_S, STOP, TIMED_OUT, UNKNOWN, UNNAVIGATED_CAPTURE_TIMEOUT_S, UPLOADS, BrowserControl) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/website_browser/control.py b/packages/agentenv-protocol/src/agentenv_protocol/website_browser/control.py index b0da15399..65545482b 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/website_browser/control.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/website_browser/control.py @@ -26,7 +26,7 @@ AGENTENV, UNKNOWN, EXIT, STOP = "agentenv", "unknown", "exit", "stop" #: How long the active tab's check waits on each page. ACTIVE_CHECK_TIMEOUT_S = 2.0 -#: How long a tab opened headless may take to be attached and held to its window's size, which is also how long a page +#: How long a tab opened alone may take to be attached and held to its window's size, which is also how long a page #: of agentenv's own may take to reach an origin, and how often it's asked meanwhile. PIN_TIMEOUT_S = 10.0 ARRIVAL_POLL_S = 0.05 @@ -75,7 +75,7 @@ def __init__(self, devtools: DevToolsPipe, dialogs: str, viewport: Optional[tupl self._closing: set[str] = set() self._open_dialogs: dict[str, dict] = {} self._answering: set[str] = set() - #: The file chooser each page's session opened that waits, headless, and the tab agentenv last brought to the + #: The file chooser each page's session opened that waits, alone, and the tab agentenv last brought to the #: front. self._choosers: dict[str, dict] = {} self._front: Optional[str] = None @@ -132,7 +132,7 @@ async def tab_list(self) -> list[dict]: async def open_tab(self, url: Optional[str] = None, background: bool = False) -> str: """A new tab at ``url``, else blank, brought to the front, or with ``background`` behind the active tab, which - stays active, brought back only if the new tab took its place; its id. Headless, it opens blank and goes to + stays active, brought back only if the new tab took its place; its id. Alone, it opens blank and goes to ``url`` once it's held to the window's size, so its page loads at that size.""" active = await self.active_tab() if background else None held = self.viewport is not None and url @@ -151,7 +151,7 @@ async def open_tab(self, url: Optional[str] = None, background: bool = False) -> async def load_start_tabs(self, urls: list[str], site_of: Callable[[str], Optional[str]], own_timeout_s: float, live_timeout_s: float = LIVE_START_TAB_TIMEOUT_S) -> list[dict]: """Each of ``urls`` in a start tab, in order, the first then active: in the tabs open now, in opened order, and - in tabs it opens blank after them; each held to the window's size, headless, before it goes to its URL, and + in tabs it opens blank after them; each held to the window's size, alone, before it goes to its URL, and all loading at once. A tab settles once it has loaded, once it has failed, which on one of the env's own sites, as ``site_of`` names them, an error status is too, or once it has been waited on ``own_timeout_s`` on such a site, else ``live_timeout_s``. Each tab's outcome, in order, kept though the browser has gone and the first @@ -635,8 +635,8 @@ def _on_Target_attachedToTarget(self, params: dict, session: Optional[str]) -> N self._spawn(self._let_run(params["sessionId"])) async def _let_run(self, session: str) -> None: - """A page it attached to, enabled and, headless, held to its window's size, then told to run: a page attached as - it opens, a popup say, or any new page headless, waits for that before it runs.""" + """A page it attached to, enabled and, alone, held to its window's size, then told to run: a page attached as + it opens, a popup say, or any new page alone, waits for that before it runs.""" target = self._sessions.get(session) try: await self.devtools.call("Page.enable", session_id=session) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/website_browser/environment.py b/packages/agentenv-protocol/src/agentenv_protocol/website_browser/environment.py index 7be58a335..8b708ff2b 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/website_browser/environment.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/website_browser/environment.py @@ -42,7 +42,7 @@ #: port, since the browser is reached only over its pipe. OWNED_FLAGS = ("--user-data-dir", "--remote-debugging-", "--remote-allow-origins", "--proxy-", "--no-proxy-server", "--headless", "--window-size", "--lang", "--force-webrtc-ip-handling-policy", "--disk-cache-dir", - "--disable-dev-shm-usage") + "--disable-dev-shm-usage", "--enable-unsafe-swiftshader") #: Below this much /dev/shm, as Docker's 64 MiB default, the browser keeps its shared memory in files instead. SHM_MIN_BYTES = 1 << 30 #: Chromium's content setting that denies a permission without a prompt. @@ -163,7 +163,8 @@ def cache_dir(self) -> Path: def agentenv_flags(self) -> list[str]: """The flags agentenv owns. A browser alone goes out only through its site proxy, so it won't launch without - one; in a computer it goes through one when the computer has one, loopback aside.""" + one; in a computer it goes through one when the computer has one, loopback aside. Alone it runs headful on the + X server its computer gives it, where WebGL draws with SwiftShader since no GPU is there, else headless.""" settings, alone, proxy = self.settings, _alone(), os.environ.get(SITE_PROXY_ENV) flags = [f"--user-data-dir={self.profile_dir()}", f"--disk-cache-dir={self.cache_dir()}", "--remote-debugging-pipe", "--no-first-run", "--no-default-browser-check", f"--lang={settings.locale}"] @@ -179,8 +180,7 @@ def agentenv_flags(self) -> list[str]: "names none") if alone: flags.append(f"--window-size={settings.window.width},{settings.window.height}") - if alone: - flags.append("--headless=new") + flags.append("--enable-unsafe-swiftshader" if _x_server() else "--headless=new") return flags def _launch_for(self, launch: Launch) -> Launch: @@ -304,7 +304,7 @@ async def start_tabs_loaded(self) -> bool: "is its window's, a PNG of it, and CDP input calls sent to it in order. navigate and history open a URL in " "the active tab, or go back, forward or reload; tabs/list, tabs/open, tabs/activate and tabs/close manage " "its tabs by their ids; dialog/answer answers the dialog open on the active tab; files/choose gives the file " - "chooser open on it files in ~/Uploads, headless; and page/state is the active tab's id, URL and title and " + "chooser open on it files in ~/Uploads, alone; and page/state is the active tab's id, URL and title and " "what is open on it. cookies/get, cookies/set and cookies/delete, and local_storage/get, local_storage/set and " "local_storage/clear, read and change its default context's cookies and local storage as Playwright's " "storageState has them, a cookie's max_age set as an expiry on the browser's clock; downloads/list and " @@ -600,7 +600,7 @@ async def _start_tabs_settled(self) -> list[dict]: def extension_params(self, uri: str) -> dict: if uri == BROWSER: - params: dict[str, Any] = {"chromium": self.version, "clock": "real"} + params: dict[str, Any] = {"chromium": self.version, "clock": "real", "mode": _mode()} if _alone(): params["window"] = self.settings.window.model_dump() return params @@ -650,10 +650,20 @@ def start_tab_why(tab: dict) -> str: def _alone() -> bool: - """Whether it runs on a computer with no display, a browser computer's, alone and headless.""" + """Whether it runs on a computer with no display, a browser computer's, alone: its page is that computer's screen.""" return computer_app.detect().display == "none" +def _x_server() -> bool: + """Whether an X server is there for it, as a browser computer gives its browser one no agent sees.""" + return bool(os.environ.get("DISPLAY")) + + +def _mode() -> str: + """How it runs: headless alone with no X server, else headful, on its own X server alone or on the desktop.""" + return "headless" if _alone() and not _x_server() else "headful" + + def _parsed(model: type[BaseModel], items: Any, what: str) -> list: """Each of ``items`` as ``model``; InvalidArgument naming the fields that don't fit and never their values, which for a cookie are its credentials.""" diff --git a/packages/agentenv-protocol/tests/test_website_browser.py b/packages/agentenv-protocol/tests/test_website_browser.py index d93ad9e27..b049b4051 100644 --- a/packages/agentenv-protocol/tests/test_website_browser.py +++ b/packages/agentenv-protocol/tests/test_website_browser.py @@ -1,5 +1,5 @@ -"""WebsiteBrowserEnvironment: agentenv's flags after a subclass's command, in a computer and alone, and the proxy rule -for each; a subclass refused for carrying one of them; the start tabs, the time zone and the window from its settings; +"""WebsiteBrowserEnvironment: agentenv's flags after a subclass's command, in a computer and alone, headful on an X +server alone or headless with none, and the proxy rule for each; a subclass refused for carrying one of them; the start tabs, the time zone and the window from its settings; its launch run with no desktop when alone; a CDP connection over a new debugging pipe at each launch, which its ready check asks, and a browser launched again when its server restarts; its shared memory in files only on a small /dev/shm; an empty cache and the deploy's prompt answers in its profile at each launch; its policies, card params and @@ -121,9 +121,13 @@ def _settings(monkeypatch, **values) -> None: monkeypatch.setenv(APP_SETTINGS_ENV, json.dumps({"chromium": values})) -def _alone(monkeypatch) -> None: - """On a computer with no display, a browser computer's.""" +def _alone(monkeypatch, x_server: bool = False) -> None: + """On a computer with no display, a browser computer's, with an X server only the browser uses or none.""" monkeypatch.setattr(app, "detect", lambda: Platform(os="linux", display="none")) + if x_server: + monkeypatch.setenv("DISPLAY", ":99") + else: + monkeypatch.delenv("DISPLAY", raising=False) def test_in_a_computer_its_flags_follow_its_command_then_its_start_tabs_and_its_window_is_waited_for(browser_home): @@ -161,6 +165,21 @@ def test_alone_it_goes_through_its_proxy_headless_at_its_window_and_its_launch_w assert browser._computer_app.desktop is False and browser._computer_app.timeout_s == 30 +def test_alone_on_an_x_server_it_runs_headful_at_its_window_with_webgl_and_its_launch_waits_on_no_window(monkeypatch): + _alone(monkeypatch, x_server=True) + monkeypatch.setenv(SITE_PROXY_ENV, PROXY) + _settings(monkeypatch, window={"width": 1024, "height": 768}) + + browser = Browser() + launch = browser.launch + + assert f"--proxy-bypass-list={ALONE_BYPASS}" in launch.command + assert launch.command[-3:] == ["--window-size=1024,768", "--enable-unsafe-swiftshader", "about:blank"] + assert not any(flag.startswith("--headless") for flag in launch.command) + assert launch.window is None and launch.ready is None + assert browser._computer_app.desktop is False + + def test_alone_its_launch_keeps_the_port_and_command_it_waits_on_and_their_timeout(monkeypatch): @linux(launch=Launch(command=["chromium/chrome"], window={"class": "Chromium"}, ready=Ready(window=True, port=9000, command=["true"], timeout_s=45))) @@ -186,7 +205,8 @@ def test_alone_it_refuses_to_launch_without_a_site_proxy(monkeypatch): @pytest.mark.parametrize("flag", ["--remote-debugging-port=9222", "--remote-allow-origins=*", "--headless", "--proxy-server=http://elsewhere:3128", "--user-data-dir=/tmp/profile", - "--disk-cache-dir=/tmp/cache", "--disable-dev-shm-usage"]) + "--disk-cache-dir=/tmp/cache", "--disable-dev-shm-usage", + "--enable-unsafe-swiftshader"]) def test_a_subclass_whose_command_carries_a_flag_agentenv_owns_is_refused_when_declared(flag): @linux(launch=Launch(command=["chromium/chrome", flag])) class Widened(WebsiteBrowserEnvironment): @@ -434,14 +454,18 @@ def test_its_policies_are_agentenvs_with_its_own_over_them_where_its_build_reads assert [step.name for step in Browser.declaration().install_steps] == ["write_policies"] -def test_its_card_declares_browser_v1_with_its_build_and_clock_and_alone_its_window(monkeypatch): +def test_its_card_declares_browser_v1_with_its_build_clock_and_mode_and_alone_its_window(monkeypatch): in_computer = Browser().extension_params(BROWSER) - _alone(monkeypatch) monkeypatch.setenv(SITE_PROXY_ENV, PROXY) - alone = Browser().extension_params(BROWSER) - - assert in_computer == {"chromium": "156.0.8078.4", "clock": "real"} - assert alone == {"chromium": "156.0.8078.4", "clock": "real", "window": {"width": 1280, "height": 800}} + _alone(monkeypatch) + headless = Browser().extension_params(BROWSER) + _alone(monkeypatch, x_server=True) + headful = Browser().extension_params(BROWSER) + + window = {"width": 1280, "height": 800} + assert in_computer == {"chromium": "156.0.8078.4", "clock": "real", "mode": "headful"} + assert headless == {"chromium": "156.0.8078.4", "clock": "real", "mode": "headless", "window": window} + assert headful == {"chromium": "156.0.8078.4", "clock": "real", "mode": "headful", "window": window} @pytest.mark.asyncio diff --git a/src/agent_env/browsers/README.md b/src/agent_env/browsers/README.md index d2a807398..b9b9ed31c 100644 --- a/src/agent_env/browsers/README.md +++ b/src/agent_env/browsers/README.md @@ -1,7 +1,7 @@ The website browsers that come with agent-env, today `chromium`, a `computer_app` env a computer env lists in its -`apps`. Deployed alone, `chromium` is a browser computer, `chromium-driver`: the `ubuntu-24.04-headless` base, with no -display, and the `page` driver, whose screen is the browser's active tab, headless; the browser on it keeps its name, -`chromium`. +`apps`. Deployed alone, `chromium` is a browser computer, `chromium-driver`: the `ubuntu-24.04-browser` base, with no +desktop, and the `page` driver, whose screen is the browser's active tab; the browser runs headful there, on an X +server only it uses, and keeps its name, `chromium`. `envs/chromium/` is Chromium for an X11 desktop or a computer with no display on Linux (`amd64` and `arm64`): Chrome for Testing 156.0.8078.4, the build Playwright 1.64.1 pins, fetched by sha256 when a computer env that lists it is put and unpacked into diff --git a/src/agent_env/browsers/envs/chromium/env.toml b/src/agent_env/browsers/envs/chromium/env.toml index 05625f6a1..5216351b4 100644 --- a/src/agent_env/browsers/envs/chromium/env.toml +++ b/src/agent_env/browsers/envs/chromium/env.toml @@ -1,6 +1,6 @@ type = "computer_app" module = "chromium.py:Chromium" -computer = "ubuntu-24.04-headless" +computer = "ubuntu-24.04-browser" driver = "page" [settings] diff --git a/src/agent_env/cli/run.py b/src/agent_env/cli/run.py index 5052e5209..fba1f76bc 100644 --- a/src/agent_env/cli/run.py +++ b/src/agent_env/cli/run.py @@ -86,7 +86,7 @@ def run(ctx: click.Context, bundle: str | None, tasks: tuple[str, ...], evals: t --set sets a key of one of the bundle's envs for this run, as its env.toml would: --set envs/desktop.driver=cua-driver runs the env in envs/desktop with that driver, and --set - envs/desktop.computer=ubuntu-24.04-headless composes it on that base. The envs it changes, the envs and tasks + envs/desktop.computer=ubuntu-24.04-browser composes it on that base. The envs it changes, the envs and tasks that name them and the evals that name those tasks are written under ids of their own, ending in the settings, such as desktop(desktop.driver cua-driver), and compose and reuse their images as any env does, leaving the bundle as written. A key or value the env's type refuses, or a run whose tasks deploy no env it changes, is diff --git a/src/agent_env/computers/README.md b/src/agent_env/computers/README.md index 38125d0bb..f1a450fcb 100644 --- a/src/agent_env/computers/README.md +++ b/src/agent_env/computers/README.md @@ -1,5 +1,5 @@ Base computers that come with agent-env, each a `computer` artifact a computer env composes a driver and apps onto. -`agent-env run --set envs/desktop.computer=ubuntu-24.04-headless` composes a bundle's `desktop` env on +`agent-env run --set envs/desktop.computer=ubuntu-24.04-browser` composes a bundle's `desktop` env on another of them for one run, the bundle left as written. `artifacts/ubuntu-24.04/` is Ubuntu 24.04 (`amd64` and `arm64`) with XFCE on Xvfb, D-Bus and AT-SPI. tini starts @@ -10,16 +10,20 @@ container form's image is built from the folder's `Dockerfile`, and a release pu building it. Name it from another bundle as `ubuntu-24.04`, the alias `aliases.toml` gives it, or by its id, `@local/agentenv-framework/computers/ubuntu-24.04`. -`artifacts/ubuntu-24.04-headless/` is Ubuntu 24.04 (`amd64` and `arm64`) with no display server or desktop -(`display = "none"`), for one website browser, which runs headless there, and the `page` driver, whose screen is the -browser's page: a browser computer, which agentenv's `chromium` deploys alone as. supervisord runs every program as the -session user, among them a stand-in site proxy on `127.0.0.1:3128`, the browser's only route out when the computer +`artifacts/ubuntu-24.04-browser/` is Ubuntu 24.04 (`amd64` and `arm64`) with no desktop (`display = "none"`), for one +website browser and the `page` driver, whose screen is the browser's page: a browser computer, which agentenv's +`chromium` deploys alone as. The browser runs headful on an X server of its own, Xvfb on `:99` at 1920x1080, which no +agent sees, records or drives, so the sites it visits see a browser with a window, not a headless one; the page's size +is the browser's `window` setting. supervisord runs every program as the session user, among them that X server and a +stand-in site proxy on `127.0.0.1:3128`, the browser's only route out when the computer routes no sites. It forwards a request or a `CONNECT` tunnel to any host, never decrypting a tunnel and refusing nothing: the sandbox's network policy is all that limits it. Every line it writes to `/var/log/agentenv/site-proxy.log` is JSON, one per connection and one per connection that fails, in the same shapes as the shared site proxy's. A computer that routes sites names the shared site proxy's forward port, `127.0.0.1:3129`, to the browser instead: it serves the run's sites, refuses every other host and runs out of the agent's reach. It has a -container form only. Name it from another bundle as `ubuntu-24.04-headless`, or by its id, -`@local/agentenv-framework/computers/ubuntu-24.04-headless`. +container form only. Name it from another bundle as `ubuntu-24.04-browser`, or by its id, +`@local/agentenv-framework/computers/ubuntu-24.04-browser`. It was `ubuntu-24.04-headless`, a name nothing resolves +any more, so what still names it fails rather than gets a headful browser; its image, `agentenv-ubuntu-24.04-headless`, +stays published for what pins it. It also has a disk form, a disk a VM boots from UEFI, which is the container form's root filesystem on the Linux boot recipe in `disk/linux/`: a GPT disk of an EFI system partition holding one UKI, Ubuntu's kernel with a busybox initramfs, diff --git a/src/agent_env/computers/aliases.toml b/src/agent_env/computers/aliases.toml index 81e385b56..5e3e33252 100644 --- a/src/agent_env/computers/aliases.toml +++ b/src/agent_env/computers/aliases.toml @@ -1,7 +1,7 @@ # Ids a task can name these computers by; a store's artifact of the id wins over an alias. [artifacts] "ubuntu-24.04" = ["ubuntu-24.04"] -"ubuntu-24.04-headless" = ["ubuntu-24.04-headless"] +"ubuntu-24.04-browser" = ["ubuntu-24.04-browser"] "macos-26" = ["macos-26"] "windows-11" = ["windows-11"] "omarchy-4.5" = ["omarchy-4.5"] diff --git a/src/agent_env/computers/artifacts/ubuntu-24.04-headless/Dockerfile b/src/agent_env/computers/artifacts/ubuntu-24.04-browser/Dockerfile similarity index 69% rename from src/agent_env/computers/artifacts/ubuntu-24.04-headless/Dockerfile rename to src/agent_env/computers/artifacts/ubuntu-24.04-browser/Dockerfile index 2314fdd2b..bac2e1a7c 100644 --- a/src/agent_env/computers/artifacts/ubuntu-24.04-headless/Dockerfile +++ b/src/agent_env/computers/artifacts/ubuntu-24.04-browser/Dockerfile @@ -1,13 +1,15 @@ -# The ubuntu-24.04-headless base computer: Ubuntu 24.04 with no display server or desktop (XDG_SESSION_TYPE=none), -# for one website browser, which runs headless there, and a page driver whose screen is the browser's page. supervisord -# runs every program as the session user, and the browser goes out only through the site proxy on 127.0.0.1:3128. +# The ubuntu-24.04-browser base computer: Ubuntu 24.04 with no desktop (XDG_SESSION_TYPE=none), for one website +# browser and a page driver whose screen is the browser's page. The browser runs headful on an X server of its own, +# Xvfb on :99, which no agent sees, records or drives, so sites see a browser with a window rather than a headless one. +# supervisord runs every program as the session user, and the browser goes out only through the site proxy on +# 127.0.0.1:3128. # Files a page's file chooser may be given go in ~/Uploads. Composing a driver or an app onto it adds its services as # programs in /etc/supervisor/conf.d. The trust files agentenv adds are in /etc/agentenv from boot: ca.pem, empty until # a deploy adds a CA, and ca-bundle.pem, the public roots and those CAs. FROM public.ecr.aws/docker/library/ubuntu:24.04@sha256:24737cef17f4e22b1977df7d9a05e77c4e28b7da5b8911f8d8ccda136bf3573c ENV DEBIAN_FRONTEND=noninteractive RUN apt-get update && apt-get install -y --no-install-recommends \ - supervisor tini python3 python3-venv fonts-dejavu-core tzdata ca-certificates procps \ + supervisor tini python3 python3-venv fonts-dejavu-core tzdata ca-certificates procps xvfb \ && rm -rf /var/lib/apt/lists/* \ && userdel --remove ubuntu \ && useradd --create-home --uid 1000 --shell /bin/bash user \ @@ -21,6 +23,6 @@ COPY start-computer /usr/local/bin/start-computer COPY site-proxy.py /usr/local/bin/agentenv-site-proxy USER user WORKDIR /home/user -ENV LANG=C.UTF-8 XDG_SESSION_TYPE=none AGENTENV_SITE_PROXY=http://127.0.0.1:3128 +ENV LANG=C.UTF-8 XDG_SESSION_TYPE=none DISPLAY=:99 AGENTENV_SITE_PROXY=http://127.0.0.1:3128 ENTRYPOINT ["/usr/bin/tini", "--"] CMD ["/usr/local/bin/start-computer"] diff --git a/src/agent_env/computers/artifacts/ubuntu-24.04-headless/artifact.toml b/src/agent_env/computers/artifacts/ubuntu-24.04-browser/artifact.toml similarity index 61% rename from src/agent_env/computers/artifacts/ubuntu-24.04-headless/artifact.toml rename to src/agent_env/computers/artifacts/ubuntu-24.04-browser/artifact.toml index dfcd4d670..5ed1e677b 100644 --- a/src/agent_env/computers/artifacts/ubuntu-24.04-headless/artifact.toml +++ b/src/agent_env/computers/artifacts/ubuntu-24.04-browser/artifact.toml @@ -1,5 +1,5 @@ type = "computer" -description = "Ubuntu 24.04 with no display, for one website browser that runs headless and a page driver whose screen is its page; run by supervisord as the session user" +description = "Ubuntu 24.04 with no desktop, for one website browser, headful on an X server only it uses, and a page driver whose screen is its page; run by supervisord as the session user" arch = ["amd64", "arm64"] os = { type = "linux", distribution = "ubuntu", version = "24.04", display = "none" } session = { user = "user" } diff --git a/src/agent_env/computers/artifacts/ubuntu-24.04-headless/site-proxy.py b/src/agent_env/computers/artifacts/ubuntu-24.04-browser/site-proxy.py similarity index 100% rename from src/agent_env/computers/artifacts/ubuntu-24.04-headless/site-proxy.py rename to src/agent_env/computers/artifacts/ubuntu-24.04-browser/site-proxy.py diff --git a/src/agent_env/computers/artifacts/ubuntu-24.04-headless/start-computer b/src/agent_env/computers/artifacts/ubuntu-24.04-browser/start-computer similarity index 100% rename from src/agent_env/computers/artifacts/ubuntu-24.04-headless/start-computer rename to src/agent_env/computers/artifacts/ubuntu-24.04-browser/start-computer diff --git a/src/agent_env/computers/artifacts/ubuntu-24.04-browser/supervisord.conf b/src/agent_env/computers/artifacts/ubuntu-24.04-browser/supervisord.conf new file mode 100644 index 000000000..1e251d8fa --- /dev/null +++ b/src/agent_env/computers/artifacts/ubuntu-24.04-browser/supervisord.conf @@ -0,0 +1,37 @@ +; The base computer's own programs, the website browser's X server and the site proxy, run as the session user. A +; composed driver or app adds its own in conf.d. +[supervisord] +nodaemon=true +logfile=/var/log/agentenv/supervisord.log +pidfile=/run/agentenv/supervisord.pid +childlogdir=/var/log/agentenv + +[unix_http_server] +file=/run/agentenv/supervisor.sock + +[rpcinterface:supervisor] +supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface + +[supervisorctl] +serverurl=unix:///run/agentenv/supervisor.sock + +; The browser's own X server, at a common desktop size: the page's own size is the browser's window setting, which it +; holds the page to whatever the screen. supervisord runs one X at a time, so a lock is one a killed X left, which X +; refuses to start over while the pid it names is taken. +[program:xvfb] +command=/bin/sh -c 'rm -f /tmp/.X99-lock; exec Xvfb :99 -screen 0 1920x1080x24 -nolisten tcp -ac' +priority=5 +autorestart=true +startsecs=1 +redirect_stderr=true +stdout_logfile=/var/log/agentenv/xvfb.log + +[program:site-proxy] +command=/usr/bin/python3 /usr/local/bin/agentenv-site-proxy +priority=10 +autorestart=true +redirect_stderr=true +stdout_logfile=/var/log/agentenv/site-proxy.log + +[include] +files = /etc/supervisor/conf.d/*.conf diff --git a/src/agent_env/computers/artifacts/ubuntu-24.04-headless/supervisord.conf b/src/agent_env/computers/artifacts/ubuntu-24.04-headless/supervisord.conf deleted file mode 100644 index da2860323..000000000 --- a/src/agent_env/computers/artifacts/ubuntu-24.04-headless/supervisord.conf +++ /dev/null @@ -1,26 +0,0 @@ -; The base computer's own program, the site proxy, run as the session user. A composed driver or app adds its own in -; conf.d. -[supervisord] -nodaemon=true -logfile=/var/log/agentenv/supervisord.log -pidfile=/run/agentenv/supervisord.pid -childlogdir=/var/log/agentenv - -[unix_http_server] -file=/run/agentenv/supervisor.sock - -[rpcinterface:supervisor] -supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface - -[supervisorctl] -serverurl=unix:///run/agentenv/supervisor.sock - -[program:site-proxy] -command=/usr/bin/python3 /usr/local/bin/agentenv-site-proxy -priority=10 -autorestart=true -redirect_stderr=true -stdout_logfile=/var/log/agentenv/site-proxy.log - -[include] -files = /etc/supervisor/conf.d/*.conf diff --git a/src/agent_env/drivers/README.md b/src/agent_env/drivers/README.md index 04ab85777..cf8b845c0 100644 --- a/src/agent_env/drivers/README.md +++ b/src/agent_env/drivers/README.md @@ -13,7 +13,7 @@ pointer and keyboard groups, and `computer-ready`, `computer-display`, `computer `x11grab` through PyAV at up to 30 fps. Name it from another bundle as `reference`, the alias `aliases.toml` gives it, or by its id, `@local/agentenv-framework/drivers/reference`. -`envs/page/` is the page driver, for a computer with no display on Linux (`amd64` and `arm64`), such as `ubuntu-24.04-headless`: +`envs/page/` is the page driver, for a computer with no display on Linux (`amd64` and `arm64`), such as `ubuntu-24.04-browser`: its screen is the computer's one website browser's active tab, its `screen` feature `page`. A screenshot is the page, and input reaches it as CDP input events, both through `browser/v1`'s `page/*` operations on the browser's own server; a tab that paints nothing shows white. It serves the page group too: `computer_navigate`, `computer_history`, diff --git a/src/agent_env/drivers/envs/page/driver.py b/src/agent_env/drivers/envs/page/driver.py index ca1fe8d3f..fdbc8ae68 100644 --- a/src/agent_env/drivers/envs/page/driver.py +++ b/src/agent_env/drivers/envs/page/driver.py @@ -1,4 +1,4 @@ -"""The page driver, for a computer with no display whose one website browser runs headless: its screen is the browser's +"""The page driver, for a computer with no display whose one website browser runs alone: its screen is the browser's active tab. A screenshot is the page, and input reaches the page as CDP input events, both through the browser's own server on its socket (``browser/v1``'s ``page/*`` operations), since only that server holds the browser's debugging pipe. The browser ending on its own is an infrastructure failure, since nothing here launches it again. Needs diff --git a/src/agent_env/env/published_images.py b/src/agent_env/env/published_images.py index 58e6773b3..e65744d74 100644 --- a/src/agent_env/env/published_images.py +++ b/src/agent_env/env/published_images.py @@ -72,8 +72,8 @@ def ref(self) -> str: EXAMPLE_IMAGES = (PublishedImage("agentenv-mail-mcp", _EXAMPLES / "mail-mcp" / "envs" / "mail" / "Dockerfile"),) # The base computers the computers bundle ships, so a run pulls a base rather than building its desktop. COMPUTER_IMAGES = (PublishedImage("agentenv-ubuntu-24.04", _COMPUTERS / "ubuntu-24.04" / "Dockerfile"), - PublishedImage("agentenv-ubuntu-24.04-headless", - _COMPUTERS / "ubuntu-24.04-headless" / "Dockerfile")) + PublishedImage("agentenv-ubuntu-24.04-browser", + _COMPUTERS / "ubuntu-24.04-browser" / "Dockerfile")) # The shipped bundles' published images, whose requirements keep a protocol floor. BUNDLE_IMAGES = AGENT_IMAGES + EXAMPLE_IMAGES + COMPUTER_IMAGES # What makes and boots a Linux disk form: the disk tools a build runs on its own platform, and the QEMU launcher. diff --git a/tst/integration/computer/browser_computer_test.py b/tst/integration/computer/browser_computer_test.py index d24312ce2..9662419b5 100644 --- a/tst/integration/computer/browser_computer_test.py +++ b/tst/integration/computer/browser_computer_test.py @@ -1,10 +1,11 @@ -"""The browser as a computer: agentenv's chromium deployed alone, on the ubuntu-24.04-headless base with no display -and the page driver, whose screen is chromium's active tab, headless. Gate 3 makes this machine's arch ready with it, +"""The browser as a computer: agentenv's chromium deployed alone, on the ubuntu-24.04-browser base with no desktop +and the page driver, whose screen is chromium's active tab, headful on the X server only the browser uses. Gate 3 makes this machine's arch ready with it, its input checks and page tools verified on the HTML desk; deployed through the computer provider like any computer, its computer_* tools see and drive the page, and its page tools navigate, move in history, answer a dialog, use tabs -and upload; nothing runs X and nothing listens but the driver and the site proxy; a killed app server or driver is -started again by supervisord and the computer is ready again; and chromium killed fails it for good, computer-ready/v1 -saying why and the tools answering with it. Through its cdp address, Playwright and Puppeteer drive it two at once, +and upload; the browser's X server runs but no desktop, and nothing listens but the driver and the site proxy; a +killed app server or driver is started again by supervisord and the computer is ready again; chromium killed fails it +for good, computer-ready/v1 saying why and the tools answering with it, and so does its X server killed, which takes +chromium with it though supervisord starts X again. Through its cdp address, Playwright and Puppeteer drive it two at once, each on its own session, by http_url and by ws_url; a wrong token is refused, and so is code in the computer without one; one client's close leaves the browser up for the other, which can't open file: URLs; the address outlives a relaunch by its app server's restart, and the browser's reset and the computer's end it. browser_connect_cdp hands the echo agent its address, with the computer's @@ -148,8 +149,8 @@ def log_message(self, *args): fields = line.split() if fields[3] == "0A": ports.add(int(fields[1].rsplit(":", 1)[1], 16)) -x = [p for p in glob.glob("/proc/[0-9]*/comm") if open(p).read().strip() in ("Xvfb", "Xorg", "xfce4-session")] -print(sorted(ports), bool(x)) +x = sorted({open(p).read().strip() for p in glob.glob("/proc/[0-9]*/comm")} & {"Xvfb", "Xorg", "xfce4-session"}) +print(sorted(ports), ",".join(x)) """ @@ -207,8 +208,9 @@ async def test_its_tools_see_and_drive_the_page_that_is_its_screen(browser_compu assert clicked["c"] == 1 and doubled["d"] == 1 and typed["v"] == "Hello, World!" and replaced["v"] == "x" assert "C-a" in replaced["k"] and "Enter" in entered["k"] and scrolled["y"] >= 300 assert after.getpixel((200, 150)) != (255, 0, 0) - assert json.loads(listening) == [3128, 18765] and x_server.strip() == "False" - assert "--headless=new" in command and "--window-size=1280,800" in command + assert json.loads(listening) == [3128, 18765] and x_server.strip() == "Xvfb" + assert "--headless" not in command and "--window-size=1280,800" in command + assert "--enable-unsafe-swiftshader" in command assert not report.failed @@ -243,6 +245,28 @@ async def test_a_killed_app_server_or_driver_comes_back_and_a_killed_browser_fai assert not report.failed +@pytest.mark.asyncio +async def test_a_killed_x_server_comes_back_but_takes_the_browser_with_it_which_fails_the_computer_for_good( + browser_computer): + with namespace_routing(): + deployed = await Env.get(browser_computer.id).deploy( + env_params={"app_settings": {"chromium": {"start_urls": [PAGE_URL]}}}) + context = TaskStepContext(deployed_envs=[deployed]) + try: + before = await _screenshot(deployed) + await _kill(deployed, "^Xvfb :99", oldest=True) + failed = await _until(deployed, lambda status: "failed" in status) + x_back = await _until_x(deployed) + finally: + report = await teardown_run(context) + + assert before.size == (1280, 800) + browser = next(check for check in failed["checks"] if check["name"] == "browser") + assert failed["ready"] is False and browser["final"] is True and "infrastructure failure" in browser["detail"] + assert x_back + assert not report.failed + + @pytest.mark.asyncio async def test_an_agent_navigates_moves_in_history_answers_a_dialog_uses_tabs_and_uploads_with_its_page_tools( browser_computer): @@ -526,6 +550,17 @@ async def _ready(deployed) -> dict: return await _until(deployed, lambda status: status["ready"]) +async def _until_x(deployed) -> bool: + """Whether the browser's X server, which supervisord starts again, answers within the wait.""" + deadline = time.monotonic() + WAIT_S + while time.monotonic() < deadline: + if (await inside.run(deployed, ["sh", "-c", "pgrep -x Xvfb >/dev/null && test -S /tmp/.X11-unix/X99 && " + "echo up || echo down"], timeout_s=30))["stdout"].strip() == "up": + return True + await asyncio.sleep(POLL_S) + return False + + #: Loads its URL through the proxy it's given, as a browser alone does, printing the status, the seconds it took and the #: first line of the body, or the error. THROUGH_PROXY = """import sys, time, urllib.request diff --git a/tst/integration/computer/conftest.py b/tst/integration/computer/conftest.py index a95538ded..d18e9fcb6 100644 --- a/tst/integration/computer/conftest.py +++ b/tst/integration/computer/conftest.py @@ -83,7 +83,7 @@ def website_browser_desktop(tmp_path_factory): @pytest.fixture(scope="module") def browser_computer(tmp_path_factory): - """agentenv's chromium, which deploys alone as a browser computer: the ubuntu-24.04-headless base, with no display, and the + """agentenv's chromium, which deploys alone as a browser computer: the ubuntu-24.04-browser base, with no desktop, and the page driver, whose screen is the browser's page; composed, through the browser-computer test bundle's task that deploys it, as ``desktop`` is.""" yield from _composed(tmp_path_factory.mktemp("browser-computer"), parse_bundle(BROWSER_COMPUTER), CHROMIUM_ALONE) diff --git a/tst/unit/bundle/variants_test.py b/tst/unit/bundle/variants_test.py index 88a62bdff..adb6159e3 100644 --- a/tst/unit/bundle/variants_test.py +++ b/tst/unit/bundle/variants_test.py @@ -87,9 +87,9 @@ def test_a_reference_by_id_is_repointed_at_the_variant(tmp_path): steps = [{**DEPLOY, "env_id": f"{ROOT}/desktop"}, {**AGENT, "env_ids": [f"{ROOT}/desktop"]}, PROMPT] root = _bundle(tmp_path / "desk", steps, {"evals/e.toml": f'tasks = ["{ROOT}/t"]\n'}) - bundle = _varied(root, "envs/desktop.computer=ubuntu-24.04-headless", "envs/desktop.tools=standard") + bundle = _varied(root, "envs/desktop.computer=ubuntu-24.04-browser", "envs/desktop.tools=standard") - label = "desktop.computer ubuntu-24.04-headless, desktop.tools standard" + label = "desktop.computer ubuntu-24.04-browser, desktop.tools standard" assert [step.get("env_id") or step.get("env_ids") for step in _entry(bundle, "t").config] == [ f"{ROOT}/desktop({label})", [f"{ROOT}/desktop({label})"], None] assert _entry(bundle, "e").config["tasks"] == [f"{ROOT}/t({label})"] diff --git a/tst/unit/env/envs/test_computer_stand_in_proxy.py b/tst/unit/env/envs/test_computer_stand_in_proxy.py index 7e7728ec5..9671ad635 100644 --- a/tst/unit/env/envs/test_computer_stand_in_proxy.py +++ b/tst/unit/env/envs/test_computer_stand_in_proxy.py @@ -22,13 +22,13 @@ pytestmark = pytest.mark.usefixtures("socket_enabled") -STAND_IN = Path(agent_env.__file__).parent / "computers" / "artifacts" / "ubuntu-24.04-headless" / "site-proxy.py" +STAND_IN = Path(agent_env.__file__).parent / "computers" / "artifacts" / "ubuntu-24.04-browser" / "site-proxy.py" GITHUB = "140.82.112.3" OK = b"HTTP/1.1 200 OK\r\nContent-Length: 5\r\nConnection: close\r\n\r\nhello" def _stand_in(): - spec = importlib.util.spec_from_file_location("headless_site_proxy", STAND_IN) + spec = importlib.util.spec_from_file_location("browser_site_proxy", STAND_IN) module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) return module diff --git a/tst/unit/test_infra_images_script.py b/tst/unit/test_infra_images_script.py index da8dc78d7..864e5dacf 100644 --- a/tst/unit/test_infra_images_script.py +++ b/tst/unit/test_infra_images_script.py @@ -76,7 +76,7 @@ def test_plan_only_lists_the_infra_images_the_agents_images_or_the_disk_forms(sc assert "agentenv-gateway" in names["infra"] and bundles == [ "agentenv-browser-use", "agentenv-claude-code", "agentenv-codex", "agentenv-gemini", "agentenv-hermes", "agentenv-letta-code", "agentenv-opencode", "agentenv-openhands", "agentenv-pi", "agentenv-proxy", - "agentenv-pydantic-ai", "agentenv-mail-mcp", "agentenv-ubuntu-24.04", "agentenv-ubuntu-24.04-headless"] + "agentenv-pydantic-ai", "agentenv-mail-mcp", "agentenv-ubuntu-24.04", "agentenv-ubuntu-24.04-browser"] def test_plan_missing_lists_the_images_the_registry_lacks_or_refuses_to_show(script, monkeypatch, capsys):