feat(computer): a browser alone runs headful on an X server only it uses, on ubuntu-24.04-browser (FD-4972)

.github/dependabot.yml

@@ -57,7 +57,7 @@ updates:
       - "/src/agent_env/agents/agents/pydantic-ai"
       - "/src/agent_env/examples/mail-mcp/envs/mail"
       - "/src/agent_env/computers/artifacts/ubuntu-24.04"
-      - "/src/agent_env/computers/artifacts/ubuntu-24.04-headless"
+      - "/src/agent_env/computers/artifacts/ubuntu-24.04-browser"
       - "/src/agent_env/computers/disk/linux/boot"
       - "/src/agent_env/computers/disk/linux/tools"
       - "/src/agent_env/computers/disk/linux/launcher"

.github/scripts/infra_build.py

@@ -57,7 +57,7 @@
     "agentenv-pydantic-ai": "src/agent_env/agents/agents/pydantic-ai/Dockerfile",
     "agentenv-mail-mcp": "src/agent_env/examples/mail-mcp/envs/mail/Dockerfile",
     "agentenv-ubuntu-24.04": "src/agent_env/computers/artifacts/ubuntu-24.04/Dockerfile",
-    "agentenv-ubuntu-24.04-headless": "src/agent_env/computers/artifacts/ubuntu-24.04-headless/Dockerfile",
+    "agentenv-ubuntu-24.04-browser": "src/agent_env/computers/artifacts/ubuntu-24.04-browser/Dockerfile",
     "agentenv-disk-tools": "src/agent_env/computers/disk/linux/tools/Dockerfile",
     "agentenv-qemu-launcher": "src/agent_env/computers/disk/linux/launcher/Dockerfile",
     "agentenv-ubuntu-24.04-disk": "src/agent_env/computers/disk/linux/boot/Dockerfile",
@@ -125,7 +125,7 @@
                               " s=$(supervisorctl -c /etc/supervisor/supervisord.conf status);"
                               " echo \"$s\" | grep -q ' RUNNING ' && ! echo \"$s\" | grep -qv ' RUNNING ' && exit 0;"
                               " done; supervisorctl -c /etc/supervisor/supervisord.conf status; exit 1"],
-    "agentenv-ubuntu-24.04-headless": [
+    "agentenv-ubuntu-24.04-browser": [
         "docker", "run", "--rm", "--entrypoint", "sh", "{image}", "-c",
         "start-computer & for _ in $(seq 60); do sleep 1;"
         " s=$(supervisorctl -c /etc/supervisor/supervisord.conf status);"

packages/agentenv-protocol/src/agentenv_protocol/website_browser/__init__.py

@@ -2,9 +2,9 @@
 its build installs and starts; this class adds what agentenv owns, the same in both: its flags after the subclass's
 command, the unattended-use policies, a CDP connection over the browser's debugging pipe, the settings every browser
 takes, a fresh cache and the deploy's prompt answers at every launch, another window for a launch while it runs, its
-skill and ``browser/v1`` on its card. It runs alone, headless, on a computer with no display, a browser computer's,
-whose screen is its page; which site proxy it goes through comes from a variable agentenv sets, never from its
-settings."""
+skill and ``browser/v1`` on its card. It runs alone on a computer with no display, a browser computer's, whose screen
+is its page, headful on an X server that computer gives it alone, else headless; which site proxy it goes through comes
+from a variable agentenv sets, never from its settings."""
 from .control import (ACTIVE_CHECK_TIMEOUT_S, AGENT, AGENTENV, CAPTURE_TIMEOUT_S, DOWNLOADS, EXIT, FAILED,
                       HASH_CHUNK_BYTES, LIVE_START_TAB_TIMEOUT_S, LOADED, PAGE_INPUT, PAGES, PIN_TIMEOUT_S, STOP,
                       TIMED_OUT, UNKNOWN, UNNAVIGATED_CAPTURE_TIMEOUT_S, UPLOADS, BrowserControl)

packages/agentenv-protocol/src/agentenv_protocol/website_browser/control.py

@@ -26,7 +26,7 @@
 AGENTENV, UNKNOWN, EXIT, STOP = "agentenv", "unknown", "exit", "stop"
 #: How long the active tab's check waits on each page.
 ACTIVE_CHECK_TIMEOUT_S = 2.0
-#: How long a tab opened headless may take to be attached and held to its window's size, which is also how long a page
+#: How long a tab opened alone may take to be attached and held to its window's size, which is also how long a page
 #: of agentenv's own may take to reach an origin, and how often it's asked meanwhile.
 PIN_TIMEOUT_S = 10.0
 ARRIVAL_POLL_S = 0.05
@@ -75,7 +75,7 @@ def __init__(self, devtools: DevToolsPipe, dialogs: str, viewport: Optional[tupl
         self._closing: set[str] = set()
         self._open_dialogs: dict[str, dict] = {}
         self._answering: set[str] = set()
-        #: The file chooser each page's session opened that waits, headless, and the tab agentenv last brought to the
+        #: The file chooser each page's session opened that waits, alone, and the tab agentenv last brought to the
         #: front.
         self._choosers: dict[str, dict] = {}
         self._front: Optional[str] = None
@@ -132,7 +132,7 @@ async def tab_list(self) -> list[dict]:
 
     async def open_tab(self, url: Optional[str] = None, background: bool = False) -> str:
         """A new tab at ``url``, else blank, brought to the front, or with ``background`` behind the active tab, which
-        stays active, brought back only if the new tab took its place; its id. Headless, it opens blank and goes to
+        stays active, brought back only if the new tab took its place; its id. Alone, it opens blank and goes to
         ``url`` once it's held to the window's size, so its page loads at that size."""
         active = await self.active_tab() if background else None
         held = self.viewport is not None and url
@@ -151,7 +151,7 @@ async def open_tab(self, url: Optional[str] = None, background: bool = False) ->
     async def load_start_tabs(self, urls: list[str], site_of: Callable[[str], Optional[str]], own_timeout_s: float,
                               live_timeout_s: float = LIVE_START_TAB_TIMEOUT_S) -> list[dict]:
         """Each of ``urls`` in a start tab, in order, the first then active: in the tabs open now, in opened order, and
-        in tabs it opens blank after them; each held to the window's size, headless, before it goes to its URL, and
+        in tabs it opens blank after them; each held to the window's size, alone, before it goes to its URL, and
         all loading at once. A tab settles once it has loaded, once it has failed, which on one of the env's own sites,
         as ``site_of`` names them, an error status is too, or once it has been waited on ``own_timeout_s`` on such a
         site, else ``live_timeout_s``. Each tab's outcome, in order, kept though the browser has gone and the first
@@ -635,8 +635,8 @@ def _on_Target_attachedToTarget(self, params: dict, session: Optional[str]) -> N
             self._spawn(self._let_run(params["sessionId"]))
 
     async def _let_run(self, session: str) -> None:
-        """A page it attached to, enabled and, headless, held to its window's size, then told to run: a page attached as
-        it opens, a popup say, or any new page headless, waits for that before it runs."""
+        """A page it attached to, enabled and, alone, held to its window's size, then told to run: a page attached as
+        it opens, a popup say, or any new page alone, waits for that before it runs."""
         target = self._sessions.get(session)
         try:
             await self.devtools.call("Page.enable", session_id=session)

packages/agentenv-protocol/src/agentenv_protocol/website_browser/environment.py

@@ -42,7 +42,7 @@
 #: port, since the browser is reached only over its pipe.
 OWNED_FLAGS = ("--user-data-dir", "--remote-debugging-", "--remote-allow-origins", "--proxy-", "--no-proxy-server",
                "--headless", "--window-size", "--lang", "--force-webrtc-ip-handling-policy", "--disk-cache-dir",
-               "--disable-dev-shm-usage")
+               "--disable-dev-shm-usage", "--enable-unsafe-swiftshader")
 #: Below this much /dev/shm, as Docker's 64 MiB default, the browser keeps its shared memory in files instead.
 SHM_MIN_BYTES = 1 << 30
 #: Chromium's content setting that denies a permission without a prompt.
@@ -163,7 +163,8 @@ def cache_dir(self) -> Path:
 
     def agentenv_flags(self) -> list[str]:
         """The flags agentenv owns. A browser alone goes out only through its site proxy, so it won't launch without
-        one; in a computer it goes through one when the computer has one, loopback aside."""
+        one; in a computer it goes through one when the computer has one, loopback aside. Alone it runs headful on the
+        X server its computer gives it, where WebGL draws with SwiftShader since no GPU is there, else headless."""
         settings, alone, proxy = self.settings, _alone(), os.environ.get(SITE_PROXY_ENV)
         flags = [f"--user-data-dir={self.profile_dir()}", f"--disk-cache-dir={self.cache_dir()}",
                  "--remote-debugging-pipe", "--no-first-run", "--no-default-browser-check", f"--lang={settings.locale}"]
@@ -179,8 +180,7 @@ def agentenv_flags(self) -> list[str]:
                              "names none")
         if alone:
             flags.append(f"--window-size={settings.window.width},{settings.window.height}")
-        if alone:
-            flags.append("--headless=new")
+            flags.append("--enable-unsafe-swiftshader" if _x_server() else "--headless=new")
         return flags
 
     def _launch_for(self, launch: Launch) -> Launch:
@@ -304,7 +304,7 @@ async def start_tabs_loaded(self) -> bool:
         "is its window's, a PNG of it, and CDP input calls sent to it in order. navigate and history open a URL in "
         "the active tab, or go back, forward or reload; tabs/list, tabs/open, tabs/activate and tabs/close manage "
         "its tabs by their ids; dialog/answer answers the dialog open on the active tab; files/choose gives the file "
-        "chooser open on it files in ~/Uploads, headless; and page/state is the active tab's id, URL and title and "
+        "chooser open on it files in ~/Uploads, alone; and page/state is the active tab's id, URL and title and "
         "what is open on it. cookies/get, cookies/set and cookies/delete, and local_storage/get, local_storage/set and "
         "local_storage/clear, read and change its default context's cookies and local storage as Playwright's "
         "storageState has them, a cookie's max_age set as an expiry on the browser's clock; downloads/list and "
@@ -600,7 +600,7 @@ async def _start_tabs_settled(self) -> list[dict]:
 
     def extension_params(self, uri: str) -> dict:
         if uri == BROWSER:
-            params: dict[str, Any] = {"chromium": self.version, "clock": "real"}
+            params: dict[str, Any] = {"chromium": self.version, "clock": "real", "mode": _mode()}
             if _alone():
                 params["window"] = self.settings.window.model_dump()
             return params
@@ -650,10 +650,20 @@ def start_tab_why(tab: dict) -> str:
 
 
 def _alone() -> bool:
-    """Whether it runs on a computer with no display, a browser computer's, alone and headless."""
+    """Whether it runs on a computer with no display, a browser computer's, alone: its page is that computer's screen."""
     return computer_app.detect().display == "none"
 
 
+def _x_server() -> bool:
+    """Whether an X server is there for it, as a browser computer gives its browser one no agent sees."""
+    return bool(os.environ.get("DISPLAY"))
+
+
+def _mode() -> str:
+    """How it runs: headless alone with no X server, else headful, on its own X server alone or on the desktop."""
+    return "headless" if _alone() and not _x_server() else "headful"
+
+
 def _parsed(model: type[BaseModel], items: Any, what: str) -> list:
     """Each of ``items`` as ``model``; InvalidArgument naming the fields that don't fit and never their values, which
     for a cookie are its credentials."""

packages/agentenv-protocol/tests/test_website_browser.py

@@ -1,5 +1,5 @@
-"""WebsiteBrowserEnvironment: agentenv's flags after a subclass's command, in a computer and alone, and the proxy rule
-for each; a subclass refused for carrying one of them; the start tabs, the time zone and the window from its settings;
+"""WebsiteBrowserEnvironment: agentenv's flags after a subclass's command, in a computer and alone, headful on an X
+server alone or headless with none, and the proxy rule for each; a subclass refused for carrying one of them; the start tabs, the time zone and the window from its settings;
 its launch run with no desktop when alone; a CDP connection over a new debugging pipe at each launch, which its ready
 check asks, and a browser launched again when its server restarts; its shared memory in files only on a small
 /dev/shm; an empty cache and the deploy's prompt answers in its profile at each launch; its policies, card params and
@@ -121,9 +121,13 @@ def _settings(monkeypatch, **values) -> None:
     monkeypatch.setenv(APP_SETTINGS_ENV, json.dumps({"chromium": values}))
 
 
-def _alone(monkeypatch) -> None:
-    """On a computer with no display, a browser computer's."""
+def _alone(monkeypatch, x_server: bool = False) -> None:
+    """On a computer with no display, a browser computer's, with an X server only the browser uses or none."""
     monkeypatch.setattr(app, "detect", lambda: Platform(os="linux", display="none"))
+    if x_server:
+        monkeypatch.setenv("DISPLAY", ":99")
+    else:
+        monkeypatch.delenv("DISPLAY", raising=False)
 
 
 def test_in_a_computer_its_flags_follow_its_command_then_its_start_tabs_and_its_window_is_waited_for(browser_home):
@@ -161,6 +165,21 @@ def test_alone_it_goes_through_its_proxy_headless_at_its_window_and_its_launch_w
     assert browser._computer_app.desktop is False and browser._computer_app.timeout_s == 30
 
 
+def test_alone_on_an_x_server_it_runs_headful_at_its_window_with_webgl_and_its_launch_waits_on_no_window(monkeypatch):
+    _alone(monkeypatch, x_server=True)
+    monkeypatch.setenv(SITE_PROXY_ENV, PROXY)
+    _settings(monkeypatch, window={"width": 1024, "height": 768})
+
+    browser = Browser()
+    launch = browser.launch
+
+    assert f"--proxy-bypass-list={ALONE_BYPASS}" in launch.command
+    assert launch.command[-3:] == ["--window-size=1024,768", "--enable-unsafe-swiftshader", "about:blank"]
+    assert not any(flag.startswith("--headless") for flag in launch.command)
+    assert launch.window is None and launch.ready is None
+    assert browser._computer_app.desktop is False
+
+
 def test_alone_its_launch_keeps_the_port_and_command_it_waits_on_and_their_timeout(monkeypatch):
     @linux(launch=Launch(command=["chromium/chrome"], window={"class": "Chromium"},
                          ready=Ready(window=True, port=9000, command=["true"], timeout_s=45)))
@@ -186,7 +205,8 @@ def test_alone_it_refuses_to_launch_without_a_site_proxy(monkeypatch):
 
 @pytest.mark.parametrize("flag", ["--remote-debugging-port=9222", "--remote-allow-origins=*", "--headless",
                                   "--proxy-server=http://elsewhere:3128", "--user-data-dir=/tmp/profile",
-                                  "--disk-cache-dir=/tmp/cache", "--disable-dev-shm-usage"])
+                                  "--disk-cache-dir=/tmp/cache", "--disable-dev-shm-usage",
+                                  "--enable-unsafe-swiftshader"])
 def test_a_subclass_whose_command_carries_a_flag_agentenv_owns_is_refused_when_declared(flag):
     @linux(launch=Launch(command=["chromium/chrome", flag]))
     class Widened(WebsiteBrowserEnvironment):
@@ -434,14 +454,18 @@ def test_its_policies_are_agentenvs_with_its_own_over_them_where_its_build_reads
     assert [step.name for step in Browser.declaration().install_steps] == ["write_policies"]
 
 
-def test_its_card_declares_browser_v1_with_its_build_and_clock_and_alone_its_window(monkeypatch):
+def test_its_card_declares_browser_v1_with_its_build_clock_and_mode_and_alone_its_window(monkeypatch):
     in_computer = Browser().extension_params(BROWSER)
-    _alone(monkeypatch)
     monkeypatch.setenv(SITE_PROXY_ENV, PROXY)
-    alone = Browser().extension_params(BROWSER)
-
-    assert in_computer == {"chromium": "156.0.8078.4", "clock": "real"}
-    assert alone == {"chromium": "156.0.8078.4", "clock": "real", "window": {"width": 1280, "height": 800}}
+    _alone(monkeypatch)
+    headless = Browser().extension_params(BROWSER)
+    _alone(monkeypatch, x_server=True)
+    headful = Browser().extension_params(BROWSER)
+
+    window = {"width": 1280, "height": 800}
+    assert in_computer == {"chromium": "156.0.8078.4", "clock": "real", "mode": "headful"}
+    assert headless == {"chromium": "156.0.8078.4", "clock": "real", "mode": "headless", "window": window}
+    assert headful == {"chromium": "156.0.8078.4", "clock": "real", "mode": "headful", "window": window}
 
 
 @pytest.mark.asyncio

src/agent_env/browsers/README.md

@@ -1,7 +1,7 @@
 The website browsers that come with agent-env, today `chromium`, a `computer_app` env a computer env lists in its
-`apps`. Deployed alone, `chromium` is a browser computer, `chromium-driver`: the `ubuntu-24.04-headless` base, with no
-display, and the `page` driver, whose screen is the browser's active tab, headless; the browser on it keeps its name,
-`chromium`.
+`apps`. Deployed alone, `chromium` is a browser computer, `chromium-driver`: the `ubuntu-24.04-browser` base, with no
+desktop, and the `page` driver, whose screen is the browser's active tab; the browser runs headful there, on an X
+server only it uses, and keeps its name, `chromium`.
 
 `envs/chromium/` is Chromium for an X11 desktop or a computer with no display on Linux (`amd64` and `arm64`): Chrome for Testing 156.0.8078.4, the
 build Playwright 1.64.1 pins, fetched by sha256 when a computer env that lists it is put and unpacked into

src/agent_env/browsers/envs/chromium/env.toml

@@ -1,6 +1,6 @@
 type = "computer_app"
 module = "chromium.py:Chromium"
-computer = "ubuntu-24.04-headless"
+computer = "ubuntu-24.04-browser"
 driver = "page"
 
 [settings]

src/agent_env/cli/run.py

@@ -86,7 +86,7 @@ def run(ctx: click.Context, bundle: str | None, tasks: tuple[str, ...], evals: t
 
     --set sets a key of one of the bundle's envs for this run, as its env.toml would: --set
     envs/desktop.driver=cua-driver runs the env in envs/desktop with that driver, and --set
-    envs/desktop.computer=ubuntu-24.04-headless composes it on that base. The envs it changes, the envs and tasks
+    envs/desktop.computer=ubuntu-24.04-browser composes it on that base. The envs it changes, the envs and tasks
     that name them and the evals that name those tasks are written under ids of their own, ending in the settings,
     such as desktop(desktop.driver cua-driver), and compose and reuse their images as any env does, leaving the
     bundle as written. A key or value the env's type refuses, or a run whose tasks deploy no env it changes, is