From b97883cfd53355b058a81c5965565f8631a23d8e Mon Sep 17 00:00:00 2001 From: Bren Huber <111620803+Brenhuber@users.noreply.github.com> Date: Tue, 10 Nov 2026 12:06:33 -0800 Subject: [PATCH 1/2] feat(drivers): aside, At Inc.'s Aside Computer Use as a macOS driver Aside's CLI serves its Computer Use as an MCP server, aside mcp --computer-use: one repl tool whose JavaScript reaches native apps through the computer global, in the background. The driver installs At Inc.'s published, signed releases from releases.aside.com, the daemon component (the CRX Aside's browsers update in the background) and the CLI, under /opt/agentenv/aside; runs the daemon as a service with usage analytics off from its first start; keeps one REPL per agent session; passes allowed_apps as --app; and checks the pinned executables, the daemon, analytics off, Computer Use on, the server against mcp.json, and the TCC grants. It pins Aside 1.26.1111.1046, components and CLI. --- src/agent_env/drivers/README.md | 25 ++ src/agent_env/drivers/aliases.toml | 1 + src/agent_env/drivers/envs/aside/driver.py | 316 +++++++++++++++++ src/agent_env/drivers/envs/aside/env.toml | 20 ++ src/agent_env/drivers/envs/aside/install.sh | 20 ++ src/agent_env/drivers/envs/aside/mcp.json | 29 ++ .../drivers/envs/aside/requirements-macos.txt | 4 + tst/unit/bundle/shipped_bundles_test.py | 1 + tst/unit/drivers/aside_driver_test.py | 335 ++++++++++++++++++ 9 files changed, 751 insertions(+) create mode 100644 src/agent_env/drivers/envs/aside/driver.py create mode 100644 src/agent_env/drivers/envs/aside/env.toml create mode 100755 src/agent_env/drivers/envs/aside/install.sh create mode 100644 src/agent_env/drivers/envs/aside/mcp.json create mode 100644 src/agent_env/drivers/envs/aside/requirements-macos.txt create mode 100644 tst/unit/drivers/aside_driver_test.py diff --git a/src/agent_env/drivers/README.md b/src/agent_env/drivers/README.md index 04ab85777..1202fee26 100644 --- a/src/agent_env/drivers/README.md +++ b/src/agent_env/drivers/README.md @@ -194,5 +194,30 @@ another ChatGPT version, set `VERSION`, pin the zip's URL and sha256 in `env.tom path), and take `mcp.json` again from what that version's `cua_repl` says at connect and lists. Name it as `codex-computer-use`. +`envs/aside/` is At Inc.'s Aside Computer Use as a driver, for the same macOS computers. Its MCP server is Aside's CLI, +`aside mcp --computer-use`, with one tool, `repl`. `repl` runs the agent's JavaScript in a QuickJS sandbox whose only +app API is the `computer` global, with no file system, network or shell. `computer` reads apps' accessibility trees and +acts on them in the background through Aside's daemon and its helper app. `driver.py` pins two of At Inc.'s published +releases: the daemon component as `VERSION`, the CRX Aside's browsers update in the background, and the CLI as +`CLI_VERSION`. The install downloads both from `releases.aside.com` and checks them by sha256, and `install.sh` unpacks +them to `/opt/agentenv/aside/`, links `bin/aside` into the CLI app, and writes the account's settings with usage +analytics off before the daemon first starts. The daemon runs as a direct LaunchAgent of its own executable, since macOS +gives an app's TCC permissions only to the executable launchd starts itself. `[install.macos]` grants Accessibility, +Post Event and Screen Recording to the helper app, to the copy the daemon runs from its home, and to the driver's own +Python. The driver starts one server for each agent session, since the REPL keeps its bindings between calls, with the +session user's environment, Aside's home and the CLI's update check off. A server that stops is started afresh on the +next call, and nothing says when a session ends, so one left unused for 30 minutes, or past the eight kept, is closed, +the least recently used first. Each server gets an `--app` for each app its `allowed_apps` setting names, by name or +bundle identifier, case aside, so `computer.getApp` opens only those, or every app when it names none. Agents get the +server's instructions and its `repl` tool, as `mcp.json` pins them, marked as acting on apps. No Aside sign-in is +needed. Its `computer_*` tools are agent-desktop's, from `MacOSDriver`. `computer-ready` checks the executables' sha256 +and that the daemon's health endpoint answers ready. It turns usage analytics off (`aside settings analytics disable`) +and Computer Use on (`aside settings computer-use enable`), then checks that the server gives the instructions and lists +the tool `mcp.json` pins and answers `computer.listApps()`, and the TCC permissions. To move it to other Aside releases, +set `VERSION` and `CLI_VERSION`, pin both downloads' URLs and sha256 in `env.toml` and the executables' in `driver.py`'s +`EXECUTABLES`, and take `mcp.json` again from what that version's server says at connect and lists. A unit test checks +that `env.toml`, `install.sh` and `driver.py` agree on the versions, the folder, the helper and the daemon. Name it as +`aside`. + This bundle has no tasks of its own: other bundles pull its drivers by alias, and `agent-env run drivers` has nothing to run. diff --git a/src/agent_env/drivers/aliases.toml b/src/agent_env/drivers/aliases.toml index a93822b44..1a88d2fd2 100644 --- a/src/agent_env/drivers/aliases.toml +++ b/src/agent_env/drivers/aliases.toml @@ -6,3 +6,4 @@ page = ["page"] agent-desktop = ["agent-desktop"] macos-harness = ["macos-harness"] codex-computer-use = ["codex-computer-use"] +aside = ["aside"] diff --git a/src/agent_env/drivers/envs/aside/driver.py b/src/agent_env/drivers/envs/aside/driver.py new file mode 100644 index 000000000..953f95136 --- /dev/null +++ b/src/agent_env/drivers/envs/aside/driver.py @@ -0,0 +1,316 @@ +"""Aside Computer Use as a computer driver on macOS: Aside's CLI, ``aside mcp --computer-use``, serves one tool, +``repl``, which runs the JavaScript an agent sends in a QuickJS sandbox whose only app API is the ``computer`` +global, which reads apps' accessibility trees and acts on them in the background through Aside's daemon and its +helper app. The pinned releases are unpacked to ``/opt/agentenv/aside/``; the daemon runs there as a service of its +own, with Aside's home in ``home/`` there, where the daemon runs its copy of the helper app from, and the driver +turns usage analytics off and Computer Use on once it answers. The driver starts the CLI's server with a minimal +environment and an ``--app`` for each app the ``allowed_apps`` setting names, or none for every app, one server for +each agent session, since the REPL keeps its bindings between calls, closed once it's left unused or past the most +kept, since nothing says when a session ends. Agents get the server's instructions and its ``repl`` tool as pinned +(``mcp.json``), marked as acting on apps. No Aside sign-in is needed. The computer/v1 operations are MacOSDriver's; +this process's Python is granted Accessibility, Post Event and Screen Recording, as the helper app is. It needs +``agentenv-framework-protocol[computer,macos]`` and the pinned releases.""" +from __future__ import annotations + +import asyncio +import contextlib +import functools +import hashlib +import json +import os +import time +import urllib.request +from pathlib import Path +from typing import Any, AsyncIterator, Callable + +from mcp import ClientSession, McpError, StdioServerParameters +from mcp.client.stdio import stdio_client +from mcp.types import CONNECTION_CLOSED, CallToolResult, InitializeResult, TextContent +from pydantic import Field + +from agentenv_protocol import EnvironmentTool, ForwardedTool, ToolContext, server_instructions +from agentenv_protocol.computer import ComputerSettings, MacOSDriver, ReadyCheck, computer_driver + +HERE = Path(__file__).resolve().parent +#: The pinned Aside releases, as At Inc. publishes them and install.sh unpacks them: the daemon component, the daemon +#: and the helper app that Aside's browsers update in the background, and the CLI; and where the CLI and the helper are. +VERSION = "1.26.1111.1046" +CLI_VERSION = "1.26.1111.1046" +PREFIX = Path("/opt/agentenv/aside") +CLI = Path("bin", "aside") +HELPER = "Aside Computer Use.app" +#: The daemon app's executable, which env.toml's daemon service runs. +DAEMON = Path("Aside Daemon.app", "Contents", "MacOS", "aside-daemon") +#: Aside's home, ``ASIDE_HOME`` for the daemon service and every CLI the driver starts, fixed so the daemon's runtime +#: copy of the helper, which install.sh places and env.toml grants, has a fixed path. +ASIDE_HOME = PREFIX / "home" +#: The sha256 of each executable at the pinned version, by its path under the prefix. +EXECUTABLES = { + str(DAEMON): "40b91cd799b8ad404d71137a62535b8335b1c2368acdf87f3cb9945e4860572b", + f"{HELPER}/Contents/MacOS/aside-computer-use": "768662135dac8727e41791469b59c3a0312551d727c2d1f70cfe907182033d3c", + "bin/aside": "0c13c719966302a8bf6cb5382c41765512139d3dcd360d067a4fec2c56dcf374", +} +#: What ``aside mcp --computer-use`` says at connect and lists at the pinned version: its instructions and its tool. +MANIFEST = HERE / "mcp.json" +#: The hints agents get, where Aside gives none: ``repl`` acts on apps, and through them on anything. +ANNOTATIONS = {"repl": {"readOnlyHint": False, "destructiveHint": True, "openWorldHint": True}} +#: The daemon's health endpoint, which answers ``"ready": true`` once it serves, and how long it has to answer. +HEALTH, HEALTH_TIMEOUT_S = "http://127.0.0.1:21420/health", 5 +#: How long an ``aside settings`` command has to finish. +SETTINGS_TIMEOUT_S = 30 +#: How long a session's REPL is kept unused before it's closed, since nothing says when a session ends; how many are +#: kept at most, the least recently used closed first; and how often the idle ones are looked for. +IDLE_S, MAX_REPLS, SWEEP_S = 1800, 8, 60 +#: How long a closed REPL's server has to end before it's cancelled. +CLOSE_TIMEOUT_S = 10 +#: What of the driver's environment the CLI gets: the session user's own; none of a deploy's values. +KEPT = ("HOME", "USER", "LOGNAME", "TMPDIR", "LANG") +#: The PATH it runs with. +PATH = "/usr/bin:/bin:/usr/sbin:/sbin" +#: The JavaScript the readiness check runs: an inventory, which needs no app, printed as the number of apps. +PROBE = "console.log((await computer.listApps({ emit: false })).length)" + + +@computer_driver(name="aside", + supports=[{"os": {"type": "macos", "display": ["aqua"]}, "arch": ["arm64"]}]) +class AsideDriver(MacOSDriver): + class Settings(ComputerSettings): + #: The apps ``computer.getApp`` may open, by name or bundle identifier, case aside; None for every app, and + #: never empty, which would pass no ``--app`` and so allow every app too. + allowed_apps: list[str] | None = Field(default=None, min_length=1) + + def __init__(self) -> None: + super().__init__() + self._pinned: list[ReadyCheck] | None = None + #: Each session's REPL, the least recently used first. + self._repls: dict[str | None, _Repl] = {} + self._sweeper: asyncio.Task | None = None + #: The closes under way, held so they aren't collected before they end. + self._closing: set[asyncio.Task] = set() + + def forwarded_tools(self) -> list[ForwardedTool]: + return [ForwardedTool(EnvironmentTool(name=t["name"], description=t["description"], + inputSchema=t["inputSchema"]), + functools.partial(self._forward, t["name"]), + {**(t.get("annotations") or {}), **ANNOTATIONS.get(t["name"], {})}) + for t in _manifest()["tools"]] + + @server_instructions + def aside_instructions(self) -> str: + return _manifest()["instructions"] + + async def ready_checks(self) -> list[ReadyCheck]: + """The pinned executables, the daemon, analytics off and Computer Use on, what the server says and lists and + that it answers, checked until they pass once, and MacOSDriver's TCC permissions every time.""" + if self._pinned is None or not all(check.ok for check in self._pinned): + self._pinned = await self._pinned_checks() + return [*self._pinned, *await super().ready_checks()] + + async def _forward(self, tool: str, /, **arguments: Any) -> CallToolResult: + """A call to Aside's tool on the caller's own REPL, answered as Aside answers it. Other sessions' REPLs left + idle, or past the most kept, are closed. Nothing here awaits before the call is queued, so calls a session + makes together find one REPL.""" + repl = self._repls.pop(caller := ToolContext.current().caller.session, None) or _Repl(self._server) + self._evict(keep=MAX_REPLS - 1) + self._repls[caller] = repl + if self._sweeper is None or self._sweeper.done(): + self._sweeper = asyncio.create_task(self._sweep()) + try: + return await repl.call(tool, arguments) + except Exception as e: + return CallToolResult(isError=True, content=[TextContent(type="text", text=( + f"Aside's MCP server failed before it answered, so {tool} may have run: {type(e).__name__}: {e}"))]) + + def _evict(self, keep: int = MAX_REPLS) -> None: + """Take out the REPLs idle for ``IDLE_S`` and, past ``keep``, the least recently used, none of them in a call, + and close them in the background.""" + now = time.monotonic() + free = [caller for caller, repl in self._repls.items() if not repl.busy] + idle = [caller for caller in free if now - self._repls[caller].used > IDLE_S] + over = max(0, len(self._repls) - len(idle) - keep) + spare = [caller for caller in free if caller not in idle][:over] + for caller in [*idle, *spare]: + closing = asyncio.create_task(self._repls.pop(caller).close()) + self._closing.add(closing) + closing.add_done_callback(self._closing.discard) + + async def _sweep(self) -> None: + """Close idle REPLs every ``SWEEP_S`` while there are any.""" + while self._repls: + await asyncio.sleep(SWEEP_S) + self._evict() + + @contextlib.asynccontextmanager + async def _server(self) -> AsyncIterator[tuple[ClientSession, InitializeResult]]: + """``aside mcp --computer-use`` for the allowed apps, connected and initialized, with what it said at connect; + it ends with the block, as closing its stdin ends it.""" + server = StdioServerParameters(command=str(PREFIX / CLI), args=_arguments(self.settings.allowed_apps), + env=_environment()) + async with stdio_client(server) as (read, write), ClientSession(read, write) as session: + yield session, await session.initialize() + + async def _pinned_checks(self) -> list[ReadyCheck]: + digests = await asyncio.to_thread(_digests) + differ = sorted(path for path, digest in digests.items() if digest != EXECUTABLES[path]) + pinned = f"Aside {VERSION}, CLI {CLI_VERSION}" + checks = [ReadyCheck("Aside Computer Use", not differ, f"{pinned}, as pinned" if not differ else + f"these differ from the pinned {pinned}: {', '.join(differ)}")] + if differ: + return checks + try: + healthy = (await asyncio.to_thread(_health)).get("ready") is True + except Exception as e: + return [*checks, ReadyCheck("Aside daemon", False, f"{HEALTH} didn't answer: {type(e).__name__}: {e}")] + checks.append(ReadyCheck("Aside daemon", healthy, "it answers ready" if healthy else + f"{HEALTH} doesn't say it's ready")) + if not healthy: + return checks + for name, words in (("Analytics off", ("analytics", "disable")), + ("Computer Use on", ("computer-use", "enable"))): + code, said = await _setting(*words) + checks.append(ReadyCheck(name, code == 0, said or f"it exited {code}")) + if code != 0: + return checks + try: + async with self._server() as (session, initialized): + listed = (await session.list_tools()).tools + probed = await session.call_tool("repl", {"code": PROBE}) + except Exception as e: + return [*checks, ReadyCheck("Aside tools", False, f"aside mcp didn't answer: {type(e).__name__}: {e}")] + running = {t.name: t.model_dump(mode="json", exclude_none=True) for t in listed} + pinned = {t["name"]: t for t in _manifest()["tools"]} + changed = ["instructions"] if initialized.instructions != _manifest()["instructions"] else [] + changed += sorted(name for name in running.keys() | pinned.keys() if name not in running or name not in pinned + or any(running[name].get(key) != value for key, value in pinned[name].items())) + answer = " ".join(c.text for c in probed.content if isinstance(c, TextContent)).strip() + listed_apps = not probed.isError and answer.isdigit() and int(answer) > 0 + return [*checks, + ReadyCheck("Aside tools", not changed, f"its instructions and {len(running)} tool, as pinned" + if not changed else f"these differ from {MANIFEST.name}: {', '.join(changed)}"), + ReadyCheck("Aside answers", listed_apps, f"it read the computer's {answer} apps" if listed_apps + else f"computer.listApps() gave no apps: {answer}")] + + +class _Repl: + """A session's ``aside mcp``, one server at a time, each held open by a task of its own, since a stdio client is + entered and left in one task, and reached from any through that server's queue, one call at a time. A server that + stops is let go at once, while it still ends, and the next call starts one afresh, without the last one's bindings; + ``close`` ends them for good.""" + + def __init__(self, open_server: Callable[[], contextlib.AbstractAsyncContextManager[ + tuple[ClientSession, InitializeResult]]]) -> None: + self._open = open_server + #: The queue of the server taking calls, None when none is; and the tasks of every server not yet ended. + self._calls: asyncio.Queue | None = None + self._serving: set[asyncio.Task] = set() + #: When a call last ended, on the monotonic clock, and how many are queued or running. + self.used, self.busy = time.monotonic(), 0 + + async def call(self, tool: str, arguments: dict) -> CallToolResult: + if self._calls is None: + self._calls = asyncio.Queue() + serving = asyncio.create_task(self._serve(self._calls)) + self._serving.add(serving) + serving.add_done_callback(self._serving.discard) + answer = asyncio.get_running_loop().create_future() + self.busy += 1 + try: + self._calls.put_nowait((tool, arguments, answer)) + return await answer + finally: + self.busy -= 1 + self.used = time.monotonic() + + async def close(self) -> None: + """Every server ended, its process with it, waiting at most ``CLOSE_TIMEOUT_S`` before cancelling one.""" + if self._calls is not None: + self._calls.put_nowait(None) + self._calls = None + if serving := set(self._serving): + _, late = await asyncio.wait(serving, timeout=CLOSE_TIMEOUT_S) + for task in late: + task.cancel() + + async def _serve(self, calls: asyncio.Queue) -> None: + stopped: BaseException = ConnectionError("aside mcp stopped") + try: + async with self._open() as (session, _): + try: + while (call := await calls.get()) is not None: + tool, arguments, answer = call + try: + result = await session.call_tool(tool, arguments) + except McpError as e: + _settle(answer, error=e) + if e.error.code == CONNECTION_CLOSED: + raise + continue + except Exception as e: + _settle(answer, error=e) + raise + _settle(answer, result=result) + finally: + self._let_go(calls) + except Exception as e: + stopped = e + finally: + self._let_go(calls) + while not calls.empty(): + if (call := calls.get_nowait()) is not None: + _settle(call[-1], error=stopped) + + def _let_go(self, calls: asyncio.Queue) -> None: + """The next call to a new server, not to this one, which takes no more.""" + if self._calls is calls: + self._calls = None + + +def _settle(answer: asyncio.Future, *, result: Any = None, error: BaseException | None = None) -> None: + if answer.done(): + return + if error is not None: + answer.set_exception(error) + else: + answer.set_result(result) + + +@functools.cache +def _manifest() -> dict: + return json.loads(MANIFEST.read_text()) + + +def _arguments(allowed_apps: list[str] | None) -> list[str]: + """The CLI's arguments for a server that may open ``allowed_apps``, every app for None.""" + return ["mcp", "--computer-use", *(word for app in allowed_apps or [] for word in ("--app", app))] + + +def _environment() -> dict[str, str]: + """The CLI's environment: the session user's own, the system PATH and Aside's home, with no update check.""" + return {**{name: os.environ[name] for name in KEPT if name in os.environ}, "PATH": PATH, + "ASIDE_HOME": str(ASIDE_HOME), "ASIDE_CLI_UPDATE_CHECK": "0"} + + +def _health() -> dict: + """What the daemon's health endpoint answers.""" + with urllib.request.urlopen(HEALTH, timeout=HEALTH_TIMEOUT_S) as response: + return json.load(response) + + +async def _setting(*words: str) -> tuple[int, str]: + """``aside settings ``, which needs the daemon and is idempotent: its exit code and what it printed.""" + process = await asyncio.create_subprocess_exec( + str(PREFIX / CLI), "settings", *words, stdin=asyncio.subprocess.DEVNULL, + stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.STDOUT, env=_environment()) + try: + out, _ = await asyncio.wait_for(process.communicate(), SETTINGS_TIMEOUT_S) + except TimeoutError: + process.kill() + await process.wait() + return 124, f"aside settings {' '.join(words)} didn't finish within {SETTINGS_TIMEOUT_S} s" + return process.returncode, out.decode(errors="replace").strip() + + +def _digests() -> dict[str, str | None]: + """Each pinned executable's sha256, None for one that isn't there.""" + return {path: hashlib.sha256((PREFIX / path).read_bytes()).hexdigest() if (PREFIX / path).is_file() else None + for path in EXECUTABLES} diff --git a/src/agent_env/drivers/envs/aside/env.toml b/src/agent_env/drivers/envs/aside/env.toml new file mode 100644 index 000000000..14ec76905 --- /dev/null +++ b/src/agent_env/drivers/envs/aside/env.toml @@ -0,0 +1,20 @@ +type = "computer_driver" +module = "driver.py:AsideDriver" +port = 18765 + +[install.macos] +python = "requirements-macos.txt" +downloads = [ + { url = "https://releases.aside.com/dev-updater/components/1.26.1111.1046/AsideDaemon/mac-arm64/AsideDaemon-1.26.1111.1046.crx", sha256 = "47f54f506f1b8e7abb75d1ca263bb388e5d4e1bd295c3b52e64b5959082744c6", to = "aside-daemon.crx", arch = "arm64" }, + { url = "https://releases.aside.com/cli/AsideCLI-darwin-arm64-1.26.1111.1046.zip", sha256 = "f4b234e8a63eb2a721a9094b7ded1c286d4f253ede3b4eb347c2d3e0ef04765c", to = "aside-cli.zip", arch = "arm64" }, +] +run = [{ script = "install.sh", scope = "session" }] +grants = [ + { executable = "venv/bin/python", services = ["accessibility", "post_event", "screen_capture"] }, + { app = "/opt/agentenv/aside/Aside Computer Use.app", services = ["accessibility", "post_event", "screen_capture"] }, + { app = "/opt/agentenv/aside/home/runtime-computer-use/Aside Computer Use.app", services = ["accessibility", "post_event", "screen_capture"] }, +] +services = [ + { name = "daemon", command = "'/opt/agentenv/aside/Aside Daemon.app/Contents/MacOS/aside-daemon'", direct = true, environment = { ASIDE_HOME = "/opt/agentenv/aside/home" } }, + { name = "driver", command = "venv/bin/python -m agentenv_protocol.computer.serve driver.py:AsideDriver" }, +] diff --git a/src/agent_env/drivers/envs/aside/install.sh b/src/agent_env/drivers/envs/aside/install.sh new file mode 100755 index 000000000..c06590ca8 --- /dev/null +++ b/src/agent_env/drivers/envs/aside/install.sh @@ -0,0 +1,20 @@ +#!/bin/sh +# The pinned daemon component and CLI, unpacked to where the grants and the daemon service name them, as the session +# user. The helper's runtime copy is placed before the daemon first starts, since macOS keys its approval to that path. +set -eu +daemon=aside-daemon.crx +cli=aside-cli.zip +dest=/opt/agentenv/aside +helper="Aside Computer Use.app" +rm -rf "$dest" +mkdir -p "$dest/bin" "$dest/home/runtime-computer-use" +# A CRX is a signature header and then a zip, which bsdtar reads. +tar -xf "$daemon" -C "$dest" +ditto -x -k "$cli" "$dest" +ln -s "../Aside CLI.app/Contents/MacOS/aside" "$dest/bin/aside" +rm -f "$daemon" "$cli" +xattr -dr com.apple.quarantine "$dest" || true +ditto "$dest/$helper" "$dest/home/runtime-computer-use/$helper" +# Usage analytics are off from the daemon's first start. +mkdir -p "$dest/home/u/0" +printf '%s\n' '{"analytics":{"enabled":false}}' > "$dest/home/u/0/settings.json" diff --git a/src/agent_env/drivers/envs/aside/mcp.json b/src/agent_env/drivers/envs/aside/mcp.json new file mode 100644 index 000000000..7554e8287 --- /dev/null +++ b/src/agent_env/drivers/envs/aside/mcp.json @@ -0,0 +1,29 @@ +{ + "instructions": "Operate native apps on this computer with the repl tool. Start with `await computer.getApp('')` and follow the guide it prints.", + "tools": [ + { + "name": "repl", + "description": "Run JavaScript in a sandbox whose only API is the `computer` global, which reads and controls native apps on this computer. There is no file system, network, shell, or browser. Variables persist between calls. Start with `await computer.getApp('')`; it prints the app's state and the full usage guide.", + "inputSchema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "properties": { + "code": { + "description": "JavaScript to run. Top-level await works.", + "type": "string" + }, + "title": { + "description": "Action title, e.g. \"Type the note\".", + "type": "string" + } + }, + "required": [ + "code" + ], + "type": "object" + }, + "execution": { + "taskSupport": "forbidden" + } + } + ] +} diff --git a/src/agent_env/drivers/envs/aside/requirements-macos.txt b/src/agent_env/drivers/envs/aside/requirements-macos.txt new file mode 100644 index 000000000..a8c357df2 --- /dev/null +++ b/src/agent_env/drivers/envs/aside/requirements-macos.txt @@ -0,0 +1,4 @@ +# What a composed macOS computer installs into the driver's venv: the protocol with MacOSDriver, whose screen and input +# need its macos extra, and the MCP client the forwarded tool reaches Aside's MCP server with. A release pins it to the +# version it ships. Aside is the pinned download, whose executables' sha256 the driver checks. +agentenv-framework-protocol[computer,macos]>=0.1.459 diff --git a/tst/unit/bundle/shipped_bundles_test.py b/tst/unit/bundle/shipped_bundles_test.py index 357d5708f..666041e8d 100644 --- a/tst/unit/bundle/shipped_bundles_test.py +++ b/tst/unit/bundle/shipped_bundles_test.py @@ -64,6 +64,7 @@ def test_the_shipped_env_aliases_name_the_mail_env_the_validator_deploys_the_dri "agent-desktop": "@local/agentenv-framework/drivers/agent-desktop", "macos-harness": "@local/agentenv-framework/drivers/macos-harness", "codex-computer-use": "@local/agentenv-framework/drivers/codex-computer-use", + "aside": "@local/agentenv-framework/drivers/aside", "chromium": "@local/agentenv-framework/browsers/chromium"} assert A2AAgent.VALIDATION_ENV_ID == "mail" diff --git a/tst/unit/drivers/aside_driver_test.py b/tst/unit/drivers/aside_driver_test.py new file mode 100644 index 000000000..9eedad6aa --- /dev/null +++ b/tst/unit/drivers/aside_driver_test.py @@ -0,0 +1,335 @@ +"""The aside driver against a fake aside CLI: agents get its repl as pinned, with its instructions, marked as acting on +apps; allowed_apps becomes one --app each, none for every app, and never empty; each agent session keeps a REPL of +its own across calls, one that stops starting afresh; the CLI gets the session user's environment, Aside's home and +no update check, and none of the driver's; readiness checks the pinned executables, the daemon's health, turns usage +analytics off and Computer Use on, and checks what the server says and lists and that it answers; and the driver's +files agree on the pinned version, folder, helper, daemon and Aside's home, where the helper's runtime copy is placed +and granted.""" +from __future__ import annotations + +import asyncio +import contextlib +import hashlib +import importlib.util +import json +import os +import shlex +import sys +import textwrap +import tomllib +from pathlib import Path + +import pytest +from mcp.shared.memory import create_connected_server_and_client_session +from pydantic import ValidationError + +import agent_env +from agentenv_protocol import SESSION_META_KEY +from agentenv_protocol.computer import Platform, _macos + +DRIVER = Path(agent_env.__file__).parent / "drivers" / "envs" / "aside" / "driver.py" +FAKE = textwrap.dedent('''\ + import json, os, sys + from pathlib import Path + state = Path(os.environ["HOME"]) / "fake" + if sys.argv[1:] == ["settings", "computer-use", "enable"]: + with open(state / "enabled", "a") as out: + out.write("enabled\\n") + (state / "on").touch() + print("Computer Use is on.") + sys.exit(0) + if sys.argv[1:] == ["settings", "analytics", "disable"]: + print("Analytics are off.") + sys.exit(0) + import anyio + import mcp.types as types + from mcp.server.lowlevel import Server + from mcp.server.stdio import stdio_server + (state / f"env-{os.getpid()}.json").write_text(json.dumps(dict(os.environ))) + manifest = json.loads((state / "tools.json").read_text()) + server = Server("aside", instructions=manifest["instructions"]) + bindings = {} + + @server.list_tools() + async def tools(): + return [types.Tool(**t) for t in manifest["tools"]] + + @server.call_tool(validate_input=False) + async def call(name, arguments): + if not (state / "on").exists(): + raise RuntimeError("Computer Use is turned off. Turn it on with 'aside settings computer-use enable'.") + code = arguments["code"] + if code.startswith("set "): + bindings["x"] = code[4:] + text = "set" + elif code == "get": + text = f"{bindings.get('x')} in {os.getpid()}" + elif code == "args": + text = json.dumps(sys.argv[1:]) + elif code == "die": + os._exit(1) + elif "computer.listApps" in code: + text = "3" + else: + text = "ran" + return [types.TextContent(type="text", text=text)] + + async def main(): + async with stdio_server() as (read, write): + await server.run(read, write, server.create_initialization_options()) + + anyio.run(main) +''') +MACOS = Platform(os="macos", version="26.5", arch="arm64", display="aqua") + + +class Granted: + def missing(self) -> list[str]: + return [] + + +@pytest.fixture +def pinned(monkeypatch, tmp_path): + """The driver's module, as it ships.""" + monkeypatch.delenv("AGENTENV_COMPUTER_SETTINGS", raising=False) + monkeypatch.setenv("HOME", str(tmp_path)) + monkeypatch.setenv("AGENTENV_CONTROL_TOKEN", "a-deploys-secret") + monkeypatch.setattr(_macos, "Quartz", Granted) + monkeypatch.setattr(sys, "platform", "darwin") + monkeypatch.setattr("agentenv_protocol.computer.driver.detect", lambda: MACOS) + spec = importlib.util.spec_from_file_location("aside_driver", DRIVER) + loaded = importlib.util.module_from_spec(spec) + spec.loader.exec_module(loaded) + return loaded + + +@pytest.fixture +def module(pinned, monkeypatch, tmp_path): + """The driver's module on a fake Aside whose CLI is the fake server, its daemon healthy and Computer Use on.""" + (tmp_path / "fake").mkdir() + (tmp_path / "fake" / "tools.json").write_text(pinned.MANIFEST.read_text()) + (tmp_path / "fake" / "on").touch() + prefix = tmp_path / "aside" + cli = prefix / pinned.CLI + cli.parent.mkdir(parents=True) + cli.write_text(f"#!{sys.executable}\n{FAKE}") + cli.chmod(0o755) + digests = {} + for path in pinned.EXECUTABLES: + file = prefix / path + if not file.exists(): + file.parent.mkdir(parents=True, exist_ok=True) + file.write_text(f"the pinned {file.name}") + digests[path] = hashlib.sha256(file.read_bytes()).hexdigest() + monkeypatch.setattr(pinned, "PREFIX", prefix) + monkeypatch.setattr(pinned, "EXECUTABLES", digests) + monkeypatch.setattr(pinned, "_health", lambda: {"ready": True}) + return pinned + + +def _driver(module, monkeypatch, **settings): + monkeypatch.setenv("AGENTENV_COMPUTER_SETTINGS", json.dumps(settings)) + return module.AsideDriver() + + +@contextlib.asynccontextmanager +async def mcp_session(driver): + async with create_connected_server_and_client_session(driver.mcp or driver.create_app()) as session: + yield session + + +def _text(result) -> str: + return " ".join(c.text for c in result.content if c.type == "text") + + +async def _gone(pid: int) -> bool: + for _ in range(200): + try: + os.kill(pid, 0) + except ProcessLookupError: + return True + await asyncio.sleep(0.01) + return False + + +@pytest.mark.asyncio +async def test_agents_get_asides_repl_as_pinned_with_its_instructions_marked_as_acting_on_apps(module, monkeypatch): + driver = _driver(module, monkeypatch) + async with mcp_session(driver) as session: + listed = {t.name: t for t in (await session.list_tools()).tools} + answered = await session.call_tool("repl", {"code": "1 + 1"}) + + (pinned,) = module._manifest()["tools"] + assert listed["repl"].inputSchema == pinned["inputSchema"] and listed["repl"].description == pinned["description"] + hints = listed["repl"].annotations + assert (hints.readOnlyHint, hints.destructiveHint, hints.openWorldHint) == (False, True, True) + assert driver.mcp.instructions.startswith(module._manifest()["instructions"]) + assert not answered.isError and _text(answered) == "ran" + + +@pytest.mark.asyncio +async def test_allowed_apps_become_one_app_flag_each_and_none_for_every_app(module, monkeypatch): + async with mcp_session(_driver(module, monkeypatch, allowed_apps=["Calculator", "com.apple.TextEdit"])) as session: + named = json.loads(_text(await session.call_tool("repl", {"code": "args"}))) + async with mcp_session(_driver(module, monkeypatch)) as session: + every = json.loads(_text(await session.call_tool("repl", {"code": "args"}))) + + assert named == ["mcp", "--computer-use", "--app", "Calculator", "--app", "com.apple.TextEdit"] + assert every == ["mcp", "--computer-use"] + with pytest.raises(ValidationError): + module.AsideDriver.Settings(allowed_apps=[]) + + +@pytest.mark.asyncio +async def test_a_failing_cell_reaches_the_agent_as_aside_answers_it(module, monkeypatch, tmp_path): + (tmp_path / "fake" / "on").unlink() + async with mcp_session(_driver(module, monkeypatch)) as session: + failed = await session.call_tool("repl", {"code": "1 + 1"}) + + assert failed.isError and "Computer Use is turned off" in _text(failed) + + +@pytest.mark.asyncio +async def test_each_agent_session_keeps_a_repl_of_its_own_and_one_that_stops_starts_afresh_while_it_still_ends( + module, monkeypatch): + """A server that stops takes a while to end; the session's next call starts a new one rather than wait on it.""" + serve = module.AsideDriver._server + + @contextlib.asynccontextmanager + async def slow_to_end(self): + async with serve(self) as opened: + try: + yield opened + finally: + await asyncio.sleep(0.5) + + monkeypatch.setattr(module.AsideDriver, "_server", slow_to_end) + async with mcp_session(_driver(module, monkeypatch)) as session: + for caller, value in (("a", "1"), ("b", "2")): + await session.call_tool("repl", {"code": f"set {value}"}, meta={SESSION_META_KEY: caller}) + a = _text(await session.call_tool("repl", {"code": "get"}, meta={SESSION_META_KEY: "a"})) + b = _text(await session.call_tool("repl", {"code": "get"}, meta={SESSION_META_KEY: "b"})) + died = await session.call_tool("repl", {"code": "die"}, meta={SESSION_META_KEY: "a"}) + after = _text(await session.call_tool("repl", {"code": "get"}, meta={SESSION_META_KEY: "a"})) + + (a_value, _, a_pid), (b_value, _, b_pid) = a.partition(" in "), b.partition(" in ") + assert (a_value, b_value) == ("1", "2") and a_pid != b_pid + assert died.isError and "may have run" in _text(died) + assert after.startswith("None in ") and after != f"None in {a_pid}" + + +@pytest.mark.asyncio +async def test_the_cli_gets_the_session_users_environment_and_none_of_the_drivers(module, monkeypatch, tmp_path): + async with mcp_session(_driver(module, monkeypatch)) as session: + await session.call_tool("repl", {"code": "1 + 1"}) + + (env_file,) = (tmp_path / "fake").glob("env-*.json") + env = json.loads(env_file.read_text()) + assert "AGENTENV_CONTROL_TOKEN" not in env and "AGENTENV_COMPUTER_SETTINGS" not in env + assert env["PATH"] == "/usr/bin:/bin:/usr/sbin:/sbin" and env["HOME"] == str(tmp_path) + assert env["ASIDE_HOME"] == str(module.ASIDE_HOME) and env["ASIDE_CLI_UPDATE_CHECK"] == "0" + + +@pytest.mark.asyncio +async def test_readiness_checks_the_pinned_executables_the_daemon_turns_computer_use_on_and_checks_the_server( + module, monkeypatch, tmp_path): + driver = _driver(module, monkeypatch) + manifest = module._manifest() + (tmp_path / "fake" / "on").unlink() + + async def checks(served: dict | None = None): + if served is not None: + (tmp_path / "fake" / "tools.json").write_text(json.dumps(served)) + driver._pinned = None + return await driver.ready_checks() + + ready = await driver.ready_checks() + again = await driver.ready_checks() + described = (await checks({"instructions": "other", "tools": [ + {**t, "description": "other"} for t in manifest["tools"]]}))[4] + extra = (await checks({**manifest, "tools": [*manifest["tools"], {**manifest["tools"][0], "name": "exec"}]}))[4] + monkeypatch.setattr(module, "_health", lambda: {"ready": False}) + unready = await checks(manifest) + (module.PREFIX / module.HELPER / "Contents" / "MacOS" / "aside-computer-use").write_text("another build") + (changed, *rest) = await checks() + + assert [(c.name, c.ok) for c in ready] == [("Aside Computer Use", True), ("Aside daemon", True), + ("Analytics off", True), ("Computer Use on", True), + ("Aside tools", True), + ("Aside answers", True), ("macOS permissions", True)] + assert (ready[2].detail, ready[3].detail) == ("Analytics are off.", "Computer Use is on.") and again == ready + assert (tmp_path / "fake" / "enabled").read_text().count("enabled") == 3 + assert (described.ok, described.detail) == (False, "these differ from mcp.json: instructions, repl") + assert (extra.ok, extra.detail) == (False, "these differ from mcp.json: exec") + assert [(c.name, c.ok) for c in unready] == [("Aside Computer Use", True), ("Aside daemon", False), + ("macOS permissions", True)] + assert not changed.ok and "Aside Computer Use.app/Contents/MacOS/aside-computer-use" in changed.detail + assert [c.name for c in rest] == ["macOS permissions"] + + +def test_env_toml_install_sh_and_the_driver_agree_on_the_pinned_version_folder_helper_daemon_and_home(pinned): + """A move to other Aside releases, or of their folder, helper, daemon or home, has to change them together; the + helper's runtime copy, which install.sh places in the home, is granted as the helper is.""" + folder = DRIVER.parent + macos = tomllib.loads((folder / "env.toml").read_text())["install"]["macos"] + install = (folder / "install.sh").read_text() + runtime = pinned.ASIDE_HOME / "runtime-computer-use" / pinned.HELPER + + daemon_download, cli_download = macos["downloads"] + assert daemon_download["url"] == ("https://releases.aside.com/dev-updater/components/" + f"{pinned.VERSION}/AsideDaemon/mac-arm64/AsideDaemon-{pinned.VERSION}.crx") + assert cli_download["url"] == f"https://releases.aside.com/cli/AsideCLI-darwin-arm64-{pinned.CLI_VERSION}.zip" + assert all(len(download["sha256"]) == 64 for download in macos["downloads"]) + assert f"daemon={daemon_download['to']}\n" in install and f"cli={cli_download['to']}\n" in install + assert [grant["app"] for grant in macos["grants"] if "app" in grant] == [str(pinned.PREFIX / pinned.HELPER), + str(runtime)] + assert f"dest={pinned.PREFIX}\n" in install and f'helper="{pinned.HELPER}"\n' in install + assert f'"$dest/{runtime.relative_to(pinned.PREFIX).parent}/$helper"' in install + (daemon,) = [service for service in macos["services"] if service.get("direct")] + assert shlex.split(daemon["command"]) == [str(pinned.PREFIX / pinned.DAEMON)] + assert daemon["environment"] == {"ASIDE_HOME": str(pinned.ASIDE_HOME)} + assert {str(pinned.DAEMON), f"{pinned.HELPER}/Contents/MacOS/aside-computer-use", str(pinned.CLI)} <= set( + pinned.EXECUTABLES) + assert all(len(digest) == 64 for digest in pinned.EXECUTABLES.values()) + + +@pytest.mark.asyncio +async def test_past_the_most_repls_kept_the_least_recently_used_is_closed_and_the_others_keep_their_bindings( + module, monkeypatch): + monkeypatch.setattr(module, "MAX_REPLS", 2) + async with mcp_session(_driver(module, monkeypatch)) as session: + async def repl(caller: str, code: str) -> str: + return _text(await session.call_tool("repl", {"code": code}, meta={SESSION_META_KEY: caller})) + + for caller in "abc": + await repl(caller, f"set {caller}") + b, c, a = await repl("b", "get"), await repl("c", "get"), await repl("a", "get") + + assert a.startswith("None in ") and b.startswith("b in ") and c.startswith("c in ") + + +@pytest.mark.asyncio +async def test_a_repl_left_idle_is_closed_without_another_call_and_its_session_starts_afresh(module, monkeypatch): + monkeypatch.setattr(module, "SWEEP_S", 0.05) + monkeypatch.setattr(module, "IDLE_S", 0.2) + async with mcp_session(_driver(module, monkeypatch)) as session: + await session.call_tool("repl", {"code": "set 1"}, meta={SESSION_META_KEY: "a"}) + before = _text(await session.call_tool("repl", {"code": "get"}, meta={SESSION_META_KEY: "a"})) + closed = await _gone(int(before.rpartition(" in ")[2])) + after = _text(await session.call_tool("repl", {"code": "get"}, meta={SESSION_META_KEY: "a"})) + + assert before.startswith("1 in ") and closed and after.startswith("None in ") + + +@pytest.mark.asyncio +async def test_calls_a_session_makes_together_while_another_repl_closes_share_one_repl(module, monkeypatch): + monkeypatch.setattr(module, "MAX_REPLS", 1) + async with mcp_session(_driver(module, monkeypatch)) as session: + async def repl(caller: str, code: str) -> str: + return _text(await session.call_tool("repl", {"code": code}, meta={SESSION_META_KEY: caller})) + + await repl("a", "set a") + _, got = await asyncio.gather(repl("b", "set b"), repl("b", "get")) + after = await repl("b", "get") + + assert got.partition(" in ")[2] == after.partition(" in ")[2] and after.startswith("b in ") From f481ea1137670af77e019875083540f95d6ce11b Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Tue, 10 Nov 2026 18:33:25 -0800 Subject: [PATCH 2/2] fix(drivers): aside drops a call its caller gave up while it waited A repl call cancelled while it waited behind another stayed in its REPL's queue, and _serve sent it to Aside once the call ahead ended, so an abandoned action could still change an app. _serve now skips a queued call whose answer is already settled, and a test cancels a waiting call and checks it never reaches Aside. Co-Authored-By: Claude Opus 5.5 --- src/agent_env/drivers/envs/aside/driver.py | 8 +++-- tst/unit/drivers/aside_driver_test.py | 34 +++++++++++++++++++++- 2 files changed, 38 insertions(+), 4 deletions(-) diff --git a/src/agent_env/drivers/envs/aside/driver.py b/src/agent_env/drivers/envs/aside/driver.py index 953f95136..81ca5329f 100644 --- a/src/agent_env/drivers/envs/aside/driver.py +++ b/src/agent_env/drivers/envs/aside/driver.py @@ -193,9 +193,9 @@ async def _pinned_checks(self) -> list[ReadyCheck]: class _Repl: """A session's ``aside mcp``, one server at a time, each held open by a task of its own, since a stdio client is - entered and left in one task, and reached from any through that server's queue, one call at a time. A server that - stops is let go at once, while it still ends, and the next call starts one afresh, without the last one's bindings; - ``close`` ends them for good.""" + entered and left in one task, and reached from any through that server's queue, one call at a time; a call its + caller gave up while it waited is dropped, never sent. A server that stops is let go at once, while it still ends, + and the next call starts one afresh, without the last one's bindings; ``close`` ends them for good.""" def __init__(self, open_server: Callable[[], contextlib.AbstractAsyncContextManager[ tuple[ClientSession, InitializeResult]]]) -> None: @@ -238,6 +238,8 @@ async def _serve(self, calls: asyncio.Queue) -> None: try: while (call := await calls.get()) is not None: tool, arguments, answer = call + if answer.done(): + continue try: result = await session.call_tool(tool, arguments) except McpError as e: diff --git a/tst/unit/drivers/aside_driver_test.py b/tst/unit/drivers/aside_driver_test.py index 9eedad6aa..f5ec3c9ef 100644 --- a/tst/unit/drivers/aside_driver_test.py +++ b/tst/unit/drivers/aside_driver_test.py @@ -1,6 +1,7 @@ """The aside driver against a fake aside CLI: agents get its repl as pinned, with its instructions, marked as acting on apps; allowed_apps becomes one --app each, none for every app, and never empty; each agent session keeps a REPL of -its own across calls, one that stops starting afresh; the CLI gets the session user's environment, Aside's home and +its own across calls, one that stops starting afresh, and a call its caller gave up while it waited never reaches +Aside; the CLI gets the session user's environment, Aside's home and no update check, and none of the driver's; readiness checks the pinned executables, the daemon's health, turns usage analytics off and Computer Use on, and checks what the server says and lists and that it answers; and the driver's files agree on the pinned version, folder, helper, daemon and Aside's home, where the helper's runtime copy is placed @@ -218,6 +219,37 @@ async def slow_to_end(self): assert after.startswith("None in ") and after != f"None in {a_pid}" +@pytest.mark.asyncio +async def test_a_call_cancelled_while_it_waits_behind_another_never_reaches_aside(pinned): + sent, release = [], asyncio.Event() + + class Session: + async def call_tool(self, tool, arguments): + sent.append(arguments["code"]) + if arguments["code"] == "slow": + await release.wait() + return arguments["code"] + + @contextlib.asynccontextmanager + async def open_server(): + yield Session(), None + + repl = pinned._Repl(open_server) + slow = asyncio.create_task(repl.call("repl", {"code": "slow"})) + while not sent: + await asyncio.sleep(0) + abandoned = asyncio.create_task(repl.call("repl", {"code": "abandoned"})) + await asyncio.sleep(0) + abandoned.cancel() + with pytest.raises(asyncio.CancelledError): + await abandoned + release.set() + answers = (await slow, await repl.call("repl", {"code": "after"})) + await repl.close() + + assert answers == ("slow", "after") and sent == ["slow", "after"] and repl.busy == 0 + + @pytest.mark.asyncio async def test_the_cli_gets_the_session_users_environment_and_none_of_the_drivers(module, monkeypatch, tmp_path): async with mcp_session(_driver(module, monkeypatch)) as session: