diff --git a/src/agent_env/drivers/README.md b/src/agent_env/drivers/README.md index 04ab85777..1202fee26 100644 --- a/src/agent_env/drivers/README.md +++ b/src/agent_env/drivers/README.md @@ -194,5 +194,30 @@ another ChatGPT version, set `VERSION`, pin the zip's URL and sha256 in `env.tom path), and take `mcp.json` again from what that version's `cua_repl` says at connect and lists. Name it as `codex-computer-use`. +`envs/aside/` is At Inc.'s Aside Computer Use as a driver, for the same macOS computers. Its MCP server is Aside's CLI, +`aside mcp --computer-use`, with one tool, `repl`. `repl` runs the agent's JavaScript in a QuickJS sandbox whose only +app API is the `computer` global, with no file system, network or shell. `computer` reads apps' accessibility trees and +acts on them in the background through Aside's daemon and its helper app. `driver.py` pins two of At Inc.'s published +releases: the daemon component as `VERSION`, the CRX Aside's browsers update in the background, and the CLI as +`CLI_VERSION`. The install downloads both from `releases.aside.com` and checks them by sha256, and `install.sh` unpacks +them to `/opt/agentenv/aside/`, links `bin/aside` into the CLI app, and writes the account's settings with usage +analytics off before the daemon first starts. The daemon runs as a direct LaunchAgent of its own executable, since macOS +gives an app's TCC permissions only to the executable launchd starts itself. `[install.macos]` grants Accessibility, +Post Event and Screen Recording to the helper app, to the copy the daemon runs from its home, and to the driver's own +Python. The driver starts one server for each agent session, since the REPL keeps its bindings between calls, with the +session user's environment, Aside's home and the CLI's update check off. A server that stops is started afresh on the +next call, and nothing says when a session ends, so one left unused for 30 minutes, or past the eight kept, is closed, +the least recently used first. Each server gets an `--app` for each app its `allowed_apps` setting names, by name or +bundle identifier, case aside, so `computer.getApp` opens only those, or every app when it names none. Agents get the +server's instructions and its `repl` tool, as `mcp.json` pins them, marked as acting on apps. No Aside sign-in is +needed. Its `computer_*` tools are agent-desktop's, from `MacOSDriver`. `computer-ready` checks the executables' sha256 +and that the daemon's health endpoint answers ready. It turns usage analytics off (`aside settings analytics disable`) +and Computer Use on (`aside settings computer-use enable`), then checks that the server gives the instructions and lists +the tool `mcp.json` pins and answers `computer.listApps()`, and the TCC permissions. To move it to other Aside releases, +set `VERSION` and `CLI_VERSION`, pin both downloads' URLs and sha256 in `env.toml` and the executables' in `driver.py`'s +`EXECUTABLES`, and take `mcp.json` again from what that version's server says at connect and lists. A unit test checks +that `env.toml`, `install.sh` and `driver.py` agree on the versions, the folder, the helper and the daemon. Name it as +`aside`. + This bundle has no tasks of its own: other bundles pull its drivers by alias, and `agent-env run drivers` has nothing to run. diff --git a/src/agent_env/drivers/aliases.toml b/src/agent_env/drivers/aliases.toml index a93822b44..1a88d2fd2 100644 --- a/src/agent_env/drivers/aliases.toml +++ b/src/agent_env/drivers/aliases.toml @@ -6,3 +6,4 @@ page = ["page"] agent-desktop = ["agent-desktop"] macos-harness = ["macos-harness"] codex-computer-use = ["codex-computer-use"] +aside = ["aside"] diff --git a/src/agent_env/drivers/envs/aside/driver.py b/src/agent_env/drivers/envs/aside/driver.py new file mode 100644 index 000000000..81ca5329f --- /dev/null +++ b/src/agent_env/drivers/envs/aside/driver.py @@ -0,0 +1,318 @@ +"""Aside Computer Use as a computer driver on macOS: Aside's CLI, ``aside mcp --computer-use``, serves one tool, +``repl``, which runs the JavaScript an agent sends in a QuickJS sandbox whose only app API is the ``computer`` +global, which reads apps' accessibility trees and acts on them in the background through Aside's daemon and its +helper app. The pinned releases are unpacked to ``/opt/agentenv/aside/``; the daemon runs there as a service of its +own, with Aside's home in ``home/`` there, where the daemon runs its copy of the helper app from, and the driver +turns usage analytics off and Computer Use on once it answers. The driver starts the CLI's server with a minimal +environment and an ``--app`` for each app the ``allowed_apps`` setting names, or none for every app, one server for +each agent session, since the REPL keeps its bindings between calls, closed once it's left unused or past the most +kept, since nothing says when a session ends. Agents get the server's instructions and its ``repl`` tool as pinned +(``mcp.json``), marked as acting on apps. No Aside sign-in is needed. The computer/v1 operations are MacOSDriver's; +this process's Python is granted Accessibility, Post Event and Screen Recording, as the helper app is. It needs +``agentenv-framework-protocol[computer,macos]`` and the pinned releases.""" +from __future__ import annotations + +import asyncio +import contextlib +import functools +import hashlib +import json +import os +import time +import urllib.request +from pathlib import Path +from typing import Any, AsyncIterator, Callable + +from mcp import ClientSession, McpError, StdioServerParameters +from mcp.client.stdio import stdio_client +from mcp.types import CONNECTION_CLOSED, CallToolResult, InitializeResult, TextContent +from pydantic import Field + +from agentenv_protocol import EnvironmentTool, ForwardedTool, ToolContext, server_instructions +from agentenv_protocol.computer import ComputerSettings, MacOSDriver, ReadyCheck, computer_driver + +HERE = Path(__file__).resolve().parent +#: The pinned Aside releases, as At Inc. publishes them and install.sh unpacks them: the daemon component, the daemon +#: and the helper app that Aside's browsers update in the background, and the CLI; and where the CLI and the helper are. +VERSION = "1.26.1111.1046" +CLI_VERSION = "1.26.1111.1046" +PREFIX = Path("/opt/agentenv/aside") +CLI = Path("bin", "aside") +HELPER = "Aside Computer Use.app" +#: The daemon app's executable, which env.toml's daemon service runs. +DAEMON = Path("Aside Daemon.app", "Contents", "MacOS", "aside-daemon") +#: Aside's home, ``ASIDE_HOME`` for the daemon service and every CLI the driver starts, fixed so the daemon's runtime +#: copy of the helper, which install.sh places and env.toml grants, has a fixed path. +ASIDE_HOME = PREFIX / "home" +#: The sha256 of each executable at the pinned version, by its path under the prefix. +EXECUTABLES = { + str(DAEMON): "40b91cd799b8ad404d71137a62535b8335b1c2368acdf87f3cb9945e4860572b", + f"{HELPER}/Contents/MacOS/aside-computer-use": "768662135dac8727e41791469b59c3a0312551d727c2d1f70cfe907182033d3c", + "bin/aside": "0c13c719966302a8bf6cb5382c41765512139d3dcd360d067a4fec2c56dcf374", +} +#: What ``aside mcp --computer-use`` says at connect and lists at the pinned version: its instructions and its tool. +MANIFEST = HERE / "mcp.json" +#: The hints agents get, where Aside gives none: ``repl`` acts on apps, and through them on anything. +ANNOTATIONS = {"repl": {"readOnlyHint": False, "destructiveHint": True, "openWorldHint": True}} +#: The daemon's health endpoint, which answers ``"ready": true`` once it serves, and how long it has to answer. +HEALTH, HEALTH_TIMEOUT_S = "http://127.0.0.1:21420/health", 5 +#: How long an ``aside settings`` command has to finish. +SETTINGS_TIMEOUT_S = 30 +#: How long a session's REPL is kept unused before it's closed, since nothing says when a session ends; how many are +#: kept at most, the least recently used closed first; and how often the idle ones are looked for. +IDLE_S, MAX_REPLS, SWEEP_S = 1800, 8, 60 +#: How long a closed REPL's server has to end before it's cancelled. +CLOSE_TIMEOUT_S = 10 +#: What of the driver's environment the CLI gets: the session user's own; none of a deploy's values. +KEPT = ("HOME", "USER", "LOGNAME", "TMPDIR", "LANG") +#: The PATH it runs with. +PATH = "/usr/bin:/bin:/usr/sbin:/sbin" +#: The JavaScript the readiness check runs: an inventory, which needs no app, printed as the number of apps. +PROBE = "console.log((await computer.listApps({ emit: false })).length)" + + +@computer_driver(name="aside", + supports=[{"os": {"type": "macos", "display": ["aqua"]}, "arch": ["arm64"]}]) +class AsideDriver(MacOSDriver): + class Settings(ComputerSettings): + #: The apps ``computer.getApp`` may open, by name or bundle identifier, case aside; None for every app, and + #: never empty, which would pass no ``--app`` and so allow every app too. + allowed_apps: list[str] | None = Field(default=None, min_length=1) + + def __init__(self) -> None: + super().__init__() + self._pinned: list[ReadyCheck] | None = None + #: Each session's REPL, the least recently used first. + self._repls: dict[str | None, _Repl] = {} + self._sweeper: asyncio.Task | None = None + #: The closes under way, held so they aren't collected before they end. + self._closing: set[asyncio.Task] = set() + + def forwarded_tools(self) -> list[ForwardedTool]: + return [ForwardedTool(EnvironmentTool(name=t["name"], description=t["description"], + inputSchema=t["inputSchema"]), + functools.partial(self._forward, t["name"]), + {**(t.get("annotations") or {}), **ANNOTATIONS.get(t["name"], {})}) + for t in _manifest()["tools"]] + + @server_instructions + def aside_instructions(self) -> str: + return _manifest()["instructions"] + + async def ready_checks(self) -> list[ReadyCheck]: + """The pinned executables, the daemon, analytics off and Computer Use on, what the server says and lists and + that it answers, checked until they pass once, and MacOSDriver's TCC permissions every time.""" + if self._pinned is None or not all(check.ok for check in self._pinned): + self._pinned = await self._pinned_checks() + return [*self._pinned, *await super().ready_checks()] + + async def _forward(self, tool: str, /, **arguments: Any) -> CallToolResult: + """A call to Aside's tool on the caller's own REPL, answered as Aside answers it. Other sessions' REPLs left + idle, or past the most kept, are closed. Nothing here awaits before the call is queued, so calls a session + makes together find one REPL.""" + repl = self._repls.pop(caller := ToolContext.current().caller.session, None) or _Repl(self._server) + self._evict(keep=MAX_REPLS - 1) + self._repls[caller] = repl + if self._sweeper is None or self._sweeper.done(): + self._sweeper = asyncio.create_task(self._sweep()) + try: + return await repl.call(tool, arguments) + except Exception as e: + return CallToolResult(isError=True, content=[TextContent(type="text", text=( + f"Aside's MCP server failed before it answered, so {tool} may have run: {type(e).__name__}: {e}"))]) + + def _evict(self, keep: int = MAX_REPLS) -> None: + """Take out the REPLs idle for ``IDLE_S`` and, past ``keep``, the least recently used, none of them in a call, + and close them in the background.""" + now = time.monotonic() + free = [caller for caller, repl in self._repls.items() if not repl.busy] + idle = [caller for caller in free if now - self._repls[caller].used > IDLE_S] + over = max(0, len(self._repls) - len(idle) - keep) + spare = [caller for caller in free if caller not in idle][:over] + for caller in [*idle, *spare]: + closing = asyncio.create_task(self._repls.pop(caller).close()) + self._closing.add(closing) + closing.add_done_callback(self._closing.discard) + + async def _sweep(self) -> None: + """Close idle REPLs every ``SWEEP_S`` while there are any.""" + while self._repls: + await asyncio.sleep(SWEEP_S) + self._evict() + + @contextlib.asynccontextmanager + async def _server(self) -> AsyncIterator[tuple[ClientSession, InitializeResult]]: + """``aside mcp --computer-use`` for the allowed apps, connected and initialized, with what it said at connect; + it ends with the block, as closing its stdin ends it.""" + server = StdioServerParameters(command=str(PREFIX / CLI), args=_arguments(self.settings.allowed_apps), + env=_environment()) + async with stdio_client(server) as (read, write), ClientSession(read, write) as session: + yield session, await session.initialize() + + async def _pinned_checks(self) -> list[ReadyCheck]: + digests = await asyncio.to_thread(_digests) + differ = sorted(path for path, digest in digests.items() if digest != EXECUTABLES[path]) + pinned = f"Aside {VERSION}, CLI {CLI_VERSION}" + checks = [ReadyCheck("Aside Computer Use", not differ, f"{pinned}, as pinned" if not differ else + f"these differ from the pinned {pinned}: {', '.join(differ)}")] + if differ: + return checks + try: + healthy = (await asyncio.to_thread(_health)).get("ready") is True + except Exception as e: + return [*checks, ReadyCheck("Aside daemon", False, f"{HEALTH} didn't answer: {type(e).__name__}: {e}")] + checks.append(ReadyCheck("Aside daemon", healthy, "it answers ready" if healthy else + f"{HEALTH} doesn't say it's ready")) + if not healthy: + return checks + for name, words in (("Analytics off", ("analytics", "disable")), + ("Computer Use on", ("computer-use", "enable"))): + code, said = await _setting(*words) + checks.append(ReadyCheck(name, code == 0, said or f"it exited {code}")) + if code != 0: + return checks + try: + async with self._server() as (session, initialized): + listed = (await session.list_tools()).tools + probed = await session.call_tool("repl", {"code": PROBE}) + except Exception as e: + return [*checks, ReadyCheck("Aside tools", False, f"aside mcp didn't answer: {type(e).__name__}: {e}")] + running = {t.name: t.model_dump(mode="json", exclude_none=True) for t in listed} + pinned = {t["name"]: t for t in _manifest()["tools"]} + changed = ["instructions"] if initialized.instructions != _manifest()["instructions"] else [] + changed += sorted(name for name in running.keys() | pinned.keys() if name not in running or name not in pinned + or any(running[name].get(key) != value for key, value in pinned[name].items())) + answer = " ".join(c.text for c in probed.content if isinstance(c, TextContent)).strip() + listed_apps = not probed.isError and answer.isdigit() and int(answer) > 0 + return [*checks, + ReadyCheck("Aside tools", not changed, f"its instructions and {len(running)} tool, as pinned" + if not changed else f"these differ from {MANIFEST.name}: {', '.join(changed)}"), + ReadyCheck("Aside answers", listed_apps, f"it read the computer's {answer} apps" if listed_apps + else f"computer.listApps() gave no apps: {answer}")] + + +class _Repl: + """A session's ``aside mcp``, one server at a time, each held open by a task of its own, since a stdio client is + entered and left in one task, and reached from any through that server's queue, one call at a time; a call its + caller gave up while it waited is dropped, never sent. A server that stops is let go at once, while it still ends, + and the next call starts one afresh, without the last one's bindings; ``close`` ends them for good.""" + + def __init__(self, open_server: Callable[[], contextlib.AbstractAsyncContextManager[ + tuple[ClientSession, InitializeResult]]]) -> None: + self._open = open_server + #: The queue of the server taking calls, None when none is; and the tasks of every server not yet ended. + self._calls: asyncio.Queue | None = None + self._serving: set[asyncio.Task] = set() + #: When a call last ended, on the monotonic clock, and how many are queued or running. + self.used, self.busy = time.monotonic(), 0 + + async def call(self, tool: str, arguments: dict) -> CallToolResult: + if self._calls is None: + self._calls = asyncio.Queue() + serving = asyncio.create_task(self._serve(self._calls)) + self._serving.add(serving) + serving.add_done_callback(self._serving.discard) + answer = asyncio.get_running_loop().create_future() + self.busy += 1 + try: + self._calls.put_nowait((tool, arguments, answer)) + return await answer + finally: + self.busy -= 1 + self.used = time.monotonic() + + async def close(self) -> None: + """Every server ended, its process with it, waiting at most ``CLOSE_TIMEOUT_S`` before cancelling one.""" + if self._calls is not None: + self._calls.put_nowait(None) + self._calls = None + if serving := set(self._serving): + _, late = await asyncio.wait(serving, timeout=CLOSE_TIMEOUT_S) + for task in late: + task.cancel() + + async def _serve(self, calls: asyncio.Queue) -> None: + stopped: BaseException = ConnectionError("aside mcp stopped") + try: + async with self._open() as (session, _): + try: + while (call := await calls.get()) is not None: + tool, arguments, answer = call + if answer.done(): + continue + try: + result = await session.call_tool(tool, arguments) + except McpError as e: + _settle(answer, error=e) + if e.error.code == CONNECTION_CLOSED: + raise + continue + except Exception as e: + _settle(answer, error=e) + raise + _settle(answer, result=result) + finally: + self._let_go(calls) + except Exception as e: + stopped = e + finally: + self._let_go(calls) + while not calls.empty(): + if (call := calls.get_nowait()) is not None: + _settle(call[-1], error=stopped) + + def _let_go(self, calls: asyncio.Queue) -> None: + """The next call to a new server, not to this one, which takes no more.""" + if self._calls is calls: + self._calls = None + + +def _settle(answer: asyncio.Future, *, result: Any = None, error: BaseException | None = None) -> None: + if answer.done(): + return + if error is not None: + answer.set_exception(error) + else: + answer.set_result(result) + + +@functools.cache +def _manifest() -> dict: + return json.loads(MANIFEST.read_text()) + + +def _arguments(allowed_apps: list[str] | None) -> list[str]: + """The CLI's arguments for a server that may open ``allowed_apps``, every app for None.""" + return ["mcp", "--computer-use", *(word for app in allowed_apps or [] for word in ("--app", app))] + + +def _environment() -> dict[str, str]: + """The CLI's environment: the session user's own, the system PATH and Aside's home, with no update check.""" + return {**{name: os.environ[name] for name in KEPT if name in os.environ}, "PATH": PATH, + "ASIDE_HOME": str(ASIDE_HOME), "ASIDE_CLI_UPDATE_CHECK": "0"} + + +def _health() -> dict: + """What the daemon's health endpoint answers.""" + with urllib.request.urlopen(HEALTH, timeout=HEALTH_TIMEOUT_S) as response: + return json.load(response) + + +async def _setting(*words: str) -> tuple[int, str]: + """``aside settings ``, which needs the daemon and is idempotent: its exit code and what it printed.""" + process = await asyncio.create_subprocess_exec( + str(PREFIX / CLI), "settings", *words, stdin=asyncio.subprocess.DEVNULL, + stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.STDOUT, env=_environment()) + try: + out, _ = await asyncio.wait_for(process.communicate(), SETTINGS_TIMEOUT_S) + except TimeoutError: + process.kill() + await process.wait() + return 124, f"aside settings {' '.join(words)} didn't finish within {SETTINGS_TIMEOUT_S} s" + return process.returncode, out.decode(errors="replace").strip() + + +def _digests() -> dict[str, str | None]: + """Each pinned executable's sha256, None for one that isn't there.""" + return {path: hashlib.sha256((PREFIX / path).read_bytes()).hexdigest() if (PREFIX / path).is_file() else None + for path in EXECUTABLES} diff --git a/src/agent_env/drivers/envs/aside/env.toml b/src/agent_env/drivers/envs/aside/env.toml new file mode 100644 index 000000000..14ec76905 --- /dev/null +++ b/src/agent_env/drivers/envs/aside/env.toml @@ -0,0 +1,20 @@ +type = "computer_driver" +module = "driver.py:AsideDriver" +port = 18765 + +[install.macos] +python = "requirements-macos.txt" +downloads = [ + { url = "https://releases.aside.com/dev-updater/components/1.26.1111.1046/AsideDaemon/mac-arm64/AsideDaemon-1.26.1111.1046.crx", sha256 = "47f54f506f1b8e7abb75d1ca263bb388e5d4e1bd295c3b52e64b5959082744c6", to = "aside-daemon.crx", arch = "arm64" }, + { url = "https://releases.aside.com/cli/AsideCLI-darwin-arm64-1.26.1111.1046.zip", sha256 = "f4b234e8a63eb2a721a9094b7ded1c286d4f253ede3b4eb347c2d3e0ef04765c", to = "aside-cli.zip", arch = "arm64" }, +] +run = [{ script = "install.sh", scope = "session" }] +grants = [ + { executable = "venv/bin/python", services = ["accessibility", "post_event", "screen_capture"] }, + { app = "/opt/agentenv/aside/Aside Computer Use.app", services = ["accessibility", "post_event", "screen_capture"] }, + { app = "/opt/agentenv/aside/home/runtime-computer-use/Aside Computer Use.app", services = ["accessibility", "post_event", "screen_capture"] }, +] +services = [ + { name = "daemon", command = "'/opt/agentenv/aside/Aside Daemon.app/Contents/MacOS/aside-daemon'", direct = true, environment = { ASIDE_HOME = "/opt/agentenv/aside/home" } }, + { name = "driver", command = "venv/bin/python -m agentenv_protocol.computer.serve driver.py:AsideDriver" }, +] diff --git a/src/agent_env/drivers/envs/aside/install.sh b/src/agent_env/drivers/envs/aside/install.sh new file mode 100755 index 000000000..c06590ca8 --- /dev/null +++ b/src/agent_env/drivers/envs/aside/install.sh @@ -0,0 +1,20 @@ +#!/bin/sh +# The pinned daemon component and CLI, unpacked to where the grants and the daemon service name them, as the session +# user. The helper's runtime copy is placed before the daemon first starts, since macOS keys its approval to that path. +set -eu +daemon=aside-daemon.crx +cli=aside-cli.zip +dest=/opt/agentenv/aside +helper="Aside Computer Use.app" +rm -rf "$dest" +mkdir -p "$dest/bin" "$dest/home/runtime-computer-use" +# A CRX is a signature header and then a zip, which bsdtar reads. +tar -xf "$daemon" -C "$dest" +ditto -x -k "$cli" "$dest" +ln -s "../Aside CLI.app/Contents/MacOS/aside" "$dest/bin/aside" +rm -f "$daemon" "$cli" +xattr -dr com.apple.quarantine "$dest" || true +ditto "$dest/$helper" "$dest/home/runtime-computer-use/$helper" +# Usage analytics are off from the daemon's first start. +mkdir -p "$dest/home/u/0" +printf '%s\n' '{"analytics":{"enabled":false}}' > "$dest/home/u/0/settings.json" diff --git a/src/agent_env/drivers/envs/aside/mcp.json b/src/agent_env/drivers/envs/aside/mcp.json new file mode 100644 index 000000000..7554e8287 --- /dev/null +++ b/src/agent_env/drivers/envs/aside/mcp.json @@ -0,0 +1,29 @@ +{ + "instructions": "Operate native apps on this computer with the repl tool. Start with `await computer.getApp('')` and follow the guide it prints.", + "tools": [ + { + "name": "repl", + "description": "Run JavaScript in a sandbox whose only API is the `computer` global, which reads and controls native apps on this computer. There is no file system, network, shell, or browser. Variables persist between calls. Start with `await computer.getApp('')`; it prints the app's state and the full usage guide.", + "inputSchema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "properties": { + "code": { + "description": "JavaScript to run. Top-level await works.", + "type": "string" + }, + "title": { + "description": "Action title, e.g. \"Type the note\".", + "type": "string" + } + }, + "required": [ + "code" + ], + "type": "object" + }, + "execution": { + "taskSupport": "forbidden" + } + } + ] +} diff --git a/src/agent_env/drivers/envs/aside/requirements-macos.txt b/src/agent_env/drivers/envs/aside/requirements-macos.txt new file mode 100644 index 000000000..a8c357df2 --- /dev/null +++ b/src/agent_env/drivers/envs/aside/requirements-macos.txt @@ -0,0 +1,4 @@ +# What a composed macOS computer installs into the driver's venv: the protocol with MacOSDriver, whose screen and input +# need its macos extra, and the MCP client the forwarded tool reaches Aside's MCP server with. A release pins it to the +# version it ships. Aside is the pinned download, whose executables' sha256 the driver checks. +agentenv-framework-protocol[computer,macos]>=0.1.459 diff --git a/tst/unit/bundle/shipped_bundles_test.py b/tst/unit/bundle/shipped_bundles_test.py index 357d5708f..666041e8d 100644 --- a/tst/unit/bundle/shipped_bundles_test.py +++ b/tst/unit/bundle/shipped_bundles_test.py @@ -64,6 +64,7 @@ def test_the_shipped_env_aliases_name_the_mail_env_the_validator_deploys_the_dri "agent-desktop": "@local/agentenv-framework/drivers/agent-desktop", "macos-harness": "@local/agentenv-framework/drivers/macos-harness", "codex-computer-use": "@local/agentenv-framework/drivers/codex-computer-use", + "aside": "@local/agentenv-framework/drivers/aside", "chromium": "@local/agentenv-framework/browsers/chromium"} assert A2AAgent.VALIDATION_ENV_ID == "mail" diff --git a/tst/unit/drivers/aside_driver_test.py b/tst/unit/drivers/aside_driver_test.py new file mode 100644 index 000000000..f5ec3c9ef --- /dev/null +++ b/tst/unit/drivers/aside_driver_test.py @@ -0,0 +1,367 @@ +"""The aside driver against a fake aside CLI: agents get its repl as pinned, with its instructions, marked as acting on +apps; allowed_apps becomes one --app each, none for every app, and never empty; each agent session keeps a REPL of +its own across calls, one that stops starting afresh, and a call its caller gave up while it waited never reaches +Aside; the CLI gets the session user's environment, Aside's home and +no update check, and none of the driver's; readiness checks the pinned executables, the daemon's health, turns usage +analytics off and Computer Use on, and checks what the server says and lists and that it answers; and the driver's +files agree on the pinned version, folder, helper, daemon and Aside's home, where the helper's runtime copy is placed +and granted.""" +from __future__ import annotations + +import asyncio +import contextlib +import hashlib +import importlib.util +import json +import os +import shlex +import sys +import textwrap +import tomllib +from pathlib import Path + +import pytest +from mcp.shared.memory import create_connected_server_and_client_session +from pydantic import ValidationError + +import agent_env +from agentenv_protocol import SESSION_META_KEY +from agentenv_protocol.computer import Platform, _macos + +DRIVER = Path(agent_env.__file__).parent / "drivers" / "envs" / "aside" / "driver.py" +FAKE = textwrap.dedent('''\ + import json, os, sys + from pathlib import Path + state = Path(os.environ["HOME"]) / "fake" + if sys.argv[1:] == ["settings", "computer-use", "enable"]: + with open(state / "enabled", "a") as out: + out.write("enabled\\n") + (state / "on").touch() + print("Computer Use is on.") + sys.exit(0) + if sys.argv[1:] == ["settings", "analytics", "disable"]: + print("Analytics are off.") + sys.exit(0) + import anyio + import mcp.types as types + from mcp.server.lowlevel import Server + from mcp.server.stdio import stdio_server + (state / f"env-{os.getpid()}.json").write_text(json.dumps(dict(os.environ))) + manifest = json.loads((state / "tools.json").read_text()) + server = Server("aside", instructions=manifest["instructions"]) + bindings = {} + + @server.list_tools() + async def tools(): + return [types.Tool(**t) for t in manifest["tools"]] + + @server.call_tool(validate_input=False) + async def call(name, arguments): + if not (state / "on").exists(): + raise RuntimeError("Computer Use is turned off. Turn it on with 'aside settings computer-use enable'.") + code = arguments["code"] + if code.startswith("set "): + bindings["x"] = code[4:] + text = "set" + elif code == "get": + text = f"{bindings.get('x')} in {os.getpid()}" + elif code == "args": + text = json.dumps(sys.argv[1:]) + elif code == "die": + os._exit(1) + elif "computer.listApps" in code: + text = "3" + else: + text = "ran" + return [types.TextContent(type="text", text=text)] + + async def main(): + async with stdio_server() as (read, write): + await server.run(read, write, server.create_initialization_options()) + + anyio.run(main) +''') +MACOS = Platform(os="macos", version="26.5", arch="arm64", display="aqua") + + +class Granted: + def missing(self) -> list[str]: + return [] + + +@pytest.fixture +def pinned(monkeypatch, tmp_path): + """The driver's module, as it ships.""" + monkeypatch.delenv("AGENTENV_COMPUTER_SETTINGS", raising=False) + monkeypatch.setenv("HOME", str(tmp_path)) + monkeypatch.setenv("AGENTENV_CONTROL_TOKEN", "a-deploys-secret") + monkeypatch.setattr(_macos, "Quartz", Granted) + monkeypatch.setattr(sys, "platform", "darwin") + monkeypatch.setattr("agentenv_protocol.computer.driver.detect", lambda: MACOS) + spec = importlib.util.spec_from_file_location("aside_driver", DRIVER) + loaded = importlib.util.module_from_spec(spec) + spec.loader.exec_module(loaded) + return loaded + + +@pytest.fixture +def module(pinned, monkeypatch, tmp_path): + """The driver's module on a fake Aside whose CLI is the fake server, its daemon healthy and Computer Use on.""" + (tmp_path / "fake").mkdir() + (tmp_path / "fake" / "tools.json").write_text(pinned.MANIFEST.read_text()) + (tmp_path / "fake" / "on").touch() + prefix = tmp_path / "aside" + cli = prefix / pinned.CLI + cli.parent.mkdir(parents=True) + cli.write_text(f"#!{sys.executable}\n{FAKE}") + cli.chmod(0o755) + digests = {} + for path in pinned.EXECUTABLES: + file = prefix / path + if not file.exists(): + file.parent.mkdir(parents=True, exist_ok=True) + file.write_text(f"the pinned {file.name}") + digests[path] = hashlib.sha256(file.read_bytes()).hexdigest() + monkeypatch.setattr(pinned, "PREFIX", prefix) + monkeypatch.setattr(pinned, "EXECUTABLES", digests) + monkeypatch.setattr(pinned, "_health", lambda: {"ready": True}) + return pinned + + +def _driver(module, monkeypatch, **settings): + monkeypatch.setenv("AGENTENV_COMPUTER_SETTINGS", json.dumps(settings)) + return module.AsideDriver() + + +@contextlib.asynccontextmanager +async def mcp_session(driver): + async with create_connected_server_and_client_session(driver.mcp or driver.create_app()) as session: + yield session + + +def _text(result) -> str: + return " ".join(c.text for c in result.content if c.type == "text") + + +async def _gone(pid: int) -> bool: + for _ in range(200): + try: + os.kill(pid, 0) + except ProcessLookupError: + return True + await asyncio.sleep(0.01) + return False + + +@pytest.mark.asyncio +async def test_agents_get_asides_repl_as_pinned_with_its_instructions_marked_as_acting_on_apps(module, monkeypatch): + driver = _driver(module, monkeypatch) + async with mcp_session(driver) as session: + listed = {t.name: t for t in (await session.list_tools()).tools} + answered = await session.call_tool("repl", {"code": "1 + 1"}) + + (pinned,) = module._manifest()["tools"] + assert listed["repl"].inputSchema == pinned["inputSchema"] and listed["repl"].description == pinned["description"] + hints = listed["repl"].annotations + assert (hints.readOnlyHint, hints.destructiveHint, hints.openWorldHint) == (False, True, True) + assert driver.mcp.instructions.startswith(module._manifest()["instructions"]) + assert not answered.isError and _text(answered) == "ran" + + +@pytest.mark.asyncio +async def test_allowed_apps_become_one_app_flag_each_and_none_for_every_app(module, monkeypatch): + async with mcp_session(_driver(module, monkeypatch, allowed_apps=["Calculator", "com.apple.TextEdit"])) as session: + named = json.loads(_text(await session.call_tool("repl", {"code": "args"}))) + async with mcp_session(_driver(module, monkeypatch)) as session: + every = json.loads(_text(await session.call_tool("repl", {"code": "args"}))) + + assert named == ["mcp", "--computer-use", "--app", "Calculator", "--app", "com.apple.TextEdit"] + assert every == ["mcp", "--computer-use"] + with pytest.raises(ValidationError): + module.AsideDriver.Settings(allowed_apps=[]) + + +@pytest.mark.asyncio +async def test_a_failing_cell_reaches_the_agent_as_aside_answers_it(module, monkeypatch, tmp_path): + (tmp_path / "fake" / "on").unlink() + async with mcp_session(_driver(module, monkeypatch)) as session: + failed = await session.call_tool("repl", {"code": "1 + 1"}) + + assert failed.isError and "Computer Use is turned off" in _text(failed) + + +@pytest.mark.asyncio +async def test_each_agent_session_keeps_a_repl_of_its_own_and_one_that_stops_starts_afresh_while_it_still_ends( + module, monkeypatch): + """A server that stops takes a while to end; the session's next call starts a new one rather than wait on it.""" + serve = module.AsideDriver._server + + @contextlib.asynccontextmanager + async def slow_to_end(self): + async with serve(self) as opened: + try: + yield opened + finally: + await asyncio.sleep(0.5) + + monkeypatch.setattr(module.AsideDriver, "_server", slow_to_end) + async with mcp_session(_driver(module, monkeypatch)) as session: + for caller, value in (("a", "1"), ("b", "2")): + await session.call_tool("repl", {"code": f"set {value}"}, meta={SESSION_META_KEY: caller}) + a = _text(await session.call_tool("repl", {"code": "get"}, meta={SESSION_META_KEY: "a"})) + b = _text(await session.call_tool("repl", {"code": "get"}, meta={SESSION_META_KEY: "b"})) + died = await session.call_tool("repl", {"code": "die"}, meta={SESSION_META_KEY: "a"}) + after = _text(await session.call_tool("repl", {"code": "get"}, meta={SESSION_META_KEY: "a"})) + + (a_value, _, a_pid), (b_value, _, b_pid) = a.partition(" in "), b.partition(" in ") + assert (a_value, b_value) == ("1", "2") and a_pid != b_pid + assert died.isError and "may have run" in _text(died) + assert after.startswith("None in ") and after != f"None in {a_pid}" + + +@pytest.mark.asyncio +async def test_a_call_cancelled_while_it_waits_behind_another_never_reaches_aside(pinned): + sent, release = [], asyncio.Event() + + class Session: + async def call_tool(self, tool, arguments): + sent.append(arguments["code"]) + if arguments["code"] == "slow": + await release.wait() + return arguments["code"] + + @contextlib.asynccontextmanager + async def open_server(): + yield Session(), None + + repl = pinned._Repl(open_server) + slow = asyncio.create_task(repl.call("repl", {"code": "slow"})) + while not sent: + await asyncio.sleep(0) + abandoned = asyncio.create_task(repl.call("repl", {"code": "abandoned"})) + await asyncio.sleep(0) + abandoned.cancel() + with pytest.raises(asyncio.CancelledError): + await abandoned + release.set() + answers = (await slow, await repl.call("repl", {"code": "after"})) + await repl.close() + + assert answers == ("slow", "after") and sent == ["slow", "after"] and repl.busy == 0 + + +@pytest.mark.asyncio +async def test_the_cli_gets_the_session_users_environment_and_none_of_the_drivers(module, monkeypatch, tmp_path): + async with mcp_session(_driver(module, monkeypatch)) as session: + await session.call_tool("repl", {"code": "1 + 1"}) + + (env_file,) = (tmp_path / "fake").glob("env-*.json") + env = json.loads(env_file.read_text()) + assert "AGENTENV_CONTROL_TOKEN" not in env and "AGENTENV_COMPUTER_SETTINGS" not in env + assert env["PATH"] == "/usr/bin:/bin:/usr/sbin:/sbin" and env["HOME"] == str(tmp_path) + assert env["ASIDE_HOME"] == str(module.ASIDE_HOME) and env["ASIDE_CLI_UPDATE_CHECK"] == "0" + + +@pytest.mark.asyncio +async def test_readiness_checks_the_pinned_executables_the_daemon_turns_computer_use_on_and_checks_the_server( + module, monkeypatch, tmp_path): + driver = _driver(module, monkeypatch) + manifest = module._manifest() + (tmp_path / "fake" / "on").unlink() + + async def checks(served: dict | None = None): + if served is not None: + (tmp_path / "fake" / "tools.json").write_text(json.dumps(served)) + driver._pinned = None + return await driver.ready_checks() + + ready = await driver.ready_checks() + again = await driver.ready_checks() + described = (await checks({"instructions": "other", "tools": [ + {**t, "description": "other"} for t in manifest["tools"]]}))[4] + extra = (await checks({**manifest, "tools": [*manifest["tools"], {**manifest["tools"][0], "name": "exec"}]}))[4] + monkeypatch.setattr(module, "_health", lambda: {"ready": False}) + unready = await checks(manifest) + (module.PREFIX / module.HELPER / "Contents" / "MacOS" / "aside-computer-use").write_text("another build") + (changed, *rest) = await checks() + + assert [(c.name, c.ok) for c in ready] == [("Aside Computer Use", True), ("Aside daemon", True), + ("Analytics off", True), ("Computer Use on", True), + ("Aside tools", True), + ("Aside answers", True), ("macOS permissions", True)] + assert (ready[2].detail, ready[3].detail) == ("Analytics are off.", "Computer Use is on.") and again == ready + assert (tmp_path / "fake" / "enabled").read_text().count("enabled") == 3 + assert (described.ok, described.detail) == (False, "these differ from mcp.json: instructions, repl") + assert (extra.ok, extra.detail) == (False, "these differ from mcp.json: exec") + assert [(c.name, c.ok) for c in unready] == [("Aside Computer Use", True), ("Aside daemon", False), + ("macOS permissions", True)] + assert not changed.ok and "Aside Computer Use.app/Contents/MacOS/aside-computer-use" in changed.detail + assert [c.name for c in rest] == ["macOS permissions"] + + +def test_env_toml_install_sh_and_the_driver_agree_on_the_pinned_version_folder_helper_daemon_and_home(pinned): + """A move to other Aside releases, or of their folder, helper, daemon or home, has to change them together; the + helper's runtime copy, which install.sh places in the home, is granted as the helper is.""" + folder = DRIVER.parent + macos = tomllib.loads((folder / "env.toml").read_text())["install"]["macos"] + install = (folder / "install.sh").read_text() + runtime = pinned.ASIDE_HOME / "runtime-computer-use" / pinned.HELPER + + daemon_download, cli_download = macos["downloads"] + assert daemon_download["url"] == ("https://releases.aside.com/dev-updater/components/" + f"{pinned.VERSION}/AsideDaemon/mac-arm64/AsideDaemon-{pinned.VERSION}.crx") + assert cli_download["url"] == f"https://releases.aside.com/cli/AsideCLI-darwin-arm64-{pinned.CLI_VERSION}.zip" + assert all(len(download["sha256"]) == 64 for download in macos["downloads"]) + assert f"daemon={daemon_download['to']}\n" in install and f"cli={cli_download['to']}\n" in install + assert [grant["app"] for grant in macos["grants"] if "app" in grant] == [str(pinned.PREFIX / pinned.HELPER), + str(runtime)] + assert f"dest={pinned.PREFIX}\n" in install and f'helper="{pinned.HELPER}"\n' in install + assert f'"$dest/{runtime.relative_to(pinned.PREFIX).parent}/$helper"' in install + (daemon,) = [service for service in macos["services"] if service.get("direct")] + assert shlex.split(daemon["command"]) == [str(pinned.PREFIX / pinned.DAEMON)] + assert daemon["environment"] == {"ASIDE_HOME": str(pinned.ASIDE_HOME)} + assert {str(pinned.DAEMON), f"{pinned.HELPER}/Contents/MacOS/aside-computer-use", str(pinned.CLI)} <= set( + pinned.EXECUTABLES) + assert all(len(digest) == 64 for digest in pinned.EXECUTABLES.values()) + + +@pytest.mark.asyncio +async def test_past_the_most_repls_kept_the_least_recently_used_is_closed_and_the_others_keep_their_bindings( + module, monkeypatch): + monkeypatch.setattr(module, "MAX_REPLS", 2) + async with mcp_session(_driver(module, monkeypatch)) as session: + async def repl(caller: str, code: str) -> str: + return _text(await session.call_tool("repl", {"code": code}, meta={SESSION_META_KEY: caller})) + + for caller in "abc": + await repl(caller, f"set {caller}") + b, c, a = await repl("b", "get"), await repl("c", "get"), await repl("a", "get") + + assert a.startswith("None in ") and b.startswith("b in ") and c.startswith("c in ") + + +@pytest.mark.asyncio +async def test_a_repl_left_idle_is_closed_without_another_call_and_its_session_starts_afresh(module, monkeypatch): + monkeypatch.setattr(module, "SWEEP_S", 0.05) + monkeypatch.setattr(module, "IDLE_S", 0.2) + async with mcp_session(_driver(module, monkeypatch)) as session: + await session.call_tool("repl", {"code": "set 1"}, meta={SESSION_META_KEY: "a"}) + before = _text(await session.call_tool("repl", {"code": "get"}, meta={SESSION_META_KEY: "a"})) + closed = await _gone(int(before.rpartition(" in ")[2])) + after = _text(await session.call_tool("repl", {"code": "get"}, meta={SESSION_META_KEY: "a"})) + + assert before.startswith("1 in ") and closed and after.startswith("None in ") + + +@pytest.mark.asyncio +async def test_calls_a_session_makes_together_while_another_repl_closes_share_one_repl(module, monkeypatch): + monkeypatch.setattr(module, "MAX_REPLS", 1) + async with mcp_session(_driver(module, monkeypatch)) as session: + async def repl(caller: str, code: str) -> str: + return _text(await session.call_tool("repl", {"code": code}, meta={SESSION_META_KEY: caller})) + + await repl("a", "set a") + _, got = await asyncio.gather(repl("b", "set b"), repl("b", "get")) + after = await repl("b", "get") + + assert got.partition(" in ")[2] == after.partition(" in ")[2] and after.startswith("b in ")