diff --git a/packages/agentenv-protocol/README.md b/packages/agentenv-protocol/README.md index fec7ad0dc..38841a1aa 100644 --- a/packages/agentenv-protocol/README.md +++ b/packages/agentenv-protocol/README.md @@ -960,7 +960,8 @@ are the profile's template. The base class adds: `notifications`, `deny` (the default), which answers a page's prompt for it, or `ask`; and `dialogs`, who answers a page's JavaScript dialogs: `agent` (the default), or agentenv, accepting (`accept`, a `prompt()` with its default text) or dismissing (`dismiss`) each. Either way agentenv leaves a page that asks before - it's left (`beforeunload`), so none of its answers blocks a navigation. + it's left (`beforeunload`), so none of its answers blocks a navigation. And `network`, on by default, whether + `data/get` exports its network log and `browser/v1` `network` reads it. - **The flags agentenv owns**, after the subclass's command and before `about:blank`, the one tab its command line opens: its profile, its HTTP cache under `~/.cache` (`--disk-cache-dir`), `--remote-debugging-pipe`, `--no-first-run`, @@ -1032,10 +1033,28 @@ are the profile's template. The base class adds: - A driver passes an agent's tool call to the browser in the `AgentEnv-Tool-Call` header (`tool_call_header`), with the caller's `AgentEnv-Role`; the page driver sends it on every call it makes for a tool. `data/get` writes the log beside `browser-state.json` as `browser-log.jsonl`, which `data/add` passes over. +- **Its network log** is the one the proxy it goes out through keeps, in the file `AGENTENV_NETWORK_LOG` names + (`website_browser.network`): on a browser alone routing sites, the shared site proxy's, + `/etc/agentenv/sites/network.jsonl`, which root keeps and the session user reads; else the base's stand-in's, + `/var/log/agentenv/network.jsonl`. One JSON object a line, numbered (`seq`) and timed (`at`) as it's written, a + proxy started again numbering after the file's last: an `open` line once a request has gone upstream or a tunnel + has opened, and an `end` line, naming the same `id`, once it's answered and done or has failed. The shared site + proxy keeps each exchange whole: the request as the browser sent it, its headers, the values of the sign-in headers + a site declares as ``, and its body up to 10 MB; the answer as the browser got it, its status, its headers + and, for text, JSON, forms, XML and scripts, its body up to 1 MB, as it came, in base64, any other type's size and + sha256 alone; a WebSocket's messages and bytes each way; and each tunnel to a live host, refusal and failure. The + stand-in, which decrypts nothing, keeps each connection: its host, port, address and bytes each way, and a plain + request's method and URL. A browser on a computer with a desktop has none, since no proxy logs for it and nothing + names one. When it has one and its `network` setting is on, `data/get` writes it as HAR 1.2, `network.har`, + which `data/add` passes over: an entry + for each request and tunnel in the order they started, one still going marked `_open`, bodies decoded from gzip and + deflate, and br and zstd with the `app` extra's `brotli` and `zstandard`, up to 1 MB, and agentenv's own fields + named with `_`: `_tunnel`, `_websocket`, `_failed`, `_refused`, `_truncated`, `_sha256`. - **`browser/v1`** on its card: `chromium` (its `version`), `clock: "real"` and, alone, `window`; `log {after}` - (GET), framework-only, its log's lines after `after`; `calls/begin {user, acts?, within_s?}`, framework-only, - sent with the tool-call header, which begins the entry of an agent's tool call that `user`'s CDP connections - serve and answers its `entry`, and `calls/end {entry, ok, error?}`, which ends it, a call not ended within + (GET), framework-only, its log's lines after `after`; `network {after}` (GET), framework-only, its network log's + lines after `after`, 500 at most and `more` when a read left some; `calls/begin {user, acts?, within_s?}`, + framework-only, sent with the tool-call header, which begins the entry of an agent's tool call that `user`'s CDP + connections serve and answers its `entry`, and `calls/end {entry, ok, error?}`, which ends it, a call not ended within `within_s` (600 s by default) written as never said to have ended; and `record` (GET), framework-only, which returns that record with `launches`, the app server's log of each launch (with its cause: `boot`, `server_restart`, `launch`, `data/add` or `data/reset`), each stop it made diff --git a/packages/agentenv-protocol/pyproject.toml b/packages/agentenv-protocol/pyproject.toml index 57df2fb1f..c621e36f6 100644 --- a/packages/agentenv-protocol/pyproject.toml +++ b/packages/agentenv-protocol/pyproject.toml @@ -38,9 +38,11 @@ agent = [ # A computer app's own server: ComputerAppEnvironment serves its card and computer-app/v1 over mcp's FastMCP, on a Unix # socket through uvicorn, and a website browser's CDP relay over WebSockets. app = [ + "brotli>=1.2", # a website browser's network log: br answer bodies, decoded for network.har "mcp>=1.25,<2", "uvicorn>=0.30", "websockets>=13", + "zstandard>=0.23", # and zstd ones ] # A computer driver: ComputerDriverEnvironment serves its tools over mcp's FastMCP, crops and encodes frames, records # the display as H.264 with PyAV (computer-recording/v1), and relays WebSockets to its apps on /svc//. @@ -76,6 +78,8 @@ dev = [ "Pillow>=10.4", # the computer extra's frame crops and encodings "av>=17", # the computer extra's encoder, and the tests' decoder "websockets>=13", # the computer extra's /svc// relay + "brotli>=1.2", # the app extra's network.har decoding + "zstandard>=0.23", ] [project.urls] Homepage = "https://www.agentenvframework.com" diff --git a/packages/agentenv-protocol/src/agentenv_protocol/website_browser/__init__.py b/packages/agentenv-protocol/src/agentenv_protocol/website_browser/__init__.py index 7c92d8b01..e6a90481d 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/website_browser/__init__.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/website_browser/__init__.py @@ -12,17 +12,18 @@ from .environment import (ALONE_BYPASS, BLOCK, BROWSER, COMPUTER_BYPASS, OWNED_FLAGS, POLICIES, READY_POLL_S, SHM_MIN_BYTES, SITE_PROXY_ENV, SITES, SKILL, BrowserSettings, WebsiteBrowserEnvironment, Window, start_tab_why) +from .network import DEFAULT_NETWORK_LOG, HAR_FILE, NETWORK_LOG_ENV, NetworkLog, har from .recording import BROWSER_RECORDING, BrowserRecordingManifest, BrowserRecordings, RecordedFrame from .shapes import (BrowserRecord, ClosedTab, Cookie, CookieToSet, Dialog, Download, LaunchEntry, OpenTab, OriginStorage, PartitionKey, SiteNames, StartTab, StartTabsLaunch, StorageItem, origin_of) __all__ = ["ACTIVE_CHECK_TIMEOUT_S", "AGENT", "AGENTENV", "ALONE_BYPASS", "BLOCK", "BROWSER", "BROWSER_RECORDING", - "CAPTURE_TIMEOUT_S", - "COMPUTER_BYPASS", "DOWNLOADS", "EXIT", "FAILED", "HASH_CHUNK_BYTES", "LIVE_START_TAB_TIMEOUT_S", "LOADED", - "MAX_MESSAGE_BYTES", "OWNED_FLAGS", "PAGE_INPUT", "PAGES", "PIN_TIMEOUT_S", "PIPE_IN", "PIPE_OUT", + "CAPTURE_TIMEOUT_S", "COMPUTER_BYPASS", "DEFAULT_NETWORK_LOG", "DOWNLOADS", "EXIT", "FAILED", "HAR_FILE", + "HASH_CHUNK_BYTES", "LIVE_START_TAB_TIMEOUT_S", "LOADED", "MAX_MESSAGE_BYTES", "NETWORK_LOG_ENV", + "OWNED_FLAGS", "PAGE_INPUT", "PAGES", "PIN_TIMEOUT_S", "PIPE_IN", "PIPE_OUT", "POLICIES", "READY_POLL_S", "SHM_MIN_BYTES", "SITE_PROXY_ENV", "SITES", "SKILL", "STOP", "TIMED_OUT", "UNKNOWN", "UNNAVIGATED_CAPTURE_TIMEOUT_S", "UPLOADS", "BrowserControl", "BrowserSettings", "DevToolsError", - "DevToolsPipe", "WebsiteBrowserEnvironment", "Window", "BrowserRecord", "ClosedTab", "Cookie", "CookieToSet", - "Dialog", "Download", "LaunchEntry", "OpenTab", "OriginStorage", "PartitionKey", "SiteNames", "StartTab", - "StartTabsLaunch", "StorageItem", "origin_of", "start_tab_why", "BrowserRecordingManifest", "BrowserRecordings", - "RecordedFrame"] + "DevToolsPipe", "NetworkLog", "WebsiteBrowserEnvironment", "Window", "BrowserRecord", "ClosedTab", "Cookie", + "CookieToSet", "Dialog", "Download", "LaunchEntry", "OpenTab", "OriginStorage", "PartitionKey", "SiteNames", + "StartTab", "StartTabsLaunch", "StorageItem", "har", "origin_of", "start_tab_why", + "BrowserRecordingManifest", "BrowserRecordings", "RecordedFrame"] diff --git a/packages/agentenv-protocol/src/agentenv_protocol/website_browser/data.py b/packages/agentenv-protocol/src/agentenv_protocol/website_browser/data.py index f9fc1ee81..a85d7ebae 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/website_browser/data.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/website_browser/data.py @@ -13,6 +13,7 @@ from ..computer.dataplane import part_bytes from ..types import DataPart, FilePart, uploaded_object_path from .control import DOWNLOADS, UPLOADS +from .network import HAR_FILE #: The browser's typed state, as ``data/get`` returns it beside its files and ``data/add`` reads it back. STATE_FILE = "browser-state.json" @@ -54,7 +55,7 @@ async def sorted_load(parts: list, profile: str) -> Load: under = PurePosixPath(path) if under in (PurePosixPath(profile), PurePosixPath(DOWNLOADS), PurePosixPath(UPLOADS)): raise ValueError(f"{path!r} is a folder of the browser's, not a file in it") - if path == LOG_FILE: + if path in (LOG_FILE, HAR_FILE): continue if path == STATE_FILE: if load.state is not None: diff --git a/packages/agentenv-protocol/src/agentenv_protocol/website_browser/environment.py b/packages/agentenv-protocol/src/agentenv_protocol/website_browser/environment.py index 0ca7e4345..c83699351 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/website_browser/environment.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/website_browser/environment.py @@ -21,6 +21,7 @@ from .cdp import ROUTE, Relay, Tokens, reached, under, websocket_form from .log import (ACTOR, ERROR_CHARS, TOOL_CALL_HEADER, UNLOGGED_OPS, BrowserLog, op_acts, read_tool_call, redact_text, redacted_args, redacted_op) +from .network import HAR_FILE, NetworkLog, log_path from ..computer import AppDeclaration, ComputerAppEnvironment, Launch, install_step, ready_check from ..computer import app as computer_app from ..computer import app_data @@ -107,6 +108,9 @@ class BrowserSettings(BaseModel): notifications: Literal["deny", "ask"] = "deny" #: Who answers a page's JavaScript dialogs: the agent, or agentenv accepting or dismissing each. dialogs: Literal["agent", "accept", "dismiss"] = "agent" + #: Whether ``data/get`` exports its network log, ``network.har``, and ``browser/v1`` ``network`` reads it: off, a + #: run's export holds none of the cookies and form bodies the log keeps. + network: bool = True @field_validator("start_urls") @classmethod @@ -152,6 +156,8 @@ def __init__(self) -> None: self.log = BrowserLog.for_app(os.environ.get("ENVIRONMENT_NAME") or type(self).__name__.lower(), SOCKET.parent) self.control.log = self.log + #: Its network log, as the proxy it goes out through writes it; None when no proxy logs for it. + self.network = NetworkLog(path) if (path := log_path()) is not None else None #: The CDP addresses it has handed out, and the relay agents' connections to them go through. self.cdp_tokens = Tokens(SOCKET.parent / "cdp-tokens.json") self.relay = Relay(self.devtools, dialogs_answered=self.settings.dialogs != AGENT, log=self.log, @@ -334,7 +340,13 @@ async def start_tabs_loaded(self) -> bool: "the env's site and the outcome, loaded, failed or timed_out. log returns its log's lines after the sequence " "number after, in order: an action entry for each CDP command, MCP tool call and pixel action an agent took, " "marked acting or reading, with who, the tab and its URL and, acting, its parameters with secrets taken out; " - "and a tab event each time a tab opened, navigated or closed, with what caused it.")) + "and a tab event each time a tab opened, navigated or closed, with what caused it. network returns its " + "network log's lines after the sequence number after, in order, as the proxy it goes out through writes them: " + "an open line once a request has gone upstream or a tunnel has opened and an end line, with the same id, once " + "it's done or has failed. The shared site proxy writes a whole exchange, the request and the answer with their " + "headers and bodies, the sign-in headers a site declares removed, and a WebSocket's message counts; the base's " + "stand-in writes each connection, its host, address and bytes each way. With the network setting off, or on a " + "computer with a desktop, where no proxy logs for it, network returns no lines.")) async def browser_record(self) -> dict: return BrowserRecord(**await self.control.record(), launches=self._computer_app.launches(), cdp_connections=list(self.relay.connections), @@ -344,6 +356,12 @@ async def browser_record(self) -> dict: async def browser_log(self, after: int = 0) -> dict: return await self.log.read_async(after) + @extension(BROWSER, op="network", method="GET") + async def browser_network(self, after: int = 0) -> dict: + if self.network is None or not self.settings.network: + return {"lines": [], "last": after, "more": False} + return await self.network.read_async(after) + @extension(BROWSER, op="sites/set") async def browser_sites_set(self, sites: list[dict]) -> dict: """The env's own sites, each by its name and domains, which it keeps for each launch after: a start tab on one @@ -560,23 +578,28 @@ def _resets_failed_add(self) -> bool: def declared_extensions(self) -> list[EnvironmentExtension]: """Its data plane moves files by their path, a file tree: its state as ``browser-state.json``, its log, its - profile, its downloads and files for agents to upload. A load takes its state as a data part too.""" + network log as HAR, its profile, its downloads and files for agents to upload. A load takes its state as a data + part too.""" tree = IntakeDeclaration(get=[IntakeFormat(part="file", format="file-tree")], add=[IntakeFormat(part="file", format="file-tree", load="replace"), IntakeFormat(part="data", format="json", load="replace")]) return [*super().declared_extensions(), intake_extension(tree, ( f"data/get returns the browser's state as {data.STATE_FILE}: its cookies and local storage as Playwright's " f"storageState has them, its tabs open and closed, its downloads and its clock; beside it, its log as " - f"{data.LOG_FILE}, which data/add passes over, its profile " + f"{data.LOG_FILE} and, when a proxy logs for it and its network setting is on, its network log as HAR " + f"1.2 in {HAR_FILE}, an " + f"entry for each request and tunnel in the order they started, both of which data/add passes over, its " + f"profile " f"without caches, and its finished downloads. data/add applies such a state, while the browser runs, and " f"writes files by their path: its profile, restored with the browser stopped, ~/{DOWNLOADS} and " f"~/{UPLOADS}. data/reset puts the profile back as installed and relaunches it at its start tabs."))] @get_data async def browser_get_data(self, write_namespace: Optional[WriteNamespaceGrant] = None) -> list: - """Its state, as ``browser-state.json``, its profile without caches, copied with the browser paused so its files - are of one moment, and its finished downloads: each a file uploaded under ``write_namespace`` when there is - one, else inline.""" + """Its state, as ``browser-state.json``, its log, its network log as HAR when it has one and its ``network`` + setting is on, its profile without caches, copied + with the browser paused so its files are of one moment, and its finished downloads: each a file uploaded under + ``write_namespace`` when there is one, else inline.""" home = Path.home() state = BrowserState(cookies=await self.control.cookies(), local_storage=await self.control.local_storage(), tabs=await self.control.tab_list(), @@ -589,9 +612,13 @@ async def browser_get_data(self, write_namespace: Optional[WriteNamespaceGrant] root = Path(staging) (root / data.STATE_FILE).write_text(json.dumps(written, indent=1), encoding="utf-8") (root / data.LOG_FILE).write_text(await self.log.complete_text(), encoding="utf-8") + exported = [data.STATE_FILE, data.LOG_FILE] + if self.network is not None and self.settings.network: + (root / HAR_FILE).write_text(await self.network.har_text(), encoding="utf-8") + exported.append(HAR_FILE) with self._computer_app.paused(): await app_data.in_thread(data.stage_profile, home, self.profile, root) - parts = await file_parts(root, [data.STATE_FILE, data.LOG_FILE, self.profile], write_namespace) + parts = await file_parts(root, [*exported, self.profile], write_namespace) return parts + await file_parts(home, data.finished_downloads(home), write_namespace) @add_data diff --git a/packages/agentenv-protocol/src/agentenv_protocol/website_browser/log.py b/packages/agentenv-protocol/src/agentenv_protocol/website_browser/log.py index c9dd0b8b2..1a25df719 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/website_browser/log.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/website_browser/log.py @@ -75,9 +75,9 @@ ACTING_OPS = frozenset({"page/input", "navigate", "history", "tabs/open", "tabs/activate", "tabs/close", "dialog/answer", "files/choose", "cookies/set", "cookies/delete", "local_storage/set", "local_storage/clear", "sites/set"}) -#: The ``browser/v1`` ops an agent's calls are never logged for: the log itself, and agentenv's own bookkeeping, a -#: driver's calls that begin and end a tool call's own entry among them. -UNLOGGED_OPS = frozenset({"log", "record", "calls/begin", "calls/end"}) +#: The ``browser/v1`` ops an agent's calls are never logged for: the logs themselves, and agentenv's own bookkeeping, +#: a driver's calls that begin and end a tool call's own entry among them. +UNLOGGED_OPS = frozenset({"log", "network", "record", "calls/begin", "calls/end"}) #: The CDP methods and ops whose text is typed into the page. TYPING_METHODS = frozenset({"Input.insertText", "Input.dispatchKeyEvent", "Input.imeSetComposition"}) #: What a key event says of the key it types, all of it dropped when a password field has focus. diff --git a/packages/agentenv-protocol/src/agentenv_protocol/website_browser/network.py b/packages/agentenv-protocol/src/agentenv_protocol/website_browser/network.py new file mode 100644 index 000000000..a300e4b86 --- /dev/null +++ b/packages/agentenv-protocol/src/agentenv_protocol/website_browser/network.py @@ -0,0 +1,368 @@ +"""A website browser's network log, as the proxy it goes out through writes it and as ``browser/v1`` and ``data/get`` +read it. That proxy, the shared site proxy or the base's stand-in, appends JSON lines to the file ``NETWORK_LOG_ENV`` +names, each numbered (``seq``) and timed (``at``) as it is written: an ``open`` line once a request has gone upstream +or a tunnel has opened, and an ``end`` line, naming the same ``id``, once it is answered and done, or has failed. + +``har`` makes HAR 1.2 of the lines: an entry for each request and each tunnel, in the order they started, with what it +ended with, or ``_open`` while it hasn't ended. A body arrives as the proxy saw it, base64 (``Body``), and leaves +decoded, up to ``DECODED_LIMIT``: gzip and deflate always, br and zstd when their libraries are installed.""" +from __future__ import annotations + +import asyncio +import base64 +import json +import os +import re +import zlib +from datetime import datetime, timezone +from email.utils import parsedate_to_datetime +from importlib.metadata import PackageNotFoundError, version +from pathlib import Path +from typing import Any, Optional +from urllib.parse import parse_qsl, urlsplit + +try: + import brotli +except ImportError: + brotli = None +try: + import zstandard +except ImportError: + zstandard = None + +#: The environment variable naming the network log of the proxy the browser goes out through. +NETWORK_LOG_ENV = "AGENTENV_NETWORK_LOG" +#: The stand-in's, which its base names when no site routing names the shared proxy's instead. +DEFAULT_NETWORK_LOG = Path("/var/log/agentenv/network.jsonl") +#: The network log as ``data/get`` returns it, beside the browser's state; ``data/add`` passes over it. +HAR_FILE = "network.har" +#: How many lines, and how many of their bytes, one read returns at most. +READ_LIMIT = 500 +READ_BYTES = 32 << 20 +#: How much of an answer's body, and of a request's, an entry keeps once decoded; decoding stops there. +DECODED_LIMIT, REQUEST_DECODED_LIMIT = 1 << 20, 10 << 20 +#: A ``charset`` in a ``Content-Type``. +CHARSET = re.compile(r"charset=\"?([\w.:-]+)", re.I) +#: What undoing a coding raises on a body that isn't what its coding says. +DECODE_ERRORS = (zlib.error, ValueError, *((brotli.error,) if brotli else ()), + *((zstandard.ZstdError,) if zstandard else ())) + + +def log_path() -> Optional[Path]: + """The network log of the proxy this browser goes out through, or None for a browser none logs for, as on a + computer with a desktop, which names none.""" + return Path(named) if (named := os.environ.get(NETWORK_LOG_ENV)) else None + + +class NetworkLog: + """The network log in ``path``, read in the order its lines were numbered, which is the order they were written: + where the last read ended is kept, so a poller reads only what came after.""" + + def __init__(self, path: Path) -> None: + self.path = path + self._mark = (0, 0) + + def read(self, after: int = 0, limit: int = READ_LIMIT) -> dict: + """The lines numbered after ``after``, up to ``limit`` of them and ``READ_BYTES`` of their text: ``lines``, + ``last``, the number of the last one, and ``more`` when a limit left some. A line still being written is left + for the next read.""" + mark_seq, mark_offset = self._mark + lines: list[dict] = [] + size, more = 0, False + try: + with self.path.open("rb") as file: + fits = mark_offset <= os.fstat(file.fileno()).st_size + file.seek(mark_offset if after >= mark_seq and fits else 0) + while (raw := file.readline()).endswith(b"\n"): + try: + line = json.loads(raw) + except ValueError: + continue + seq = line.get("seq", 0) if isinstance(line, dict) else 0 + if seq > after: + if len(lines) >= limit or (lines and size + len(raw) > READ_BYTES): + more = True + break + lines.append(line) + size += len(raw) + self._mark = (seq, file.tell()) + except FileNotFoundError: + pass + return {"lines": lines, "last": lines[-1]["seq"] if lines else after, "more": more} + + async def read_async(self, after: int = 0, limit: int = READ_LIMIT) -> dict: + return await asyncio.to_thread(self.read, after, limit) + + def every_line(self) -> list[dict]: + lines, after = [], 0 + while (page := self.read(after, limit=1 << 30))["lines"]: + lines += page["lines"] + after = page["last"] + if not page["more"]: + break + return lines + + async def har_text(self) -> str: + """Every line as HAR, as ``data/get`` writes it, made off the event loop.""" + return await asyncio.to_thread(lambda: json.dumps(har(self.every_line()), indent=1)) + + +def har(lines: list[dict]) -> dict: + """``lines`` as HAR 1.2: each request or tunnel one entry, its ``open`` and ``end`` lines joined by ``id``, the + entries in the order they started, and those that started in the same millisecond in the order they were first + written. A line that isn't one, or an entry that can't be read, is left out and counted in ``_unreadable``, so one + bad line can't cost an export the rest.""" + opened: dict[str, Optional[dict]] = {} + ended: dict[str, dict] = {} + unreadable = 0 + for line in lines: + kind = "open" if isinstance(line.get("open"), dict) else "end" if isinstance(line.get("end"), dict) else None + key = line[kind].get("id") if kind else None + if not isinstance(key, str): + unreadable += 1 + elif kind == "open": + if opened.get(key) is None: + opened[key] = line["open"] + else: + ended.setdefault(key, line["end"]) + opened.setdefault(key, None) + entries = [] + for key, record in opened.items(): + try: + entries.append(_entry(record, ended.get(key))) + except (KeyError, TypeError, ValueError, AttributeError, IndexError, OverflowError): + unreadable += 1 + entries.sort(key=lambda entry: entry["startedDateTime"]) + log = {"version": "1.2", "creator": {"name": "agentenv", "version": _version()}, "entries": entries} + return {"log": {**log, **({"_unreadable": unreadable} if unreadable else {})}} + + +def _entry(opened: Optional[dict], ended: Optional[dict]) -> dict: + record = {**(ended or {}), **(opened or {})} + finished = (ended or {}).get("ended") + started = record.get("started") or finished or "" + if not isinstance(started, str): + raise TypeError("started") + answered = (ended or {}).get("answered") or finished + entry: dict[str, Any] = {"startedDateTime": started, "time": _ms(started, finished)} + if record.get("kind") == "tunnel": + entry.update(_tunnel(record, ended)) + else: + request = record.get("request") or {"method": record.get("method", ""), "url": _url(record)} + response = (ended or {}).get("response") + entry["request"] = _request(request) + entry["response"] = _response(response) + if response and response.get("status") == 101: + entry["request"]["url"] = re.sub(r"^http", "ws", entry["request"]["url"]) + entry["_resourceType"] = "websocket" + if (ended or {}).get("websocket"): + entry["_websocket"] = ended["websocket"] + if record.get("address"): + entry["serverIPAddress"] = record["address"] + entry["cache"] = {} + entry["timings"] = {"send": 0, "wait": _ms(started, answered), "receive": _ms(answered, finished)} + for key in ("failed", "refused", "unparsed"): + if (ended or {}).get(key): + entry[f"_{key}"] = ended[key] + if ended is None: + entry["_open"] = True + return entry + + +def _url(record: dict) -> str: + """The URL a request the proxy refused, by its host alone, was for: https for a ``CONNECT``.""" + url = record.get("url", "") + if not url or "://" in url: + return url + return f"{'https' if record.get('method') == 'CONNECT' else 'http'}://{url}/" + + +def _tunnel(record: dict, ended: Optional[dict]) -> dict: + """A connection the proxy kept only at connection level: a CONNECT tunnel, answered 200 once it opened, or a plain + request the base's stand-in forwarded, by its method and URL, its answer unread.""" + host, port = record.get("host", ""), record.get("port", 443) + url = record.get("url") or f"https://{host}:{port}/" + request = {"method": record.get("method", "CONNECT"), "url": url, "httpVersion": "HTTP/1.1", "cookies": [], + "headers": [], "queryString": [{"name": name, "value": value} + for name, value in parse_qsl(urlsplit(url).query, keep_blank_values=True)], + "headersSize": -1, "bodySize": 0} + failed, connect = (ended or {}).get("failed"), "url" not in record + response = {"status": 200 if connect and not failed else 0, + "statusText": "Connection established" if connect and not failed else "", + "httpVersion": "HTTP/1.1", "cookies": [], "headers": [], "content": {"size": 0, "mimeType": ""}, + "redirectURL": "", "headersSize": -1, "bodySize": 0} + fields = {"request": request, "response": response, + "_tunnel": {key: record[key] for key in ("host", "port", "address", "bytes_sent", "bytes_received") + if key in record}} + if record.get("address"): + fields["serverIPAddress"] = record["address"] + return fields + + +def _request(request: dict) -> dict: + headers = request.get("headers") or [] + url = request.get("url", "") + body = request.get("body") + har_request = {"method": request.get("method", ""), "url": url, + "httpVersion": request.get("http_version", "HTTP/1.1"), "cookies": _cookies(headers), + "headers": _headers(headers), + "queryString": [{"name": name, "value": value} + for name, value in parse_qsl(urlsplit(url).query, keep_blank_values=True)], + "headersSize": -1, "bodySize": body["size"] if body else 0} + if body and body.get("size"): + text, extra = _text(body, _header(headers, "content-encoding"), _header(headers, "content-type"), + REQUEST_DECODED_LIMIT) + har_request["postData"] = {"mimeType": _header(headers, "content-type") or "", "text": text, **extra} + return har_request + + +def _response(response: Optional[dict]) -> dict: + if response is None: + return {"status": 0, "statusText": "", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [], + "content": {"size": 0, "mimeType": ""}, "redirectURL": "", "headersSize": -1, "bodySize": 0} + headers = response.get("headers") or [] + status, mime = response.get("status", 0), _header(headers, "content-type") or "" + body = response.get("body") + content: dict[str, Any] = {"size": body["size"] if body else 0, "mimeType": mime} + if body and "base64" in body: + text, extra = _text(body, _header(headers, "content-encoding"), mime, DECODED_LIMIT) + content.update(text=text, **extra) + elif body: + content["_sha256"] = body.get("sha256") + return {"status": status, "statusText": response.get("reason", ""), + "httpVersion": response.get("http_version", "HTTP/1.1"), "cookies": _set_cookies(headers), + "headers": _headers(headers), "content": content, + "redirectURL": (_header(headers, "location") or "") if 300 <= status < 400 else "", + "headersSize": -1, "bodySize": body["size"] if body else 0} + + +def _text(body: dict, coding: Optional[str], mime: Optional[str], limit: int) -> tuple[str, dict]: + """A body's text, decoded from ``coding`` and its charset, up to ``limit``, and what a HAR says beside it: that the + proxy or the limit cut it, how much compression saved, or, for a coding that can't be undone here, the raw body as + base64 with the coding named.""" + raw = base64.b64decode(body.get("base64") or "") + extra: dict[str, Any] = {} + if body.get("truncated"): + extra["_truncated"] = True + if coding and coding.strip().lower() not in ("", "identity"): + decoded = _decompressed(raw, coding, limit) + if decoded is None: + return base64.b64encode(raw).decode(), {**extra, "encoding": "base64", "_undecoded": coding} + extra["compression"] = max(0, len(decoded) - len(raw)) + raw = decoded + if len(raw) > limit: + raw, extra["_truncated"] = raw[:limit], True + charset = (CHARSET.search(mime or "") or [None, "utf-8"])[1] + try: + return raw.decode(charset, errors="replace"), extra + except LookupError: + return raw.decode("utf-8", errors="replace"), extra + + +def _decompressed(raw: bytes, coding: str, limit: int) -> Optional[bytes]: + """``raw`` with each of ``coding``'s codings undone, last first, as far as it goes, so a body the proxy cut decodes + up to the cut, and each stopping past ``limit``, so a small body can't expand without bound; None for a coding + this can't undo.""" + data, most = raw, limit + 1 + for name in reversed([part.strip().lower() for part in coding.split(",") if part.strip()]): + try: + if name in ("gzip", "x-gzip"): + data = zlib.decompressobj(16 + zlib.MAX_WBITS).decompress(data, most) + elif name == "deflate": + try: + data = zlib.decompressobj(zlib.MAX_WBITS).decompress(data, most) + except zlib.error: + data = zlib.decompressobj(-zlib.MAX_WBITS).decompress(data, most) + elif name == "br" and brotli is not None: + data = brotli.Decompressor().process(data, output_buffer_limit=most)[:most] + elif name == "zstd" and zstandard is not None: + data = _zstd(data, most) + elif name != "identity": + return None + except DECODE_ERRORS: + return None + return data + + +def _zstd(data: bytes, most: int) -> bytes: + out = bytearray() + with zstandard.ZstdDecompressor().stream_reader(data) as reader: + while len(out) < most and (chunk := reader.read(most - len(out))): + out += chunk + return bytes(out) + + +def _headers(headers: list) -> list[dict]: + return [{"name": name, "value": value} for name, value in headers] + + +def _header(headers: list, name: str) -> Optional[str]: + return next((value for key, value in headers if key.lower() == name), None) + + +def _cookies(headers: list) -> list[dict]: + """The cookies a request sent, from its ``Cookie`` headers.""" + cookies = [] + for key, value in headers: + if key.lower() == "cookie": + for pair in value.split(";"): + name, equals, cookie = pair.strip().partition("=") + if name and equals: + cookies.append({"name": name, "value": cookie}) + return cookies + + +def _set_cookies(headers: list) -> list[dict]: + """The cookies an answer set, from its ``Set-Cookie`` headers, with the attributes HAR names.""" + cookies = [] + for key, value in headers: + if key.lower() != "set-cookie": + continue + first, *attributes = value.split(";") + name, _, cookie_value = first.strip().partition("=") + cookie: dict[str, Any] = {"name": name, "value": cookie_value, "httpOnly": False, "secure": False} + for attribute in attributes: + attr, _, attr_value = attribute.strip().partition("=") + lower = attr.lower() + if lower in ("path", "domain"): + cookie[lower] = attr_value + elif lower == "expires" and (expires := _http_date(attr_value)): + cookie["expires"] = expires + elif lower == "httponly": + cookie["httpOnly"] = True + elif lower == "secure": + cookie["secure"] = True + elif lower == "samesite": + cookie["sameSite"] = attr_value + cookies.append(cookie) + return cookies + + +def _http_date(text: str) -> Optional[str]: + """An HTTP date, as a cookie's ``Expires`` has it, in ISO 8601 as HAR wants; None for one that doesn't parse.""" + try: + moment = parsedate_to_datetime(text) + moment = moment if moment.tzinfo else moment.replace(tzinfo=timezone.utc) + return moment.astimezone(timezone.utc).isoformat(timespec="milliseconds").replace("+00:00", "Z") + except (TypeError, ValueError, IndexError, OverflowError): + return None + + +def _ms(start: Optional[str], end: Optional[str]) -> float: + if not start or not end: + return 0 + try: + return max(0.0, round((_when(end) - _when(start)).total_seconds() * 1000, 3)) + except ValueError: + return 0 + + +def _when(text: str) -> datetime: + return datetime.fromisoformat(text.replace("Z", "+00:00")) + + +def _version() -> str: + try: + return version("agentenv-framework-protocol") + except PackageNotFoundError: + return "unknown" diff --git a/packages/agentenv-protocol/tests/test_browser_data.py b/packages/agentenv-protocol/tests/test_browser_data.py index aacf4f444..08b92c53e 100644 --- a/packages/agentenv-protocol/tests/test_browser_data.py +++ b/packages/agentenv-protocol/tests/test_browser_data.py @@ -92,6 +92,7 @@ def browser(tmp_path, monkeypatch): monkeypatch.setenv("ENVIRONMENT_NAME", "chromium") monkeypatch.setenv("AGENTENV_APP_SETTINGS", json.dumps({"chromium": {"start_urls": ["https://shop.test/"]}})) monkeypatch.setattr(browser_log, "LOG_DIR", tmp_path / "logs") + monkeypatch.setenv("AGENTENV_NETWORK_LOG", str(tmp_path / "network.jsonl")) made = Browser() made.control = FakeControl() launches = [] @@ -147,6 +148,7 @@ async def test_data_get_is_its_state_its_profile_without_caches_or_locks_and_its files = _files(await browser.browser_get_data()) state = json.loads(files.pop("browser-state.json")) + assert json.loads(files.pop("network.har"))["log"]["entries"] == [] assert set(state) == {"cookies", "local_storage", "tabs", "closed_tabs", "downloads", "clock", "start_tabs"} assert state["cookies"] == FakeControl().cookie_list and "partitionKey" not in state["cookies"][0] assert state["clock"]["mode"] == "real" and state["tabs"][0]["id"] == "t1" and state["closed_tabs"][0]["id"] == "t0" @@ -170,6 +172,55 @@ async def test_data_get_exports_the_log_written_so_far_and_data_add_passes_over_ assert ("cookies", ["sid"]) in browser.control.did +@pytest.mark.asyncio +async def test_data_get_exports_the_network_log_as_har_and_data_add_passes_over_it(browser, tmp_path): + lines = [{"seq": 1, "at": "2026-11-11T10:00:00.000Z", "open": { + "id": "1-1", "kind": "exchange", "started": "2026-11-11T10:00:00.000Z", + "request": {"method": "GET", "url": "https://shop.test/cart?n=2", "http_version": "HTTP/1.1", + "headers": [["Host", "shop.test"]]}}}, + {"seq": 2, "at": "2026-11-11T10:00:00.250Z", "end": { + "id": "1-1", "ended": "2026-11-11T10:00:00.250Z", "answered": "2026-11-11T10:00:00.200Z", + "response": {"status": 200, "reason": "OK", "http_version": "HTTP/1.1", + "headers": [["Content-Type", "application/json"]], + "body": {"size": 2, "sha256": "x", "base64": base64.b64encode(b"{}").decode()}}}}] + (tmp_path / "network.jsonl").write_text("".join(json.dumps(line) + "\n" for line in lines)) + + files = _files(await browser.browser_get_data()) + [entry] = json.loads(files["network.har"])["log"]["entries"] + + assert (entry["request"]["url"], entry["response"]["status"], entry["response"]["content"]["text"]) == ( + "https://shop.test/cart?n=2", 200, "{}") + assert entry["time"] == 250 and entry["timings"]["wait"] == 200 + assert (await browser.browser_network(after=1)) == {"lines": [lines[1]], "last": 2, "more": False} + await browser.browser_add_data(_parts(_file("browser-state.json", _snapshot_state()), + _file("network.har", files["network.har"]))) + assert ("cookies", ["sid"]) in browser.control.did + + +@pytest.mark.asyncio +async def test_a_browser_no_proxy_logs_for_exports_no_har_and_network_reads_nothing(tmp_path, monkeypatch, browser): + monkeypatch.delenv("AGENTENV_NETWORK_LOG") + made = Browser() + made.control, made._computer_app = browser.control, browser._computer_app + + files = _files(await made.browser_get_data()) + + assert made.network is None and "network.har" not in files and "browser-state.json" in files + assert await made.browser_network(after=3) == {"lines": [], "last": 3, "more": False} + + +@pytest.mark.asyncio +async def test_with_its_network_setting_off_data_get_exports_no_har_and_network_reads_nothing(browser, tmp_path): + (tmp_path / "network.jsonl").write_text(json.dumps({"seq": 1, "at": "t", "end": {"id": "1-1", "ended": "t"}}) + + "\n") + browser.settings = browser.settings.model_copy(update={"network": False}) + + files = _files(await browser.browser_get_data()) + + assert "network.har" not in files and "browser-state.json" in files + assert await browser.browser_network(after=0) == {"lines": [], "last": 0, "more": False} + + @pytest.mark.asyncio async def test_data_get_reads_the_state_then_copies_the_profile_with_the_browser_paused(browser, monkeypatch): stage = website_browser.data.stage_profile @@ -200,7 +251,7 @@ def removed_meanwhile(source, target): monkeypatch.setattr(website_browser.data.shutil, "copy2", removed_meanwhile) files = _files(await browser.browser_get_data()) - assert set(files) == {"browser-state.json", "browser-log.jsonl", f"{PROFILE}/Default/Cookies", + assert set(files) == {"browser-state.json", "browser-log.jsonl", "network.har", f"{PROFILE}/Default/Cookies", f"{PROFILE}/Default/Session Storage/LOG"} diff --git a/packages/agentenv-protocol/tests/test_browser_network.py b/packages/agentenv-protocol/tests/test_browser_network.py new file mode 100644 index 000000000..0f1d20f47 --- /dev/null +++ b/packages/agentenv-protocol/tests/test_browser_network.py @@ -0,0 +1,336 @@ +"""A website browser's network log as browser/v1 reads it and data/get exports it: lines read in order after a number, +a page at a time, a line still being written left for the next read; and HAR 1.2 made of them, each request and tunnel +one entry, its open and end lines joined by id, in the order they started, with the fields a grader reads, and bodies +decoded from gzip, deflate, br and zstd, cut at a limit, or left base64 when they can't be.""" +from __future__ import annotations + +import base64 +import gzip +import hashlib +import json +import zlib + +import brotli +import pytest +import zstandard + +from agentenv_protocol.website_browser import network +from agentenv_protocol.website_browser.network import NetworkLog, har + +T0, T1, T2, T3 = "2026-11-11T10:00:00.000Z", "2026-11-11T10:00:00.100Z", "2026-11-11T10:00:00.300Z", \ + "2026-11-11T10:00:01.000Z" + + +def _line(seq: int, kind: str, record: dict) -> dict: + return {"seq": seq, "at": T0, kind: record} + + +def _write(path, lines, tail: str = "") -> None: + path.write_text("".join(json.dumps(line) + "\n" for line in lines) + tail) + + +def _body(data: bytes, **extra) -> dict: + return {"size": len(data), "sha256": hashlib.sha256(data).hexdigest(), "base64": base64.b64encode(data).decode(), + **extra} + + +def _exchange(key: str, request: dict, response: dict | None = None, started: str = T0, **end) -> list[dict]: + opened = _line(1, "open", {"id": key, "kind": "exchange", "started": started, "request": request}) + ended = {"id": key, "ended": T2, **end} + if response is not None: + ended.update(answered=T1, response=response) + return [opened, _line(2, "end", ended)] + + +def _get(url="https://shop.test/", **fields) -> dict: + return {"method": "GET", "url": url, "http_version": "HTTP/1.1", "headers": [["Host", "shop.test"]], **fields} + + +def _ok(headers, body: dict | None = None, status=200, reason="OK") -> dict: + return {"status": status, "reason": reason, "http_version": "HTTP/1.1", "headers": headers, + **({"body": body} if body else {})} + + +def test_a_read_is_the_lines_after_a_number_in_order_a_page_at_a_time(tmp_path): + path = tmp_path / "network.jsonl" + _write(path, [_line(n, "end", {"id": str(n), "ended": T0}) for n in range(1, 6)]) + log = NetworkLog(path) + + first = log.read(after=0, limit=2) + second = log.read(after=first["last"], limit=2) + rest = log.read(after=second["last"]) + + assert ([line["seq"] for line in first["lines"]], first["last"], first["more"]) == ([1, 2], 2, True) + assert ([line["seq"] for line in second["lines"]], second["more"]) == ([3, 4], True) + assert ([line["seq"] for line in rest["lines"]], rest["last"], rest["more"]) == ([5], 5, False) + assert log.read(after=5) == {"lines": [], "last": 5, "more": False} + assert [line["seq"] for line in log.read(after=3)["lines"]] == [4, 5] + + +def test_a_line_still_being_written_is_left_for_the_next_read_and_a_bad_line_passed_over(tmp_path): + path = tmp_path / "network.jsonl" + _write(path, [_line(1, "end", {"id": "1", "ended": T0})], tail='not json\n{"seq":2,"at":"') + log = NetworkLog(path) + + assert [line["seq"] for line in log.read()["lines"]] == [1] + with path.open("a") as file: + file.write(f'{T0}","end":{{"id":"2","ended":"{T0}"}}}}\n') + assert [line["seq"] for line in log.read(after=1)["lines"]] == [2] + + +def test_a_log_begun_again_shorter_is_read_from_its_start(tmp_path): + path = tmp_path / "network.jsonl" + _write(path, [_line(n, "end", {"id": str(n), "ended": T0, "pad": "x" * 200}) for n in range(1, 4)]) + log = NetworkLog(path) + log.read() + _write(path, [_line(4, "end", {"id": "4", "ended": T0})]) + + assert [line["seq"] for line in log.read(after=3)["lines"]] == [4] + + +def test_a_missing_log_reads_as_empty_and_exports_as_har_with_no_entries(tmp_path): + log = NetworkLog(tmp_path / "none.jsonl") + + assert log.read(after=4) == {"lines": [], "last": 4, "more": False} + assert har(log.every_line())["log"] == {"version": "1.2", "creator": {"name": "agentenv", + "version": network._version()}, + "entries": []} + + +def test_a_read_stops_at_its_byte_limit_but_always_returns_one_line(tmp_path, monkeypatch): + path = tmp_path / "network.jsonl" + _write(path, [_line(n, "end", {"id": str(n), "ended": T0, "pad": "x" * 500}) for n in range(1, 4)]) + monkeypatch.setattr(network, "READ_BYTES", 100) + + page = NetworkLog(path).read() + + assert ([line["seq"] for line in page["lines"]], page["more"]) == ([1], True) + assert [line["seq"] for line in NetworkLog(path).every_line()] == [1, 2, 3] + + +def test_an_exchange_is_one_entry_with_the_fields_a_grader_reads(): + form = b"user=amy&next=%2Fcart" + request = _get("https://shop.test/login?from=home&q=", method="POST", headers=[ + ["Host", "shop.test"], ["Cookie", "sid=s1; theme=dark"], ["Referer", "https://shop.test/"], + ["Content-Type", "application/x-www-form-urlencoded"], ["X-Shop-Key", ""]], body=_body(form)) + response = _ok([["Location", "https://shop.test/cart"], ["Content-Type", "text/html; charset=utf-8"], + ["Set-Cookie", "sid=s2; Path=/; HttpOnly; Secure; SameSite=Lax"]], + _body(b"moved"), status=303, reason="See Other") + + [entry] = har(_exchange("1-1", request, response))["log"]["entries"] + + assert (entry["startedDateTime"], entry["time"], entry["timings"]) == (T0, 300, {"send": 0, "wait": 100, + "receive": 200}) + sent = entry["request"] + assert (sent["method"], sent["url"], sent["httpVersion"]) == ("POST", "https://shop.test/login?from=home&q=", + "HTTP/1.1") + assert sent["queryString"] == [{"name": "from", "value": "home"}, {"name": "q", "value": ""}] + assert sent["cookies"] == [{"name": "sid", "value": "s1"}, {"name": "theme", "value": "dark"}] + assert {"name": "X-Shop-Key", "value": ""} in sent["headers"] + assert sent["postData"] == {"mimeType": "application/x-www-form-urlencoded", "text": "user=amy&next=%2Fcart"} + assert sent["bodySize"] == len(form) + answer = entry["response"] + assert (answer["status"], answer["statusText"], answer["redirectURL"]) == (303, "See Other", + "https://shop.test/cart") + assert answer["cookies"] == [{"name": "sid", "value": "s2", "httpOnly": True, "secure": True, "path": "/", + "sameSite": "Lax"}] + assert answer["content"] == {"size": 12, "mimeType": "text/html; charset=utf-8", "text": "moved"} + + +def test_a_location_on_an_answer_not_a_redirect_is_no_redirect_url(): + [entry] = har(_exchange("1-1", _get(), _ok([["Location", "https://shop.test/x"]])))["log"]["entries"] + + assert entry["response"]["redirectURL"] == "" + + +@pytest.mark.parametrize("coding, encode", [ + ("gzip", gzip.compress), + ("deflate", zlib.compress), + ("deflate", lambda data: (lambda c: c.compress(data) + c.flush())(zlib.compressobj(wbits=-zlib.MAX_WBITS))), + ("br", brotli.compress), + ("zstd", lambda data: zstandard.ZstdCompressor().compress(data)), + ("gzip, br", lambda data: brotli.compress(gzip.compress(data))), +], ids=["gzip", "deflate", "raw-deflate", "br", "zstd", "gzip-then-br"]) +def test_an_answer_body_is_decoded_from_its_coding(coding, encode): + text = json.dumps({"items": list(range(200))}).encode() + wire = encode(text) + response = _ok([["Content-Type", "application/json"], ["Content-Encoding", coding]], _body(wire)) + + [entry] = har(_exchange("1-1", _get(), response))["log"]["entries"] + + content = entry["response"]["content"] + assert json.loads(content["text"]) == {"items": list(range(200))} + assert content["size"] == len(wire) and content["compression"] == len(text) - len(wire) + + +@pytest.mark.parametrize("coding, wire", [("compress", b"\x1f\x9d..."), ("gzip", b"not gzip at all")], + ids=["unknown-coding", "broken-gzip"]) +def test_an_answer_body_that_cant_be_decoded_stays_base64_with_its_coding_named(coding, wire): + response = _ok([["Content-Type", "text/plain"], ["Content-Encoding", coding]], _body(wire)) + + [entry] = har(_exchange("1-1", _get(), response))["log"]["entries"] + + content = entry["response"]["content"] + assert (content["encoding"], content["_undecoded"], base64.b64decode(content["text"])) == ("base64", coding, wire) + + +def test_an_answer_body_is_cut_at_the_limit_once_decoded_and_says_so(monkeypatch): + monkeypatch.setattr(network, "DECODED_LIMIT", 1000) + wire = gzip.compress(b"a" * 5000) + response = _ok([["Content-Type", "text/plain"], ["Content-Encoding", "gzip"]], _body(wire)) + + [entry] = har(_exchange("1-1", _get(), response))["log"]["entries"] + + content = entry["response"]["content"] + assert (content["text"], content["_truncated"]) == ("a" * 1000, True) + + +def test_a_body_the_proxy_cut_says_so_and_decodes_as_far_as_it_goes(): + text = b"x" * 4000 + wire = gzip.compress(text) + response = _ok([["Content-Type", "text/plain"], ["Content-Encoding", "gzip"]], + {**_body(wire[:len(wire) // 2]), "size": len(wire), "truncated": True}) + + [entry] = har(_exchange("1-1", _get(), response))["log"]["entries"] + + content = entry["response"]["content"] + assert content["_truncated"] is True and content["size"] == len(wire) + + +def test_a_body_is_decoded_in_its_charset(): + response = _ok([["Content-Type", "text/plain; charset=iso-8859-1"]], _body("café".encode("latin-1"))) + + [entry] = har(_exchange("1-1", _get(), response))["log"]["entries"] + + assert entry["response"]["content"]["text"] == "café" + + +def test_an_answer_the_proxy_kept_no_bytes_of_has_its_size_and_sha256(): + image = bytes(range(256)) * 4 + body = {"size": len(image), "sha256": hashlib.sha256(image).hexdigest()} + + [entry] = har(_exchange("1-1", _get(), _ok([["Content-Type", "image/png"]], body)))["log"]["entries"] + + assert entry["response"]["content"] == {"size": 1024, "mimeType": "image/png", + "_sha256": hashlib.sha256(image).hexdigest()} + + +def test_a_websocket_is_its_ws_url_its_resource_type_and_its_message_counts(): + request = _get("https://chat.test/socket", headers=[["Upgrade", "websocket"], ["Connection", "Upgrade"]]) + counts = {"messages_sent": 1, "messages_received": 2, "bytes_sent": 14, "bytes_received": 9} + + [entry] = har(_exchange("1-1", request, _ok([["Upgrade", "websocket"]], status=101, + reason="Switching Protocols"), websocket=counts))["log"]["entries"] + + assert (entry["request"]["url"], entry["_resourceType"], entry["_websocket"]) == ( + "wss://chat.test/socket", "websocket", counts) + + +def test_a_tunnel_is_a_connect_answered_as_established_with_its_address_and_bytes(): + lines = [_line(1, "open", {"id": "1-1", "kind": "tunnel", "host": "cdn.test", "port": 443, "started": T0, + "address": "198.51.100.7"}), + _line(2, "end", {"id": "1-1", "ended": T3, "bytes_sent": 812, "bytes_received": 40210})] + + [entry] = har(lines)["log"]["entries"] + + assert (entry["request"]["method"], entry["request"]["url"]) == ("CONNECT", "https://cdn.test:443/") + assert (entry["response"]["status"], entry["response"]["statusText"]) == (200, "Connection established") + assert entry["serverIPAddress"] == "198.51.100.7" and entry["time"] == 1000 + assert entry["_tunnel"] == {"host": "cdn.test", "port": 443, "address": "198.51.100.7", "bytes_sent": 812, + "bytes_received": 40210} + + +def test_a_plain_request_the_stand_in_forwarded_is_its_method_and_url_its_answer_unread(): + lines = [_line(1, "open", {"id": "1-1", "kind": "tunnel", "host": "shop.test", "port": 80, "method": "GET", + "url": "http://shop.test/a?b=1", "started": T0, "address": "198.51.100.7"}), + _line(2, "end", {"id": "1-1", "ended": T2, "bytes_sent": 80, "bytes_received": 500})] + + [entry] = har(lines)["log"]["entries"] + + assert (entry["request"]["method"], entry["request"]["url"]) == ("GET", "http://shop.test/a?b=1") + assert entry["request"]["queryString"] == [{"name": "b", "value": "1"}] + assert entry["response"]["status"] == 0 and entry["_tunnel"]["bytes_received"] == 500 + + +def test_a_failed_a_refused_and_an_open_connection_are_entries_that_say_so(): + lines = [_line(1, "end", {"id": "1-1", "kind": "tunnel", "host": "gone.test", "port": 443, "started": T0, + "ended": T1, "failed": {"reason": "doesn't resolve", "error": "gaierror"}}), + _line(2, "end", {"id": "1-2", "kind": "refused", "method": "GET", "url": "evil.test", "started": T1, + "ended": T1, "refused": {"reason": "no site of this computer"}}), + _line(3, "open", {"id": "1-3", "kind": "exchange", "started": T2, "request": _get()})] + + failed, refused, still = har(lines)["log"]["entries"] + + assert failed["_failed"] == {"reason": "doesn't resolve", "error": "gaierror"} + assert (failed["response"]["status"], failed["request"]["url"]) == (0, "https://gone.test:443/") + assert refused["_refused"] == {"reason": "no site of this computer"} + assert (refused["request"]["method"], refused["request"]["url"]) == ("GET", "http://evil.test/") + assert still["_open"] is True and still["response"]["status"] == 0 and still["time"] == 0 + + +def test_entries_are_in_the_order_they_started_not_the_order_they_ended(): + slow = _exchange("1-1", _get("https://shop.test/slow"), _ok([]), started=T0) + fast = _exchange("1-2", _get("https://shop.test/fast"), _ok([]), started=T1) + lines = [slow[0], fast[0], fast[1], slow[1]] + + assert [entry["request"]["url"] for entry in har(lines)["log"]["entries"]] == [ + "https://shop.test/slow", "https://shop.test/fast"] + + +@pytest.mark.asyncio +async def test_the_har_text_is_every_line_as_har(tmp_path): + path = tmp_path / "network.jsonl" + _write(path, _exchange("1-1", _get(), _ok([["Content-Type", "text/plain"]], _body(b"hi")))) + + exported = json.loads(await NetworkLog(path).har_text()) + + assert exported["log"]["entries"][0]["response"]["content"]["text"] == "hi" + + +@pytest.mark.parametrize("coding, encode", [ + ("gzip", gzip.compress), + ("br", brotli.compress), + ("zstd", lambda data: zstandard.ZstdCompressor().compress(data)), +], ids=["gzip", "br", "zstd"]) +def test_a_small_body_that_decodes_huge_stops_at_the_limit_answer_and_request_alike(coding, encode): + wire = encode(b"a" * (64 << 20)) + response = _ok([["Content-Type", "text/plain"], ["Content-Encoding", coding]], _body(wire)) + request = _get(method="POST", headers=[["Content-Type", "text/plain"], ["Content-Encoding", coding]], + body=_body(wire)) + + [entry] = har(_exchange("1-1", request, response))["log"]["entries"] + + answer, sent = entry["response"]["content"], entry["request"]["postData"] + assert (len(answer["text"]), answer["_truncated"]) == (network.DECODED_LIMIT, True) + assert (len(sent["text"]), sent["_truncated"]) == (network.REQUEST_DECODED_LIMIT, True) + + +def test_a_line_that_cant_be_an_entry_is_left_out_and_counted_and_the_rest_export(): + good = _exchange("1-1", _get(), _ok([["Content-Type", "text/plain"]], _body(b"hi"))) + lines = [*good, {"seq": 3, "at": T0, "open": {"kind": "exchange"}}, + {"seq": 4, "at": T0, "end": {"id": 7}}, {"seq": 5, "at": T0, "open": {"id": "1-2", "started": 5}}, + {"seq": 6, "at": T0, "end": {"id": "1-3", "started": T0, "response": {"status": "x", "headers": 3}}}, + {"seq": 7, "at": T0}] + + exported = har(lines)["log"] + + assert [entry["response"]["content"]["text"] for entry in exported["entries"]] == ["hi"] + assert exported["_unreadable"] == 5 + + +def test_a_cookies_expiry_is_iso_8601_and_one_that_doesnt_parse_is_left_out(): + response = _ok([["Set-Cookie", "a=1; Expires=Wed, 09 Jun 2027 10:18:14 GMT"], + ["Set-Cookie", "b=2; Expires=someday"], ["Set-Cookie", "c=3; Expires=Fri, 31 Dec 9999 23:59:59 -0100"]]) + + [entry] = har(_exchange("1-1", _get(), response))["log"]["entries"] + + first, second, third = entry["response"]["cookies"] + assert first["expires"] == "2027-06-09T10:18:14.000Z" and "expires" not in second and "expires" not in third + + +def test_an_answer_the_proxy_couldnt_read_says_so(): + [entry] = har(_exchange("1-1", _get("http://live.test/"), None, + unparsed={"reason": "an answer the proxy can't read as HTTP", "error": "bad"}))["log"][ + "entries"] + + assert entry["_unparsed"]["error"] == "bad" and entry["response"]["status"] == 0 diff --git a/pyproject.toml b/pyproject.toml index ba03791cc..97ee9f98b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -156,6 +156,8 @@ dev = [ "fastmcp>=3.4,<4", # the standalone-fastmcp mount path of the protocol; <4 keeps mcp 1.x "Pillow>=10.4", # the protocol's computer extra, whose tests the unit tier runs "av>=17", # the protocol's computer extra, whose tests the unit tier runs and which the recording tests decode with + "brotli>=1.2", # the protocol's app extra, which decodes a website browser's network log; its tests import it + "zstandard>=0.23", "griffelib==2.3.0", # the plugin API check; exact, since what it reports as a break changes between releases "tomli-w==1.2.0", # the Codex agent's config writer, which only its image installs; its tests import it "hypothesis>=6.100", # the property tests of the judge score parse diff --git a/src/agent_env/browsers/README.md b/src/agent_env/browsers/README.md index 04204faf8..68fa1ddaa 100644 --- a/src/agent_env/browsers/README.md +++ b/src/agent_env/browsers/README.md @@ -15,7 +15,8 @@ sees only its pages' requests, but for one Google account check at launch. It ru without Chromium's sandbox, which a container can't give it. Its env.toml spells out its settings and their defaults under `[settings]`: `start_urls`, opened at boot in order, the first tab active, each waited on to load; `window`, alone only; `locale`; `time_zone`; and -`geolocation` and `notifications`, `deny` or `ask`; and `dialogs`, `agent`, `accept` or `dismiss`. A computer env sets its own beside it +`geolocation` and `notifications`, `deny` or `ask`; `dialogs`, `agent`, `accept` or `dismiss`; and `network`, whether +`data/get` exports its network log, on by default. A computer env sets its own beside it (`apps = [{ env = "chromium", settings = { start_urls = ["https://example.test/"] } }]`), and a deploy overrides any of them (`env_params.app_settings = {chromium = {start_urls = [...]}}`). Its data is its profile, `~/.config/chromium`, which the image has none of, so each deploy and each reset starts with a clean one. Name it from another bundle as `chromium`, @@ -29,6 +30,14 @@ and pixel action an agent takes, and an event for each tab opened, navigated or taken out (`agentenv_protocol.website_browser.log`); `browser/v1` `log` reads it and `data/get` exports it as `browser-log.jsonl`. It lasts the computer's boot: `reset_env` boots the computer afresh with a new one, so a task that needs the log before a reset takes it first with `snapshot_env`. +Its proxy keeps its network log (`agentenv_protocol.website_browser.network`), which `browser/v1` `network` reads and +`data/get` exports as HAR 1.2, `network.har`, the file a network grader such as WebArena-Verified's reads: routing +sites, the shared site proxy's, `/etc/agentenv/sites/network.jsonl`, each exchange whole, a site's sign-in headers +taken out; else the base's stand-in's, `/var/log/agentenv/network.jsonl`, each connection, since it decrypts none. On +a computer with a desktop it has none, and its export no `network.har`. +Like its log, it lasts the computer's boot. It keeps cookies and form bodies as the site saw them, sign-in passwords +among them, so a run that needs no network grader turns it off with `network = false`: its export then has no +`network.har` and `network` returns no lines. A task changes a browser's state with the steps in `agent_env.task_step.task_steps.browser`, each a call to one `browser/v1` operation on the browser in the env `env_id`, the one `browser` names when the env has several: diff --git a/src/agent_env/browsers/envs/chromium/env.toml b/src/agent_env/browsers/envs/chromium/env.toml index 5216351b4..fef58a60a 100644 --- a/src/agent_env/browsers/envs/chromium/env.toml +++ b/src/agent_env/browsers/envs/chromium/env.toml @@ -11,3 +11,4 @@ time_zone = "UTC" geolocation = "deny" notifications = "deny" dialogs = "agent" +network = true diff --git a/src/agent_env/computers/README.md b/src/agent_env/computers/README.md index b9fd9ff67..07171af04 100644 --- a/src/agent_env/computers/README.md +++ b/src/agent_env/computers/README.md @@ -18,7 +18,9 @@ is the browser's `window` setting. supervisord runs every program as the session stand-in site proxy on `127.0.0.1:3128`, the browser's only route out when the computer routes no sites. It forwards a request or a `CONNECT` tunnel to any host, never decrypting a tunnel and refusing nothing: the sandbox's network policy is all that limits it. Every line it writes to `/var/log/agentenv/site-proxy.log` -is JSON, one per connection and one per connection that fails, in the same shapes as the shared site proxy's. A computer that routes sites names the shared site proxy's forward port, +is JSON, one per connection and one per connection that fails, in the same shapes as the shared site proxy's. It +keeps the browser's network log too, in `/var/log/agentenv/network.jsonl`, which `AGENTENV_NETWORK_LOG` names: an +`open` and an `end` line for each connection, numbered, in the shared site proxy's shapes. A computer that routes sites names the shared site proxy's forward port, `127.0.0.1:3129`, to the browser instead: it serves the run's sites, refuses every other host and runs out of the agent's reach. It has a container form only. Name it from another bundle as `ubuntu-24.04-browser`, or by its id, `@local/agentenv-framework/computers/ubuntu-24.04-browser`. It was `ubuntu-24.04-headless`, a name nothing resolves diff --git a/src/agent_env/computers/artifacts/ubuntu-24.04-browser/Dockerfile b/src/agent_env/computers/artifacts/ubuntu-24.04-browser/Dockerfile index bac2e1a7c..98d2063f0 100644 --- a/src/agent_env/computers/artifacts/ubuntu-24.04-browser/Dockerfile +++ b/src/agent_env/computers/artifacts/ubuntu-24.04-browser/Dockerfile @@ -2,7 +2,7 @@ # browser and a page driver whose screen is the browser's page. The browser runs headful on an X server of its own, # Xvfb on :99, which no agent sees, records or drives, so sites see a browser with a window rather than a headless one. # supervisord runs every program as the session user, and the browser goes out only through the site proxy on -# 127.0.0.1:3128. +# 127.0.0.1:3128, which writes its network log to /var/log/agentenv/network.jsonl. # Files a page's file chooser may be given go in ~/Uploads. Composing a driver or an app onto it adds its services as # programs in /etc/supervisor/conf.d. The trust files agentenv adds are in /etc/agentenv from boot: ca.pem, empty until # a deploy adds a CA, and ca-bundle.pem, the public roots and those CAs. @@ -23,6 +23,7 @@ COPY start-computer /usr/local/bin/start-computer COPY site-proxy.py /usr/local/bin/agentenv-site-proxy USER user WORKDIR /home/user -ENV LANG=C.UTF-8 XDG_SESSION_TYPE=none DISPLAY=:99 AGENTENV_SITE_PROXY=http://127.0.0.1:3128 +ENV LANG=C.UTF-8 XDG_SESSION_TYPE=none DISPLAY=:99 AGENTENV_SITE_PROXY=http://127.0.0.1:3128 \ + AGENTENV_NETWORK_LOG=/var/log/agentenv/network.jsonl ENTRYPOINT ["/usr/bin/tini", "--"] CMD ["/usr/local/bin/start-computer"] diff --git a/src/agent_env/computers/artifacts/ubuntu-24.04-browser/site-proxy.py b/src/agent_env/computers/artifacts/ubuntu-24.04-browser/site-proxy.py index 3fc860305..daa48af7f 100644 --- a/src/agent_env/computers/artifacts/ubuntu-24.04-browser/site-proxy.py +++ b/src/agent_env/computers/artifacts/ubuntu-24.04-browser/site-proxy.py @@ -3,19 +3,30 @@ The sandbox's own network policy still decides where a connection can go. Every line it logs is JSON, in the same shapes as the shared site proxy's: ``connection`` (host, port, address, bytes each way, start, duration) when one ends, and ``failed`` (method, host, port, ``reason`` "doesn't resolve" or "no answer", the error, and when it was -asked for) for one that couldn't open.""" +asked for) for one that couldn't open. + +It also keeps the browser's network log, in the file ``AGENTENV_NETWORK_LOG`` names, in the shared site proxy's lines: +an ``open`` line when a connection opens and an ``end`` line, naming the same ``id``, when it ends or fails to open, +each numbered after the last the file holds. Seeing no request inside a tunnel, it keeps each at connection level.""" import http.server +import itertools import json +import os +import re import selectors import socket import socketserver +import threading import time import urllib.parse +from datetime import datetime, timezone PORT = 3128 CONNECT_TIMEOUT_S = 30 IDLE_S = 300 HOP = {"connection", "keep-alive", "proxy-connection", "proxy-authorization"} +NETWORK_LOG_ENV, NETWORK_LOG = "AGENTENV_NETWORK_LOG", "/var/log/agentenv/network.jsonl" +NUMBERED = re.compile(rb'^\{"seq":(\d+),') def log(**fields) -> None: @@ -26,6 +37,55 @@ def _when(at: float) -> str: return time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(at)) +def _stamp() -> str: + return datetime.now(timezone.utc).isoformat(timespec="milliseconds").replace("+00:00", "Z") + + +class NetworkLog: + """The network log in ``path``, each line numbered after the last the file holds, so a proxy started again + carries on, and written whole by one thread at a time.""" + + def __init__(self, path: str): + seq, partial = 0, False + try: + with open(path, "rb") as existing: + for raw in existing: + partial = not raw.endswith(b"\n") + if not partial and (numbered := NUMBERED.match(raw)): + seq = max(seq, int(numbered.group(1))) + except FileNotFoundError: + pass + self.file = open(path, "ab") + if partial: + self.file.write(b"\n") + self.seq, self.lock, self.ids = seq, threading.Lock(), itertools.count(1) + + def key(self) -> str: + return f"{os.getpid()}-{next(self.ids)}" + + def write(self, kind: str, record: dict) -> None: + with self.lock: + self.seq += 1 + line = json.dumps({"seq": self.seq, "at": _stamp(), kind: record}, separators=(",", ":")) + "\n" + try: + self.file.write(line.encode()) + self.file.flush() + except (OSError, ValueError) as error: + log(error={"connection": "network log", "error": _error(error)}) + + +NETWORK: NetworkLog | None = None + + +def _note(kind: str, record: dict) -> None: + if NETWORK is not None: + NETWORK.write(kind, record) + + +def _key() -> str: + return NETWORK.key() if NETWORK is not None else "" + + def _error(error: BaseException) -> str: return f"{type(error).__name__}: {error}".removesuffix(": ") @@ -39,40 +99,49 @@ def do_CONNECT(self): if not port.isdigit(): self.send_error(400) return - if (opened := self._open(host, int(port), "CONNECT")) is None: + key, started, record = _key(), _stamp(), {"kind": "tunnel", "host": host, "port": int(port)} + if (opened := self._open(host, int(port), "CONNECT", key, started, record)) is None: return - self.send_response(200, "Connection established") - self.end_headers() self.close_connection = True - self._relay(*opened, host, int(port), b"") + self._relay(*opened, host, int(port), b"", key, {**record, "id": key, "started": started}, established=True) def forward(self): url = urllib.parse.urlsplit(self.path) if not url.hostname: self.send_error(400) return + try: + length = int(self.headers.get("Content-Length") or 0) + except ValueError: + self.send_error(400) + return port = url.port or 80 - if (opened := self._open(url.hostname, port, self.command)) is None: + key, started = _key(), _stamp() + record = {"kind": "tunnel", "host": url.hostname, "port": port, "method": self.command, "url": self.path} + if (opened := self._open(url.hostname, port, self.command, key, started, record)) is None: return target = (url.path or "/") + (f"?{url.query}" if url.query else "") headers = "".join(f"{k}: {v}\r\n" for k, v in self.headers.items() if k.lower() not in HOP) head = f"{self.command} {target} HTTP/1.1\r\n{headers}Connection: close\r\n\r\n".encode("latin-1") - body = self.rfile.read(length) if (length := int(self.headers.get("Content-Length") or 0)) else b"" self.close_connection = True - self._relay(*opened, url.hostname, port, head + body) + self._relay(*opened, url.hostname, port, head, key, {**record, "id": key, "started": started}, length=length) do_GET = do_POST = do_PUT = do_PATCH = do_DELETE = do_HEAD = do_OPTIONS = forward - def _open(self, host: str, port: int, method: str) -> tuple[socket.socket, str] | None: + def _open(self, host: str, port: int, method: str, key: str, started: str, + record: dict) -> tuple[socket.socket, str] | None: """A connection to an address ``host`` resolves to, tried in turn within one ``CONNECT_TIMEOUT_S`` for them all, and that address; or None once the browser has its answer, a 502 for a host that doesn't resolve or - answer. The connection keeps an ``IDLE_S`` timeout, so a write to a peer that stopped reading ends too.""" + answer, which ends ``record`` in the network log. The connection keeps an ``IDLE_S`` timeout, so a write to a + peer that stopped reading ends too.""" asked, deadline = time.time(), time.monotonic() + CONNECT_TIMEOUT_S try: found = socket.getaddrinfo(host, port, type=socket.SOCK_STREAM) except OSError as error: log(failed={"method": method, "host": host, "port": port, "reason": "doesn't resolve", "error": _error(error), "started": _when(asked)}) + _note("end", {**record, "id": key, "started": started, "ended": _stamp(), + "failed": {"reason": "doesn't resolve", "error": _error(error)}}) self._answer(502, f"{host} doesn't resolve\n") return None last = None @@ -86,8 +155,11 @@ def _open(self, host: str, port: int, method: str) -> tuple[socket.socket, str] continue upstream.settimeout(IDLE_S) return upstream, address - log(failed={"method": method, "host": host, "port": port, "reason": "no answer", - "error": _error(last) if last else f"none within {CONNECT_TIMEOUT_S} s", "started": _when(asked)}) + why = _error(last) if last else f"none within {CONNECT_TIMEOUT_S} s" + log(failed={"method": method, "host": host, "port": port, "reason": "no answer", "error": why, + "started": _when(asked)}) + _note("end", {**record, "id": key, "started": started, "ended": _stamp(), + "failed": {"reason": "no answer", "error": why}}) self._answer(502, f"{host}:{port} didn't answer\n") return None @@ -102,16 +174,29 @@ def _answer(self, status: int, body: str) -> None: self.wfile.write(data) self.close_connection = True - def _relay(self, upstream: socket.socket, address: str, host: str, port: int, first: bytes) -> None: + def _relay(self, upstream: socket.socket, address: str, host: str, port: int, first: bytes, key: str, + opening: dict, *, established: bool = False, length: int = 0) -> None: + """The connection to ``upstream`` relayed both ways, in the network log from its ``open`` line to its ``end`` + line, which is written however it ends: a tunnel's ``established`` answer, or a request's ``length`` bytes of + body after its head ``first``, are sent within it.""" started, began = time.time(), time.monotonic() - sent, received = len(first), 0 - self.connection.settimeout(IDLE_S) + sent = received = 0 try: + _note("open", {**opening, "address": address}) + if established: + self.send_response(200, "Connection established") + self.end_headers() + if length: + first += self.rfile.read(length) + self.connection.settimeout(IDLE_S) if first: upstream.sendall(first) + sent = len(first) more, received = _pipe(self.connection, upstream) sent += more finally: + upstream.close() + _note("end", {"id": key, "ended": _stamp(), "bytes_sent": sent, "bytes_received": received}) log(connection={"host": host, "port": port, "address": address, "bytes_sent": sent, "bytes_received": received, "started": _when(started), @@ -157,5 +242,9 @@ class Server(socketserver.ThreadingMixIn, http.server.HTTPServer): if __name__ == "__main__": + try: + NETWORK = NetworkLog(os.environ.get(NETWORK_LOG_ENV) or NETWORK_LOG) + except OSError as error: + log(error={"connection": "network log", "error": _error(error)}) log(listening={"address": "127.0.0.1", "port": PORT}) Server(("127.0.0.1", PORT), Proxy).serve_forever() diff --git a/src/agent_env/env/envs/computer/utils/site_proxy.py b/src/agent_env/env/envs/computer/utils/site_proxy.py index 0e9ea16b9..acce65a09 100644 --- a/src/agent_env/env/envs/computer/utils/site_proxy.py +++ b/src/agent_env/env/envs/computer/utils/site_proxy.py @@ -24,6 +24,7 @@ import base64 import hashlib import ipaddress +import itertools import json import os import re @@ -37,6 +38,7 @@ import urllib.error import urllib.request from dataclasses import dataclass, field +from datetime import datetime, timezone from pathlib import Path from typing import Awaitable, Optional from urllib.parse import urlsplit, urlunsplit @@ -85,6 +87,15 @@ RESTART_DELAY_S = 1.0 SERVE_USER = "nobody" FAILED = (("X-Agentenv-Site-Proxy", "failed"),) +#: The most of a request's body, and of an answer's, the network log keeps, and the answers it keeps any of, by their +#: type: text, JSON, forms, XML and scripts. +REQUEST_BODY_LIMIT, ANSWER_BODY_LIMIT = 10 << 20, 1 << 20 +KEPT_ANSWERS = re.compile( + r"^\s*(text/|application/(json|x-www-form-urlencoded|xml|javascript|ecmascript)\b|[^;]*\+(json|xml)\b)", re.I) +#: What the network log keeps of a header a site names as its sign-in header. +REMOVED = "" +#: How a network log line begins, so a proxy started again finds its last number without reading every body. +NUMBERED = re.compile(rb'^\{"seq":(\d+),') class BadRequest(Exception): @@ -120,6 +131,11 @@ def when(at: float) -> str: return time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(at)) +def stamp() -> str: + """Now, to the millisecond, in UTC, as the network log, and HAR after it, write times.""" + return datetime.now(timezone.utc).isoformat(timespec="milliseconds").replace("+00:00", "Z") + + def described(error: BaseException) -> str: return f"{type(error).__name__}: {error}".removesuffix(": ") @@ -130,6 +146,108 @@ class Count: def __init__(self, start: int = 0): self.bytes = start + def add(self, data: bytes) -> None: + self.bytes += len(data) + + +class Capture: + """A body as it is relayed: its size and, when it is ``logged``, its sha256 and, when ``keep``, its first ``limit`` + bytes; with no network log to write it to, its size alone, so nothing is held or hashed for nothing.""" + + def __init__(self, limit: int, keep: bool = True, logged: bool = True): + self.limit, self.keep, self.size = limit, keep and logged, 0 + self.data, self.digest = bytearray(), hashlib.sha256() if logged else None + + def add(self, data: bytes) -> None: + self.size += len(data) + if self.digest is not None: + self.digest.update(data) + if self.keep and len(self.data) < self.limit: + self.data += data[:self.limit - len(self.data)] + + def record(self) -> dict: + """The body as the network log keeps it: its bytes base64, kept or not, and cut or not.""" + if self.digest is None: + return {"size": self.size} + kept = {"base64": base64.b64encode(bytes(self.data)).decode(), "truncated": self.size > self.limit} \ + if self.keep else {} + return {"size": self.size, "sha256": self.digest.hexdigest(), **kept} + + +class Frames: + """One direction of a WebSocket, its messages counted as its bytes go by: a message ends at a data frame marked + final, and control frames aren't messages.""" + + def __init__(self): + self.messages = self.bytes = self._payload = 0 + self._head = b"" + + def add(self, data: bytes) -> None: + self.bytes += len(data) + data, at = self._head + data, 0 + self._head = b"" + while at < len(data): + if self._payload: + taken = min(self._payload, len(data) - at) + self._payload -= taken + at += taken + continue + if len(data) - at < 2: + break + first, second = data[at], data[at + 1] + size, length = second & 0x7F, 2 + (2 if second & 0x7F == 126 else 8 if second & 0x7F == 127 else 0) + if len(data) - at < length + (4 if second & 0x80 else 0): + break + if size >= 126: + size = int.from_bytes(data[at + 2:at + length], "big") + if first & 0x80 and first & 0x0F < 8: + self.messages += 1 + at += length + (4 if second & 0x80 else 0) + self._payload = size + self._head = data[at:] if not self._payload else b"" + + +class NetworkLog: + """The network log in ``path``: JSON lines appended as the proxy runs, each numbered after the last the file holds, + so a proxy started again carries on, and timed. ``open`` opens it as root, readable by every user, before + ``serve`` drops privileges, so the session user can read the log and can't write it.""" + + def __init__(self, file, seq: int): + self.file, self.seq, self.ids = file, seq, itertools.count(1) + + @classmethod + def open(cls, path: str) -> NetworkLog: + seq, partial = 0, False + try: + with open(path, "rb") as existing: + for raw in existing: + partial = not raw.endswith(b"\n") + if not partial and (numbered := NUMBERED.match(raw)): + seq = max(seq, int(numbered.group(1))) + except FileNotFoundError: + pass + descriptor = os.open(path, os.O_WRONLY | os.O_APPEND | os.O_CREAT | os.O_NOFOLLOW, 0o644) + os.fchmod(descriptor, 0o644) + file = os.fdopen(descriptor, "ab") + if partial: + file.write(b"\n") + return cls(file, seq) + + def key(self) -> str: + """An id for a request or tunnel, which its ``open`` and ``end`` lines both name: unique to this process's + log, and, by its pid, to a proxy started again.""" + return f"{os.getpid()}-{next(self.ids)}" + + def write(self, kind: str, record: dict) -> None: + """A line of ``kind``, ``open`` or ``end``, holding ``record``.""" + self.seq += 1 + line = json.dumps({"seq": self.seq, "at": stamp(), kind: record}, separators=(",", ":")) + "\n" + try: + self.file.write(line.encode()) + self.file.flush() + except (OSError, ValueError) as e: + log_event(error={"connection": "network log", "error": described(e)}) + @dataclass class Site: @@ -142,6 +260,7 @@ class Site: cert: Optional[str] = None key: Optional[str] = None upstream_headers: dict[str, str] = field(default_factory=dict) + sign_in_headers: list[str] = field(default_factory=list) scheme: str = field(init=False) host: str = field(init=False) port: int = field(init=False) @@ -179,6 +298,7 @@ class Config: dns_port: Optional[int] = None nameservers: list[str] = field(default_factory=list) direct: list[str] = field(default_factory=list) + network_log: Optional[str] = None @classmethod def load(cls, path: str) -> Config: @@ -192,12 +312,14 @@ def load(cls, path: str) -> Config: except PermissionError: headers = {} sites = [Site(name=s["name"], upstream=s["upstream"], domains=list(s["domains"]), cert=s.get("cert"), - key=s.get("key"), upstream_headers=dict(headers.get(s["name"]) or {})) for s in data["sites"]] + key=s.get("key"), upstream_headers=dict(headers.get(s["name"]) or {}), + sign_in_headers=list(s.get("sign_in_headers") or [])) for s in data["sites"]] return cls(user=data.get("user"), sites=sites, serve_user=data.get("serve_user"), listen=data.get("listen", LOOPBACK), https_port=data.get("https_port", 443), http_port=data.get("http_port", 80), proxy_port=data.get("proxy_port"), dns_port=data.get("dns_port"), - nameservers=list(data.get("nameservers") or []), direct=list(data.get("direct") or [])) + nameservers=list(data.get("nameservers") or []), direct=list(data.get("direct") or []), + network_log=data.get("network_log")) def site_for(self, host: str) -> Optional[Site]: """The site ``host`` names: one naming it exactly before one whose wildcard covers it.""" @@ -324,6 +446,15 @@ def direct_head(start: str, headers: list[tuple[str, str]]) -> bytes: def response_head(status_line: str, headers: list[tuple[str, str]], site: Site, *, scheme: str, host: str, connection: str) -> bytes: _, _, status = status_line.partition(" ") + lines = [f"HTTP/1.1 {status}", *(f"{key}: {value}" for key, value in + public_headers(headers, site, scheme=scheme, host=host, connection=connection))] + return ("\r\n".join(lines) + "\r\n\r\n").encode("latin-1") + + +def public_headers(headers: list[tuple[str, str]], site: Site, *, scheme: str, host: str, + connection: str) -> list[tuple[str, str]]: + """An answer's headers as the browser gets them: the proxy's own dropped, ``Location`` and cookies naming the route + moved onto the site's name, and ``connection``.""" kept = [] for key, value in headers: lower = key.lower() @@ -335,8 +466,36 @@ def response_head(status_line: str, headers: list[tuple[str, str]], site: Site, value = public_cookie(value, site) kept.append((key, value)) kept.append(("Connection", connection)) - lines = [f"HTTP/1.1 {status}", *(f"{key}: {value}" for key, value in kept)] - return ("\r\n".join(lines) + "\r\n\r\n").encode("latin-1") + return kept + + +def logged_headers(headers: list[tuple[str, str]], removed: list[str]) -> list[list[str]]: + """A request's headers as the network log keeps them: each as the browser sent it, but for those named in + ``removed``, a site's sign-in headers, whose values are taken out.""" + names = {name.lower() for name in removed} + return [[key, REMOVED if key.lower() in names else value] for key, value in headers] + + +def kept_answer(headers: list[tuple[str, str]]) -> bool: + """Whether the network log keeps any of an answer's body, by its ``Content-Type``.""" + return bool(KEPT_ANSWERS.match(header(headers, "content-type") or "")) + + +def status_parts(status_line: str) -> tuple[str, int, str]: + """An answer's status line as its version, its code and its reason.""" + version, _, rest = status_line.partition(" ") + code, _, reason = rest.partition(" ") + try: + return version, int(code), reason + except ValueError: + raise BadRequest(f"a malformed status line {status_line!r}") from None + + +def answered(status_line: str, headers: list[tuple[str, str]]) -> dict: + """An answer as the network log keeps it, before its body: its status, its reason and its headers.""" + version, code, reason = status_parts(status_line) + return {"status": code, "reason": reason, "http_version": version, + "headers": [[key, value] for key, value in headers]} def simple_response(status: str, body: str, extra: tuple[tuple[str, str], ...] = ()) -> bytes: @@ -358,18 +517,22 @@ async def read_head(reader: asyncio.StreamReader) -> Optional[bytes]: raise BadRequest("a head over the limit") from None -async def relay_exact(reader: asyncio.StreamReader, writer: asyncio.StreamWriter, size: int) -> None: +async def relay_exact(reader: asyncio.StreamReader, writer: asyncio.StreamWriter, size: int, + capture: Optional[Capture] = None) -> None: while size > 0: data = await reader.read(min(size, CHUNK)) if not data: raise BadRequest("the body ended early") writer.write(data) await writer.drain() + if capture is not None: + capture.add(data) size -= len(data) -async def relay_chunked(reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None: - """A chunked body relayed as framed, to its last chunk and trailers.""" +async def relay_chunked(reader: asyncio.StreamReader, writer: asyncio.StreamWriter, + capture: Optional[Capture] = None) -> None: + """A chunked body relayed as framed, to its last chunk and trailers; ``capture`` takes the chunks' data alone.""" while True: line = await reader.readuntil(b"\r\n") writer.write(line) @@ -383,34 +546,36 @@ async def relay_chunked(reader: asyncio.StreamReader, writer: asyncio.StreamWrit writer.write(trailer) await writer.drain() return - await relay_exact(reader, writer, size + 2) + await relay_exact(reader, writer, size, capture) + await relay_exact(reader, writer, 2) -async def relay_until_eof(reader: asyncio.StreamReader, writer: asyncio.StreamWriter, - count: Optional[Count] = None) -> None: +async def relay_until_eof(reader: asyncio.StreamReader, writer: asyncio.StreamWriter, count=None) -> None: + """Everything ``reader`` sends until it ends, each piece handed to ``count`` (a ``Count``, a ``Capture`` or + ``Frames``) when given.""" while data := await reader.read(CHUNK): writer.write(data) await writer.drain() if count is not None: - count.bytes += len(data) + count.add(data) async def relay_body(reader: asyncio.StreamReader, writer: asyncio.StreamWriter, headers: list[tuple[str, str]], - *, to_eof: bool) -> None: + *, to_eof: bool, capture: Optional[Capture] = None) -> None: if "chunked" in tokens(headers, "transfer-encoding"): - await relay_chunked(reader, writer) + await relay_chunked(reader, writer, capture) elif (length := header(headers, "content-length")) is not None: try: - await relay_exact(reader, writer, int(length)) + await relay_exact(reader, writer, int(length), capture) except ValueError: raise BadRequest(f"a malformed Content-Length {length!r}") from None elif to_eof: - await relay_until_eof(reader, writer) + await relay_until_eof(reader, writer, capture) -async def splice(reader: asyncio.StreamReader, writer: asyncio.StreamWriter, count: Optional[Count] = None) -> None: +async def splice(reader: asyncio.StreamReader, writer: asyncio.StreamWriter, count=None) -> None: """One direction of an upgraded connection, closing the other side when it ends, which ends the other direction; - the bytes it carries counted in ``count`` when given.""" + each piece it carries handed to ``count`` when given.""" try: await relay_until_eof(reader, writer, count) except (ConnectionError, OSError, ssl.SSLError): @@ -421,12 +586,23 @@ async def splice(reader: asyncio.StreamReader, writer: asyncio.StreamWriter, cou class SiteProxy: """The proxy's HTTP side: one connection's requests, each forwarded on a connection of its own to its site's - route.""" + route; with a ``network`` log, each request and each tunnel in it as it begins and as it ends.""" - def __init__(self, config: Config, tls: Optional[ssl.SSLContext] = None): + def __init__(self, config: Config, tls: Optional[ssl.SSLContext] = None, network: Optional[NetworkLog] = None): self.config = config self.tls = tls self.upstream_tls = ssl.create_default_context() + self.network = network + + def _id(self) -> str: + return self.network.key() if self.network is not None else "" + + def _note(self, kind: str, record: dict) -> None: + if self.network is not None: + self.network.write(kind, record) + + def _capture(self, limit: int, keep: bool = True) -> Capture: + return Capture(limit, keep, logged=self.network is not None) async def handle(self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter, scheme: str) -> None: await self._serving(writer, scheme, self._requests(reader, writer, scheme, _peer(writer), forward=False)) @@ -453,7 +629,7 @@ async def _proxied(self, reader: asyncio.StreamReader, writer: asyncio.StreamWri target = urlsplit(rest.rpartition(" ")[0]) if target.hostname and self.config.site_for(target.hostname) is None and self.config.is_direct( target.hostname): - await self._direct(reader, writer, target.hostname, target.port or 80, direct_head(start, headers)) + await self._direct_exchange(reader, writer, start, headers, target.hostname, target.port or 80) return if not await self.exchange(reader, writer, "http", peer, head=head, forward=True): return @@ -480,42 +656,150 @@ async def _tunnel(self, reader: asyncio.StreamReader, writer: asyncio.StreamWrit await writer.start_tls(self.tls) await self._requests(reader, writer, "https" if tls else "http", peer, forward=True) - @staticmethod - async def _direct(reader: asyncio.StreamReader, writer: asyncio.StreamWriter, host: str, port: int, - head: Optional[bytes] = None) -> None: - """``host``, a live site's, reached straight at ``port``: a tunnel, answered as established, for a ``CONNECT``, - else ``head`` sent on; then both ends relayed until either closes, and the connection logged (``log_event``). - One that doesn't resolve or answer within ``UPSTREAM_TIMEOUT_S``, the lookup and every address's attempt - together, is a 502, logged too.""" + async def _connect_direct(self, writer: asyncio.StreamWriter, method: str, host: str, port: int, key: str, + started: str, record: dict) -> Optional[tuple]: + """A connection to ``host``, a live site's, at ``port``, its reader, writer and address; or None once the browser + has a 502 for one that doesn't resolve or answer within ``UPSTREAM_TIMEOUT_S``, the lookup and every address's + attempt together, which is logged (``log_event``) and ends ``record`` in the network log.""" asked = time.time() try: - up_reader, up_writer = await asyncio.wait_for(asyncio.open_connection(host, port), UPSTREAM_TIMEOUT_S) + up_reader, up_writer = await asyncio.wait_for(asyncio.open_connection(host, port, limit=HEAD_LIMIT), + UPSTREAM_TIMEOUT_S) except (OSError, asyncio.TimeoutError) as e: unresolved = isinstance(e, socket.gaierror) - log_event(failed={"method": "CONNECT" if head is None else head.split(b" ", 1)[0].decode("latin-1"), - "host": host, "port": port, "reason": "doesn't resolve" if unresolved else "no answer", - "error": described(e), "started": when(asked)}) + reason = "doesn't resolve" if unresolved else "no answer" + log_event(failed={"method": method, "host": host, "port": port, "reason": reason, "error": described(e), + "started": when(asked)}) + self._note("end", {**record, "id": key, "started": started, "ended": stamp(), + "failed": {"reason": reason, "error": described(e)}}) writer.write(simple_response("502 Bad Gateway", f"{host} doesn't resolve\n" if unresolved else f"{host}:{port} didn't answer\n", FAILED)) + return None + return up_reader, up_writer, (up_writer.get_extra_info("peername") or ("",))[0] + + async def _direct(self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter, host: str, port: int) -> None: + """A ``CONNECT`` to ``host``, a live site's, tunnelled straight to it at ``port`` and answered as established; + then both ends relayed until either closes. The tunnel is in the network log as it opens and as it ends, and + the connection is logged (``log_event``) as it ends.""" + key, started = self._id(), stamp() + tunnel = {"kind": "tunnel", "host": host, "port": port} + if (opened := await self._connect_direct(writer, "CONNECT", host, port, key, started, tunnel)) is None: return - address = (up_writer.get_extra_info("peername") or ("",))[0] - started, began, sent, received = time.time(), time.monotonic(), Count(len(head or b"")), Count() - if head is None: - writer.write(b"HTTP/1.1 200 Connection established\r\n\r\n") - else: - up_writer.write(head) + up_reader, up_writer, address = opened + self._note("open", {**tunnel, "id": key, "started": started, "address": address}) + wall, began, sent, received = time.time(), time.monotonic(), Count(), Count() + writer.write(b"HTTP/1.1 200 Connection established\r\n\r\n") try: await asyncio.gather(writer.drain(), up_writer.drain()) await asyncio.gather(splice(reader, up_writer, sent), splice(up_reader, writer, received)) finally: + self._note("end", {"id": key, "ended": stamp(), "bytes_sent": sent.bytes, "bytes_received": received.bytes}) + log_event(connection={"host": host, "port": port, "address": address, "bytes_sent": sent.bytes, + "bytes_received": received.bytes, "started": when(wall), + "duration_s": round(time.monotonic() - began, 3)}) + + async def _direct_exchange(self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter, start: str, + headers: list[tuple[str, str]], host: str, port: int) -> None: + """A request a browser alone named by its URL on a live site's ``host``, sent straight to it at ``port`` with its + path alone and a connection that closes after it. Its body and the answer go both ways at once, as a tunnel's + bytes do, so an answer that comes before the body is done, a ``100 Continue`` or an early 413, reaches the + browser as it comes. The exchange is in the network log in full; an answer the proxy can't read as HTTP is + relayed as it came, its entry saying so (``unparsed``). The connection is logged (``log_event``) as a tunnel's + is.""" + method, _, rest = start.partition(" ") + target, _, version = rest.rpartition(" ") + key, started = self._id(), stamp() + request = {"method": method, "url": target, "http_version": version, "headers": logged_headers(headers, [])} + record = {"kind": "exchange", "request": request} + if (opened := await self._connect_direct(writer, method, host, port, key, started, record)) is None: + return + up_reader, up_writer, address = opened + head = direct_head(start, headers) + wall, began, sent, received = time.time(), time.monotonic(), Count(len(head)), Count() + body, kept, ended, noted = self._capture(REQUEST_BODY_LIMIT), None, {"id": key}, [] + + def sent_request() -> dict: + return {**request, **({"body": body.record()} if body.size else {})} + + async def upload() -> None: + try: + up_writer.write(head) + await relay_body(reader, up_writer, headers, to_eof=False, capture=body) + await up_writer.drain() + except (BadRequest, ConnectionError, OSError, asyncio.IncompleteReadError) as e: + ended.setdefault("failed", {"reason": "the request broke off", "error": described(e)}) + up_writer.close() + return + self._note("open", {**record, "id": key, "started": started, "address": address, + "request": sent_request()}) + noted.append(True) + + sending = asyncio.ensure_future(upload()) + try: + while True: + try: + answer = await up_reader.readuntil(b"\r\n\r\n") + status_line, answer_headers = parse_head(answer) + code = status_parts(status_line)[1] + except asyncio.IncompleteReadError as e: + if not e.partial.strip(): + writer.write(simple_response("502 Bad Gateway", f"{host} closed without answering\n", + FAILED)) + ended.setdefault("failed", {"reason": "no answer", + "error": f"{host} closed without answering"}) + return + await self._unparsed(up_reader, writer, e.partial, received, ended, "the answer's head ended early") + return + except asyncio.LimitOverrunError: + await self._unparsed(up_reader, writer, await up_reader.read(HEAD_LIMIT), received, ended, + "a head over the limit") + return + except BadRequest as e: + await self._unparsed(up_reader, writer, answer, received, ended, str(e)) + return + received.bytes += len(answer) + writer.write(answer) + if not 100 <= code < 200: + break + await writer.drain() + ended.update(answered=stamp(), response=answered(status_line, answer_headers)) + kept = self._capture(ANSWER_BODY_LIMIT, kept_answer(answer_headers)) + if method != "HEAD" and code not in (204, 304): + await relay_body(up_reader, writer, answer_headers, to_eof=True, capture=kept) + await writer.drain() + received.bytes += kept.size + except BaseException as e: + ended.setdefault("failed", {"reason": "the exchange broke off", "error": described(e)}) + raise + finally: + if not sending.done(): + sending.cancel() + await asyncio.gather(sending, return_exceptions=True) + sent.bytes += body.size + if kept is not None and kept.size: + ended["response"]["body"] = kept.record() + alone = {} if noted else {**record, "request": sent_request(), "started": started, "address": address} + self._note("end", {**ended, "ended": stamp(), **alone}) log_event(connection={"host": host, "port": port, "address": address, "bytes_sent": sent.bytes, - "bytes_received": received.bytes, - "started": when(started), + "bytes_received": received.bytes, "started": when(wall), "duration_s": round(time.monotonic() - began, 3)}) + up_writer.close() @staticmethod - def _refuse(writer: asyncio.StreamWriter, method: str, host: str) -> None: + async def _unparsed(up_reader: asyncio.StreamReader, writer: asyncio.StreamWriter, first: bytes, received: Count, + ended: dict, why: str) -> None: + """An answer the proxy can't read as HTTP, relayed as it came, from ``first`` to its end.""" + ended["unparsed"] = {"reason": "an answer the proxy can't read as HTTP", "error": why} + received.add(first) + writer.write(first) + await writer.drain() + await relay_until_eof(up_reader, writer, received) + + def _refuse(self, writer: asyncio.StreamWriter, method: str, host: str) -> None: log_event(refused={"method": method, "host": host, "reason": "no site of this computer"}) + now = stamp() + self._note("end", {"id": self._id(), "kind": "refused", "started": now, "ended": now, "method": method, + "url": host, "refused": {"reason": "no site of this computer"}}) writer.write(simple_response("403 Forbidden", f"{host} is no site of this computer, and the browser reaches " "its sites alone\n", FAILED)) @@ -558,11 +842,17 @@ async def exchange(self, reader: asyncio.StreamReader, writer: asyncio.StreamWri writer.write(simple_response("421 Misdirected Request", f"{host or 'no host'} is no site of this " "computer\n", FAILED)) return False + key, started = self._id(), stamp() + request = {"method": method, "url": target if "://" in target else f"{scheme}://{host}{target}", + "http_version": version, "headers": logged_headers(headers, site.sign_in_headers)} if scheme == "http" and site.tls(split_host(host).lower()): parts = urlsplit(target) path = target if target.startswith("/") else urlunsplit(("", "", parts.path or "/", parts.query, "")) location = f"https://{split_host(host)}{path}" writer.write(simple_response("308 Permanent Redirect", f"{location}\n", (("Location", location),))) + self._note("end", {"id": key, "kind": "exchange", "started": started, "ended": stamp(), "request": request, + "response": {"status": 308, "reason": "Permanent Redirect", "http_version": "HTTP/1.1", + "headers": [["Location", location]]}}) return False upgrade = header(headers, "upgrade") if "upgrade" in tokens(headers, "connection") else None if "100-continue" in tokens(headers, "expect"): @@ -574,49 +864,80 @@ async def exchange(self, reader: asyncio.StreamReader, writer: asyncio.StreamWri except (OSError, asyncio.TimeoutError, ssl.SSLError) as e: writer.write(simple_response("502 Bad Gateway", f"the site {site.name} can't be reached at " f"{site.upstream}: {type(e).__name__}: {e}\n", FAILED)) + self._note("end", {"id": key, "kind": "exchange", "started": started, "ended": stamp(), "request": request, + "failed": {"reason": "the site can't be reached", "error": described(e)}}) return False try: return await self._forward(reader, writer, up_reader, up_writer, method, target, version, headers, site, - scheme=scheme, host=host, peer=peer, upgrade=upgrade) + scheme=scheme, host=host, peer=peer, upgrade=upgrade, key=key, started=started, + request=request) finally: up_writer.close() async def _forward(self, reader, writer, up_reader, up_writer, method, target, version, headers, site, *, - scheme, host, peer, upgrade) -> bool: - up_writer.write(upstream_request(method, target, headers, site, scheme=scheme, host=host, peer=peer, - upgrade=upgrade)) - await relay_body(reader, up_writer, headers, to_eof=False) - await up_writer.drain() - while True: - answer = await read_head(up_reader) - if answer is None: - writer.write(simple_response("502 Bad Gateway", f"the site {site.name} closed without answering\n", - FAILED)) - return False - status_line, answer_headers = parse_head(answer) - code = int(status_line.split(" ")[1]) - if code == 101 and upgrade: - writer.write(response_head(status_line, answer_headers, site, scheme=scheme, host=host, - connection="Upgrade")) - await writer.drain() - await asyncio.gather(splice(reader, up_writer), splice(up_reader, writer)) - return False - if 100 <= code < 200: - continue - break - bodiless = method == "HEAD" or code in (204, 304) - framed = bodiless or "chunked" in tokens(answer_headers, "transfer-encoding") \ - or header(answer_headers, "content-length") is not None - keep = framed and version == "HTTP/1.1" and "close" not in tokens(headers, "connection") - writer.write(response_head(status_line, answer_headers, site, scheme=scheme, host=host, - connection="keep-alive" if keep else "close")) - if not bodiless: - try: - await relay_body(up_reader, writer, answer_headers, to_eof=True) - except (BadRequest, asyncio.IncompleteReadError): - return False - await writer.drain() - return keep + scheme, host, peer, upgrade, key, started, request) -> bool: + """The request sent on to the site's route and its answer relayed back; whether the connection takes another. + The request is in the network log once it has gone upstream, and its answer once it is done.""" + body, kept, ended, noted = self._capture(REQUEST_BODY_LIMIT), None, {"id": key}, False + try: + up_writer.write(upstream_request(method, target, headers, site, scheme=scheme, host=host, peer=peer, + upgrade=upgrade)) + await relay_body(reader, up_writer, headers, to_eof=False, capture=body) + await up_writer.drain() + self._note("open", {"id": key, "kind": "exchange", "started": started, + "request": {**request, **({"body": body.record()} if body.size else {})}}) + noted = True + while True: + answer = await read_head(up_reader) + if answer is None: + writer.write(simple_response("502 Bad Gateway", + f"the site {site.name} closed without answering\n", FAILED)) + ended["failed"] = {"reason": "no answer", "error": f"the site {site.name} closed without answering"} + return False + status_line, answer_headers = parse_head(answer) + code = status_parts(status_line)[1] + if code == 101 and upgrade: + shown = public_headers(answer_headers, site, scheme=scheme, host=host, connection="Upgrade") + ended.update(answered=stamp(), response=answered(status_line, shown)) + writer.write(response_head(status_line, answer_headers, site, scheme=scheme, host=host, + connection="Upgrade")) + await writer.drain() + out, back = Frames(), Frames() + try: + await asyncio.gather(splice(reader, up_writer, out), splice(up_reader, writer, back)) + finally: + ended["websocket"] = {"messages_sent": out.messages, "messages_received": back.messages, + "bytes_sent": out.bytes, "bytes_received": back.bytes} + return False + if 100 <= code < 200: + continue + break + bodiless = method == "HEAD" or code in (204, 304) + framed = bodiless or "chunked" in tokens(answer_headers, "transfer-encoding") \ + or header(answer_headers, "content-length") is not None + keep = framed and version == "HTTP/1.1" and "close" not in tokens(headers, "connection") + connection = "keep-alive" if keep else "close" + shown = public_headers(answer_headers, site, scheme=scheme, host=host, connection=connection) + ended.update(answered=stamp(), response=answered(status_line, shown)) + writer.write(response_head(status_line, answer_headers, site, scheme=scheme, host=host, + connection=connection)) + kept = self._capture(ANSWER_BODY_LIMIT, kept_answer(answer_headers)) + if not bodiless: + try: + await relay_body(up_reader, writer, answer_headers, to_eof=True, capture=kept) + except (BadRequest, asyncio.IncompleteReadError) as e: + ended["failed"] = {"reason": "the answer ended early", "error": described(e)} + return False + await writer.drain() + return keep + except BaseException as e: + ended.setdefault("failed", {"reason": "the exchange broke off", "error": described(e)}) + raise + finally: + if kept is not None and kept.size: + ended["response"]["body"] = kept.record() + extra = {} if noted else {"kind": "exchange", "started": started, "request": request} + self._note("end", {**ended, "ended": stamp(), **extra}) class Resolver(asyncio.DatagramProtocol): @@ -764,9 +1085,11 @@ def pick(connection: ssl.SSLObject, name: Optional[str], _: ssl.SSLContext) -> N async def serve(config: Config) -> None: - """Listen, drop to the user it serves as, and serve until cancelled.""" + """Listen, open the network log when the config names one, drop to the user it serves as, and serve until + cancelled. The log is opened before the drop, so the user it serves as writes it through this process alone.""" context = tls_context(config) - proxy = SiteProxy(config, tls=context) + network = NetworkLog.open(config.network_log) if config.network_log else None + proxy = SiteProxy(config, tls=context, network=network) servers = [] if context is not None: servers.append(await asyncio.start_server(lambda r, w: proxy.handle(r, w, "https"), config.listen, @@ -1100,8 +1423,8 @@ def install(config_path: str, paths: Paths = Paths()) -> dict: for name, value in given["trust_environment"].items())) trust["environment"] = str(paths.profile) serving = {"user": user, "serve_user": serve_user(user), "sites": sites, - **{key: given[key] for key in ("listen", "https_port", "http_port", "proxy_port", "direct") - if key in given}} + **{key: given[key] for key in ("listen", "https_port", "http_port", "proxy_port", "direct", + "network_log") if key in given}} if wildcard: serving.update(dns_port=given.get("dns_port", 53), nameservers=nameservers) if upstream_headers: diff --git a/src/agent_env/env/envs/computer/utils/sites.py b/src/agent_env/env/envs/computer/utils/sites.py index 98eba5325..040bfe734 100644 --- a/src/agent_env/env/envs/computer/utils/sites.py +++ b/src/agent_env/env/envs/computer/utils/sites.py @@ -35,7 +35,7 @@ from agentenv_protocol import client as protocol_client from agentenv_protocol.website import WEBSITE_EXTENSION_URI as WEBSITE from agentenv_protocol.website import WEBSITE_LIVE_PARAM -from agentenv_protocol.website_browser import SITE_PROXY_ENV +from agentenv_protocol.website_browser import NETWORK_LOG_ENV, SITE_PROXY_ENV from cryptography import x509 from cryptography.hazmat.primitives import hashes, serialization from cryptography.hazmat.primitives.asymmetric import ec @@ -64,6 +64,9 @@ FORWARD_PROXY = f"http://127.0.0.1:{FORWARD_PROXY_PORT}" ALONE_DISPLAY = "none" LOG = FOLDER / "site-proxy.log" +#: The network log a browser alone's site proxy writes, in the folder root keeps, so the session user can read it and +#: can't change or replace it. +NETWORK_LOG = FOLDER / "network.jsonl" SYNTHETIC, ALLOWLIST = "synthetic", "allowlist" RANKS = (SYNTHETIC, ALLOWLIST) VALID_MARGIN = timedelta(days=1) @@ -298,9 +301,10 @@ def trust_environment(os_type: str, sites: list[Site], *, later: bool = False) - def browser_environment(display: str) -> dict[str, str]: - """For a computer routing sites, the variable naming the forward port to a browser alone, which runs on a computer - with no display; none on one with a display, whose apps reach the sites by name on the transparent listeners.""" - return {SITE_PROXY_ENV: FORWARD_PROXY} if display == ALONE_DISPLAY else {} + """For a computer routing sites, the variables naming the forward port and its network log to a browser alone, + which runs on a computer with no display; none on one with a display, whose apps reach the sites by name on the + transparent listeners.""" + return {SITE_PROXY_ENV: FORWARD_PROXY, NETWORK_LOG_ENV: str(NETWORK_LOG)} if display == ALONE_DISPLAY else {} def proxy_config(table: SiteTable, issued: Issued, *, user: Optional[str], os_type: str, forward_port: bool, @@ -309,7 +313,7 @@ def proxy_config(table: SiteTable, issued: Issued, *, user: Optional[str], os_ty the variables naming them, which its profile script sets as the computer's boot environment does, each route with its certificate and key, and the headers its gateway's sandbox provider asks requests to it to carry (``upstream_headers``, by site), which the proxy sends upstream alone, and with ``forward_port``, for a browser - alone, the forward port and the live sites' hosts it takes straight to them (``direct``).""" + alone, the forward port, the live sites' hosts it takes straight to them (``direct``) and its network log.""" sites = [] for route in table.routes: site = {"name": route.site, "upstream": route.url, "domains": list(route.domains)} @@ -320,7 +324,8 @@ def proxy_config(table: SiteTable, issued: Issued, *, user: Optional[str], os_ty sites.append(site) return {"user": user, "ca_pem": issued.ca_pem, "sites": sites, "log": str(LOG), "trust_files": trust_files(os_type), "trust_environment": dict(TRUST_ENVIRONMENTS[os_type]), - **({"proxy_port": FORWARD_PROXY_PORT, "direct": list(direct)} if forward_port else {})} + **({"proxy_port": FORWARD_PROXY_PORT, "direct": list(direct), "network_log": str(NETWORK_LOG)} + if forward_port else {})} def reachable_problem(policy: Optional[NetworkPolicy], routes: list[Route]) -> Optional[str]: diff --git a/tst/integration/computer/browser_computer_test.py b/tst/integration/computer/browser_computer_test.py index a6e9f246b..140cf63f9 100644 --- a/tst/integration/computer/browser_computer_test.py +++ b/tst/integration/computer/browser_computer_test.py @@ -24,7 +24,10 @@ loads for well over a minute after it's ready, but for one Google account check at launch. Routing sites, it goes out only through the shared site proxy's forward port: start tabs on a site over HTTPS and on a .localhost one load through it, an event stream comes through as it's sent, any other host and loopback are refused and logged, the session user can't stop the port, killed it comes back, and -after a reset it serves the sites and refuses the rest again. As a multi env's website browser, ``browser_envs = +after a reset it serves the sites and refuses the rest again; its network log has each exchange whole, so that +WebArena-Verified's own loader, reading the network.har its data/get exports, sees an agent's sign-in form, redirect, +cookie, JSON and multipart as its grader reads them, and the live hosts' tunnels and plain requests beside them; alone, +it has each connection the base's stand-in made, and each that failed. As a multi env's website browser, ``browser_envs = [chromium]``, it deploys on a sandbox of its own as one more computer child beside a synthetic site, with no playwright-mcp: its start tab loads the site by name through the gateway, an agent drives it through the gateway's /mcp under ``chromium_``, its CDP address asked through ``/svc/chromium/`` names the gateway and reaches the site by name, a @@ -39,16 +42,21 @@ localhost:5000, and Node 20 or later and npm for the CDP clients.""" import asyncio +import base64 import calendar import contextlib +import gzip +import http.server import itertools import json import os import platform import re import shlex +import shutil import ssl import subprocess +import threading import time import urllib.parse import uuid @@ -67,7 +75,8 @@ from agentenv_protocol.computer.conformance.client import image from agentenv_protocol.tool_access import ENABLE, ENABLE_TOOL_EXTENSION_URI from agentenv_protocol.tool_context import ROLE_HEADER -from agentenv_protocol.website_browser import BROWSER +from agentenv_protocol.types import uploaded_object_path +from agentenv_protocol.website_browser import BROWSER, HAR_FILE import agent_env from agent_env.a2a_agent import conversation_store @@ -654,6 +663,24 @@ async def call(method: str, params: dict, session: str | None = None) -> dict: return [(method, target) for method in sent] +async def _network_log(deployed, after: int = 0) -> dict: + card = (await _get(deployed, "/.well-known/agent-env.json")).json() + app = next(child for child in card["children_environments"] if child["name"] == "chromium") + path = protocol_v1.find_extension_method(app, BROWSER, "network")["endpoint"] + return (await _get(deployed, path, headers=deployed.control_headers, params={"after": after})).json() + + +async def _har(deployed) -> bytes: + """The browser's network log as its data/get exports it, ``network.har``.""" + answer = await protocol_v1.get_data(f"{deployed.environment_url}/svc/chromium", timeout=120, + headers=deployed.control_headers) + return next(base64.b64decode(part.file.bytes) for part in answer.parts if uploaded_object_path(part) == HAR_FILE) + + +async def _har_entries(deployed) -> list[dict]: + return json.loads(await _har(deployed))["log"]["entries"] + + async def _browser_log(deployed, after: int = 0) -> dict: card = (await _get(deployed, "/.well-known/agent-env.json")).json() app = next(child for child in card["children_environments"] if child["name"] == "chromium") @@ -874,6 +901,7 @@ async def test_routing_sites_beside_live_ones_its_forward_port_takes_the_live_ho "https://github.com/", "http://github.com/", "https://github.githubassets.com/favicons/favicon.svg", "https://shop.test/alpha", "https://example.com/")} tabs = await tabs_of(deployed) + entries = await _har_entries(deployed) finally: report = await teardown_run(context) @@ -883,6 +911,160 @@ async def test_routing_sites_beside_live_ones_its_forward_port_takes_the_live_ho "https://shop.test/alpha")), pages assert pages["https://example.com/"].startswith("error") and "403" in pages["https://example.com/"], pages assert any(tab["url"].startswith("https://github.com/") and "GitHub" in (tab.get("title") or "") for tab in tabs), tabs + by_url = {entry["request"]["url"]: entry for entry in entries} + assert by_url["https://shop.test/alpha"]["response"]["content"]["text"] == "alpha", by_url.keys() + assert by_url["http://github.com/"]["response"]["status"] in (200, 301, 308) and \ + by_url["http://github.com/"]["request"]["method"] == "GET" + github = [entry for entry in entries if entry["request"]["url"] == "https://github.com:443/" and "_open" not in entry] + assert any(entry["request"]["method"] == "CONNECT" and entry["response"]["status"] == 200 and + entry["_tunnel"]["bytes_received"] > 0 for entry in github), github + assert by_url["https://example.com:443/"]["_refused"] == {"reason": "no site of this computer"} + assert not report.failed + + +#: A shop's pages under its route: a form that signs in by POST, answered with a redirect that sets a cookie, onto a +#: cart whose script saves an item as JSON, answered gzipped, then uploads a file as multipart, each answer saying in +#: the title what came of it. +SHOP_FORM = ("form
" + "
").encode() +SHOP_CART = b"""cart""" +SAVED = json.dumps({"id": 7, "name": "desk"}).encode() +#: What WebArena-Verified's own loader reads of each entry of a HAR it grades, as JSON. +WEBARENA_READS = """import json, sys +from pathlib import Path +from webarena_verified.types.tracing import NetworkTrace +trace = NetworkTrace.from_har(Path(sys.argv[1])) +print(json.dumps([{"method": e.http_method, "url": e.url, "status": e.request_status, "post_data": e.post_data, + "content": e.response_content, "redirect": e.redirect_url, "cookies": e.response_cookies, + "navigation": e.is_navigation_event, "referer": e.referer} for e in trace.evaluation_events])) +""" +WEBARENA_VERIFIED = "webarena-verified==1.2.3" + + +@contextlib.contextmanager +def _shop_pages(): + """``SHOP_FORM`` and the rest served on every address of this machine under ``/website/shop/``, at the port it + yields.""" + + class Shop(http.server.BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + + def _send(self, status: int, body: bytes = b"", **headers) -> None: + self.send_response(status) + for name, value in {"Content-Length": str(len(body)), **headers}.items(): + self.send_header(name.replace("_", "-"), value) + self.end_headers() + self.wfile.write(body) + + def do_GET(self): + pages = {"/website/shop/form": SHOP_FORM, "/website/shop/cart": SHOP_CART} + if (path := self.path.partition("?")[0]) in pages: + self._send(200, pages[path], Content_Type="text/html; charset=utf-8") + else: + self._send(404) + + def do_POST(self): + self.rfile.read(int(self.headers.get("Content-Length") or 0)) + if self.path == "/website/shop/login": + self._send(303, Location="/website/shop/cart", Set_Cookie="sid=s2; Path=/website/shop/; HttpOnly") + elif self.path == "/website/shop/api/items": + self._send(201, gzip.compress(SAVED), Content_Type="application/json", Content_Encoding="gzip") + else: + self._send(204) + + def log_message(self, *args): + pass + + server = http.server.ThreadingHTTPServer(("0.0.0.0", 0), Shop) + server.daemon_threads = True + threading.Thread(target=server.serve_forever, daemon=True).start() + try: + yield server.server_address[1] + finally: + server.shutdown() + server.server_close() + + +async def _cdp_agent_shops(ws_url: str) -> str: + """A CDP agent that opens the shop's form, signs in with a click and waits on the cart to save: the title it ends + on.""" + async with connect(ws_url, max_size=None, open_timeout=60) as ws: + ids = itertools.count(1) + + async def call(method: str, params: dict, session: str | None = None) -> dict: + message_id = next(ids) + await ws.send(json.dumps({"id": message_id, "method": method, "params": params, + **({"sessionId": session} if session else {})})) + while (reply := json.loads(await asyncio.wait_for(ws.recv(), 90))).get("id") != message_id: + pass + return reply + + async def title() -> str: + evaluated = await call("Runtime.evaluate", {"expression": "document.title", "returnByValue": True}, session) + return evaluated.get("result", {}).get("result", {}).get("value") or "" + + target = (await call("Target.createTarget", {"url": "https://shop.test/form"}))["result"]["targetId"] + session = (await call("Target.attachToTarget", {"targetId": target, "flatten": True}))["result"]["sessionId"] + deadline = time.monotonic() + 60 + while await title() != "form" and time.monotonic() < deadline: + await asyncio.sleep(0.5) + await call("Runtime.evaluate", {"expression": "document.querySelector('button').click()", + "userGesture": True}, session) + shown = "" + while not shown.startswith("saved") and time.monotonic() < deadline: + await asyncio.sleep(0.5) + shown = await title() + return shown + + +@pytest.mark.asyncio +async def test_routing_sites_its_network_har_has_what_webarena_verifieds_grader_reads(browser_computer, tmp_path): + """M8.20: an agent signs in to a site by a form, follows the redirect and saves an item as JSON and a file as + multipart; WebArena-Verified's own loader, reading the network.har data/get exports, sees each request's method, + URL, form, JSON and multipart fields and referer, each answer's status, redirect, cookie and decoded JSON, and the + page loads as navigations.""" + if shutil.which("uvx") is None: + pytest.skip(f"uvx runs {WEBARENA_VERIFIED} in an environment of its own") + with _shop_pages() as port, namespace_routing(): + params = {"site_routing": True, + "sites": [{"name": "shop", "url": f"http://127.0.0.1:{port}/website/shop/", "domains": ["shop.test"]}], + "app_settings": {"chromium": {"start_urls": [DATA_PAGE]}}} + deployed = await Env.get(browser_computer.id).deploy(env_params=params) + context = TaskStepContext(deployed_envs=[deployed]) + try: + shown = await _cdp_agent_shops((await _cdp(deployed, "ada"))["ws_url"]) + await asyncio.sleep(1) + exported = await _har(deployed) + lines = (await _network_log(deployed))["lines"] + finally: + report = await teardown_run(context) + + (tmp_path / HAR_FILE).write_bytes(exported) + read = subprocess.run(["uvx", "--from", WEBARENA_VERIFIED, "python", "-I", "-c", WEBARENA_READS, + str(tmp_path / HAR_FILE)], capture_output=True, text=True, timeout=600) + assert read.returncode == 0, read.stderr[-3000:] + events = {(event["method"], event["url"]): event for event in json.loads(read.stdout)} + assert shown == "saved 7", shown + form, login = events[("GET", "https://shop.test/form")], events[("POST", "https://shop.test/login")] + cart = events[("GET", "https://shop.test/cart")] + saved, uploaded = events[("POST", "https://shop.test/api/items")], events[("POST", "https://shop.test/upload")] + assert (form["status"], form["navigation"]) == (200, True), form + assert (login["post_data"], login["status"], login["redirect"], login["cookies"]) == ( + {"user": "amy", "note": "caf\u00e9 & tea"}, 303, "/cart", {"sid": "s2"}), login + assert (cart["status"], cart["navigation"], cart["referer"]) == (200, True, "https://shop.test/form"), cart + assert (saved["post_data"], saved["status"], saved["content"]) == ({"name": "desk", "qty": 2}, 201, + {"id": 7, "name": "desk"}), saved + assert (uploaded["post_data"], uploaded["status"]) == ({"title": "Trip", "file": "notes"}, 204), uploaded + assert [line["seq"] for line in lines] == list(range(1, len(lines) + 1)) assert not report.failed @@ -898,6 +1080,8 @@ async def test_alone_on_the_open_web_it_loads_a_public_site_through_its_proxy_an through = {url: await _through_port(deployed, url, "http://127.0.0.1:3128") for url in (public, nowhere)} seen = await _cdp_agent_loads((await _cdp(deployed, "ada"))["ws_url"], [public]) logged = _as_root(await _container_of(deployed), "cat /var/log/agentenv/site-proxy.log") + lines = (await _network_log(deployed))["lines"] + exported = await _har_entries(deployed) finally: report = await teardown_run(context) @@ -907,6 +1091,13 @@ async def test_alone_on_the_open_web_it_loads_a_public_site_through_its_proxy_an github = [entry["connection"] for entry in entries if entry.get("connection", {}).get("host") == "github.com"] assert github and all(entry["bytes_received"] > 0 for entry in github), logged[-2000:] assert any(entry.get("failed", {}).get("host") == "agentenv-no-such-host.invalid" for entry in entries), logged[-2000:] + assert [line["seq"] for line in lines] == list(range(1, len(lines) + 1)), lines[:5] + tunnels = [entry for entry in exported if entry["request"]["url"] == "https://github.com:443/"] + assert tunnels and all(entry["request"]["method"] == "CONNECT" and entry["_tunnel"]["bytes_received"] > 0 + and entry["serverIPAddress"] for entry in tunnels if "_open" not in entry), tunnels + assert any(entry["request"]["url"] == "https://agentenv-no-such-host.invalid:443/" and + entry.get("_failed", {}).get("reason") == "doesn't resolve" for entry in exported), exported[-5:] + assert [entry["startedDateTime"] for entry in exported] == sorted(entry["startedDateTime"] for entry in exported) assert not report.failed diff --git a/tst/integration/computer/website_browser_test.py b/tst/integration/computer/website_browser_test.py index da702a8cf..62ea16e7a 100644 --- a/tst/integration/computer/website_browser_test.py +++ b/tst/integration/computer/website_browser_test.py @@ -158,7 +158,7 @@ async def test_chromium_runs_with_agentenvs_flags_on_its_pipe_and_a_reset_relaun "156.0.8078.4", "real", ["calls/begin", "calls/end", "cdp", "cookies/delete", "cookies/get", "cookies/set", "dialog/answer", "downloads/get", "downloads/list", "files/choose", "history", "local_storage/clear", "local_storage/get", "local_storage/set", "log", "navigate", - "page/capture", "page/input", "page/state", "page/viewport", "record", "sites/set", + "network", "page/capture", "page/input", "page/state", "page/viewport", "record", "sites/set", "tabs/activate", "tabs/close", "tabs/list", "tabs/open"]) assert readiness["ready"] is True, readiness assert relaunched["pid"] != browser["pid"] and relaunched["listening"] == 0 diff --git a/tst/unit/env/envs/test_computer_network_log.py b/tst/unit/env/envs/test_computer_network_log.py new file mode 100644 index 000000000..6cfff00c2 --- /dev/null +++ b/tst/unit/env/envs/test_computer_network_log.py @@ -0,0 +1,478 @@ +"""A browser's network log as its proxies write it: the shared site proxy keeps each exchange whole, the request as +the browser sent it, a site's sign-in headers removed, and the answer as the browser got it, its body up to a limit +by its type, a WebSocket's messages counted, and each tunnel, failure and refusal; the base's stand-in keeps each +connection. Both number their lines after the last the file holds, and the protocol's HAR of what they wrote has the +fields a grader reads. The proxies are network programs, so these tests open sockets, on loopback alone.""" + +from __future__ import annotations + +import asyncio +import gzip +import hashlib +import json +import os +import socket +import ssl +import struct + +import pytest +from agentenv_protocol.website_browser.network import NetworkLog as NetworkReader +from agentenv_protocol.website_browser.network import har + +from agent_env.env.envs.computer.utils import site_proxy +from agent_env.env.envs.computer.utils.site_proxy import NetworkLog +from tst.unit.env.envs.test_computer_site_proxy import (Upstream, _config, _exchange, _forward_port, _live_hosts_at, + _proxy) +from tst.unit.env.envs.test_computer_stand_in_proxy import GITHUB, _ask, _serving, _stand_in + +pytestmark = pytest.mark.usefixtures("socket_enabled") + +PAGE = b"

caf\xe9

" +BIG = b"a" * (1 << 20) + b"b" * 1000 +IMAGE = bytes(range(256)) * 8 +API = json.dumps({"ok": True, "items": [1, 2, 3]}).encode() + + +def _chunked(data: bytes, size: int = 7) -> bytes: + return b"".join(b"%x\r\n%s\r\n" % (len(data[at:at + size]), data[at:at + size]) + for at in range(0, len(data), size)) + b"0\r\n\r\n" + + +def _masked(opcode: int, payload: bytes, final: bool = True) -> bytes: + mask = b"\x01\x02\x03\x04" + return bytes([(0x80 if final else 0) | opcode, 0x80 | len(payload)]) + mask + bytes( + byte ^ mask[at % 4] for at, byte in enumerate(payload)) + + +def _site(upstream_port): + """A site's answers by path: a page in Latin-1 setting a cookie under the route, a sign-in redirecting onto the + route, gzipped JSON sent chunked, a text answer past the limit, an image, and a WebSocket that answers twice.""" + async def respond(start, headers, body, reader, writer): + path = start.split(" ")[1].removeprefix("/website/notes") + if path.startswith("/page"): + writer.write(b"HTTP/1.1 200 OK\r\nContent-Type: text/html; charset=iso-8859-1\r\nContent-Length: %d\r\n" + b"Set-Cookie: sid=s2; Path=/website/notes/; HttpOnly\r\n\r\n%s" % (len(PAGE), PAGE)) + elif path == "/login": + writer.write(f"HTTP/1.1 302 Found\r\nLocation: http://127.0.0.1:{upstream_port()}/website/notes/home\r\n" + f"Content-Length: 0\r\n\r\n".encode()) + elif path == "/api": + writer.write(b"HTTP/1.1 201 Created\r\nContent-Type: application/json\r\nContent-Encoding: gzip\r\n" + b"Transfer-Encoding: chunked\r\n\r\n" + _chunked(gzip.compress(API))) + elif path == "/big": + writer.write(b"HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nContent-Length: %d\r\n\r\n%s" % ( + len(BIG), BIG)) + elif path == "/logo.png": + writer.write(b"HTTP/1.1 200 OK\r\nContent-Type: image/png\r\nContent-Length: %d\r\n\r\n%s" % ( + len(IMAGE), IMAGE)) + elif path == "/socket": + writer.write(b"HTTP/1.1 101 Switching Protocols\r\nUpgrade: websocket\r\nConnection: Upgrade\r\n\r\n") + await writer.drain() + await reader.readexactly(len(_masked(1, b"he", final=False)) + len(_masked(0, b"y")) + + len(_masked(9, b""))) + writer.write(b"\x81\x02hi" + b"\x81\x7e" + struct.pack(">H", 200) + b"z" * 200) + else: + writer.write(b"HTTP/1.1 204 No Content\r\n\r\n") + return respond + + +async def _over_tls(port, issued, request: bytes): + answer = await _exchange(port, request, server_hostname="notes.test", cadata=issued.ca_pem) + answer[3][1].close() + return answer + + +async def _ended(path, timeout_s: float = 5.0) -> None: + """Until the log at ``path`` has an entry and every entry in it has ended, as a connection's does once both its + ends have closed.""" + for _ in range(int(timeout_s / 0.02)): + if (entries := _entries(path)) and not any("_open" in entry for entry in entries): + return + await asyncio.sleep(0.02) + + +def _entries(path) -> list[dict]: + return har(NetworkReader(path).every_line())["log"]["entries"] + + +def _headers(entry_part) -> dict: + return {header["name"].lower(): header["value"] for header in entry_part["headers"]} + + +@pytest.mark.asyncio +async def test_the_site_proxy_keeps_each_exchange_whole_as_the_browser_sent_and_got_it(tmp_path): + path = tmp_path / "network.jsonl" + log = NetworkLog.open(str(path)) + form = b"user=amy&password=pw" + multipart = (b"--XyZ\r\nContent-Disposition: form-data; name=\"title\"\r\n\r\nTrip\r\n--XyZ\r\n" + b"Content-Disposition: form-data; name=\"file\"; filename=\"a.txt\"\r\nContent-Type: text/plain" + b"\r\n\r\nnotes\r\n--XyZ--\r\n") + async with Upstream() as upstream: + upstream.respond = _site(lambda: upstream.port) + config, issued = _config(tmp_path, upstream.port) + config.sites[0].sign_in_headers = ["X-Notes-Key"] + async with _proxy(config, network=log) as port: + page = await _over_tls(port, issued, b"GET /page?q=caf%C3%A9&n= HTTP/1.1\r\nHost: notes.test\r\n" + b"Cookie: sid=s1; theme=dark\r\nX-Notes-Key: secret\r\n" + b"Referer: https://notes.test/\r\nSec-Fetch-Mode: navigate\r\n\r\n") + login = await _over_tls(port, issued, b"POST /login HTTP/1.1\r\nHost: notes.test\r\nContent-Type: " + b"application/x-www-form-urlencoded\r\nContent-Length: %d\r\n\r\n%s" + % (len(form), form)) + api = await _over_tls(port, issued, b"POST /api HTTP/1.1\r\nHost: notes.test\r\nContent-Type: " + b"application/json\r\nTransfer-Encoding: chunked\r\n\r\n" + + _chunked(b'{"title":"Trip"}')) + await _over_tls(port, issued, b"POST /upload HTTP/1.1\r\nHost: notes.test\r\nContent-Type: " + b"multipart/form-data; boundary=XyZ\r\nContent-Length: %d\r\n\r\n%s" + % (len(multipart), multipart)) + await _over_tls(port, issued, b"GET /big HTTP/1.1\r\nHost: notes.test\r\n\r\n") + await _over_tls(port, issued, b"GET /logo.png HTTP/1.1\r\nHost: notes.test\r\n\r\n") + async with _proxy(config, scheme="http", network=log) as port: + plain = await _exchange(port, b"GET /page HTTP/1.1\r\nHost: notes.test\r\n\r\n") + plain[3][1].close() + + entries = _entries(path) + assert [(entry["request"]["method"], entry["request"]["url"], entry["response"]["status"]) for entry in entries] \ + == [("GET", "https://notes.test/page?q=caf%C3%A9&n=", 200), ("POST", "https://notes.test/login", 302), + ("POST", "https://notes.test/api", 201), ("POST", "https://notes.test/upload", 204), + ("GET", "https://notes.test/big", 200), ("GET", "https://notes.test/logo.png", 200), + ("GET", "http://notes.test/page", 308)] + first = entries[0] + assert first["request"]["queryString"] == [{"name": "q", "value": "café"}, {"name": "n", "value": ""}] + assert first["request"]["cookies"] == [{"name": "sid", "value": "s1"}, {"name": "theme", "value": "dark"}] + assert _headers(first["request"])["x-notes-key"] == "" + assert _headers(first["request"])["referer"] == "https://notes.test/" + assert _headers(first["request"])["sec-fetch-mode"] == "navigate" + assert _headers(first["response"]) == page[1] + assert first["response"]["cookies"] == [{"name": "sid", "value": "s2", "httpOnly": True, "secure": False, + "path": "/"}] + assert first["response"]["content"] == {"size": len(PAGE), "mimeType": "text/html; charset=iso-8859-1", + "text": "

café

"} + assert entries[1]["request"]["postData"] == {"mimeType": "application/x-www-form-urlencoded", + "text": "user=amy&password=pw"} + assert entries[1]["response"]["redirectURL"] == login[1]["location"] == "https://notes.test/home" + assert entries[2]["request"]["postData"]["text"] == '{"title":"Trip"}' + assert json.loads(entries[2]["response"]["content"]["text"]) == json.loads(API) == json.loads(gzip.decompress( + api[2])) + assert entries[3]["request"]["postData"] == {"mimeType": "multipart/form-data; boundary=XyZ", + "text": multipart.decode()} + big = entries[4]["response"]["content"] + assert (big["size"], len(big["text"]), big["_truncated"]) == (len(BIG), 1 << 20, True) + assert entries[5]["response"]["content"] == {"size": len(IMAGE), "mimeType": "image/png", + "_sha256": hashlib.sha256(IMAGE).hexdigest()} + assert entries[6]["response"]["redirectURL"] == plain[1]["location"] == "https://notes.test/page" + assert all(entry["timings"]["wait"] <= entry["time"] for entry in entries) + assert not any(key in entry for entry in entries for key in ("_open", "_failed", "_refused")) + + +@pytest.mark.asyncio +async def test_the_site_proxy_counts_a_websockets_messages_each_way(tmp_path): + path = tmp_path / "network.jsonl" + async with Upstream() as upstream: + upstream.respond = _site(lambda: upstream.port) + config, issued = _config(tmp_path, upstream.port) + async with _proxy(config, network=NetworkLog.open(str(path))) as port: + reader, writer = await asyncio.open_connection("127.0.0.1", port, server_hostname="notes.test", + ssl=ssl.create_default_context(cadata=issued.ca_pem)) + writer.write(b"GET /socket HTTP/1.1\r\nHost: notes.test\r\nConnection: Upgrade\r\nUpgrade: websocket\r\n" + b"Sec-WebSocket-Version: 13\r\nSec-WebSocket-Key: dGhlIHNhbXBsZSBub25jZQ==\r\n\r\n") + assert (await reader.readuntil(b"\r\n\r\n")).startswith(b"HTTP/1.1 101 ") + writer.write(_masked(1, b"he", final=False) + _masked(0, b"y") + _masked(9, b"")) + back = await reader.read() + writer.close() + await _ended(path) + + [entry] = _entries(path) + assert back == b"\x81\x02hi" + b"\x81\x7e" + struct.pack(">H", 200) + b"z" * 200 + assert (entry["request"]["url"], entry["response"]["status"], entry["_resourceType"]) == ( + "wss://notes.test/socket", 101, "websocket") + assert entry["_websocket"] == {"messages_sent": 1, "messages_received": 2, "bytes_sent": 8 + 7 + 6, + "bytes_received": len(back)} + + +@pytest.mark.parametrize("cut", [1, 2, 3, 5, 9]) +def test_websocket_frames_are_counted_across_any_cut_and_control_frames_arent_messages(cut): + stream = (_masked(1, b"x" * 100, final=False) + _masked(10, b"") + _masked(0, b"y" * 20) + + b"\x82\x7f" + struct.pack(">Q", 70000) + b"\0" * 70000 + b"\x88\x00") + frames = site_proxy.Frames() + for at in range(0, len(stream), cut): + frames.add(stream[at:at + cut]) + + assert (frames.messages, frames.bytes) == (2, len(stream)) + + +@pytest.mark.asyncio +async def test_the_forward_port_keeps_a_sites_tunnel_by_its_requests_a_live_hosts_whole_and_refusals_and_failures( + tmp_path, monkeypatch): + path = tmp_path / "network.jsonl" + async with Upstream() as upstream, Upstream() as live: + config, issued = _config(tmp_path, upstream.port) + config.direct = ["github.com", "gone.github.com"] + with _live_hosts_at(monkeypatch, live.port): + real = site_proxy.asyncio.open_connection + + async def gone_or_live(host, port, **kwargs): + if host == "gone.github.com": + raise socket.gaierror(-2, "Name or service not known") + return await real(host, port, **kwargs) + + monkeypatch.setattr(site_proxy.asyncio, "open_connection", gone_or_live) + async with _forward_port(config, network=NetworkLog.open(str(path))) as port: + reader, writer = await asyncio.open_connection("127.0.0.1", port) + writer.write(b"CONNECT notes.test:443 HTTP/1.1\r\nHost: notes.test:443\r\n\r\n") + await reader.readuntil(b"\r\n\r\n") + await writer.start_tls(ssl.create_default_context(cadata=issued.ca_pem), server_hostname="notes.test") + await _exchange(port, b"GET /a?b=1 HTTP/1.1\r\nHost: notes.test\r\n\r\n", reader_writer=(reader, writer)) + writer.close() + reader, writer = await asyncio.open_connection("127.0.0.1", port) + writer.write(b"CONNECT github.com:443 HTTP/1.1\r\nHost: github.com:443\r\n\r\n") + await reader.readuntil(b"\r\n\r\n") + await _exchange(port, b"GET /x HTTP/1.1\r\nHost: github.com\r\n\r\n", reader_writer=(reader, writer)) + writer.close() + for request in (b"POST http://github.com/a?b=1 HTTP/1.1\r\nHost: github.com\r\nContent-Length: 2\r\n" + b"Cookie: gh=1\r\n\r\nhi", + b"CONNECT gone.github.com:443 HTTP/1.1\r\nHost: gone.github.com:443\r\n\r\n", + b"CONNECT evil.test:443 HTTP/1.1\r\nHost: evil.test:443\r\n\r\n"): + (await _exchange(port, request))[3][1].close() + await asyncio.sleep(0.3) + + site, tunnel, plain, gone, refused = _entries(path) + assert (site["request"]["url"], site["response"]["content"]["size"]) == ("https://notes.test/a?b=1", 5) + assert (tunnel["request"]["method"], tunnel["request"]["url"], tunnel["serverIPAddress"]) == ( + "CONNECT", "https://github.com:443/", "127.0.0.1") + assert tunnel["_tunnel"]["bytes_sent"] > 0 and tunnel["_tunnel"]["bytes_received"] > 0 + assert (plain["request"]["method"], plain["request"]["url"], plain["request"]["postData"]["text"]) == ( + "POST", "http://github.com/a?b=1", "hi") + assert plain["request"]["cookies"] == [{"name": "gh", "value": "1"}] + assert (plain["response"]["status"], plain["response"]["content"].get("text")) == (200, None) + assert plain["response"]["content"]["size"] == 5 and plain["serverIPAddress"] == "127.0.0.1" + assert gone["_failed"] == {"reason": "doesn't resolve", "error": "gaierror: [Errno -2] Name or service not known"} + assert (refused["_refused"], refused["request"]["url"]) == ({"reason": "no site of this computer"}, + "https://evil.test:443/") + assert not any("_open" in entry for entry in (site, tunnel, plain, gone, refused)) + + +@pytest.mark.asyncio +async def test_a_request_still_going_is_an_open_entry_until_it_ends(tmp_path): + path = tmp_path / "network.jsonl" + answered = asyncio.Event() + + async def slow(start, headers, body, reader, writer): + await answered.wait() + writer.write(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n") + + async with Upstream(slow) as upstream: + config, issued = _config(tmp_path, upstream.port) + async with _proxy(config, network=NetworkLog.open(str(path))) as port: + asked = asyncio.create_task(_over_tls(port, issued, b"GET /slow HTTP/1.1\r\nHost: notes.test\r\n\r\n")) + for _ in range(100): + if path.stat().st_size: + break + await asyncio.sleep(0.02) + [going] = _entries(path) + answered.set() + await asyncio.wait_for(asked, 5) + + [done] = _entries(path) + assert (going["_open"], going["response"]["status"]) == (True, 0) + assert "_open" not in done and done["response"]["status"] == 200 + + +def test_the_site_proxy_numbers_after_the_last_line_the_file_holds_and_ends_a_cut_one(tmp_path): + path = tmp_path / "network.jsonl" + path.write_bytes(b'{"seq":1,"at":"t","end":{"id":"a"}}\n{"seq":7,"at":"t","end":{"id":"b"}}\n{"seq":8,"at":') + log = NetworkLog.open(str(path)) + log.write("end", {"id": "c", "ended": "t"}) + + assert [line["seq"] for line in NetworkReader(path).every_line()] == [1, 7, 8] + assert path.read_bytes().splitlines()[2] == b'{"seq":8,"at":' + + +def test_the_site_proxy_opens_its_log_readable_by_all_and_never_through_a_link(tmp_path): + path, elsewhere = tmp_path / "network.jsonl", tmp_path / "elsewhere" + previous = os.umask(0o077) + try: + NetworkLog.open(str(path)).write("end", {"id": "a"}) + finally: + os.umask(previous) + elsewhere.write_text("") + (tmp_path / "linked.jsonl").symlink_to(elsewhere) + + assert path.stat().st_mode & 0o777 == 0o644 + with pytest.raises(OSError): + NetworkLog.open(str(tmp_path / "linked.jsonl")) + assert elsewhere.read_text() == "" + + +def test_the_site_proxy_logs_a_write_it_couldnt_make_and_carries_on(tmp_path, capsys): + log = NetworkLog.open(str(tmp_path / "network.jsonl")) + log.file.close() + log.write("end", {"id": "a"}) + + [logged] = [json.loads(line) for line in capsys.readouterr().err.splitlines() if "network log" in line] + assert logged["error"]["connection"] == "network log" + + +def test_the_stand_in_keeps_each_connection_and_each_one_that_failed(monkeypatch, tmp_path): + path = tmp_path / "network.jsonl" + with _serving(monkeypatch, names={"nowhere.test": []}) as proxy: + module = proxy.module + monkeypatch.setattr(module, "NETWORK", module.NetworkLog(str(path))) + _ask(proxy.port, b"CONNECT site.test:443 HTTP/1.1\r\nHost: site.test:443\r\n\r\n", + then=b"GET / HTTP/1.1\r\nHost: site.test\r\n\r\n") + _ask(proxy.port, b"GET http://site.test/a?b=1 HTTP/1.1\r\nHost: site.test\r\n\r\n") + _ask(proxy.port, b"CONNECT nowhere.test:443 HTTP/1.1\r\nHost: nowhere.test:443\r\n\r\n") + + lines = NetworkReader(path).every_line() + assert [line["seq"] for line in lines] == [1, 2, 3, 4, 5] + assert [next(kind for kind in ("open", "end") if kind in line) for line in lines] == [ + "open", "end", "open", "end", "end"] + tunnel, forwarded, failed = har(lines)["log"]["entries"] + assert (tunnel["request"]["method"], tunnel["request"]["url"], tunnel["serverIPAddress"]) == ( + "CONNECT", "https://site.test:443/", GITHUB) + assert tunnel["_tunnel"]["bytes_sent"] > 0 and tunnel["_tunnel"]["bytes_received"] > 0 + assert (forwarded["request"]["method"], forwarded["request"]["url"]) == ("GET", "http://site.test/a?b=1") + assert failed["_failed"] == {"reason": "doesn't resolve", "error": "gaierror: [Errno -2] Name or service not known"} + assert failed["request"]["url"] == "https://nowhere.test:443/" + + +def test_the_stand_in_numbers_after_the_last_line_the_file_holds_and_ends_a_cut_one(tmp_path): + path = tmp_path / "network.jsonl" + path.write_bytes(b'{"seq":4,"at":"t","end":{"id":"a"}}\n{"seq":5,"at":') + module = _stand_in() + module.NetworkLog(str(path)).write("end", {"id": "b"}) + + assert [line["seq"] for line in NetworkReader(path).every_line()] == [4, 5] + + + +async def _live(handle): + """A live host on loopback, each connection given to ``handle(reader, writer)``.""" + server = await asyncio.start_server(handle, "127.0.0.1", 0) + return server, server.sockets[0].getsockname()[1] + + +async def _head(reader) -> bytes: + return await asyncio.wait_for(reader.readuntil(b"\r\n\r\n"), 5) + + +@pytest.mark.asyncio +async def test_a_live_host_s_100_continue_reaches_the_browser_before_it_sends_its_body(tmp_path, monkeypatch): + path = tmp_path / "network.jsonl" + + async def handle(reader, writer): + await reader.readuntil(b"\r\n\r\n") + writer.write(b"HTTP/1.1 100 Continue\r\n\r\n") + await writer.drain() + body = await reader.readexactly(5) + writer.write(b"HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nContent-Length: 5\r\n\r\n" + body) + await writer.drain() + writer.close() + + server, port = await _live(handle) + async with Upstream() as upstream: + config, _ = _config(tmp_path, upstream.port) + config.direct = ["live.test"] + with _live_hosts_at(monkeypatch, port): + async with _forward_port(config, network=NetworkLog.open(str(path))) as proxy: + reader, writer = await asyncio.open_connection("127.0.0.1", proxy) + writer.write(b"POST http://live.test/up HTTP/1.1\r\nHost: live.test\r\nContent-Length: 5\r\n" + b"Expect: 100-continue\r\n\r\n") + await writer.drain() + go_on = await _head(reader) + writer.write(b"hello") + answered = await _head(reader) + echoed = await reader.readexactly(5) + writer.close() + await _ended(path) + server.close() + + [entry] = _entries(path) + assert go_on.startswith(b"HTTP/1.1 100 ") and answered.startswith(b"HTTP/1.1 200 ") and echoed == b"hello" + assert (entry["response"]["status"], entry["request"]["postData"]["text"], + entry["response"]["content"]["text"]) == (200, "hello", "hello") + + +@pytest.mark.asyncio +async def test_a_live_host_that_turns_an_upload_down_early_is_answered_at_once(tmp_path, monkeypatch): + path = tmp_path / "network.jsonl" + done = asyncio.Event() + + async def handle(reader, writer): + await reader.readuntil(b"\r\n\r\n") + writer.write(b"HTTP/1.1 413 Payload Too Large\r\nContent-Length: 0\r\n\r\n") + await writer.drain() + await done.wait() + writer.close() + + server, port = await _live(handle) + async with Upstream() as upstream: + config, _ = _config(tmp_path, upstream.port) + config.direct = ["live.test"] + with _live_hosts_at(monkeypatch, port): + async with _forward_port(config, network=NetworkLog.open(str(path))) as proxy: + reader, writer = await asyncio.open_connection("127.0.0.1", proxy) + writer.write(b"POST http://live.test/up HTTP/1.1\r\nHost: live.test\r\nContent-Length: %d\r\n\r\n" + % (64 << 20)) + sending = asyncio.ensure_future(_send_slowly(writer, 64 << 20)) + answered = await _head(reader) + sending.cancel() + writer.close() + await _ended(path) + done.set() + server.close() + + [entry] = _entries(path) + assert answered.startswith(b"HTTP/1.1 413 ") + assert entry["response"]["status"] == 413 and entry["request"]["bodySize"] < 64 << 20 + + +async def _send_slowly(writer, size: int) -> None: + for _ in range(size // 65536): + writer.write(b"x" * 65536) + await writer.drain() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("answer", [b"HTTP/1.1 200 OK\r\nNoColonHere\r\n\r\nhello", + b"HTTP/1.1 200 OK\nContent-Length: 5\n\nhello"], ids=["no-colon", "bare-lf"]) +async def test_a_live_host_s_answer_the_proxy_cant_read_is_relayed_as_it_came(tmp_path, monkeypatch, answer): + path = tmp_path / "network.jsonl" + + async def handle(reader, writer): + await reader.readuntil(b"\r\n\r\n") + writer.write(answer) + await writer.drain() + writer.close() + + server, port = await _live(handle) + async with Upstream() as upstream: + config, _ = _config(tmp_path, upstream.port) + config.direct = ["live.test"] + with _live_hosts_at(monkeypatch, port): + async with _forward_port(config, network=NetworkLog.open(str(path))) as proxy: + reader, writer = await asyncio.open_connection("127.0.0.1", proxy) + writer.write(b"GET http://live.test/ HTTP/1.1\r\nHost: live.test\r\n\r\n") + got = await asyncio.wait_for(reader.read(), 5) + writer.close() + await _ended(path) + server.close() + + [entry] = _entries(path) + assert got == answer + assert entry["_unparsed"]["reason"] == "an answer the proxy can't read as HTTP" + + +def test_with_no_network_log_a_body_is_counted_but_neither_kept_nor_hashed(): + capture = site_proxy.SiteProxy(site_proxy.Config(user=None, sites=[]))._capture(1 << 20) + capture.add(b"x" * 4096) + + assert capture.record() == {"size": 4096} and not capture.data and capture.digest is None + + +def test_the_stand_in_turns_down_a_bad_content_length_before_it_connects(monkeypatch, tmp_path): + path = tmp_path / "network.jsonl" + with _serving(monkeypatch) as proxy: + monkeypatch.setattr(proxy.module, "NETWORK", proxy.module.NetworkLog(str(path))) + answer = _ask(proxy.port, b"POST http://site.test/a HTTP/1.1\r\nHost: site.test\r\nContent-Length: abc\r\n\r\n") + + assert answer.startswith(b"HTTP/1.1 400 ") and proxy.opened == [] + assert NetworkReader(path).every_line() == [] diff --git a/tst/unit/env/envs/test_computer_site_proxy.py b/tst/unit/env/envs/test_computer_site_proxy.py index 3e23bd6a0..5fd2000a2 100644 --- a/tst/unit/env/envs/test_computer_site_proxy.py +++ b/tst/unit/env/envs/test_computer_site_proxy.py @@ -83,8 +83,8 @@ async def _body(reader, headers): @contextlib.asynccontextmanager -async def _proxy(config, scheme="https"): - proxy = SiteProxy(config) +async def _proxy(config, scheme="https", network=None): + proxy = SiteProxy(config, network=network) context = site_proxy.tls_context(config) if scheme == "https" else None server = await asyncio.start_server(lambda r, w: proxy.handle(r, w, scheme), "127.0.0.1", 0, ssl=context) try: @@ -94,8 +94,8 @@ async def _proxy(config, scheme="https"): @contextlib.asynccontextmanager -async def _forward_port(config): - proxy = SiteProxy(config, tls=site_proxy.tls_context(config)) +async def _forward_port(config, network=None): + proxy = SiteProxy(config, tls=site_proxy.tls_context(config), network=network) server = await asyncio.start_server(proxy.handle_forward, "127.0.0.1", 0) try: yield server.sockets[0].getsockname()[1] @@ -1225,5 +1225,6 @@ def test_the_proxy_is_standard_library_alone_so_the_bases_own_python_runs_it(): imported = {line.split()[1].split(".")[0] for line in source.splitlines() if line.startswith(("import ", "from ")) and not line.startswith("from __future__")} - assert imported <= {"asyncio", "base64", "hashlib", "ipaddress", "json", "os", "pwd", "re", "shutil", "socket", "ssl", - "struct", "subprocess", "sys", "time", "urllib", "dataclasses", "pathlib", "typing"} + assert imported <= {"asyncio", "base64", "datetime", "hashlib", "ipaddress", "itertools", "json", "os", "pwd", "re", + "shutil", "socket", "ssl", "struct", "subprocess", "sys", "time", "urllib", "dataclasses", + "pathlib", "typing"} diff --git a/tst/unit/env/envs/test_computer_sites.py b/tst/unit/env/envs/test_computer_sites.py index 5815c8c71..bf90a5485 100644 --- a/tst/unit/env/envs/test_computer_sites.py +++ b/tst/unit/env/envs/test_computer_sites.py @@ -663,7 +663,8 @@ def test_the_trust_a_routing_linux_computer_boots_with_is_the_local_providers_so def test_a_browser_alone_routing_sites_is_named_the_forward_port_and_a_computer_with_a_display_is_named_none(): - assert sites.browser_environment("none") == {"AGENTENV_SITE_PROXY": "http://127.0.0.1:3129"} + assert sites.browser_environment("none") == {"AGENTENV_SITE_PROXY": "http://127.0.0.1:3129", + "AGENTENV_NETWORK_LOG": "/etc/agentenv/sites/network.jsonl"} assert sites.browser_environment("x11") == {} diff --git a/tst/unit/env/envs/test_computer_stand_in_proxy.py b/tst/unit/env/envs/test_computer_stand_in_proxy.py index 9671ad635..c74f9a444 100644 --- a/tst/unit/env/envs/test_computer_stand_in_proxy.py +++ b/tst/unit/env/envs/test_computer_stand_in_proxy.py @@ -93,7 +93,7 @@ def create_connection(address, *args, **kwargs): threading.Thread(target=server.serve_forever, daemon=True).start() try: yield SimpleNamespace(port=server.server_address[1], opened=opened, read=read, - upstream_port=upstream.getsockname()[1]) + upstream_port=upstream.getsockname()[1], module=module) finally: server.shutdown() upstream.close() diff --git a/uv.lock b/uv.lock index dd63b2605..6d54931a2 100644 --- a/uv.lock +++ b/uv.lock @@ -3,8 +3,8 @@ revision = 5 requires-python = ">=3.11.4" resolution-markers = [ "python_full_version >= '3.15' and sys_platform == 'win32'", - "python_full_version == '3.14.*' and sys_platform == 'win32'", "python_full_version >= '3.15' and sys_platform != 'win32'", + "python_full_version == '3.14.*' and sys_platform == 'win32'", "python_full_version == '3.14.*' and sys_platform != 'win32'", "python_full_version >= '3.13.14' and python_full_version < '3.14' and sys_platform == 'win32'", "python_full_version >= '3.13.14' and python_full_version < '3.14' and sys_platform != 'win32'", @@ -103,6 +103,7 @@ dev = [ { name = "boto3" }, { name = "botocore" }, { name = "boxd" }, + { name = "brotli" }, { name = "cwsandbox" }, { name = "datasets" }, { name = "daytona" }, @@ -137,6 +138,7 @@ dev = [ { name = "urllib3" }, { name = "vercel-sandbox" }, { name = "websockets" }, + { name = "zstandard" }, ] gcp = [ { name = "google-api-core" }, @@ -216,6 +218,7 @@ requires-dist = [ { name = "boto3", marker = "extra == 'aws'", specifier = ">=1.35.0" }, { name = "botocore", marker = "extra == 'aws'", specifier = ">=1.35.0" }, { name = "boxd", marker = "extra == 'boxd'", specifier = "~=0.2.17" }, + { name = "brotli", marker = "extra == 'dev'", specifier = ">=1.2" }, { name = "certifi", specifier = ">=2024.2.2" }, { name = "click", specifier = ">=8.0.0" }, { name = "cryptography", specifier = ">=42" }, @@ -276,6 +279,7 @@ requires-dist = [ { name = "uvicorn", specifier = ">=0.47" }, { name = "vercel-sandbox", marker = "extra == 'vercel'", specifier = ">=0.7.0,<0.8" }, { name = "websockets", marker = "extra == 'dev'", specifier = ">=15" }, + { name = "zstandard", marker = "extra == 'dev'", specifier = ">=0.23" }, ] provides-extras = ["openenv", "harness", "trl", "nemo-gym", "verl", "skyrl", "miles", "slime", "aws", "gcp", "sail", "vercel", "tensorlake", "tenki", "boxd", "daytona", "boat", "coreweave", "dev", "smol"] @@ -298,9 +302,11 @@ agent = [ { name = "uvicorn" }, ] app = [ + { name = "brotli" }, { name = "mcp" }, { name = "uvicorn" }, { name = "websockets" }, + { name = "zstandard" }, ] computer = [ { name = "av", version = "18.1.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, @@ -314,6 +320,7 @@ dev = [ { name = "a2a-sdk", extra = ["http-server"] }, { name = "av", version = "18.1.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, { name = "av", version = "19.0.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, + { name = "brotli" }, { name = "fastmcp" }, { name = "mcp" }, { name = "pillow" }, @@ -322,6 +329,7 @@ dev = [ { name = "python-multipart" }, { name = "regex" }, { name = "websockets" }, + { name = "zstandard" }, ] macos = [ { name = "pyobjc-framework-quartz", marker = "sys_platform == 'darwin'" }, @@ -341,6 +349,8 @@ requires-dist = [ { name = "a2a-sdk", extras = ["http-server"], marker = "extra == 'dev'", specifier = ">=0.3.26,<0.4" }, { name = "av", marker = "extra == 'computer'", specifier = ">=17" }, { name = "av", marker = "extra == 'dev'", specifier = ">=17" }, + { name = "brotli", marker = "extra == 'app'", specifier = ">=1.2" }, + { name = "brotli", marker = "extra == 'dev'", specifier = ">=1.2" }, { name = "fastmcp", marker = "extra == 'dev'", specifier = ">=3.4,<4" }, { name = "httpx", specifier = ">=0.27" }, { name = "mcp", marker = "extra == 'agent'", specifier = ">=1.25,<2" }, @@ -368,6 +378,8 @@ requires-dist = [ { name = "websockets", marker = "extra == 'app'", specifier = ">=13" }, { name = "websockets", marker = "extra == 'computer'", specifier = ">=13" }, { name = "websockets", marker = "extra == 'dev'", specifier = ">=13" }, + { name = "zstandard", marker = "extra == 'app'", specifier = ">=0.23" }, + { name = "zstandard", marker = "extra == 'dev'", specifier = ">=0.23" }, ] provides-extras = ["agent", "app", "computer", "macos", "x11", "windows", "dev"] @@ -683,8 +695,8 @@ version = "4.15.1" source = { registry = "https://pypi.org/simple" } resolution-markers = [ "python_full_version >= '3.15' and sys_platform == 'win32'", - "python_full_version == '3.14.*' and sys_platform == 'win32'", "python_full_version >= '3.15' and sys_platform != 'win32'", + "python_full_version == '3.14.*' and sys_platform == 'win32'", "python_full_version == '3.14.*' and sys_platform != 'win32'", "python_full_version >= '3.13.14' and python_full_version < '3.14' and sys_platform == 'win32'", "python_full_version >= '3.13.14' and python_full_version < '3.14' and sys_platform != 'win32'", @@ -824,8 +836,8 @@ version = "19.0.2" source = { registry = "https://pypi.org/simple" } resolution-markers = [ "python_full_version >= '3.15' and sys_platform == 'win32'", - "python_full_version == '3.14.*' and sys_platform == 'win32'", "python_full_version >= '3.15' and sys_platform != 'win32'", + "python_full_version == '3.14.*' and sys_platform == 'win32'", "python_full_version == '3.14.*' and sys_platform != 'win32'", "python_full_version >= '3.13.14' and python_full_version < '3.14' and sys_platform == 'win32'", "python_full_version >= '3.13.14' and python_full_version < '3.14' and sys_platform != 'win32'", @@ -4226,8 +4238,8 @@ version = "2.5.4" source = { registry = "https://pypi.org/simple" } resolution-markers = [ "python_full_version >= '3.15' and sys_platform == 'win32'", - "python_full_version == '3.14.*' and sys_platform == 'win32'", "python_full_version >= '3.15' and sys_platform != 'win32'", + "python_full_version == '3.14.*' and sys_platform == 'win32'", "python_full_version == '3.14.*' and sys_platform != 'win32'", "python_full_version >= '3.13.14' and python_full_version < '3.14' and sys_platform == 'win32'", "python_full_version >= '3.13.14' and python_full_version < '3.14' and sys_platform != 'win32'", @@ -4402,8 +4414,8 @@ version = "2.44.0" source = { registry = "https://pypi.org/simple" } resolution-markers = [ "python_full_version >= '3.15' and sys_platform == 'win32'", - "python_full_version == '3.14.*' and sys_platform == 'win32'", "python_full_version >= '3.15' and sys_platform != 'win32'", + "python_full_version == '3.14.*' and sys_platform == 'win32'", "python_full_version == '3.14.*' and sys_platform != 'win32'", "python_full_version >= '3.13.14' and python_full_version < '3.14' and sys_platform == 'win32'", "python_full_version >= '3.13.14' and python_full_version < '3.14' and sys_platform != 'win32'", @@ -4707,8 +4719,8 @@ version = "26.3" source = { registry = "https://pypi.org/simple" } resolution-markers = [ "python_full_version >= '3.15' and sys_platform == 'win32'", - "python_full_version == '3.14.*' and sys_platform == 'win32'", "python_full_version >= '3.15' and sys_platform != 'win32'", + "python_full_version == '3.14.*' and sys_platform == 'win32'", "python_full_version == '3.14.*' and sys_platform != 'win32'", "python_full_version >= '3.13.14' and python_full_version < '3.14' and sys_platform == 'win32'", "python_full_version >= '3.13.14' and python_full_version < '3.14' and sys_platform != 'win32'", @@ -5234,8 +5246,8 @@ version = "25.0.1" source = { registry = "https://pypi.org/simple" } resolution-markers = [ "python_full_version >= '3.15' and sys_platform == 'win32'", - "python_full_version == '3.14.*' and sys_platform == 'win32'", "python_full_version >= '3.15' and sys_platform != 'win32'", + "python_full_version == '3.14.*' and sys_platform == 'win32'", "python_full_version == '3.14.*' and sys_platform != 'win32'", "python_full_version >= '3.13.14' and python_full_version < '3.14' and sys_platform == 'win32'", "python_full_version >= '3.13.14' and python_full_version < '3.14' and sys_platform != 'win32'", @@ -8230,3 +8242,77 @@ sdist = { url = "https://files.pythonhosted.org/packages/b9/d8/eab98a517c14134c0 wheels = [ { url = "https://files.pythonhosted.org/packages/3a/13/547360d81e6d88d58492968ffda9f9542854f11310ee556fef14260cc886/zipp-4.1.0-py3-none-any.whl", hash = "sha256:25ad4e16390cd314347dd8f1de67a2ac538ae658ed4ab9db16029c07c188e97f", size = 10238, upload-time = "2026-05-18T20:08:57.045Z" }, ] + +[[package]] +name = "zstandard" +version = "0.25.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/fd/aa/3e0508d5a5dd96529cdc5a97011299056e14c6505b678fd58938792794b1/zstandard-0.25.0.tar.gz", hash = "sha256:7713e1179d162cf5c7906da876ec2ccb9c3a9dcbdffef0cc7f70c3667a205f0b", size = 711513, upload-time = "2025-09-14T22:15:54.002Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2a/83/c3ca27c363d104980f1c9cee1101cc8ba724ac8c28a033ede6aab89585b1/zstandard-0.25.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:933b65d7680ea337180733cf9e87293cc5500cc0eb3fc8769f4d3c88d724ec5c", size = 795254, upload-time = "2025-09-14T22:16:26.137Z" }, + { url = "https://files.pythonhosted.org/packages/ac/4d/e66465c5411a7cf4866aeadc7d108081d8ceba9bc7abe6b14aa21c671ec3/zstandard-0.25.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:a3f79487c687b1fc69f19e487cd949bf3aae653d181dfb5fde3bf6d18894706f", size = 640559, upload-time = "2025-09-14T22:16:27.973Z" }, + { url = "https://files.pythonhosted.org/packages/12/56/354fe655905f290d3b147b33fe946b0f27e791e4b50a5f004c802cb3eb7b/zstandard-0.25.0-cp311-cp311-manylinux2010_i686.manylinux2014_i686.manylinux_2_12_i686.manylinux_2_17_i686.whl", hash = "sha256:0bbc9a0c65ce0eea3c34a691e3c4b6889f5f3909ba4822ab385fab9057099431", size = 5348020, upload-time = "2025-09-14T22:16:29.523Z" }, + { url = "https://files.pythonhosted.org/packages/3b/13/2b7ed68bd85e69a2069bcc72141d378f22cae5a0f3b353a2c8f50ef30c1b/zstandard-0.25.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:01582723b3ccd6939ab7b3a78622c573799d5d8737b534b86d0e06ac18dbde4a", size = 5058126, upload-time = "2025-09-14T22:16:31.811Z" }, + { url = "https://files.pythonhosted.org/packages/c9/dd/fdaf0674f4b10d92cb120ccff58bbb6626bf8368f00ebfd2a41ba4a0dc99/zstandard-0.25.0-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:5f1ad7bf88535edcf30038f6919abe087f606f62c00a87d7e33e7fc57cb69fcc", size = 5405390, upload-time = "2025-09-14T22:16:33.486Z" }, + { url = "https://files.pythonhosted.org/packages/0f/67/354d1555575bc2490435f90d67ca4dd65238ff2f119f30f72d5cde09c2ad/zstandard-0.25.0-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:06acb75eebeedb77b69048031282737717a63e71e4ae3f77cc0c3b9508320df6", size = 5452914, upload-time = "2025-09-14T22:16:35.277Z" }, + { url = "https://files.pythonhosted.org/packages/bb/1f/e9cfd801a3f9190bf3e759c422bbfd2247db9d7f3d54a56ecde70137791a/zstandard-0.25.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:9300d02ea7c6506f00e627e287e0492a5eb0371ec1670ae852fefffa6164b072", size = 5559635, upload-time = "2025-09-14T22:16:37.141Z" }, + { url = "https://files.pythonhosted.org/packages/21/88/5ba550f797ca953a52d708c8e4f380959e7e3280af029e38fbf47b55916e/zstandard-0.25.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:bfd06b1c5584b657a2892a6014c2f4c20e0db0208c159148fa78c65f7e0b0277", size = 5048277, upload-time = "2025-09-14T22:16:38.807Z" }, + { url = "https://files.pythonhosted.org/packages/46/c0/ca3e533b4fa03112facbe7fbe7779cb1ebec215688e5df576fe5429172e0/zstandard-0.25.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:f373da2c1757bb7f1acaf09369cdc1d51d84131e50d5fa9863982fd626466313", size = 5574377, upload-time = "2025-09-14T22:16:40.523Z" }, + { url = "https://files.pythonhosted.org/packages/12/9b/3fb626390113f272abd0799fd677ea33d5fc3ec185e62e6be534493c4b60/zstandard-0.25.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:6c0e5a65158a7946e7a7affa6418878ef97ab66636f13353b8502d7ea03c8097", size = 4961493, upload-time = "2025-09-14T22:16:43.3Z" }, + { url = "https://files.pythonhosted.org/packages/cb/d3/23094a6b6a4b1343b27ae68249daa17ae0651fcfec9ed4de09d14b940285/zstandard-0.25.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:c8e167d5adf59476fa3e37bee730890e389410c354771a62e3c076c86f9f7778", size = 5269018, upload-time = "2025-09-14T22:16:45.292Z" }, + { url = "https://files.pythonhosted.org/packages/8c/a7/bb5a0c1c0f3f4b5e9d5b55198e39de91e04ba7c205cc46fcb0f95f0383c1/zstandard-0.25.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:98750a309eb2f020da61e727de7d7ba3c57c97cf6213f6f6277bb7fb42a8e065", size = 5443672, upload-time = "2025-09-14T22:16:47.076Z" }, + { url = "https://files.pythonhosted.org/packages/27/22/503347aa08d073993f25109c36c8d9f029c7d5949198050962cb568dfa5e/zstandard-0.25.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:22a086cff1b6ceca18a8dd6096ec631e430e93a8e70a9ca5efa7561a00f826fa", size = 5822753, upload-time = "2025-09-14T22:16:49.316Z" }, + { url = "https://files.pythonhosted.org/packages/e2/be/94267dc6ee64f0f8ba2b2ae7c7a2df934a816baaa7291db9e1aa77394c3c/zstandard-0.25.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:72d35d7aa0bba323965da807a462b0966c91608ef3a48ba761678cb20ce5d8b7", size = 5366047, upload-time = "2025-09-14T22:16:51.328Z" }, + { url = "https://files.pythonhosted.org/packages/7b/a3/732893eab0a3a7aecff8b99052fecf9f605cf0fb5fb6d0290e36beee47a4/zstandard-0.25.0-cp311-cp311-win32.whl", hash = "sha256:f5aeea11ded7320a84dcdd62a3d95b5186834224a9e55b92ccae35d21a8b63d4", size = 436484, upload-time = "2025-09-14T22:16:55.005Z" }, + { url = "https://files.pythonhosted.org/packages/43/a3/c6155f5c1cce691cb80dfd38627046e50af3ee9ddc5d0b45b9b063bfb8c9/zstandard-0.25.0-cp311-cp311-win_amd64.whl", hash = "sha256:daab68faadb847063d0c56f361a289c4f268706b598afbf9ad113cbe5c38b6b2", size = 506183, upload-time = "2025-09-14T22:16:52.753Z" }, + { url = "https://files.pythonhosted.org/packages/8c/3e/8945ab86a0820cc0e0cdbf38086a92868a9172020fdab8a03ac19662b0e5/zstandard-0.25.0-cp311-cp311-win_arm64.whl", hash = "sha256:22a06c5df3751bb7dc67406f5374734ccee8ed37fc5981bf1ad7041831fa1137", size = 462533, upload-time = "2025-09-14T22:16:53.878Z" }, + { url = "https://files.pythonhosted.org/packages/82/fc/f26eb6ef91ae723a03e16eddb198abcfce2bc5a42e224d44cc8b6765e57e/zstandard-0.25.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7b3c3a3ab9daa3eed242d6ecceead93aebbb8f5f84318d82cee643e019c4b73b", size = 795738, upload-time = "2025-09-14T22:16:56.237Z" }, + { url = "https://files.pythonhosted.org/packages/aa/1c/d920d64b22f8dd028a8b90e2d756e431a5d86194caa78e3819c7bf53b4b3/zstandard-0.25.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:913cbd31a400febff93b564a23e17c3ed2d56c064006f54efec210d586171c00", size = 640436, upload-time = "2025-09-14T22:16:57.774Z" }, + { url = "https://files.pythonhosted.org/packages/53/6c/288c3f0bd9fcfe9ca41e2c2fbfd17b2097f6af57b62a81161941f09afa76/zstandard-0.25.0-cp312-cp312-manylinux2010_i686.manylinux2014_i686.manylinux_2_12_i686.manylinux_2_17_i686.whl", hash = "sha256:011d388c76b11a0c165374ce660ce2c8efa8e5d87f34996aa80f9c0816698b64", size = 5343019, upload-time = "2025-09-14T22:16:59.302Z" }, + { url = "https://files.pythonhosted.org/packages/1e/15/efef5a2f204a64bdb5571e6161d49f7ef0fffdbca953a615efbec045f60f/zstandard-0.25.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:6dffecc361d079bb48d7caef5d673c88c8988d3d33fb74ab95b7ee6da42652ea", size = 5063012, upload-time = "2025-09-14T22:17:01.156Z" }, + { url = "https://files.pythonhosted.org/packages/b7/37/a6ce629ffdb43959e92e87ebdaeebb5ac81c944b6a75c9c47e300f85abdf/zstandard-0.25.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:7149623bba7fdf7e7f24312953bcf73cae103db8cae49f8154dd1eadc8a29ecb", size = 5394148, upload-time = "2025-09-14T22:17:03.091Z" }, + { url = "https://files.pythonhosted.org/packages/e3/79/2bf870b3abeb5c070fe2d670a5a8d1057a8270f125ef7676d29ea900f496/zstandard-0.25.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:6a573a35693e03cf1d67799fd01b50ff578515a8aeadd4595d2a7fa9f3ec002a", size = 5451652, upload-time = "2025-09-14T22:17:04.979Z" }, + { url = "https://files.pythonhosted.org/packages/53/60/7be26e610767316c028a2cbedb9a3beabdbe33e2182c373f71a1c0b88f36/zstandard-0.25.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:5a56ba0db2d244117ed744dfa8f6f5b366e14148e00de44723413b2f3938a902", size = 5546993, upload-time = "2025-09-14T22:17:06.781Z" }, + { url = "https://files.pythonhosted.org/packages/85/c7/3483ad9ff0662623f3648479b0380d2de5510abf00990468c286c6b04017/zstandard-0.25.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:10ef2a79ab8e2974e2075fb984e5b9806c64134810fac21576f0668e7ea19f8f", size = 5046806, upload-time = "2025-09-14T22:17:08.415Z" }, + { url = "https://files.pythonhosted.org/packages/08/b3/206883dd25b8d1591a1caa44b54c2aad84badccf2f1de9e2d60a446f9a25/zstandard-0.25.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:aaf21ba8fb76d102b696781bddaa0954b782536446083ae3fdaa6f16b25a1c4b", size = 5576659, upload-time = "2025-09-14T22:17:10.164Z" }, + { url = "https://files.pythonhosted.org/packages/9d/31/76c0779101453e6c117b0ff22565865c54f48f8bd807df2b00c2c404b8e0/zstandard-0.25.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:1869da9571d5e94a85a5e8d57e4e8807b175c9e4a6294e3b66fa4efb074d90f6", size = 4953933, upload-time = "2025-09-14T22:17:11.857Z" }, + { url = "https://files.pythonhosted.org/packages/18/e1/97680c664a1bf9a247a280a053d98e251424af51f1b196c6d52f117c9720/zstandard-0.25.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:809c5bcb2c67cd0ed81e9229d227d4ca28f82d0f778fc5fea624a9def3963f91", size = 5268008, upload-time = "2025-09-14T22:17:13.627Z" }, + { url = "https://files.pythonhosted.org/packages/1e/73/316e4010de585ac798e154e88fd81bb16afc5c5cb1a72eeb16dd37e8024a/zstandard-0.25.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:f27662e4f7dbf9f9c12391cb37b4c4c3cb90ffbd3b1fb9284dadbbb8935fa708", size = 5433517, upload-time = "2025-09-14T22:17:16.103Z" }, + { url = "https://files.pythonhosted.org/packages/5b/60/dd0f8cfa8129c5a0ce3ea6b7f70be5b33d2618013a161e1ff26c2b39787c/zstandard-0.25.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:99c0c846e6e61718715a3c9437ccc625de26593fea60189567f0118dc9db7512", size = 5814292, upload-time = "2025-09-14T22:17:17.827Z" }, + { url = "https://files.pythonhosted.org/packages/fc/5f/75aafd4b9d11b5407b641b8e41a57864097663699f23e9ad4dbb91dc6bfe/zstandard-0.25.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:474d2596a2dbc241a556e965fb76002c1ce655445e4e3bf38e5477d413165ffa", size = 5360237, upload-time = "2025-09-14T22:17:19.954Z" }, + { url = "https://files.pythonhosted.org/packages/ff/8d/0309daffea4fcac7981021dbf21cdb2e3427a9e76bafbcdbdf5392ff99a4/zstandard-0.25.0-cp312-cp312-win32.whl", hash = "sha256:23ebc8f17a03133b4426bcc04aabd68f8236eb78c3760f12783385171b0fd8bd", size = 436922, upload-time = "2025-09-14T22:17:24.398Z" }, + { url = "https://files.pythonhosted.org/packages/79/3b/fa54d9015f945330510cb5d0b0501e8253c127cca7ebe8ba46a965df18c5/zstandard-0.25.0-cp312-cp312-win_amd64.whl", hash = "sha256:ffef5a74088f1e09947aecf91011136665152e0b4b359c42be3373897fb39b01", size = 506276, upload-time = "2025-09-14T22:17:21.429Z" }, + { url = "https://files.pythonhosted.org/packages/ea/6b/8b51697e5319b1f9ac71087b0af9a40d8a6288ff8025c36486e0c12abcc4/zstandard-0.25.0-cp312-cp312-win_arm64.whl", hash = "sha256:181eb40e0b6a29b3cd2849f825e0fa34397f649170673d385f3598ae17cca2e9", size = 462679, upload-time = "2025-09-14T22:17:23.147Z" }, + { url = "https://files.pythonhosted.org/packages/35/0b/8df9c4ad06af91d39e94fa96cc010a24ac4ef1378d3efab9223cc8593d40/zstandard-0.25.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:ec996f12524f88e151c339688c3897194821d7f03081ab35d31d1e12ec975e94", size = 795735, upload-time = "2025-09-14T22:17:26.042Z" }, + { url = "https://files.pythonhosted.org/packages/3f/06/9ae96a3e5dcfd119377ba33d4c42a7d89da1efabd5cb3e366b156c45ff4d/zstandard-0.25.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:a1a4ae2dec3993a32247995bdfe367fc3266da832d82f8438c8570f989753de1", size = 640440, upload-time = "2025-09-14T22:17:27.366Z" }, + { url = "https://files.pythonhosted.org/packages/d9/14/933d27204c2bd404229c69f445862454dcc101cd69ef8c6068f15aaec12c/zstandard-0.25.0-cp313-cp313-manylinux2010_i686.manylinux2014_i686.manylinux_2_12_i686.manylinux_2_17_i686.whl", hash = "sha256:e96594a5537722fdfb79951672a2a63aec5ebfb823e7560586f7484819f2a08f", size = 5343070, upload-time = "2025-09-14T22:17:28.896Z" }, + { url = "https://files.pythonhosted.org/packages/6d/db/ddb11011826ed7db9d0e485d13df79b58586bfdec56e5c84a928a9a78c1c/zstandard-0.25.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:bfc4e20784722098822e3eee42b8e576b379ed72cca4a7cb856ae733e62192ea", size = 5063001, upload-time = "2025-09-14T22:17:31.044Z" }, + { url = "https://files.pythonhosted.org/packages/db/00/87466ea3f99599d02a5238498b87bf84a6348290c19571051839ca943777/zstandard-0.25.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:457ed498fc58cdc12fc48f7950e02740d4f7ae9493dd4ab2168a47c93c31298e", size = 5394120, upload-time = "2025-09-14T22:17:32.711Z" }, + { url = "https://files.pythonhosted.org/packages/2b/95/fc5531d9c618a679a20ff6c29e2b3ef1d1f4ad66c5e161ae6ff847d102a9/zstandard-0.25.0-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:fd7a5004eb1980d3cefe26b2685bcb0b17989901a70a1040d1ac86f1d898c551", size = 5451230, upload-time = "2025-09-14T22:17:34.41Z" }, + { url = "https://files.pythonhosted.org/packages/63/4b/e3678b4e776db00f9f7b2fe58e547e8928ef32727d7a1ff01dea010f3f13/zstandard-0.25.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:8e735494da3db08694d26480f1493ad2cf86e99bdd53e8e9771b2752a5c0246a", size = 5547173, upload-time = "2025-09-14T22:17:36.084Z" }, + { url = "https://files.pythonhosted.org/packages/4e/d5/ba05ed95c6b8ec30bd468dfeab20589f2cf709b5c940483e31d991f2ca58/zstandard-0.25.0-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:3a39c94ad7866160a4a46d772e43311a743c316942037671beb264e395bdd611", size = 5046736, upload-time = "2025-09-14T22:17:37.891Z" }, + { url = "https://files.pythonhosted.org/packages/50/d5/870aa06b3a76c73eced65c044b92286a3c4e00554005ff51962deef28e28/zstandard-0.25.0-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:172de1f06947577d3a3005416977cce6168f2261284c02080e7ad0185faeced3", size = 5576368, upload-time = "2025-09-14T22:17:40.206Z" }, + { url = "https://files.pythonhosted.org/packages/5d/35/398dc2ffc89d304d59bc12f0fdd931b4ce455bddf7038a0a67733a25f550/zstandard-0.25.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:3c83b0188c852a47cd13ef3bf9209fb0a77fa5374958b8c53aaa699398c6bd7b", size = 4954022, upload-time = "2025-09-14T22:17:41.879Z" }, + { url = "https://files.pythonhosted.org/packages/9a/5c/36ba1e5507d56d2213202ec2b05e8541734af5f2ce378c5d1ceaf4d88dc4/zstandard-0.25.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:1673b7199bbe763365b81a4f3252b8e80f44c9e323fc42940dc8843bfeaf9851", size = 5267889, upload-time = "2025-09-14T22:17:43.577Z" }, + { url = "https://files.pythonhosted.org/packages/70/e8/2ec6b6fb7358b2ec0113ae202647ca7c0e9d15b61c005ae5225ad0995df5/zstandard-0.25.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:0be7622c37c183406f3dbf0cba104118eb16a4ea7359eeb5752f0794882fc250", size = 5433952, upload-time = "2025-09-14T22:17:45.271Z" }, + { url = "https://files.pythonhosted.org/packages/7b/01/b5f4d4dbc59ef193e870495c6f1275f5b2928e01ff5a81fecb22a06e22fb/zstandard-0.25.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:5f5e4c2a23ca271c218ac025bd7d635597048b366d6f31f420aaeb715239fc98", size = 5814054, upload-time = "2025-09-14T22:17:47.08Z" }, + { url = "https://files.pythonhosted.org/packages/b2/e5/fbd822d5c6f427cf158316d012c5a12f233473c2f9c5fe5ab1ae5d21f3d8/zstandard-0.25.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:4f187a0bb61b35119d1926aee039524d1f93aaf38a9916b8c4b78ac8514a0aaf", size = 5360113, upload-time = "2025-09-14T22:17:48.893Z" }, + { url = "https://files.pythonhosted.org/packages/8e/e0/69a553d2047f9a2c7347caa225bb3a63b6d7704ad74610cb7823baa08ed7/zstandard-0.25.0-cp313-cp313-win32.whl", hash = "sha256:7030defa83eef3e51ff26f0b7bfb229f0204b66fe18e04359ce3474ac33cbc09", size = 436936, upload-time = "2025-09-14T22:17:52.658Z" }, + { url = "https://files.pythonhosted.org/packages/d9/82/b9c06c870f3bd8767c201f1edbdf9e8dc34be5b0fbc5682c4f80fe948475/zstandard-0.25.0-cp313-cp313-win_amd64.whl", hash = "sha256:1f830a0dac88719af0ae43b8b2d6aef487d437036468ef3c2ea59c51f9d55fd5", size = 506232, upload-time = "2025-09-14T22:17:50.402Z" }, + { url = "https://files.pythonhosted.org/packages/d4/57/60c3c01243bb81d381c9916e2a6d9e149ab8627c0c7d7abb2d73384b3c0c/zstandard-0.25.0-cp313-cp313-win_arm64.whl", hash = "sha256:85304a43f4d513f5464ceb938aa02c1e78c2943b29f44a750b48b25ac999a049", size = 462671, upload-time = "2025-09-14T22:17:51.533Z" }, + { url = "https://files.pythonhosted.org/packages/3d/5c/f8923b595b55fe49e30612987ad8bf053aef555c14f05bb659dd5dbe3e8a/zstandard-0.25.0-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:e29f0cf06974c899b2c188ef7f783607dbef36da4c242eb6c82dcd8b512855e3", size = 795887, upload-time = "2025-09-14T22:17:54.198Z" }, + { url = "https://files.pythonhosted.org/packages/8d/09/d0a2a14fc3439c5f874042dca72a79c70a532090b7ba0003be73fee37ae2/zstandard-0.25.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:05df5136bc5a011f33cd25bc9f506e7426c0c9b3f9954f056831ce68f3b6689f", size = 640658, upload-time = "2025-09-14T22:17:55.423Z" }, + { url = "https://files.pythonhosted.org/packages/5d/7c/8b6b71b1ddd517f68ffb55e10834388d4f793c49c6b83effaaa05785b0b4/zstandard-0.25.0-cp314-cp314-manylinux2010_i686.manylinux_2_12_i686.manylinux_2_28_i686.whl", hash = "sha256:f604efd28f239cc21b3adb53eb061e2a205dc164be408e553b41ba2ffe0ca15c", size = 5379849, upload-time = "2025-09-14T22:17:57.372Z" }, + { url = "https://files.pythonhosted.org/packages/a4/86/a48e56320d0a17189ab7a42645387334fba2200e904ee47fc5a26c1fd8ca/zstandard-0.25.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:223415140608d0f0da010499eaa8ccdb9af210a543fac54bce15babbcfc78439", size = 5058095, upload-time = "2025-09-14T22:17:59.498Z" }, + { url = "https://files.pythonhosted.org/packages/f8/ad/eb659984ee2c0a779f9d06dbfe45e2dc39d99ff40a319895df2d3d9a48e5/zstandard-0.25.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2e54296a283f3ab5a26fc9b8b5d4978ea0532f37b231644f367aa588930aa043", size = 5551751, upload-time = "2025-09-14T22:18:01.618Z" }, + { url = "https://files.pythonhosted.org/packages/61/b3/b637faea43677eb7bd42ab204dfb7053bd5c4582bfe6b1baefa80ac0c47b/zstandard-0.25.0-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:ca54090275939dc8ec5dea2d2afb400e0f83444b2fc24e07df7fdef677110859", size = 6364818, upload-time = "2025-09-14T22:18:03.769Z" }, + { url = "https://files.pythonhosted.org/packages/31/dc/cc50210e11e465c975462439a492516a73300ab8caa8f5e0902544fd748b/zstandard-0.25.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e09bb6252b6476d8d56100e8147b803befa9a12cea144bbe629dd508800d1ad0", size = 5560402, upload-time = "2025-09-14T22:18:05.954Z" }, + { url = "https://files.pythonhosted.org/packages/c9/ae/56523ae9c142f0c08efd5e868a6da613ae76614eca1305259c3bf6a0ed43/zstandard-0.25.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:a9ec8c642d1ec73287ae3e726792dd86c96f5681eb8df274a757bf62b750eae7", size = 4955108, upload-time = "2025-09-14T22:18:07.68Z" }, + { url = "https://files.pythonhosted.org/packages/98/cf/c899f2d6df0840d5e384cf4c4121458c72802e8bda19691f3b16619f51e9/zstandard-0.25.0-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:a4089a10e598eae6393756b036e0f419e8c1d60f44a831520f9af41c14216cf2", size = 5269248, upload-time = "2025-09-14T22:18:09.753Z" }, + { url = "https://files.pythonhosted.org/packages/1b/c0/59e912a531d91e1c192d3085fc0f6fb2852753c301a812d856d857ea03c6/zstandard-0.25.0-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:f67e8f1a324a900e75b5e28ffb152bcac9fbed1cc7b43f99cd90f395c4375344", size = 5430330, upload-time = "2025-09-14T22:18:11.966Z" }, + { url = "https://files.pythonhosted.org/packages/a0/1d/7e31db1240de2df22a58e2ea9a93fc6e38cc29353e660c0272b6735d6669/zstandard-0.25.0-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:9654dbc012d8b06fc3d19cc825af3f7bf8ae242226df5f83936cb39f5fdc846c", size = 5811123, upload-time = "2025-09-14T22:18:13.907Z" }, + { url = "https://files.pythonhosted.org/packages/f6/49/fac46df5ad353d50535e118d6983069df68ca5908d4d65b8c466150a4ff1/zstandard-0.25.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:4203ce3b31aec23012d3a4cf4a2ed64d12fea5269c49aed5e4c3611b938e4088", size = 5359591, upload-time = "2025-09-14T22:18:16.465Z" }, + { url = "https://files.pythonhosted.org/packages/c2/38/f249a2050ad1eea0bb364046153942e34abba95dd5520af199aed86fbb49/zstandard-0.25.0-cp314-cp314-win32.whl", hash = "sha256:da469dc041701583e34de852d8634703550348d5822e66a0c827d39b05365b12", size = 444513, upload-time = "2025-09-14T22:18:20.61Z" }, + { url = "https://files.pythonhosted.org/packages/3a/43/241f9615bcf8ba8903b3f0432da069e857fc4fd1783bd26183db53c4804b/zstandard-0.25.0-cp314-cp314-win_amd64.whl", hash = "sha256:c19bcdd826e95671065f8692b5a4aa95c52dc7a02a4c5a0cac46deb879a017a2", size = 516118, upload-time = "2025-09-14T22:18:17.849Z" }, + { url = "https://files.pythonhosted.org/packages/f0/ef/da163ce2450ed4febf6467d77ccb4cd52c4c30ab45624bad26ca0a27260c/zstandard-0.25.0-cp314-cp314-win_arm64.whl", hash = "sha256:d7541afd73985c630bafcd6338d2518ae96060075f9463d7dc14cfb33514383d", size = 476940, upload-time = "2025-09-14T22:18:19.088Z" }, +]