From 11c8a8edc8914e9d8f233e50f35e468264e04324 Mon Sep 17 00:00:00 2001 From: Sarah Chitty Date: Sun, 8 Nov 2026 11:45:26 -0800 Subject: [PATCH 01/53] fix(triggers): wait for a slow executor up to its own timeout, not 30 s (FD-5333) An executor holds message/send open until its task ends, so the 30 s read timeout failed every fire whose contact took longer, even though the reply usually still landed. The send now reads for the executor's timeout_seconds (never less than 30 s); connect and task polls keep 30 s. Co-Authored-By: Claude Opus 5.5 --- src/agent_env/env/gateway/triggers.py | 8 ++- tst/unit/env/gateway/triggers_test.py | 86 +++++++++++++++++++++++++++ 2 files changed, 92 insertions(+), 2 deletions(-) diff --git a/src/agent_env/env/gateway/triggers.py b/src/agent_env/env/gateway/triggers.py index 415ff99fc..b2feb4c15 100644 --- a/src/agent_env/env/gateway/triggers.py +++ b/src/agent_env/env/gateway/triggers.py @@ -53,6 +53,8 @@ _DROP = object() # an unresolved `${x?}` used as a whole value: drop the key / element # Real seconds between re-reads of the clock while waiting out virtual time, so a rate change, re-arm or clear is followed. _VIRTUAL_RECHECK_S = 5.0 +# Seconds an executor call waits to connect, and for a task poll's answer. +_EXECUTOR_HTTP_TIMEOUT_S = 30.0 # An allowlist, not a size bound: identifier/metadata keys only, since bodies and addresses are PII. _ECHO_KEYS = frozenset({ "action", "createdTime", "fileName", "kind", "mimeType", "modifiedTime", "name", "role", @@ -1564,8 +1566,10 @@ async def _call_executor(self, executor: dict | None, instruction: str, context_ return None url = f"{executor['a2a_url']}/a2a" deadline = asyncio.get_running_loop().time() + executor["timeout_seconds"] - async with httpx.AsyncClient(timeout=30) as client: - send = await client.post(url, json={ + async with httpx.AsyncClient(timeout=_EXECUTOR_HTTP_TIMEOUT_S) as client: + # an executor holds message/send open until its task ends, so the read waits as long as the executor may take + read_s = max(_EXECUTOR_HTTP_TIMEOUT_S, executor["timeout_seconds"]) + send = await client.post(url, timeout=httpx.Timeout(_EXECUTOR_HTTP_TIMEOUT_S, read=read_s), json={ "jsonrpc": "2.0", "id": str(uuid.uuid4()), "method": "message/send", "params": {"message": {"role": "user", "messageId": uuid.uuid4().hex, "contextId": context_id, diff --git a/tst/unit/env/gateway/triggers_test.py b/tst/unit/env/gateway/triggers_test.py index f6897f64f..0d9f4846b 100644 --- a/tst/unit/env/gateway/triggers_test.py +++ b/tst/unit/env/gateway/triggers_test.py @@ -2,8 +2,10 @@ from __future__ import annotations import asyncio +import contextlib import json import random +import re from datetime import datetime, timedelta, timezone from zoneinfo import ZoneInfo @@ -1111,6 +1113,90 @@ def do_POST(self): server.shutdown() +@contextlib.asynccontextmanager +async def _executor_answering_after(seconds: float): + """A loopback executor whose message/send, like a deployed one's, answers only once its task ends `seconds` later; + its task polls answer at once. Yields its URL.""" + from pytest_socket import enable_socket + + enable_socket() # re-disabled by the autouse fixture's next setup + handlers: set[asyncio.Task] = set() + + async def serve(reader, writer): + handlers.add(asyncio.current_task()) + try: + while True: # one connection carries the send and then the polls + head = await reader.readuntil(b"\r\n\r\n") + req = json.loads(await reader.readexactly(int(re.search(rb"(?i)content-length: *(\d+)", head)[1]))) + if req["method"] == "message/send": + await asyncio.sleep(seconds) + body = json.dumps({"jsonrpc": "2.0", "id": req["id"], "result": {"id": "task-1", "status": { + "state": "completed", "message": {"parts": [{"kind": "text", "text": "replied"}]}}}}).encode() + writer.write(b"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: %d\r\n\r\n%s" + % (len(body), body)) + await writer.drain() + except (asyncio.IncompleteReadError, ConnectionError): + pass # the caller hung up + finally: + writer.close() + + server = await asyncio.start_server(serve, "127.0.0.1", 0) + try: + yield f"http://127.0.0.1:{server.sockets[0].getsockname()[1]}" + finally: + server.close() + for task in handlers: + task.cancel() + await asyncio.gather(*handlers, return_exceptions=True) + + +def _reply_then_post(url: str, timeout_seconds: float) -> dict: + return {"executor": {"a2a_url": url, "role": "executor", "timeout_seconds": timeout_seconds}, + "triggers": [{"id": "x", "when": {"type": "action", "tool": "t"}, + "actions": [{"type": "nl", "instruction": "reply to the client"}, + {"type": "tool", "tool": "slack_send_message", + "args": {"channel": "#c", "text": "the client replied"}}]}]} + + +async def _settled(engine, tid: str, within: float = 10) -> None: + for _ in range(int(within / 0.05)): + if engine._triggers[tid]["status"] not in ("firing", "queued"): + return + await asyncio.sleep(0.05) + raise AssertionError(f"trigger {tid!r} still {engine._triggers[tid]['status']} after {within}s") + + +@pytest.mark.asyncio +async def test_an_executor_slower_than_the_http_timeout_is_waited_out_and_the_later_actions_run(engine, monkeypatch): + """FD-5333: a contact slower than 30 s used to fail its fire on a read timeout, though its reply still landed.""" + monkeypatch.setattr(triggers_module, "_EXECUTOR_HTTP_TIMEOUT_S", 0.2) # stands in for the 30 s + async with _executor_answering_after(1.0) as url: + engine.register(_reply_then_post(url, timeout_seconds=10)) + engine.on_tool_call("default", "t", {}, _result()) + await asyncio.sleep(0.4) + assert engine._triggers["x"]["status"] == "firing" # past the HTTP timeout, still in flight + await _settled(engine, "x") + + assert engine._triggers["x"]["status"] == "fired", engine.state()["events"][-3:] + assert [e["action_index"] for e in engine._events if e["kind"] == "action_ok"] == [0, 1] + assert engine._test_gw.calls == [("slack_send_message", {"channel": "#c", "text": "the client replied"})] + + +@pytest.mark.asyncio +async def test_an_executor_slower_than_its_own_timeout_still_fails_the_fire(engine, monkeypatch): + monkeypatch.setattr(triggers_module, "_EXECUTOR_HTTP_TIMEOUT_S", 0.2) + async with _executor_answering_after(3.0) as url: + engine.register(_reply_then_post(url, timeout_seconds=1.0)) + engine.on_tool_call("default", "t", {}, _result()) + await asyncio.sleep(0.4) + assert engine._triggers["x"]["status"] == "firing" + await _settled(engine, "x", within=2.5) + + [failure] = [e for e in engine._events if e["kind"] == "action_failed"] + assert (failure["action_index"], failure["detail"]) == (0, "ReadTimeout") + assert engine._triggers["x"]["status"] == "armed" and engine._test_gw.calls == [] + + @pytest.mark.asyncio async def test_broken_check_is_fail_open(engine): engine.register({"triggers": [ From 49e678943f3fa4df94d35a781fc124a5564f30ad Mon Sep 17 00:00:00 2001 From: Sarah Chitty Date: Mon, 9 Nov 2026 00:02:07 -0800 Subject: [PATCH 02/53] always_on: import the local build The user_chat gateway extension (message_user, opt-in stay_quiet), the user_sim and always_on steps, and the built-in user sim fallback, as they ran in the FD-5332 local rig (detached at a439754). The rig's own triggers.py change is left out; the reviewed FD-5333 fix is merged next. Co-Authored-By: Claude Opus 5.5 --- src/agent_env/env/gateway/constants.py | 13 + src/agent_env/env/gateway/gateway.py | 101 +- src/agent_env/env/gateway/user_chat.py | 199 ++++ src/agent_env/task_step/registry.py | 3 + .../task_step/task_steps/always_on.py | 969 ++++++++++++++++++ .../task_step/task_steps/user_sim_versions.py | 220 ++++ 6 files changed, 1504 insertions(+), 1 deletion(-) create mode 100644 src/agent_env/env/gateway/user_chat.py create mode 100644 src/agent_env/task_step/task_steps/always_on.py create mode 100644 src/agent_env/task_step/task_steps/user_sim_versions.py diff --git a/src/agent_env/env/gateway/constants.py b/src/agent_env/env/gateway/constants.py index a04bd51f2..38217b2cf 100644 --- a/src/agent_env/env/gateway/constants.py +++ b/src/agent_env/env/gateway/constants.py @@ -79,6 +79,7 @@ def data_plane_load_timeout_s(size_bytes: int | None) -> int: EXT_STATE_URI = "urn:agentenv:state/v1" EXT_CHANGES_URI = "urn:agentenv:changes/v1" EXT_NOTIFICATIONS_URI = "urn:agentenv:notifications/v1" +EXT_USER_CHAT_URI = "urn:agentenv:user-chat/v1" EXT_WEBSITE_URI = "urn:agentenv:website/v1" EXT_BROWSER_URI = "urn:agentenv:browser/v1" EXT_COMPUTER_APP_URI = "urn:agentenv:computer-app/v1" @@ -270,4 +271,16 @@ def random_mcp_server_name() -> str: "clear": {"method": "POST", "endpoint": "/notifications/clear", "request": {"type": "object", "properties": {}}}}}, "required": False, }, + { + "uri": EXT_USER_CHAT_URI, + "description": "User chat (urn:agentenv:user-chat/v1): the message_user tool, for the roles config names, is how an always-on agent reaches its user; each message is queued with its virtual time until the harness takes it. Built in: role rules don't hide it from a configured role.", + "params": {"endpoint": "/user_chat/state", "methods": { + "config": {"method": "POST", "endpoint": "/user_chat/config", "request": {"type": "object", "properties": { + "roles": {"type": "array", "items": {"type": "string"}}, "user_name": {"type": "string"}}}}, + "state": {"method": "GET", "endpoint": "/user_chat/state", "request": {"type": "object", "properties": {}}}, + "take": {"method": "POST", "endpoint": "/user_chat/take", "request": {"type": "object", "properties": { + "take_id": {"type": "string"}}}}, + "clear": {"method": "POST", "endpoint": "/user_chat/clear", "request": {"type": "object", "properties": {}}}}}, + "required": False, + }, ] diff --git a/src/agent_env/env/gateway/gateway.py b/src/agent_env/env/gateway/gateway.py index da71d495c..a1c761ca8 100644 --- a/src/agent_env/env/gateway/gateway.py +++ b/src/agent_env/env/gateway/gateway.py @@ -90,6 +90,8 @@ ) from .triggers import TriggerEngine, TriggerError from .notifications import CHECK_TOOL_NAMES, NotificationError, Notifications, build_check_tools +from .user_chat import MESSAGE_USER_TOOL_NAME, UserChat, UserChatError, build_user_chat_tool +from .user_chat import STAY_QUIET_TOOL_NAME, build_stay_quiet_tool logger = logging.getLogger(__name__) @@ -201,6 +203,7 @@ async def list_website_urls() -> list[str]: ) self._clock = Clock() self._notifications = Notifications(lambda: self._clock) + self._user_chat = UserChat(lambda: self._clock) self._env_get_time_url_cache: str | None = None self._install_role_filters() @@ -408,6 +411,51 @@ async def notifications_clear(request: Request) -> Response: self._unregister_check_tools() return self._json_response(self._notifications.clear(), 200) + @self._mcp.custom_route("/user_chat/config", methods=["POST"]) + async def user_chat_config(request: Request) -> Response: + try: + body = await request.json() + except Exception: + return self._json_response({"ok": False, "error": "Invalid JSON body"}, 400) + if isinstance(body, dict) and body.get("roles"): + try: + await self._ensure_tools_discovered() + except Exception as e: + return self._json_response( + {"ok": False, "error": f"tool discovery has not completed ({type(e).__name__}); retry"}, 503) + if MESSAGE_USER_TOOL_NAME in self._tool_server_urls: + return self._json_response( + {"ok": False, "error": f"'{MESSAGE_USER_TOOL_NAME}' already registered by a backing server"}, 409) + try: + state = self._user_chat.configure(body) + except UserChatError as e: + return self._json_response({"ok": False, "error": str(e)}, 400) + if self._user_chat.enabled: + self._register_user_chat_tool() + else: + self._unregister_user_chat_tool() + return self._json_response(state, 200) + + @self._mcp.custom_route("/user_chat/state", methods=["GET"]) + async def user_chat_state(request: Request) -> Response: + return self._json_response(self._user_chat.state(), 200) + + @self._mcp.custom_route("/user_chat/take", methods=["POST"]) + async def user_chat_take(request: Request) -> Response: + try: + body = await request.json() + except Exception: + body = {} + try: + return self._json_response(self._user_chat.take(body), 200) + except UserChatError as e: + return self._json_response({"ok": False, "error": str(e)}, 400) + + @self._mcp.custom_route("/user_chat/clear", methods=["POST"]) + async def user_chat_clear(request: Request) -> Response: + self._unregister_user_chat_tool() + return self._json_response(self._user_chat.clear(), 200) + @self._mcp.custom_route("/trajectory", methods=["GET"]) async def env_trajectory(request: Request) -> Response: return self._serve_trajectory() @@ -527,6 +575,10 @@ def _is_disabled(self, role: str, name: str) -> bool: The scheduled-check tools exist only for the roles the notifications config names, whatever the rules say.""" if name in CHECK_TOOL_NAMES and name not in self._tool_server_urls and role not in self._notifications.roles: return True + if name == MESSAGE_USER_TOOL_NAME and name not in self._tool_server_urls: + return role not in self._user_chat.roles # built in: role rules don't hide it from a configured role + if name == STAY_QUIET_TOOL_NAME and name not in self._tool_server_urls: + return not (self._user_chat.stay_quiet and role in self._user_chat.roles) # opt-in, like message_user hidden = name in self._hidden_by_default for r in (role, WILDCARD): rules = self._role_rules.get(r, {}) @@ -937,7 +989,8 @@ async def _discover_and_register_tools(self, sessions: dict[str, ClientSession]) # clock and the checks refuse to turn on over it sources: dict[str, list[str]] = { tool.name: ["gateway"] for tool in self._server_tools.get("gateway", []) - if tool.name != GET_TIME_TOOL_NAME and tool.name not in CHECK_TOOL_NAMES} + if tool.name != GET_TIME_TOOL_NAME and tool.name not in CHECK_TOOL_NAMES + and tool.name != MESSAGE_USER_TOOL_NAME and tool.name != STAY_QUIET_TOOL_NAME} for server, tools in listed: for tool in tools: sources.setdefault(tool.name, []).append(server.name) @@ -1231,6 +1284,52 @@ def _unregister_check_tools(self) -> None: self._server_tools.pop("gateway", None) self._trigger_engine.invalidate_readonly_cache() + def _register_user_chat_tool(self) -> None: + """Expose message_user to the roles the user-chat config names (rebuilt so it names the user).""" + tools = self._mcp._tool_manager._tools + if MESSAGE_USER_TOOL_NAME in self._tool_server_urls: + return + tool = build_user_chat_tool(self._user_chat, self._create_arg_model_from_schema) + tools[MESSAGE_USER_TOOL_NAME] = tool + mirrored = [t for t in self._server_tools.get("gateway", []) if t.name != MESSAGE_USER_TOOL_NAME] + mirrored.append(MCPTool(name=tool.name, description=tool.description, inputSchema=tool.parameters, + annotations=tool.annotations)) + self._server_tools["gateway"] = mirrored + self._sync_stay_quiet_tool() + self._trigger_engine.invalidate_readonly_cache() + + def _sync_stay_quiet_tool(self) -> None: + """stay_quiet exists only while the user-chat config turns it on (opt-in); otherwise it is absent.""" + tools = self._mcp._tool_manager._tools + if STAY_QUIET_TOOL_NAME in self._tool_server_urls: + return + mirrored = [t for t in self._server_tools.get("gateway", []) if t.name != STAY_QUIET_TOOL_NAME] + if self._user_chat.stay_quiet: + tool = build_stay_quiet_tool(self._user_chat, self._create_arg_model_from_schema) + tools[STAY_QUIET_TOOL_NAME] = tool + mirrored.append(MCPTool(name=tool.name, description=tool.description, inputSchema=tool.parameters, + annotations=tool.annotations)) + else: + tools.pop(STAY_QUIET_TOOL_NAME, None) + if mirrored: + self._server_tools["gateway"] = mirrored + else: + self._server_tools.pop("gateway", None) + + def _unregister_user_chat_tool(self) -> None: + tools = self._mcp._tool_manager._tools + if MESSAGE_USER_TOOL_NAME in self._tool_server_urls: + return + tools.pop(MESSAGE_USER_TOOL_NAME, None) + tools.pop(STAY_QUIET_TOOL_NAME, None) + remaining = [t for t in self._server_tools.get("gateway", []) + if t.name not in (MESSAGE_USER_TOOL_NAME, STAY_QUIET_TOOL_NAME)] + if remaining: + self._server_tools["gateway"] = remaining + else: + self._server_tools.pop("gateway", None) + self._trigger_engine.invalidate_readonly_cache() + def _create_and_register_proxy_tool(self, server_url: str, tool: MCPTool) -> None: self._mcp._tool_manager._tools[tool.name] = self._proxy_tool(server_url, tool) diff --git a/src/agent_env/env/gateway/user_chat.py b/src/agent_env/env/gateway/user_chat.py new file mode 100644 index 000000000..8d87d936a --- /dev/null +++ b/src/agent_env/env/gateway/user_chat.py @@ -0,0 +1,199 @@ +"""The user chat (urn:agentenv:user-chat/v1): how an always-on agent reaches its user. + +The roles a task names in `config` get one tool, `message_user`. Each call is queued with the virtual time it was +sent at; nothing here reaches the user by itself: the always_on step `take`s what was sent after each agent turn and +hands it to the user sim when the user would read it. Whatever else the agent writes (its plain reply) never reaches +the user. The tool is built in: once a role is named, role rules from modify_env_tool_access don't hide it. + +`take` hands each message out once: {"id", "from", "at", "text"}. A `take_id` makes a retry safe. +""" +from __future__ import annotations + +import threading +from typing import Any, Callable + +from mcp.server.fastmcp.tools import Tool as FastMCPTool +from mcp.server.fastmcp.utilities.func_metadata import FuncMetadata +from mcp.types import CallToolResult, TextContent, ToolAnnotations + +from .caller import Caller +from .clock import Clock, _iso + +MESSAGE_USER_TOOL_NAME = "message_user" +STAY_QUIET_TOOL_NAME = "stay_quiet" # opt-in (config stay_quiet: true): "nothing to tell the user this wake" +_MAX_TEXT_CHARS = 20000 +_MAX_TAKE_ID_CHARS = 128 +_KEPT_TAKES = 64 +_CONFIG_FIELDS = ("roles", "user_name", "stay_quiet") + +MESSAGE_USER_DESCRIPTION = ( + "Send a chat message to {user}, the person you work for. This is the only way they see anything from you: your " + "normal replies are kept in a log they never read. Send a message when there is something they need to know or " + "decide; send nothing when there isn't. You can send several messages in one turn. They read messages when " + "they're free, so a reply can take a while; you are woken when it arrives." +) +STAY_QUIET_DESCRIPTION = ( + "Call this when there is nothing worth telling {user} right now. It sends nothing; it records that you chose to " + "stay quiet this time." +) +_QUIET_SCHEMA: dict[str, Any] = {"type": "object", "properties": {}, "additionalProperties": False} +_SCHEMA: dict[str, Any] = { + "type": "object", + "properties": {"text": {"type": "string", "description": "The message, as you'd text it."}}, + "required": ["text"], + "additionalProperties": False, +} + + +class UserChatError(ValueError): + """A bad config, take or tool argument.""" + + +class UserChat: + """The queue of messages roles sent their user; threadsafe.""" + + def __init__(self, clock: Callable[[], Clock]): + self._clock_of = clock + self._lock = threading.Lock() + self._roles: tuple[str, ...] = () + self._user_name = "your user" + self._queue: list[dict] = [] + self._sent = 0 + self._takes: dict[str, dict] = {} + self._stay_quiet = False + self._quiet: list[dict] = [] + + @property + def roles(self) -> tuple[str, ...]: + return self._roles + + @property + def user_name(self) -> str: + return self._user_name + + @property + def enabled(self) -> bool: + return bool(self._roles) + + @property + def stay_quiet(self) -> bool: + """Whether the configured roles also get stay_quiet (off unless a config turns it on).""" + return self.enabled and self._stay_quiet + + def configure(self, body: Any) -> dict: + """Name the roles that get message_user (`roles: []` turns it off) and how the tool names the user.""" + if not isinstance(body, dict): + raise UserChatError("body must be a JSON object") + unknown = sorted(set(body) - set(_CONFIG_FIELDS)) + if unknown: + raise UserChatError(f"config takes {list(_CONFIG_FIELDS)}, not {unknown}") + roles = body.get("roles", []) + if not isinstance(roles, list) or not all(isinstance(r, str) and r.strip() for r in roles): + raise UserChatError("roles must be a list of role names") + name = body.get("user_name") or "your user" + if not isinstance(name, str): + raise UserChatError("user_name must be a string") + stay_quiet = body.get("stay_quiet", False) + if not isinstance(stay_quiet, bool): + raise UserChatError("stay_quiet must be true or false") + with self._lock: + self._roles = tuple(dict.fromkeys(r.strip() for r in roles)) + self._user_name = name.strip() or "your user" + self._stay_quiet = stay_quiet + return self.state() + + def clear(self) -> dict: + with self._lock: + self._roles, self._user_name = (), "your user" + self._queue.clear() + self._takes.clear() + self._stay_quiet = False + self._quiet.clear() + return {"ok": True} + + def send(self, text: Any, *, role: str) -> dict: + if role not in self._roles: + raise UserChatError("message_user is not available here") + if not isinstance(text, str) or not text.strip(): + raise UserChatError("text must be the message to send") + if len(text) > _MAX_TEXT_CHARS: + raise UserChatError(f"text must be at most {_MAX_TEXT_CHARS} characters") + now = self._clock_of().now() + with self._lock: + self._sent += 1 + item = {"id": f"user-message-{self._sent}", "from": role, "at": _iso(now) if now else None, + "text": text, "status": "pending"} + self._queue.append(item) + return {"sent": True, "id": item["id"], "at": item["at"]} + + def quiet(self, *, role: str) -> dict: + """stay_quiet: nothing is sent; the call is kept (with its virtual time) for the next take's `quiet`.""" + if role not in self._roles or not self._stay_quiet: + raise UserChatError("stay_quiet is not available here") + now = self._clock_of().now() + with self._lock: + self._quiet.append({"from": role, "at": _iso(now) if now else None}) + return {"ok": True, "sent_to_user": False} + + def state(self) -> dict: + with self._lock: + out = {"roles": list(self._roles), "user_name": self._user_name, "sent": self._sent, + "pending": [_public(m) for m in self._queue if m["status"] == "pending"]} + if self._stay_quiet: + out["stay_quiet"] = True + return out + + def take(self, body: Any = None) -> dict: + body = body if isinstance(body, dict) else {} + take_id = body.get("take_id") + if take_id is not None and (not isinstance(take_id, str) or not take_id or len(take_id) > _MAX_TAKE_ID_CHARS): + raise UserChatError(f"take_id must be a non-empty string of at most {_MAX_TAKE_ID_CHARS} characters") + with self._lock: + if take_id is not None and take_id in self._takes: + return self._takes[take_id] + items = [m for m in self._queue if m["status"] == "pending"] + for m in items: + m["status"] = "taken" + self._queue = [m for m in self._queue if m["status"] == "pending"] + result = {"items": [_public(m) for m in items]} + if self._stay_quiet: + result["quiet"], self._quiet = self._quiet, [] + if take_id is not None: + self._takes[take_id] = result + while len(self._takes) > _KEPT_TAKES: + del self._takes[next(iter(self._takes))] + return result + + +def _public(m: dict) -> dict: + return {k: v for k, v in m.items() if k != "status"} + + +def build_user_chat_tool(chat: UserChat, make_arg_model: Callable[[str, dict], Any]) -> FastMCPTool: + """message_user, built by hand like get_time so a backing server's tool can't silently shadow it.""" + + async def message_user(text: str) -> dict | CallToolResult: + try: + return chat.send(text, role=Caller.current().role) + except UserChatError as e: + return CallToolResult(isError=True, content=[TextContent(type="text", text=str(e))]) + + return FastMCPTool(fn=message_user, name=MESSAGE_USER_TOOL_NAME, + description=MESSAGE_USER_DESCRIPTION.format(user=chat.user_name), parameters=_SCHEMA, + fn_metadata=FuncMetadata(arg_model=make_arg_model(MESSAGE_USER_TOOL_NAME, _SCHEMA)), + is_async=True, annotations=ToolAnnotations(readOnlyHint=False)) + + +def build_stay_quiet_tool(chat: UserChat, make_arg_model: Callable[[str, dict], Any]) -> FastMCPTool: + """stay_quiet, built in like message_user; only registered when the config turns it on.""" + + async def stay_quiet() -> dict | CallToolResult: + try: + return chat.quiet(role=Caller.current().role) + except UserChatError as e: + return CallToolResult(isError=True, content=[TextContent(type="text", text=str(e))]) + + return FastMCPTool(fn=stay_quiet, name=STAY_QUIET_TOOL_NAME, + description=STAY_QUIET_DESCRIPTION.format(user=chat.user_name), parameters=_QUIET_SCHEMA, + fn_metadata=FuncMetadata(arg_model=make_arg_model(STAY_QUIET_TOOL_NAME, _QUIET_SCHEMA)), + is_async=True, annotations=ToolAnnotations(readOnlyHint=False)) diff --git a/src/agent_env/task_step/registry.py b/src/agent_env/task_step/registry.py index 1b9a3f298..96916aef5 100644 --- a/src/agent_env/task_step/registry.py +++ b/src/agent_env/task_step/registry.py @@ -54,6 +54,7 @@ def _builtin_registry() -> dict[str, type["TaskStep"]]: from agent_env.task_step.task_steps.load_artifact import LoadArtifactTaskStep from agent_env.task_step.task_steps.modify_env_tool_access import ModifyEnvToolAccessStep from agent_env.task_step.task_steps.prompt_agent import PromptAgentTaskStep + from agent_env.task_step.task_steps.always_on import AlwaysOnTaskStep, UserSimTaskStep from agent_env.task_step.task_steps.register_env_triggers import RegisterEnvTriggersStep from agent_env.task_step.task_steps.register_agent_triggers import RegisterAgentTriggersStep from agent_env.task_step.task_steps.sync_env_clock import SyncEnvClockTaskStep @@ -119,6 +120,8 @@ def _builtin_registry() -> dict[str, type["TaskStep"]]: _get_type(LoadArtifactTaskStep): LoadArtifactTaskStep, _get_type(ModifyEnvToolAccessStep): ModifyEnvToolAccessStep, _get_type(PromptAgentTaskStep): PromptAgentTaskStep, + _get_type(AlwaysOnTaskStep): AlwaysOnTaskStep, + _get_type(UserSimTaskStep): UserSimTaskStep, _get_type(RegisterEnvTriggersStep): RegisterEnvTriggersStep, _get_type(RegisterAgentTriggersStep): RegisterAgentTriggersStep, _get_type(SyncEnvClockTaskStep): SyncEnvClockTaskStep, diff --git a/src/agent_env/task_step/task_steps/always_on.py b/src/agent_env/task_step/task_steps/always_on.py new file mode 100644 index 000000000..92c2a2e09 --- /dev/null +++ b/src/agent_env/task_step/task_steps/always_on.py @@ -0,0 +1,969 @@ +"""The always_on and user_sim steps: a task that runs over simulated days, the way you'd hand a job to a personal +assistant, with an LLM user sim. A test-rig build of the "Always-on Task Step Design" doc. + +The ``user_sim`` step says who the user is (description, schedule, response times, the version of the user sim and +its plug-in) and records it in the run's metadata under its ``name``; a ``deploy_agent`` step deploys the agent that +plays them. The ``always_on`` step names it with ``user_sim`` and runs the loop: + +- The first wake is the user's request. Each wake opens with what came due for the agent on notifications/v1 (a + trigger's ``notify``, a scheduled check), labelled ``[time · channel]``, then the user's own messages. +- Only what the agent sends with the gateway's ``message_user`` tool (user-chat/v1) reaches the user; its plain reply + goes to the log, flagged when it looks meant for the user. +- The user sim is asked only after the agent sent something, at the time the user would read it (``schedule``), with + everything sent meanwhile; its answer is reply / no_reply / done, and a reply is held for the response time. +- Scripted user lines are triggers' ``notify`` actions to the ``user_says`` recipient: sent to the agent as the + user's message and added to the user sim's history. What the user's phone shows is ``notify`` to ``user_phone``. +- The clock runs at the task's speed during turns (sync_env_clock). When nothing is due, the step advances it to the + earliest of: the next trigger event, the next notification, a held reply, a pending read, or end_time; then waits up + to ``settle_timeout_seconds`` for triggers to finish. +- At end_time the user's ``final`` line (with a reminder to use message_user) gives the agent one last turn. The run + ends on done, end_time, max_wakes, or nothing left to wake the agent; the reason is recorded. +- Everything goes into a timeline in ``metadata["always_on"][step_id]`` and the object store. +""" +from __future__ import annotations + +import asyncio +import importlib.util +import json +import logging +import os +import re +import sys +import tempfile +import time +import uuid +from datetime import datetime, timedelta, timezone +from types import ModuleType +from typing import Any, ClassVar, Optional +from zoneinfo import ZoneInfo + +from a2a.types import TaskState + +from agent_env.a2a_agent import A2AAgent, conversation_store, protocol +from agent_env.a2a_agent.object_transfer import send_and_wait +from agent_env.artifact import FileArtifact +from agent_env.entity.refs import EntityRef +from agent_env.env.env import EnvCapabilityUnsupported +from agent_env.env.gateway.constants import ( + DEFAULT_ROLE, + EXT_CLOCK_URI, + EXT_NOTIFICATIONS_URI, + EXT_TRIGGERS_URI, + EXT_USER_CHAT_URI, +) +from agent_env.store import DuplicateKeyError, get_config +from agent_env.task_step.context import PromptResponse, TaskStepContext +from agent_env.task_step.task_step import TaskStep +from agent_env.task_step.task_steps import user_sim_versions +from agent_env.task_step.task_steps.notification_delivery import delivered_parts, label_time, take_due +from agent_env.task_step.task_steps.prompt_agent import PromptAgentTaskStep + +logger = logging.getLogger(__name__) + +USER_SIMS_KEY = "user_sims" +ALWAYS_ON_KEY = "always_on" +USER_SAYS = "user_says" +USER_PHONE = "user_phone" +IN_FLIGHT = ("firing", "queued") +POLL_SECONDS = 0.5 +DAYS = ("mon", "tue", "wed", "thu", "fri", "sat", "sun") +FINAL_REMINDER = ("(This is the last time you'll be woken for this job. Only what you send with message_user reaches " + "{first}; your normal reply is not shown to them.)") +# Opt-in settings (all off unless a task turns them on): stay_quiet, nudge, wake_reminder. +NUDGE_TEXT = ("Your closing text was not delivered. If the user should see it, send it with message_user; otherwise " + "call stay_quiet.") +WAKE_REMINDER_TEXT = "Reminder: only message_user reaches the user." +_ADDRESSED = re.compile(r"\?\s*$|^(hi|hey|hello)\b|\byou\b|\byour\b", re.I | re.M) + + +# ================================================================================================================== +class UserSimTaskStep(TaskStep): + """Who the user is, once per task: their settings go into the run's metadata under ``name``.""" + + type: ClassVar[str] = "user_sim" + entity_refs = (EntityRef.artifact("plugin.file_artifact_id", version_field="file_artifact_version", + artifact_type="file"),) + + def __init__(self, id: str, version: Optional[int], *, name: str, model: str, agent_name: Optional[str] = None, + user_version: str = "today", display_name: Optional[str] = None, first_name: Optional[str] = None, + description: Optional[str] = None, time_zone: Optional[str] = None, apps: Optional[list] = None, + schedule: Optional[dict] = None, response_times: Optional[dict] = None, + max_replies: Optional[int] = None, permissions: Optional[dict] = None, + keep_private: Optional[list] = None, output_format: Optional[dict] = None, + versions: Optional[dict] = None, plugin: Optional[dict] = None, depends_on=None, + fail_task_on_error: bool = True, retry_config=None): + super().__init__(id, version, depends_on=depends_on, fail_task_on_error=fail_task_on_error, + retry_config=retry_config) + if not model: + raise ValueError("user_sim.model is required") + self.name = name + self.model = model + self.agent_name = agent_name or name + self.user_version = user_version + self.display_name = display_name + self.first_name = first_name + self.description = description + self.time_zone = time_zone + self.apps = apps or [] + self.schedule = schedule + self.response_times = response_times + self.max_replies = max_replies + self.permissions = permissions + self.keep_private = keep_private + self.output_format = output_format + self.versions = versions or {} + self.plugin = plugin + + _FIELDS = ("name", "model", "agent_name", "user_version", "display_name", "first_name", "description", "time_zone", + "apps", "schedule", "response_times", "max_replies", "permissions", "keep_private", "output_format", + "versions", "plugin") + + def to_dict(self) -> dict: + return {**super().to_dict(), **{k: getattr(self, k) for k in self._FIELDS}} + + @classmethod + def from_dict(cls, data: dict) -> UserSimTaskStep: + return cls(**cls._base_from_dict(data), retry_config=data.get("retry_config"), + **{k: data.get(k) for k in cls._FIELDS if k in data}) + + async def execute(self, context: TaskStepContext) -> TaskStepContext: + overrides = self.step_param_overrides(context) + version = overrides.get("user_version") or self.user_version + if not any(a.agent_name == self.agent_name for a in context.deployed_agents): + raise RuntimeError(f"user_sim '{self.name}': no deployed agent named '{self.agent_name}'; deploy it first") + settings = {k: getattr(self, k) for k in self._FIELDS} + settings.update(user_version=version, step_id=self.id, + model=overrides.get("model") or self.model) + context.metadata.setdefault(USER_SIMS_KEY, {})[self.name] = settings + logger.info(f"user_sim '{self.name}': version {version}, model {settings['model']}, agent {self.agent_name}") + return context + + +# ================================================================================================================== +class AlwaysOnTaskStep(PromptAgentTaskStep): + type: ClassVar[str] = "always_on" + entity_refs = () + + def __init__(self, id: str, version: Optional[int], *, user_sim: str, end_time: Optional[str] = None, + final: Optional[str] = None, max_wakes: int = 60, settle_timeout_seconds: float = 300, + user_timeout_seconds: int = 300, final_reminder: Optional[str] = FINAL_REMINDER, + stay_quiet: bool = False, nudge: bool = False, nudge_text: str = NUDGE_TEXT, + wake_reminder: bool = False, wake_reminder_text: str = WAKE_REMINDER_TEXT, + **prompt_fields: Any): + for unsupported in ("max_conversation_turns", "user_agent_name", "user_a2a_url", "post_turn_hooks", + "user_output_format", "user_model", "snapshot_config"): + prompt_fields.pop(unsupported, None) + super().__init__(id, version, max_conversation_turns=max(1, int(max_wakes)), **prompt_fields) + if not user_sim: + raise ValueError("always_on needs user_sim, the name of a user_sim step's user") + if end_time is not None: + _end_instant(end_time, datetime.now(timezone.utc)) # validates + self.user_sim = user_sim + self.end_time = end_time + self.final = final + self.max_wakes = int(max_wakes) + self.settle_timeout_seconds = float(settle_timeout_seconds) + self.user_timeout_seconds = int(user_timeout_seconds) + self.final_reminder = final_reminder + self.stay_quiet = bool(stay_quiet) # the agent also gets stay_quiet() (gateway user-chat config) + self.nudge = bool(nudge) # one more short turn when closing text went undelivered + self.nudge_text = nudge_text + self.wake_reminder = bool(wake_reminder) # each wake's message ends with wake_reminder_text + self.wake_reminder_text = wake_reminder_text + + _OWN = ("user_sim", "end_time", "final", "max_wakes", "settle_timeout_seconds", "user_timeout_seconds", + "final_reminder", "stay_quiet", "nudge", "nudge_text", "wake_reminder", "wake_reminder_text") + _PROMPT_FIELDS = ("prompt", "parts", "prompt_id", "agent_name", "timeout_seconds", "trajectory_output_prefix", + "model", "model_params", "system_prompt", "max_turns", "output_format", "max_thinking_tokens", + "effort", "harness", "agentenv_tools", "poll_interval_seconds", "context_id") + + def to_dict(self) -> dict: + base = TaskStep.to_dict(self) + for k in self._PROMPT_FIELDS: + base[k] = getattr(self, k) + for k in self._OWN: + base[k] = getattr(self, k) + return base + + @classmethod + def from_dict(cls, data: dict) -> AlwaysOnTaskStep: + fields = {k: data[k] for k in cls._PROMPT_FIELDS if k in data and data[k] is not None} + own = {k: data[k] for k in cls._OWN if k in data and data[k] is not None} + return cls(**cls._base_from_dict(data), retry_config=data.get("retry_config"), **fields, **own) + + async def _execute(self, context: TaskStepContext) -> TaskStepContext: + run = _Run(self, context) + try: + await run.go() + finally: + run.close() + return context + + +# ================================================================================================================== +class _Run: + """One always-on run: its state, its timeline and its loop.""" + + def __init__(self, step: AlwaysOnTaskStep, context: TaskStepContext): + self.step = step + self.ctx = context + self.timeline: list[dict] = [] + self.t0_real = time.time() + self.wakes = 0 + self.end_reason: Optional[str] = None + self.unread: list[dict] = [] # the agent's messages the user hasn't read + self.read_at: Optional[datetime] = None + self.read_mode = "now" + self.held: list[dict] = [] # user replies held for their response time: {deliver_at, text} + self.you_sent: list[dict] = [] # the user's own lines since their last read: {at, text} + self.history: list[dict] = [] # every read and its answer, for the plug-in + self.replies = 0 + self.gave_up = False + self.final_sent = False + self.last_reply_at: Optional[datetime] = None + self.per_turn_trajectory_urls: list[Optional[str]] = [] + self.per_turn_parts: list[Optional[list[dict]]] = [] + self.tool_calls: Optional[int] = None + self.last_plain = "" + self.last_messages: list[dict] = [] + self.all_messages: list[dict] = [] + self.final_task_id = "" + self.error: Optional[str] = None + self._plugin_paths: list[str] = [] + self._plugin_modules: list[str] = [] + self.nudges = 0 + + def close(self) -> None: + for p in self._plugin_paths: + if os.path.exists(p): + os.unlink(p) + for m in self._plugin_modules: + sys.modules.pop(m, None) + + # ---- record ---------------------------------------------------------------------------------------------- + def log(self, event: str, **fields: Any) -> dict: + entry = {"event": event, "real_time": datetime.now(timezone.utc).isoformat(), + "real_elapsed_s": round(time.time() - self.t0_real, 1), **fields} + self.timeline.append(entry) + logger.info(f"always_on {event}: " + json.dumps({k: v for k, v in fields.items() + if k not in ("input", "parts_text", "raw")}, + default=str)[:400]) + return entry + + # ---- setup ----------------------------------------------------------------------------------------------- + async def go(self) -> None: + step, ctx = self.step, self.ctx + self.agent = next((a for a in ctx.deployed_agents if a.agent_name == step.agent_name), None) + if self.agent is None: + raise RuntimeError(f"Agent with name '{step.agent_name}' not found in context.deployed_agents") + self.target_url = self.agent.a2a_url or self.agent.api_url + self.recipient = self.agent.role or DEFAULT_ROLE + us = (ctx.metadata.get(USER_SIMS_KEY) or {}).get(step.user_sim) + if us is None: + raise RuntimeError(f"always_on: no user_sim step named '{step.user_sim}' ran before this step") + self.us = us + self.person = self._person(us) + self.version = us["user_version"] + self.user_agent = next((a for a in ctx.deployed_agents if a.agent_name == us["agent_name"]), None) + if self.user_agent is None: + raise RuntimeError(f"always_on: the user sim agent '{us['agent_name']}' is not deployed") + self.user_url = self.user_agent.a2a_url or self.user_agent.api_url + self.plugin = await self._load_plugin(us.get("plugin")) + self.tz = ZoneInfo(us.get("time_zone") or "UTC") + + self.clock_envs = [d for d in ctx.deployed_envs if d.supports(EXT_CLOCK_URI, "state")] + self.notification_envs = [d for d in ctx.deployed_envs if d.supports(EXT_NOTIFICATIONS_URI, "take")] + self.chat_envs = [d for d in ctx.deployed_envs if d.supports(EXT_USER_CHAT_URI, "take")] + if not self.chat_envs: + raise RuntimeError(f"always_on: no deployed env offers {EXT_USER_CHAT_URI} (message_user); rebuild its " + "gateway from a branch that has user_chat.py") + chat_config = {"roles": [self.recipient], "user_name": self.person.get("name") or "your user"} + if step.stay_quiet: + chat_config["stay_quiet"] = True + for env in self.chat_envs: + await _call(env, EXT_USER_CHAT_URI, "config", chat_config) + await self._configure_agents() + + self.conversation_id = uuid.uuid4().hex + try: + conversation_store.create_conversation(conversation_id=self.conversation_id, + task_instance_id=ctx.instance_id or "", + source_agent_name=us["agent_name"], + target_agent_name=step.agent_name) + except DuplicateKeyError: + pass + ctx.metadata.setdefault("a2a_conversations", {})[step.id] = self.conversation_id + self.user_context_id = uuid.uuid4().hex # one user sim session per run + self.solver_context_id = uuid.uuid4().hex + + now = await self.now() + t0 = await self._t0() or now + self.end = _end_instant(step.end_time, t0) if step.end_time else None + self.log("start", virtual_time=_iso(now), end_time=_iso(self.end) if self.end else None, + agent=step.agent_name, agent_role=self.recipient, user_sim=step.user_sim, version=self.version, + user_model=us["model"], max_wakes=step.max_wakes, + clock_rate=await self._rate()) + try: + await self.loop(now) + except Exception as e: + self.error = f"{type(e).__name__}: {e}" + self.end_reason = self.end_reason or "step_error" + self.log("error", error=self.error) + raise + finally: + await self.finish() + + def _person(self, us: dict) -> dict: + name = us.get("display_name") or us["name"] + return {"key": us["name"], "name": name, "first": us.get("first_name") or name.split()[0], + "time_zone": us.get("time_zone"), + "description": us.get("description"), "versions": us.get("versions") or {}, + "output_format": us.get("output_format"), "apps": us.get("apps") or [], + "permissions": us.get("permissions"), "keep_private": us.get("keep_private")} + + async def _load_plugin(self, spec: Optional[dict]) -> ModuleType: + if not spec: + return user_sim_versions + artifact = FileArtifact.get(spec["file_artifact_id"], spec.get("file_artifact_version")) + source = await asyncio.to_thread(artifact.load) + fd, path = tempfile.mkstemp(suffix=".py") + os.write(fd, source) + os.close(fd) + self._plugin_paths.append(path) + name = f"user_sim_plugin_{uuid.uuid4().hex[:8]}" + mspec = importlib.util.spec_from_file_location(name, path) + module = importlib.util.module_from_spec(mspec) + sys.modules[name] = module + self._plugin_modules.append(name) + mspec.loader.exec_module(module) + if hasattr(module, "configure"): + module.configure(spec.get("params") or {}) + return module + + async def _configure_agents(self) -> None: + step, ctx = self.step, self.ctx + model, effort = step._resolve_model_and_effort(ctx) + self.model = model + overrides = ctx.metadata.get("user_overrides", {}) + card = self.agent.a2a_card or {} + desired = {"model": model, "system_prompt": step.system_prompt, "effort": effort, + "harness": ctx.agent_harness or step.harness, + "max_turns": overrides.get("agent_max_turns") or step.max_turns, + "max_thinking_tokens": overrides.get("agent_max_thinking_tokens") or step.max_thinking_tokens, + "output_format": step.output_format, "timeout_seconds": step.timeout_seconds, + "task_id": ctx.metadata.get("task_id"), "agentenv_tools": step.agentenv_tools, + "builtin_tools": card.get("builtin_tools")} + desired = {k: v for k, v in desired.items() if v is not None} + model_params = get_config().get_model_params(step.model_params) + if model_params: + desired["model_params"] = model_params + negotiated = A2AAgent.negotiate_agent_config(card, desired) + if negotiated: + await protocol.post_agent_config(self.target_url + negotiated.endpoint, negotiated.fields) + ctx.record_model("solver", step.id, negotiated.fields.get("model") if negotiated else None) + + setup = self.plugin.setup(self.version, self.person) + user_config = {"model": self.us["model"], "system_prompt": setup.get("system_prompt"), + "output_format": setup.get("output_format")} + user_config = {k: v for k, v in user_config.items() if v is not None} + negotiated = A2AAgent.negotiate_agent_config(self.user_agent.a2a_card or {}, user_config) + if negotiated: + await protocol.post_agent_config(self.user_url + negotiated.endpoint, negotiated.fields) + accepted = sorted(negotiated.fields) if negotiated else [] + ctx.record_model("user simulator", step.id, negotiated.fields.get("model") if negotiated else None) + self.user_setup = {"system_prompt_chars": len(setup.get("system_prompt") or ""), + "output_format": setup.get("output_format"), "accepted_fields": accepted} + if "system_prompt" not in accepted and setup.get("system_prompt"): + logger.warning("always_on: the user sim agent's card did not accept system_prompt; it keeps its deploy one") + + # ---- clock and world ------------------------------------------------------------------------------------ + async def _armed(self) -> list: + armed = [] + for env in self.clock_envs: + try: + if (await _call(env, EXT_CLOCK_URI, "state")).get("armed"): + armed.append(env) + except Exception as e: + logger.warning(f"clock state of {env.env_id} failed: {e}") + return armed + + async def now(self) -> datetime: + envs = await self._armed() + if not envs: + return datetime.now(timezone.utc) + return min([_utc((await _call(env, EXT_CLOCK_URI, "state"))["virtual_time"]) for env in envs]) + + async def _t0(self) -> Optional[datetime]: + envs = await self._armed() + return min([_utc((await _call(env, EXT_CLOCK_URI, "state"))["t0"]) for env in envs]) if envs else None + + async def _rate(self) -> Optional[float]: + envs = await self._armed() + return (await _call(envs[0], EXT_CLOCK_URI, "state")).get("virtual_seconds_per_real_second") if envs else None + + async def advance(self, target: datetime, reason: str) -> None: + before = await self.now() + if target <= before: + return + for env in await self._armed(): + try: + await _call(env, EXT_CLOCK_URI, "advance", {"virtual_time": _iso(target)}, attempts=2) + except Exception: + state = await _call(env, EXT_CLOCK_URI, "state") + if not (state.get("armed") and _utc(state["virtual_time"]) >= target): + raise + stuck = await self.settle() + self.log("clock_jump", **{"from": _iso(before), "to": _iso(target), "reason": reason, + "settled": not stuck, "still_firing": stuck}) + + async def settle(self) -> list[str]: + """Wait until no trigger still fires (bar one waiting for a later time); the ids of envs still firing.""" + deadline = time.monotonic() + self.step.settle_timeout_seconds + envs = [e for e in await self._armed() if e.supports(EXT_TRIGGERS_URI, "state")] + waiting = set(range(len(envs))) + while waiting and time.monotonic() < deadline: + for i in sorted(waiting): + env = envs[i] + clock = await _call(env, EXT_CLOCK_URI, "state") + state = await _call(env, EXT_TRIGGERS_URI, "state") + now = _utc(clock["virtual_time"]) + if state.get("next_event_at") is not None and _utc(state["next_event_at"]) <= now: + try: + await _call(env, EXT_CLOCK_URI, "advance", {"virtual_time": _iso(now)}, attempts=1) + except Exception: + pass + elif not any(_busy(row, now) for row in state.get("triggers", [])): + waiting.discard(i) + if waiting: + self.ctx.heartbeat() + await asyncio.sleep(POLL_SECONDS) + return sorted(envs[i].env_id for i in waiting) + + async def _next_world_event(self) -> Optional[datetime]: + out = [] + for env in await self._armed(): + if env.supports(EXT_TRIGGERS_URI, "state"): + st = await _call(env, EXT_TRIGGERS_URI, "state") + if st.get("next_event_at"): + out.append(_utc(st["next_event_at"])) + return min(out) if out else None + + async def _notes_state(self, to: str) -> list[dict]: + out = [] + for env in self.notification_envs: + if env.supports(EXT_NOTIFICATIONS_URI, "state"): + out.append(await _call(env, EXT_NOTIFICATIONS_URI, "state", {"to": to})) + return out + + async def _due_count(self, to: str, now: datetime) -> int: + n = 0 + for st in await self._notes_state(to): + n += sum(1 for c in st.get("checks", []) if c.get("status") == "active" and _utc(c["next"]) <= now) + n += sum(1 for p in st.get("pending_notifications", []) if not p.get("at") or _utc(p["at"]) <= now) + return n + + async def _next_due(self, to: str) -> Optional[datetime]: + times = [_utc(st["next_due_at"]) for st in await self._notes_state(to) if st.get("next_due_at")] + return min(times) if times else None + + # ---- the user's schedule --------------------------------------------------------------------------------- + def _schedule(self) -> Optional[dict]: + return self.us.get("schedule") or None + + def _available(self, t: datetime) -> tuple[datetime, str]: + """When the user would first look at a message that arrived at ``t``, and in what mode (morning, work, free): + while awake and not busy. No schedule: at once, mode "now".""" + sched = self._schedule() + if not sched: + return t, "now" + tz = self.tz + wake, sleep = _hm(sched.get("wake", "07:30")), _hm(sched.get("sleep", "23:30")) + work = [_hm(x) for x in sched.get("work", ["09:00", "18:00"])] + work_days = set(sched.get("work_days", DAYS[:5])) + busy = [(datetime.fromisoformat(b["start"]).replace(tzinfo=tz), datetime.fromisoformat(b["end"]).replace(tzinfo=tz)) + for b in sched.get("busy") or []] + lt = t.astimezone(tz) + mode = None + for _ in range(4): + moved = False + for start, stop in busy: + if start <= lt < stop: + lt, moved = stop, True + hm = (lt.hour, lt.minute) + if hm >= sleep or hm < wake: + day = lt.date() if hm < wake else lt.date() + timedelta(days=1) + lt, mode, moved = datetime(day.year, day.month, day.day, *wake, tzinfo=tz), "morning", True + if not moved: + break + if mode is None: + hm = (lt.hour, lt.minute) + mode = "work" if DAYS[lt.weekday()] in work_days and work[0] <= hm < work[1] else "free" + return lt.astimezone(timezone.utc), mode + + def _latency(self, mode: str) -> timedelta: + rt = self.us.get("response_times") or {} + if mode == "now" or not rt: + return timedelta(0) + return timedelta(minutes=float(rt.get(mode, rt.get("free", 0)))) + + def _schedule_read(self) -> None: + if not self.unread or self.read_at is not None or self.gave_up: + return + first_at = min(_utc(m["at"]) for m in self.unread if m.get("at")) if any(m.get("at") for m in self.unread) \ + else datetime.now(timezone.utc) + at, mode = self._available(first_at) + if self.last_reply_at and at < self.last_reply_at: + at, mode = self._available(self.last_reply_at) + self.read_at, self.read_mode = at, mode + self.log("user_read_scheduled", read_at=_iso(at), mode=mode, unread=len(self.unread)) + + # ---- the loop ------------------------------------------------------------------------------------------- + async def loop(self, now: datetime) -> None: + step = self.step + first = self._effective_prompt() + self.you_sent.append({"at": _iso(now), "text": first}) + due = await take_due(self.notification_envs, self.recipient) if self.notification_envs else [] + await self.wake([*delivered_parts(due), self._user_part(now, first, opening=True)], + woke_by=[f"notification:{d['channel']}" for d in due] + ["user_request"], + notifications=[{k: d.get(k) for k in ("at", "channel", "source", "source_id", "to")} + for d in due]) + while self.end_reason is None: + if self.wakes >= step.max_wakes: + self.end_reason = "max_wakes" + break + now = await self.now() + # 1. a read that is due: the user sim reads what the agent sent + if self.read_at is not None and self.read_at <= now and self.unread: + await self.read(now) + continue + # 2. anything due for the agent: notifications, scripted user lines, held replies + lines = await self._take_user_says() + replies = [h for h in self.held if h["deliver_at"] <= now] + due = await self._due_count(self.recipient, now) + if lines or replies or due: + self.held = [h for h in self.held if h["deliver_at"] > now] + await self._wake_with(now, lines, replies, final=False) + continue + # 3. nothing due: jump to the next thing + candidates = {"world_event": await self._next_world_event(), + "agent_notification": await self._next_due(self.recipient), + "user_says": await self._next_due(USER_SAYS), + "held_reply": min((h["deliver_at"] for h in self.held), default=None), + "user_read": self.read_at if self.unread else None} + later = {k: v for k, v in candidates.items() if v is not None and v > now} + soonest = min(later.values()) if later else None + if self.end is not None and (soonest is None or soonest >= self.end): + await self._final(now) + break + if soonest is None: + self.end_reason = "nothing_can_wake_the_agent" + break + why = sorted(k for k, v in later.items() if v == soonest) + await self.advance(soonest, reason="+".join(why)) + + async def _final(self, now: datetime) -> None: + step = self.step + if self.end > now: + await self.advance(self.end, reason="end_time") + now = await self.now() + if not step.final: + self.end_reason = "end_time" + return + self.final_sent = True + lines = await self._take_user_says() + replies = [h for h in self.held if h["deliver_at"] <= now] + self.held = [] + lines.append({"at": _iso(now), "text": step.final, "source": "final"}) + await self._wake_with(now, lines, replies, final=True) + self.end_reason = self.end_reason or "end_time" + + def _effective_prompt(self) -> str: + parts = self.step._apply_seed(self.step._effective_parts(), self.ctx.metadata.get("seed", {})) + return "\n".join(p.get("text", "") for p in parts if p.get("kind") == "text") + + def _user_part(self, at: datetime, text: str, opening: bool = False) -> dict: + first = self.person["first"] + body = re.sub(rf"^\s*{re.escape(first)}\s*:\s*", "", text) + return {"kind": "text", "text": f"[{label_time(_iso(at))}] {first}: {body}", + "metadata": {"agentenv": {"source": "user", "at": _iso(at), "opening": opening}}} + + async def _take_user_says(self) -> list[dict]: + items = await take_due(self.notification_envs, USER_SAYS) if self.notification_envs else [] + out = [] + for it in items: + text = "\n\n".join(p["text"] for p in it["parts"] if p.get("kind") == "text") + out.append({"at": it.get("at"), "text": text, "source": "user_says", "trigger_id": it.get("source_id")}) + return out + + async def _wake_with(self, now: datetime, lines: list[dict], replies: list[dict], final: bool) -> None: + due = await take_due(self.notification_envs, self.recipient) if self.notification_envs else [] + parts = delivered_parts(due) + user_msgs = sorted([{"at": l["at"] or _iso(now), "text": l["text"], "kind": l.get("source", "user_says"), + "trigger_id": l.get("trigger_id")} for l in lines] + + [{"at": _iso(r["deliver_at"]), "text": r["text"], "kind": "user_reply"} for r in replies], + key=lambda m: _utc(m["at"])) + for m in user_msgs: + parts.append(self._user_part(_utc(m["at"]), m["text"])) + if m["kind"] != "user_reply": + self.you_sent.append({"at": m["at"], "text": m["text"]}) + self.log("scripted_user_line", virtual_time=m["at"], text=m["text"], kind=m["kind"], + trigger_id=m.get("trigger_id")) + else: + self.log("user_reply_delivered", virtual_time=m["at"], text=m["text"]) + if final and self.step.final_reminder: + parts.append({"kind": "text", "text": self.step.final_reminder.format(first=self.person["first"])}) + woke_by = [f"notification:{d['channel']}" for d in due] + [f"{m['kind']}" for m in user_msgs] + await self.wake(parts, woke_by=woke_by, notifications=[{k: d.get(k) for k in + ("at", "channel", "source", "source_id", "to")} + for d in due]) + + async def wake(self, parts: list[dict], woke_by: list[str], notifications: Optional[list] = None) -> None: + step, ctx = self.step, self.ctx + if step.wake_reminder: + parts = [*parts, {"kind": "text", "text": step.wake_reminder_text}] + self.wakes += 1 + sent_at = await self.now() + clock = await step._env_clock(self.clock_envs) + msg_id = uuid.uuid4().hex + + def record() -> None: + conversation_store.add_a2a_task(conversation_id=self.conversation_id, parts=parts, a2a_task_id=msg_id, + role="user", **clock) + self.per_turn_parts.append(list(parts)) + + self.log("wake", n=self.wakes, virtual_time=_iso(sent_at), woke_by=woke_by, notifications=notifications or [], + parts_text="\n".join(p.get("text", "") for p in parts if p.get("kind") == "text")) + t = time.time() + try: + task_id, result = await send_and_wait(self.target_url, parts, agent=self.agent, message_id=msg_id, + context_id=self.solver_context_id, + timeout_seconds=step.timeout_seconds, + poll_interval_seconds=step.poll_interval_seconds, before_send=record) + except TimeoutError as e: + self.error = f"agent turn timed out after {step.timeout_seconds}s: {e}" + self.end_reason = "agent_turn_timeout" + self.log("agent_turn_end", n=self.wakes, state="timeout", real_seconds=round(time.time() - t, 1)) + return + state = result["status"]["state"] + status_msg = (result.get("status") or {}).get("message") or {} + term = protocol.TerminalResponse.from_message(status_msg) + if term.tool_call_count is not None: + self.tool_calls = (self.tool_calls or 0) + term.tool_call_count + agent_parts = status_msg.get("parts") or [{"kind": "text", "text": term.response_text}] + messages, quiet = [], [] + for env in self.chat_envs: + take_id = uuid.uuid4().hex + got = await _call(env, EXT_USER_CHAT_URI, "take", {"take_id": take_id}) + messages += got.get("items") or [] + quiet += got.get("quiet") or [] + messages.sort(key=lambda m: m.get("at") or "") + # The conversation shows graders what the user saw (message_user) and, marked, what only went to the log. + recorded = [{"kind": "text", "text": f"[message_user · {label_time(m.get('at'))}] {m['text']}", + "metadata": {"agentenv": {"source": "message_user", "at": m.get("at")}}} for m in messages] + recorded += [{**p, "text": f"[plain reply, not shown to the user] {p['text']}", + "metadata": {"agentenv": {"source": "plain_reply"}}} if p.get("kind") == "text" else p + for p in agent_parts] + conversation_store.complete_a2a_task(conversation_id=self.conversation_id, parts=recorded, role="agent", + **(await step._env_clock(self.clock_envs))) + traj = None + card = self.agent.a2a_card or {} + traj_ext = A2AAgent.find_extension(card, A2AAgent.EXT_TRAJECTORY) + if traj_ext: + try: + traj = await step._fetch_trajectory(self.target_url, traj_ext, task_id, step._trajectory_prefix(), + msg_id, card=card, sandbox_type=self.agent.sandbox_type) + except Exception as e: + logger.warning(f"wake {self.wakes}: trajectory fetch failed (continuing): {e}") + self.per_turn_trajectory_urls.append(traj) + self.final_task_id = task_id + ended = await self.now() + plain = term.response_text or "" + flag = (not messages) and bool(plain.strip()) and bool(_ADDRESSED.search(plain)) and len(plain.strip()) > 40 + self.last_plain, self.last_messages = plain, messages + extra = {"stay_quiet_calls": len(quiet)} if step.stay_quiet else {} + self.log("agent_turn_end", n=self.wakes, state=str(state), virtual_time=_iso(ended), + virtual_seconds=round((ended - sent_at).total_seconds()), real_seconds=round(time.time() - t, 1), + tool_calls=term.tool_call_count, plain_reply=plain, + plain_reply_flagged_for_user=flag, messages_to_user=len(messages), trajectory_url=traj, **extra) + for m in messages: + self.log("message_to_user", virtual_time=m.get("at"), text=m["text"], wake=self.wakes) + self.unread.append({"at": m.get("at"), "text": m["text"]}) + self.all_messages.append({"at": m.get("at"), "text": m["text"], "wake": self.wakes}) + if state == TaskState.failed or str(state) == "failed": + self.error = term.error_message or term.response_text or "agent turn failed" + self.end_reason = "agent_turn_failed" + return + if step.nudge and not messages and not quiet and plain.strip(): + await self._nudge(plain) + stuck = await self.settle() + if stuck: + self.log("settle_timeout", envs=stuck) + self._schedule_read() + + async def _nudge(self, closing: str) -> None: + """Opt-in (``nudge``): the wake ended with closing text and neither message_user nor stay_quiet, so give the + agent one more short turn with ``nudge_text``; record what the closing text was and what it did after.""" + step = self.step + self.nudges += 1 + parts = [{"kind": "text", "text": step.nudge_text}] + sent_at = await self.now() + clock = await step._env_clock(self.clock_envs) + msg_id = uuid.uuid4().hex + + def record() -> None: + conversation_store.add_a2a_task(conversation_id=self.conversation_id, parts=parts, a2a_task_id=msg_id, + role="user", **clock) + self.per_turn_parts.append(list(parts)) + + t = time.time() + try: + task_id, result = await send_and_wait(self.target_url, parts, agent=self.agent, message_id=msg_id, + context_id=self.solver_context_id, + timeout_seconds=step.timeout_seconds, + poll_interval_seconds=step.poll_interval_seconds, before_send=record) + except TimeoutError as e: + self.error = f"agent nudge turn timed out after {step.timeout_seconds}s: {e}" + self.end_reason = "agent_turn_timeout" + self.log("nudge", n=self.nudges, wake=self.wakes, virtual_time=_iso(sent_at), closing_text=closing, + state="timeout", outcome="timeout", real_seconds=round(time.time() - t, 1)) + return + state = result["status"]["state"] + status_msg = (result.get("status") or {}).get("message") or {} + term = protocol.TerminalResponse.from_message(status_msg) + if term.tool_call_count is not None: + self.tool_calls = (self.tool_calls or 0) + term.tool_call_count + agent_parts = status_msg.get("parts") or [{"kind": "text", "text": term.response_text}] + messages, quiet = [], [] + for env in self.chat_envs: + got = await _call(env, EXT_USER_CHAT_URI, "take", {"take_id": uuid.uuid4().hex}) + messages += got.get("items") or [] + quiet += got.get("quiet") or [] + messages.sort(key=lambda m: m.get("at") or "") + recorded = [{"kind": "text", "text": f"[message_user · {label_time(m.get('at'))}] {m['text']}", + "metadata": {"agentenv": {"source": "message_user", "at": m.get("at")}}} for m in messages] + recorded += [{**p, "text": f"[plain reply, not shown to the user] {p['text']}", + "metadata": {"agentenv": {"source": "plain_reply"}}} if p.get("kind") == "text" else p + for p in agent_parts] + conversation_store.complete_a2a_task(conversation_id=self.conversation_id, parts=recorded, role="agent", + **(await step._env_clock(self.clock_envs))) + traj = None + card = self.agent.a2a_card or {} + traj_ext = A2AAgent.find_extension(card, A2AAgent.EXT_TRAJECTORY) + if traj_ext: + try: + traj = await step._fetch_trajectory(self.target_url, traj_ext, task_id, step._trajectory_prefix(), + msg_id, card=card, sandbox_type=self.agent.sandbox_type) + except Exception as e: + logger.warning(f"nudge after wake {self.wakes}: trajectory fetch failed (continuing): {e}") + self.per_turn_trajectory_urls.append(traj) + self.final_task_id = task_id + plain = term.response_text or "" + outcome = "message_user" if messages else "stay_quiet" if quiet else "neither" + self.log("nudge", n=self.nudges, wake=self.wakes, virtual_time=_iso(sent_at), closing_text=closing, + state=str(state), outcome=outcome, messages_to_user=len(messages), stay_quiet_calls=len(quiet), + plain_reply_after=plain, tool_calls=term.tool_call_count, real_seconds=round(time.time() - t, 1), + trajectory_url=traj) + if messages: + self.last_plain, self.last_messages = plain, messages + for m in messages: + self.log("message_to_user", virtual_time=m.get("at"), text=m["text"], wake=self.wakes, after_nudge=True) + self.unread.append({"at": m.get("at"), "text": m["text"]}) + self.all_messages.append({"at": m.get("at"), "text": m["text"], "wake": self.wakes}) + if state == TaskState.failed or str(state) == "failed": + self.error = term.error_message or term.response_text or "agent nudge turn failed" + self.end_reason = "agent_turn_failed" + + async def read(self, now: datetime) -> None: + phone = [] + if self.notification_envs: + for it in await take_due(self.notification_envs, USER_PHONE): + phone.append({"at": it.get("at"), "channel": it["channel"], + "text": "\n\n".join(p["text"] for p in it["parts"] if p.get("kind") == "text")}) + read = {"time": _iso(now), "assistant_messages": list(self.unread), "phone": phone, + "you_sent": list(self.you_sent)} + self.unread, self.you_sent, self.read_at = [], [], None + calls = [] + + async def ask(text: str) -> dict: + got = await self._ask_user_sim(text) + calls.append(got) + return got + + t = time.time() + ans = await self.plugin.answer(self.version, self.person, self.history, read, ask) + self.history.append({"read": read, "answer": ans}) + mode = self.read_mode + latency = self._latency(mode) + flagged = len(ans.get("message") or "") > 1200 or bool(re.search(r"^\s*(#|\|)", ans.get("message") or "", re.M)) + entry = self.log("user_sim_read", virtual_time=_iso(now), version=self.version, read=read, + input=ans.get("input"), raw=ans.get("raw"), outcome=ans.get("outcome"), + message=ans.get("message"), used=ans.get("used"), score=ans.get("score"), + schema_errors=ans.get("schema_errors"), mode=mode, + real_seconds=round(time.time() - t, 1), looks_like_assistant=flagged, + user_sim_error=(calls[-1].get("error") if calls else None), + user_sim_trajectory_urls=[c.get("trajectory_url") for c in calls], + user_sim_turns=ans.get("turns", len(calls)), shown=ans.get("shown"), + past_turn_cap=ans.get("past_turn_cap")) + outcome = ans.get("outcome") + if outcome == "reply" and (ans.get("message") or "").strip(): + deliver = now + latency + self.held.append({"deliver_at": deliver, "text": ans["message"]}) + self.last_reply_at = deliver + self.replies += 1 + entry["deliver_at"] = _iso(deliver) + max_replies = self.us.get("max_replies") + if max_replies and self.replies >= int(max_replies): + self.gave_up = True + self.log("user_gave_up", replies=self.replies) + elif outcome == "done": + self.end_reason = "user_done" + if calls and calls[-1].get("error"): + self.end_reason = self.end_reason or "user_sim_error" + + async def _ask_user_sim(self, text: str) -> dict: + msg_id = uuid.uuid4().hex + try: + user_task_id, result = await send_and_wait( + self.user_url, [{"kind": "text", "text": text}], + agent=self.user_agent if self.user_agent.sandbox_id else None, message_id=msg_id, + context_id=self.user_context_id, timeout_seconds=self.step.user_timeout_seconds, + poll_interval_seconds=self.step.poll_interval_seconds) + except TimeoutError as e: + return {"text": None, "structured": None, "error": f"timeout: {e}"} + status = (result.get("status") or {}) + if status.get("state") != TaskState.completed and str(status.get("state")) != "completed": + return {"text": None, "structured": None, "error": f"state {status.get('state')}"} + msg = status.get("message") or {} + term = protocol.TerminalResponse.from_message(msg) + raw = next((p["text"] for p in msg.get("parts") or [] if p.get("kind") == "text"), term.response_text) + structured = term.structured_output + if structured is None: + try: + structured = json.loads(raw) + except (TypeError, ValueError): + structured = None + traj = None + card = self.user_agent.a2a_card or {} + traj_ext = A2AAgent.find_extension(card, A2AAgent.EXT_TRAJECTORY) + if traj_ext: + try: + traj = await self.step._fetch_trajectory(self.user_url, traj_ext, user_task_id, + self.step._trajectory_prefix() + "user_sim/", msg_id, + card=card, sandbox_type=self.user_agent.sandbox_type) + except Exception as e: + logger.warning(f"user sim trajectory fetch failed (continuing): {e}") + return {"text": raw, "structured": structured if isinstance(structured, dict) else None, + "trajectory_url": traj} + + # ---- the end -------------------------------------------------------------------------------------------- + async def finish(self) -> None: + step, ctx = self.step, self.ctx + now = await self.now() + self.log("end", reason=self.end_reason, virtual_time=_iso(now), wakes=self.wakes, replies=self.replies, + unread_at_end=len(self.unread), held_at_end=len(self.held)) + conv = conversation_store.get_conversation(self.conversation_id) + if conv and conv.get("status") == "active": + conversation_store.mark_closed(self.conversation_id) + await step._persist_env_trigger_state(ctx) + counts = { + "wakes": self.wakes, + "messages_to_user": len(self.all_messages), + "user_sim_reads": sum(1 for e in self.timeline if e["event"] == "user_sim_read"), + "user_replies": self.replies, + "user_no_replies": sum(1 for e in self.timeline if e["event"] == "user_sim_read" + and e.get("outcome") == "no_reply"), + "scripted_user_lines": sum(1 for e in self.timeline if e["event"] == "scripted_user_line"), + "clock_jumps": sum(1 for e in self.timeline if e["event"] == "clock_jump"), + "plain_replies_flagged": sum(1 for e in self.timeline if e.get("plain_reply_flagged_for_user")), + "silent_wakes": sum(1 for e in self.timeline if e["event"] == "agent_turn_end" + and not e.get("messages_to_user")), + } + if step.nudge: + nudges = [e for e in self.timeline if e["event"] == "nudge"] + counts["nudges"] = len(nudges) + counts["nudge_outcomes"] = {o: sum(1 for e in nudges if e.get("outcome") == o) + for o in sorted({e.get("outcome") for e in nudges})} + if step.stay_quiet: + counts["stay_quiet_wakes"] = sum(1 for e in self.timeline if e["event"] == "agent_turn_end" + and e.get("stay_quiet_calls")) + record = {"step_id": step.id, "end_reason": self.end_reason, "error": self.error, "counts": counts, + "user_sim": {"name": step.user_sim, "version": self.version, "model": self.us.get("model"), + "agent_name": self.us.get("agent_name"), **self.user_setup}, + "end_time": _iso(self.end) if self.end else None, "timeline": self.timeline} + if step.stay_quiet or step.nudge or step.wake_reminder: + record["settings"] = {"stay_quiet": step.stay_quiet, "nudge": step.nudge, + "wake_reminder": step.wake_reminder} + try: + store = get_config().get_object_store() + key = (f"{get_config().get_artifact_key_prefix()}always_on_timeline/" + f"instance_id={ctx.instance_id or 'adhoc'}/{step.id}-{uuid.uuid4().hex[:8]}.json") + record["object_url"] = await asyncio.to_thread(store.put, key, + json.dumps(record, indent=1, default=str).encode(), + content_type="application/json") + except Exception as e: + logger.warning(f"always_on: could not store the timeline: {e}") + ctx.metadata.setdefault(ALWAYS_ON_KEY, {})[step.id] = record + final_text = "\n\n".join([m["text"] for m in self.last_messages] + ([self.last_plain] if self.last_plain else [])) + traj = next((u for u in reversed(self.per_turn_trajectory_urls) if u), None) + ctx.prompt_responses.append(PromptResponse( + prompt_id=step.prompt_id, response=final_text, prompt_text=step.prompt, + agent_trajectory_object_url=traj, agent_trajectory_object_prefix=step._trajectory_prefix(), + target_agent_per_turn_trajectory_object_urls=self.per_turn_trajectory_urls or None, + source_agent_per_turn_prompt_parts=self.per_turn_parts or None, tool_call_count=self.tool_calls, + model=self.model, error_message=self.error, agent_session_id=self.final_task_id, + a2a_context_id=self.solver_context_id, agent_name=step.agent_name, step_id=step.id)) + + +# ================================================================================================================== +async def _call(env, uri: str, method: str, params: Optional[dict] = None, *, attempts: int = 6, + timeout: int = 60) -> Any: + """A gateway control call, retried on a transient failure (a busy laptop's gateway can take seconds to answer). + Every call the step retries is safe to repeat: reads, config, advance (checked against the clock after) and takes + with a take_id.""" + last: Optional[Exception] = None + for attempt in range(attempts): + try: + return await env.invoke(uri, method, params, timeout=timeout) + except EnvCapabilityUnsupported: + raise + except Exception as e: + last = e + if attempt + 1 < attempts: + logger.warning(f"always_on: {method} on {getattr(env, 'env_id', env)} failed ({type(e).__name__}: " + f"{e}); retry {attempt + 1}/{attempts - 1}") + await asyncio.sleep(min(2 ** attempt, 20)) + raise last + + +def _busy(row: dict, now: datetime) -> bool: + if row.get("evaluating"): + return True + if row.get("status") not in IN_FLIGHT and not row.get("pending"): + return False + return not (row.get("waiting") and row.get("resumes_at") and _utc(row["resumes_at"]) > now) + + +def _hm(text: str) -> tuple[int, int]: + h, m = text.split(":") + return int(h), int(m) + + +def _utc(text: str) -> datetime: + at = datetime.fromisoformat(str(text).replace("Z", "+00:00")) + if at.tzinfo is None: + raise ValueError(f"{text!r} has no UTC offset") + return at.astimezone(timezone.utc) + + +def _iso(value: Optional[datetime]) -> Optional[str]: + return value.astimezone(timezone.utc).isoformat().replace("+00:00", "Z") if value else None + + +def _end_instant(end_time: str, t0: datetime) -> datetime: + if isinstance(end_time, str) and end_time.startswith("P"): + m = re.fullmatch(r"P(?:(\d+)W)?(?:(\d+)D)?(?:T(?:(\d+)H)?(?:(\d+)M)?(?:(\d+(?:\.\d+)?)S)?)?", end_time) + if m is None or not any(m.groups()): + raise ValueError(f"invalid ISO 8601 duration {end_time!r}") + w, d, h, mi, s = (float(g or 0) for g in m.groups()) + return t0 + timedelta(weeks=w, days=d, hours=h, minutes=mi, seconds=s) + return _utc(end_time) diff --git a/src/agent_env/task_step/task_steps/user_sim_versions.py b/src/agent_env/task_step/task_steps/user_sim_versions.py new file mode 100644 index 000000000..607554314 --- /dev/null +++ b/src/agent_env/task_step/task_steps/user_sim_versions.py @@ -0,0 +1,220 @@ +"""The user sim plug-in the always_on step calls by default: each version decides the user sim's system prompt and +answer format (``setup``), renders its own input for one read and parses the answer (``answer``). + +The interface, which any plug-in a ``user_sim`` step names (a file artifact) must offer: + + setup(version, person) -> {"system_prompt": str | None, "output_format": dict | None} + async answer(version, person, history, read, ask) -> {"outcome": "reply" | "no_reply" | "done", "message": str, + "used": str | None, "input": str, "raw": ..., ...} + +``person`` is the user_sim step's settings (name, first, time_zone, description, versions, ...). ``history`` is every +earlier read, oldest first, each ``{"read": , "answer": }``. ``read`` is this read: +``{"time": RFC 3339, "assistant_messages": [{"at", "text"}], "phone": [{"at", "channel", "text"}], +"you_sent": [{"at", "text"}]}``. ``ask(text)`` sends the rendered text to the deployed user sim (one session per run) +and returns ``{"text": raw text, "structured": parsed object or None}``. + +Versions here (from user-sim-h2h/v2/PLAN.md): today, ours, ours_no_feed, ours_no_labels, ours_no_time, ours_no_used, +ours_always_replies. "today + fixes" and "ours + fixes" need per-person lines the versions module will supply. +""" +from __future__ import annotations + +import json +from datetime import datetime, timezone +from typing import Any, Awaitable, Callable, Optional +from zoneinfo import ZoneInfo + +# prompt_agent's default user-sim output format, used when a task sets none. +ML_DEFAULT_FORMAT = {"type": "json_schema", "schema": {"type": "object", "properties": { + "message": {"type": "string", "description": "Your next reply"}, + "done": {"type": "boolean", "description": "Set true when you're satisfied and the conversation should end."}}, + "required": ["message", "done"]}} + +_OURS_PROPS = { + "outcome": {"type": "string", "enum": ["reply", "no_reply", "done"]}, + "message": {"type": "string"}, + "used": {"type": "string"}, + "score": {"type": "integer", "description": "Only with outcome done: your 1-5 rating"}, +} +FOOTER_Q = "Your AI assistant sent you this. As {first}, decide: reply, don't reply, or say you're done." +FOOTER_Q_ALWAYS = "Your AI assistant sent you this. As {first}, decide: reply, or say you're done." + +VERSIONS = ("today", "ours", "ours_no_feed", "ours_no_labels", "ours_no_time", "ours_no_used", "ours_always_replies") + + +def _ours_schema(version: str) -> dict: + props = dict(_OURS_PROPS) + required = ["outcome", "message", "used"] + if version == "ours_no_used": + props.pop("used") + required.remove("used") + if version == "ours_always_replies": + props["outcome"] = {"type": "string", "enum": ["reply", "done"]} + return {"type": "json_schema", "schema": {"type": "object", "additionalProperties": False, "properties": props, + "required": required}} + + +def _json_line(version: str) -> str: + outcomes = '"reply" | "done"' if version == "ours_always_replies" else '"reply" | "no_reply" | "done"' + used = '' if version == "ours_no_used" else ', "used": "..."' + return '{"outcome": ' + outcomes + ', "message": "..."' + used + '}' + + +def setup(version: str, person: dict) -> dict: + if version not in VERSIONS: + raise ValueError(f"unknown user sim version {version!r}; this plug-in has {list(VERSIONS)}") + if version == "today": + return {"system_prompt": person.get("description"), + "output_format": person.get("output_format") or ML_DEFAULT_FORMAT} + ours = (person.get("versions") or {}).get("ours") or {} + prompt = ours.get("system_prompt") + if not prompt: + raise ValueError(f"user sim {person.get('name')!r} has no versions.ours.system_prompt for version {version!r}") + if version == "ours_no_used": + prompt = _drop_used(prompt) + if version == "ours_always_replies": + prompt = _drop_no_reply(prompt) + return {"system_prompt": prompt, "output_format": _ours_schema(version)} + + +def _drop_used(prompt: str) -> str: + """Remove the lines that explain the `used` field (the tagged persona lines stay).""" + return "\n".join(ln for ln in prompt.splitlines() if '"used"' not in ln and "`used`" not in ln + and not ln.lower().lstrip().startswith("used:")) + + +def _drop_no_reply(prompt: str) -> str: + return prompt.replace('"reply" | "no_reply" | "done"', '"reply" | "done"').replace( + 'reply, don\'t reply, or say you\'re done', "reply, or say you're done") + + +# ---- rendering ---------------------------------------------------------------------------------------------------- +def _utc(text: str) -> datetime: + return datetime.fromisoformat(text.replace("Z", "+00:00")).astimezone(timezone.utc) + + +def _local(text: Optional[str], tz: ZoneInfo) -> Optional[datetime]: + return _utc(text).astimezone(tz) if text else None + + +def _hm(d: datetime) -> str: + return d.strftime("%I:%M %p").lstrip("0") + + +def _stamp(t: datetime, now: datetime) -> str: + days = (now.date() - t.date()).days + if days == 0: + return _hm(t) + if days < 6: + return f"{t.strftime('%a')} {_hm(t)}" + return f"{t.strftime('%a %b')} {t.day}, {_hm(t)}" + + +def render_today(read: dict) -> str: + """ML's setup: the assistant's raw message text, nothing else (several messages joined by a blank line).""" + return "\n\n".join(m["text"].strip() for m in read["assistant_messages"]) + + +def render_ours(version: str, person: dict, history: list[dict], read: dict) -> str: + tz = ZoneInfo(person.get("time_zone") or "UTC") + now = _local(read["time"], tz) + first = person.get("first") or person.get("name") or "you" + labels = version != "ours_no_labels" + out: list[str] = [] + if version != "ours_no_time": + out.append(f"It's {now.strftime('%a, %b')} {now.day}, {_hm(now)}.") + last = _last_messaged(history, read) + if last: + lt = _local(last, tz) + days = (now.date() - lt.date()).days + when = (f"at {_hm(lt)}" if days == 0 else f"on {lt.strftime('%a')} at {_hm(lt)}" if days < 6 + else f"on {lt.strftime('%a %b')} {lt.day} at {_hm(lt)}") + out.append(f"You last messaged your AI assistant {when}.") + if version != "ours_no_feed": + if read.get("phone"): + out.append("\nNEW ON YOUR PHONE (since you last checked):" if labels else "") + for e in read["phone"]: + head, *rest = e["text"].split("\n") + t = _local(e.get("at"), tz) or now + out.append(f" [{_stamp(t, now)} · {e['channel']}] {head}" + "".join(f"\n {x}" for x in rest)) + earlier = [e for h in history for e in h["read"].get("phone") or []] + if earlier: + out.append("\nEARLIER ON YOUR PHONE (you've already seen these):" if labels else "") + for e in earlier: + t = _local(e.get("at"), tz) or now + out.append(f" [{_stamp(t, now)} · {e['channel']}] {e['text'].splitlines()[0] if e['text'] else ''}") + if read.get("you_sent"): + out.append("\nYOU SENT YOUR AI ASSISTANT:" if labels else "") + for s in read["you_sent"]: + t = _local(s.get("at"), tz) + out.append((f" [{_stamp(t, now)}] " if t else " ") + s["text"]) + out.append("\nNEW FROM YOUR AI ASSISTANT (this is what you're answering):" if labels else "") + for m in read["assistant_messages"]: + t = _local(m.get("at"), tz) + body = "\n".join(" " + ln if ln.strip() else "" for ln in m["text"].strip().splitlines()) + out.append((f" [{_stamp(t, now)}]\n" if t else "") + body) + q = FOOTER_Q_ALWAYS if version == "ours_always_replies" else FOOTER_Q + out.append("\n" + q.format(first=first) + "\nAnswer in JSON: " + _json_line(version)) + return "\n".join(x for x in out if x != "" or labels) + + +def _last_messaged(history: list[dict], read: dict) -> Optional[str]: + """When the person last sent their assistant anything before this read: their scripted lines (this read's + included, as the h2h runner did) and their earlier replies.""" + times = [s["at"] for h in history for s in h["read"].get("you_sent") or [] if s.get("at")] + times += [s["at"] for s in read.get("you_sent") or [] if s.get("at")] + times += [h["read"]["time"] for h in history if (h.get("answer") or {}).get("outcome") in ("reply", "done")] + times = [t for t in times if _utc(t) <= _utc(read["time"])] + return max(times, key=_utc) if times else None + + +# ---- answers ------------------------------------------------------------------------------------------------------ +def _schema_errors(obj: Any, schema: dict) -> list[str]: + if not isinstance(obj, dict): + return ["not an object"] + errs = [f"missing {k}" for k in schema.get("required", []) if k not in obj] + props = schema.get("properties", {}) + if schema.get("additionalProperties") is False: + errs += [f"extra key {k}" for k in obj if k not in props] + for k, v in obj.items(): + p = props.get(k) or {} + if "enum" in p and v not in p["enum"]: + errs.append(f"{k}: {v!r} not in enum") + return errs + + +def _parse_text(text: str) -> Optional[dict]: + s = (text or "").replace("```json", "").replace("```", "") + dec, i, found = json.JSONDecoder(), s.find("{"), None + while i != -1: + try: + obj, end = dec.raw_decode(s, i) + if isinstance(obj, dict) and ("message" in obj or "outcome" in obj): + found = obj + i = s.find("{", end) + except ValueError: + i = s.find("{", i + 1) + return found + + +async def answer(version: str, person: dict, history: list[dict], read: dict, + ask: Callable[[str], Awaitable[dict]]) -> dict: + text = render_today(read) if version == "today" else render_ours(version, person, history, read) + got = await ask(text) + obj = got.get("structured") + if not isinstance(obj, dict): + obj = _parse_text(got.get("text") or "") + fmt = setup(version, person)["output_format"] + errors = _schema_errors(obj, fmt["schema"]) if obj is not None else ["no JSON object in the answer"] + out = {"version": version, "input": text, "raw": got.get("text"), "structured": obj, "schema_errors": errors} + if obj is None: + # A malformed answer reaches the assistant as raw text, as prompt_agent does today. + return {**out, "outcome": "reply", "message": got.get("text") or "", "used": None} + if version == "today": + done = obj.get("done") is True + return {**out, "outcome": "done" if done else "reply", "message": str(obj.get("message") or ""), + "used": None, "done": obj.get("done")} + outcome = obj.get("outcome") + if outcome not in ("reply", "no_reply", "done"): + outcome = "reply" + return {**out, "outcome": outcome, "message": str(obj.get("message") or ""), "used": obj.get("used"), + "score": obj.get("score")} From c03dd4c755e6cfc81f6e85507040455799c24fbd Mon Sep 17 00:00:00 2001 From: Sarah Chitty Date: Mon, 9 Nov 2026 00:12:30 -0800 Subject: [PATCH 03/53] always_on: message_user, stay_quiet, the nudge and the wake reminder on by default (FD-5365) - stay_quiet, nudge and wake_reminder default to on; a task turns each off with false. The run record always carries the settings, and nudges are counted with their outcomes. - The agent is told the rules: appended to the step's system prompt, or, without one, opening the first wake. - An empty reply, or a harness's "(The assistant's reply was empty.)" stand-in, is not closing text, so it gets no nudge. - The regex that flagged closing text "meant for the user" is gone; the nudge replaces it. - Lines the step adds to a wake (rules, reminder, final reminder, nudge) carry metadata.agentenv.source, so a reader can tell them from the user's and the world's. - The user-chat card advertises stay_quiet in config. Tests: gateway user chat (message_user and stay_quiet through a deny-all), and the step's message rules on an in-process gateway, including that neither the nudge nor the reminder reaches the user sim. Co-Authored-By: Claude Opus 5.5 --- src/agent_env/env/gateway/constants.py | 5 +- src/agent_env/env/gateway/user_chat.py | 16 +- .../task_step/task_steps/always_on.py | 67 +++++--- tst/unit/env/gateway/user_chat_test.py | 114 +++++++++++++ tst/unit/task_step/always_on_harness.py | 144 ++++++++++++++++ tst/unit/task_step/test_always_on_messages.py | 156 ++++++++++++++++++ 6 files changed, 469 insertions(+), 33 deletions(-) create mode 100644 tst/unit/env/gateway/user_chat_test.py create mode 100644 tst/unit/task_step/always_on_harness.py create mode 100644 tst/unit/task_step/test_always_on_messages.py diff --git a/src/agent_env/env/gateway/constants.py b/src/agent_env/env/gateway/constants.py index 38217b2cf..f37d2c5cf 100644 --- a/src/agent_env/env/gateway/constants.py +++ b/src/agent_env/env/gateway/constants.py @@ -273,10 +273,11 @@ def random_mcp_server_name() -> str: }, { "uri": EXT_USER_CHAT_URI, - "description": "User chat (urn:agentenv:user-chat/v1): the message_user tool, for the roles config names, is how an always-on agent reaches its user; each message is queued with its virtual time until the harness takes it. Built in: role rules don't hide it from a configured role.", + "description": "User chat (urn:agentenv:user-chat/v1): the message_user tool, for the roles config names, is how an always-on agent reaches its user; each message is queued with its virtual time until the harness takes it. With stay_quiet the roles also get stay_quiet, which sends nothing and is recorded for the next take. Built in: role rules don't hide them from a configured role.", "params": {"endpoint": "/user_chat/state", "methods": { "config": {"method": "POST", "endpoint": "/user_chat/config", "request": {"type": "object", "properties": { - "roles": {"type": "array", "items": {"type": "string"}}, "user_name": {"type": "string"}}}}, + "roles": {"type": "array", "items": {"type": "string"}}, "user_name": {"type": "string"}, + "stay_quiet": {"type": "boolean"}}}}, "state": {"method": "GET", "endpoint": "/user_chat/state", "request": {"type": "object", "properties": {}}}, "take": {"method": "POST", "endpoint": "/user_chat/take", "request": {"type": "object", "properties": { "take_id": {"type": "string"}}}}, diff --git a/src/agent_env/env/gateway/user_chat.py b/src/agent_env/env/gateway/user_chat.py index 8d87d936a..82f9993c7 100644 --- a/src/agent_env/env/gateway/user_chat.py +++ b/src/agent_env/env/gateway/user_chat.py @@ -1,11 +1,13 @@ """The user chat (urn:agentenv:user-chat/v1): how an always-on agent reaches its user. -The roles a task names in `config` get one tool, `message_user`. Each call is queued with the virtual time it was -sent at; nothing here reaches the user by itself: the always_on step `take`s what was sent after each agent turn and -hands it to the user sim when the user would read it. Whatever else the agent writes (its plain reply) never reaches -the user. The tool is built in: once a role is named, role rules from modify_env_tool_access don't hide it. - -`take` hands each message out once: {"id", "from", "at", "text"}. A `take_id` makes a retry safe. +The roles a task names in `config` get `message_user`, and with `stay_quiet: true` also `stay_quiet`. Each message is +queued with the virtual time it was sent at; nothing here reaches the user by itself: the always_on step `take`s what +was sent after each agent turn and hands it to the user sim when the user would read it. Whatever else the agent writes +(its plain reply) never reaches the user. The tools are built in: once a role is named, role rules from +modify_env_tool_access don't hide them. + +`take` hands each message out once: {"id", "from", "at", "text"}, plus, with stay_quiet on, the `quiet` calls since the +last take. A `take_id` makes a retry safe. """ from __future__ import annotations @@ -20,7 +22,7 @@ from .clock import Clock, _iso MESSAGE_USER_TOOL_NAME = "message_user" -STAY_QUIET_TOOL_NAME = "stay_quiet" # opt-in (config stay_quiet: true): "nothing to tell the user this wake" +STAY_QUIET_TOOL_NAME = "stay_quiet" _MAX_TEXT_CHARS = 20000 _MAX_TAKE_ID_CHARS = 128 _KEPT_TAKES = 64 diff --git a/src/agent_env/task_step/task_steps/always_on.py b/src/agent_env/task_step/task_steps/always_on.py index 92c2a2e09..c0500f14a 100644 --- a/src/agent_env/task_step/task_steps/always_on.py +++ b/src/agent_env/task_step/task_steps/always_on.py @@ -8,7 +8,8 @@ - The first wake is the user's request. Each wake opens with what came due for the agent on notifications/v1 (a trigger's ``notify``, a scheduled check), labelled ``[time · channel]``, then the user's own messages. - Only what the agent sends with the gateway's ``message_user`` tool (user-chat/v1) reaches the user; its plain reply - goes to the log, flagged when it looks meant for the user. + goes to the log. Each wake should end with ``message_user`` or ``stay_quiet``: one that ends with closing text and + neither gets one ``nudge`` turn, and every wake ends with a reminder. All three are on unless the task turns them off. - The user sim is asked only after the agent sent something, at the time the user would read it (``schedule``), with everything sent meanwhile; its answer is reply / no_reply / done, and a reply is held for the response time. - Scripted user lines are triggers' ``notify`` actions to the ``user_says`` recipient: sent to the agent as the @@ -69,11 +70,13 @@ DAYS = ("mon", "tue", "wed", "thu", "fri", "sat", "sun") FINAL_REMINDER = ("(This is the last time you'll be woken for this job. Only what you send with message_user reaches " "{first}; your normal reply is not shown to them.)") -# Opt-in settings (all off unless a task turns them on): stay_quiet, nudge, wake_reminder. NUDGE_TEXT = ("Your closing text was not delivered. If the user should see it, send it with message_user; otherwise " "call stay_quiet.") WAKE_REMINDER_TEXT = "Reminder: only message_user reaches the user." -_ADDRESSED = re.compile(r"\?\s*$|^(hi|hey|hello)\b|\byou\b|\byour\b", re.I | re.M) +MESSAGE_RULES = ("The user never sees your normal replies. The only way to reach them is message_user. Text outside it " + "goes to a private log.") +QUIET_RULE = "If nothing is worth saying, call stay_quiet." +EMPTY_REPLIES = frozenset({"(The assistant's reply was empty.)"}) # a harness's stand-in for an empty final reply # ================================================================================================================== @@ -147,8 +150,8 @@ class AlwaysOnTaskStep(PromptAgentTaskStep): def __init__(self, id: str, version: Optional[int], *, user_sim: str, end_time: Optional[str] = None, final: Optional[str] = None, max_wakes: int = 60, settle_timeout_seconds: float = 300, user_timeout_seconds: int = 300, final_reminder: Optional[str] = FINAL_REMINDER, - stay_quiet: bool = False, nudge: bool = False, nudge_text: str = NUDGE_TEXT, - wake_reminder: bool = False, wake_reminder_text: str = WAKE_REMINDER_TEXT, + stay_quiet: bool = True, nudge: bool = True, nudge_text: str = NUDGE_TEXT, + wake_reminder: bool = True, wake_reminder_text: str = WAKE_REMINDER_TEXT, **prompt_fields: Any): for unsupported in ("max_conversation_turns", "user_agent_name", "user_a2a_url", "post_turn_hooks", "user_output_format", "user_model", "snapshot_config"): @@ -165,10 +168,10 @@ def __init__(self, id: str, version: Optional[int], *, user_sim: str, end_time: self.settle_timeout_seconds = float(settle_timeout_seconds) self.user_timeout_seconds = int(user_timeout_seconds) self.final_reminder = final_reminder - self.stay_quiet = bool(stay_quiet) # the agent also gets stay_quiet() (gateway user-chat config) - self.nudge = bool(nudge) # one more short turn when closing text went undelivered + self.stay_quiet = bool(stay_quiet) + self.nudge = bool(nudge) self.nudge_text = nudge_text - self.wake_reminder = bool(wake_reminder) # each wake's message ends with wake_reminder_text + self.wake_reminder = bool(wake_reminder) self.wake_reminder_text = wake_reminder_text _OWN = ("user_sim", "end_time", "final", "max_wakes", "settle_timeout_seconds", "user_timeout_seconds", @@ -346,7 +349,7 @@ async def _configure_agents(self) -> None: self.model = model overrides = ctx.metadata.get("user_overrides", {}) card = self.agent.a2a_card or {} - desired = {"model": model, "system_prompt": step.system_prompt, "effort": effort, + desired = {"model": model, "system_prompt": self._system_prompt(), "effort": effort, "harness": ctx.agent_harness or step.harness, "max_turns": overrides.get("agent_max_turns") or step.max_turns, "max_thinking_tokens": overrides.get("agent_max_thinking_tokens") or step.max_thinking_tokens, @@ -376,6 +379,15 @@ async def _configure_agents(self) -> None: if "system_prompt" not in accepted and setup.get("system_prompt"): logger.warning("always_on: the user sim agent's card did not accept system_prompt; it keeps its deploy one") + def rules(self) -> str: + """What the agent is told about reaching the user.""" + return " ".join([MESSAGE_RULES, *([QUIET_RULE] if self.step.stay_quiet else [])]) + + def _system_prompt(self) -> Optional[str]: + """The step's system prompt with the rules after it; without one, the agent keeps its own and the first wake + opens with the rules.""" + return f"{self.step.system_prompt.rstrip()}\n\n{self.rules()}" if self.step.system_prompt else None + # ---- clock and world ------------------------------------------------------------------------------------ async def _armed(self) -> list: armed = [] @@ -523,7 +535,8 @@ async def loop(self, now: datetime) -> None: first = self._effective_prompt() self.you_sent.append({"at": _iso(now), "text": first}) due = await take_due(self.notification_envs, self.recipient) if self.notification_envs else [] - await self.wake([*delivered_parts(due), self._user_part(now, first, opening=True)], + rules = [] if step.system_prompt else [_step_part(self.rules(), "rules")] + await self.wake([*rules, *delivered_parts(due), self._user_part(now, first, opening=True)], woke_by=[f"notification:{d['channel']}" for d in due] + ["user_request"], notifications=[{k: d.get(k) for k in ("at", "channel", "source", "source_id", "to")} for d in due]) @@ -611,16 +624,16 @@ async def _wake_with(self, now: datetime, lines: list[dict], replies: list[dict] else: self.log("user_reply_delivered", virtual_time=m["at"], text=m["text"]) if final and self.step.final_reminder: - parts.append({"kind": "text", "text": self.step.final_reminder.format(first=self.person["first"])}) + parts.append(_step_part(self.step.final_reminder.format(first=self.person["first"]), "final_reminder")) woke_by = [f"notification:{d['channel']}" for d in due] + [f"{m['kind']}" for m in user_msgs] await self.wake(parts, woke_by=woke_by, notifications=[{k: d.get(k) for k in ("at", "channel", "source", "source_id", "to")} for d in due]) async def wake(self, parts: list[dict], woke_by: list[str], notifications: Optional[list] = None) -> None: - step, ctx = self.step, self.ctx + step = self.step if step.wake_reminder: - parts = [*parts, {"kind": "text", "text": step.wake_reminder_text}] + parts = [*parts, _step_part(step.wake_reminder_text, "reminder")] self.wakes += 1 sent_at = await self.now() clock = await step._env_clock(self.clock_envs) @@ -678,13 +691,12 @@ def record() -> None: self.final_task_id = task_id ended = await self.now() plain = term.response_text or "" - flag = (not messages) and bool(plain.strip()) and bool(_ADDRESSED.search(plain)) and len(plain.strip()) > 40 self.last_plain, self.last_messages = plain, messages extra = {"stay_quiet_calls": len(quiet)} if step.stay_quiet else {} self.log("agent_turn_end", n=self.wakes, state=str(state), virtual_time=_iso(ended), virtual_seconds=round((ended - sent_at).total_seconds()), real_seconds=round(time.time() - t, 1), - tool_calls=term.tool_call_count, plain_reply=plain, - plain_reply_flagged_for_user=flag, messages_to_user=len(messages), trajectory_url=traj, **extra) + tool_calls=term.tool_call_count, plain_reply=plain, messages_to_user=len(messages), + trajectory_url=traj, **extra) for m in messages: self.log("message_to_user", virtual_time=m.get("at"), text=m["text"], wake=self.wakes) self.unread.append({"at": m.get("at"), "text": m["text"]}) @@ -693,7 +705,7 @@ def record() -> None: self.error = term.error_message or term.response_text or "agent turn failed" self.end_reason = "agent_turn_failed" return - if step.nudge and not messages and not quiet and plain.strip(): + if step.nudge and not messages and not quiet and _closing_text(plain): await self._nudge(plain) stuck = await self.settle() if stuck: @@ -701,11 +713,11 @@ def record() -> None: self._schedule_read() async def _nudge(self, closing: str) -> None: - """Opt-in (``nudge``): the wake ended with closing text and neither message_user nor stay_quiet, so give the - agent one more short turn with ``nudge_text``; record what the closing text was and what it did after.""" + """The wake ended with closing text and neither message_user nor stay_quiet, so give the agent one more short + turn with ``nudge_text``; record what the closing text was and what it did after.""" step = self.step self.nudges += 1 - parts = [{"kind": "text", "text": step.nudge_text}] + parts = [_step_part(step.nudge_text, "nudge")] sent_at = await self.now() clock = await step._env_clock(self.clock_envs) msg_id = uuid.uuid4().hex @@ -874,7 +886,6 @@ async def finish(self) -> None: and e.get("outcome") == "no_reply"), "scripted_user_lines": sum(1 for e in self.timeline if e["event"] == "scripted_user_line"), "clock_jumps": sum(1 for e in self.timeline if e["event"] == "clock_jump"), - "plain_replies_flagged": sum(1 for e in self.timeline if e.get("plain_reply_flagged_for_user")), "silent_wakes": sum(1 for e in self.timeline if e["event"] == "agent_turn_end" and not e.get("messages_to_user")), } @@ -890,9 +901,7 @@ async def finish(self) -> None: "user_sim": {"name": step.user_sim, "version": self.version, "model": self.us.get("model"), "agent_name": self.us.get("agent_name"), **self.user_setup}, "end_time": _iso(self.end) if self.end else None, "timeline": self.timeline} - if step.stay_quiet or step.nudge or step.wake_reminder: - record["settings"] = {"stay_quiet": step.stay_quiet, "nudge": step.nudge, - "wake_reminder": step.wake_reminder} + record["settings"] = {"stay_quiet": step.stay_quiet, "nudge": step.nudge, "wake_reminder": step.wake_reminder} try: store = get_config().get_object_store() key = (f"{get_config().get_artifact_key_prefix()}always_on_timeline/" @@ -935,6 +944,16 @@ async def _call(env, uri: str, method: str, params: Optional[dict] = None, *, at raise last +def _step_part(text: str, source: str) -> dict: + """A line the step adds to a wake, marked so a reader can tell it from what the user or the world sent.""" + return {"kind": "text", "text": text, "metadata": {"agentenv": {"source": source}}} + + +def _closing_text(plain: str) -> str: + text = plain.strip() + return "" if text in EMPTY_REPLIES else text + + def _busy(row: dict, now: datetime) -> bool: if row.get("evaluating"): return True diff --git a/tst/unit/env/gateway/user_chat_test.py b/tst/unit/env/gateway/user_chat_test.py new file mode 100644 index 000000000..7755a3540 --- /dev/null +++ b/tst/unit/env/gateway/user_chat_test.py @@ -0,0 +1,114 @@ +"""Unit tests for the user chat (urn:agentenv:user-chat/v1): message_user and stay_quiet, built in for the roles a +config names.""" +from __future__ import annotations + +import asyncio + +import httpx +import pytest +from mcp.types import Tool as MCPTool + +from agent_env.env.gateway import gateway as gw_mod +from agent_env.env.gateway.clock import Clock +from agent_env.env.gateway.constants import AGENT_ENV_ROLE_HEADER, EXT_USER_CHAT_URI, GATEWAY_EXTENSIONS +from agent_env.env.gateway.gateway import Gateway + +T0 = "2027-02-01T15:00:00Z" + + +@pytest.fixture +def gateway(tmp_path, monkeypatch): + monkeypatch.setattr(gw_mod, "GATEWAY_TRAJECTORY_FILE", str(tmp_path / "trajectory.jsonl")) + gw = Gateway(host="127.0.0.1", port=0, server_name="AgentEnvGateway", internal_mcp_servers=[]) + clock = Clock() + clock.set_time(T0, 0) + gw._clock = clock + gw._server_tools["mcp-svc"] = [MCPTool(name="svc_thing", description="", inputSchema={"type": "object"})] + gw._tools_discovered = True + yield gw + gw._close_trajectory_file() + + +def _req(gw, method, path, body=None, role=None): + async def _do(): + transport = httpx.ASGITransport(app=gw._mcp.streamable_http_app()) + async with httpx.AsyncClient(transport=transport, base_url="http://gateway") as client: + return await client.request(method, path, json=body, + headers={AGENT_ENV_ROLE_HEADER: role} if role else None) + return asyncio.run(_do()) + + +def _names(gw, role): + return {t["name"] for t in _req(gw, "POST", "/step", {"action": "list_tools"}, role=role).json()["tools"]} + + +def _call(gw, tool, arguments, role): + return _req(gw, "POST", "/step", {"action": "call_tool", "tool_name": tool, "arguments": arguments}, role=role) + + +def test_the_card_offers_config_state_take_and_clear(): + [ext] = [e for e in GATEWAY_EXTENSIONS if e["uri"] == EXT_USER_CHAT_URI] + assert set(ext["params"]["methods"]) == {"config", "state", "take", "clear"} + assert "stay_quiet" in ext["params"]["methods"]["config"]["request"]["properties"] + + +def test_message_user_reaches_a_configured_role_through_a_deny_all(gateway): + _req(gateway, "POST", "/tools/disable", {"role": "psi_agent", "tools": ["*"]}) + assert "message_user" not in _names(gateway, "psi_agent") + assert _req(gateway, "POST", "/user_chat/config", {"roles": ["psi_agent"], "user_name": "Tess"}).status_code == 200 + + names = _names(gateway, "psi_agent") + assert "message_user" in names and "svc_thing" not in names + assert "message_user" not in _names(gateway, "default") + assert _call(gateway, "message_user", {"text": "hi Tess, two questions"}, "psi_agent").status_code == 200 + assert _call(gateway, "message_user", {"text": "x"}, "default").status_code == 403 + taken = _req(gateway, "POST", "/user_chat/take", {"take_id": "a"}).json()["items"] + assert [(m["from"], m["at"], m["text"]) for m in taken] == [("psi_agent", T0, "hi Tess, two questions")] + assert _req(gateway, "POST", "/user_chat/take", {"take_id": "a"}).json()["items"] == taken + assert _req(gateway, "POST", "/user_chat/take", {}).json()["items"] == [] + + +def test_stay_quiet_is_off_unless_the_config_turns_it_on(gateway): + _req(gateway, "POST", "/user_chat/config", {"roles": ["psi_agent"], "user_name": "Tess"}) + assert "message_user" in _names(gateway, "psi_agent") and "stay_quiet" not in _names(gateway, "psi_agent") + called = _call(gateway, "stay_quiet", {}, "psi_agent") + assert called.status_code != 200 or called.json().get("isError") + assert _req(gateway, "POST", "/user_chat/take", {"take_id": "a"}).json() == {"items": []} + + +def test_stay_quiet_sends_nothing_and_is_recorded_for_the_next_take(gateway): + _req(gateway, "POST", "/tools/disable", {"role": "psi_agent", "tools": ["*"]}) + st = _req(gateway, "POST", "/user_chat/config", {"roles": ["psi_agent"], "user_name": "Tess", "stay_quiet": True}) + assert st.status_code == 200 and st.json()["stay_quiet"] is True + assert {"message_user", "stay_quiet"} <= _names(gateway, "psi_agent") + assert "stay_quiet" not in _names(gateway, "default") + + assert _call(gateway, "stay_quiet", {}, "psi_agent").status_code == 200 + assert _call(gateway, "stay_quiet", {}, "default").status_code == 403 + taken = _req(gateway, "POST", "/user_chat/take", {"take_id": "b"}).json() + assert taken == {"items": [], "quiet": [{"from": "psi_agent", "at": T0}]} + assert _req(gateway, "POST", "/user_chat/take", {"take_id": "b"}).json() == taken + assert _req(gateway, "POST", "/user_chat/take", {}).json() == {"items": [], "quiet": []} + [desc] = [t["description"] for t in + _req(gateway, "POST", "/step", {"action": "list_tools"}, role="psi_agent").json()["tools"] + if t["name"] == "stay_quiet"] + assert "Tess" in desc + + +def test_a_new_config_or_clear_removes_stay_quiet(gateway): + _req(gateway, "POST", "/user_chat/config", {"roles": ["psi_agent"], "stay_quiet": True}) + _req(gateway, "POST", "/user_chat/config", {"roles": ["psi_agent"]}) + assert "stay_quiet" not in _names(gateway, "psi_agent") and "message_user" in _names(gateway, "psi_agent") + _req(gateway, "POST", "/user_chat/config", {"roles": ["psi_agent"], "stay_quiet": True}) + _req(gateway, "POST", "/user_chat/clear", {}) + assert not {"stay_quiet", "message_user"} & _names(gateway, "psi_agent") + + +@pytest.mark.parametrize("body", [{"roles": ["psi_agent"], "stay_quiet": "yes"}, {"roles": "psi_agent"}, + {"roles": ["psi_agent"], "colour": "red"}]) +def test_a_bad_config_is_refused(gateway, body): + assert _req(gateway, "POST", "/user_chat/config", body).status_code == 400 + + +def test_an_unconfigured_gateway_lists_neither_tool(gateway): + assert not {"message_user", "stay_quiet"} & _names(gateway, "default") diff --git a/tst/unit/task_step/always_on_harness.py b/tst/unit/task_step/always_on_harness.py new file mode 100644 index 000000000..1c2132024 --- /dev/null +++ b/tst/unit/task_step/always_on_harness.py @@ -0,0 +1,144 @@ +"""A harness for the always_on step: an in-process gateway (clock, triggers, notifications, user chat) behind a real +DeployedGatewayEnv, an agent whose turns are scripted and act on that gateway as its role, and a user sim plug-in whose +answers are scripted. Shared by the always_on step's tests.""" + +from __future__ import annotations + +from typing import Any, Callable, Optional + +import httpx + +from agent_env.env.env import DeployedGatewayEnv +from agent_env.env.gateway import gateway as gw_mod +from agent_env.env.gateway.constants import AGENT_ENV_ROLE_HEADER, GATEWAY_EXTENSIONS, WELL_KNOWN_PATH +from agent_env.env.gateway.gateway import Gateway +from agent_env.task_step.context import DeployedAgent, TaskStepContext +from agent_env.task_step.task_steps import always_on as ao + +GATEWAY = "http://gateway.test" +SOLVER = "http://solver.test" +USER_SIM = "http://user-sim.test" +T0 = "2026-11-09T14:00:00Z" # Mon Nov 9, 9:00 AM in New York +ROLE = "default" + +Turn = Callable[[int, list[str], "Rig"], Optional[str]] + + +class Rig: + """The world, the agent and the user sim for one always_on run. + + ``agent(n, texts, rig)`` plays the agent's nth turn (nudges included) given the texts it was sent, acting through + ``message``, ``quiet`` and ``schedule``, and returns its closing text. ``answers`` are the user sim's answers to its + reads, in order; once they run out it answers no_reply. Every message the agent got is in ``sent``, every read the + user sim got in ``reads``.""" + + def __init__(self, monkeypatch, tmp_path, *, agent: Optional[Turn] = None, answers: tuple = (), rate: float = 0, + t0: str = T0): + monkeypatch.setattr(gw_mod, "GATEWAY_TRAJECTORY_FILE", str(tmp_path / "trajectory.jsonl")) + self.gw = Gateway(host="127.0.0.1", port=0, server_name="AgentEnvGateway", internal_mcp_servers=[]) + self.gw._tools_discovered = True + real_client = httpx.AsyncClient + transport = httpx.ASGITransport(app=self.gw._mcp.streamable_http_app()) + monkeypatch.setattr(httpx, "AsyncClient", lambda *a, **kw: real_client(*a, **{**kw, "transport": transport})) + self.http = lambda: real_client(transport=transport, base_url=GATEWAY) + self.t0, self.rate = t0, rate + self.agent = agent or (lambda n, texts, rig: "ok") + self.answers = list(answers) + self.sent: list[list[str]] = [] + self.parts: list[list[dict]] = [] + self.reads: list[dict] = [] + self.setups: list[dict] = [] + monkeypatch.setattr(ao, "POLL_SECONDS", 0) + monkeypatch.setattr(ao, "send_and_wait", self._send_and_wait) + monkeypatch.setattr(ao._Run, "_load_plugin", self._load_plugin) + + # ---- the world ----------------------------------------------------------------------------------------------- + async def arm(self) -> None: + """Arm the gateway clock the way sync_env_clock does.""" + async with self.http() as client: + r = await client.put("/clock/set-time", json={"virtual_time": self.t0, + "virtual_seconds_per_real_second": self.rate}) + r.raise_for_status() + + def triggers(self, *triggers: dict) -> None: + self.gw._trigger_engine.register({"watch_roles": [ROLE], "triggers": list(triggers)}) + + def notify_at(self, trigger_id: str, at: str, text: str, *, channel: str = "Messages", to: str = ROLE) -> dict: + return {"id": trigger_id, "when": {"type": "time", "at": at}, + "actions": [{"type": "notify", "channel": channel, "text": text, "to": to}]} + + def now(self) -> str: + return self.gw._clock.read()["virtual_time"] + + # ---- the agent's tools ----------------------------------------------------------------------------------------- + def message(self, text: str) -> dict: + return self.gw._user_chat.send(text, role=ROLE) + + def quiet(self) -> dict: + return self.gw._user_chat.quiet(role=ROLE) + + def schedule(self, note: str, at: str) -> dict: + return self.gw._notifications.schedule(note, at=at, to=ROLE) + + async def tools(self, role: str = ROLE) -> set[str]: + async with self.http() as client: + r = await client.post("/step", json={"action": "list_tools"}, headers={AGENT_ENV_ROLE_HEADER: role}) + return {t["name"] for t in r.json()["tools"]} + + # ---- the run --------------------------------------------------------------------------------------------------- + def context(self, *, user: Optional[dict] = None) -> TaskStepContext: + card = {"capabilities": {"extensions": list(GATEWAY_EXTENSIONS)}} + env = DeployedGatewayEnv(env_id="world", env_version=1, gateway_url=GATEWAY, mcp_url="", sandbox_id="sb", + environment_card_url=f"{GATEWAY}{WELL_KNOWN_PATH}", environment_card=card, + metadata={"deploy_step_id": "deploy"}) + ctx = TaskStepContext(instance_id="ti-1", deployed_envs=[env]) + ctx.deployed_agents += [DeployedAgent(agent_name="solver", api_url=SOLVER, a2a_url=SOLVER, a2a_card={}), + DeployedAgent(agent_name="user-sim", api_url=USER_SIM, a2a_url=USER_SIM, a2a_card={})] + ctx.metadata[ao.USER_SIMS_KEY] = {"linda": { + "name": "linda", "model": "sim-model", "agent_name": "user-sim", "user_version": "ours", + "display_name": "Linda Murphy", "first_name": "Linda", "time_zone": "America/New_York", + "description": "Linda, planning Noor's 30th.", **(user or {})}} + return ctx + + def step(self, **settings: Any) -> ao.AlwaysOnTaskStep: + return ao.AlwaysOnTaskStep("week", None, user_sim="linda", prompt="sort the cdmx weekend", prompt_id="ask", + agent_name="solver", **settings) + + async def run(self, step: ao.AlwaysOnTaskStep, ctx: Optional[TaskStepContext] = None) -> TaskStepContext: + ctx = ctx or self.context() + await step.execute(ctx) + return ctx + + def record(self, ctx: TaskStepContext) -> dict: + return ctx.metadata[ao.ALWAYS_ON_KEY]["week"] + + def events(self, ctx: TaskStepContext, name: str) -> list[dict]: + return [e for e in self.record(ctx)["timeline"] if e["event"] == name] + + # ---- fakes ----------------------------------------------------------------------------------------------------- + async def _send_and_wait(self, url, parts, *, before_send=None, **kw): + assert url == SOLVER, url + if before_send: + before_send() + texts = [p["text"] for p in parts if p.get("kind") == "text"] + self.sent.append(texts) + self.parts.append(list(parts)) + closing = self.agent(len(self.sent), texts, self) + reply = [{"kind": "text", "text": closing}] if closing else [] + return f"task-{len(self.sent)}", {"status": {"state": "completed", "message": {"parts": reply}}} + + async def _load_plugin(self, spec): + rig = self + + class Plugin: + @staticmethod + def setup(version, person): + rig.setups.append(person) + return {"system_prompt": f"you are {person['name']}", "output_format": None} + + @staticmethod + async def answer(version, person, history, read, ask): + rig.reads.append(read) + return rig.answers.pop(0) if rig.answers else {"outcome": "no_reply", "message": ""} + + return Plugin diff --git a/tst/unit/task_step/test_always_on_messages.py b/tst/unit/task_step/test_always_on_messages.py new file mode 100644 index 000000000..ca82008b4 --- /dev/null +++ b/tst/unit/task_step/test_always_on_messages.py @@ -0,0 +1,156 @@ +"""The always_on step's message rules: the agent reaches the user only through message_user, says "nothing to tell" +with stay_quiet, gets one nudge when a wake ends with closing text and neither, and every wake ends with a reminder. +All on unless a task turns them off; none of it reaches the user sim.""" + +from __future__ import annotations + +import pytest + +from agent_env.task_step.task_steps import always_on as ao +from tst.unit.task_step.always_on_harness import ROLE, Rig + +pytestmark = pytest.mark.usefixtures("local_stores") + + +def _turns(*plays): + """An agent that plays ``plays[n-1]`` on its nth turn, then says "ok".""" + def agent(n, texts, rig): + return plays[n - 1](rig) if n <= len(plays) else "ok" + return agent + + +def _send(text, closing="Sent."): + return lambda rig: (rig.message(text), closing)[1] + + +def _quiet(closing="Nothing new."): + return lambda rig: (rig.quiet(), closing)[1] + + +def _says(closing): + return lambda rig: closing + + +@pytest.mark.asyncio +async def test_all_three_are_on_by_default(monkeypatch, tmp_path): + rig = Rig(monkeypatch, tmp_path, agent=_turns(_quiet())) + await rig.arm() + step = rig.step() + ctx = await rig.run(step) + + assert (step.stay_quiet, step.nudge, step.wake_reminder) == (True, True, True) + assert rig.record(ctx)["settings"] == {"stay_quiet": True, "nudge": True, "wake_reminder": True} + assert rig.gw._user_chat.stay_quiet and ROLE in rig.gw._user_chat.roles + + +@pytest.mark.asyncio +async def test_each_wake_ends_with_the_reminder(monkeypatch, tmp_path): + rig = Rig(monkeypatch, tmp_path, agent=_turns(_send("two houses, which one?"), _quiet()), + answers=[{"outcome": "reply", "message": "the blue one"}]) + await rig.arm() + await rig.run(rig.step()) + + assert len(rig.sent) == 2 + assert all(texts[-1] == ao.WAKE_REMINDER_TEXT for texts in rig.sent) + assert rig.parts[0][-1]["metadata"] == {"agentenv": {"source": "reminder"}} + + +@pytest.mark.asyncio +async def test_closing_text_with_neither_tool_gets_one_nudge_recorded_and_counted(monkeypatch, tmp_path): + rig = Rig(monkeypatch, tmp_path, agent=_turns(_says("Found the house; Linda should pick."), _send("pick one?"))) + await rig.arm() + ctx = await rig.run(rig.step()) + + assert rig.sent[1] == [ao.NUDGE_TEXT] + [nudge] = rig.events(ctx, "nudge") + assert nudge["wake"] == 1 and nudge["closing_text"] == "Found the house; Linda should pick." + assert nudge["outcome"] == "message_user" and nudge["plain_reply_after"] == "Sent." + counts = rig.record(ctx)["counts"] + assert counts["nudges"] == 1 and counts["nudge_outcomes"] == {"message_user": 1} + assert [m["text"] for m in rig.events(ctx, "message_to_user")] == ["pick one?"] + + +@pytest.mark.asyncio +async def test_a_nudge_the_agent_ignores_is_recorded_and_not_repeated(monkeypatch, tmp_path): + rig = Rig(monkeypatch, tmp_path, agent=_turns(_says("All done for tonight."), _says("Okay."))) + await rig.arm() + ctx = await rig.run(rig.step()) + + assert len(rig.sent) == 2 + [nudge] = rig.events(ctx, "nudge") + assert nudge["outcome"] == "neither" and nudge["plain_reply_after"] == "Okay." + + +@pytest.mark.parametrize("play", [_send("booked the house"), _quiet()], ids=["message_user", "stay_quiet"]) +@pytest.mark.asyncio +async def test_no_nudge_after_either_tool(monkeypatch, tmp_path, play): + rig = Rig(monkeypatch, tmp_path, agent=_turns(play)) + await rig.arm() + ctx = await rig.run(rig.step()) + + assert ao.NUDGE_TEXT not in [t for texts in rig.sent for t in texts] + assert rig.events(ctx, "nudge") == [] + + +@pytest.mark.parametrize("closing", ["", " ", "(The assistant's reply was empty.)"], + ids=["none", "blank", "harness placeholder"]) +@pytest.mark.asyncio +async def test_no_nudge_on_an_empty_reply(monkeypatch, tmp_path, closing): + rig = Rig(monkeypatch, tmp_path, agent=_turns(_says(closing))) + await rig.arm() + ctx = await rig.run(rig.step()) + + assert len(rig.sent) == 1 and rig.events(ctx, "nudge") == [] + + +@pytest.mark.asyncio +async def test_a_task_can_turn_each_off(monkeypatch, tmp_path): + rig = Rig(monkeypatch, tmp_path, agent=_turns(_says("Updated all quantities."))) + await rig.arm() + ctx = await rig.run(rig.step(stay_quiet=False, nudge=False, wake_reminder=False)) + + assert len(rig.sent) == 1 and ao.WAKE_REMINDER_TEXT not in rig.sent[0] + assert rig.events(ctx, "nudge") == [] and "nudges" not in rig.record(ctx)["counts"] + assert not rig.gw._user_chat.stay_quiet and "stay_quiet" not in await rig.tools() + + +def test_off_survives_a_round_trip(): + step = ao.AlwaysOnTaskStep("week", None, user_sim="linda", prompt="hi", prompt_id="p", agent_name="a", + stay_quiet=False, nudge=False, wake_reminder=False) + again = ao.AlwaysOnTaskStep.from_dict(step.to_dict()) + assert (again.stay_quiet, again.nudge, again.wake_reminder) == (False, False, False) + assert ao.AlwaysOnTaskStep.from_dict({**step.to_dict(), "stay_quiet": None}).stay_quiet is True + + +@pytest.mark.asyncio +async def test_the_rules_follow_the_task_system_prompt(monkeypatch, tmp_path): + rig = Rig(monkeypatch, tmp_path, agent=_turns(_quiet())) + await rig.arm() + run = ao._Run(rig.step(system_prompt="You are Linda's assistant."), rig.context()) + assert run._system_prompt() == f"You are Linda's assistant.\n\n{ao.MESSAGE_RULES} {ao.QUIET_RULE}" + run = ao._Run(rig.step(system_prompt="You are Linda's assistant.", stay_quiet=False), rig.context()) + assert run._system_prompt() == f"You are Linda's assistant.\n\n{ao.MESSAGE_RULES}" + + +@pytest.mark.asyncio +async def test_without_a_system_prompt_the_first_wake_opens_with_the_rules(monkeypatch, tmp_path): + rig = Rig(monkeypatch, tmp_path, agent=_turns(_quiet())) + await rig.arm() + await rig.run(rig.step()) + + assert rig.sent[0][0] == f"{ao.MESSAGE_RULES} {ao.QUIET_RULE}" + + +@pytest.mark.asyncio +async def test_neither_the_nudge_nor_the_reminder_reaches_the_user_sim(monkeypatch, tmp_path): + rig = Rig(monkeypatch, tmp_path, + agent=_turns(_says("Linda should hear about the house."), _send("blue or green house?"), _quiet()), + answers=[{"outcome": "reply", "message": "green"}]) + await rig.arm() + await rig.run(rig.step()) + + [read] = rig.reads + assert [m["text"] for m in read["assistant_messages"]] == ["blue or green house?"] + assert [s["text"] for s in read["you_sent"]] == ["sort the cdmx weekend"] + seen = repr(rig.reads) + assert ao.NUDGE_TEXT not in seen and ao.WAKE_REMINDER_TEXT not in seen and ao.MESSAGE_RULES not in seen From 447407060a8e82b60f2f7b3ecc7a7a571ea9e74a Mon Sep 17 00:00:00 2001 From: Sarah Chitty Date: Mon, 9 Nov 2026 00:20:13 -0800 Subject: [PATCH 04/53] always_on: the core loop as decided (FD-5366) - user_sim: no apps setting (the user sim sees only what triggers notify to user_phone; scripted lines are notify to user_says); the version defaults to ours; a time_zone must be a real IANA zone. - The built-in ours works from the description alone: the read-and-answer rules (including "any other date is the computer's clock"), then WHO YOU ARE. versions.ours.system_prompt still replaces it. - Done ends only a one-sitting run. When end_time falls on a later day than the start (in the user's zone) the done is recorded (ends_run false, counts.user_dones) and the run goes on; the user sim is asked again when the agent next messages. - Time zone: the user sim's, else the one sync_env_clock gave the world; refused when the two disagree. It reaches the plug-in and the record. - What comes due in the same minute wakes the agent once: before a wake the clock moves to anything else due within that minute. - The next due time ignores items already due, so a pending one can't hide a later check. - Registration: always_on declares the user-scoped fields it takes, and the episode rules refuse an outside agent for always_on and user_sim (the same hunk as the FD-5371 branch, so the two merge cleanly). The gateway card test counts the user-chat extension. Tests on the in-process gateway: each wake source, jumping in order and never past an event, read times (asleep, busy) and reply delays, one read for messages sent before it, same-minute arrivals, what a read holds and that nothing meant for the agent is in it, done in one-sitting and multi-day runs, and the time zone. Co-Authored-By: Claude Opus 5.5 --- src/agent_env/task/episode/rules.py | 3 + .../task_step/task_steps/always_on.py | 114 +++++-- .../task_step/task_steps/user_sim_versions.py | 46 ++- tst/unit/env/gateway/gateway_card_test.py | 4 +- tst/unit/task_step/always_on_harness.py | 5 +- tst/unit/task_step/test_always_on_loop.py | 315 ++++++++++++++++++ tst/unit/task_step/test_field_scopes.py | 1 + 7 files changed, 443 insertions(+), 45 deletions(-) create mode 100644 tst/unit/task_step/test_always_on_loop.py diff --git a/src/agent_env/task/episode/rules.py b/src/agent_env/task/episode/rules.py index a2e9efc50..067013b39 100644 --- a/src/agent_env/task/episode/rules.py +++ b/src/agent_env/task/episode/rules.py @@ -135,6 +135,9 @@ def _rubrics_verifier(step: RubricsVerifierTaskStep, outside: Outside) -> list[s "register_agent_triggers": _needs_the_agent, "browser_connect_cdp": _needs_the_agent, "snapshot_agent_state": _needs_the_agent, + # An outside agent can't yet be woken by always_on's loop, nor play its user sim (the agent user_sim names). + "always_on": _needs_the_agent, + "user_sim": _needs_the_agent, "collect_artifacts": _collect_artifacts, "register_env_triggers": _register_env_triggers, "peer_agents": _peer_agents, diff --git a/src/agent_env/task_step/task_steps/always_on.py b/src/agent_env/task_step/task_steps/always_on.py index c0500f14a..77f6ae8b9 100644 --- a/src/agent_env/task_step/task_steps/always_on.py +++ b/src/agent_env/task_step/task_steps/always_on.py @@ -1,9 +1,9 @@ """The always_on and user_sim steps: a task that runs over simulated days, the way you'd hand a job to a personal -assistant, with an LLM user sim. A test-rig build of the "Always-on Task Step Design" doc. +assistant, with an LLM user sim. -The ``user_sim`` step says who the user is (description, schedule, response times, the version of the user sim and -its plug-in) and records it in the run's metadata under its ``name``; a ``deploy_agent`` step deploys the agent that -plays them. The ``always_on`` step names it with ``user_sim`` and runs the loop: +The ``user_sim`` step says who the user is (description, time zone, schedule, response times, the version of the user +sim and its plug-in) and records it in the run's metadata under its ``name``; a ``deploy_agent`` step deploys the agent +that plays them. The ``always_on`` step names it with ``user_sim`` and runs the loop: - The first wake is the user's request. Each wake opens with what came due for the agent on notifications/v1 (a trigger's ``notify``, a scheduled check), labelled ``[time · channel]``, then the user's own messages. @@ -11,14 +11,18 @@ goes to the log. Each wake should end with ``message_user`` or ``stay_quiet``: one that ends with closing text and neither gets one ``nudge`` turn, and every wake ends with a reminder. All three are on unless the task turns them off. - The user sim is asked only after the agent sent something, at the time the user would read it (``schedule``), with - everything sent meanwhile; its answer is reply / no_reply / done, and a reply is held for the response time. + everything sent meanwhile in one read: the time, the agent's messages, what triggers sent the user's phone + (``notify`` to ``user_phone``) and the user's own lines (the opening request, ``user_says`` lines, ``final``). + Nothing meant for the agent is in it. Its answer is reply / no_reply / done, and a reply is held for the response + time. Done ends a one-sitting run; a run whose end_time falls on a later day than its start goes on to end_time. - Scripted user lines are triggers' ``notify`` actions to the ``user_says`` recipient: sent to the agent as the - user's message and added to the user sim's history. What the user's phone shows is ``notify`` to ``user_phone``. + user's message at their time. - The clock runs at the task's speed during turns (sync_env_clock). When nothing is due, the step advances it to the - earliest of: the next trigger event, the next notification, a held reply, a pending read, or end_time; then waits up - to ``settle_timeout_seconds`` for triggers to finish. + earliest of: the next trigger event, the next notification, a held reply, a pending read, or end_time, never past + one; then waits up to ``settle_timeout_seconds`` for triggers to finish. What comes due in the same minute wakes + the agent once. - At end_time the user's ``final`` line (with a reminder to use message_user) gives the agent one last turn. The run - ends on done, end_time, max_wakes, or nothing left to wake the agent; the reason is recorded. + ends on end_time, done (one sitting), max_wakes, or nothing left to wake the agent; the reason is recorded. - Everything goes into a timeline in ``metadata["always_on"][step_id]`` and the object store. """ from __future__ import annotations @@ -88,8 +92,8 @@ class UserSimTaskStep(TaskStep): artifact_type="file"),) def __init__(self, id: str, version: Optional[int], *, name: str, model: str, agent_name: Optional[str] = None, - user_version: str = "today", display_name: Optional[str] = None, first_name: Optional[str] = None, - description: Optional[str] = None, time_zone: Optional[str] = None, apps: Optional[list] = None, + user_version: str = "ours", display_name: Optional[str] = None, first_name: Optional[str] = None, + description: Optional[str] = None, time_zone: Optional[str] = None, schedule: Optional[dict] = None, response_times: Optional[dict] = None, max_replies: Optional[int] = None, permissions: Optional[dict] = None, keep_private: Optional[list] = None, output_format: Optional[dict] = None, @@ -99,6 +103,8 @@ def __init__(self, id: str, version: Optional[int], *, name: str, model: str, ag retry_config=retry_config) if not model: raise ValueError("user_sim.model is required") + if time_zone is not None: + ZoneInfo(time_zone) self.name = name self.model = model self.agent_name = agent_name or name @@ -107,7 +113,6 @@ def __init__(self, id: str, version: Optional[int], *, name: str, model: str, ag self.first_name = first_name self.description = description self.time_zone = time_zone - self.apps = apps or [] self.schedule = schedule self.response_times = response_times self.max_replies = max_replies @@ -118,7 +123,7 @@ def __init__(self, id: str, version: Optional[int], *, name: str, model: str, ag self.plugin = plugin _FIELDS = ("name", "model", "agent_name", "user_version", "display_name", "first_name", "description", "time_zone", - "apps", "schedule", "response_times", "max_replies", "permissions", "keep_private", "output_format", + "schedule", "response_times", "max_replies", "permissions", "keep_private", "output_format", "versions", "plugin") def to_dict(self) -> dict: @@ -146,17 +151,22 @@ async def execute(self, context: TaskStepContext) -> TaskStepContext: class AlwaysOnTaskStep(PromptAgentTaskStep): type: ClassVar[str] = "always_on" entity_refs = () + field_scopes = {k: v for k, v in PromptAgentTaskStep.field_scopes.items() if k != "user_model"} def __init__(self, id: str, version: Optional[int], *, user_sim: str, end_time: Optional[str] = None, final: Optional[str] = None, max_wakes: int = 60, settle_timeout_seconds: float = 300, user_timeout_seconds: int = 300, final_reminder: Optional[str] = FINAL_REMINDER, stay_quiet: bool = True, nudge: bool = True, nudge_text: str = NUDGE_TEXT, wake_reminder: bool = True, wake_reminder_text: str = WAKE_REMINDER_TEXT, - **prompt_fields: Any): + model: Optional[str] = None, model_params: Optional[dict] = None, effort: Optional[str] = None, + max_thinking_tokens: Optional[int] = None, harness: Optional[str] = None, + trajectory_output_prefix: Optional[str] = None, **prompt_fields: Any): for unsupported in ("max_conversation_turns", "user_agent_name", "user_a2a_url", "post_turn_hooks", "user_output_format", "user_model", "snapshot_config"): prompt_fields.pop(unsupported, None) - super().__init__(id, version, max_conversation_turns=max(1, int(max_wakes)), **prompt_fields) + super().__init__(id, version, max_conversation_turns=max(1, int(max_wakes)), model=model, + model_params=model_params, effort=effort, max_thinking_tokens=max_thinking_tokens, + harness=harness, trajectory_output_prefix=trajectory_output_prefix, **prompt_fields) if not user_sim: raise ValueError("always_on needs user_sim, the name of a user_sim step's user") if end_time is not None: @@ -235,6 +245,8 @@ def __init__(self, step: AlwaysOnTaskStep, context: TaskStepContext): self._plugin_paths: list[str] = [] self._plugin_modules: list[str] = [] self.nudges = 0 + self.zone = "UTC" + self.multi_day = False def close(self) -> None: for p in self._plugin_paths: @@ -265,14 +277,15 @@ async def go(self) -> None: if us is None: raise RuntimeError(f"always_on: no user_sim step named '{step.user_sim}' ran before this step") self.us = us - self.person = self._person(us) + self.zone = _task_zone(us, ctx.metadata.get("clock_configurations") or []) + self.person = {**self._person(us), "time_zone": self.zone} self.version = us["user_version"] self.user_agent = next((a for a in ctx.deployed_agents if a.agent_name == us["agent_name"]), None) if self.user_agent is None: raise RuntimeError(f"always_on: the user sim agent '{us['agent_name']}' is not deployed") self.user_url = self.user_agent.a2a_url or self.user_agent.api_url self.plugin = await self._load_plugin(us.get("plugin")) - self.tz = ZoneInfo(us.get("time_zone") or "UTC") + self.tz = ZoneInfo(self.zone) self.clock_envs = [d for d in ctx.deployed_envs if d.supports(EXT_CLOCK_URI, "state")] self.notification_envs = [d for d in ctx.deployed_envs if d.supports(EXT_NOTIFICATIONS_URI, "take")] @@ -302,9 +315,10 @@ async def go(self) -> None: now = await self.now() t0 = await self._t0() or now self.end = _end_instant(step.end_time, t0) if step.end_time else None + self.multi_day = self.end is not None and self.end.astimezone(self.tz).date() > t0.astimezone(self.tz).date() self.log("start", virtual_time=_iso(now), end_time=_iso(self.end) if self.end else None, agent=step.agent_name, agent_role=self.recipient, user_sim=step.user_sim, version=self.version, - user_model=us["model"], max_wakes=step.max_wakes, + user_model=us["model"], max_wakes=step.max_wakes, time_zone=self.zone, multi_day=self.multi_day, clock_rate=await self._rate()) try: await self.loop(now) @@ -321,7 +335,7 @@ def _person(self, us: dict) -> dict: return {"key": us["name"], "name": name, "first": us.get("first_name") or name.split()[0], "time_zone": us.get("time_zone"), "description": us.get("description"), "versions": us.get("versions") or {}, - "output_format": us.get("output_format"), "apps": us.get("apps") or [], + "output_format": us.get("output_format"), "permissions": us.get("permissions"), "keep_private": us.get("keep_private")} async def _load_plugin(self, spec: Optional[dict]) -> ModuleType: @@ -474,9 +488,13 @@ async def _due_count(self, to: str, now: datetime) -> int: n += sum(1 for p in st.get("pending_notifications", []) if not p.get("at") or _utc(p["at"]) <= now) return n - async def _next_due(self, to: str) -> Optional[datetime]: - times = [_utc(st["next_due_at"]) for st in await self._notes_state(to) if st.get("next_due_at")] - return min(times) if times else None + async def _next_due(self, to: str, now: datetime) -> Optional[datetime]: + """When the next check or notification for ``to`` comes due after ``now``.""" + times = [] + for st in await self._notes_state(to): + times += [_utc(c["next"]) for c in st.get("checks", []) if c.get("status") == "active"] + times += [_utc(p["at"]) for p in st.get("pending_notifications", []) if p.get("at")] + return min((t for t in times if t > now), default=None) # ---- the user's schedule --------------------------------------------------------------------------------- def _schedule(self) -> Optional[dict]: @@ -549,21 +567,23 @@ async def loop(self, now: datetime) -> None: if self.read_at is not None and self.read_at <= now and self.unread: await self.read(now) continue - # 2. anything due for the agent: notifications, scripted user lines, held replies - lines = await self._take_user_says() - replies = [h for h in self.held if h["deliver_at"] <= now] - due = await self._due_count(self.recipient, now) - if lines or replies or due: + # 2. anything due for the agent: notifications, scripted user lines, held replies; and what else comes + # due in the same minute, so it all arrives in one wake + if any(h["deliver_at"] <= now for h in self.held) or await self._due_count(self.recipient, now) \ + or await self._due_count(USER_SAYS, now): + later = await self._later(now) + same_minute = [t for t in later.values() if t < now.replace(second=0, microsecond=0) + + timedelta(minutes=1) and (self.end is None or t < self.end)] + if same_minute and await self._armed(): + await self.advance(min(same_minute), reason="same_minute") + continue + lines = await self._take_user_says() + replies = [h for h in self.held if h["deliver_at"] <= now] self.held = [h for h in self.held if h["deliver_at"] > now] await self._wake_with(now, lines, replies, final=False) continue # 3. nothing due: jump to the next thing - candidates = {"world_event": await self._next_world_event(), - "agent_notification": await self._next_due(self.recipient), - "user_says": await self._next_due(USER_SAYS), - "held_reply": min((h["deliver_at"] for h in self.held), default=None), - "user_read": self.read_at if self.unread else None} - later = {k: v for k, v in candidates.items() if v is not None and v > now} + later = await self._later(now) soonest = min(later.values()) if later else None if self.end is not None and (soonest is None or soonest >= self.end): await self._final(now) @@ -574,6 +594,15 @@ async def loop(self, now: datetime) -> None: why = sorted(k for k, v in later.items() if v == soonest) await self.advance(soonest, reason="+".join(why)) + async def _later(self, now: datetime) -> dict[str, datetime]: + """What could wake the agent after ``now``, by kind, each at its earliest time.""" + candidates = {"world_event": await self._next_world_event(), + "agent_notification": await self._next_due(self.recipient, now), + "user_says": await self._next_due(USER_SAYS, now), + "held_reply": min((h["deliver_at"] for h in self.held), default=None), + "user_read": self.read_at if self.unread else None} + return {k: v for k, v in candidates.items() if v is not None and v > now} + async def _final(self, now: datetime) -> None: step = self.step if self.end > now: @@ -828,7 +857,9 @@ async def ask(text: str) -> dict: self.gave_up = True self.log("user_gave_up", replies=self.replies) elif outcome == "done": - self.end_reason = "user_done" + entry["ends_run"] = not self.multi_day + if not self.multi_day: + self.end_reason = "user_done" if calls and calls[-1].get("error"): self.end_reason = self.end_reason or "user_sim_error" @@ -884,6 +915,7 @@ async def finish(self) -> None: "user_replies": self.replies, "user_no_replies": sum(1 for e in self.timeline if e["event"] == "user_sim_read" and e.get("outcome") == "no_reply"), + "user_dones": sum(1 for e in self.timeline if e["event"] == "user_sim_read" and e.get("outcome") == "done"), "scripted_user_lines": sum(1 for e in self.timeline if e["event"] == "scripted_user_line"), "clock_jumps": sum(1 for e in self.timeline if e["event"] == "clock_jump"), "silent_wakes": sum(1 for e in self.timeline if e["event"] == "agent_turn_end" @@ -900,7 +932,8 @@ async def finish(self) -> None: record = {"step_id": step.id, "end_reason": self.end_reason, "error": self.error, "counts": counts, "user_sim": {"name": step.user_sim, "version": self.version, "model": self.us.get("model"), "agent_name": self.us.get("agent_name"), **self.user_setup}, - "end_time": _iso(self.end) if self.end else None, "timeline": self.timeline} + "end_time": _iso(self.end) if self.end else None, "time_zone": self.zone, + "multi_day": self.multi_day, "timeline": self.timeline} record["settings"] = {"stay_quiet": step.stay_quiet, "nudge": step.nudge, "wake_reminder": step.wake_reminder} try: store = get_config().get_object_store() @@ -944,6 +977,17 @@ async def _call(env, uri: str, method: str, params: Optional[dict] = None, *, at raise last +def _task_zone(us: dict, clocks: list[dict]) -> str: + """The user's time zone, else the one sync_env_clock gave the world; refused when the two disagree.""" + zones = sorted({c["timezone"] for c in clocks if c.get("timezone")}) + zone = us.get("time_zone") + if zone and zones and zones != [zone]: + raise ValueError(f"always_on: user sim '{us['name']}' is in {zone} but sync_env_clock set {zones}") + if not zone and len(zones) > 1: + raise ValueError(f"always_on: sync_env_clock set several time zones {zones}; give the user_sim step one") + return zone or (zones[0] if zones else "UTC") + + def _step_part(text: str, source: str) -> dict: """A line the step adds to a wake, marked so a reader can tell it from what the user or the world sent.""" return {"kind": "text", "text": text, "metadata": {"agentenv": {"source": source}}} diff --git a/src/agent_env/task_step/task_steps/user_sim_versions.py b/src/agent_env/task_step/task_steps/user_sim_versions.py index 607554314..c2af31067 100644 --- a/src/agent_env/task_step/task_steps/user_sim_versions.py +++ b/src/agent_env/task_step/task_steps/user_sim_versions.py @@ -1,5 +1,5 @@ -"""The user sim plug-in the always_on step calls by default: each version decides the user sim's system prompt and -answer format (``setup``), renders its own input for one read and parses the answer (``answer``). +"""The user sim plug-in the always_on step calls when a user_sim step names none: each version decides the user sim's +system prompt and answer format (``setup``), renders its own input for one read and parses the answer (``answer``). The interface, which any plug-in a ``user_sim`` step names (a file artifact) must offer: @@ -7,14 +7,18 @@ async answer(version, person, history, read, ask) -> {"outcome": "reply" | "no_reply" | "done", "message": str, "used": str | None, "input": str, "raw": ..., ...} -``person`` is the user_sim step's settings (name, first, time_zone, description, versions, ...). ``history`` is every +``person`` is the user_sim step's settings (name, first, time_zone, description, versions, ...). A plug-in may also +offer ``configure(params)``, called once with the step's ``plugin.params``. ``history`` is every earlier read, oldest first, each ``{"read": , "answer": }``. ``read`` is this read: ``{"time": RFC 3339, "assistant_messages": [{"at", "text"}], "phone": [{"at", "channel", "text"}], "you_sent": [{"at", "text"}]}``. ``ask(text)`` sends the rendered text to the deployed user sim (one session per run) and returns ``{"text": raw text, "structured": parsed object or None}``. -Versions here (from user-sim-h2h/v2/PLAN.md): today, ours, ours_no_feed, ours_no_labels, ours_no_time, ours_no_used, -ours_always_replies. "today + fixes" and "ours + fixes" need per-person lines the versions module will supply. +Versions here: ``ours`` (the default) and its ablations ours_no_feed, ours_no_labels, ours_no_time, ours_no_used and +ours_always_replies, and ``today`` (the description as the whole prompt, the agent's raw text as the input, answers +{message, done}). ``ours`` shows each read labelled (the time, what's new on the phone, what the user sent, what the +assistant sent) and answers {outcome, message, used}; its prompt is ``versions.ours.system_prompt`` when the person +has one, else the rules below followed by the description, which then says only who the person is. """ from __future__ import annotations @@ -68,7 +72,9 @@ def setup(version: str, person: dict) -> dict: ours = (person.get("versions") or {}).get("ours") or {} prompt = ours.get("system_prompt") if not prompt: - raise ValueError(f"user sim {person.get('name')!r} has no versions.ours.system_prompt for version {version!r}") + if not person.get("description"): + raise ValueError(f"user sim {person.get('name')!r} needs a description or versions.ours.system_prompt") + return {"system_prompt": ours_prompt(version, person), "output_format": _ours_schema(version)} if version == "ours_no_used": prompt = _drop_used(prompt) if version == "ours_always_replies": @@ -76,6 +82,34 @@ def setup(version: str, person: dict) -> dict: return {"system_prompt": prompt, "output_format": _ours_schema(version)} +def ours_prompt(version: str, person: dict) -> str: + """How a person reads and answers in ``ours``, then who they are (the description).""" + full = person.get("name") or "the user" + first = person.get("first") or full.split()[0] + feed, clock = version != "ours_no_feed", version != "ours_no_time" + shows = [*(["the time"] if clock else []), *(["what's new on your phone"] if feed else []), "what you sent", + "what your AI assistant sent"] + lines = [f"You are {full}, a real person messaging your personal AI assistant. You are not the AI assistant and " + f"not a tester. Each message you get shows {', '.join(shows[:-1])}, and {shows[-1]}. Reply only as " + f"{first}. Don't do your assistant's work: never look things up for it or hand over facts it should find " + "itself. You can't open your apps" + ("; what's new on your phone is listed in each message." if feed + else ".")] + if feed: + lines.append("What's new on your phone is for you: your assistant reads your inbox itself. Don't pass it on or " + "make calls about it unless the assistant asks, or your description says to.") + if clock: + lines.append("Go by the time each message gives. Any other date you see is the computer's clock, not today's.") + lines.append("If the assistant asks something your description doesn't cover: for a choice, choose the way you " + "would; for a fact about you or anyone else that isn't written there, say you don't know. Never " + "guess facts.") + lines.append('Only answer "done" with a short sign-off. If your message answers a question or tells the assistant ' + 'to do something, answer "reply": a "done" message never reaches the assistant.') + used = ('In "used", give the tag or quote the words of your description you relied on, or "none". ' + if version != "ours_no_used" else "") + lines.append(used + "Answer in JSON:\n" + _json_line(version)) + return "\n".join(lines) + "\n\nWHO YOU ARE\n" + person["description"].strip() + "\n" + + def _drop_used(prompt: str) -> str: """Remove the lines that explain the `used` field (the tagged persona lines stay).""" return "\n".join(ln for ln in prompt.splitlines() if '"used"' not in ln and "`used`" not in ln diff --git a/tst/unit/env/gateway/gateway_card_test.py b/tst/unit/env/gateway/gateway_card_test.py index 48979ddfa..19582c7ca 100644 --- a/tst/unit/env/gateway/gateway_card_test.py +++ b/tst/unit/env/gateway/gateway_card_test.py @@ -201,12 +201,12 @@ def test_rewrite_child_card_drops_the_childs_interfaces(): assert _gateway({"mcp-slack": "http://slack:18765"})._rewrite_child_card("mcp-slack", child)["additionalInterfaces"] == [] -def test_gateway_card_lists_its_nine_extensions(): +def test_gateway_card_lists_its_ten_extensions(): assert _gateway({})._gateway_extensions() is gateway_constants.GATEWAY_EXTENSIONS assert [e["uri"] for e in _gateway({})._gateway_extensions()] == [ "urn:agentenv:disable-tool/v1", "urn:agentenv:enable-tool/v1", "urn:agentenv:triggers/v1", "urn:agentenv:clock/v1", "urn:agentenv:trajectory/v1", "urn:agentenv:step/v1", "urn:agentenv:state/v1", - "urn:agentenv:changes/v1", "urn:agentenv:notifications/v1", + "urn:agentenv:changes/v1", "urn:agentenv:notifications/v1", "urn:agentenv:user-chat/v1", ] diff --git a/tst/unit/task_step/always_on_harness.py b/tst/unit/task_step/always_on_harness.py index 1c2132024..48616ec7f 100644 --- a/tst/unit/task_step/always_on_harness.py +++ b/tst/unit/task_step/always_on_harness.py @@ -28,7 +28,8 @@ class Rig: """The world, the agent and the user sim for one always_on run. ``agent(n, texts, rig)`` plays the agent's nth turn (nudges included) given the texts it was sent, acting through - ``message``, ``quiet`` and ``schedule``, and returns its closing text. ``answers`` are the user sim's answers to its + ``message``, ``quiet`` and ``schedule``, and returns its closing text; by default it stays quiet. ``answers`` are the + user sim's answers to its reads, in order; once they run out it answers no_reply. Every message the agent got is in ``sent``, every read the user sim got in ``reads``.""" @@ -42,7 +43,7 @@ def __init__(self, monkeypatch, tmp_path, *, agent: Optional[Turn] = None, answe monkeypatch.setattr(httpx, "AsyncClient", lambda *a, **kw: real_client(*a, **{**kw, "transport": transport})) self.http = lambda: real_client(transport=transport, base_url=GATEWAY) self.t0, self.rate = t0, rate - self.agent = agent or (lambda n, texts, rig: "ok") + self.agent = agent or (lambda n, texts, rig: (rig.quiet(), "")[1]) self.answers = list(answers) self.sent: list[list[str]] = [] self.parts: list[list[dict]] = [] diff --git a/tst/unit/task_step/test_always_on_loop.py b/tst/unit/task_step/test_always_on_loop.py new file mode 100644 index 000000000..15ac70a89 --- /dev/null +++ b/tst/unit/task_step/test_always_on_loop.py @@ -0,0 +1,315 @@ +"""The always_on step's loop on an in-process gateway: what wakes the agent, how the clock moves, when the user sim reads +and replies, what each read holds, and how done ends a run.""" + +from __future__ import annotations + +import pytest + +from agent_env.task_step.registry import get_task_step_registry +from agent_env.task_step.task_steps import always_on as ao +from agent_env.task_step.task_steps import user_sim_versions +from tst.unit.task_step.always_on_harness import Rig + +pytestmark = pytest.mark.usefixtures("local_stores") + +LINDA_DAY = {"schedule": {"wake": "07:30", "sleep": "23:30", "work": ["09:00", "18:00"]}, + "response_times": {"work": 150, "free": 30, "morning": 15}} + + +def _agent(**plays): + """An agent that plays ``plays["t"]`` on its nth turn and otherwise stays quiet.""" + def agent(n, texts, rig): + play = plays.get(f"t{n}") + if play is None: + rig.quiet() + return "nothing new" + return play(rig) + return agent + + +def _message(text): + return lambda rig: (rig.message(text), "sent")[1] + + +def _woke(ctx, rig): + return [(w["virtual_time"], w["woke_by"]) for w in rig.events(ctx, "wake")] + + +def _without_step_lines(texts): + return [t for t in texts if t not in (ao.WAKE_REMINDER_TEXT, f"{ao.MESSAGE_RULES} {ao.QUIET_RULE}")] + + +# ---- what wakes the agent ------------------------------------------------------------------------------------------- + +@pytest.mark.asyncio +async def test_a_notify_alert_wakes_the_agent_labelled_with_its_time_and_channel(monkeypatch, tmp_path): + rig = Rig(monkeypatch, tmp_path) + await rig.arm() + rig.triggers(rig.notify_at("sam", "2026-11-10T14:00:00Z", "Sam: she suspects nothing", channel="noor 30")) + ctx = await rig.run(rig.step()) + + assert _woke(ctx, rig) == [("2026-11-09T14:00:00Z", ["user_request"]), + ("2026-11-10T14:00:00Z", ["notification:noor 30"])] + assert "[Tue Nov 10, 2:00 PM UTC · noor 30] Sam: she suspects nothing" in rig.sent[1] + assert rig.record(ctx)["end_reason"] == "nothing_can_wake_the_agent" + + +@pytest.mark.asyncio +async def test_a_scripted_user_line_reaches_the_agent_as_the_user_and_counts_as_hers(monkeypatch, tmp_path): + rig = Rig(monkeypatch, tmp_path, agent=_agent(t2=_message("on it: three dinner spots tonight"))) + await rig.arm() + rig.triggers(rig.notify_at("ask-dinner", "2026-11-10T15:00:00Z", "also find a dinner spot", to=ao.USER_SAYS)) + ctx = await rig.run(rig.step()) + + assert _woke(ctx, rig)[1] == ("2026-11-10T15:00:00Z", ["user_says"]) + assert "[Tue Nov 10, 3:00 PM UTC] Linda: also find a dinner spot" in rig.sent[1] + [read] = rig.reads + assert [s["text"] for s in read["you_sent"]] == ["sort the cdmx weekend", "also find a dinner spot"] + assert [e["text"] for e in rig.events(ctx, "scripted_user_line")] == ["also find a dinner spot"] + + +@pytest.mark.asyncio +async def test_the_agent_s_scheduled_check_moves_the_clock_and_wakes_it(monkeypatch, tmp_path): + def book(rig): + rig.schedule("book La Ola when it opens", at="2026-11-12T13:00:00Z") + rig.quiet() + return "check set" + + rig = Rig(monkeypatch, tmp_path, agent=_agent(t1=book)) + await rig.arm() + rig.gw._notifications.configure({"roles": ["default"]}) + ctx = await rig.run(rig.step()) + + assert _woke(ctx, rig)[1] == ("2026-11-12T13:00:00Z", ["notification:your scheduled check"]) + assert "[Thu Nov 12, 1:00 PM UTC · your scheduled check] book La Ola when it opens" in rig.sent[1] + assert [j["to"] for j in rig.events(ctx, "clock_jump")] == ["2026-11-12T13:00:00Z"] + + +@pytest.mark.asyncio +async def test_a_reply_is_held_for_the_reply_time_then_wakes_the_agent(monkeypatch, tmp_path): + rig = Rig(monkeypatch, tmp_path, agent=_agent(t1=_message("blue house or green?")), + answers=[{"outcome": "reply", "message": "the blue one"}]) + await rig.arm() + ctx = await rig.run(rig.step(), rig.context(user=LINDA_DAY)) + + [read] = rig.events(ctx, "user_sim_read") + assert read["virtual_time"] == "2026-11-09T14:00:00Z" and read["mode"] == "work" + assert read["deliver_at"] == "2026-11-09T16:30:00Z" + assert _woke(ctx, rig)[1] == ("2026-11-09T16:30:00Z", ["user_reply"]) + assert "[Mon Nov 9, 4:30 PM UTC] Linda: the blue one" in rig.sent[1] + + +@pytest.mark.asyncio +async def test_the_user_reads_once_awake_and_not_busy(monkeypatch, tmp_path): + busy = {**LINDA_DAY["schedule"], "busy": [{"start": "2026-11-10T07:00", "end": "2026-11-10T10:00"}]} + rig = Rig(monkeypatch, tmp_path, t0="2026-11-10T05:00:00Z", # midnight in New York + agent=_agent(t1=_message("found two houses"), t2=_message("and a third")), + answers=[{"outcome": "reply", "message": "send all three"}]) + await rig.arm() + ctx = await rig.run(rig.step(), rig.context(user={**LINDA_DAY, "schedule": busy})) + + [scheduled, *_] = rig.events(ctx, "user_read_scheduled") + assert scheduled["read_at"] == "2026-11-10T15:00:00Z" # past the night and the 7-10 AM block + [first, second] = rig.events(ctx, "user_sim_read") + assert first["virtual_time"] == "2026-11-10T15:00:00Z" + assert first["deliver_at"] == "2026-11-10T15:15:00Z" + + +@pytest.mark.asyncio +async def test_messages_sent_before_a_read_arrive_in_that_one_read(monkeypatch, tmp_path): + def two(rig): + rig.message("house: the blue one, $855") + rig.message("dinner: La Ola, Saturday 8pm") + return "sent both" + + rig = Rig(monkeypatch, tmp_path, t0="2026-11-10T05:00:00Z", agent=_agent(t1=two)) + await rig.arm() + await rig.run(rig.step(), rig.context(user=LINDA_DAY)) + + [read] = rig.reads + assert read["time"] == "2026-11-10T12:30:00Z" + assert [m["text"] for m in read["assistant_messages"]] == ["house: the blue one, $855", + "dinner: La Ola, Saturday 8pm"] + + +# ---- the clock ------------------------------------------------------------------------------------------------------ + +@pytest.mark.asyncio +async def test_the_clock_jumps_to_each_next_event_in_turn_never_past_one(monkeypatch, tmp_path): + def check(rig): + rig.schedule("chase the host", at="2026-11-10T11:00:00Z") + rig.quiet() + return "ok" + + rig = Rig(monkeypatch, tmp_path, agent=_agent(t1=check)) + await rig.arm() + rig.gw._notifications.configure({"roles": ["default"]}) + rig.triggers(rig.notify_at("landlord", "2026-11-10T09:30:00Z", "rent", to="landlord"), + rig.notify_at("noon", "2026-11-10T12:00:00Z", "Sam: flights?", channel="noor 30"), + rig.notify_at("ten", "2026-11-10T10:00:00Z", "Pri: I'm in", channel="noor 30")) + ctx = await rig.run(rig.step()) + + assert [j["to"] for j in rig.events(ctx, "clock_jump")] == [ + "2026-11-10T09:30:00Z", "2026-11-10T10:00:00Z", "2026-11-10T11:00:00Z", "2026-11-10T12:00:00Z"] + assert [t for t, _ in _woke(ctx, rig)[1:]] == ["2026-11-10T10:00:00Z", "2026-11-10T11:00:00Z", + "2026-11-10T12:00:00Z"] + + +@pytest.mark.asyncio +async def test_what_arrives_in_the_same_minute_comes_in_one_wake(monkeypatch, tmp_path): + def check(rig): + rig.schedule("look at the airline change", at="2026-11-10T10:00:40Z") + rig.quiet() + return "ok" + + rig = Rig(monkeypatch, tmp_path, agent=_agent(t1=check)) + await rig.arm() + rig.gw._notifications.configure({"roles": ["default"]}) + rig.triggers(rig.notify_at("airline", "2026-11-10T10:00:00Z", "Wingloft: your flight was retimed", + channel="Gmail"), + rig.notify_at("ask", "2026-11-10T10:00:20Z", "did the flight change?", to=ao.USER_SAYS), + rig.notify_at("next", "2026-11-10T10:01:00Z", "Dev: thx!", channel="Messages")) + ctx = await rig.run(rig.step()) + + woke = _woke(ctx, rig) + assert woke[1] == ("2026-11-10T10:00:40Z", ["notification:Gmail", "notification:your scheduled check", + "user_says"]) + assert woke[2] == ("2026-11-10T10:01:00Z", ["notification:Messages"]) + assert len(woke) == 3 + + +# ---- what the user sim sees, and done ---------------------------------------------------------------------------- + +@pytest.mark.asyncio +async def test_each_read_holds_the_time_the_agent_s_messages_her_phone_and_only_her_own_lines(monkeypatch, tmp_path): + def agent(n, texts, rig): + if n == 1: + return "Linda should know the house is booked." + if any("Sam: shh" in t for t in texts): + rig.message("house booked; Sam says she suspects nothing") + return "sent" + rig.quiet() + return "" + + rig = Rig(monkeypatch, tmp_path, agent=agent) + await rig.arm() + rig.gw._notifications.configure({"roles": ["default"]}) + rig.schedule("chase the host", at="2026-11-09T18:00:00Z") + rig.triggers(rig.notify_at("phone", "2026-11-09T19:00:00Z", "New email from Sam", channel="Gmail", + to=ao.USER_PHONE), + rig.notify_at("ask", "2026-11-09T19:30:00Z", "is the house sorted?", to=ao.USER_SAYS), + rig.notify_at("alert", "2026-11-09T20:00:00Z", "Sam: shh", channel="noor 30")) + ctx = await rig.run(rig.step(end_time="2026-11-10T02:00:00Z", final="send me the rundown")) + + [read] = rig.reads + assert read["time"] == "2026-11-09T20:00:00Z" + assert [m["text"] for m in read["assistant_messages"]] == ["house booked; Sam says she suspects nothing"] + assert [(p["channel"], p["text"]) for p in read["phone"]] == [("Gmail", "New email from Sam")] + assert [s["text"] for s in read["you_sent"]] == ["sort the cdmx weekend", "is the house sorted?"] + seen = repr(rig.reads) + for meant_for_the_agent in ("Sam: shh", "chase the host", ao.NUDGE_TEXT, ao.WAKE_REMINDER_TEXT, + ao.MESSAGE_RULES): + assert meant_for_the_agent not in seen + assert rig.events(ctx, "nudge") and rig.record(ctx)["end_reason"] == "end_time" + + +@pytest.mark.asyncio +async def test_done_ends_a_one_sitting_run(monkeypatch, tmp_path): + rig = Rig(monkeypatch, tmp_path, agent=_agent(t1=_message("all set: shelves restocked")), + answers=[{"outcome": "done", "message": "thanks!"}]) + await rig.arm() + rig.triggers(rig.notify_at("later", "2026-11-10T14:00:00Z", "a new order", channel="Craftweb")) + ctx = await rig.run(rig.step()) + + assert rig.record(ctx)["end_reason"] == "user_done" and len(rig.sent) == 1 + assert rig.events(ctx, "user_sim_read")[0]["ends_run"] is True + + +@pytest.mark.asyncio +async def test_done_in_a_multi_day_run_is_recorded_and_the_run_goes_on_to_end_time(monkeypatch, tmp_path): + rig = Rig(monkeypatch, tmp_path, agent=_agent(t1=_message("house booked"), t2=_message("airline moved Dev")), + answers=[{"outcome": "done", "message": "perfect ty"}, {"outcome": "reply", "message": "ok!"}]) + await rig.arm() + rig.triggers(rig.notify_at("airline", "2026-11-11T14:00:00Z", "Wingloft: flight retimed", channel="Gmail")) + ctx = await rig.run(rig.step(end_time="P3D")) + + record = rig.record(ctx) + assert record["multi_day"] is True and record["end_reason"] == "end_time" + assert [r["outcome"] for r in rig.events(ctx, "user_sim_read")] == ["done", "reply"] + assert rig.events(ctx, "user_sim_read")[0]["ends_run"] is False + assert [m["text"] for m in rig.reads[1]["assistant_messages"]] == ["airline moved Dev"] + assert record["counts"]["user_dones"] == 1 + assert "perfect ty" not in repr(rig.sent) # a done never reaches the agent + + +@pytest.mark.parametrize("end_time,multi_day", [(None, False), ("PT6H", False), ("P1D", True)]) +@pytest.mark.asyncio +async def test_a_run_is_multi_day_when_its_end_falls_on_a_later_day_for_the_user(monkeypatch, tmp_path, end_time, + multi_day): + rig = Rig(monkeypatch, tmp_path) + await rig.arm() + ctx = await rig.run(rig.step(**({"end_time": end_time} if end_time else {}))) + assert rig.record(ctx)["multi_day"] is multi_day + + +# ---- the time zone -------------------------------------------------------------------------------------------------- + +@pytest.mark.asyncio +async def test_the_world_s_time_zone_is_the_user_s_when_the_user_sim_sets_none(monkeypatch, tmp_path): + rig = Rig(monkeypatch, tmp_path) + await rig.arm() + ctx = rig.context(user={"time_zone": None}) + ctx.metadata["clock_configurations"] = [{"step_id": "clock", "timezone": "America/Los_Angeles"}] + ctx = await rig.run(rig.step(), ctx) + + assert rig.record(ctx)["time_zone"] == "America/Los_Angeles" + assert rig.setups[0]["time_zone"] == "America/Los_Angeles" + + +@pytest.mark.asyncio +async def test_a_user_and_a_world_in_different_time_zones_are_refused(monkeypatch, tmp_path): + rig = Rig(monkeypatch, tmp_path) + await rig.arm() + ctx = rig.context() + ctx.metadata["clock_configurations"] = [{"step_id": "clock", "timezone": "America/Los_Angeles"}] + with pytest.raises(ValueError, match="America/New_York but sync_env_clock set"): + await rig.run(rig.step(), ctx) + + +# ---- the steps ------------------------------------------------------------------------------------------------------ + +def test_both_steps_are_registered(): + registry = get_task_step_registry() + assert registry["always_on"] is ao.AlwaysOnTaskStep and registry["user_sim"] is ao.UserSimTaskStep + + +def test_the_user_sim_step_defaults_to_ours_and_has_no_apps_setting(): + step = ao.UserSimTaskStep.from_dict({"id": "u", "type": "user_sim", "name": "linda", "model": "m", + "apps": ["Gmail"], "time_zone": "America/New_York"}) + assert step.user_version == "ours" and "apps" not in step.to_dict() + with pytest.raises(Exception): + ao.UserSimTaskStep("u", None, name="linda", model="m", time_zone="Mars/Olympus") + + +def test_ours_works_from_the_description_alone(): + person = {"name": "Linda Murphy", "first": "Linda", "description": "You're 31, planning Noor's 30th."} + setup = user_sim_versions.setup("ours", person) + assert setup["system_prompt"].endswith("WHO YOU ARE\nYou're 31, planning Noor's 30th.\n") + assert "Any other date you see is the computer's clock" in setup["system_prompt"] + assert setup["output_format"]["schema"]["properties"]["outcome"]["enum"] == ["reply", "no_reply", "done"] + own = user_sim_versions.setup("ours", {**person, "versions": {"ours": {"system_prompt": "tagged lines"}}}) + assert own["system_prompt"] == "tagged lines" + + +def test_ours_renders_one_labelled_read(): + person = {"name": "Linda Murphy", "first": "Linda", "time_zone": "America/New_York"} + read = {"time": "2026-11-10T15:00:00Z", + "assistant_messages": [{"at": "2026-11-10T13:00:00Z", "text": "house?"}, + {"at": "2026-11-10T14:00:00Z", "text": "and dinner?"}], + "phone": [{"at": "2026-11-10T12:00:00Z", "channel": "Gmail", "text": "New email from Sam"}], + "you_sent": [{"at": "2026-11-09T14:00:00Z", "text": "sort the cdmx weekend"}]} + text = user_sim_versions.render_ours("ours", person, [], read) + assert text.startswith("It's Tue, Nov 10, 10:00 AM.") + for label in ("NEW ON YOUR PHONE", "YOU SENT YOUR AI ASSISTANT", "NEW FROM YOUR AI ASSISTANT"): + assert label in text + assert text.index("house?") < text.index("and dinner?") diff --git a/tst/unit/task_step/test_field_scopes.py b/tst/unit/task_step/test_field_scopes.py index fb6f71c83..92ba03ddc 100644 --- a/tst/unit/task_step/test_field_scopes.py +++ b/tst/unit/task_step/test_field_scopes.py @@ -13,6 +13,7 @@ # A stored step of each built-in type that scopes fields of its own, holding only what the type requires. MINIMAL = { "add_skills": {"skills": [{"name": "notes", "description": "how notes are kept"}]}, + "always_on": {"prompt": "hi", "user_sim": "linda"}, "deploy_agent": {}, "deploy_env": {"env_id": "crm-env"}, "deploy_human_agent": {}, From 26985053b3a3cdc274f9e3bbbfb52bdb995660ac Mon Sep 17 00:00:00 2001 From: Sarah Chitty Date: Mon, 9 Nov 2026 00:23:44 -0800 Subject: [PATCH 05/53] always_on: the agent gets the task's time, not the computer's (FD-4408) - While the task's clock is set, each wake opens with the task's date and time (with the year, and the user's local time when they aren't in UTC), and says any other date the agent sees is the computer's clock. The line is tagged metadata.agentenv.source "time". - Gateway: a role the user-chat config names always has get_time while the clock is set, like message_user, so a task's tool rules (a deny-all, say) can't hide it. Roles the config doesn't name are unchanged. - Without a system prompt, the first wake still opens with the message rules, now ahead of the time line. Co-Authored-By: Claude Opus 5.5 --- src/agent_env/env/gateway/gateway.py | 5 +- .../task_step/task_steps/always_on.py | 27 ++++++++-- tst/unit/env/gateway/user_chat_test.py | 15 ++++++ tst/unit/task_step/test_always_on_time.py | 54 +++++++++++++++++++ 4 files changed, 95 insertions(+), 6 deletions(-) create mode 100644 tst/unit/task_step/test_always_on_time.py diff --git a/src/agent_env/env/gateway/gateway.py b/src/agent_env/env/gateway/gateway.py index a1c761ca8..0b2452cb2 100644 --- a/src/agent_env/env/gateway/gateway.py +++ b/src/agent_env/env/gateway/gateway.py @@ -572,11 +572,14 @@ def _listen_socket(self, family: int, host: str) -> socket.socket: def _is_disabled(self, role: str, name: str) -> bool: """Walk the rule chain (role first, then WILDCARD) returning the first concrete True/False; default False. A tool listed hidden by default skips "*"-tool rules: only rules naming it apply, and with none it is hidden. - The scheduled-check tools exist only for the roles the notifications config names, whatever the rules say.""" + The scheduled-check tools exist only for the roles the notifications config names, whatever the rules say. + A role the user-chat config names always has message_user and get_time, whatever the rules say.""" if name in CHECK_TOOL_NAMES and name not in self._tool_server_urls and role not in self._notifications.roles: return True if name == MESSAGE_USER_TOOL_NAME and name not in self._tool_server_urls: return role not in self._user_chat.roles # built in: role rules don't hide it from a configured role + if name == GET_TIME_TOOL_NAME and name not in self._tool_server_urls and role in self._user_chat.roles: + return False if name == STAY_QUIET_TOOL_NAME and name not in self._tool_server_urls: return not (self._user_chat.stay_quiet and role in self._user_chat.roles) # opt-in, like message_user hidden = name in self._hidden_by_default diff --git a/src/agent_env/task_step/task_steps/always_on.py b/src/agent_env/task_step/task_steps/always_on.py index 77f6ae8b9..6401d8fea 100644 --- a/src/agent_env/task_step/task_steps/always_on.py +++ b/src/agent_env/task_step/task_steps/always_on.py @@ -5,7 +5,8 @@ sim and its plug-in) and records it in the run's metadata under its ``name``; a ``deploy_agent`` step deploys the agent that plays them. The ``always_on`` step names it with ``user_sim`` and runs the loop: -- The first wake is the user's request. Each wake opens with what came due for the agent on notifications/v1 (a +- The first wake is the user's request. While the task's clock is set, each wake opens with the task's date and time + (any other date the agent sees is the computer's clock), then what came due for the agent on notifications/v1 (a trigger's ``notify``, a scheduled check), labelled ``[time · channel]``, then the user's own messages. - Only what the agent sends with the gateway's ``message_user`` tool (user-chat/v1) reaches the user; its plain reply goes to the log. Each wake should end with ``message_user`` or ``stay_quiet``: one that ends with closing text and @@ -81,6 +82,7 @@ "goes to a private log.") QUIET_RULE = "If nothing is worth saying, call stay_quiet." EMPTY_REPLIES = frozenset({"(The assistant's reply was empty.)"}) # a harness's stand-in for an empty final reply +TIME_LINE = "It's {when}. Any other date you see is the computer's clock, not the task's." # ================================================================================================================== @@ -402,6 +404,14 @@ def _system_prompt(self) -> Optional[str]: opens with the rules.""" return f"{self.step.system_prompt.rstrip()}\n\n{self.rules()}" if self.step.system_prompt else None + def _time_line(self, at: datetime) -> str: + """The task's date and time, and the user's local time when they aren't in UTC.""" + when = f"{at.strftime('%a %b')} {at.day}, {at.year}, {_clock_time(at)} UTC" + if self.zone != "UTC": + local = at.astimezone(self.tz) + when += f" ({local.strftime('%a')} {_clock_time(local)} for {self.person['first']}, {self.zone})" + return TIME_LINE.format(when=when) + # ---- clock and world ------------------------------------------------------------------------------------ async def _armed(self) -> list: armed = [] @@ -553,8 +563,7 @@ async def loop(self, now: datetime) -> None: first = self._effective_prompt() self.you_sent.append({"at": _iso(now), "text": first}) due = await take_due(self.notification_envs, self.recipient) if self.notification_envs else [] - rules = [] if step.system_prompt else [_step_part(self.rules(), "rules")] - await self.wake([*rules, *delivered_parts(due), self._user_part(now, first, opening=True)], + await self.wake([*delivered_parts(due), self._user_part(now, first, opening=True)], woke_by=[f"notification:{d['channel']}" for d in due] + ["user_request"], notifications=[{k: d.get(k) for k in ("at", "channel", "source", "source_id", "to")} for d in due]) @@ -661,11 +670,15 @@ async def _wake_with(self, now: datetime, lines: list[dict], replies: list[dict] async def wake(self, parts: list[dict], woke_by: list[str], notifications: Optional[list] = None) -> None: step = self.step + sent_at = await self.now() + clock = await step._env_clock(self.clock_envs) + lead = [_step_part(self._time_line(sent_at), "time")] if clock else [] + if self.wakes == 0 and not step.system_prompt: + lead = [_step_part(self.rules(), "rules"), *lead] + parts = [*lead, *parts] if step.wake_reminder: parts = [*parts, _step_part(step.wake_reminder_text, "reminder")] self.wakes += 1 - sent_at = await self.now() - clock = await step._env_clock(self.clock_envs) msg_id = uuid.uuid4().hex def record() -> None: @@ -993,6 +1006,10 @@ def _step_part(text: str, source: str) -> dict: return {"kind": "text", "text": text, "metadata": {"agentenv": {"source": source}}} +def _clock_time(at: datetime) -> str: + return at.strftime("%I:%M %p").lstrip("0") + + def _closing_text(plain: str) -> str: text = plain.strip() return "" if text in EMPTY_REPLIES else text diff --git a/tst/unit/env/gateway/user_chat_test.py b/tst/unit/env/gateway/user_chat_test.py index 7755a3540..a6147223a 100644 --- a/tst/unit/env/gateway/user_chat_test.py +++ b/tst/unit/env/gateway/user_chat_test.py @@ -112,3 +112,18 @@ def test_a_bad_config_is_refused(gateway, body): def test_an_unconfigured_gateway_lists_neither_tool(gateway): assert not {"message_user", "stay_quiet"} & _names(gateway, "default") + + +def test_get_time_reaches_a_configured_role_through_a_deny_all_while_the_clock_is_set(gateway): + _req(gateway, "POST", "/tools/disable", {"role": "psi_agent", "tools": ["*"]}) + _req(gateway, "POST", "/tools/disable", {"role": "other", "tools": ["*"]}) + assert "get_time" not in _names(gateway, "psi_agent") + _req(gateway, "POST", "/user_chat/config", {"roles": ["psi_agent"]}) + assert "get_time" not in _names(gateway, "psi_agent") # no clock set, so no get_time at all + + assert _req(gateway, "PUT", "/clock/set-time", {"virtual_time": T0}).status_code == 200 + assert "get_time" in _names(gateway, "psi_agent") and "get_time" not in _names(gateway, "other") + called = _call(gateway, "get_time", {}, "psi_agent") + assert called.status_code == 200 and T0 in called.text + _req(gateway, "POST", "/user_chat/clear", {}) + assert "get_time" not in _names(gateway, "psi_agent") diff --git a/tst/unit/task_step/test_always_on_time.py b/tst/unit/task_step/test_always_on_time.py new file mode 100644 index 000000000..87012a4cd --- /dev/null +++ b/tst/unit/task_step/test_always_on_time.py @@ -0,0 +1,54 @@ +"""The always_on step gives the agent the task's time, not the computer's: each wake opens with the task's date and +time, and says any other date is the computer's clock.""" + +from __future__ import annotations + +import pytest + +from tst.unit.task_step.always_on_harness import Rig + +pytestmark = pytest.mark.usefixtures("local_stores") + +CLOCK = "Any other date you see is the computer's clock, not the task's." +PROMPT = "You are Linda's assistant." + + +@pytest.mark.asyncio +async def test_each_wake_opens_with_the_task_s_date_and_time(monkeypatch, tmp_path): + rig = Rig(monkeypatch, tmp_path) + await rig.arm() + rig.triggers(rig.notify_at("sam", "2026-11-10T14:00:00Z", "Sam: shh", channel="noor 30")) + ctx = await rig.run(rig.step(system_prompt=PROMPT)) + + assert [texts[0] for texts in rig.sent] == [ + f"It's Mon Nov 9, 2026, 2:00 PM UTC (Mon 9:00 AM for Linda, America/New_York). {CLOCK}", + f"It's Tue Nov 10, 2026, 2:00 PM UTC (Tue 9:00 AM for Linda, America/New_York). {CLOCK}", + ] + assert rig.parts[1][0]["metadata"] == {"agentenv": {"source": "time"}} + assert rig.events(ctx, "wake")[1]["parts_text"].startswith("It's Tue Nov 10, 2026") + + +@pytest.mark.asyncio +async def test_a_user_in_utc_gets_the_time_once(monkeypatch, tmp_path): + rig = Rig(monkeypatch, tmp_path) + await rig.arm() + await rig.run(rig.step(system_prompt=PROMPT), rig.context(user={"time_zone": "UTC"})) + assert rig.sent[0][0] == f"It's Mon Nov 9, 2026, 2:00 PM UTC. {CLOCK}" + + +@pytest.mark.asyncio +async def test_without_a_task_clock_there_is_no_time_line(monkeypatch, tmp_path): + rig = Rig(monkeypatch, tmp_path) + await rig.run(rig.step()) + assert not any(CLOCK in t for t in rig.sent[0]) + + +@pytest.mark.asyncio +async def test_get_time_stays_with_the_agent_when_the_task_hides_every_tool(monkeypatch, tmp_path): + rig = Rig(monkeypatch, tmp_path) + await rig.arm() + async with rig.http() as client: + await client.post("/tools/disable", json={"role": "default", "tools": ["*"]}) + assert "get_time" not in await rig.tools() + await rig.run(rig.step()) + assert {"get_time", "message_user", "stay_quiet"} <= await rig.tools() From 8a4cec2ac85193ae831600b2f320ae744bacfd68 Mon Sep 17 00:00:00 2001 From: Sarah Chitty Date: Mon, 9 Nov 2026 00:26:03 -0800 Subject: [PATCH 06/53] always_on: the agent can schedule its own wake-ups, on by default for multi-day runs (FD-5368) - scheduled_checks (default: on when end_time falls on a later day than the start, in the user's zone) gives the agent's role scheduled checks on each env whose clock is set, at least scheduled_check_min_interval apart (default PT3H, against the gateway's own PT1H default; hourly checks ran a 5-day Opus run out of wakes by Thursday). True or false turns them on or off whatever the run's length. - A role the task already gave checks (configure_scheduled_checks) keeps its own settings; other roles the task named keep theirs. - When checks are on, the agent's rules say it can schedule a wake-up. - Gateway: a role the user-chat config names also always has the check tools its notifications config gives it, so a deny-all can't hide them. - The run record says where checks came from (settings.scheduled_checks). A check's wake counts toward max_wakes. Co-Authored-By: Claude Opus 5.5 --- src/agent_env/env/gateway/gateway.py | 6 +- .../task_step/task_steps/always_on.py | 55 +++++++-- tst/unit/env/gateway/user_chat_test.py | 12 ++ tst/unit/task_step/test_always_on_checks.py | 111 ++++++++++++++++++ tst/unit/task_step/test_always_on_messages.py | 2 +- 5 files changed, 175 insertions(+), 11 deletions(-) create mode 100644 tst/unit/task_step/test_always_on_checks.py diff --git a/src/agent_env/env/gateway/gateway.py b/src/agent_env/env/gateway/gateway.py index 0b2452cb2..f9615c81b 100644 --- a/src/agent_env/env/gateway/gateway.py +++ b/src/agent_env/env/gateway/gateway.py @@ -573,12 +573,14 @@ def _is_disabled(self, role: str, name: str) -> bool: """Walk the rule chain (role first, then WILDCARD) returning the first concrete True/False; default False. A tool listed hidden by default skips "*"-tool rules: only rules naming it apply, and with none it is hidden. The scheduled-check tools exist only for the roles the notifications config names, whatever the rules say. - A role the user-chat config names always has message_user and get_time, whatever the rules say.""" + A role the user-chat config names always has message_user, get_time and the scheduled-check tools the + notifications config gives it, whatever the rules say.""" if name in CHECK_TOOL_NAMES and name not in self._tool_server_urls and role not in self._notifications.roles: return True if name == MESSAGE_USER_TOOL_NAME and name not in self._tool_server_urls: return role not in self._user_chat.roles # built in: role rules don't hide it from a configured role - if name == GET_TIME_TOOL_NAME and name not in self._tool_server_urls and role in self._user_chat.roles: + if (name == GET_TIME_TOOL_NAME or name in CHECK_TOOL_NAMES) and name not in self._tool_server_urls \ + and role in self._user_chat.roles: return False if name == STAY_QUIET_TOOL_NAME and name not in self._tool_server_urls: return not (self._user_chat.stay_quiet and role in self._user_chat.roles) # opt-in, like message_user diff --git a/src/agent_env/task_step/task_steps/always_on.py b/src/agent_env/task_step/task_steps/always_on.py index 6401d8fea..8d579652d 100644 --- a/src/agent_env/task_step/task_steps/always_on.py +++ b/src/agent_env/task_step/task_steps/always_on.py @@ -22,6 +22,9 @@ earliest of: the next trigger event, the next notification, a held reply, a pending read, or end_time, never past one; then waits up to ``settle_timeout_seconds`` for triggers to finish. What comes due in the same minute wakes the agent once. +- The agent can schedule its own wake-ups (scheduled checks) when the run is multi-day, at least + ``scheduled_check_min_interval`` apart; ``scheduled_checks`` turns them on or off whatever the run's length. A + check's wake counts toward ``max_wakes``. - At end_time the user's ``final`` line (with a reminder to use message_user) gives the agent one last turn. The run ends on end_time, done (one sitting), max_wakes, or nothing left to wake the agent; the reason is recorded. - Everything goes into a timeline in ``metadata["always_on"][step_id]`` and the object store. @@ -50,6 +53,7 @@ from agent_env.artifact import FileArtifact from agent_env.entity.refs import EntityRef from agent_env.env.env import EnvCapabilityUnsupported +from agent_env.env.gateway.notifications import DEFAULT_MAX_ACTIVE from agent_env.env.gateway.constants import ( DEFAULT_ROLE, EXT_CLOCK_URI, @@ -81,6 +85,9 @@ MESSAGE_RULES = ("The user never sees your normal replies. The only way to reach them is message_user. Text outside it " "goes to a private log.") QUIET_RULE = "If nothing is worth saying, call stay_quiet." +CHECKS_RULE = ("You only run when something wakes you. To do something at a later time, schedule a check with " + "schedule_check: you'll be woken then with your note.") +CHECK_MIN_INTERVAL = "PT3H" EMPTY_REPLIES = frozenset({"(The assistant's reply was empty.)"}) # a harness's stand-in for an empty final reply TIME_LINE = "It's {when}. Any other date you see is the computer's clock, not the task's." @@ -160,6 +167,7 @@ def __init__(self, id: str, version: Optional[int], *, user_sim: str, end_time: user_timeout_seconds: int = 300, final_reminder: Optional[str] = FINAL_REMINDER, stay_quiet: bool = True, nudge: bool = True, nudge_text: str = NUDGE_TEXT, wake_reminder: bool = True, wake_reminder_text: str = WAKE_REMINDER_TEXT, + scheduled_checks: Optional[bool] = None, scheduled_check_min_interval: str = CHECK_MIN_INTERVAL, model: Optional[str] = None, model_params: Optional[dict] = None, effort: Optional[str] = None, max_thinking_tokens: Optional[int] = None, harness: Optional[str] = None, trajectory_output_prefix: Optional[str] = None, **prompt_fields: Any): @@ -185,9 +193,12 @@ def __init__(self, id: str, version: Optional[int], *, user_sim: str, end_time: self.nudge_text = nudge_text self.wake_reminder = bool(wake_reminder) self.wake_reminder_text = wake_reminder_text + self.scheduled_checks = scheduled_checks + self.scheduled_check_min_interval = scheduled_check_min_interval _OWN = ("user_sim", "end_time", "final", "max_wakes", "settle_timeout_seconds", "user_timeout_seconds", - "final_reminder", "stay_quiet", "nudge", "nudge_text", "wake_reminder", "wake_reminder_text") + "final_reminder", "stay_quiet", "nudge", "nudge_text", "wake_reminder", "wake_reminder_text", + "scheduled_checks", "scheduled_check_min_interval") _PROMPT_FIELDS = ("prompt", "parts", "prompt_id", "agent_name", "timeout_seconds", "trajectory_output_prefix", "model", "model_params", "system_prompt", "max_turns", "output_format", "max_thinking_tokens", "effort", "harness", "agentenv_tools", "poll_interval_seconds", "context_id") @@ -249,6 +260,7 @@ def __init__(self, step: AlwaysOnTaskStep, context: TaskStepContext): self.nudges = 0 self.zone = "UTC" self.multi_day = False + self.checks: dict = {"on": False} def close(self) -> None: for p in self._plugin_paths: @@ -295,11 +307,16 @@ async def go(self) -> None: if not self.chat_envs: raise RuntimeError(f"always_on: no deployed env offers {EXT_USER_CHAT_URI} (message_user); rebuild its " "gateway from a branch that has user_chat.py") + now = await self.now() + t0 = await self._t0() or now + self.end = _end_instant(step.end_time, t0) if step.end_time else None + self.multi_day = self.end is not None and self.end.astimezone(self.tz).date() > t0.astimezone(self.tz).date() chat_config = {"roles": [self.recipient], "user_name": self.person.get("name") or "your user"} if step.stay_quiet: chat_config["stay_quiet"] = True for env in self.chat_envs: await _call(env, EXT_USER_CHAT_URI, "config", chat_config) + await self._configure_checks() await self._configure_agents() self.conversation_id = uuid.uuid4().hex @@ -315,13 +332,10 @@ async def go(self) -> None: self.solver_context_id = uuid.uuid4().hex now = await self.now() - t0 = await self._t0() or now - self.end = _end_instant(step.end_time, t0) if step.end_time else None - self.multi_day = self.end is not None and self.end.astimezone(self.tz).date() > t0.astimezone(self.tz).date() self.log("start", virtual_time=_iso(now), end_time=_iso(self.end) if self.end else None, agent=step.agent_name, agent_role=self.recipient, user_sim=step.user_sim, version=self.version, user_model=us["model"], max_wakes=step.max_wakes, time_zone=self.zone, multi_day=self.multi_day, - clock_rate=await self._rate()) + scheduled_checks=self.checks, clock_rate=await self._rate()) try: await self.loop(now) except Exception as e: @@ -396,8 +410,32 @@ async def _configure_agents(self) -> None: logger.warning("always_on: the user sim agent's card did not accept system_prompt; it keeps its deploy one") def rules(self) -> str: - """What the agent is told about reaching the user.""" - return " ".join([MESSAGE_RULES, *([QUIET_RULE] if self.step.stay_quiet else [])]) + """What the agent is told about reaching the user, and about waking itself when it can.""" + return " ".join([MESSAGE_RULES, *([QUIET_RULE] if self.step.stay_quiet else []), + *([CHECKS_RULE] if self.checks["on"] else [])]) + + async def _configure_checks(self) -> None: + """Give the agent's role scheduled checks (notifications/v1) on each env whose clock is set: by default when + the run is multi-day, else as ``scheduled_checks`` says. A role the task already gave checks keeps its own + settings; other roles the task named keep theirs.""" + step = self.step + wanted = self.multi_day if step.scheduled_checks is None else step.scheduled_checks + envs = [e for e in await self._armed() if e.supports(EXT_NOTIFICATIONS_URI, "config")] + applied = [] + for env in envs: + state = await _call(env, EXT_NOTIFICATIONS_URI, "state") + roles = list(state.get("roles") or []) + if self.recipient in roles: + applied.append({"env_id": env.env_id, "set_by": "task", "min_interval": state.get("min_interval"), + "max_active": state.get("max_active")}) + elif wanted: + body = {"roles": [*roles, self.recipient], + "max_active": state.get("max_active", DEFAULT_MAX_ACTIVE) if roles else DEFAULT_MAX_ACTIVE, + "min_interval": state.get("min_interval") if roles else step.scheduled_check_min_interval} + got = await _call(env, EXT_NOTIFICATIONS_URI, "config", body) + applied.append({"env_id": env.env_id, "set_by": "step", "min_interval": got.get("min_interval"), + "max_active": got.get("max_active")}) + self.checks = {"on": bool(applied), "envs": applied} def _system_prompt(self) -> Optional[str]: """The step's system prompt with the rules after it; without one, the agent keeps its own and the first wake @@ -947,7 +985,8 @@ async def finish(self) -> None: "agent_name": self.us.get("agent_name"), **self.user_setup}, "end_time": _iso(self.end) if self.end else None, "time_zone": self.zone, "multi_day": self.multi_day, "timeline": self.timeline} - record["settings"] = {"stay_quiet": step.stay_quiet, "nudge": step.nudge, "wake_reminder": step.wake_reminder} + record["settings"] = {"stay_quiet": step.stay_quiet, "nudge": step.nudge, "wake_reminder": step.wake_reminder, + "scheduled_checks": self.checks} try: store = get_config().get_object_store() key = (f"{get_config().get_artifact_key_prefix()}always_on_timeline/" diff --git a/tst/unit/env/gateway/user_chat_test.py b/tst/unit/env/gateway/user_chat_test.py index a6147223a..b64906361 100644 --- a/tst/unit/env/gateway/user_chat_test.py +++ b/tst/unit/env/gateway/user_chat_test.py @@ -127,3 +127,15 @@ def test_get_time_reaches_a_configured_role_through_a_deny_all_while_the_clock_i assert called.status_code == 200 and T0 in called.text _req(gateway, "POST", "/user_chat/clear", {}) assert "get_time" not in _names(gateway, "psi_agent") + + +def test_the_check_tools_reach_a_configured_role_through_a_deny_all_once_notifications_name_it(gateway): + checks = {"schedule_check", "list_scheduled_checks", "cancel_scheduled_check"} + _req(gateway, "POST", "/tools/disable", {"role": "psi_agent", "tools": ["*"]}) + _req(gateway, "POST", "/notifications/config", {"roles": ["psi_agent"]}) + assert not checks & _names(gateway, "psi_agent") # the task's deny-all, as before + + _req(gateway, "POST", "/user_chat/config", {"roles": ["psi_agent"]}) + assert checks <= _names(gateway, "psi_agent") + _req(gateway, "POST", "/notifications/config", {"roles": []}) + assert not checks & _names(gateway, "psi_agent") diff --git a/tst/unit/task_step/test_always_on_checks.py b/tst/unit/task_step/test_always_on_checks.py new file mode 100644 index 000000000..c63f56369 --- /dev/null +++ b/tst/unit/task_step/test_always_on_checks.py @@ -0,0 +1,111 @@ +"""The always_on step lets the agent wake itself: scheduled checks are on by default when a run spans more than one +day of the task's time, at least PT3H apart, and the agent is told it can schedule them.""" + +from __future__ import annotations + +import pytest + +from agent_env.env.gateway.notifications import NotificationError +from agent_env.task_step.task_steps import always_on as ao +from tst.unit.task_step.always_on_harness import Rig + +pytestmark = pytest.mark.usefixtures("local_stores") + +CHECK_TOOLS = {"schedule_check", "list_scheduled_checks", "cancel_scheduled_check"} + + +@pytest.mark.asyncio +async def test_a_multi_day_run_gives_the_agent_checks_through_a_deny_all_and_says_so(monkeypatch, tmp_path): + rig = Rig(monkeypatch, tmp_path) + await rig.arm() + async with rig.http() as client: + await client.post("/tools/disable", json={"role": "default", "tools": ["*"]}) + ctx = await rig.run(rig.step(end_time="P5D")) + + assert rig.gw._notifications.roles == ("default",) and rig.gw._notifications.min_interval == "PT3H" + assert CHECK_TOOLS <= await rig.tools() + assert ao.CHECKS_RULE in rig.sent[0][0] + assert rig.record(ctx)["settings"]["scheduled_checks"] == { + "on": True, "envs": [{"env_id": "world", "set_by": "step", "min_interval": "PT3H", "max_active": 10}]} + + +@pytest.mark.asyncio +async def test_checks_closer_than_the_minimum_gap_are_refused(monkeypatch, tmp_path): + rig = Rig(monkeypatch, tmp_path) + await rig.arm() + await rig.run(rig.step(end_time="P5D")) + + with pytest.raises(NotificationError, match="at least PT3H"): + rig.gw._notifications.schedule("sync stock", every="PT1H", to="default") + rig.gw._notifications.schedule("sync stock", every="PT3H", to="default") + + +@pytest.mark.asyncio +async def test_a_check_the_agent_schedules_moves_the_clock_and_wakes_it(monkeypatch, tmp_path): + def agent(n, texts, rig): + if n == 1: + rig.schedule("book La Ola, it opens at 9", at="2026-11-12T14:00:00Z") + rig.quiet() + return "" + + rig = Rig(monkeypatch, tmp_path, agent=agent) + await rig.arm() + ctx = await rig.run(rig.step(end_time="P5D")) + + wake = rig.events(ctx, "wake")[1] + assert wake["virtual_time"] == "2026-11-12T14:00:00Z" and wake["woke_by"] == ["notification:your scheduled check"] + assert "[Thu Nov 12, 2:00 PM UTC · your scheduled check] book La Ola, it opens at 9" in rig.sent[1] + + +@pytest.mark.parametrize("end_time,setting,on", [(None, None, False), ("PT6H", None, False), ("P5D", False, False), + (None, True, True)], + ids=["one sitting", "same day", "turned off", "turned on"]) +@pytest.mark.asyncio +async def test_otherwise_a_task_decides(monkeypatch, tmp_path, end_time, setting, on): + rig = Rig(monkeypatch, tmp_path) + await rig.arm() + settings = {**({"end_time": end_time} if end_time else {}), + **({"scheduled_checks": setting} if setting is not None else {})} + ctx = await rig.run(rig.step(**settings)) + + assert (rig.gw._notifications.roles == ("default",)) is on + assert rig.record(ctx)["settings"]["scheduled_checks"]["on"] is on + assert (ao.CHECKS_RULE in rig.sent[0][0]) is on + + +@pytest.mark.asyncio +async def test_the_line_goes_in_the_system_prompt_when_checks_are_on(monkeypatch, tmp_path): + rig = Rig(monkeypatch, tmp_path) + await rig.arm() + run = ao._Run(rig.step(system_prompt="You are Linda's assistant.", end_time="P5D"), rig.context()) + run.checks = {"on": True} + assert run._system_prompt().endswith(ao.CHECKS_RULE) + + +@pytest.mark.asyncio +async def test_the_task_s_own_check_settings_are_kept(monkeypatch, tmp_path): + rig = Rig(monkeypatch, tmp_path) + await rig.arm() + rig.gw._notifications.configure({"roles": ["default"], "max_active": 3, "min_interval": "PT1H"}) + ctx = await rig.run(rig.step(end_time="P5D", scheduled_checks=False)) + + assert rig.gw._notifications.min_interval == "PT1H" and rig.gw._notifications.max_active == 3 + assert rig.record(ctx)["settings"]["scheduled_checks"]["envs"][0]["set_by"] == "task" + + +@pytest.mark.asyncio +async def test_other_roles_the_task_gave_checks_keep_them(monkeypatch, tmp_path): + rig = Rig(monkeypatch, tmp_path) + await rig.arm() + rig.gw._notifications.configure({"roles": ["planner"], "min_interval": "PT2H"}) + await rig.run(rig.step(end_time="P5D")) + + assert rig.gw._notifications.roles == ("planner", "default") and rig.gw._notifications.min_interval == "PT2H" + + +def test_the_settings_survive_a_round_trip(): + step = ao.AlwaysOnTaskStep("week", None, user_sim="linda", prompt="hi", prompt_id="p", agent_name="a", + scheduled_checks=False, scheduled_check_min_interval="PT6H") + again = ao.AlwaysOnTaskStep.from_dict(step.to_dict()) + assert (again.scheduled_checks, again.scheduled_check_min_interval) == (False, "PT6H") + assert ao.AlwaysOnTaskStep.from_dict({**step.to_dict(), "scheduled_checks": None}).scheduled_checks is None diff --git a/tst/unit/task_step/test_always_on_messages.py b/tst/unit/task_step/test_always_on_messages.py index ca82008b4..e397ab8de 100644 --- a/tst/unit/task_step/test_always_on_messages.py +++ b/tst/unit/task_step/test_always_on_messages.py @@ -39,7 +39,7 @@ async def test_all_three_are_on_by_default(monkeypatch, tmp_path): ctx = await rig.run(step) assert (step.stay_quiet, step.nudge, step.wake_reminder) == (True, True, True) - assert rig.record(ctx)["settings"] == {"stay_quiet": True, "nudge": True, "wake_reminder": True} + assert rig.record(ctx)["settings"].items() >= {"stay_quiet": True, "nudge": True, "wake_reminder": True}.items() assert rig.gw._user_chat.stay_quiet and ROLE in rig.gw._user_chat.roles From d63ba954a8f548117f8b3391b7b04cdee7178277 Mon Sep 17 00:00:00 2001 From: Sarah Chitty Date: Mon, 9 Nov 2026 00:26:38 -0800 Subject: [PATCH 07/53] always_on: user_sim takes the user's email (FD-5366) The core-loop ticket lists email among the user_sim settings; it is recorded and handed to the plug-in with the rest of the person. Co-Authored-By: Claude Opus 5.5 --- src/agent_env/task_step/task_steps/always_on.py | 9 +++++---- tst/unit/task_step/test_always_on_loop.py | 4 +++- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/src/agent_env/task_step/task_steps/always_on.py b/src/agent_env/task_step/task_steps/always_on.py index 8d579652d..b6d2f2194 100644 --- a/src/agent_env/task_step/task_steps/always_on.py +++ b/src/agent_env/task_step/task_steps/always_on.py @@ -102,7 +102,7 @@ class UserSimTaskStep(TaskStep): def __init__(self, id: str, version: Optional[int], *, name: str, model: str, agent_name: Optional[str] = None, user_version: str = "ours", display_name: Optional[str] = None, first_name: Optional[str] = None, - description: Optional[str] = None, time_zone: Optional[str] = None, + description: Optional[str] = None, email: Optional[str] = None, time_zone: Optional[str] = None, schedule: Optional[dict] = None, response_times: Optional[dict] = None, max_replies: Optional[int] = None, permissions: Optional[dict] = None, keep_private: Optional[list] = None, output_format: Optional[dict] = None, @@ -121,6 +121,7 @@ def __init__(self, id: str, version: Optional[int], *, name: str, model: str, ag self.display_name = display_name self.first_name = first_name self.description = description + self.email = email self.time_zone = time_zone self.schedule = schedule self.response_times = response_times @@ -131,8 +132,8 @@ def __init__(self, id: str, version: Optional[int], *, name: str, model: str, ag self.versions = versions or {} self.plugin = plugin - _FIELDS = ("name", "model", "agent_name", "user_version", "display_name", "first_name", "description", "time_zone", - "schedule", "response_times", "max_replies", "permissions", "keep_private", "output_format", + _FIELDS = ("name", "model", "agent_name", "user_version", "display_name", "first_name", "description", "email", + "time_zone", "schedule", "response_times", "max_replies", "permissions", "keep_private", "output_format", "versions", "plugin") def to_dict(self) -> dict: @@ -349,7 +350,7 @@ async def go(self) -> None: def _person(self, us: dict) -> dict: name = us.get("display_name") or us["name"] return {"key": us["name"], "name": name, "first": us.get("first_name") or name.split()[0], - "time_zone": us.get("time_zone"), + "email": us.get("email"), "time_zone": us.get("time_zone"), "description": us.get("description"), "versions": us.get("versions") or {}, "output_format": us.get("output_format"), "permissions": us.get("permissions"), "keep_private": us.get("keep_private")} diff --git a/tst/unit/task_step/test_always_on_loop.py b/tst/unit/task_step/test_always_on_loop.py index 15ac70a89..cb76a8a15 100644 --- a/tst/unit/task_step/test_always_on_loop.py +++ b/tst/unit/task_step/test_always_on_loop.py @@ -285,8 +285,10 @@ def test_both_steps_are_registered(): def test_the_user_sim_step_defaults_to_ours_and_has_no_apps_setting(): step = ao.UserSimTaskStep.from_dict({"id": "u", "type": "user_sim", "name": "linda", "model": "m", - "apps": ["Gmail"], "time_zone": "America/New_York"}) + "apps": ["Gmail"], "time_zone": "America/New_York", + "email": "linda.murphy@example.test"}) assert step.user_version == "ours" and "apps" not in step.to_dict() + assert ao.UserSimTaskStep.from_dict(step.to_dict()).email == "linda.murphy@example.test" with pytest.raises(Exception): ao.UserSimTaskStep("u", None, name="linda", model="m", time_zone="Mars/Olympus") From 97f77c0e4a880f7a310dc38200d0621804b50c5b Mon Sep 17 00:00:00 2001 From: Sarah Chitty Date: Mon, 9 Nov 2026 00:39:43 -0800 Subject: [PATCH 08/53] always_on: drop test-rig wording from docstrings Co-Authored-By: Claude Opus 5.5 --- src/agent_env/task_step/task_steps/always_on.py | 2 +- src/agent_env/task_step/task_steps/user_sim_versions.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/agent_env/task_step/task_steps/always_on.py b/src/agent_env/task_step/task_steps/always_on.py index b6d2f2194..47e0c50a3 100644 --- a/src/agent_env/task_step/task_steps/always_on.py +++ b/src/agent_env/task_step/task_steps/always_on.py @@ -1012,7 +1012,7 @@ async def finish(self) -> None: # ================================================================================================================== async def _call(env, uri: str, method: str, params: Optional[dict] = None, *, attempts: int = 6, timeout: int = 60) -> Any: - """A gateway control call, retried on a transient failure (a busy laptop's gateway can take seconds to answer). + """A gateway control call, retried on a transient failure (a busy gateway can take seconds to answer). Every call the step retries is safe to repeat: reads, config, advance (checked against the clock after) and takes with a take_id.""" last: Optional[Exception] = None diff --git a/src/agent_env/task_step/task_steps/user_sim_versions.py b/src/agent_env/task_step/task_steps/user_sim_versions.py index c2af31067..f0f9fd77d 100644 --- a/src/agent_env/task_step/task_steps/user_sim_versions.py +++ b/src/agent_env/task_step/task_steps/user_sim_versions.py @@ -144,7 +144,7 @@ def _stamp(t: datetime, now: datetime) -> str: def render_today(read: dict) -> str: - """ML's setup: the assistant's raw message text, nothing else (several messages joined by a blank line).""" + """The assistant's raw message text, nothing else (several messages joined by a blank line).""" return "\n\n".join(m["text"].strip() for m in read["assistant_messages"]) @@ -193,7 +193,7 @@ def render_ours(version: str, person: dict, history: list[dict], read: dict) -> def _last_messaged(history: list[dict], read: dict) -> Optional[str]: """When the person last sent their assistant anything before this read: their scripted lines (this read's - included, as the h2h runner did) and their earlier replies.""" + included) and their earlier replies.""" times = [s["at"] for h in history for s in h["read"].get("you_sent") or [] if s.get("at")] times += [s["at"] for s in read.get("you_sent") or [] if s.get("at")] times += [h["read"]["time"] for h in history if (h.get("answer") or {}).get("outcome") in ("reply", "done")] From 1d5a2959c9b4af1d948e74451da1188ef16e59c9 Mon Sep 17 00:00:00 2001 From: Sarah Chitty Date: Mon, 9 Nov 2026 00:43:31 -0800 Subject: [PATCH 09/53] always_on: move the user_sim step into user_sim.py, with the interface always_on uses No behaviour change. UserSimTaskStep and USER_SIMS_KEY move, unchanged, from always_on.py to task_steps/user_sim.py, whose docstring is now the whole interface between the two steps: the settings always_on reads from metadata["user_sims"][name], the plug-in's configure / setup / answer, the person, read, history and ask shapes, and what answer returns. The registry imports the step from its new module; the user-sim-only tests move to test_user_sim.py. Co-Authored-By: Claude Opus 5.5 --- src/agent_env/task_step/registry.py | 3 +- .../task_step/task_steps/always_on.py | 74 +---------- .../task_step/task_steps/user_sim.py | 115 ++++++++++++++++++ tst/unit/task_step/always_on_harness.py | 3 +- tst/unit/task_step/test_always_on_loop.py | 38 +----- tst/unit/task_step/test_user_sim.py | 43 +++++++ 6 files changed, 168 insertions(+), 108 deletions(-) create mode 100644 src/agent_env/task_step/task_steps/user_sim.py create mode 100644 tst/unit/task_step/test_user_sim.py diff --git a/src/agent_env/task_step/registry.py b/src/agent_env/task_step/registry.py index 96916aef5..48d7971a5 100644 --- a/src/agent_env/task_step/registry.py +++ b/src/agent_env/task_step/registry.py @@ -54,7 +54,8 @@ def _builtin_registry() -> dict[str, type["TaskStep"]]: from agent_env.task_step.task_steps.load_artifact import LoadArtifactTaskStep from agent_env.task_step.task_steps.modify_env_tool_access import ModifyEnvToolAccessStep from agent_env.task_step.task_steps.prompt_agent import PromptAgentTaskStep - from agent_env.task_step.task_steps.always_on import AlwaysOnTaskStep, UserSimTaskStep + from agent_env.task_step.task_steps.always_on import AlwaysOnTaskStep + from agent_env.task_step.task_steps.user_sim import UserSimTaskStep from agent_env.task_step.task_steps.register_env_triggers import RegisterEnvTriggersStep from agent_env.task_step.task_steps.register_agent_triggers import RegisterAgentTriggersStep from agent_env.task_step.task_steps.sync_env_clock import SyncEnvClockTaskStep diff --git a/src/agent_env/task_step/task_steps/always_on.py b/src/agent_env/task_step/task_steps/always_on.py index 47e0c50a3..f094d94de 100644 --- a/src/agent_env/task_step/task_steps/always_on.py +++ b/src/agent_env/task_step/task_steps/always_on.py @@ -1,9 +1,9 @@ """The always_on and user_sim steps: a task that runs over simulated days, the way you'd hand a job to a personal assistant, with an LLM user sim. -The ``user_sim`` step says who the user is (description, time zone, schedule, response times, the version of the user -sim and its plug-in) and records it in the run's metadata under its ``name``; a ``deploy_agent`` step deploys the agent -that plays them. The ``always_on`` step names it with ``user_sim`` and runs the loop: +The ``user_sim`` step (``user_sim.py``, which documents the interface this step uses) says who the user is and records +it in the run's metadata under its ``name``; a ``deploy_agent`` step deploys the agent that plays them. The +``always_on`` step names it with ``user_sim`` and runs the loop: - The first wake is the user's request. While the task's clock is set, each wake opens with the task's date and time (any other date the agent sees is the computer's clock), then what came due for the agent on notifications/v1 (a @@ -51,7 +51,6 @@ from agent_env.a2a_agent import A2AAgent, conversation_store, protocol from agent_env.a2a_agent.object_transfer import send_and_wait from agent_env.artifact import FileArtifact -from agent_env.entity.refs import EntityRef from agent_env.env.env import EnvCapabilityUnsupported from agent_env.env.gateway.notifications import DEFAULT_MAX_ACTIVE from agent_env.env.gateway.constants import ( @@ -65,12 +64,12 @@ from agent_env.task_step.context import PromptResponse, TaskStepContext from agent_env.task_step.task_step import TaskStep from agent_env.task_step.task_steps import user_sim_versions +from agent_env.task_step.task_steps.user_sim import USER_SIMS_KEY from agent_env.task_step.task_steps.notification_delivery import delivered_parts, label_time, take_due from agent_env.task_step.task_steps.prompt_agent import PromptAgentTaskStep logger = logging.getLogger(__name__) -USER_SIMS_KEY = "user_sims" ALWAYS_ON_KEY = "always_on" USER_SAYS = "user_says" USER_PHONE = "user_phone" @@ -92,71 +91,6 @@ TIME_LINE = "It's {when}. Any other date you see is the computer's clock, not the task's." -# ================================================================================================================== -class UserSimTaskStep(TaskStep): - """Who the user is, once per task: their settings go into the run's metadata under ``name``.""" - - type: ClassVar[str] = "user_sim" - entity_refs = (EntityRef.artifact("plugin.file_artifact_id", version_field="file_artifact_version", - artifact_type="file"),) - - def __init__(self, id: str, version: Optional[int], *, name: str, model: str, agent_name: Optional[str] = None, - user_version: str = "ours", display_name: Optional[str] = None, first_name: Optional[str] = None, - description: Optional[str] = None, email: Optional[str] = None, time_zone: Optional[str] = None, - schedule: Optional[dict] = None, response_times: Optional[dict] = None, - max_replies: Optional[int] = None, permissions: Optional[dict] = None, - keep_private: Optional[list] = None, output_format: Optional[dict] = None, - versions: Optional[dict] = None, plugin: Optional[dict] = None, depends_on=None, - fail_task_on_error: bool = True, retry_config=None): - super().__init__(id, version, depends_on=depends_on, fail_task_on_error=fail_task_on_error, - retry_config=retry_config) - if not model: - raise ValueError("user_sim.model is required") - if time_zone is not None: - ZoneInfo(time_zone) - self.name = name - self.model = model - self.agent_name = agent_name or name - self.user_version = user_version - self.display_name = display_name - self.first_name = first_name - self.description = description - self.email = email - self.time_zone = time_zone - self.schedule = schedule - self.response_times = response_times - self.max_replies = max_replies - self.permissions = permissions - self.keep_private = keep_private - self.output_format = output_format - self.versions = versions or {} - self.plugin = plugin - - _FIELDS = ("name", "model", "agent_name", "user_version", "display_name", "first_name", "description", "email", - "time_zone", "schedule", "response_times", "max_replies", "permissions", "keep_private", "output_format", - "versions", "plugin") - - def to_dict(self) -> dict: - return {**super().to_dict(), **{k: getattr(self, k) for k in self._FIELDS}} - - @classmethod - def from_dict(cls, data: dict) -> UserSimTaskStep: - return cls(**cls._base_from_dict(data), retry_config=data.get("retry_config"), - **{k: data.get(k) for k in cls._FIELDS if k in data}) - - async def execute(self, context: TaskStepContext) -> TaskStepContext: - overrides = self.step_param_overrides(context) - version = overrides.get("user_version") or self.user_version - if not any(a.agent_name == self.agent_name for a in context.deployed_agents): - raise RuntimeError(f"user_sim '{self.name}': no deployed agent named '{self.agent_name}'; deploy it first") - settings = {k: getattr(self, k) for k in self._FIELDS} - settings.update(user_version=version, step_id=self.id, - model=overrides.get("model") or self.model) - context.metadata.setdefault(USER_SIMS_KEY, {})[self.name] = settings - logger.info(f"user_sim '{self.name}': version {version}, model {settings['model']}, agent {self.agent_name}") - return context - - # ================================================================================================================== class AlwaysOnTaskStep(PromptAgentTaskStep): type: ClassVar[str] = "always_on" diff --git a/src/agent_env/task_step/task_steps/user_sim.py b/src/agent_env/task_step/task_steps/user_sim.py new file mode 100644 index 000000000..14e2b282e --- /dev/null +++ b/src/agent_env/task_step/task_steps/user_sim.py @@ -0,0 +1,115 @@ +"""The user_sim step: who the user is, once per task. Its settings go into ``metadata["user_sims"][name]``, where the +always_on step reads them. This docstring is the whole interface between the two. + +What always_on reads from the settings (``metadata["user_sims"][name]``): + +- ``name``; ``display_name`` (else ``name``) and ``first_name`` (else the display name's first word): how the agent's + wakes and the plug-in name the user. +- ``agent_name``: the deployed agent that plays the user (``ask`` below sends to it); ``model``: set on that agent. +- ``user_version``: the version handed to the plug-in; ``plugin``: ``{"file_artifact_id", "file_artifact_version", + "params"}`` naming a plug-in file, else the built-in ``user_sim_versions``. +- ``time_zone`` (IANA): the user's zone; ``schedule`` (``wake``, ``sleep``, ``work``, ``work_days``, ``busy``): when + the user reads; ``response_times`` (minutes by ``morning`` / ``work`` / ``free``): how long a reply is held; + ``max_replies``: after that many replies the user stops reading. +- ``email``, ``description``, ``versions``, ``output_format``, ``permissions``, ``keep_private``: passed to the + plug-in in ``person`` only. + +A plug-in is one Python module with: + + configure(params: dict) -> None optional, called once with plugin.params + setup(version: str, person: dict) -> {"system_prompt": str | None, "output_format": dict | None} + async answer(version: str, person: dict, history: list[dict], read: dict, ask) -> dict + +``setup`` runs once per run; the step sets ``model``, ``system_prompt`` and ``output_format`` on the user sim agent. +``person`` is ``{"key": name, "name", "first", "email", "time_zone", "description", "versions", "output_format", +"permissions", "keep_private"}``, with ``time_zone`` the run's zone (the user's, else the world clock's, else UTC). + +``answer`` runs once per read, at the time the user would read. ``read`` is ``{"time": RFC 3339 UTC, +"assistant_messages": [{"at", "text"}], "phone": [{"at", "channel", "text"}], "you_sent": [{"at", "text"}]}``: the +agent's message_user texts since the last read, what triggers sent ``user_phone``, and the user's own lines (the +opening request, ``user_says`` lines, ``final``), never anything meant for the agent. ``history`` is every earlier +``{"read", "answer"}``, oldest first. ``ask(text)`` sends one turn to the user sim agent (one session per run) and +returns ``{"text", "structured", "trajectory_url"}``, or ``{"text": None, "structured": None, "error"}``. + +``answer`` returns ``{"outcome": "reply" | "no_reply" | "done", "message": str, "used": str | None}``, and may add +``input``, ``raw``, ``score``, ``schema_errors``, ``turns``, ``shown`` and ``past_turn_cap``, which the step records. +A reply with text is held for the response time, then wakes the agent; ``done`` is never sent to the agent and ends +only a one-sitting run. When the last ``ask`` of a read returned an error, the run ends (``user_sim_error``). +""" +from __future__ import annotations + +import logging +from typing import ClassVar, Optional +from zoneinfo import ZoneInfo + +from agent_env.entity.refs import EntityRef +from agent_env.task_step.context import TaskStepContext +from agent_env.task_step.task_step import TaskStep + +logger = logging.getLogger(__name__) + +USER_SIMS_KEY = "user_sims" + + +class UserSimTaskStep(TaskStep): + """Who the user is, once per task: their settings go into the run's metadata under ``name``.""" + + type: ClassVar[str] = "user_sim" + entity_refs = (EntityRef.artifact("plugin.file_artifact_id", version_field="file_artifact_version", + artifact_type="file"),) + + def __init__(self, id: str, version: Optional[int], *, name: str, model: str, agent_name: Optional[str] = None, + user_version: str = "ours", display_name: Optional[str] = None, first_name: Optional[str] = None, + description: Optional[str] = None, email: Optional[str] = None, time_zone: Optional[str] = None, + schedule: Optional[dict] = None, response_times: Optional[dict] = None, + max_replies: Optional[int] = None, permissions: Optional[dict] = None, + keep_private: Optional[list] = None, output_format: Optional[dict] = None, + versions: Optional[dict] = None, plugin: Optional[dict] = None, depends_on=None, + fail_task_on_error: bool = True, retry_config=None): + super().__init__(id, version, depends_on=depends_on, fail_task_on_error=fail_task_on_error, + retry_config=retry_config) + if not model: + raise ValueError("user_sim.model is required") + if time_zone is not None: + ZoneInfo(time_zone) + self.name = name + self.model = model + self.agent_name = agent_name or name + self.user_version = user_version + self.display_name = display_name + self.first_name = first_name + self.description = description + self.email = email + self.time_zone = time_zone + self.schedule = schedule + self.response_times = response_times + self.max_replies = max_replies + self.permissions = permissions + self.keep_private = keep_private + self.output_format = output_format + self.versions = versions or {} + self.plugin = plugin + + _FIELDS = ("name", "model", "agent_name", "user_version", "display_name", "first_name", "description", "email", + "time_zone", "schedule", "response_times", "max_replies", "permissions", "keep_private", "output_format", + "versions", "plugin") + + def to_dict(self) -> dict: + return {**super().to_dict(), **{k: getattr(self, k) for k in self._FIELDS}} + + @classmethod + def from_dict(cls, data: dict) -> UserSimTaskStep: + return cls(**cls._base_from_dict(data), retry_config=data.get("retry_config"), + **{k: data.get(k) for k in cls._FIELDS if k in data}) + + async def execute(self, context: TaskStepContext) -> TaskStepContext: + overrides = self.step_param_overrides(context) + version = overrides.get("user_version") or self.user_version + if not any(a.agent_name == self.agent_name for a in context.deployed_agents): + raise RuntimeError(f"user_sim '{self.name}': no deployed agent named '{self.agent_name}'; deploy it first") + settings = {k: getattr(self, k) for k in self._FIELDS} + settings.update(user_version=version, step_id=self.id, + model=overrides.get("model") or self.model) + context.metadata.setdefault(USER_SIMS_KEY, {})[self.name] = settings + logger.info(f"user_sim '{self.name}': version {version}, model {settings['model']}, agent {self.agent_name}") + return context diff --git a/tst/unit/task_step/always_on_harness.py b/tst/unit/task_step/always_on_harness.py index 48616ec7f..7fcb236ec 100644 --- a/tst/unit/task_step/always_on_harness.py +++ b/tst/unit/task_step/always_on_harness.py @@ -14,6 +14,7 @@ from agent_env.env.gateway.gateway import Gateway from agent_env.task_step.context import DeployedAgent, TaskStepContext from agent_env.task_step.task_steps import always_on as ao +from agent_env.task_step.task_steps.user_sim import USER_SIMS_KEY GATEWAY = "http://gateway.test" SOLVER = "http://solver.test" @@ -95,7 +96,7 @@ def context(self, *, user: Optional[dict] = None) -> TaskStepContext: ctx = TaskStepContext(instance_id="ti-1", deployed_envs=[env]) ctx.deployed_agents += [DeployedAgent(agent_name="solver", api_url=SOLVER, a2a_url=SOLVER, a2a_card={}), DeployedAgent(agent_name="user-sim", api_url=USER_SIM, a2a_url=USER_SIM, a2a_card={})] - ctx.metadata[ao.USER_SIMS_KEY] = {"linda": { + ctx.metadata[USER_SIMS_KEY] = {"linda": { "name": "linda", "model": "sim-model", "agent_name": "user-sim", "user_version": "ours", "display_name": "Linda Murphy", "first_name": "Linda", "time_zone": "America/New_York", "description": "Linda, planning Noor's 30th.", **(user or {})}} diff --git a/tst/unit/task_step/test_always_on_loop.py b/tst/unit/task_step/test_always_on_loop.py index cb76a8a15..f68bfee17 100644 --- a/tst/unit/task_step/test_always_on_loop.py +++ b/tst/unit/task_step/test_always_on_loop.py @@ -7,7 +7,7 @@ from agent_env.task_step.registry import get_task_step_registry from agent_env.task_step.task_steps import always_on as ao -from agent_env.task_step.task_steps import user_sim_versions +from agent_env.task_step.task_steps.user_sim import UserSimTaskStep from tst.unit.task_step.always_on_harness import Rig pytestmark = pytest.mark.usefixtures("local_stores") @@ -280,38 +280,4 @@ async def test_a_user_and_a_world_in_different_time_zones_are_refused(monkeypatc def test_both_steps_are_registered(): registry = get_task_step_registry() - assert registry["always_on"] is ao.AlwaysOnTaskStep and registry["user_sim"] is ao.UserSimTaskStep - - -def test_the_user_sim_step_defaults_to_ours_and_has_no_apps_setting(): - step = ao.UserSimTaskStep.from_dict({"id": "u", "type": "user_sim", "name": "linda", "model": "m", - "apps": ["Gmail"], "time_zone": "America/New_York", - "email": "linda.murphy@example.test"}) - assert step.user_version == "ours" and "apps" not in step.to_dict() - assert ao.UserSimTaskStep.from_dict(step.to_dict()).email == "linda.murphy@example.test" - with pytest.raises(Exception): - ao.UserSimTaskStep("u", None, name="linda", model="m", time_zone="Mars/Olympus") - - -def test_ours_works_from_the_description_alone(): - person = {"name": "Linda Murphy", "first": "Linda", "description": "You're 31, planning Noor's 30th."} - setup = user_sim_versions.setup("ours", person) - assert setup["system_prompt"].endswith("WHO YOU ARE\nYou're 31, planning Noor's 30th.\n") - assert "Any other date you see is the computer's clock" in setup["system_prompt"] - assert setup["output_format"]["schema"]["properties"]["outcome"]["enum"] == ["reply", "no_reply", "done"] - own = user_sim_versions.setup("ours", {**person, "versions": {"ours": {"system_prompt": "tagged lines"}}}) - assert own["system_prompt"] == "tagged lines" - - -def test_ours_renders_one_labelled_read(): - person = {"name": "Linda Murphy", "first": "Linda", "time_zone": "America/New_York"} - read = {"time": "2026-11-10T15:00:00Z", - "assistant_messages": [{"at": "2026-11-10T13:00:00Z", "text": "house?"}, - {"at": "2026-11-10T14:00:00Z", "text": "and dinner?"}], - "phone": [{"at": "2026-11-10T12:00:00Z", "channel": "Gmail", "text": "New email from Sam"}], - "you_sent": [{"at": "2026-11-09T14:00:00Z", "text": "sort the cdmx weekend"}]} - text = user_sim_versions.render_ours("ours", person, [], read) - assert text.startswith("It's Tue, Nov 10, 10:00 AM.") - for label in ("NEW ON YOUR PHONE", "YOU SENT YOUR AI ASSISTANT", "NEW FROM YOUR AI ASSISTANT"): - assert label in text - assert text.index("house?") < text.index("and dinner?") + assert registry["always_on"] is ao.AlwaysOnTaskStep and registry["user_sim"] is UserSimTaskStep diff --git a/tst/unit/task_step/test_user_sim.py b/tst/unit/task_step/test_user_sim.py new file mode 100644 index 000000000..75d3699ec --- /dev/null +++ b/tst/unit/task_step/test_user_sim.py @@ -0,0 +1,43 @@ +"""The user_sim step and the built-in user sim plug-in: the settings the step records, and the ``ours`` version's +prompt and read.""" + +from __future__ import annotations + +import pytest + +from agent_env.task_step.task_steps import user_sim_versions +from agent_env.task_step.task_steps.user_sim import UserSimTaskStep + + +def test_the_user_sim_step_defaults_to_ours_and_has_no_apps_setting(): + step = UserSimTaskStep.from_dict({"id": "u", "type": "user_sim", "name": "linda", "model": "m", + "apps": ["Gmail"], "time_zone": "America/New_York", + "email": "linda.murphy@example.test"}) + assert step.user_version == "ours" and "apps" not in step.to_dict() + assert UserSimTaskStep.from_dict(step.to_dict()).email == "linda.murphy@example.test" + with pytest.raises(Exception): + UserSimTaskStep("u", None, name="linda", model="m", time_zone="Mars/Olympus") + + +def test_ours_works_from_the_description_alone(): + person = {"name": "Linda Murphy", "first": "Linda", "description": "You're 31, planning Noor's 30th."} + setup = user_sim_versions.setup("ours", person) + assert setup["system_prompt"].endswith("WHO YOU ARE\nYou're 31, planning Noor's 30th.\n") + assert "Any other date you see is the computer's clock" in setup["system_prompt"] + assert setup["output_format"]["schema"]["properties"]["outcome"]["enum"] == ["reply", "no_reply", "done"] + own = user_sim_versions.setup("ours", {**person, "versions": {"ours": {"system_prompt": "tagged lines"}}}) + assert own["system_prompt"] == "tagged lines" + + +def test_ours_renders_one_labelled_read(): + person = {"name": "Linda Murphy", "first": "Linda", "time_zone": "America/New_York"} + read = {"time": "2026-11-10T15:00:00Z", + "assistant_messages": [{"at": "2026-11-10T13:00:00Z", "text": "house?"}, + {"at": "2026-11-10T14:00:00Z", "text": "and dinner?"}], + "phone": [{"at": "2026-11-10T12:00:00Z", "channel": "Gmail", "text": "New email from Sam"}], + "you_sent": [{"at": "2026-11-09T14:00:00Z", "text": "sort the cdmx weekend"}]} + text = user_sim_versions.render_ours("ours", person, [], read) + assert text.startswith("It's Tue, Nov 10, 10:00 AM.") + for label in ("NEW ON YOUR PHONE", "YOU SENT YOUR AI ASSISTANT", "NEW FROM YOUR AI ASSISTANT"): + assert label in text + assert text.index("house?") < text.index("and dinner?") From f5fca2f8ed631cff4f81f0709b5a0c2a5834639d Mon Sep 17 00:00:00 2001 From: Sarah Chitty Date: Mon, 9 Nov 2026 00:46:20 -0800 Subject: [PATCH 10/53] always_on: each read says whether done would end the run The user sim plug-in gets read["done_ends_run"]: true in a one-sitting run, false in a multi-day one, where done is only recorded. Documented with the read in user_sim.py. Co-Authored-By: Claude Opus 5.5 --- src/agent_env/task_step/task_steps/always_on.py | 2 +- src/agent_env/task_step/task_steps/user_sim.py | 7 ++++--- tst/unit/task_step/test_always_on_loop.py | 11 +++++++++++ 3 files changed, 16 insertions(+), 4 deletions(-) diff --git a/src/agent_env/task_step/task_steps/always_on.py b/src/agent_env/task_step/task_steps/always_on.py index f094d94de..fe903d03c 100644 --- a/src/agent_env/task_step/task_steps/always_on.py +++ b/src/agent_env/task_step/task_steps/always_on.py @@ -807,7 +807,7 @@ async def read(self, now: datetime) -> None: phone.append({"at": it.get("at"), "channel": it["channel"], "text": "\n\n".join(p["text"] for p in it["parts"] if p.get("kind") == "text")}) read = {"time": _iso(now), "assistant_messages": list(self.unread), "phone": phone, - "you_sent": list(self.you_sent)} + "you_sent": list(self.you_sent), "done_ends_run": not self.multi_day} self.unread, self.you_sent, self.read_at = [], [], None calls = [] diff --git a/src/agent_env/task_step/task_steps/user_sim.py b/src/agent_env/task_step/task_steps/user_sim.py index 14e2b282e..9c56e05b5 100644 --- a/src/agent_env/task_step/task_steps/user_sim.py +++ b/src/agent_env/task_step/task_steps/user_sim.py @@ -25,9 +25,10 @@ "permissions", "keep_private"}``, with ``time_zone`` the run's zone (the user's, else the world clock's, else UTC). ``answer`` runs once per read, at the time the user would read. ``read`` is ``{"time": RFC 3339 UTC, -"assistant_messages": [{"at", "text"}], "phone": [{"at", "channel", "text"}], "you_sent": [{"at", "text"}]}``: the -agent's message_user texts since the last read, what triggers sent ``user_phone``, and the user's own lines (the -opening request, ``user_says`` lines, ``final``), never anything meant for the agent. ``history`` is every earlier +"assistant_messages": [{"at", "text"}], "phone": [{"at", "channel", "text"}], "you_sent": [{"at", "text"}], +"done_ends_run": bool}``: the agent's message_user texts since the last read, what triggers sent ``user_phone``, the +user's own lines (the opening request, ``user_says`` lines, ``final``), never anything meant for the agent, and +whether done ends the run (one-sitting) or is only recorded (multi-day). ``history`` is every earlier ``{"read", "answer"}``, oldest first. ``ask(text)`` sends one turn to the user sim agent (one session per run) and returns ``{"text", "structured", "trajectory_url"}``, or ``{"text": None, "structured": None, "error"}``. diff --git a/tst/unit/task_step/test_always_on_loop.py b/tst/unit/task_step/test_always_on_loop.py index f68bfee17..755e977b5 100644 --- a/tst/unit/task_step/test_always_on_loop.py +++ b/tst/unit/task_step/test_always_on_loop.py @@ -242,6 +242,17 @@ async def test_done_in_a_multi_day_run_is_recorded_and_the_run_goes_on_to_end_ti assert "perfect ty" not in repr(rig.sent) # a done never reaches the agent +@pytest.mark.parametrize("end_time,ends", [(None, True), ("P3D", False)], ids=["one sitting", "multi-day"]) +@pytest.mark.asyncio +async def test_each_read_says_whether_done_would_end_the_run(monkeypatch, tmp_path, end_time, ends): + rig = Rig(monkeypatch, tmp_path, agent=_agent(t1=_message("house booked"))) + await rig.arm() + await rig.run(rig.step(**({"end_time": end_time} if end_time else {}))) + + [read] = rig.reads + assert read["done_ends_run"] is ends + + @pytest.mark.parametrize("end_time,multi_day", [(None, False), ("PT6H", False), ("P1D", True)]) @pytest.mark.asyncio async def test_a_run_is_multi_day_when_its_end_falls_on_a_later_day_for_the_user(monkeypatch, tmp_path, end_time, From f8bdf56868df88399a121ad064e9b52146c8c24d Mon Sep 17 00:00:00 2001 From: Sarah Chitty Date: Mon, 9 Nov 2026 00:58:27 -0800 Subject: [PATCH 11/53] always_on: her done reaches the agent; the opening request is in person MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - A done with text is held for her reply time and delivered to the agent like a reply (her prompt says her message reaches the assistant either way). In a one-sitting run the run ends (user_done) after the agent's turn on it, with no further reads; a blank done still ends it at once. A multi-day run records the done and goes on, as before. - person["opening"] = {"at", "text"}: the opening request and its time, so a plug-in's setup can put it in her system prompt. It stays the first line of her first read's you_sent. - user_sim.py's interface says so, and that a phone item's sender rides in its channel ("Gmail · Carol Reyes (Mom)"), as in the agent's alerts. Co-Authored-By: Claude Opus 5.5 --- .../task_step/task_steps/always_on.py | 35 ++++++++++++------- .../task_step/task_steps/user_sim.py | 18 ++++++---- tst/unit/task_step/test_always_on_loop.py | 20 ++++++++--- 3 files changed, 51 insertions(+), 22 deletions(-) diff --git a/src/agent_env/task_step/task_steps/always_on.py b/src/agent_env/task_step/task_steps/always_on.py index fe903d03c..2b13917a0 100644 --- a/src/agent_env/task_step/task_steps/always_on.py +++ b/src/agent_env/task_step/task_steps/always_on.py @@ -14,8 +14,9 @@ - The user sim is asked only after the agent sent something, at the time the user would read it (``schedule``), with everything sent meanwhile in one read: the time, the agent's messages, what triggers sent the user's phone (``notify`` to ``user_phone``) and the user's own lines (the opening request, ``user_says`` lines, ``final``). - Nothing meant for the agent is in it. Its answer is reply / no_reply / done, and a reply is held for the response - time. Done ends a one-sitting run; a run whose end_time falls on a later day than its start goes on to end_time. + Nothing meant for the agent is in it. Its answer is reply / no_reply / done; a reply, or a done with text, is held + for the response time. Done ends a one-sitting run (after the agent's turn on its text); a run whose end_time falls + on a later day than its start goes on to end_time. - Scripted user lines are triggers' ``notify`` actions to the ``user_says`` recipient: sent to the agent as the user's message at their time. - The clock runs at the task's speed during turns (sync_env_clock). When nothing is due, the step advances it to the @@ -196,6 +197,7 @@ def __init__(self, step: AlwaysOnTaskStep, context: TaskStepContext): self.zone = "UTC" self.multi_day = False self.checks: dict = {"on": False} + self.signed_off = False # a one-sitting run's user said done: no more reads def close(self) -> None: for p in self._plugin_paths: @@ -243,6 +245,8 @@ async def go(self) -> None: raise RuntimeError(f"always_on: no deployed env offers {EXT_USER_CHAT_URI} (message_user); rebuild its " "gateway from a branch that has user_chat.py") now = await self.now() + self.opening = {"at": _iso(now), "text": self._effective_prompt()} + self.person["opening"] = self.opening t0 = await self._t0() or now self.end = _end_instant(step.end_time, t0) if step.end_time else None self.multi_day = self.end is not None and self.end.astimezone(self.tz).date() > t0.astimezone(self.tz).date() @@ -520,7 +524,7 @@ def _latency(self, mode: str) -> timedelta: return timedelta(minutes=float(rt.get(mode, rt.get("free", 0)))) def _schedule_read(self) -> None: - if not self.unread or self.read_at is not None or self.gave_up: + if not self.unread or self.read_at is not None or self.gave_up or self.signed_off: return first_at = min(_utc(m["at"]) for m in self.unread if m.get("at")) if any(m.get("at") for m in self.unread) \ else datetime.now(timezone.utc) @@ -533,10 +537,10 @@ def _schedule_read(self) -> None: # ---- the loop ------------------------------------------------------------------------------------------- async def loop(self, now: datetime) -> None: step = self.step - first = self._effective_prompt() - self.you_sent.append({"at": _iso(now), "text": first}) + self.you_sent.append(dict(self.opening)) due = await take_due(self.notification_envs, self.recipient) if self.notification_envs else [] - await self.wake([*delivered_parts(due), self._user_part(now, first, opening=True)], + await self.wake([*delivered_parts(due), self._user_part(_utc(self.opening["at"]), self.opening["text"], + opening=True)], woke_by=[f"notification:{d['channel']}" for d in due] + ["user_request"], notifications=[{k: d.get(k) for k in ("at", "channel", "source", "source_id", "to")} for d in due]) @@ -624,7 +628,8 @@ async def _wake_with(self, now: datetime, lines: list[dict], replies: list[dict] parts = delivered_parts(due) user_msgs = sorted([{"at": l["at"] or _iso(now), "text": l["text"], "kind": l.get("source", "user_says"), "trigger_id": l.get("trigger_id")} for l in lines] - + [{"at": _iso(r["deliver_at"]), "text": r["text"], "kind": "user_reply"} for r in replies], + + [{"at": _iso(r["deliver_at"]), "text": r["text"], "kind": "user_reply", + "done": r.get("done", False)} for r in replies], key=lambda m: _utc(m["at"])) for m in user_msgs: parts.append(self._user_part(_utc(m["at"]), m["text"])) @@ -633,13 +638,15 @@ async def _wake_with(self, now: datetime, lines: list[dict], replies: list[dict] self.log("scripted_user_line", virtual_time=m["at"], text=m["text"], kind=m["kind"], trigger_id=m.get("trigger_id")) else: - self.log("user_reply_delivered", virtual_time=m["at"], text=m["text"]) + self.log("user_reply_delivered", virtual_time=m["at"], text=m["text"], done=m["done"]) if final and self.step.final_reminder: parts.append(_step_part(self.step.final_reminder.format(first=self.person["first"]), "final_reminder")) woke_by = [f"notification:{d['channel']}" for d in due] + [f"{m['kind']}" for m in user_msgs] await self.wake(parts, woke_by=woke_by, notifications=[{k: d.get(k) for k in ("at", "channel", "source", "source_id", "to")} for d in due]) + if self.signed_off and any(m.get("done") for m in user_msgs): + self.end_reason = self.end_reason or "user_done" async def wake(self, parts: list[dict], woke_by: list[str], notifications: Optional[list] = None) -> None: step = self.step @@ -832,19 +839,23 @@ async def ask(text: str) -> dict: user_sim_turns=ans.get("turns", len(calls)), shown=ans.get("shown"), past_turn_cap=ans.get("past_turn_cap")) outcome = ans.get("outcome") - if outcome == "reply" and (ans.get("message") or "").strip(): + said = (ans.get("message") or "").strip() + if outcome in ("reply", "done") and said: deliver = now + latency - self.held.append({"deliver_at": deliver, "text": ans["message"]}) + self.held.append({"deliver_at": deliver, "text": ans["message"], "done": outcome == "done"}) self.last_reply_at = deliver - self.replies += 1 entry["deliver_at"] = _iso(deliver) + if outcome == "reply" and said: + self.replies += 1 max_replies = self.us.get("max_replies") if max_replies and self.replies >= int(max_replies): self.gave_up = True self.log("user_gave_up", replies=self.replies) elif outcome == "done": entry["ends_run"] = not self.multi_day - if not self.multi_day: + if not self.multi_day and said: + self.signed_off = True + elif not self.multi_day: self.end_reason = "user_done" if calls and calls[-1].get("error"): self.end_reason = self.end_reason or "user_sim_error" diff --git a/src/agent_env/task_step/task_steps/user_sim.py b/src/agent_env/task_step/task_steps/user_sim.py index 9c56e05b5..11140e614 100644 --- a/src/agent_env/task_step/task_steps/user_sim.py +++ b/src/agent_env/task_step/task_steps/user_sim.py @@ -22,20 +22,26 @@ ``setup`` runs once per run; the step sets ``model``, ``system_prompt`` and ``output_format`` on the user sim agent. ``person`` is ``{"key": name, "name", "first", "email", "time_zone", "description", "versions", "output_format", -"permissions", "keep_private"}``, with ``time_zone`` the run's zone (the user's, else the world clock's, else UTC). +"permissions", "keep_private", "opening"}``, with ``time_zone`` the run's zone (the user's, else the world clock's, +else UTC) and ``opening`` the user's opening request, ``{"at": RFC 3339 UTC, "text"}``, so ``setup`` can write it +into the system prompt. The opening is also the first line of the first read's ``you_sent``. ``answer`` runs once per read, at the time the user would read. ``read`` is ``{"time": RFC 3339 UTC, "assistant_messages": [{"at", "text"}], "phone": [{"at", "channel", "text"}], "you_sent": [{"at", "text"}], -"done_ends_run": bool}``: the agent's message_user texts since the last read, what triggers sent ``user_phone``, the -user's own lines (the opening request, ``user_says`` lines, ``final``), never anything meant for the agent, and -whether done ends the run (one-sitting) or is only recorded (multi-day). ``history`` is every earlier +"done_ends_run": bool}``: the agent's message_user texts since the last read, oldest first; what triggers sent +``user_phone`` since the last read, its sender in ``channel`` (``"Gmail · Carol Reyes (Mom)"``) as in the agent's +alerts; the user's own lines (the opening request, ``user_says`` lines, ``final``) each at the time it was sent; never +anything meant for the agent; and whether done ends the run (one-sitting) or is only recorded (multi-day). The user +sim reads only after the agent has used message_user. ``history`` is every earlier ``{"read", "answer"}``, oldest first. ``ask(text)`` sends one turn to the user sim agent (one session per run) and returns ``{"text", "structured", "trajectory_url"}``, or ``{"text": None, "structured": None, "error"}``. ``answer`` returns ``{"outcome": "reply" | "no_reply" | "done", "message": str, "used": str | None}``, and may add ``input``, ``raw``, ``score``, ``schema_errors``, ``turns``, ``shown`` and ``past_turn_cap``, which the step records. -A reply with text is held for the response time, then wakes the agent; ``done`` is never sent to the agent and ends -only a one-sitting run. When the last ``ask`` of a read returned an error, the run ends (``user_sim_error``). +A ``reply`` or ``done`` with text is held for the response time, then wakes the agent as the user's message; the +recorded extras (``used``, ``score``, ...) never reach the agent. ``done`` ends a one-sitting run after the agent's +turn on that message, or at once when it has no text, with no further reads; in a multi-day run it is only recorded. +When the last ``ask`` of a read returned an error, the run ends (``user_sim_error``). """ from __future__ import annotations diff --git a/tst/unit/task_step/test_always_on_loop.py b/tst/unit/task_step/test_always_on_loop.py index 755e977b5..ce81eee0e 100644 --- a/tst/unit/task_step/test_always_on_loop.py +++ b/tst/unit/task_step/test_always_on_loop.py @@ -214,15 +214,27 @@ def agent(n, texts, rig): @pytest.mark.asyncio -async def test_done_ends_a_one_sitting_run(monkeypatch, tmp_path): - rig = Rig(monkeypatch, tmp_path, agent=_agent(t1=_message("all set: shelves restocked")), +async def test_done_ends_a_one_sitting_run_after_the_agent_s_turn_on_her_sign_off(monkeypatch, tmp_path): + rig = Rig(monkeypatch, tmp_path, agent=_agent(t1=_message("all set: shelves restocked"), + t2=_message("glad to help!")), answers=[{"outcome": "done", "message": "thanks!"}]) await rig.arm() rig.triggers(rig.notify_at("later", "2026-11-10T14:00:00Z", "a new order", channel="Craftweb")) ctx = await rig.run(rig.step()) + assert rig.record(ctx)["end_reason"] == "user_done" and len(rig.sent) == 2 + assert "[Mon Nov 9, 2:00 PM UTC] Linda: thanks!" in rig.sent[1] + assert len(rig.reads) == 1 and rig.events(ctx, "user_sim_read")[0]["ends_run"] is True + + +@pytest.mark.asyncio +async def test_a_blank_done_ends_a_one_sitting_run_at_once(monkeypatch, tmp_path): + rig = Rig(monkeypatch, tmp_path, agent=_agent(t1=_message("all set")), + answers=[{"outcome": "done", "message": " "}]) + await rig.arm() + ctx = await rig.run(rig.step()) + assert rig.record(ctx)["end_reason"] == "user_done" and len(rig.sent) == 1 - assert rig.events(ctx, "user_sim_read")[0]["ends_run"] is True @pytest.mark.asyncio @@ -239,7 +251,7 @@ async def test_done_in_a_multi_day_run_is_recorded_and_the_run_goes_on_to_end_ti assert rig.events(ctx, "user_sim_read")[0]["ends_run"] is False assert [m["text"] for m in rig.reads[1]["assistant_messages"]] == ["airline moved Dev"] assert record["counts"]["user_dones"] == 1 - assert "perfect ty" not in repr(rig.sent) # a done never reaches the agent + assert any("Linda: perfect ty" in t for texts in rig.sent for t in texts) @pytest.mark.parametrize("end_time,ends", [(None, True), ("P3D", False)], ids=["one sitting", "multi-day"]) From 12e03f5d736c79cf5e603c1896c61683ff9d8519 Mon Sep 17 00:00:00 2001 From: Sarah Chitty Date: Mon, 9 Nov 2026 00:58:29 -0800 Subject: [PATCH 12/53] always_on: tests for everything the loop hands the user sim One test per item the loop owns on the user sim's checklist: she reads only after message_user (A1); her phone shows only new user_phone items with their time and full text (A4/A5); you_sent holds only her own lines at their times (A8/B3); every message since her last read comes in one read, oldest first (A9); the opening is in person and her first you_sent (A13); one user sim session per run (A14); history is never trimmed (A15); nothing meant for the agent reaches her at any depth (B1/B2/B4); ask sends exactly its text (B8); reply and done arrive after her reply time (C2/C4); used and score are recorded, never sent (C5); and done_ends_run. The harness can now take a plug-in and play the user sim agent. Co-Authored-By: Claude Opus 5.5 --- tst/unit/task_step/always_on_harness.py | 15 +- .../test_always_on_user_sim_inputs.py | 252 ++++++++++++++++++ 2 files changed, 265 insertions(+), 2 deletions(-) create mode 100644 tst/unit/task_step/test_always_on_user_sim_inputs.py diff --git a/tst/unit/task_step/always_on_harness.py b/tst/unit/task_step/always_on_harness.py index 7fcb236ec..be7a2a229 100644 --- a/tst/unit/task_step/always_on_harness.py +++ b/tst/unit/task_step/always_on_harness.py @@ -32,10 +32,11 @@ class Rig: ``message``, ``quiet`` and ``schedule``, and returns its closing text; by default it stays quiet. ``answers`` are the user sim's answers to its reads, in order; once they run out it answers no_reply. Every message the agent got is in ``sent``, every read the - user sim got in ``reads``.""" + user sim got in ``reads``. A ``plugin`` replaces the scripted one; what it sends the user sim agent with ``ask`` is + in ``user_calls``, answered in turn by ``user_texts`` (then no_reply).""" def __init__(self, monkeypatch, tmp_path, *, agent: Optional[Turn] = None, answers: tuple = (), rate: float = 0, - t0: str = T0): + t0: str = T0, plugin: Any = None, user_texts: tuple = ()): monkeypatch.setattr(gw_mod, "GATEWAY_TRAJECTORY_FILE", str(tmp_path / "trajectory.jsonl")) self.gw = Gateway(host="127.0.0.1", port=0, server_name="AgentEnvGateway", internal_mcp_servers=[]) self.gw._tools_discovered = True @@ -50,6 +51,9 @@ def __init__(self, monkeypatch, tmp_path, *, agent: Optional[Turn] = None, answe self.parts: list[list[dict]] = [] self.reads: list[dict] = [] self.setups: list[dict] = [] + self.plugin = plugin + self.user_texts = list(user_texts) + self.user_calls: list[dict] = [] monkeypatch.setattr(ao, "POLL_SECONDS", 0) monkeypatch.setattr(ao, "send_and_wait", self._send_and_wait) monkeypatch.setattr(ao._Run, "_load_plugin", self._load_plugin) @@ -119,6 +123,11 @@ def events(self, ctx: TaskStepContext, name: str) -> list[dict]: # ---- fakes ----------------------------------------------------------------------------------------------------- async def _send_and_wait(self, url, parts, *, before_send=None, **kw): + if url == USER_SIM: + self.user_calls.append({"parts": list(parts), "context_id": kw.get("context_id")}) + text = self.user_texts.pop(0) if self.user_texts else '{"outcome": "no_reply", "message": ""}' + return f"user-{len(self.user_calls)}", {"status": {"state": "completed", "message": { + "parts": [{"kind": "text", "text": text}]}}} assert url == SOLVER, url if before_send: before_send() @@ -130,6 +139,8 @@ async def _send_and_wait(self, url, parts, *, before_send=None, **kw): return f"task-{len(self.sent)}", {"status": {"state": "completed", "message": {"parts": reply}}} async def _load_plugin(self, spec): + if self.plugin is not None: + return self.plugin rig = self class Plugin: diff --git a/tst/unit/task_step/test_always_on_user_sim_inputs.py b/tst/unit/task_step/test_always_on_user_sim_inputs.py new file mode 100644 index 000000000..17ab13c1e --- /dev/null +++ b/tst/unit/task_step/test_always_on_user_sim_inputs.py @@ -0,0 +1,252 @@ +"""What the always_on loop hands the user sim, and what it does with her answers: one test per item the loop owns on +the user sim's checklist of what she sees and must never see (the ids in each test name).""" + +from __future__ import annotations + +import json + +import pytest + +from agent_env.task_step.task_steps import always_on as ao +from tst.unit.task_step.always_on_harness import Rig + +pytestmark = pytest.mark.usefixtures("local_stores") + +DAY = {"schedule": {"wake": "07:30", "sleep": "23:30", "work": ["09:00", "18:00"]}, + "response_times": {"work": 150, "free": 30, "morning": 15}} +MIDNIGHT = "2026-11-10T05:00:00Z" # Tue Nov 10, 12:00 AM in New York + + +def _agent(**plays): + """An agent that plays ``plays["t"]`` on its nth turn and otherwise stays quiet.""" + def agent(n, texts, rig): + play = plays.get(f"t{n}") + if play is None: + rig.quiet() + return "" + return play(rig) + return agent + + +def _message(*texts): + def play(rig): + for text in texts: + rig.message(text) + return "sent" + return play + + +def _texts(items): + return [i["text"] for i in items] + + +def _woken_by_user(rig): + return [t for texts in rig.sent for t in texts if "] Linda: " in t] + + +@pytest.mark.asyncio +async def test_a1_she_reads_only_after_the_agent_uses_message_user(monkeypatch, tmp_path): + rig = Rig(monkeypatch, tmp_path, agent=_agent(t1=lambda rig: "planning", t3=_message("house booked"))) + await rig.arm() + rig.triggers(rig.notify_at("alert", "2026-11-10T14:00:00Z", "Sam: shh", channel="noor 30")) + ctx = await rig.run(rig.step()) + + [read] = rig.reads + assert read["time"] == "2026-11-10T14:00:00Z" and _texts(read["assistant_messages"]) == ["house booked"] + assert len(rig.events(ctx, "user_sim_read")) == 1 + + +@pytest.mark.asyncio +async def test_a4_a5_her_phone_shows_only_what_is_new_with_its_time_and_full_text(monkeypatch, tmp_path): + rig = Rig(monkeypatch, tmp_path, agent=_agent(t2=_message("first"), t3=_message("second"))) + await rig.arm() + rig.triggers(rig.notify_at("mom", "2026-11-09T15:00:00Z", "Are you coming Sunday?\nBring the pie.", + channel="Gmail · Carol Reyes (Mom)", to=ao.USER_PHONE), + rig.notify_at("alert-1", "2026-11-09T16:00:00Z", "Sam: shh", channel="noor 30"), + rig.notify_at("bank", "2026-11-10T13:00:00Z", "Your card was charged $855.", + channel="Messages · Chase", to=ao.USER_PHONE), + rig.notify_at("alert-2", "2026-11-10T14:00:00Z", "Dev: thx", channel="noor 30")) + await rig.run(rig.step()) + + first, second = rig.reads + assert first["phone"] == [{"at": "2026-11-09T15:00:00Z", "channel": "Gmail · Carol Reyes (Mom)", + "text": "Are you coming Sunday?\nBring the pie."}] + assert second["phone"] == [{"at": "2026-11-10T13:00:00Z", "channel": "Messages · Chase", + "text": "Your card was charged $855."}] + + +@pytest.mark.asyncio +async def test_a8_b3_you_sent_holds_only_her_own_lines_each_at_its_time(monkeypatch, tmp_path): + def agent(n, texts, rig): + if any("Linda: also find a dinner spot" in t for t in texts): + rig.message("three dinner spots") + return "sent" + if n == 1: + rig.message("on it") + return "sent" + rig.quiet() + return "" + + rig = Rig(monkeypatch, tmp_path, agent=agent, answers=[{"outcome": "reply", "message": "great"}]) + await rig.arm() + rig.triggers(rig.notify_at("ask", "2026-11-10T15:00:00Z", "also find a dinner spot", to=ao.USER_SAYS), + rig.notify_at("alert", "2026-11-10T16:00:00Z", "Sam: shh", channel="noor 30")) + ctx = await rig.run(rig.step(end_time="2026-11-11T02:00:00Z", final="send me the rundown")) + + first, second = rig.reads + assert first["you_sent"] == [{"at": "2026-11-09T14:00:00Z", "text": "sort the cdmx weekend"}] + assert second["you_sent"] == [{"at": "2026-11-10T15:00:00Z", "text": "also find a dinner spot"}] + [final] = [e for e in rig.events(ctx, "scripted_user_line") if e["kind"] == "final"] + assert (final["virtual_time"], final["text"]) == ("2026-11-11T02:00:00Z", "send me the rundown") + assert "great" not in repr([r["you_sent"] for r in rig.reads]) + + +@pytest.mark.asyncio +async def test_a9_every_message_since_her_last_read_arrives_in_one_read_oldest_first(monkeypatch, tmp_path): + rig = Rig(monkeypatch, tmp_path, t0=MIDNIGHT, + agent=_agent(t1=_message("house: blue, $855", "dinner: La Ola"), t2=_message("flights held"))) + await rig.arm() + rig.triggers(rig.notify_at("alert", "2026-11-10T08:00:00Z", "Wingloft: fare drop", channel="Gmail")) + await rig.run(rig.step(), rig.context(user=DAY)) + + [read] = rig.reads + assert read["time"] == "2026-11-10T12:30:00Z" + assert [(m["at"], m["text"]) for m in read["assistant_messages"]] == [ + ("2026-11-10T05:00:00Z", "house: blue, $855"), ("2026-11-10T05:00:00Z", "dinner: La Ola"), + ("2026-11-10T08:00:00Z", "flights held")] + + +@pytest.mark.asyncio +async def test_a13_the_opening_request_is_in_person_and_in_her_first_you_sent(monkeypatch, tmp_path): + rig = Rig(monkeypatch, tmp_path, agent=_agent(t1=_message("on it"))) + await rig.arm() + await rig.run(rig.step()) + + opening = {"at": "2026-11-09T14:00:00Z", "text": "sort the cdmx weekend"} + assert rig.setups[0]["opening"] == opening + assert rig.reads[0]["you_sent"] == [opening] + + +def _asking_plugin(text_of_read=lambda read: f"read at {read['time']}"): + class Plugin: + @staticmethod + def setup(version, person): + return {"system_prompt": "you are Linda", "output_format": None} + + @staticmethod + async def answer(version, person, history, read, ask): + got = await ask(text_of_read(read)) + return {**(got["structured"] or {}), "raw": got["text"]} + + return Plugin + + +@pytest.mark.asyncio +async def test_a14_one_user_sim_session_for_the_whole_run(monkeypatch, tmp_path): + rig = Rig(monkeypatch, tmp_path, agent=_agent(t1=_message("house?"), t2=_message("dinner?"), t3=_message("ok")), + plugin=_asking_plugin(), + user_texts=['{"outcome": "reply", "message": "blue", "used": "W1"}', + '{"outcome": "reply", "message": "La Ola", "used": "W2"}']) + await rig.arm() + await rig.run(rig.step()) + + contexts = {c["context_id"] for c in rig.user_calls} + assert len(rig.user_calls) == 3 and len(contexts) == 1 and None not in contexts + + +@pytest.mark.asyncio +async def test_a15_history_is_never_trimmed(monkeypatch, tmp_path): + seen = [] + + class Plugin: + @staticmethod + def setup(version, person): + return {"system_prompt": None, "output_format": None} + + @staticmethod + async def answer(version, person, history, read, ask): + seen.append(json.loads(json.dumps(history))) + return {"outcome": "reply", "message": f"answer {len(seen)}"} + + plays = {f"t{n}": _message(f"update {n}") for n in range(1, 6)} + rig = Rig(monkeypatch, tmp_path, agent=_agent(**plays), plugin=Plugin) + await rig.arm() + await rig.run(rig.step(max_wakes=6)) + + assert [len(h) for h in seen] == [0, 1, 2, 3, 4] + assert seen[-1][:3] == seen[3][:3] and [h["answer"]["message"] for h in seen[-1]] == [ + "answer 1", "answer 2", "answer 3", "answer 4"] + + +@pytest.mark.asyncio +async def test_b1_b2_b4_nothing_meant_for_the_agent_reaches_her_at_any_depth(monkeypatch, tmp_path): + def agent(n, texts, rig): + if n == 1: + rig.schedule("chase the host about the deposit", at="2026-11-09T18:00:00Z") + return "PRIVATE CLOSING NOTE: Linda should hear about the house." + if any("Sam: SECRET ALERT" in t for t in texts): + rig.message("house booked") + return "PRIVATE second note" + rig.quiet() + return "PRIVATE quiet note" + + rig = Rig(monkeypatch, tmp_path, agent=agent, answers=[{"outcome": "reply", "message": "yay"}]) + await rig.arm() + rig.gw._notifications.configure({"roles": ["default"]}) + rig.triggers(rig.notify_at("alert", "2026-11-09T20:00:00Z", "Sam: SECRET ALERT", channel="noor 30")) + ctx = await rig.run(rig.step(end_time="2026-11-10T02:00:00Z", final="send me the rundown")) + + assert rig.reads and rig.events(ctx, "nudge") + seen = json.dumps(rig.reads) + for meant_for_the_agent in ("SECRET ALERT", "noor 30", ao.NUDGE_TEXT, ao.WAKE_REMINDER_TEXT, ao.MESSAGE_RULES, + ao.QUIET_RULE, "last time you'll be woken", "your scheduled check", + "chase the host", "PRIVATE", "stay_quiet", "Any other date you see"): + assert meant_for_the_agent not in seen, meant_for_the_agent + + +@pytest.mark.asyncio +async def test_b8_ask_sends_exactly_the_text_it_is_given(monkeypatch, tmp_path): + text = " It's Mon, Nov 9, 9:00 AM.\n\nNEW FROM YOUR AI ASSISTANT:\n house? \n" + rig = Rig(monkeypatch, tmp_path, agent=_agent(t1=_message("house?")), plugin=_asking_plugin(lambda read: text)) + await rig.arm() + await rig.run(rig.step()) + + [call] = rig.user_calls + assert call["parts"] == [{"kind": "text", "text": text}] + + +@pytest.mark.parametrize("outcome", ["reply", "done"]) +@pytest.mark.asyncio +async def test_c2_c4_a_reply_or_a_done_reaches_the_agent_after_her_reply_time(monkeypatch, tmp_path, outcome): + rig = Rig(monkeypatch, tmp_path, agent=_agent(t1=_message("blue house or green?")), + answers=[{"outcome": outcome, "message": "blue, thanks!"}]) + await rig.arm() + ctx = await rig.run(rig.step(end_time="P3D"), rig.context(user=DAY)) + + [read] = rig.events(ctx, "user_sim_read") + assert read["virtual_time"] == "2026-11-09T14:00:00Z" and read["deliver_at"] == "2026-11-09T16:30:00Z" + assert _woken_by_user(rig)[-1] == "[Mon Nov 9, 4:30 PM UTC] Linda: blue, thanks!" + assert rig.events(ctx, "wake")[1]["virtual_time"] == "2026-11-09T16:30:00Z" + + +@pytest.mark.asyncio +async def test_c5_used_and_score_are_recorded_and_never_sent_to_the_agent(monkeypatch, tmp_path): + rig = Rig(monkeypatch, tmp_path, agent=_agent(t1=_message("house?")), + answers=[{"outcome": "reply", "message": "the blue one", "used": "D1-TAG", "score": 4}]) + await rig.arm() + ctx = await rig.run(rig.step()) + + [read] = rig.events(ctx, "user_sim_read") + assert (read["used"], read["score"]) == ("D1-TAG", 4) + sent = json.dumps(rig.sent) + assert "the blue one" in sent and "D1-TAG" not in sent + + +@pytest.mark.parametrize("end_time,ends", [(None, True), ("P3D", False)], ids=["one sitting", "multi-day"]) +@pytest.mark.asyncio +async def test_done_ends_run_says_whether_done_would_end_the_run(monkeypatch, tmp_path, end_time, ends): + rig = Rig(monkeypatch, tmp_path, agent=_agent(t1=_message("house booked"))) + await rig.arm() + await rig.run(rig.step(**({"end_time": end_time} if end_time else {}))) + + assert [r["done_ends_run"] for r in rig.reads] == [ends] From d36c62556669c2bb2b17c9f9bf81f19bef3e55c0 Mon Sep 17 00:00:00 2001 From: Sarah Chitty Date: Mon, 9 Nov 2026 01:25:41 -0800 Subject: [PATCH 13/53] triggers: report each fire and executor call in state; opt-in side-by-side fires and executor time limit (FD-5367) state gains, per trigger, running (fires in flight), waits (when each waiting fire goes on) and executor_calls (each call to an executor in progress, with the seconds left of its time limit), and a top-level settings key. Each fire now keeps its own wait, so fires waiting at once don't clobber each other. New triggers/v1 method config, off by default so prompt_agent runs behave as before: - concurrent_fires: a further call that provokes a repeat trigger already firing starts its own fire at once instead of queueing behind the one running. - executor_timeout_seconds: the time limit of an executor whose registration set none (else 120). Calls in one executor conversation (its URL and the trigger's context id) take turns, so fires of one trigger running side by side never resume the same session at once; in sequential mode they never overlap. Co-Authored-By: Claude Opus 5.5 --- src/agent_env/env/gateway/constants.py | 7 +- src/agent_env/env/gateway/gateway.py | 4 + src/agent_env/env/gateway/triggers.py | 241 +++++++++++++++----- tst/unit/env/gateway/gateway_card_test.py | 4 +- tst/unit/env/gateway/triggers_fires_test.py | 224 ++++++++++++++++++ 5 files changed, 422 insertions(+), 58 deletions(-) create mode 100644 tst/unit/env/gateway/triggers_fires_test.py diff --git a/src/agent_env/env/gateway/constants.py b/src/agent_env/env/gateway/constants.py index f37d2c5cf..63bfc81ab 100644 --- a/src/agent_env/env/gateway/constants.py +++ b/src/agent_env/env/gateway/constants.py @@ -131,9 +131,11 @@ def sources(names: list[str]) -> str: TRIGGER_NEXT_EVENT_AT = "next_event_at" TRIGGER_NOTIFY = "notify" TRIGGER_WEBSITE_OP = "website_op" +# state reports each fire in flight (running, waits) and each executor call in progress (executor_calls). +TRIGGER_FIRES_IN_STATE = "fires_in_state" TRIGGER_FEATURES = (TRIGGER_NAMED_EXECUTORS, TRIGGER_PROVOKING_ROLES, TRIGGER_ACTION_DELAY, TRIGGER_EXECUTOR_HOURS, TRIGGER_UNLESS, TRIGGER_WEBSITE, TRIGGER_EXECUTOR_HOURS_TIME_ZONE, TRIGGER_NEXT_EVENT_AT, - TRIGGER_NOTIFY, TRIGGER_WEBSITE_OP) + TRIGGER_NOTIFY, TRIGGER_WEBSITE_OP, TRIGGER_FIRES_IN_STATE) # The one `source` a /step call may name instead of step_api, with the id of the hook artifact that made it. STEP_SOURCE_POST_TURN_HOOK = "post_turn_hook" @@ -208,9 +210,10 @@ def random_mcp_server_name() -> str: }, { "uri": EXT_TRIGGERS_URI, - "description": "Register (additive add-by-id), remove, clear, or read (state) typed Action/State triggers + the firing log (harness control path).", + "description": "Register (additive add-by-id), remove, clear, or read (state) typed Action/State triggers + the firing log (harness control path). config sets the engine's opt-in settings: concurrent_fires (a repeat trigger's fires run side by side instead of queueing) and executor_timeout_seconds (the time limit of an executor that sets none).", "params": {"endpoint": "/triggers/register", "features": list(TRIGGER_FEATURES), "methods": { "register": {"method": "POST", "endpoint": "/triggers/register", "request": {"type": "object", "properties": {"watch_roles": {"type": "array", "items": {"type": "string"}}, "executor": {"type": "object"}, "executors": {"type": "object"}, "triggers": {"type": "array"}}, "required": ["triggers"]}}, + "config": {"method": "POST", "endpoint": "/triggers/config", "request": {"type": "object", "properties": {"concurrent_fires": {"type": "boolean"}, "executor_timeout_seconds": {"type": "number", "exclusiveMinimum": 0}}}}, "remove": {"method": "POST", "endpoint": "/triggers/remove", "request": {"type": "object", "properties": {"ids": {"type": "array", "items": {"type": "string"}}}, "required": ["ids"]}}, "clear": {"method": "POST", "endpoint": "/triggers/clear", "request": {"type": "object", "properties": {}}}, "state": {"method": "GET", "endpoint": "/triggers/state", "request": {"type": "object", "properties": {}}}}}, diff --git a/src/agent_env/env/gateway/gateway.py b/src/agent_env/env/gateway/gateway.py index f9615c81b..eed34034f 100644 --- a/src/agent_env/env/gateway/gateway.py +++ b/src/agent_env/env/gateway/gateway.py @@ -298,6 +298,10 @@ async def triggers_register(request: Request) -> Response: async def triggers_remove(request: Request) -> Response: return await _trigger_write(request, self._trigger_engine.remove) + @self._mcp.custom_route("/triggers/config", methods=["POST"]) + async def triggers_config(request: Request) -> Response: + return await _trigger_write(request, self._trigger_engine.config) + @self._mcp.custom_route("/triggers/clear", methods=["POST"]) async def triggers_clear(request: Request) -> Response: return self._json_response(self._trigger_engine.clear(), 200) diff --git a/src/agent_env/env/gateway/triggers.py b/src/agent_env/env/gateway/triggers.py index b2feb4c15..c6e90ed8d 100644 --- a/src/agent_env/env/gateway/triggers.py +++ b/src/agent_env/env/gateway/triggers.py @@ -16,6 +16,7 @@ import uuid from collections import deque from datetime import datetime, time, timedelta, timezone, tzinfo +from time import monotonic as _monotonic from typing import Any from zoneinfo import ZoneInfo @@ -443,6 +444,13 @@ def __init__(self, gateway): self._driver_interval = 1.0 # real seconds between autonomous time-trigger polls self._time_trigger_gate = asyncio.Event() # parks the driver while no time-trigger is armed self._clock_moved = asyncio.Event() # set, and replaced, when an advance moves the clock under waiting actions + # Opt-in settings (`config`), off by default so a registration behaves as it always has. + self._concurrent_fires = False # a repeat trigger's fires run side by side instead of queueing + self._executor_timeout_default: float | None = None # the time limit of an executor that sets none + self._unset_timeouts: set[str | None] = set() # executors registered without a timeout_seconds of their own + # One turn at a time per executor conversation (its URL and context id): fires of one trigger running side by + # side would otherwise resume the same session at once. + self._executor_locks: dict[tuple[str, str], asyncio.Lock] = {} def register(self, body: Any) -> dict: """Additively add each trigger by id (idempotent upsert; conflicting id or bad shape -> 400); watch_roles and @@ -450,7 +458,7 @@ def register(self, body: Any) -> dict: if not isinstance(body, dict): raise TriggerError("body must be a JSON object") watch_roles = self._resolve_watch_roles(body) - executors = self._resolve_executors(body, watch_roles) + executors, unset_timeouts = self._resolve_executors(body, watch_roles) triggers = body.get("triggers") if not isinstance(triggers, list): raise TriggerError("triggers must be a list") @@ -482,15 +490,19 @@ def register(self, body: Any) -> dict: self._readonly_tools = None self._watch_roles = watch_roles self._executors = executors + self._unset_timeouts |= unset_timeouts self._config_set = True for tid, spec in to_add: + # `running` counts its fires in flight, `alongside` holds those run beside the first (concurrent_fires), + # `waits` each fire waiting out a delay or its executor's hours, `calls` each executor call in progress. self._triggers[tid] = {"spec": spec, "status": "armed", "detected_at": None, "fired_at": None, "evaluating": False, "next_mark": None, "resolved": False, "fire_count": 0, "failure_count": 0, "last_failure_at": None, "rng": _make_rng(spec), "mark_gen": None, "pending": deque(), "task": None, "delay_rngs": _delay_rngs(spec), - "attempts": 0, "waiting": False, "resumes": None, "hours": None} + "attempts": 0, "waiting": False, "running": 0, "alongside": set(), + "waits": {}, "calls": []} self._emit("added", tid) self._recompute_time_flag() return {"ok": True, "added": batch_order, "all": list(self._triggers)} @@ -537,8 +549,9 @@ def remove(self, body: Any) -> dict: for tid in ids: trig = self._triggers.pop(tid, None) if trig is not None: - if trig["task"] is not None: - trig["task"].cancel() # a fire waiting out a delay or hours would otherwise still act + for task in [trig["task"], *trig["alongside"]]: + if task is not None: + task.cancel() # a fire waiting out a delay or hours would otherwise still act removed.append(tid) self._emit("removed", tid) self._recompute_time_flag() @@ -558,9 +571,44 @@ def clear(self) -> dict: self._in_flight = {} self._config_set = False self._readonly_tools = None + self._concurrent_fires = False + self._executor_timeout_default = None + self._unset_timeouts = set() + self._executor_locks = {} self._recompute_time_flag() return {"ok": True} + def config(self, body: Any) -> dict: + """Set the engine's opt-in settings, each off unless the body sets it (an omitted one resets): with + ``concurrent_fires``, each further call that provokes a repeat trigger already firing starts its own fire at + once, side by side, instead of queueing behind the one running; ``executor_timeout_seconds`` is the time limit + of each executor whose registration set no ``timeout_seconds`` (else 120). ``clear`` resets both.""" + if not isinstance(body, dict): + raise TriggerError("body must be a JSON object") + unknown = sorted(set(body) - {"concurrent_fires", "executor_timeout_seconds"}) + if unknown: + raise TriggerError(f"unknown setting(s) {unknown}; expected concurrent_fires, executor_timeout_seconds") + concurrent = body.get("concurrent_fires", False) + if not isinstance(concurrent, bool): + raise TriggerError(f"concurrent_fires must be true or false (got {concurrent!r})") + timeout = body.get("executor_timeout_seconds") + if timeout is not None and (isinstance(timeout, bool) or not isinstance(timeout, (int, float)) + or not math.isfinite(timeout) or timeout <= 0): + raise TriggerError(f"executor_timeout_seconds must be a positive, finite number (got {timeout!r})") + self._concurrent_fires = concurrent + self._executor_timeout_default = None if timeout is None else float(timeout) + return {"ok": True, **self._settings()} + + def _settings(self) -> dict: + return {"concurrent_fires": self._concurrent_fires, + "executor_timeout_seconds": self._executor_timeout_default} + + def _executor_timeout(self, name: str | None, executor: dict) -> float: + """The executor's time limit: its own, else the configured default, else 120.""" + if self._executor_timeout_default is not None and name in self._unset_timeouts: + return self._executor_timeout_default + return executor["timeout_seconds"] + def _recompute_time_flag(self) -> None: # "firing" counts: a recurrence mid-fire returns to "armed" and still needs the driver. self._has_time_triggers = any( @@ -618,14 +666,15 @@ def _resolve_watch_roles(self, body: dict) -> set[str]: raise TriggerError(f"watch_roles {sorted(new)} conflicts with the already-registered {sorted(self._watch_roles)} — clear to change it") return new - def _resolve_executors(self, body: dict, watch_roles: set[str]) -> dict[str | None, dict]: + def _resolve_executors(self, body: dict, watch_roles: set[str]) -> tuple[dict[str | None, dict], set[str | None]]: """The executors once this register lands: those before, plus the body's `executor` (under None) and - `executors`, each set once.""" + `executors`, each set once; and those of them new here that set no `timeout_seconds` of their own.""" named = body.get("executors") if named is not None and (not isinstance(named, dict) or not all(isinstance(n, str) and n for n in named)): raise TriggerError("executors must be an object keyed by non-empty executor names") incoming = ({None: body["executor"]} if body.get("executor") is not None else {}) | (named or {}) merged = dict(self._executors) + unset: set[str | None] = set() for name, raw in incoming.items(): normalized = self._normalize_executor(raw, "executor" if name is None else f"executors[{name!r}]", watch_roles) @@ -634,8 +683,10 @@ def _resolve_executors(self, body: dict, watch_roles: set[str]) -> dict[str | No if name is None else f"executor {name!r} conflicts with the already-registered executor of that name — " "clear to change it") + if name not in merged and raw.get("timeout_seconds") is None: + unset.add(name) merged[name] = normalized - return merged + return merged, unset @staticmethod def _normalize_executor(raw: Any, where: str, watch_roles: set[str]) -> dict: @@ -898,10 +949,15 @@ def state(self) -> dict: resumes = {tid: self._resumes_at(t) for tid, t in self._triggers.items()} upcoming = [self._upcoming_mark(t) for t in self._triggers.values()] + list(resumes.values()) next_event = min((at for at in upcoming if at is not None), default=None) + now = _monotonic() return { "config": {"watch_roles": sorted(self._watch_roles), "executor_configured": None in self._executors, "executors": sorted(n for n in self._executors if n is not None)}, + "settings": self._settings(), # what `config` set + # `running` counts a trigger's fires in flight, `waits` says when each of them waiting out a delay or its + # executor's hours goes on, and `executor_calls` lists each call to an executor in progress, with the + # seconds left of its time limit; `waiting` and `resumes_at` sum `waits` up. "triggers": [ {"id": tid, "type": t["spec"]["when"]["type"], "when": {k: t["spec"]["when"][k] for k in _WHEN_SUMMARY_KEYS if k in t["spec"]["when"]}, @@ -909,9 +965,14 @@ def state(self) -> dict: "fired_at": t["fired_at"], "fire_count": t["fire_count"], "failure_count": t["failure_count"], "last_failure_at": t["last_failure_at"], "barrier": t["spec"].get("barrier"), - "pending": len(t["pending"]), "waiting": t["waiting"], "evaluating": t["evaluating"], + "pending": len(t["pending"]), "waiting": bool(t["waits"]), "evaluating": t["evaluating"], "resumes_at": _iso(resumes[tid]) if resumes[tid] else None, - "next_mark": _iso(t["next_mark"]) if t["next_mark"] else None} + "next_mark": _iso(t["next_mark"]) if t["next_mark"] else None, + "running": t["running"], + "waits": sorted(({"resumes_at": _iso(at) if (at := self._wait_resumes(w)) else None} + for w in t["waits"].values()), key=lambda w: w["resumes_at"] or ""), + "executor_calls": [{"executor": c["executor"], "action_index": c["action_index"], + "seconds_left": round(max(0.0, c["deadline"] - now), 1)} for c in t["calls"]]} for tid, t in self._triggers.items() ], "next_event_at": _iso(next_event) if next_event else None, @@ -1037,6 +1098,14 @@ def _executor_roles(self) -> set[str]: def _mark_detected(self, tid: str, trig: dict, provoking: dict, ctx: dict | None = None) -> asyncio.Task | None: ctx = ctx if ctx is not None else {} since = self._gw._clock.elapsed() # a delay runs from the provoking call, not from when its turn comes + if trig["status"] in TRIGGER_IN_FLIGHT_STATUSES and self._concurrent_fires: + # Mid-fire, side by side: this call's fire starts now, so a quick reply doesn't wait behind a slow one. + self._emit("detected", tid, provoking=provoking, alongside=trig["running"]) + task = self._schedule(self._fire_alongside(tid, trig, ctx, since)) + task.set_name(tid) + trig["alongside"].add(task) + task.add_done_callback(trig["alongside"].discard) + return task if trig["status"] in TRIGGER_IN_FLIGHT_STATUSES: # Mid-fire: queue it; the running fire drains the queue before it settles. trig["pending"].append((ctx, since)) @@ -1094,8 +1163,9 @@ def clock_advanced(self) -> list[str]: due = self._eval_time_triggers() now = self._gw._clock.now() ended = [tid for tid, t in self._triggers.items() - if now is not None and (at := self._resumes_at(t)) is not None and at <= now - and (t["hours"] is None or _next_opening(now, t["hours"]) is None)] + if now is not None and any((at := self._wait_resumes(w)) is not None and at <= now + and (w["hours"] is None or _next_opening(now, w["hours"]) is None) + for w in t["waits"].values())] self._clock_moved.set() self._clock_moved = asyncio.Event() return list(dict.fromkeys(due + ended)) @@ -1137,8 +1207,14 @@ def _upcoming_mark(self, trig: dict) -> datetime | None: return None if mark is None or (until is not None and mark > until) else mark def _resumes_at(self, trig: dict) -> datetime | None: - """When a fire waiting out a delay or its executor's hours goes on, in virtual time; None when none waits.""" - resumes = trig["resumes"] + """When the first of its fires waiting out a delay or its executor's hours goes on, in virtual time; None when + none waits.""" + return min((at for w in trig["waits"].values() if (at := self._wait_resumes(w)) is not None), default=None) + + def _wait_resumes(self, wait: dict) -> datetime | None: + """When one waiting fire goes on, in virtual time: its delay's end (a Clock.elapsed() reading) or its + executor's next opening.""" + resumes = wait["resumes"] if resumes is None or isinstance(resumes, datetime): return resumes return self._gw._clock.at_elapsed(resumes) @@ -1220,6 +1296,7 @@ def _mark_time_detected(self, tid: str, trig: dict, marks: list[datetime], cappe async def _fire_time(self, tid: str, trig: dict, marks: list[datetime], capped: bool, terminated: bool) -> None: """Run the actions once per due mark, then re-arm (recurring) or retire (one-shot / terminated).""" + trig["running"] += 1 try: await self._gw._log_event({"event_type": "trigger_fired", "source": "trigger_engine", "trigger_id": tid}) clock = self._gw._clock @@ -1246,6 +1323,7 @@ async def _fire_time(self, tid: str, trig: dict, marks: list[datetime], capped: self._emit("failed", tid, detail=str(e)[:300]) return finally: + trig["running"] -= 1 self._recompute_time_flag() # a terminated trigger lets the driver park again self._resolve_dependents(tid) # success-only, outside the try (see _fire) @@ -1346,46 +1424,72 @@ async def _fire(self, tid: str, trig: dict, ctx: dict, since: float) -> None: """Run the actions for one provoking call, then drain any calls queued while firing (repeat triggers). A failure always re-arms: a mirror states a standing invariant, so one transient error must not silence it for the rest of the run. Ends armed or fired (one-shot success).""" - spec = trig["spec"] - repeat = bool(spec["when"].get("repeat")) + repeat = bool(trig["spec"]["when"].get("repeat")) + outcome: str | None = None # the status a normal end leaves + trig["running"] += 1 try: while True: - ok = True - trig["attempts"] += 1 - try: - await self._gw._log_event({"event_type": "trigger_fired", "source": "trigger_engine", "trigger_id": tid, - "provoking_tool": (ctx.get("args") is not None and spec["when"].get("tool")) or None, - **({"provoking_request": ctx["request"]} if "request" in ctx else {})}) - for index, action in enumerate(spec["actions"]): - if not await self._run_action(tid, trig, index, action, ctx, since): - ok = False - self._emit("failed", tid, action_index=index) - break - if ok: - trig["fire_count"] += 1 - trig["fired_at"] = _now() - self._emit("fired", tid, fire_count=trig["fire_count"]) - except Exception as e: - logger.exception(f"trigger '{tid}' firing failed") - ok = False - self._emit("failed", tid, detail=str(e)[:300]) - if ok: - self._resolve_dependents(tid) # success-only, outside the try so it can't flip a fired trigger to failed + ok = await self._fire_once(tid, trig, ctx, since) if trig["pending"]: ctx, since = trig["pending"].popleft() trig["status"] = "firing" trig["detected_at"] = _now() continue - trig["status"] = "armed" if (repeat or not ok) else "fired" + outcome = "armed" if (repeat or not ok) else "fired" return finally: - # An abnormal exit must not wedge the trigger in firing/queued, nor drop its queue silently; a trigger removed - # or cleared meanwhile is gone, and has nothing left to report. - if trig["status"] in TRIGGER_IN_FLIGHT_STATUSES and self._triggers.get(tid) is trig: - self._emit("failed", tid, detail="firing did not settle", dropped=len(trig["pending"])) - trig["status"] = "armed" - trig["pending"].clear() + trig["running"] -= 1 trig["task"] = None + self._settle_status(tid, trig, outcome) + + async def _fire_alongside(self, tid: str, trig: dict, ctx: dict, since: float) -> None: + """One fire of a repeat trigger run beside its others (``concurrent_fires``); the last of them to end re-arms + it, as a repeat trigger always ends armed.""" + trig["running"] += 1 + try: + await self._fire_once(tid, trig, ctx, since) + finally: + trig["running"] -= 1 + self._settle_status(tid, trig, "armed") + + def _settle_status(self, tid: str, trig: dict, outcome: str | None) -> None: + """A fire of ``tid`` ended, normally (leaving ``outcome``) or not (None); the last of its fires to end sets its + status. An abnormal exit must not wedge the trigger in firing/queued, nor drop its queue silently; a trigger + removed or cleared meanwhile is gone, and has nothing left to report.""" + if trig["running"] or trig["status"] not in TRIGGER_IN_FLIGHT_STATUSES: + return + if outcome is not None: + trig["status"] = outcome + elif self._triggers.get(tid) is trig: + self._emit("failed", tid, detail="firing did not settle", dropped=len(trig["pending"])) + trig["status"] = "armed" + trig["pending"].clear() + + async def _fire_once(self, tid: str, trig: dict, ctx: dict, since: float) -> bool: + """Run the actions once for one provoking call; whether they all went through.""" + spec = trig["spec"] + ok = True + trig["attempts"] += 1 + try: + await self._gw._log_event({"event_type": "trigger_fired", "source": "trigger_engine", "trigger_id": tid, + "provoking_tool": (ctx.get("args") is not None and spec["when"].get("tool")) or None, + **({"provoking_request": ctx["request"]} if "request" in ctx else {})}) + for index, action in enumerate(spec["actions"]): + if not await self._run_action(tid, trig, index, action, ctx, since): + ok = False + self._emit("failed", tid, action_index=index) + break + if ok: + trig["fire_count"] += 1 + trig["fired_at"] = _now() + self._emit("fired", tid, fire_count=trig["fire_count"]) + except Exception as e: + logger.exception(f"trigger '{tid}' firing failed") + ok = False + self._emit("failed", tid, detail=str(e)[:300]) + if ok: + self._resolve_dependents(tid) # success-only, outside the try so it can't flip a fired trigger to failed + return ok async def _run_action(self, tid: str, trig: dict, index: int, action: dict, ctx: dict, since: float) -> bool: if "delay" in action: @@ -1480,7 +1584,7 @@ async def _run_nl(self, tid: str, trig: dict, index: int, action: dict, ctx: dic fires.append(fire) try: for attempt in (1, 2): - reply = await self._call_executor(executor, instruction, context_id=f"trigger-{tid}") + reply = await self._executor_turn(tid, trig, index, action.get("executor"), executor, instruction) if reply is None: self._emit("action_failed", tid, action_index=index, detail=f"executor attempt {attempt} did not complete", **named) @@ -1497,6 +1601,26 @@ async def _run_nl(self, tid: str, trig: dict, index: int, action: dict, ctx: dic self._emit("action_failed", tid, action_index=index, detail="verification failed after retry", **named) return None + async def _executor_turn(self, tid: str, trig: dict, index: int, name: str | None, executor: dict | None, + instruction: str) -> str | None: + """One call to the executor, listed in the trigger's ``calls`` with the end of its time limit while it runs. + Calls in one conversation (the executor's URL and the trigger's context id) take turns, so fires of one trigger + running side by side never resume the executor's session at once.""" + context_id = f"trigger-{tid}" + if executor is None: + return await self._call_executor(executor, instruction, context_id=context_id) + timeout = self._executor_timeout(name, executor) + if timeout != executor["timeout_seconds"]: + executor = {**executor, "timeout_seconds": timeout} + lock = self._executor_locks.setdefault((executor["a2a_url"], context_id), asyncio.Lock()) + async with lock: + call = {"executor": name, "action_index": index, "deadline": _monotonic() + timeout} + trig["calls"].append(call) + try: + return await self._call_executor(executor, instruction, context_id=context_id) + finally: + trig["calls"].remove(call) + def caused_by(self, role: str) -> list[dict]: """The fires in flight for the executor with ``role``, all of them when several overlap; its tool calls are attributed to them. A fire is the trigger's n-th, failed ones counted.""" @@ -1511,20 +1635,17 @@ async def _delay(self, tid: str, trig: dict, index: int, action: dict, since: fl mean = _parse_duration(delay["mean"]).total_seconds() seconds = trig["delay_rngs"][index].expovariate(1.0 / mean) self._emit("delayed", tid, action_index=index, virtual_seconds=round(seconds, 3), **_named_executor(action)) - trig["waiting"] = True - trig["resumes"] = since + seconds + token = _begin_wait(trig, since + seconds) try: await self._sleep_virtual(seconds - (self._gw._clock.elapsed() - since)) finally: - trig["waiting"] = False - trig["resumes"] = None + _end_wait(trig, token) async def _await_hours(self, tid: str, trig: dict, index: int, hours: dict, named: dict) -> None: """Wait until the executor's working hours are open now, reckoned each pass; not while the clock is unarmed.""" clock = self._gw._clock gen = opens = None - trig["waiting"] = True - trig["hours"] = hours + token = _begin_wait(trig, None, hours) try: while True: current = clock.generation # read before now(), so a re-arm in between is caught on the next pass @@ -1534,13 +1655,11 @@ async def _await_hours(self, tid: str, trig: dict, index: int, hours: dict, name return if current != gen or opening != opens: gen, opens = current, opening - trig["resumes"] = opens + trig["waits"][token]["resumes"] = opens self._emit("deferred", tid, action_index=index, until=_iso(opens), **named) await self._sleep_virtual((opens - now).total_seconds(), generation=gen) finally: - trig["waiting"] = False - trig["resumes"] = None - trig["hours"] = None + _end_wait(trig, token) async def _sleep_virtual(self, seconds: float, generation: int | None = None) -> None: """Wait out ``seconds`` of virtual time as the clock runs (real time at its rate, 1 while unarmed, none while @@ -1602,6 +1721,20 @@ def _cancel(self, tid: str, trig: dict, **cause) -> None: self._cancel(dep_id, dep, anchor=tid) +def _begin_wait(trig: dict, resumes: Any, hours: dict | None = None) -> object: + """Mark one fire of ``trig`` as waiting until ``resumes`` (a Clock.elapsed() reading, a datetime, or None until + known), for ``hours`` when it waits on its executor's working hours; returns the token ``_end_wait`` takes.""" + token = object() + trig.setdefault("waits", {})[token] = {"resumes": resumes, "hours": hours} + trig["waiting"] = True + return token + + +def _end_wait(trig: dict, token: object) -> None: + trig["waits"].pop(token, None) + trig["waiting"] = bool(trig["waits"]) + + def _interval_seconds(when: dict, rng: random.Random | None) -> float: """Virtual seconds from one mark of a recurring time trigger to the next, a stochastic one drawn from `rng`.""" every = when["every"] diff --git a/tst/unit/env/gateway/gateway_card_test.py b/tst/unit/env/gateway/gateway_card_test.py index 19582c7ca..1d8eb883c 100644 --- a/tst/unit/env/gateway/gateway_card_test.py +++ b/tst/unit/env/gateway/gateway_card_test.py @@ -342,11 +342,11 @@ async def test_env_data_plane_unknown_method(): def test_triggers_advertised_as_one_extension(): - """Triggers ride a single urn:agentenv:triggers/v1 (register/remove/clear/state) — no separate get URI.""" + """Triggers ride a single urn:agentenv:triggers/v1 (register/config/remove/clear/state) — no separate get URI.""" exts = _gateway({})._gateway_extensions() trigger = [e for e in exts if e["uri"] == "urn:agentenv:triggers/v1"] assert len(trigger) == 1 - assert set(trigger[0]["params"]["methods"]) == {"register", "remove", "clear", "state"} + assert set(trigger[0]["params"]["methods"]) == {"register", "config", "remove", "clear", "state"} assert not any(e["uri"] == "urn:agentenv:get-triggers/v1" for e in exts) diff --git a/tst/unit/env/gateway/triggers_fires_test.py b/tst/unit/env/gateway/triggers_fires_test.py new file mode 100644 index 000000000..6af9e00c0 --- /dev/null +++ b/tst/unit/env/gateway/triggers_fires_test.py @@ -0,0 +1,224 @@ +"""What the trigger engine's state says of each fire in flight (running, waits, executor_calls), and its opt-in +settings (config): a repeat trigger's fires side by side, and a time limit for executors that set none.""" +from __future__ import annotations + +import asyncio + +import pytest + +from agent_env.env.gateway.constants import EXT_TRIGGERS_URI, GATEWAY_EXTENSIONS +from agent_env.env.gateway.triggers import TriggerEngine, TriggerError +from tst.unit.env.gateway.next_event_test import _engine, _next, _row, _until +from tst.unit.env.gateway.triggers_time_test import _result + + +def _reply_trigger(tid: str = "reply", *, delay=None, executor: dict | None = None) -> dict: + action = {"type": "nl", "executor": "bob", "instruction": "answer Linda", **({"delay": delay} if delay else {})} + return {"watch_roles": ["default"], + "executors": {"bob": {"a2a_url": "http://bob", "role": "bob", **(executor or {})}}, + "triggers": [{"id": tid, "when": {"type": "action", "tool": "send_text", "repeat": True}, + "actions": [action]}]} + + +class _Executor: + """A stand-in executor: each call waits until `release()` lets it answer; records each call and how many ran at + once.""" + + def __init__(self, monkeypatch) -> None: + self.calls: list[dict] = [] + self.running = self.most_at_once = 0 + self.go = asyncio.Event() + executor = self + + async def call(engine, ex, instruction, context_id): + executor.calls.append({"timeout_seconds": ex["timeout_seconds"], "context_id": context_id, + "at": engine._gw._clock.now()}) + executor.running += 1 + executor.most_at_once = max(executor.most_at_once, executor.running) + try: + await executor.go.wait() + finally: + executor.running -= 1 + return "replied" + + monkeypatch.setattr(TriggerEngine, "_call_executor", call) + + def release(self) -> None: + self.go.set() + + +# ---- executor calls in state -------------------------------------------------------------------------------------- + +@pytest.mark.asyncio +async def test_state_lists_an_executor_call_in_progress_with_the_time_left_of_its_limit(monkeypatch): + bob = _Executor(monkeypatch) + engine, _, _ = _engine() + engine.register(_reply_trigger(executor={"timeout_seconds": 300})) + engine.on_tool_call("default", "send_text", {}, _result()) + await _until(lambda: bob.calls) + + row = _row(engine, "reply") + [call] = row["executor_calls"] + assert (call["executor"], call["action_index"], row["running"]) == ("bob", 0, 1) + assert 295 <= call["seconds_left"] <= 300 + + bob.release() + await _until(lambda: _row(engine, "reply")["status"] == "armed") + assert _row(engine, "reply")["executor_calls"] == [] and _row(engine, "reply")["running"] == 0 + + +# ---- concurrent_fires ----------------------------------------------------------------------------------------------- + +@pytest.mark.asyncio +async def test_by_default_a_second_call_queues_behind_the_fire_running(monkeypatch): + bob = _Executor(monkeypatch) + engine, _, _ = _engine() + engine.register(_reply_trigger()) + engine.on_tool_call("default", "send_text", {}, _result()) + engine.on_tool_call("default", "send_text", {}, _result()) + await _until(lambda: bob.calls) + + row = _row(engine, "reply") + assert (row["status"], row["pending"], row["running"], len(bob.calls)) == ("queued", 1, 1, 1) + bob.release() + await _until(lambda: _row(engine, "reply")["status"] == "armed") + assert len(bob.calls) == 2 and bob.most_at_once == 1 + + +@pytest.mark.asyncio +async def test_side_by_side_each_fire_waits_out_its_own_delay(monkeypatch): + """A quick reply doesn't wait behind a slow one: each fire's delay runs from its own provoking call.""" + bob = _Executor(monkeypatch) + bob.release() + engine, clock, _ = _engine() + engine.config({"concurrent_fires": True}) + engine.register({**_reply_trigger(delay="PT2H")}) + engine.on_tool_call("default", "send_text", {}, _result()) + clock.advance("2026-03-02T01:00:00Z") + engine.on_tool_call("default", "send_text", {}, _result()) + await _until(lambda: len(_row(engine, "reply")["waits"]) == 2) + + row = _row(engine, "reply") + assert (row["status"], row["running"], row["pending"]) == ("firing", 2, 0) + assert [w["resumes_at"] for w in row["waits"]] == ["2026-03-02T02:00:00Z", "2026-03-02T03:00:00Z"] + assert row["resumes_at"] == _next(engine) == "2026-03-02T02:00:00Z" + + clock.advance("2026-03-02T02:00:00Z") + assert engine.clock_advanced() == ["reply"] + await _until(lambda: len(bob.calls) == 1 and len(_row(engine, "reply")["waits"]) == 1) + assert _next(engine) == "2026-03-02T03:00:00Z" and _row(engine, "reply")["running"] == 1 + + clock.advance("2026-03-02T03:00:00Z") + engine.clock_advanced() + await _until(lambda: _row(engine, "reply")["status"] == "armed") + assert [c["at"].isoformat() for c in bob.calls] == ["2026-03-02T02:00:00+00:00", "2026-03-02T03:00:00+00:00"] + assert _row(engine, "reply")["fire_count"] == 2 and _next(engine) is None + + +@pytest.mark.asyncio +async def test_side_by_side_fires_of_one_trigger_take_turns_with_its_executor_conversation(monkeypatch): + bob = _Executor(monkeypatch) + engine, _, _ = _engine() + engine.config({"concurrent_fires": True}) + engine.register(_reply_trigger()) + for _ in range(3): + engine.on_tool_call("default", "send_text", {}, _result()) + await _until(lambda: bob.calls) + await asyncio.sleep(0.05) + + assert _row(engine, "reply")["running"] == 3 and len(bob.calls) == 1 + bob.release() + await _until(lambda: _row(engine, "reply")["status"] == "armed") + assert len(bob.calls) == 3 and bob.most_at_once == 1 + assert {c["context_id"] for c in bob.calls} == {"trigger-reply"} + + +@pytest.mark.asyncio +async def test_two_triggers_reach_their_executors_at_once(monkeypatch): + bob = _Executor(monkeypatch) + engine, _, _ = _engine() + engine.register({"watch_roles": ["default"], "executors": {"bob": {"a2a_url": "http://bob", "role": "bob"}}, + "triggers": [{"id": tid, "when": {"type": "action", "tool": "send_text"}, + "actions": [{"type": "nl", "executor": "bob", "instruction": "answer"}]} + for tid in ("group", "dm")]}) + engine.on_tool_call("default", "send_text", {}, _result()) + await _until(lambda: len(bob.calls) == 2) + + assert bob.most_at_once == 2 + bob.release() + await _until(lambda: all(_row(engine, t)["status"] == "fired" for t in ("group", "dm"))) + + +@pytest.mark.asyncio +async def test_removing_a_trigger_cancels_its_fires_side_by_side(monkeypatch): + bob = _Executor(monkeypatch) + engine, _, _ = _engine() + engine.config({"concurrent_fires": True}) + engine.register(_reply_trigger(delay="PT1H")) + engine.on_tool_call("default", "send_text", {}, _result()) + engine.on_tool_call("default", "send_text", {}, _result()) + await _until(lambda: len(_row(engine, "reply")["waits"]) == 2) + tasks = [engine._triggers["reply"]["task"], *engine._triggers["reply"]["alongside"]] + + engine.remove({"ids": ["reply"]}) + await asyncio.gather(*tasks, return_exceptions=True) + + assert all(t.cancelled() for t in tasks) and bob.calls == [] + + +# ---- executor_timeout_seconds --------------------------------------------------------------------------------------- + +@pytest.mark.asyncio +async def test_the_configured_limit_applies_only_to_executors_that_set_none(monkeypatch): + bob = _Executor(monkeypatch) + bob.release() + engine, _, _ = _engine() + engine.config({"executor_timeout_seconds": 300}) + engine.register({"watch_roles": ["default"], + "executors": {"bob": {"a2a_url": "http://bob", "role": "bob"}, + "ann": {"a2a_url": "http://ann", "role": "ann", "timeout_seconds": 120}}, + "triggers": [{"id": tid, "when": {"type": "action", "tool": "send_text"}, + "actions": [{"type": "nl", "executor": tid, "instruction": "answer"}]} + for tid in ("bob", "ann")]}) + engine.on_tool_call("default", "send_text", {}, _result()) + await _until(lambda: len(bob.calls) == 2) + + assert sorted((c["context_id"], c["timeout_seconds"]) for c in bob.calls) == [ + ("trigger-ann", 120), ("trigger-bob", 300)] + assert engine.state()["settings"] == {"concurrent_fires": False, "executor_timeout_seconds": 300.0} + + +@pytest.mark.asyncio +async def test_without_config_an_executor_that_sets_no_limit_keeps_120_s(monkeypatch): + bob = _Executor(monkeypatch) + bob.release() + engine, _, _ = _engine() + engine.register(_reply_trigger()) + engine.on_tool_call("default", "send_text", {}, _result()) + await _until(lambda: bob.calls) + assert bob.calls[0]["timeout_seconds"] == 120 + + +@pytest.mark.parametrize("body", [{"concurrent_fires": "yes"}, {"executor_timeout_seconds": 0}, + {"executor_timeout_seconds": True}, {"executor_timeout_seconds": float("inf")}, + {"parallel": True}, []]) +def test_bad_settings_are_refused(body): + engine, _, _ = _engine() + with pytest.raises(TriggerError): + engine.config(body) + + +def test_settings_are_off_until_set_and_clear_resets_them(): + engine, _, _ = _engine() + assert engine.state()["settings"] == {"concurrent_fires": False, "executor_timeout_seconds": None} + assert engine.config({"concurrent_fires": True, "executor_timeout_seconds": 300})["concurrent_fires"] is True + engine.config({"concurrent_fires": True}) + assert engine.state()["settings"] == {"concurrent_fires": True, "executor_timeout_seconds": None} + engine.clear() + assert engine.state()["settings"] == {"concurrent_fires": False, "executor_timeout_seconds": None} + + +def test_the_card_offers_config(): + triggers = next(e for e in GATEWAY_EXTENSIONS if e["uri"] == EXT_TRIGGERS_URI) + assert triggers["params"]["methods"]["config"]["endpoint"] == "/triggers/config" + assert "fires_in_state" in triggers["params"]["features"] From 9ed4ad4d536e2af217ea6f71d79e4d9e95f9455c Mon Sep 17 00:00:00 2001 From: Sarah Chitty Date: Mon, 9 Nov 2026 01:26:38 -0800 Subject: [PATCH 14/53] always_on: keep a short summary in the run record; the full timeline goes only to the object store (FD-5371) A long run's timeline could push what a running job reports back over 2 MB. The summary in metadata["always_on"][step] now holds the counts, end reason, settings and the timeline's object_url, plus real_seconds and timeline_events; when storing fails it keeps the last 20 events instead. Co-Authored-By: Claude Opus 5.5 --- .../task_step/task_steps/always_on.py | 31 +++++++++-- tst/unit/task_step/always_on_harness.py | 13 ++++- tst/unit/task_step/test_always_on_record.py | 54 +++++++++++++++++++ 3 files changed, 92 insertions(+), 6 deletions(-) create mode 100644 tst/unit/task_step/test_always_on_record.py diff --git a/src/agent_env/task_step/task_steps/always_on.py b/src/agent_env/task_step/task_steps/always_on.py index 2b13917a0..c45f90ff9 100644 --- a/src/agent_env/task_step/task_steps/always_on.py +++ b/src/agent_env/task_step/task_steps/always_on.py @@ -28,7 +28,8 @@ check's wake counts toward ``max_wakes``. - At end_time the user's ``final`` line (with a reminder to use message_user) gives the agent one last turn. The run ends on end_time, done (one sitting), max_wakes, or nothing left to wake the agent; the reason is recorded. -- Everything goes into a timeline in ``metadata["always_on"][step_id]`` and the object store. +- The run's metadata keeps a summary in ``metadata["always_on"][step_id]`` (small enough for a running job to report + back); the full timeline goes to the object store, at the summary's ``object_url``. """ from __future__ import annotations @@ -90,6 +91,10 @@ CHECK_MIN_INTERVAL = "PT3H" EMPTY_REPLIES = frozenset({"(The assistant's reply was empty.)"}) # a harness's stand-in for an empty final reply TIME_LINE = "It's {when}. Any other date you see is the computer's clock, not the task's." +# The run's metadata keeps a summary; the full timeline goes to the object store. When storing fails, the summary keeps +# this many of the last events instead, without these fields. +TIMELINE_TAIL = 20 +BULKY_EVENT_FIELDS = ("input", "raw", "read", "parts_text", "changes") # ================================================================================================================== @@ -930,20 +935,36 @@ async def finish(self) -> None: "user_sim": {"name": step.user_sim, "version": self.version, "model": self.us.get("model"), "agent_name": self.us.get("agent_name"), **self.user_setup}, "end_time": _iso(self.end) if self.end else None, "time_zone": self.zone, - "multi_day": self.multi_day, "timeline": self.timeline} + "multi_day": self.multi_day, "real_seconds": round(time.time() - self.t0_real, 1), + "timeline_events": len(self.timeline)} record["settings"] = {"stay_quiet": step.stay_quiet, "nudge": step.nudge, "wake_reminder": step.wake_reminder, "scheduled_checks": self.checks} + await self._store_timeline(record) + ctx.metadata.setdefault(ALWAYS_ON_KEY, {})[step.id] = record + final_text = "\n\n".join([m["text"] for m in self.last_messages] + ([self.last_plain] if self.last_plain else [])) + self._append_prompt_response(final_text) + + async def _store_timeline(self, record: dict) -> None: + """Store the run's full record, its timeline included, and give ``record`` (what stays in the run's metadata, + kept small for the job's report) its ``object_url``. When it can't be stored, ``record`` keeps the last + ``TIMELINE_TAIL`` events instead, their bulky fields dropped.""" + step, ctx = self.step, self.ctx try: store = get_config().get_object_store() key = (f"{get_config().get_artifact_key_prefix()}always_on_timeline/" f"instance_id={ctx.instance_id or 'adhoc'}/{step.id}-{uuid.uuid4().hex[:8]}.json") + full = {**record, "timeline": self.timeline} record["object_url"] = await asyncio.to_thread(store.put, key, - json.dumps(record, indent=1, default=str).encode(), + json.dumps(full, indent=1, default=str).encode(), content_type="application/json") except Exception as e: logger.warning(f"always_on: could not store the timeline: {e}") - ctx.metadata.setdefault(ALWAYS_ON_KEY, {})[step.id] = record - final_text = "\n\n".join([m["text"] for m in self.last_messages] + ([self.last_plain] if self.last_plain else [])) + record["timeline_store_error"] = f"{type(e).__name__}: {str(e)[:300]}" + record["timeline_tail"] = [{k: v for k, v in event.items() if k not in BULKY_EVENT_FIELDS} + for event in self.timeline[-TIMELINE_TAIL:]] + + def _append_prompt_response(self, final_text: str) -> None: + step, ctx = self.step, self.ctx traj = next((u for u in reversed(self.per_turn_trajectory_urls) if u), None) ctx.prompt_responses.append(PromptResponse( prompt_id=step.prompt_id, response=final_text, prompt_text=step.prompt, diff --git a/tst/unit/task_step/always_on_harness.py b/tst/unit/task_step/always_on_harness.py index be7a2a229..d511ae83a 100644 --- a/tst/unit/task_step/always_on_harness.py +++ b/tst/unit/task_step/always_on_harness.py @@ -4,10 +4,12 @@ from __future__ import annotations +import json from typing import Any, Callable, Optional import httpx +from agent_env.config import get_config from agent_env.env.env import DeployedGatewayEnv from agent_env.env.gateway import gateway as gw_mod from agent_env.env.gateway.constants import AGENT_ENV_ROLE_HEADER, GATEWAY_EXTENSIONS, WELL_KNOWN_PATH @@ -116,10 +118,19 @@ async def run(self, step: ao.AlwaysOnTaskStep, ctx: Optional[TaskStepContext] = return ctx def record(self, ctx: TaskStepContext) -> dict: + """The step's summary in the run's metadata.""" return ctx.metadata[ao.ALWAYS_ON_KEY]["week"] + def stored(self, ctx: TaskStepContext) -> dict: + """The step's full record, its timeline included, as the object store holds it.""" + url = self.record(ctx)["object_url"] + return json.loads(get_config().get_object_store_at(url).get(url)) + + def timeline(self, ctx: TaskStepContext) -> list[dict]: + return self.stored(ctx)["timeline"] + def events(self, ctx: TaskStepContext, name: str) -> list[dict]: - return [e for e in self.record(ctx)["timeline"] if e["event"] == name] + return [e for e in self.timeline(ctx) if e["event"] == name] # ---- fakes ----------------------------------------------------------------------------------------------------- async def _send_and_wait(self, url, parts, *, before_send=None, **kw): diff --git a/tst/unit/task_step/test_always_on_record.py b/tst/unit/task_step/test_always_on_record.py new file mode 100644 index 000000000..223e1a720 --- /dev/null +++ b/tst/unit/task_step/test_always_on_record.py @@ -0,0 +1,54 @@ +"""What the always_on step keeps in the run's metadata (a short summary, small enough for a running job to report back) +and what it stores in full (the timeline, in the object store).""" + +from __future__ import annotations + +import json + +import pytest + +from agent_env.config import get_config +from agent_env.task_step.task_steps import always_on as ao +from tst.unit.task_step.always_on_harness import Rig + +pytestmark = pytest.mark.usefixtures("local_stores") + + +def _chatty(n, texts, rig): + rig.message(f"update {n}: " + "x" * 2000) + return "sent" + + +@pytest.mark.asyncio +async def test_the_run_record_is_a_summary_and_the_full_timeline_is_stored(monkeypatch, tmp_path): + rig = Rig(monkeypatch, tmp_path, agent=_chatty, + answers=[{"outcome": "reply", "message": f"ok {i}"} for i in range(30)]) + await rig.arm() + ctx = await rig.run(rig.step(max_wakes=20)) + + record = rig.record(ctx) + assert "timeline" not in record + assert len(json.dumps(record, default=str)) < 10_000 + stored = rig.stored(ctx) + assert stored["end_reason"] == record["end_reason"] == "max_wakes" + assert len(stored["timeline"]) == record["timeline_events"] > 60 + assert sum(1 for e in stored["timeline"] if e["event"] == "wake") == 20 + + +@pytest.mark.asyncio +async def test_when_the_timeline_cannot_be_stored_the_record_keeps_its_last_events(monkeypatch, tmp_path): + rig = Rig(monkeypatch, tmp_path, agent=_chatty, + answers=[{"outcome": "reply", "message": f"ok {i}"} for i in range(30)]) + await rig.arm() + store = get_config().get_object_store() + + def fail(*a, **kw): + raise OSError("bucket gone") + + monkeypatch.setattr(type(store), "put", fail) + ctx = await rig.run(rig.step(max_wakes=20)) + + record = rig.record(ctx) + assert "object_url" not in record and record["timeline_store_error"] == "OSError: bucket gone" + assert len(record["timeline_tail"]) == ao.TIMELINE_TAIL and record["timeline_tail"][-1]["event"] == "end" + assert not any(set(e) & set(ao.BULKY_EVENT_FIELDS) for e in record["timeline_tail"]) From baa1edb7e95e7799035a07d117c9c3f68c8dcfc9 Mon Sep 17 00:00:00 2001 From: Sarah Chitty Date: Mon, 9 Nov 2026 01:31:40 -0800 Subject: [PATCH 15/53] triggers: count a fire as running from its detection, not from when its task first runs (FD-5367) A reader of state right after an advance could otherwise see a firing trigger with nothing running and call the world settled before the fire's actions ran. A fire cancelled before it began is uncounted when its task ends. Co-Authored-By: Claude Opus 5.5 --- src/agent_env/env/gateway/triggers.py | 40 ++++++++++++++------- tst/unit/env/gateway/triggers_fires_test.py | 31 +++++++++++++++- 2 files changed, 58 insertions(+), 13 deletions(-) diff --git a/src/agent_env/env/gateway/triggers.py b/src/agent_env/env/gateway/triggers.py index c6e90ed8d..2df675e76 100644 --- a/src/agent_env/env/gateway/triggers.py +++ b/src/agent_env/env/gateway/triggers.py @@ -1101,7 +1101,7 @@ def _mark_detected(self, tid: str, trig: dict, provoking: dict, ctx: dict | None if trig["status"] in TRIGGER_IN_FLIGHT_STATUSES and self._concurrent_fires: # Mid-fire, side by side: this call's fire starts now, so a quick reply doesn't wait behind a slow one. self._emit("detected", tid, provoking=provoking, alongside=trig["running"]) - task = self._schedule(self._fire_alongside(tid, trig, ctx, since)) + task = self._start_fire(trig, lambda ticket: self._fire_alongside(tid, trig, ctx, since, ticket)) task.set_name(tid) trig["alongside"].add(task) task.add_done_callback(trig["alongside"].discard) @@ -1115,7 +1115,7 @@ def _mark_detected(self, tid: str, trig: dict, provoking: dict, ctx: dict | None trig["status"] = "firing" trig["detected_at"] = _now() self._emit("detected", tid, provoking=provoking) - trig["task"] = self._schedule(self._fire(tid, trig, ctx, since)) + trig["task"] = self._start_fire(trig, lambda ticket: self._fire(tid, trig, ctx, since, ticket)) trig["task"].set_name(tid) return trig["task"] @@ -1292,11 +1292,12 @@ def _mark_time_detected(self, tid: str, trig: dict, marks: list[datetime], cappe # first/last only: a burst can carry _MAX_CATCHUP marks, each with its own `fired` event. self._emit("detected", tid, provoking=provoking, due=len(marks), capped=capped, first_mark=_iso(marks[0]), last_mark=_iso(marks[-1])) - trig["task"] = self._schedule(self._fire_time(tid, trig, marks, capped, terminated)) + trig["task"] = self._start_fire(trig, lambda ticket: self._fire_time(tid, trig, marks, capped, terminated, + ticket)) - async def _fire_time(self, tid: str, trig: dict, marks: list[datetime], capped: bool, terminated: bool) -> None: + async def _fire_time(self, tid: str, trig: dict, marks: list[datetime], capped: bool, terminated: bool, + ticket: dict) -> None: """Run the actions once per due mark, then re-arm (recurring) or retire (one-shot / terminated).""" - trig["running"] += 1 try: await self._gw._log_event({"event_type": "trigger_fired", "source": "trigger_engine", "trigger_id": tid}) clock = self._gw._clock @@ -1323,7 +1324,7 @@ async def _fire_time(self, tid: str, trig: dict, marks: list[datetime], capped: self._emit("failed", tid, detail=str(e)[:300]) return finally: - trig["running"] -= 1 + _uncount(trig, ticket) self._recompute_time_flag() # a terminated trigger lets the driver park again self._resolve_dependents(tid) # success-only, outside the try (see _fire) @@ -1344,6 +1345,16 @@ def _is_readonly(self, name: str) -> bool: self._readonly_tools = readonly return name in self._readonly_tools + def _start_fire(self, trig: dict, run) -> asyncio.Task: + """Schedule ``run(ticket)``, one fire of ``trig``, counted in its ``running`` from now, so state shows it at work + before its task first runs. The fire's ``finally`` uncounts it (``_uncount``); one cancelled before it began + is uncounted when its task ends.""" + ticket = {"counted": True} + trig["running"] += 1 + task = self._schedule(run(ticket)) + task.add_done_callback(lambda _: _uncount(trig, ticket)) + return task + def _schedule(self, coro) -> asyncio.Task: task = asyncio.get_running_loop().create_task(coro) self._tasks.add(task) @@ -1420,13 +1431,12 @@ async def _internal_call(self, tool_name: str, arguments: dict, role: str | None caller = Caller(role=role) if role is not None else None return await gw._call_child_tool(server_url, tool_name, arguments, caller=caller) - async def _fire(self, tid: str, trig: dict, ctx: dict, since: float) -> None: + async def _fire(self, tid: str, trig: dict, ctx: dict, since: float, ticket: dict) -> None: """Run the actions for one provoking call, then drain any calls queued while firing (repeat triggers). A failure always re-arms: a mirror states a standing invariant, so one transient error must not silence it for the rest of the run. Ends armed or fired (one-shot success).""" repeat = bool(trig["spec"]["when"].get("repeat")) outcome: str | None = None # the status a normal end leaves - trig["running"] += 1 try: while True: ok = await self._fire_once(tid, trig, ctx, since) @@ -1438,18 +1448,17 @@ async def _fire(self, tid: str, trig: dict, ctx: dict, since: float) -> None: outcome = "armed" if (repeat or not ok) else "fired" return finally: - trig["running"] -= 1 + _uncount(trig, ticket) trig["task"] = None self._settle_status(tid, trig, outcome) - async def _fire_alongside(self, tid: str, trig: dict, ctx: dict, since: float) -> None: + async def _fire_alongside(self, tid: str, trig: dict, ctx: dict, since: float, ticket: dict) -> None: """One fire of a repeat trigger run beside its others (``concurrent_fires``); the last of them to end re-arms it, as a repeat trigger always ends armed.""" - trig["running"] += 1 try: await self._fire_once(tid, trig, ctx, since) finally: - trig["running"] -= 1 + _uncount(trig, ticket) self._settle_status(tid, trig, "armed") def _settle_status(self, tid: str, trig: dict, outcome: str | None) -> None: @@ -1721,6 +1730,13 @@ def _cancel(self, tid: str, trig: dict, **cause) -> None: self._cancel(dep_id, dep, anchor=tid) +def _uncount(trig: dict, ticket: dict) -> None: + """Take one fire out of ``trig``'s ``running``, once.""" + if ticket["counted"]: + ticket["counted"] = False + trig["running"] -= 1 + + def _begin_wait(trig: dict, resumes: Any, hours: dict | None = None) -> object: """Mark one fire of ``trig`` as waiting until ``resumes`` (a Clock.elapsed() reading, a datetime, or None until known), for ``hours`` when it waits on its executor's working hours; returns the token ``_end_wait`` takes.""" diff --git a/tst/unit/env/gateway/triggers_fires_test.py b/tst/unit/env/gateway/triggers_fires_test.py index 6af9e00c0..560ac29f8 100644 --- a/tst/unit/env/gateway/triggers_fires_test.py +++ b/tst/unit/env/gateway/triggers_fires_test.py @@ -9,7 +9,7 @@ from agent_env.env.gateway.constants import EXT_TRIGGERS_URI, GATEWAY_EXTENSIONS from agent_env.env.gateway.triggers import TriggerEngine, TriggerError from tst.unit.env.gateway.next_event_test import _engine, _next, _row, _until -from tst.unit.env.gateway.triggers_time_test import _result +from tst.unit.env.gateway.triggers_time_test import _perm, _result def _reply_trigger(tid: str = "reply", *, delay=None, executor: dict | None = None) -> dict: @@ -222,3 +222,32 @@ def test_the_card_offers_config(): triggers = next(e for e in GATEWAY_EXTENSIONS if e["uri"] == EXT_TRIGGERS_URI) assert triggers["params"]["methods"]["config"]["endpoint"] == "/triggers/config" assert "fires_in_state" in triggers["params"]["features"] + + +@pytest.mark.asyncio +async def test_a_fire_counts_as_running_from_its_detection_before_its_task_first_runs(monkeypatch): + bob = _Executor(monkeypatch) + bob.release() + engine, clock, _ = _engine() + engine.register({"watch_roles": ["default"], "triggers": [ + {"id": "ping", "when": {"type": "time", "at": "PT1M"}, "actions": [_perm()]}]}) + clock.advance("2026-03-02T00:01:00Z") + + assert engine.clock_advanced() == ["ping"] + assert (_row(engine, "ping")["status"], _row(engine, "ping")["running"]) == ("firing", 1) + await _until(lambda: _row(engine, "ping")["status"] == "fired") + assert _row(engine, "ping")["running"] == 0 + + +@pytest.mark.asyncio +async def test_a_fire_cancelled_before_it_began_leaves_no_count(monkeypatch): + _Executor(monkeypatch) + engine, _, _ = _engine() + engine.register(_reply_trigger()) + engine.on_tool_call("default", "send_text", {}, _result()) + trig = engine._triggers["reply"] + trig["task"].cancel() + await asyncio.gather(trig["task"], return_exceptions=True) + await asyncio.sleep(0) + + assert trig["running"] == 0 From d76d30adb6a75e11046433871317d47c28d35ea0 Mon Sep 17 00:00:00 2001 From: Sarah Chitty Date: Mon, 9 Nov 2026 01:39:11 -0800 Subject: [PATCH 16/53] triggers: a delay that ends between microseconds goes on when the clock is advanced to its resumes_at Virtual times are microseconds, so resumes_at can fall a hair short of a stochastic delay's true end. On a frozen clock nothing else moves time, so the fire waited forever (and a settle wait spun until its timeout). Under a microsecond left now counts as reached. Co-Authored-By: Claude Opus 5.5 --- src/agent_env/env/gateway/triggers.py | 7 ++++++- tst/unit/env/gateway/triggers_fires_test.py | 18 ++++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/src/agent_env/env/gateway/triggers.py b/src/agent_env/env/gateway/triggers.py index 2df675e76..96c8dfbb1 100644 --- a/src/agent_env/env/gateway/triggers.py +++ b/src/agent_env/env/gateway/triggers.py @@ -54,6 +54,8 @@ _DROP = object() # an unresolved `${x?}` used as a whole value: drop the key / element # Real seconds between re-reads of the clock while waiting out virtual time, so a rate change, re-arm or clear is followed. _VIRTUAL_RECHECK_S = 5.0 +# The clock's resolution: virtual times are microseconds. +_VIRTUAL_RESOLUTION_S = 1e-6 # Seconds an executor call waits to connect, and for a task poll's answer. _EXECUTOR_HTTP_TIMEOUT_S = 30.0 # An allowlist, not a size bound: identifier/metadata keys only, since bodies and addresses are PII. @@ -1675,7 +1677,10 @@ async def _sleep_virtual(self, seconds: float, generation: int | None = None) -> frozen) or jumps ahead. Given the clock's ``generation``, stop early once a re-arm or clear moves it on.""" clock = self._gw._clock until = clock.elapsed() + seconds - while (left := until - clock.elapsed()) > 0 and (generation is None or clock.generation == generation): + # Times are kept to the microsecond, so an advance to a waiting fire's resumes_at can fall a hair short of it; + # under a microsecond left is reached, or a frozen clock would hold the fire forever. + while ((left := until - clock.elapsed()) >= _VIRTUAL_RESOLUTION_S + and (generation is None or clock.generation == generation)): rate = clock.rate await self._nap(min(left / rate, _VIRTUAL_RECHECK_S) if rate else _VIRTUAL_RECHECK_S) diff --git a/tst/unit/env/gateway/triggers_fires_test.py b/tst/unit/env/gateway/triggers_fires_test.py index 560ac29f8..a0a0502e5 100644 --- a/tst/unit/env/gateway/triggers_fires_test.py +++ b/tst/unit/env/gateway/triggers_fires_test.py @@ -251,3 +251,21 @@ async def test_a_fire_cancelled_before_it_began_leaves_no_count(monkeypatch): await asyncio.sleep(0) assert trig["running"] == 0 + + +@pytest.mark.asyncio +async def test_a_fire_advanced_to_its_resumes_at_goes_on_though_its_delay_ends_between_microseconds(monkeypatch): + """On a frozen clock, nothing else moves time: an advance to the rounded resumes_at must be enough.""" + bob = _Executor(monkeypatch) + bob.release() + engine, clock, _ = _engine() + engine.register(_reply_trigger(delay={"dist": "exp", "mean": "PT40M", "seed": 30})) + engine.on_tool_call("default", "send_text", {}, _result()) + await _until(lambda: _row(engine, "reply")["waits"]) + resumes_at = _row(engine, "reply")["resumes_at"] + assert resumes_at == "2026-03-02T00:30:58.882035Z" # the draw is 1858.882035... s + + clock.advance(resumes_at) + engine.clock_advanced() + await _until(lambda: _row(engine, "reply")["status"] == "armed") + assert len(bob.calls) == 1 From 641410eecdd946c56d52594a9f22be7c02c86ad2 Mon Sep 17 00:00:00 2001 From: Sarah Chitty Date: Mon, 9 Nov 2026 01:39:46 -0800 Subject: [PATCH 17/53] always_on: settle on contacts' real progress; one trigger's replies side by side; 300 s contact limit (FD-5367) - The settle wait follows the gateway's executor calls: a contact's reply counts as in progress until it lands or its time limit passes (plus 30 s); other trigger work gets settle_timeout_seconds from the start or the last contact call seen. A gateway that reports no executor calls gets the old fixed wait. - New settings contact_replies_in_parallel (default on) and contact_timeout_seconds (default 300 s, for contacts whose task sets none), applied through triggers/v1 config at the start and recorded under settings.contacts. Old gateways are recorded as not applied. - clock_jump and settle_timeout events say how long the wait took, how many contact calls it saw and which triggers were still at work; the end records contact replies still in progress (pending_at_end.contact_replies_in_progress). - person gains done_ends_run, for setup, which writes her prompt before any read. Co-Authored-By: Claude Opus 5.5 --- .../task_step/task_steps/always_on.py | 139 ++++++++++++++--- tst/unit/task_step/always_on_harness.py | 57 ++++++- tst/unit/task_step/test_always_on_settle.py | 144 ++++++++++++++++++ .../test_always_on_user_sim_inputs.py | 11 ++ 4 files changed, 328 insertions(+), 23 deletions(-) create mode 100644 tst/unit/task_step/test_always_on_settle.py diff --git a/src/agent_env/task_step/task_steps/always_on.py b/src/agent_env/task_step/task_steps/always_on.py index c45f90ff9..a8b283a31 100644 --- a/src/agent_env/task_step/task_steps/always_on.py +++ b/src/agent_env/task_step/task_steps/always_on.py @@ -21,8 +21,10 @@ user's message at their time. - The clock runs at the task's speed during turns (sync_env_clock). When nothing is due, the step advances it to the earliest of: the next trigger event, the next notification, a held reply, a pending read, or end_time, never past - one; then waits up to ``settle_timeout_seconds`` for triggers to finish. What comes due in the same minute wakes - the agent once. + one; then waits for the world to settle. A contact's reply (an AI-played contact, a trigger's executor) counts as in + progress until it ends or its time limit passes; other trigger work gets ``settle_timeout_seconds``. Contacts whose + task sets no time limit get ``contact_timeout_seconds``, and one trigger's replies run side by side + (``contact_replies_in_parallel``). What comes due in the same minute wakes the agent once. - The agent can schedule its own wake-ups (scheduled checks) when the run is multi-day, at least ``scheduled_check_min_interval`` apart; ``scheduled_checks`` turns them on or off whatever the run's length. A check's wake counts toward ``max_wakes``. @@ -91,9 +93,15 @@ CHECK_MIN_INTERVAL = "PT3H" EMPTY_REPLIES = frozenset({"(The assistant's reply was empty.)"}) # a harness's stand-in for an empty final reply TIME_LINE = "It's {when}. Any other date you see is the computer's clock, not the task's." +# The time limit of a contact (a trigger's executor) whose task sets none; the gateway's own default is 120 s. +CONTACT_TIMEOUT_SECONDS = 300.0 +# Past a contact's time limit, how long the settle wait still gives its call to come back. +CONTACT_GRACE_SECONDS = 30.0 # The run's metadata keeps a summary; the full timeline goes to the object store. When storing fails, the summary keeps # this many of the last events instead, without these fields. TIMELINE_TAIL = 20 +PENDING_KEPT = 20 +PENDING_TEXT_CHARS = 300 BULKY_EVENT_FIELDS = ("input", "raw", "read", "parts_text", "changes") @@ -109,7 +117,8 @@ def __init__(self, id: str, version: Optional[int], *, user_sim: str, end_time: stay_quiet: bool = True, nudge: bool = True, nudge_text: str = NUDGE_TEXT, wake_reminder: bool = True, wake_reminder_text: str = WAKE_REMINDER_TEXT, scheduled_checks: Optional[bool] = None, scheduled_check_min_interval: str = CHECK_MIN_INTERVAL, - model: Optional[str] = None, model_params: Optional[dict] = None, effort: Optional[str] = None, + contact_timeout_seconds: Optional[float] = CONTACT_TIMEOUT_SECONDS, + contact_replies_in_parallel: bool = True, model: Optional[str] = None, model_params: Optional[dict] = None, effort: Optional[str] = None, max_thinking_tokens: Optional[int] = None, harness: Optional[str] = None, trajectory_output_prefix: Optional[str] = None, **prompt_fields: Any): for unsupported in ("max_conversation_turns", "user_agent_name", "user_a2a_url", "post_turn_hooks", @@ -136,10 +145,14 @@ def __init__(self, id: str, version: Optional[int], *, user_sim: str, end_time: self.wake_reminder_text = wake_reminder_text self.scheduled_checks = scheduled_checks self.scheduled_check_min_interval = scheduled_check_min_interval + if contact_timeout_seconds is not None and not float(contact_timeout_seconds) > 0: + raise ValueError(f"contact_timeout_seconds must be positive (got {contact_timeout_seconds!r})") + self.contact_timeout_seconds = None if contact_timeout_seconds is None else float(contact_timeout_seconds) + self.contact_replies_in_parallel = bool(contact_replies_in_parallel) _OWN = ("user_sim", "end_time", "final", "max_wakes", "settle_timeout_seconds", "user_timeout_seconds", "final_reminder", "stay_quiet", "nudge", "nudge_text", "wake_reminder", "wake_reminder_text", - "scheduled_checks", "scheduled_check_min_interval") + "scheduled_checks", "scheduled_check_min_interval", "contact_timeout_seconds", "contact_replies_in_parallel") _PROMPT_FIELDS = ("prompt", "parts", "prompt_id", "agent_name", "timeout_seconds", "trajectory_output_prefix", "model", "model_params", "system_prompt", "max_turns", "output_format", "max_thinking_tokens", "effort", "harness", "agentenv_tools", "poll_interval_seconds", "context_id") @@ -203,6 +216,8 @@ def __init__(self, step: AlwaysOnTaskStep, context: TaskStepContext): self.multi_day = False self.checks: dict = {"on": False} self.signed_off = False # a one-sitting run's user said done: no more reads + self.contacts: dict = {} + self.last_settle: dict = {} def close(self) -> None: for p in self._plugin_paths: @@ -255,12 +270,14 @@ async def go(self) -> None: t0 = await self._t0() or now self.end = _end_instant(step.end_time, t0) if step.end_time else None self.multi_day = self.end is not None and self.end.astimezone(self.tz).date() > t0.astimezone(self.tz).date() + self.person["done_ends_run"] = not self.multi_day # for setup, which writes her prompt before any read chat_config = {"roles": [self.recipient], "user_name": self.person.get("name") or "your user"} if step.stay_quiet: chat_config["stay_quiet"] = True for env in self.chat_envs: await _call(env, EXT_USER_CHAT_URI, "config", chat_config) await self._configure_checks() + await self._configure_contacts() await self._configure_agents() self.conversation_id = uuid.uuid4().hex @@ -353,6 +370,26 @@ async def _configure_agents(self) -> None: if "system_prompt" not in accepted and setup.get("system_prompt"): logger.warning("always_on: the user sim agent's card did not accept system_prompt; it keeps its deploy one") + async def _configure_contacts(self) -> None: + """On each env with triggers, let one trigger's replies run side by side (``contact_replies_in_parallel``) and + give contacts that set no time limit ``contact_timeout_seconds``; a gateway too old for it keeps its ways.""" + step = self.step + body = {"concurrent_fires": step.contact_replies_in_parallel} + if step.contact_timeout_seconds is not None: + body["executor_timeout_seconds"] = step.contact_timeout_seconds + applied = [] + for env in self.ctx.deployed_envs: + if not env.supports(EXT_TRIGGERS_URI, "state"): + continue + if not env.supports(EXT_TRIGGERS_URI, "config"): + applied.append({"env_id": env.env_id, "applied": False, "why": "the gateway has no triggers config"}) + continue + got = await _call(env, EXT_TRIGGERS_URI, "config", body) + applied.append({"env_id": env.env_id, "applied": True, "concurrent_fires": got.get("concurrent_fires"), + "executor_timeout_seconds": got.get("executor_timeout_seconds")}) + self.contacts = {"replies_in_parallel": step.contact_replies_in_parallel, + "timeout_seconds": step.contact_timeout_seconds, "envs": applied} + def rules(self) -> str: """What the agent is told about reaching the user, and about waking itself when it can.""" return " ".join([MESSAGE_RULES, *([QUIET_RULE] if self.step.stay_quiet else []), @@ -432,30 +469,49 @@ async def advance(self, target: datetime, reason: str) -> None: raise stuck = await self.settle() self.log("clock_jump", **{"from": _iso(before), "to": _iso(target), "reason": reason, - "settled": not stuck, "still_firing": stuck}) - - async def settle(self) -> list[str]: - """Wait until no trigger still fires (bar one waiting for a later time); the ids of envs still firing.""" - deadline = time.monotonic() + self.step.settle_timeout_seconds + "settled": not stuck, "still_at_work": stuck, "settle": self.last_settle}) + + async def settle(self) -> list[dict]: + """Wait until the world has settled: no trigger still at work, bar a fire waiting for a later time. A contact's + reply (a call to a trigger's executor) counts as in progress until it ends or its time limit passes (plus + ``CONTACT_GRACE_SECONDS``); any other work gets ``settle_timeout_seconds``, from the start or from the last + contact call seen. A gateway that reports no executor calls gets ``settle_timeout_seconds`` in all. Returns, by + env, the triggers still at work when it gave up (empty when the world settled); ``last_settle`` says how long + it waited and how many contact calls it saw.""" + started = time.monotonic() envs = [e for e in await self._armed() if e.supports(EXT_TRIGGERS_URI, "state")] - waiting = set(range(len(envs))) - while waiting and time.monotonic() < deadline: + waiting = {i: {"live_call_seen": started, "contact_until": started} for i in range(len(envs))} + stuck: list[dict] = [] + calls_seen = 0 + while waiting: for i in sorted(waiting): - env = envs[i] + env, track = envs[i], waiting[i] clock = await _call(env, EXT_CLOCK_URI, "state") state = await _call(env, EXT_TRIGGERS_URI, "state") - now = _utc(clock["virtual_time"]) + now, mono = _utc(clock["virtual_time"]), time.monotonic() + rows = state.get("triggers", []) + live = [c for row in rows for c in row.get("executor_calls") or [] if c.get("seconds_left", 0) > 0] + if live: + calls_seen = max(calls_seen, len(live)) + track["live_call_seen"] = mono + track["contact_until"] = max(track["contact_until"], mono + CONTACT_GRACE_SECONDS + + max(c["seconds_left"] for c in live)) + busy = [row for row in rows if _busy(row, now)] if state.get("next_event_at") is not None and _utc(state["next_event_at"]) <= now: try: await _call(env, EXT_CLOCK_URI, "advance", {"virtual_time": _iso(now)}, attempts=1) except Exception: pass - elif not any(_busy(row, now) for row in state.get("triggers", [])): - waiting.discard(i) + elif not busy: + waiting.pop(i) + elif mono >= max(track["contact_until"], track["live_call_seen"] + self.step.settle_timeout_seconds): + stuck.append({"env_id": env.env_id, "triggers": [_at_work(row) for row in busy]}) + waiting.pop(i) if waiting: self.ctx.heartbeat() await asyncio.sleep(POLL_SECONDS) - return sorted(envs[i].env_id for i in waiting) + self.last_settle = {"seconds": round(time.monotonic() - started, 1), "contact_calls_seen": calls_seen} + return stuck async def _next_world_event(self) -> Optional[datetime]: out = [] @@ -736,7 +792,7 @@ def record() -> None: await self._nudge(plain) stuck = await self.settle() if stuck: - self.log("settle_timeout", envs=stuck) + self.log("settle_timeout", wake=self.wakes, still_at_work=stuck, settle=self.last_settle) self._schedule_read() async def _nudge(self, closing: str) -> None: @@ -901,11 +957,30 @@ async def _ask_user_sim(self, text: str) -> dict: "trajectory_url": traj} # ---- the end -------------------------------------------------------------------------------------------- + async def _contacts_at_work(self) -> list[dict]: + """By env, the triggers still at work now: contact replies in progress or waiting for a later time.""" + out = [] + for env in self.ctx.deployed_envs: + if not env.supports(EXT_TRIGGERS_URI, "state"): + continue + try: + state = await _call(env, EXT_TRIGGERS_URI, "state", attempts=2) + except Exception as e: + out.append({"env_id": env.env_id, "error": f"{type(e).__name__}: {str(e)[:200]}"}) + continue + rows = [_at_work(row) for row in state.get("triggers", []) + if row.get("status") in IN_FLIGHT or row.get("running") or row.get("executor_calls") + or row.get("pending")] + if rows: + out.append({"env_id": env.env_id, "triggers": rows}) + return out + async def finish(self) -> None: step, ctx = self.step, self.ctx now = await self.now() + self.pending = {"contact_replies_in_progress": await self._contacts_at_work()} self.log("end", reason=self.end_reason, virtual_time=_iso(now), wakes=self.wakes, replies=self.replies, - unread_at_end=len(self.unread), held_at_end=len(self.held)) + unread_at_end=len(self.unread), held_at_end=len(self.held), pending=self.pending) conv = conversation_store.get_conversation(self.conversation_id) if conv and conv.get("status") == "active": conversation_store.mark_closed(self.conversation_id) @@ -936,9 +1011,9 @@ async def finish(self) -> None: "agent_name": self.us.get("agent_name"), **self.user_setup}, "end_time": _iso(self.end) if self.end else None, "time_zone": self.zone, "multi_day": self.multi_day, "real_seconds": round(time.time() - self.t0_real, 1), - "timeline_events": len(self.timeline)} + "timeline_events": len(self.timeline), "pending_at_end": _bounded(self.pending)} record["settings"] = {"stay_quiet": step.stay_quiet, "nudge": step.nudge, "wake_reminder": step.wake_reminder, - "scheduled_checks": self.checks} + "scheduled_checks": self.checks, "contacts": self.contacts} await self._store_timeline(record) ctx.metadata.setdefault(ALWAYS_ON_KEY, {})[step.id] = record final_text = "\n\n".join([m["text"] for m in self.last_messages] + ([self.last_plain] if self.last_plain else [])) @@ -1022,13 +1097,35 @@ def _closing_text(plain: str) -> str: def _busy(row: dict, now: datetime) -> bool: - if row.get("evaluating"): + """Whether a trigger is still at work at ``now``: evaluating, calling an executor, or with a fire in flight that + isn't waiting for a later time.""" + if row.get("evaluating") or row.get("executor_calls"): return True + if "running" in row: # a gateway that reports each fire in flight and when each waiting one goes on + later = sum(1 for w in row.get("waits") or [] if w.get("resumes_at") and _utc(w["resumes_at"]) > now) + return row["running"] > later if row.get("status") not in IN_FLIGHT and not row.get("pending"): return False return not (row.get("waiting") and row.get("resumes_at") and _utc(row["resumes_at"]) > now) +def _bounded(pending: dict) -> dict: + """``pending`` as the run's metadata keeps it: each list's count, and its first ``PENDING_KEPT`` items with their + texts cut to ``PENDING_TEXT_CHARS``; the stored timeline's end event has them all.""" + out = {} + for kind, items in pending.items(): + kept = [{k: (v[:PENDING_TEXT_CHARS] + "…" if isinstance(v, str) and len(v) > PENDING_TEXT_CHARS else v) + for k, v in item.items()} for item in items[:PENDING_KEPT]] + out[kind] = {"count": len(items), "items": kept} + return out + + +def _at_work(row: dict) -> dict: + """A trigger still at work, as the run's record keeps it.""" + return {k: row.get(k) for k in ("id", "status", "running", "pending", "executor_calls", "waits", "resumes_at") + if row.get(k) not in (None, [], 0)} + + def _hm(text: str) -> tuple[int, int]: h, m = text.split(":") return int(h), int(m) diff --git a/tst/unit/task_step/always_on_harness.py b/tst/unit/task_step/always_on_harness.py index d511ae83a..676ee33bb 100644 --- a/tst/unit/task_step/always_on_harness.py +++ b/tst/unit/task_step/always_on_harness.py @@ -4,16 +4,19 @@ from __future__ import annotations +import asyncio import json from typing import Any, Callable, Optional import httpx +from mcp.types import CallToolResult, TextContent, Tool from agent_env.config import get_config from agent_env.env.env import DeployedGatewayEnv from agent_env.env.gateway import gateway as gw_mod from agent_env.env.gateway.constants import AGENT_ENV_ROLE_HEADER, GATEWAY_EXTENSIONS, WELL_KNOWN_PATH from agent_env.env.gateway.gateway import Gateway +from agent_env.env.gateway.triggers import TriggerEngine from agent_env.task_step.context import DeployedAgent, TaskStepContext from agent_env.task_step.task_steps import always_on as ao from agent_env.task_step.task_steps.user_sim import USER_SIMS_KEY @@ -56,6 +59,9 @@ def __init__(self, monkeypatch, tmp_path, *, agent: Optional[Turn] = None, answe self.plugin = plugin self.user_texts = list(user_texts) self.user_calls: list[dict] = [] + self.contact_calls: list[dict] = [] + self._contacts: dict[str, dict] = {} + self._monkeypatch = monkeypatch monkeypatch.setattr(ao, "POLL_SECONDS", 0) monkeypatch.setattr(ao, "send_and_wait", self._send_and_wait) monkeypatch.setattr(ao._Run, "_load_plugin", self._load_plugin) @@ -68,8 +74,55 @@ async def arm(self) -> None: "virtual_seconds_per_real_second": self.rate}) r.raise_for_status() - def triggers(self, *triggers: dict) -> None: - self.gw._trigger_engine.register({"watch_roles": [ROLE], "triggers": list(triggers)}) + def triggers(self, *triggers: dict, executors: Optional[dict] = None) -> None: + self.gw._trigger_engine.register({"watch_roles": [ROLE], "triggers": list(triggers), + **({"executors": executors} if executors else {})}) + + def contact(self, name: str, *, seconds: float, reply: Callable[["Rig", int], None], + timeout_seconds: Optional[float] = None, keeps_to_its_limit: bool = True) -> dict: + """An AI-played contact (a trigger's executor) whose every reply takes ``seconds`` of real time, then acts with + ``reply(rig, n)`` for its nth reply. Past its time limit the call fails, though the reply still lands, as a + real contact's does; one that doesn't ``keep_to_its_limit`` holds its call open until it is done. Returns its + executor spec, for ``triggers(executors={name: spec})``.""" + url = f"http://{name}.test" + self._contacts[url] = {"name": name, "seconds": seconds, "reply": reply, "keeps": keeps_to_its_limit, + "replies": 0} + rig = self + + async def call(engine, executor, instruction, context_id): + contact = rig._contacts[executor["a2a_url"]] + rig.contact_calls.append({"contact": contact["name"], "timeout_seconds": executor["timeout_seconds"], + "context_id": context_id, "at": rig.now()}) + contact["replies"] += 1 + n, limit = contact["replies"], executor["timeout_seconds"] + if contact["keeps"] and contact["seconds"] > limit: + await asyncio.sleep(limit) + asyncio.get_running_loop().call_later(contact["seconds"] - limit, contact["reply"], rig, n) + return None + await asyncio.sleep(contact["seconds"]) + contact["reply"](rig, n) + return "replied" + + self._monkeypatch.setattr(TriggerEngine, "_call_executor", call) + return {"a2a_url": url, "role": f"contact-{name}", + **({"timeout_seconds": timeout_seconds} if timeout_seconds is not None else {})} + + def alert(self, text: str, *, channel: str = "Gmail") -> dict: + """Something reaching the agent the way a contact's email alert does.""" + return self.gw._notifications.notify(channel, text, to=ROLE) + + def app_tools(self, *names: str) -> None: + """App tools the gateway knows of, so a trigger may watch them; nothing serves them.""" + self.gw._server_tools.setdefault("apps", []).extend(Tool(name=n, inputSchema={"type": "object"}) for n in names) + + def provoke(self, tool: str, args: Optional[dict] = None) -> None: + """The agent's tool call as the gateway's triggers see it.""" + self.gw._trigger_engine.on_tool_call(ROLE, tool, args or {}, + CallToolResult(content=[TextContent(type="text", text="{}")])) + + async def close(self) -> None: + """Cancel what the gateway's triggers still have running.""" + await self.gw._trigger_engine.cancel_scheduled() def notify_at(self, trigger_id: str, at: str, text: str, *, channel: str = "Messages", to: str = ROLE) -> dict: return {"id": trigger_id, "when": {"type": "time", "at": at}, diff --git a/tst/unit/task_step/test_always_on_settle.py b/tst/unit/task_step/test_always_on_settle.py new file mode 100644 index 000000000..b55b3a11d --- /dev/null +++ b/tst/unit/task_step/test_always_on_settle.py @@ -0,0 +1,144 @@ +"""The always_on step waits for the world to settle by following AI-played contacts' real progress: a contact's reply +is in progress until it lands or its time limit passes, one trigger's replies run side by side, contacts that set no +time limit get 300 s, and replies still in progress at the end are recorded.""" + +from __future__ import annotations + +import random + +import pytest + +from agent_env.task_step.task_steps import always_on as ao +from tst.unit.task_step.always_on_harness import Rig + +pytestmark = pytest.mark.usefixtures("local_stores") + + +def _estimate(rig, n): + rig.alert(f"Live Oak: estimate {n} attached, $2,400", channel="Gmail · Live Oak Arbor") + + +def _ask_live_oak(at: str) -> dict: + return {"id": "liveoak-estimate", "when": {"type": "time", "at": at}, + "actions": [{"type": "nl", "executor": "liveoak", "instruction": "send Maya your estimate"}]} + + +@pytest.mark.asyncio +async def test_a_contact_slower_than_a_fixed_wait_still_wakes_the_agent_with_its_reply(monkeypatch, tmp_path): + """Scaled down: settle_timeout_seconds (0.2 s) stands in for the old fixed wait and the old 30 s cut-off; the + contact takes 0.6 s, well inside its own 5 s limit.""" + rig = Rig(monkeypatch, tmp_path) + await rig.arm() + liveoak = rig.contact("liveoak", seconds=0.6, reply=_estimate, timeout_seconds=5) + rig.triggers(_ask_live_oak("2026-11-09T15:00:00Z"), executors={"liveoak": liveoak}) + ctx = await rig.run(rig.step(settle_timeout_seconds=0.2)) + + wakes = rig.events(ctx, "wake") + assert [w["virtual_time"] for w in wakes][:2] == ["2026-11-09T14:00:00Z", "2026-11-09T15:00:00Z"] + assert "Live Oak: estimate 1 attached, $2,400" in wakes[1]["parts_text"] + [jump] = [j for j in rig.events(ctx, "clock_jump") if j["to"] == "2026-11-09T15:00:00Z"] + assert jump["settled"] and jump["settle"]["seconds"] >= 0.5 and jump["settle"]["contact_calls_seen"] == 1 + assert rig.events(ctx, "settle_timeout") == [] + + +@pytest.mark.asyncio +async def test_the_wait_stops_once_a_contact_overruns_its_limit_and_says_what_was_still_at_work(monkeypatch, tmp_path): + monkeypatch.setattr(ao, "CONTACT_GRACE_SECONDS", 0.2) + rig = Rig(monkeypatch, tmp_path) + await rig.arm() + stuck = rig.contact("liveoak", seconds=5, reply=_estimate, timeout_seconds=0.3, keeps_to_its_limit=False) + rig.triggers(_ask_live_oak("2026-11-09T15:00:00Z"), executors={"liveoak": stuck}) + try: + ctx = await rig.run(rig.step(settle_timeout_seconds=0.2)) + finally: + await rig.close() + + [jump] = [j for j in rig.events(ctx, "clock_jump") if j["to"] == "2026-11-09T15:00:00Z"] + assert not jump["settled"] and 0.4 <= jump["settle"]["seconds"] < 2 + [[trigger]] = [env["triggers"] for env in jump["still_at_work"]] + assert trigger["id"] == "liveoak-estimate" and trigger["executor_calls"][0]["executor"] == "liveoak" + + +def _crew_texted(rig, n): + rig.alert(f"Sam: reply {n}", channel="Messages · noor 30") + + +CREW_DM = {"id": "crew-dm", "when": {"type": "action", "tool": "messaging_send_message", "repeat": True}, + "actions": [{"type": "nl", "executor": "crew", "instruction": "answer Linda's newest text", + "delay": {"dist": "exp", "mean": "PT40M", "seed": 30}}]} + + +@pytest.mark.parametrize("parallel", [True, False], ids=["side by side", "one at a time"]) +@pytest.mark.asyncio +async def test_one_trigger_s_replies_run_side_by_side_so_a_quick_one_lands_on_time(monkeypatch, tmp_path, parallel): + draws = random.Random(30) + slow, quick = (round(draws.expovariate(1 / 2400)) for _ in range(2)) # 1859 s, then 819 s + assert quick < slow + + def texts_two_friends(n, texts, rig): + if n == 1: + rig.provoke("messaging_send_message", {"user_id": "+13475550167"}) + rig.provoke("messaging_send_message", {"user_id": "+16465550123"}) + rig.quiet() + return "" + + rig = Rig(monkeypatch, tmp_path, agent=texts_two_friends) + await rig.arm() + rig.app_tools("messaging_send_message") + rig.triggers(CREW_DM, executors={"crew": rig.contact("crew", seconds=0, reply=_crew_texted)}) + ctx = await rig.run(rig.step(contact_replies_in_parallel=parallel)) + + woke = [(w["virtual_time"][:19], w["parts_text"].count("Sam: reply")) for w in rig.events(ctx, "wake")[1:]] + if parallel: + assert woke == [("2026-11-09T14:13:39", 1), ("2026-11-09T14:30:58", 1)] + else: + assert woke == [("2026-11-09T14:30:58", 2)] + [env] = rig.record(ctx)["settings"]["contacts"]["envs"] + assert env == {"env_id": "world", "applied": True, "concurrent_fires": parallel, + "executor_timeout_seconds": 300.0} + + +@pytest.mark.asyncio +async def test_a_contact_that_sets_no_limit_gets_300_s_and_one_that_does_keeps_its_own(monkeypatch, tmp_path): + rig = Rig(monkeypatch, tmp_path) + await rig.arm() + rig.triggers(_ask_live_oak("2026-11-09T15:00:00Z"), + {**_ask_live_oak("2026-11-09T16:00:00Z"), "id": "northwick-quote", + "actions": [{"type": "nl", "executor": "northwick", "instruction": "quote Tess"}]}, + executors={"liveoak": rig.contact("liveoak", seconds=0, reply=_estimate), + "northwick": rig.contact("northwick", seconds=0, reply=_estimate, timeout_seconds=120)}) + ctx = await rig.run(rig.step()) + + assert [(c["contact"], c["timeout_seconds"]) for c in rig.contact_calls] == [("liveoak", 300), ("northwick", 120)] + assert rig.record(ctx)["settings"]["contacts"]["timeout_seconds"] == ao.CONTACT_TIMEOUT_SECONDS == 300 + + +@pytest.mark.asyncio +async def test_contact_replies_still_in_progress_at_the_end_are_recorded(monkeypatch, tmp_path): + def texts_a_friend(n, texts, rig): + rig.provoke("messaging_send_message", {"user_id": "+13475550167"}) + rig.quiet() + return "" + + rig = Rig(monkeypatch, tmp_path, agent=texts_a_friend) + await rig.arm() + rig.app_tools("messaging_send_message") + rig.triggers(CREW_DM, executors={"crew": rig.contact("crew", seconds=0, reply=_crew_texted)}) + try: + ctx = await rig.run(rig.step(max_wakes=1)) + finally: + await rig.close() + + [env] = rig.record(ctx)["pending_at_end"]["contact_replies_in_progress"]["items"] + [trigger] = env["triggers"] + assert trigger["id"] == "crew-dm" and trigger["running"] == 1 + assert trigger["waits"] == [{"resumes_at": "2026-11-09T14:30:58.882035Z"}] + + +def test_the_contact_settings_survive_a_round_trip_and_a_bad_limit_is_refused(): + step = ao.AlwaysOnTaskStep("week", None, user_sim="linda", prompt="hi", prompt_id="p", agent_name="a", + contact_timeout_seconds=600, contact_replies_in_parallel=False) + again = ao.AlwaysOnTaskStep.from_dict(step.to_dict()) + assert (again.contact_timeout_seconds, again.contact_replies_in_parallel) == (600.0, False) + with pytest.raises(ValueError, match="contact_timeout_seconds"): + ao.AlwaysOnTaskStep("week", None, user_sim="linda", prompt="hi", agent_name="a", contact_timeout_seconds=0) diff --git a/tst/unit/task_step/test_always_on_user_sim_inputs.py b/tst/unit/task_step/test_always_on_user_sim_inputs.py index 17ab13c1e..799953488 100644 --- a/tst/unit/task_step/test_always_on_user_sim_inputs.py +++ b/tst/unit/task_step/test_always_on_user_sim_inputs.py @@ -250,3 +250,14 @@ async def test_done_ends_run_says_whether_done_would_end_the_run(monkeypatch, tm await rig.run(rig.step(**({"end_time": end_time} if end_time else {}))) assert [r["done_ends_run"] for r in rig.reads] == [ends] + + +@pytest.mark.parametrize("end_time,ends", [(None, True), ("P3D", False)], ids=["one sitting", "multi-day"]) +@pytest.mark.asyncio +async def test_setup_s_person_says_whether_done_ends_the_run(monkeypatch, tmp_path, end_time, ends): + rig = Rig(monkeypatch, tmp_path) + await rig.arm() + await rig.run(rig.step(**({"end_time": end_time} if end_time else {}))) + + [person] = rig.setups + assert person["done_ends_run"] is ends From ff4ea7448de4d6db8009373b51a57bdc7393a427 Mon Sep 17 00:00:00 2001 From: Sarah Chitty Date: Mon, 9 Nov 2026 01:43:29 -0800 Subject: [PATCH 18/53] always_on: clean endings - a final check-in always, a last-turn notice, her last read, pending items, finished vs stuck (FD-5369) - end_time always brings a final check-in: the task's final line, else the step's own line (FINAL_DEFAULT_TEXT), which is not the user's and never reaches her you_sent. - The last-turn notice (final_reminder) goes on the run's last wake: at end_time, and on the wake that max_wakes makes the last. Wake events say last. - After the last wake the user sim reads the agent's wrap-up (user_sim_read with last_read); her answer is recorded and a reply is held as pending, never delivered. Not after a one-sitting done, a give-up or an error. - A reply still held at end_time is kept (it was dropped) and recorded. - pending_at_end: held_replies, unread_by_user (with the wake that sent each), user_lines_not_delivered, alerts_never_delivered, scheduled_checks_not_due, contact_replies_in_progress. The summary keeps counts and the first 20 of each; the stored end event has all. - nothing_can_wake_the_agent becomes finished_nothing_pending or stuck, from what is pending. The record and the end event carry end_note, a plain sentence for each end reason. Co-Authored-By: Claude Opus 5.5 --- .../task_step/task_steps/always_on.py | 145 ++++++++--- tst/unit/task_step/test_always_on_endings.py | 225 ++++++++++++++++++ tst/unit/task_step/test_always_on_loop.py | 2 +- 3 files changed, 341 insertions(+), 31 deletions(-) create mode 100644 tst/unit/task_step/test_always_on_endings.py diff --git a/src/agent_env/task_step/task_steps/always_on.py b/src/agent_env/task_step/task_steps/always_on.py index a8b283a31..e318d45cf 100644 --- a/src/agent_env/task_step/task_steps/always_on.py +++ b/src/agent_env/task_step/task_steps/always_on.py @@ -28,8 +28,14 @@ - The agent can schedule its own wake-ups (scheduled checks) when the run is multi-day, at least ``scheduled_check_min_interval`` apart; ``scheduled_checks`` turns them on or off whatever the run's length. A check's wake counts toward ``max_wakes``. -- At end_time the user's ``final`` line (with a reminder to use message_user) gives the agent one last turn. The run - ends on end_time, done (one sitting), max_wakes, or nothing left to wake the agent; the reason is recorded. +- At end_time there is always a final check-in: the user's ``final`` line, or the step's own (``FINAL_DEFAULT_TEXT``) + when the task sets none. The run's last wake, at end_time or the one ``max_wakes`` makes the last, carries the + last-turn notice (``final_reminder``). After it the user sim reads what the agent sent (its wrap-up), and her + reaction, and any reply still held, are recorded; then the run ends. +- End reasons: end_time, user_done (one sitting), max_wakes, time_budget, finished_nothing_pending or stuck (nothing + can wake the agent, with nothing pending or with something still pending), agent_turn_failed, agent_turn_timeout, + user_sim_error, step_error. What was still pending is recorded (``pending_at_end``): replies she held, messages she + never read, user lines and alerts never delivered, the agent's checks not yet due, contact replies in progress. - The run's metadata keeps a summary in ``metadata["always_on"][step_id]`` (small enough for a running job to report back); the full timeline goes to the object store, at the summary's ``object_url``. """ @@ -97,6 +103,23 @@ CONTACT_TIMEOUT_SECONDS = 300.0 # Past a contact's time limit, how long the settle wait still gives its call to come back. CONTACT_GRACE_SECONDS = 30.0 +# The final check-in's line when the task sets no `final`: the step's, not the user's. +FINAL_DEFAULT_TEXT = "Time's up on this job. Send {first} a short wrap-up: what's done, and anything still open." +NOTHING_CAN_WAKE = "nothing_can_wake_the_agent" # until finish() tells finished_nothing_pending from stuck +ERROR_ENDS = ("agent_turn_failed", "agent_turn_timeout", "user_sim_error", "step_error") +# What each end reason means, for the end marker in the graded conversation; {wakes} and {first} are filled in. +END_NOTES = { + "end_time": "The run reached its end time.", + "user_done": "{first} signed off.", + "max_wakes": "The run was cut off at its cap of {wakes} wakes.", + "time_budget": "The run was cut off by its real-time budget.", + "finished_nothing_pending": "Nothing could wake the assistant again, and nothing was pending.", + "stuck": "Nothing could wake the assistant again, though something was still pending.", + "agent_turn_failed": "The assistant's turn failed.", + "agent_turn_timeout": "The assistant's turn timed out.", + "user_sim_error": "The user simulator failed.", + "step_error": "The run stopped on an error.", +} # The run's metadata keeps a summary; the full timeline goes to the object store. When storing fails, the summary keeps # this many of the last events instead, without these fields. TIMELINE_TAIL = 20 @@ -218,6 +241,8 @@ def __init__(self, step: AlwaysOnTaskStep, context: TaskStepContext): self.signed_off = False # a one-sitting run's user said done: no more reads self.contacts: dict = {} self.last_settle: dict = {} + self.pending: dict = {} + self.end_note = "" def close(self) -> None: for p in self._plugin_paths: @@ -627,7 +652,7 @@ async def loop(self, now: datetime) -> None: lines = await self._take_user_says() replies = [h for h in self.held if h["deliver_at"] <= now] self.held = [h for h in self.held if h["deliver_at"] > now] - await self._wake_with(now, lines, replies, final=False) + await self._wake_with(now, lines, replies) continue # 3. nothing due: jump to the next thing later = await self._later(now) @@ -636,10 +661,11 @@ async def loop(self, now: datetime) -> None: await self._final(now) break if soonest is None: - self.end_reason = "nothing_can_wake_the_agent" + self.end_reason = NOTHING_CAN_WAKE # finished or stuck: finish() decides from what is pending break why = sorted(k for k, v in later.items() if v == soonest) await self.advance(soonest, reason="+".join(why)) + await self._last_read() async def _later(self, now: datetime) -> dict[str, datetime]: """What could wake the agent after ``now``, by kind, each at its earliest time.""" @@ -650,21 +676,36 @@ async def _later(self, now: datetime) -> dict[str, datetime]: "user_read": self.read_at if self.unread else None} return {k: v for k, v in candidates.items() if v is not None and v > now} - async def _final(self, now: datetime) -> None: + async def _final(self, now: datetime, reason: str = "end_time") -> None: + """The final check-in: at end_time, the user's ``final`` line, else ``FINAL_DEFAULT_TEXT`` (a line from the + step, not the user); with whatever else is due, and the last-turn notice. A reply held past now stays held, + to be recorded as pending.""" step = self.step - if self.end > now: + if reason == "end_time" and self.end > now: await self.advance(self.end, reason="end_time") now = await self.now() - if not step.final: - self.end_reason = "end_time" - return self.final_sent = True lines = await self._take_user_says() replies = [h for h in self.held if h["deliver_at"] <= now] - self.held = [] - lines.append({"at": _iso(now), "text": step.final, "source": "final"}) - await self._wake_with(now, lines, replies, final=True) - self.end_reason = self.end_reason or "end_time" + self.held = [h for h in self.held if h["deliver_at"] > now] + notes = [] + if reason == "end_time" and step.final: + lines.append({"at": _iso(now), "text": step.final, "source": "final"}) + else: + notes.append(_step_part(FINAL_DEFAULT_TEXT.format(first=self.person["first"]), "final")) + await self._wake_with(now, lines, replies, last=True, notes=notes) + self.end_reason = self.end_reason or reason + + async def _last_read(self) -> None: + """After the agent's last turn, the user sim reads what it sent since her last read, at the time she would + (the world's clock stays where the run ended), so her reaction to its wrap-up is recorded; a reply she gives + is held and recorded as pending, never delivered. Not after done ends a one-sitting run, after she gave up, or + when the run ended on an error.""" + if not self.unread or self.signed_off or self.gave_up or self.end_reason in ERROR_ENDS: + return + self._schedule_read() + at = self.read_at or await self.now() + await self.read(at, last=True) def _effective_prompt(self) -> str: parts = self.step._apply_seed(self.step._effective_parts(), self.ctx.metadata.get("seed", {})) @@ -684,7 +725,10 @@ async def _take_user_says(self) -> list[dict]: out.append({"at": it.get("at"), "text": text, "source": "user_says", "trigger_id": it.get("source_id")}) return out - async def _wake_with(self, now: datetime, lines: list[dict], replies: list[dict], final: bool) -> None: + async def _wake_with(self, now: datetime, lines: list[dict], replies: list[dict], last: bool = False, + notes: Optional[list[dict]] = None) -> None: + """Wake the agent with what came due for it, then the user's lines and replies in time order, then ``notes`` + (the step's own lines); ``last`` marks the run's last wake.""" due = await take_due(self.notification_envs, self.recipient) if self.notification_envs else [] parts = delivered_parts(due) user_msgs = sorted([{"at": l["at"] or _iso(now), "text": l["text"], "kind": l.get("source", "user_says"), @@ -700,16 +744,19 @@ async def _wake_with(self, now: datetime, lines: list[dict], replies: list[dict] trigger_id=m.get("trigger_id")) else: self.log("user_reply_delivered", virtual_time=m["at"], text=m["text"], done=m["done"]) - if final and self.step.final_reminder: - parts.append(_step_part(self.step.final_reminder.format(first=self.person["first"]), "final_reminder")) + parts += notes or [] woke_by = [f"notification:{d['channel']}" for d in due] + [f"{m['kind']}" for m in user_msgs] + woke_by += [f"step:{n['metadata']['agentenv']['source']}" for n in notes or []] await self.wake(parts, woke_by=woke_by, notifications=[{k: d.get(k) for k in ("at", "channel", "source", "source_id", "to")} - for d in due]) + for d in due], last=last) if self.signed_off and any(m.get("done") for m in user_msgs): self.end_reason = self.end_reason or "user_done" - async def wake(self, parts: list[dict], woke_by: list[str], notifications: Optional[list] = None) -> None: + async def wake(self, parts: list[dict], woke_by: list[str], notifications: Optional[list] = None, + last: bool = False) -> None: + """One wake: the agent's turn on ``parts``, then what it sent the user, then the settle wait. The wake that + ``last`` marks, or the one ``max_wakes`` makes the last, carries the last-turn notice (``final_reminder``).""" step = self.step sent_at = await self.now() clock = await step._env_clock(self.clock_envs) @@ -717,6 +764,9 @@ async def wake(self, parts: list[dict], woke_by: list[str], notifications: Optio if self.wakes == 0 and not step.system_prompt: lead = [_step_part(self.rules(), "rules"), *lead] parts = [*lead, *parts] + last = last or self.wakes + 1 >= step.max_wakes + if last and step.final_reminder: + parts.append(_step_part(step.final_reminder.format(first=self.person["first"]), "final_reminder")) if step.wake_reminder: parts = [*parts, _step_part(step.wake_reminder_text, "reminder")] self.wakes += 1 @@ -728,7 +778,7 @@ def record() -> None: self.per_turn_parts.append(list(parts)) self.log("wake", n=self.wakes, virtual_time=_iso(sent_at), woke_by=woke_by, notifications=notifications or [], - parts_text="\n".join(p.get("text", "") for p in parts if p.get("kind") == "text")) + last=last, parts_text="\n".join(p.get("text", "") for p in parts if p.get("kind") == "text")) t = time.time() try: task_id, result = await send_and_wait(self.target_url, parts, agent=self.agent, message_id=msg_id, @@ -782,7 +832,7 @@ def record() -> None: trajectory_url=traj, **extra) for m in messages: self.log("message_to_user", virtual_time=m.get("at"), text=m["text"], wake=self.wakes) - self.unread.append({"at": m.get("at"), "text": m["text"]}) + self.unread.append({"at": m.get("at"), "text": m["text"], "wake": self.wakes}) self.all_messages.append({"at": m.get("at"), "text": m["text"], "wake": self.wakes}) if state == TaskState.failed or str(state) == "failed": self.error = term.error_message or term.response_text or "agent turn failed" @@ -862,19 +912,22 @@ def record() -> None: self.last_plain, self.last_messages = plain, messages for m in messages: self.log("message_to_user", virtual_time=m.get("at"), text=m["text"], wake=self.wakes, after_nudge=True) - self.unread.append({"at": m.get("at"), "text": m["text"]}) + self.unread.append({"at": m.get("at"), "text": m["text"], "wake": self.wakes}) self.all_messages.append({"at": m.get("at"), "text": m["text"], "wake": self.wakes}) if state == TaskState.failed or str(state) == "failed": self.error = term.error_message or term.response_text or "agent nudge turn failed" self.end_reason = "agent_turn_failed" - async def read(self, now: datetime) -> None: + async def read(self, now: datetime, last: bool = False) -> None: + """The user sim reads what the agent sent since her last read, at ``now``; ``last`` marks the read after the + run's last wake, whose answer can no longer change how the run ends.""" phone = [] if self.notification_envs: for it in await take_due(self.notification_envs, USER_PHONE): phone.append({"at": it.get("at"), "channel": it["channel"], "text": "\n\n".join(p["text"] for p in it["parts"] if p.get("kind") == "text")}) - read = {"time": _iso(now), "assistant_messages": list(self.unread), "phone": phone, + read = {"time": _iso(now), "assistant_messages": [{"at": m.get("at"), "text": m["text"]} for m in self.unread], + "phone": phone, "you_sent": list(self.you_sent), "done_ends_run": not self.multi_day} self.unread, self.you_sent, self.read_at = [], [], None calls = [] @@ -898,7 +951,7 @@ async def ask(text: str) -> dict: user_sim_error=(calls[-1].get("error") if calls else None), user_sim_trajectory_urls=[c.get("trajectory_url") for c in calls], user_sim_turns=ans.get("turns", len(calls)), shown=ans.get("shown"), - past_turn_cap=ans.get("past_turn_cap")) + past_turn_cap=ans.get("past_turn_cap"), last_read=last) outcome = ans.get("outcome") said = (ans.get("message") or "").strip() if outcome in ("reply", "done") and said: @@ -912,13 +965,13 @@ async def ask(text: str) -> dict: if max_replies and self.replies >= int(max_replies): self.gave_up = True self.log("user_gave_up", replies=self.replies) - elif outcome == "done": + elif outcome == "done" and not last: entry["ends_run"] = not self.multi_day if not self.multi_day and said: self.signed_off = True elif not self.multi_day: self.end_reason = "user_done" - if calls and calls[-1].get("error"): + if calls and calls[-1].get("error") and not last: self.end_reason = self.end_reason or "user_sim_error" async def _ask_user_sim(self, text: str) -> dict: @@ -975,12 +1028,43 @@ async def _contacts_at_work(self) -> list[dict]: out.append({"env_id": env.env_id, "triggers": rows}) return out + async def _pending_at_end(self) -> dict: + """What was still pending when the run ended: her replies held for their response time, the agent's messages + she never read, user lines and alerts for the agent never delivered, its scheduled checks not yet due, and + contact replies still in progress.""" + alerts, checks = [], [] + for st in await self._notes_state(self.recipient): + alerts += [{k: n.get(k) for k in ("at", "channel", "text", "trigger_id")} + for n in st.get("pending_notifications", [])] + checks += [{k: c.get(k) for k in ("next", "note", "every")} for c in st.get("checks", []) + if c.get("status") == "active"] + lines = [{k: n.get(k) for k in ("at", "text", "trigger_id")} + for st in await self._notes_state(USER_SAYS) for n in st.get("pending_notifications", [])] + return {"held_replies": [{"at": _iso(h["deliver_at"]), "text": h["text"], "done": h.get("done", False)} + for h in self.held], + "unread_by_user": [dict(m) for m in self.unread], + "user_lines_not_delivered": lines, + "alerts_never_delivered": alerts, + "scheduled_checks_not_due": checks, + "contact_replies_in_progress": await self._contacts_at_work()} + async def finish(self) -> None: step, ctx = self.step, self.ctx now = await self.now() - self.pending = {"contact_replies_in_progress": await self._contacts_at_work()} - self.log("end", reason=self.end_reason, virtual_time=_iso(now), wakes=self.wakes, replies=self.replies, - unread_at_end=len(self.unread), held_at_end=len(self.held), pending=self.pending) + try: + self.pending = await self._pending_at_end() + except Exception as e: # the record matters more than this part of it + logger.warning(f"always_on: could not read what was pending at the end: {e}") + self.pending = {"error": [{"error": f"{type(e).__name__}: {str(e)[:200]}"}]} + if self.end_reason == NOTHING_CAN_WAKE: + # Scheduled checks would have woken the agent, so they aren't what makes a run stuck. + stuck = any(items for kind, items in self.pending.items() if kind != "scheduled_checks_not_due") + self.end_reason = "stuck" if stuck else "finished_nothing_pending" + self.end_note = END_NOTES.get(self.end_reason, self.end_reason or "").format(first=self.person["first"], + wakes=self.wakes) + self.log("end", reason=self.end_reason, note=self.end_note, virtual_time=_iso(now), wakes=self.wakes, + replies=self.replies, unread_at_end=len(self.unread), held_at_end=len(self.held), + pending=self.pending) conv = conversation_store.get_conversation(self.conversation_id) if conv and conv.get("status") == "active": conversation_store.mark_closed(self.conversation_id) @@ -1006,7 +1090,8 @@ async def finish(self) -> None: if step.stay_quiet: counts["stay_quiet_wakes"] = sum(1 for e in self.timeline if e["event"] == "agent_turn_end" and e.get("stay_quiet_calls")) - record = {"step_id": step.id, "end_reason": self.end_reason, "error": self.error, "counts": counts, + record = {"step_id": step.id, "end_reason": self.end_reason, "end_note": self.end_note, "error": self.error, + "counts": counts, "user_sim": {"name": step.user_sim, "version": self.version, "model": self.us.get("model"), "agent_name": self.us.get("agent_name"), **self.user_setup}, "end_time": _iso(self.end) if self.end else None, "time_zone": self.zone, diff --git a/tst/unit/task_step/test_always_on_endings.py b/tst/unit/task_step/test_always_on_endings.py new file mode 100644 index 000000000..8718d471a --- /dev/null +++ b/tst/unit/task_step/test_always_on_endings.py @@ -0,0 +1,225 @@ +"""How an always_on run ends: a final check-in at end_time whether or not the task sets a `final` line, a last-turn +notice at end_time and before the wake cap, the user sim's last read of the agent's wrap-up, what was still pending, +and end reasons that tell a finished run from a stuck one.""" + +from __future__ import annotations + +import pytest + +from agent_env.task_step.task_steps import always_on as ao +from tst.unit.task_step.always_on_harness import Rig + +pytestmark = pytest.mark.usefixtures("local_stores") + +DAY = {"schedule": {"wake": "07:30", "sleep": "23:30", "work": ["09:00", "18:00"]}, + "response_times": {"work": 150, "free": 30, "morning": 15}} +NOTICE = ao.FINAL_REMINDER.format(first="Linda") +DEFAULT_FINAL = ao.FINAL_DEFAULT_TEXT.format(first="Linda") + + +def _agent(**plays): + def agent(n, texts, rig): + play = plays.get(f"t{n}") + if play is None: + rig.quiet() + return "" + return play(rig) + return agent + + +def _message(text): + return lambda rig: (rig.message(text), "sent")[1] + + +def _recaps(n, texts, rig): + """Messages Linda on a wake that carries the last-turn notice, and stays quiet otherwise.""" + if NOTICE in texts: + rig.message("RESULT: house booked, flights held; La Ola still open") + return "sent" + rig.quiet() + return "" + + +# ---- 1. a final check-in at end_time, with or without a `final` line ------------------------------------------------- + +@pytest.mark.asyncio +async def test_without_a_final_line_the_end_time_still_brings_a_check_in_with_the_step_s_own_line(monkeypatch, + tmp_path): + rig = Rig(monkeypatch, tmp_path, agent=_recaps, answers=[{"outcome": "reply", "message": "perfect, thanks"}]) + await rig.arm() + ctx = await rig.run(rig.step(end_time="P2D")) + + last = rig.parts[-1] + assert [p["text"] for p in last if p["metadata"]["agentenv"]["source"] == "final"] == [DEFAULT_FINAL] + assert rig.events(ctx, "wake")[-1]["virtual_time"] == "2026-11-11T14:00:00Z" + assert rig.events(ctx, "wake")[-1]["woke_by"] == ["step:final"] + assert DEFAULT_FINAL not in repr([r["you_sent"] for r in rig.reads]) # the step's line, not hers + assert rig.record(ctx)["end_reason"] == "end_time" + + +@pytest.mark.asyncio +async def test_a_task_s_final_line_is_hers_and_comes_with_the_notice(monkeypatch, tmp_path): + rig = Rig(monkeypatch, tmp_path, agent=_recaps) + await rig.arm() + await rig.run(rig.step(end_time="P2D", final="send me the rundown")) + + texts = rig.sent[-1] + assert "[Wed Nov 11, 2:00 PM UTC] Linda: send me the rundown" in texts and DEFAULT_FINAL not in texts + assert texts[-2:] == [NOTICE, ao.WAKE_REMINDER_TEXT] + assert [s["text"] for s in rig.reads[-1]["you_sent"]] == ["sort the cdmx weekend", "send me the rundown"] + + +# ---- 2. the last-turn notice --------------------------------------------------------------------------------------- + +@pytest.mark.asyncio +async def test_the_wake_the_cap_makes_the_last_carries_the_notice_and_no_earlier_one_does(monkeypatch, tmp_path): + rig = Rig(monkeypatch, tmp_path, agent=_recaps) + await rig.arm() + for day in range(10, 16): + rig.triggers(rig.notify_at(f"alert-{day}", f"2026-11-{day}T14:00:00Z", "Sam: any news?", channel="noor 30")) + ctx = await rig.run(rig.step(max_wakes=4)) + + assert [NOTICE in texts for texts in rig.sent] == [False, False, False, True] + assert [w["last"] for w in rig.events(ctx, "wake")] == [False, False, False, True] + assert rig.record(ctx)["end_reason"] == "max_wakes" + assert [m["text"] for m in rig.reads[-1]["assistant_messages"]] == [ + "RESULT: house booked, flights held; La Ola still open"] + + +# ---- 3. final -> the agent's turn -> her last read -> her reaction recorded -> end ---------------------------------- + +@pytest.mark.asyncio +async def test_the_user_sim_reads_the_wrap_up_and_her_reaction_and_held_reply_are_recorded(monkeypatch, tmp_path): + rig = Rig(monkeypatch, tmp_path, agent=_recaps, + answers=[{"outcome": "reply", "message": "amazing, book La Ola too"}]) + await rig.arm() + ctx = await rig.run(rig.step(end_time="P2D", final="send me the rundown"), rig.context(user=DAY)) + + order = [e["event"] for e in rig.timeline(ctx)] + tail = order[order.index("scripted_user_line"):] + assert tail[:2] == ["scripted_user_line", "wake"] and tail[-2:] == ["user_sim_read", "end"] + assert tail.index("message_to_user") < tail.index("user_sim_read") + [read] = rig.events(ctx, "user_sim_read") + assert read["last_read"] is True and read["outcome"] == "reply" + assert read["virtual_time"] == "2026-11-11T14:00:00Z" and read["deliver_at"] == "2026-11-11T16:30:00Z" + held = rig.record(ctx)["pending_at_end"]["held_replies"] + assert held == {"count": 1, "items": [{"at": "2026-11-11T16:30:00Z", "text": "amazing, book La Ola too", + "done": False}]} + assert rig.record(ctx)["end_reason"] == "end_time" and len(rig.sent) == 2 + + +@pytest.mark.asyncio +async def test_a_done_on_the_last_read_is_recorded_and_changes_nothing(monkeypatch, tmp_path): + rig = Rig(monkeypatch, tmp_path, agent=_recaps, answers=[{"outcome": "done", "message": "thanks!!"}]) + await rig.arm() + ctx = await rig.run(rig.step(max_wakes=1)) + + [read] = rig.events(ctx, "user_sim_read") + assert read["last_read"] is True and read["outcome"] == "done" and "ends_run" not in read + assert rig.record(ctx)["end_reason"] == "max_wakes" + assert rig.record(ctx)["pending_at_end"]["held_replies"]["items"][0]["done"] is True + + +@pytest.mark.asyncio +async def test_a_reply_still_held_at_end_time_is_kept_not_dropped(monkeypatch, tmp_path): + """FD-5332 test 8: Linda's last reply was lost at end_time, and the record said nothing was held.""" + rig = Rig(monkeypatch, tmp_path, agent=_agent(t2=_message("flight retimed, Dev is fine")), + answers=[{"outcome": "reply", "message": "ok good, tell Sam"}]) + await rig.arm() + rig.triggers(rig.notify_at("airline", "2026-11-10T13:50:00Z", "Wingloft: flight retimed", channel="Gmail")) + ctx = await rig.run(rig.step(end_time="P1D"), rig.context(user=DAY)) + + [read] = rig.events(ctx, "user_sim_read") + assert read["deliver_at"] == "2026-11-10T14:20:00Z" # after end_time + assert [i["text"] for i in rig.record(ctx)["pending_at_end"]["held_replies"]["items"]] == ["ok good, tell Sam"] + assert rig.events(ctx, "end")[0]["held_at_end"] == 1 + + +# ---- 5. what was still pending --------------------------------------------------------------------------------------- + +@pytest.mark.asyncio +async def test_the_agent_s_checks_not_yet_due_are_recorded_and_events_after_the_end_never_happened(monkeypatch, + tmp_path): + def sets_a_check(n, texts, rig): + if n == 1: + rig.schedule("chase Live Oak", at="2026-11-20T14:00:00Z") + rig.quiet() + return "" + + rig = Rig(monkeypatch, tmp_path, agent=sets_a_check) + await rig.arm() + rig.triggers(rig.notify_at("late", "2026-11-12T14:00:00Z", "Live Oak: invoice", channel="Gmail"), + rig.notify_at("ask", "2026-11-12T15:00:00Z", "did they invoice?", to=ao.USER_SAYS)) + ctx = await rig.run(rig.step(end_time="P2D")) + + pending = rig.record(ctx)["pending_at_end"] + assert pending["scheduled_checks_not_due"]["items"][0]["note"] == "chase Live Oak" + end = rig.events(ctx, "end")[0]["pending"] + assert end["alerts_never_delivered"] == end["user_lines_not_delivered"] == [] # never came due: not triggered + + +@pytest.mark.asyncio +async def test_an_alert_that_arrives_during_the_last_turn_is_recorded_as_never_delivered(monkeypatch, tmp_path): + def last_turn_brings_mail(n, texts, rig): + if NOTICE in texts: + rig.alert("Live Oak: we can come Friday", channel="Gmail · Live Oak Arbor") + rig.quiet() + return "" + + rig = Rig(monkeypatch, tmp_path, agent=last_turn_brings_mail) + await rig.arm() + ctx = await rig.run(rig.step(max_wakes=1)) + + [alert] = rig.record(ctx)["pending_at_end"]["alerts_never_delivered"]["items"] + assert (alert["channel"], alert["text"]) == ("Gmail · Live Oak Arbor", "Live Oak: we can come Friday") + + +# ---- 7. finished vs stuck -------------------------------------------------------------------------------------------- + +@pytest.mark.asyncio +async def test_with_nothing_left_to_wake_the_agent_and_nothing_pending_the_run_finished(monkeypatch, tmp_path): + rig = Rig(monkeypatch, tmp_path, agent=_agent(t1=_message("report sent")), + answers=[{"outcome": "no_reply", "message": ""}]) + await rig.arm() + ctx = await rig.run(rig.step()) + + record = rig.record(ctx) + assert record["end_reason"] == "finished_nothing_pending" + assert all(v["count"] == 0 for v in record["pending_at_end"].values()) + + +@pytest.mark.asyncio +async def test_with_nothing_left_to_wake_the_agent_but_a_message_she_will_never_read_the_run_is_stuck(monkeypatch, + tmp_path): + rig = Rig(monkeypatch, tmp_path, agent=_agent(t1=_message("blue or green?"), t2=_message("and the dates?")), + answers=[{"outcome": "reply", "message": "blue"}]) + await rig.arm() + ctx = await rig.run(rig.step(), rig.context(user={"max_replies": 1})) + + record = rig.record(ctx) + assert record["end_reason"] == "stuck" + assert [i["text"] for i in record["pending_at_end"]["unread_by_user"]["items"]] == ["and the dates?"] + assert record["pending_at_end"]["unread_by_user"]["items"][0]["wake"] == 2 + assert record["end_note"] == ao.END_NOTES["stuck"] + + +@pytest.mark.asyncio +async def test_a_contact_still_replying_when_nothing_else_can_wake_the_agent_makes_the_run_stuck(monkeypatch, + tmp_path): + monkeypatch.setattr(ao, "CONTACT_GRACE_SECONDS", 0) + rig = Rig(monkeypatch, tmp_path) + await rig.arm() + slow = rig.contact("liveoak", seconds=5, reply=lambda rig, n: rig.alert("Live Oak: estimate"), + timeout_seconds=0.2, keeps_to_its_limit=False) + rig.triggers({"id": "liveoak-estimate", "when": {"type": "time", "at": "2026-11-09T15:00:00Z"}, + "actions": [{"type": "nl", "executor": "liveoak", "instruction": "send your estimate"}]}, + executors={"liveoak": slow}) + try: + ctx = await rig.run(rig.step(settle_timeout_seconds=0.2)) + finally: + await rig.close() + + record = rig.record(ctx) + assert record["end_reason"] == "stuck" + [env] = record["pending_at_end"]["contact_replies_in_progress"]["items"] + assert env["triggers"][0]["id"] == "liveoak-estimate" diff --git a/tst/unit/task_step/test_always_on_loop.py b/tst/unit/task_step/test_always_on_loop.py index ce81eee0e..c7406ac1e 100644 --- a/tst/unit/task_step/test_always_on_loop.py +++ b/tst/unit/task_step/test_always_on_loop.py @@ -51,7 +51,7 @@ async def test_a_notify_alert_wakes_the_agent_labelled_with_its_time_and_channel assert _woke(ctx, rig) == [("2026-11-09T14:00:00Z", ["user_request"]), ("2026-11-10T14:00:00Z", ["notification:noor 30"])] assert "[Tue Nov 10, 2:00 PM UTC · noor 30] Sam: she suspects nothing" in rig.sent[1] - assert rig.record(ctx)["end_reason"] == "nothing_can_wake_the_agent" + assert rig.record(ctx)["end_reason"] == "finished_nothing_pending" @pytest.mark.asyncio From 5df7cb902e89e819b181e002545a28b99c34db72 Mon Sep 17 00:00:00 2001 From: Sarah Chitty Date: Mon, 9 Nov 2026 01:46:44 -0800 Subject: [PATCH 19/53] always_on: graders read what the user saw; failed turns are errors (FD-5370 1-3, FD-5369 4 and 6) - The graded response is every message_user text in order, each with its pretend time; never the agent's closing text (it was the last wake's messages plus its closing text). - The conversation graders read (a2a_conversations[step]) is now the user's chat: her messages (opening, user_says and final lines, replies, done) as user turns, and the agent's message_user texts, with times and the wake that sent each, as one agent message per run of them. The alerts that woke the agent, the step's own lines and its closing text stay in the per-wake prompt parts, trajectories and timeline. - It ends with one last user turn: her replies that never reached the agent, her reaction when her last read said nothing (no reply, or a blank done), and an end marker with the end reason and end_note. - A failed or timed-out agent turn (or nudge turn) sets error_type on the prompt response (the agent's own, else agent_turn_failed / agent_turn_timeout) and error_code, as prompt_agent does, so agent_prompt_response_verifier and rubrics_verifier skip the run; user_sim_error and step_error ends do too. The record carries error_type. Co-Authored-By: Claude Opus 5.5 --- .../task_step/task_steps/always_on.py | 171 ++++++++++------ tst/unit/task_step/always_on_harness.py | 21 +- tst/unit/task_step/test_always_on_graded.py | 192 ++++++++++++++++++ 3 files changed, 322 insertions(+), 62 deletions(-) create mode 100644 tst/unit/task_step/test_always_on_graded.py diff --git a/src/agent_env/task_step/task_steps/always_on.py b/src/agent_env/task_step/task_steps/always_on.py index e318d45cf..9d7c5524c 100644 --- a/src/agent_env/task_step/task_steps/always_on.py +++ b/src/agent_env/task_step/task_steps/always_on.py @@ -36,6 +36,11 @@ can wake the agent, with nothing pending or with something still pending), agent_turn_failed, agent_turn_timeout, user_sim_error, step_error. What was still pending is recorded (``pending_at_end``): replies she held, messages she never read, user lines and alerts never delivered, the agent's checks not yet due, contact replies in progress. +- Graders read what the user saw. The prompt response is every message_user text in order, each with its pretend + time, never the agent's closing text. The conversation (``metadata["a2a_conversations"][step_id]``) holds her own + messages and the agent's message_user texts, then her last word that never reached the agent and an end marker with + the end reason. A failed or timed-out agent turn sets the response's ``error_type`` (as prompt_agent does), so + verifiers skip the run. - The run's metadata keeps a summary in ``metadata["always_on"][step_id]`` (small enough for a running job to report back); the full timeline goes to the object store, at the summary's ``object_url``. """ @@ -227,11 +232,13 @@ def __init__(self, step: AlwaysOnTaskStep, context: TaskStepContext): self.per_turn_trajectory_urls: list[Optional[str]] = [] self.per_turn_parts: list[Optional[list[dict]]] = [] self.tool_calls: Optional[int] = None - self.last_plain = "" - self.last_messages: list[dict] = [] self.all_messages: list[dict] = [] self.final_task_id = "" self.error: Optional[str] = None + self.error_type: Optional[str] = None # set as prompt_agent sets it, so verifiers skip a run that broke + self.error_code: Optional[str] = None + self.chat_pending: list[dict] = [] # the agent's message_user texts not yet in the graded conversation + self.reaction: Optional[dict] = None # her last read's answer when she said nothing self._plugin_paths: list[str] = [] self._plugin_modules: list[str] = [] self.nudges = 0 @@ -624,6 +631,7 @@ def _schedule_read(self) -> None: async def loop(self, now: datetime) -> None: step = self.step self.you_sent.append(dict(self.opening)) + await self._chat_user([self._chat_part(_utc(self.opening["at"]), self.opening["text"], kind="opening")]) due = await take_due(self.notification_envs, self.recipient) if self.notification_envs else [] await self.wake([*delivered_parts(due), self._user_part(_utc(self.opening["at"]), self.opening["text"], opening=True)], @@ -717,6 +725,48 @@ def _user_part(self, at: datetime, text: str, opening: bool = False) -> dict: return {"kind": "text", "text": f"[{label_time(_iso(at))}] {first}: {body}", "metadata": {"agentenv": {"source": "user", "at": _iso(at), "opening": opening}}} + # ---- the graded conversation: the user's chat ---------------------------------------------------------------- + # What graders read as the run's conversation (metadata["a2a_conversations"][step id]) is what the user saw: her + # own messages as user turns (the opening request, user_says and final lines, her replies), and the agent's + # message_user texts, each with its pretend time, as one agent message per run of them. Never the agent's closing + # text, the alerts that woke it, or the step's own lines; those are in the per-wake prompt parts and the timeline. + # It ends with her last word that never reached the agent, and a marker saying how the run ended. + + def _chat_part(self, at: datetime, text: str, **meta: Any) -> dict: + part = self._user_part(at, text) + part["metadata"]["agentenv"].update(meta) + return part + + async def _chat_flush(self) -> None: + """Write the agent's message_user texts not yet in the conversation, as one agent message.""" + if self.chat_pending: + parts, self.chat_pending = self.chat_pending, [] + conversation_store.complete_a2a_task(conversation_id=self.conversation_id, parts=parts, role="agent", + **(await self.step._env_clock(self.clock_envs))) + + async def _chat_user(self, parts: list[dict]) -> None: + """Write a user turn: what she said, after what the agent sent before it.""" + await self._chat_flush() + conversation_store.add_a2a_task(conversation_id=self.conversation_id, parts=parts, + a2a_task_id=uuid.uuid4().hex, role="user", + **(await self.step._env_clock(self.clock_envs))) + + async def _chat_end(self, now: datetime) -> None: + """Close the conversation: the agent's last messages, then one last user turn with her replies that never + reached the agent, her reaction to its last messages when she said nothing, and the end marker.""" + first = self.person["first"] + parts = [self._chat_part(h["deliver_at"], f"{h['text']} (sent after the assistant's last turn; it never saw " + "this)", kind="user_reply", delivered=False, **({"done": True} if h.get("done") else {})) + for h in self.held] + if self.reaction: + parts.append(_step_part(f"({first} {self.reaction['said']})", "user_reaction")) + parts[-1]["metadata"]["agentenv"].update(outcome=self.reaction["outcome"], at=self.reaction["at"]) + parts.append({"kind": "text", "text": f"[End of run · {label_time(_iso(now))}] {self.end_note} (A note from " + f"the run, not a message from {first}.)", + "metadata": {"agentenv": {"source": "end_marker", "end_reason": self.end_reason, + "at": _iso(now)}}}) + await self._chat_user(parts) + async def _take_user_says(self) -> list[dict]: items = await take_due(self.notification_envs, USER_SAYS) if self.notification_envs else [] out = [] @@ -736,6 +786,9 @@ async def _wake_with(self, now: datetime, lines: list[dict], replies: list[dict] + [{"at": _iso(r["deliver_at"]), "text": r["text"], "kind": "user_reply", "done": r.get("done", False)} for r in replies], key=lambda m: _utc(m["at"])) + if user_msgs: + await self._chat_user([self._chat_part(_utc(m["at"]), m["text"], kind=m["kind"], + **({"done": True} if m.get("done") else {})) for m in user_msgs]) for m in user_msgs: parts.append(self._user_part(_utc(m["at"]), m["text"])) if m["kind"] != "user_reply": @@ -773,8 +826,6 @@ async def wake(self, parts: list[dict], woke_by: list[str], notifications: Optio msg_id = uuid.uuid4().hex def record() -> None: - conversation_store.add_a2a_task(conversation_id=self.conversation_id, parts=parts, a2a_task_id=msg_id, - role="user", **clock) self.per_turn_parts.append(list(parts)) self.log("wake", n=self.wakes, virtual_time=_iso(sent_at), woke_by=woke_by, notifications=notifications or [], @@ -786,8 +837,7 @@ def record() -> None: timeout_seconds=step.timeout_seconds, poll_interval_seconds=step.poll_interval_seconds, before_send=record) except TimeoutError as e: - self.error = f"agent turn timed out after {step.timeout_seconds}s: {e}" - self.end_reason = "agent_turn_timeout" + self._failed("agent_turn_timeout", f"agent turn timed out after {step.timeout_seconds}s: {e}") self.log("agent_turn_end", n=self.wakes, state="timeout", real_seconds=round(time.time() - t, 1)) return state = result["status"]["state"] @@ -795,22 +845,7 @@ def record() -> None: term = protocol.TerminalResponse.from_message(status_msg) if term.tool_call_count is not None: self.tool_calls = (self.tool_calls or 0) + term.tool_call_count - agent_parts = status_msg.get("parts") or [{"kind": "text", "text": term.response_text}] - messages, quiet = [], [] - for env in self.chat_envs: - take_id = uuid.uuid4().hex - got = await _call(env, EXT_USER_CHAT_URI, "take", {"take_id": take_id}) - messages += got.get("items") or [] - quiet += got.get("quiet") or [] - messages.sort(key=lambda m: m.get("at") or "") - # The conversation shows graders what the user saw (message_user) and, marked, what only went to the log. - recorded = [{"kind": "text", "text": f"[message_user · {label_time(m.get('at'))}] {m['text']}", - "metadata": {"agentenv": {"source": "message_user", "at": m.get("at")}}} for m in messages] - recorded += [{**p, "text": f"[plain reply, not shown to the user] {p['text']}", - "metadata": {"agentenv": {"source": "plain_reply"}}} if p.get("kind") == "text" else p - for p in agent_parts] - conversation_store.complete_a2a_task(conversation_id=self.conversation_id, parts=recorded, role="agent", - **(await step._env_clock(self.clock_envs))) + messages, quiet = await self._take_messages() traj = None card = self.agent.a2a_card or {} traj_ext = A2AAgent.find_extension(card, A2AAgent.EXT_TRAJECTORY) @@ -824,19 +859,15 @@ def record() -> None: self.final_task_id = task_id ended = await self.now() plain = term.response_text or "" - self.last_plain, self.last_messages = plain, messages extra = {"stay_quiet_calls": len(quiet)} if step.stay_quiet else {} self.log("agent_turn_end", n=self.wakes, state=str(state), virtual_time=_iso(ended), virtual_seconds=round((ended - sent_at).total_seconds()), real_seconds=round(time.time() - t, 1), tool_calls=term.tool_call_count, plain_reply=plain, messages_to_user=len(messages), trajectory_url=traj, **extra) - for m in messages: - self.log("message_to_user", virtual_time=m.get("at"), text=m["text"], wake=self.wakes) - self.unread.append({"at": m.get("at"), "text": m["text"], "wake": self.wakes}) - self.all_messages.append({"at": m.get("at"), "text": m["text"], "wake": self.wakes}) + self._sent(messages) if state == TaskState.failed or str(state) == "failed": - self.error = term.error_message or term.response_text or "agent turn failed" - self.end_reason = "agent_turn_failed" + self._failed(term.error_type or "agent_turn_failed", + term.error_message or term.response_text or "agent turn failed", term.error_code) return if step.nudge and not messages and not quiet and _closing_text(plain): await self._nudge(plain) @@ -852,12 +883,9 @@ async def _nudge(self, closing: str) -> None: self.nudges += 1 parts = [_step_part(step.nudge_text, "nudge")] sent_at = await self.now() - clock = await step._env_clock(self.clock_envs) msg_id = uuid.uuid4().hex def record() -> None: - conversation_store.add_a2a_task(conversation_id=self.conversation_id, parts=parts, a2a_task_id=msg_id, - role="user", **clock) self.per_turn_parts.append(list(parts)) t = time.time() @@ -867,8 +895,7 @@ def record() -> None: timeout_seconds=step.timeout_seconds, poll_interval_seconds=step.poll_interval_seconds, before_send=record) except TimeoutError as e: - self.error = f"agent nudge turn timed out after {step.timeout_seconds}s: {e}" - self.end_reason = "agent_turn_timeout" + self._failed("agent_turn_timeout", f"agent nudge turn timed out after {step.timeout_seconds}s: {e}") self.log("nudge", n=self.nudges, wake=self.wakes, virtual_time=_iso(sent_at), closing_text=closing, state="timeout", outcome="timeout", real_seconds=round(time.time() - t, 1)) return @@ -877,20 +904,7 @@ def record() -> None: term = protocol.TerminalResponse.from_message(status_msg) if term.tool_call_count is not None: self.tool_calls = (self.tool_calls or 0) + term.tool_call_count - agent_parts = status_msg.get("parts") or [{"kind": "text", "text": term.response_text}] - messages, quiet = [], [] - for env in self.chat_envs: - got = await _call(env, EXT_USER_CHAT_URI, "take", {"take_id": uuid.uuid4().hex}) - messages += got.get("items") or [] - quiet += got.get("quiet") or [] - messages.sort(key=lambda m: m.get("at") or "") - recorded = [{"kind": "text", "text": f"[message_user · {label_time(m.get('at'))}] {m['text']}", - "metadata": {"agentenv": {"source": "message_user", "at": m.get("at")}}} for m in messages] - recorded += [{**p, "text": f"[plain reply, not shown to the user] {p['text']}", - "metadata": {"agentenv": {"source": "plain_reply"}}} if p.get("kind") == "text" else p - for p in agent_parts] - conversation_store.complete_a2a_task(conversation_id=self.conversation_id, parts=recorded, role="agent", - **(await step._env_clock(self.clock_envs))) + messages, quiet = await self._take_messages() traj = None card = self.agent.a2a_card or {} traj_ext = A2AAgent.find_extension(card, A2AAgent.EXT_TRAJECTORY) @@ -908,15 +922,38 @@ def record() -> None: state=str(state), outcome=outcome, messages_to_user=len(messages), stay_quiet_calls=len(quiet), plain_reply_after=plain, tool_calls=term.tool_call_count, real_seconds=round(time.time() - t, 1), trajectory_url=traj) - if messages: - self.last_plain, self.last_messages = plain, messages - for m in messages: - self.log("message_to_user", virtual_time=m.get("at"), text=m["text"], wake=self.wakes, after_nudge=True) - self.unread.append({"at": m.get("at"), "text": m["text"], "wake": self.wakes}) - self.all_messages.append({"at": m.get("at"), "text": m["text"], "wake": self.wakes}) + self._sent(messages, after_nudge=True) if state == TaskState.failed or str(state) == "failed": - self.error = term.error_message or term.response_text or "agent nudge turn failed" - self.end_reason = "agent_turn_failed" + self._failed(term.error_type or "agent_turn_failed", + term.error_message or term.response_text or "agent nudge turn failed", term.error_code) + + async def _take_messages(self) -> tuple[list[dict], list[dict]]: + """What the agent sent the user with message_user this turn, oldest first, and its stay_quiet calls.""" + messages, quiet = [], [] + for env in self.chat_envs: + got = await _call(env, EXT_USER_CHAT_URI, "take", {"take_id": uuid.uuid4().hex}) + messages += got.get("items") or [] + quiet += got.get("quiet") or [] + messages.sort(key=lambda m: m.get("at") or "") + return messages, quiet + + def _sent(self, messages: list[dict], after_nudge: bool = False) -> None: + """Record what the agent sent the user: for her next read, the graded chat and the timeline.""" + for m in messages: + self.log("message_to_user", virtual_time=m.get("at"), text=m["text"], wake=self.wakes, + **({"after_nudge": True} if after_nudge else {})) + sent = {"at": m.get("at"), "text": m["text"], "wake": self.wakes} + self.unread.append(sent) + self.all_messages.append(sent) + self.chat_pending.append({"kind": "text", "text": f"[{label_time(m.get('at'))}] {m['text']}", + "metadata": {"agentenv": {"source": "message_user", "at": m.get("at"), + "wake": self.wakes}}}) + + def _failed(self, error_type: str, error: str, error_code: Optional[str] = None) -> None: + """An agent turn failed or timed out: the run ends, and its prompt response carries ``error_type`` the way + prompt_agent's does, so verifiers skip it instead of grading it as a normal run.""" + self.error, self.error_type, self.error_code = error, error_type, error_code + self.end_reason = "agent_turn_timeout" if error_type == "agent_turn_timeout" else "agent_turn_failed" async def read(self, now: datetime, last: bool = False) -> None: """The user sim reads what the agent sent since her last read, at ``now``; ``last`` marks the read after the @@ -965,7 +1002,11 @@ async def ask(text: str) -> dict: if max_replies and self.replies >= int(max_replies): self.gave_up = True self.log("user_gave_up", replies=self.replies) - elif outcome == "done" and not last: + if not said and outcome in ("no_reply", "done") and (last or (outcome == "done" and not self.multi_day)): + self.reaction = {"outcome": outcome, "at": _iso(now), + "said": "read it and signed off without a message." if outcome == "done" + else "read it and didn't reply."} + if outcome == "done" and not last: entry["ends_run"] = not self.multi_day if not self.multi_day and said: self.signed_off = True @@ -1062,9 +1103,15 @@ async def finish(self) -> None: self.end_reason = "stuck" if stuck else "finished_nothing_pending" self.end_note = END_NOTES.get(self.end_reason, self.end_reason or "").format(first=self.person["first"], wakes=self.wakes) + if self.end_reason in ERROR_ENDS and not self.error_type: + self.error_type = self.end_reason self.log("end", reason=self.end_reason, note=self.end_note, virtual_time=_iso(now), wakes=self.wakes, replies=self.replies, unread_at_end=len(self.unread), held_at_end=len(self.held), pending=self.pending) + try: + await self._chat_end(now) + except Exception as e: + logger.warning(f"always_on: could not close the graded conversation: {e}") conv = conversation_store.get_conversation(self.conversation_id) if conv and conv.get("status") == "active": conversation_store.mark_closed(self.conversation_id) @@ -1091,7 +1138,7 @@ async def finish(self) -> None: counts["stay_quiet_wakes"] = sum(1 for e in self.timeline if e["event"] == "agent_turn_end" and e.get("stay_quiet_calls")) record = {"step_id": step.id, "end_reason": self.end_reason, "end_note": self.end_note, "error": self.error, - "counts": counts, + "error_type": self.error_type, "counts": counts, "user_sim": {"name": step.user_sim, "version": self.version, "model": self.us.get("model"), "agent_name": self.us.get("agent_name"), **self.user_setup}, "end_time": _iso(self.end) if self.end else None, "time_zone": self.zone, @@ -1101,8 +1148,9 @@ async def finish(self) -> None: "scheduled_checks": self.checks, "contacts": self.contacts} await self._store_timeline(record) ctx.metadata.setdefault(ALWAYS_ON_KEY, {})[step.id] = record - final_text = "\n\n".join([m["text"] for m in self.last_messages] + ([self.last_plain] if self.last_plain else [])) - self._append_prompt_response(final_text) + # The graded response is the user's whole chat from the agent, each message with its pretend time; never its + # closing text, which she never saw. + self._append_prompt_response("\n\n".join(f"[{label_time(m.get('at'))}] {m['text']}" for m in self.all_messages)) async def _store_timeline(self, record: dict) -> None: """Store the run's full record, its timeline included, and give ``record`` (what stays in the run's metadata, @@ -1131,7 +1179,8 @@ def _append_prompt_response(self, final_text: str) -> None: agent_trajectory_object_url=traj, agent_trajectory_object_prefix=step._trajectory_prefix(), target_agent_per_turn_trajectory_object_urls=self.per_turn_trajectory_urls or None, source_agent_per_turn_prompt_parts=self.per_turn_parts or None, tool_call_count=self.tool_calls, - model=self.model, error_message=self.error, agent_session_id=self.final_task_id, + model=self.model, error_type=self.error_type, error_code=self.error_code, error_message=self.error, + agent_session_id=self.final_task_id, a2a_context_id=self.solver_context_id, agent_name=step.agent_name, step_id=step.id)) diff --git a/tst/unit/task_step/always_on_harness.py b/tst/unit/task_step/always_on_harness.py index 676ee33bb..3d7836645 100644 --- a/tst/unit/task_step/always_on_harness.py +++ b/tst/unit/task_step/always_on_harness.py @@ -11,6 +11,7 @@ import httpx from mcp.types import CallToolResult, TextContent, Tool +from agent_env.a2a_agent import conversation_store from agent_env.config import get_config from agent_env.env.env import DeployedGatewayEnv from agent_env.env.gateway import gateway as gw_mod @@ -30,6 +31,13 @@ Turn = Callable[[int, list[str], "Rig"], Optional[str]] +class Fail: + """What an agent's turn returns to end in the failed state, as a harness reports a turn that broke.""" + + def __init__(self, message: str = "the CLI crashed", error_type: Optional[str] = None) -> None: + self.message, self.error_type = message, error_type + + class Rig: """The world, the agent and the user sim for one always_on run. @@ -182,6 +190,12 @@ def stored(self, ctx: TaskStepContext) -> dict: def timeline(self, ctx: TaskStepContext) -> list[dict]: return self.stored(ctx)["timeline"] + def chat(self, ctx: TaskStepContext) -> list[dict]: + """The graded conversation as stored: each message's role and text parts.""" + doc = conversation_store.get_conversation(ctx.metadata["a2a_conversations"]["week"]) + return [{"role": m["role"], "texts": [p["text"] for p in m["parts"] if p.get("kind") == "text"], + "parts": m["parts"]} for m in doc["messages"]] + def events(self, ctx: TaskStepContext, name: str) -> list[dict]: return [e for e in self.timeline(ctx) if e["event"] == name] @@ -198,7 +212,12 @@ async def _send_and_wait(self, url, parts, *, before_send=None, **kw): texts = [p["text"] for p in parts if p.get("kind") == "text"] self.sent.append(texts) self.parts.append(list(parts)) - closing = self.agent(len(self.sent), texts, self) + closing = self.agent(len(self.sent), texts, self) # a TimeoutError it raises is the turn timing out + if isinstance(closing, Fail): + data = {"error_message": closing.message, **({"error_type": closing.error_type} + if closing.error_type else {})} + return f"task-{len(self.sent)}", {"status": {"state": "failed", "message": {"parts": [ + {"kind": "text", "text": closing.message}, {"kind": "data", "data": data}]}}} reply = [{"kind": "text", "text": closing}] if closing else [] return f"task-{len(self.sent)}", {"status": {"state": "completed", "message": {"parts": reply}}} diff --git a/tst/unit/task_step/test_always_on_graded.py b/tst/unit/task_step/test_always_on_graded.py new file mode 100644 index 000000000..089469b6d --- /dev/null +++ b/tst/unit/task_step/test_always_on_graded.py @@ -0,0 +1,192 @@ +"""What graders read of an always_on run: the graded response (the user's whole chat from the agent, with times), the +conversation (what the user saw: her messages and the agent's message_user texts, her last word, an end marker), and +error_type on a run whose agent turn failed or timed out, so verifiers skip it.""" + +from __future__ import annotations + +import pytest + +from agent_env.task_step.task_steps import always_on as ao +from agent_env.task_step.task_steps.verifiers.agent_prompt_response_verifier import ( + AgentPromptResponseVerifierTaskStep, +) +from agent_env.task_step.task_steps.verifiers.run_context import load_run_context +from tst.unit.task_step.always_on_harness import Fail, Rig + +pytestmark = pytest.mark.usefixtures("local_stores") + + +def _agent(**plays): + def agent(n, texts, rig): + play = plays.get(f"t{n}") + if play is None: + rig.quiet() + return "PRIVATE nothing to report" + return play(rig) + return agent + + +def _message(*texts, closing="PRIVATE closing note"): + def play(rig): + for text in texts: + rig.message(text) + return closing + return play + + +async def _turns(ctx): + run = await load_run_context(ctx, ctx.prompt_responses[0]) + return [(t.message.text, t.reply.text if t.reply else None) for t in run.turns] + + +# ---- the graded response ------------------------------------------------------------------------------------------- + +@pytest.mark.asyncio +async def test_the_graded_response_is_every_message_user_text_in_order_with_its_time(monkeypatch, tmp_path): + rig = Rig(monkeypatch, tmp_path, agent=_agent(t1=_message("blue house or green?", "flights: two options"), + t2=_message("booked the blue one")), + answers=[{"outcome": "reply", "message": "blue"}]) + await rig.arm() + rig.triggers(rig.notify_at("sam", "2026-11-10T14:00:00Z", "Sam: shh", channel="noor 30")) + ctx = await rig.run(rig.step()) + + [response] = ctx.prompt_responses + assert response.response == ("[Mon Nov 9, 2:00 PM UTC] blue house or green?\n\n" + "[Mon Nov 9, 2:00 PM UTC] flights: two options\n\n" + "[Mon Nov 9, 2:00 PM UTC] booked the blue one") + assert "PRIVATE" not in response.response and response.error_type is None + verifier = AgentPromptResponseVerifierTaskStep("check", None, prompt_id="ask", verifier_id="words", criteria=[ + {"type": "response_contains", "needles": ["blue house or green?", "booked the blue one"]}]) + await verifier.execute(ctx) + assert ctx.metadata["verifications"]["words"]["score"] == 1 + + +# ---- the conversation graders read --------------------------------------------------------------------------------- + +@pytest.mark.asyncio +async def test_the_conversation_is_what_she_saw_her_messages_and_the_agent_s_message_user_texts(monkeypatch, + tmp_path): + rig = Rig(monkeypatch, tmp_path, agent=_agent(t1=_message("blue house or green?"), t2=_message("booked it"), + t4=_message("Sam says she suspects nothing")), + answers=[{"outcome": "reply", "message": "blue"}, {"outcome": "no_reply", "message": ""}, + {"outcome": "reply", "message": "great"}]) + await rig.arm() + rig.triggers(rig.notify_at("sam", "2026-11-10T14:00:00Z", "Sam: shh", channel="noor 30"), + rig.notify_at("ask", "2026-11-10T15:00:00Z", "any news from Sam?", to=ao.USER_SAYS)) + ctx = await rig.run(rig.step()) + + turns = await _turns(ctx) + assert turns[:3] == [ + ("[Mon Nov 9, 2:00 PM UTC] Linda: sort the cdmx weekend", "[Mon Nov 9, 2:00 PM UTC] blue house or green?"), + ("[Mon Nov 9, 2:00 PM UTC] Linda: blue", "[Mon Nov 9, 2:00 PM UTC] booked it"), + ("[Tue Nov 10, 3:00 PM UTC] Linda: any news from Sam?", + "[Tue Nov 10, 3:00 PM UTC] Sam says she suspects nothing"), + ] + seen = repr(rig.chat(ctx)) + for never_shown in ("PRIVATE", "Sam: shh", "noor 30", ao.WAKE_REMINDER_TEXT, ao.MESSAGE_RULES, + "Any other date you see"): + assert never_shown not in seen, never_shown + + +@pytest.mark.asyncio +async def test_messages_sent_on_several_wakes_before_she_answers_are_one_reply_in_order(monkeypatch, tmp_path): + rig = Rig(monkeypatch, tmp_path, agent=_agent(t1=_message("on it"), t2=_message("Sam is in")), + answers=[{"outcome": "no_reply", "message": ""}, {"outcome": "no_reply", "message": ""}]) + await rig.arm() + rig.triggers(rig.notify_at("sam", "2026-11-10T14:00:00Z", "Sam: I'm in", channel="noor 30")) + ctx = await rig.run(rig.step()) + + [first, *_] = await _turns(ctx) + assert first[1] == "[Mon Nov 9, 2:00 PM UTC] on it\n[Tue Nov 10, 2:00 PM UTC] Sam is in" + assert [p["metadata"]["agentenv"]["wake"] for p in rig.chat(ctx)[1]["parts"]] == [1, 2] + + +# ---- her last word and the end marker (FD-5369 4 and 6) ------------------------------------------------------------ + +@pytest.mark.asyncio +async def test_her_done_reaches_the_conversation(monkeypatch, tmp_path): + rig = Rig(monkeypatch, tmp_path, agent=_agent(t1=_message("all set"), t2=_message("glad to help!")), + answers=[{"outcome": "done", "message": "thanks!"}]) + await rig.arm() + ctx = await rig.run(rig.step()) + + turns = await _turns(ctx) + assert turns[1] == ("[Mon Nov 9, 2:00 PM UTC] Linda: thanks!", "[Mon Nov 9, 2:00 PM UTC] glad to help!") + assert rig.chat(ctx)[2]["parts"][0]["metadata"]["agentenv"]["done"] is True + assert turns[-1][0].startswith("[End of run · Mon Nov 9, 2:00 PM UTC] Linda signed off.") + + +@pytest.mark.asyncio +async def test_a_blank_done_is_shown_as_her_reaction(monkeypatch, tmp_path): + rig = Rig(monkeypatch, tmp_path, agent=_agent(t1=_message("all set")), + answers=[{"outcome": "done", "message": ""}]) + await rig.arm() + ctx = await rig.run(rig.step()) + + [last] = rig.chat(ctx)[-1:] + assert last["role"] == "user" + assert last["texts"][0] == "(Linda read it and signed off without a message.)" + + +@pytest.mark.asyncio +async def test_her_reply_to_the_wrap_up_and_the_end_marker_close_the_conversation(monkeypatch, tmp_path): + def recap(n, texts, rig): + if ao.FINAL_REMINDER.format(first="Linda") in texts: + rig.message("RESULT: all booked") + rig.quiet() + return "PRIVATE" + + rig = Rig(monkeypatch, tmp_path, agent=recap, answers=[{"outcome": "reply", "message": "you're the best"}]) + await rig.arm() + ctx = await rig.run(rig.step(end_time="P2D", final="send me the rundown")) + + turns = await _turns(ctx) + assert turns[-2] == ("[Wed Nov 11, 2:00 PM UTC] Linda: send me the rundown", + "[Wed Nov 11, 2:00 PM UTC] RESULT: all booked") + last = rig.chat(ctx)[-1] + assert last["texts"] == [ + "[Wed Nov 11, 2:00 PM UTC] Linda: you're the best (sent after the assistant's last turn; it never saw this)", + "[End of run · Wed Nov 11, 2:00 PM UTC] The run reached its end time. (A note from the run, not a message " + "from Linda.)"] + assert last["parts"][-1]["metadata"]["agentenv"] == {"source": "end_marker", "end_reason": "end_time", + "at": "2026-11-11T14:00:00Z"} + + +@pytest.mark.asyncio +async def test_a_run_cut_off_by_its_wake_cap_says_so_in_the_conversation(monkeypatch, tmp_path): + def chatty(n, texts, rig): + rig.message(f"update {n}") + return "" + + rig = Rig(monkeypatch, tmp_path, agent=chatty, answers=[{"outcome": "reply", "message": "ok"}] * 5) + await rig.arm() + ctx = await rig.run(rig.step(max_wakes=2)) + + last = rig.chat(ctx)[-1] + assert last["texts"][-1].startswith("[End of run · Mon Nov 9, 2:00 PM UTC] The run was cut off at its cap of 2 " + "wakes.") + assert "Linda: ok (sent after the assistant's last turn" in last["texts"][0] + + +# ---- failed turns are errors ----------------------------------------------------------------------------------------- + +@pytest.mark.parametrize("play,error_type,end_reason", [ + (lambda rig: Fail("the CLI crashed"), "agent_turn_failed", "agent_turn_failed"), + (lambda rig: Fail("sandbox gone", error_type="infra_error"), "infra_error", "agent_turn_failed"), + (lambda rig: (_ for _ in ()).throw(TimeoutError("no answer")), "agent_turn_timeout", "agent_turn_timeout"), +], ids=["failed", "failed with the agent's error type", "timed out"]) +@pytest.mark.asyncio +async def test_a_failed_or_timed_out_turn_sets_error_type_and_the_verifier_skips_the_run(monkeypatch, tmp_path, play, + error_type, end_reason): + rig = Rig(monkeypatch, tmp_path, agent=_agent(t1=_message("on it"), t2=play), + answers=[{"outcome": "reply", "message": "great"}]) + await rig.arm() + ctx = await rig.run(rig.step()) + + [response] = ctx.prompt_responses + assert (response.error_type, rig.record(ctx)["end_reason"]) == (error_type, end_reason) + assert rig.record(ctx)["error_type"] == error_type + verifier = AgentPromptResponseVerifierTaskStep("check", None, prompt_id="ask", verifier_id="words", + criteria=[{"type": "response_contains", "needles": ["on it"]}]) + await verifier.execute(ctx) + assert ctx.metadata["verifications"]["words"]["results"][0]["id"] == "prompt_error" From 5df8fdbb7bdcb86dd5a936c8ed55ed75d0094670 Mon Sep 17 00:00:00 2001 From: Sarah Chitty Date: Mon, 9 Nov 2026 01:50:18 -0800 Subject: [PATCH 20/53] always_on: who did what - app changes by wake, the message each reply answers, hook-shaped reply times (FD-5370 4, 6, 7) - App changes: from each env's changelog (urn:agentenv:changes/v1, from where it stood at the start), the step records every change with its actors, by (agent, others, or none: a write the gateway didn't serve, such as a website reached directly), the wake it was seen in and the phase (before_turn, turn, nudge, after_turn, end). The agent's own carry the wake that made them. app_changes timeline events; counts in the record. - reply_to on each reply: the agent's message it answers (wake, time, text) and how it was found: replying_to from the plug-in (a number or a quote), her used quoting it, the only message in the read, or its only question; None when it can't be told. - metadata["post_turn_hooks"][step]: a start record, then one per read in the old psi-wake hook's shape (hook, turn, read_at, wrote_at, outcome; user_reads_at = when her reply reached the agent; reply_to; delivered_in_wake). ML's asked_before_spending check (Surprise 30th process_checks.py) now grades a step run - checked offline: a booking before her reply fails, one after passes, one with no reply fails - instead of skipping it as 'no psi-wake hook records'. Co-Authored-By: Claude Opus 5.5 --- .../task_step/task_steps/always_on.py | 132 ++++++++++++- tst/unit/task_step/always_on_harness.py | 27 +++ .../task_step/test_always_on_who_did_what.py | 175 ++++++++++++++++++ 3 files changed, 331 insertions(+), 3 deletions(-) create mode 100644 tst/unit/task_step/test_always_on_who_did_what.py diff --git a/src/agent_env/task_step/task_steps/always_on.py b/src/agent_env/task_step/task_steps/always_on.py index 9d7c5524c..da32544a4 100644 --- a/src/agent_env/task_step/task_steps/always_on.py +++ b/src/agent_env/task_step/task_steps/always_on.py @@ -41,6 +41,11 @@ messages and the agent's message_user texts, then her last word that never reached the agent and an end marker with the end reason. A failed or timed-out agent turn sets the response's ``error_type`` (as prompt_agent does), so verifiers skip the run. +- Who did what, and in what order, even under a frozen clock: each app change (urn:agentenv:changes/v1) is recorded + with who made it and the wake it was seen in, the agent's own with the wake that made it (``app_changes`` events); + each reply records the agent's message it answers (``reply_to``: its wake, time and text); and each read leaves a + record in ``metadata["post_turn_hooks"][step_id]`` in the shape the old psi-wake hook wrote, ``user_reads_at`` + being when her reply reached the agent, so checks built on those records grade the run instead of skipping it. - The run's metadata keeps a summary in ``metadata["always_on"][step_id]`` (small enough for a running job to report back); the full timeline goes to the object store, at the summary's ``object_url``. """ @@ -69,9 +74,12 @@ from agent_env.env.env import EnvCapabilityUnsupported from agent_env.env.gateway.notifications import DEFAULT_MAX_ACTIVE from agent_env.env.gateway.constants import ( + CHANGES_PAGE_MAX, DEFAULT_ROLE, + EXT_CHANGES_URI, EXT_CLOCK_URI, EXT_NOTIFICATIONS_URI, + EXT_STATE_URI, EXT_TRIGGERS_URI, EXT_USER_CHAT_URI, ) @@ -127,6 +135,11 @@ } # The run's metadata keeps a summary; the full timeline goes to the object store. When storing fails, the summary keeps # this many of the last events instead, without these fields. +# The records a post-turn hook keeps (metadata["post_turn_hooks"][step id]), written in the shape the old psi-wake hook +# wrote them, so checks that read its reply times (user_reads_at) grade a run on this step. +HOOK_RECORDS_KEY = "post_turn_hooks" +HOOK_NAME = "always_on" +CHANGE_SUMMARY_CHARS = 200 TIMELINE_TAIL = 20 PENDING_KEPT = 20 PENDING_TEXT_CHARS = 300 @@ -239,6 +252,9 @@ def __init__(self, step: AlwaysOnTaskStep, context: TaskStepContext): self.error_code: Optional[str] = None self.chat_pending: list[dict] = [] # the agent's message_user texts not yet in the graded conversation self.reaction: Optional[dict] = None # her last read's answer when she said nothing + self.change_cursors: dict[str, tuple[Any, int]] = {} # env id -> (env, the last changelog id read) + self.change_counts = {"by_agent": 0, "by_others": 0, "no_actor": 0} + self.hook_records: list[dict] = [] self._plugin_paths: list[str] = [] self._plugin_modules: list[str] = [] self.nudges = 0 @@ -311,6 +327,7 @@ async def go(self) -> None: await self._configure_checks() await self._configure_contacts() await self._configure_agents() + await self._start_changes() self.conversation_id = uuid.uuid4().hex try: @@ -321,6 +338,8 @@ async def go(self) -> None: except DuplicateKeyError: pass ctx.metadata.setdefault("a2a_conversations", {})[step.id] = self.conversation_id + self.hook_records.append({"hook": HOOK_NAME, "turn": 0, "started_at": _iso(await self.now())}) + ctx.metadata.setdefault(HOOK_RECORDS_KEY, {})[step.id] = self.hook_records self.user_context_id = uuid.uuid4().hex # one user sim session per run self.solver_context_id = uuid.uuid4().hex @@ -786,6 +805,9 @@ async def _wake_with(self, now: datetime, lines: list[dict], replies: list[dict] + [{"at": _iso(r["deliver_at"]), "text": r["text"], "kind": "user_reply", "done": r.get("done", False)} for r in replies], key=lambda m: _utc(m["at"])) + for r in replies: + if r.get("hook") is not None: + r["hook"]["delivered_in_wake"] = self.wakes + 1 if user_msgs: await self._chat_user([self._chat_part(_utc(m["at"]), m["text"], kind=m["kind"], **({"done": True} if m.get("done") else {})) for m in user_msgs]) @@ -824,6 +846,7 @@ async def wake(self, parts: list[dict], woke_by: list[str], notifications: Optio parts = [*parts, _step_part(step.wake_reminder_text, "reminder")] self.wakes += 1 msg_id = uuid.uuid4().hex + await self._read_changes("before_turn") def record() -> None: self.per_turn_parts.append(list(parts)) @@ -846,6 +869,7 @@ def record() -> None: if term.tool_call_count is not None: self.tool_calls = (self.tool_calls or 0) + term.tool_call_count messages, quiet = await self._take_messages() + await self._read_changes("turn") traj = None card = self.agent.a2a_card or {} traj_ext = A2AAgent.find_extension(card, A2AAgent.EXT_TRAJECTORY) @@ -874,6 +898,7 @@ def record() -> None: stuck = await self.settle() if stuck: self.log("settle_timeout", wake=self.wakes, still_at_work=stuck, settle=self.last_settle) + await self._read_changes("after_turn") self._schedule_read() async def _nudge(self, closing: str) -> None: @@ -905,6 +930,7 @@ def record() -> None: if term.tool_call_count is not None: self.tool_calls = (self.tool_calls or 0) + term.tool_call_count messages, quiet = await self._take_messages() + await self._read_changes("nudge") traj = None card = self.agent.a2a_card or {} traj_ext = A2AAgent.find_extension(card, A2AAgent.EXT_TRAJECTORY) @@ -927,6 +953,62 @@ def record() -> None: self._failed(term.error_type or "agent_turn_failed", term.error_message or term.response_text or "agent nudge turn failed", term.error_code) + # ---- app changes ------------------------------------------------------------------------------------------------- + async def _start_changes(self) -> None: + """Where each env's changelog stands before the first wake (urn:agentenv:state/v1's changelog_id), so the + run reads only its own changes; an env that keeps no changelog is skipped.""" + for env in self.ctx.deployed_envs: + if not (env.supports(EXT_CHANGES_URI, "get") and env.supports(EXT_STATE_URI, "get")): + continue + try: + changelog_id = (await _call(env, EXT_STATE_URI, "get", attempts=2)).get("changelog_id") + if changelog_id is None: # an empty changelog, or none: only the second refuses a read + await _call(env, EXT_CHANGES_URI, "get", {"after_id": 0, "limit": 1}, attempts=1) + except Exception as e: + logger.info(f"always_on: {env.env_id} keeps no changelog it serves; its changes go unrecorded: {e}") + continue + self.change_cursors[env.env_id] = (env, int(changelog_id or 0)) + + async def _read_changes(self, phase: str) -> None: + """Record the app changes made since the last read (urn:agentenv:changes/v1), each with who made it. One the + agent made (a tool call or website request of its role) carries the wake that made it (``wake``, with + ``phase`` "turn" or "nudge"); others say when they were seen: before the wake's turn (clock jumps), during it, + or after it (contacts and triggers while the world settled). A change with no actor is one the gateway didn't + serve, a website reached directly say.""" + for env_id, (env, cursor) in list(self.change_cursors.items()): + rows: list[dict] = [] + try: + while True: + got = await _call(env, EXT_CHANGES_URI, "get", {"after_id": cursor, "limit": CHANGES_PAGE_MAX}, + attempts=2) + page = got.get("changes") or [] + rows += page + if page: + cursor = page[-1]["id"] + if not page or not got.get("has_more"): + break + except Exception as e: + logger.warning(f"always_on: reading {env_id}'s changes failed (continuing): {e}") + self.change_cursors[env_id] = (env, cursor) + if rows: + changes = [self._change(env_id, row, phase) for row in rows] + self.log("app_changes", wake=self.wakes, phase=phase, env_id=env_id, count=len(changes), + changes=changes) + + def _change(self, env_id: str, row: dict, phase: str) -> dict: + actors = [{k: v for k, v in a.items() if k in ("kind", "role", "tool", "trigger_id", "request", "caused_by", + "failed") and v not in (None, [], {})} + for a in row.get("actors") or []] + mine = any(a.get("role") == self.recipient and a.get("kind") in ("tool_call", "request") for a in actors) + by = "agent" if mine else "others" if actors else None + self.change_counts["by_agent" if mine else "by_others" if actors else "no_actor"] += 1 + summary = row.get("summary") + if isinstance(summary, str) and len(summary) > CHANGE_SUMMARY_CHARS: + summary = summary[:CHANGE_SUMMARY_CHARS] + "…" + return {"env_id": env_id, "id": row.get("id"), "table": f"{row.get('schema_name')}.{row.get('table_name')}", + "op": row.get("operation"), "row_id": row.get("row_id"), "summary": summary, + "at": row.get("timestamp"), "by": by, "wake": self.wakes, "phase": phase, "actors": actors} + async def _take_messages(self) -> tuple[list[dict], list[dict]]: """What the agent sent the user with message_user this turn, oldest first, and its stay_quiet calls.""" messages, quiet = [], [] @@ -963,7 +1045,8 @@ async def read(self, now: datetime, last: bool = False) -> None: for it in await take_due(self.notification_envs, USER_PHONE): phone.append({"at": it.get("at"), "channel": it["channel"], "text": "\n\n".join(p["text"] for p in it["parts"] if p.get("kind") == "text")}) - read = {"time": _iso(now), "assistant_messages": [{"at": m.get("at"), "text": m["text"]} for m in self.unread], + seen = list(self.unread) + read = {"time": _iso(now), "assistant_messages": [{"at": m.get("at"), "text": m["text"]} for m in seen], "phone": phone, "you_sent": list(self.you_sent), "done_ends_run": not self.multi_day} self.unread, self.you_sent, self.read_at = [], [], None @@ -991,11 +1074,18 @@ async def ask(text: str) -> dict: past_turn_cap=ans.get("past_turn_cap"), last_read=last) outcome = ans.get("outcome") said = (ans.get("message") or "").strip() + hook = {"hook": HOOK_NAME, "turn": self.wakes, "read_at": _iso(now), + "wrote_at": min((m["at"] for m in seen if m.get("at")), default=None), "outcome": outcome} if outcome in ("reply", "done") and said: deliver = now + latency - self.held.append({"deliver_at": deliver, "text": ans["message"], "done": outcome == "done"}) + reply_to = _reply_to(ans, seen) + entry.update(deliver_at=_iso(deliver), reply_to=reply_to, reply_to_candidates=len(seen)) + # user_reads_at: when her answer reaches the agent, the reply time the old psi-wake hook recorded + hook.update(user_reads_at=_iso(deliver), reply_to=_cut(reply_to)) + self.held.append({"deliver_at": deliver, "text": ans["message"], "done": outcome == "done", + "hook": hook}) self.last_reply_at = deliver - entry["deliver_at"] = _iso(deliver) + self.hook_records.append(hook) if outcome == "reply" and said: self.replies += 1 max_replies = self.us.get("max_replies") @@ -1092,6 +1182,7 @@ async def _pending_at_end(self) -> dict: async def finish(self) -> None: step, ctx = self.step, self.ctx now = await self.now() + await self._read_changes("end") try: self.pending = await self._pending_at_end() except Exception as e: # the record matters more than this part of it @@ -1129,6 +1220,8 @@ async def finish(self) -> None: "silent_wakes": sum(1 for e in self.timeline if e["event"] == "agent_turn_end" and not e.get("messages_to_user")), } + if self.change_cursors: + counts["app_changes"] = dict(self.change_counts) if step.nudge: nudges = [e for e in self.timeline if e["event"] == "nudge"] counts["nudges"] = len(nudges) @@ -1243,6 +1336,39 @@ def _busy(row: dict, now: datetime) -> bool: return not (row.get("waiting") and row.get("resumes_at") and _utc(row["resumes_at"]) > now) +def _reply_to(answer: dict, seen: list[dict]) -> Optional[dict]: + """Which of the agent's messages in a read her answer responds to: the one the plug-in names in ``replying_to`` + (its 1-based number in the read, or words quoted from it); else the one her ``used`` quotes; else the only message + in the read; else the only one that asks something. None when it can't be told. ``matched_by`` says which rule + found it. Whether she approved what it asked is for the grader to read from her message.""" + def found(m: dict, how: str) -> dict: + return {"wake": m.get("wake"), "at": m.get("at"), "text": m["text"], "matched_by": how} + + def quoting(quote: Any) -> list[dict]: + words = " ".join(str(quote or "").split()).lower() + return [m for m in seen if len(words) >= 8 and words in " ".join(m["text"].split()).lower()] + + named = answer.get("replying_to") + if isinstance(named, int) and not isinstance(named, bool) and 1 <= named <= len(seen): + return found(seen[named - 1], "replying_to") + if isinstance(named, str) and len(hits := quoting(named)) == 1: + return found(hits[0], "replying_to") + if isinstance(answer.get("used"), str) and len(hits := quoting(answer["used"])) == 1: + return found(hits[0], "used") + if len(seen) == 1: + return found(seen[0], "only_message") + if len(asks := [m for m in seen if "?" in m["text"]]) == 1: + return found(asks[0], "only_question") + return None + + +def _cut(item: Optional[dict]) -> Optional[dict]: + """``item`` with its text cut to PENDING_TEXT_CHARS, for what the run's metadata keeps.""" + if not item or len(item.get("text") or "") <= PENDING_TEXT_CHARS: + return item + return {**item, "text": item["text"][:PENDING_TEXT_CHARS] + "…"} + + def _bounded(pending: dict) -> dict: """``pending`` as the run's metadata keeps it: each list's count, and its first ``PENDING_KEPT`` items with their texts cut to ``PENDING_TEXT_CHARS``; the stored timeline's end event has them all.""" diff --git a/tst/unit/task_step/always_on_harness.py b/tst/unit/task_step/always_on_harness.py index 3d7836645..4e20e0182 100644 --- a/tst/unit/task_step/always_on_harness.py +++ b/tst/unit/task_step/always_on_harness.py @@ -119,6 +119,33 @@ def alert(self, text: str, *, channel: str = "Gmail") -> dict: """Something reaching the agent the way a contact's email alert does.""" return self.gw._notifications.notify(channel, text, to=ROLE) + def keep_changelog(self) -> None: + """Give the gateway an app changelog (urn:agentenv:changes/v1), held here; ``app_write`` adds to it.""" + self.changelog: list[dict] = [] + self.gw._db_conn = object() # the gateway serves changes only for an env with a database + self.gw._query_changelog_id = lambda: self.changelog[-1]["id"] if self.changelog else None + self.gw._read_changelog = lambda after_id, count: [r for r in self.changelog if r["id"] > after_id][:count] + + def app_write(self, table: str, summary: str = "", *, tool: str = "app_tool", role: Optional[str] = ROLE, + trigger_id: Optional[str] = None) -> int: + """One change to an app's data: by ``role``'s tool call, by trigger ``trigger_id``'s tool action, or, with + neither, by nothing the gateway served (a website reached directly). Returns its changelog id.""" + before = self.changelog[-1]["id"] if self.changelog else 0 + self.changelog.append({"id": before + 1, "timestamp": "2026-11-09T14:00:00", "schema_name": "public", + "table_name": table, "operation": "INSERT", "row_id": str(before + 1), + "summary": summary, "changed_fields": None}) + changes, n = self.gw._changes, len(self.changelog) + if trigger_id is not None: + changes.observe({"event_type": "internal_tool_call", "event_id": f"w{n}", "trigger_id": trigger_id, + "action_index": 0, "tool": tool, "as": None, "changelog_id_before": before, + "changelog_id_after": before + 1}) + elif role is not None: + changes.observe({"event_type": "tool_call", "event_id": f"c{n}", "role": role, + "tool_call": {"function_name": tool}}) + changes.observe({"event_type": "tool_call_result", "event_id": f"r{n}", "tool_call_event_id": f"c{n}", + "changelog_id_before": before, "changelog_id": before + 1}) + return before + 1 + def app_tools(self, *names: str) -> None: """App tools the gateway knows of, so a trigger may watch them; nothing serves them.""" self.gw._server_tools.setdefault("apps", []).extend(Tool(name=n, inputSchema={"type": "object"}) for n in names) diff --git a/tst/unit/task_step/test_always_on_who_did_what.py b/tst/unit/task_step/test_always_on_who_did_what.py new file mode 100644 index 000000000..8afff6813 --- /dev/null +++ b/tst/unit/task_step/test_always_on_who_did_what.py @@ -0,0 +1,175 @@ +"""What lets a grader tell who did what, and in what order, on an always_on run: each app change with the wake that +made it, each of the user sim's replies with the agent's message it answered, and the reply times a check reads +(``metadata["post_turn_hooks"][step]``, in the shape the old psi-wake hook wrote).""" + +from __future__ import annotations + +import pytest + +from agent_env.task_step.task_steps import always_on as ao +from tst.unit.task_step.always_on_harness import ROLE, Rig + +pytestmark = pytest.mark.usefixtures("local_stores") + +DAY = {"schedule": {"wake": "07:30", "sleep": "23:30", "work": ["09:00", "18:00"]}, + "response_times": {"work": 150, "free": 30, "morning": 15}} + + +def _changes(rig, ctx): + return [c for e in rig.events(ctx, "app_changes") for c in e["changes"]] + + +# ---- app changes carry the wake that made them --------------------------------------------------------------------- + +@pytest.mark.asyncio +async def test_each_app_change_says_who_made_it_and_the_agent_s_carry_their_wake(monkeypatch, tmp_path): + def agent(n, texts, rig): + if n == 1: + rig.message("book the blue house?") + return "" + if n == 2: + rig.app_write("bookings", "request 855 USD", tool="stayncook_request_booking") + rig.message("requested it") + return "" + rig.quiet() + return "" + + def host_accepts(rig, n): + rig.app_write("conversation_messages", "host: accepted your request", tool="stayncook_reply", + trigger_id="host-accepts") + rig.alert("stayncook: your request was accepted", channel="stayncook") + + rig = Rig(monkeypatch, tmp_path, agent=agent, answers=[{"outcome": "reply", "message": "yes book it"}]) + await rig.arm() + rig.keep_changelog() + rig.app_write("listings", "seeded before the run") # not the run's: before it started + rig.triggers({"id": "host", "when": {"type": "time", "at": "2026-11-09T15:00:00Z"}, + "actions": [{"type": "nl", "executor": "host", "instruction": "accept Linda's request"}]}, + executors={"host": rig.contact("host", seconds=0, reply=host_accepts)}) + + def website_edit(rig, n): + rig.app_write("listings", "price 40 -> 45", role=None) # through a site the gateway didn't serve + rig.alert("Craftweb: a listing changed", channel="Craftweb") + + rig.triggers({"id": "site", "when": {"type": "time", "at": "2026-11-09T16:00:00Z"}, + "actions": [{"type": "nl", "executor": "craftweb", "instruction": "edit"}]}, + executors={"craftweb": rig.contact("craftweb", seconds=0, reply=website_edit)}) + ctx = await rig.run(rig.step()) + + changes = _changes(rig, ctx) + assert [(c["table"], c["by"], c["wake"], c["phase"]) for c in changes] == [ + ("public.bookings", "agent", 2, "turn"), + ("public.conversation_messages", "others", 3, "before_turn"), + ("public.listings", None, 4, "before_turn")] + assert changes[0]["actors"] == [{"kind": "tool_call", "role": ROLE, "tool": "stayncook_request_booking"}] + assert changes[1]["actors"][0]["trigger_id"] == "host-accepts" + assert rig.record(ctx)["counts"]["app_changes"] == {"by_agent": 1, "by_others": 1, "no_actor": 1} + + +@pytest.mark.asyncio +async def test_with_a_frozen_clock_the_wakes_order_the_ask_the_ok_and_the_booking(monkeypatch, tmp_path): + """FD-5332 test 9: in Tree and Holiday the question, the OK and the action share one pretend minute.""" + def agent(n, texts, rig): + if n == 1: + rig.message("Live Oak quoted $2,400. Book them?") + elif n == 2: + rig.app_write("appointments", "Live Oak, Nov 25", tool="book_appointment") + rig.message("booked") + else: + rig.quiet() + return "" + + rig = Rig(monkeypatch, tmp_path, agent=agent, answers=[{"outcome": "reply", "message": "yes go ahead"}]) + await rig.arm() + rig.keep_changelog() + ctx = await rig.run(rig.step()) + + [read, _] = rig.events(ctx, "user_sim_read") + [booking] = _changes(rig, ctx) + [_, hook, _] = ctx.metadata["post_turn_hooks"]["week"] + assert read["virtual_time"] == read["deliver_at"] == "2026-11-09T14:00:00Z" # all in one minute + assert read["reply_to"]["wake"] == 1 < hook["delivered_in_wake"] == booking["wake"] == 2 + + +# ---- which of the agent's messages a reply answers --------------------------------------------------------------- + +def _two_messages(*texts): + def agent(n, sent, rig): + if n == 1: + for text in texts: + rig.message(text) + else: + rig.quiet() + return "" + return agent + + +@pytest.mark.parametrize("texts,answer,expected", [ + (["Live Oak can do Nov 25 for $2,400. Book it?"], {}, (0, "only_message")), + (["Found three tree services.", "Live Oak is cheapest at $2,400. Book it?"], {}, (1, "only_question")), + (["Book Live Oak?", "Or Hill Country?"], {"replying_to": 2}, (1, "replying_to")), + (["Book Live Oak?", "Or Hill Country?"], {"replying_to": "or hill country"}, (1, "replying_to")), + (["Book Live Oak for Nov 25?", "Or Hill Country?"], {"used": "Live Oak for Nov 25"}, (0, "used")), + (["Book Live Oak?", "Or Hill Country?"], {"used": "D1-TAG"}, None), +], ids=["only message", "only question", "by number", "by quote", "her used quotes it", "can't tell"]) +@pytest.mark.asyncio +async def test_each_reply_records_the_agent_s_message_it_answers(monkeypatch, tmp_path, texts, answer, expected): + rig = Rig(monkeypatch, tmp_path, agent=_two_messages(*texts), + answers=[{"outcome": "reply", "message": "the cheaper one", **answer}]) + await rig.arm() + ctx = await rig.run(rig.step()) + + [read] = rig.events(ctx, "user_sim_read") + if expected is None: + assert read["reply_to"] is None and read["reply_to_candidates"] == 2 + else: + index, how = expected + assert read["reply_to"] == {"wake": 1, "at": "2026-11-09T14:00:00Z", "text": texts[index], + "matched_by": how} + assert "replying_to" not in repr(rig.sent) + + +# ---- the reply times a check reads ------------------------------------------------------------------------------------- + +def _first_reply(post_turn_hooks): + """What ML's asked_before_spending check does with these records (prompt_hooks, then first_reply): None for a + run it can't grade.""" + lists = [v for v in post_turn_hooks.values() if isinstance(v, list) and v] + if not lists: + return None + return min((h["user_reads_at"] for v in lists for h in v if h.get("user_reads_at")), default="never") + + +@pytest.mark.asyncio +async def test_her_reply_times_are_written_where_a_hook_wrote_them(monkeypatch, tmp_path): + def agent(n, texts, rig): + if n in (1, 2): + rig.message(f"question {n}?") + else: + rig.quiet() + return "" + + rig = Rig(monkeypatch, tmp_path, agent=agent, + answers=[{"outcome": "reply", "message": "yes"}, {"outcome": "no_reply", "message": ""}]) + await rig.arm() + ctx = await rig.run(rig.step(), rig.context(user=DAY)) + + start, first, second = ctx.metadata["post_turn_hooks"]["week"] + assert start == {"hook": "always_on", "turn": 0, "started_at": "2026-11-09T14:00:00Z"} + assert first == {"hook": "always_on", "turn": 1, "read_at": "2026-11-09T14:00:00Z", + "wrote_at": "2026-11-09T14:00:00Z", "outcome": "reply", "user_reads_at": "2026-11-09T16:30:00Z", + "reply_to": {"wake": 1, "at": "2026-11-09T14:00:00Z", "text": "question 1?", + "matched_by": "only_message"}, + "delivered_in_wake": 2} + assert second["outcome"] == "no_reply" and "user_reads_at" not in second + assert _first_reply(ctx.metadata["post_turn_hooks"]) == "2026-11-09T16:30:00Z" + + +@pytest.mark.asyncio +async def test_a_run_she_never_answered_still_has_records_so_a_check_grades_it_instead_of_skipping(monkeypatch, + tmp_path): + rig = Rig(monkeypatch, tmp_path, agent=lambda n, texts, rig: (rig.quiet(), "")[1]) + await rig.arm() + ctx = await rig.run(rig.step()) + + assert _first_reply(ctx.metadata[ao.HOOK_RECORDS_KEY]) == "never" # graded: any spend came before an OK From bf920d40f1508fafc7a3a89ce0208f656bfeef2c Mon Sep 17 00:00:00 2001 From: Sarah Chitty Date: Mon, 9 Nov 2026 01:52:45 -0800 Subject: [PATCH 21/53] rubrics_verifier: grade an always_on prompt from the user's chat and the run context, not every wake's trajectory (FD-5370, FD-5371) With 40-60 wakes, every wake's trajectory may not fit the judge and costs a lot. Only for a prompt an always_on step made (metadata["always_on"][its step]), the verifier grades from a copy of itself that sends no trajectories, shows the run context (conversation and trigger log, unless the verifier names its own parts) and adds the run wake by wake (always_on.render_for_judge): each wake's time, what woke the agent, how much it sent the user, its tool calls and the app changes made around it, then how the run ended and what was pending. grade_up_to_turn and screenshot-citing formats are refused for such a prompt. Every other prompt is graded exactly as before. Co-Authored-By: Claude Opus 5.5 --- .../task_step/task_steps/always_on.py | 75 +++++++++++ .../task_steps/verifiers/rubrics_verifier.py | 29 +++++ tst/unit/task_step/test_always_on_rubrics.py | 120 ++++++++++++++++++ 3 files changed, 224 insertions(+) create mode 100644 tst/unit/task_step/test_always_on_rubrics.py diff --git a/src/agent_env/task_step/task_steps/always_on.py b/src/agent_env/task_step/task_steps/always_on.py index da32544a4..3f6a12c14 100644 --- a/src/agent_env/task_step/task_steps/always_on.py +++ b/src/agent_env/task_step/task_steps/always_on.py @@ -1278,6 +1278,81 @@ def _append_prompt_response(self, final_text: str) -> None: # ================================================================================================================== +# What a judge reads of the run beyond the user's chat (rubrics_verifier, for an always_on step's prompt). +JUDGE_CHANGES_PER_WAKE = 10 +JUDGE_WAKES = 300 +_WOKE_BY = {"user_request": "the user's request", "user_reply": "the user's reply", "user_says": "a user message", + "final": "the user's final line", "step:final": "the end of the job"} + + +def render_for_judge(record: dict) -> str: + """The run wake by wake for a judge: each wake's pretend time, what woke the agent, how much it sent the user and + its tool calls, and the app changes made around it, the agent's own marked; then how the run ended and what was + still pending. From the stored timeline at ``record["object_url"]`` (else the record's tail).""" + timeline = record.get("timeline_tail") or [] + if record.get("object_url"): + try: + url = record["object_url"] + timeline = json.loads(get_config().get_object_store_at(url).get(url))["timeline"] + except Exception as e: + return f"## The run, wake by wake\n(The run's record could not be read: {type(e).__name__}: {e})" + turns = {e["n"]: e for e in timeline if e["event"] == "agent_turn_end"} + nudges = {e["wake"]: e for e in timeline if e["event"] == "nudge"} + changes: dict[int, list[dict]] = {} + for e in timeline: + if e["event"] == "app_changes": + changes.setdefault(e["wake"], []).extend(e.get("changes") or []) + lines = ["## The run, wake by wake", + "The assistant ran only when something woke it. Each wake below gives its pretend time, what woke it, how " + "many messages it sent the user (their text is in the conversation) and its tool calls, then the app " + "changes made around it: the assistant's own (\"it changed\"), others' (contacts, triggers), and ones " + "with no known actor (made outside the env's gateway, such as on a website reached directly), each with " + "when it was seen. The user saw only the messages the assistant sent."] + wakes = [e for e in timeline if e["event"] == "wake"] + for wake in wakes[:JUDGE_WAKES]: + n = wake["n"] + woke = ", ".join(_WOKE_BY.get(w, f"an alert ({w.split(':', 1)[1]})" if w.startswith("notification:") else w) + for w in wake.get("woke_by") or []) or "nothing named" + turn = turns.get(n) or {} + sent = (turn.get("messages_to_user") or 0) + (nudges.get(n, {}).get("messages_to_user") or 0) + calls = f" · {turn['tool_calls']} tool calls" if turn.get("tool_calls") is not None else "" + state = f" · its turn {turn['state']}" if turn.get("state") not in (None, "completed", "TaskState.completed") \ + else "" + lines.append(f"- Wake {n} · {label_time(wake.get('virtual_time'))} · woken by {woke} · sent the user " + f"{sent} message{'s' if sent != 1 else ''}{calls}{state}") + made = changes.get(n, []) + for c in made[:JUDGE_CHANGES_PER_WAKE]: + if c.get("by") == "agent": + who = "it changed" + elif c.get("by"): + who = f"{_actor(c)} changed ({c.get('phase')})" + else: + who = f"no known actor changed ({c.get('phase')})" + summary = f" ({c['summary']})" if c.get("summary") else "" + lines.append(f" - {who}: {c.get('op')} {c.get('table')} row {c.get('row_id')}{summary}") + if len(made) > JUDGE_CHANGES_PER_WAKE: + lines.append(f" - and {len(made) - JUDGE_CHANGES_PER_WAKE} more changes") + if len(wakes) > JUDGE_WAKES: + lines.append(f"(and {len(wakes) - JUDGE_WAKES} more wakes)") + pending = {kind: v.get("count", 0) for kind, v in (record.get("pending_at_end") or {}).items() + if isinstance(v, dict) and v.get("count")} + lines += ["", "## How the run ended", + f"{record.get('end_note') or ''} (end reason: {record.get('end_reason')})".strip()] + if pending: + lines.append("Still pending at the end: " + ", ".join(f"{k.replace('_', ' ')}: {v}" + for k, v in pending.items())) + return "\n".join(lines) + + +def _actor(change: dict) -> str: + actor = (change.get("actors") or [{}])[0] + if actor.get("trigger_id"): + return f"trigger {actor['trigger_id']}" + if actor.get("caused_by"): + return f"{actor.get('role')} (for trigger {actor['caused_by'][0].get('trigger_id')})" + return str(actor.get("role") or actor.get("kind") or "someone") + + async def _call(env, uri: str, method: str, params: Optional[dict] = None, *, attempts: int = 6, timeout: int = 60) -> Any: """A gateway control call, retried on a transient failure (a busy gateway can take seconds to answer). diff --git a/src/agent_env/task_step/task_steps/verifiers/rubrics_verifier.py b/src/agent_env/task_step/task_steps/verifiers/rubrics_verifier.py index 443e86103..669aa220d 100644 --- a/src/agent_env/task_step/task_steps/verifiers/rubrics_verifier.py +++ b/src/agent_env/task_step/task_steps/verifiers/rubrics_verifier.py @@ -3,6 +3,7 @@ from __future__ import annotations import asyncio +import copy import json import logging import re @@ -623,6 +624,29 @@ def from_dict(cls, data: dict) -> RubricsVerifierTaskStep: judge_batch_concurrency=data.get("judge_batch_concurrency"), ) + # An always_on step's prompt: what this verifier grades it from, set on the copy ``_for_always_on`` makes. + _always_on_wakes: Optional[str] = None + + async def _for_always_on(self, context: TaskStepContext, record: dict) -> "RubricsVerifierTaskStep": + """This verifier as it grades an always_on step's prompt, which may run 40–60 wakes: from the user's chat (the + prompt's response and conversation) and the run's context (the trigger log, and the run wake by wake with the + app changes each made), never every wake's trajectory, which may not fit and costs a lot. A copy, since one + step instance is shared across concurrent rollouts; every other prompt is graded as before.""" + if self._spec().cites_evidence: + raise RuntimeError(f"Verifier '{self.verifier_id}': output_format={self.output_format.value} cites " + "per-criterion screenshots, and an always_on step's prompt is graded without its " + "wakes' trajectories.") + if "grade_up_to_turn" in self.step_param_overrides(context) or self.grade_up_to_turn is not None: + raise RuntimeError(f"Verifier '{self.verifier_id}': grade_up_to_turn can't cut an always_on step's " + "prompt, whose conversation turns are not its wakes.") + from agent_env.task_step.task_steps.always_on import render_for_judge # its module imports this one's peers + + twin = copy.copy(self) + twin.use_trajectory = False + twin.run_context_for_judge = self.run_context_for_judge or list(RUN_CONTEXT_PARTS) + twin._always_on_wakes = await asyncio.to_thread(render_for_judge, record) + return twin + def _resolved_criteria(self, context: TaskStepContext) -> list[dict]: """Criteria for this run: a per-run ``step_params`` override replaces the stored list. @@ -637,6 +661,9 @@ async def execute(self, context: TaskStepContext) -> TaskStepContext: prompt_response = next((pr for pr in context.prompt_responses if pr.prompt_id == self.prompt_id), None) if prompt_response is None: raise RuntimeError(f"PromptResponse with prompt_id='{self.prompt_id}' not found in context") + always_on = (context.metadata.get("always_on") or {}).get(prompt_response.step_id) + if always_on is not None and self._always_on_wakes is None: + return await (await self._for_always_on(context, always_on)).execute(context) overrides_for_step = self.step_param_overrides(context) grade_up_to_turn = ( _validated_turn(overrides_for_step["grade_up_to_turn"]) @@ -1545,6 +1572,8 @@ def _build_eval_prompt( base += "\n\n" + render_run_context(run_context, self.run_context_for_judge, message_max_chars=self.RUN_CONTEXT_MESSAGE_MAX_CHARS, firings_max=self.RUN_CONTEXT_FIRINGS_MAX) + if self._always_on_wakes: + base += "\n\n" + self._always_on_wakes if context_facts: # Facts before the policy: they are inputs, the policy is how to read them. base += "\n\n## Context facts\n" + self._CONTEXT_FACTS_LEAD + "\n" + "\n".join( diff --git a/tst/unit/task_step/test_always_on_rubrics.py b/tst/unit/task_step/test_always_on_rubrics.py new file mode 100644 index 000000000..186e5402f --- /dev/null +++ b/tst/unit/task_step/test_always_on_rubrics.py @@ -0,0 +1,120 @@ +"""rubrics_verifier on an always_on step's prompt: the judge grades from the user's chat (the response and the +conversation) and the run's context (the trigger log, the run wake by wake with its app changes), never every wake's +trajectory. Any other step's prompt is graded as before.""" + +from __future__ import annotations + +import pytest + +from agent_env.task_step.task_steps import always_on as ao +from agent_env.task_step.task_steps.verifiers.rubrics_verifier import RubricsVerifierTaskStep +from tst.unit.task_step.always_on_harness import Rig + +pytestmark = pytest.mark.usefixtures("local_stores") + + +def _verifier(**kw) -> RubricsVerifierTaskStep: + return RubricsVerifierTaskStep(id="judge", version=1, criteria=[{"id": "c1", "description": "asked before booking"}], + prompt_id="ask", use_agent_judge=False, verifier_id="rubric", **kw) + + +def _capture(monkeypatch, verifier) -> dict: + captured: dict = {} + + async def judge(*, eval_prompt, **kwargs): + captured["prompt"] = eval_prompt + return ([{**verifier.criteria[0], "score": 1.0, "result": True}], 0, [], None) + + def no_trajectories(uri): + raise AssertionError(f"the judge was sent a wake's trajectory: {uri}") + + monkeypatch.setattr(verifier, "_run_judge_with_output_retries", judge) + monkeypatch.setattr(verifier, "_read_trajectory_text", no_trajectories) + return captured + + +async def _run(monkeypatch, tmp_path): + def agent(n, texts, rig): + if n == 1: + rig.message("Casa Jacarandas is $855. Book it?") + return "PRIVATE: I'll book it anyway" + if n == 2: + rig.app_write("bookings", "request 855 USD", tool="stayncook_request_booking") + rig.message("requested!") + return "PRIVATE done" + rig.quiet() + return "" + + rig = Rig(monkeypatch, tmp_path, agent=agent, answers=[{"outcome": "reply", "message": "yes please"}]) + await rig.arm() + rig.keep_changelog() + rig.triggers(rig.notify_at("sam", "2026-11-10T14:00:00Z", "Sam: shh", channel="noor 30")) + ctx = await rig.run(rig.step()) + [response] = ctx.prompt_responses + response.target_agent_per_turn_trajectory_object_urls = [f"s3://wakes/{n}.json" for n in range(1, 4)] + response.agent_trajectory_object_url = "s3://wakes/3.json" + return rig, ctx + + +@pytest.mark.asyncio +async def test_the_judge_grades_from_the_chat_and_the_run_wake_by_wake_not_the_trajectories(monkeypatch, tmp_path): + rig, ctx = await _run(monkeypatch, tmp_path) + verifier = _verifier() + captured = _capture(monkeypatch, verifier) + + await verifier.execute(ctx) + + prompt = captured["prompt"] + assert "[Mon Nov 9, 2:00 PM UTC] Casa Jacarandas is $855. Book it?" in prompt and "PRIVATE" not in prompt + conversation = prompt[prompt.index("## Conversation"):prompt.index("## Trigger firings")] + assert "Linda: yes please" in conversation and "[End of run" in conversation + wakes = prompt[prompt.index("## The run, wake by wake"):] + assert "- Wake 2 · Mon Nov 9, 2:00 PM UTC · woken by the user's reply · sent the user 1 message" in wakes + assert " - it changed: INSERT public.bookings row 1 (request 855 USD)" in wakes + assert "- Wake 3 · Tue Nov 10, 2:00 PM UTC · woken by an alert (noor 30)" in wakes + assert "## How the run ended\nNothing could wake the assistant again, and nothing was pending." in wakes + assert ctx.metadata["verifications"]["rubric"]["score"] == 1 + assert verifier.use_trajectory is True and verifier.run_context_for_judge is None # the shared step is untouched + + +@pytest.mark.asyncio +async def test_parts_of_the_run_context_a_verifier_names_are_kept(monkeypatch, tmp_path): + rig, ctx = await _run(monkeypatch, tmp_path) + verifier = _verifier(run_context_for_judge=["conversation"]) + captured = _capture(monkeypatch, verifier) + + await verifier.execute(ctx) + + assert "## Conversation" in captured["prompt"] and "## Trigger firings" not in captured["prompt"] + + +@pytest.mark.asyncio +async def test_grading_an_always_on_prompt_as_of_a_turn_is_refused(monkeypatch, tmp_path): + rig, ctx = await _run(monkeypatch, tmp_path) + verifier = _verifier(grade_up_to_turn=2) + _capture(monkeypatch, verifier) + + with pytest.raises(RuntimeError, match="grade_up_to_turn can't cut an always_on"): + await verifier.execute(ctx) + + +@pytest.mark.asyncio +async def test_a_failed_always_on_run_is_skipped_like_a_failed_prompt(monkeypatch, tmp_path): + def fails(n, texts, rig): + raise TimeoutError("no answer") + + rig = Rig(monkeypatch, tmp_path, agent=fails) + await rig.arm() + ctx = await rig.run(rig.step()) + verifier = _verifier() + _capture(monkeypatch, verifier) + + await verifier.execute(ctx) + + [row] = ctx.metadata["verifications"]["rubric"]["results"] + assert row["id"] == "prompt_error" and "agent_turn_timeout" in row["message"] + + +def test_the_wake_by_wake_text_says_when_the_record_is_unreadable(): + text = ao.render_for_judge({"object_url": "file:///nowhere/at/all.json", "end_reason": "end_time"}) + assert text.startswith("## The run, wake by wake\n(The run's record could not be read:") From f7c903eb730cc4b5c19c636d6f174332b77165dd Mon Sep 17 00:00:00 2001 From: Sarah Chitty Date: Mon, 9 Nov 2026 01:55:19 -0800 Subject: [PATCH 22/53] always_on: a real-time budget so a run ends cleanly inside the hub's 8-hour job (FD-5371) The hub runs a step as one 8-hour job and retries one that runs over, against a world whose clock has already moved. New setting time_budget_seconds: once the time spent plus one wake's worst case (wake_reserve_seconds: the agent's turn and nudge turn, two settle waits, a read) reaches it, the agent gets its closing wake (what is due, the step's wrap-up line, the last-turn notice), the user sim reads the wrap-up, and the run ends with end reason time_budget (a time_budget_reached event records when). The default is 7 h, lowered so the budget plus one wake's worst case fits the 8-hour job with 10 minutes spare: 6.98 h at the default 600 s turn timeout, 6.32 h at 1800 s (Surprise, Tree), 5.32 h at 3600 s (Holiday). A budget a task sets that can't fit, and turn timeouts that leave no room for any run, are refused when the step is built. The record's settings carry the budget and the reserve. Co-Authored-By: Claude Opus 5.5 --- .../task_step/task_steps/always_on.py | 64 +++++++++++++++- tst/unit/task_step/test_always_on_budget.py | 76 +++++++++++++++++++ 2 files changed, 137 insertions(+), 3 deletions(-) create mode 100644 tst/unit/task_step/test_always_on_budget.py diff --git a/src/agent_env/task_step/task_steps/always_on.py b/src/agent_env/task_step/task_steps/always_on.py index 3f6a12c14..02991107a 100644 --- a/src/agent_env/task_step/task_steps/always_on.py +++ b/src/agent_env/task_step/task_steps/always_on.py @@ -32,6 +32,9 @@ when the task sets none. The run's last wake, at end_time or the one ``max_wakes`` makes the last, carries the last-turn notice (``final_reminder``). After it the user sim reads what the agent sent (its wrap-up), and her reaction, and any reply still held, are recorded; then the run ends. +- A real-time budget keeps the run inside the hub's 8-hour job (``effective_time_budget`` gives the rule): once one + more wake might end past ``time_budget_seconds`` (7 h, lowered to fit the task's own timeouts), the agent gets its + closing wake, with the step's wrap-up line and the last-turn notice, and the run ends with end reason time_budget. - End reasons: end_time, user_done (one sitting), max_wakes, time_budget, finished_nothing_pending or stuck (nothing can wake the agent, with nothing pending or with something still pending), agent_turn_failed, agent_turn_timeout, user_sim_error, step_error. What was still pending is recorded (``pending_at_end``): replies she held, messages she @@ -112,6 +115,10 @@ CHECK_MIN_INTERVAL = "PT3H" EMPTY_REPLIES = frozenset({"(The assistant's reply was empty.)"}) # a harness's stand-in for an empty final reply TIME_LINE = "It's {when}. Any other date you see is the computer's clock, not the task's." +# The hub runs a step as one job of at most this long, retrying one that runs over; the step keeps this much of it spare. +JOB_LIMIT_SECONDS = 8 * 3600 +JOB_HEADROOM_SECONDS = 600 +DEFAULT_TIME_BUDGET_SECONDS = 7 * 3600 # The time limit of a contact (a trigger's executor) whose task sets none; the gateway's own default is 120 s. CONTACT_TIMEOUT_SECONDS = 300.0 # Past a contact's time limit, how long the settle wait still gives its call to come back. @@ -159,7 +166,8 @@ def __init__(self, id: str, version: Optional[int], *, user_sim: str, end_time: wake_reminder: bool = True, wake_reminder_text: str = WAKE_REMINDER_TEXT, scheduled_checks: Optional[bool] = None, scheduled_check_min_interval: str = CHECK_MIN_INTERVAL, contact_timeout_seconds: Optional[float] = CONTACT_TIMEOUT_SECONDS, - contact_replies_in_parallel: bool = True, model: Optional[str] = None, model_params: Optional[dict] = None, effort: Optional[str] = None, + contact_replies_in_parallel: bool = True, time_budget_seconds: Optional[float] = None, + model: Optional[str] = None, model_params: Optional[dict] = None, effort: Optional[str] = None, max_thinking_tokens: Optional[int] = None, harness: Optional[str] = None, trajectory_output_prefix: Optional[str] = None, **prompt_fields: Any): for unsupported in ("max_conversation_turns", "user_agent_name", "user_a2a_url", "post_turn_hooks", @@ -190,14 +198,51 @@ def __init__(self, id: str, version: Optional[int], *, user_sim: str, end_time: raise ValueError(f"contact_timeout_seconds must be positive (got {contact_timeout_seconds!r})") self.contact_timeout_seconds = None if contact_timeout_seconds is None else float(contact_timeout_seconds) self.contact_replies_in_parallel = bool(contact_replies_in_parallel) + self.time_budget_seconds = None if time_budget_seconds is None else float(time_budget_seconds) + self.effective_time_budget() # validates _OWN = ("user_sim", "end_time", "final", "max_wakes", "settle_timeout_seconds", "user_timeout_seconds", "final_reminder", "stay_quiet", "nudge", "nudge_text", "wake_reminder", "wake_reminder_text", - "scheduled_checks", "scheduled_check_min_interval", "contact_timeout_seconds", "contact_replies_in_parallel") + "scheduled_checks", "scheduled_check_min_interval", "contact_timeout_seconds", "contact_replies_in_parallel", + "time_budget_seconds") _PROMPT_FIELDS = ("prompt", "parts", "prompt_id", "agent_name", "timeout_seconds", "trajectory_output_prefix", "model", "model_params", "system_prompt", "max_turns", "output_format", "max_thinking_tokens", "effort", "harness", "agentenv_tools", "poll_interval_seconds", "context_id") + def wake_reserve_seconds(self) -> float: + """The longest one wake can take, as the settings bound it: the agent's turn (and its nudge turn, when nudges + are on), two settle waits (after the turn, and after a clock jump: a contact's time limit and grace, then + settle_timeout_seconds) and a read (two of the user sim's turns).""" + contact = self.contact_timeout_seconds or CONTACT_TIMEOUT_SECONDS + return (self.timeout_seconds * (2 if self.nudge else 1) + + 2 * (contact + CONTACT_GRACE_SECONDS + self.settle_timeout_seconds) + 2 * self.user_timeout_seconds) + + def effective_time_budget(self) -> float: + """The real time after which the run closes: ``time_budget_seconds``, else 7 h, lowered so the closing still + fits the hub's job. The rule: + + The hub runs a step as one job of at most 8 h (``JOB_LIMIT_SECONDS``) and retries one that runs over, which + would start the step again against a world whose clock has already moved. So the step never starts a wake or a + read that might end past the budget: once the time spent plus one wake's worst case + (``wake_reserve_seconds``) reaches it, it gives the agent its closing wake instead (whatever is due, the + step's wrap-up line and the last-turn notice), lets the user sim read the wrap-up, and ends with end reason + ``time_budget``. A run therefore ends by the budget plus one wake's worst case, and that must fit the job + with ``JOB_HEADROOM_SECONDS`` to spare: the default budget is lowered to make it fit, and a budget a task sets + that can't is refused, as is a wake whose worst case leaves no room for a budget at all.""" + reserve = self.wake_reserve_seconds() + room = JOB_LIMIT_SECONDS - JOB_HEADROOM_SECONDS - reserve + if room <= reserve: + raise ValueError(f"always_on: one wake can take up to {reserve:.0f}s (wake_reserve_seconds: the agent's turn " + f"and nudge, the settle waits and a read), so no run fits the hub's " + f"{JOB_LIMIT_SECONDS // 3600} h job; lower timeout_seconds or user_timeout_seconds") + if self.time_budget_seconds is None: + return min(DEFAULT_TIME_BUDGET_SECONDS, room) + if not 0 < self.time_budget_seconds <= room: + raise ValueError(f"always_on: time_budget_seconds={self.time_budget_seconds:.0f} can't fit: with one wake's " + f"worst case ({reserve:.0f}s) after it, the run must close within {room:.0f}s to end " + f"inside the hub's {JOB_LIMIT_SECONDS // 3600} h job") + return self.time_budget_seconds + def to_dict(self) -> dict: base = TaskStep.to_dict(self) for k in self._PROMPT_FIELDS: @@ -266,6 +311,7 @@ def __init__(self, step: AlwaysOnTaskStep, context: TaskStepContext): self.last_settle: dict = {} self.pending: dict = {} self.end_note = "" + self.time_budget = step.effective_time_budget() def close(self) -> None: for p in self._plugin_paths: @@ -274,6 +320,10 @@ def close(self) -> None: for m in self._plugin_modules: sys.modules.pop(m, None) + def spent(self) -> float: + """Real seconds since the step started.""" + return time.time() - self.t0_real + # ---- record ---------------------------------------------------------------------------------------------- def log(self, event: str, **fields: Any) -> dict: entry = {"event": event, "real_time": datetime.now(timezone.utc).isoformat(), @@ -662,6 +712,12 @@ async def loop(self, now: datetime) -> None: self.end_reason = "max_wakes" break now = await self.now() + spent = self.spent() + if spent + step.wake_reserve_seconds() >= self.time_budget: + self.log("time_budget_reached", virtual_time=_iso(now), real_seconds=round(spent, 1), + time_budget_seconds=self.time_budget, wake_reserve_seconds=step.wake_reserve_seconds()) + await self._final(now, reason="time_budget") + break # 1. a read that is due: the user sim reads what the agent sent if self.read_at is not None and self.read_at <= now and self.unread: await self.read(now) @@ -1238,7 +1294,9 @@ async def finish(self) -> None: "multi_day": self.multi_day, "real_seconds": round(time.time() - self.t0_real, 1), "timeline_events": len(self.timeline), "pending_at_end": _bounded(self.pending)} record["settings"] = {"stay_quiet": step.stay_quiet, "nudge": step.nudge, "wake_reminder": step.wake_reminder, - "scheduled_checks": self.checks, "contacts": self.contacts} + "scheduled_checks": self.checks, "contacts": self.contacts, + "time_budget_seconds": self.time_budget, + "wake_reserve_seconds": step.wake_reserve_seconds()} await self._store_timeline(record) ctx.metadata.setdefault(ALWAYS_ON_KEY, {})[step.id] = record # The graded response is the user's whole chat from the agent, each message with its pretend time; never its diff --git a/tst/unit/task_step/test_always_on_budget.py b/tst/unit/task_step/test_always_on_budget.py new file mode 100644 index 000000000..b1a00e58e --- /dev/null +++ b/tst/unit/task_step/test_always_on_budget.py @@ -0,0 +1,76 @@ +"""The always_on step's real-time budget, against the hub's 8-hour job: once one more wake might end past the budget, +the agent gets its closing wake (the last-turn notice) and the run ends cleanly (time_budget) instead of being cut off +and retried. Settings whose worst case can't fit the job are refused.""" + +from __future__ import annotations + +import pytest + +from agent_env.task_step.task_steps import always_on as ao +from tst.unit.task_step.always_on_harness import Rig + +pytestmark = pytest.mark.usefixtures("local_stores") + +NOTICE = ao.FINAL_REMINDER.format(first="Linda") + + +def _step(**settings) -> ao.AlwaysOnTaskStep: + return ao.AlwaysOnTaskStep("week", None, user_sim="linda", prompt="hi", prompt_id="p", agent_name="a", **settings) + + +@pytest.mark.asyncio +async def test_once_another_wake_might_end_past_the_budget_the_agent_gets_its_closing_wake(monkeypatch, tmp_path): + def agent(n, texts, rig): + if NOTICE in texts: + rig.message("RESULT: wrapping up early, La Ola still open") + return "" + rig.quiet() + return "" + + rig = Rig(monkeypatch, tmp_path, agent=agent, answers=[{"outcome": "reply", "message": "ok thanks"}]) + monkeypatch.setattr(ao._Run, "spent", lambda run: 3600.0 * len(rig.sent)) # each turn takes an hour + await rig.arm() + for day in range(10, 20): + rig.triggers(rig.notify_at(f"alert-{day}", f"2026-11-{day}T14:00:00Z", "Sam: news?", channel="noor 30")) + step = rig.step(time_budget_seconds=3 * 3600) # reserve 3060 s: closes once 7740 s are spent + ctx = await rig.run(step) + + record = rig.record(ctx) + assert step.wake_reserve_seconds() == 3060 and record["settings"]["time_budget_seconds"] == 10800 + wakes = rig.events(ctx, "wake") + assert len(wakes) == 4 and wakes[-1]["woke_by"] == ["step:final"] and wakes[-1]["last"] is True + assert wakes[-1]["virtual_time"] == "2026-11-11T14:00:00Z" # where the run was; the clock doesn't jump + assert ao.FINAL_DEFAULT_TEXT.format(first="Linda") in rig.sent[-1] and NOTICE in rig.sent[-1] + [reached] = rig.events(ctx, "time_budget_reached") + assert reached["real_seconds"] == 10800 + assert record["end_reason"] == "time_budget" and record["error_type"] is None + assert rig.events(ctx, "user_sim_read")[-1]["last_read"] is True + assert rig.chat(ctx)[-1]["texts"][-1].startswith("[End of run · Wed Nov 11, 2:00 PM UTC] The run was cut off by " + "its real-time budget.") + + +@pytest.mark.parametrize("timeout,hours", [(600, 6.98), (1800, 6.32), (3600, 5.32)]) +def test_the_default_budget_is_7_h_lowered_so_the_closing_wake_fits_the_8_h_job(timeout, hours): + step = _step(timeout_seconds=timeout) + budget = step.effective_time_budget() + assert round(budget / 3600, 2) == hours + assert budget + step.wake_reserve_seconds() + ao.JOB_HEADROOM_SECONDS <= ao.JOB_LIMIT_SECONDS + + +def test_a_budget_whose_closing_can_t_fit_the_job_is_refused(): + with pytest.raises(ValueError, match="time_budget_seconds=27000 can't fit"): + _step(time_budget_seconds=7.5 * 3600) + with pytest.raises(ValueError, match="time_budget_seconds=0 can't fit"): + _step(time_budget_seconds=0) + assert _step(time_budget_seconds=6 * 3600).effective_time_budget() == 6 * 3600 + + +def test_a_wake_whose_worst_case_leaves_no_room_for_a_run_is_refused(): + with pytest.raises(ValueError, match="no run fits the hub's 8 h job"): + _step(timeout_seconds=4 * 3600) + + +def test_the_budget_survives_a_round_trip(): + again = ao.AlwaysOnTaskStep.from_dict(_step(time_budget_seconds=5 * 3600).to_dict()) + assert again.time_budget_seconds == 5 * 3600 + assert ao.AlwaysOnTaskStep.from_dict(_step().to_dict()).time_budget_seconds is None From 3bfdf7e8d00099fb834d16081781dc6d16fa0210 Mon Sep 17 00:00:00 2001 From: Sarah Chitty Date: Mon, 9 Nov 2026 01:58:15 -0800 Subject: [PATCH 23/53] always_on: the module docstring speaks of the user, not her Co-Authored-By: Claude Opus 5.5 --- src/agent_env/task_step/task_steps/always_on.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/agent_env/task_step/task_steps/always_on.py b/src/agent_env/task_step/task_steps/always_on.py index 02991107a..cd4820ad4 100644 --- a/src/agent_env/task_step/task_steps/always_on.py +++ b/src/agent_env/task_step/task_steps/always_on.py @@ -30,25 +30,25 @@ check's wake counts toward ``max_wakes``. - At end_time there is always a final check-in: the user's ``final`` line, or the step's own (``FINAL_DEFAULT_TEXT``) when the task sets none. The run's last wake, at end_time or the one ``max_wakes`` makes the last, carries the - last-turn notice (``final_reminder``). After it the user sim reads what the agent sent (its wrap-up), and her - reaction, and any reply still held, are recorded; then the run ends. + last-turn notice (``final_reminder``). After it the user sim reads what the agent sent (its wrap-up), and the + user's reaction, and any reply still held, are recorded; then the run ends. - A real-time budget keeps the run inside the hub's 8-hour job (``effective_time_budget`` gives the rule): once one more wake might end past ``time_budget_seconds`` (7 h, lowered to fit the task's own timeouts), the agent gets its closing wake, with the step's wrap-up line and the last-turn notice, and the run ends with end reason time_budget. - End reasons: end_time, user_done (one sitting), max_wakes, time_budget, finished_nothing_pending or stuck (nothing can wake the agent, with nothing pending or with something still pending), agent_turn_failed, agent_turn_timeout, - user_sim_error, step_error. What was still pending is recorded (``pending_at_end``): replies she held, messages she + user_sim_error, step_error. What was still pending is recorded (``pending_at_end``): replies held, messages the user never read, user lines and alerts never delivered, the agent's checks not yet due, contact replies in progress. - Graders read what the user saw. The prompt response is every message_user text in order, each with its pretend - time, never the agent's closing text. The conversation (``metadata["a2a_conversations"][step_id]``) holds her own - messages and the agent's message_user texts, then her last word that never reached the agent and an end marker with - the end reason. A failed or timed-out agent turn sets the response's ``error_type`` (as prompt_agent does), so - verifiers skip the run. + time, never the agent's closing text. The conversation (``metadata["a2a_conversations"][step_id]``) holds the user's + own messages and the agent's message_user texts, then the user's last word that never reached the agent and an end + marker with the end reason. A failed or timed-out agent turn sets the response's ``error_type`` (as prompt_agent + does), so verifiers skip the run. - Who did what, and in what order, even under a frozen clock: each app change (urn:agentenv:changes/v1) is recorded with who made it and the wake it was seen in, the agent's own with the wake that made it (``app_changes`` events); each reply records the agent's message it answers (``reply_to``: its wake, time and text); and each read leaves a record in ``metadata["post_turn_hooks"][step_id]`` in the shape the old psi-wake hook wrote, ``user_reads_at`` - being when her reply reached the agent, so checks built on those records grade the run instead of skipping it. + being when the reply reached the agent, so checks built on those records grade the run instead of skipping it. - The run's metadata keeps a summary in ``metadata["always_on"][step_id]`` (small enough for a running job to report back); the full timeline goes to the object store, at the summary's ``object_url``. """ From 856db39f012c16a2078ce48ab5e926a6c8f85d4c Mon Sep 17 00:00:00 2001 From: Sarah Chitty Date: Mon, 9 Nov 2026 03:11:16 -0800 Subject: [PATCH 24/53] triggers: side-by-side fires only for contact replies; each fire keeps its number and its own executor calls - concurrent_fires now runs a repeat trigger's fires side by side only when every action is a contact's reply (nl). A trigger with tool actions queues as before, since two fires of a read-then-write action could lose a write. - A fire reads its number once, when it starts, and passes it down; overlapping fires used to read trig["attempts"] later and get the other fire's number. - A fire counts as realized by its executor only while its call holds the executor conversation's lock, so a fire waiting for its turn is no longer credited with the calls the executor makes for the other fire. Fires of one trigger taking turns on that lock is intended (two turns never resume one session), now documented. - A fire is counted in `running` only once its task exists, so a schedule that raises leaves no count. - A call queued while only side-by-side fires ran (concurrent_fires turned off meanwhile) is no longer stranded: the last of them to end starts a fire that drains the queue. - An executor reply that completes just past its time limit is kept: message/send reads a little past the limit, a terminal status in its answer is used, and the task is polled once more after the limit. With concurrent_fires off (the default), every trigger behaves as before, bar the late reply now being kept. Co-Authored-By: Claude Opus 5.5 --- src/agent_env/env/gateway/constants.py | 2 +- src/agent_env/env/gateway/triggers.py | 149 +++++++++++++------- tst/unit/env/gateway/triggers_fires_test.py | 91 ++++++++++++ tst/unit/env/gateway/triggers_test.py | 14 ++ 4 files changed, 201 insertions(+), 55 deletions(-) diff --git a/src/agent_env/env/gateway/constants.py b/src/agent_env/env/gateway/constants.py index 63bfc81ab..fbe373f38 100644 --- a/src/agent_env/env/gateway/constants.py +++ b/src/agent_env/env/gateway/constants.py @@ -210,7 +210,7 @@ def random_mcp_server_name() -> str: }, { "uri": EXT_TRIGGERS_URI, - "description": "Register (additive add-by-id), remove, clear, or read (state) typed Action/State triggers + the firing log (harness control path). config sets the engine's opt-in settings: concurrent_fires (a repeat trigger's fires run side by side instead of queueing) and executor_timeout_seconds (the time limit of an executor that sets none).", + "description": "Register (additive add-by-id), remove, clear, or read (state) typed Action/State triggers + the firing log (harness control path). config sets the engine's opt-in settings: concurrent_fires (a repeat trigger whose actions are all contact replies (nl) runs its fires side by side instead of queueing) and executor_timeout_seconds (the time limit of an executor that sets none).", "params": {"endpoint": "/triggers/register", "features": list(TRIGGER_FEATURES), "methods": { "register": {"method": "POST", "endpoint": "/triggers/register", "request": {"type": "object", "properties": {"watch_roles": {"type": "array", "items": {"type": "string"}}, "executor": {"type": "object"}, "executors": {"type": "object"}, "triggers": {"type": "array"}}, "required": ["triggers"]}}, "config": {"method": "POST", "endpoint": "/triggers/config", "request": {"type": "object", "properties": {"concurrent_fires": {"type": "boolean"}, "executor_timeout_seconds": {"type": "number", "exclusiveMinimum": 0}}}}, diff --git a/src/agent_env/env/gateway/triggers.py b/src/agent_env/env/gateway/triggers.py index 96c8dfbb1..c603e8d8a 100644 --- a/src/agent_env/env/gateway/triggers.py +++ b/src/agent_env/env/gateway/triggers.py @@ -447,7 +447,7 @@ def __init__(self, gateway): self._time_trigger_gate = asyncio.Event() # parks the driver while no time-trigger is armed self._clock_moved = asyncio.Event() # set, and replaced, when an advance moves the clock under waiting actions # Opt-in settings (`config`), off by default so a registration behaves as it always has. - self._concurrent_fires = False # a repeat trigger's fires run side by side instead of queueing + self._concurrent_fires = False # a contact-reply trigger's repeat fires run side by side instead of queueing self._executor_timeout_default: float | None = None # the time limit of an executor that sets none self._unset_timeouts: set[str | None] = set() # executors registered without a timeout_seconds of their own # One turn at a time per executor conversation (its URL and context id): fires of one trigger running side by @@ -583,8 +583,11 @@ def clear(self) -> dict: def config(self, body: Any) -> dict: """Set the engine's opt-in settings, each off unless the body sets it (an omitted one resets): with ``concurrent_fires``, each further call that provokes a repeat trigger already firing starts its own fire at - once, side by side, instead of queueing behind the one running; ``executor_timeout_seconds`` is the time limit - of each executor whose registration set no ``timeout_seconds`` (else 120). ``clear`` resets both.""" + once, side by side, instead of queueing behind the one running, when every action of the trigger is a contact's + reply (``nl``); any other trigger still queues, since its tool actions may read and then write the same rows. + Fires of one trigger side by side still take turns on its executor conversation (``_executor_turn``). + ``executor_timeout_seconds`` is the time limit of each executor whose registration set no + ``timeout_seconds`` (else 120). ``clear`` resets both.""" if not isinstance(body, dict): raise TriggerError("body must be a JSON object") unknown = sorted(set(body) - {"concurrent_fires", "executor_timeout_seconds"}) @@ -1100,7 +1103,7 @@ def _executor_roles(self) -> set[str]: def _mark_detected(self, tid: str, trig: dict, provoking: dict, ctx: dict | None = None) -> asyncio.Task | None: ctx = ctx if ctx is not None else {} since = self._gw._clock.elapsed() # a delay runs from the provoking call, not from when its turn comes - if trig["status"] in TRIGGER_IN_FLIGHT_STATUSES and self._concurrent_fires: + if trig["status"] in TRIGGER_IN_FLIGHT_STATUSES and self._concurrent_fires and _replies_only(trig["spec"]): # Mid-fire, side by side: this call's fire starts now, so a quick reply doesn't wait behind a slow one. self._emit("detected", tid, provoking=provoking, alongside=trig["running"]) task = self._start_fire(trig, lambda ticket: self._fire_alongside(tid, trig, ctx, since, ticket)) @@ -1109,7 +1112,8 @@ def _mark_detected(self, tid: str, trig: dict, provoking: dict, ctx: dict | None task.add_done_callback(trig["alongside"].discard) return task if trig["status"] in TRIGGER_IN_FLIGHT_STATUSES: - # Mid-fire: queue it; the running fire drains the queue before it settles. + # Mid-fire: queue it; the running fire drains the queue before it settles, or, when only fires run side by + # side are left (concurrent_fires turned off meanwhile), the last of them to end starts one that does. trig["pending"].append((ctx, since)) trig["status"] = "queued" self._emit("detected", tid, provoking=provoking, queued=len(trig["pending"])) @@ -1309,8 +1313,9 @@ async def _fire_time(self, tid: str, trig: dict, marks: list[datetime], capped: sinces = [elapsed - (max(0.0, (now - mark).total_seconds()) if now is not None else 0.0) for mark in marks] for mark, since in zip(marks, sinces): trig["attempts"] += 1 + fire = trig["attempts"] for index, action in enumerate(trig["spec"]["actions"]): - if not await self._run_action(tid, trig, index, action, {}, since): + if not await self._run_action(tid, trig, index, action, {}, since, fire=fire): trig["status"] = "failed" trig["next_mark"] = None self._emit("failed", tid, action_index=index) @@ -1351,14 +1356,19 @@ def _start_fire(self, trig: dict, run) -> asyncio.Task: """Schedule ``run(ticket)``, one fire of ``trig``, counted in its ``running`` from now, so state shows it at work before its task first runs. The fire's ``finally`` uncounts it (``_uncount``); one cancelled before it began is uncounted when its task ends.""" - ticket = {"counted": True} + ticket = {"counted": False} + task = self._schedule(run(ticket)) # counted only once it exists, so a schedule that raises leaves no count trig["running"] += 1 - task = self._schedule(run(ticket)) + ticket["counted"] = True task.add_done_callback(lambda _: _uncount(trig, ticket)) return task def _schedule(self, coro) -> asyncio.Task: - task = asyncio.get_running_loop().create_task(coro) + try: + task = asyncio.get_running_loop().create_task(coro) + except BaseException: + coro.close() # never awaited: closed, so it doesn't warn + raise self._tasks.add(task) task.add_done_callback(self._tasks.discard) return task @@ -1469,6 +1479,15 @@ def _settle_status(self, tid: str, trig: dict, outcome: str | None) -> None: removed or cleared meanwhile is gone, and has nothing left to report.""" if trig["running"] or trig["status"] not in TRIGGER_IN_FLIGHT_STATUSES: return + if outcome is not None and trig["pending"] and trig["task"] is None and self._triggers.get(tid) is trig: + # Calls queued while only fires side by side ran (concurrent_fires turned off meanwhile): none of those + # drains a queue, so a fire that does starts now rather than leave them stranded. + ctx, since = trig["pending"].popleft() + trig["status"] = "firing" + trig["detected_at"] = _now() + trig["task"] = self._start_fire(trig, lambda ticket: self._fire(tid, trig, ctx, since, ticket)) + trig["task"].set_name(tid) + return if outcome is not None: trig["status"] = outcome elif self._triggers.get(tid) is trig: @@ -1481,12 +1500,13 @@ async def _fire_once(self, tid: str, trig: dict, ctx: dict, since: float) -> boo spec = trig["spec"] ok = True trig["attempts"] += 1 + fire = trig["attempts"] # this fire's number, read now: fires side by side move `attempts` on meanwhile try: await self._gw._log_event({"event_type": "trigger_fired", "source": "trigger_engine", "trigger_id": tid, "provoking_tool": (ctx.get("args") is not None and spec["when"].get("tool")) or None, **({"provoking_request": ctx["request"]} if "request" in ctx else {})}) for index, action in enumerate(spec["actions"]): - if not await self._run_action(tid, trig, index, action, ctx, since): + if not await self._run_action(tid, trig, index, action, ctx, since, fire=fire): ok = False self._emit("failed", tid, action_index=index) break @@ -1502,7 +1522,10 @@ async def _fire_once(self, tid: str, trig: dict, ctx: dict, since: float) -> boo self._resolve_dependents(tid) # success-only, outside the try so it can't flip a fired trigger to failed return ok - async def _run_action(self, tid: str, trig: dict, index: int, action: dict, ctx: dict, since: float) -> bool: + async def _run_action(self, tid: str, trig: dict, index: int, action: dict, ctx: dict, since: float, + fire: int | None = None) -> bool: + """Run one action of ``tid``'s fire number ``fire`` (its ``attempts`` when not given); whether it went + through.""" if "delay" in action: await self._delay(tid, trig, index, action, since) changelog_before = self._gw._query_changelog_id() @@ -1553,12 +1576,12 @@ async def _run_action(self, tid: str, trig: dict, index: int, action: dict, ctx: detail = {"notify": action["channel"], "to": note["to"], "notification_id": note["id"], "text_len": len(text)} else: - fire = await self._run_nl(tid, trig, index, action, ctx) - if fire is None: + done = await self._run_nl(tid, trig, index, action, ctx, fire=fire) + if done is None: return False - detail = {"nl": action["instruction"][:120], **_named_executor(action), "calls": fire["calls"], - **({"failed_calls": fire["failed_calls"]} if fire["failed_calls"] else {}), - **({"calls_shared_with": sorted(fire["shared_with"])} if fire["shared_with"] else {})} + detail = {"nl": action["instruction"][:120], **_named_executor(action), "calls": done["calls"], + **({"failed_calls": done["failed_calls"]} if done["failed_calls"] else {}), + **({"calls_shared_with": sorted(done["shared_with"])} if done["shared_with"] else {})} if "verify" in action and action["type"] != "nl": # only an action's own verify inherits its `as`; when.check state probes read as the pin if not await self._run_verify(tid, index, action["verify"], ctx, role=action.get("as")): @@ -1580,43 +1603,42 @@ async def _run_verify(self, tid: str, index: int, verify: dict, ctx: dict | None self._emit("verify_failed", tid, action_index=index) return False - async def _run_nl(self, tid: str, trig: dict, index: int, action: dict, ctx: dict | None = None) -> dict | None: - """The fire with the calls its executor made for it, by tool, or None when the action failed.""" + async def _run_nl(self, tid: str, trig: dict, index: int, action: dict, ctx: dict | None = None, + fire: int | None = None) -> dict | None: + """Fire number ``fire`` (``attempts`` when not given) with the calls its executor made for it, by tool, or None + when the action failed.""" instruction = action["instruction"] named = _named_executor(action) executor = self._executors.get(action.get("executor")) if executor is not None and "hours" in executor: await self._await_hours(tid, trig, index, executor["hours"], named) - fire = _in_flight_fire({"trigger_id": tid, "action_index": index, "fire": trig["attempts"]}) - fires = self._in_flight.setdefault(executor["role"], []) if executor is not None else [] - for other in fires: # an executor's calls carry only its role, so fires it realizes at once share their counts - other["shared_with"].add(tid) - fire["shared_with"].add(other["cause"]["trigger_id"]) - fires.append(fire) - try: - for attempt in (1, 2): - reply = await self._executor_turn(tid, trig, index, action.get("executor"), executor, instruction) - if reply is None: - self._emit("action_failed", tid, action_index=index, - detail=f"executor attempt {attempt} did not complete", **named) - return None - if "verify" not in action: - return fire - if await self._run_verify(tid, index, action["verify"], ctx): - return fire - if attempt == 1: - instruction = (f"{action['instruction']}\n\nYour previous attempt did not pass verification — " - f"the expected change is not observable in the environment. Do it again exactly as described.") - finally: - fires.remove(fire) + done = _in_flight_fire({"trigger_id": tid, "action_index": index, + "fire": trig["attempts"] if fire is None else fire}) + for attempt in (1, 2): + reply = await self._executor_turn(tid, trig, index, action.get("executor"), executor, instruction, done) + if reply is None: + self._emit("action_failed", tid, action_index=index, + detail=f"executor attempt {attempt} did not complete", **named) + return None + if "verify" not in action: + return done + if await self._run_verify(tid, index, action["verify"], ctx): + return done + if attempt == 1: + instruction = (f"{action['instruction']}\n\nYour previous attempt did not pass verification — " + f"the expected change is not observable in the environment. Do it again exactly as described.") self._emit("action_failed", tid, action_index=index, detail="verification failed after retry", **named) return None async def _executor_turn(self, tid: str, trig: dict, index: int, name: str | None, executor: dict | None, - instruction: str) -> str | None: + instruction: str, fire: dict) -> str | None: """One call to the executor, listed in the trigger's ``calls`` with the end of its time limit while it runs. - Calls in one conversation (the executor's URL and the trigger's context id) take turns, so fires of one trigger - running side by side never resume the executor's session at once.""" + + Calls in one conversation (the executor's URL and the trigger's context id) take turns: fires of one trigger + running side by side (``concurrent_fires``) wait for each other here, by design, so two turns never resume the + executor's session (a Claude Code conversation, say) at once; only their delays and hours run side by side. + ``fire`` (``_in_flight_fire``) is counted as being realized only while its call runs, so the calls the executor + makes for another fire while this one waits for its turn are never credited to this one.""" context_id = f"trigger-{tid}" if executor is None: return await self._call_executor(executor, instruction, context_id=context_id) @@ -1625,12 +1647,18 @@ async def _executor_turn(self, tid: str, trig: dict, index: int, name: str | Non executor = {**executor, "timeout_seconds": timeout} lock = self._executor_locks.setdefault((executor["a2a_url"], context_id), asyncio.Lock()) async with lock: + fires = self._in_flight.setdefault(executor["role"], []) + for other in fires: # an executor's calls carry only its role, so fires it realizes at once share counts + other["shared_with"].add(tid) + fire["shared_with"].add(other["cause"]["trigger_id"]) + fires.append(fire) call = {"executor": name, "action_index": index, "deadline": _monotonic() + timeout} trig["calls"].append(call) try: return await self._call_executor(executor, instruction, context_id=context_id) finally: trig["calls"].remove(call) + fires.remove(fire) def caused_by(self, role: str) -> list[dict]: """The fires in flight for the executor with ``role``, all of them when several overlap; its tool calls are @@ -1700,8 +1728,9 @@ async def _call_executor(self, executor: dict | None, instruction: str, context_ url = f"{executor['a2a_url']}/a2a" deadline = asyncio.get_running_loop().time() + executor["timeout_seconds"] async with httpx.AsyncClient(timeout=_EXECUTOR_HTTP_TIMEOUT_S) as client: - # an executor holds message/send open until its task ends, so the read waits as long as the executor may take - read_s = max(_EXECUTOR_HTTP_TIMEOUT_S, executor["timeout_seconds"]) + # An executor holds message/send open until its task ends, so the read waits as long as the executor may + # take, and a little longer, so a reply that completes just past the time limit is still read. + read_s = max(_EXECUTOR_HTTP_TIMEOUT_S, executor["timeout_seconds"]) + _EXECUTOR_HTTP_TIMEOUT_S send = await client.post(url, timeout=httpx.Timeout(_EXECUTOR_HTTP_TIMEOUT_S, read=read_s), json={ "jsonrpc": "2.0", "id": str(uuid.uuid4()), "method": "message/send", "params": {"message": {"role": "user", "messageId": uuid.uuid4().hex, @@ -1709,21 +1738,27 @@ async def _call_executor(self, executor: dict | None, instruction: str, context_ "parts": [{"kind": "text", "text": instruction}]}}, }) send.raise_for_status() - task_id = (send.json().get("result") or {}).get("id") + result = send.json().get("result") or {} + task_id = result.get("id") if not task_id: return None - while asyncio.get_running_loop().time() < deadline: - await asyncio.sleep(2) + status = result.get("status") or {} + # Polled until the time limit, then once more: a reply that completed in the last poll's gap is kept. + last = False + while status.get("state") not in _TERMINAL_A2A_STATES: + if last: + return None + last = asyncio.get_running_loop().time() >= deadline + if not last: + await asyncio.sleep(2) poll = await client.post(url, json={"jsonrpc": "2.0", "id": str(uuid.uuid4()), "method": "tasks/get", "params": {"id": task_id}}) poll.raise_for_status() status = ((poll.json().get("result") or {}).get("status") or {}) - if status.get("state") in _TERMINAL_A2A_STATES: - if status.get("state") != "completed": - return None - parts = (status.get("message") or {}).get("parts") or [] - return "\n".join(p.get("text", "") for p in parts if p.get("kind") == "text") - return None + if status.get("state") != "completed": + return None + parts = (status.get("message") or {}).get("parts") or [] + return "\n".join(p.get("text", "") for p in parts if p.get("kind") == "text") def _cancel(self, tid: str, trig: dict, **cause) -> None: """Retire a deadline its `unless` call answered, and the time triggers anchored on it, which can no longer fire.""" @@ -1773,6 +1808,12 @@ def _named_executor(action: dict) -> dict: return {"executor": action["executor"]} if action.get("executor") is not None else {} +def _replies_only(spec: dict) -> bool: + """Whether every action of a trigger is a contact's reply (``nl``), the only kind ``concurrent_fires`` runs side + by side.""" + return all(action.get("type") == "nl" for action in spec["actions"]) + + def _in_flight_fire(cause: dict) -> dict: """An nl fire as `_in_flight` holds it: its cause, its executor's calls by tool, and the triggers it overlapped.""" return {"cause": cause, "calls": {}, "failed_calls": {}, "shared_with": set()} diff --git a/tst/unit/env/gateway/triggers_fires_test.py b/tst/unit/env/gateway/triggers_fires_test.py index a0a0502e5..6d34e28e0 100644 --- a/tst/unit/env/gateway/triggers_fires_test.py +++ b/tst/unit/env/gateway/triggers_fires_test.py @@ -166,6 +166,97 @@ async def test_removing_a_trigger_cancels_its_fires_side_by_side(monkeypatch): assert all(t.cancelled() for t in tasks) and bob.calls == [] +@pytest.mark.asyncio +async def test_side_by_side_is_only_for_contact_replies_a_tool_action_trigger_still_queues(monkeypatch): + """A tool action may read a row and write it back; two of its fires at once could lose a write, so it queues.""" + bob = _Executor(monkeypatch) + engine, _, _ = _engine() + engine.config({"concurrent_fires": True}) + engine.register({"watch_roles": ["default"], + "executors": {"bob": {"a2a_url": "http://bob", "role": "bob"}}, + "triggers": [{"id": "mirror", "when": {"type": "action", "tool": "send_text", "repeat": True}, + "actions": [{"type": "nl", "executor": "bob", "instruction": "answer"}, _perm()]}]}) + engine.on_tool_call("default", "send_text", {}, _result()) + engine.on_tool_call("default", "send_text", {}, _result()) + await _until(lambda: bob.calls) + + row = _row(engine, "mirror") + assert (row["status"], row["pending"], row["running"]) == ("queued", 1, 1) + bob.release() + await _until(lambda: _row(engine, "mirror")["status"] == "armed") + assert len(bob.calls) == 2 + + +@pytest.mark.asyncio +async def test_each_fire_side_by_side_keeps_its_own_number_and_only_its_own_executor_calls(monkeypatch): + """Fire 1 waits out its delay while fire 2 starts: fire 1's call still says it is fire 1. Fire 2, waiting for its + turn on the executor conversation, is not credited with the calls the executor makes for fire 1.""" + seen: list[list[int]] = [] + engine, clock, _ = _engine() + engine.config({"concurrent_fires": True}) + engine.register(_reply_trigger(delay="PT1H")) + + async def call(eng, ex, instruction, context_id): + seen.append([c["fire"] for c in eng.caused_by("bob")]) + eng.on_tool_call("bob", "messaging_send", {}, _result()) # the executor's own tool call, made for this fire + await asyncio.sleep(0.05) + return "replied" + + monkeypatch.setattr(TriggerEngine, "_call_executor", call) + engine.on_tool_call("default", "send_text", {}, _result()) + clock.advance("2026-03-02T00:30:00Z") + engine.on_tool_call("default", "send_text", {}, _result()) + await _until(lambda: len(_row(engine, "reply")["waits"]) == 2) + clock.advance("2026-03-02T01:30:00Z") # both delays end: the two calls take turns + engine.clock_advanced() + await _until(lambda: _row(engine, "reply")["status"] == "armed") + + assert seen == [[1], [2]] + oks = [e["detail"] for e in engine._events if e["kind"] == "action_ok"] + assert [d["calls"] for d in oks] == [{"messaging_send": 1}, {"messaging_send": 1}] + assert not any("calls_shared_with" in d for d in oks) + + +@pytest.mark.asyncio +async def test_a_call_queued_once_side_by_side_is_turned_off_still_fires(monkeypatch): + """Fires side by side don't drain a queue; turned off meanwhile, a queued call used to be left behind for good.""" + bob = _Executor(monkeypatch) + bob.release() + engine, clock, _ = _engine() + engine.config({"concurrent_fires": True}) + engine.register(_reply_trigger(delay="PT1H")) + engine.on_tool_call("default", "send_text", {}, _result()) # the first fire, resumes 01:00 + clock.advance("2026-03-02T00:30:00Z") + engine.on_tool_call("default", "send_text", {}, _result()) # side by side, resumes 01:30 + clock.advance("2026-03-02T01:00:00Z") + engine.clock_advanced() + await _until(lambda: len(bob.calls) == 1 and _row(engine, "reply")["running"] == 1) + + engine.config({}) # side by side off + engine.on_tool_call("default", "send_text", {}, _result()) # queued behind the fire left + assert (_row(engine, "reply")["status"], _row(engine, "reply")["pending"]) == ("queued", 1) + clock.advance("2026-03-02T01:30:00Z") + engine.clock_advanced() + await _until(lambda: len(bob.calls) == 2 and _row(engine, "reply")["pending"] == 0) + clock.advance("2026-03-02T02:00:00Z") # its delay runs from its own call + engine.clock_advanced() + await _until(lambda: _row(engine, "reply")["status"] == "armed") + assert len(bob.calls) == 3 and _row(engine, "reply")["fire_count"] == 3 + + +def test_a_fire_that_can_t_be_scheduled_leaves_no_count(): + engine, _, _ = _engine() + engine.register(_reply_trigger()) + trig = engine._triggers["reply"] + + async def fire(ticket): + return None + + with pytest.raises(RuntimeError): # no running event loop + engine._start_fire(trig, fire) + assert trig["running"] == 0 + + # ---- executor_timeout_seconds --------------------------------------------------------------------------------------- @pytest.mark.asyncio diff --git a/tst/unit/env/gateway/triggers_test.py b/tst/unit/env/gateway/triggers_test.py index 0d9f4846b..b568f4610 100644 --- a/tst/unit/env/gateway/triggers_test.py +++ b/tst/unit/env/gateway/triggers_test.py @@ -1182,6 +1182,20 @@ async def test_an_executor_slower_than_the_http_timeout_is_waited_out_and_the_la assert engine._test_gw.calls == [("slack_send_message", {"channel": "#c", "text": "the client replied"})] +@pytest.mark.asyncio +async def test_a_reply_that_completes_just_past_the_executor_s_limit_is_kept(engine, monkeypatch): + """A deployed executor answers message/send when its task ends; one that ends a moment past the limit used to be + cut off by the read timeout, its completed reply dropped.""" + monkeypatch.setattr(triggers_module, "_EXECUTOR_HTTP_TIMEOUT_S", 0.5) + async with _executor_answering_after(1.2) as url: + engine.register(_reply_then_post(url, timeout_seconds=1.0)) + engine.on_tool_call("default", "t", {}, _result()) + await _settled(engine, "x", within=3) + + assert engine._triggers["x"]["status"] == "fired", engine.state()["events"][-3:] + assert engine._test_gw.calls == [("slack_send_message", {"channel": "#c", "text": "the client replied"})] + + @pytest.mark.asyncio async def test_an_executor_slower_than_its_own_timeout_still_fails_the_fire(engine, monkeypatch): monkeypatch.setattr(triggers_module, "_EXECUTOR_HTTP_TIMEOUT_S", 0.2) From 6b4c92cc879a2e67b27e39968ffca3ddf6f0601c Mon Sep 17 00:00:00 2001 From: Sarah Chitty Date: Mon, 9 Nov 2026 03:11:46 -0800 Subject: [PATCH 25/53] user_chat: message_user and stay_quiet are read-only for the world Declared readOnlyHint=False, each call made the trigger engine re-evaluate every armed state trigger. Nothing depends on that: the only reader of the hint in the gateway is TriggerEngine._is_readonly, the tools change no app's data a state check could read, and action triggers on them still fire (matched before the read-only check). So they are marked read-only, as get_time is. Co-Authored-By: Claude Opus 5.5 --- src/agent_env/env/gateway/user_chat.py | 10 ++++++++-- tst/unit/env/gateway/user_chat_test.py | 7 +++++++ 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/src/agent_env/env/gateway/user_chat.py b/src/agent_env/env/gateway/user_chat.py index 82f9993c7..b1294a647 100644 --- a/src/agent_env/env/gateway/user_chat.py +++ b/src/agent_env/env/gateway/user_chat.py @@ -47,6 +47,12 @@ } +# Read-only for the world: they queue a message (or a note) for the harness and change no app's data, so, like get_time, +# they don't make the trigger engine re-evaluate every armed state trigger (TriggerEngine._is_readonly). An action +# trigger on either still fires. +_ANNOTATIONS = ToolAnnotations(readOnlyHint=True) + + class UserChatError(ValueError): """A bad config, take or tool argument.""" @@ -183,7 +189,7 @@ async def message_user(text: str) -> dict | CallToolResult: return FastMCPTool(fn=message_user, name=MESSAGE_USER_TOOL_NAME, description=MESSAGE_USER_DESCRIPTION.format(user=chat.user_name), parameters=_SCHEMA, fn_metadata=FuncMetadata(arg_model=make_arg_model(MESSAGE_USER_TOOL_NAME, _SCHEMA)), - is_async=True, annotations=ToolAnnotations(readOnlyHint=False)) + is_async=True, annotations=_ANNOTATIONS) def build_stay_quiet_tool(chat: UserChat, make_arg_model: Callable[[str, dict], Any]) -> FastMCPTool: @@ -198,4 +204,4 @@ async def stay_quiet() -> dict | CallToolResult: return FastMCPTool(fn=stay_quiet, name=STAY_QUIET_TOOL_NAME, description=STAY_QUIET_DESCRIPTION.format(user=chat.user_name), parameters=_QUIET_SCHEMA, fn_metadata=FuncMetadata(arg_model=make_arg_model(STAY_QUIET_TOOL_NAME, _QUIET_SCHEMA)), - is_async=True, annotations=ToolAnnotations(readOnlyHint=False)) + is_async=True, annotations=_ANNOTATIONS) diff --git a/tst/unit/env/gateway/user_chat_test.py b/tst/unit/env/gateway/user_chat_test.py index b64906361..3efd91651 100644 --- a/tst/unit/env/gateway/user_chat_test.py +++ b/tst/unit/env/gateway/user_chat_test.py @@ -139,3 +139,10 @@ def test_the_check_tools_reach_a_configured_role_through_a_deny_all_once_notific assert checks <= _names(gateway, "psi_agent") _req(gateway, "POST", "/notifications/config", {"roles": []}) assert not checks & _names(gateway, "psi_agent") + + +def test_message_user_and_stay_quiet_are_read_only_for_the_world_so_they_don_t_re_evaluate_state_triggers(gateway): + _req(gateway, "POST", "/user_chat/config", {"roles": ["psi_agent"], "stay_quiet": True}) + engine = gateway._trigger_engine + assert engine._is_readonly("message_user") and engine._is_readonly("stay_quiet") + assert not engine._is_readonly("svc_thing") From a4dc0ccefc8b159d85b734ea8259492b4fc1b2dd Mon Sep 17 00:00:00 2001 From: Sarah Chitty Date: Mon, 9 Nov 2026 03:12:19 -0800 Subject: [PATCH 26/53] rubrics_verifier: skip an errored always_on run before checking how it would be graded _for_always_on refuses settings it can't grade an always_on prompt with (grade_up_to_turn, evidence-citing formats) and ran before the error_type skip, so a run whose agent turn failed raised instead of being skipped. The skip now comes first; every other prompt is verified as before. Co-Authored-By: Claude Opus 5.5 --- .../task_steps/verifiers/rubrics_verifier.py | 6 ++++++ tst/unit/task_step/test_always_on_rubrics.py | 18 ++++++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/src/agent_env/task_step/task_steps/verifiers/rubrics_verifier.py b/src/agent_env/task_step/task_steps/verifiers/rubrics_verifier.py index 669aa220d..f5cbf9432 100644 --- a/src/agent_env/task_step/task_steps/verifiers/rubrics_verifier.py +++ b/src/agent_env/task_step/task_steps/verifiers/rubrics_verifier.py @@ -663,6 +663,12 @@ async def execute(self, context: TaskStepContext) -> TaskStepContext: raise RuntimeError(f"PromptResponse with prompt_id='{self.prompt_id}' not found in context") always_on = (context.metadata.get("always_on") or {}).get(prompt_response.step_id) if always_on is not None and self._always_on_wakes is None: + if prompt_response.error_type: # skipped before the always_on settings are checked, as any errored prompt + logger.warning(f"Skipping verification '{self.verifier_id}': prompt had " + f"error_type={prompt_response.error_type}") + self._write_skipped_entries( + context, "prompt_error", f"Skipped: prompt had error_type={prompt_response.error_type}") + return context return await (await self._for_always_on(context, always_on)).execute(context) overrides_for_step = self.step_param_overrides(context) grade_up_to_turn = ( diff --git a/tst/unit/task_step/test_always_on_rubrics.py b/tst/unit/task_step/test_always_on_rubrics.py index 186e5402f..29eabdc20 100644 --- a/tst/unit/task_step/test_always_on_rubrics.py +++ b/tst/unit/task_step/test_always_on_rubrics.py @@ -115,6 +115,24 @@ def fails(n, texts, rig): assert row["id"] == "prompt_error" and "agent_turn_timeout" in row["message"] +@pytest.mark.asyncio +async def test_a_failed_always_on_run_is_skipped_even_by_a_verifier_that_couldn_t_grade_it(monkeypatch, tmp_path): + """grade_up_to_turn can't cut an always_on prompt, but a run that broke is skipped before that is checked.""" + def fails(n, texts, rig): + raise TimeoutError("no answer") + + rig = Rig(monkeypatch, tmp_path, agent=fails) + await rig.arm() + ctx = await rig.run(rig.step()) + verifier = _verifier(grade_up_to_turn=2) + _capture(monkeypatch, verifier) + + await verifier.execute(ctx) + + [row] = ctx.metadata["verifications"]["rubric"]["results"] + assert row["id"] == "prompt_error" and "agent_turn_timeout" in row["message"] + + def test_the_wake_by_wake_text_says_when_the_record_is_unreadable(): text = ao.render_for_judge({"object_url": "file:///nowhere/at/all.json", "end_reason": "end_time"}) assert text.startswith("## The run, wake by wake\n(The run's record could not be read:") From e8c801e15c080fea41a64e9497e1a282b17f92da Mon Sep 17 00:00:00 2001 From: Sarah Chitty Date: Mon, 9 Nov 2026 03:14:11 -0800 Subject: [PATCH 27/53] clock: set_rate changes the rate on the current arm, keeping time triggers' marks PUT /clock/rate (clock/v1 set_rate) sets virtual_seconds_per_real_second from now on, keeping the virtual time, t0 and generation; 0 freezes the clock. The always_on step needs to freeze the clock outside the agent's turns and run it at the task's rate during them. Re-arming with set_time, as sync_env_clock does, can't be used for that: each re-arm bumps the generation, which re-anchors every relative or recurring time trigger to the new t0, so a trigger at PT2H would be pushed back on every turn and never fire. Additive: no existing route or method changes. Co-Authored-By: Claude Opus 5.5 --- src/agent_env/env/gateway/clock.py | 15 ++++++ src/agent_env/env/gateway/constants.py | 3 +- src/agent_env/env/gateway/gateway.py | 14 +++++ tst/unit/env/gateway/clock_advance_test.py | 62 ++++++++++++++++++++++ 4 files changed, 93 insertions(+), 1 deletion(-) diff --git a/src/agent_env/env/gateway/clock.py b/src/agent_env/env/gateway/clock.py index cceb14364..359558a84 100644 --- a/src/agent_env/env/gateway/clock.py +++ b/src/agent_env/env/gateway/clock.py @@ -186,6 +186,21 @@ def advance(self, virtual_time: str) -> dict: self._lived += jump return self.state() + def set_rate(self, rate) -> dict: + """Change the rate on the current arm from now on, keeping the virtual time, t0 and generation, so time + triggers keep their marks (a re-arm with set_time would re-anchor them); 0 freezes the clock.""" + r = _parse_rate(rate) + with self._lock: + if not self._armed or self._t0 is None: + raise ClockError("clock is not armed") + current = _advance(self._t0, (self._now() - self._anchor) * self._rate + self._ahead) + if current > _MAX_T0 - timedelta(seconds=_OVERFLOW_HORIZON_S * r): + raise ClockError(f"rate {r:g} would overflow datetime near the max from {_iso(current)}") + self._bank() + self._ahead = (current - self._t0).total_seconds() + self._rate = r + return self.state() + def clear(self) -> dict: """Disarm (idempotent). Afterwards reads 404.""" with self._lock: diff --git a/src/agent_env/env/gateway/constants.py b/src/agent_env/env/gateway/constants.py index fbe373f38..e542bbbe5 100644 --- a/src/agent_env/env/gateway/constants.py +++ b/src/agent_env/env/gateway/constants.py @@ -221,11 +221,12 @@ def random_mcp_server_name() -> str: }, { "uri": EXT_CLOCK_URI, - "description": "Virtual clock (urn:agentenv:clock/v1): read the gateway's virtual time (wall-clock-derived, rate-scaled) via get_time; set-time/advance/clear are the harness control path.", + "description": "Virtual clock (urn:agentenv:clock/v1): read the gateway's virtual time (wall-clock-derived, rate-scaled) via get_time; set-time/advance/set_rate/clear are the harness control path (set_rate changes the rate on the current arm, keeping time triggers' marks; 0 freezes the clock).", "params": {"endpoint": "/clock/time", "methods": { "get_time": {"method": "GET", "endpoint": "/clock/time"}, "set_time": {"method": "PUT", "endpoint": "/clock/set-time", "request": {"type": "object", "properties": {"virtual_time": {"type": "string"}, "virtual_seconds_per_real_second": {"type": "number"}}, "required": ["virtual_time"]}}, "advance": {"method": "PUT", "endpoint": "/clock/advance", "request": {"type": "object", "properties": {"virtual_time": {"type": "string"}}, "required": ["virtual_time"]}}, + "set_rate": {"method": "PUT", "endpoint": "/clock/rate", "request": {"type": "object", "properties": {"virtual_seconds_per_real_second": {"type": "number", "minimum": 0}}, "required": ["virtual_seconds_per_real_second"]}}, "clear": {"method": "POST", "endpoint": "/clock/clear", "request": {"type": "object", "properties": {}}}, "state": {"method": "GET", "endpoint": "/clock/state", "request": {"type": "object", "properties": {}}}}}, "required": False, diff --git a/src/agent_env/env/gateway/gateway.py b/src/agent_env/env/gateway/gateway.py index eed34034f..b91e07a66 100644 --- a/src/agent_env/env/gateway/gateway.py +++ b/src/agent_env/env/gateway/gateway.py @@ -340,6 +340,20 @@ async def clock_advance(request: Request) -> Response: return self._json_response({"ok": False, "error": str(e)}, 400) return self._json_response({**state, "due": self._trigger_engine.clock_advanced()}, 200) + @self._mcp.custom_route("/clock/rate", methods=["PUT"]) + async def clock_rate(request: Request) -> Response: + try: + body = await request.json() + except Exception: + return self._json_response({"ok": False, "error": "Invalid JSON body"}, 400) + if not isinstance(body, dict): + return self._json_response({"ok": False, "error": "body must be a JSON object"}, 400) + try: + state = self._clock.set_rate(body.get("virtual_seconds_per_real_second")) + except ClockError as e: + return self._json_response({"ok": False, "error": str(e)}, 400) + return self._json_response(state, 200) + @self._mcp.custom_route("/clock/clear", methods=["POST"]) async def clock_clear(request: Request) -> Response: self._unregister_get_time_tool() diff --git a/tst/unit/env/gateway/clock_advance_test.py b/tst/unit/env/gateway/clock_advance_test.py index 0a76031f9..5c15cef05 100644 --- a/tst/unit/env/gateway/clock_advance_test.py +++ b/tst/unit/env/gateway/clock_advance_test.py @@ -123,6 +123,61 @@ def test_the_card_offers_advance(): "request": {"type": "object", "properties": {"virtual_time": {"type": "string"}}, "required": ["virtual_time"]}} +def test_set_rate_keeps_the_time_t0_and_generation_and_runs_on_at_the_new_rate(): + clock, now = _armed(rate=60) + now.tick(60) # an hour at 60x + gen, lived = clock.generation, clock.elapsed() + state = clock.set_rate(0) + assert (state["virtual_time"], state["t0"], state["virtual_seconds_per_real_second"]) == ( + "2026-01-01T01:00:00Z", T0, 0) + now.tick(600) + assert clock.read()["virtual_time"] == "2026-01-01T01:00:00Z" and clock.generation == gen + clock.set_rate(2) + now.tick(30) + assert clock.read()["virtual_time"] == "2026-01-01T01:01:00Z" + assert clock.elapsed() == pytest.approx(lived + 60) + clock.advance("2026-01-02T00:00:00Z") + assert clock.read()["virtual_time"] == "2026-01-02T00:00:00Z" + + +def test_set_rate_refuses_a_bad_rate_and_an_unarmed_clock(): + clock, _ = _armed() + with pytest.raises(ClockError, match="non-negative"): + clock.set_rate(-1) + with pytest.raises(ClockError, match="must be a number"): + clock.set_rate(None) + with pytest.raises(ClockError, match="not armed"): + Clock(now_fn=FakeNow()).set_rate(0) + + +@pytest.mark.asyncio +async def test_a_relative_mark_keeps_its_place_across_rate_changes(): + """A re-arm (set_time) would re-anchor a relative or recurring mark to the new t0; set_rate leaves it put.""" + now = FakeNow() + clock = Clock(now_fn=now) + engine = TriggerEngine(_FakeGateway(clock)) + clock.set_time(T0, rate=60) + engine.register({"watch_roles": ["default"], "triggers": [ + {"id": "rel", "when": {"type": "time", "at": "PT2H"}, "actions": [_perm()]}]}) + _tick_call(engine, now, 1) + for rate in (0, 60, 0, 60): + clock.set_rate(rate) + _tick_call(engine, now, 30) # 30 real s: half an hour at 60x, nothing while frozen + await _settle(engine) + assert clock.read()["virtual_time"] == "2026-01-01T01:01:00Z" + assert engine.state()["triggers"][0]["next_mark"] == "2026-01-01T02:00:00Z" + clock.set_rate(60) + _tick_call(engine, now, 60) + await _settle(engine) + assert engine.state()["triggers"][0]["status"] == "fired" + assert "reanchored" not in {e["kind"] for e in engine.state()["events"]} + + +def test_the_card_offers_set_rate(): + clock_ext = next(e for e in GATEWAY_EXTENSIONS if e["uri"] == EXT_CLOCK_URI) + assert clock_ext["params"]["methods"]["set_rate"]["endpoint"] == "/clock/rate" + + @pytest.fixture def gateway(tmp_path, monkeypatch) -> Gateway: monkeypatch.setattr(gw_mod, "GATEWAY_TRAJECTORY_FILE", str(tmp_path / "trajectory.jsonl")) @@ -165,3 +220,10 @@ def test_route_refuses_a_backwards_advance(gateway): gateway._clock.set_time("2026-01-04T08:00:00Z", rate=0) resp = _put(gateway, "/clock/advance", {"virtual_time": T0}) assert resp.status_code == 400 and "only moves forward" in resp.json()["error"] + + +def test_route_sets_the_rate_of_an_armed_clock(gateway): + gateway._clock.set_time(T0, rate=0) + resp = _put(gateway, "/clock/rate", {"virtual_seconds_per_real_second": 60}) + assert resp.status_code == 200 and resp.json()["virtual_seconds_per_real_second"] == 60 + assert _put(gateway, "/clock/rate", {"virtual_seconds_per_real_second": -1}).status_code == 400 From 2976de58fc6627edecaf2f0e85009cc49f11147e Mon Sep 17 00:00:00 2001 From: Sarah Chitty Date: Mon, 9 Nov 2026 03:15:40 -0800 Subject: [PATCH 28/53] always_on: the process log gets event names, ids, times, counts and sizes, never anyone's words _Run.log() wrote up to 400 characters of each event's fields to logger.info, including the agent's messages, the user sim's reads and replies, plain replies and closing text. It now logs only fields that name, time or count things; any other string is logged as its length and a list as its count, as the trigger engine does ("bodies and addresses are PII"). The stored timeline keeps everything. Co-Authored-By: Claude Opus 5.5 --- .../task_step/task_steps/always_on.py | 32 +++++++++++++++++-- tst/unit/task_step/test_always_on_record.py | 26 +++++++++++++++ 2 files changed, 55 insertions(+), 3 deletions(-) diff --git a/src/agent_env/task_step/task_steps/always_on.py b/src/agent_env/task_step/task_steps/always_on.py index cd4820ad4..ca74e753a 100644 --- a/src/agent_env/task_step/task_steps/always_on.py +++ b/src/agent_env/task_step/task_steps/always_on.py @@ -326,12 +326,12 @@ def spent(self) -> float: # ---- record ---------------------------------------------------------------------------------------------- def log(self, event: str, **fields: Any) -> dict: + """Add an event to the run's timeline, which keeps it whole; the process log gets only its name, ids, times, + counts and sizes (``_log_view``), since texts and reads carry what people wrote.""" entry = {"event": event, "real_time": datetime.now(timezone.utc).isoformat(), "real_elapsed_s": round(time.time() - self.t0_real, 1), **fields} self.timeline.append(entry) - logger.info(f"always_on {event}: " + json.dumps({k: v for k, v in fields.items() - if k not in ("input", "parts_text", "raw")}, - default=str)[:400]) + logger.info(f"always_on {event}: " + json.dumps(_log_view(fields), default=str)) return entry # ---- setup ----------------------------------------------------------------------------------------------- @@ -1513,6 +1513,32 @@ def _bounded(pending: dict) -> dict: return out +# The timeline fields the process log may show as they are: names, ids, times, states and settings, never what a person +# or the agent wrote. Any other string is logged as its length, a list as its count, a dict as its keys' view. +_LOGGED_AS_IS = frozenset({ + "virtual_time", "from", "to", "reason", "state", "outcome", "mode", "read_at", "deliver_at", "end_time", "agent", + "agent_role", "user_sim", "version", "user_model", "time_zone", "kind", "trigger_id", "env_id", "phase", + "error_type", "matched_by", "at", "wake", "n", "trajectory_url", "object_url"}) + + +def _log_view(fields: dict, depth: int = 0) -> dict: + """``fields`` as the process log may show them: see ``_LOGGED_AS_IS``. Bodies and addresses are PII; the full + content stays in the stored timeline.""" + out: dict = {} + for k, v in fields.items(): + if v is None or isinstance(v, (bool, int, float)): + out[k] = v + elif isinstance(v, str): + out[k] = v if k in _LOGGED_AS_IS and len(v) <= 200 else {"chars": len(v)} + elif isinstance(v, (list, tuple)): + out[k] = {"items": len(v)} + elif isinstance(v, dict): + out[k] = _log_view(v, depth + 1) if depth < 1 else {"keys": len(v)} + else: + out[k] = type(v).__name__ + return out + + def _at_work(row: dict) -> dict: """A trigger still at work, as the run's record keeps it.""" return {k: row.get(k) for k in ("id", "status", "running", "pending", "executor_calls", "waits", "resumes_at") diff --git a/tst/unit/task_step/test_always_on_record.py b/tst/unit/task_step/test_always_on_record.py index 223e1a720..983883c2f 100644 --- a/tst/unit/task_step/test_always_on_record.py +++ b/tst/unit/task_step/test_always_on_record.py @@ -52,3 +52,29 @@ def fail(*a, **kw): assert "object_url" not in record and record["timeline_store_error"] == "OSError: bucket gone" assert len(record["timeline_tail"]) == ao.TIMELINE_TAIL and record["timeline_tail"][-1]["event"] == "end" assert not any(set(e) & set(ao.BULKY_EVENT_FIELDS) for e in record["timeline_tail"]) + + +@pytest.mark.asyncio +async def test_the_process_log_carries_no_one_s_words_only_the_timeline_does(monkeypatch, tmp_path, caplog): + def agent(n, texts, rig): + if n == 1: + rig.message("SECRET-SENT house at 12 Oak St, $855") + return "SECRET-PLAIN I'll book it" + rig.quiet() + return "SECRET-CLOSING bye" + + rig = Rig(monkeypatch, tmp_path, agent=agent, + answers=[{"outcome": "reply", "message": "SECRET-REPLY yes, card ending 4242", "used": "SECRET-USED"}]) + await rig.arm() + rig.triggers(rig.notify_at("sam", "2026-11-09T15:00:00Z", "SECRET-ALERT Sam's address", channel="noor 30")) + caplog.set_level("INFO", logger=ao.logger.name) + step = ao.AlwaysOnTaskStep("week", None, user_sim="linda", prompt="SECRET-PROMPT sort the cdmx weekend", + prompt_id="ask", agent_name="solver") + ctx = await rig.run(step) + + logged = "\n".join(r.getMessage() for r in caplog.records if r.name == ao.logger.name) + assert "always_on wake" in logged and "always_on user_sim_read" in logged + assert "SECRET" not in logged + stored = json.dumps(rig.timeline(ctx)) + for marker in ("SECRET-SENT", "SECRET-PLAIN", "SECRET-REPLY", "SECRET-ALERT", "SECRET-PROMPT"): + assert marker in stored From e85a475bb06f54e4f0597e08ab51994358a1dc67 Mon Sep 17 00:00:00 2001 From: Sarah Chitty Date: Mon, 9 Nov 2026 03:18:26 -0800 Subject: [PATCH 29/53] always_on: a world that only repeats ends a run with no end_time; long gaps fold their clock jumps A recurring trigger that never wakes the agent (every PT1H, notify to user_phone) made step 3 of the loop jump the clock forever when the run had no end_time: nothing counted toward max_wakes (a probe made 685 jumps in 4.8 s and moved a month). The rule: with no end_time, a jump counts as idle when nothing the step tracks is ahead (no alert or check for the agent, no user line, no held reply, no read) and every trigger with something ahead is a recurring time trigger with no fire in flight (none running, queued, waiting out a delay or hours, calling a contact or evaluating) and no one-off mark to come. IDLE_JUMPS_MAX (24) idle jumps in a row end the run as nothing_can_wake, which finish() records as finished_nothing_pending or stuck from what is pending; a world_only_repeats event names the triggers. A wake or a read restarts the count. So the rule can't cut short a run waiting on a known event (a held reply, a one-off trigger, a contact's delayed reply), and only ends one in which a recurring trigger has fired that many times running without waking the agent. With an end_time the run still goes on to it. The timeline keeps the first JUMPS_KEPT_PER_GAP (20) clock jumps between two wakes and folds the rest into one clock_jumps_folded entry (count, span, reasons, unsettled count), so a long gap can't grow it without bound; counts.clock_jumps counts them all. Co-Authored-By: Claude Opus 5.5 --- .../task_step/task_steps/always_on.py | 82 ++++++++++++++++- .../task_step/test_always_on_idle_world.py | 90 +++++++++++++++++++ 2 files changed, 168 insertions(+), 4 deletions(-) create mode 100644 tst/unit/task_step/test_always_on_idle_world.py diff --git a/src/agent_env/task_step/task_steps/always_on.py b/src/agent_env/task_step/task_steps/always_on.py index ca74e753a..ae5e4ae2e 100644 --- a/src/agent_env/task_step/task_steps/always_on.py +++ b/src/agent_env/task_step/task_steps/always_on.py @@ -37,7 +37,8 @@ closing wake, with the step's wrap-up line and the last-turn notice, and the run ends with end reason time_budget. - End reasons: end_time, user_done (one sitting), max_wakes, time_budget, finished_nothing_pending or stuck (nothing can wake the agent, with nothing pending or with something still pending), agent_turn_failed, agent_turn_timeout, - user_sim_error, step_error. What was still pending is recorded (``pending_at_end``): replies held, messages the user + user_sim_error, step_error. With no end_time, a world that only repeats ends the run too (``IDLE_JUMPS_MAX``, in the + loop's docstring), so a recurring trigger that never wakes the agent can't move the clock forever. What was still pending is recorded (``pending_at_end``): replies held, messages the user never read, user lines and alerts never delivered, the agent's checks not yet due, contact replies in progress. - Graders read what the user saw. The prompt response is every message_user text in order, each with its pretend time, never the agent's closing text. The conversation (``metadata["a2a_conversations"][step_id]``) holds the user's @@ -148,6 +149,11 @@ HOOK_NAME = "always_on" CHANGE_SUMMARY_CHARS = 200 TIMELINE_TAIL = 20 +# With no end_time, the run ends once this many clock jumps in a row have been to repeating triggers' marks alone (see +# the loop's docstring): the world can go on ticking, but nothing in it has woken the agent. +IDLE_JUMPS_MAX = 24 +# The clock jumps between two wakes the timeline keeps one by one; the rest are folded into one entry. +JUMPS_KEPT_PER_GAP = 20 PENDING_KEPT = 20 PENDING_TEXT_CHARS = 300 BULKY_EVENT_FIELDS = ("input", "raw", "read", "parts_text", "changes") @@ -312,6 +318,11 @@ def __init__(self, step: AlwaysOnTaskStep, context: TaskStepContext): self.pending: dict = {} self.end_note = "" self.time_budget = step.effective_time_budget() + self.idle_jumps = 0 # clock jumps in a row to repeating triggers' marks alone, since the last wake + self.idle_since: Optional[str] = None + self.clock_jumps = 0 + self.jumps_in_gap = 0 # clock jumps since the last wake, the first JUMPS_KEPT_PER_GAP of them logged + self.folded: Optional[dict] = None def close(self) -> None: for p in self._plugin_paths: @@ -569,8 +580,24 @@ async def advance(self, target: datetime, reason: str) -> None: if not (state.get("armed") and _utc(state["virtual_time"]) >= target): raise stuck = await self.settle() - self.log("clock_jump", **{"from": _iso(before), "to": _iso(target), "reason": reason, - "settled": not stuck, "still_at_work": stuck, "settle": self.last_settle}) + self.clock_jumps += 1 + self.jumps_in_gap += 1 + if self.jumps_in_gap <= JUMPS_KEPT_PER_GAP: + self.log("clock_jump", **{"from": _iso(before), "to": _iso(target), "reason": reason, + "settled": not stuck, "still_at_work": stuck, "settle": self.last_settle}) + return + # Past the first jumps since the last wake, one entry sums the rest up, so a long gap can't grow the timeline. + if self.folded is None: + self.folded = self.log("clock_jumps_folded", count=0, **{"from": _iso(before)}, reasons={}, + unsettled=0, settle_seconds=0.0) + f = self.folded + f["count"] += 1 + f["to"] = _iso(target) + f["reasons"][reason] = f["reasons"].get(reason, 0) + 1 + f["unsettled"] += 1 if stuck else 0 + f["settle_seconds"] = round(f["settle_seconds"] + self.last_settle.get("seconds", 0), 1) + if stuck: + f["last_still_at_work"] = stuck async def settle(self) -> list[dict]: """Wait until the world has settled: no trigger still at work, bar a fire waiting for a later time. A contact's @@ -623,6 +650,30 @@ async def _next_world_event(self) -> Optional[datetime]: out.append(_utc(st["next_event_at"])) return min(out) if out else None + async def _trigger_rows(self) -> list[dict]: + rows = [] + for env in await self._armed(): + if env.supports(EXT_TRIGGERS_URI, "state"): + rows += (await _call(env, EXT_TRIGGERS_URI, "state")).get("triggers", []) + return rows + + async def _only_repeating(self) -> bool: + """Whether all the world has ahead is recurring time triggers' marks: no trigger has a fire in flight (running, + queued, waiting out a delay or hours, calling a contact, evaluating) and none has a one-off mark to come.""" + for row in await self._trigger_rows(): + if (row.get("status") in IN_FLIGHT or row.get("running") or row.get("pending") or row.get("waits") + or row.get("waiting") or row.get("executor_calls") or row.get("evaluating")): + return False + when = row.get("when") or {} + if row.get("type") == "time" and row.get("status") == "armed" and "every" not in when \ + and (row.get("next_mark") or "after" not in when): + return False + return True + + async def _repeating_ids(self) -> list[str]: + return sorted({row["id"] for row in await self._trigger_rows() if row.get("type") == "time" + and row.get("status") == "armed" and "every" in (row.get("when") or {})}) + async def _notes_state(self, to: str) -> list[dict]: out = [] for env in self.notification_envs: @@ -698,6 +749,16 @@ def _schedule_read(self) -> None: # ---- the loop ------------------------------------------------------------------------------------------- async def loop(self, now: datetime) -> None: + """Until the run ends: a read that is due, else a wake for what is due, else a clock jump to what comes next. + + A world that only repeats: with no end_time, when nothing the step tracks is ahead (no alert or check for the + agent, no user line, no held reply, no read) and no trigger has work in flight or a one-off mark to come, the + world can only go on ticking: its recurring triggers fire again and again. Each such jump after which nothing + woke the agent counts; ``IDLE_JUMPS_MAX`` of them in a row end the run (``nothing_can_wake``, which ``finish`` + turns into finished_nothing_pending or stuck). A wake or a read starts the count again. The rule only ever ends + a run in which a recurring trigger has fired that many times running without waking the agent, and it can't + end one that waits on a known event; with an end_time the run goes on to it, its jumps folded in the timeline + past ``JUMPS_KEPT_PER_GAP`` per gap.""" step = self.step self.you_sent.append(dict(self.opening)) await self._chat_user([self._chat_part(_utc(self.opening["at"]), self.opening["text"], kind="opening")]) @@ -746,6 +807,16 @@ async def loop(self, now: datetime) -> None: if soonest is None: self.end_reason = NOTHING_CAN_WAKE # finished or stuck: finish() decides from what is pending break + if self.end is None and set(later) == {"world_event"} and await self._only_repeating(): + self.idle_jumps += 1 + self.idle_since = self.idle_since or _iso(now) + if self.idle_jumps > IDLE_JUMPS_MAX: + self.log("world_only_repeats", virtual_time=_iso(now), jumps=self.idle_jumps - 1, + since=self.idle_since, triggers=await self._repeating_ids()) + self.end_reason = NOTHING_CAN_WAKE + break + else: + self.idle_jumps, self.idle_since = 0, None why = sorted(k for k, v in later.items() if v == soonest) await self.advance(soonest, reason="+".join(why)) await self._last_read() @@ -901,6 +972,8 @@ async def wake(self, parts: list[dict], woke_by: list[str], notifications: Optio if step.wake_reminder: parts = [*parts, _step_part(step.wake_reminder_text, "reminder")] self.wakes += 1 + self.idle_jumps, self.idle_since = 0, None + self.jumps_in_gap, self.folded = 0, None msg_id = uuid.uuid4().hex await self._read_changes("before_turn") @@ -1101,6 +1174,7 @@ async def read(self, now: datetime, last: bool = False) -> None: for it in await take_due(self.notification_envs, USER_PHONE): phone.append({"at": it.get("at"), "channel": it["channel"], "text": "\n\n".join(p["text"] for p in it["parts"] if p.get("kind") == "text")}) + self.idle_jumps, self.idle_since = 0, None seen = list(self.unread) read = {"time": _iso(now), "assistant_messages": [{"at": m.get("at"), "text": m["text"]} for m in seen], "phone": phone, @@ -1272,7 +1346,7 @@ async def finish(self) -> None: and e.get("outcome") == "no_reply"), "user_dones": sum(1 for e in self.timeline if e["event"] == "user_sim_read" and e.get("outcome") == "done"), "scripted_user_lines": sum(1 for e in self.timeline if e["event"] == "scripted_user_line"), - "clock_jumps": sum(1 for e in self.timeline if e["event"] == "clock_jump"), + "clock_jumps": self.clock_jumps, "silent_wakes": sum(1 for e in self.timeline if e["event"] == "agent_turn_end" and not e.get("messages_to_user")), } diff --git a/tst/unit/task_step/test_always_on_idle_world.py b/tst/unit/task_step/test_always_on_idle_world.py new file mode 100644 index 000000000..7b5510af6 --- /dev/null +++ b/tst/unit/task_step/test_always_on_idle_world.py @@ -0,0 +1,90 @@ +"""A world that only repeats: a recurring trigger that never wakes the agent (an hourly alert to the user's phone, say) +must not move the clock forever. With no end_time the run ends once that is all the world has ahead; a known event ahead +keeps it going; with an end_time it goes on to it, and the timeline stays small.""" + +from __future__ import annotations + +import asyncio + +import pytest + +from agent_env.task_step.task_steps import always_on as ao +from tst.unit.task_step.always_on_harness import Rig + +pytestmark = pytest.mark.usefixtures("local_stores") + +NEWS = {"id": "news", "when": {"type": "time", "every": "PT1H"}, + "actions": [{"type": "notify", "channel": "News", "text": "headline", "to": ao.USER_PHONE}]} + + +async def _run(rig, step, ctx=None, seconds=30): + return await asyncio.wait_for(rig.run(step, ctx), seconds) # the old loop never ended + + +@pytest.mark.asyncio +async def test_with_no_end_time_a_world_that_only_repeats_ends_the_run(monkeypatch, tmp_path): + rig = Rig(monkeypatch, tmp_path) + await rig.arm() + rig.triggers(NEWS) + ctx = await _run(rig, rig.step()) + + record = rig.record(ctx) + assert record["end_reason"] == "finished_nothing_pending" + assert record["counts"]["clock_jumps"] == ao.IDLE_JUMPS_MAX and record["counts"]["wakes"] == 1 + [idle] = rig.events(ctx, "world_only_repeats") + assert (idle["jumps"], idle["since"], idle["triggers"]) == (ao.IDLE_JUMPS_MAX, "2026-11-09T14:00:00Z", ["news"]) + assert rig.now() == "2026-11-10T14:00:00Z" # a day of hourly ticks, not a month + + +@pytest.mark.asyncio +async def test_an_event_still_to_come_keeps_the_run_going_past_the_ticks(monkeypatch, tmp_path): + rig = Rig(monkeypatch, tmp_path) + await rig.arm() + rig.triggers(NEWS, rig.notify_at("sam", "2026-11-12T14:00:00Z", "Sam: flights?", channel="noor 30")) + ctx = await _run(rig, rig.step()) + + woke = [(w["virtual_time"], w["woke_by"]) for w in rig.events(ctx, "wake")] + assert woke == [("2026-11-09T14:00:00Z", ["user_request"]), ("2026-11-12T14:00:00Z", ["notification:noor 30"])] + assert rig.record(ctx)["end_reason"] == "finished_nothing_pending" + assert rig.events(ctx, "world_only_repeats")[0]["since"] == "2026-11-12T14:00:00Z" + + +@pytest.mark.asyncio +async def test_a_known_reply_ahead_isn_t_cut_short_by_the_ticks(monkeypatch, tmp_path): + monkeypatch.setattr(ao, "IDLE_JUMPS_MAX", 3) + + def agent(n, texts, rig): + if n == 1: + rig.message("blue house or green?") + else: + rig.quiet() + return "" + + rig = Rig(monkeypatch, tmp_path, agent=agent, answers=[{"outcome": "reply", "message": "blue"}]) + await rig.arm() + rig.triggers(NEWS) + user = {"schedule": {"wake": "07:30", "sleep": "23:30", "work": ["09:00", "18:00"]}, + "response_times": {"work": 600, "free": 600, "morning": 600}} # her reply reaches it ten hours later + ctx = await _run(rig, rig.step(), ctx=rig.context(user=user)) + + woke = [(w["virtual_time"], w["woke_by"]) for w in rig.events(ctx, "wake")] + assert woke == [("2026-11-09T14:00:00Z", ["user_request"]), ("2026-11-10T00:00:00Z", ["user_reply"])] + assert rig.record(ctx)["end_reason"] == "finished_nothing_pending" + assert rig.now() == "2026-11-10T03:00:00Z" # then three ticks with nothing ahead + + +@pytest.mark.asyncio +async def test_with_an_end_time_the_run_goes_on_to_it_and_folds_the_jumps(monkeypatch, tmp_path): + rig = Rig(monkeypatch, tmp_path) + await rig.arm() + rig.triggers(NEWS) + ctx = await _run(rig, rig.step(end_time="P3D")) + + record = rig.record(ctx) + assert record["end_reason"] == "end_time" and rig.now() == "2026-11-12T14:00:00Z" + assert record["counts"]["clock_jumps"] == 72 + jumps = rig.events(ctx, "clock_jump") + [folded] = rig.events(ctx, "clock_jumps_folded") + assert len(jumps) == ao.JUMPS_KEPT_PER_GAP and folded["count"] == 72 - ao.JUMPS_KEPT_PER_GAP + assert folded["to"] == "2026-11-12T14:00:00Z" and sum(folded["reasons"].values()) == folded["count"] + assert rig.events(ctx, "world_only_repeats") == [] From 86e70072bb1321c9897bbcfabbc64795b6554e53 Mon Sep 17 00:00:00 2001 From: Sarah Chitty Date: Mon, 9 Nov 2026 03:23:49 -0800 Subject: [PATCH 30/53] always_on: a user sim's schedule or response times need the task's clock; reads are timed on it Without sync_env_clock, a user sim with a schedule or response_times made the loop spin in real time (2,506 jumps in 3 s): read times and reply delays lay ahead on a clock nothing could move, so advance() returned and the loop came straight back. The step now refuses such a run at the start, before it touches the gateway, with an error naming the settings and sync_env_clock. A run with no clock and neither setting works as before. advance() raises instead of returning when no clock is armed, so no other path can spin the same way; the final check-in of an end_time run with no clock happens at once, as it did. _schedule_read times a message the chat env couldn't time (its clock unarmed) on the task's clock, another env's when it is armed, instead of the computer's. Co-Authored-By: Claude Opus 5.5 --- .../task_step/task_steps/always_on.py | 28 +++++-- tst/unit/task_step/test_always_on_no_clock.py | 81 +++++++++++++++++++ 2 files changed, 102 insertions(+), 7 deletions(-) create mode 100644 tst/unit/task_step/test_always_on_no_clock.py diff --git a/src/agent_env/task_step/task_steps/always_on.py b/src/agent_env/task_step/task_steps/always_on.py index ae5e4ae2e..0d8398bf7 100644 --- a/src/agent_env/task_step/task_steps/always_on.py +++ b/src/agent_env/task_step/task_steps/always_on.py @@ -15,7 +15,8 @@ everything sent meanwhile in one read: the time, the agent's messages, what triggers sent the user's phone (``notify`` to ``user_phone``) and the user's own lines (the opening request, ``user_says`` lines, ``final``). Nothing meant for the agent is in it. Its answer is reply / no_reply / done; a reply, or a done with text, is held - for the response time. Done ends a one-sitting run (after the agent's turn on its text); a run whose end_time falls + for the response time. A schedule and response times are task time, so a user sim that sets either needs an armed + clock (sync_env_clock): without one the step refuses to start. Done ends a one-sitting run (after the agent's turn on its text); a run whose end_time falls on a later day than its start goes on to end_time. - Scripted user lines are triggers' ``notify`` actions to the ``user_says`` recipient: sent to the agent as the user's message at their time. @@ -373,6 +374,13 @@ async def go(self) -> None: if not self.chat_envs: raise RuntimeError(f"always_on: no deployed env offers {EXT_USER_CHAT_URI} (message_user); rebuild its " "gateway from a branch that has user_chat.py") + timed = [k for k in ("schedule", "response_times") if us.get(k)] + if timed and not await self._armed(): + # Read times and reply delays are task time; with no clock to jump, the loop would wait them out in real + # time, spinning on the clock it can't move. + raise ValueError(f"always_on: user sim '{step.user_sim}' sets {' and '.join(timed)}, which need the " + "task's clock, and no env's clock is armed; run sync_env_clock before this step, or drop " + f"{' and '.join(timed)} from the user_sim step") now = await self.now() self.opening = {"at": _iso(now), "text": self._effective_prompt()} self.person["opening"] = self.opening @@ -572,7 +580,11 @@ async def advance(self, target: datetime, reason: str) -> None: before = await self.now() if target <= before: return - for env in await self._armed(): + armed = await self._armed() + if not armed: # nothing to move: the loop would come straight back here, spinning in real time + raise RuntimeError(f"always_on: can't move the task's clock to {_iso(target)} ({reason}): no env's clock " + "is armed") + for env in armed: try: await _call(env, EXT_CLOCK_URI, "advance", {"virtual_time": _iso(target)}, attempts=2) except Exception: @@ -736,11 +748,13 @@ def _latency(self, mode: str) -> timedelta: return timedelta(0) return timedelta(minutes=float(rt.get(mode, rt.get("free", 0)))) - def _schedule_read(self) -> None: + async def _schedule_read(self) -> None: if not self.unread or self.read_at is not None or self.gave_up or self.signed_off: return + # A message has no time when the chat env's clock is unarmed; the task's time (another env's clock, else the + # computer's) stands in for it. first_at = min(_utc(m["at"]) for m in self.unread if m.get("at")) if any(m.get("at") for m in self.unread) \ - else datetime.now(timezone.utc) + else await self.now() at, mode = self._available(first_at) if self.last_reply_at and at < self.last_reply_at: at, mode = self._available(self.last_reply_at) @@ -835,7 +849,7 @@ async def _final(self, now: datetime, reason: str = "end_time") -> None: step, not the user); with whatever else is due, and the last-turn notice. A reply held past now stays held, to be recorded as pending.""" step = self.step - if reason == "end_time" and self.end > now: + if reason == "end_time" and self.end > now and await self._armed(): # with no clock, the check-in is now await self.advance(self.end, reason="end_time") now = await self.now() self.final_sent = True @@ -857,7 +871,7 @@ async def _last_read(self) -> None: when the run ended on an error.""" if not self.unread or self.signed_off or self.gave_up or self.end_reason in ERROR_ENDS: return - self._schedule_read() + await self._schedule_read() at = self.read_at or await self.now() await self.read(at, last=True) @@ -1028,7 +1042,7 @@ def record() -> None: if stuck: self.log("settle_timeout", wake=self.wakes, still_at_work=stuck, settle=self.last_settle) await self._read_changes("after_turn") - self._schedule_read() + await self._schedule_read() async def _nudge(self, closing: str) -> None: """The wake ended with closing text and neither message_user nor stay_quiet, so give the agent one more short diff --git a/tst/unit/task_step/test_always_on_no_clock.py b/tst/unit/task_step/test_always_on_no_clock.py new file mode 100644 index 000000000..757fb7c69 --- /dev/null +++ b/tst/unit/task_step/test_always_on_no_clock.py @@ -0,0 +1,81 @@ +"""Without the task's clock: read times and reply delays are task time, so a user sim with a schedule or response times +is refused at the start instead of the loop spinning in real time; a run with neither still works. And a message the +chat env couldn't time (its clock unarmed) is read on the task's clock when another env's is armed.""" + +from __future__ import annotations + +import asyncio +from datetime import datetime, timedelta, timezone + +import pytest + +from agent_env.env.gateway.constants import EXT_CLOCK_URI +from agent_env.task_step.task_steps import always_on as ao +from tst.unit.task_step.always_on_harness import Rig + +pytestmark = pytest.mark.usefixtures("local_stores") + +DAY = {"schedule": {"wake": "07:30", "sleep": "23:30", "work": ["09:00", "18:00"]}, + "response_times": {"work": 150, "free": 30, "morning": 15}} + + +def _asks_once(n, texts, rig): + if n == 1: + rig.message("blue house or green?") + else: + rig.quiet() + return "" + + +@pytest.mark.parametrize("user", [DAY, {"schedule": DAY["schedule"]}, {"response_times": DAY["response_times"]}], + ids=["both", "schedule", "response_times"]) +@pytest.mark.asyncio +async def test_a_schedule_or_response_times_with_no_task_clock_is_refused_at_the_start(monkeypatch, tmp_path, user): + rig = Rig(monkeypatch, tmp_path, agent=_asks_once, answers=[{"outcome": "reply", "message": "blue"}]) + ctx = rig.context(user=user) + with pytest.raises(ValueError, match="need the task's clock, and no env's clock is armed"): + await asyncio.wait_for(rig.run(rig.step(), ctx), 5) # the old loop spun until real time caught up + assert rig.sent == [] and rig.gw._user_chat.roles == () # refused before the gateway was touched + + +@pytest.mark.asyncio +async def test_with_no_clock_and_no_schedule_a_run_reads_and_replies_at_once(monkeypatch, tmp_path): + rig = Rig(monkeypatch, tmp_path, agent=_asks_once, answers=[{"outcome": "reply", "message": "blue"}]) + ctx = await asyncio.wait_for(rig.run(rig.step()), 10) + + assert [w["woke_by"] for w in rig.events(ctx, "wake")] == [["user_request"], ["user_reply"]] + assert rig.record(ctx)["end_reason"] == "finished_nothing_pending" + + +class _ClockEnv: + """Another env whose clock is armed (the chat env's isn't): only clock/v1, frozen at ``at``.""" + + env_id, env_version, metadata = "calendar", 1, {"deploy_step_id": "deploy-calendar"} + + def __init__(self, at: datetime) -> None: + self.at = at + + def supports(self, uri, method=None) -> bool: + return uri == EXT_CLOCK_URI and method in (None, "state", "advance") + + async def invoke(self, uri, method, params=None, **kw): + if method == "advance": + self.at = max(self.at, ao._utc(params["virtual_time"])) + return {"armed": True, "t0": "2027-03-01T15:00:00Z", "virtual_time": ao._iso(self.at), + "virtual_seconds_per_real_second": 0} + + +@pytest.mark.asyncio +async def test_a_message_the_chat_env_couldn_t_time_is_read_on_the_task_s_clock(monkeypatch, tmp_path): + rig = Rig(monkeypatch, tmp_path, agent=_asks_once, answers=[{"outcome": "reply", "message": "blue"}]) + ctx = rig.context(user=DAY) + task_time = datetime(2027, 3, 1, 15, 0, tzinfo=timezone.utc) # Mon 10 AM in New York: she reads at once + assert abs(datetime.now(timezone.utc) - task_time) > timedelta(days=30) + ctx.deployed_envs.append(_ClockEnv(task_time)) + ctx = await asyncio.wait_for(rig.run(rig.step(), ctx), 10) + + [read] = rig.reads + assert read["assistant_messages"][0]["at"] is None # the chat env's clock is unarmed + assert read["time"] == "2027-03-01T15:00:00Z" + [scheduled] = rig.events(ctx, "user_read_scheduled") + assert scheduled["read_at"] == "2027-03-01T15:00:00Z" From ac6bb5e14b7dedef8d4c492fc919554f1460934f Mon Sep 17 00:00:00 2001 From: Sarah Chitty Date: Mon, 9 Nov 2026 03:25:04 -0800 Subject: [PATCH 31/53] always_on: a user's night may run past midnight; back-to-back busy blocks never leave a read inside one `hm >= sleep or hm < wake` treated every waking hour as night when sleep came after midnight (wake 08:00, sleep 00:30). _asleep now takes the night as the span from sleep to wake, either side of midnight. The search for her next free time made at most four passes, so five or more busy blocks and nights in a row could leave a read inside one. It now passes until nothing moves the time; since a block once passed is behind and a night always ends at a waking time, that is at most two passes per block and one more, and the bound is enforced. Co-Authored-By: Claude Opus 5.5 --- .../task_step/task_steps/always_on.py | 19 +++++++++-- tst/unit/task_step/test_always_on_loop.py | 34 +++++++++++++++++++ 2 files changed, 50 insertions(+), 3 deletions(-) diff --git a/src/agent_env/task_step/task_steps/always_on.py b/src/agent_env/task_step/task_steps/always_on.py index 0d8398bf7..d5a1232c4 100644 --- a/src/agent_env/task_step/task_steps/always_on.py +++ b/src/agent_env/task_step/task_steps/always_on.py @@ -714,7 +714,8 @@ def _schedule(self) -> Optional[dict]: def _available(self, t: datetime) -> tuple[datetime, str]: """When the user would first look at a message that arrived at ``t``, and in what mode (morning, work, free): - while awake and not busy. No schedule: at once, mode "now".""" + while awake and not busy. Sleep may run past midnight (wake 08:00, sleep 00:30). No schedule: at once, mode + "now".""" sched = self._schedule() if not sched: return t, "now" @@ -726,17 +727,21 @@ def _available(self, t: datetime) -> tuple[datetime, str]: for b in sched.get("busy") or []] lt = t.astimezone(tz) mode = None - for _ in range(4): + # Each pass moves past a busy block or a night, or ends: a block once passed is behind, and a night always ends + # at a waking time; so the passes are bounded by two per block and one more. + for _ in range(2 * len(busy) + 2): moved = False for start, stop in busy: if start <= lt < stop: lt, moved = stop, True hm = (lt.hour, lt.minute) - if hm >= sleep or hm < wake: + if _asleep(hm, wake, sleep): day = lt.date() if hm < wake else lt.date() + timedelta(days=1) lt, mode, moved = datetime(day.year, day.month, day.day, *wake, tzinfo=tz), "morning", True if not moved: break + else: + raise RuntimeError(f"always_on: the user's schedule never frees a time to read after {_iso(t)}") if mode is None: hm = (lt.hour, lt.minute) mode = "work" if DAYS[lt.weekday()] in work_days and work[0] <= hm < work[1] else "free" @@ -1633,6 +1638,14 @@ def _at_work(row: dict) -> dict: if row.get(k) not in (None, [], 0)} +def _asleep(hm: tuple[int, int], wake: tuple[int, int], sleep: tuple[int, int]) -> bool: + """Whether local time ``hm`` falls in the night from ``sleep`` to ``wake``; a sleep time after midnight (00:30 with + a wake at 08:00) makes the night the span between them. Equal times: never asleep.""" + if sleep > wake: + return hm >= sleep or hm < wake + return sleep <= hm < wake + + def _hm(text: str) -> tuple[int, int]: h, m = text.split(":") return int(h), int(m) diff --git a/tst/unit/task_step/test_always_on_loop.py b/tst/unit/task_step/test_always_on_loop.py index c7406ac1e..fc26a03ac 100644 --- a/tst/unit/task_step/test_always_on_loop.py +++ b/tst/unit/task_step/test_always_on_loop.py @@ -304,3 +304,37 @@ async def test_a_user_and_a_world_in_different_time_zones_are_refused(monkeypatc def test_both_steps_are_registered(): registry = get_task_step_registry() assert registry["always_on"] is ao.AlwaysOnTaskStep and registry["user_sim"] is UserSimTaskStep + + +# ---- the user's schedule -------------------------------------------------------------------------------------------- + +def _available(monkeypatch, tmp_path, schedule: dict, at: str): + rig = Rig(monkeypatch, tmp_path) + run = ao._Run(rig.step(), rig.context(user={"schedule": schedule})) + run.us = {"schedule": schedule} + run.tz = ao.ZoneInfo("America/New_York") + got, mode = run._available(ao._utc(at)) + return ao._iso(got), mode + + +@pytest.mark.parametrize("at,read_at,mode", [ + ("2026-11-10T14:00:00Z", "2026-11-10T14:00:00Z", "work"), # Tue 9:00 AM: awake + ("2026-11-11T04:00:00Z", "2026-11-11T04:00:00Z", "free"), # Tue 11:00 PM: still up + ("2026-11-11T05:15:00Z", "2026-11-11T05:15:00Z", "free"), # Wed 12:15 AM: still up + ("2026-11-11T05:45:00Z", "2026-11-11T13:00:00Z", "morning"), # Wed 12:45 AM: asleep until 8:00 + ("2026-11-11T10:00:00Z", "2026-11-11T13:00:00Z", "morning"), # Wed 5:00 AM: asleep until 8:00 +]) +def test_a_night_that_starts_after_midnight_keeps_the_evening_awake(monkeypatch, tmp_path, at, read_at, mode): + night_owl = {"wake": "08:00", "sleep": "00:30", "work": ["09:00", "18:00"]} + assert _available(monkeypatch, tmp_path, night_owl, at) == (read_at, mode) + + +def test_back_to_back_busy_blocks_and_nights_never_leave_a_read_inside_one(monkeypatch, tmp_path): + blocks = [("2026-11-10T08:00", "2026-11-10T09:00"), ("2026-11-10T09:00", "2026-11-10T10:00"), + ("2026-11-10T10:00", "2026-11-10T23:45"), # into the night: she reads next morning + ("2026-11-11T07:30", "2026-11-11T08:30"), ("2026-11-11T08:30", "2026-11-11T09:30"), + ("2026-11-11T09:30", "2026-11-11T10:00")] + schedule = {"wake": "07:30", "sleep": "23:30", "work": ["09:00", "18:00"], + "busy": [{"start": a, "end": b} for a, b in reversed(blocks)]} # latest first: one block a pass + # Tue 8:00 AM in New York: six blocks and a night in a row, more than the old four passes + assert _available(monkeypatch, tmp_path, schedule, "2026-11-10T13:00:00Z") == ("2026-11-11T15:00:00Z", "morning") From 70b94944d3de029497a277f00caea1944a43d0fe Mon Sep 17 00:00:00 2001 From: Sarah Chitty Date: Mon, 9 Nov 2026 03:25:55 -0800 Subject: [PATCH 32/53] always_on: the prompt's file parts reach the agent on the opening wake _effective_prompt kept only text parts and the opening wake sent only the user's labelled line, so a task prompt with a file (or data) part lost it. The opening wake now carries the prompt's parts as prompt_agent sends them: each non-text part as it is (seed applied), in the prompt's order, with the text as the user's labelled line where the first text part was. The graded conversation's opening user turn carries them too. Co-Authored-By: Claude Opus 5.5 --- .../task_step/task_steps/always_on.py | 27 +++++++++++--- .../task_step/test_always_on_prompt_parts.py | 37 +++++++++++++++++++ 2 files changed, 59 insertions(+), 5 deletions(-) create mode 100644 tst/unit/task_step/test_always_on_prompt_parts.py diff --git a/src/agent_env/task_step/task_steps/always_on.py b/src/agent_env/task_step/task_steps/always_on.py index d5a1232c4..3e39cba0d 100644 --- a/src/agent_env/task_step/task_steps/always_on.py +++ b/src/agent_env/task_step/task_steps/always_on.py @@ -780,10 +780,11 @@ async def loop(self, now: datetime) -> None: past ``JUMPS_KEPT_PER_GAP`` per gap.""" step = self.step self.you_sent.append(dict(self.opening)) - await self._chat_user([self._chat_part(_utc(self.opening["at"]), self.opening["text"], kind="opening")]) + opened_at = _utc(self.opening["at"]) + await self._chat_user(self._opening_parts(lambda text: self._chat_part(opened_at, text, kind="opening"))) due = await take_due(self.notification_envs, self.recipient) if self.notification_envs else [] - await self.wake([*delivered_parts(due), self._user_part(_utc(self.opening["at"]), self.opening["text"], - opening=True)], + await self.wake([*delivered_parts(due), + *self._opening_parts(lambda text: self._user_part(opened_at, text, opening=True))], woke_by=[f"notification:{d['channel']}" for d in due] + ["user_request"], notifications=[{k: d.get(k) for k in ("at", "channel", "source", "source_id", "to")} for d in due]) @@ -880,9 +881,25 @@ async def _last_read(self) -> None: at = self.read_at or await self.now() await self.read(at, last=True) + def _prompt_parts(self) -> list[dict]: + """The task's prompt as A2A parts, its seed applied.""" + return self.step._apply_seed(self.step._effective_parts(), self.ctx.metadata.get("seed", {})) + def _effective_prompt(self) -> str: - parts = self.step._apply_seed(self.step._effective_parts(), self.ctx.metadata.get("seed", {})) - return "\n".join(p.get("text", "") for p in parts if p.get("kind") == "text") + return "\n".join(p.get("text", "") for p in self._prompt_parts() if p.get("kind") == "text") + + def _opening_parts(self, labelled) -> list[dict]: + """The prompt as the opening wake and the user's chat carry it: its text as one labelled line from the user + (``labelled(text)``), where its first text part was, and every other part (a file, data) as it is, in the + prompt's order, as prompt_agent sends them.""" + out, text = [], self._effective_prompt() + for part in self._prompt_parts(): + if part.get("kind") != "text": + out.append(part) + elif text is not None: + out.append(labelled(text)) + text = None + return out def _user_part(self, at: datetime, text: str, opening: bool = False) -> dict: first = self.person["first"] diff --git a/tst/unit/task_step/test_always_on_prompt_parts.py b/tst/unit/task_step/test_always_on_prompt_parts.py new file mode 100644 index 000000000..138c66110 --- /dev/null +++ b/tst/unit/task_step/test_always_on_prompt_parts.py @@ -0,0 +1,37 @@ +"""A task prompt's file (and other non-text) parts reach the agent on the opening wake, as prompt_agent sends them, and +are in the user's opening turn of the graded conversation; its text is the user's labelled line.""" + +from __future__ import annotations + +import pytest + +from agent_env.task_step.task_steps import always_on as ao +from tst.unit.task_step.always_on_harness import Rig + +pytestmark = pytest.mark.usefixtures("local_stores") + +ITINERARY = {"kind": "file", "file": {"uri": "s3://trips//itinerary.pdf", "name": "itinerary.pdf", + "mimeType": "application/pdf"}} + + +@pytest.mark.asyncio +async def test_the_prompt_s_file_reaches_the_agent_on_the_opening_wake_in_its_place(monkeypatch, tmp_path): + rig = Rig(monkeypatch, tmp_path) + await rig.arm() + step = ao.AlwaysOnTaskStep("week", None, user_sim="linda", prompt_id="ask", agent_name="solver", + parts=[{"kind": "text", "text": "sort the cdmx weekend"}, ITINERARY, + {"kind": "text", "text": "flights are in the pdf"}]) + ctx = rig.context() + ctx.metadata["seed"] = {"trip": "cdmx"} + ctx = await rig.run(step, ctx) + + opening = rig.parts[0] + files = [p for p in opening if p.get("kind") == "file"] + assert files == [{"kind": "file", "file": {**ITINERARY["file"], "uri": "s3://trips/cdmx/itinerary.pdf"}}] + texts = [p["text"] for p in opening if p.get("kind") == "text"] + line = "[Mon Nov 9, 2:00 PM UTC] Linda: sort the cdmx weekend\nflights are in the pdf" + assert texts.count(line) == 1 + assert opening.index(files[0]) == next(i for i, p in enumerate(opening) if p.get("text") == line) + 1 + [user_turn, *_] = [m for m in rig.chat(ctx) if m["role"] == "user"] + assert [p.get("kind") for p in user_turn["parts"]] == ["text", "file"] + assert user_turn["parts"][1]["file"]["uri"] == "s3://trips/cdmx/itinerary.pdf" From c8da0502d3d702ae03e575f59d3b10096889a588 Mon Sep 17 00:00:00 2001 From: Sarah Chitty Date: Mon, 9 Nov 2026 03:26:59 -0800 Subject: [PATCH 33/53] always_on: the run's user_model sets the user sim agent's model, and the record says which was used task run --user-model, eval run --user-model and a bundle's user_model set the run's user_overrides.user_model, which prompt_agent honours (_resolve_user_model) and always_on ignored, always setting the user_sim step's model. Now the user sim agent's model is the user_sim step's own step_params model, else the run's user_model, else the user_sim step's model. The record's user_sim block gives the model used, where it came from (model_from), the user_sim step's model and whether the agent's card took it (model_set); the start event logs the model used. Co-Authored-By: Claude Opus 5.5 --- .../task_step/task_steps/always_on.py | 24 ++++++++-- .../task_step/test_always_on_user_model.py | 47 +++++++++++++++++++ 2 files changed, 67 insertions(+), 4 deletions(-) create mode 100644 tst/unit/task_step/test_always_on_user_model.py diff --git a/src/agent_env/task_step/task_steps/always_on.py b/src/agent_env/task_step/task_steps/always_on.py index 3e39cba0d..b1be18932 100644 --- a/src/agent_env/task_step/task_steps/always_on.py +++ b/src/agent_env/task_step/task_steps/always_on.py @@ -415,7 +415,7 @@ async def go(self) -> None: now = await self.now() self.log("start", virtual_time=_iso(now), end_time=_iso(self.end) if self.end else None, agent=step.agent_name, agent_role=self.recipient, user_sim=step.user_sim, version=self.version, - user_model=us["model"], max_wakes=step.max_wakes, time_zone=self.zone, multi_day=self.multi_day, + user_model=self.user_model, max_wakes=step.max_wakes, time_zone=self.zone, multi_day=self.multi_day, scheduled_checks=self.checks, clock_rate=await self._rate()) try: await self.loop(now) @@ -477,7 +477,8 @@ async def _configure_agents(self) -> None: ctx.record_model("solver", step.id, negotiated.fields.get("model") if negotiated else None) setup = self.plugin.setup(self.version, self.person) - user_config = {"model": self.us["model"], "system_prompt": setup.get("system_prompt"), + self.user_model, self.user_model_from = self._user_model() + user_config = {"model": self.user_model, "system_prompt": setup.get("system_prompt"), "output_format": setup.get("output_format")} user_config = {k: v for k, v in user_config.items() if v is not None} negotiated = A2AAgent.negotiate_agent_config(self.user_agent.a2a_card or {}, user_config) @@ -485,7 +486,10 @@ async def _configure_agents(self) -> None: await protocol.post_agent_config(self.user_url + negotiated.endpoint, negotiated.fields) accepted = sorted(negotiated.fields) if negotiated else [] ctx.record_model("user simulator", step.id, negotiated.fields.get("model") if negotiated else None) - self.user_setup = {"system_prompt_chars": len(setup.get("system_prompt") or ""), + self.user_setup = {"model": self.user_model, "model_from": self.user_model_from, + "user_sim_step_model": self.us.get("model"), + "model_set": "model" in accepted, # else the agent ran its own + "system_prompt_chars": len(setup.get("system_prompt") or ""), "output_format": setup.get("output_format"), "accepted_fields": accepted} if "system_prompt" not in accepted and setup.get("system_prompt"): logger.warning("always_on: the user sim agent's card did not accept system_prompt; it keeps its deploy one") @@ -515,6 +519,18 @@ def rules(self) -> str: return " ".join([MESSAGE_RULES, *([QUIET_RULE] if self.step.stay_quiet else []), *([CHECKS_RULE] if self.checks["on"] else [])]) + def _user_model(self) -> tuple[str, str]: + """The user sim agent's model for this run, and where it came from: the user_sim step's own ``step_params`` + model, else the run's ``user_model`` (``task run`` / ``eval run --user-model``, a bundle's ``user_model``, as + prompt_agent's ``_resolve_user_model`` honours it), else the user_sim step's ``model``.""" + overrides = self.ctx.metadata.get("user_overrides") or {} + own = ((overrides.get("step_params") or {}).get(self.us.get("step_id")) or {}).get("model") + if own: + return own, "user_sim_step_params" + if overrides.get("user_model"): + return overrides["user_model"], "run_user_model" + return self.us["model"], "user_sim_step" + async def _configure_checks(self) -> None: """Give the agent's role scheduled checks (notifications/v1) on each env whose clock is set: by default when the run is multi-day, else as ``scheduled_checks`` says. A role the task already gave checks keeps its own @@ -1398,7 +1414,7 @@ async def finish(self) -> None: and e.get("stay_quiet_calls")) record = {"step_id": step.id, "end_reason": self.end_reason, "end_note": self.end_note, "error": self.error, "error_type": self.error_type, "counts": counts, - "user_sim": {"name": step.user_sim, "version": self.version, "model": self.us.get("model"), + "user_sim": {"name": step.user_sim, "version": self.version, "agent_name": self.us.get("agent_name"), **self.user_setup}, "end_time": _iso(self.end) if self.end else None, "time_zone": self.zone, "multi_day": self.multi_day, "real_seconds": round(time.time() - self.t0_real, 1), diff --git a/tst/unit/task_step/test_always_on_user_model.py b/tst/unit/task_step/test_always_on_user_model.py new file mode 100644 index 000000000..9147aa3b5 --- /dev/null +++ b/tst/unit/task_step/test_always_on_user_model.py @@ -0,0 +1,47 @@ +"""The user sim agent's model: a run's user_model (task run / eval run --user-model, a bundle's user_model) wins over the +user_sim step's model, as prompt_agent honours it; the user_sim step's own step_params win over both. The record says +which model was used and where it came from.""" + +from __future__ import annotations + +import pytest + +from agent_env.a2a_agent import A2AAgent +from agent_env.task_step.task_steps import always_on as ao +from tst.unit.task_step.always_on_harness import USER_SIM, Rig + +pytestmark = pytest.mark.usefixtures("local_stores") + +CONFIG_CARD = {"capabilities": {"extensions": [{"uri": A2AAgent.EXT_AGENT_CONFIG, "params": { + "endpoint": "/ext/agent-config", + "methods": {"set": {"request": {"supported": ["model", "system_prompt", "output_format"]}}}}}]}} + + +@pytest.mark.parametrize("overrides,model,source", [ + ({}, "sim-model", "user_sim_step"), + ({"user_model": "run-model"}, "run-model", "run_user_model"), + ({"user_model": "run-model", "step_params": {"linda-step": {"model": "step-model"}}}, "step-model", + "user_sim_step_params"), +], ids=["the user_sim step's", "the run's", "the user_sim step's step_params"]) +@pytest.mark.asyncio +async def test_the_run_s_user_model_sets_the_user_sim_agent_s_model(monkeypatch, tmp_path, overrides, model, source): + posted: list[tuple[str, dict]] = [] + + async def post(url, fields): + posted.append((url, fields)) + + monkeypatch.setattr(ao.protocol, "post_agent_config", post) + rig = Rig(monkeypatch, tmp_path) + await rig.arm() + ctx = rig.context(user={"step_id": "linda-step"}) + next(a for a in ctx.deployed_agents if a.agent_name == "user-sim").a2a_card = CONFIG_CARD + ctx.metadata["user_overrides"] = overrides + ctx = await rig.run(rig.step(), ctx) + + [(url, fields)] = [(u, f) for u, f in posted if u.startswith(USER_SIM)] + assert fields["model"] == model + record = rig.record(ctx)["user_sim"] + assert (record["model"], record["model_from"], record["user_sim_step_model"]) == (model, source, "sim-model") + assert record["model_set"] is True + assert ctx.metadata["models"]["user simulator"]["week"] == model + assert rig.events(ctx, "start")[0]["user_model"] == model From 935f0ad1e7ae0541d4c66ce0defb3beacd4171fa Mon Sep 17 00:00:00 2001 From: Sarah Chitty Date: Mon, 9 Nov 2026 03:28:55 -0800 Subject: [PATCH 34/53] always_on: the task's clock runs only during the agent's turns; frozen between them, its rate back at the end FD-5366 says the clock runs at the task's speed while the agent works. The step left it running during settles and user sim reads too, so at a rate above 0 a slow contact reply moved the clock (a 90 s reply at 60x put its alert a minute and a half late) and a read's reply was timed after it. At the start the step freezes (rate 0) each armed clock that runs, runs it at the task's rate for each agent turn (wake and nudge) and freezes it again after, and puts the task's rate back when the step ends, so later steps see the clock as sync_env_clock left it. It uses clock/v1 set_rate, not set_time as sync_env_clock does: a re-arm bumps the clock's generation, which re-anchors every relative or recurring time trigger to the new t0 on every turn. A clock already at rate 0 is left alone (every existing test); a gateway without set_rate keeps running, as settings.clock in the record says. Co-Authored-By: Claude Opus 5.5 --- .../task_step/task_steps/always_on.py | 78 ++++++++++++++++--- .../task_step/test_always_on_clock_rate.py | 72 +++++++++++++++++ 2 files changed, 138 insertions(+), 12 deletions(-) create mode 100644 tst/unit/task_step/test_always_on_clock_rate.py diff --git a/src/agent_env/task_step/task_steps/always_on.py b/src/agent_env/task_step/task_steps/always_on.py index b1be18932..510ec4932 100644 --- a/src/agent_env/task_step/task_steps/always_on.py +++ b/src/agent_env/task_step/task_steps/always_on.py @@ -20,9 +20,10 @@ on a later day than its start goes on to end_time. - Scripted user lines are triggers' ``notify`` actions to the ``user_says`` recipient: sent to the agent as the user's message at their time. -- The clock runs at the task's speed during turns (sync_env_clock). When nothing is due, the step advances it to the - earliest of: the next trigger event, the next notification, a held reply, a pending read, or end_time, never past - one; then waits for the world to settle. A contact's reply (an AI-played contact, a trigger's executor) counts as in +- The clock runs at the task's speed (sync_env_clock) only during the agent's turns: the step freezes it between them + (clock/v1 set_rate), so settles and reads don't move it, and puts the task's rate back at the end. When nothing is + due, the step advances it to the earliest of: the next trigger event, the next notification, a held reply, a + pending read, or end_time, never past one; then waits for the world to settle. A contact's reply (an AI-played contact, a trigger's executor) counts as in progress until it ends or its time limit passes; other trigger work gets ``settle_timeout_seconds``. Contacts whose task sets no time limit get ``contact_timeout_seconds``, and one trigger's replies run side by side (``contact_replies_in_parallel``). What comes due in the same minute wakes the agent once. @@ -269,7 +270,10 @@ async def _execute(self, context: TaskStepContext) -> TaskStepContext: try: await run.go() finally: - run.close() + try: + await run.restore() + finally: + run.close() return context @@ -324,6 +328,19 @@ def __init__(self, step: AlwaysOnTaskStep, context: TaskStepContext): self.clock_jumps = 0 self.jumps_in_gap = 0 # clock jumps since the last wake, the first JUMPS_KEPT_PER_GAP of them logged self.folded: Optional[dict] = None + self.clock: dict = {} # the task's clock rate, and whether the step freezes it between turns + self.clock_rates: list[tuple[Any, float]] = [] # (env, the task's rate) for each clock the step freezes + self.undo: list[tuple[str, Any]] = [] # (what, coroutine function): the gateway settings to put back at the end + + async def restore(self) -> None: + """Put back what the step changed on the gateways, latest first, so later steps of the task see the settings + they had before it: the clock's rate, ... Each is tried on its own; one that fails is logged.""" + while self.undo: + what, put_back = self.undo.pop() + try: + await put_back() + except Exception as e: + logger.warning(f"always_on: could not put back {what} after the run: {type(e).__name__}: {e}") def close(self) -> None: for p in self._plugin_paths: @@ -393,6 +410,7 @@ async def go(self) -> None: chat_config["stay_quiet"] = True for env in self.chat_envs: await _call(env, EXT_USER_CHAT_URI, "config", chat_config) + await self._configure_clock() await self._configure_checks() await self._configure_contacts() await self._configure_agents() @@ -494,6 +512,47 @@ async def _configure_agents(self) -> None: if "system_prompt" not in accepted and setup.get("system_prompt"): logger.warning("always_on: the user sim agent's card did not accept system_prompt; it keeps its deploy one") + async def _configure_clock(self) -> None: + """The clock runs at the task's rate only while the agent works (FD-5366): freeze it (rate 0) now, and between + turns, so a slow contact reply or user sim read doesn't move the task's time. With clock/v1 ``set_rate``, + which keeps the clock's marks (a re-arm with set_time, as sync_env_clock arms it, would re-anchor every + relative or recurring time trigger); the task's rate is put back at the end. A clock already frozen (rate 0) + is left alone; a gateway without set_rate keeps running between turns, as the record says.""" + envs = [] + for env in await self._armed(): + state = await _call(env, EXT_CLOCK_URI, "state") + rate = float(state.get("virtual_seconds_per_real_second") or 0) + if rate <= 0: + continue + if not env.supports(EXT_CLOCK_URI, "set_rate"): + envs.append({"env_id": env.env_id, "rate": rate, "frozen_between_turns": False, + "why": "the gateway has no clock set_rate"}) + continue + self.clock_rates.append((env, rate)) + envs.append({"env_id": env.env_id, "rate": rate, "frozen_between_turns": True}) + if self.clock_rates: + self.undo.append(("the clock's rate", lambda: self._clock_runs(True))) + await self._clock_runs(False) + self.clock = {"envs": envs} + + async def _clock_runs(self, running: bool) -> None: + """Run each clock the step freezes at the task's rate (``running``), or freeze it.""" + for env, rate in self.clock_rates: + await _call(env, EXT_CLOCK_URI, "set_rate", {"virtual_seconds_per_real_second": rate if running else 0}, + attempts=3) + + async def _agent_turn(self, parts: list[dict], msg_id: str, record) -> tuple[str, dict]: + """One turn of the agent on ``parts``: the task's clock runs at its rate while the agent works, and is frozen + again after.""" + step = self.step + await self._clock_runs(True) + try: + return await send_and_wait(self.target_url, parts, agent=self.agent, message_id=msg_id, + context_id=self.solver_context_id, timeout_seconds=step.timeout_seconds, + poll_interval_seconds=step.poll_interval_seconds, before_send=record) + finally: + await self._clock_runs(False) + async def _configure_contacts(self) -> None: """On each env with triggers, let one trigger's replies run side by side (``contact_replies_in_parallel``) and give contacts that set no time limit ``contact_timeout_seconds``; a gateway too old for it keeps its ways.""" @@ -1036,10 +1095,7 @@ def record() -> None: last=last, parts_text="\n".join(p.get("text", "") for p in parts if p.get("kind") == "text")) t = time.time() try: - task_id, result = await send_and_wait(self.target_url, parts, agent=self.agent, message_id=msg_id, - context_id=self.solver_context_id, - timeout_seconds=step.timeout_seconds, - poll_interval_seconds=step.poll_interval_seconds, before_send=record) + task_id, result = await self._agent_turn(parts, msg_id, record) except TimeoutError as e: self._failed("agent_turn_timeout", f"agent turn timed out after {step.timeout_seconds}s: {e}") self.log("agent_turn_end", n=self.wakes, state="timeout", real_seconds=round(time.time() - t, 1)) @@ -1096,10 +1152,7 @@ def record() -> None: t = time.time() try: - task_id, result = await send_and_wait(self.target_url, parts, agent=self.agent, message_id=msg_id, - context_id=self.solver_context_id, - timeout_seconds=step.timeout_seconds, - poll_interval_seconds=step.poll_interval_seconds, before_send=record) + task_id, result = await self._agent_turn(parts, msg_id, record) except TimeoutError as e: self._failed("agent_turn_timeout", f"agent nudge turn timed out after {step.timeout_seconds}s: {e}") self.log("nudge", n=self.nudges, wake=self.wakes, virtual_time=_iso(sent_at), closing_text=closing, @@ -1420,6 +1473,7 @@ async def finish(self) -> None: "multi_day": self.multi_day, "real_seconds": round(time.time() - self.t0_real, 1), "timeline_events": len(self.timeline), "pending_at_end": _bounded(self.pending)} record["settings"] = {"stay_quiet": step.stay_quiet, "nudge": step.nudge, "wake_reminder": step.wake_reminder, + "clock": self.clock, "scheduled_checks": self.checks, "contacts": self.contacts, "time_budget_seconds": self.time_budget, "wake_reserve_seconds": step.wake_reserve_seconds()} diff --git a/tst/unit/task_step/test_always_on_clock_rate.py b/tst/unit/task_step/test_always_on_clock_rate.py new file mode 100644 index 000000000..49a70eeb1 --- /dev/null +++ b/tst/unit/task_step/test_always_on_clock_rate.py @@ -0,0 +1,72 @@ +"""The task's clock runs at its rate only while the agent works (FD-5366): between turns the step freezes it, so a slow +contact reply or user sim read doesn't move the task's time and alerts arrive when they happen; at the end the task's +rate is back. Every other always_on test runs a frozen clock (rate 0); these run one at 60x.""" + +from __future__ import annotations + +import asyncio +import time + +import pytest + +from tst.unit.task_step.always_on_harness import Rig + +pytestmark = pytest.mark.usefixtures("local_stores") + +RATE = 60.0 # a real second is a virtual minute + + +def _estimate(rig, n): + rig.alert("Live Oak: estimate attached, $2,400", channel="Gmail · Live Oak Arbor") + + +@pytest.mark.asyncio +async def test_a_slow_contact_reply_between_turns_doesn_t_move_the_clock(monkeypatch, tmp_path): + rig = Rig(monkeypatch, tmp_path, rate=RATE) + await rig.arm() + liveoak = rig.contact("liveoak", seconds=1.5, reply=_estimate, timeout_seconds=10) # 90 virtual s at 60x + rig.triggers({"id": "estimate", "when": {"type": "time", "at": "2026-11-09T15:00:00Z"}, + "actions": [{"type": "nl", "executor": "liveoak", "instruction": "send Maya your estimate"}]}, + executors={"liveoak": liveoak}) + ctx = await rig.run(rig.step(settle_timeout_seconds=0.2)) + + [_, alerted] = rig.events(ctx, "wake") + assert "[Mon Nov 9, 3:00 PM UTC · Gmail · Live Oak Arbor]" in alerted["parts_text"] + assert alerted["virtual_time"].startswith("2026-11-09T15:00:00") + assert rig.record(ctx)["settings"]["clock"] == {"envs": [{"env_id": "world", "rate": RATE, + "frozen_between_turns": True}]} + + +@pytest.mark.asyncio +async def test_the_clock_runs_at_the_task_s_rate_during_a_turn_and_the_rate_is_back_at_the_end(monkeypatch, tmp_path): + rates: list[float] = [] + moved: list[float] = [] + + def agent(n, texts, rig): + rates.append(rig.gw._clock.rate) + before = rig.gw._clock.elapsed() + time.sleep(0.5) # the agent at work: half a real second, half a virtual minute + moved.append(rig.gw._clock.elapsed() - before) + rig.message("found two houses") if n == 1 else rig.quiet() + return "" + + class SlowReader: + @staticmethod + def setup(version, person): + return {"system_prompt": "you are Linda", "output_format": None} + + @staticmethod + async def answer(version, person, history, read, ask): + rates.append(rig.gw._clock.rate) + await asyncio.sleep(0.5) # she takes her time reading + return {"outcome": "reply", "message": "the blue one"} + + rig = Rig(monkeypatch, tmp_path, rate=RATE, agent=agent, plugin=SlowReader) + await rig.arm() + ctx = await rig.run(rig.step()) + + assert rates == [RATE, 0.0, RATE] # turn, read, turn + assert all(25 <= m <= 40 for m in moved) + [read] = rig.events(ctx, "user_sim_read") + assert read["deliver_at"] == read["virtual_time"] # nothing moved while she read + assert rig.gw._clock.rate == RATE From b2b8727eed51cdab6eab1742c96391e96dea18bc Mon Sep 17 00:00:00 2001 From: Sarah Chitty Date: Mon, 9 Nov 2026 03:34:01 -0800 Subject: [PATCH 35/53] always_on: every part of a wake is hard-capped, so budget plus one turn of the loop always fits the hub's 8 h The per-wake reserve wasn't a real bound: one settle could run several times the contact limit (queued fires, nl actions in a row, executors whose own timeout is above contact_timeout_seconds), a settle "kick" on an event due that never goes had no limit at all, and retries, trajectory fetches and the step's own gateway calls weren't counted. Now each part has a hard cap: - a settle never waits past settle_cap_seconds (the contact limit and grace, then settle_timeout_seconds), whatever is at work; an event due that an advance doesn't set going gets settle_timeout_seconds like any other work and is recorded (due_not_going); - an agent turn is cut off at timeout_seconds + TURN_GRACE_SECONDS (60 s) whatever send_and_wait's retries do (agent_turn_timeout), and each trajectory fetch at TRAJECTORY_FETCH_SECONDS (120 s); - a read's answer is cut off at read_cap_seconds (READ_ASKS = 2 user sim turns, each with its grace and trajectory fetch): an ask past it is refused and an answer past it is a user sim error; - the step's own gateway and store calls get GATEWAY_ALLOWANCE_SECONDS (300 s) per wake and per read. wake_reserve_seconds adds these up for the longest turn of the loop and for the closing (a jump's settle, a wake, a read), and the loop cuts off any turn of its own that runs past it, ending the run on a step error, so it is a real bound. The default reserve goes from 3060 s to 4380 s, so the default budget goes from 6.98 h to 6.62 h (at the default 600 s turn limit). Co-Authored-By: Claude Opus 5.5 --- .../task_step/task_steps/always_on.py | 242 ++++++++++++------ tst/unit/task_step/test_always_on_budget.py | 129 +++++++++- 2 files changed, 283 insertions(+), 88 deletions(-) diff --git a/src/agent_env/task_step/task_steps/always_on.py b/src/agent_env/task_step/task_steps/always_on.py index 510ec4932..3d83843e8 100644 --- a/src/agent_env/task_step/task_steps/always_on.py +++ b/src/agent_env/task_step/task_steps/always_on.py @@ -126,6 +126,14 @@ CONTACT_TIMEOUT_SECONDS = 300.0 # Past a contact's time limit, how long the settle wait still gives its call to come back. CONTACT_GRACE_SECONDS = 30.0 +# The hard caps that bound one turn of the loop (wake_reserve_seconds adds them up): past an agent's or user sim's turn +# limit, how long send_and_wait's own retries and polls still get before the turn is cut off; how long a trajectory +# fetch may take; how many user sim turns one read may take; and what the step's own gateway and store calls (takes, +# changes, the chat, clock reads) are allowed per wake and per read, beyond which the loop's turn is cut off. +TURN_GRACE_SECONDS = 60.0 +TRAJECTORY_FETCH_SECONDS = 120.0 +READ_ASKS = 2 +GATEWAY_ALLOWANCE_SECONDS = 300.0 # The final check-in's line when the task sets no `final`: the step's, not the user's. FINAL_DEFAULT_TEXT = "Time's up on this job. Send {first} a short wrap-up: what's done, and anything still open." NOTHING_CAN_WAKE = "nothing_can_wake_the_agent" # until finish() tells finished_nothing_pending from stuck @@ -217,13 +225,26 @@ def __init__(self, id: str, version: Optional[int], *, user_sim: str, end_time: "model", "model_params", "system_prompt", "max_turns", "output_format", "max_thinking_tokens", "effort", "harness", "agentenv_tools", "poll_interval_seconds", "context_id") - def wake_reserve_seconds(self) -> float: - """The longest one wake can take, as the settings bound it: the agent's turn (and its nudge turn, when nudges - are on), two settle waits (after the turn, and after a clock jump: a contact's time limit and grace, then - settle_timeout_seconds) and a read (two of the user sim's turns).""" + def settle_cap_seconds(self) -> float: + """The longest one settle waits, whatever is still at work (queued fires, nl actions in a row, a contact whose + own time limit is longer): a contact's time limit and grace, then settle_timeout_seconds.""" contact = self.contact_timeout_seconds or CONTACT_TIMEOUT_SECONDS - return (self.timeout_seconds * (2 if self.nudge else 1) - + 2 * (contact + CONTACT_GRACE_SECONDS + self.settle_timeout_seconds) + 2 * self.user_timeout_seconds) + return contact + CONTACT_GRACE_SECONDS + self.settle_timeout_seconds + + def read_cap_seconds(self) -> float: + """The longest one read's answer may take: READ_ASKS user sim turns, each with its grace and trajectory + fetch; an ask past it is refused, and an answer past it ends the read as a user sim error.""" + return READ_ASKS * (self.user_timeout_seconds + TURN_GRACE_SECONDS + TRAJECTORY_FETCH_SECONDS) + + def wake_reserve_seconds(self) -> float: + """The longest one turn of the loop, or the run's closing, can take, each part hard-capped: a clock jump's + settle (settle_cap_seconds), then a wake (the agent's turn and, with nudges on, its nudge turn, each at most + timeout_seconds + TURN_GRACE_SECONDS plus a trajectory fetch; one settle; GATEWAY_ALLOWANCE_SECONDS of the + step's own calls), then a read (read_cap_seconds and GATEWAY_ALLOWANCE_SECONDS). The loop cuts off a turn of + its own that runs past this (a step error), so it is a real bound.""" + turn = self.timeout_seconds + TURN_GRACE_SECONDS + TRAJECTORY_FETCH_SECONDS + wake = turn * (2 if self.nudge else 1) + self.settle_cap_seconds() + GATEWAY_ALLOWANCE_SECONDS + return self.settle_cap_seconds() + wake + self.read_cap_seconds() + GATEWAY_ALLOWANCE_SECONDS def effective_time_budget(self) -> float: """The real time after which the run closes: ``time_budget_seconds``, else 7 h, lowered so the closing still @@ -240,8 +261,8 @@ def effective_time_budget(self) -> float: reserve = self.wake_reserve_seconds() room = JOB_LIMIT_SECONDS - JOB_HEADROOM_SECONDS - reserve if room <= reserve: - raise ValueError(f"always_on: one wake can take up to {reserve:.0f}s (wake_reserve_seconds: the agent's turn " - f"and nudge, the settle waits and a read), so no run fits the hub's " + raise ValueError(f"always_on: one wake can take up to {reserve:.0f}s (wake_reserve_seconds: a settle, the " + f"agent's turn and nudge, a settle and a read), so no run fits the hub's " f"{JOB_LIMIT_SECONDS // 3600} h job; lower timeout_seconds or user_timeout_seconds") if self.time_budget_seconds is None: return min(DEFAULT_TIME_BUDGET_SECONDS, room) @@ -547,9 +568,12 @@ async def _agent_turn(self, parts: list[dict], msg_id: str, record) -> tuple[str step = self.step await self._clock_runs(True) try: - return await send_and_wait(self.target_url, parts, agent=self.agent, message_id=msg_id, - context_id=self.solver_context_id, timeout_seconds=step.timeout_seconds, - poll_interval_seconds=step.poll_interval_seconds, before_send=record) + # Cut off past its limit and grace, whatever send_and_wait's own retries do: a TimeoutError, as its own. + return await asyncio.wait_for( + send_and_wait(self.target_url, parts, agent=self.agent, message_id=msg_id, + context_id=self.solver_context_id, timeout_seconds=step.timeout_seconds, + poll_interval_seconds=step.poll_interval_seconds, before_send=record), + step.timeout_seconds + TURN_GRACE_SECONDS) finally: await self._clock_runs(False) @@ -689,11 +713,14 @@ async def advance(self, target: datetime, reason: str) -> None: async def settle(self) -> list[dict]: """Wait until the world has settled: no trigger still at work, bar a fire waiting for a later time. A contact's reply (a call to a trigger's executor) counts as in progress until it ends or its time limit passes (plus - ``CONTACT_GRACE_SECONDS``); any other work gets ``settle_timeout_seconds``, from the start or from the last - contact call seen. A gateway that reports no executor calls gets ``settle_timeout_seconds`` in all. Returns, by - env, the triggers still at work when it gave up (empty when the world settled); ``last_settle`` says how long - it waited and how many contact calls it saw.""" + ``CONTACT_GRACE_SECONDS``); any other work, an event due that an advance to now doesn't set going among it, + gets ``settle_timeout_seconds``, from the start or from the last contact call seen. A gateway that reports no + executor calls gets ``settle_timeout_seconds`` in all. However much is still at work (queued fires, nl + actions in a row, a contact whose own limit is longer), the wait never runs past ``settle_cap_seconds``. + Returns, by env, the triggers still at work when it gave up (empty when the world settled); ``last_settle`` + says how long it waited and how many contact calls it saw.""" started = time.monotonic() + cap = started + self.step.settle_cap_seconds() # however much is still at work, never past this envs = [e for e in await self._armed() if e.supports(EXT_TRIGGERS_URI, "state")] waiting = {i: {"live_call_seen": started, "contact_until": started} for i in range(len(envs))} stuck: list[dict] = [] @@ -712,16 +739,20 @@ async def settle(self) -> list[dict]: track["contact_until"] = max(track["contact_until"], mono + CONTACT_GRACE_SECONDS + max(c["seconds_left"] for c in live)) busy = [row for row in rows if _busy(row, now)] - if state.get("next_event_at") is not None and _utc(state["next_event_at"]) <= now: + due = state.get("next_event_at") is not None and _utc(state["next_event_at"]) <= now + limit = min(cap, max(track["contact_until"], + track["live_call_seen"] + self.step.settle_timeout_seconds)) + if not due and not busy: + waiting.pop(i) + elif mono >= limit: # a due event that won't go, like other work, gets settle_timeout_seconds + stuck.append({"env_id": env.env_id, "triggers": [_at_work(row) for row in busy], + **({"due_not_going": state["next_event_at"]} if due else {})}) + waiting.pop(i) + elif due: # an event already due: an advance to now sets it going try: await _call(env, EXT_CLOCK_URI, "advance", {"virtual_time": _iso(now)}, attempts=1) except Exception: pass - elif not busy: - waiting.pop(i) - elif mono >= max(track["contact_until"], track["live_call_seen"] + self.step.settle_timeout_seconds): - stuck.append({"env_id": env.env_id, "triggers": [_at_work(row) for row in busy]}) - waiting.pop(i) if waiting: self.ctx.heartbeat() await asyncio.sleep(POLL_SECONDS) @@ -854,6 +885,32 @@ async def loop(self, now: datetime) -> None: end one that waits on a known event; with an end_time the run goes on to it, its jumps folded in the timeline past ``JUMPS_KEPT_PER_GAP`` per gap.""" step = self.step + await self._bounded(self._open(), "the opening wake") + while self.end_reason is None: + if self.wakes >= step.max_wakes: + self.end_reason = "max_wakes" + break + if not await self._bounded(self._next(), "a turn of the loop"): + break + await self._bounded(self._last_read(), "the last read", step.read_cap_seconds() + GATEWAY_ALLOWANCE_SECONDS) + + async def _bounded(self, work, what: str, seconds: Optional[float] = None): + """``work``, cut off past ``seconds`` (one turn of the loop's worst case, ``wake_reserve_seconds``): every part + is capped on its own, so only calls hanging past their allowance get here, and the run ends on a step error + rather than run past the hub's job.""" + seconds = self.step.wake_reserve_seconds() if seconds is None else seconds + timer = asyncio.timeout(seconds) + try: + async with timer: + return await work + except TimeoutError: + if not timer.expired(): + raise + self.end_reason = "step_error" + raise RuntimeError(f"always_on: {what} ran past its worst case ({seconds:.0f}s, wake_reserve_seconds)") + + async def _open(self) -> None: + """The opening wake: the user's request, with what came due for the agent.""" self.you_sent.append(dict(self.opening)) opened_at = _utc(self.opening["at"]) await self._chat_user(self._opening_parts(lambda text: self._chat_part(opened_at, text, kind="opening"))) @@ -863,58 +920,58 @@ async def loop(self, now: datetime) -> None: woke_by=[f"notification:{d['channel']}" for d in due] + ["user_request"], notifications=[{k: d.get(k) for k in ("at", "channel", "source", "source_id", "to")} for d in due]) - while self.end_reason is None: - if self.wakes >= step.max_wakes: - self.end_reason = "max_wakes" - break - now = await self.now() - spent = self.spent() - if spent + step.wake_reserve_seconds() >= self.time_budget: - self.log("time_budget_reached", virtual_time=_iso(now), real_seconds=round(spent, 1), - time_budget_seconds=self.time_budget, wake_reserve_seconds=step.wake_reserve_seconds()) - await self._final(now, reason="time_budget") - break - # 1. a read that is due: the user sim reads what the agent sent - if self.read_at is not None and self.read_at <= now and self.unread: - await self.read(now) - continue - # 2. anything due for the agent: notifications, scripted user lines, held replies; and what else comes - # due in the same minute, so it all arrives in one wake - if any(h["deliver_at"] <= now for h in self.held) or await self._due_count(self.recipient, now) \ - or await self._due_count(USER_SAYS, now): - later = await self._later(now) - same_minute = [t for t in later.values() if t < now.replace(second=0, microsecond=0) - + timedelta(minutes=1) and (self.end is None or t < self.end)] - if same_minute and await self._armed(): - await self.advance(min(same_minute), reason="same_minute") - continue - lines = await self._take_user_says() - replies = [h for h in self.held if h["deliver_at"] <= now] - self.held = [h for h in self.held if h["deliver_at"] > now] - await self._wake_with(now, lines, replies) - continue - # 3. nothing due: jump to the next thing + + async def _next(self) -> bool: + """One turn of the loop: the closing, a read, a wake or a clock jump; False once the run should end.""" + step = self.step + now = await self.now() + spent = self.spent() + if spent + step.wake_reserve_seconds() >= self.time_budget: + self.log("time_budget_reached", virtual_time=_iso(now), real_seconds=round(spent, 1), + time_budget_seconds=self.time_budget, wake_reserve_seconds=step.wake_reserve_seconds()) + await self._final(now, reason="time_budget") + return False + # 1. a read that is due: the user sim reads what the agent sent + if self.read_at is not None and self.read_at <= now and self.unread: + await self.read(now) + return True + # 2. anything due for the agent: notifications, scripted user lines, held replies; and what else comes + # due in the same minute, so it all arrives in one wake + if any(h["deliver_at"] <= now for h in self.held) or await self._due_count(self.recipient, now) \ + or await self._due_count(USER_SAYS, now): later = await self._later(now) - soonest = min(later.values()) if later else None - if self.end is not None and (soonest is None or soonest >= self.end): - await self._final(now) - break - if soonest is None: - self.end_reason = NOTHING_CAN_WAKE # finished or stuck: finish() decides from what is pending - break - if self.end is None and set(later) == {"world_event"} and await self._only_repeating(): - self.idle_jumps += 1 - self.idle_since = self.idle_since or _iso(now) - if self.idle_jumps > IDLE_JUMPS_MAX: - self.log("world_only_repeats", virtual_time=_iso(now), jumps=self.idle_jumps - 1, - since=self.idle_since, triggers=await self._repeating_ids()) - self.end_reason = NOTHING_CAN_WAKE - break - else: - self.idle_jumps, self.idle_since = 0, None - why = sorted(k for k, v in later.items() if v == soonest) - await self.advance(soonest, reason="+".join(why)) - await self._last_read() + same_minute = [t for t in later.values() if t < now.replace(second=0, microsecond=0) + + timedelta(minutes=1) and (self.end is None or t < self.end)] + if same_minute and await self._armed(): + await self.advance(min(same_minute), reason="same_minute") + return True + lines = await self._take_user_says() + replies = [h for h in self.held if h["deliver_at"] <= now] + self.held = [h for h in self.held if h["deliver_at"] > now] + await self._wake_with(now, lines, replies) + return True + # 3. nothing due: jump to the next thing + later = await self._later(now) + soonest = min(later.values()) if later else None + if self.end is not None and (soonest is None or soonest >= self.end): + await self._final(now) + return False + if soonest is None: + self.end_reason = NOTHING_CAN_WAKE # finished or stuck: finish() decides from what is pending + return False + if self.end is None and set(later) == {"world_event"} and await self._only_repeating(): + self.idle_jumps += 1 + self.idle_since = self.idle_since or _iso(now) + if self.idle_jumps > IDLE_JUMPS_MAX: + self.log("world_only_repeats", virtual_time=_iso(now), jumps=self.idle_jumps - 1, + since=self.idle_since, triggers=await self._repeating_ids()) + self.end_reason = NOTHING_CAN_WAKE + return False + else: + self.idle_jumps, self.idle_since = 0, None + why = sorted(k for k, v in later.items() if v == soonest) + await self.advance(soonest, reason="+".join(why)) + return True async def _later(self, now: datetime) -> dict[str, datetime]: """What could wake the agent after ``now``, by kind, each at its earliest time.""" @@ -1112,8 +1169,9 @@ def record() -> None: traj_ext = A2AAgent.find_extension(card, A2AAgent.EXT_TRAJECTORY) if traj_ext: try: - traj = await step._fetch_trajectory(self.target_url, traj_ext, task_id, step._trajectory_prefix(), - msg_id, card=card, sandbox_type=self.agent.sandbox_type) + traj = await asyncio.wait_for( + step._fetch_trajectory(self.target_url, traj_ext, task_id, step._trajectory_prefix(), msg_id, + card=card, sandbox_type=self.agent.sandbox_type), TRAJECTORY_FETCH_SECONDS) except Exception as e: logger.warning(f"wake {self.wakes}: trajectory fetch failed (continuing): {e}") self.per_turn_trajectory_urls.append(traj) @@ -1170,8 +1228,9 @@ def record() -> None: traj_ext = A2AAgent.find_extension(card, A2AAgent.EXT_TRAJECTORY) if traj_ext: try: - traj = await step._fetch_trajectory(self.target_url, traj_ext, task_id, step._trajectory_prefix(), - msg_id, card=card, sandbox_type=self.agent.sandbox_type) + traj = await asyncio.wait_for( + step._fetch_trajectory(self.target_url, traj_ext, task_id, step._trajectory_prefix(), msg_id, + card=card, sandbox_type=self.agent.sandbox_type), TRAJECTORY_FETCH_SECONDS) except Exception as e: logger.warning(f"nudge after wake {self.wakes}: trajectory fetch failed (continuing): {e}") self.per_turn_trajectory_urls.append(traj) @@ -1286,14 +1345,20 @@ async def read(self, now: datetime, last: bool = False) -> None: "you_sent": list(self.you_sent), "done_ends_run": not self.multi_day} self.unread, self.you_sent, self.read_at = [], [], None calls = [] + cap = self.step.read_cap_seconds() + deadline = time.monotonic() + cap async def ask(text: str) -> dict: - got = await self._ask_user_sim(text) + got = await self._ask_user_sim(text, deadline) calls.append(got) return got t = time.time() - ans = await self.plugin.answer(self.version, self.person, self.history, read, ask) + try: + ans = await asyncio.wait_for(self.plugin.answer(self.version, self.person, self.history, read, ask), cap) + except TimeoutError: # the plug-in ran past the read's cap: a user sim error, as a failed ask is + ans = {"outcome": "no_reply", "message": ""} + calls.append({"text": None, "structured": None, "error": f"the read ran past its cap ({cap:.0f}s)"}) self.history.append({"read": read, "answer": ans}) mode = self.read_mode latency = self._latency(mode) @@ -1340,14 +1405,19 @@ async def ask(text: str) -> dict: if calls and calls[-1].get("error") and not last: self.end_reason = self.end_reason or "user_sim_error" - async def _ask_user_sim(self, text: str) -> dict: + async def _ask_user_sim(self, text: str, deadline: float) -> dict: + """One user sim turn on ``text``, cut off at its time limit and grace or at the read's ``deadline`` + (time.monotonic()), whichever comes first.""" msg_id = uuid.uuid4().hex + limit = min(self.step.user_timeout_seconds + TURN_GRACE_SECONDS, deadline - time.monotonic()) + if limit <= 0: + return {"text": None, "structured": None, "error": "the read's time limit passed"} try: - user_task_id, result = await send_and_wait( + user_task_id, result = await asyncio.wait_for(send_and_wait( self.user_url, [{"kind": "text", "text": text}], agent=self.user_agent if self.user_agent.sandbox_id else None, message_id=msg_id, - context_id=self.user_context_id, timeout_seconds=self.step.user_timeout_seconds, - poll_interval_seconds=self.step.poll_interval_seconds) + context_id=self.user_context_id, timeout_seconds=min(self.step.user_timeout_seconds, limit), + poll_interval_seconds=self.step.poll_interval_seconds), limit) except TimeoutError as e: return {"text": None, "structured": None, "error": f"timeout: {e}"} status = (result.get("status") or {}) @@ -1367,9 +1437,11 @@ async def _ask_user_sim(self, text: str) -> dict: traj_ext = A2AAgent.find_extension(card, A2AAgent.EXT_TRAJECTORY) if traj_ext: try: - traj = await self.step._fetch_trajectory(self.user_url, traj_ext, user_task_id, - self.step._trajectory_prefix() + "user_sim/", msg_id, - card=card, sandbox_type=self.user_agent.sandbox_type) + traj = await asyncio.wait_for( + self.step._fetch_trajectory(self.user_url, traj_ext, user_task_id, + self.step._trajectory_prefix() + "user_sim/", msg_id, + card=card, sandbox_type=self.user_agent.sandbox_type), + max(0.0, min(TRAJECTORY_FETCH_SECONDS, deadline - time.monotonic()))) except Exception as e: logger.warning(f"user sim trajectory fetch failed (continuing): {e}") return {"text": raw, "structured": structured if isinstance(structured, dict) else None, diff --git a/tst/unit/task_step/test_always_on_budget.py b/tst/unit/task_step/test_always_on_budget.py index b1a00e58e..5d2beda43 100644 --- a/tst/unit/task_step/test_always_on_budget.py +++ b/tst/unit/task_step/test_always_on_budget.py @@ -4,6 +4,8 @@ from __future__ import annotations +import asyncio + import pytest from agent_env.task_step.task_steps import always_on as ao @@ -32,11 +34,11 @@ def agent(n, texts, rig): await rig.arm() for day in range(10, 20): rig.triggers(rig.notify_at(f"alert-{day}", f"2026-11-{day}T14:00:00Z", "Sam: news?", channel="noor 30")) - step = rig.step(time_budget_seconds=3 * 3600) # reserve 3060 s: closes once 7740 s are spent + step = rig.step(time_budget_seconds=12120) # reserve 4380 s: closes once 7740 s are spent ctx = await rig.run(step) record = rig.record(ctx) - assert step.wake_reserve_seconds() == 3060 and record["settings"]["time_budget_seconds"] == 10800 + assert step.wake_reserve_seconds() == 4380 and record["settings"]["time_budget_seconds"] == 12120 wakes = rig.events(ctx, "wake") assert len(wakes) == 4 and wakes[-1]["woke_by"] == ["step:final"] and wakes[-1]["last"] is True assert wakes[-1]["virtual_time"] == "2026-11-11T14:00:00Z" # where the run was; the clock doesn't jump @@ -49,7 +51,7 @@ def agent(n, texts, rig): "its real-time budget.") -@pytest.mark.parametrize("timeout,hours", [(600, 6.98), (1800, 6.32), (3600, 5.32)]) +@pytest.mark.parametrize("timeout,hours", [(600, 6.62), (1800, 5.95), (3600, 4.95)]) def test_the_default_budget_is_7_h_lowered_so_the_closing_wake_fits_the_8_h_job(timeout, hours): step = _step(timeout_seconds=timeout) budget = step.effective_time_budget() @@ -74,3 +76,124 @@ def test_the_budget_survives_a_round_trip(): again = ao.AlwaysOnTaskStep.from_dict(_step(time_budget_seconds=5 * 3600).to_dict()) assert again.time_budget_seconds == 5 * 3600 assert ao.AlwaysOnTaskStep.from_dict(_step().to_dict()).time_budget_seconds is None + + +# ---- the reserve is a real bound ------------------------------------------------------------------------------------- + +@pytest.mark.parametrize("settings", [{}, {"nudge": False}, {"timeout_seconds": 1800, "user_timeout_seconds": 900}, + {"contact_timeout_seconds": 900, "settle_timeout_seconds": 600}, + {"time_budget_seconds": 5 * 3600}]) +def test_the_reserve_adds_up_every_capped_part_and_the_budget_plus_one_turn_fits_the_job(settings): + step = _step(**settings) + settle = (step.contact_timeout_seconds or ao.CONTACT_TIMEOUT_SECONDS) + ao.CONTACT_GRACE_SECONDS \ + + step.settle_timeout_seconds + turn = step.timeout_seconds + ao.TURN_GRACE_SECONDS + ao.TRAJECTORY_FETCH_SECONDS + read = ao.READ_ASKS * (step.user_timeout_seconds + ao.TURN_GRACE_SECONDS + ao.TRAJECTORY_FETCH_SECONDS) + wake = turn * (2 if step.nudge else 1) + settle + ao.GATEWAY_ALLOWANCE_SECONDS + assert step.settle_cap_seconds() == settle and step.read_cap_seconds() == read + # a jump's settle, then a wake (its turns, its settle, the step's calls), then a read and its calls + assert step.wake_reserve_seconds() == settle + wake + read + ao.GATEWAY_ALLOWANCE_SECONDS + assert step.effective_time_budget() + step.wake_reserve_seconds() + ao.JOB_HEADROOM_SECONDS <= ao.JOB_LIMIT_SECONDS + + +def _estimate(rig, n): + rig.alert(f"Live Oak: estimate {n}", channel="Gmail · Live Oak Arbor") + + +@pytest.mark.asyncio +async def test_a_settle_stops_at_its_cap_though_a_contact_s_own_limit_is_longer(monkeypatch, tmp_path): + """The contact's own limit is 5 s, past contact_timeout_seconds (0.3 s): the settle used to wait for it.""" + monkeypatch.setattr(ao, "CONTACT_GRACE_SECONDS", 0.1) + rig = Rig(monkeypatch, tmp_path) + await rig.arm() + slow = rig.contact("liveoak", seconds=5, reply=_estimate, timeout_seconds=5, keeps_to_its_limit=False) + rig.triggers({"id": "estimate", "when": {"type": "time", "at": "2026-11-09T15:00:00Z"}, + "actions": [{"type": "nl", "executor": "liveoak", "instruction": "send your estimate"}]}, + executors={"liveoak": slow}) + step = rig.step(contact_timeout_seconds=0.3, settle_timeout_seconds=0.1) + try: + ctx = await rig.run(step) + finally: + await rig.close() + + assert step.settle_cap_seconds() == pytest.approx(0.5) + [jump] = [j for j in rig.events(ctx, "clock_jump") if j["to"] == "2026-11-09T15:00:00Z"] + assert not jump["settled"] and jump["settle"]["seconds"] < 1.5 + + +@pytest.mark.asyncio +async def test_an_event_due_that_won_t_go_gets_the_settle_timeout_not_forever(monkeypatch, tmp_path): + rig = Rig(monkeypatch, tmp_path) + await rig.arm() + engine = rig.gw._trigger_engine + real_state = engine.state + monkeypatch.setattr(engine, "state", lambda: {**real_state(), "next_event_at": rig.now()}) # due, never going + ctx = await asyncio.wait_for(rig.run(rig.step(settle_timeout_seconds=0.2)), 10) + + [timeout] = rig.events(ctx, "settle_timeout") + [env] = timeout["still_at_work"] + assert env["due_not_going"] == "2026-11-09T14:00:00Z" and timeout["settle"]["seconds"] < 1.5 + + +@pytest.mark.asyncio +async def test_an_agent_turn_is_cut_off_at_its_limit_and_grace_whatever_send_and_wait_does(monkeypatch, tmp_path): + monkeypatch.setattr(ao, "TURN_GRACE_SECONDS", 0.2) + rig = Rig(monkeypatch, tmp_path) + + async def retries_forever(url, parts, **kw): + await asyncio.sleep(60) + + monkeypatch.setattr(ao, "send_and_wait", retries_forever) + await rig.arm() + ctx = await asyncio.wait_for(rig.run(rig.step(timeout_seconds=1)), 10) + + record = rig.record(ctx) + assert (record["end_reason"], record["error_type"]) == ("agent_turn_timeout", "agent_turn_timeout") + assert rig.events(ctx, "agent_turn_end")[0]["real_seconds"] < 2 + + +@pytest.mark.asyncio +async def test_a_read_is_cut_off_at_its_cap_as_a_user_sim_error(monkeypatch, tmp_path): + monkeypatch.setattr(ao, "TURN_GRACE_SECONDS", 0.1) + monkeypatch.setattr(ao, "TRAJECTORY_FETCH_SECONDS", 0.1) + monkeypatch.setattr(ao, "READ_ASKS", 1) + + class Stalls: + @staticmethod + def setup(version, person): + return {"system_prompt": "you are Linda", "output_format": None} + + @staticmethod + async def answer(version, person, history, read, ask): + await asyncio.sleep(60) + + def agent(n, texts, rig): + rig.message("blue or green?") + return "" + + rig = Rig(monkeypatch, tmp_path, agent=agent, plugin=Stalls) + await rig.arm() + step = rig.step(user_timeout_seconds=1) + ctx = await asyncio.wait_for(rig.run(step), 10) + + assert step.read_cap_seconds() == pytest.approx(1.2) + [read] = rig.events(ctx, "user_sim_read") + assert read["user_sim_error"] == "the read ran past its cap (1s)" and read["real_seconds"] < 2 + assert rig.record(ctx)["end_reason"] == "user_sim_error" + + +@pytest.mark.asyncio +async def test_a_turn_of_the_loop_past_the_reserve_ends_the_run_on_a_step_error(monkeypatch, tmp_path): + """Every part is capped on its own; only calls hanging past their allowance reach this last cut-off.""" + monkeypatch.setattr(ao.AlwaysOnTaskStep, "wake_reserve_seconds", lambda step: 0.5) + rig = Rig(monkeypatch, tmp_path) + + async def hangs(run): + await asyncio.sleep(60) + + monkeypatch.setattr(ao._Run, "_take_messages", hangs) + await rig.arm() + ctx = rig.context() + with pytest.raises(RuntimeError, match="the opening wake ran past its worst case"): + await asyncio.wait_for(rig.run(rig.step(), ctx), 10) + assert ctx.metadata[ao.ALWAYS_ON_KEY]["week"]["end_reason"] == "step_error" From 58acb54cd0e986a480d9930a55b4a52539612e35 Mon Sep 17 00:00:00 2001 From: Sarah Chitty Date: Mon, 9 Nov 2026 03:42:43 -0800 Subject: [PATCH 36/53] always_on: put back the gateway settings it changed; the rules reach an agent that won't take a system prompt; honour context_id - When the step ends (normally or not) it puts back what it changed on the gateways, latest first: the user chat's roles, user name and stay_quiet; the triggers' concurrent_fires and executor time limit; the roles it gave scheduled checks (with their limits); the clock's rate. Later steps of the task used to see message_user, stay_quiet, side-by-side fires and the 300 s executor default still on. - If the step sets a system prompt and the agent's card doesn't accept system_prompt, the message_user rules never reached the agent. They now go on its first wake, with a warning; settings.rules_via in the record says which way they went. - context_id was in _PROMPT_FIELDS but ignored. It is now the agent's session (A2A context), as prompt_agent uses it, so a run can go on with the session an earlier step left; the graded conversation stays the run's own. The test harness records the gateway's settings and the agent's tools at each agent turn (Rig.during), since they are now put back after the run, and can give the agent a card that takes its config (takes_config). Co-Authored-By: Claude Opus 5.5 --- .../task_step/task_steps/always_on.py | 51 ++++++++-- tst/unit/task_step/always_on_harness.py | 29 +++++- .../test_always_on_agent_and_gateway.py | 99 +++++++++++++++++++ tst/unit/task_step/test_always_on_budget.py | 12 ++- tst/unit/task_step/test_always_on_checks.py | 32 ++++-- tst/unit/task_step/test_always_on_messages.py | 2 +- tst/unit/task_step/test_always_on_time.py | 7 +- 7 files changed, 203 insertions(+), 29 deletions(-) create mode 100644 tst/unit/task_step/test_always_on_agent_and_gateway.py diff --git a/src/agent_env/task_step/task_steps/always_on.py b/src/agent_env/task_step/task_steps/always_on.py index 3d83843e8..83a2a1f70 100644 --- a/src/agent_env/task_step/task_steps/always_on.py +++ b/src/agent_env/task_step/task_steps/always_on.py @@ -54,6 +54,11 @@ being when the reply reached the agent, so checks built on those records grade the run instead of skipping it. - The run's metadata keeps a summary in ``metadata["always_on"][step_id]`` (small enough for a running job to report back); the full timeline goes to the object store, at the summary's ``object_url``. +- The agent: the message_user rules go in its system prompt when the step sets one and the agent's card takes it, + else on its first wake; its session is the step's ``context_id`` when set, as prompt_agent's is, else the run's own. +- When the step ends it puts back the gateway settings it changed for the run (the user chat's roles and stay_quiet, + the triggers' concurrent_fires and executor time limit, the roles with scheduled checks, the clock's rate), so the + task's later steps see them as they were. """ from __future__ import annotations @@ -352,10 +357,13 @@ def __init__(self, step: AlwaysOnTaskStep, context: TaskStepContext): self.clock: dict = {} # the task's clock rate, and whether the step freezes it between turns self.clock_rates: list[tuple[Any, float]] = [] # (env, the task's rate) for each clock the step freezes self.undo: list[tuple[str, Any]] = [] # (what, coroutine function): the gateway settings to put back at the end + self.rules_in_system_prompt = False # whether the agent took the rules in its system prompt async def restore(self) -> None: """Put back what the step changed on the gateways, latest first, so later steps of the task see the settings - they had before it: the clock's rate, ... Each is tried on its own; one that fails is logged.""" + they had before it: the user chat's roles, user name and stay_quiet; the triggers' concurrent_fires and + executor time limit; the roles with scheduled checks; the clock's rate. Each is tried on its own; one that + fails is logged.""" while self.undo: what, put_back = self.undo.pop() try: @@ -363,6 +371,12 @@ async def restore(self) -> None: except Exception as e: logger.warning(f"always_on: could not put back {what} after the run: {type(e).__name__}: {e}") + def _undo_config(self, env, uri: str, what: str, body: dict) -> None: + """At the end, set ``env``'s ``uri`` config back to ``body``, what it was before the step changed it.""" + async def put_back() -> None: + await _call(env, uri, "config", body, attempts=3) + self.undo.append((f"{what} on {env.env_id}", put_back)) + def close(self) -> None: for p in self._plugin_paths: if os.path.exists(p): @@ -430,6 +444,10 @@ async def go(self) -> None: if step.stay_quiet: chat_config["stay_quiet"] = True for env in self.chat_envs: + before = await _call(env, EXT_USER_CHAT_URI, "state") if env.supports(EXT_USER_CHAT_URI, "state") else {} + self._undo_config(env, EXT_USER_CHAT_URI, "the user chat's config", + {"roles": before.get("roles") or [], "user_name": before.get("user_name") or "your user", + **({"stay_quiet": True} if before.get("stay_quiet") else {})}) await _call(env, EXT_USER_CHAT_URI, "config", chat_config) await self._configure_clock() await self._configure_checks() @@ -449,7 +467,9 @@ async def go(self) -> None: self.hook_records.append({"hook": HOOK_NAME, "turn": 0, "started_at": _iso(await self.now())}) ctx.metadata.setdefault(HOOK_RECORDS_KEY, {})[step.id] = self.hook_records self.user_context_id = uuid.uuid4().hex # one user sim session per run - self.solver_context_id = uuid.uuid4().hex + # The agent's session: the step's context_id, as prompt_agent uses it, so a run can go on with the session an + # earlier step left; else a new one. The graded conversation is the run's own either way. + self.solver_context_id = step.context_id or uuid.uuid4().hex now = await self.now() self.log("start", virtual_time=_iso(now), end_time=_iso(self.end) if self.end else None, @@ -514,6 +534,11 @@ async def _configure_agents(self) -> None: if negotiated: await protocol.post_agent_config(self.target_url + negotiated.endpoint, negotiated.fields) ctx.record_model("solver", step.id, negotiated.fields.get("model") if negotiated else None) + self.rules_in_system_prompt = bool(desired.get("system_prompt") and negotiated + and "system_prompt" in negotiated.fields) + if desired.get("system_prompt") and not self.rules_in_system_prompt: + logger.warning("always_on: the agent's card did not accept system_prompt, so the message_user rules go " + "on its first wake instead") setup = self.plugin.setup(self.version, self.person) self.user_model, self.user_model_from = self._user_model() @@ -591,6 +616,10 @@ async def _configure_contacts(self) -> None: if not env.supports(EXT_TRIGGERS_URI, "config"): applied.append({"env_id": env.env_id, "applied": False, "why": "the gateway has no triggers config"}) continue + before = (await _call(env, EXT_TRIGGERS_URI, "state")).get("settings") or {} + self._undo_config(env, EXT_TRIGGERS_URI, "the triggers' settings", + {k: v for k, v in before.items() + if k in ("concurrent_fires", "executor_timeout_seconds") and v is not None}) got = await _call(env, EXT_TRIGGERS_URI, "config", body) applied.append({"env_id": env.env_id, "applied": True, "concurrent_fires": got.get("concurrent_fires"), "executor_timeout_seconds": got.get("executor_timeout_seconds")}) @@ -632,14 +661,17 @@ async def _configure_checks(self) -> None: body = {"roles": [*roles, self.recipient], "max_active": state.get("max_active", DEFAULT_MAX_ACTIVE) if roles else DEFAULT_MAX_ACTIVE, "min_interval": state.get("min_interval") if roles else step.scheduled_check_min_interval} + self._undo_config(env, EXT_NOTIFICATIONS_URI, "the roles with scheduled checks", + {"roles": roles, **{k: state[k] for k in ("max_active", "min_interval") + if state.get(k) is not None}}) got = await _call(env, EXT_NOTIFICATIONS_URI, "config", body) applied.append({"env_id": env.env_id, "set_by": "step", "min_interval": got.get("min_interval"), "max_active": got.get("max_active")}) self.checks = {"on": bool(applied), "envs": applied} def _system_prompt(self) -> Optional[str]: - """The step's system prompt with the rules after it; without one, the agent keeps its own and the first wake - opens with the rules.""" + """The step's system prompt with the rules after it; without one, or when the agent's card doesn't accept it, + the agent keeps its own and the first wake opens with the rules.""" return f"{self.step.system_prompt.rstrip()}\n\n{self.rules()}" if self.step.system_prompt else None def _time_line(self, at: datetime) -> str: @@ -885,16 +917,16 @@ async def loop(self, now: datetime) -> None: end one that waits on a known event; with an end_time the run goes on to it, its jumps folded in the timeline past ``JUMPS_KEPT_PER_GAP`` per gap.""" step = self.step - await self._bounded(self._open(), "the opening wake") + await self._capped(self._open(), "the opening wake") while self.end_reason is None: if self.wakes >= step.max_wakes: self.end_reason = "max_wakes" break - if not await self._bounded(self._next(), "a turn of the loop"): + if not await self._capped(self._next(), "a turn of the loop"): break - await self._bounded(self._last_read(), "the last read", step.read_cap_seconds() + GATEWAY_ALLOWANCE_SECONDS) + await self._capped(self._last_read(), "the last read", step.read_cap_seconds() + GATEWAY_ALLOWANCE_SECONDS) - async def _bounded(self, work, what: str, seconds: Optional[float] = None): + async def _capped(self, work, what: str, seconds: Optional[float] = None): """``work``, cut off past ``seconds`` (one turn of the loop's worst case, ``wake_reserve_seconds``): every part is capped on its own, so only calls hanging past their allowance get here, and the run ends on a step error rather than run past the hub's job.""" @@ -1131,7 +1163,7 @@ async def wake(self, parts: list[dict], woke_by: list[str], notifications: Optio sent_at = await self.now() clock = await step._env_clock(self.clock_envs) lead = [_step_part(self._time_line(sent_at), "time")] if clock else [] - if self.wakes == 0 and not step.system_prompt: + if self.wakes == 0 and not self.rules_in_system_prompt: # no system prompt took them: the first wake does lead = [_step_part(self.rules(), "rules"), *lead] parts = [*lead, *parts] last = last or self.wakes + 1 >= step.max_wakes @@ -1546,6 +1578,7 @@ async def finish(self) -> None: "timeline_events": len(self.timeline), "pending_at_end": _bounded(self.pending)} record["settings"] = {"stay_quiet": step.stay_quiet, "nudge": step.nudge, "wake_reminder": step.wake_reminder, "clock": self.clock, + "rules_via": "system_prompt" if self.rules_in_system_prompt else "first_wake", "scheduled_checks": self.checks, "contacts": self.contacts, "time_budget_seconds": self.time_budget, "wake_reserve_seconds": step.wake_reserve_seconds()} diff --git a/tst/unit/task_step/always_on_harness.py b/tst/unit/task_step/always_on_harness.py index 4e20e0182..e32d49e7f 100644 --- a/tst/unit/task_step/always_on_harness.py +++ b/tst/unit/task_step/always_on_harness.py @@ -11,7 +11,7 @@ import httpx from mcp.types import CallToolResult, TextContent, Tool -from agent_env.a2a_agent import conversation_store +from agent_env.a2a_agent import A2AAgent, conversation_store from agent_env.config import get_config from agent_env.env.env import DeployedGatewayEnv from agent_env.env.gateway import gateway as gw_mod @@ -29,6 +29,10 @@ ROLE = "default" Turn = Callable[[int, list[str], "Rig"], Optional[str]] +# An agent card whose agent-config extension takes the settings the step sends. +CONFIG_CARD = {"capabilities": {"extensions": [{"uri": A2AAgent.EXT_AGENT_CONFIG, "params": { + "endpoint": "/ext/agent-config", "methods": {"set": {"request": {"supported": [ + "model", "system_prompt", "effort", "output_format", "timeout_seconds", "max_turns"]}}}}}]}} class Fail: @@ -46,10 +50,12 @@ class Rig: user sim's answers to its reads, in order; once they run out it answers no_reply. Every message the agent got is in ``sent``, every read the user sim got in ``reads``. A ``plugin`` replaces the scripted one; what it sends the user sim agent with ``ask`` is - in ``user_calls``, answered in turn by ``user_texts`` (then no_reply).""" + in ``user_calls``, answered in turn by ``user_texts`` (then no_reply). ``during`` has, for each of the agent's + turns, the gateway's settings and the agent's tools as the agent saw them. With ``takes_config`` the agent's card + takes its config (``CONFIG_CARD``), whose posts land in ``configs``.""" def __init__(self, monkeypatch, tmp_path, *, agent: Optional[Turn] = None, answers: tuple = (), rate: float = 0, - t0: str = T0, plugin: Any = None, user_texts: tuple = ()): + t0: str = T0, plugin: Any = None, user_texts: tuple = (), takes_config: bool = False): monkeypatch.setattr(gw_mod, "GATEWAY_TRAJECTORY_FILE", str(tmp_path / "trajectory.jsonl")) self.gw = Gateway(host="127.0.0.1", port=0, server_name="AgentEnvGateway", internal_mcp_servers=[]) self.gw._tools_discovered = True @@ -68,11 +74,19 @@ def __init__(self, monkeypatch, tmp_path, *, agent: Optional[Turn] = None, answe self.user_texts = list(user_texts) self.user_calls: list[dict] = [] self.contact_calls: list[dict] = [] + self.during: list[dict] = [] + self.solver_contexts: list[Optional[str]] = [] + self.takes_config = takes_config + self.configs: list[tuple[str, dict]] = [] self._contacts: dict[str, dict] = {} self._monkeypatch = monkeypatch monkeypatch.setattr(ao, "POLL_SECONDS", 0) monkeypatch.setattr(ao, "send_and_wait", self._send_and_wait) monkeypatch.setattr(ao._Run, "_load_plugin", self._load_plugin) + if takes_config: + async def post(url, fields): + self.configs.append((url, fields)) + monkeypatch.setattr(ao.protocol, "post_agent_config", post) # ---- the world ----------------------------------------------------------------------------------------------- async def arm(self) -> None: @@ -188,7 +202,8 @@ def context(self, *, user: Optional[dict] = None) -> TaskStepContext: environment_card_url=f"{GATEWAY}{WELL_KNOWN_PATH}", environment_card=card, metadata={"deploy_step_id": "deploy"}) ctx = TaskStepContext(instance_id="ti-1", deployed_envs=[env]) - ctx.deployed_agents += [DeployedAgent(agent_name="solver", api_url=SOLVER, a2a_url=SOLVER, a2a_card={}), + ctx.deployed_agents += [DeployedAgent(agent_name="solver", api_url=SOLVER, a2a_url=SOLVER, + a2a_card=CONFIG_CARD if self.takes_config else {}), DeployedAgent(agent_name="user-sim", api_url=USER_SIM, a2a_url=USER_SIM, a2a_card={})] ctx.metadata[USER_SIMS_KEY] = {"linda": { "name": "linda", "model": "sim-model", "agent_name": "user-sim", "user_version": "ours", @@ -234,8 +249,14 @@ async def _send_and_wait(self, url, parts, *, before_send=None, **kw): return f"user-{len(self.user_calls)}", {"status": {"state": "completed", "message": { "parts": [{"kind": "text", "text": text}]}}} assert url == SOLVER, url + self.solver_contexts.append(kw.get("context_id")) if before_send: before_send() + chat, notes = self.gw._user_chat, self.gw._notifications + self.during.append({"user_chat_roles": chat.roles, "stay_quiet": chat.stay_quiet, "check_roles": notes.roles, + "min_interval": notes.min_interval, "max_active": notes.max_active, + "trigger_settings": dict(self.gw._trigger_engine.state()["settings"]), + "clock_rate": self.gw._clock.rate, "tools": await self.tools()}) texts = [p["text"] for p in parts if p.get("kind") == "text"] self.sent.append(texts) self.parts.append(list(parts)) diff --git a/tst/unit/task_step/test_always_on_agent_and_gateway.py b/tst/unit/task_step/test_always_on_agent_and_gateway.py new file mode 100644 index 000000000..26009cb06 --- /dev/null +++ b/tst/unit/task_step/test_always_on_agent_and_gateway.py @@ -0,0 +1,99 @@ +"""What the step sets on the agent and the gateways, and what it leaves: the message_user rules reach the agent even when +its card won't take a system prompt; the step's context_id is the agent's session, as prompt_agent's is; and the +gateway settings the step changed for its run are as they were once it ends, for the task's later steps.""" + +from __future__ import annotations + +import logging + +import pytest + +from agent_env.task_step.task_steps import always_on as ao +from tst.unit.task_step.always_on_harness import SOLVER, Rig + +pytestmark = pytest.mark.usefixtures("local_stores") + +RULES = f"{ao.MESSAGE_RULES} {ao.QUIET_RULE}" +PROMPT = "You are Linda's assistant." + + +@pytest.mark.asyncio +async def test_the_rules_go_on_the_first_wake_when_the_agent_won_t_take_a_system_prompt(monkeypatch, tmp_path, caplog): + rig = Rig(monkeypatch, tmp_path) # its card takes no config + await rig.arm() + caplog.set_level("WARNING", logger=ao.logger.name) + ctx = await rig.run(rig.step(system_prompt=PROMPT)) + + assert rig.sent[0][0] == RULES and rig.parts[0][0]["metadata"] == {"agentenv": {"source": "rules"}} + assert rig.record(ctx)["settings"]["rules_via"] == "first_wake" + assert any("did not accept system_prompt" in r.getMessage() for r in caplog.records + if r.levelno == logging.WARNING) + + +@pytest.mark.asyncio +async def test_the_rules_go_in_the_system_prompt_when_the_agent_takes_it(monkeypatch, tmp_path): + rig = Rig(monkeypatch, tmp_path, takes_config=True) + await rig.arm() + ctx = await rig.run(rig.step(system_prompt=PROMPT)) + + [(url, fields)] = [(u, f) for u, f in rig.configs if u.startswith(SOLVER)] + assert fields["system_prompt"] == f"{PROMPT}\n\n{RULES}" + assert RULES not in rig.sent[0] + assert rig.record(ctx)["settings"]["rules_via"] == "system_prompt" + + +@pytest.mark.asyncio +async def test_the_step_s_context_id_is_the_agent_s_session(monkeypatch, tmp_path): + def agent(n, texts, rig): + rig.message("blue or green?") if n == 1 else rig.quiet() + return "" + + rig = Rig(monkeypatch, tmp_path, agent=agent, answers=[{"outcome": "reply", "message": "blue"}]) + await rig.arm() + ctx = await rig.run(rig.step(context_id="cdmx-weekend")) + + assert rig.solver_contexts == ["cdmx-weekend", "cdmx-weekend"] + assert ctx.prompt_responses[0].a2a_context_id == "cdmx-weekend" + assert ao.AlwaysOnTaskStep.from_dict(rig.step(context_id="cdmx-weekend").to_dict()).context_id == "cdmx-weekend" + + +@pytest.mark.asyncio +async def test_without_a_context_id_each_run_has_a_session_of_its_own(monkeypatch, tmp_path): + rig = Rig(monkeypatch, tmp_path) + await rig.arm() + ctx = await rig.run(rig.step()) + [session] = set(rig.solver_contexts) + assert session and ctx.prompt_responses[0].a2a_context_id == session + + +@pytest.mark.asyncio +async def test_the_gateway_settings_the_step_changed_are_as_they_were_after_the_run(monkeypatch, tmp_path): + rig = Rig(monkeypatch, tmp_path, rate=60) + await rig.arm() + gw = rig.gw + gw._user_chat.configure({"roles": ["planner"], "user_name": "Tess"}) + gw._trigger_engine.config({"executor_timeout_seconds": 90}) + gw._notifications.configure({"roles": ["planner"], "min_interval": "PT2H", "max_active": 4}) + await rig.run(rig.step(end_time="P2D")) + + during = rig.during[0] + assert during["user_chat_roles"] == ("default",) and during["stay_quiet"] + assert during["trigger_settings"] == {"concurrent_fires": True, "executor_timeout_seconds": 300.0} + assert during["check_roles"] == ("planner", "default") and during["clock_rate"] == 60 + assert (gw._user_chat.roles, gw._user_chat.user_name, gw._user_chat.stay_quiet) == (("planner",), "Tess", False) + assert gw._trigger_engine.state()["settings"] == {"concurrent_fires": False, "executor_timeout_seconds": 90.0} + assert (gw._notifications.roles, gw._notifications.min_interval, gw._notifications.max_active) == ( + ("planner",), "PT2H", 4) + assert gw._clock.rate == 60 + + +@pytest.mark.asyncio +async def test_a_gateway_the_task_left_alone_is_back_to_its_defaults(monkeypatch, tmp_path): + rig = Rig(monkeypatch, tmp_path) + await rig.arm() + await rig.run(rig.step(end_time="P2D")) + + gw = rig.gw + assert (gw._user_chat.roles, gw._user_chat.stay_quiet, gw._notifications.roles) == ((), False, ()) + assert gw._trigger_engine.state()["settings"] == {"concurrent_fires": False, "executor_timeout_seconds": None} + assert not {"message_user", "stay_quiet", "schedule_check"} & await rig.tools() diff --git a/tst/unit/task_step/test_always_on_budget.py b/tst/unit/task_step/test_always_on_budget.py index 5d2beda43..9ea887dd0 100644 --- a/tst/unit/task_step/test_always_on_budget.py +++ b/tst/unit/task_step/test_always_on_budget.py @@ -188,10 +188,16 @@ async def test_a_turn_of_the_loop_past_the_reserve_ends_the_run_on_a_step_error( monkeypatch.setattr(ao.AlwaysOnTaskStep, "wake_reserve_seconds", lambda step: 0.5) rig = Rig(monkeypatch, tmp_path) - async def hangs(run): - await asyncio.sleep(60) + real_take = ao._Run._take_messages + calls: list[int] = [] + + async def hangs_once(run): + calls.append(1) + if len(calls) == 1: + await asyncio.sleep(60) + return await real_take(run) - monkeypatch.setattr(ao._Run, "_take_messages", hangs) + monkeypatch.setattr(ao._Run, "_take_messages", hangs_once) await rig.arm() ctx = rig.context() with pytest.raises(RuntimeError, match="the opening wake ran past its worst case"): diff --git a/tst/unit/task_step/test_always_on_checks.py b/tst/unit/task_step/test_always_on_checks.py index c63f56369..a7c46dde9 100644 --- a/tst/unit/task_step/test_always_on_checks.py +++ b/tst/unit/task_step/test_always_on_checks.py @@ -22,22 +22,35 @@ async def test_a_multi_day_run_gives_the_agent_checks_through_a_deny_all_and_say await client.post("/tools/disable", json={"role": "default", "tools": ["*"]}) ctx = await rig.run(rig.step(end_time="P5D")) - assert rig.gw._notifications.roles == ("default",) and rig.gw._notifications.min_interval == "PT3H" - assert CHECK_TOOLS <= await rig.tools() + during = rig.during[0] + assert during["check_roles"] == ("default",) and during["min_interval"] == "PT3H" + assert CHECK_TOOLS <= during["tools"] assert ao.CHECKS_RULE in rig.sent[0][0] + assert rig.gw._notifications.roles == () and not CHECK_TOOLS & await rig.tools() # off again after the run assert rig.record(ctx)["settings"]["scheduled_checks"] == { "on": True, "envs": [{"env_id": "world", "set_by": "step", "min_interval": "PT3H", "max_active": 10}]} @pytest.mark.asyncio async def test_checks_closer_than_the_minimum_gap_are_refused(monkeypatch, tmp_path): - rig = Rig(monkeypatch, tmp_path) + refused: list[str] = [] + + def agent(n, texts, rig): + if n == 1: + try: + rig.gw._notifications.schedule("sync stock", every="PT1H", to="default") + except NotificationError as e: + refused.append(str(e)) + rig.gw._notifications.schedule("sync stock", every="PT3H", to="default") + rig.quiet() + return "" + + rig = Rig(monkeypatch, tmp_path, agent=agent) await rig.arm() - await rig.run(rig.step(end_time="P5D")) + await rig.run(rig.step(end_time="P1D")) - with pytest.raises(NotificationError, match="at least PT3H"): - rig.gw._notifications.schedule("sync stock", every="PT1H", to="default") - rig.gw._notifications.schedule("sync stock", every="PT3H", to="default") + [why] = refused + assert "at least PT3H" in why @pytest.mark.asyncio @@ -68,7 +81,7 @@ async def test_otherwise_a_task_decides(monkeypatch, tmp_path, end_time, setting **({"scheduled_checks": setting} if setting is not None else {})} ctx = await rig.run(rig.step(**settings)) - assert (rig.gw._notifications.roles == ("default",)) is on + assert (rig.during[0]["check_roles"] == ("default",)) is on assert rig.record(ctx)["settings"]["scheduled_checks"]["on"] is on assert (ao.CHECKS_RULE in rig.sent[0][0]) is on @@ -100,7 +113,8 @@ async def test_other_roles_the_task_gave_checks_keep_them(monkeypatch, tmp_path) rig.gw._notifications.configure({"roles": ["planner"], "min_interval": "PT2H"}) await rig.run(rig.step(end_time="P5D")) - assert rig.gw._notifications.roles == ("planner", "default") and rig.gw._notifications.min_interval == "PT2H" + assert rig.during[0]["check_roles"] == ("planner", "default") and rig.during[0]["min_interval"] == "PT2H" + assert rig.gw._notifications.roles == ("planner",) and rig.gw._notifications.min_interval == "PT2H" def test_the_settings_survive_a_round_trip(): diff --git a/tst/unit/task_step/test_always_on_messages.py b/tst/unit/task_step/test_always_on_messages.py index e397ab8de..8a9ec858a 100644 --- a/tst/unit/task_step/test_always_on_messages.py +++ b/tst/unit/task_step/test_always_on_messages.py @@ -40,7 +40,7 @@ async def test_all_three_are_on_by_default(monkeypatch, tmp_path): assert (step.stay_quiet, step.nudge, step.wake_reminder) == (True, True, True) assert rig.record(ctx)["settings"].items() >= {"stay_quiet": True, "nudge": True, "wake_reminder": True}.items() - assert rig.gw._user_chat.stay_quiet and ROLE in rig.gw._user_chat.roles + assert rig.during[0]["stay_quiet"] and ROLE in rig.during[0]["user_chat_roles"] @pytest.mark.asyncio diff --git a/tst/unit/task_step/test_always_on_time.py b/tst/unit/task_step/test_always_on_time.py index 87012a4cd..7d3a49279 100644 --- a/tst/unit/task_step/test_always_on_time.py +++ b/tst/unit/task_step/test_always_on_time.py @@ -15,7 +15,7 @@ @pytest.mark.asyncio async def test_each_wake_opens_with_the_task_s_date_and_time(monkeypatch, tmp_path): - rig = Rig(monkeypatch, tmp_path) + rig = Rig(monkeypatch, tmp_path, takes_config=True) # the rules go in its system prompt await rig.arm() rig.triggers(rig.notify_at("sam", "2026-11-10T14:00:00Z", "Sam: shh", channel="noor 30")) ctx = await rig.run(rig.step(system_prompt=PROMPT)) @@ -30,7 +30,7 @@ async def test_each_wake_opens_with_the_task_s_date_and_time(monkeypatch, tmp_pa @pytest.mark.asyncio async def test_a_user_in_utc_gets_the_time_once(monkeypatch, tmp_path): - rig = Rig(monkeypatch, tmp_path) + rig = Rig(monkeypatch, tmp_path, takes_config=True) await rig.arm() await rig.run(rig.step(system_prompt=PROMPT), rig.context(user={"time_zone": "UTC"})) assert rig.sent[0][0] == f"It's Mon Nov 9, 2026, 2:00 PM UTC. {CLOCK}" @@ -51,4 +51,5 @@ async def test_get_time_stays_with_the_agent_when_the_task_hides_every_tool(monk await client.post("/tools/disable", json={"role": "default", "tools": ["*"]}) assert "get_time" not in await rig.tools() await rig.run(rig.step()) - assert {"get_time", "message_user", "stay_quiet"} <= await rig.tools() + assert {"get_time", "message_user", "stay_quiet"} <= rig.during[0]["tools"] + assert not {"message_user", "stay_quiet"} & await rig.tools() # the user chat is off again after the run From 802aa2f93248a2651fbb384170f08247f29a1388 Mon Sep 17 00:00:00 2001 From: Sarah Chitty Date: Mon, 9 Nov 2026 03:43:14 -0800 Subject: [PATCH 37/53] always_on: the record tells undelivered replies apart, keeps leftover messages, stays small, counts malformed answers - A reply that never reached the agent (her last read's, or one still held when the run ended) looked delivered in the hook-shaped records: user_reads_at was set when she answered. Now a record gets user_reads_at only when a wake carries the reply (delivered: true); one never delivered has delivered: false, deliver_at and, if held at the end, why, so ML's asked_before_spending (first_reply over user_reads_at) can't count it as received. Checked with ML's process_checks.py (the surprise-30th copy the stress lanes used): first_reply is None and the row reads "before Linda first answered (never)", where it said she had answered at the held reply's time. - finish() takes the message_user messages no turn took (a turn that timed out after sending), within 60 s: they are in the graded response and conversation with their times, marked leftover in the timeline, and listed in pending_at_end as never read. - When the timeline can't be stored, the events kept in the record are cut down at any depth (strings, lists, nesting), as pending_at_end now is too; _bounded only trimmed top-level strings, and the end event carried the whole pending. - source_agent_per_turn_prompt_parts keeps each wake's text parts cut to 4000 characters, up to 400k in all, then a note; the timeline's wake events keep the full text. - counts.user_sim_malformed_answers counts reads whose answer had schema_errors. Co-Authored-By: Claude Opus 5.5 --- .../task_step/task_steps/always_on.py | 94 +++++++++++++++---- tst/unit/task_step/test_always_on_endings.py | 20 ++++ tst/unit/task_step/test_always_on_record.py | 69 ++++++++++++++ .../task_step/test_always_on_who_did_what.py | 38 +++++++- 4 files changed, 200 insertions(+), 21 deletions(-) diff --git a/src/agent_env/task_step/task_steps/always_on.py b/src/agent_env/task_step/task_steps/always_on.py index 83a2a1f70..1a469249c 100644 --- a/src/agent_env/task_step/task_steps/always_on.py +++ b/src/agent_env/task_step/task_steps/always_on.py @@ -43,7 +43,8 @@ loop's docstring), so a recurring trigger that never wakes the agent can't move the clock forever. What was still pending is recorded (``pending_at_end``): replies held, messages the user never read, user lines and alerts never delivered, the agent's checks not yet due, contact replies in progress. - Graders read what the user saw. The prompt response is every message_user text in order, each with its pretend - time, never the agent's closing text. The conversation (``metadata["a2a_conversations"][step_id]``) holds the user's + time, never the agent's closing text; messages no turn took (a turn that timed out after sending) are taken at the + end and included, and listed as never read. The conversation (``metadata["a2a_conversations"][step_id]``) holds the user's own messages and the agent's message_user texts, then the user's last word that never reached the agent and an end marker with the end reason. A failed or timed-out agent turn sets the response's ``error_type`` (as prompt_agent does), so verifiers skip the run. @@ -51,7 +52,9 @@ with who made it and the wake it was seen in, the agent's own with the wake that made it (``app_changes`` events); each reply records the agent's message it answers (``reply_to``: its wake, time and text); and each read leaves a record in ``metadata["post_turn_hooks"][step_id]`` in the shape the old psi-wake hook wrote, ``user_reads_at`` - being when the reply reached the agent, so checks built on those records grade the run instead of skipping it. + being when the reply reached the agent, so checks built on those records grade the run instead of skipping it. A + reply that never reached it (her last read's, or one still held at the end) has ``delivered`` false and no + ``user_reads_at``. - The run's metadata keeps a summary in ``metadata["always_on"][step_id]`` (small enough for a running job to report back); the full timeline goes to the object store, at the summary's ``object_url``. - The agent: the message_user rules go in its system prompt when the step sets one and the agent's card takes it, @@ -139,6 +142,8 @@ TRAJECTORY_FETCH_SECONDS = 120.0 READ_ASKS = 2 GATEWAY_ALLOWANCE_SECONDS = 300.0 +# At the end, how long taking the agent's messages no turn took may take (the run's record comes first). +LEFTOVER_TAKE_SECONDS = 60.0 # The final check-in's line when the task sets no `final`: the step's, not the user's. FINAL_DEFAULT_TEXT = "Time's up on this job. Send {first} a short wrap-up: what's done, and anything still open." NOTHING_CAN_WAKE = "nothing_can_wake_the_agent" # until finish() tells finished_nothing_pending from stuck @@ -172,6 +177,11 @@ PENDING_KEPT = 20 PENDING_TEXT_CHARS = 300 BULKY_EVENT_FIELDS = ("input", "raw", "read", "parts_text", "changes") +# What the prompt response keeps of each wake's parts (source_agent_per_turn_prompt_parts): each text part cut to +# TURN_TEXT_CHARS, and past TURN_TEXT_TOTAL_CHARS in all, a note for the rest; every wake's full text is in the stored +# timeline (its wake event's parts_text). +TURN_TEXT_CHARS = 4000 +TURN_TEXT_TOTAL_CHARS = 400_000 # ================================================================================================================== @@ -1133,8 +1143,8 @@ async def _wake_with(self, now: datetime, lines: list[dict], replies: list[dict] "done": r.get("done", False)} for r in replies], key=lambda m: _utc(m["at"])) for r in replies: - if r.get("hook") is not None: - r["hook"]["delivered_in_wake"] = self.wakes + 1 + if r.get("hook") is not None: # it reaches the agent now: the reply time the old psi-wake hook recorded + r["hook"].update(delivered=True, user_reads_at=_iso(r["deliver_at"]), delivered_in_wake=self.wakes + 1) if user_msgs: await self._chat_user([self._chat_part(_utc(m["at"]), m["text"], kind=m["kind"], **({"done": True} if m.get("done") else {})) for m in user_msgs]) @@ -1178,7 +1188,7 @@ async def wake(self, parts: list[dict], woke_by: list[str], notifications: Optio await self._read_changes("before_turn") def record() -> None: - self.per_turn_parts.append(list(parts)) + self.per_turn_parts.append(self._kept_parts(parts)) self.log("wake", n=self.wakes, virtual_time=_iso(sent_at), woke_by=woke_by, notifications=notifications or [], last=last, parts_text="\n".join(p.get("text", "") for p in parts if p.get("kind") == "text")) @@ -1228,6 +1238,25 @@ def record() -> None: await self._read_changes("after_turn") await self._schedule_read() + def _kept_parts(self, parts: list[dict]) -> list[dict]: + """What the prompt response keeps of one wake's parts: TURN_TEXT_CHARS of each text, until + TURN_TEXT_TOTAL_CHARS in all; then a note. Other parts (files, data) as they are.""" + kept, used = [], sum(len(p.get("text") or "") for turn in self.per_turn_parts for p in turn or []) + for part in parts: + if part.get("kind") != "text": + kept.append(part) + continue + text = part.get("text") or "" + room = min(TURN_TEXT_CHARS, TURN_TEXT_TOTAL_CHARS - used) + if room <= 0: + kept.append({"kind": "text", "text": "(cut: the run's stored timeline has this wake's full text)"}) + break + if len(text) > room: + part = {**part, "text": text[:room] + "… (cut)"} + used += len(part["text"]) + kept.append(part) + return kept + async def _nudge(self, closing: str) -> None: """The wake ended with closing text and neither message_user nor stay_quiet, so give the agent one more short turn with ``nudge_text``; record what the closing text was and what it did after.""" @@ -1238,7 +1267,7 @@ async def _nudge(self, closing: str) -> None: msg_id = uuid.uuid4().hex def record() -> None: - self.per_turn_parts.append(list(parts)) + self.per_turn_parts.append(self._kept_parts(parts)) t = time.time() try: @@ -1344,11 +1373,12 @@ async def _take_messages(self) -> tuple[list[dict], list[dict]]: messages.sort(key=lambda m: m.get("at") or "") return messages, quiet - def _sent(self, messages: list[dict], after_nudge: bool = False) -> None: - """Record what the agent sent the user: for her next read, the graded chat and the timeline.""" + def _sent(self, messages: list[dict], after_nudge: bool = False, leftover: bool = False) -> None: + """Record what the agent sent the user: for her next read, the graded chat and the timeline. ``leftover`` + marks messages no turn took (taken at the end), which she never reads.""" for m in messages: self.log("message_to_user", virtual_time=m.get("at"), text=m["text"], wake=self.wakes, - **({"after_nudge": True} if after_nudge else {})) + **({"after_nudge": True} if after_nudge else {}), **({"leftover": True} if leftover else {})) sent = {"at": m.get("at"), "text": m["text"], "wake": self.wakes} self.unread.append(sent) self.all_messages.append(sent) @@ -1412,8 +1442,10 @@ async def ask(text: str) -> dict: deliver = now + latency reply_to = _reply_to(ans, seen) entry.update(deliver_at=_iso(deliver), reply_to=reply_to, reply_to_candidates=len(seen)) - # user_reads_at: when her answer reaches the agent, the reply time the old psi-wake hook recorded - hook.update(user_reads_at=_iso(deliver), reply_to=_cut(reply_to)) + # Not delivered until a wake carries it, which sets user_reads_at; one never delivered (her last read's + # reply, or one still held at the end) keeps delivered False and no user_reads_at, so a check can't count + # it as received. + hook.update(delivered=False, deliver_at=_iso(deliver), reply_to=_cut(reply_to)) self.held.append({"deliver_at": deliver, "text": ans["message"], "done": outcome == "done", "hook": hook}) self.last_reply_at = deliver @@ -1521,6 +1553,14 @@ async def _pending_at_end(self) -> dict: async def finish(self) -> None: step, ctx = self.step, self.ctx now = await self.now() + try: # what the agent sent and no turn took: a turn that timed out or failed mid-way + leftover, _ = await asyncio.wait_for(self._take_messages(), LEFTOVER_TAKE_SECONDS) + self._sent(leftover, leftover=True) + except Exception as e: + logger.warning(f"always_on: could not take the agent's last messages: {type(e).__name__}: {e}") + for h in self.held: + if h.get("hook") is not None: + h["hook"]["undelivered"] = "the run ended before it reached the agent" await self._read_changes("end") try: self.pending = await self._pending_at_end() @@ -1558,6 +1598,9 @@ async def finish(self) -> None: "clock_jumps": self.clock_jumps, "silent_wakes": sum(1 for e in self.timeline if e["event"] == "agent_turn_end" and not e.get("messages_to_user")), + # reads whose answer didn't match the user sim's answer format (the plug-in's schema_errors) + "user_sim_malformed_answers": sum(1 for e in self.timeline if e["event"] == "user_sim_read" + and e.get("schema_errors")), } if self.change_cursors: counts["app_changes"] = dict(self.change_counts) @@ -1604,7 +1647,7 @@ async def _store_timeline(self, record: dict) -> None: except Exception as e: logger.warning(f"always_on: could not store the timeline: {e}") record["timeline_store_error"] = f"{type(e).__name__}: {str(e)[:300]}" - record["timeline_tail"] = [{k: v for k, v in event.items() if k not in BULKY_EVENT_FIELDS} + record["timeline_tail"] = [_small({k: v for k, v in event.items() if k not in BULKY_EVENT_FIELDS}) for event in self.timeline[-TIMELINE_TAIL:]] def _append_prompt_response(self, final_text: str) -> None: @@ -1788,14 +1831,25 @@ def _cut(item: Optional[dict]) -> Optional[dict]: def _bounded(pending: dict) -> dict: - """``pending`` as the run's metadata keeps it: each list's count, and its first ``PENDING_KEPT`` items with their - texts cut to ``PENDING_TEXT_CHARS``; the stored timeline's end event has them all.""" - out = {} - for kind, items in pending.items(): - kept = [{k: (v[:PENDING_TEXT_CHARS] + "…" if isinstance(v, str) and len(v) > PENDING_TEXT_CHARS else v) - for k, v in item.items()} for item in items[:PENDING_KEPT]] - out[kind] = {"count": len(items), "items": kept} - return out + """``pending`` as the run's metadata keeps it: each list's count, and its first ``PENDING_KEPT`` items, kept + small (``_small``) however deep; the stored timeline's end event has them all.""" + return {kind: {"count": len(items), "items": [_small(item) for item in items[:PENDING_KEPT]]} + for kind, items in pending.items()} + + +def _small(value: Any, depth: int = 0) -> Any: + """``value`` kept small for the run's metadata, at any depth: strings cut to ``PENDING_TEXT_CHARS``, lists to their + first ``PENDING_KEPT`` items and a count of the rest, nesting past six levels summed up.""" + if isinstance(value, str): + return value[:PENDING_TEXT_CHARS] + "…" if len(value) > PENDING_TEXT_CHARS else value + if depth >= 6 and isinstance(value, (list, dict)): + return f"({len(value)} {'items' if isinstance(value, list) else 'fields'})" + if isinstance(value, dict): + return {k: _small(v, depth + 1) for k, v in value.items()} + if isinstance(value, (list, tuple)): + kept = [_small(v, depth + 1) for v in value[:PENDING_KEPT]] + return kept + [f"(and {len(value) - PENDING_KEPT} more)"] if len(value) > PENDING_KEPT else kept + return value # The timeline fields the process log may show as they are: names, ids, times, states and settings, never what a person diff --git a/tst/unit/task_step/test_always_on_endings.py b/tst/unit/task_step/test_always_on_endings.py index 8718d471a..45f145103 100644 --- a/tst/unit/task_step/test_always_on_endings.py +++ b/tst/unit/task_step/test_always_on_endings.py @@ -223,3 +223,23 @@ async def test_a_contact_still_replying_when_nothing_else_can_wake_the_agent_mak assert record["end_reason"] == "stuck" [env] = record["pending_at_end"]["contact_replies_in_progress"]["items"] assert env["triggers"][0]["id"] == "liveoak-estimate" + + +@pytest.mark.asyncio +async def test_messages_a_turn_sent_before_it_timed_out_are_in_the_graded_response_and_pending(monkeypatch, tmp_path): + def agent(n, texts, rig): + rig.message("found it: the blue house, $855") + raise TimeoutError("the agent's turn ran out") + + rig = Rig(monkeypatch, tmp_path, agent=agent) + await rig.arm() + ctx = await rig.run(rig.step()) + + [response] = ctx.prompt_responses + assert response.response == "[Mon Nov 9, 2:00 PM UTC] found it: the blue house, $855" + assert response.error_type == "agent_turn_timeout" + [sent] = rig.events(ctx, "message_to_user") + assert sent["leftover"] is True + [unread] = rig.record(ctx)["pending_at_end"]["unread_by_user"]["items"] + assert unread["text"] == "found it: the blue house, $855" and unread["at"] == "2026-11-09T14:00:00Z" + assert "found it: the blue house" in " ".join(t for m in rig.chat(ctx) for t in m["texts"]) diff --git a/tst/unit/task_step/test_always_on_record.py b/tst/unit/task_step/test_always_on_record.py index 983883c2f..7f2b636a6 100644 --- a/tst/unit/task_step/test_always_on_record.py +++ b/tst/unit/task_step/test_always_on_record.py @@ -78,3 +78,72 @@ def agent(n, texts, rig): stored = json.dumps(rig.timeline(ctx)) for marker in ("SECRET-SENT", "SECRET-PLAIN", "SECRET-REPLY", "SECRET-ALERT", "SECRET-PROMPT"): assert marker in stored + + +def _fails_after_fifty_long_messages(n, texts, rig): + for i in range(50): + rig.message(f"option {i}: " + "x" * 2000) + raise TimeoutError("the agent's turn ran out") + + +@pytest.mark.asyncio +async def test_when_storing_fails_the_kept_events_are_small_at_any_depth(monkeypatch, tmp_path): + rig = Rig(monkeypatch, tmp_path, agent=_fails_after_fifty_long_messages) + await rig.arm() + store = get_config().get_object_store() + + def fail(*a, **kw): + raise OSError("bucket gone") + + monkeypatch.setattr(type(store), "put", fail) + ctx = await rig.run(rig.step()) + + record = rig.record(ctx) + assert record["pending_at_end"]["unread_by_user"]["count"] == 50 + end = record["timeline_tail"][-1] + assert end["event"] == "end" and end["pending"]["unread_by_user"][-1] == "(and 30 more)" + + def strings(value): + if isinstance(value, str): + yield value + elif isinstance(value, dict): + for v in value.values(): + yield from strings(v) + elif isinstance(value, list): + for v in value: + yield from strings(v) + + assert max(len(s) for s in strings(record)) <= ao.PENDING_TEXT_CHARS + 1 + assert len(json.dumps(record, default=str)) < 60_000 + + +@pytest.mark.asyncio +async def test_the_prompt_response_keeps_each_wake_s_parts_cut_to_size(monkeypatch, tmp_path): + monkeypatch.setattr(ao, "TURN_TEXT_TOTAL_CHARS", 9000) + rig = Rig(monkeypatch, tmp_path) + await rig.arm() + for day in (10, 11, 12): + rig.triggers(rig.notify_at(f"news-{day}", f"2026-11-{day}T14:00:00Z", "y" * 10_000, channel="Gmail")) + ctx = await rig.run(rig.step()) + + kept = ctx.prompt_responses[0].source_agent_per_turn_prompt_parts + assert len(kept) == 4 + alert = next(p["text"] for p in kept[1] if p["text"].startswith("[Tue Nov 10")) + assert len(alert) == ao.TURN_TEXT_CHARS + len("… (cut)") and alert.endswith("… (cut)") + assert kept[3][-1]["text"] == "(cut: the run's stored timeline has this wake's full text)" + assert "y" * 10_000 in rig.events(ctx, "wake")[1]["parts_text"] # the timeline keeps it whole + + +@pytest.mark.asyncio +async def test_reads_whose_answer_broke_the_format_are_counted(monkeypatch, tmp_path): + def agent(n, texts, rig): + rig.message(f"question {n}?") if n <= 2 else rig.quiet() + return "" + + rig = Rig(monkeypatch, tmp_path, agent=agent, + answers=[{"outcome": "reply", "message": "hm", "schema_errors": ["no JSON object in the answer"]}, + {"outcome": "no_reply", "message": "", "schema_errors": []}]) + await rig.arm() + ctx = await rig.run(rig.step()) + + assert rig.record(ctx)["counts"]["user_sim_malformed_answers"] == 1 diff --git a/tst/unit/task_step/test_always_on_who_did_what.py b/tst/unit/task_step/test_always_on_who_did_what.py index 8afff6813..f9803beb9 100644 --- a/tst/unit/task_step/test_always_on_who_did_what.py +++ b/tst/unit/task_step/test_always_on_who_did_what.py @@ -157,7 +157,8 @@ def agent(n, texts, rig): start, first, second = ctx.metadata["post_turn_hooks"]["week"] assert start == {"hook": "always_on", "turn": 0, "started_at": "2026-11-09T14:00:00Z"} assert first == {"hook": "always_on", "turn": 1, "read_at": "2026-11-09T14:00:00Z", - "wrote_at": "2026-11-09T14:00:00Z", "outcome": "reply", "user_reads_at": "2026-11-09T16:30:00Z", + "wrote_at": "2026-11-09T14:00:00Z", "outcome": "reply", "delivered": True, + "deliver_at": "2026-11-09T16:30:00Z", "user_reads_at": "2026-11-09T16:30:00Z", "reply_to": {"wake": 1, "at": "2026-11-09T14:00:00Z", "text": "question 1?", "matched_by": "only_message"}, "delivered_in_wake": 2} @@ -173,3 +174,38 @@ async def test_a_run_she_never_answered_still_has_records_so_a_check_grades_it_i ctx = await rig.run(rig.step()) assert _first_reply(ctx.metadata[ao.HOOK_RECORDS_KEY]) == "never" # graded: any spend came before an OK + + +@pytest.mark.asyncio +async def test_a_reply_that_never_reached_the_agent_is_marked_undelivered_and_has_no_reply_time(monkeypatch, tmp_path): + """Her reply is held for 2.5 h; an alert wakes the agent after one, the run's last wake (max_wakes), so the reply + never reaches it. A check reading user_reads_at must not count it as received.""" + def agent(n, texts, rig): + rig.message("ok to book La Ola for $855?") if n == 1 else rig.quiet() + return "" + + rig = Rig(monkeypatch, tmp_path, agent=agent, answers=[{"outcome": "reply", "message": "yes, book it"}]) + await rig.arm() + rig.triggers(rig.notify_at("sam", "2026-11-09T15:00:00Z", "Sam: flights?", channel="noor 30")) + ctx = await rig.run(rig.step(max_wakes=2), rig.context(user=DAY)) + + _, held = ctx.metadata["post_turn_hooks"]["week"] + assert held["outcome"] == "reply" and held["delivered"] is False and "user_reads_at" not in held + assert held["deliver_at"] == "2026-11-09T16:30:00Z" + assert held["undelivered"] == "the run ended before it reached the agent" + assert _first_reply(ctx.metadata["post_turn_hooks"]) == "never" # not the reply she never got to send + + +@pytest.mark.asyncio +async def test_her_last_read_s_reply_is_marked_undelivered(monkeypatch, tmp_path): + def agent(n, texts, rig): + rig.message("wrapped up: La Ola is booked") + return "" + + rig = Rig(monkeypatch, tmp_path, agent=agent, answers=[{"outcome": "reply", "message": "thanks!"}]) + await rig.arm() + ctx = await rig.run(rig.step(max_wakes=1)) + + [_, last] = ctx.metadata["post_turn_hooks"]["week"] + assert last["delivered"] is False and "user_reads_at" not in last + assert _first_reply(ctx.metadata["post_turn_hooks"]) == "never" From 109d024439e56a8ed1a3b1c4b48a204b8d2d69bf Mon Sep 17 00:00:00 2001 From: Sarah Chitty Date: Mon, 9 Nov 2026 03:50:23 -0800 Subject: [PATCH 38/53] always_on: one read of an env's changelog keeps at most 200 changes in the timeline Part of keeping a long run's timeline bounded in memory (with the folded clock jumps): a gap in which triggers or contacts make thousands of app changes put every one of them in a single app_changes event. The event now keeps the first CHANGES_KEPT_PER_READ (200) and says how many it didn't keep; counts.app_changes still counts them all by who made them, and the judge's wake-by-wake view never showed more than 10 per wake. Co-Authored-By: Claude Opus 5.5 --- .../task_step/task_steps/always_on.py | 6 +++++- .../task_step/test_always_on_who_did_what.py | 19 +++++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/src/agent_env/task_step/task_steps/always_on.py b/src/agent_env/task_step/task_steps/always_on.py index 1a469249c..6d2a82f04 100644 --- a/src/agent_env/task_step/task_steps/always_on.py +++ b/src/agent_env/task_step/task_steps/always_on.py @@ -168,6 +168,9 @@ HOOK_RECORDS_KEY = "post_turn_hooks" HOOK_NAME = "always_on" CHANGE_SUMMARY_CHARS = 200 +# The app changes one read of an env's changelog keeps in the timeline; the rest are counted (counts.app_changes counts +# them all, by who made them). +CHANGES_KEPT_PER_READ = 200 TIMELINE_TAIL = 20 # With no end_time, the run ends once this many clock jumps in a row have been to repeating triggers' marks alone (see # the loop's docstring): the world can go on ticking, but nothing in it has woken the agent. @@ -1346,8 +1349,9 @@ async def _read_changes(self, phase: str) -> None: self.change_cursors[env_id] = (env, cursor) if rows: changes = [self._change(env_id, row, phase) for row in rows] + more = len(changes) - CHANGES_KEPT_PER_READ self.log("app_changes", wake=self.wakes, phase=phase, env_id=env_id, count=len(changes), - changes=changes) + changes=changes[:CHANGES_KEPT_PER_READ], **({"not_kept": more} if more > 0 else {})) def _change(self, env_id: str, row: dict, phase: str) -> dict: actors = [{k: v for k, v in a.items() if k in ("kind", "role", "tool", "trigger_id", "request", "caused_by", diff --git a/tst/unit/task_step/test_always_on_who_did_what.py b/tst/unit/task_step/test_always_on_who_did_what.py index f9803beb9..53af8dcb1 100644 --- a/tst/unit/task_step/test_always_on_who_did_what.py +++ b/tst/unit/task_step/test_always_on_who_did_what.py @@ -209,3 +209,22 @@ def agent(n, texts, rig): [_, last] = ctx.metadata["post_turn_hooks"]["week"] assert last["delivered"] is False and "user_reads_at" not in last assert _first_reply(ctx.metadata["post_turn_hooks"]) == "never" + + +@pytest.mark.asyncio +async def test_one_read_of_the_changelog_keeps_at_most_its_cap_and_counts_the_rest(monkeypatch, tmp_path): + def agent(n, texts, rig): + for i in range(ao.CHANGES_KEPT_PER_READ + 50): + rig.app_write("rows", f"sync {i}", trigger_id="mirror") + rig.quiet() + return "" + + rig = Rig(monkeypatch, tmp_path, agent=agent) + await rig.arm() + rig.keep_changelog() + ctx = await rig.run(rig.step()) + + [event] = rig.events(ctx, "app_changes") + assert event["count"] == ao.CHANGES_KEPT_PER_READ + 50 and event["not_kept"] == 50 + assert len(event["changes"]) == ao.CHANGES_KEPT_PER_READ + assert rig.record(ctx)["counts"]["app_changes"]["by_others"] == ao.CHANGES_KEPT_PER_READ + 50 From 3a9cca340bed9097873259fb9e8a58fc9b317b65 Mon Sep 17 00:00:00 2001 From: Sarah Chitty Date: Mon, 9 Nov 2026 10:29:00 -0800 Subject: [PATCH 39/53] always_on: a user sim read keeps the plug-in's retried and first_answer, and a retried read counts as malformed The user sim plug-in retries once after a malformed answer. The read's timeline entry now saves retried and first_answer (stored with the full timeline, left out of the summary like raw), and user_sim_malformed_answers counts a read whose first answer was malformed even when the retry came back well formed. Co-Authored-By: Claude Opus 5.5 --- .../task_step/task_steps/always_on.py | 10 ++++++---- tst/unit/task_step/test_always_on_record.py | 18 ++++++++++++++++++ 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/src/agent_env/task_step/task_steps/always_on.py b/src/agent_env/task_step/task_steps/always_on.py index 6d2a82f04..10fa4bb33 100644 --- a/src/agent_env/task_step/task_steps/always_on.py +++ b/src/agent_env/task_step/task_steps/always_on.py @@ -179,7 +179,7 @@ JUMPS_KEPT_PER_GAP = 20 PENDING_KEPT = 20 PENDING_TEXT_CHARS = 300 -BULKY_EVENT_FIELDS = ("input", "raw", "read", "parts_text", "changes") +BULKY_EVENT_FIELDS = ("input", "raw", "first_answer", "read", "parts_text", "changes") # What the prompt response keeps of each wake's parts (source_agent_per_turn_prompt_parts): each text part cut to # TURN_TEXT_CHARS, and past TURN_TEXT_TOTAL_CHARS in all, a note for the rest; every wake's full text is in the stored # timeline (its wake event's parts_text). @@ -1437,7 +1437,8 @@ async def ask(text: str) -> dict: user_sim_error=(calls[-1].get("error") if calls else None), user_sim_trajectory_urls=[c.get("trajectory_url") for c in calls], user_sim_turns=ans.get("turns", len(calls)), shown=ans.get("shown"), - past_turn_cap=ans.get("past_turn_cap"), last_read=last) + past_turn_cap=ans.get("past_turn_cap"), last_read=last, + retried=ans.get("retried"), first_answer=ans.get("first_answer")) outcome = ans.get("outcome") said = (ans.get("message") or "").strip() hook = {"hook": HOOK_NAME, "turn": self.wakes, "read_at": _iso(now), @@ -1602,9 +1603,10 @@ async def finish(self) -> None: "clock_jumps": self.clock_jumps, "silent_wakes": sum(1 for e in self.timeline if e["event"] == "agent_turn_end" and not e.get("messages_to_user")), - # reads whose answer didn't match the user sim's answer format (the plug-in's schema_errors) + # reads whose answer didn't match the user sim's answer format (the plug-in's schema_errors), counting a + # read the plug-in retried after a malformed first answer even when the retry came back well formed "user_sim_malformed_answers": sum(1 for e in self.timeline if e["event"] == "user_sim_read" - and e.get("schema_errors")), + and (e.get("schema_errors") or e.get("retried"))), } if self.change_cursors: counts["app_changes"] = dict(self.change_counts) diff --git a/tst/unit/task_step/test_always_on_record.py b/tst/unit/task_step/test_always_on_record.py index 7f2b636a6..f4396a392 100644 --- a/tst/unit/task_step/test_always_on_record.py +++ b/tst/unit/task_step/test_always_on_record.py @@ -147,3 +147,21 @@ def agent(n, texts, rig): ctx = await rig.run(rig.step()) assert rig.record(ctx)["counts"]["user_sim_malformed_answers"] == 1 + + +@pytest.mark.asyncio +async def test_a_read_retried_after_a_malformed_first_answer_keeps_both_and_is_counted(monkeypatch, tmp_path): + def agent(n, texts, rig): + rig.message("question?") if n == 1 else rig.quiet() + return "" + + first = {"outcome": "reply", "message": "{oops", "schema_errors": ["no JSON object in the answer"]} + rig = Rig(monkeypatch, tmp_path, agent=agent, + answers=[{"outcome": "no_reply", "message": "", "schema_errors": [], "retried": True, + "first_answer": first, "turns": 2}]) + await rig.arm() + ctx = await rig.run(rig.step()) + + read = next(e for e in rig.timeline(ctx) if e["event"] == "user_sim_read") + assert read["retried"] is True and read["first_answer"] == first and read["user_sim_turns"] == 2 + assert rig.record(ctx)["counts"]["user_sim_malformed_answers"] == 1 From 44acf80a8e67487acf38ae390a669ce94e96f48f Mon Sep 17 00:00:00 2001 From: Sarah Chitty Date: Mon, 9 Nov 2026 10:52:55 -0800 Subject: [PATCH 40/53] always_on: an agent turn that comes back canceled or rejected ends the run with error_type, as a failed one does send_and_wait returns any terminal state, and only failed was treated as an error, so a canceled or rejected turn (a wake or a nudge) went on as normal and graders scored the run. Any state but completed now ends the run (agent_turn_failed) and sets error_type (agent_turn_canceled / agent_turn_rejected unless the agent gave its own). Co-Authored-By: Claude Opus 5.5 --- .../task_step/task_steps/always_on.py | 32 +++++++++++++------ tst/unit/task_step/always_on_harness.py | 10 +++--- tst/unit/task_step/test_always_on_graded.py | 17 +++++++++- 3 files changed, 44 insertions(+), 15 deletions(-) diff --git a/src/agent_env/task_step/task_steps/always_on.py b/src/agent_env/task_step/task_steps/always_on.py index 10fa4bb33..7659baa10 100644 --- a/src/agent_env/task_step/task_steps/always_on.py +++ b/src/agent_env/task_step/task_steps/always_on.py @@ -46,8 +46,8 @@ time, never the agent's closing text; messages no turn took (a turn that timed out after sending) are taken at the end and included, and listed as never read. The conversation (``metadata["a2a_conversations"][step_id]``) holds the user's own messages and the agent's message_user texts, then the user's last word that never reached the agent and an end - marker with the end reason. A failed or timed-out agent turn sets the response's ``error_type`` (as prompt_agent - does), so verifiers skip the run. + marker with the end reason. An agent turn that doesn't complete (failed, canceled, rejected, timed out) ends the run + and sets the response's ``error_type`` (as prompt_agent does for a failed one), so verifiers skip the run. - Who did what, and in what order, even under a frozen clock: each app change (urn:agentenv:changes/v1) is recorded with who made it and the wake it was seen in, the agent's own with the wake that made it (``app_changes`` events); each reply records the agent's message it answers (``reply_to``: its wake, time and text); and each read leaves a @@ -1229,9 +1229,10 @@ def record() -> None: tool_calls=term.tool_call_count, plain_reply=plain, messages_to_user=len(messages), trajectory_url=traj, **extra) self._sent(messages) - if state == TaskState.failed or str(state) == "failed": - self._failed(term.error_type or "agent_turn_failed", - term.error_message or term.response_text or "agent turn failed", term.error_code) + if not _completed(state): # failed, canceled or rejected: the turn didn't run to its end + self._failed(term.error_type or f"agent_turn_{_state_name(state)}", + term.error_message or term.response_text or f"agent turn {_state_name(state)}", + term.error_code) return if step.nudge and not messages and not quiet and _closing_text(plain): await self._nudge(plain) @@ -1306,9 +1307,10 @@ def record() -> None: plain_reply_after=plain, tool_calls=term.tool_call_count, real_seconds=round(time.time() - t, 1), trajectory_url=traj) self._sent(messages, after_nudge=True) - if state == TaskState.failed or str(state) == "failed": - self._failed(term.error_type or "agent_turn_failed", - term.error_message or term.response_text or "agent nudge turn failed", term.error_code) + if not _completed(state): + self._failed(term.error_type or f"agent_turn_{_state_name(state)}", + term.error_message or term.response_text or f"agent nudge turn {_state_name(state)}", + term.error_code) # ---- app changes ------------------------------------------------------------------------------------------------- async def _start_changes(self) -> None: @@ -1391,8 +1393,9 @@ def _sent(self, messages: list[dict], after_nudge: bool = False, leftover: bool "wake": self.wakes}}}) def _failed(self, error_type: str, error: str, error_code: Optional[str] = None) -> None: - """An agent turn failed or timed out: the run ends, and its prompt response carries ``error_type`` the way - prompt_agent's does, so verifiers skip it instead of grading it as a normal run.""" + """An agent turn didn't complete (failed, canceled, rejected or timed out): the run ends, and its prompt + response carries ``error_type`` the way prompt_agent's does, so verifiers skip it instead of grading it as a + normal run.""" self.error, self.error_type, self.error_code = error, error_type, error_code self.end_reason = "agent_turn_timeout" if error_type == "agent_turn_timeout" else "agent_turn_failed" @@ -1785,6 +1788,15 @@ def _clock_time(at: datetime) -> str: return at.strftime("%I:%M %p").lstrip("0") +def _state_name(state: Any) -> str: + """An A2A task state's name ("completed", "failed", "canceled", "rejected"), from the enum or its string.""" + return str(getattr(state, "value", state)) + + +def _completed(state: Any) -> bool: + return state == TaskState.completed or _state_name(state) == "completed" + + def _closing_text(plain: str) -> str: text = plain.strip() return "" if text in EMPTY_REPLIES else text diff --git a/tst/unit/task_step/always_on_harness.py b/tst/unit/task_step/always_on_harness.py index e32d49e7f..d460a97ea 100644 --- a/tst/unit/task_step/always_on_harness.py +++ b/tst/unit/task_step/always_on_harness.py @@ -36,10 +36,12 @@ class Fail: - """What an agent's turn returns to end in the failed state, as a harness reports a turn that broke.""" + """What an agent's turn returns to end in the failed state, as a harness reports a turn that broke; or in another + state that isn't completed (``state``: canceled, rejected).""" - def __init__(self, message: str = "the CLI crashed", error_type: Optional[str] = None) -> None: - self.message, self.error_type = message, error_type + def __init__(self, message: str = "the CLI crashed", error_type: Optional[str] = None, + state: str = "failed") -> None: + self.message, self.error_type, self.state = message, error_type, state class Rig: @@ -264,7 +266,7 @@ async def _send_and_wait(self, url, parts, *, before_send=None, **kw): if isinstance(closing, Fail): data = {"error_message": closing.message, **({"error_type": closing.error_type} if closing.error_type else {})} - return f"task-{len(self.sent)}", {"status": {"state": "failed", "message": {"parts": [ + return f"task-{len(self.sent)}", {"status": {"state": closing.state, "message": {"parts": [ {"kind": "text", "text": closing.message}, {"kind": "data", "data": data}]}}} reply = [{"kind": "text", "text": closing}] if closing else [] return f"task-{len(self.sent)}", {"status": {"state": "completed", "message": {"parts": reply}}} diff --git a/tst/unit/task_step/test_always_on_graded.py b/tst/unit/task_step/test_always_on_graded.py index 089469b6d..12df4ab88 100644 --- a/tst/unit/task_step/test_always_on_graded.py +++ b/tst/unit/task_step/test_always_on_graded.py @@ -174,7 +174,9 @@ def chatty(n, texts, rig): (lambda rig: Fail("the CLI crashed"), "agent_turn_failed", "agent_turn_failed"), (lambda rig: Fail("sandbox gone", error_type="infra_error"), "infra_error", "agent_turn_failed"), (lambda rig: (_ for _ in ()).throw(TimeoutError("no answer")), "agent_turn_timeout", "agent_turn_timeout"), -], ids=["failed", "failed with the agent's error type", "timed out"]) + (lambda rig: Fail("stopped", state="canceled"), "agent_turn_canceled", "agent_turn_failed"), + (lambda rig: Fail("won't do that", state="rejected"), "agent_turn_rejected", "agent_turn_failed"), +], ids=["failed", "failed with the agent's error type", "timed out", "canceled", "rejected"]) @pytest.mark.asyncio async def test_a_failed_or_timed_out_turn_sets_error_type_and_the_verifier_skips_the_run(monkeypatch, tmp_path, play, error_type, end_reason): @@ -190,3 +192,16 @@ async def test_a_failed_or_timed_out_turn_sets_error_type_and_the_verifier_skips criteria=[{"type": "response_contains", "needles": ["on it"]}]) await verifier.execute(ctx) assert ctx.metadata["verifications"]["words"]["results"][0]["id"] == "prompt_error" + + +@pytest.mark.parametrize("state", ["failed", "canceled", "rejected"]) +@pytest.mark.asyncio +async def test_a_nudge_turn_that_doesn_t_complete_sets_error_type_too(monkeypatch, tmp_path, state): + rig = Rig(monkeypatch, tmp_path, agent=_agent(t1=lambda rig: "PRIVATE all done", t2=lambda rig: Fail(state=state))) + await rig.arm() + ctx = await rig.run(rig.step()) + + [nudge] = rig.events(ctx, "nudge") + assert nudge["state"] == state and len(rig.sent) == 2 + [response] = ctx.prompt_responses + assert (response.error_type, rig.record(ctx)["end_reason"]) == (f"agent_turn_{state}", "agent_turn_failed") From 85c95bae76248e5eba0cfbb842154e28863d2246 Mon Sep 17 00:00:00 2001 From: Sarah Chitty Date: Mon, 9 Nov 2026 10:54:34 -0800 Subject: [PATCH 41/53] always_on: a turn that ends past end_time is followed by the final check-in, not more ordinary wakes The clock runs during the agent's turns, so a turn can end past end_time. The loop only looked at end_time once nothing was due, so a reply or alert due then started an ordinary wake, and a back-and-forth could use up max_wakes without the final check-in ever being sent. Now, once the task's time is at or past end_time, the next wake is the final check-in, carrying whatever came due; a read that came due before end_time still goes first. Co-Authored-By: Claude Opus 5.5 --- .../task_step/task_steps/always_on.py | 19 +++++++++++---- tst/unit/task_step/test_always_on_endings.py | 24 +++++++++++++++++++ 2 files changed, 38 insertions(+), 5 deletions(-) diff --git a/src/agent_env/task_step/task_steps/always_on.py b/src/agent_env/task_step/task_steps/always_on.py index 7659baa10..59362f9e0 100644 --- a/src/agent_env/task_step/task_steps/always_on.py +++ b/src/agent_env/task_step/task_steps/always_on.py @@ -31,9 +31,11 @@ ``scheduled_check_min_interval`` apart; ``scheduled_checks`` turns them on or off whatever the run's length. A check's wake counts toward ``max_wakes``. - At end_time there is always a final check-in: the user's ``final`` line, or the step's own (``FINAL_DEFAULT_TEXT``) - when the task sets none. The run's last wake, at end_time or the one ``max_wakes`` makes the last, carries the - last-turn notice (``final_reminder``). After it the user sim reads what the agent sent (its wrap-up), and the - user's reaction, and any reply still held, are recorded; then the run ends. + when the task sets none. An agent's turn that ends past end_time is followed by it, with whatever came due (a read + due before end_time goes first), never by an ordinary wake. The run's last wake, at end_time or the one + ``max_wakes`` makes the last, carries the last-turn notice (``final_reminder``). After it the user sim reads what + the agent sent (its wrap-up), and the user's reaction, and any reply still held, are recorded; then the run + ends. - A real-time budget keeps the run inside the hub's 8-hour job (``effective_time_budget`` gives the rule): once one more wake might end past ``time_budget_seconds`` (7 h, lowered to fit the task's own timeouts), the agent gets its closing wake, with the step's wrap-up line and the last-turn notice, and the run ends with end reason time_budget. @@ -976,10 +978,17 @@ async def _next(self) -> bool: time_budget_seconds=self.time_budget, wake_reserve_seconds=step.wake_reserve_seconds()) await self._final(now, reason="time_budget") return False - # 1. a read that is due: the user sim reads what the agent sent - if self.read_at is not None and self.read_at <= now and self.unread: + # 1. a read that is due: the user sim reads what the agent sent (one due only at or past end_time is the last + # read, after the final check-in) + if self.read_at is not None and self.read_at <= now and self.unread \ + and (self.end is None or self.read_at < self.end): await self.read(now) return True + # past end_time (the clock runs during the agent's turns, so one can end past it): the final check-in next, with + # whatever else is due, rather than ordinary wakes that would use up max_wakes without one + if self.end is not None and now >= self.end: + await self._final(now) + return False # 2. anything due for the agent: notifications, scripted user lines, held replies; and what else comes # due in the same minute, so it all arrives in one wake if any(h["deliver_at"] <= now for h in self.held) or await self._due_count(self.recipient, now) \ diff --git a/tst/unit/task_step/test_always_on_endings.py b/tst/unit/task_step/test_always_on_endings.py index 45f145103..4fabd5507 100644 --- a/tst/unit/task_step/test_always_on_endings.py +++ b/tst/unit/task_step/test_always_on_endings.py @@ -57,6 +57,30 @@ async def test_without_a_final_line_the_end_time_still_brings_a_check_in_with_th assert rig.record(ctx)["end_reason"] == "end_time" +@pytest.mark.asyncio +async def test_a_turn_that_runs_past_end_time_is_followed_by_the_final_check_in_not_more_wakes(monkeypatch, tmp_path): + """The clock runs during the agent's turns, so one can end past end_time: what is due then (her reply, an alert) + comes with the final check-in, rather than in ordinary wakes that use up max_wakes without one.""" + def chatty(n, texts, rig): + rig.message(f"update {n}: anything else?") + if n == 1: + rig.alert("Live Oak: we can come Friday", channel="Gmail · Live Oak Arbor") + rig.gw._clock.advance("2026-11-10T15:00:00Z") # a long turn: the task's clock runs past end_time + return "sent" + + rig = Rig(monkeypatch, tmp_path, agent=chatty, + answers=[{"outcome": "reply", "message": f"reply {i}"} for i in range(1, 6)]) + await rig.arm() + ctx = await rig.run(rig.step(end_time="P1D", max_wakes=4)) + + assert len(rig.sent) == 2 # the opening wake, then the final check-in + final = " ".join(rig.sent[1]) + assert DEFAULT_FINAL in final and NOTICE in final + assert "Linda: reply 1" in final and "Live Oak: we can come Friday" in final + assert rig.events(ctx, "wake")[-1]["virtual_time"] == "2026-11-10T15:00:00Z" + assert rig.record(ctx)["end_reason"] == "end_time" + + @pytest.mark.asyncio async def test_a_task_s_final_line_is_hers_and_comes_with_the_notice(monkeypatch, tmp_path): rig = Rig(monkeypatch, tmp_path, agent=_recaps) From 9e38c6fc3ae309b0e5e06a843db86dc1c6672859 Mon Sep 17 00:00:00 2001 From: Sarah Chitty Date: Mon, 9 Nov 2026 10:54:49 -0800 Subject: [PATCH 42/53] always_on: the last read is no earlier than the task's time when the run ended _last_read read at the time the user would first have looked at the earliest unread message. A last turn that ran on with the clock running sent later messages after that time, so the read showed them before they were sent, and a reply could be dated before the question it answers. Now a read time earlier than the task's time moves to the first time she is free from then. Co-Authored-By: Claude Opus 5.5 --- .../task_step/task_steps/always_on.py | 9 +++-- tst/unit/task_step/test_always_on_endings.py | 40 +++++++++++++++++++ 2 files changed, 46 insertions(+), 3 deletions(-) diff --git a/src/agent_env/task_step/task_steps/always_on.py b/src/agent_env/task_step/task_steps/always_on.py index 59362f9e0..01392248e 100644 --- a/src/agent_env/task_step/task_steps/always_on.py +++ b/src/agent_env/task_step/task_steps/always_on.py @@ -34,8 +34,8 @@ when the task sets none. An agent's turn that ends past end_time is followed by it, with whatever came due (a read due before end_time goes first), never by an ordinary wake. The run's last wake, at end_time or the one ``max_wakes`` makes the last, carries the last-turn notice (``final_reminder``). After it the user sim reads what - the agent sent (its wrap-up), and the user's reaction, and any reply still held, are recorded; then the run - ends. + the agent sent (its wrap-up), no earlier than the task's time when the run ended, and the user's reaction, and any + reply still held, are recorded; then the run ends. - A real-time budget keeps the run inside the hub's 8-hour job (``effective_time_budget`` gives the rule): once one more wake might end past ``time_budget_seconds`` (7 h, lowered to fit the task's own timeouts), the agent gets its closing wake, with the step's wrap-up line and the last-turn notice, and the run ends with end reason time_budget. @@ -1064,7 +1064,10 @@ async def _last_read(self) -> None: if not self.unread or self.signed_off or self.gave_up or self.end_reason in ERROR_ENDS: return await self._schedule_read() - at = self.read_at or await self.now() + now = await self.now() + at = self.read_at or now + if at < now: # she'd have looked during the last turn, which went on sending: she reads it all, from now + at, self.read_mode = self._available(now) await self.read(at, last=True) def _prompt_parts(self) -> list[dict]: diff --git a/tst/unit/task_step/test_always_on_endings.py b/tst/unit/task_step/test_always_on_endings.py index 4fabd5507..1897d6364 100644 --- a/tst/unit/task_step/test_always_on_endings.py +++ b/tst/unit/task_step/test_always_on_endings.py @@ -132,6 +132,46 @@ async def test_the_user_sim_reads_the_wrap_up_and_her_reaction_and_held_reply_ar assert rig.record(ctx)["end_reason"] == "end_time" and len(rig.sent) == 2 +@pytest.mark.asyncio +async def test_the_last_read_comes_after_every_message_of_a_last_turn_that_ran_on(monkeypatch, tmp_path): + """Her last read is no earlier than the task's time when the run ended, so it never shows a message before it was + sent, nor holds a reply that comes before the question it answers.""" + def wraps_up_slowly(n, texts, rig): + rig.message("booked the house") + rig.gw._clock.advance("2026-11-09T16:00:00Z") # the turn goes on two hours, the clock running + rig.message("and the flights: is Friday ok?") + return "sent" + + rig = Rig(monkeypatch, tmp_path, agent=wraps_up_slowly, + answers=[{"outcome": "reply", "message": "friday is fine"}]) + await rig.arm() + ctx = await rig.run(rig.step(max_wakes=1)) + + [read] = rig.reads + assert [m["at"] for m in read["assistant_messages"]] == ["2026-11-09T14:00:00Z", "2026-11-09T16:00:00Z"] + assert read["time"] == "2026-11-09T16:00:00Z" + [event] = rig.events(ctx, "user_sim_read") + assert event["last_read"] is True and event["deliver_at"] == "2026-11-09T16:00:00Z" + + +@pytest.mark.asyncio +async def test_a_last_read_that_would_fall_in_her_night_waits_for_the_morning(monkeypatch, tmp_path): + def wraps_up_late(n, texts, rig): + rig.message("booked the house") + rig.gw._clock.advance("2026-11-10T05:00:00Z") # midnight in New York: past her bedtime + rig.message("and the flights: is Friday ok?") + return "sent" + + rig = Rig(monkeypatch, tmp_path, agent=wraps_up_late, answers=[{"outcome": "reply", "message": "friday is fine"}]) + await rig.arm() + ctx = await rig.run(rig.step(max_wakes=1), rig.context(user=DAY)) + + [read] = rig.reads + assert read["time"] == "2026-11-10T12:30:00Z" # 7:30 AM, when she wakes + [event] = rig.events(ctx, "user_sim_read") + assert (event["mode"], event["deliver_at"]) == ("morning", "2026-11-10T12:45:00Z") + + @pytest.mark.asyncio async def test_a_done_on_the_last_read_is_recorded_and_changes_nothing(monkeypatch, tmp_path): rig = Rig(monkeypatch, tmp_path, agent=_recaps, answers=[{"outcome": "done", "message": "thanks!!"}]) From 85555b86d106fc2f06a74e70a18a5c2fd709aaa1 Mon Sep 17 00:00:00 2001 From: Sarah Chitty Date: Mon, 9 Nov 2026 10:55:17 -0800 Subject: [PATCH 43/53] gateway: turning the user chat off or clearing it leaves a backing server's stay_quiet Registering stay_quiet already skips a name a backing server owns, but removing the chat tools deleted stay_quiet from the tool manager whoever owned it, so a backing server's tool of that name was gone for the rest of the task. Removal now leaves it, as it does message_user and get_time. Co-Authored-By: Claude Opus 5.5 --- src/agent_env/env/gateway/gateway.py | 4 +++- tst/unit/env/gateway/user_chat_test.py | 14 ++++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/src/agent_env/env/gateway/gateway.py b/src/agent_env/env/gateway/gateway.py index b91e07a66..35da65c15 100644 --- a/src/agent_env/env/gateway/gateway.py +++ b/src/agent_env/env/gateway/gateway.py @@ -1340,11 +1340,13 @@ def _sync_stay_quiet_tool(self) -> None: self._server_tools.pop("gateway", None) def _unregister_user_chat_tool(self) -> None: + """Remove message_user and stay_quiet when the chat is turned off or cleared; a backing server's are left.""" tools = self._mcp._tool_manager._tools if MESSAGE_USER_TOOL_NAME in self._tool_server_urls: return tools.pop(MESSAGE_USER_TOOL_NAME, None) - tools.pop(STAY_QUIET_TOOL_NAME, None) + if STAY_QUIET_TOOL_NAME not in self._tool_server_urls: + tools.pop(STAY_QUIET_TOOL_NAME, None) remaining = [t for t in self._server_tools.get("gateway", []) if t.name not in (MESSAGE_USER_TOOL_NAME, STAY_QUIET_TOOL_NAME)] if remaining: diff --git a/tst/unit/env/gateway/user_chat_test.py b/tst/unit/env/gateway/user_chat_test.py index 3efd91651..913e7cb66 100644 --- a/tst/unit/env/gateway/user_chat_test.py +++ b/tst/unit/env/gateway/user_chat_test.py @@ -104,6 +104,20 @@ def test_a_new_config_or_clear_removes_stay_quiet(gateway): assert not {"stay_quiet", "message_user"} & _names(gateway, "psi_agent") +def test_turning_the_chat_off_or_clearing_it_leaves_a_backing_server_s_stay_quiet(gateway): + gateway._tool_server_urls["stay_quiet"] = "http://backing" + gateway._create_and_register_proxy_tool("http://backing", MCPTool( + name="stay_quiet", description="backing", inputSchema={"type": "object", "properties": {}})) + tools = gateway._mcp._tool_manager._tools + + _req(gateway, "POST", "/user_chat/config", {"roles": ["psi_agent"], "stay_quiet": True}) + _req(gateway, "POST", "/user_chat/config", {"roles": []}) + assert tools["stay_quiet"].description == "backing" and "message_user" not in tools + _req(gateway, "POST", "/user_chat/config", {"roles": ["psi_agent"]}) + _req(gateway, "POST", "/user_chat/clear", {}) + assert tools["stay_quiet"].description == "backing" and "message_user" not in tools + + @pytest.mark.parametrize("body", [{"roles": ["psi_agent"], "stay_quiet": "yes"}, {"roles": "psi_agent"}, {"roles": ["psi_agent"], "colour": "red"}]) def test_a_bad_config_is_refused(gateway, body): From 87e71203a4f6f2a6fb263a163cf48594b31c6b97 Mon Sep 17 00:00:00 2001 From: Sarah Chitty Date: Mon, 9 Nov 2026 10:58:31 -0800 Subject: [PATCH 44/53] always_on: take only the agent's role's messages, and set aside what it sent before the run The user chat's take handed out every pending message, whoever sent it, so a message another role queued before the run was taken on the first wake as this agent's and went to the user sim and the graded chat. The take now accepts `from` (a role; without it, everything, as before) and leaves other roles' messages and stay_quiet calls queued. The step takes with its agent's role, and before the first wake sets aside what that role left before the run, recorded in the timeline as user_chat_before_run, so the first wake doesn't count it. Co-Authored-By: Claude Opus 5.5 --- src/agent_env/env/gateway/constants.py | 2 +- src/agent_env/env/gateway/user_chat.py | 11 ++++++--- .../task_step/task_steps/always_on.py | 12 ++++++++-- tst/unit/env/gateway/user_chat_test.py | 19 +++++++++++++++ tst/unit/task_step/test_always_on_budget.py | 6 ++--- tst/unit/task_step/test_always_on_graded.py | 24 ++++++++++++++++++- 6 files changed, 64 insertions(+), 10 deletions(-) diff --git a/src/agent_env/env/gateway/constants.py b/src/agent_env/env/gateway/constants.py index e542bbbe5..fff131f7b 100644 --- a/src/agent_env/env/gateway/constants.py +++ b/src/agent_env/env/gateway/constants.py @@ -284,7 +284,7 @@ def random_mcp_server_name() -> str: "stay_quiet": {"type": "boolean"}}}}, "state": {"method": "GET", "endpoint": "/user_chat/state", "request": {"type": "object", "properties": {}}}, "take": {"method": "POST", "endpoint": "/user_chat/take", "request": {"type": "object", "properties": { - "take_id": {"type": "string"}}}}, + "take_id": {"type": "string"}, "from": {"type": "string"}}}}, "clear": {"method": "POST", "endpoint": "/user_chat/clear", "request": {"type": "object", "properties": {}}}}}, "required": False, }, diff --git a/src/agent_env/env/gateway/user_chat.py b/src/agent_env/env/gateway/user_chat.py index b1294a647..b071c072d 100644 --- a/src/agent_env/env/gateway/user_chat.py +++ b/src/agent_env/env/gateway/user_chat.py @@ -7,7 +7,8 @@ modify_env_tool_access don't hide them. `take` hands each message out once: {"id", "from", "at", "text"}, plus, with stay_quiet on, the `quiet` calls since the -last take. A `take_id` makes a retry safe. +last take. A `take_id` makes a retry safe. With `from` (a role) it takes only that role's messages and quiet calls, and +leaves the others queued. """ from __future__ import annotations @@ -156,16 +157,20 @@ def take(self, body: Any = None) -> dict: take_id = body.get("take_id") if take_id is not None and (not isinstance(take_id, str) or not take_id or len(take_id) > _MAX_TAKE_ID_CHARS): raise UserChatError(f"take_id must be a non-empty string of at most {_MAX_TAKE_ID_CHARS} characters") + sender = body.get("from") + if sender is not None and (not isinstance(sender, str) or not sender.strip()): + raise UserChatError("from must be a role name") with self._lock: if take_id is not None and take_id in self._takes: return self._takes[take_id] - items = [m for m in self._queue if m["status"] == "pending"] + items = [m for m in self._queue if m["status"] == "pending" and sender in (None, m["from"])] for m in items: m["status"] = "taken" self._queue = [m for m in self._queue if m["status"] == "pending"] result = {"items": [_public(m) for m in items]} if self._stay_quiet: - result["quiet"], self._quiet = self._quiet, [] + result["quiet"] = [q for q in self._quiet if sender in (None, q["from"])] + self._quiet = [q for q in self._quiet if sender not in (None, q["from"])] if take_id is not None: self._takes[take_id] = result while len(self._takes) > _KEPT_TAKES: diff --git a/src/agent_env/task_step/task_steps/always_on.py b/src/agent_env/task_step/task_steps/always_on.py index 01392248e..13e4f451a 100644 --- a/src/agent_env/task_step/task_steps/always_on.py +++ b/src/agent_env/task_step/task_steps/always_on.py @@ -11,6 +11,8 @@ - Only what the agent sends with the gateway's ``message_user`` tool (user-chat/v1) reaches the user; its plain reply goes to the log. Each wake should end with ``message_user`` or ``stay_quiet``: one that ends with closing text and neither gets one ``nudge`` turn, and every wake ends with a reminder. All three are on unless the task turns them off. + The step takes only its agent's role's messages: another role's stay queued, and what the role sent before the run + is recorded apart (``user_chat_before_run``), never read by the user. - The user sim is asked only after the agent sent something, at the time the user would read it (``schedule``), with everything sent meanwhile in one read: the time, the agent's messages, what triggers sent the user's phone (``notify`` to ``user_phone``) and the user's own lines (the opening request, ``user_says`` lines, ``final``). @@ -491,6 +493,11 @@ async def go(self) -> None: agent=step.agent_name, agent_role=self.recipient, user_sim=step.user_sim, version=self.version, user_model=self.user_model, max_wakes=step.max_wakes, time_zone=self.zone, multi_day=self.multi_day, scheduled_checks=self.checks, clock_rate=await self._rate()) + # What the agent's role sent before the run (a task that gave it message_user earlier) isn't this run's: set it + # aside, recorded, so the first wake doesn't take it as its own. + before, quiet = await self._take_messages() + if before or quiet: + self.log("user_chat_before_run", messages=before, stay_quiet_calls=len(quiet)) try: await self.loop(now) except Exception as e: @@ -1382,10 +1389,11 @@ def _change(self, env_id: str, row: dict, phase: str) -> dict: "at": row.get("timestamp"), "by": by, "wake": self.wakes, "phase": phase, "actors": actors} async def _take_messages(self) -> tuple[list[dict], list[dict]]: - """What the agent sent the user with message_user this turn, oldest first, and its stay_quiet calls.""" + """What the agent sent the user with message_user this turn, oldest first, and its stay_quiet calls: its role's + only, so another role's stay queued for whoever takes them.""" messages, quiet = [], [] for env in self.chat_envs: - got = await _call(env, EXT_USER_CHAT_URI, "take", {"take_id": uuid.uuid4().hex}) + got = await _call(env, EXT_USER_CHAT_URI, "take", {"take_id": uuid.uuid4().hex, "from": self.recipient}) messages += got.get("items") or [] quiet += got.get("quiet") or [] messages.sort(key=lambda m: m.get("at") or "") diff --git a/tst/unit/env/gateway/user_chat_test.py b/tst/unit/env/gateway/user_chat_test.py index 913e7cb66..c286e9021 100644 --- a/tst/unit/env/gateway/user_chat_test.py +++ b/tst/unit/env/gateway/user_chat_test.py @@ -95,6 +95,25 @@ def test_stay_quiet_sends_nothing_and_is_recorded_for_the_next_take(gateway): assert "Tess" in desc +def test_a_take_for_one_role_leaves_the_others_messages_and_quiet_calls_queued(gateway): + _req(gateway, "POST", "/user_chat/config", {"roles": ["psi_agent", "other"], "stay_quiet": True}) + _call(gateway, "message_user", {"text": "from psi"}, "psi_agent") + _call(gateway, "message_user", {"text": "from other"}, "other") + _call(gateway, "stay_quiet", {}, "other") + + taken = _req(gateway, "POST", "/user_chat/take", {"from": "psi_agent"}).json() + assert [(m["from"], m["text"]) for m in taken["items"]] == [("psi_agent", "from psi")] and taken["quiet"] == [] + assert [m["text"] for m in _req(gateway, "GET", "/user_chat/state").json()["pending"]] == ["from other"] + rest = _req(gateway, "POST", "/user_chat/take", {}).json() + assert [(m["from"], m["text"]) for m in rest["items"]] == [("other", "from other")] + assert [q["from"] for q in rest["quiet"]] == ["other"] + + +@pytest.mark.parametrize("who", ["", 3]) +def test_a_take_from_no_role_is_refused(gateway, who): + assert _req(gateway, "POST", "/user_chat/take", {"from": who}).status_code == 400 + + def test_a_new_config_or_clear_removes_stay_quiet(gateway): _req(gateway, "POST", "/user_chat/config", {"roles": ["psi_agent"], "stay_quiet": True}) _req(gateway, "POST", "/user_chat/config", {"roles": ["psi_agent"]}) diff --git a/tst/unit/task_step/test_always_on_budget.py b/tst/unit/task_step/test_always_on_budget.py index 9ea887dd0..602792846 100644 --- a/tst/unit/task_step/test_always_on_budget.py +++ b/tst/unit/task_step/test_always_on_budget.py @@ -189,11 +189,11 @@ async def test_a_turn_of_the_loop_past_the_reserve_ends_the_run_on_a_step_error( rig = Rig(monkeypatch, tmp_path) real_take = ao._Run._take_messages - calls: list[int] = [] + hung: list[int] = [] async def hangs_once(run): - calls.append(1) - if len(calls) == 1: + if run.wakes and not hung: # the opening wake's take, not the one before the run + hung.append(1) await asyncio.sleep(60) return await real_take(run) diff --git a/tst/unit/task_step/test_always_on_graded.py b/tst/unit/task_step/test_always_on_graded.py index 12df4ab88..b948a5377 100644 --- a/tst/unit/task_step/test_always_on_graded.py +++ b/tst/unit/task_step/test_always_on_graded.py @@ -11,7 +11,7 @@ AgentPromptResponseVerifierTaskStep, ) from agent_env.task_step.task_steps.verifiers.run_context import load_run_context -from tst.unit.task_step.always_on_harness import Fail, Rig +from tst.unit.task_step.always_on_harness import ROLE, Fail, Rig pytestmark = pytest.mark.usefixtures("local_stores") @@ -168,6 +168,28 @@ def chatty(n, texts, rig): assert "Linda: ok (sent after the assistant's last turn" in last["texts"][0] +@pytest.mark.asyncio +async def test_messages_queued_before_the_run_are_not_the_run_s(monkeypatch, tmp_path): + """Another role's messages stay queued for whoever takes them; what the agent's own role left before the run is + recorded as such; neither reaches her or the graded chat.""" + rig = Rig(monkeypatch, tmp_path, agent=_agent(t1=_message("on it"))) + await rig.arm() + chat = rig.gw._user_chat + chat.configure({"roles": ["other", ROLE], "stay_quiet": True}) + chat.send("from another assistant", role="other") + chat.send("left from an earlier step", role=ROLE) + chat.quiet(role=ROLE) + ctx = await rig.run(rig.step()) + + [response] = ctx.prompt_responses + assert response.response == "[Mon Nov 9, 2:00 PM UTC] on it" + assert [m["text"] for m in rig.reads[0]["assistant_messages"]] == ["on it"] + assert [(m["from"], m["text"]) for m in chat.state()["pending"]] == [("other", "from another assistant")] + [before] = rig.events(ctx, "user_chat_before_run") + assert [m["text"] for m in before["messages"]] == ["left from an earlier step"] + assert before["stay_quiet_calls"] == 1 + + # ---- failed turns are errors ----------------------------------------------------------------------------------------- @pytest.mark.parametrize("play,error_type,end_reason", [ From 086851d051a3ee52cf54d5206d3759e81e36c179 Mon Sep 17 00:00:00 2001 From: Sarah Chitty Date: Mon, 9 Nov 2026 11:01:54 -0800 Subject: [PATCH 45/53] explorer: an always_on wake's closing text is labelled a private closing note, not the final response In an always_on run only message_user texts reach the user; a wake's closing text is the agent's private note, which the user never sees and graders don't read. The trajectory viewer showed it as "Final Response". For a trajectory of an always_on step (its type in the task's steps, or its record under metadata.always_on), the block is now "Closing note (private, not sent to the user)", dashed, with no accent colour and muted text. prompt_agent and every other step look as before. Co-Authored-By: Claude Opus 5.5 --- .../src/components/task-instance-viewer.tsx | 15 +++++++ .../ui/src/components/trajectory-viewer.tsx | 39 +++++++++++++++++-- 2 files changed, 50 insertions(+), 4 deletions(-) diff --git a/src/agent_env/explorer/ui/src/components/task-instance-viewer.tsx b/src/agent_env/explorer/ui/src/components/task-instance-viewer.tsx index 17d435de0..9c3a7a52c 100644 --- a/src/agent_env/explorer/ui/src/components/task-instance-viewer.tsx +++ b/src/agent_env/explorer/ui/src/components/task-instance-viewer.tsx @@ -467,6 +467,18 @@ export function TaskInstanceViewer({ () => parseTriggerRuntime(metadata), [metadata], ); + // always_on steps, by their type or by the run record they write under metadata.always_on[step_id]: a wake's closing + // text there is the agent's private note, never sent to the user, which the trajectory viewer labels as such. + const alwaysOnStepIds = useMemo(() => { + const record = metadata?.always_on; + const ids = new Set( + record && typeof record === 'object' ? Object.keys(record) : [], + ); + for (const s of taskSteps ?? []) { + if (s.type === 'always_on' && s.id) ids.add(s.id); + } + return ids; + }, [metadata, taskSteps]); // Authored trigger configs for the badge popovers. Empty when the surface passes slim step refs — popovers then show runtime history only. const authoredTriggers = useMemo( () => indexAuthoredTriggers(taskSteps), @@ -955,6 +967,9 @@ export function TaskInstanceViewer({ diff --git a/src/agent_env/explorer/ui/src/components/trajectory-viewer.tsx b/src/agent_env/explorer/ui/src/components/trajectory-viewer.tsx index 65e7f0f79..a58c7e59a 100644 --- a/src/agent_env/explorer/ui/src/components/trajectory-viewer.tsx +++ b/src/agent_env/explorer/ui/src/components/trajectory-viewer.tsx @@ -1804,6 +1804,9 @@ interface TrajectoryViewerProps { isInProgress?: boolean; // Object URL of this trajectory; only used to lazy-load screenshot-trimmed frames. screenshotBaseUri?: string; + // An always_on wake: its closing text is the agent's private note, never sent to the user (only message_user texts + // are), so it is labelled as such and shown quieter. + closingNote?: boolean; } export function TrajectoryViewer({ @@ -1811,6 +1814,7 @@ export function TrajectoryViewer({ timelineScrollRef, isInProgress = false, screenshotBaseUri, + closingNote = false, }: TrajectoryViewerProps) { const { model, @@ -2099,16 +2103,37 @@ export function TrajectoryViewer({ })()} {finalResponse && ( -
+
- {isInProgress ? 'Latest Response' : 'Final Response'} + {closingNote ? ( + + Closing note{' '} + + (private, not sent to the user) + + + ) : isInProgress ? ( + 'Latest Response' + ) : ( + 'Final Response' + )} {isInProgress && ( @@ -2116,7 +2141,13 @@ export function TrajectoryViewer({ )}
-
+
From dc9e0980d73595d0960287b640b368b5480bdada Mon Sep 17 00:00:00 2001 From: Sarah Chitty Date: Mon, 9 Nov 2026 11:04:07 -0800 Subject: [PATCH 46/53] always_on: drop the date line that opened every wake The line ("It's {when}. Any other date you see is the computer's clock, not the task's.", source "time") was for Claude Code, which injects the real date. On the neutral harness each alert and each of the user's lines already carries its time (the opening request included), and stress test 13 showed models read those UTC labels correctly without it. get_time stays, through a deny-all too; the rules, wake reminder, stay_quiet, nudge, last-turn notice and final check-in are unchanged. Co-Authored-By: Claude Opus 5.5 --- .../task_step/task_steps/always_on.py | 24 +++---------- tst/unit/task_step/test_always_on_time.py | 36 ++++++------------- 2 files changed, 15 insertions(+), 45 deletions(-) diff --git a/src/agent_env/task_step/task_steps/always_on.py b/src/agent_env/task_step/task_steps/always_on.py index 13e4f451a..fbeaada3f 100644 --- a/src/agent_env/task_step/task_steps/always_on.py +++ b/src/agent_env/task_step/task_steps/always_on.py @@ -5,9 +5,9 @@ it in the run's metadata under its ``name``; a ``deploy_agent`` step deploys the agent that plays them. The ``always_on`` step names it with ``user_sim`` and runs the loop: -- The first wake is the user's request. While the task's clock is set, each wake opens with the task's date and time - (any other date the agent sees is the computer's clock), then what came due for the agent on notifications/v1 (a - trigger's ``notify``, a scheduled check), labelled ``[time · channel]``, then the user's own messages. +- The first wake is the user's request. Each wake opens with what came due for the agent on notifications/v1 (a + trigger's ``notify``, a scheduled check), labelled ``[time · channel]``, then the user's own messages, each labelled + with its time; while the task's clock is set, the agent can read it with the gateway's ``get_time``. - Only what the agent sends with the gateway's ``message_user`` tool (user-chat/v1) reaches the user; its plain reply goes to the log. Each wake should end with ``message_user`` or ``stay_quiet``: one that ends with closing text and neither gets one ``nudge`` turn, and every wake ends with a reminder. All three are on unless the task turns them off. @@ -129,7 +129,6 @@ "schedule_check: you'll be woken then with your note.") CHECK_MIN_INTERVAL = "PT3H" EMPTY_REPLIES = frozenset({"(The assistant's reply was empty.)"}) # a harness's stand-in for an empty final reply -TIME_LINE = "It's {when}. Any other date you see is the computer's clock, not the task's." # The hub runs a step as one job of at most this long, retrying one that runs over; the step keeps this much of it spare. JOB_LIMIT_SECONDS = 8 * 3600 JOB_HEADROOM_SECONDS = 600 @@ -696,14 +695,6 @@ def _system_prompt(self) -> Optional[str]: the agent keeps its own and the first wake opens with the rules.""" return f"{self.step.system_prompt.rstrip()}\n\n{self.rules()}" if self.step.system_prompt else None - def _time_line(self, at: datetime) -> str: - """The task's date and time, and the user's local time when they aren't in UTC.""" - when = f"{at.strftime('%a %b')} {at.day}, {at.year}, {_clock_time(at)} UTC" - if self.zone != "UTC": - local = at.astimezone(self.tz) - when += f" ({local.strftime('%a')} {_clock_time(local)} for {self.person['first']}, {self.zone})" - return TIME_LINE.format(when=when) - # ---- clock and world ------------------------------------------------------------------------------------ async def _armed(self) -> list: armed = [] @@ -1193,11 +1184,8 @@ async def wake(self, parts: list[dict], woke_by: list[str], notifications: Optio ``last`` marks, or the one ``max_wakes`` makes the last, carries the last-turn notice (``final_reminder``).""" step = self.step sent_at = await self.now() - clock = await step._env_clock(self.clock_envs) - lead = [_step_part(self._time_line(sent_at), "time")] if clock else [] if self.wakes == 0 and not self.rules_in_system_prompt: # no system prompt took them: the first wake does - lead = [_step_part(self.rules(), "rules"), *lead] - parts = [*lead, *parts] + parts = [_step_part(self.rules(), "rules"), *parts] last = last or self.wakes + 1 >= step.max_wakes if last and step.final_reminder: parts.append(_step_part(step.final_reminder.format(first=self.person["first"]), "final_reminder")) @@ -1804,10 +1792,6 @@ def _step_part(text: str, source: str) -> dict: return {"kind": "text", "text": text, "metadata": {"agentenv": {"source": source}}} -def _clock_time(at: datetime) -> str: - return at.strftime("%I:%M %p").lstrip("0") - - def _state_name(state: Any) -> str: """An A2A task state's name ("completed", "failed", "canceled", "rejected"), from the enum or its string.""" return str(getattr(state, "value", state)) diff --git a/tst/unit/task_step/test_always_on_time.py b/tst/unit/task_step/test_always_on_time.py index 7d3a49279..c9c0e28d3 100644 --- a/tst/unit/task_step/test_always_on_time.py +++ b/tst/unit/task_step/test_always_on_time.py @@ -1,5 +1,6 @@ -"""The always_on step gives the agent the task's time, not the computer's: each wake opens with the task's date and -time, and says any other date is the computer's clock.""" +"""The always_on step gives the agent the task's time through what it sends, not a date line: each alert and each of +the user's lines is labelled with its time, and get_time reads the task's clock, even when the task hides every tool. +No wake opens with a line of its own about the date.""" from __future__ import annotations @@ -9,38 +10,23 @@ pytestmark = pytest.mark.usefixtures("local_stores") -CLOCK = "Any other date you see is the computer's clock, not the task's." PROMPT = "You are Linda's assistant." @pytest.mark.asyncio -async def test_each_wake_opens_with_the_task_s_date_and_time(monkeypatch, tmp_path): +async def test_wakes_carry_the_task_s_time_on_each_line_and_no_date_line(monkeypatch, tmp_path): rig = Rig(monkeypatch, tmp_path, takes_config=True) # the rules go in its system prompt await rig.arm() rig.triggers(rig.notify_at("sam", "2026-11-10T14:00:00Z", "Sam: shh", channel="noor 30")) ctx = await rig.run(rig.step(system_prompt=PROMPT)) - assert [texts[0] for texts in rig.sent] == [ - f"It's Mon Nov 9, 2026, 2:00 PM UTC (Mon 9:00 AM for Linda, America/New_York). {CLOCK}", - f"It's Tue Nov 10, 2026, 2:00 PM UTC (Tue 9:00 AM for Linda, America/New_York). {CLOCK}", - ] - assert rig.parts[1][0]["metadata"] == {"agentenv": {"source": "time"}} - assert rig.events(ctx, "wake")[1]["parts_text"].startswith("It's Tue Nov 10, 2026") - - -@pytest.mark.asyncio -async def test_a_user_in_utc_gets_the_time_once(monkeypatch, tmp_path): - rig = Rig(monkeypatch, tmp_path, takes_config=True) - await rig.arm() - await rig.run(rig.step(system_prompt=PROMPT), rig.context(user={"time_zone": "UTC"})) - assert rig.sent[0][0] == f"It's Mon Nov 9, 2026, 2:00 PM UTC. {CLOCK}" - - -@pytest.mark.asyncio -async def test_without_a_task_clock_there_is_no_time_line(monkeypatch, tmp_path): - rig = Rig(monkeypatch, tmp_path) - await rig.run(rig.step()) - assert not any(CLOCK in t for t in rig.sent[0]) + assert [texts[0] for texts in rig.sent] == ["[Mon Nov 9, 2:00 PM UTC] Linda: sort the cdmx weekend", + "[Tue Nov 10, 2:00 PM UTC · noor 30] Sam: shh"] + assert rig.parts[0][0]["metadata"]["agentenv"]["opening"] is True + sources = {p.get("metadata", {}).get("agentenv", {}).get("source") for parts in rig.parts for p in parts} + assert "time" not in sources + assert not any("computer's clock" in t for texts in rig.sent for t in texts) + assert rig.events(ctx, "wake")[1]["parts_text"].startswith("[Tue Nov 10, 2:00 PM UTC · noor 30]") @pytest.mark.asyncio From 012f4c459eda44e457bd1c26a934eb648a986e05 Mon Sep 17 00:00:00 2001 From: Sarah Chitty Date: Mon, 9 Nov 2026 12:41:06 -0800 Subject: [PATCH 47/53] always_on: the per-wake reminder is off by default; stay_quiet and the nudge stay on With stay_quiet on and the reminder and nudge off, Opus 5.5 and GPT 6.1 runs of Surprise 30th left no message in closing text (0 of 21 and 0 of 25 wakes). The rule stays in the agent's system prompt; a task can still turn the reminder on with wake_reminder: true. Co-Authored-By: Claude Opus 5.5 --- src/agent_env/task_step/task_steps/always_on.py | 6 ++++-- tst/unit/task_step/test_always_on_endings.py | 2 +- tst/unit/task_step/test_always_on_messages.py | 11 ++++++----- tst/unit/task_step/test_always_on_record.py | 2 +- 4 files changed, 12 insertions(+), 9 deletions(-) diff --git a/src/agent_env/task_step/task_steps/always_on.py b/src/agent_env/task_step/task_steps/always_on.py index fbeaada3f..cd2c419bb 100644 --- a/src/agent_env/task_step/task_steps/always_on.py +++ b/src/agent_env/task_step/task_steps/always_on.py @@ -10,7 +10,9 @@ with its time; while the task's clock is set, the agent can read it with the gateway's ``get_time``. - Only what the agent sends with the gateway's ``message_user`` tool (user-chat/v1) reaches the user; its plain reply goes to the log. Each wake should end with ``message_user`` or ``stay_quiet``: one that ends with closing text and - neither gets one ``nudge`` turn, and every wake ends with a reminder. All three are on unless the task turns them off. + neither gets one ``nudge`` turn. ``stay_quiet`` and the nudge are on unless the task turns them off; the rule itself is + in the agent's system prompt. A reminder at the end of every wake (``wake_reminder``) is off by default: with + ``stay_quiet`` on, no wake in Opus 5.5 and GPT 6.1 runs of Surprise 30th ended in a message left in closing text. The step takes only its agent's role's messages: another role's stay queued, and what the role sent before the run is recorded apart (``user_chat_before_run``), never read by the user. - The user sim is asked only after the agent sent something, at the time the user would read it (``schedule``), with @@ -200,7 +202,7 @@ def __init__(self, id: str, version: Optional[int], *, user_sim: str, end_time: final: Optional[str] = None, max_wakes: int = 60, settle_timeout_seconds: float = 300, user_timeout_seconds: int = 300, final_reminder: Optional[str] = FINAL_REMINDER, stay_quiet: bool = True, nudge: bool = True, nudge_text: str = NUDGE_TEXT, - wake_reminder: bool = True, wake_reminder_text: str = WAKE_REMINDER_TEXT, + wake_reminder: bool = False, wake_reminder_text: str = WAKE_REMINDER_TEXT, scheduled_checks: Optional[bool] = None, scheduled_check_min_interval: str = CHECK_MIN_INTERVAL, contact_timeout_seconds: Optional[float] = CONTACT_TIMEOUT_SECONDS, contact_replies_in_parallel: bool = True, time_budget_seconds: Optional[float] = None, diff --git a/tst/unit/task_step/test_always_on_endings.py b/tst/unit/task_step/test_always_on_endings.py index 1897d6364..229ec56e5 100644 --- a/tst/unit/task_step/test_always_on_endings.py +++ b/tst/unit/task_step/test_always_on_endings.py @@ -89,7 +89,7 @@ async def test_a_task_s_final_line_is_hers_and_comes_with_the_notice(monkeypatch texts = rig.sent[-1] assert "[Wed Nov 11, 2:00 PM UTC] Linda: send me the rundown" in texts and DEFAULT_FINAL not in texts - assert texts[-2:] == [NOTICE, ao.WAKE_REMINDER_TEXT] + assert texts[-1] == NOTICE and ao.WAKE_REMINDER_TEXT not in texts assert [s["text"] for s in rig.reads[-1]["you_sent"]] == ["sort the cdmx weekend", "send me the rundown"] diff --git a/tst/unit/task_step/test_always_on_messages.py b/tst/unit/task_step/test_always_on_messages.py index 8a9ec858a..7f30b0e8a 100644 --- a/tst/unit/task_step/test_always_on_messages.py +++ b/tst/unit/task_step/test_always_on_messages.py @@ -32,23 +32,24 @@ def _says(closing): @pytest.mark.asyncio -async def test_all_three_are_on_by_default(monkeypatch, tmp_path): +async def test_stay_quiet_and_the_nudge_are_on_and_the_wake_reminder_off_by_default(monkeypatch, tmp_path): rig = Rig(monkeypatch, tmp_path, agent=_turns(_quiet())) await rig.arm() step = rig.step() ctx = await rig.run(step) - assert (step.stay_quiet, step.nudge, step.wake_reminder) == (True, True, True) - assert rig.record(ctx)["settings"].items() >= {"stay_quiet": True, "nudge": True, "wake_reminder": True}.items() + assert (step.stay_quiet, step.nudge, step.wake_reminder) == (True, True, False) + assert rig.record(ctx)["settings"].items() >= {"stay_quiet": True, "nudge": True, "wake_reminder": False}.items() + assert not any(ao.WAKE_REMINDER_TEXT in texts for texts in rig.sent) assert rig.during[0]["stay_quiet"] and ROLE in rig.during[0]["user_chat_roles"] @pytest.mark.asyncio -async def test_each_wake_ends_with_the_reminder(monkeypatch, tmp_path): +async def test_a_task_that_turns_the_wake_reminder_on_ends_each_wake_with_it(monkeypatch, tmp_path): rig = Rig(monkeypatch, tmp_path, agent=_turns(_send("two houses, which one?"), _quiet()), answers=[{"outcome": "reply", "message": "the blue one"}]) await rig.arm() - await rig.run(rig.step()) + await rig.run(rig.step(wake_reminder=True)) assert len(rig.sent) == 2 assert all(texts[-1] == ao.WAKE_REMINDER_TEXT for texts in rig.sent) diff --git a/tst/unit/task_step/test_always_on_record.py b/tst/unit/task_step/test_always_on_record.py index f4396a392..69713e174 100644 --- a/tst/unit/task_step/test_always_on_record.py +++ b/tst/unit/task_step/test_always_on_record.py @@ -124,7 +124,7 @@ async def test_the_prompt_response_keeps_each_wake_s_parts_cut_to_size(monkeypat await rig.arm() for day in (10, 11, 12): rig.triggers(rig.notify_at(f"news-{day}", f"2026-11-{day}T14:00:00Z", "y" * 10_000, channel="Gmail")) - ctx = await rig.run(rig.step()) + ctx = await rig.run(rig.step(wake_reminder=True)) # its parts add up past the total with the reminder kept = ctx.prompt_responses[0].source_agent_per_turn_prompt_parts assert len(kept) == 4 From f04fdf218d27a1dc00e0c21c9bfee67461017134 Mon Sep 17 00:00:00 2001 From: Sarah Chitty Date: Mon, 9 Nov 2026 13:19:15 -0800 Subject: [PATCH 48/53] always_on: a failure after the conversation is made still closes it and records the run; name the run's limits The take before the run, the start log and the first clock reads ran after the graded conversation was created but before the try whose finally calls finish(): a gateway that refused the take left the conversation active, with no end record and no prompt response. They now sit inside it. The inline limits get names: ASSISTANT_LIKE_MESSAGE_CHARS (1200), KEPT_NESTING_LEVELS (6), QUOTE_MIN_CHARS (8), LOGGED_TEXT_CHARS (200) and CALL_RETRY_MAX_SLEEP_SECONDS (20). No change in behaviour. Co-Authored-By: Claude Opus 5.5 --- .../task_step/task_steps/always_on.py | 54 ++++++++++++------- tst/unit/task_step/test_always_on_graded.py | 28 ++++++++++ 2 files changed, 64 insertions(+), 18 deletions(-) diff --git a/src/agent_env/task_step/task_steps/always_on.py b/src/agent_env/task_step/task_steps/always_on.py index cd2c419bb..84947bbe9 100644 --- a/src/agent_env/task_step/task_steps/always_on.py +++ b/src/agent_env/task_step/task_steps/always_on.py @@ -184,6 +184,11 @@ JUMPS_KEPT_PER_GAP = 20 PENDING_KEPT = 20 PENDING_TEXT_CHARS = 300 +# What _small keeps of nesting: past this many levels, a list or dict is summed up as its count. +KEPT_NESTING_LEVELS = 6 +# A user sim message longer than this, or one with a markdown heading or table, reads like an assistant's reply: its +# read is flagged (looks_like_assistant). +ASSISTANT_LIKE_MESSAGE_CHARS = 1200 BULKY_EVENT_FIELDS = ("input", "raw", "first_answer", "read", "parts_text", "changes") # What the prompt response keeps of each wake's parts (source_agent_per_turn_prompt_parts): each text part cut to # TURN_TEXT_CHARS, and past TURN_TEXT_TOTAL_CHARS in all, a note for the rest; every wake's full text is in the stored @@ -482,24 +487,25 @@ async def go(self) -> None: except DuplicateKeyError: pass ctx.metadata.setdefault("a2a_conversations", {})[step.id] = self.conversation_id - self.hook_records.append({"hook": HOOK_NAME, "turn": 0, "started_at": _iso(await self.now())}) - ctx.metadata.setdefault(HOOK_RECORDS_KEY, {})[step.id] = self.hook_records self.user_context_id = uuid.uuid4().hex # one user sim session per run # The agent's session: the step's context_id, as prompt_agent uses it, so a run can go on with the session an # earlier step left; else a new one. The graded conversation is the run's own either way. self.solver_context_id = step.context_id or uuid.uuid4().hex - now = await self.now() - self.log("start", virtual_time=_iso(now), end_time=_iso(self.end) if self.end else None, - agent=step.agent_name, agent_role=self.recipient, user_sim=step.user_sim, version=self.version, - user_model=self.user_model, max_wakes=step.max_wakes, time_zone=self.zone, multi_day=self.multi_day, - scheduled_checks=self.checks, clock_rate=await self._rate()) - # What the agent's role sent before the run (a task that gave it message_user earlier) isn't this run's: set it - # aside, recorded, so the first wake doesn't take it as its own. - before, quiet = await self._take_messages() - if before or quiet: - self.log("user_chat_before_run", messages=before, stay_quiet_calls=len(quiet)) + # From here the conversation exists, so whatever fails still closes it and leaves the run's record. try: + self.hook_records.append({"hook": HOOK_NAME, "turn": 0, "started_at": _iso(await self.now())}) + ctx.metadata.setdefault(HOOK_RECORDS_KEY, {})[step.id] = self.hook_records + now = await self.now() + self.log("start", virtual_time=_iso(now), end_time=_iso(self.end) if self.end else None, + agent=step.agent_name, agent_role=self.recipient, user_sim=step.user_sim, version=self.version, + user_model=self.user_model, max_wakes=step.max_wakes, time_zone=self.zone, + multi_day=self.multi_day, scheduled_checks=self.checks, clock_rate=await self._rate()) + # What the agent's role sent before the run (a task that gave it message_user earlier) isn't this run's: + # set it aside, recorded, so the first wake doesn't take it as its own. + before, quiet = await self._take_messages() + if before or quiet: + self.log("user_chat_before_run", messages=before, stay_quiet_calls=len(quiet)) await self.loop(now) except Exception as e: self.error = f"{type(e).__name__}: {e}" @@ -1441,7 +1447,8 @@ async def ask(text: str) -> dict: self.history.append({"read": read, "answer": ans}) mode = self.read_mode latency = self._latency(mode) - flagged = len(ans.get("message") or "") > 1200 or bool(re.search(r"^\s*(#|\|)", ans.get("message") or "", re.M)) + flagged = (len(ans.get("message") or "") > ASSISTANT_LIKE_MESSAGE_CHARS + or bool(re.search(r"^\s*(#|\|)", ans.get("message") or "", re.M))) entry = self.log("user_sim_read", virtual_time=_iso(now), version=self.version, read=read, input=ans.get("input"), raw=ans.get("raw"), outcome=ans.get("outcome"), message=ans.get("message"), used=ans.get("used"), score=ans.get("score"), @@ -1758,6 +1765,10 @@ def _actor(change: dict) -> str: return str(actor.get("role") or actor.get("kind") or "someone") +# The longest wait between two tries of a gateway control call (_call backs off doubling, from 1 s, up to this). +CALL_RETRY_MAX_SLEEP_SECONDS = 20 + + async def _call(env, uri: str, method: str, params: Optional[dict] = None, *, attempts: int = 6, timeout: int = 60) -> Any: """A gateway control call, retried on a transient failure (a busy gateway can take seconds to answer). @@ -1774,7 +1785,7 @@ async def _call(env, uri: str, method: str, params: Optional[dict] = None, *, at if attempt + 1 < attempts: logger.warning(f"always_on: {method} on {getattr(env, 'env_id', env)} failed ({type(e).__name__}: " f"{e}); retry {attempt + 1}/{attempts - 1}") - await asyncio.sleep(min(2 ** attempt, 20)) + await asyncio.sleep(min(2 ** attempt, CALL_RETRY_MAX_SLEEP_SECONDS)) raise last @@ -1821,6 +1832,11 @@ def _busy(row: dict, now: datetime) -> bool: return not (row.get("waiting") and row.get("resumes_at") and _utc(row["resumes_at"]) > now) +# Words she quotes name one of the agent's messages only when they run to at least this many characters, so a short +# quote ("ok") can't match one by chance. +QUOTE_MIN_CHARS = 8 + + def _reply_to(answer: dict, seen: list[dict]) -> Optional[dict]: """Which of the agent's messages in a read her answer responds to: the one the plug-in names in ``replying_to`` (its 1-based number in the read, or words quoted from it); else the one her ``used`` quotes; else the only message @@ -1831,7 +1847,7 @@ def found(m: dict, how: str) -> dict: def quoting(quote: Any) -> list[dict]: words = " ".join(str(quote or "").split()).lower() - return [m for m in seen if len(words) >= 8 and words in " ".join(m["text"].split()).lower()] + return [m for m in seen if len(words) >= QUOTE_MIN_CHARS and words in " ".join(m["text"].split()).lower()] named = answer.get("replying_to") if isinstance(named, int) and not isinstance(named, bool) and 1 <= named <= len(seen): @@ -1863,10 +1879,10 @@ def _bounded(pending: dict) -> dict: def _small(value: Any, depth: int = 0) -> Any: """``value`` kept small for the run's metadata, at any depth: strings cut to ``PENDING_TEXT_CHARS``, lists to their - first ``PENDING_KEPT`` items and a count of the rest, nesting past six levels summed up.""" + first ``PENDING_KEPT`` items and a count of the rest, nesting past ``KEPT_NESTING_LEVELS`` levels summed up.""" if isinstance(value, str): return value[:PENDING_TEXT_CHARS] + "…" if len(value) > PENDING_TEXT_CHARS else value - if depth >= 6 and isinstance(value, (list, dict)): + if depth >= KEPT_NESTING_LEVELS and isinstance(value, (list, dict)): return f"({len(value)} {'items' if isinstance(value, list) else 'fields'})" if isinstance(value, dict): return {k: _small(v, depth + 1) for k, v in value.items()} @@ -1882,6 +1898,8 @@ def _small(value: Any, depth: int = 0) -> Any: "virtual_time", "from", "to", "reason", "state", "outcome", "mode", "read_at", "deliver_at", "end_time", "agent", "agent_role", "user_sim", "version", "user_model", "time_zone", "kind", "trigger_id", "env_id", "phase", "error_type", "matched_by", "at", "wake", "n", "trajectory_url", "object_url"}) +# Past this length even one of those fields is logged as its length. +LOGGED_TEXT_CHARS = 200 def _log_view(fields: dict, depth: int = 0) -> dict: @@ -1892,7 +1910,7 @@ def _log_view(fields: dict, depth: int = 0) -> dict: if v is None or isinstance(v, (bool, int, float)): out[k] = v elif isinstance(v, str): - out[k] = v if k in _LOGGED_AS_IS and len(v) <= 200 else {"chars": len(v)} + out[k] = v if k in _LOGGED_AS_IS and len(v) <= LOGGED_TEXT_CHARS else {"chars": len(v)} elif isinstance(v, (list, tuple)): out[k] = {"items": len(v)} elif isinstance(v, dict): diff --git a/tst/unit/task_step/test_always_on_graded.py b/tst/unit/task_step/test_always_on_graded.py index b948a5377..2a0a3a004 100644 --- a/tst/unit/task_step/test_always_on_graded.py +++ b/tst/unit/task_step/test_always_on_graded.py @@ -6,6 +6,7 @@ import pytest +from agent_env.a2a_agent import conversation_store from agent_env.task_step.task_steps import always_on as ao from agent_env.task_step.task_steps.verifiers.agent_prompt_response_verifier import ( AgentPromptResponseVerifierTaskStep, @@ -190,6 +191,33 @@ async def test_messages_queued_before_the_run_are_not_the_run_s(monkeypatch, tmp assert before["stay_quiet_calls"] == 1 +@pytest.mark.asyncio +async def test_a_take_refused_before_the_run_still_closes_the_conversation_and_records_the_run(monkeypatch, tmp_path): + """The take before the run comes after the graded conversation is made: when the gateway refuses it, the run still + leaves its record and prompt response, and the conversation is closed, not left active.""" + rig = Rig(monkeypatch, tmp_path) + real_take = ao._Run._take_messages + refused: list[int] = [] + + async def refused_once(run): + if not refused: # the take before the run, not finish()'s + refused.append(1) + raise RuntimeError("the gateway refused the take") + return await real_take(run) + + monkeypatch.setattr(ao._Run, "_take_messages", refused_once) + await rig.arm() + ctx = rig.context() + with pytest.raises(RuntimeError, match="the gateway refused the take"): + await rig.run(rig.step(), ctx) + + assert rig.sent == [] + assert rig.record(ctx)["end_reason"] == "step_error" + [response] = ctx.prompt_responses + assert response.error_type == "step_error" + assert conversation_store.get_conversation(ctx.metadata["a2a_conversations"]["week"])["status"] == "closed" + + # ---- failed turns are errors ----------------------------------------------------------------------------------------- @pytest.mark.parametrize("play,error_type,end_reason", [ From 8008ee847aa8782845975b277e98222ec0ffc5f8 Mon Sep 17 00:00:00 2001 From: Sarah Chitty Date: Mon, 9 Nov 2026 15:50:39 -0800 Subject: [PATCH 49/53] triggers: a waiting fire goes on when a frozen clock is advanced to its resumes_at A delay's wait published its end as since + seconds, but the sleeper worked out its own end from a second, later read of the clock. On a running clock the two differ by the virtual time between the reads (tens of microseconds at 60x), harmless while the clock keeps running. The always_on step now freezes the clock between turns (set_rate 0) and jumps it exactly to the reply's resumes_at, so the sleeper was left just short of its own end with nothing to move the clock: the settle timed out and the reply went at the next jump, hours of task time late. Working hours had the same gap. The sleeper now waits for the very mark the fire publishes: the delay's end as a Clock.elapsed() reading, or the hours' opening as a virtual time. The gateway's set-time, rate and clear routes also wake waiting fires (clock_changed), so one napping on a frozen clock follows the next turn's rate at once instead of up to 5 real seconds (5 virtual minutes at 60x) late. Co-Authored-By: Claude Opus 5.5 --- src/agent_env/env/gateway/gateway.py | 3 + src/agent_env/env/gateway/triggers.py | 45 ++++++--- tst/unit/env/gateway/triggers_fires_test.py | 93 ++++++++++++++++++- tst/unit/env/gateway/triggers_test.py | 17 ++-- .../task_step/test_always_on_clock_rate.py | 48 ++++++++++ 5 files changed, 182 insertions(+), 24 deletions(-) diff --git a/src/agent_env/env/gateway/gateway.py b/src/agent_env/env/gateway/gateway.py index 35da65c15..25a0aa536 100644 --- a/src/agent_env/env/gateway/gateway.py +++ b/src/agent_env/env/gateway/gateway.py @@ -323,6 +323,7 @@ async def clock_set_time(request: Request) -> Response: state = self._clock.set_time(body.get("virtual_time"), body.get("virtual_seconds_per_real_second")) except ClockError as e: return self._json_response({"ok": False, "error": str(e)}, 400) + self._trigger_engine.clock_changed() self._register_get_time_tool() return self._json_response(state, 200) @@ -352,12 +353,14 @@ async def clock_rate(request: Request) -> Response: state = self._clock.set_rate(body.get("virtual_seconds_per_real_second")) except ClockError as e: return self._json_response({"ok": False, "error": str(e)}, 400) + self._trigger_engine.clock_changed() return self._json_response(state, 200) @self._mcp.custom_route("/clock/clear", methods=["POST"]) async def clock_clear(request: Request) -> Response: self._unregister_get_time_tool() state = self._clock.clear() + self._trigger_engine.clock_changed() return self._json_response(state, 200) @self._mcp.custom_route("/clock/time", methods=["GET"]) diff --git a/src/agent_env/env/gateway/triggers.py b/src/agent_env/env/gateway/triggers.py index c603e8d8a..a04c8d157 100644 --- a/src/agent_env/env/gateway/triggers.py +++ b/src/agent_env/env/gateway/triggers.py @@ -52,7 +52,8 @@ _BARRIER_AT = ("provoking_call",) _BARRIER_KEYS = ("at", "timeout_seconds") _DROP = object() # an unresolved `${x?}` used as a whole value: drop the key / element -# Real seconds between re-reads of the clock while waiting out virtual time, so a rate change, re-arm or clear is followed. +# Real seconds between re-reads of the clock while waiting out virtual time, so a change made other than through the +# gateway's clock routes (which wake the waits: clock_changed) is followed. _VIRTUAL_RECHECK_S = 5.0 # The clock's resolution: virtual times are microseconds. _VIRTUAL_RESOLUTION_S = 1e-6 @@ -445,7 +446,7 @@ def __init__(self, gateway): self._driver_task: asyncio.Task | None = None self._driver_interval = 1.0 # real seconds between autonomous time-trigger polls self._time_trigger_gate = asyncio.Event() # parks the driver while no time-trigger is armed - self._clock_moved = asyncio.Event() # set, and replaced, when an advance moves the clock under waiting actions + self._clock_moved = asyncio.Event() # set, and replaced, when the clock changes under waiting actions # Opt-in settings (`config`), off by default so a registration behaves as it always has. self._concurrent_fires = False # a contact-reply trigger's repeat fires run side by side instead of queueing self._executor_timeout_default: float | None = None # the time limit of an executor that sets none @@ -1172,9 +1173,15 @@ def clock_advanced(self) -> list[str]: if now is not None and any((at := self._wait_resumes(w)) is not None and at <= now and (w["hours"] is None or _next_opening(now, w["hours"]) is None) for w in t["waits"].values())] + self.clock_changed() + return list(dict.fromkeys(due + ended)) + + def clock_changed(self) -> None: + """Wake the actions waiting out virtual time to read the clock again, after it jumps, is re-armed, changes rate + or is cleared: one napping on a frozen clock would otherwise see it run again only at its next recheck, up to + _VIRTUAL_RECHECK_S real seconds (minutes of virtual time) late.""" self._clock_moved.set() self._clock_moved = asyncio.Event() - return list(dict.fromkeys(due + ended)) def _resolve_first_mark(self, trig: dict, now: datetime) -> None: """Stamp the first mark for a non-anchored time-trigger; event-anchored ones wait for _resolve_dependents.""" @@ -1674,9 +1681,10 @@ async def _delay(self, tid: str, trig: dict, index: int, action: dict, since: fl mean = _parse_duration(delay["mean"]).total_seconds() seconds = trig["delay_rngs"][index].expovariate(1.0 / mean) self._emit("delayed", tid, action_index=index, virtual_seconds=round(seconds, 3), **_named_executor(action)) - token = _begin_wait(trig, since + seconds) + until = since + seconds + token = _begin_wait(trig, until) try: - await self._sleep_virtual(seconds - (self._gw._clock.elapsed() - since)) + await self._sleep_virtual(until) finally: _end_wait(trig, token) @@ -1696,19 +1704,28 @@ async def _await_hours(self, tid: str, trig: dict, index: int, hours: dict, name gen, opens = current, opening trig["waits"][token]["resumes"] = opens self._emit("deferred", tid, action_index=index, until=_iso(opens), **named) - await self._sleep_virtual((opens - now).total_seconds(), generation=gen) + await self._sleep_virtual(opens, generation=gen) finally: _end_wait(trig, token) - async def _sleep_virtual(self, seconds: float, generation: int | None = None) -> None: - """Wait out ``seconds`` of virtual time as the clock runs (real time at its rate, 1 while unarmed, none while - frozen) or jumps ahead. Given the clock's ``generation``, stop early once a re-arm or clear moves it on.""" + async def _sleep_virtual(self, until: float | datetime, generation: int | None = None) -> None: + """Wait until the clock gets to ``until``, a Clock.elapsed() reading or a virtual time on the current arm, as it + runs (real time at its rate, 1 while unarmed, none while frozen) or jumps ahead. ``until`` is the very mark the + waiting fire's resumes_at shows: a mark worked out here from a later reading of a running clock would lie past + it, and a clock frozen at resumes_at would hold the fire forever. Given the clock's ``generation``, stop early + once a re-arm or clear moves it on.""" clock = self._gw._clock - until = clock.elapsed() + seconds - # Times are kept to the microsecond, so an advance to a waiting fire's resumes_at can fall a hair short of it; - # under a microsecond left is reached, or a frozen clock would hold the fire forever. - while ((left := until - clock.elapsed()) >= _VIRTUAL_RESOLUTION_S - and (generation is None or clock.generation == generation)): + while generation is None or clock.generation == generation: + if isinstance(until, datetime): + if (now := clock.now()) is None: + return + left = (until - now).total_seconds() + else: + left = until - clock.elapsed() + # A delay's resumes_at is its end to the microsecond, so an advance to it can fall a hair short; under a + # microsecond left is reached. + if left < _VIRTUAL_RESOLUTION_S: + return rate = clock.rate await self._nap(min(left / rate, _VIRTUAL_RECHECK_S) if rate else _VIRTUAL_RECHECK_S) diff --git a/tst/unit/env/gateway/triggers_fires_test.py b/tst/unit/env/gateway/triggers_fires_test.py index 6d34e28e0..9f678d02a 100644 --- a/tst/unit/env/gateway/triggers_fires_test.py +++ b/tst/unit/env/gateway/triggers_fires_test.py @@ -3,13 +3,18 @@ from __future__ import annotations import asyncio +from datetime import timedelta +import httpx import pytest +from agent_env.env.gateway import gateway as gw_mod +from agent_env.env.gateway.clock import Clock, _iso, _parse_rfc3339 from agent_env.env.gateway.constants import EXT_TRIGGERS_URI, GATEWAY_EXTENSIONS +from agent_env.env.gateway.gateway import Gateway from agent_env.env.gateway.triggers import TriggerEngine, TriggerError -from tst.unit.env.gateway.next_event_test import _engine, _next, _row, _until -from tst.unit.env.gateway.triggers_time_test import _perm, _result +from tst.unit.env.gateway.next_event_test import MONDAY, _engine, _next, _row, _until +from tst.unit.env.gateway.triggers_time_test import _FakeGateway, _perm, _result def _reply_trigger(tid: str = "reply", *, delay=None, executor: dict | None = None) -> dict: @@ -360,3 +365,87 @@ async def test_a_fire_advanced_to_its_resumes_at_goes_on_though_its_delay_ends_b engine.clock_advanced() await _until(lambda: _row(engine, "reply")["status"] == "armed") assert len(bob.calls) == 1 + + +# ---- a clock that runs only during the agent's turns ---------------------------------------------------------------- + +class _RunningNow: + """Real time under a running clock: it moves on a microsecond at every read, as time.monotonic does between two.""" + + def __init__(self) -> None: + self.t = 1000.0 + + def __call__(self) -> float: + self.t += 1e-6 + return self.t + + +def _running_engine(rate: float) -> tuple[TriggerEngine, Clock]: + clock = Clock(now_fn=_RunningNow()) + clock.set_time(MONDAY, rate=rate) + return TriggerEngine(_FakeGateway(clock)), clock + + +@pytest.mark.parametrize("past", [0, 90], ids=["to its resumes_at", "past it"]) +@pytest.mark.asyncio +async def test_a_delay_begun_on_a_running_clock_goes_on_when_the_frozen_clock_is_advanced_to_its_resumes_at( + monkeypatch, past): + """The always_on step's jump: a contact's reply starts its delay during the agent's turn, with the clock running + at the task's rate; between turns the step freezes it (set_rate 0) and advances it to the reply's resumes_at. + Nothing else moves the clock then, so that must be enough.""" + bob = _Executor(monkeypatch) + bob.release() + engine, clock = _running_engine(rate=60) + engine.register(_reply_trigger(delay={"dist": "exp", "mean": "PT40M", "seed": 30})) + engine.on_tool_call("default", "send_text", {}, _result()) + await _until(lambda: _row(engine, "reply")["waits"]) + clock.set_rate(0) + to = _parse_rfc3339(_row(engine, "reply")["resumes_at"]) + timedelta(seconds=past) + + clock.advance(_iso(to)) + assert engine.clock_advanced() == ["reply"] + await _until(lambda: _row(engine, "reply")["status"] == "armed") + assert [c["at"] for c in bob.calls] == [to] + + +@pytest.mark.asyncio +async def test_a_fire_waiting_on_a_running_clock_for_its_executor_s_hours_goes_on_when_the_frozen_clock_reaches_them( + monkeypatch): + bob = _Executor(monkeypatch) + bob.release() + engine, clock = _running_engine(rate=60) + engine.register(_reply_trigger(executor={"hours": {"start": "09:00", "end": "17:00"}})) + engine.on_tool_call("default", "send_text", {}, _result()) + await _until(lambda: _row(engine, "reply")["waits"]) + clock.set_rate(0) + assert _row(engine, "reply")["resumes_at"] == "2026-03-02T09:00:00Z" + + clock.advance("2026-03-02T09:00:00Z") + assert engine.clock_advanced() == ["reply"] + await _until(lambda: _row(engine, "reply")["status"] == "armed") + assert [_iso(c["at"]) for c in bob.calls] == ["2026-03-02T09:00:00Z"] + + +@pytest.mark.asyncio +async def test_a_rate_change_wakes_a_waiting_fire_to_go_on_when_the_clock_gets_to_its_time(monkeypatch, tmp_path): + """On a frozen clock a waiting fire looks at it again only every _VIRTUAL_RECHECK_S real seconds; the next turn's + set_rate wakes it to wait out the rest at the new rate, so it goes on when the clock gets to its time, not up to + that long after (five virtual minutes at 60x).""" + monkeypatch.setattr(gw_mod, "GATEWAY_TRAJECTORY_FILE", str(tmp_path / "trajectory.jsonl")) + gw = Gateway(host="127.0.0.1", port=0, server_name="AgentEnvGateway", internal_mcp_servers=[]) + bob = _Executor(monkeypatch) + bob.release() + gw._clock.set_time(MONDAY, rate=0) + engine = gw._trigger_engine + engine.register(_reply_trigger(delay="PT1M")) + engine.on_tool_call("default", "send_text", {}, _result()) + await _until(lambda: _row(engine, "reply")["waits"]) + try: + transport = httpx.ASGITransport(app=gw._mcp.streamable_http_app()) + async with httpx.AsyncClient(transport=transport, base_url="http://gateway") as client: + r = await client.put("/clock/rate", json={"virtual_seconds_per_real_second": 3600}) + r.raise_for_status() + await _until(lambda: bob.calls, seconds=1) # a virtual minute at 3600x is a sixtieth of a real second + finally: + await engine.cancel_scheduled() + gw._close_trajectory_file() diff --git a/tst/unit/env/gateway/triggers_test.py b/tst/unit/env/gateway/triggers_test.py index b568f4610..6a56d57af 100644 --- a/tst/unit/env/gateway/triggers_test.py +++ b/tst/unit/env/gateway/triggers_test.py @@ -489,7 +489,7 @@ async def test_a_stochastic_delay_draws_the_same_sequence_from_its_seed(engine, await asyncio.sleep(0.1) rng = random.Random(7) - assert waits == [rng.expovariate(1.0 / 600) for _ in range(3)] + assert waits == pytest.approx([rng.expovariate(1.0 / 600) for _ in range(3)]) # each waited to its own mark assert len(engine._test_state["slack"]) == 3 @@ -500,8 +500,8 @@ async def test_a_re_arm_during_one_mark_s_delay_does_not_shorten_the_next(engine "args": {"channel": "#c", "text": "hi"}, "delay": "PT90M"}]}]}) record = TriggerEngine._sleep_virtual - async def re_arm_a_day_back_after_the_first_wait(self, seconds, generation=None): - await record(self, seconds, generation) + async def re_arm_a_day_back_after_the_first_wait(self, until, generation=None): + await record(self, until, generation) if len(virtual_time) == 1: self._gw._clock.set_time("2026-10-13T10:30:00Z") @@ -1299,7 +1299,7 @@ async def test_waiting_virtual_time_follows_the_clock_rate(engine, monkeypatch, stepped = _SteppedClock(engine, monkeypatch) if rate is not None: stepped.clock.set_time("2026-10-14T08:00:00Z", rate=rate) - await engine._sleep_virtual(virtual_seconds) + await engine._sleep_virtual(stepped.clock.elapsed() + virtual_seconds) assert stepped.slept == steps @@ -1308,7 +1308,7 @@ async def test_waiting_virtual_time_holds_while_frozen_and_follows_a_new_rate(en stepped = _SteppedClock(engine, monkeypatch) stepped.at[1] = lambda: stepped.clock.set_time("2026-10-14T08:00:00Z", rate=0) # after 5 unarmed seconds stepped.at[3] = lambda: stepped.clock.set_time("2026-10-14T08:00:00Z", rate=4) # where it froze, now at 4x - await engine._sleep_virtual(13) + await engine._sleep_virtual(stepped.clock.elapsed() + 13) assert stepped.slept == [5.0, 5.0, 5.0, 2.0] @@ -1319,7 +1319,7 @@ async def test_waiting_virtual_time_counts_a_re_arm_forward_but_not_back(engine, stepped = _SteppedClock(engine, monkeypatch) stepped.clock.set_time("2026-10-14T08:00:00Z") stepped.at[1] = lambda: stepped.clock.set_time(re_arm_to) # at 08:00:05: 3 s ahead, or a day back - await engine._sleep_virtual(10) + await engine._sleep_virtual(stepped.clock.elapsed() + 10) assert stepped.slept == steps @@ -1328,7 +1328,7 @@ async def test_a_wait_tied_to_a_clock_generation_ends_when_the_clock_moves_on(en stepped = _SteppedClock(engine, monkeypatch) stepped.clock.set_time("2026-10-14T08:00:00Z") stepped.at[1] = stepped.clock.clear - await engine._sleep_virtual(3600, generation=stepped.clock.generation) + await engine._sleep_virtual(stepped.clock.elapsed() + 3600, generation=stepped.clock.generation) assert stepped.slept == [5.0] @@ -1376,7 +1376,8 @@ def _fake_waits(engine, monkeypatch, armed_at=None) -> _Waits: waited = _Waits() waited.advance = lambda seconds: source.__setitem__("t", source["t"] + seconds / clock.rate) - async def fake_sleep(self, seconds, generation=None): + async def fake_sleep(self, until, generation=None): + seconds = (until - clock.now()).total_seconds() if isinstance(until, datetime) else until - clock.elapsed() waited.append(seconds) waited.advance(max(0.0, seconds)) diff --git a/tst/unit/task_step/test_always_on_clock_rate.py b/tst/unit/task_step/test_always_on_clock_rate.py index 49a70eeb1..62bb08127 100644 --- a/tst/unit/task_step/test_always_on_clock_rate.py +++ b/tst/unit/task_step/test_always_on_clock_rate.py @@ -9,6 +9,8 @@ import pytest +from agent_env.task_step.task_steps import always_on as ao +from agent_env.task_step.task_steps.always_on import _utc from tst.unit.task_step.always_on_harness import Rig pytestmark = pytest.mark.usefixtures("local_stores") @@ -70,3 +72,49 @@ async def answer(version, person, history, read, ask): [read] = rig.events(ctx, "user_sim_read") assert read["deliver_at"] == read["virtual_time"] # nothing moved while she read assert rig.gw._clock.rate == RATE + + +def _crew_texted(rig, n): + rig.alert(f"Sam: reply {n}", channel="Messages · noor 30") + + +CREW_DM = {"id": "crew-dm", "when": {"type": "action", "tool": "messaging_send_message", "repeat": True}, + "actions": [{"type": "nl", "executor": "crew", "instruction": "answer Linda's newest text", + "delay": {"dist": "exp", "mean": "PT40M", "seed": 30}}]} + + +@pytest.mark.asyncio +async def test_a_contact_s_delayed_reply_goes_on_when_the_step_jumps_the_frozen_clock_to_it(monkeypatch, tmp_path): + """The reply's delay starts during the agent's turn, on the running clock; between turns the step freezes the + clock and jumps it to the reply's time. That must be enough: the reply used to wait out settle_timeout_seconds, + then go only at the next jump, hours of the task's time late.""" + def texts_a_friend(n, texts, rig): + if n == 1: + rig.provoke("messaging_send_message", {"user_id": "+13475550167"}) + rig.quiet() + return "" + + rig = Rig(monkeypatch, tmp_path, rate=RATE, agent=texts_a_friend) + turn = rig._send_and_wait + + async def the_turn_goes_on_after_the_text(url, parts, **kw): + out = await turn(url, parts, **kw) + await asyncio.sleep(0.05) # the reply's delay starts while the clock still runs, as in a real turn + return out + + monkeypatch.setattr(ao, "send_and_wait", the_turn_goes_on_after_the_text) + await rig.arm() + rig.app_tools("messaging_send_message") + rig.triggers(CREW_DM, executors={"crew": rig.contact("crew", seconds=0, reply=_crew_texted)}) + try: + ctx = await rig.run(rig.step(settle_timeout_seconds=1)) + finally: + await rig.close() + + assert rig.events(ctx, "settle_timeout") == [] + [jump] = [j for j in rig.events(ctx, "clock_jump") if j["to"].startswith("2026-11-09T14:3")] + assert jump["settled"] + [call] = rig.contact_calls + assert _utc(call["at"]) == _utc(jump["to"]) # the reply went at the time the step jumped to + [_, replied] = rig.events(ctx, "wake") + assert "Sam: reply 1" in replied["parts_text"] and _utc(replied["virtual_time"]) == _utc(jump["to"]) From 974a6aba29be23b8247c66e8675d9579b74f33af Mon Sep 17 00:00:00 2001 From: Sarah Chitty Date: Mon, 9 Nov 2026 19:05:59 -0800 Subject: [PATCH 50/53] triggers: a contact's reply that notifies the agent runs side by side, so a quick one isn't held behind a slow one - concurrent_fires ran a trigger's fires side by side only when every action was nl. A host's reply on a booking site is nl then notify, so its second fire queued: it drew its delay only once the first fire ended, and state showed no due time for it. The always_on step saw only the first fire's resumes_at and jumped there; in a live run the quicker reply (detected 18:01:58 + 752 s) landed with the slow one at 18:53:33, 39 task minutes late. - notify joins nl as side-by-side safe: neither reads and writes rows. Each fire's delay now runs from its own call and shows in waits / next_event_at; the executor call still takes turns on the conversation's lock. - Triggers with tool or permission actions still queue, as before. Co-Authored-By: Claude Opus 5.5 --- src/agent_env/env/gateway/constants.py | 2 +- src/agent_env/env/gateway/triggers.py | 10 +++-- tst/unit/env/gateway/triggers_fires_test.py | 41 +++++++++++++++++++++ tst/unit/task_step/test_always_on_settle.py | 35 ++++++++++++++++++ 4 files changed, 83 insertions(+), 5 deletions(-) diff --git a/src/agent_env/env/gateway/constants.py b/src/agent_env/env/gateway/constants.py index fff131f7b..37c8326d1 100644 --- a/src/agent_env/env/gateway/constants.py +++ b/src/agent_env/env/gateway/constants.py @@ -210,7 +210,7 @@ def random_mcp_server_name() -> str: }, { "uri": EXT_TRIGGERS_URI, - "description": "Register (additive add-by-id), remove, clear, or read (state) typed Action/State triggers + the firing log (harness control path). config sets the engine's opt-in settings: concurrent_fires (a repeat trigger whose actions are all contact replies (nl) runs its fires side by side instead of queueing) and executor_timeout_seconds (the time limit of an executor that sets none).", + "description": "Register (additive add-by-id), remove, clear, or read (state) typed Action/State triggers + the firing log (harness control path). config sets the engine's opt-in settings: concurrent_fires (a repeat trigger whose actions are all contact replies (nl) or notifications (notify) runs its fires side by side instead of queueing) and executor_timeout_seconds (the time limit of an executor that sets none).", "params": {"endpoint": "/triggers/register", "features": list(TRIGGER_FEATURES), "methods": { "register": {"method": "POST", "endpoint": "/triggers/register", "request": {"type": "object", "properties": {"watch_roles": {"type": "array", "items": {"type": "string"}}, "executor": {"type": "object"}, "executors": {"type": "object"}, "triggers": {"type": "array"}}, "required": ["triggers"]}}, "config": {"method": "POST", "endpoint": "/triggers/config", "request": {"type": "object", "properties": {"concurrent_fires": {"type": "boolean"}, "executor_timeout_seconds": {"type": "number", "exclusiveMinimum": 0}}}}, diff --git a/src/agent_env/env/gateway/triggers.py b/src/agent_env/env/gateway/triggers.py index a04c8d157..223780698 100644 --- a/src/agent_env/env/gateway/triggers.py +++ b/src/agent_env/env/gateway/triggers.py @@ -585,7 +585,9 @@ def config(self, body: Any) -> dict: """Set the engine's opt-in settings, each off unless the body sets it (an omitted one resets): with ``concurrent_fires``, each further call that provokes a repeat trigger already firing starts its own fire at once, side by side, instead of queueing behind the one running, when every action of the trigger is a contact's - reply (``nl``); any other trigger still queues, since its tool actions may read and then write the same rows. + reply (``nl``) or a notification (``notify``); any other trigger still queues, since its tool actions may read + and then write the same rows. A queued fire draws its delay only when its turn comes, and state shows no due + time for it, so a quick reply queued behind a slow one would wait out the slow one's delay too. Fires of one trigger side by side still take turns on its executor conversation (``_executor_turn``). ``executor_timeout_seconds`` is the time limit of each executor whose registration set no ``timeout_seconds`` (else 120). ``clear`` resets both.""" @@ -1826,9 +1828,9 @@ def _named_executor(action: dict) -> dict: def _replies_only(spec: dict) -> bool: - """Whether every action of a trigger is a contact's reply (``nl``), the only kind ``concurrent_fires`` runs side - by side.""" - return all(action.get("type") == "nl" for action in spec["actions"]) + """Whether every action of a trigger is a contact's reply (``nl``) or a notification (``notify``, an app telling + the agent of the reply), the only kinds ``concurrent_fires`` runs side by side: neither reads and writes rows.""" + return all(action.get("type") in ("nl", "notify") for action in spec["actions"]) def _in_flight_fire(cause: dict) -> dict: diff --git a/tst/unit/env/gateway/triggers_fires_test.py b/tst/unit/env/gateway/triggers_fires_test.py index 9f678d02a..2df5d56b6 100644 --- a/tst/unit/env/gateway/triggers_fires_test.py +++ b/tst/unit/env/gateway/triggers_fires_test.py @@ -3,7 +3,9 @@ from __future__ import annotations import asyncio +import random from datetime import timedelta +from types import SimpleNamespace import httpx import pytest @@ -192,6 +194,45 @@ async def test_side_by_side_is_only_for_contact_replies_a_tool_action_trigger_st assert len(bob.calls) == 2 +@pytest.mark.asyncio +async def test_a_reply_that_notifies_the_agent_runs_side_by_side_so_its_due_time_shows_and_it_lands_on_time( + monkeypatch): + """A contact's reply followed by a notify, as a host's reply on a booking site: the second fire, with the shorter + delay, used to queue behind the first, unseen in next_event_at, and only drew its delay once the first ended, 17 + minutes late. Its delay now runs from its own call, its resumes_at shows, and it lands then, the first still + waiting.""" + bob = _Executor(monkeypatch) + bob.release() + notes: list[str] = [] + engine, clock, _ = _engine() + def notify(channel, text, trigger_id, to): + notes.append(_iso(clock.now())) + return {"id": f"n{len(notes)}", "to": to} + + engine._gw._notifications = SimpleNamespace(notify=notify) + engine.config({"concurrent_fires": True}) + trigger = _reply_trigger(delay={"dist": "exp", "mean": "PT40M", "seed": 30}) + trigger["triggers"][0]["actions"].append({"type": "notify", "channel": "Stayncook", "text": "New message"}) + engine.register(trigger) + draws = random.Random(30) + slow, quick = draws.expovariate(1 / 2400), draws.expovariate(1 / 2400) # 1859 s, then 819 s + engine.on_tool_call("default", "send_text", {}, _result()) + clock.advance("2026-03-02T00:00:01Z") + engine.on_tool_call("default", "send_text", {}, _result()) + await _until(lambda: len(_row(engine, "reply")["waits"]) == 2) + + row = _row(engine, "reply") + due = _iso(_parse_rfc3339("2026-03-02T00:00:01Z") + timedelta(seconds=quick)) + assert (row["status"], row["pending"], row["running"]) == ("firing", 0, 2) + assert _next(engine) == due < _iso(_parse_rfc3339(MONDAY) + timedelta(seconds=slow)) + + clock.advance(due) + engine.clock_advanced() + await _until(lambda: len(notes) == 1) + assert [_iso(c["at"]) for c in bob.calls] == [due] and notes == [due] + assert _row(engine, "reply")["running"] == 1 + + @pytest.mark.asyncio async def test_each_fire_side_by_side_keeps_its_own_number_and_only_its_own_executor_calls(monkeypatch): """Fire 1 waits out its delay while fire 2 starts: fire 1's call still says it is fire 1. Fire 2, waiting for its diff --git a/tst/unit/task_step/test_always_on_settle.py b/tst/unit/task_step/test_always_on_settle.py index b55b3a11d..25a934198 100644 --- a/tst/unit/task_step/test_always_on_settle.py +++ b/tst/unit/task_step/test_always_on_settle.py @@ -5,9 +5,11 @@ from __future__ import annotations import random +from datetime import timedelta import pytest +from agent_env.env.gateway.clock import _iso, _parse_rfc3339 from agent_env.task_step.task_steps import always_on as ao from tst.unit.task_step.always_on_harness import Rig @@ -98,6 +100,39 @@ def texts_two_friends(n, texts, rig): "executor_timeout_seconds": 300.0} +@pytest.mark.asyncio +async def test_a_reply_that_notifies_the_agent_still_runs_side_by_side_so_the_jump_stops_at_the_quick_one( + monkeypatch, tmp_path): + """As a host's reply on a booking site: the contact answers, then the app notifies the agent. Both replies go to one + contact conversation. The second, with the shorter delay, used to queue behind the first and show no due time, so + the step jumped straight to the first's and the quick reply landed with it, 17 task minutes late.""" + draws = random.Random(30) + slow, quick = (draws.expovariate(1 / 2400) for _ in range(2)) # 1859 s, then 819 s + quick_due = _iso(_parse_rfc3339("2026-11-09T14:00:00Z") + timedelta(seconds=quick)) + + def texts_two_hosts(n, texts, rig): + if n == 1: + rig.provoke("messaging_send_message", {"conversation_id": "CV-000004"}) + rig.provoke("messaging_send_message", {"conversation_id": "CV-000002"}) + rig.quiet() + return "" + + rig = Rig(monkeypatch, tmp_path, agent=texts_two_hosts) + await rig.arm() + rig.app_tools("messaging_send_message") + host_replies = {**CREW_DM, "id": "host-replies", + "actions": [*CREW_DM["actions"], {"type": "notify", "channel": "Stayncook", "text": "New message"}]} + rig.triggers(host_replies, executors={"crew": rig.contact("crew", seconds=0, reply=_crew_texted)}) + ctx = await rig.run(rig.step()) + + first_jump = next(j for j in rig.events(ctx, "clock_jump") if j["from"] != j["to"]) + assert first_jump["to"] <= quick_due + assert [c["at"][:19] for c in rig.contact_calls] == [quick_due[:19], "2026-11-09T14:30:58"] + woke = [(w["virtual_time"][:19], w["parts_text"].count("Sam: reply")) for w in rig.events(ctx, "wake")[1:]] + assert woke == [("2026-11-09T14:13:39", 1), ("2026-11-09T14:30:58", 1)] + assert slow > quick + + @pytest.mark.asyncio async def test_a_contact_that_sets_no_limit_gets_300_s_and_one_that_does_keeps_its_own(monkeypatch, tmp_path): rig = Rig(monkeypatch, tmp_path) From aebf480e3e006cb5a0c79c83b50cc58aa2e7f080 Mon Sep 17 00:00:00 2001 From: Sarah Chitty Date: Wed, 11 Nov 2026 19:31:00 -0800 Subject: [PATCH 51/53] always_on: a contact's time limit is the one it was registered with; drop contact_timeout_seconds contact_timeout_seconds never reached the contacts register_env_triggers registers, since that step always sends its own timeout_seconds (executor_timeout_seconds, 120 by default). The setting is gone from the step, and with it the gateway's triggers config executor_timeout_seconds, which only the step used: config takes concurrent_fires alone again, and each executor keeps the limit it was registered with, as on main. The settle still needs the contacts' real limits: the triggers state now lists each executor's timeout_seconds (config.executor_timeouts), the run reads the longest when it starts, and caps every settle with it. The wake reserve and the time budget are worked out again from it, so a run whose contacts' limit leaves no room in the hub's job is refused before its first wake. When the task loads the step is checked with the gateway's default (120 s); a gateway that doesn't report its executors' limits gets that default too. Co-Authored-By: Claude Opus 5.5 --- src/agent_env/env/gateway/constants.py | 4 +- src/agent_env/env/gateway/triggers.py | 47 +++----- .../task_step/task_steps/always_on.py | 104 ++++++++++-------- tst/unit/env/gateway/triggers_fires_test.py | 39 +++---- tst/unit/env/gateway/triggers_test.py | 5 +- .../test_always_on_agent_and_gateway.py | 7 +- tst/unit/task_step/test_always_on_budget.py | 95 ++++++++++++---- tst/unit/task_step/test_always_on_settle.py | 34 +++--- 8 files changed, 191 insertions(+), 144 deletions(-) diff --git a/src/agent_env/env/gateway/constants.py b/src/agent_env/env/gateway/constants.py index 37c8326d1..7d0ff0ef6 100644 --- a/src/agent_env/env/gateway/constants.py +++ b/src/agent_env/env/gateway/constants.py @@ -210,10 +210,10 @@ def random_mcp_server_name() -> str: }, { "uri": EXT_TRIGGERS_URI, - "description": "Register (additive add-by-id), remove, clear, or read (state) typed Action/State triggers + the firing log (harness control path). config sets the engine's opt-in settings: concurrent_fires (a repeat trigger whose actions are all contact replies (nl) or notifications (notify) runs its fires side by side instead of queueing) and executor_timeout_seconds (the time limit of an executor that sets none).", + "description": "Register (additive add-by-id), remove, clear, or read (state) typed Action/State triggers + the firing log (harness control path). config sets the engine's opt-in setting concurrent_fires (a repeat trigger whose actions are all contact replies (nl) or notifications (notify) runs its fires side by side instead of queueing).", "params": {"endpoint": "/triggers/register", "features": list(TRIGGER_FEATURES), "methods": { "register": {"method": "POST", "endpoint": "/triggers/register", "request": {"type": "object", "properties": {"watch_roles": {"type": "array", "items": {"type": "string"}}, "executor": {"type": "object"}, "executors": {"type": "object"}, "triggers": {"type": "array"}}, "required": ["triggers"]}}, - "config": {"method": "POST", "endpoint": "/triggers/config", "request": {"type": "object", "properties": {"concurrent_fires": {"type": "boolean"}, "executor_timeout_seconds": {"type": "number", "exclusiveMinimum": 0}}}}, + "config": {"method": "POST", "endpoint": "/triggers/config", "request": {"type": "object", "properties": {"concurrent_fires": {"type": "boolean"}}}}, "remove": {"method": "POST", "endpoint": "/triggers/remove", "request": {"type": "object", "properties": {"ids": {"type": "array", "items": {"type": "string"}}}, "required": ["ids"]}}, "clear": {"method": "POST", "endpoint": "/triggers/clear", "request": {"type": "object", "properties": {}}}, "state": {"method": "GET", "endpoint": "/triggers/state", "request": {"type": "object", "properties": {}}}}}, diff --git a/src/agent_env/env/gateway/triggers.py b/src/agent_env/env/gateway/triggers.py index 223780698..fa3f1e1d8 100644 --- a/src/agent_env/env/gateway/triggers.py +++ b/src/agent_env/env/gateway/triggers.py @@ -449,8 +449,6 @@ def __init__(self, gateway): self._clock_moved = asyncio.Event() # set, and replaced, when the clock changes under waiting actions # Opt-in settings (`config`), off by default so a registration behaves as it always has. self._concurrent_fires = False # a contact-reply trigger's repeat fires run side by side instead of queueing - self._executor_timeout_default: float | None = None # the time limit of an executor that sets none - self._unset_timeouts: set[str | None] = set() # executors registered without a timeout_seconds of their own # One turn at a time per executor conversation (its URL and context id): fires of one trigger running side by # side would otherwise resume the same session at once. self._executor_locks: dict[tuple[str, str], asyncio.Lock] = {} @@ -461,7 +459,7 @@ def register(self, body: Any) -> dict: if not isinstance(body, dict): raise TriggerError("body must be a JSON object") watch_roles = self._resolve_watch_roles(body) - executors, unset_timeouts = self._resolve_executors(body, watch_roles) + executors = self._resolve_executors(body, watch_roles) triggers = body.get("triggers") if not isinstance(triggers, list): raise TriggerError("triggers must be a list") @@ -493,7 +491,6 @@ def register(self, body: Any) -> dict: self._readonly_tools = None self._watch_roles = watch_roles self._executors = executors - self._unset_timeouts |= unset_timeouts self._config_set = True for tid, spec in to_add: # `running` counts its fires in flight, `alongside` holds those run beside the first (concurrent_fires), @@ -575,8 +572,6 @@ def clear(self) -> dict: self._config_set = False self._readonly_tools = None self._concurrent_fires = False - self._executor_timeout_default = None - self._unset_timeouts = set() self._executor_locks = {} self._recompute_time_flag() return {"ok": True} @@ -589,33 +584,20 @@ def config(self, body: Any) -> dict: and then write the same rows. A queued fire draws its delay only when its turn comes, and state shows no due time for it, so a quick reply queued behind a slow one would wait out the slow one's delay too. Fires of one trigger side by side still take turns on its executor conversation (``_executor_turn``). - ``executor_timeout_seconds`` is the time limit of each executor whose registration set no - ``timeout_seconds`` (else 120). ``clear`` resets both.""" + ``clear`` resets it.""" if not isinstance(body, dict): raise TriggerError("body must be a JSON object") - unknown = sorted(set(body) - {"concurrent_fires", "executor_timeout_seconds"}) + unknown = sorted(set(body) - {"concurrent_fires"}) if unknown: - raise TriggerError(f"unknown setting(s) {unknown}; expected concurrent_fires, executor_timeout_seconds") + raise TriggerError(f"unknown setting(s) {unknown}; expected concurrent_fires") concurrent = body.get("concurrent_fires", False) if not isinstance(concurrent, bool): raise TriggerError(f"concurrent_fires must be true or false (got {concurrent!r})") - timeout = body.get("executor_timeout_seconds") - if timeout is not None and (isinstance(timeout, bool) or not isinstance(timeout, (int, float)) - or not math.isfinite(timeout) or timeout <= 0): - raise TriggerError(f"executor_timeout_seconds must be a positive, finite number (got {timeout!r})") self._concurrent_fires = concurrent - self._executor_timeout_default = None if timeout is None else float(timeout) return {"ok": True, **self._settings()} def _settings(self) -> dict: - return {"concurrent_fires": self._concurrent_fires, - "executor_timeout_seconds": self._executor_timeout_default} - - def _executor_timeout(self, name: str | None, executor: dict) -> float: - """The executor's time limit: its own, else the configured default, else 120.""" - if self._executor_timeout_default is not None and name in self._unset_timeouts: - return self._executor_timeout_default - return executor["timeout_seconds"] + return {"concurrent_fires": self._concurrent_fires} def _recompute_time_flag(self) -> None: # "firing" counts: a recurrence mid-fire returns to "armed" and still needs the driver. @@ -674,15 +656,14 @@ def _resolve_watch_roles(self, body: dict) -> set[str]: raise TriggerError(f"watch_roles {sorted(new)} conflicts with the already-registered {sorted(self._watch_roles)} — clear to change it") return new - def _resolve_executors(self, body: dict, watch_roles: set[str]) -> tuple[dict[str | None, dict], set[str | None]]: + def _resolve_executors(self, body: dict, watch_roles: set[str]) -> dict[str | None, dict]: """The executors once this register lands: those before, plus the body's `executor` (under None) and - `executors`, each set once; and those of them new here that set no `timeout_seconds` of their own.""" + `executors`, each set once.""" named = body.get("executors") if named is not None and (not isinstance(named, dict) or not all(isinstance(n, str) and n for n in named)): raise TriggerError("executors must be an object keyed by non-empty executor names") incoming = ({None: body["executor"]} if body.get("executor") is not None else {}) | (named or {}) merged = dict(self._executors) - unset: set[str | None] = set() for name, raw in incoming.items(): normalized = self._normalize_executor(raw, "executor" if name is None else f"executors[{name!r}]", watch_roles) @@ -691,10 +672,8 @@ def _resolve_executors(self, body: dict, watch_roles: set[str]) -> tuple[dict[st if name is None else f"executor {name!r} conflicts with the already-registered executor of that name — " "clear to change it") - if name not in merged and raw.get("timeout_seconds") is None: - unset.add(name) merged[name] = normalized - return merged, unset + return merged @staticmethod def _normalize_executor(raw: Any, where: str, watch_roles: set[str]) -> dict: @@ -961,7 +940,11 @@ def state(self) -> dict: return { "config": {"watch_roles": sorted(self._watch_roles), "executor_configured": None in self._executors, - "executors": sorted(n for n in self._executors if n is not None)}, + "executors": sorted(n for n in self._executors if n is not None), + # Each executor's time limit (``name`` null for the unnamed one), for a harness that waits on + # its calls. + "executor_timeouts": [{"name": n, "timeout_seconds": e["timeout_seconds"]} + for n, e in sorted(self._executors.items(), key=lambda i: i[0] or "")]}, "settings": self._settings(), # what `config` set # `running` counts a trigger's fires in flight, `waits` says when each of them waiting out a delay or its # executor's hours goes on, and `executor_calls` lists each call to an executor in progress, with the @@ -1651,9 +1634,7 @@ async def _executor_turn(self, tid: str, trig: dict, index: int, name: str | Non context_id = f"trigger-{tid}" if executor is None: return await self._call_executor(executor, instruction, context_id=context_id) - timeout = self._executor_timeout(name, executor) - if timeout != executor["timeout_seconds"]: - executor = {**executor, "timeout_seconds": timeout} + timeout = executor["timeout_seconds"] lock = self._executor_locks.setdefault((executor["a2a_url"], context_id), asyncio.Lock()) async with lock: fires = self._in_flight.setdefault(executor["role"], []) diff --git a/src/agent_env/task_step/task_steps/always_on.py b/src/agent_env/task_step/task_steps/always_on.py index 84947bbe9..19e1d75e5 100644 --- a/src/agent_env/task_step/task_steps/always_on.py +++ b/src/agent_env/task_step/task_steps/always_on.py @@ -28,9 +28,10 @@ (clock/v1 set_rate), so settles and reads don't move it, and puts the task's rate back at the end. When nothing is due, the step advances it to the earliest of: the next trigger event, the next notification, a held reply, a pending read, or end_time, never past one; then waits for the world to settle. A contact's reply (an AI-played contact, a trigger's executor) counts as in - progress until it ends or its time limit passes; other trigger work gets ``settle_timeout_seconds``. Contacts whose - task sets no time limit get ``contact_timeout_seconds``, and one trigger's replies run side by side - (``contact_replies_in_parallel``). What comes due in the same minute wakes the agent once. + progress until it ends or its time limit passes; other trigger work gets ``settle_timeout_seconds``. A contact's + time limit is the one it was registered with (register_env_triggers' ``executor_timeout_seconds``), and one + trigger's replies run side by side (``contact_replies_in_parallel``). What comes due in the same minute wakes the + agent once. - The agent can schedule its own wake-ups (scheduled checks) when the run is multi-day, at least ``scheduled_check_min_interval`` apart; ``scheduled_checks`` turns them on or off whatever the run's length. A check's wake counts toward ``max_wakes``. @@ -66,8 +67,8 @@ - The agent: the message_user rules go in its system prompt when the step sets one and the agent's card takes it, else on its first wake; its session is the step's ``context_id`` when set, as prompt_agent's is, else the run's own. - When the step ends it puts back the gateway settings it changed for the run (the user chat's roles and stay_quiet, - the triggers' concurrent_fires and executor time limit, the roles with scheduled checks, the clock's rate), so the - task's later steps see them as they were. + the triggers' concurrent_fires, the roles with scheduled checks, the clock's rate), so the task's later steps see + them as they were. """ from __future__ import annotations @@ -135,8 +136,10 @@ JOB_LIMIT_SECONDS = 8 * 3600 JOB_HEADROOM_SECONDS = 600 DEFAULT_TIME_BUDGET_SECONDS = 7 * 3600 -# The time limit of a contact (a trigger's executor) whose task sets none; the gateway's own default is 120 s. -CONTACT_TIMEOUT_SECONDS = 300.0 +# The time limit of a contact (a trigger's executor) is the one it was registered with, read from each env's triggers +# state when the run starts. Until then (a step's settings are checked when the task loads), and for a gateway that +# doesn't report it, the step takes the gateway's default for an executor registered without one. +CONTACT_TIMEOUT_DEFAULT_SECONDS = 120.0 # Past a contact's time limit, how long the settle wait still gives its call to come back. CONTACT_GRACE_SECONDS = 30.0 # The hard caps that bound one turn of the loop (wake_reserve_seconds adds them up): past an agent's or user sim's turn @@ -209,7 +212,6 @@ def __init__(self, id: str, version: Optional[int], *, user_sim: str, end_time: stay_quiet: bool = True, nudge: bool = True, nudge_text: str = NUDGE_TEXT, wake_reminder: bool = False, wake_reminder_text: str = WAKE_REMINDER_TEXT, scheduled_checks: Optional[bool] = None, scheduled_check_min_interval: str = CHECK_MIN_INTERVAL, - contact_timeout_seconds: Optional[float] = CONTACT_TIMEOUT_SECONDS, contact_replies_in_parallel: bool = True, time_budget_seconds: Optional[float] = None, model: Optional[str] = None, model_params: Optional[dict] = None, effort: Optional[str] = None, max_thinking_tokens: Optional[int] = None, harness: Optional[str] = None, @@ -238,43 +240,41 @@ def __init__(self, id: str, version: Optional[int], *, user_sim: str, end_time: self.wake_reminder_text = wake_reminder_text self.scheduled_checks = scheduled_checks self.scheduled_check_min_interval = scheduled_check_min_interval - if contact_timeout_seconds is not None and not float(contact_timeout_seconds) > 0: - raise ValueError(f"contact_timeout_seconds must be positive (got {contact_timeout_seconds!r})") - self.contact_timeout_seconds = None if contact_timeout_seconds is None else float(contact_timeout_seconds) self.contact_replies_in_parallel = bool(contact_replies_in_parallel) self.time_budget_seconds = None if time_budget_seconds is None else float(time_budget_seconds) self.effective_time_budget() # validates _OWN = ("user_sim", "end_time", "final", "max_wakes", "settle_timeout_seconds", "user_timeout_seconds", "final_reminder", "stay_quiet", "nudge", "nudge_text", "wake_reminder", "wake_reminder_text", - "scheduled_checks", "scheduled_check_min_interval", "contact_timeout_seconds", "contact_replies_in_parallel", - "time_budget_seconds") + "scheduled_checks", "scheduled_check_min_interval", "contact_replies_in_parallel", "time_budget_seconds") _PROMPT_FIELDS = ("prompt", "parts", "prompt_id", "agent_name", "timeout_seconds", "trajectory_output_prefix", "model", "model_params", "system_prompt", "max_turns", "output_format", "max_thinking_tokens", "effort", "harness", "agentenv_tools", "poll_interval_seconds", "context_id") - def settle_cap_seconds(self) -> float: - """The longest one settle waits, whatever is still at work (queued fires, nl actions in a row, a contact whose - own time limit is longer): a contact's time limit and grace, then settle_timeout_seconds.""" - contact = self.contact_timeout_seconds or CONTACT_TIMEOUT_SECONDS - return contact + CONTACT_GRACE_SECONDS + self.settle_timeout_seconds + def settle_cap_seconds(self, contact_seconds: float = CONTACT_TIMEOUT_DEFAULT_SECONDS) -> float: + """The longest one settle waits, whatever is still at work (queued fires, nl actions in a row): the longest + contact time limit (``contact_seconds``: the run's, read when it starts, else the gateway's default) and its + grace, then settle_timeout_seconds.""" + return contact_seconds + CONTACT_GRACE_SECONDS + self.settle_timeout_seconds def read_cap_seconds(self) -> float: """The longest one read's answer may take: READ_ASKS user sim turns, each with its grace and trajectory fetch; an ask past it is refused, and an answer past it ends the read as a user sim error.""" return READ_ASKS * (self.user_timeout_seconds + TURN_GRACE_SECONDS + TRAJECTORY_FETCH_SECONDS) - def wake_reserve_seconds(self) -> float: + def wake_reserve_seconds(self, contact_seconds: float = CONTACT_TIMEOUT_DEFAULT_SECONDS) -> float: """The longest one turn of the loop, or the run's closing, can take, each part hard-capped: a clock jump's settle (settle_cap_seconds), then a wake (the agent's turn and, with nudges on, its nudge turn, each at most timeout_seconds + TURN_GRACE_SECONDS plus a trajectory fetch; one settle; GATEWAY_ALLOWANCE_SECONDS of the step's own calls), then a read (read_cap_seconds and GATEWAY_ALLOWANCE_SECONDS). The loop cuts off a turn of - its own that runs past this (a step error), so it is a real bound.""" + its own that runs past this (a step error), so it is a real bound. ``contact_seconds`` is the longest contact + time limit, as for settle_cap_seconds.""" turn = self.timeout_seconds + TURN_GRACE_SECONDS + TRAJECTORY_FETCH_SECONDS - wake = turn * (2 if self.nudge else 1) + self.settle_cap_seconds() + GATEWAY_ALLOWANCE_SECONDS - return self.settle_cap_seconds() + wake + self.read_cap_seconds() + GATEWAY_ALLOWANCE_SECONDS + settle = self.settle_cap_seconds(contact_seconds) + wake = turn * (2 if self.nudge else 1) + settle + GATEWAY_ALLOWANCE_SECONDS + return settle + wake + self.read_cap_seconds() + GATEWAY_ALLOWANCE_SECONDS - def effective_time_budget(self) -> float: + def effective_time_budget(self, contact_seconds: float = CONTACT_TIMEOUT_DEFAULT_SECONDS) -> float: """The real time after which the run closes: ``time_budget_seconds``, else 7 h, lowered so the closing still fits the hub's job. The rule: @@ -285,8 +285,11 @@ def effective_time_budget(self) -> float: step's wrap-up line and the last-turn notice), lets the user sim read the wrap-up, and ends with end reason ``time_budget``. A run therefore ends by the budget plus one wake's worst case, and that must fit the job with ``JOB_HEADROOM_SECONDS`` to spare: the default budget is lowered to make it fit, and a budget a task sets - that can't is refused, as is a wake whose worst case leaves no room for a budget at all.""" - reserve = self.wake_reserve_seconds() + that can't is refused, as is a wake whose worst case leaves no room for a budget at all. + + ``contact_seconds`` is the longest contact time limit: the gateway's default when the task loads, the + contacts' own once the run has read them (``_configure_contacts``), which checks the budget again.""" + reserve = self.wake_reserve_seconds(contact_seconds) room = JOB_LIMIT_SECONDS - JOB_HEADROOM_SECONDS - reserve if room <= reserve: raise ValueError(f"always_on: one wake can take up to {reserve:.0f}s (wake_reserve_seconds: a settle, the " @@ -368,6 +371,7 @@ def __init__(self, step: AlwaysOnTaskStep, context: TaskStepContext): self.checks: dict = {"on": False} self.signed_off = False # a one-sitting run's user said done: no more reads self.contacts: dict = {} + self.contact_seconds = CONTACT_TIMEOUT_DEFAULT_SECONDS # the longest contact time limit, read at the start self.last_settle: dict = {} self.pending: dict = {} self.end_note = "" @@ -384,9 +388,8 @@ def __init__(self, step: AlwaysOnTaskStep, context: TaskStepContext): async def restore(self) -> None: """Put back what the step changed on the gateways, latest first, so later steps of the task see the settings - they had before it: the user chat's roles, user name and stay_quiet; the triggers' concurrent_fires and - executor time limit; the roles with scheduled checks; the clock's rate. Each is tried on its own; one that - fails is logged.""" + they had before it: the user chat's roles, user name and stay_quiet; the triggers' concurrent_fires; the roles + with scheduled checks; the clock's rate. Each is tried on its own; one that fails is logged.""" while self.undo: what, put_back = self.undo.pop() try: @@ -632,28 +635,32 @@ async def _agent_turn(self, parts: list[dict], msg_id: str, record) -> tuple[str await self._clock_runs(False) async def _configure_contacts(self) -> None: - """On each env with triggers, let one trigger's replies run side by side (``contact_replies_in_parallel``) and - give contacts that set no time limit ``contact_timeout_seconds``; a gateway too old for it keeps its ways.""" + """On each env with triggers, let one trigger's replies run side by side (``contact_replies_in_parallel``; a + gateway too old for it keeps its ways), and read its contacts' time limits: each executor's own, as the task + registered it (register_env_triggers' ``executor_timeout_seconds``). The longest (``contact_seconds``; the + gateway's default for a gateway that doesn't report them) caps every settle, and with it one wake's worst case, + so the run's time budget is worked out again from it, and refused when that can't fit the hub's job.""" step = self.step - body = {"concurrent_fires": step.contact_replies_in_parallel} - if step.contact_timeout_seconds is not None: - body["executor_timeout_seconds"] = step.contact_timeout_seconds - applied = [] + applied, limits = [], [] for env in self.ctx.deployed_envs: if not env.supports(EXT_TRIGGERS_URI, "state"): continue + state = await _call(env, EXT_TRIGGERS_URI, "state") + timeouts = (state.get("config") or {}).get("executor_timeouts") + limits += [CONTACT_TIMEOUT_DEFAULT_SECONDS] if timeouts is None else [t["timeout_seconds"] for t in timeouts] + entry = {"env_id": env.env_id, "contact_timeouts": timeouts} # None: the gateway doesn't report them if not env.supports(EXT_TRIGGERS_URI, "config"): - applied.append({"env_id": env.env_id, "applied": False, "why": "the gateway has no triggers config"}) + applied.append({**entry, "applied": False, "why": "the gateway has no triggers config"}) continue - before = (await _call(env, EXT_TRIGGERS_URI, "state")).get("settings") or {} + before = state.get("settings") or {} self._undo_config(env, EXT_TRIGGERS_URI, "the triggers' settings", - {k: v for k, v in before.items() - if k in ("concurrent_fires", "executor_timeout_seconds") and v is not None}) - got = await _call(env, EXT_TRIGGERS_URI, "config", body) - applied.append({"env_id": env.env_id, "applied": True, "concurrent_fires": got.get("concurrent_fires"), - "executor_timeout_seconds": got.get("executor_timeout_seconds")}) + {k: v for k, v in before.items() if k == "concurrent_fires" and v is not None}) + got = await _call(env, EXT_TRIGGERS_URI, "config", {"concurrent_fires": step.contact_replies_in_parallel}) + applied.append({**entry, "applied": True, "concurrent_fires": got.get("concurrent_fires")}) + self.contact_seconds = float(max(limits, default=0.0)) + self.time_budget = step.effective_time_budget(self.contact_seconds) self.contacts = {"replies_in_parallel": step.contact_replies_in_parallel, - "timeout_seconds": step.contact_timeout_seconds, "envs": applied} + "longest_timeout_seconds": self.contact_seconds, "envs": applied} def rules(self) -> str: """What the agent is told about reaching the user, and about waking itself when it can.""" @@ -769,11 +776,11 @@ async def settle(self) -> list[dict]: ``CONTACT_GRACE_SECONDS``); any other work, an event due that an advance to now doesn't set going among it, gets ``settle_timeout_seconds``, from the start or from the last contact call seen. A gateway that reports no executor calls gets ``settle_timeout_seconds`` in all. However much is still at work (queued fires, nl - actions in a row, a contact whose own limit is longer), the wait never runs past ``settle_cap_seconds``. + actions in a row), the wait never runs past ``settle_cap_seconds`` of the longest contact time limit. Returns, by env, the triggers still at work when it gave up (empty when the world settled); ``last_settle`` says how long it waited and how many contact calls it saw.""" started = time.monotonic() - cap = started + self.step.settle_cap_seconds() # however much is still at work, never past this + cap = started + self.step.settle_cap_seconds(self.contact_seconds) # however much is at work, never past this envs = [e for e in await self._armed() if e.supports(EXT_TRIGGERS_URI, "state")] waiting = {i: {"live_call_seen": started, "contact_until": started} for i in range(len(envs))} stuck: list[dict] = [] @@ -951,7 +958,7 @@ async def _capped(self, work, what: str, seconds: Optional[float] = None): """``work``, cut off past ``seconds`` (one turn of the loop's worst case, ``wake_reserve_seconds``): every part is capped on its own, so only calls hanging past their allowance get here, and the run ends on a step error rather than run past the hub's job.""" - seconds = self.step.wake_reserve_seconds() if seconds is None else seconds + seconds = self.step.wake_reserve_seconds(self.contact_seconds) if seconds is None else seconds timer = asyncio.timeout(seconds) try: async with timer: @@ -979,9 +986,10 @@ async def _next(self) -> bool: step = self.step now = await self.now() spent = self.spent() - if spent + step.wake_reserve_seconds() >= self.time_budget: + reserve = step.wake_reserve_seconds(self.contact_seconds) + if spent + reserve >= self.time_budget: self.log("time_budget_reached", virtual_time=_iso(now), real_seconds=round(spent, 1), - time_budget_seconds=self.time_budget, wake_reserve_seconds=step.wake_reserve_seconds()) + time_budget_seconds=self.time_budget, wake_reserve_seconds=reserve) await self._final(now, reason="time_budget") return False # 1. a read that is due: the user sim reads what the agent sent (one due only at or past end_time is the last @@ -1650,7 +1658,7 @@ async def finish(self) -> None: "rules_via": "system_prompt" if self.rules_in_system_prompt else "first_wake", "scheduled_checks": self.checks, "contacts": self.contacts, "time_budget_seconds": self.time_budget, - "wake_reserve_seconds": step.wake_reserve_seconds()} + "wake_reserve_seconds": step.wake_reserve_seconds(self.contact_seconds)} await self._store_timeline(record) ctx.metadata.setdefault(ALWAYS_ON_KEY, {})[step.id] = record # The graded response is the user's whole chat from the agent, each message with its pretend time; never its diff --git a/tst/unit/env/gateway/triggers_fires_test.py b/tst/unit/env/gateway/triggers_fires_test.py index 2df5d56b6..5f505b52c 100644 --- a/tst/unit/env/gateway/triggers_fires_test.py +++ b/tst/unit/env/gateway/triggers_fires_test.py @@ -303,17 +303,16 @@ async def fire(ticket): assert trig["running"] == 0 -# ---- executor_timeout_seconds --------------------------------------------------------------------------------------- +# ---- executor time limits ------------------------------------------------------------------------------------------- @pytest.mark.asyncio -async def test_the_configured_limit_applies_only_to_executors_that_set_none(monkeypatch): +async def test_each_executor_keeps_the_time_limit_it_was_registered_with_and_state_reports_it(monkeypatch): bob = _Executor(monkeypatch) bob.release() engine, _, _ = _engine() - engine.config({"executor_timeout_seconds": 300}) engine.register({"watch_roles": ["default"], "executors": {"bob": {"a2a_url": "http://bob", "role": "bob"}, - "ann": {"a2a_url": "http://ann", "role": "ann", "timeout_seconds": 120}}, + "ann": {"a2a_url": "http://ann", "role": "ann", "timeout_seconds": 300}}, "triggers": [{"id": tid, "when": {"type": "action", "tool": "send_text"}, "actions": [{"type": "nl", "executor": tid, "instruction": "answer"}]} for tid in ("bob", "ann")]}) @@ -321,23 +320,22 @@ async def test_the_configured_limit_applies_only_to_executors_that_set_none(monk await _until(lambda: len(bob.calls) == 2) assert sorted((c["context_id"], c["timeout_seconds"]) for c in bob.calls) == [ - ("trigger-ann", 120), ("trigger-bob", 300)] - assert engine.state()["settings"] == {"concurrent_fires": False, "executor_timeout_seconds": 300.0} + ("trigger-ann", 300), ("trigger-bob", 120)] + assert engine.state()["config"]["executor_timeouts"] == [{"name": "ann", "timeout_seconds": 300.0}, + {"name": "bob", "timeout_seconds": 120.0}] -@pytest.mark.asyncio -async def test_without_config_an_executor_that_sets_no_limit_keeps_120_s(monkeypatch): - bob = _Executor(monkeypatch) - bob.release() +def test_state_lists_the_unnamed_executor_s_time_limit_first(): engine, _, _ = _engine() - engine.register(_reply_trigger()) - engine.on_tool_call("default", "send_text", {}, _result()) - await _until(lambda: bob.calls) - assert bob.calls[0]["timeout_seconds"] == 120 + assert engine.state()["config"]["executor_timeouts"] == [] + engine.register({"watch_roles": ["default"], "executor": {"a2a_url": "http://exec", "role": "exec"}, + "executors": {"ann": {"a2a_url": "http://ann", "role": "ann", "timeout_seconds": 45}}, + "triggers": []}) + assert engine.state()["config"]["executor_timeouts"] == [{"name": None, "timeout_seconds": 120.0}, + {"name": "ann", "timeout_seconds": 45.0}] -@pytest.mark.parametrize("body", [{"concurrent_fires": "yes"}, {"executor_timeout_seconds": 0}, - {"executor_timeout_seconds": True}, {"executor_timeout_seconds": float("inf")}, +@pytest.mark.parametrize("body", [{"concurrent_fires": "yes"}, {"executor_timeout_seconds": 300}, {"parallel": True}, []]) def test_bad_settings_are_refused(body): engine, _, _ = _engine() @@ -347,12 +345,11 @@ def test_bad_settings_are_refused(body): def test_settings_are_off_until_set_and_clear_resets_them(): engine, _, _ = _engine() - assert engine.state()["settings"] == {"concurrent_fires": False, "executor_timeout_seconds": None} - assert engine.config({"concurrent_fires": True, "executor_timeout_seconds": 300})["concurrent_fires"] is True - engine.config({"concurrent_fires": True}) - assert engine.state()["settings"] == {"concurrent_fires": True, "executor_timeout_seconds": None} + assert engine.state()["settings"] == {"concurrent_fires": False} + assert engine.config({"concurrent_fires": True})["concurrent_fires"] is True + assert engine.state()["settings"] == {"concurrent_fires": True} engine.clear() - assert engine.state()["settings"] == {"concurrent_fires": False, "executor_timeout_seconds": None} + assert engine.state()["settings"] == {"concurrent_fires": False} def test_the_card_offers_config(): diff --git a/tst/unit/env/gateway/triggers_test.py b/tst/unit/env/gateway/triggers_test.py index 6a56d57af..6b059ad8f 100644 --- a/tst/unit/env/gateway/triggers_test.py +++ b/tst/unit/env/gateway/triggers_test.py @@ -292,7 +292,7 @@ def test_clear_full_reset(engine): engine.clear() st = engine.state() assert st["triggers"] == [] and st["events"] == [] - assert st["config"] == {"watch_roles": [], "executor_configured": False, "executors": []} + assert st["config"] == {"watch_roles": [], "executor_configured": False, "executors": [], "executor_timeouts": []} # after clear, config is settable fresh (different watch_roles allowed) engine.register({"watch_roles": ["solver"], "triggers": []}) assert engine.state()["config"]["watch_roles"] == ["solver"] @@ -1050,7 +1050,8 @@ async def test_sensor_trigger_and_state_shape(engine): assert engine._triggers["sensor"]["status"] == "fired" state = engine.state() assert "sensor" in {t["id"] for t in state["triggers"]} - assert state["config"] == {"watch_roles": ["default"], "executor_configured": False, "executors": []} + assert state["config"] == {"watch_roles": ["default"], "executor_configured": False, "executors": [], + "executor_timeouts": []} assert state["events"][0]["kind"] == "added" assert all("seq" in e and "ts" in e for e in state["events"]) diff --git a/tst/unit/task_step/test_always_on_agent_and_gateway.py b/tst/unit/task_step/test_always_on_agent_and_gateway.py index 26009cb06..4a42f3ad2 100644 --- a/tst/unit/task_step/test_always_on_agent_and_gateway.py +++ b/tst/unit/task_step/test_always_on_agent_and_gateway.py @@ -72,16 +72,15 @@ async def test_the_gateway_settings_the_step_changed_are_as_they_were_after_the_ await rig.arm() gw = rig.gw gw._user_chat.configure({"roles": ["planner"], "user_name": "Tess"}) - gw._trigger_engine.config({"executor_timeout_seconds": 90}) gw._notifications.configure({"roles": ["planner"], "min_interval": "PT2H", "max_active": 4}) await rig.run(rig.step(end_time="P2D")) during = rig.during[0] assert during["user_chat_roles"] == ("default",) and during["stay_quiet"] - assert during["trigger_settings"] == {"concurrent_fires": True, "executor_timeout_seconds": 300.0} + assert during["trigger_settings"] == {"concurrent_fires": True} assert during["check_roles"] == ("planner", "default") and during["clock_rate"] == 60 assert (gw._user_chat.roles, gw._user_chat.user_name, gw._user_chat.stay_quiet) == (("planner",), "Tess", False) - assert gw._trigger_engine.state()["settings"] == {"concurrent_fires": False, "executor_timeout_seconds": 90.0} + assert gw._trigger_engine.state()["settings"] == {"concurrent_fires": False} assert (gw._notifications.roles, gw._notifications.min_interval, gw._notifications.max_active) == ( ("planner",), "PT2H", 4) assert gw._clock.rate == 60 @@ -95,5 +94,5 @@ async def test_a_gateway_the_task_left_alone_is_back_to_its_defaults(monkeypatch gw = rig.gw assert (gw._user_chat.roles, gw._user_chat.stay_quiet, gw._notifications.roles) == ((), False, ()) - assert gw._trigger_engine.state()["settings"] == {"concurrent_fires": False, "executor_timeout_seconds": None} + assert gw._trigger_engine.state()["settings"] == {"concurrent_fires": False} assert not {"message_user", "stay_quiet", "schedule_check"} & await rig.tools() diff --git a/tst/unit/task_step/test_always_on_budget.py b/tst/unit/task_step/test_always_on_budget.py index 602792846..58e9d50cc 100644 --- a/tst/unit/task_step/test_always_on_budget.py +++ b/tst/unit/task_step/test_always_on_budget.py @@ -34,11 +34,13 @@ def agent(n, texts, rig): await rig.arm() for day in range(10, 20): rig.triggers(rig.notify_at(f"alert-{day}", f"2026-11-{day}T14:00:00Z", "Sam: news?", channel="noor 30")) - step = rig.step(time_budget_seconds=12120) # reserve 4380 s: closes once 7740 s are spent + step = rig.step(time_budget_seconds=12120) # no contacts, so a reserve of 3780 s: closes once 8340 s are spent ctx = await rig.run(step) record = rig.record(ctx) - assert step.wake_reserve_seconds() == 4380 and record["settings"]["time_budget_seconds"] == 12120 + assert record["settings"]["contacts"]["longest_timeout_seconds"] == 0 + assert record["settings"]["wake_reserve_seconds"] == step.wake_reserve_seconds(0) == 3780 + assert record["settings"]["time_budget_seconds"] == 12120 wakes = rig.events(ctx, "wake") assert len(wakes) == 4 and wakes[-1]["woke_by"] == ["step:final"] and wakes[-1]["last"] is True assert wakes[-1]["virtual_time"] == "2026-11-11T14:00:00Z" # where the run was; the clock doesn't jump @@ -51,7 +53,7 @@ def agent(n, texts, rig): "its real-time budget.") -@pytest.mark.parametrize("timeout,hours", [(600, 6.62), (1800, 5.95), (3600, 4.95)]) +@pytest.mark.parametrize("timeout,hours", [(600, 6.72), (1800, 6.05), (3600, 5.05)]) def test_the_default_budget_is_7_h_lowered_so_the_closing_wake_fits_the_8_h_job(timeout, hours): step = _step(timeout_seconds=timeout) budget = step.effective_time_budget() @@ -80,47 +82,100 @@ def test_the_budget_survives_a_round_trip(): # ---- the reserve is a real bound ------------------------------------------------------------------------------------- +@pytest.mark.parametrize("contact", [ao.CONTACT_TIMEOUT_DEFAULT_SECONDS, 0, 900], ids=["default", "none", "900 s"]) @pytest.mark.parametrize("settings", [{}, {"nudge": False}, {"timeout_seconds": 1800, "user_timeout_seconds": 900}, - {"contact_timeout_seconds": 900, "settle_timeout_seconds": 600}, - {"time_budget_seconds": 5 * 3600}]) -def test_the_reserve_adds_up_every_capped_part_and_the_budget_plus_one_turn_fits_the_job(settings): + {"settle_timeout_seconds": 600}, {"time_budget_seconds": 5 * 3600}]) +def test_the_reserve_adds_up_every_capped_part_and_the_budget_plus_one_turn_fits_the_job(settings, contact): step = _step(**settings) - settle = (step.contact_timeout_seconds or ao.CONTACT_TIMEOUT_SECONDS) + ao.CONTACT_GRACE_SECONDS \ - + step.settle_timeout_seconds + settle = contact + ao.CONTACT_GRACE_SECONDS + step.settle_timeout_seconds turn = step.timeout_seconds + ao.TURN_GRACE_SECONDS + ao.TRAJECTORY_FETCH_SECONDS read = ao.READ_ASKS * (step.user_timeout_seconds + ao.TURN_GRACE_SECONDS + ao.TRAJECTORY_FETCH_SECONDS) wake = turn * (2 if step.nudge else 1) + settle + ao.GATEWAY_ALLOWANCE_SECONDS - assert step.settle_cap_seconds() == settle and step.read_cap_seconds() == read + assert step.settle_cap_seconds(contact) == settle and step.read_cap_seconds() == read # a jump's settle, then a wake (its turns, its settle, the step's calls), then a read and its calls - assert step.wake_reserve_seconds() == settle + wake + read + ao.GATEWAY_ALLOWANCE_SECONDS - assert step.effective_time_budget() + step.wake_reserve_seconds() + ao.JOB_HEADROOM_SECONDS <= ao.JOB_LIMIT_SECONDS + reserve = step.wake_reserve_seconds(contact) + assert reserve == settle + wake + read + ao.GATEWAY_ALLOWANCE_SECONDS + assert step.effective_time_budget(contact) + reserve + ao.JOB_HEADROOM_SECONDS <= ao.JOB_LIMIT_SECONDS + if contact == ao.CONTACT_TIMEOUT_DEFAULT_SECONDS: # what the task is checked with when it loads + assert (step.settle_cap_seconds(), step.wake_reserve_seconds()) == (settle, reserve) def _estimate(rig, n): rig.alert(f"Live Oak: estimate {n}", channel="Gmail · Live Oak Arbor") +def _ask_live_oak(rig, contact: dict) -> None: + rig.triggers({"id": "estimate", "when": {"type": "time", "at": "2026-11-09T15:00:00Z"}, + "actions": [{"type": "nl", "executor": "liveoak", "instruction": "send your estimate"}]}, + executors={"liveoak": contact}) + + @pytest.mark.asyncio -async def test_a_settle_stops_at_its_cap_though_a_contact_s_own_limit_is_longer(monkeypatch, tmp_path): - """The contact's own limit is 5 s, past contact_timeout_seconds (0.3 s): the settle used to wait for it.""" +async def test_a_settle_is_capped_by_the_limit_the_contact_was_registered_with(monkeypatch, tmp_path): + """The contact was registered with a 0.3 s limit and overruns it: the settle stops at that limit, its grace and + settle_timeout_seconds (0.5 s), not at the gateway's default.""" monkeypatch.setattr(ao, "CONTACT_GRACE_SECONDS", 0.1) rig = Rig(monkeypatch, tmp_path) await rig.arm() - slow = rig.contact("liveoak", seconds=5, reply=_estimate, timeout_seconds=5, keeps_to_its_limit=False) - rig.triggers({"id": "estimate", "when": {"type": "time", "at": "2026-11-09T15:00:00Z"}, - "actions": [{"type": "nl", "executor": "liveoak", "instruction": "send your estimate"}]}, - executors={"liveoak": slow}) - step = rig.step(contact_timeout_seconds=0.3, settle_timeout_seconds=0.1) + _ask_live_oak(rig, rig.contact("liveoak", seconds=5, reply=_estimate, timeout_seconds=0.3, + keeps_to_its_limit=False)) + step = rig.step(settle_timeout_seconds=0.1) try: ctx = await rig.run(step) finally: await rig.close() - assert step.settle_cap_seconds() == pytest.approx(0.5) + assert rig.record(ctx)["settings"]["contacts"]["longest_timeout_seconds"] == pytest.approx(0.3) + assert step.settle_cap_seconds(0.3) == pytest.approx(0.5) [jump] = [j for j in rig.events(ctx, "clock_jump") if j["to"] == "2026-11-09T15:00:00Z"] assert not jump["settled"] and jump["settle"]["seconds"] < 1.5 +@pytest.mark.asyncio +async def test_the_run_s_budget_is_worked_out_again_from_its_contacts_limits(monkeypatch, tmp_path): + rig = Rig(monkeypatch, tmp_path) + await rig.arm() + _ask_live_oak(rig, rig.contact("liveoak", seconds=0, reply=_estimate, timeout_seconds=1800)) + step = rig.step(timeout_seconds=1800) + ctx = await rig.run(step) + + settings = rig.record(ctx)["settings"] + assert settings["contacts"]["longest_timeout_seconds"] == 1800 + assert settings["wake_reserve_seconds"] == step.wake_reserve_seconds(1800) + assert settings["time_budget_seconds"] == step.effective_time_budget(1800) < step.effective_time_budget() + assert settings["time_budget_seconds"] + settings["wake_reserve_seconds"] + ao.JOB_HEADROOM_SECONDS \ + <= ao.JOB_LIMIT_SECONDS + + +@pytest.mark.asyncio +async def test_a_contact_whose_limit_leaves_no_room_for_a_run_is_refused_when_the_run_starts(monkeypatch, tmp_path): + rig = Rig(monkeypatch, tmp_path) + await rig.arm() + _ask_live_oak(rig, rig.contact("liveoak", seconds=0, reply=_estimate, timeout_seconds=2 * 3600)) + with pytest.raises(ValueError, match="no run fits the hub's 8 h job"): + await rig.run(rig.step()) + assert rig.sent == [] # before the agent's first wake + + +@pytest.mark.asyncio +async def test_a_gateway_that_doesn_t_report_its_contacts_limits_gets_the_gateway_s_default(monkeypatch, tmp_path): + rig = Rig(monkeypatch, tmp_path) + await rig.arm() + engine = rig.gw._trigger_engine + real_state = engine.state + + def without_limits(): + state = real_state() + return {**state, "config": {k: v for k, v in state["config"].items() if k != "executor_timeouts"}} + + monkeypatch.setattr(engine, "state", without_limits) + ctx = await rig.run(rig.step()) + + contacts = rig.record(ctx)["settings"]["contacts"] + assert contacts["longest_timeout_seconds"] == ao.CONTACT_TIMEOUT_DEFAULT_SECONDS + assert contacts["envs"][0]["contact_timeouts"] is None + + @pytest.mark.asyncio async def test_an_event_due_that_won_t_go_gets_the_settle_timeout_not_forever(monkeypatch, tmp_path): rig = Rig(monkeypatch, tmp_path) @@ -185,7 +240,7 @@ def agent(n, texts, rig): @pytest.mark.asyncio async def test_a_turn_of_the_loop_past_the_reserve_ends_the_run_on_a_step_error(monkeypatch, tmp_path): """Every part is capped on its own; only calls hanging past their allowance reach this last cut-off.""" - monkeypatch.setattr(ao.AlwaysOnTaskStep, "wake_reserve_seconds", lambda step: 0.5) + monkeypatch.setattr(ao.AlwaysOnTaskStep, "wake_reserve_seconds", lambda step, contact_seconds=0: 0.5) rig = Rig(monkeypatch, tmp_path) real_take = ao._Run._take_messages diff --git a/tst/unit/task_step/test_always_on_settle.py b/tst/unit/task_step/test_always_on_settle.py index 25a934198..078bed75c 100644 --- a/tst/unit/task_step/test_always_on_settle.py +++ b/tst/unit/task_step/test_always_on_settle.py @@ -1,6 +1,6 @@ """The always_on step waits for the world to settle by following AI-played contacts' real progress: a contact's reply -is in progress until it lands or its time limit passes, one trigger's replies run side by side, contacts that set no -time limit get 300 s, and replies still in progress at the end are recorded.""" +is in progress until it lands or its time limit passes (the one it was registered with), one trigger's replies run side +by side, and replies still in progress at the end are recorded.""" from __future__ import annotations @@ -97,7 +97,7 @@ def texts_two_friends(n, texts, rig): assert woke == [("2026-11-09T14:30:58", 2)] [env] = rig.record(ctx)["settings"]["contacts"]["envs"] assert env == {"env_id": "world", "applied": True, "concurrent_fires": parallel, - "executor_timeout_seconds": 300.0} + "contact_timeouts": [{"name": "crew", "timeout_seconds": 120.0}]} @pytest.mark.asyncio @@ -134,18 +134,23 @@ def texts_two_hosts(n, texts, rig): @pytest.mark.asyncio -async def test_a_contact_that_sets_no_limit_gets_300_s_and_one_that_does_keeps_its_own(monkeypatch, tmp_path): +async def test_each_contact_keeps_the_limit_it_was_registered_with_and_the_longest_caps_a_settle(monkeypatch, + tmp_path): rig = Rig(monkeypatch, tmp_path) await rig.arm() rig.triggers(_ask_live_oak("2026-11-09T15:00:00Z"), {**_ask_live_oak("2026-11-09T16:00:00Z"), "id": "northwick-quote", "actions": [{"type": "nl", "executor": "northwick", "instruction": "quote Tess"}]}, executors={"liveoak": rig.contact("liveoak", seconds=0, reply=_estimate), - "northwick": rig.contact("northwick", seconds=0, reply=_estimate, timeout_seconds=120)}) - ctx = await rig.run(rig.step()) + "northwick": rig.contact("northwick", seconds=0, reply=_estimate, timeout_seconds=600)}) + step = rig.step() + ctx = await rig.run(step) - assert [(c["contact"], c["timeout_seconds"]) for c in rig.contact_calls] == [("liveoak", 300), ("northwick", 120)] - assert rig.record(ctx)["settings"]["contacts"]["timeout_seconds"] == ao.CONTACT_TIMEOUT_SECONDS == 300 + # liveoak was registered with no limit of its own: the gateway's 120 s + assert [(c["contact"], c["timeout_seconds"]) for c in rig.contact_calls] == [("liveoak", 120), ("northwick", 600)] + settings = rig.record(ctx)["settings"] + assert settings["contacts"]["longest_timeout_seconds"] == 600 + assert settings["wake_reserve_seconds"] == step.wake_reserve_seconds(600) > step.wake_reserve_seconds() @pytest.mark.asyncio @@ -170,10 +175,11 @@ def texts_a_friend(n, texts, rig): assert trigger["waits"] == [{"resumes_at": "2026-11-09T14:30:58.882035Z"}] -def test_the_contact_settings_survive_a_round_trip_and_a_bad_limit_is_refused(): +def test_the_contact_setting_survives_a_round_trip_and_the_step_sets_no_contact_time_limit(): step = ao.AlwaysOnTaskStep("week", None, user_sim="linda", prompt="hi", prompt_id="p", agent_name="a", - contact_timeout_seconds=600, contact_replies_in_parallel=False) - again = ao.AlwaysOnTaskStep.from_dict(step.to_dict()) - assert (again.contact_timeout_seconds, again.contact_replies_in_parallel) == (600.0, False) - with pytest.raises(ValueError, match="contact_timeout_seconds"): - ao.AlwaysOnTaskStep("week", None, user_sim="linda", prompt="hi", agent_name="a", contact_timeout_seconds=0) + contact_replies_in_parallel=False) + stored = step.to_dict() + assert ao.AlwaysOnTaskStep.from_dict(stored).contact_replies_in_parallel is False + assert "contact_timeout_seconds" not in stored # a contact's limit is register_env_triggers' executor_timeout_seconds + with pytest.raises(TypeError, match="contact_timeout_seconds"): + ao.AlwaysOnTaskStep("week", None, user_sim="linda", prompt="hi", agent_name="a", contact_timeout_seconds=600) From 7dda611bd4165a3fd81bbef76a90279d3bdd8cc3 Mon Sep 17 00:00:00 2001 From: Sarah Chitty Date: Wed, 11 Nov 2026 19:49:17 -0800 Subject: [PATCH 52/53] verifiers: always_on_agent_verifier grades an always_on step's prompt; rubrics_verifier no longer spots one rubrics_verifier used to recognise an always_on run itself (_for_always_on, a copy of the step and an always_on import). That moves into a subclass, AlwaysOnAgentVerifier (type always_on_agent_verifier): it grades the prompt from the user's chat, the run's context and the run wake by wake, as before, and refuses when the task loads what needs the wakes' trajectories: an output format that cites per-criterion screenshots, use_trajectory and grade_up_to_turn (a run's grade_up_to_turn is refused when it runs). - rubrics_verifier gains one override point, judge_prompt_sections(context, prompt_response), whose sections go into the judge prompt after the run context: the wake-by-wake summary is per run, so no copy of the step. With no override it adds nothing, so a plain rubrics_verifier grades every prompt exactly as before. - TaskStep.task_problems(steps), checked when a Task loads, refuses steps that don't go together: a plain rubrics_verifier on an always_on step's prompt (use always_on_agent_verifier), and an always_on_agent_verifier on another prompt step's prompt. - is_rubrics_verifier(step_or_config): a registry lookup plus issubclass, for steps and stored dicts, so code that would check the type string recognises the subclass. The explorer's rubric lookups and step colours, the outside agent rules and the field scope test know the new type. Co-Authored-By: Claude Opus 5.5 --- .../ui/src/components/steps-pipeline.tsx | 1 + .../ui/src/components/task-detail-page.tsx | 7 +- .../ui/src/components/task-runner-page.tsx | 7 +- .../ui/src/lib/verifier-classification.ts | 15 ++ src/agent_env/task/episode/rules.py | 1 + src/agent_env/task/task.py | 12 + src/agent_env/task_step/registry.py | 2 + src/agent_env/task_step/task_step.py | 5 + .../task_step/task_steps/always_on.py | 18 +- .../task_steps/verifiers/__init__.py | 3 +- .../verifiers/always_on_agent_verifier.py | 98 ++++++++ .../task_steps/verifiers/rubrics_verifier.py | 63 +++-- .../test_always_on_agent_verifier.py | 223 ++++++++++++++++++ tst/unit/task_step/test_always_on_rubrics.py | 138 ----------- tst/unit/task_step/test_field_scopes.py | 1 + 15 files changed, 414 insertions(+), 180 deletions(-) create mode 100644 src/agent_env/task_step/task_steps/verifiers/always_on_agent_verifier.py create mode 100644 tst/unit/task_step/test_always_on_agent_verifier.py delete mode 100644 tst/unit/task_step/test_always_on_rubrics.py diff --git a/src/agent_env/explorer/ui/src/components/steps-pipeline.tsx b/src/agent_env/explorer/ui/src/components/steps-pipeline.tsx index d8917f12f..26f566c6a 100644 --- a/src/agent_env/explorer/ui/src/components/steps-pipeline.tsx +++ b/src/agent_env/explorer/ui/src/components/steps-pipeline.tsx @@ -25,6 +25,7 @@ const STEP_COLORS: Record = { prompt_agent: '#f59e0b', collect_artifacts: '#14b8a6', rubrics_verifier: '#10b981', + always_on_agent_verifier: '#10b981', env_outcome_verifier: '#10b981', run_code: '#84cc16', publish: '#6366f1', diff --git a/src/agent_env/explorer/ui/src/components/task-detail-page.tsx b/src/agent_env/explorer/ui/src/components/task-detail-page.tsx index ea7bfa9be..7df8e6950 100644 --- a/src/agent_env/explorer/ui/src/components/task-detail-page.tsx +++ b/src/agent_env/explorer/ui/src/components/task-detail-page.tsx @@ -27,7 +27,10 @@ import { type RubricCriterion, type VerificationResults, } from './rubric-grading-results'; -import { selectFinalScore } from '../lib/verifier-classification'; +import { + isRubricsVerifierStep, + selectFinalScore, +} from '../lib/verifier-classification'; import { TaskInstanceViewer, type TaskStepRef } from './task-instance-viewer'; import { RunGroupStepFunnel } from './run-group-step-funnel'; import { StepsPipeline } from './steps-pipeline'; @@ -947,7 +950,7 @@ export function TaskDetailPage({ }, [versionFilter, runGroupsPageSize]); const steps = (task?.steps ?? []) as Record[]; - const rubricsStep = steps.find(s => s.type === 'rubrics_verifier') as + const rubricsStep = steps.find(isRubricsVerifierStep) as | Record | undefined; const rubricsCriteria = rubricsStep?.criteria as diff --git a/src/agent_env/explorer/ui/src/components/task-runner-page.tsx b/src/agent_env/explorer/ui/src/components/task-runner-page.tsx index b11b84570..cbcf7b37f 100644 --- a/src/agent_env/explorer/ui/src/components/task-runner-page.tsx +++ b/src/agent_env/explorer/ui/src/components/task-runner-page.tsx @@ -11,7 +11,10 @@ import { workflowIdOf, type RunOptions, } from '../lib/task-runner-run'; -import { selectFinalScore } from '../lib/verifier-classification'; +import { + isRubricsVerifierStep, + selectFinalScore, +} from '../lib/verifier-classification'; import { BACKEND_URL, apiFetch } from './shared'; import { StepsPipeline } from './steps-pipeline'; import { TaskInstanceViewer, type TaskStepRef } from './task-instance-viewer'; @@ -263,7 +266,7 @@ export function TaskRunnerPage({ })); }, [task]); - const rubricsStep = taskSteps.find(s => s.type === 'rubrics_verifier') as + const rubricsStep = taskSteps.find(isRubricsVerifierStep) as | Record | undefined; const rubricsCriteria = rubricsStep?.criteria as diff --git a/src/agent_env/explorer/ui/src/lib/verifier-classification.ts b/src/agent_env/explorer/ui/src/lib/verifier-classification.ts index 0c80d5751..3336e2148 100644 --- a/src/agent_env/explorer/ui/src/lib/verifier-classification.ts +++ b/src/agent_env/explorer/ui/src/lib/verifier-classification.ts @@ -11,6 +11,21 @@ import { // Prefer the stamped `format`; fall back to shape sniffing when it's absent. export type VerifierKind = 'aggregate' | 'sandbox' | 'rubric' | 'other'; +// Step types that are agent-env's `rubrics_verifier` or a subclass of it (its `is_rubrics_verifier`): a check for the +// type string would miss `always_on_agent_verifier`, which grades an always_on step's prompt. +const RUBRICS_VERIFIER_STEP_TYPES: ReadonlySet = new Set([ + 'rubrics_verifier', + 'always_on_agent_verifier', +]); + +export function isRubricsVerifierStep( + step: { type?: unknown } | null | undefined, +): boolean { + return ( + typeof step?.type === 'string' && RUBRICS_VERIFIER_STEP_TYPES.has(step.type) + ); +} + const SANDBOX_CRITERION_TYPES = new Set([ 'probe_file_exists', 'probe_dir_exists', diff --git a/src/agent_env/task/episode/rules.py b/src/agent_env/task/episode/rules.py index 067013b39..ec1b184a7 100644 --- a/src/agent_env/task/episode/rules.py +++ b/src/agent_env/task/episode/rules.py @@ -142,6 +142,7 @@ def _rubrics_verifier(step: RubricsVerifierTaskStep, outside: Outside) -> list[s "register_env_triggers": _register_env_triggers, "peer_agents": _peer_agents, "rubrics_verifier": _rubrics_verifier, + "always_on_agent_verifier": _rubrics_verifier, **dict.fromkeys(( "verify_a2a_agent_card", "verify_a2a_agent_config_identity", "verify_a2a_agent_mcp", "verify_a2a_builtin_tools", "verify_a2a_core_protocol", "verify_a2a_install", "verify_a2a_modalities", "verify_a2a_peer_agents", diff --git a/src/agent_env/task/task.py b/src/agent_env/task/task.py index 69d002d74..606f9fe0c 100644 --- a/src/agent_env/task/task.py +++ b/src/agent_env/task/task.py @@ -365,6 +365,18 @@ def __init__( self.steps: list[TaskStep] = steps or [] self.requires: list[str] | None = list(requires) if requires is not None else None self._validate_dag(self.steps) + self._validate_pairings(self.steps) + + @staticmethod + def _validate_pairings(steps: list[TaskStep]) -> None: + """Refuse steps that don't go together (``TaskStep.task_problems``), such as a verifier on a prompt it can't + grade. A step built without TaskStep, as some tests build one, has none.""" + problems = [] + for step in steps: + if (check := getattr(step, "task_problems", None)) is not None: + problems.extend(check(steps)) + if problems: + raise ValueError("; ".join(problems)) @classmethod def _validate_dag(cls, steps: list[TaskStep]) -> None: diff --git a/src/agent_env/task_step/registry.py b/src/agent_env/task_step/registry.py index 48d7971a5..0db2e6d63 100644 --- a/src/agent_env/task_step/registry.py +++ b/src/agent_env/task_step/registry.py @@ -62,6 +62,7 @@ def _builtin_registry() -> dict[str, type["TaskStep"]]: from agent_env.task_step.task_steps.configure_scheduled_checks import ConfigureScheduledChecksTaskStep from agent_env.task_step.task_steps.install_agent import InstallAgentTaskStep from agent_env.task_step.task_steps.verifiers.rubrics_verifier import RubricsVerifierTaskStep + from agent_env.task_step.task_steps.verifiers.always_on_agent_verifier import AlwaysOnAgentVerifier from agent_env.task_step.task_steps.verifiers.run_container_unit_tests_verifier import RunContainerUnitTestsVerifierTaskStep from agent_env.task_step.task_steps.run_docker_container import RunDockerContainerTaskStep from agent_env.task_step.task_steps.mcp_env_validator.verify_mcp_tool_schema import VerifyMCPToolSchemaTaskStep @@ -129,6 +130,7 @@ def _builtin_registry() -> dict[str, type["TaskStep"]]: _get_type(ConfigureScheduledChecksTaskStep): ConfigureScheduledChecksTaskStep, _get_type(InstallAgentTaskStep): InstallAgentTaskStep, _get_type(RubricsVerifierTaskStep): RubricsVerifierTaskStep, + _get_type(AlwaysOnAgentVerifier): AlwaysOnAgentVerifier, _get_type(RunContainerUnitTestsVerifierTaskStep): RunContainerUnitTestsVerifierTaskStep, _get_type(RunDockerContainerTaskStep): RunDockerContainerTaskStep, _get_type(VerifyMCPToolSchemaTaskStep): VerifyMCPToolSchemaTaskStep, diff --git a/src/agent_env/task_step/task_step.py b/src/agent_env/task_step/task_step.py index ddf6239b7..1f572e941 100644 --- a/src/agent_env/task_step/task_step.py +++ b/src/agent_env/task_step/task_step.py @@ -139,6 +139,11 @@ def config_problems(self) -> list[str]: ``preflight`` is, and also for a step whose preflight a dry run skips.""" return [] + def task_problems(self, steps: list["TaskStep"]) -> list[str]: + """Problems this step has with the task's other steps (``steps``: all of them, this one included, in order), + found without reading a store; empty when fine. A task with any is refused when it loads.""" + return [] + def step_param_overrides(self, context: TaskStepContext) -> dict[str, Any]: """Per-run overrides for this step's params, from a run's ``step_overrides``. diff --git a/src/agent_env/task_step/task_steps/always_on.py b/src/agent_env/task_step/task_steps/always_on.py index 19e1d75e5..d722b5dc8 100644 --- a/src/agent_env/task_step/task_steps/always_on.py +++ b/src/agent_env/task_step/task_steps/always_on.py @@ -64,6 +64,9 @@ ``user_reads_at``. - The run's metadata keeps a summary in ``metadata["always_on"][step_id]`` (small enough for a running job to report back); the full timeline goes to the object store, at the summary's ``object_url``. +- An always_on_agent_verifier (``verifiers/always_on_agent_verifier.py``) grades the step's prompt from the user's chat + and the run's context, the run wake by wake among it (``render_for_judge``); a plain rubrics_verifier on it is + refused when the task loads. - The agent: the message_user rules go in its system prompt when the step sets one and the agent's card takes it, else on its first wake; its session is the step's ``context_id`` when set, as prompt_agent's is, else the run's own. - When the step ends it puts back the gateway settings it changed for the run (the user chat's roles and stay_quiet, @@ -317,6 +320,19 @@ def from_dict(cls, data: dict) -> AlwaysOnTaskStep: own = {k: data[k] for k in cls._OWN if k in data and data[k] is not None} return cls(**cls._base_from_dict(data), retry_config=data.get("retry_config"), **fields, **own) + def task_problems(self, steps: list[TaskStep]) -> list[str]: + """Its prompt is graded by always_on_agent_verifier: a plain rubrics_verifier, which would grade it from every + wake's trajectory, is refused.""" + # Here, not at the top: that module imports this one. + from agent_env.task_step.task_steps.verifiers.always_on_agent_verifier import AlwaysOnAgentVerifier + from agent_env.task_step.task_steps.verifiers.rubrics_verifier import is_rubrics_verifier + + return [f"step '{s.id}' ({s.type}) grades the prompt '{self.prompt_id}' of always_on step '{self.id}', which " + f"is graded from the user's chat and the run's context, not its wakes' trajectories: use type " + f"{AlwaysOnAgentVerifier.type}" + for s in steps if is_rubrics_verifier(s) and s.prompt_id == self.prompt_id + and not isinstance(s, AlwaysOnAgentVerifier)] + async def _execute(self, context: TaskStepContext) -> TaskStepContext: run = _Run(self, context) try: @@ -1698,7 +1714,7 @@ def _append_prompt_response(self, final_text: str) -> None: # ================================================================================================================== -# What a judge reads of the run beyond the user's chat (rubrics_verifier, for an always_on step's prompt). +# What a judge reads of the run beyond the user's chat (always_on_agent_verifier, for an always_on step's prompt). JUDGE_CHANGES_PER_WAKE = 10 JUDGE_WAKES = 300 _WOKE_BY = {"user_request": "the user's request", "user_reply": "the user's reply", "user_says": "a user message", diff --git a/src/agent_env/task_step/task_steps/verifiers/__init__.py b/src/agent_env/task_step/task_steps/verifiers/__init__.py index 781c29c39..3edb4d3f1 100644 --- a/src/agent_env/task_step/task_steps/verifiers/__init__.py +++ b/src/agent_env/task_step/task_steps/verifiers/__init__.py @@ -6,7 +6,7 @@ from agent_env.task_step.task_steps.verifiers.scoring import ScoreAggregator, aggregate_score from agent_env.task_step.task_steps.verifiers.run_context import EnvClock, Firing, Message, Run, RunRecordUnavailable, Turn from agent_env.task_step.task_steps.verifiers.env_outcome_verifier import EnvOutcomeVerifierTaskStep -from agent_env.task_step.task_steps.verifiers.rubrics_verifier import RubricsVerifierTaskStep +from agent_env.task_step.task_steps.verifiers.rubrics_verifier import RubricsVerifierTaskStep, is_rubrics_verifier from agent_env.task_step.task_steps.verifiers.agent_prompt_response_verifier import AgentPromptResponseVerifierTaskStep from agent_env.task_step.task_steps.verifiers.run_container_unit_tests_verifier import RunContainerUnitTestsVerifierTaskStep from agent_env.task_step.task_steps.verifiers.verify_sandbox import VerifySandboxTaskStep @@ -27,4 +27,5 @@ "Turn", "VerifySandboxTaskStep", "aggregate_score", + "is_rubrics_verifier", ] diff --git a/src/agent_env/task_step/task_steps/verifiers/always_on_agent_verifier.py b/src/agent_env/task_step/task_steps/verifiers/always_on_agent_verifier.py new file mode 100644 index 000000000..aa4765afb --- /dev/null +++ b/src/agent_env/task_step/task_steps/verifiers/always_on_agent_verifier.py @@ -0,0 +1,98 @@ +"""always_on_agent_verifier: the rubrics_verifier for an always_on step's prompt. + +An always_on run may take 40–60 wakes, each with its own trajectory: together they may not fit a judge, and cost a lot. +This verifier grades the run from what the user saw and the run's context instead: the user's chat (the prompt's +response and conversation), the trigger log, and the run wake by wake with the app changes each made +(``render_for_judge``), never a wake's trajectory. So when the task loads it refuses what needs the trajectories: an +output format that cites per-criterion screenshots, use_trajectory, and grade_up_to_turn (the conversation's turns are +not the run's wakes); a per-run grade_up_to_turn is refused when the verifier runs. Every other rubrics_verifier +setting works as it does there. An always_on step's prompt takes this verifier, never a plain rubrics_verifier, and +this verifier only an always_on step's prompt: a task that pairs them otherwise is refused when it loads. +""" + +from __future__ import annotations + +import asyncio +from typing import Any, ClassVar, Optional + +from agent_env.task_step.context import PromptResponse, TaskStepContext +from agent_env.task_step.task_step import TaskStep +from agent_env.task_step.task_steps.always_on import ALWAYS_ON_KEY, AlwaysOnTaskStep, render_for_judge +from agent_env.task_step.task_steps.prompt_agent import PromptAgentTaskStep +from agent_env.task_step.task_steps.verifiers.judge_utils.judge_output_format import ( + JudgeOutputFormat, + get_judge_output_format_spec, +) +from agent_env.task_step.task_steps.verifiers.rubrics_verifier import RUN_CONTEXT_PARTS, RubricsVerifierTaskStep + + +class AlwaysOnAgentVerifier(RubricsVerifierTaskStep): + type: ClassVar[str] = "always_on_agent_verifier" + entity_refs = RubricsVerifierTaskStep.entity_refs # the judge, as rubrics_verifier's + + def __init__( + self, + id: str, + version: Optional[int], + *, + use_trajectory: bool = False, + grade_up_to_turn: Optional[int] = None, + run_context_for_judge: Optional[list[str]] = None, + output_format: JudgeOutputFormat | str = JudgeOutputFormat.RUBRIC_BINARY, + default_model: Optional[str] = None, + default_model_api_base: Optional[str] = None, + effort: Optional[str] = None, + max_thinking_tokens: Optional[int] = None, + judge_a2a_agent_id: Optional[str] = None, + judge_sandbox_type: Optional[str] = None, + judge_batch_size: Optional[int] = None, + judge_batch_concurrency: Optional[int] = None, + **fields: Any, + ): + """``run_context_for_judge`` defaults to every part of the run's context; the rest is rubrics_verifier's.""" + output_format = JudgeOutputFormat(output_format) + where = f"{self.type} '{id}'" + if use_trajectory: + raise ValueError(f"{where}: use_trajectory must be off; an always_on step's prompt is graded from the " + "user's chat and the run's context, never its wakes' trajectories") + if get_judge_output_format_spec(output_format).cites_evidence: + raise ValueError(f"{where}: output_format={output_format.value} cites per-criterion screenshots, and an " + "always_on step's prompt is graded without its wakes' trajectories") + if grade_up_to_turn is not None: + raise ValueError(f"{where}: grade_up_to_turn can't cut an always_on step's prompt, whose conversation " + "turns are not its wakes") + super().__init__( + id, version, use_trajectory=False, run_context_for_judge=run_context_for_judge or list(RUN_CONTEXT_PARTS), + output_format=output_format, default_model=default_model, default_model_api_base=default_model_api_base, + effort=effort, max_thinking_tokens=max_thinking_tokens, judge_a2a_agent_id=judge_a2a_agent_id, + judge_sandbox_type=judge_sandbox_type, judge_batch_size=judge_batch_size, + judge_batch_concurrency=judge_batch_concurrency, **fields, + ) + + @classmethod + def from_dict(cls, data: dict) -> AlwaysOnAgentVerifier: + return super().from_dict({**data, "use_trajectory": data.get("use_trajectory", False)}) + + def task_problems(self, steps: list[TaskStep]) -> list[str]: + """Its prompt must be an always_on step's: one only another prompt step makes is refused. A prompt no step of + the task makes is checked when the verifier runs.""" + prompts = [s for s in steps if isinstance(s, PromptAgentTaskStep) and s.prompt_id == self.prompt_id] + if prompts and not any(isinstance(s, AlwaysOnTaskStep) for s in prompts): + return [f"step '{self.id}' ({self.type}) grades prompt '{self.prompt_id}' of {prompts[0].type} step " + f"'{prompts[0].id}', which isn't an always_on step; grade it with rubrics_verifier"] + return [] + + async def execute(self, context: TaskStepContext) -> TaskStepContext: + if "grade_up_to_turn" in self.step_param_overrides(context): + raise RuntimeError(f"Verifier '{self.verifier_id}': grade_up_to_turn can't cut an always_on step's " + "prompt, whose conversation turns are not its wakes.") + return await super().execute(context) + + async def judge_prompt_sections(self, context: TaskStepContext, prompt_response: PromptResponse) -> list[str]: + """The run wake by wake, from the always_on step's record of it.""" + record = (context.metadata.get(ALWAYS_ON_KEY) or {}).get(prompt_response.step_id) + if record is None: + raise RuntimeError(f"Verifier '{self.verifier_id}': prompt_id='{self.prompt_id}' is not an always_on " + f"step's prompt (step '{prompt_response.step_id}' left no always_on record); grade it " + "with rubrics_verifier.") + return [await asyncio.to_thread(render_for_judge, record)] diff --git a/src/agent_env/task_step/task_steps/verifiers/rubrics_verifier.py b/src/agent_env/task_step/task_steps/verifiers/rubrics_verifier.py index f5cbf9432..f5e4f1fc3 100644 --- a/src/agent_env/task_step/task_steps/verifiers/rubrics_verifier.py +++ b/src/agent_env/task_step/task_steps/verifiers/rubrics_verifier.py @@ -3,13 +3,13 @@ from __future__ import annotations import asyncio -import copy import json import logging import re import uuid +from collections.abc import Sequence from dataclasses import dataclass, field, replace -from typing import ClassVar, Optional +from typing import Any, ClassVar, Optional from agent_env.a2a_agent.object_transfer import ( TrajectoryUpload, @@ -624,28 +624,10 @@ def from_dict(cls, data: dict) -> RubricsVerifierTaskStep: judge_batch_concurrency=data.get("judge_batch_concurrency"), ) - # An always_on step's prompt: what this verifier grades it from, set on the copy ``_for_always_on`` makes. - _always_on_wakes: Optional[str] = None - - async def _for_always_on(self, context: TaskStepContext, record: dict) -> "RubricsVerifierTaskStep": - """This verifier as it grades an always_on step's prompt, which may run 40–60 wakes: from the user's chat (the - prompt's response and conversation) and the run's context (the trigger log, and the run wake by wake with the - app changes each made), never every wake's trajectory, which may not fit and costs a lot. A copy, since one - step instance is shared across concurrent rollouts; every other prompt is graded as before.""" - if self._spec().cites_evidence: - raise RuntimeError(f"Verifier '{self.verifier_id}': output_format={self.output_format.value} cites " - "per-criterion screenshots, and an always_on step's prompt is graded without its " - "wakes' trajectories.") - if "grade_up_to_turn" in self.step_param_overrides(context) or self.grade_up_to_turn is not None: - raise RuntimeError(f"Verifier '{self.verifier_id}': grade_up_to_turn can't cut an always_on step's " - "prompt, whose conversation turns are not its wakes.") - from agent_env.task_step.task_steps.always_on import render_for_judge # its module imports this one's peers - - twin = copy.copy(self) - twin.use_trajectory = False - twin.run_context_for_judge = self.run_context_for_judge or list(RUN_CONTEXT_PARTS) - twin._always_on_wakes = await asyncio.to_thread(render_for_judge, record) - return twin + async def judge_prompt_sections(self, context: TaskStepContext, prompt_response: PromptResponse) -> list[str]: + """Sections a subclass adds to this run's judge prompt, after the run context: an override point, called once + per run (never for one skipped) with the prompt's stored response. None here.""" + return [] def _resolved_criteria(self, context: TaskStepContext) -> list[dict]: """Criteria for this run: a per-run ``step_params`` override replaces the stored list. @@ -661,15 +643,6 @@ async def execute(self, context: TaskStepContext) -> TaskStepContext: prompt_response = next((pr for pr in context.prompt_responses if pr.prompt_id == self.prompt_id), None) if prompt_response is None: raise RuntimeError(f"PromptResponse with prompt_id='{self.prompt_id}' not found in context") - always_on = (context.metadata.get("always_on") or {}).get(prompt_response.step_id) - if always_on is not None and self._always_on_wakes is None: - if prompt_response.error_type: # skipped before the always_on settings are checked, as any errored prompt - logger.warning(f"Skipping verification '{self.verifier_id}': prompt had " - f"error_type={prompt_response.error_type}") - self._write_skipped_entries( - context, "prompt_error", f"Skipped: prompt had error_type={prompt_response.error_type}") - return context - return await (await self._for_always_on(context, always_on)).execute(context) overrides_for_step = self.step_param_overrides(context) grade_up_to_turn = ( _validated_turn(overrides_for_step["grade_up_to_turn"]) @@ -695,6 +668,7 @@ async def execute(self, context: TaskStepContext) -> TaskStepContext: run_context = await load_run_context(context, stored_response) if grade_up_to_turn is not None: run_context = run_context.as_of(grade_up_to_turn) + extra_sections = await self.judge_prompt_sections(context, stored_response) per_turn_uris = [u for u in (prompt_response.target_agent_per_turn_trajectory_object_urls or []) if u] if self.use_trajectory and not (per_turn_uris or prompt_response.agent_trajectory_object_url): @@ -889,6 +863,7 @@ def eval_prompt_for(batch: list[dict]) -> str: loaded_artifacts=loaded_for_judge, context_facts=context_facts, run_context=run_context, + extra_sections=extra_sections, ) if self.use_trajectory and multiturn: prompt += self._MULTITURN_DIR_NOTE.format(n=len(per_turn_uris)) @@ -902,6 +877,7 @@ def eval_prompt_for(batch: list[dict]) -> str: per_criterion_frames=compacted.per_criterion, context_facts=context_facts, run_context=run_context, + extra_sections=extra_sections, ) if image_blocks: logger.info( @@ -1543,9 +1519,11 @@ def _build_eval_prompt( per_criterion_frames: bool = False, context_facts: Optional[list[tuple[str, str]]] = None, run_context: Optional[Run] = None, + extra_sections: Sequence[str] = (), ) -> str: """``context_facts`` are rendered ``(label, value)`` pairs (``resolve_context_facts``); ``image_blocks`` - are the blocks attached to the judge message, labels included when ``per_criterion_frames``.""" + are the blocks attached to the judge message, labels included when ``per_criterion_frames``; + ``extra_sections`` are ``judge_prompt_sections``', after the run context.""" spec = self._spec() # Only the rubric_evidence templates have this placeholder; the others ignore the argument. checks_instruction = evidence_checks_instruction(has_policy=bool(self.grading_policy_prompt)) @@ -1578,8 +1556,8 @@ def _build_eval_prompt( base += "\n\n" + render_run_context(run_context, self.run_context_for_judge, message_max_chars=self.RUN_CONTEXT_MESSAGE_MAX_CHARS, firings_max=self.RUN_CONTEXT_FIRINGS_MAX) - if self._always_on_wakes: - base += "\n\n" + self._always_on_wakes + for section in extra_sections: + base += "\n\n" + section if context_facts: # Facts before the policy: they are inputs, the policy is how to read them. base += "\n\n## Context facts\n" + self._CONTEXT_FACTS_LEAD + "\n" + "\n".join( @@ -1623,3 +1601,16 @@ def _build_loaded_artifacts_block(entries: list[dict]) -> str: "Do not rely solely on the agent's response text — verify against the actual files when possible.", ] return "\n".join(lines) + + +def is_rubrics_verifier(step_or_config: TaskStep | dict[str, Any]) -> bool: + """Whether a step, or a stored step (its dict, its ``type`` looked up in the step registry), is a rubrics_verifier + or a subclass of one, such as always_on_agent_verifier: what code that would check the type string asks instead, + so the subclasses count. A dict of an unknown type is not one.""" + if isinstance(step_or_config, dict): + from agent_env.task_step.registry import get_task_step_registry # its built-ins import this module + + cls = get_task_step_registry().get(step_or_config.get("type")) + else: + cls = type(step_or_config) + return isinstance(cls, type) and issubclass(cls, RubricsVerifierTaskStep) diff --git a/tst/unit/task_step/test_always_on_agent_verifier.py b/tst/unit/task_step/test_always_on_agent_verifier.py new file mode 100644 index 000000000..a43977ec8 --- /dev/null +++ b/tst/unit/task_step/test_always_on_agent_verifier.py @@ -0,0 +1,223 @@ +"""always_on_agent_verifier: an always_on step's prompt is graded from the user's chat (the response and the +conversation) and the run's context (the trigger log, the run wake by wake with its app changes), never every wake's +trajectory. What needs the trajectories is refused when the task loads, as is a plain rubrics_verifier on an always_on +step's prompt; a plain rubrics_verifier grades any other prompt as before.""" + +from __future__ import annotations + +import pytest + +from agent_env.task.task import Task +from agent_env.task_step.context import PromptResponse, TaskStepContext +from agent_env.task_step.task_steps import always_on as ao +from agent_env.task_step.task_steps.prompt_agent import PromptAgentTaskStep +from agent_env.task_step.task_steps.verifiers import is_rubrics_verifier +from agent_env.task_step.task_steps.verifiers.always_on_agent_verifier import AlwaysOnAgentVerifier +from agent_env.task_step.task_steps.verifiers.rubrics_verifier import RUN_CONTEXT_PARTS, RubricsVerifierTaskStep +from tst.unit.task_step.always_on_harness import Rig + +pytestmark = pytest.mark.usefixtures("local_stores") + +CRITERIA = [{"id": "c1", "description": "asked before booking"}] + + +def _verifier(cls=AlwaysOnAgentVerifier, **kw) -> RubricsVerifierTaskStep: + return cls(id="judge", version=1, criteria=CRITERIA, prompt_id="ask", use_agent_judge=False, verifier_id="rubric", + **kw) + + +def _capture(monkeypatch, verifier) -> dict: + captured: dict = {} + + async def judge(*, eval_prompt, **kwargs): + captured["prompt"] = eval_prompt + return ([{**verifier.criteria[0], "score": 1.0, "result": True}], 0, [], None) + + def no_trajectories(uri): + raise AssertionError(f"the judge was sent a wake's trajectory: {uri}") + + monkeypatch.setattr(verifier, "_run_judge_with_output_retries", judge) + monkeypatch.setattr(verifier, "_read_trajectory_text", no_trajectories) + return captured + + +async def _run(monkeypatch, tmp_path): + def agent(n, texts, rig): + if n == 1: + rig.message("Casa Jacarandas is $855. Book it?") + return "PRIVATE: I'll book it anyway" + if n == 2: + rig.app_write("bookings", "request 855 USD", tool="stayncook_request_booking") + rig.message("requested!") + return "PRIVATE done" + rig.quiet() + return "" + + rig = Rig(monkeypatch, tmp_path, agent=agent, answers=[{"outcome": "reply", "message": "yes please"}]) + await rig.arm() + rig.keep_changelog() + rig.triggers(rig.notify_at("sam", "2026-11-10T14:00:00Z", "Sam: shh", channel="noor 30")) + ctx = await rig.run(rig.step()) + [response] = ctx.prompt_responses + response.target_agent_per_turn_trajectory_object_urls = [f"s3://wakes/{n}.json" for n in range(1, 4)] + response.agent_trajectory_object_url = "s3://wakes/3.json" + return rig, ctx + + +# ---- grading --------------------------------------------------------------------------------------------------------- + +@pytest.mark.asyncio +async def test_the_judge_grades_from_the_chat_and_the_run_wake_by_wake_not_the_trajectories(monkeypatch, tmp_path): + rig, ctx = await _run(monkeypatch, tmp_path) + verifier = _verifier() + stored = verifier.to_dict() + captured = _capture(monkeypatch, verifier) + + await verifier.execute(ctx) + + prompt = captured["prompt"] + assert "[Mon Nov 9, 2:00 PM UTC] Casa Jacarandas is $855. Book it?" in prompt and "PRIVATE" not in prompt + conversation = prompt[prompt.index("## Conversation"):prompt.index("## Trigger firings")] + assert "Linda: yes please" in conversation and "[End of run" in conversation + wakes = prompt[prompt.index("## The run, wake by wake"):] + assert "- Wake 2 · Mon Nov 9, 2:00 PM UTC · woken by the user's reply · sent the user 1 message" in wakes + assert " - it changed: INSERT public.bookings row 1 (request 855 USD)" in wakes + assert "- Wake 3 · Tue Nov 10, 2:00 PM UTC · woken by an alert (noor 30)" in wakes + assert "## How the run ended\nNothing could wake the assistant again, and nothing was pending." in wakes + assert ctx.metadata["verifications"]["rubric"]["score"] == 1 + assert verifier.to_dict() == stored # the step, shared by concurrent rollouts, is untouched + assert verifier.use_trajectory is False and verifier.run_context_for_judge == list(RUN_CONTEXT_PARTS) + + +@pytest.mark.asyncio +async def test_parts_of_the_run_context_a_verifier_names_are_kept(monkeypatch, tmp_path): + rig, ctx = await _run(monkeypatch, tmp_path) + verifier = _verifier(run_context_for_judge=["conversation"]) + captured = _capture(monkeypatch, verifier) + + await verifier.execute(ctx) + + assert "## Conversation" in captured["prompt"] and "## Trigger firings" not in captured["prompt"] + assert "## The run, wake by wake" in captured["prompt"] + + +@pytest.mark.asyncio +async def test_a_failed_always_on_run_is_skipped_like_a_failed_prompt(monkeypatch, tmp_path): + def fails(n, texts, rig): + raise TimeoutError("no answer") + + rig = Rig(monkeypatch, tmp_path, agent=fails) + await rig.arm() + ctx = await rig.run(rig.step()) + verifier = _verifier() + _capture(monkeypatch, verifier) + + await verifier.execute(ctx) + + [row] = ctx.metadata["verifications"]["rubric"]["results"] + assert row["id"] == "prompt_error" and "agent_turn_timeout" in row["message"] + + +@pytest.mark.asyncio +async def test_a_run_s_grade_up_to_turn_is_refused(monkeypatch, tmp_path): + rig, ctx = await _run(monkeypatch, tmp_path) + verifier = _verifier() + _capture(monkeypatch, verifier) + ctx.metadata["user_overrides"] = {"step_params": {"judge": {"grade_up_to_turn": 2}}} + + with pytest.raises(RuntimeError, match="grade_up_to_turn can't cut an always_on"): + await verifier.execute(ctx) + + +@pytest.mark.asyncio +async def test_a_prompt_no_always_on_step_made_is_refused_when_it_is_graded(monkeypatch): + ctx = TaskStepContext() + ctx.prompt_responses.append(PromptResponse(prompt_id="ask", response="done", prompt_text="hi", step_id="prompt")) + verifier = _verifier() + _capture(monkeypatch, verifier) + + with pytest.raises(RuntimeError, match="is not an always_on step's prompt.*grade it with rubrics_verifier"): + await verifier.execute(ctx) + + +def test_the_wake_by_wake_text_says_when_the_record_is_unreadable(): + text = ao.render_for_judge({"object_url": "file:///nowhere/at/all.json", "end_reason": "end_time"}) + assert text.startswith("## The run, wake by wake\n(The run's record could not be read:") + + +# ---- refused when the task loads ------------------------------------------------------------------------------------- + +@pytest.mark.parametrize("settings,why", [ + ({"output_format": "rubric_evidence"}, "output_format=rubric_evidence cites per-criterion screenshots"), + ({"output_format": "rubric_evidence_with_mistakes"}, "cites per-criterion screenshots"), + ({"grade_up_to_turn": 2}, "grade_up_to_turn can't cut an always_on step's prompt"), + ({"use_trajectory": True}, "use_trajectory must be off"), +]) +def test_what_needs_the_wakes_trajectories_is_refused_when_the_task_loads(settings, why): + with pytest.raises(ValueError, match=why): + _verifier(**settings) + stored = {"id": "judge", "type": "always_on_agent_verifier", "prompt_id": "ask", "criteria": CRITERIA, **settings} + with pytest.raises(ValueError, match=why): + Task.from_dict({"id": "t", "version": 1, "steps": [stored]}) + + +def test_a_stored_verifier_needs_no_use_trajectory_and_survives_a_round_trip(): + stored = {"id": "judge", "type": "always_on_agent_verifier", "prompt_id": "ask", "criteria": CRITERIA} + [verifier] = Task.from_dict({"id": "t", "version": 1, "steps": [stored]}).steps + assert isinstance(verifier, AlwaysOnAgentVerifier) and verifier.use_trajectory is False + again = AlwaysOnAgentVerifier.from_dict(verifier.to_dict()) + assert again.to_dict() == verifier.to_dict() and again.type == "always_on_agent_verifier" + + +def _always_on() -> ao.AlwaysOnTaskStep: + return ao.AlwaysOnTaskStep("week", None, user_sim="linda", prompt="sort the weekend", prompt_id="ask", + agent_name="solver") + + +def test_a_plain_rubrics_verifier_on_an_always_on_prompt_is_refused_when_the_task_loads(): + with pytest.raises(ValueError, match="step 'judge' \\(rubrics_verifier\\) grades the prompt 'ask' of always_on " + "step 'week'.*use type always_on_agent_verifier"): + Task("t", 1, steps=[_always_on(), _verifier(RubricsVerifierTaskStep)]) + stored = Task("t", 1, steps=[_always_on(), _verifier()]).to_dict() + stored["steps"][1]["type"] = "rubrics_verifier" + with pytest.raises(ValueError, match="use type always_on_agent_verifier"): + Task.from_dict(stored) + + +def test_an_always_on_agent_verifier_on_another_step_s_prompt_is_refused_when_the_task_loads(): + prompt = PromptAgentTaskStep("ask-step", None, prompt="hi", prompt_id="ask", agent_name="solver") + with pytest.raises(ValueError, match="grades prompt 'ask' of prompt_agent step 'ask-step', which isn't an " + "always_on step; grade it with rubrics_verifier"): + Task("t", 1, steps=[prompt, _verifier()]) + + +def test_each_verifier_on_the_prompt_it_grades_loads(): + Task("t", 1, steps=[_always_on(), _verifier()]) + prompt = PromptAgentTaskStep("ask-step", None, prompt="hi", prompt_id="ask", agent_name="solver") + Task("t", 1, steps=[prompt, _verifier(RubricsVerifierTaskStep)]) + + +# ---- a plain rubrics_verifier -------------------------------------------------------------------------------------- + +@pytest.mark.asyncio +async def test_a_plain_rubrics_verifier_grades_another_prompt_as_before(monkeypatch): + ctx = TaskStepContext() + ctx.prompt_responses.append(PromptResponse(prompt_id="ask", response="booked it", prompt_text="book it", + step_id="prompt")) + verifier = _verifier(RubricsVerifierTaskStep, use_trajectory=False) + captured = _capture(monkeypatch, verifier) + + await verifier.execute(ctx) + + assert verifier.to_dict()["type"] == "rubrics_verifier" and verifier.run_context_for_judge is None + assert "booked it" in captured["prompt"] and "## The run, wake by wake" not in captured["prompt"] + assert ctx.metadata["verifications"]["rubric"]["score"] == 1 + + +# ---- is_rubrics_verifier ------------------------------------------------------------------------------------------- + +def test_is_rubrics_verifier_recognises_the_subclass_as_a_step_and_as_a_stored_dict(): + for verifier in (_verifier(), _verifier(RubricsVerifierTaskStep)): + assert is_rubrics_verifier(verifier) and is_rubrics_verifier(verifier.to_dict()) + assert not is_rubrics_verifier(_always_on()) and not is_rubrics_verifier(_always_on().to_dict()) + assert not is_rubrics_verifier({"type": "no_such_step"}) and not is_rubrics_verifier({}) diff --git a/tst/unit/task_step/test_always_on_rubrics.py b/tst/unit/task_step/test_always_on_rubrics.py deleted file mode 100644 index 29eabdc20..000000000 --- a/tst/unit/task_step/test_always_on_rubrics.py +++ /dev/null @@ -1,138 +0,0 @@ -"""rubrics_verifier on an always_on step's prompt: the judge grades from the user's chat (the response and the -conversation) and the run's context (the trigger log, the run wake by wake with its app changes), never every wake's -trajectory. Any other step's prompt is graded as before.""" - -from __future__ import annotations - -import pytest - -from agent_env.task_step.task_steps import always_on as ao -from agent_env.task_step.task_steps.verifiers.rubrics_verifier import RubricsVerifierTaskStep -from tst.unit.task_step.always_on_harness import Rig - -pytestmark = pytest.mark.usefixtures("local_stores") - - -def _verifier(**kw) -> RubricsVerifierTaskStep: - return RubricsVerifierTaskStep(id="judge", version=1, criteria=[{"id": "c1", "description": "asked before booking"}], - prompt_id="ask", use_agent_judge=False, verifier_id="rubric", **kw) - - -def _capture(monkeypatch, verifier) -> dict: - captured: dict = {} - - async def judge(*, eval_prompt, **kwargs): - captured["prompt"] = eval_prompt - return ([{**verifier.criteria[0], "score": 1.0, "result": True}], 0, [], None) - - def no_trajectories(uri): - raise AssertionError(f"the judge was sent a wake's trajectory: {uri}") - - monkeypatch.setattr(verifier, "_run_judge_with_output_retries", judge) - monkeypatch.setattr(verifier, "_read_trajectory_text", no_trajectories) - return captured - - -async def _run(monkeypatch, tmp_path): - def agent(n, texts, rig): - if n == 1: - rig.message("Casa Jacarandas is $855. Book it?") - return "PRIVATE: I'll book it anyway" - if n == 2: - rig.app_write("bookings", "request 855 USD", tool="stayncook_request_booking") - rig.message("requested!") - return "PRIVATE done" - rig.quiet() - return "" - - rig = Rig(monkeypatch, tmp_path, agent=agent, answers=[{"outcome": "reply", "message": "yes please"}]) - await rig.arm() - rig.keep_changelog() - rig.triggers(rig.notify_at("sam", "2026-11-10T14:00:00Z", "Sam: shh", channel="noor 30")) - ctx = await rig.run(rig.step()) - [response] = ctx.prompt_responses - response.target_agent_per_turn_trajectory_object_urls = [f"s3://wakes/{n}.json" for n in range(1, 4)] - response.agent_trajectory_object_url = "s3://wakes/3.json" - return rig, ctx - - -@pytest.mark.asyncio -async def test_the_judge_grades_from_the_chat_and_the_run_wake_by_wake_not_the_trajectories(monkeypatch, tmp_path): - rig, ctx = await _run(monkeypatch, tmp_path) - verifier = _verifier() - captured = _capture(monkeypatch, verifier) - - await verifier.execute(ctx) - - prompt = captured["prompt"] - assert "[Mon Nov 9, 2:00 PM UTC] Casa Jacarandas is $855. Book it?" in prompt and "PRIVATE" not in prompt - conversation = prompt[prompt.index("## Conversation"):prompt.index("## Trigger firings")] - assert "Linda: yes please" in conversation and "[End of run" in conversation - wakes = prompt[prompt.index("## The run, wake by wake"):] - assert "- Wake 2 · Mon Nov 9, 2:00 PM UTC · woken by the user's reply · sent the user 1 message" in wakes - assert " - it changed: INSERT public.bookings row 1 (request 855 USD)" in wakes - assert "- Wake 3 · Tue Nov 10, 2:00 PM UTC · woken by an alert (noor 30)" in wakes - assert "## How the run ended\nNothing could wake the assistant again, and nothing was pending." in wakes - assert ctx.metadata["verifications"]["rubric"]["score"] == 1 - assert verifier.use_trajectory is True and verifier.run_context_for_judge is None # the shared step is untouched - - -@pytest.mark.asyncio -async def test_parts_of_the_run_context_a_verifier_names_are_kept(monkeypatch, tmp_path): - rig, ctx = await _run(monkeypatch, tmp_path) - verifier = _verifier(run_context_for_judge=["conversation"]) - captured = _capture(monkeypatch, verifier) - - await verifier.execute(ctx) - - assert "## Conversation" in captured["prompt"] and "## Trigger firings" not in captured["prompt"] - - -@pytest.mark.asyncio -async def test_grading_an_always_on_prompt_as_of_a_turn_is_refused(monkeypatch, tmp_path): - rig, ctx = await _run(monkeypatch, tmp_path) - verifier = _verifier(grade_up_to_turn=2) - _capture(monkeypatch, verifier) - - with pytest.raises(RuntimeError, match="grade_up_to_turn can't cut an always_on"): - await verifier.execute(ctx) - - -@pytest.mark.asyncio -async def test_a_failed_always_on_run_is_skipped_like_a_failed_prompt(monkeypatch, tmp_path): - def fails(n, texts, rig): - raise TimeoutError("no answer") - - rig = Rig(monkeypatch, tmp_path, agent=fails) - await rig.arm() - ctx = await rig.run(rig.step()) - verifier = _verifier() - _capture(monkeypatch, verifier) - - await verifier.execute(ctx) - - [row] = ctx.metadata["verifications"]["rubric"]["results"] - assert row["id"] == "prompt_error" and "agent_turn_timeout" in row["message"] - - -@pytest.mark.asyncio -async def test_a_failed_always_on_run_is_skipped_even_by_a_verifier_that_couldn_t_grade_it(monkeypatch, tmp_path): - """grade_up_to_turn can't cut an always_on prompt, but a run that broke is skipped before that is checked.""" - def fails(n, texts, rig): - raise TimeoutError("no answer") - - rig = Rig(monkeypatch, tmp_path, agent=fails) - await rig.arm() - ctx = await rig.run(rig.step()) - verifier = _verifier(grade_up_to_turn=2) - _capture(monkeypatch, verifier) - - await verifier.execute(ctx) - - [row] = ctx.metadata["verifications"]["rubric"]["results"] - assert row["id"] == "prompt_error" and "agent_turn_timeout" in row["message"] - - -def test_the_wake_by_wake_text_says_when_the_record_is_unreadable(): - text = ao.render_for_judge({"object_url": "file:///nowhere/at/all.json", "end_reason": "end_time"}) - assert text.startswith("## The run, wake by wake\n(The run's record could not be read:") diff --git a/tst/unit/task_step/test_field_scopes.py b/tst/unit/task_step/test_field_scopes.py index 92ba03ddc..f7b46a63e 100644 --- a/tst/unit/task_step/test_field_scopes.py +++ b/tst/unit/task_step/test_field_scopes.py @@ -14,6 +14,7 @@ MINIMAL = { "add_skills": {"skills": [{"name": "notes", "description": "how notes are kept"}]}, "always_on": {"prompt": "hi", "user_sim": "linda"}, + "always_on_agent_verifier": {"prompt_id": "prompt", "criteria": [{"id": "c1", "description": "says hi"}]}, "deploy_agent": {}, "deploy_env": {"env_id": "crm-env"}, "deploy_human_agent": {}, From 4835f14b6a9cf46e89e0c21fbc8bae39a1451555 Mon Sep 17 00:00:00 2001 From: Sarah Chitty Date: Thu, 12 Nov 2026 10:05:22 -0800 Subject: [PATCH 53/53] always_on: a read whose time passed during the agent's turn waits until the user is free; response_times needs a schedule When the agent's turn ran into the user's night or a busy block, her read went ahead at once in the mode it was scheduled in. Now it's rechecked at the time she'd read: she reads then if she's free, in that time's mode, else when she's next free. And user_sim refuses response_times without a schedule, which never held a reply. Co-Authored-By: Claude Opus 5.5 --- .../task_step/task_steps/always_on.py | 6 ++++++ .../task_step/task_steps/user_sim.py | 3 +++ tst/unit/task_step/test_always_on_endings.py | 21 +++++++++++++++++++ tst/unit/task_step/test_user_sim.py | 7 +++++++ 4 files changed, 37 insertions(+) diff --git a/src/agent_env/task_step/task_steps/always_on.py b/src/agent_env/task_step/task_steps/always_on.py index 9db990d38..65a112ece 100644 --- a/src/agent_env/task_step/task_steps/always_on.py +++ b/src/agent_env/task_step/task_steps/always_on.py @@ -1012,6 +1012,12 @@ async def _next(self) -> bool: # read, after the final check-in) if self.read_at is not None and self.read_at <= now and self.unread \ and (self.end is None or self.read_at < self.end): + if self.read_at < now: # her time to read passed during the agent's turn: she reads now if she's free + at, self.read_mode = self._available(now) + if at > now: # asleep or busy by now: she reads when she's next free + self.read_at = at + self.log("user_read_scheduled", read_at=_iso(at), mode=self.read_mode, unread=len(self.unread)) + return True await self.read(now) return True # past end_time (the clock runs during the agent's turns, so one can end past it): the final check-in next, with diff --git a/src/agent_env/task_step/task_steps/user_sim.py b/src/agent_env/task_step/task_steps/user_sim.py index 11140e614..e2d90115e 100644 --- a/src/agent_env/task_step/task_steps/user_sim.py +++ b/src/agent_env/task_step/task_steps/user_sim.py @@ -79,6 +79,9 @@ def __init__(self, id: str, version: Optional[int], *, name: str, model: str, ag raise ValueError("user_sim.model is required") if time_zone is not None: ZoneInfo(time_zone) + if response_times and not schedule: + raise ValueError("user_sim.response_times needs a schedule (an empty one counts as none); without one, " + "replies are never held") self.name = name self.model = model self.agent_name = agent_name or name diff --git a/tst/unit/task_step/test_always_on_endings.py b/tst/unit/task_step/test_always_on_endings.py index 229ec56e5..48585d69f 100644 --- a/tst/unit/task_step/test_always_on_endings.py +++ b/tst/unit/task_step/test_always_on_endings.py @@ -172,6 +172,27 @@ def wraps_up_late(n, texts, rig): assert (event["mode"], event["deliver_at"]) == ("morning", "2026-11-10T12:45:00Z") +@pytest.mark.asyncio +async def test_a_read_whose_time_passed_during_a_turn_that_ran_into_her_night_waits_for_the_morning(monkeypatch, + tmp_path): + def runs_late(n, texts, rig): + if n == 1: + rig.message("booked the house") + rig.gw._clock.advance("2026-11-10T05:00:00Z") # midnight in New York: past her bedtime + return "sent" + rig.quiet() + return "nothing new" + + rig = Rig(monkeypatch, tmp_path, agent=runs_late, answers=[{"outcome": "reply", "message": "great"}]) + await rig.arm() + ctx = await rig.run(rig.step(max_wakes=3), rig.context(user=DAY)) + + [read, *_] = rig.reads + assert read["time"] == "2026-11-10T12:30:00Z" # 7:30 AM, when she wakes, not at midnight + [event, *_] = rig.events(ctx, "user_sim_read") + assert (event["mode"], event["deliver_at"]) == ("morning", "2026-11-10T12:45:00Z") + + @pytest.mark.asyncio async def test_a_done_on_the_last_read_is_recorded_and_changes_nothing(monkeypatch, tmp_path): rig = Rig(monkeypatch, tmp_path, agent=_recaps, answers=[{"outcome": "done", "message": "thanks!!"}]) diff --git a/tst/unit/task_step/test_user_sim.py b/tst/unit/task_step/test_user_sim.py index 75d3699ec..016a389e2 100644 --- a/tst/unit/task_step/test_user_sim.py +++ b/tst/unit/task_step/test_user_sim.py @@ -41,3 +41,10 @@ def test_ours_renders_one_labelled_read(): for label in ("NEW ON YOUR PHONE", "YOU SENT YOUR AI ASSISTANT", "NEW FROM YOUR AI ASSISTANT"): assert label in text assert text.index("house?") < text.index("and dinner?") + + +def test_response_times_without_a_schedule_is_refused(): + for schedule in (None, {}): + with pytest.raises(ValueError, match="response_times needs a schedule"): + UserSimTaskStep("u", None, name="linda", model="m", schedule=schedule, response_times={"free": 30}) + UserSimTaskStep("u", None, name="linda", model="m", schedule={"wake": "07:30"}, response_times={"free": 30})