Add verified browser action plans

Author: rgarcia

docs/agent-tool-configuration-spec.md

@@ -10,7 +10,19 @@ export type { BrowserRefState } from "./translator/browser-ref-lifecycle";
 export type {
 	BatchExecutionResult,
 	BatchReadResult,
+	BrowserActExpectationEvidence,
+	BrowserActExpectationStatus,
+	BrowserActObservedSuccessor,
+	BrowserActOutcome,
+	BrowserActResult,
+	BrowserActStepResult,
+	BrowserActStopReason,
+	BrowserActSuccessor,
+	BrowserActUnavailableSuccessor,
 	BrowserExpectationEvidence,
+	BrowserExpectationState,
+	BrowserObservationDiff,
+	BrowserObservationDiffEntry,
 	BrowserWaitForResult,
 	BrowserWaitReason,
 } from "./translator/types";

packages/agent/src/index.ts

@@ -16,7 +16,11 @@ import {
 } from "@onkernel/cua-ai";
 import { InternalComputerTranslator, type KernelBrowser } from "./translator/translator";
 import type { AgentTool, AgentToolResult } from "@earendil-works/pi-agent-core";
-import type { BrowserWaitForResult } from "./translator/types";
+import type { BrowserActResult, BrowserWaitForResult } from "./translator/types";
+
+const BROWSER_ACT_DIFF_ENTRY_LIMIT = 200;
+const BROWSER_ACT_DIFF_CHAR_LIMIT = 20_000;
+const BROWSER_ACT_RESULT_CHAR_LIMIT = 50_000;
 
 export interface ComputerToolOptions {
 	browser: KernelBrowser;
@@ -41,6 +45,7 @@ export interface BatchDetails {
 		| { type: "cursor_position"; x: number; y: number }
 		| { type: "browser_text"; label: string; bytes: number }
 		| { type: "browser_wait_for"; result: BrowserWaitForResult }
+		| { type: "browser_act"; result: BrowserActResult }
 	>;
 }
 
@@ -179,6 +184,9 @@ async function executeBatchTool(
 			} else if (read.type === "browser_wait_for") {
 				readResults.push(read);
 				content.push({ type: "text", text: formatBrowserWaitResult(read.result) });
+			} else if (read.type === "browser_act") {
+				readResults.push(read);
+				content.push({ type: "text", text: formatBrowserActResult(read.result) });
 			} else {
 				readResults.push({ type: "screenshot", bytes: read.data.length });
 				content.push({ type: "image", data: read.data.toString("base64"), mimeType: read.mimeType });
@@ -194,13 +202,18 @@ async function executeBatchTool(
 	} catch (err) {
 		throw new Error(`Actions failed: ${errorMessage(err)}`, { cause: err });
 	}
+	const acts = readResults.flatMap((read) => read.type === "browser_act" ? [read.result] : []);
 	const waits = readResults.flatMap((read) => read.type === "browser_wait_for" ? [read.result] : []);
 	const failedWait = ["interrupted", "timed_out", "unverifiable"].find((status) => waits.some((wait) => wait.status === status));
-	let statusText = waits.length === 0
-		? "Actions executed successfully."
+	let statusText = acts.some((act) => act.outcome === "didnt")
+		? "Browser action plan did not satisfy its expectations."
 		: failedWait
 			? `Browser condition ${failedWait}.`
-			: "Browser condition satisfied.";
+			: acts.some((act) => act.outcome === "unknown")
+				? "Browser action plan outcome is unknown."
+				: acts.length > 0
+					? "Browser action plan worked."
+					: waits.length > 0 ? "Browser condition satisfied." : "Actions executed successfully.";
 	if (skippedActions) {
 		const skipped = `${skippedActions} subsequent action${skippedActions === 1 ? " was" : "s were"} skipped.`;
 		statusText = `${statusText} ${skipped}`;
@@ -290,6 +303,52 @@ function formatBrowserWaitResult(result: BrowserWaitForResult): string {
 	return [`wait_for: ${result.status}/${result.evidence}${reason} after ${result.elapsed_ms}ms`, ...result.details].join("\n");
 }
 
+/** Format model-facing plan feedback with bounded diff entries, diff characters, and total characters. */
+export function formatBrowserActResult(result: BrowserActResult): string {
+	const lines = [`browser_act: ${result.outcome}`];
+	if (result.stopped_at !== undefined) lines.push(`stopped_at: ${result.stopped_at} (${result.stop_reason ?? "unknown"})`);
+	for (const step of result.steps) {
+		lines.push(`step ${step.index} ${step.type}: ${step.outcome} — ${step.diagnostics.join("; ")}`);
+		for (const diagnostic of step.expectation?.diagnostics ?? []) lines.push(`  ${diagnostic}`);
+	}
+	if (result.final_expectation) {
+		lines.push(`final expectation: ${result.final_expectation.status}`);
+		for (const diagnostic of result.final_expectation.diagnostics) lines.push(`  ${diagnostic}`);
+	}
+	if (result.successor.status === "unavailable") {
+		lines.push(`successor unavailable: ${result.successor.error}`);
+		return boundedBrowserActOutput(lines);
+	}
+	const { diff } = result.successor;
+	const addedCount = diff.added.reduce((total, entry) => total + entry.count, 0);
+	const removedCount = diff.removed.reduce((total, entry) => total + entry.count, 0);
+	lines.push(`successor: ${result.successor.title} (${result.successor.url})`);
+	lines.push(`diff: ${diff.changed ? `+${addedCount} -${removedCount}` : "unchanged"}`);
+	if (diff.url) lines.push(`  url: ${diff.url.before} -> ${diff.url.after}`);
+	if (diff.title) lines.push(`  title: ${diff.title.before} -> ${diff.title.after}`);
+	const changes = [
+		...diff.added.map((entry) => `  + ${entry.line}${entry.count === 1 ? "" : ` ×${entry.count}`}`),
+		...diff.removed.map((entry) => `  - ${entry.line}${entry.count === 1 ? "" : ` ×${entry.count}`}`),
+	];
+	let emittedChars = 0;
+	let emittedEntries = 0;
+	for (const change of changes) {
+		if (emittedEntries >= BROWSER_ACT_DIFF_ENTRY_LIMIT || emittedChars + change.length > BROWSER_ACT_DIFF_CHAR_LIMIT) break;
+		lines.push(change);
+		emittedEntries += 1;
+		emittedChars += change.length;
+	}
+	if (emittedEntries < changes.length) lines.push(`  … ${changes.length - emittedEntries} more diff entries omitted`);
+	lines.push(result.successor.text);
+	return boundedBrowserActOutput(lines);
+}
+
+function boundedBrowserActOutput(lines: readonly string[]): string {
+	const output = lines.join("\n");
+	if (output.length <= BROWSER_ACT_RESULT_CHAR_LIMIT) return output;
+	return `${output.slice(0, BROWSER_ACT_RESULT_CHAR_LIMIT)}\n… browser_act output truncated at ${BROWSER_ACT_RESULT_CHAR_LIMIT} characters`;
+}
+
 function formatPlaywrightResult(result: unknown): string {
 	return typeof result === "string" ? result : JSON.stringify(result);
 }