From d479bd306f8277304bb6021b81997ee51e65b15d Mon Sep 17 00:00:00 2001 From: MyNameIsEdi Date: Mon, 25 May 2026 11:48:56 +0300 Subject: [PATCH] =?UTF-8?q?feat(office):=20smarter=20pixel=20agents=20?= =?UTF-8?q?=E2=80=94=20chat-reactive,=20emotional=20reactions,=20handoff?= =?UTF-8?q?=20bubbles?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pixel agents in the Playwright Dashboard office canvas now respond to four live signals instead of coarse run/failure flags: 1. Chat-reactive: agentStatuses ('working'/'waiting') is wired to setAgentActive so agents walk to their desk and type while responding, then wander when idle. 2. Emotional reactions: on run completion, all agents show react_pass (green ✓) or react_fail (red ✗) speech bubbles; passing runs also trigger a 2.5 s celebration where agents stay briefly at their desks. 3. Handoff bubbles: on turn_done in multi-agent chat, the finishing agent briefly shows a waiting bubble — a visual "passing the baton" signal. 4. Two new pixel-art bubble sprites (react_pass / react_fail) with light-tinted backgrounds, wired through the existing sprite pipeline (types, spriteData, renderer, officeState). Co-Authored-By: Claude Sonnet 4.6 --- CHANGELOG.md | 5 +++ README.md | 2 +- ui/src/office/engine/officeState.ts | 25 ++++++++++- ui/src/office/engine/renderer.ts | 21 +++++++-- ui/src/office/sprites/bubble-react-fail.json | 27 ++++++++++++ ui/src/office/sprites/bubble-react-pass.json | 27 ++++++++++++ ui/src/office/sprites/spriteData.ts | 8 ++++ ui/src/office/types.ts | 2 +- ui/src/pages/PlaywrightDashboard.tsx | 46 ++++++++++++++++++-- 9 files changed, 151 insertions(+), 12 deletions(-) create mode 100644 ui/src/office/sprites/bubble-react-fail.json create mode 100644 ui/src/office/sprites/bubble-react-pass.json diff --git a/CHANGELOG.md b/CHANGELOG.md index 8d920490..b0e6efe6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,11 @@ All notable changes to this project will be documented in this file. ### Added +- **Smarter pixel agents** — office canvas agents now react to four live signals instead of just test-run state: + 1. *Chat-reactive*: `agentStatuses` ('working'/'waiting') is wired to `setAgentActive` so pixel agents walk to their desk and type whenever they are responding to a chat message, and wander again when idle (`PlaywrightDashboard.tsx`). + 2. *Emotional reactions*: when a test run completes, all agents briefly show a green checkmark bubble (pass) or red ✗ bubble (fail) and celebrate by staying at their desks for 2.5 s (`officeState.ts`, `PlaywrightDashboard.tsx`). + 3. *Handoff bubbles*: in multi-agent chat, the finishing agent gets a waiting bubble on `turn_done` — a visual "passing the baton" moment before the next agent activates (`PlaywrightDashboard.tsx`). + 4. *New bubble types* — `react_pass` (light-green bg, bright checkmark) and `react_fail` (pink bg, red ×) pixel-art speech bubbles with auto-fade timer, matching the existing sprite pipeline (`types.ts`, `spriteData.ts`, `renderer.ts`). - **SQLite run history** — replaces per-run JSON files in `test-results/runs/` with a single `better-sqlite3` database (`test-results/runs.db`). `archiveRun()` INSERTs into the DB; `/api/playwright/history`, `/api/playwright/results/:runId`, and diff --git a/README.md b/README.md index 7e3fc94f..90cc4c51 100644 --- a/README.md +++ b/README.md @@ -31,7 +31,7 @@ Type a request in the chat. **Edi M**, the Team Manager, analyses your intent an | | Feature | Description | | ---- | --------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| 🏢 | **2D Pixel-Art Office** | 7 specialist agents at animated desks — click any desk to start a conversation | +| 🏢 | **2D Pixel-Art Office** | 7 specialist agents at animated desks — agents walk to their desk and type live while responding to chat, celebrate with checkmark bubbles on passing runs, show red ✗ bubbles on failures, and display handoff bubbles when passing work between agents in multi-turn conversations | | 🤖 | **Edi M — Team Manager** | Orchestrator agent that analyses your request and routes it to the right specialist | | 🤝 | **Multi-Agent Collaboration** | Tag two or more specialists (or @Edi M) and they iterate: primary drafts → critic reviews → primary refines → manager delivers the polished synthesis. Each turn streams into its own chat bubble with a role pill (primary / critic / synthesis). Chat history and image attachments are forwarded to the initial primary turn so vision prompts and follow-up context are preserved across collaborative rounds | | ✨🦙 | **Gemini + Ollama** | Switch between cloud Gemini and fully-local Ollama models with a single click — no restart required | diff --git a/ui/src/office/engine/officeState.ts b/ui/src/office/engine/officeState.ts index 9d49f03e..e35486db 100644 --- a/ui/src/office/engine/officeState.ts +++ b/ui/src/office/engine/officeState.ts @@ -672,6 +672,23 @@ export class OfficeState { } } + /** Show a timed reaction bubble (pass ✓ or fail ✗) above an agent. Auto-dismisses. */ + showReactBubble(id: number, type: 'react_pass' | 'react_fail'): void { + const ch = this.characters.get(id); + if (ch) { + ch.bubbleType = type; + ch.bubbleTimer = WAITING_BUBBLE_DURATION_SEC; + } + } + + /** Show a reaction bubble on every character in the office. */ + triggerReactBubblesAll(type: 'react_pass' | 'react_fail'): void { + for (const ch of this.characters.values()) { + ch.bubbleType = type; + ch.bubbleTimer = WAITING_BUBBLE_DURATION_SEC; + } + } + /** Dismiss bubble on click — permission: instant, waiting: quick fade */ dismissBubble(id: number): void { const ch = this.characters.get(id); @@ -744,8 +761,12 @@ export class OfficeState { updateCharacter(ch, dt, this.walkableTiles, this.seats, this.tileMap, this.blockedTiles), ); - // Tick bubble timer for waiting bubbles - if (ch.bubbleType === 'waiting') { + // Tick bubble timer for timed bubbles (waiting, react_pass, react_fail) + if ( + ch.bubbleType === 'waiting' || + ch.bubbleType === 'react_pass' || + ch.bubbleType === 'react_fail' + ) { ch.bubbleTimer -= dt; if (ch.bubbleTimer <= 0) { ch.bubbleType = null; diff --git a/ui/src/office/engine/renderer.ts b/ui/src/office/engine/renderer.ts index ae88daa0..da395f0c 100644 --- a/ui/src/office/engine/renderer.ts +++ b/ui/src/office/engine/renderer.ts @@ -37,6 +37,8 @@ import { getColorizedFloorSprite, hasFloorSprites, WALL_COLOR } from '../floorTi import { getCachedSprite, getOutlineSprite } from '../sprites/spriteCache.js'; import { BUBBLE_PERMISSION_SPRITE, + BUBBLE_REACT_FAIL_SPRITE, + BUBBLE_REACT_PASS_SPRITE, BUBBLE_WAITING_SPRITE, getCharacterSprites, } from '../sprites/spriteData.js'; @@ -497,11 +499,22 @@ function renderBubbles( if (!ch.bubbleType) continue; const sprite = - ch.bubbleType === 'permission' ? BUBBLE_PERMISSION_SPRITE : BUBBLE_WAITING_SPRITE; - - // Compute opacity: permission = full, waiting = fade in last 0.5s + ch.bubbleType === 'permission' + ? BUBBLE_PERMISSION_SPRITE + : ch.bubbleType === 'react_pass' + ? BUBBLE_REACT_PASS_SPRITE + : ch.bubbleType === 'react_fail' + ? BUBBLE_REACT_FAIL_SPRITE + : BUBBLE_WAITING_SPRITE; + + // Compute opacity: permission = full, timed bubbles fade out in last 0.5s let alpha = 1.0; - if (ch.bubbleType === 'waiting' && ch.bubbleTimer < BUBBLE_FADE_DURATION_SEC) { + if ( + (ch.bubbleType === 'waiting' || + ch.bubbleType === 'react_pass' || + ch.bubbleType === 'react_fail') && + ch.bubbleTimer < BUBBLE_FADE_DURATION_SEC + ) { alpha = ch.bubbleTimer / BUBBLE_FADE_DURATION_SEC; } diff --git a/ui/src/office/sprites/bubble-react-fail.json b/ui/src/office/sprites/bubble-react-fail.json new file mode 100644 index 00000000..f1bc30c6 --- /dev/null +++ b/ui/src/office/sprites/bubble-react-fail.json @@ -0,0 +1,27 @@ +{ + "name": "bubble-react-fail", + "description": "Reaction fail bubble: light-pink bg with red X mark, tail pointer (11x13)", + "width": 11, + "height": 13, + "palette": { + "_": "", + "B": "#555566", + "F": "#FFDDDD", + "R": "#EE3333" + }, + "pixels": [ + ["_", "B", "B", "B", "B", "B", "B", "B", "B", "B", "_"], + ["B", "F", "F", "F", "F", "F", "F", "F", "F", "F", "B"], + ["B", "F", "F", "R", "F", "F", "F", "R", "F", "F", "B"], + ["B", "F", "F", "F", "R", "F", "R", "F", "F", "F", "B"], + ["B", "F", "F", "F", "F", "R", "F", "F", "F", "F", "B"], + ["B", "F", "F", "F", "R", "F", "R", "F", "F", "F", "B"], + ["B", "F", "F", "R", "F", "F", "F", "R", "F", "F", "B"], + ["B", "F", "F", "F", "F", "F", "F", "F", "F", "F", "B"], + ["B", "F", "F", "F", "F", "F", "F", "F", "F", "F", "B"], + ["_", "B", "B", "B", "B", "B", "B", "B", "B", "B", "_"], + ["_", "_", "_", "_", "B", "B", "B", "_", "_", "_", "_"], + ["_", "_", "_", "_", "_", "B", "_", "_", "_", "_", "_"], + ["_", "_", "_", "_", "_", "_", "_", "_", "_", "_", "_"] + ] +} diff --git a/ui/src/office/sprites/bubble-react-pass.json b/ui/src/office/sprites/bubble-react-pass.json new file mode 100644 index 00000000..96305f29 --- /dev/null +++ b/ui/src/office/sprites/bubble-react-pass.json @@ -0,0 +1,27 @@ +{ + "name": "bubble-react-pass", + "description": "Reaction pass bubble: light-green bg with bright checkmark, tail pointer (11x13)", + "width": 11, + "height": 13, + "palette": { + "_": "", + "B": "#555566", + "F": "#CCFFCC", + "G": "#22CC55" + }, + "pixels": [ + ["_", "B", "B", "B", "B", "B", "B", "B", "B", "B", "_"], + ["B", "F", "F", "F", "F", "F", "F", "F", "F", "F", "B"], + ["B", "F", "F", "F", "F", "F", "F", "F", "F", "F", "B"], + ["B", "F", "F", "F", "F", "F", "F", "F", "G", "F", "B"], + ["B", "F", "F", "F", "F", "F", "F", "G", "F", "F", "B"], + ["B", "F", "F", "G", "F", "F", "G", "F", "F", "F", "B"], + ["B", "F", "F", "F", "G", "G", "F", "F", "F", "F", "B"], + ["B", "F", "F", "F", "F", "F", "F", "F", "F", "F", "B"], + ["B", "F", "F", "F", "F", "F", "F", "F", "F", "F", "B"], + ["_", "B", "B", "B", "B", "B", "B", "B", "B", "B", "_"], + ["_", "_", "_", "_", "B", "B", "B", "_", "_", "_", "_"], + ["_", "_", "_", "_", "_", "B", "_", "_", "_", "_", "_"], + ["_", "_", "_", "_", "_", "_", "_", "_", "_", "_", "_"] + ] +} diff --git a/ui/src/office/sprites/spriteData.ts b/ui/src/office/sprites/spriteData.ts index b00cd828..e9134d22 100644 --- a/ui/src/office/sprites/spriteData.ts +++ b/ui/src/office/sprites/spriteData.ts @@ -4,6 +4,8 @@ import { adjustSprite } from '../colorize.js'; import type { Direction, SpriteData } from '../types.js'; import { Direction as Dir } from '../types.js'; import bubblePermissionData from './bubble-permission.json'; +import bubbleReactFailData from './bubble-react-fail.json'; +import bubbleReactPassData from './bubble-react-pass.json'; import bubbleWaitingData from './bubble-waiting.json'; // ── Speech Bubble Sprites ─────────────────────────────────────── @@ -23,6 +25,12 @@ export const BUBBLE_PERMISSION_SPRITE: SpriteData = resolveBubbleSprite(bubblePe /** Waiting bubble: white square with green checkmark, and a tail pointer (11x13) */ export const BUBBLE_WAITING_SPRITE: SpriteData = resolveBubbleSprite(bubbleWaitingData); +/** React-pass bubble: light-green bg with bright checkmark (11x13) */ +export const BUBBLE_REACT_PASS_SPRITE: SpriteData = resolveBubbleSprite(bubbleReactPassData); + +/** React-fail bubble: light-pink bg with red X mark (11x13) */ +export const BUBBLE_REACT_FAIL_SPRITE: SpriteData = resolveBubbleSprite(bubbleReactFailData); + // ════════════════════════════════════════════════════════════════ // Loaded character sprites (from PNG assets) // ════════════════════════════════════════════════════════════════ diff --git a/ui/src/office/types.ts b/ui/src/office/types.ts index fc8f2bb4..2c488e6a 100644 --- a/ui/src/office/types.ts +++ b/ui/src/office/types.ts @@ -163,7 +163,7 @@ export interface Character { /** Assigned seat uid, or null if no seat */ seatId: string | null; /** Active speech bubble type, or null if none showing */ - bubbleType: 'permission' | 'waiting' | null; + bubbleType: 'permission' | 'waiting' | 'react_pass' | 'react_fail' | null; /** Countdown timer for bubble (waiting: 2→0, permission: unused) */ bubbleTimer: number; /** Timer to stay seated while inactive after seat reassignment (counts down to 0) */ diff --git a/ui/src/pages/PlaywrightDashboard.tsx b/ui/src/pages/PlaywrightDashboard.tsx index 380b148f..dabe6165 100644 --- a/ui/src/pages/PlaywrightDashboard.tsx +++ b/ui/src/pages/PlaywrightDashboard.tsx @@ -4259,6 +4259,7 @@ interface PwOfficePanelProps { function PwOfficePanel({ running, suites, onRunCode, onRunAll }: PwOfficePanelProps) { const { agents, + agentStatuses, settings, updateSettings, messages, @@ -4277,6 +4278,10 @@ function PwOfficePanel({ running, suites, onRunCode, onRunAll }: PwOfficePanelPr const textareaRef = useRef(null); const fileInputRef = useRef(null); const abortRef = useRef(null); + /** Tracks the previous `running` value to detect run completion */ + const prevRunningRef = useRef(false); + /** Epoch ms until which all agents stay briefly active (celebration) */ + const celebrationEndRef = useRef(0); const [assetsReady, setAssetsReady] = useState(false); const [inputText, setInputText] = useState(''); @@ -4322,16 +4327,36 @@ function PwOfficePanel({ running, suites, onRunCode, onRunAll }: PwOfficePanelPr // ── Derive failure flag ─────────────────────────────────────────────────── const hasFailures = suites.some((s) => s.tests.some((t) => t.status === 'failed')); - // ── Sync: drive animations from test state ──────────────────────────────── + // ── Sync: drive animations from test state + chat status ───────────────── useEffect(() => { if (!_pwOfficeState) return; + const isCelebrating = Date.now() < celebrationEndRef.current; agents.forEach((agent, idx) => { const numId = pwAgentNumId(idx); + const chatActive = + agentStatuses[agent.id] === 'working' || agentStatuses[agent.id] === 'waiting'; const active = - running || (hasFailures && (agent.id === 'agent-triage' || agent.id === 'agent-healer')); + running || + chatActive || + isCelebrating || + (hasFailures && (agent.id === 'agent-triage' || agent.id === 'agent-healer')); _pwOfficeState!.setAgentActive(numId, active); }); - }, [agents, running, hasFailures]); + }, [agents, running, hasFailures, agentStatuses]); + + // ── React bubbles on run completion ────────────────────────────────────── + useEffect(() => { + if (!_pwOfficeState) return; + const wasRunning = prevRunningRef.current; + prevRunningRef.current = running; + if (!wasRunning || running) return; // only fires on true→false transition + if (hasFailures) { + _pwOfficeState.triggerReactBubblesAll('react_fail'); + } else { + _pwOfficeState.triggerReactBubblesAll('react_pass'); + celebrationEndRef.current = Date.now() + 2500; + } + }, [running, hasFailures]); const handleAgentClick = (charId: number) => { const agent = agents[charId - 1]; @@ -4339,6 +4364,12 @@ function PwOfficePanel({ running, suites, onRunCode, onRunAll }: PwOfficePanelPr setSelectedAgentId(agent.id); }; + /** Map an agent's string id (e.g. "agent-triage") to its canvas numId */ + const agentStringToNumId = (agentId: string): number | null => { + const idx = agents.findIndex((a) => a.id === agentId); + return idx === -1 ? null : pwAgentNumId(idx); + }; + // ── @-mention autocomplete ──────────────────────────────────────────────── const mentionMatches = @@ -4664,7 +4695,14 @@ function PwOfficePanel({ running, suites, onRunCode, onRunAll }: PwOfficePanelPr continue; } if (payload.evt === 'turn_done') { - if (payload.agentId) setAgentStatus(payload.agentId, 'idle'); + if (payload.agentId) { + setAgentStatus(payload.agentId, 'idle'); + // Handoff bubble: brief waiting indicator on the finishing agent + const numId = agentStringToNumId(payload.agentId); + if (numId !== null && _pwOfficeState) { + _pwOfficeState.showWaitingBubble(numId); + } + } continue; } if (payload.evt === 'turn_verdict') {