LOU-O: debug & observability console

apps/agent-forge/server/__tests__/app.test.ts

@@ -162,4 +162,46 @@ describe('LOU-N HTTP API', () => {
     const res = await request(app).post('/agents/foo/approve').send({ approvalId: 'x' });
     expect(res.status).toBe(400);
   });
+
+  it('GET /agents/:id/debug reports no breakpoints/not paused for an agent that never ran', async () => {
+    const res = await request(app).get('/agents/never-debugged/debug');
+    expect(res.status).toBe(200);
+    expect(res.body).toMatchObject({ agentId: 'never-debugged', paused: false, breakpoints: [] });
+  });
+
+  it('PUT /agents/:id/debug/breakpoints sets breakpoints, reflected by GET /agents/:id/debug', async () => {
+    const put = await request(app)
+      .put('/agents/debug-bp/debug/breakpoints')
+      .send({ breakpoints: ['llm:before', 'tool:current-date:after'] });
+    expect(put.status).toBe(200);
+    expect(put.body.breakpoints.sort()).toEqual(['llm:before', 'tool:current-date:after']);
+
+    const get = await request(app).get('/agents/debug-bp/debug');
+    expect(get.body.breakpoints.sort()).toEqual(['llm:before', 'tool:current-date:after']);
+  });
+
+  it('PUT /agents/:id/debug/breakpoints rejects a non-array body', async () => {
+    const res = await request(app).put('/agents/debug-bp/debug/breakpoints').send({ breakpoints: 'nope' });
+    expect(res.status).toBe(400);
+  });
+
+  it('a run paused at a breakpoint resumes via POST /agents/:id/debug/continue', async () => {
+    await request(app)
+      .put('/agents/debug-run/debug/breakpoints')
+      .send({ breakpoints: ['llm:before'] });
+
+    await request(app).post('/agents/debug-run/run').send({ input: 'hi', spec: SPEC });
+    await new Promise((resolve) => setTimeout(resolve, 50));
+
+    const pausedState = await request(app).get('/agents/debug-run/debug');
+    expect(pausedState.body.paused).toBe(true);
+
+    // Clear breakpoints so a follow-up llm turn (if any) doesn't re-pause,
+    // then continue until the run reaches a terminal status.
+    await request(app).put('/agents/debug-run/debug/breakpoints').send({ breakpoints: [] });
+    const cont = await request(app).post('/agents/debug-run/debug/continue');
+    expect(cont.status).toBe(200);
+
+    await waitForStatus(runManager, 'debug-run', (s) => s.status === 'stopped');
+  });
 });

apps/agent-forge/server/__tests__/debugController.test.ts

@@ -0,0 +1,101 @@
+import { describe, it, expect, vi } from 'vitest';
+import { DebugSession, llmBreakpointKey, toolBreakpointKey } from '../debugController';
+
+describe('DebugSession', () => {
+  it('does not pause when no breakpoint matches and no step is armed', async () => {
+    const onChange = vi.fn();
+    const session = new DebugSession([], onChange);
+    await session.hooks().onLLMRequest?.({ model: 'x', messages: [] });
+    expect(onChange).not.toHaveBeenCalled();
+    expect(session.snapshot().paused).toBe(false);
+  });
+
+  it('pauses at a matching llm:before breakpoint and resumes on continue()', async () => {
+    const states: boolean[] = [];
+    const session = new DebugSession([llmBreakpointKey('before')], (s) => states.push(s.paused));
+
+    const hookPromise = session.hooks().onLLMRequest?.({ model: 'x', messages: [{ role: 'user', content: 'hi' } as any] });
+
+    // Give the pause a tick to take effect before asserting/continuing.
+    await new Promise((r) => setTimeout(r, 10));
+    expect(session.snapshot().paused).toBe(true);
+    expect(session.snapshot().atBreakpoint).toEqual({ phase: 'llm', boundary: 'before' });
+    expect(session.snapshot().messages).toHaveLength(1);
+
+    session.continue();
+    await hookPromise;
+    expect(session.snapshot().paused).toBe(false);
+    expect(states).toEqual([true, false]);
+  });
+
+  it('pauses on tool:<name>:after breakpoints keyed by the tool call name', async () => {
+    const session = new DebugSession([toolBreakpointKey('current-date', 'after')], () => {});
+    const toolCall = { id: '1', type: 'function' as const, function: { name: 'current-date', arguments: '{}' } };
+
+    const hookPromise = session.hooks().onToolResult?.(toolCall, undefined, 5);
+    await new Promise((r) => setTimeout(r, 10));
+    expect(session.snapshot().paused).toBe(true);
+    session.continue();
+    await hookPromise;
+  });
+
+  it('step() arms a one-shot pause at the next hook boundary regardless of breakpoints', async () => {
+    const session = new DebugSession([], () => {});
+    session.step();
+    const hookPromise = session.hooks().onLLMRequest?.({ model: 'x', messages: [] });
+    await new Promise((r) => setTimeout(r, 10));
+    expect(session.snapshot().paused).toBe(true);
+    session.continue();
+    await hookPromise;
+
+    // Step is one-shot: the next hook call does NOT pause again.
+    const secondPromise = session.hooks().onLLMResponse?.({} as any, 1);
+    await secondPromise;
+    expect(session.snapshot().paused).toBe(false);
+  });
+
+  it('setBreakpoints() updates the live set and notifies onChange', () => {
+    const onChange = vi.fn();
+    const session = new DebugSession([], onChange);
+    session.setBreakpoints([llmBreakpointKey('after')]);
+    expect(session.snapshot().breakpoints).toEqual(['llm:after']);
+    expect(onChange).toHaveBeenCalledWith(expect.objectContaining({ breakpoints: ['llm:after'] }));
+  });
+
+  it('step() while already paused releases the current pause instead of stacking', async () => {
+    const session = new DebugSession([llmBreakpointKey('before')], () => {});
+    const hookPromise = session.hooks().onLLMRequest?.({ model: 'x', messages: [] });
+    await new Promise((r) => setTimeout(r, 10));
+    expect(session.snapshot().paused).toBe(true);
+    session.step();
+    await hookPromise;
+    expect(session.snapshot().paused).toBe(false);
+  });
+
+  it('auto-resumes (and flags autoResumed) if nobody continues/steps before pauseTimeoutMs elapses', async () => {
+    const states: Array<{ paused: boolean; autoResumed?: boolean }> = [];
+    // A tiny pauseTimeoutMs stands in for an abandoned pause (e.g. the only
+    // WS client disconnected) - this is the safety net that stops a run
+    // staying paused forever with no way to resume it.
+    const session = new DebugSession([llmBreakpointKey('before')], (s) => states.push({ paused: s.paused, autoResumed: s.autoResumed }), 15);
+
+    const hookPromise = session.hooks().onLLMRequest?.({ model: 'x', messages: [] });
+    await new Promise((r) => setTimeout(r, 10));
+    expect(session.snapshot().paused).toBe(true);
+
+    // Nobody calls continue()/step() - just wait past the timeout.
+    await hookPromise;
+    expect(session.snapshot().paused).toBe(false);
+    expect(states.at(-1)).toEqual({ paused: false, autoResumed: true });
+  });
+
+  it('pauseTimeoutMs: 0 disables the safety net (pause never auto-resumes on its own)', async () => {
+    const session = new DebugSession([llmBreakpointKey('before')], () => {}, 0);
+    const hookPromise = session.hooks().onLLMRequest?.({ model: 'x', messages: [] });
+    await new Promise((r) => setTimeout(r, 30));
+    expect(session.snapshot().paused).toBe(true);
+    session.continue();
+    await hookPromise;
+    expect(session.snapshot().paused).toBe(false);
+  });
+});

apps/agent-forge/server/__tests__/runRegistry.test.ts

@@ -117,6 +117,65 @@ describe('RunManager', () => {
     expect(afterResume).toBeNull();
   });
 
+  it('emits structured log entries derived from the ExecutionEvent stream (O1)', async () => {
+    const logs: any[] = [];
+    runManager.on('log', (agentId: string, entry: any) => {
+      if (agentId === 'agent-logs') logs.push(entry);
+    });
+
+    await runManager.run('agent-logs', 'please use current-date', SPEC);
+    await waitForStatus(runManager, 'agent-logs', (s) => s.status === 'stopped');
+
+    expect(logs.length).toBeGreaterThan(0);
+    expect(logs.every((l) => typeof l.id === 'string' && typeof l.timestamp === 'string')).toBe(true);
+    expect(logs.some((l) => l.phase === 'trigger' && l.message.includes('started'))).toBe(true);
+    expect(logs.some((l) => l.phase === 'tool' && l.toolName === 'current-date')).toBe(true);
+  });
+
+  it('emits real span start/end events from the SDK TraceExporter hook (O2)', async () => {
+    const spans: any[] = [];
+    runManager.on('span', (agentId: string, span: any) => {
+      if (agentId === 'agent-spans') spans.push(span);
+    });
+
+    await runManager.run('agent-spans', 'please use current-date', SPEC);
+    await waitForStatus(runManager, 'agent-spans', (s) => s.status === 'stopped');
+
+    const runSpans = spans.filter((s) => s.name === 'agent.run');
+    expect(runSpans).toHaveLength(2); // start + end
+    const [startSpan, endSpan] = runSpans;
+    expect(startSpan.endTime).toBeUndefined();
+    expect(endSpan.endTime).toBeGreaterThanOrEqual(endSpan.startTime);
+    expect(spans.some((s) => s.name === 'llm.generate')).toBe(true);
+    expect(spans.some((s) => s.name === 'tool.call' && s.parentId === startSpan.id)).toBe(true);
+  });
+
+  it('pauses a run at a configured breakpoint and resumes on continueRun() (O3)', async () => {
+    const debugStates: any[] = [];
+    let pauseCount = 0;
+    // The fixture SPEC's prompt drives the mock provider through a tool
+    // call then a final response - two provider.generate() calls, so a
+    // persistent 'llm:before' breakpoint pauses twice. continueRun() every
+    // time the run reports paused, same as a real client would on seeing
+    // `{type:'debug', payload:{paused:true}}` over the WS stream.
+    runManager.on('debug', (agentId: string, state: any) => {
+      if (agentId !== 'agent-debug') return;
+      debugStates.push(state);
+      if (state.paused) {
+        pauseCount += 1;
+        runManager.continueRun('agent-debug');
+      }
+    });
+
+    runManager.setBreakpoints('agent-debug', ['llm:before']);
+    await runManager.run('agent-debug', 'please use current-date', SPEC);
+    await waitForStatus(runManager, 'agent-debug', (s) => s.status === 'stopped');
+
+    expect(pauseCount).toBeGreaterThanOrEqual(1);
+    expect(runManager.debugState('agent-debug').paused).toBe(false);
+    expect(debugStates.some((s) => s.paused === true && s.atBreakpoint?.phase === 'llm')).toBe(true);
+  });
+
   it('surfaces a thrown provider error as status "error" without crashing the process', async () => {
     const errorSpec: AgentSpec = {
       name: 'erroring-agent',

apps/agent-forge/server/app.ts

@@ -167,6 +167,36 @@ export function createApp({ agentStore, runManager }: CreateAppOptions): Express
     })
   );
 
+  // O3: step-through debugger controls (Topbar's "Debug" button). See
+  // debugController.ts for exactly what "breakpoint"/"paused" mean given
+  // AgentExecutor's real control surface.
+  app.get('/agents/:id/debug', (req, res) => {
+    res.json(runManager.debugState(paramId(req)));
+  });
+
+  app.put(
+    '/agents/:id/debug/breakpoints',
+    (req, res) => {
+      const { breakpoints } = req.body as { breakpoints?: unknown };
+      if (!Array.isArray(breakpoints) || !breakpoints.every((b) => typeof b === 'string')) {
+        res.status(400).json({ error: "Request body must include a 'breakpoints' string array" });
+        return;
+      }
+      runManager.setBreakpoints(paramId(req), breakpoints);
+      res.json(runManager.debugState(paramId(req)));
+    }
+  );
+
+  app.post('/agents/:id/debug/continue', (req, res) => {
+    runManager.continueRun(paramId(req));
+    res.json(runManager.debugState(paramId(req)));
+  });
+
+  app.post('/agents/:id/debug/step', (req, res) => {
+    runManager.stepRun(paramId(req));
+    res.json(runManager.debugState(paramId(req)));
+  });
+
   // eslint-disable-next-line @typescript-eslint/no-unused-vars
   app.use((err: Error, _req: Request, res: Response, _next: NextFunction) => {
     res.status(500).json({ error: err.message });