LOU-D5: scripted mockModel for deterministic agent tests

README.md

@@ -194,6 +194,7 @@ console.log(result.usage.totalTokens, result.finishReason, result.steps);
 - [Configuration](docs/configuration.md) - agent spec fields, provider env vars, `AgentExecutor.execute()` options, CLI flags
 - [Deployment](docs/deployment.md) - `loushy build` targets: Node server, Docker, Cloudflare Workers
 - [API Overview](docs/api-overview.md) - the main exports; `npm run docs:build` generates the full TypeDoc reference
+- [Testing](docs/testing.md) - unit-test agents deterministically with the scripted `mockModel`
 - [Agent Forge](docs/agent-forge.md) - the visual dashboard (`loushy studio`): quickstart, first-agent walkthrough, hook authoring
 - Full guides site: [linuxdevil.github.io/agent-sdk-docs](https://linuxdevil.github.io/agent-sdk-docs/)
 

docs/api-overview.md

@@ -36,6 +36,15 @@ npm run docs:build   # writes docs/api/index.html
 | `OpenAIProvider`, `AnthropicProvider`, `OllamaProvider`, `OpenRouterProvider` | Provider classes. |
 | `createMockProvider()`, `MockLLMProvider` | Deterministic mock provider for tests and demos.   |
 
+## Testing
+
+Exported from `@loushy/build-ai-agent/testing` (see [Testing agents](testing.md)).
+
+| Export                | Description                                                                              |
+| --------------------- | ---------------------------------------------------------------------------------------- |
+| `mockModel(script)`   | Scripted, deterministic `LLMProvider` that records every request (`calls`, `lastCall`, `reset()`, `assertExhausted()`). |
+| `Mock*Repository`     | In-memory repository mocks (`MockAgentRepository`, `MockSessionRepository`, ...).        |
+
 ## Tools
 
 | Export                                       | Description                                   |

docs/testing.md

@@ -0,0 +1,155 @@
+# Testing agents
+
+`@loushy/build-ai-agent/testing` ships `mockModel`: a scripted, deterministic
+`LLMProvider` for unit tests. You write down what the model should say on each
+turn, run your agent, and then assert on exactly what the agent sent to the
+model. No network, no API keys, no flakiness.
+
+```bash
+npm install --save-dev vitest
+```
+
+The same subpath still exports the in-memory repository mocks
+(`MockAgentRepository`, `MockSessionRepository`, ...).
+
+## A text reply
+
+A bare string is shorthand for `{ text }`.
+
+```ts
+import { createAgent } from '@loushy/build-ai-agent';
+import { mockModel } from '@loushy/build-ai-agent/testing';
+
+const model = mockModel(['Hello! How can I help?']);
+const agent = createAgent({ prompt: 'You are friendly.', provider: model });
+
+const result = await agent.send('hi');
+
+console.log(result.text); // Hello! How can I help?
+console.log(model.calls.length); // 1
+model.assertExhausted();
+```
+
+## A tool-call flow
+
+Each element of the script is one model turn. Turn 1 asks for a tool; the agent
+runs it and calls the model again; turn 2 produces the final answer. Tool-call
+ids are generated deterministically (`call_1`, `call_2`, ...) unless you pass
+`id`.
+
+```ts no-verify
+import { describe, it, expect, vi } from 'vitest';
+import { z } from 'zod';
+import { createAgent } from '@loushy/build-ai-agent';
+import { mockModel } from '@loushy/build-ai-agent/testing';
+
+describe('weather agent', () => {
+  it('calls get_weather and reports the result', async () => {
+    const execute = vi.fn(async ({ city }: { city: string }) => ({ city, tempC: 21 }));
+    const model = mockModel([
+      { toolCalls: [{ name: 'get_weather', args: { city: 'Paris' } }] }, // turn 1
+      { text: 'It is 21°C in Paris.' },                                  // turn 2
+    ]);
+    const agent = createAgent({
+      prompt: 'You report the weather.',
+      provider: model,
+      tools: {
+        get_weather: {
+          displayName: 'Get weather',
+          tool: {
+            description: 'Get the weather for a city',
+            parameters: z.object({ city: z.string() }),
+            execute,
+          },
+        },
+      },
+    });
+
+    const result = await agent.send('Weather in Paris?');
+
+    expect(result.text).toBe('It is 21°C in Paris.');
+    expect(execute).toHaveBeenCalledWith({ city: 'Paris' }, expect.anything());
+    expect(model.calls).toHaveLength(2);
+    expect(model.calls[1].messages.at(-1)).toMatchObject({ role: 'tool', toolCallId: 'call_1' });
+    model.assertExhausted(); // fails if scripted turns were never used
+  });
+});
+```
+
+## An error turn
+
+`{ error }` makes the model call reject, so you can test how your agent behaves
+when the provider fails.
+
+```ts no-verify
+import { it, expect } from 'vitest';
+import { createAgent } from '@loushy/build-ai-agent';
+import { mockModel } from '@loushy/build-ai-agent/testing';
+
+it('surfaces a provider failure', async () => {
+  const model = mockModel([{ error: new Error('upstream 503') }]);
+  const agent = createAgent({ prompt: 'x', provider: model });
+
+  await expect(agent.send('hi')).rejects.toThrow();
+  expect(model.calls).toHaveLength(1); // the failed request is still recorded
+});
+```
+
+## Asserting requests
+
+`model.calls` holds a deep-frozen snapshot of every request, in order, so later
+mutation by the agent can never rewrite history. Use `model.lastCall` for the
+most recent one.
+
+```ts no-verify
+expect(model.calls[0].messages[0]).toMatchObject({ role: 'system' });
+expect(model.lastCall?.tools?.map((t) => t.function.name)).toContain('get_weather');
+expect(model.calls[0].temperature).toBe(0.2);
+```
+
+## Dynamic turns
+
+Pass a function to compute a turn from the request. It may be async and may
+return a string or any turn object.
+
+```ts
+import { mockModel } from '@loushy/build-ai-agent/testing';
+
+const model = mockModel([(req) => `You said: ${req.messages.at(-1)?.content}`]);
+const result = await model.generate({ messages: [{ role: 'user', content: 'ping' }] });
+
+console.log(result.text); // You said: ping
+```
+
+## Turn reference
+
+| Field          | Meaning                                                                        |
+| -------------- | ------------------------------------------------------------------------------ |
+| `text`         | Assistant text (default `''`).                                                 |
+| `toolCalls`    | `[{ name, args?, id? }]`. Args are JSON-encoded; ids default to `call_N`.      |
+| `error`        | Reject `generate()` / `stream()` with this error.                              |
+| `usage`        | `{ inputTokens, outputTokens }` (default zero).                                |
+| `finishReason` | Defaults to `'tool_calls'` when there are tool calls, otherwise `'stop'`.      |
+| `delayMs`      | Wait before answering (works with `vi.useFakeTimers()`).                       |
+
+## Running out of turns
+
+If the agent calls the model more often than you scripted, `mockModel` throws
+an error that names the unexpected call number, shows the last message of the
+request and tells you to add a turn. To replay the final turn forever (handy for
+loop and step-limit tests), pass `{ onExhausted: 'repeat-last' }`:
+
+```ts
+import { mockModel } from '@loushy/build-ai-agent/testing';
+
+const looping = mockModel([{ toolCalls: [{ name: 'again' }] }], { onExhausted: 'repeat-last' });
+```
+
+Other helpers: `model.reset()` rewinds the script and clears recorded calls, and
+`model.assertExhausted()` throws if scripted turns remain unused.
+
+## Streaming
+
+`mockModel` also implements `stream()`: the scripted text is emitted as
+`text-delta` chunks (split on word boundaries), followed by any `tool-call`
+chunks and a final `finish` chunk, so streaming consumers can be tested too.

package.json

@@ -44,9 +44,9 @@
       "require": "./dist/types/index.js"
     },
     "./testing": {
-      "types": "./dist/data/mocks.d.ts",
-      "import": "./dist/data/mocks.mjs",
-      "require": "./dist/data/mocks.js"
+      "types": "./dist/testing/index.d.ts",
+      "import": "./dist/testing/index.mjs",
+      "require": "./dist/testing/index.js"
     },
     "./otel": {
       "types": "./dist/execution/otel.d.ts",

src/execution/DelegationTool.test.ts

@@ -3,15 +3,8 @@ import { createDelegateTool, DelegationDepthExceededError } from './DelegationTo
 import { AgentExecutor } from './AgentExecutor';
 import { AgentType } from '../types';
 import { ToolRegistry } from '../tools';
-import type { LLMProvider, GenerateResult } from '../providers';
-
-function makeGenerateResult(text: string): GenerateResult {
-  return {
-    text,
-    finishReason: 'stop',
-    usage: { promptTokens: 1, completionTokens: 1, totalTokens: 2 },
-  };
-}
+import type { LLMProvider } from '../providers';
+import { mockModel } from '../testing';
 
 function makeMockProvider(generate: LLMProvider['generate']): LLMProvider {
   return {
@@ -26,8 +19,7 @@ function makeMockProvider(generate: LLMProvider['generate']): LLMProvider {
 
 describe('createDelegateTool', () => {
   it('invokes the child agent with only the delegated task as input', async () => {
-    const generate = vi.fn().mockResolvedValue(makeGenerateResult('child response'));
-    const provider = makeMockProvider(generate);
+    const provider = mockModel([{ text: 'child response', usage: { inputTokens: 1, outputTokens: 1 } }]);
 
     const childAgent = {
       name: 'Child Agent',
@@ -42,13 +34,13 @@ describe('createDelegateTool', () => {
 
     const result = await delegateTool.tool.execute!({ task: 'Do the thing' }, {} as any);
 
-    expect(generate).toHaveBeenCalledTimes(1);
-    const callArgs = generate.mock.calls[0][0];
+    expect(provider.calls).toHaveLength(1);
+    const callArgs = provider.calls[0];
 
     // messages should contain the system prompt (from agent.prompt) plus
     // exactly one user message with the delegated task - no unrelated
     // parent conversation history leaking in.
-    const userMessages = callArgs.messages.filter((m: any) => m.role === 'user');
+    const userMessages = callArgs.messages.filter((m) => m.role === 'user');
     expect(userMessages).toEqual([{ role: 'user', content: 'Do the thing' }]);
 
     expect(result).toEqual({
@@ -58,8 +50,7 @@ describe('createDelegateTool', () => {
   });
 
   it('works with an agent that has no system prompt', async () => {
-    const generate = vi.fn().mockResolvedValue(makeGenerateResult('ok'));
-    const provider = makeMockProvider(generate);
+    const provider = mockModel(['ok']);
 
     const childAgent = {
       name: 'Bare Agent',
@@ -69,21 +60,12 @@ describe('createDelegateTool', () => {
     const delegateTool = createDelegateTool({ agent: childAgent, provider });
     await delegateTool.tool.execute!({ task: 'task 1' }, {} as any);
 
-    // `callArgs.messages` is the same array reference AgentExecutor mutates
-    // in place, so after execute() resolves it also reflects the final
-    // assistant reply that gets pushed once the no-tool-calls exit path
-    // runs (see AgentExecutor.ts) - not just what was sent on the wire.
-    const callArgs = generate.mock.calls[0][0];
-    expect(callArgs.messages).toEqual([
-      { role: 'user', content: 'task 1' },
-      { role: 'assistant', content: 'ok' },
-    ]);
+    expect(provider.calls[0].messages).toEqual([{ role: 'user', content: 'task 1' }]);
   });
 
   describe('contextMode: full-history', () => {
     it('passes context history then the task message, in order', async () => {
-      const generate = vi.fn().mockResolvedValue(makeGenerateResult('done'));
-      const provider = makeMockProvider(generate);
+      const provider = mockModel(['done']);
 
       const agent = { name: 'Contextual Agent', agentType: AgentType.SmartAssistant };
       const delegateTool = createDelegateTool({ agent, provider, contextMode: 'full-history' });
@@ -95,33 +77,22 @@ describe('createDelegateTool', () => {
 
       await delegateTool.tool.execute!({ task: 'follow-up task', context }, {} as any);
 
-      // See the "no system prompt" test above re: this array reference
-      // reflecting the post-call assistant push too.
-      const callArgs = generate.mock.calls[0][0];
-      expect(callArgs.messages).toEqual([
+      expect(provider.calls[0].messages).toEqual([
         ...context,
         { role: 'user', content: 'follow-up task' },
-        { role: 'assistant', content: 'done' },
       ]);
     });
 
     it('falls back to task-only input when contextMode is the default "none"', async () => {
-      const generate = vi.fn().mockResolvedValue(makeGenerateResult('done'));
-      const provider = makeMockProvider(generate);
+      const provider = mockModel(['done']);
 
       const agent = { name: 'Contextual Agent', agentType: AgentType.SmartAssistant };
       const delegateTool = createDelegateTool({ agent, provider });
 
       const context = [{ role: 'user' as const, content: 'earlier question' }];
       await delegateTool.tool.execute!({ task: 'follow-up task', context }, {} as any);
 
-      // See the "no system prompt" test above re: this array reference
-      // reflecting the post-call assistant push too.
-      const callArgs = generate.mock.calls[0][0];
-      expect(callArgs.messages).toEqual([
-        { role: 'user', content: 'follow-up task' },
-        { role: 'assistant', content: 'done' },
-      ]);
+      expect(provider.calls[0].messages).toEqual([{ role: 'user', content: 'follow-up task' }]);
     });
   });
 
@@ -282,29 +253,15 @@ describe('createDelegateTool', () => {
       const maxDepth = 2;
       const maxSteps = 5;
 
-      function makeDelegateToolCall(toolName: string, task: string): GenerateResult {
-        return {
-          text: '',
-          finishReason: 'tool_calls',
-          usage: { promptTokens: 1, completionTokens: 1, totalTokens: 2 },
-          toolCalls: [
-            {
-              id: `call-${Math.random().toString(36).slice(2)}`,
-              type: 'function',
-              function: { name: toolName, arguments: JSON.stringify({ task }) },
-            },
-          ],
-        };
-      }
-
       // Always requests delegation to the other agent - never stops on its
       // own. Exactly the "neither agent ever stops" pathological case from
       // the bug report.
-      const generateA = vi.fn(async () => makeDelegateToolCall('delegate', 'go to B'));
-      const generateB = vi.fn(async () => makeDelegateToolCall('delegate', 'go to A'));
-
-      const providerA = makeMockProvider(generateA);
-      const providerB = makeMockProvider(generateB);
+      const delegating = (task: string) => ({
+        toolCalls: [{ name: 'delegate', args: { task } }],
+        usage: { inputTokens: 1, outputTokens: 1 },
+      });
+      const providerA = mockModel([delegating('go to B')], { onExhausted: 'repeat-last' });
+      const providerB = mockModel([delegating('go to A')], { onExhausted: 'repeat-last' });
 
       const agentA = { name: 'Agent A', agentType: AgentType.SmartAssistant, tools: { delegate: { tool: 'delegate' } } };
       const agentB = { name: 'Agent B', agentType: AgentType.SmartAssistant, tools: { delegate: { tool: 'delegate' } } };
@@ -353,7 +310,7 @@ describe('createDelegateTool', () => {
       // fix, QA measured 155 calls for maxDepth=2/maxSteps=5 (and 780 for
       // maxDepth=3/maxSteps=5); after the fix this should be a handful of
       // calls (one per hop until the guard fires), well under 20.
-      const totalCalls = generateA.mock.calls.length + generateB.mock.calls.length;
+      const totalCalls = providerA.calls.length + providerB.calls.length;
       expect(totalCalls).toBeLessThan(20);
     });
   });