diff --git a/README.md b/README.md index 85bcf7b9d..a649dc533 100644 --- a/README.md +++ b/README.md @@ -194,6 +194,7 @@ console.log(result.usage.totalTokens, result.finishReason, result.steps); - [Configuration](docs/configuration.md) - agent spec fields, provider env vars, `AgentExecutor.execute()` options, CLI flags - [Deployment](docs/deployment.md) - `loushy build` targets: Node server, Docker, Cloudflare Workers - [API Overview](docs/api-overview.md) - the main exports; `npm run docs:build` generates the full TypeDoc reference +- [Testing](docs/testing.md) - unit-test agents deterministically with the scripted `mockModel` - [Agent Forge](docs/agent-forge.md) - the visual dashboard (`loushy studio`): quickstart, first-agent walkthrough, hook authoring - Full guides site: [linuxdevil.github.io/agent-sdk-docs](https://linuxdevil.github.io/agent-sdk-docs/) diff --git a/docs/api-overview.md b/docs/api-overview.md index 69cce8d2a..983a71cfe 100644 --- a/docs/api-overview.md +++ b/docs/api-overview.md @@ -36,6 +36,15 @@ npm run docs:build # writes docs/api/index.html | `OpenAIProvider`, `AnthropicProvider`, `OllamaProvider`, `OpenRouterProvider` | Provider classes. | | `createMockProvider()`, `MockLLMProvider` | Deterministic mock provider for tests and demos. | +## Testing + +Exported from `@loushy/build-ai-agent/testing` (see [Testing agents](testing.md)). + +| Export | Description | +| --------------------- | ---------------------------------------------------------------------------------------- | +| `mockModel(script)` | Scripted, deterministic `LLMProvider` that records every request (`calls`, `lastCall`, `reset()`, `assertExhausted()`). | +| `Mock*Repository` | In-memory repository mocks (`MockAgentRepository`, `MockSessionRepository`, ...). | + ## Tools | Export | Description | diff --git a/docs/testing.md b/docs/testing.md new file mode 100644 index 000000000..c6ccf9676 --- /dev/null +++ b/docs/testing.md @@ -0,0 +1,155 @@ +# Testing agents + +`@loushy/build-ai-agent/testing` ships `mockModel`: a scripted, deterministic +`LLMProvider` for unit tests. You write down what the model should say on each +turn, run your agent, and then assert on exactly what the agent sent to the +model. No network, no API keys, no flakiness. + +```bash +npm install --save-dev vitest +``` + +The same subpath still exports the in-memory repository mocks +(`MockAgentRepository`, `MockSessionRepository`, ...). + +## A text reply + +A bare string is shorthand for `{ text }`. + +```ts +import { createAgent } from '@loushy/build-ai-agent'; +import { mockModel } from '@loushy/build-ai-agent/testing'; + +const model = mockModel(['Hello! How can I help?']); +const agent = createAgent({ prompt: 'You are friendly.', provider: model }); + +const result = await agent.send('hi'); + +console.log(result.text); // Hello! How can I help? +console.log(model.calls.length); // 1 +model.assertExhausted(); +``` + +## A tool-call flow + +Each element of the script is one model turn. Turn 1 asks for a tool; the agent +runs it and calls the model again; turn 2 produces the final answer. Tool-call +ids are generated deterministically (`call_1`, `call_2`, ...) unless you pass +`id`. + +```ts no-verify +import { describe, it, expect, vi } from 'vitest'; +import { z } from 'zod'; +import { createAgent } from '@loushy/build-ai-agent'; +import { mockModel } from '@loushy/build-ai-agent/testing'; + +describe('weather agent', () => { + it('calls get_weather and reports the result', async () => { + const execute = vi.fn(async ({ city }: { city: string }) => ({ city, tempC: 21 })); + const model = mockModel([ + { toolCalls: [{ name: 'get_weather', args: { city: 'Paris' } }] }, // turn 1 + { text: 'It is 21°C in Paris.' }, // turn 2 + ]); + const agent = createAgent({ + prompt: 'You report the weather.', + provider: model, + tools: { + get_weather: { + displayName: 'Get weather', + tool: { + description: 'Get the weather for a city', + parameters: z.object({ city: z.string() }), + execute, + }, + }, + }, + }); + + const result = await agent.send('Weather in Paris?'); + + expect(result.text).toBe('It is 21°C in Paris.'); + expect(execute).toHaveBeenCalledWith({ city: 'Paris' }, expect.anything()); + expect(model.calls).toHaveLength(2); + expect(model.calls[1].messages.at(-1)).toMatchObject({ role: 'tool', toolCallId: 'call_1' }); + model.assertExhausted(); // fails if scripted turns were never used + }); +}); +``` + +## An error turn + +`{ error }` makes the model call reject, so you can test how your agent behaves +when the provider fails. + +```ts no-verify +import { it, expect } from 'vitest'; +import { createAgent } from '@loushy/build-ai-agent'; +import { mockModel } from '@loushy/build-ai-agent/testing'; + +it('surfaces a provider failure', async () => { + const model = mockModel([{ error: new Error('upstream 503') }]); + const agent = createAgent({ prompt: 'x', provider: model }); + + await expect(agent.send('hi')).rejects.toThrow(); + expect(model.calls).toHaveLength(1); // the failed request is still recorded +}); +``` + +## Asserting requests + +`model.calls` holds a deep-frozen snapshot of every request, in order, so later +mutation by the agent can never rewrite history. Use `model.lastCall` for the +most recent one. + +```ts no-verify +expect(model.calls[0].messages[0]).toMatchObject({ role: 'system' }); +expect(model.lastCall?.tools?.map((t) => t.function.name)).toContain('get_weather'); +expect(model.calls[0].temperature).toBe(0.2); +``` + +## Dynamic turns + +Pass a function to compute a turn from the request. It may be async and may +return a string or any turn object. + +```ts +import { mockModel } from '@loushy/build-ai-agent/testing'; + +const model = mockModel([(req) => `You said: ${req.messages.at(-1)?.content}`]); +const result = await model.generate({ messages: [{ role: 'user', content: 'ping' }] }); + +console.log(result.text); // You said: ping +``` + +## Turn reference + +| Field | Meaning | +| -------------- | ------------------------------------------------------------------------------ | +| `text` | Assistant text (default `''`). | +| `toolCalls` | `[{ name, args?, id? }]`. Args are JSON-encoded; ids default to `call_N`. | +| `error` | Reject `generate()` / `stream()` with this error. | +| `usage` | `{ inputTokens, outputTokens }` (default zero). | +| `finishReason` | Defaults to `'tool_calls'` when there are tool calls, otherwise `'stop'`. | +| `delayMs` | Wait before answering (works with `vi.useFakeTimers()`). | + +## Running out of turns + +If the agent calls the model more often than you scripted, `mockModel` throws +an error that names the unexpected call number, shows the last message of the +request and tells you to add a turn. To replay the final turn forever (handy for +loop and step-limit tests), pass `{ onExhausted: 'repeat-last' }`: + +```ts +import { mockModel } from '@loushy/build-ai-agent/testing'; + +const looping = mockModel([{ toolCalls: [{ name: 'again' }] }], { onExhausted: 'repeat-last' }); +``` + +Other helpers: `model.reset()` rewinds the script and clears recorded calls, and +`model.assertExhausted()` throws if scripted turns remain unused. + +## Streaming + +`mockModel` also implements `stream()`: the scripted text is emitted as +`text-delta` chunks (split on word boundaries), followed by any `tool-call` +chunks and a final `finish` chunk, so streaming consumers can be tested too. diff --git a/package.json b/package.json index 491554dcd..618537c84 100644 --- a/package.json +++ b/package.json @@ -44,9 +44,9 @@ "require": "./dist/types/index.js" }, "./testing": { - "types": "./dist/data/mocks.d.ts", - "import": "./dist/data/mocks.mjs", - "require": "./dist/data/mocks.js" + "types": "./dist/testing/index.d.ts", + "import": "./dist/testing/index.mjs", + "require": "./dist/testing/index.js" }, "./otel": { "types": "./dist/execution/otel.d.ts", diff --git a/src/execution/DelegationTool.test.ts b/src/execution/DelegationTool.test.ts index 1f87d1dd3..460f799a4 100644 --- a/src/execution/DelegationTool.test.ts +++ b/src/execution/DelegationTool.test.ts @@ -3,15 +3,8 @@ import { createDelegateTool, DelegationDepthExceededError } from './DelegationTo import { AgentExecutor } from './AgentExecutor'; import { AgentType } from '../types'; import { ToolRegistry } from '../tools'; -import type { LLMProvider, GenerateResult } from '../providers'; - -function makeGenerateResult(text: string): GenerateResult { - return { - text, - finishReason: 'stop', - usage: { promptTokens: 1, completionTokens: 1, totalTokens: 2 }, - }; -} +import type { LLMProvider } from '../providers'; +import { mockModel } from '../testing'; function makeMockProvider(generate: LLMProvider['generate']): LLMProvider { return { @@ -26,8 +19,7 @@ function makeMockProvider(generate: LLMProvider['generate']): LLMProvider { describe('createDelegateTool', () => { it('invokes the child agent with only the delegated task as input', async () => { - const generate = vi.fn().mockResolvedValue(makeGenerateResult('child response')); - const provider = makeMockProvider(generate); + const provider = mockModel([{ text: 'child response', usage: { inputTokens: 1, outputTokens: 1 } }]); const childAgent = { name: 'Child Agent', @@ -42,13 +34,13 @@ describe('createDelegateTool', () => { const result = await delegateTool.tool.execute!({ task: 'Do the thing' }, {} as any); - expect(generate).toHaveBeenCalledTimes(1); - const callArgs = generate.mock.calls[0][0]; + expect(provider.calls).toHaveLength(1); + const callArgs = provider.calls[0]; // messages should contain the system prompt (from agent.prompt) plus // exactly one user message with the delegated task - no unrelated // parent conversation history leaking in. - const userMessages = callArgs.messages.filter((m: any) => m.role === 'user'); + const userMessages = callArgs.messages.filter((m) => m.role === 'user'); expect(userMessages).toEqual([{ role: 'user', content: 'Do the thing' }]); expect(result).toEqual({ @@ -58,8 +50,7 @@ describe('createDelegateTool', () => { }); it('works with an agent that has no system prompt', async () => { - const generate = vi.fn().mockResolvedValue(makeGenerateResult('ok')); - const provider = makeMockProvider(generate); + const provider = mockModel(['ok']); const childAgent = { name: 'Bare Agent', @@ -69,21 +60,12 @@ describe('createDelegateTool', () => { const delegateTool = createDelegateTool({ agent: childAgent, provider }); await delegateTool.tool.execute!({ task: 'task 1' }, {} as any); - // `callArgs.messages` is the same array reference AgentExecutor mutates - // in place, so after execute() resolves it also reflects the final - // assistant reply that gets pushed once the no-tool-calls exit path - // runs (see AgentExecutor.ts) - not just what was sent on the wire. - const callArgs = generate.mock.calls[0][0]; - expect(callArgs.messages).toEqual([ - { role: 'user', content: 'task 1' }, - { role: 'assistant', content: 'ok' }, - ]); + expect(provider.calls[0].messages).toEqual([{ role: 'user', content: 'task 1' }]); }); describe('contextMode: full-history', () => { it('passes context history then the task message, in order', async () => { - const generate = vi.fn().mockResolvedValue(makeGenerateResult('done')); - const provider = makeMockProvider(generate); + const provider = mockModel(['done']); const agent = { name: 'Contextual Agent', agentType: AgentType.SmartAssistant }; const delegateTool = createDelegateTool({ agent, provider, contextMode: 'full-history' }); @@ -95,19 +77,14 @@ describe('createDelegateTool', () => { await delegateTool.tool.execute!({ task: 'follow-up task', context }, {} as any); - // See the "no system prompt" test above re: this array reference - // reflecting the post-call assistant push too. - const callArgs = generate.mock.calls[0][0]; - expect(callArgs.messages).toEqual([ + expect(provider.calls[0].messages).toEqual([ ...context, { role: 'user', content: 'follow-up task' }, - { role: 'assistant', content: 'done' }, ]); }); it('falls back to task-only input when contextMode is the default "none"', async () => { - const generate = vi.fn().mockResolvedValue(makeGenerateResult('done')); - const provider = makeMockProvider(generate); + const provider = mockModel(['done']); const agent = { name: 'Contextual Agent', agentType: AgentType.SmartAssistant }; const delegateTool = createDelegateTool({ agent, provider }); @@ -115,13 +92,7 @@ describe('createDelegateTool', () => { const context = [{ role: 'user' as const, content: 'earlier question' }]; await delegateTool.tool.execute!({ task: 'follow-up task', context }, {} as any); - // See the "no system prompt" test above re: this array reference - // reflecting the post-call assistant push too. - const callArgs = generate.mock.calls[0][0]; - expect(callArgs.messages).toEqual([ - { role: 'user', content: 'follow-up task' }, - { role: 'assistant', content: 'done' }, - ]); + expect(provider.calls[0].messages).toEqual([{ role: 'user', content: 'follow-up task' }]); }); }); @@ -282,29 +253,15 @@ describe('createDelegateTool', () => { const maxDepth = 2; const maxSteps = 5; - function makeDelegateToolCall(toolName: string, task: string): GenerateResult { - return { - text: '', - finishReason: 'tool_calls', - usage: { promptTokens: 1, completionTokens: 1, totalTokens: 2 }, - toolCalls: [ - { - id: `call-${Math.random().toString(36).slice(2)}`, - type: 'function', - function: { name: toolName, arguments: JSON.stringify({ task }) }, - }, - ], - }; - } - // Always requests delegation to the other agent - never stops on its // own. Exactly the "neither agent ever stops" pathological case from // the bug report. - const generateA = vi.fn(async () => makeDelegateToolCall('delegate', 'go to B')); - const generateB = vi.fn(async () => makeDelegateToolCall('delegate', 'go to A')); - - const providerA = makeMockProvider(generateA); - const providerB = makeMockProvider(generateB); + const delegating = (task: string) => ({ + toolCalls: [{ name: 'delegate', args: { task } }], + usage: { inputTokens: 1, outputTokens: 1 }, + }); + const providerA = mockModel([delegating('go to B')], { onExhausted: 'repeat-last' }); + const providerB = mockModel([delegating('go to A')], { onExhausted: 'repeat-last' }); const agentA = { name: 'Agent A', agentType: AgentType.SmartAssistant, tools: { delegate: { tool: 'delegate' } } }; const agentB = { name: 'Agent B', agentType: AgentType.SmartAssistant, tools: { delegate: { tool: 'delegate' } } }; @@ -353,7 +310,7 @@ describe('createDelegateTool', () => { // fix, QA measured 155 calls for maxDepth=2/maxSteps=5 (and 780 for // maxDepth=3/maxSteps=5); after the fix this should be a handful of // calls (one per hop until the guard fires), well under 20. - const totalCalls = generateA.mock.calls.length + generateB.mock.calls.length; + const totalCalls = providerA.calls.length + providerB.calls.length; expect(totalCalls).toBeLessThan(20); }); }); diff --git a/src/execution/sandbox-wiring.test.ts b/src/execution/sandbox-wiring.test.ts index 9e838ee3c..9e28d1c76 100644 --- a/src/execution/sandbox-wiring.test.ts +++ b/src/execution/sandbox-wiring.test.ts @@ -4,46 +4,11 @@ import { ToolRegistry } from '../tools'; import { AgentBuilder } from '../core'; import { AgentType } from '../types'; import { SandboxAdapter } from '../security/sandbox'; +import { mockModel } from '../testing'; -/** - * Builds a scripted provider that emits exactly one tool call (for - * `toolName`) and then stops - enough to drive AgentExecutor through a - * single tool execution without depending on MockLLMProvider's - * content-sniffing heuristics. - */ +/** A model that emits exactly one tool call (for `toolName`) and then stops. */ function makeSingleToolCallProvider(toolName: string) { - let call = 0; - return { - name: 'scripted', - supportsTools: () => true, - supportsStreaming: () => false, - getModels: async () => ['scripted'], - stream: async () => { - throw new Error('not implemented'); - }, - generate: async () => { - call++; - if (call === 1) { - return { - text: '', - finishReason: 'tool_calls' as const, - usage: { promptTokens: 1, completionTokens: 1, totalTokens: 2 }, - toolCalls: [ - { - id: 'call-1', - type: 'function' as const, - function: { name: toolName, arguments: '{}' }, - }, - ], - }; - } - return { - text: 'done', - finishReason: 'stop' as const, - usage: { promptTokens: 1, completionTokens: 1, totalTokens: 2 }, - }; - }, - }; + return mockModel([{ toolCalls: [{ name: toolName }] }, 'done']); } describe('AgentExecutor sandbox wiring (LOU-F5 / LOU-F fix)', () => { @@ -85,7 +50,7 @@ describe('AgentExecutor sandbox wiring (LOU-F5 / LOU-F fix)', () => { const result = await AgentExecutor.execute({ agent, input: 'go', - provider: makeSingleToolCallProvider('sandboxed') as any, + provider: makeSingleToolCallProvider('sandboxed'), toolRegistry, sandbox: spySandbox, onToolResult, @@ -143,7 +108,7 @@ describe('AgentExecutor sandbox wiring (LOU-F5 / LOU-F fix)', () => { const result = await AgentExecutor.execute({ agent, input: 'go', - provider: makeSingleToolCallProvider('unsandboxable') as any, + provider: makeSingleToolCallProvider('unsandboxable'), toolRegistry, sandbox: spySandbox, onToolResult, @@ -196,7 +161,7 @@ describe('AgentExecutor sandbox wiring (LOU-F5 / LOU-F fix)', () => { await AgentExecutor.execute({ agent, input: 'go', - provider: makeSingleToolCallProvider('plain') as any, + provider: makeSingleToolCallProvider('plain'), toolRegistry, sandbox: spySandbox, }); @@ -235,7 +200,7 @@ describe('AgentExecutor sandbox wiring (LOU-F5 / LOU-F fix)', () => { const result = await AgentExecutor.execute({ agent, input: 'go', - provider: makeSingleToolCallProvider('sandboxed-noop') as any, + provider: makeSingleToolCallProvider('sandboxed-noop'), toolRegistry, }); diff --git a/src/testing/index.ts b/src/testing/index.ts new file mode 100644 index 000000000..ffe4b51d8 --- /dev/null +++ b/src/testing/index.ts @@ -0,0 +1,18 @@ +/** + * `@loushy/build-ai-agent/testing` - test doubles for agents. + * + * Re-exports the in-memory repository mocks plus the scripted `mockModel`. + */ + +export * from '../data/mocks'; +export { mockModel } from './mockModel'; +export type { + DeepReadonly, + MockModel, + MockModelOptions, + MockRequest, + MockStaticTurn, + MockToolCall, + MockTurn, + MockTurnObject, +} from './mockModel'; diff --git a/src/testing/mockModel.test.ts b/src/testing/mockModel.test.ts new file mode 100644 index 000000000..c9e8e3bfa --- /dev/null +++ b/src/testing/mockModel.test.ts @@ -0,0 +1,235 @@ +import { describe, it, expect, vi } from 'vitest'; +import { z } from 'zod'; +import { mockModel } from './index'; +import { createAgent } from '../createAgent'; +import type { GenerateOptions, StreamChunk } from '../providers/llm'; + +const request = (content = 'hi'): GenerateOptions => ({ messages: [{ role: 'user', content }] }); + +describe('mockModel turns', () => { + it('treats a bare string as { text } and reports zero usage by default', async () => { + const model = mockModel(['Hello!']); + + const result = await model.generate(request()); + + expect(result).toEqual({ + text: 'Hello!', + finishReason: 'stop', + usage: { promptTokens: 0, completionTokens: 0, totalTokens: 0 }, + }); + }); + + it('emits tool calls with stable generated ids and JSON-encoded args', async () => { + const model = mockModel([ + { toolCalls: [{ name: 'a', args: { x: 1 } }, { name: 'b' }] }, + { toolCalls: [{ name: 'c', id: 'custom' }, { name: 'd' }] }, + ]); + + const first = await model.generate(request()); + const second = await model.generate(request()); + + expect(first.finishReason).toBe('tool_calls'); + expect(first.toolCalls).toEqual([ + { id: 'call_1', type: 'function', function: { name: 'a', arguments: '{"x":1}' } }, + { id: 'call_2', type: 'function', function: { name: 'b', arguments: '{}' } }, + ]); + expect(second.toolCalls?.map((c) => c.id)).toEqual(['custom', 'call_3']); + }); + + it('honours usage, finishReason, text alongside tool calls', async () => { + const model = mockModel([ + { + text: 'thinking', + toolCalls: [{ name: 't' }], + usage: { inputTokens: 3, outputTokens: 4 }, + finishReason: 'length', + }, + ]); + + const result = await model.generate(request()); + + expect(result.text).toBe('thinking'); + expect(result.finishReason).toBe('length'); + expect(result.usage).toEqual({ promptTokens: 3, completionTokens: 4, totalTokens: 7 }); + }); + + it('rejects with the scripted error and still records the call', async () => { + const boom = new Error('rate limited'); + const model = mockModel([{ error: boom }]); + + await expect(model.generate(request())).rejects.toBe(boom); + expect(model.calls).toHaveLength(1); + }); + + it('builds dynamic turns from the request (sync and async)', async () => { + const model = mockModel([ + (req) => `echo: ${req.messages.at(-1)?.content}`, + async (req) => ({ text: `n=${req.messages.length}` }), + ]); + + expect((await model.generate(request('ping'))).text).toBe('echo: ping'); + expect((await model.generate(request())).text).toBe('n=1'); + }); + + it('waits delayMs before answering', async () => { + vi.useFakeTimers(); + try { + const model = mockModel([{ text: 'late', delayMs: 50 }]); + let settled = false; + const pending = model.generate(request()).then((r) => { + settled = true; + return r; + }); + + await vi.advanceTimersByTimeAsync(49); + expect(settled).toBe(false); + await vi.advanceTimersByTimeAsync(1); + expect((await pending).text).toBe('late'); + } finally { + vi.useRealTimers(); + } + }); +}); + +describe('mockModel recording', () => { + it('records deep-frozen snapshots that later mutation cannot change', async () => { + const model = mockModel(['a', 'b']); + const messages = [{ role: 'user' as const, content: 'first' }]; + + await model.generate({ messages, model: 'm', temperature: 0.2 }); + messages.push({ role: 'user', content: 'mutated' }); + await model.generate(request('second')); + + expect(model.calls).toHaveLength(2); + expect(model.calls[0].messages).toEqual([{ role: 'user', content: 'first' }]); + expect(model.calls[0]).toMatchObject({ model: 'm', temperature: 0.2 }); + expect(model.lastCall).toBe(model.calls[1]); + expect(Object.isFrozen(model.calls[0])).toBe(true); + expect(Object.isFrozen(model.calls[0].messages)).toBe(true); + expect(Object.isFrozen(model.calls[0].messages[0])).toBe(true); + }); + + it('reset() rewinds the script, calls and generated ids', async () => { + const model = mockModel([{ toolCalls: [{ name: 't' }] }]); + await model.generate(request()); + + model.reset(); + + expect(model.calls).toHaveLength(0); + expect(model.lastCall).toBeUndefined(); + const again = await model.generate(request()); + expect(again.toolCalls?.[0].id).toBe('call_1'); + }); +}); + +describe('mockModel exhaustion', () => { + it('throws an actionable error naming the unexpected call and last message', async () => { + const model = mockModel(['only']); + await model.generate(request()); + + const error = await model.generate(request('what next?')).catch((e: Error) => e); + + expect(error).toBeInstanceOf(Error); + const message = (error as Error).message; + expect(message).toContain('call #2'); + expect(message).toContain('1 turn'); + expect(message).toContain('user: "what next?"'); + expect(message).toContain('Add another turn'); + expect(message).toContain("onExhausted: 'repeat-last'"); + expect(model.calls).toHaveLength(2); + }); + + it("replays the final turn with onExhausted: 'repeat-last'", async () => { + const model = mockModel(['a', 'b'], { onExhausted: 'repeat-last' }); + + const texts = []; + for (let i = 0; i < 4; i++) texts.push((await model.generate(request())).text); + + expect(texts).toEqual(['a', 'b', 'b', 'b']); + }); + + it('still throws on an empty script even with repeat-last', async () => { + const model = mockModel([], { onExhausted: 'repeat-last' }); + await expect(model.generate(request())).rejects.toThrow(/call #1/); + }); + + it('assertExhausted() throws while turns remain and passes once consumed', async () => { + const model = mockModel(['a', 'b']); + await model.generate(request()); + + expect(() => model.assertExhausted()).toThrow(/1 scripted turn/); + await model.generate(request()); + expect(() => model.assertExhausted()).not.toThrow(); + }); +}); + +describe('mockModel streaming', () => { + it('streams the scripted text as chunks that rejoin to the text, then finishes', async () => { + const model = mockModel([{ text: 'Hello big world', usage: { inputTokens: 1, outputTokens: 2 } }]); + + const result = await model.stream(request()); + const chunks: StreamChunk[] = []; + for await (const chunk of result.fullStream) chunks.push(chunk); + + const deltas = chunks.filter((c) => c.type === 'text-delta').map((c) => c.textDelta); + expect(deltas).toEqual(['Hello ', 'big ', 'world']); + expect(chunks.at(-1)).toMatchObject({ type: 'finish', finishReason: 'stop' }); + expect(await result.text).toBe('Hello big world'); + expect(await result.usage).toEqual({ promptTokens: 1, completionTokens: 2, totalTokens: 3 }); + expect(model.calls).toHaveLength(1); + }); + + it('emits tool-call chunks and exposes them on the result', async () => { + const model = mockModel([{ toolCalls: [{ name: 'lookup', args: { q: 1 } }] }]); + + const result = await model.stream(request()); + const chunks: StreamChunk[] = []; + for await (const chunk of result.fullStream) chunks.push(chunk); + + expect(chunks.find((c) => c.type === 'tool-call')?.toolCall?.id).toBe('call_1'); + expect(await result.toolCalls).toHaveLength(1); + expect(await result.finishReason).toBe('tool_calls'); + }); + + it('streams textStream and rejects stream() for an error turn', async () => { + const model = mockModel(['a b', { error: new Error('nope') }]); + + const result = await model.stream(request()); + const parts: string[] = []; + for await (const part of result.textStream) parts.push(part); + expect(parts.join('')).toBe('a b'); + await expect(model.stream(request())).rejects.toThrow('nope'); + }); +}); + +describe('mockModel as an agent provider', () => { + it('drives a tool-call flow end to end', async () => { + const execute = vi.fn(async (args: { city: string }) => ({ tempC: 21, city: args.city })); + const model = mockModel([ + { toolCalls: [{ name: 'get_weather', args: { city: 'Paris' } }] }, + { text: 'It is 21°C in Paris.' }, + ]); + const agent = createAgent({ + prompt: 'You report weather.', + provider: model, + tools: { + get_weather: { + displayName: 'Get weather', + tool: { + description: 'Get the weather for a city', + parameters: z.object({ city: z.string() }), + execute, + }, + }, + }, + }); + + const result = await agent.send('Weather in Paris?'); + + expect(result.text).toBe('It is 21°C in Paris.'); + expect(execute).toHaveBeenCalledWith({ city: 'Paris' }, expect.anything()); + expect(model.calls).toHaveLength(2); + expect(model.calls[1].messages.at(-1)).toMatchObject({ role: 'tool', toolCallId: 'call_1' }); + model.assertExhausted(); + }); +}); diff --git a/src/testing/mockModel.ts b/src/testing/mockModel.ts new file mode 100644 index 000000000..36ff6b93b --- /dev/null +++ b/src/testing/mockModel.ts @@ -0,0 +1,333 @@ +/** + * Scripted mock model: a deterministic, typed `LLMProvider` test double. + * + * Script the model's replies turn by turn, run your agent, then assert on the + * exact requests the agent sent. No network, no randomness. + */ + +import type { + GenerateOptions, + GenerateResult, + LLMProvider, + StreamChunk, + StreamResult, + ToolCall, +} from '../providers/llm'; + +/** Recursively read-only version of `T` (used for recorded requests). */ +export type DeepReadonly = T extends (...args: never[]) => unknown + ? T + : T extends ReadonlyArray + ? ReadonlyArray> + : T extends object + ? { readonly [K in keyof T]: DeepReadonly } + : T; + +/** A request recorded by {@link MockModel}: a deep-frozen snapshot of the `generate()` options. */ +export type MockRequest = DeepReadonly; + +/** + * A tool call the mock model should emit. + * + * @example + * ```ts + * const call: MockToolCall = { name: 'get_weather', args: { city: 'Paris' } }; + * ``` + */ +export interface MockToolCall { + /** Name of the tool to call. */ + name: string; + /** Arguments for the tool; serialized to JSON for the agent. Defaults to `{}`. */ + args?: Readonly>; + /** Tool-call id. Defaults to a stable `call_1`, `call_2`, ... sequence. */ + id?: string; +} + +/** + * One scripted model reply (object form). + * + * @example + * ```ts + * const turn: MockTurnObject = { + * toolCalls: [{ name: 'search', args: { q: 'cats' } }], + * usage: { inputTokens: 10, outputTokens: 5 }, + * }; + * ``` + */ +export interface MockTurnObject { + /** Assistant text. Defaults to an empty string. */ + text?: string; + /** Tool calls the model requests this turn. */ + toolCalls?: readonly MockToolCall[]; + /** Make `generate()` / `stream()` reject with this error instead of replying. */ + error?: Error; + /** Token usage to report. Defaults to zero. */ + usage?: { inputTokens: number; outputTokens: number }; + /** Finish reason to report. Defaults to `'tool_calls'` when there are tool calls, else `'stop'`. */ + finishReason?: GenerateResult['finishReason']; + /** Wait this many milliseconds before answering. */ + delayMs?: number; +} + +/** A non-dynamic turn: a bare string (shorthand for `{ text }`) or a {@link MockTurnObject}. */ +export type MockStaticTurn = string | MockTurnObject; + +/** + * A scripted turn: a string, a {@link MockTurnObject}, or a function that + * builds either from the incoming request (sync or async). + * + * @example + * ```ts + * const echo: MockTurn = (req) => `You said: ${req.messages.at(-1)?.content}`; + * ``` + */ +export type MockTurn = + | MockStaticTurn + | ((request: MockRequest) => MockStaticTurn | Promise); + +/** Options for {@link mockModel}. */ +export interface MockModelOptions { + /** + * What to do when the script runs out. `'throw'` (default) rejects with a + * descriptive error; `'repeat-last'` replays the final turn forever. + */ + onExhausted?: 'throw' | 'repeat-last'; + /** Reported as the provider's `defaultModel`. Defaults to `'mock-model'`. */ + defaultModel?: string; +} + +/** + * A scripted `LLMProvider` that records every request. + * + * @example + * ```ts + * const model = mockModel(['Hello!']); + * await model.generate({ messages: [{ role: 'user', content: 'hi' }] }); + * model.assertExhausted(); + * ``` + */ +export interface MockModel extends LLMProvider { + /** Every `generate()` / `stream()` request so far, in order, as deep-frozen snapshots. */ + readonly calls: readonly MockRequest[]; + /** The most recent request, or `undefined` before the first call. */ + readonly lastCall: MockRequest | undefined; + /** Rewind the script, clear recorded calls and restart generated tool-call ids at `call_1`. */ + reset(): void; + /** + * Throw if scripted turns remain unused. + * + * @example + * ```ts + * model.assertExhausted(); // fails the test if the agent made fewer calls than scripted + * ``` + */ + assertExhausted(): void; +} + +interface ResolvedTurn { + text: string; + toolCalls: ToolCall[]; + finishReason: GenerateResult['finishReason']; + usage: GenerateResult['usage']; +} + +/** + * Copy plain objects and arrays (freezing the copies) while keeping anything + * else (functions, class instances such as zod schemas in tool definitions) + * by reference, since those cannot be structurally cloned. + */ +function cloneFrozen(value: unknown): unknown { + if (Array.isArray(value)) return Object.freeze(value.map(cloneFrozen)); + if (typeof value === 'object' && value !== null && Object.getPrototypeOf(value) === Object.prototype) { + const copy: Record = {}; + for (const [key, child] of Object.entries(value)) copy[key] = cloneFrozen(child); + return Object.freeze(copy); + } + return value; +} + +function snapshot(options: GenerateOptions): MockRequest { + return cloneFrozen(options) as MockRequest; +} + +function describeLastMessage(request: MockRequest): string { + const last = request.messages.at(-1); + if (!last) return '(no messages)'; + const content = last.content.length > 120 ? `${last.content.slice(0, 117)}...` : last.content; + return `${last.role}: ${JSON.stringify(content)}`; +} + +function toStaticTurn(turn: MockStaticTurn): MockTurnObject { + return typeof turn === 'string' ? { text: turn } : turn; +} + +function chunkText(text: string): string[] { + return text.match(/\S+\s*|\s+/g) ?? []; +} + +class ScriptedMockModel implements MockModel { + readonly name = 'mock'; + readonly defaultModel: string; + private readonly recorded: MockRequest[] = []; + private index = 0; + private idCounter = 0; + + constructor( + private readonly script: readonly MockTurn[], + private readonly options: MockModelOptions + ) { + this.defaultModel = options.defaultModel ?? 'mock-model'; + } + + get calls(): readonly MockRequest[] { + return this.recorded; + } + + get lastCall(): MockRequest | undefined { + return this.recorded.at(-1); + } + + reset(): void { + this.recorded.length = 0; + this.index = 0; + this.idCounter = 0; + } + + assertExhausted(): void { + const remaining = this.script.length - this.index; + if (remaining > 0) { + throw new Error( + `mockModel: ${remaining} scripted turn(s) were never used (${this.index} of ${this.script.length} consumed). ` + + 'Remove the extra turns or check why the agent stopped calling the model early.' + ); + } + } + + async generate(options: GenerateOptions): Promise { + const turn = await this.resolve(options); + return { + text: turn.text, + finishReason: turn.finishReason, + usage: turn.usage, + ...(turn.toolCalls.length > 0 ? { toolCalls: turn.toolCalls } : {}), + }; + } + + async stream(options: GenerateOptions): Promise { + const turn = await this.resolve(options); + const chunks = chunkText(turn.text); + const fullStream = async function* (): AsyncGenerator { + for (const textDelta of chunks) yield { type: 'text-delta', textDelta }; + for (const toolCall of turn.toolCalls) yield { type: 'tool-call', toolCall }; + yield { type: 'finish', finishReason: turn.finishReason, usage: turn.usage }; + }; + const textStream = async function* (): AsyncGenerator { + yield* chunks; + }; + return { + fullStream: fullStream(), + textStream: textStream(), + text: Promise.resolve(turn.text), + usage: Promise.resolve(turn.usage), + finishReason: Promise.resolve(turn.finishReason), + toolCalls: Promise.resolve(turn.toolCalls), + }; + } + + supportsTools(): boolean { + return true; + } + + supportsStreaming(): boolean { + return true; + } + + async getModels(): Promise { + return [this.defaultModel]; + } + + private async resolve(options: GenerateOptions): Promise { + const request = snapshot(options); + this.recorded.push(request); + const turn = toStaticTurn(await this.nextTurn(request)); + if (turn.delayMs) await new Promise((resolve) => setTimeout(resolve, turn.delayMs)); + if (turn.error) throw turn.error; + return this.toResolved(turn); + } + + private async nextTurn(request: MockRequest): Promise { + const callNumber = this.recorded.length; + let scripted = this.script[this.index]; + if (scripted === undefined) { + if (this.options.onExhausted === 'repeat-last' && this.script.length > 0) { + scripted = this.script[this.script.length - 1]; + } else { + throw this.exhaustedError(callNumber, request); + } + } else { + this.index++; + } + return typeof scripted === 'function' ? scripted(request) : scripted; + } + + private exhaustedError(callNumber: number, request: MockRequest): Error { + return new Error( + `mockModel: unexpected call #${callNumber} - the script only has ${this.script.length} turn(s). ` + + `Last message of this request: ${describeLastMessage(request)}. ` + + "Add another turn to the script, or pass { onExhausted: 'repeat-last' } to replay the final turn." + ); + } + + private toResolved(turn: MockTurnObject): ResolvedTurn { + const toolCalls = (turn.toolCalls ?? []).map((call) => this.toToolCall(call)); + const inputTokens = turn.usage?.inputTokens ?? 0; + const outputTokens = turn.usage?.outputTokens ?? 0; + return { + text: turn.text ?? '', + toolCalls, + finishReason: turn.finishReason ?? (toolCalls.length > 0 ? 'tool_calls' : 'stop'), + usage: { + promptTokens: inputTokens, + completionTokens: outputTokens, + totalTokens: inputTokens + outputTokens, + }, + }; + } + + private toToolCall(call: MockToolCall): ToolCall { + return { + id: call.id ?? `call_${++this.idCounter}`, + type: 'function', + function: { name: call.name, arguments: JSON.stringify(call.args ?? {}) }, + }; + } +} + +/** + * Create a scripted, deterministic `LLMProvider` for unit-testing agents. + * + * Each `generate()` / `stream()` call consumes the next turn of `script`. + * Tool-call ids are generated as `call_1`, `call_2`, ... unless you give one. + * Every request is recorded (deep-frozen) on `model.calls`. + * + * @param script - Turns to replay, in order. A bare string means `{ text }`. + * @param options - See {@link MockModelOptions}. + * + * @example + * ```ts + * import { mockModel } from '@loushy/build-ai-agent/testing'; + * + * const model = mockModel([ + * { toolCalls: [{ name: 'get_weather', args: { city: 'Paris' } }] }, + * { text: 'It is 21°C in Paris.' }, + * ]); + * const agent = createAgent({ prompt: '...', provider: model, tools }); + * await agent.send('Weather in Paris?'); + * + * expect(model.calls).toHaveLength(2); + * model.assertExhausted(); + * ``` + */ +export function mockModel(script: readonly MockTurn[], options: MockModelOptions = {}): MockModel { + return new ScriptedMockModel(script, options); +} diff --git a/tsup.config.ts b/tsup.config.ts index 6150b8268..f486b9193 100644 --- a/tsup.config.ts +++ b/tsup.config.ts @@ -26,7 +26,7 @@ export default defineConfig({ 'flows/index': 'src/flows/index.ts', 'data/index': 'src/data/index.ts', 'types/index': 'src/types/index.ts', - 'data/mocks': 'src/data/mocks.ts', + 'testing/index': 'src/testing/index.ts', 'cli/dev': 'src/cli/dev.ts', 'cli/build': 'src/cli/build.ts', 'cli/studio': 'src/cli/studio.ts',