From 63e4e077fce135286fd30f80ca6062a964fa259d Mon Sep 17 00:00:00 2001 From: Ali Mohammad Date: Thu, 1 Oct 2026 09:01:30 +0300 Subject: [PATCH] LOU-U12: send a structured error to the model when a tool throws When a tool's execute threw an ordinary error, the generic path set result: null, so the model saw the string "null" and could not tell the tool failed or why. The result is now { error: , toolName, message } (message only, never a stack, capped at 2,000 chars with a truncation marker), mirroring the LOU-U4 argument-validation shape. PropagatingToolError still aborts the run; events, onToolResult and postToolCall still see outcome.error. toolErrorMessage now falls back to String(error) for non-Error throws so those still count as errors. Docs: api-overview 'Argument validation' became 'Tool errors'. Co-Authored-By: Claude Fable 5.1 --- docs/api-overview.md | 21 +++- src/execution/propagatingToolError.ts | 43 +++++++- src/execution/toolCallExecution.ts | 9 +- src/execution/toolErrorResult.test.ts | 142 ++++++++++++++++++++++++++ 4 files changed, 210 insertions(+), 5 deletions(-) create mode 100644 src/execution/toolErrorResult.test.ts diff --git a/docs/api-overview.md b/docs/api-overview.md index 6ec6d583c..ba92aa105 100644 --- a/docs/api-overview.md +++ b/docs/api-overview.md @@ -50,9 +50,14 @@ import { loadMcpTools } from '@loushy/build-ai-agent/mcp'; const tools = await loadMcpTools(mcpClient, 'my-server'); ``` -### Argument validation +### Tool errors -Before a tool runs, the model's arguments are parsed with the tool's zod +A tool call can fail in two ways. In both, the run continues and the model +receives a structured JSON error as the tool result (never the string `null`), +so it can recover. `tool-result` events, tracing, `onToolResult` and +`postToolCall` hooks see the call as an error. + +**Argument validation.** Before a tool runs, the model's arguments are parsed with the tool's zod `parameters` schema. Validation happens first, so pre-tool hooks, the `needsApproval` predicate and `execute` all receive the **parsed** value (defaults, coercions and transforms applied). Tools without a zod schema are @@ -78,6 +83,18 @@ result; `preToolCall` hooks are skipped because there is no valid call): `ToolArgumentsValidationError` (with a typed `issues` array) is exported from the package root. +**Thrown errors.** If `execute` throws, the model receives the error name, +the tool name and the message only (never a stack trace). Messages are capped +at 2,000 characters and end with `... (truncated)` when cut: + +```json +{ "error": "TypeError", "toolName": "search", "message": "query must not be empty" } +``` + +Errors extending `PropagatingToolError` (for example the delegation depth +guard) are the exception: they are rethrown and abort the run instead of being +shown to the model. + ## Flows, evals, observability and security - `FlowBuilder` / `FlowExecutor` - multi-step workflow graphs. diff --git a/src/execution/propagatingToolError.ts b/src/execution/propagatingToolError.ts index c3abd2325..65480c39f 100644 --- a/src/execution/propagatingToolError.ts +++ b/src/execution/propagatingToolError.ts @@ -36,5 +36,46 @@ export function toolErrorMessage(error: unknown): string { if (error instanceof PropagatingToolError) { throw error; } - return (error as Error).message; + return (error as Error | undefined)?.message ?? String(error); +} + +/** Max characters of an error message sent to the model (see {@link toolErrorResult}). */ +const MAX_TOOL_ERROR_MESSAGE_LENGTH = 2000; + +/** + * The structured tool result the model sees when a tool's `execute` throws + * (mirrors `ToolArgumentsValidationError.toToolResult()`, minus `issues`). + * + * @example + * ```ts + * // execute: async () => { throw new TypeError('bad input') } + * // tool result the model sees: + * // { error: 'TypeError', toolName: 'search', message: 'bad input' } + * ``` + */ +interface ToolErrorResult { + /** The thrown error's `name` (`'Error'`, `'TypeError'`, ...). */ + error: string; + toolName: string; + /** The error message only (never a stack), truncated to 2,000 characters. */ + message: string; +} + +/** + * Builds the {@link ToolErrorResult} for a thrown tool error. Only the name + * and (length-capped) message are exposed - never the stack - so a huge or + * sensitive error cannot blow the context window. + */ +export function toolErrorResult(toolName: string, error: unknown): ToolErrorResult { + const err = error as { name?: unknown; message?: unknown } | null | undefined; + const raw = typeof err?.message === 'string' ? err.message : String(error); + const message = + raw.length > MAX_TOOL_ERROR_MESSAGE_LENGTH + ? `${raw.slice(0, MAX_TOOL_ERROR_MESSAGE_LENGTH)}... (truncated)` + : raw; + return { + error: typeof err?.name === 'string' && err.name ? err.name : 'Error', + toolName, + message, + }; } diff --git a/src/execution/toolCallExecution.ts b/src/execution/toolCallExecution.ts index d6ef1085f..6bd8f774d 100644 --- a/src/execution/toolCallExecution.ts +++ b/src/execution/toolCallExecution.ts @@ -10,7 +10,7 @@ import { ToolRegistry } from '../tools'; import { SandboxAdapter } from '../security/sandboxCore'; import { executeToolWithSandboxGuard } from './sandboxGuard'; import { HookRegistry, ToolCallHookContext } from './hooks'; -import { toolErrorMessage } from './propagatingToolError'; +import { toolErrorMessage, toolErrorResult } from './propagatingToolError'; import { ToolArgumentsValidationError, validateToolArguments } from './toolArgsValidation'; import type { ExecuteOptions } from './AgentExecutor'; @@ -255,6 +255,11 @@ async function doExecuteToolCall( // DelegationDepthExceededError) are rethrown by toolErrorMessage() so // they propagate out of execute() as a rejected promise instead of // becoming a conversational {error} tool-result. - return toolFailure(toolCall, toolErrorMessage(error)); + // LOU-U12: the model gets a structured error (not the string "null"), + // while `outcome.error` keeps the plain message for events/hooks. + return { + ...toolFailure(toolCall, toolErrorMessage(error)), + result: toolErrorResult(toolCall.function.name, error), + }; } } diff --git a/src/execution/toolErrorResult.test.ts b/src/execution/toolErrorResult.test.ts new file mode 100644 index 000000000..010a0db14 --- /dev/null +++ b/src/execution/toolErrorResult.test.ts @@ -0,0 +1,142 @@ +import { describe, it, expect, vi } from 'vitest'; +import { z } from 'zod'; +import { AgentExecutor, ExecutionEvent } from './AgentExecutor'; +import { PropagatingToolError } from './propagatingToolError'; +import { HookRegistry } from './hooks'; +import { ToolRegistry } from '../tools'; +import { AgentBuilder } from '../core'; +import { AgentType } from '../types'; + +const usage = { promptTokens: 1, completionTokens: 1, totalTokens: 2 }; + +/** Provider that calls `boom` for `steps` turns, then answers. */ +function scriptedProvider(steps: number) { + const seenMessages: any[][] = []; + let step = 0; + const provider = { + name: 'mock', + async generate(options: any) { + seenMessages.push(JSON.parse(JSON.stringify(options.messages))); + if (step++ >= steps) { + return { text: 'done', finishReason: 'stop' as const, usage }; + } + return { + text: '', + finishReason: 'tool_calls' as const, + usage, + toolCalls: [ + { + id: `call-${step}`, + type: 'function' as const, + function: { name: 'boom', arguments: '{}' }, + }, + ], + }; + }, + async stream() { + throw new Error('not implemented'); + }, + supportsTools: () => true, + supportsStreaming: () => false, + async getModels() { + return []; + }, + }; + return { provider: provider as any, seenMessages }; +} + +function setup(execute: () => Promise) { + const toolRegistry = new ToolRegistry(); + toolRegistry.register('boom', { + displayName: 'Boom', + tool: { parameters: z.object({}), execute } as any, + }); + const agent = AgentBuilder.create() + .setType(AgentType.SmartAssistant) + .setName('Test Agent') + .addTool('boom', { tool: 'boom', options: {} }) + .build(); + return { toolRegistry, agent }; +} + +describe('thrown tool errors reach the model (LOU-U12)', () => { + it('sends a structured error, then the model recovers', async () => { + let calls = 0; + const { toolRegistry, agent } = setup(async () => { + if (calls++ === 0) { + throw new TypeError('query must not be empty'); + } + return { ok: true }; + }); + const { provider, seenMessages } = scriptedProvider(2); + const events: ExecutionEvent[] = []; + const onToolResult = vi.fn(); + const postToolCall = vi.fn(); + const hooks = new HookRegistry(); + hooks.register({ name: 'spy', postToolCall }); + + const result = await AgentExecutor.execute({ + agent, + input: 'go', + provider, + toolRegistry, + hooks, + onToolResult, + onEvent: e => events.push(e), + }); + + expect(result.text).toBe('done'); + const toolMessage = seenMessages[1].find(m => m.role === 'tool'); + expect(toolMessage.toolCallId).toBe('call-1'); + expect(JSON.parse(toolMessage.content)).toEqual({ + error: 'TypeError', + toolName: 'boom', + message: 'query must not be empty', + }); + // The follow-up call succeeded. + const secondTools = seenMessages[2].filter(m => m.role === 'tool'); + expect(JSON.parse(secondTools[1].content)).toEqual({ ok: true }); + + // Events and callbacks still see an error. + const toolResults = events.filter(e => e.type === 'tool-result') as any[]; + expect(toolResults[0].toolResult.error).toBe('query must not be empty'); + expect(onToolResult.mock.calls[0][1].error).toBe('query must not be empty'); + expect(postToolCall.mock.calls[0][1].error).toBe('query must not be empty'); + }); + + it('truncates very long messages', async () => { + const { toolRegistry, agent } = setup(async () => { + throw new Error('x'.repeat(10_000)); + }); + const { provider, seenMessages } = scriptedProvider(1); + + await AgentExecutor.execute({ agent, input: 'go', provider, toolRegistry }); + + const payload = JSON.parse(seenMessages[1].find(m => m.role === 'tool').content); + expect(payload.message.length).toBe(2000 + '... (truncated)'.length); + expect(payload.message.endsWith('... (truncated)')).toBe(true); + }); + + it('falls back to a generic name for non-Error throws', async () => { + const { toolRegistry, agent } = setup(async () => { + throw 'plain string'; + }); + const { provider, seenMessages } = scriptedProvider(1); + + await AgentExecutor.execute({ agent, input: 'go', provider, toolRegistry }); + + const payload = JSON.parse(seenMessages[1].find(m => m.role === 'tool').content); + expect(payload).toEqual({ error: 'Error', toolName: 'boom', message: 'plain string' }); + }); + + it('still aborts the run on a PropagatingToolError', async () => { + const { toolRegistry, agent } = setup(async () => { + throw new PropagatingToolError('stop everything'); + }); + const { provider } = scriptedProvider(1); + + await expect( + AgentExecutor.execute({ agent, input: 'go', provider, toolRegistry }) + ).rejects.toThrow('stop everything'); + }); +});