diff --git a/CHANGELOG.md b/CHANGELOG.md index 10debe3b5..67c4889bd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `createAgent()` agents can pause for approval (LOU-D21): a `needsApproval` tool no longer fails the run with "requires approval but no approvalStore". The run pauses (`finishReason: 'awaiting-approval'`) in a per-agent `InMemoryApprovalStore` (or the new `approvalStore` option), and `agent.approvals.list()` / `agent.approvals.resolve({ id, approved, note? })` continue it, in its session if it paused in one. The `approve` option decides calls in code without pausing. - Tools own their contract (LOU-D22): `ToolDescriptor` gains optional `inputSchema` (zod) and `execute(args, ctx)`, which are now the canonical fields. `defineTool()` sets both and no longer calls `ai`'s `tool()`. Argument validation, the schema sent to the model, and tool execution read them first and fall back to `tool.parameters` / `tool.execute`. `ToolDescriptor.tool` (the `ai` v4 `Tool`) is now legacy: it is still built by `defineTool()` and still accepted on hand-written descriptors this release, but new code should set `inputSchema` and `execute`. +- `finishReason: 'max-steps'` (LOU-U19): a run that exhausts `maxSteps` while the model still wanted to continue (last turn ended in tool calls) now resolves with `finishReason: 'max-steps'` instead of the stale last-turn reason (usually `'tool_calls'`), on `ExecutionResult`, the `finish` event and `run.done`. Resumed runs count `initialSteps` toward the budget. A run that finishes naturally within the budget is unchanged. Code that treated `'tool_calls'` as "hit maxSteps" should check `'max-steps'` instead (the eval `completed()` message and the MCP server's agent tool do). + ### Docs - README revamped into a short front page (LOU-D52); the details it dropped moved to `docs/` (new pages: tools, approvals, providers, cli, flows, guardrails, utilities). diff --git a/docs/api-overview.md b/docs/api-overview.md index 08c797e14..fb732a3f1 100644 --- a/docs/api-overview.md +++ b/docs/api-overview.md @@ -139,6 +139,26 @@ How it behaves: conversation stays valid for the provider. - An already-aborted signal returns at once without calling the provider. +### Finish reasons + +`result.finishReason` says why a run ended: the model's own reason for its last +turn (`'stop'`, `'length'`, `'tool_calls'`, `'content_filter'`, `'error'`), +`'awaiting-approval'` (paused on a tool call that needs a human), `'aborted'` +(cancelled with `signal`), or `'max-steps'`. `'max-steps'` means the `maxSteps` +budget (default 10) ran out while the model still wanted to continue, so the +reply may be empty or partial; a run that finishes naturally within the budget +keeps its `'stop'`. Steps carried over by `initialSteps` or an approval resume +count against the budget, and `result.steps` is the number of steps taken. The +same reason is on the `finish` event and on `run.done` in +[streaming](streaming.md). + +```ts +import { createAgent, createMockProvider } from '@loushy/build-ai-agent'; +const agent = createAgent({ prompt: 'You are helpful.', provider: createMockProvider(), maxSteps: 3 }); +const result = await agent.send('Research this thoroughly'); +if (result.finishReason === 'max-steps') console.warn(`Gave up after ${result.steps} steps`); +``` + ### Parallel tool calls When the model asks for several tools in one turn, they run concurrently. diff --git a/docs/streaming.md b/docs/streaming.md index bb8403e6d..5d0b6ea87 100644 --- a/docs/streaming.md +++ b/docs/streaming.md @@ -107,9 +107,9 @@ The event types and their extra fields: | `tool.done` | `toolCallId`, `toolName`, `result: unknown`, `durationMs: number` | A tool call returned. `result` is the value as it would be JSON-encoded (`undefined` becomes `null`, a `Date` becomes a string). `durationMs` counts from its `tool.start`. | | `tool.error` | `toolCallId`, `toolName`, `error: { name: string, message: string }`, `durationMs: number` | A tool call failed: it threw, its arguments did not match its schema (`name: 'ToolArgumentsValidationError'`), or the tool does not exist. The model gets the error as the call's result and the run continues. | | `approval.requested` | `approvalId: string`, `toolCallId`, `toolName`, `args: Record` | A tool call needs a human decision. The run then stops; resume it with `agent.approvals.resolve()` or `resumeAfterApproval()` (see [Approvals](./approvals.md)). | -| `step.done` | `step: number`, `finishReason: string`, `usage?: { promptTokens, completionTokens, totalTokens }` | A step ends. `finishReason` is the model's (`'stop'`, `'tool_calls'`, `'length'`, ...), or `'awaiting-approval'`, `'aborted'` or `'error'` when the step ended that way. `usage` is this step's model call, absent when the call produced no response. | +| `step.done` | `step: number`, `finishReason: string`, `usage?: { promptTokens, completionTokens, totalTokens }` | A step ends. `finishReason` is the model's (`'stop'`, `'tool_calls'`, `'length'`, ...), or `'awaiting-approval'`, `'aborted'` or `'error'` when the step ended that way (a run that runs out of `maxSteps` still wanting to continue ends with `run.done` `'max-steps'`). `usage` is this step's model call, absent when the call produced no response. | | `error` | `error: { name: string, message: string }` | An error. If it ends the run, `run.done` with `finishReason: 'error'` follows. A provider error retried under `surfaceRetryableProviderErrors` is followed by further steps instead. | -| `run.done` | `finishReason: string`, `text: string`, `usage?: { promptTokens, completionTokens, totalTokens }` | Last event of every run, exactly once, including aborted, failed and awaiting-approval runs. `finishReason` and `text` match `run.result`; a failed run has `finishReason: 'error'`, `text: ''` and no `usage`. | +| `run.done` | `finishReason: string`, `text: string`, `usage?: { promptTokens, completionTokens, totalTokens }` | Last event of every run, exactly once, including aborted, failed and awaiting-approval runs. `finishReason` and `text` match `run.result` (`'max-steps'` when the `maxSteps` budget ran out while the model still wanted to continue); a failed run has `finishReason: 'error'`, `text: ''` and no `usage`. | Optional fields are left out when they have no value. They are never `undefined`, so `JSON.parse(JSON.stringify(event))` returns an equal object. diff --git a/llms-full.txt b/llms-full.txt index 1e2321bd0..dbaf02135 100644 --- a/llms-full.txt +++ b/llms-full.txt @@ -796,6 +796,26 @@ How it behaves: conversation stays valid for the provider. - An already-aborted signal returns at once without calling the provider. +### Finish reasons + +`result.finishReason` says why a run ended: the model's own reason for its last +turn (`'stop'`, `'length'`, `'tool_calls'`, `'content_filter'`, `'error'`), +`'awaiting-approval'` (paused on a tool call that needs a human), `'aborted'` +(cancelled with `signal`), or `'max-steps'`. `'max-steps'` means the `maxSteps` +budget (default 10) ran out while the model still wanted to continue, so the +reply may be empty or partial; a run that finishes naturally within the budget +keeps its `'stop'`. Steps carried over by `initialSteps` or an approval resume +count against the budget, and `result.steps` is the number of steps taken. The +same reason is on the `finish` event and on `run.done` in +[streaming](https://github.com/LinuxDevil/agent-sdk/blob/main/docs/streaming.md). + +```ts +import { createAgent, createMockProvider } from '@loushy/build-ai-agent'; +const agent = createAgent({ prompt: 'You are helpful.', provider: createMockProvider(), maxSteps: 3 }); +const result = await agent.send('Research this thoroughly'); +if (result.finishReason === 'max-steps') console.warn(`Gave up after ${result.steps} steps`); +``` + ### Parallel tool calls When the model asks for several tools in one turn, they run concurrently. @@ -3787,9 +3807,9 @@ The event types and their extra fields: | `tool.done` | `toolCallId`, `toolName`, `result: unknown`, `durationMs: number` | A tool call returned. `result` is the value as it would be JSON-encoded (`undefined` becomes `null`, a `Date` becomes a string). `durationMs` counts from its `tool.start`. | | `tool.error` | `toolCallId`, `toolName`, `error: { name: string, message: string }`, `durationMs: number` | A tool call failed: it threw, its arguments did not match its schema (`name: 'ToolArgumentsValidationError'`), or the tool does not exist. The model gets the error as the call's result and the run continues. | | `approval.requested` | `approvalId: string`, `toolCallId`, `toolName`, `args: Record` | A tool call needs a human decision. The run then stops; resume it with `agent.approvals.resolve()` or `resumeAfterApproval()` (see [Approvals](https://github.com/LinuxDevil/agent-sdk/blob/main/docs/approvals.md)). | -| `step.done` | `step: number`, `finishReason: string`, `usage?: { promptTokens, completionTokens, totalTokens }` | A step ends. `finishReason` is the model's (`'stop'`, `'tool_calls'`, `'length'`, ...), or `'awaiting-approval'`, `'aborted'` or `'error'` when the step ended that way. `usage` is this step's model call, absent when the call produced no response. | +| `step.done` | `step: number`, `finishReason: string`, `usage?: { promptTokens, completionTokens, totalTokens }` | A step ends. `finishReason` is the model's (`'stop'`, `'tool_calls'`, `'length'`, ...), or `'awaiting-approval'`, `'aborted'` or `'error'` when the step ended that way (a run that runs out of `maxSteps` still wanting to continue ends with `run.done` `'max-steps'`). `usage` is this step's model call, absent when the call produced no response. | | `error` | `error: { name: string, message: string }` | An error. If it ends the run, `run.done` with `finishReason: 'error'` follows. A provider error retried under `surfaceRetryableProviderErrors` is followed by further steps instead. | -| `run.done` | `finishReason: string`, `text: string`, `usage?: { promptTokens, completionTokens, totalTokens }` | Last event of every run, exactly once, including aborted, failed and awaiting-approval runs. `finishReason` and `text` match `run.result`; a failed run has `finishReason: 'error'`, `text: ''` and no `usage`. | +| `run.done` | `finishReason: string`, `text: string`, `usage?: { promptTokens, completionTokens, totalTokens }` | Last event of every run, exactly once, including aborted, failed and awaiting-approval runs. `finishReason` and `text` match `run.result` (`'max-steps'` when the `maxSteps` budget ran out while the model still wanted to continue); a failed run has `finishReason: 'error'`, `text: ''` and no `usage`. | Optional fields are left out when they have no value. They are never `undefined`, so `JSON.parse(JSON.stringify(event))` returns an equal object. diff --git a/src/evals/trajectory.test.ts b/src/evals/trajectory.test.ts index f9be9bf95..f88dae639 100644 --- a/src/evals/trajectory.test.ts +++ b/src/evals/trajectory.test.ts @@ -101,7 +101,7 @@ describe('trajectory assertions: diagnostic failures', () => { ); expect(result.passed).toBe(false); expect(failedMessages(result)).toEqual([ - "completed() failed: finishReason was 'tool_calls' (the run hit maxSteps while still calling tools)", + "completed() failed: finishReason was 'max-steps' (the run hit maxSteps while still calling tools)", ]); }); diff --git a/src/evals/trajectory.ts b/src/evals/trajectory.ts index 1dd53081e..46fc649a3 100644 --- a/src/evals/trajectory.ts +++ b/src/evals/trajectory.ts @@ -144,7 +144,7 @@ class TrajectoryContext implements EvalTestContext { const detail = this.result === undefined ? 'no run yet - call t.send() first' - : `finishReason was '${reason}'${reason === 'tool_calls' ? ' (the run hit maxSteps while still calling tools)' : ''}`; + : `finishReason was '${reason}'${reason === 'max-steps' ? ' (the run hit maxSteps while still calling tools)' : ''}`; this.gate('completed()', passed, passed ? undefined : `completed() failed: ${detail}`); } diff --git a/src/execution/AgentExecutor.ts b/src/execution/AgentExecutor.ts index beee7edca..24ab8f326 100644 --- a/src/execution/AgentExecutor.ts +++ b/src/execution/AgentExecutor.ts @@ -90,11 +90,15 @@ export type ExecutionEventType = * - `'awaiting-approval'`: paused on a tool call that needs a human * decision (see `resumeAfterApproval()`). * - `'aborted'`: cancelled through `ExecuteOptions.signal` (LOU-V1). + * - `'max-steps'`: the `maxSteps` budget ran out while the model still + * wanted to continue (LOU-U19). A run that finishes naturally within the + * budget keeps the model's own reason (usually `'stop'`). */ export type ExecutionFinishReason = | GenerateResult['finishReason'] | 'awaiting-approval' | 'aborted' + | 'max-steps' | (string & {}); /** @@ -627,7 +631,14 @@ export class AgentExecutor { } } - return signal?.aborted ? this.abortRun(options, state) : this.finishRun(options, state); + if (signal?.aborted) { + return this.abortRun(options, state); + } + // LOU-U19: every non-final turn 'continue's, so leaving the loop here + // means the step budget (counting `initialSteps` of a resumed run) is + // spent while the model still wanted to go on. + state.finishReason = 'max-steps'; + return this.finishRun(options, state); } /** diff --git a/src/execution/maxSteps.test.ts b/src/execution/maxSteps.test.ts new file mode 100644 index 000000000..126e0ba2d --- /dev/null +++ b/src/execution/maxSteps.test.ts @@ -0,0 +1,149 @@ +/** + * LOU-U19: exhausting `maxSteps` while the model still wants to continue + * reports `finishReason: 'max-steps'` (on the result and on `run.done`). + */ + +import { describe, it, expect } from 'vitest'; +import { z } from 'zod'; +import { AgentExecutor } from './AgentExecutor'; +import type { ApprovalStore, ExecutionSnapshot, PendingApproval } from './ApprovalGate'; +import type { AgentEvent } from './agentEvents'; +import { resumeAfterApproval } from './resume'; +import { createAgent } from '../createAgent'; +import { defineTool, DefinedTool } from '../tools/defineTool'; +import { ToolRegistry } from '../tools'; +import { mockModel, MockTurn } from '../testing'; +import { AgentConfig, AgentType } from '../types'; + +const ping = defineTool({ + name: 'ping', + description: 'Ping', + input: z.object({}), + execute: async () => 'pong', +}); + +const gated = defineTool({ + name: 'gated', + description: 'Needs approval', + input: z.object({}), + needsApproval: true, + execute: async () => 'done', +}); + +function registryOf(tools: DefinedTool[]): ToolRegistry { + const registry = new ToolRegistry(); + registry.registerMany(tools); + return registry; +} + +function options(tools: DefinedTool[], script: MockTurn[]) { + const config: AgentConfig['tools'] = {}; + for (const t of tools) config[t.name] = { tool: t.name }; + const agent: AgentConfig = { id: 'a', name: 'Agent', agentType: AgentType.SmartAssistant, prompt: 'p', tools: config }; + return { agent, provider: mockModel(script, { onExhausted: 'repeat-last' }), toolRegistry: registryOf(tools), input: 'go' }; +} + +function memoryApprovalStore(): ApprovalStore { + const records = new Map(); + return { + async save(pending, snapshot) { + records.set(pending.id, { pending, snapshot: JSON.parse(JSON.stringify(snapshot)) as ExecutionSnapshot }); + }, + async resolve(id) { + const record = records.get(id) ?? null; + records.delete(id); + return record; + }, + }; +} + +async function events(run: AsyncIterable): Promise { + const all: AgentEvent[] = []; + for await (const event of run) all.push(event); + return all; +} + +const alwaysPing: MockTurn = { toolCalls: [{ name: 'ping', id: 'call_ping' }] }; +const gatedCall: MockTurn = { toolCalls: [{ name: 'gated', id: 'call_gated' }] }; + +describe("finishReason 'max-steps'", () => { + it('a model that always calls a tool exhausts maxSteps: 2', async () => { + const result = await AgentExecutor.execute({ ...options([ping], [alwaysPing]), maxSteps: 2 }); + expect(result.finishReason).toBe('max-steps'); + expect(result.steps).toBe(2); + }); + + it('the finish event and run.done (AgentExecutor.stream) carry max-steps', async () => { + const finishReasons: Array = []; + const run = AgentExecutor.stream({ + ...options([ping], [alwaysPing]), + maxSteps: 2, + onEvent: (e) => { + if (e.type === 'finish') finishReasons.push(e.finishReason); + }, + }); + const all = await events(run); + expect(all.find((e) => e.type === 'run.done')).toMatchObject({ type: 'run.done', finishReason: 'max-steps' }); + expect((await run.result).finishReason).toBe('max-steps'); + expect(finishReasons).toEqual(['max-steps']); + }); + + it('agent.stream() run.done carries max-steps', async () => { + const agent = createAgent({ + instructions: 'x', + provider: mockModel([alwaysPing], { onExhausted: 'repeat-last' }), + tools: [ping], + maxSteps: 2, + }); + const all = await events(agent.stream('go')); + expect(all.find((e) => e.type === 'run.done')).toMatchObject({ finishReason: 'max-steps' }); + }); + + it('a text reply on exactly the last allowed step is not max-steps', async () => { + const result = await AgentExecutor.execute({ ...options([ping], [alwaysPing, 'all done']), maxSteps: 2 }); + expect(result.finishReason).toBe('stop'); + expect(result.steps).toBe(2); + expect(result.text).toBe('all done'); + }); + + it('a run that finishes well within the budget is unchanged', async () => { + const result = await AgentExecutor.execute({ ...options([ping], ['hi']), maxSteps: 5 }); + expect(result.finishReason).toBe('stop'); + }); + + it('initialSteps count against the budget', async () => { + const result = await AgentExecutor.execute({ ...options([ping], [alwaysPing]), maxSteps: 3, initialSteps: 2 }); + expect(result.steps).toBe(3); + expect(result.finishReason).toBe('max-steps'); + }); + + it('a resume after approval that immediately exhausts the budget reports max-steps', async () => { + const approvalStore = memoryApprovalStore(); + const paused = await AgentExecutor.execute({ ...options([gated], [gatedCall]), approvalStore, maxSteps: 1 }); + expect(paused.finishReason).toBe('awaiting-approval'); + + const resumed = await resumeAfterApproval( + { id: paused.approvalId!, approved: true }, + approvalStore, + registryOf([gated]), + mockModel(['never reached']), + { maxSteps: 1 } + ); + expect(resumed.finishReason).toBe('max-steps'); + expect(resumed.steps).toBe(1); + }); + + it('a resume after approval with budget left finishes naturally', async () => { + const approvalStore = memoryApprovalStore(); + const paused = await AgentExecutor.execute({ ...options([gated], [gatedCall]), approvalStore, maxSteps: 2 }); + const resumed = await resumeAfterApproval( + { id: paused.approvalId!, approved: true }, + approvalStore, + registryOf([gated]), + mockModel(['ok']), + { maxSteps: 2 } + ); + expect(resumed.finishReason).toBe('stop'); + expect(resumed.steps).toBe(2); + }); +}); diff --git a/src/subagents/withSubagents.ts b/src/subagents/withSubagents.ts index dafcb8bfc..3bac4170c 100644 --- a/src/subagents/withSubagents.ts +++ b/src/subagents/withSubagents.ts @@ -138,7 +138,7 @@ function failureReason(name: string, result: ExecutionResult, maxSteps: number): if (result.finishReason === 'aborted') { return `Sub-agent '${name}' was aborted before it finished.${last}`; } - if (result.steps >= maxSteps) { + if (result.finishReason === 'max-steps' || result.steps >= maxSteps) { return `Sub-agent '${name}' used all ${maxSteps} of its steps (maxSteps) without giving a final answer.${last}`; } return `Sub-agent '${name}' ended with finish reason '${result.finishReason}' without a final answer.${last}`; diff --git a/src/tools/mcp/server/buildServer.ts b/src/tools/mcp/server/buildServer.ts index 93174fda2..cd97d26a3 100644 --- a/src/tools/mcp/server/buildServer.ts +++ b/src/tools/mcp/server/buildServer.ts @@ -45,6 +45,7 @@ function agentResultToMcp(result: ExecutionResult): CallToolResult { ); } if (result.finishReason === 'aborted') return textResult('The agent run was cancelled.', true); + if (result.finishReason === 'max-steps') return textResult('The agent ran out of steps (maxSteps) before finishing.', true); if (result.finishReason === 'error') { return textResult(`The agent failed: ${result.text || 'the model reported an error'}`, true); }