From 75aa0429bf862498423929ecd8e1f9b4621e9bfc Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 1 Oct 2026 16:47:10 +0000 Subject: [PATCH] [EPIC-V][LOU-V12] agent.send(), session.send(), evals and the React hook accept content parts send()/stream() on agents and sessions, t.send() and useLoushyAgent().send() take an AgentInput (string | ContentPart[] | Message[]). SqliteStore saves Uint8Array parts as base64 like FileSessionStore. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_016bY29ibQgv3MDdjAbUT4YR --- CHANGELOG.md | 1 + docs/providers.md | 18 +++- docs/react.md | 2 +- docs/sessions.md | 2 + llms-full.txt | 22 +++-- src/createAgent.ts | 19 +++-- src/evals/trajectory.test.ts | 20 +++++ src/evals/trajectory.ts | 5 +- src/multimodalInput.test.ts | 142 +++++++++++++++++++++++++++++++ src/providers/content.ts | 30 ++++++- src/providers/index.ts | 1 + src/react/react.test-d.ts | 3 +- src/react/reducer.test.ts | 16 +++- src/react/reducer.ts | 9 +- src/react/useLoushyAgent.test.ts | 17 ++++ src/react/useLoushyAgent.ts | 5 +- src/session/AgentSession.ts | 14 +-- src/session/sessionStore.ts | 8 +- src/storage/sqlite/stores.ts | 10 +-- 19 files changed, 299 insertions(+), 45 deletions(-) create mode 100644 src/multimodalInput.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 6e7a77118..a197b4d03 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] - 2026-09-28 ### Added +- Multimodal input through the public API (LOU-V12): `agent.send()` / `agent.stream()`, `session.send()` / `stream()`, `t.send()` in evals and `useLoushyAgent().send()` take the new exported `AgentInput` (`string | ContentPart[] | Message[]`): a string or parts become one user message, a `Message[]` is passed through (in a session it is appended to the transcript). Checkpoints, `agent.resume()` and queued input hold the parts like any message. The React user bubble shows the text with an `[image]` / `[file]` marker per non-text part. `SqliteStore` sessions and checkpoints save `Uint8Array` parts as `{ "$bytes": "" }`, as `FileSessionStore` does. Finishes LOU-V11 (its note that `agent.send()` / `session.send()` take no parts is out of date). See docs/providers.md#multimodal-input. - Fork and replay (LOU-D44): `AgentExecutor.fork({ sessionId, fromStep, newSessionId?, checkpointStore, patch? })` takes the newest entry of `fromStep` in the session's checkpoint history, applies `patch` (`messages(messages)`, `businessState`, `toolResult: { toolCallId, result }` replaced in place so the transcript stays provider-valid, `appendInput` queued as a user message) and saves it as the `'in-progress'` checkpoint of `newSessionId` (default `.fork-`), leaving the original session untouched; it returns `{ sessionId, step, checkpoint }`, and `execute({ sessionId, checkpointStore, input: [] })` continues the fork. A missing step throws the new `LOUSHY_CHECKPOINT_NOT_FOUND` code. `createAgent()` agents gain `agent.fork(sessionId, { fromStep, newSessionId?, patch? })` over `store.checkpoints` (resume with `agent.resume()`). `compareTrajectories(a, b)` compares two checkpoints or transcripts step by step: each run's steps (text, tool calls and results), the first diverging step and the `loushy eval --drift` diff. New types `ForkOptions`, `ForkPatch`, `ForkResult`, `TrajectoryComparison`, `TrajectoryStep` and `DriftEntry`. See docs/durable-execution.md#fork-and-replay. - Multimodal message parts (LOU-V11): `Message.content` is now `string | ContentPart[]`, where a `ContentPart` is `{ type: 'text', text }`, `{ type: 'image', image, mimeType? }` (an `http(s)` URL, a `data:` URL or a `Uint8Array`) or `{ type: 'file', data, mimeType, filename? }` (new types `ContentPart`, `TextContentPart`, `ImageContentPart`, `FileContentPart`). The built-in providers map the parts of `user` messages to the `ai` v4 user-content parts: images go to the model with all four (the `ai` SDK downloads an image URL first for Anthropic and Ollama); file parts become a `[file () not sent]` text note with a one-time `console.warn`, since the pinned peers (`@ai-sdk/*` 0.0.x, `ollama-ai-provider`) cannot send files (a subclass sets `acceptsFileParts = true` to send them). `system`, `assistant` and `tool` messages are sent as their text. The new `textOf(message)` returns a message's text (the string, or its text parts joined), and is what `estimateTokens()` (each image or file part counts 1,000 tokens), compaction, `recordReplay()` fingerprints (cassette messages gain an optional `attachments` list: each image or file as its URL or a digest of its bytes), `mockModel()`, `MockLLMProvider`, OpenTelemetry content attributes and the React reducer (`ui.send` takes parts too) now read. `FileSessionStore` saves `Uint8Array`s as `{ "$bytes": "" }` and loads them back. Pass multimodal messages as `AgentExecutor.execute({ input: Message[] })`; `agent.send()` / `session.send()` taking parts is LOU-V12. See docs/providers.md#multimodal-input. - Declarative permission policies (LOU-X2): `permissions?: PermissionRule[]` on `createAgent()`, `ExecuteOptions` and `resumeAfterApproval()` options. A rule is `{ tool: string | string[] | RegExp | '*', when?: (args, { toolName, toolCallId, sessionId }) => boolean | Promise, action: 'allow' | 'deny' | 'ask', reason? }`; rules are checked in order for each tool call (after argument validation and `preToolCall` hooks, before `needsApproval`) and the first match decides: `allow` runs the call without approval even when `needsApproval` would ask, `deny` gives the model a tool error with the new `kind: 'denied'` (`ToolDeniedError`, plus `reason`) and streams `tool.error`, `ask` pauses for approval (or asks `approve`). No match keeps today's behaviour. New helpers `allow(tools)`, `deny(tools, reason?)`, `ask(tools)` and types `PermissionRule`, `PermissionAction`, `PermissionToolMatcher`, `PermissionContext`, `PermissionDecision`, `PermissionDecisionEntry`, `PermissionOptions`. Audit log: `onPermissionDecision(entry)` gets `{ toolName, toolCallId, decision: 'allow' | 'deny' | 'ask' | 'default', rule?: { index, reason? }, args?, at }` per call (`args` left out under `redactContent`), and streams get the same entry as the new typed `permission.decision` event (`PermissionDecisionEvent`; additive, the event schema version stays 1). Neither is produced unless `permissions` or `onPermissionDecision` is set. Sub-agents inherit the lead's rules (checked before their own) and its `onPermissionDecision`. See docs/approvals.md#permission-policies. diff --git a/docs/providers.md b/docs/providers.md index d8bd04a80..e102d756f 100644 --- a/docs/providers.md +++ b/docs/providers.md @@ -85,12 +85,14 @@ fallback model. `Message.content` is a string or a list of parts: `{ type: 'text', text }`, `{ type: 'image', image, mimeType? }` (an `http(s)` URL, a `data:` URL or the bytes as a `Uint8Array`) and `{ type: 'file', data, mimeType, filename? }`. -Pass such messages as the `input` of `AgentExecutor.execute()` or -`stream()` (`agent.send()` and `session.send()` take a string for now): +`agent.send()`, `agent.stream()`, `session.send()` / `stream()`, `t.send()` in +evals and the React hook's `send()` all take an `AgentInput`: a string, a list +of parts (sent as one user message) or a `Message[]` (passed through as it is). +`AgentExecutor.execute()` takes the same messages as its `input`: ```ts import { readFileSync } from 'node:fs'; -import { AgentBuilder, AgentExecutor, resolveProvider, textOf, type Message } from '@loushy/build-ai-agent'; +import { AgentBuilder, AgentExecutor, createAgent, resolveProvider, textOf, type Message } from '@loushy/build-ai-agent'; const agent = AgentBuilder.create().setName('vision').setPrompt('Describe images briefly.').build(); const input: Message[] = [ @@ -106,6 +108,13 @@ const input: Message[] = [ const result = await AgentExecutor.execute({ agent, input, provider: resolveProvider('openai/gpt-4o-mini') }); console.log(textOf(input[0]), '->', result.text); // textOf(): the text parts of a message + +// The same through the public API: parts become one user message. +const answer = await createAgent({ prompt: 'Describe images briefly.', model: 'openai/gpt-4o-mini' }).send([ + { type: 'text', text: 'What is in this picture?' }, + { type: 'image', image: 'https://example.com/cat.png' }, +]); +console.log(answer.text); ``` - **Images** go to the model on `user` messages with every built-in provider @@ -123,7 +132,8 @@ console.log(textOf(input[0]), '->', result.text); // textOf(): the text parts of (each image or file part counts as a flat 1,000 tokens), compaction, `recordReplay()` cassettes (which store the text and a digest of each image or file, or its URL) and `mockModel()`. `FileSessionStore` saves bytes - as base64 (`{ "$bytes": "..." }`) and loads them back as `Uint8Array`s. + as base64 (`{ "$bytes": "..." }`) and loads them back as `Uint8Array`s; + `SqliteStore` (sessions and checkpoints) does the same. ## Provider classes diff --git a/docs/react.md b/docs/react.md index c5272d5e1..213285555 100644 --- a/docs/react.md +++ b/docs/react.md @@ -71,7 +71,7 @@ on the server. | `error` | `{ name, message }` of the last `error` event or a failed request, else `null`. | | `usage` | Token usage of the last finished run (from `run.done`), else `null`. | | `lastEvent` | The last event received, for anything the projection does not cover. | -| `send(input)` | Starts a turn. If a turn is still running, it is aborted first. | +| `send(input)` | Starts a turn. If a turn is still running, it is aborted first. `input` is a string, content parts or a `Message[]` (an `AgentInput`); the user bubble shows the text with an `[image]` / `[file]` marker per other part (remote mode POSTs it as `{ "input": ... }`). | | `stop()` | Aborts the turn in flight through its `AbortSignal`; `status` goes back to `'idle'`. | | `approve(note?)`, `reject(note?)` | Decide `pendingApproval` (see below). | diff --git a/docs/sessions.md b/docs/sessions.md index 482ba444f..e307515ee 100644 --- a/docs/sessions.md +++ b/docs/sessions.md @@ -33,6 +33,8 @@ console.log(session.id, session.messages.length); Concurrent `send()` calls on one session are queued and run one after another in call order, so the transcript never interleaves. +`send()` and `stream()` take a string, content parts (`[{ type: 'text', ... }, { type: 'image', ... }]`, one user message) or a `Message[]`; the stores keep the parts, see [Multimodal input](./providers.md#multimodal-input). + A `send()` that throws (a provider error, or a tool that throws a `PropagatingToolError`) or is aborted with `signal` leaves the transcript exactly as it was before that call. An aborted `send()` resolves with diff --git a/llms-full.txt b/llms-full.txt index 709d3db4b..b72684ff2 100644 --- a/llms-full.txt +++ b/llms-full.txt @@ -4389,12 +4389,14 @@ fallback model. `Message.content` is a string or a list of parts: `{ type: 'text', text }`, `{ type: 'image', image, mimeType? }` (an `http(s)` URL, a `data:` URL or the bytes as a `Uint8Array`) and `{ type: 'file', data, mimeType, filename? }`. -Pass such messages as the `input` of `AgentExecutor.execute()` or -`stream()` (`agent.send()` and `session.send()` take a string for now): +`agent.send()`, `agent.stream()`, `session.send()` / `stream()`, `t.send()` in +evals and the React hook's `send()` all take an `AgentInput`: a string, a list +of parts (sent as one user message) or a `Message[]` (passed through as it is). +`AgentExecutor.execute()` takes the same messages as its `input`: ```ts import { readFileSync } from 'node:fs'; -import { AgentBuilder, AgentExecutor, resolveProvider, textOf, type Message } from '@loushy/build-ai-agent'; +import { AgentBuilder, AgentExecutor, createAgent, resolveProvider, textOf, type Message } from '@loushy/build-ai-agent'; const agent = AgentBuilder.create().setName('vision').setPrompt('Describe images briefly.').build(); const input: Message[] = [ @@ -4410,6 +4412,13 @@ const input: Message[] = [ const result = await AgentExecutor.execute({ agent, input, provider: resolveProvider('openai/gpt-4o-mini') }); console.log(textOf(input[0]), '->', result.text); // textOf(): the text parts of a message + +// The same through the public API: parts become one user message. +const answer = await createAgent({ prompt: 'Describe images briefly.', model: 'openai/gpt-4o-mini' }).send([ + { type: 'text', text: 'What is in this picture?' }, + { type: 'image', image: 'https://example.com/cat.png' }, +]); +console.log(answer.text); ``` - **Images** go to the model on `user` messages with every built-in provider @@ -4427,7 +4436,8 @@ console.log(textOf(input[0]), '->', result.text); // textOf(): the text parts of (each image or file part counts as a flat 1,000 tokens), compaction, `recordReplay()` cassettes (which store the text and a digest of each image or file, or its URL) and `mockModel()`. `FileSessionStore` saves bytes - as base64 (`{ "$bytes": "..." }`) and loads them back as `Uint8Array`s. + as base64 (`{ "$bytes": "..." }`) and loads them back as `Uint8Array`s; + `SqliteStore` (sessions and checkpoints) does the same. ## Provider classes @@ -4526,7 +4536,7 @@ on the server. | `error` | `{ name, message }` of the last `error` event or a failed request, else `null`. | | `usage` | Token usage of the last finished run (from `run.done`), else `null`. | | `lastEvent` | The last event received, for anything the projection does not cover. | -| `send(input)` | Starts a turn. If a turn is still running, it is aborted first. | +| `send(input)` | Starts a turn. If a turn is still running, it is aborted first. `input` is a string, content parts or a `Message[]` (an `AgentInput`); the user bubble shows the text with an `[image]` / `[file]` marker per other part (remote mode POSTs it as `{ "input": ... }`). | | `stop()` | Aborts the turn in flight through its `AbortSignal`; `status` goes back to `'idle'`. | | `approve(note?)`, `reject(note?)` | Decide `pendingApproval` (see below). | @@ -4680,6 +4690,8 @@ console.log(session.id, session.messages.length); Concurrent `send()` calls on one session are queued and run one after another in call order, so the transcript never interleaves. +`send()` and `stream()` take a string, content parts (`[{ type: 'text', ... }, { type: 'image', ... }]`, one user message) or a `Message[]`; the stores keep the parts, see [Multimodal input](https://github.com/LinuxDevil/agent-sdk/blob/main/docs/providers.md#multimodal-input). + A `send()` that throws (a provider error, or a tool that throws a `PropagatingToolError`) or is aborted with `signal` leaves the transcript exactly as it was before that call. An aborted `send()` resolves with diff --git a/src/createAgent.ts b/src/createAgent.ts index 4e7066aa6..881bb0962 100644 --- a/src/createAgent.ts +++ b/src/createAgent.ts @@ -46,6 +46,7 @@ import type { z } from 'zod'; import type { McpServerSpec } from './spec/schema'; import { agentMcp, streamAfter } from './tools/mcp/agentMcp'; import { HookRegistry, type AgentHook } from './execution/hooks'; +import { toMessages, type AgentInput } from './providers/content'; import { compactionHookFor, type AgentCompaction } from './context/agentCompaction'; /** @@ -348,8 +349,12 @@ export interface SendOptions { /** `TObject`: the type of `result.object` - `z.output` of the `output` schema. */ export interface SimpleAgent { - /** Send a single user message and get back the full execution result text. */ - send: (message: string, options?: SendOptions) => Promise>; + /** + * Send a single user message and get back the full execution result text. + * `message` is a string, content parts (one user message with an image or + * file, LOU-V12) or a `Message[]` passed through as it is. + */ + send: (message: AgentInput, options?: SendOptions) => Promise>; /** * Send a single user message and stream the run as typed events (LOU-V2): * `text.delta` chunks as the model writes, `tool.start`/`tool.done`, @@ -365,7 +370,7 @@ export interface SimpleAgent { * } * ``` */ - stream: (message: string, options?: SendOptions) => AgentRun; + stream: (message: AgentInput, options?: SendOptions) => AgentRun; /** * Start a multi-turn conversation (LOU-W4): every `send()` sees the earlier * exchanges. Kept in the agent's `store` (in memory without one); pass @@ -539,12 +544,12 @@ export function createAgent( // `object` was validated with `config.output`, so it has its output type. type Typed = z.output; const simpleAgent: SimpleAgent = { - async send(message: string, options: SendOptions = {}): Promise> { - const result = await run(message, options.signal, durable(options.sessionId)); + async send(message: AgentInput, options: SendOptions = {}): Promise> { + const result = await run(toMessages(message), options.signal, durable(options.sessionId)); return approvals.settle(result, options.signal) as Promise>; }, - stream(message: string, options: SendOptions = {}): AgentRun { - return startStream(executeOptions(message, options.signal, durable(options.sessionId))) as AgentRun; + stream(message: AgentInput, options: SendOptions = {}): AgentRun { + return startStream(executeOptions(toMessages(message), options.signal, durable(options.sessionId))) as AgentRun; }, session, async resume(sessionId: string, { signal } = {}): Promise | null> { diff --git a/src/evals/trajectory.test.ts b/src/evals/trajectory.test.ts index f88dae639..98067a6e8 100644 --- a/src/evals/trajectory.test.ts +++ b/src/evals/trajectory.test.ts @@ -58,6 +58,26 @@ function failedMessages(result: EvalResult): string[] { return result.assertions.filter((a) => !a.passed).map((a) => a.message ?? ''); } +describe('t.send() with an AgentInput (LOU-V12)', () => { + it('sends content parts to the agent as one user message', async () => { + const model = mockModel(['It is a cat.']); + const parts = [ + { type: 'text' as const, text: 'What is this?' }, + { type: 'image' as const, image: 'https://example.com/cat.png' }, + ]; + const result = await run( + async (t) => { + await t.send(parts); + t.check('names the animal', t.reply, includes('cat')); + }, + { agent: createAgent({ provider: model }) } + ); + + expect(failedMessages(result)).toEqual([]); + expect(model.calls[0].messages.filter((m) => m.role === 'user')).toEqual([{ role: 'user', content: parts }]); + }); +}); + describe('trajectory assertions: passing', () => { it('passes every assertion for a matching run and reports steps, tool calls and usage', async () => { const result = await run(async (t) => { diff --git a/src/evals/trajectory.ts b/src/evals/trajectory.ts index 46fc649a3..5b21bda2f 100644 --- a/src/evals/trajectory.ts +++ b/src/evals/trajectory.ts @@ -8,6 +8,7 @@ import type { SimpleAgent } from '../createAgent'; import type { ExecutionResult } from '../execution/AgentExecutor'; import type { LLMProvider } from '../providers/llm'; +import type { AgentInput } from '../providers/content'; import { llmJudge } from './llmJudge'; import type { Check } from './checks'; import type { AssertionKind, AssertionResult, EvalResult, EvalToolCall } from './evalResult'; @@ -50,7 +51,7 @@ export interface CalledToolOptions { */ export interface EvalTestContext { /** Runs the agent on `message`. May be called more than once; assertions look at all runs. */ - send(message: string): Promise; + send(message: AgentInput): Promise; /** Text of the latest reply (empty before the first `send()`). */ readonly reply: string; /** The latest run, or `undefined` before the first `send()`. */ @@ -130,7 +131,7 @@ class TrajectoryContext implements EvalTestContext { return this.recorded; } - async send(message: string): Promise { + async send(message: AgentInput): Promise { this.agent ??= typeof this.agentSource === 'function' ? await this.agentSource() : this.agentSource; const result = await this.agent.send(message); this.results.push(result); diff --git a/src/multimodalInput.test.ts b/src/multimodalInput.test.ts new file mode 100644 index 000000000..aaca5732b --- /dev/null +++ b/src/multimodalInput.test.ts @@ -0,0 +1,142 @@ +/** + * LOU-V12: send() / stream() take an AgentInput (a string, content parts or + * a Message[]) on agents and sessions, and sessions keep the parts. + */ +import { describe, it, expect } from 'vitest'; +import { z } from 'zod'; +import { createAgent } from './createAgent'; +import { defineTool } from './tools/defineTool'; +import { PropagatingToolError } from './execution/AgentExecutor'; +import { memoryStore, type AgentStore } from './storage/agentStore'; +import { SqliteStore } from './storage/sqlite'; +import { mockModel } from './testing'; +import { toMessages, type AgentInput } from './providers/content'; +import type { ContentPart, Message } from './providers/llm'; + +const bytes = new Uint8Array([137, 80, 78, 71, 0, 255]); +const parts: ContentPart[] = [ + { type: 'text', text: 'What is in this photo?' }, + { type: 'image', image: bytes, mimeType: 'image/png' }, +]; +const conversation = (messages: readonly Message[]) => messages.filter((m) => m.role !== 'system'); + +describe('toMessages()', () => { + it('wraps a string or parts in one user message and passes Message[] through', () => { + expect(toMessages('Hi')).toEqual([{ role: 'user', content: 'Hi' }]); + expect(toMessages(parts)).toEqual([{ role: 'user', content: parts }]); + const messages: Message[] = [{ role: 'user', content: 'a' }, { role: 'assistant', content: 'b' }]; + expect(toMessages(messages)).toBe(messages); + expect(toMessages([])).toEqual([]); + }); +}); + +describe('agent.send() / agent.stream() with an AgentInput (LOU-V12)', () => { + it('send(parts) reaches the provider as one user message with the parts', async () => { + const model = mockModel(['A cat.']); + const result = await createAgent({ prompt: 'Describe images.', provider: model }).send(parts); + + expect(result.text).toBe('A cat.'); + expect(conversation(model.calls[0].messages)).toEqual([{ role: 'user', content: parts }]); + }); + + it('send(Message[]) is passed through as the conversation', async () => { + const model = mockModel(['Paris.']); + const input: AgentInput = [ + { role: 'user', content: 'Capital of France?' }, + { role: 'assistant', content: 'Let me think.' }, + { role: 'user', content: [{ type: 'text', text: 'Answer now.' }] }, + ]; + await createAgent({ provider: model }).send(input); + + expect(conversation(model.calls[0].messages)).toEqual(input); + }); + + it('stream(parts) sends the parts too', async () => { + const model = mockModel(['A cat.']); + const run = createAgent({ provider: model }).stream(parts); + + expect((await run.result).text).toBe('A cat.'); + expect(conversation(model.calls[0].messages)).toEqual([{ role: 'user', content: parts }]); + }); + + it('a durable send(parts, { sessionId }) is finished by agent.resume() with the parts intact', async () => { + let runs = 0; + const flaky = defineTool({ + name: 'flaky', + description: 'dies once', + input: z.object({}), + execute: async () => { + if (++runs === 1) throw new PropagatingToolError('process died'); + return 'ok'; + }, + }); + const store = memoryStore(); + const call = { toolCalls: [{ name: 'flaky', id: 'call_flaky' }] }; + const crashing = createAgent({ provider: mockModel([call]), tools: [flaky], store }); + await expect(crashing.send(parts, { sessionId: 'job' })).rejects.toThrow('process died'); + + const model = mockModel(['Done.']); + const resumed = await createAgent({ provider: model, tools: [flaky], store }).resume('job'); + + expect(resumed?.text).toBe('Done.'); + expect(conversation(model.calls[0].messages)[0]).toEqual({ role: 'user', content: parts }); + }); +}); + +const stores: Array<[string, () => AgentStore & { close?: () => void }]> = [ + ['memoryStore()', () => memoryStore()], + [":memory: SqliteStore", () => new SqliteStore(':memory:')], +]; + +describe.each(stores)('session.send() with parts through %s (LOU-V12)', (_name, makeStore) => { + it('keeps the parts in the transcript and shows them to the next turn', async () => { + const store = makeStore(); + const first = mockModel(['A cat.']); + await createAgent({ provider: first, store }).session({ id: 'chat' }).send(parts); + + const saved = await store.sessions!.load('chat'); + expect(saved![0]).toEqual({ role: 'user', content: parts }); + expect((saved![0].content as ContentPart[])[1]).toMatchObject({ image: bytes }); + expect((saved![0].content as { image: unknown }[])[1].image).toBeInstanceOf(Uint8Array); + + const second = mockModel(['It is orange.']); + const session = createAgent({ provider: second, store }).session({ id: 'chat' }); + await session.send([{ type: 'text', text: 'What colour?' }]); + expect(conversation(second.calls[0].messages).map((m) => m.role)).toEqual(['user', 'assistant', 'user']); + expect(conversation(second.calls[0].messages)[0]).toEqual({ role: 'user', content: parts }); + store.close?.(); + }); + + it('session.stream(parts) records the parts as well', async () => { + const store = makeStore(); + const session = createAgent({ provider: mockModel(['A cat.']), store }).session({ id: 'chat' }); + await session.stream(parts).result; + + expect((await store.sessions!.load('chat'))![0]).toEqual({ role: 'user', content: parts }); + store.close?.(); + }); + + it('a checkpointed turn with parts survives a crash and resume()', async () => { + const store = makeStore(); + let runs = 0; + const flaky = defineTool({ + name: 'flaky', + description: 'dies once', + input: z.object({}), + execute: async () => { + if (++runs === 1) throw new PropagatingToolError('process died'); + return 'ok'; + }, + }); + const call = { toolCalls: [{ name: 'flaky', id: 'call_flaky' }] }; + await expect( + createAgent({ provider: mockModel([call]), tools: [flaky], store }).session({ id: 'chat' }).send(parts) + ).rejects.toThrow('process died'); + + const resumed = await createAgent({ provider: mockModel(['Done.']), tools: [flaky], store }).resume('chat'); + + expect(resumed?.text).toBe('Done.'); + expect((await store.sessions!.load('chat'))![0]).toEqual({ role: 'user', content: parts }); + store.close?.(); + }); +}); diff --git a/src/providers/content.ts b/src/providers/content.ts index 852118ea4..e69958664 100644 --- a/src/providers/content.ts +++ b/src/providers/content.ts @@ -2,7 +2,7 @@ * Helpers for multimodal message content (LOU-V11). */ -import type { Message } from './llm'; +import type { ContentPart, Message } from './llm'; /** * The text of a message (or of its `content`): the string itself, or its @@ -18,3 +18,31 @@ export function textOf(message: Pick | Message['content']): for (const part of content) if (part.type === 'text') text += part.text; return text; } + +/** + * What `send()` / `stream()` take (LOU-V12): a string or content parts become + * one user message; a `Message[]` is passed through as it is. + * + * @example + * agent.send([{ type: 'text', text: 'What is in this photo?' }, { type: 'image', image: url }]); + */ +export type AgentInput = string | ContentPart[] | Message[]; + +/** `input` as messages: a string or parts become one user message, `Message[]` is returned as it is. */ +export function toMessages(input: AgentInput): Message[] { + if (typeof input === 'string') return [{ role: 'user', content: input }]; + // An empty array has no parts to tell it from messages: it adds nothing either way. + return isMessages(input) ? input : [{ role: 'user', content: input }]; +} + +function isMessages(input: ContentPart[] | Message[]): input is Message[] { + return input.length === 0 || 'role' in input[0]!; +} + +/** The user text of `input` (the last user message of a `Message[]`), then an `[image]` / `[file]` marker per non-text part. */ +export function describeInput(input: AgentInput): string { + const content = toMessages(input).filter((m) => m.role === 'user').pop()?.content ?? ''; + if (typeof content === 'string') return content; + const markers = content.filter((part) => part.type !== 'text').map((part) => `[${part.type}]`); + return [textOf(content), ...markers].filter(Boolean).join(' '); +} diff --git a/src/providers/index.ts b/src/providers/index.ts index 03d0b72e2..fef0c2cb4 100644 --- a/src/providers/index.ts +++ b/src/providers/index.ts @@ -5,6 +5,7 @@ export * from './llm'; export { textOf } from './content'; +export type { AgentInput } from './content'; export * from './mock'; export * from './OpenAIProvider'; export * from './OllamaProvider'; diff --git a/src/react/react.test-d.ts b/src/react/react.test-d.ts index 8d761640c..e56f1c95a 100644 --- a/src/react/react.test-d.ts +++ b/src/react/react.test-d.ts @@ -4,6 +4,7 @@ import { describe, it, expectTypeOf } from 'vitest'; import { createAgent } from '../createAgent'; import { createMockProvider } from '../providers/mock'; +import type { AgentInput } from '../providers/content'; import type { AgentEvent, AgentEventError, AgentEventUsage } from '../execution/agentEvents'; import { initialAgentUIState, @@ -39,7 +40,7 @@ describe('@loushy/build-ai-agent/react types', () => { expectTypeOf().toEqualTypeOf(); expectTypeOf().toEqualTypeOf(); expectTypeOf().toEqualTypeOf(); - expectTypeOf().toEqualTypeOf<(input: string) => Promise>(); + expectTypeOf().toEqualTypeOf<(input: AgentInput) => Promise>(); expectTypeOf().toEqualTypeOf<(note?: string) => Promise>(); expectTypeOf().toEqualTypeOf<() => void>(); expectTypeOf().toEqualTypeOf<'user' | 'assistant'>(); diff --git a/src/react/reducer.test.ts b/src/react/reducer.test.ts index e61812664..12c671eb3 100644 --- a/src/react/reducer.test.ts +++ b/src/react/reducer.test.ts @@ -29,12 +29,24 @@ describe('reduceAgentEvents (LOU-D15)', () => { ]); }); - it('ui.send with content parts shows their text (LOU-V11)', () => { + it('ui.send with content parts shows their text and a marker per non-text part (LOU-V12)', () => { const state = reduce({ type: 'ui.send', input: [{ type: 'text', text: 'What is this?' }, { type: 'image', image: 'https://example.com/a.png' }], }); - expect(state.messages[0]).toEqual({ id: 'm0', role: 'user', text: 'What is this?', toolCalls: [] }); + expect(state.messages[0]).toEqual({ id: 'm0', role: 'user', text: 'What is this? [image]', toolCalls: [] }); + }); + + it('ui.send with a Message[] shows its last user message', () => { + const state = reduce({ + type: 'ui.send', + input: [ + { role: 'user', content: 'Earlier' }, + { role: 'assistant', content: 'Ok' }, + { role: 'user', content: [{ type: 'file', data: 'https://example.com/a.pdf', mimeType: 'application/pdf' }] }, + ], + }); + expect(state.messages[0].text).toBe('[file]'); }); it('accumulates text deltas, then run.done sets idle and usage', () => { diff --git a/src/react/reducer.ts b/src/react/reducer.ts index c2a5f53e8..afb12cd65 100644 --- a/src/react/reducer.ts +++ b/src/react/reducer.ts @@ -6,8 +6,7 @@ */ import type { AgentEvent, AgentEventError, AgentEventUsage } from '../execution/agentEvents'; -import type { ContentPart } from '../providers/llm'; -import { textOf } from '../providers/content'; +import { describeInput, type AgentInput } from '../providers/content'; /** Where a tool call stands: running, paused for approval, or finished. */ export type UIToolCallStatus = 'running' | 'awaiting-approval' | 'done' | 'error' | 'rejected'; @@ -67,8 +66,8 @@ export interface AgentUIState { /** Local actions, besides the events themselves. */ export type AgentUIAction = - /** `input` may be multimodal parts (LOU-V11); the bubble shows their text. */ - | { type: 'ui.send'; input: string | ContentPart[] } + /** `input` may be multimodal (LOU-V12); the bubble shows its text and an `[image]` / `[file]` marker per other part. */ + | { type: 'ui.send'; input: AgentInput } | { type: 'ui.decide'; approved: boolean } | { type: 'ui.resumed'; outcome: ApprovalOutcome } | { type: 'ui.stopped' } @@ -139,7 +138,7 @@ function resumed(state: AgentUIState, { text, finishReason, usage, approval }: A export function reduceAgentEvents(state: AgentUIState, event: AgentEvent | AgentUIAction): AgentUIState { switch (event.type) { case 'ui.send': { - const user: UIMessage = { id: `m${state.messages.length}`, role: 'user', text: textOf(event.input), toolCalls: [] }; + const user: UIMessage = { id: `m${state.messages.length}`, role: 'user', text: describeInput(event.input), toolCalls: [] }; const messages = onAssistant([...state.messages, user], (message) => message); return { ...state, messages, status: 'streaming', error: null, pendingApproval: null }; } diff --git a/src/react/useLoushyAgent.test.ts b/src/react/useLoushyAgent.test.ts index 4898a6825..838b433b4 100644 --- a/src/react/useLoushyAgent.test.ts +++ b/src/react/useLoushyAgent.test.ts @@ -46,6 +46,23 @@ function emailAgent(...turns: Parameters[0]) { const callEmail = { toolCalls: [{ name: 'send_email', args: { to: 'sam' }, id: 'call_email' }] }; describe('useLoushyAgent in process (LOU-D15)', () => { + it('send(parts) sends one user message with the parts and shows their text and a marker (LOU-V12)', async () => { + const model = mockModel(['A cat.']); + mount({ agent: createAgent({ provider: model }) }); + const parts = [ + { type: 'text' as const, text: 'What is this?' }, + { type: 'image' as const, image: 'https://example.com/cat.png' }, + ]; + + await act(() => hook.send(parts)); + + expect(hook.messages.map(({ role, text }) => ({ role, text }))).toEqual([ + { role: 'user', text: 'What is this? [image]' }, + { role: 'assistant', text: 'A cat.' }, + ]); + expect(model.calls[0].messages.filter((m) => m.role === 'user')).toEqual([{ role: 'user', content: parts }]); + }); + it('send() streams the reply into messages and ends idle with usage', async () => { const agent = createAgent({ provider: mockModel([{ text: 'Hello there!', usage: { inputTokens: 5, outputTokens: 3 } }]) }); mount({ agent }); diff --git a/src/react/useLoushyAgent.ts b/src/react/useLoushyAgent.ts index 1017e4517..b16dd6cb2 100644 --- a/src/react/useLoushyAgent.ts +++ b/src/react/useLoushyAgent.ts @@ -6,6 +6,7 @@ import { useCallback, useEffect, useReducer, useRef } from 'react'; import type { SimpleAgent } from '../createAgent'; +import type { AgentInput } from '../providers/content'; import type { AgentEvent, AgentEventError } from '../execution/agentEvents'; import type { AgentSession } from '../session/AgentSession'; import { parseEventStream } from './parseEventStream'; @@ -43,7 +44,7 @@ export interface UseLoushyAgentOptions { export interface UseLoushyAgentResult extends AgentUIState { /** Starts a turn (aborting one still running). */ - send(input: string): Promise; + send(input: AgentInput): Promise; /** Aborts the run in flight; it ends with `finishReason: 'aborted'`. */ stop(): void; approve(note?: string): Promise; @@ -109,7 +110,7 @@ export function useLoushyAgent(source: LoushyAgentSource, options: UseLoushyAgen }, []); const send = useCallback( - (input: string) => + (input: AgentInput) => run(async (emit, signal) => { emit({ type: 'ui.send', input }); const { source } = latest.current; diff --git a/src/session/AgentSession.ts b/src/session/AgentSession.ts index a0da87c97..54dffef9a 100644 --- a/src/session/AgentSession.ts +++ b/src/session/AgentSession.ts @@ -7,6 +7,7 @@ import { randomUUID } from 'node:crypto'; import type { Message } from '../providers/llm'; +import { toMessages, type AgentInput } from '../providers/content'; import type { ExecutionResult } from '../execution/AgentExecutor'; import type { AgentRun } from '../execution/agentRun'; import type { Checkpoint, CheckpointStore } from '../execution/checkpoint'; @@ -168,7 +169,8 @@ export class AgentSession { } /** - * Send a user message with the whole conversation so far. Concurrent calls + * Send a user message (a string, content parts or a `Message[]`, see + * `AgentInput`) with the whole conversation so far. Concurrent calls * run one after another, in call order. * * A call that throws or is aborted leaves the transcript as it was before @@ -180,7 +182,7 @@ export class AgentSession { * const result = await session.send('And in Paris?', { signal: AbortSignal.timeout(10_000) }); * ``` */ - send(input: string, options: { signal?: AbortSignal } = {}): Promise { + send(input: AgentInput, options: { signal?: AbortSignal } = {}): Promise { return this.enqueue(() => this.turn(input, options.signal)); } @@ -206,7 +208,7 @@ export class AgentSession { * } * ``` */ - stream(input: string, options: { signal?: AbortSignal } = {}): AgentRun { + stream(input: AgentInput, options: { signal?: AbortSignal } = {}): AgentRun { const streamRun = this.streamRun; if (!streamRun) { throw new SDKError( @@ -218,7 +220,7 @@ export class AgentSession { (signal, started) => this.enqueue(async () => { await this.beforeTurn(signal); - const run = streamRun([...this.history, { role: 'user', content: input }], signal, this.turnCheckpoint()); + const run = streamRun([...this.history, ...toMessages(input)], signal, this.turnCheckpoint()); started(run); return this.record(await run.result); }), @@ -291,9 +293,9 @@ export class AgentSession { this.loaded = true; } - private async turn(input: string, signal?: AbortSignal): Promise { + private async turn(input: AgentInput, signal?: AbortSignal): Promise { await this.beforeTurn(signal); - return this.record(await this.run([...this.history, { role: 'user', content: input }], signal, this.turnCheckpoint())); + return this.record(await this.run([...this.history, ...toMessages(input)], signal, this.turnCheckpoint())); } /** Loads the history and, in a checkpointed session, finishes a pending turn first. */ diff --git a/src/session/sessionStore.ts b/src/session/sessionStore.ts index 78c5041ad..8411b5d3c 100644 --- a/src/session/sessionStore.ts +++ b/src/session/sessionStore.ts @@ -12,14 +12,14 @@ import { ConfigurationError, SDKError } from '../execution/errors'; /** How bytes (image and file parts, LOU-V11) are saved in a JSON transcript: `{ "$bytes": "" }`. */ const BYTES_KEY = '$bytes'; -/** `JSON.stringify` replacer: a `Uint8Array` (a `Buffer` too, read before its `toJSON()`) becomes `{ $bytes }`. */ -function encodeBytes(this: Record, key: string, value: unknown): unknown { +/** `JSON.stringify` replacer: a `Uint8Array` (a `Buffer` too, read before its `toJSON()`) becomes `{ $bytes }`. Shared with `SqliteStore`. */ +export function encodeBytes(this: Record, key: string, value: unknown): unknown { const raw = this[key]; return raw instanceof Uint8Array ? { [BYTES_KEY]: Buffer.from(raw).toString('base64') } : value; } -/** `JSON.parse` reviver: `{ $bytes }` back to a `Uint8Array`. */ -function decodeBytes(_key: string, value: unknown): unknown { +/** `JSON.parse` reviver: `{ $bytes }` back to a `Uint8Array`. Shared with `SqliteStore`. */ +export function decodeBytes(_key: string, value: unknown): unknown { if (typeof value !== 'object' || value === null || Object.keys(value).length !== 1) return value; const base64 = (value as Record)[BYTES_KEY]; return typeof base64 === 'string' ? new Uint8Array(Buffer.from(base64, 'base64')) : value; diff --git a/src/storage/sqlite/stores.ts b/src/storage/sqlite/stores.ts index bbde192de..f8137e190 100644 --- a/src/storage/sqlite/stores.ts +++ b/src/storage/sqlite/stores.ts @@ -13,7 +13,7 @@ import type { PendingApproval, ResolvedApproval, } from '../../execution/ApprovalGate'; -import { assertSessionId, type SessionStore } from '../../session/sessionStore'; +import { assertSessionId, decodeBytes, encodeBytes, type SessionStore } from '../../session/sessionStore'; import type { Connection } from './connection'; import type { SqlStatement } from './driver'; @@ -34,7 +34,7 @@ export class Statements { } const parse = (row: { payload?: unknown } | undefined): T | undefined => - row === undefined ? undefined : (JSON.parse(String(row.payload)) as T); + row === undefined ? undefined : (JSON.parse(String(row.payload), decodeBytes) as T); // `table` and `key` are fixed identifiers from this file, never user input. const upsert = (table: string, key: string): string => @@ -56,7 +56,7 @@ export class SqliteSessionStore implements SessionStore { async save(id: string, messages: readonly Message[]): Promise { assertSessionId(id); const now = Date.now(); - this.sql.get(upsert('sessions', 'id')).run(id, JSON.stringify(messages), now, now); + this.sql.get(upsert('sessions', 'id')).run(id, JSON.stringify(messages, encodeBytes), now, now); } async delete(id: string): Promise { @@ -83,7 +83,7 @@ export class SqliteCheckpointStore implements CheckpointStore { async save(sessionId: string, checkpoint: Checkpoint): Promise { const now = Date.now(); - const payload = JSON.stringify(checkpoint); + const payload = JSON.stringify(checkpoint, encodeBytes); this.connection.transaction(() => { this.sql.get(upsert('checkpoints', 'session_id')).run(sessionId, payload, now, now); if (this.historyLimit === 0) return; @@ -120,7 +120,7 @@ export class SqliteCheckpointStore implements CheckpointStore { step: Number(row.step), savedAt: new Date(Number(row.saved_at)).toISOString(), status: String(row.status) as CheckpointHistoryEntry['status'], - checkpoint: JSON.parse(String(row.payload)) as Checkpoint, + checkpoint: JSON.parse(String(row.payload), decodeBytes) as Checkpoint, })); } }