From b899350f9059666a4f39c57356f2f188a6d73901 Mon Sep 17 00:00:00 2001 From: fforres Date: Fri, 9 Oct 2026 00:51:37 -0700 Subject: [PATCH] feat(mcp): passthrough invoke forwards upstream image and audio content blocks A successful passthrough invoke whose upstream result carries image or audio content blocks now returns those blocks as native MCP content. Text blocks stay text and keep the existing size bound, and structuredContent no longer duplicates the media bytes. Other results and the execute tool are unchanged. --- .../hosts/mcp/src/passthrough-tools.test.ts | 73 +++++++++++++++++++ packages/hosts/mcp/src/tool-server.ts | 32 +++++++- 2 files changed, 104 insertions(+), 1 deletion(-) diff --git a/packages/hosts/mcp/src/passthrough-tools.test.ts b/packages/hosts/mcp/src/passthrough-tools.test.ts index 491a623740..179d803428 100644 --- a/packages/hosts/mcp/src/passthrough-tools.test.ts +++ b/packages/hosts/mcp/src/passthrough-tools.test.ts @@ -639,6 +639,79 @@ describe("passthrough mode server", () => { ); }); + const invokeWith = async (data: unknown) => { + const recording = makeRecordingEngine({ ok: true, data }); + let result: Awaited> | undefined; + await withClient( + { engine: recording.engine, mode: "passthrough", tools: toolPort(CATALOG) }, + async (client) => { + result = await client.callTool({ + name: "invoke", + arguments: { tool: "tools.github.org.main.issues.list", arguments: {} }, + }); + }, + ); + return result!; + }; + + it("forwards an upstream image block as a native image block", async () => { + const result = await invokeWith({ + content: [{ type: "image", data: "AQID", mimeType: "image/jpeg" }], + }); + expect(result.content).toEqual([{ type: "image", data: "AQID", mimeType: "image/jpeg" }]); + expect(result.isError ?? false).toBe(false); + }); + + it("keeps mixed text and image blocks in order and keeps bytes out of structuredContent", async () => { + const result = await invokeWith({ + content: [ + { type: "text", text: "before" }, + { type: "image", data: "AQID", mimeType: "image/png" }, + { type: "text", text: "after" }, + { type: "audio", data: "BAUG", mimeType: "audio/wav" }, + ], + structuredContent: { name: "x.png" }, + }); + expect(result.content).toEqual([ + { type: "text", text: "before" }, + { type: "image", data: "AQID", mimeType: "image/png" }, + { type: "text", text: "after" }, + { type: "audio", data: "BAUG", mimeType: "audio/wav" }, + ]); + expect(result.structuredContent).toEqual({ name: "x.png" }); + expect(JSON.stringify(result.structuredContent)).not.toContain("AQID"); + }); + + it("omits structuredContent when the upstream result has none", async () => { + const result = await invokeWith({ + content: [{ type: "image", data: "AQID", mimeType: "image/png" }], + }); + expect(result.structuredContent).toBeUndefined(); + }); + + it("still bounds a large text block next to an image", async () => { + const result = await invokeWith({ + content: [ + { type: "text", text: "x".repeat(30_005) }, + { type: "image", data: "AQID", mimeType: "image/png" }, + ], + }); + expect(result.content).toEqual([ + { type: "text", text: `${"x".repeat(30_000)}\n... [truncated 5 chars]` }, + { type: "image", data: "AQID", mimeType: "image/png" }, + ]); + }); + + it("keeps a text-only upstream result on the standard rendering", async () => { + const result = await invokeWith({ content: [{ type: "text", text: "hi" }] }); + expect((result.content as Array<{ type: string }>).map((b) => b.type)).toEqual(["text"]); + expect(result.structuredContent).toEqual({ + status: "completed", + result: { content: [{ type: "text", text: "hi" }] }, + logs: [], + }); + }); + /** An engine whose tool raises the given elicitations in order and records * each answer. `source` is what the executor stamps: `policy` for its own * approval gate, `tool` for anything the tool asked for itself. */ diff --git a/packages/hosts/mcp/src/tool-server.ts b/packages/hosts/mcp/src/tool-server.ts index daa843cd4e..c0d662ea03 100644 --- a/packages/hosts/mcp/src/tool-server.ts +++ b/packages/hosts/mcp/src/tool-server.ts @@ -719,6 +719,36 @@ const toMcpResult = (result: FormattedExecuteInput): McpToolResult => { }; }; +const MAX_PASSTHROUGH_TEXT_CHARS = 30_000; + +const isMediaBlock = (block: ContentBlock): boolean => + block.type === "image" || block.type === "audio"; + +const boundText = (block: ContentBlock): ContentBlock => + block.type === "text" && block.text.length > MAX_PASSTHROUGH_TEXT_CHARS + ? { + ...block, + text: `${block.text.slice(0, MAX_PASSTHROUGH_TEXT_CHARS)}\n... [truncated ${block.text.length - MAX_PASSTHROUGH_TEXT_CHARS} chars]`, + } + : block; + +/** + * An upstream MCP tool result that carries image or audio blocks is forwarded + * as native MCP content, so the bytes reach the client as media and never as + * JSON text or `structuredContent`. Any other shape returns undefined and + * keeps the standard rendering. + */ +const upstreamMediaResult = (data: unknown): McpToolResult | undefined => { + if (!isRecord(data) || data.isError === true || !Array.isArray(data.content)) return undefined; + const blocks: unknown[] = data.content; + if (!blocks.every(isMcpContentBlock)) return undefined; + if (!blocks.some(isMediaBlock)) return undefined; + return { + content: blocks.map(boundText), + ...(isRecord(data.structuredContent) ? { structuredContent: data.structuredContent } : {}), + }; +}; + /** * A passthrough call's result IS the tool's `ToolResult`. Inside `execute` * the model reads `{ ok, data | error }` and branches; here nothing runs @@ -731,7 +761,7 @@ const toPassthroughResult = (outcome: FormattedExecuteInput): McpToolResult => { const value = outcome.result; if (outcome.error || !isToolResult(value)) return toMcpResult(outcome); if (value.ok) { - return toMcpResult({ ...outcome, result: value.data }); + return upstreamMediaResult(value.data) ?? toMcpResult({ ...outcome, result: value.data }); } const message = `${value.error.code}: ${value.error.message}`; return {