From fbb6ac6827a4525f5160c632182c8ba4f470cf37 Mon Sep 17 00:00:00 2001 From: lirenjie Date: Tue, 13 Oct 2026 21:47:55 +0800 Subject: [PATCH 1/5] fix(agent): allow tool responses to pass image content to next-round vision input --- packages/global/core/ai/llm/type.ts | 16 +- packages/global/core/chat/adapt.ts | 35 ++- packages/global/test/core/chat/adapt.test.ts | 40 +++- .../core/ai/llm/agentLoop/domain/tool.ts | 8 +- .../agentLoop/provider/fastAgent/loop/base.ts | 43 +++- .../provider/fastAgent/loop/index.ts | 3 +- .../llm/agentLoop/provider/piAgent/message.ts | 88 ++++++- .../ai/llm/agentLoop/provider/piAgent/run.ts | 25 +- .../agentLoop/provider/piAgent/tool/build.ts | 64 ++++- .../service/core/ai/llm/promptCall/index.ts | 19 +- packages/service/core/ai/llm/utils.ts | 105 ++++++++- .../application/toolCall/getFileUrl.tool.ts | 27 ++- .../ai/sandbox/application/toolCall/index.ts | 6 +- .../ai/sandbox/application/toolCall/type.ts | 10 +- .../application/runtime/workflowToolRunner.ts | 38 ++- .../ai/agentLoopCore/domain/toolProvider.ts | 11 +- .../ai/llm/agentLoop/fastAgentLoop.test.ts | 36 ++- .../ai/llm/agentLoop/piAgentMessage.test.ts | 83 ++++++- .../service/test/core/ai/llm/toolCall.test.ts | 87 +++++++ .../service/test/core/ai/llm/utils.test.ts | 222 +++++++++++++++++- .../toolCall/getFileUrl.tool.test.ts | 27 +++ .../runtime/workflowToolRunner.test.ts | 48 +++- 22 files changed, 983 insertions(+), 58 deletions(-) diff --git a/packages/global/core/ai/llm/type.ts b/packages/global/core/ai/llm/type.ts index 432cefc6f0c0..a168cb19b2e2 100644 --- a/packages/global/core/ai/llm/type.ts +++ b/packages/global/core/ai/llm/type.ts @@ -128,13 +128,25 @@ export type ChatCompletionUserMessageParam = z.infer; + /** * Tool message: 对齐 openai SDK 的 ChatCompletionToolMessageParam，新增可选 `name` - * SDK content 仅允许纯文本 part + * SDK content 允许字符串或 text/image_url part 数组 */ export const ChatCompletionToolMessageParamSchema = z.object({ role: z.literal('tool'), - content: z.union([z.string(), z.array(ChatCompletionContentPartTextSchema)]), + content: z.union([z.string(), z.array(ChatCompletionToolMessageContentPartSchema)]), tool_call_id: z.string(), name: z.string().optional() }); diff --git a/packages/global/core/chat/adapt.ts b/packages/global/core/chat/adapt.ts index 50c4eaa670c2..54c8b679d175 100644 --- a/packages/global/core/chat/adapt.ts +++ b/packages/global/core/chat/adapt.ts @@ -12,6 +12,7 @@ import type { import { ChatFileTypeEnum, ChatRoleEnum } from '../../core/chat/constants'; import type { ChatCompletionContentPart, + ChatCompletionContentPartText, ChatCompletionFunctionMessageParam, ChatCompletionMessageFunctionCall, ChatCompletionMessageParam, @@ -842,6 +843,26 @@ export const chats2GPTMessages = ({ return results; }; +/** + * 提取工具响应的纯文本部分。 + * + * tool content 允许是字符串或 parts 数组。折回 ChatItem 时只保留 text parts: + * 工具卡片展示与历史持久化都基于文本,image_url 等富媒体内容不进入这条链路, + * 避免把数组整体 JSON 化后污染展示与后续轮次读取。 + */ +const getToolResponseText = (content: ChatCompletionMessageParam['content']): string => { + if (typeof content === 'string') return content; + if (!Array.isArray(content)) return ''; + return content + .filter( + (part): part is ChatCompletionContentPartText => + typeof part === 'object' && part !== null && part.type === 'text' + ) + .map((part) => part.text) + .filter(Boolean) + .join('\n'); +}; + /** * 将 GPT messages 转回 FastGPT ChatItem。 * @@ -969,14 +990,12 @@ export const GPTMessages2Chats = ({ const toolCalls = item.tool_calls as ChatCompletionMessageToolCall[]; const tools = toolCalls.flatMap((tool) => { - let toolResponse = - normalizedMessages.find( - (msg) => - msg.role === ChatCompletionRequestMessageRoleEnum.Tool && - msg.tool_call_id === tool.id - )?.content || ''; - toolResponse = - typeof toolResponse === 'string' ? toolResponse : JSON.stringify(toolResponse); + const toolContent = normalizedMessages.find( + (msg) => + msg.role === ChatCompletionRequestMessageRoleEnum.Tool && + msg.tool_call_id === tool.id + )?.content; + const toolResponse = getToolResponseText(toolContent); const toolInfo = getToolInfo?.(tool.function.name); diff --git a/packages/global/test/core/chat/adapt.test.ts b/packages/global/test/core/chat/adapt.test.ts index f2f09dff98de..b012ff1ae736 100644 --- a/packages/global/test/core/chat/adapt.test.ts +++ b/packages/global/test/core/chat/adapt.test.ts @@ -3145,7 +3145,7 @@ describe('GPTMessages2Chats', () => { expect(toolValue.tools[0].response).toBe('{"results": []}'); }); - it('should stringify non-string tool response content', () => { + it('should keep text parts when folding non-string tool response content', () => { const messages: ChatCompletionMessageParam[] = [ { role: ChatCompletionRequestMessageRoleEnum.Assistant, @@ -3170,9 +3170,7 @@ describe('GPTMessages2Chats', () => { const result = GPTMessages2Chats({ messages, reserveTool: true }); expect(result).toHaveLength(1); - expect((result[0].value[0] as any).tools[0].response).toBe( - '[{"type":"text","text":"chunked result"}]' - ); + expect((result[0].value[0] as any).tools[0].response).toBe('chunked result'); }); it('should keep assistant content and reasoning separate from tools after chat roundtrip', () => { @@ -3642,6 +3640,40 @@ describe('GPTMessages2Chats', () => { expect(toolValue.tools[0].toolName).toBe('Custom Tool Display Name'); expect(toolValue.tools[0].toolAvatar).toBe('http://example.com/avatar.png'); }); + + it('should keep only text parts when folding tool content parts back to chat items', () => { + const messages: ChatCompletionMessageParam[] = [ + { + role: ChatCompletionRequestMessageRoleEnum.Assistant, + content: '', + tool_calls: [ + { + id: 'call_1', + type: 'function', + function: { + name: 'custom_tool', + arguments: '{}' + } + } + ] + }, + { + role: ChatCompletionRequestMessageRoleEnum.Tool, + tool_call_id: 'call_1', + content: [ + { type: 'text', text: 'Look at these images' }, + { type: 'image_url', image_url: { url: 'https://example.com/a.png' } } + ] + } + ]; + + const result = GPTMessages2Chats({ messages, reserveTool: true }); + + const toolValue = result[0].value[0] as any; + expect(toolValue.tools).toBeDefined(); + expect(toolValue.tools).toHaveLength(1); + expect(toolValue.tools[0].response).toBe('Look at these images'); + }); }); describe('chatValue2RuntimePrompt', () => { diff --git a/packages/service/core/ai/llm/agentLoop/domain/tool.ts b/packages/service/core/ai/llm/agentLoop/domain/tool.ts index 944889ba184f..130d8d309991 100644 --- a/packages/service/core/ai/llm/agentLoop/domain/tool.ts +++ b/packages/service/core/ai/llm/agentLoop/domain/tool.ts @@ -1,7 +1,8 @@ import type { ChatCompletionMessageParam, ChatCompletionMessageToolCall, - ChatCompletionTool + ChatCompletionTool, + ChatCompletionToolMessageContentPart } from '@fastgpt/global/core/ai/llm/type'; import type { SandboxClient } from '../../../sandbox/application/runtime/client'; import type { SandboxToolCallValidator } from '../../../sandbox/application/toolCall/type'; @@ -53,6 +54,11 @@ export type AgentLoopToolExecutionResult = { hideInUI?: boolean; /** 由调用方透传并在 agent-loop 外部解释的工具运行元数据。 */ metadata?: unknown; + /** + * 写入 LLM tool message 的结构化 content（text / image_url parts）。 + * 工具取到图片时可返回 image_url part，作为下一轮模型的视觉输入；缺省时按 response 字符串兜底。 + */ + content?: ChatCompletionToolMessageContentPart[]; }; export type AgentLoopReadFileExecutionResult = { diff --git a/packages/service/core/ai/llm/agentLoop/provider/fastAgent/loop/base.ts b/packages/service/core/ai/llm/agentLoop/provider/fastAgent/loop/base.ts index 5fd5f4bb2f43..4aaa05130897 100644 --- a/packages/service/core/ai/llm/agentLoop/provider/fastAgent/loop/base.ts +++ b/packages/service/core/ai/llm/agentLoop/provider/fastAgent/loop/base.ts @@ -4,6 +4,7 @@ import type { ChatCompletionMessageParam, ChatCompletionMessageToolCall, ChatCompletionTool, + ChatCompletionToolMessageContentPart, CompletionFinishReason } from '@fastgpt/global/core/ai/llm/type'; import { ChatCompletionRequestMessageRoleEnum } from '@fastgpt/global/core/ai/constants'; @@ -137,6 +138,23 @@ type RunAgentResponse = { finish_reason: CompletionFinishReason | undefined; }; +/** + * 构造写入 tool message 的 content： + * - 无结构化 parts 时直接用文本字符串； + * - 有 parts 时把最终要发送的文本作为 text part 拼在图片等 parts 前， + * 保证模型实际拿到的文本与展示/计费统计一致（文本已压缩时传压缩后的值）。 + */ +export const buildToolMessageContent = ({ + content, + text +}: { + content?: ChatCompletionToolMessageContentPart[]; + text: string; +}): string | ChatCompletionToolMessageContentPart[] => { + if (!content?.length) return text; + return [{ type: 'text', text }, ...content.filter((part) => part.type !== 'text')]; +}; + /** * 上下文压缩，内部会判断是否需要压缩 */ @@ -305,7 +323,8 @@ export const runAgentLoop = async ({ interactive, stop, errorMessage, - metadata + metadata, + content: toolContent } = await (async () => { try { return await onRunInteractiveTool({ @@ -338,11 +357,16 @@ export const runAgentLoop = async ({ }); // FastAgent 只使用 ChatItem 重建的上下文，旧交互中的 memory 快照不再参与恢复。 + // 与普通工具执行路径保持一致：结构化 parts 与最终文本统一通过 buildToolMessageContent 组装。 + const interactiveToolContent = buildToolMessageContent({ + content: toolContent, + text: normalizeToolResponseContent(response) + }); requestMessages = requestMessages.map((item) => item.role === 'tool' && item.tool_call_id === childrenInteractiveParams.toolCallId ? { ...item, - content: normalizeToolResponseContent(response) + content: interactiveToolContent } : item ); @@ -352,7 +376,7 @@ export const runAgentLoop = async ({ assistantMessages.push({ role: ChatCompletionRequestMessageRoleEnum.Tool, tool_call_id: childrenInteractiveParams.toolCallId, - content: response + content: interactiveToolContent }); assistantMessages.push(...filterEmptyAssistantMessages(toolAssistantMessages)); // 相同 tool 触发了多次交互, 调用的 toolId 认为是相同的 @@ -568,7 +592,8 @@ export const runAgentLoop = async ({ skipResponseCompress, errorMessage, hideInUI, - metadata + metadata, + content: toolContent } = await (async () => { try { return await onRunTool({ @@ -638,6 +663,14 @@ export const runAgentLoop = async ({ interactive }); + // 工具返回结构化 content（如含图片 parts）时，在保留图片的前提下， + // 把展示/计费用的最终文本（可能已压缩）作为 text part 一并发送； + // 否则直接使用最终文本字符串。 + const toolMessageContent = buildToolMessageContent({ + content: toolContent, + text: toolFinalResponse + }); + return { tool, interactive, @@ -645,7 +678,7 @@ export const runAgentLoop = async ({ toolMessage: { tool_call_id: tool.id, role: ChatCompletionRequestMessageRoleEnum.Tool, - content: toolFinalResponse + content: toolMessageContent } as ChatCompletionMessageParam, toolAssistantMessages: filterEmptyAssistantMessages(toolAssistantMessages) }; diff --git a/packages/service/core/ai/llm/agentLoop/provider/fastAgent/loop/index.ts b/packages/service/core/ai/llm/agentLoop/provider/fastAgent/loop/index.ts index b4971139bb68..0562d3241d04 100644 --- a/packages/service/core/ai/llm/agentLoop/provider/fastAgent/loop/index.ts +++ b/packages/service/core/ai/llm/agentLoop/provider/fastAgent/loop/index.ts @@ -375,7 +375,8 @@ export const runFastAgentMainLoop = async ({ }); return createToolResponse(sandboxResult.response, { skipResponseCompress: true, - errorMessage: sandboxResult.success ? undefined : sandboxResult.response + errorMessage: sandboxResult.success ? undefined : sandboxResult.response, + content: sandboxResult.content }); } diff --git a/packages/service/core/ai/llm/agentLoop/provider/piAgent/message.ts b/packages/service/core/ai/llm/agentLoop/provider/piAgent/message.ts index 75906dd3403c..bfb88a34d8f6 100644 --- a/packages/service/core/ai/llm/agentLoop/provider/piAgent/message.ts +++ b/packages/service/core/ai/llm/agentLoop/provider/piAgent/message.ts @@ -13,8 +13,11 @@ import type { ChatCompletionMessageParam, ChatCompletionMessageToolCall, ChatCompletionTool, + ChatCompletionToolMessageContentPart, CompletionFinishReason } from '@fastgpt/global/core/ai/llm/type'; +import { getImageBase64 } from '../../../../../../common/file/image/utils'; +import { maskImagePlaceholderUrl } from '../../../utils'; /** 将工具参数稳定序列化；不可序列化值回退为空对象，避免中断事件链。 */ export const stringifyJson = (value: unknown) => { @@ -81,12 +84,59 @@ const getContentText = ( content: | string | null - | Array<{ type: 'text'; text: string } | { type: 'refusal'; refusal: string }> + | Array< + | { type: 'text'; text: string } + | { type: 'refusal'; refusal: string } + | { type: 'image_url'; image_url: { url: string } } + > | undefined ) => { if (!content) return ''; if (typeof content === 'string') return content; - return content.map((item) => (item.type === 'text' ? item.text : item.refusal)).join(''); + return content + .map((item) => (item.type === 'text' ? item.text : item.type === 'refusal' ? item.refusal : '')) + .join(''); +}; + +/** + * 将工具返回的 content（字符串或 text / image_url parts）转换成 pi-agent 的 + * TextContent / ImageContent。图片 URL 取回 base64 后进入 transcript； + * 取图失败时降级为占位文本，避免中断 agent loop。 + * useVision 为 false 时图片 part 统一降级为占位文本，与 loadRequestMessages + * 的能力过滤语义一致，避免把图片发给不支持视觉的模型。 + */ +export const convertToolResponseContentToPiContent = async ( + content: string | ChatCompletionToolMessageContentPart[], + options?: { useVision?: boolean } +): Promise> => { + const useVision = options?.useVision ?? true; + if (typeof content === 'string') { + return [{ type: 'text', text: content }]; + } + + return Promise.all( + content.map(async (part): Promise => { + if (part.type === 'text') { + return { type: 'text', text: part.text }; + } + + const url = part.image_url.url; + if (!useVision) { + return { type: 'text', text: `[Image: ${maskImagePlaceholderUrl(url)}]` }; + } + const dataMatch = url.match(/^data:([^;]+);base64,(.+)$/); + if (dataMatch) { + return { type: 'image', mimeType: dataMatch[1], data: dataMatch[2] }; + } + + try { + const { base64, mime } = await getImageBase64(url); + return { type: 'image', mimeType: mime, data: base64 }; + } catch { + return { type: 'text', text: `[Image: ${maskImagePlaceholderUrl(url)}]` }; + } + }) + ); }; /** @@ -171,12 +221,32 @@ export const convertChatMessagesToPiAgentMessages = ({ } if (message.role === ChatCompletionRequestMessageRoleEnum.Tool) { + // 工具消息允许 text / image_url parts：data URI 直接还原为 ImageContent， + // 远程图片 URL 与 user 分支一致降级为占位文本，不阻塞转录恢复。 + const content: Array = []; + if (Array.isArray(message.content)) { + message.content.forEach((part) => { + if (part.type === 'text') { + content.push({ type: 'text', text: part.text }); + return; + } + const match = part.image_url.url.match(/^data:([^;]+);base64,(.+)$/); + if (match) { + content.push({ type: 'image', mimeType: match[1], data: match[2] }); + return; + } + content.push({ type: 'text', text: `[Image: ${part.image_url.url}]` }); + }); + } else { + content.push({ type: 'text', text: getContentText(message.content) }); + } + return [ { role: 'toolResult', toolCallId: message.tool_call_id, toolName: message.name ?? toolNames.get(message.tool_call_id) ?? '', - content: [{ type: 'text', text: getContentText(message.content) }], + content, details: {}, isError: false, timestamp: Date.now() @@ -295,18 +365,24 @@ export const replaceInteractiveToolResult = ({ messages, call, response, - isError + isError, + content }: { messages: AgentMessage[]; call: ChatCompletionMessageToolCall; response: string; isError: boolean; + /** 工具返回的图片 content（已转成 pi ImageContent），错误时不生效。 */ + content?: Array; }): AgentMessage[] => { + const nextContent = + !isError && content?.length ? content : [{ type: 'text' as const, text: response }]; + let replaced = false; const nextMessages = messages.map((message) => { if (message.role !== 'toolResult' || message.toolCallId !== call.id || replaced) return message; replaced = true; - return { ...message, content: [{ type: 'text' as const, text: response }], isError }; + return { ...message, content: nextContent, isError }; }); if (replaced) return nextMessages; @@ -316,7 +392,7 @@ export const replaceInteractiveToolResult = ({ role: 'toolResult', toolCallId: call.id, toolName: call.function.name, - content: [{ type: 'text', text: response }], + content: nextContent, details: {}, isError, timestamp: Date.now() diff --git a/packages/service/core/ai/llm/agentLoop/provider/piAgent/run.ts b/packages/service/core/ai/llm/agentLoop/provider/piAgent/run.ts index 3e5494631ae8..f40e252d7c47 100644 --- a/packages/service/core/ai/llm/agentLoop/provider/piAgent/run.ts +++ b/packages/service/core/ai/llm/agentLoop/provider/piAgent/run.ts @@ -2,6 +2,7 @@ import { Agent, type AgentEvent } from '@mariozechner/pi-agent-core'; import type { ChatCompletionMessageParam, ChatCompletionMessageToolCall, + ChatCompletionToolMessageContentPart, CompletionFinishReason } from '@fastgpt/global/core/ai/llm/type'; import { ChatCompletionRequestMessageRoleEnum } from '@fastgpt/global/core/ai/constants'; @@ -33,6 +34,7 @@ import { compressRequestMessages } from '../../../compress'; import { convertChatMessagesToPiAgentMessages, convertPiAgentMessagesToChatMessages, + convertToolResponseContentToPiContent, getPiAgentPrompt, isAssistantMessage, mapStopReason, @@ -65,6 +67,7 @@ export const runPiAgentLoop = async ({ }): Promise> => { const state = readPiAgentProviderState(input.providerState); const modelData = runtime.llmParams.model; + const useVision = runtime.llmParams.useVision && modelData.config.vision; const piModel = buildPiModel( modelData, runtime.llmParams.useVision, @@ -73,7 +76,7 @@ export const runPiAgentLoop = async ({ ); const requestMessages = await loadRequestMessages({ messages: input.messages, - useVision: runtime.llmParams.useVision && modelData.config.vision, + useVision, useAudio: runtime.llmParams.useAudio && modelData.config.audio, useVideo: runtime.llmParams.useVideo && modelData.config.video, extractFiles: runtime.llmParams.extractFiles, @@ -170,17 +173,20 @@ export const runPiAgentLoop = async ({ call, response, childAssistantMessages = [], - appendToAssistantMessages = true + appendToAssistantMessages = true, + content }: { call: ChatCompletionMessageToolCall; response: string; childAssistantMessages?: ChatCompletionMessageParam[]; appendToAssistantMessages?: boolean; + /** 工具返回的结构化 content（text / image_url parts），缺省时回退 response 字符串。 */ + content?: string | ChatCompletionToolMessageContentPart[]; }) => { const toolMessage: ChatCompletionMessageParam = { role: ChatCompletionRequestMessageRoleEnum.Tool, tool_call_id: call.id, - content: response + content: content ?? response }; const existingToolIndex = completeMessages.findIndex( (message) => message.role === 'tool' && message.tool_call_id === call.id @@ -319,11 +325,14 @@ export const runPiAgentLoop = async ({ metadata: result.metadata, seconds: +((Date.now() - startedAt) / 1000).toFixed(2) }); + const toolContent = result.content ?? normalizeToolResponseContent(result.response); + const piToolContent = await convertToolResponseContentToPiContent(toolContent, { useVision }); initialPiMessages = replaceInteractiveToolResult({ messages: initialPiMessages, call, response: normalizedResponse, - isError: !!result.errorMessage + isError: !!result.errorMessage, + content: piToolContent }); const providerState: PiAgentProviderState = { @@ -334,7 +343,8 @@ export const runPiAgentLoop = async ({ recordToolResult({ call, response: normalizedResponse, - childAssistantMessages: result.assistantMessages + childAssistantMessages: result.assistantMessages, + content: toolContent }); if (result.interactive) { return { @@ -397,11 +407,12 @@ export const runPiAgentLoop = async ({ }, getMessages: () => completeMessages, onToolCall: emitOrdinaryToolExecution, - onToolResult: ({ call, response, assistantMessages }) => { + onToolResult: ({ call, response, assistantMessages, content }) => { recordToolResult({ call, response, - childAssistantMessages: assistantMessages + childAssistantMessages: assistantMessages, + content }); } }); diff --git a/packages/service/core/ai/llm/agentLoop/provider/piAgent/tool/build.ts b/packages/service/core/ai/llm/agentLoop/provider/piAgent/tool/build.ts index 51a6e543fae8..b3b48cab0447 100644 --- a/packages/service/core/ai/llm/agentLoop/provider/piAgent/tool/build.ts +++ b/packages/service/core/ai/llm/agentLoop/provider/piAgent/tool/build.ts @@ -5,7 +5,8 @@ import { AgentAskPayloadSchema, type AgentAskPayload } from '@fastgpt/global/cor import { normalizeToolResponseContent } from '@fastgpt/global/core/ai/llm/utils'; import type { ChatCompletionMessageParam, - ChatCompletionMessageToolCall + ChatCompletionMessageToolCall, + ChatCompletionToolMessageContentPart } from '@fastgpt/global/core/ai/llm/type'; import { getErrText } from '@fastgpt/global/common/error/utils'; import { createAskUserAgentTool } from '../../../domain/systemTool/ask'; @@ -29,7 +30,12 @@ import { type AgentLoopUsage } from '../../../domain'; import { runSandboxTools } from '../../../../../sandbox/application/toolCall'; -import { createToolCall, normalizeToolArgs, stringifyJson } from '../message'; +import { + convertToolResponseContentToPiContent, + createToolCall, + normalizeToolArgs, + stringifyJson +} from '../message'; import { getPiAgentRuntimeTools } from './catalog'; type PlanOperationEvent = Extract; @@ -85,10 +91,14 @@ export const buildPiAgentTools = async ({ call: ChatCompletionMessageToolCall; response: string; assistantMessages?: ChatCompletionMessageParam[]; + content?: string | ChatCompletionToolMessageContentPart[]; }) => void; }): Promise => { const tools: AgentTool[] = []; + // 与 loadRequestMessages 对齐：模型不支持视觉时，工具图片 part 降级为占位文本。 + const useVision = runtime.llmParams.useVision && runtime.llmParams.model.config.vision; + /** 保证每个普通工具 start 都对应 end，并将异常变成可回填模型的工具结果。 */ const executeOrdinaryTool = async ({ call, @@ -97,6 +107,7 @@ export const buildPiAgentTools = async ({ call: ChatCompletionMessageToolCall; execute: () => Promise<{ response: string; + content?: ChatCompletionToolMessageContentPart[]; assistantMessages?: ChatCompletionMessageParam[]; assistantResponses?: AgentLoopAssistantResponse[]; usages?: AgentLoopUsage[]; @@ -135,8 +146,18 @@ export const buildPiAgentTools = async ({ metadata: result.metadata, seconds: +((Date.now() - startedAt) / 1000).toFixed(2) }); - onToolResult({ call, response: normalizedResponse, assistantMessages }); - return { ...result, response: normalizedResponse, assistantMessages, usages }; + // 工具返回结构化 content 时优先透传；否则用规范化后的 response 字符串兜底 + const toolContent = result.content?.length + ? result.content + : normalizeToolResponseContent(result.response); + onToolResult({ call, response: normalizedResponse, assistantMessages, content: toolContent }); + return { + ...result, + response: normalizedResponse, + assistantMessages, + usages, + content: toolContent + }; }; for (const tool of getPiAgentRuntimeTools(runtime)) { @@ -159,7 +180,13 @@ export const buildPiAgentTools = async ({ } else if (result.stop) { onToolStop?.(); } - return { content: [{ type: 'text' as const, text: result.response }], details: {} }; + return { + content: await convertToolResponseContentToPiContent( + result.content?.length ? result.content : normalizeToolResponseContent(result.response), + { useVision } + ), + details: {} + }; } }); } @@ -301,11 +328,18 @@ export const buildPiAgentTools = async ({ response: sandboxResult.response, assistantMessages: [], usages: [], - errorMessage: sandboxResult.success ? undefined : sandboxResult.response + errorMessage: sandboxResult.success ? undefined : sandboxResult.response, + content: sandboxResult.content }; } }); - return { content: [{ type: 'text' as const, text: result.response }], details: {} }; + return { + content: await convertToolResponseContentToPiContent( + result.content ?? normalizeToolResponseContent(result.response), + { useVision } + ), + details: {} + }; } }); } @@ -344,7 +378,13 @@ export const buildPiAgentTools = async ({ }; } }); - return { content: [{ type: 'text' as const, text: result.response }], details: {} }; + return { + content: await convertToolResponseContentToPiContent( + result.content?.length ? result.content : normalizeToolResponseContent(result.response), + { useVision } + ), + details: {} + }; } }); } @@ -389,7 +429,13 @@ export const buildPiAgentTools = async ({ }; } }); - return { content: [{ type: 'text' as const, text: result.response }], details: {} }; + return { + content: await convertToolResponseContentToPiContent( + result.content?.length ? result.content : normalizeToolResponseContent(result.response), + { useVision } + ), + details: {} + }; } }); } diff --git a/packages/service/core/ai/llm/promptCall/index.ts b/packages/service/core/ai/llm/promptCall/index.ts index 8bac89871c80..76fb5ee29a40 100644 --- a/packages/service/core/ai/llm/promptCall/index.ts +++ b/packages/service/core/ai/llm/promptCall/index.ts @@ -7,6 +7,7 @@ import type { ChatCompletionTool } from '@fastgpt/global/core/ai/llm/type'; import { getPromptToolCallPrompt } from './prompt'; +import { maskImagePlaceholderUrl } from '../utils'; import { cloneDeep } from 'lodash-es'; export const promptToolCallMessageRewrite = ( @@ -56,9 +57,25 @@ export const promptToolCallMessageRewrite = ( delete message.tool_calls; } } else if (message.role === 'tool') { + // 工具消息可能是字符串、text/image_url parts 或空值；纯图片时保留占位文本，避免响应静默丢失。 + const toolContent = + typeof message.content === 'string' + ? message.content + : Array.isArray(message.content) + ? message.content + .map((part) => { + if (part.type === 'text' && part.text) return part.text; + if (part.type === 'image_url' && part.image_url?.url) { + return `[Image: ${maskImagePlaceholderUrl(part.image_url.url)}]`; + } + return ''; + }) + .filter(Boolean) + .join('\n') + : String(message.content ?? ''); cloneMessages.splice(i, 1, { role: 'user', - content: `\n${message.content}\n` + content: `\n${toolContent}\n` }); } } diff --git a/packages/service/core/ai/llm/utils.ts b/packages/service/core/ai/llm/utils.ts index a2e5bb3073a8..174a95535a26 100644 --- a/packages/service/core/ai/llm/utils.ts +++ b/packages/service/core/ai/llm/utils.ts @@ -37,6 +37,26 @@ import { const logger = getLogger(LogCategories.MODULE.AI.LLM); +/** + * 判断 URL 是否为沙箱文件预览地址。 + * + * 预览地址带一次性 session 凭证且只在服务端网络拓扑内可达，不适合直接透传给模型服务商。 + * 这里按 env 里的代理前缀识别（与 sandbox 层 buildSandboxPreviewFileUrl 的 `/preview/` 契约一致）， + * 避免把 sandbox 模块及其 dal/redis 依赖拖进 llm 模块。 + */ +export const isSandboxPreviewFileUrl = (url: string) => { + const base = serviceEnv.AGENT_SANDBOX_PREVIEW_PROXY_URL; + if (!base) return false; + return url.startsWith(`${base.replace(/\/+$/, '')}/preview/`); +}; + +/** + * 图片占位文本里的 URL 可能是带凭证的预览地址，写入提示词前先脱敏， + * 避免凭证经纯文本路径外发给模型服务商。 + */ +export const maskImagePlaceholderUrl = (url: string) => + isSandboxPreviewFileUrl(url) ? 'sandbox preview file' : url; + /** * 判断消息是否属于系统级提示。 * 这类消息不参与普通会话窗口裁剪，避免 system/developer 指令被上下文截断。 @@ -271,6 +291,38 @@ export const loadRequestMessages = async ({ const stripSystemReminder = (input: string) => input.replace(/[\s\S]*?<\/system-reminder>\s*/g, ''); + /** + * 预览图片在发送前由服务端下载并转成 data URI。 + * 下载失败时丢弃该图片 part，避免把带凭证的预览地址透传给模型服务商。 + */ + const inlineSandboxPreviewImage = async ( + part: ChatCompletionContentPart + ): Promise => { + if ( + part.type !== 'image_url' || + !part.image_url.url || + !isSandboxPreviewFileUrl(part.image_url.url) + ) { + return part; + } + + try { + const { completeBase64: base64 } = await getImageBase64(part.image_url.url); + return { + ...part, + image_url: { + ...part.image_url, + url: base64 + } + }; + } catch (error) { + logger.warn('Failed to inline sandbox preview image, drop the image part', { + error: getErrText(error) + }); + return undefined; + } + }; + /** * 从短文本中提取候选 HTTP(S) 链接并做尾部标点与 Markdown 语法清洗。 * 具体是否为媒体文件交由统一分类入口 classifyUrlMediaType 判断，支持无后缀短链探测。 @@ -727,12 +779,63 @@ export const loadRequestMessages = async ({ ...formatAssistantItem(item, supportReason), content: formatContent }; + } else if (item.role === ChatCompletionRequestMessageRoleEnum.Tool) { + // 供应商接口的 tool content 只接受字符串或文本 parts，图片不能放进 tool 消息。 + // 文本保留在 tool 消息里；图片先按模型能力过滤、再归一化， + // 并转换成紧随 tool 消息之后的 user 视觉消息。 + if (Array.isArray(item.content)) { + const supportedContent = item.content + .map((part) => filterSupportedUserContentPart(part as ChatCompletionContentPart)) + .filter(Boolean) as ChatCompletionContentPart[]; + + const textParts = supportedContent.filter( + (part): part is ChatCompletionContentPartText => part.type === 'text' + ); + const mediaParts = supportedContent.filter((part) => part.type !== 'text'); + + const normalizedTextParts = (await normalizeMediaContentParts( + textParts + )) as ChatCompletionContentPartText[]; + const toolContent = + normalizedTextParts.length === 0 + ? 'none' + : normalizedTextParts.length === 1 + ? normalizedTextParts[0].text + : normalizedTextParts; + + const normalizedMediaParts = await normalizeMediaContentParts( + (await Promise.all(mediaParts.map(inlineSandboxPreviewImage))).filter( + (part): part is ChatCompletionContentPart => Boolean(part) + ) + ); + if (normalizedMediaParts.length === 0) { + return { + ...item, + content: toolContent + }; + } + + return [ + { + ...item, + content: toolContent + }, + { + role: ChatCompletionRequestMessageRoleEnum.User, + content: normalizedMediaParts + } as ChatCompletionMessageParam + ]; + } + + return item; } else { return item; } }) ) - ).filter(Boolean) as ChatCompletionMessageParam[]; + ) + .flat() + .filter(Boolean) as ChatCompletionMessageParam[]; return loadMessages; }; diff --git a/packages/service/core/ai/sandbox/application/toolCall/getFileUrl.tool.ts b/packages/service/core/ai/sandbox/application/toolCall/getFileUrl.tool.ts index 54266dd65261..60e3e6437c2c 100644 --- a/packages/service/core/ai/sandbox/application/toolCall/getFileUrl.tool.ts +++ b/packages/service/core/ai/sandbox/application/toolCall/getFileUrl.tool.ts @@ -11,11 +11,21 @@ import { createSandboxPreviewSession, resolveSandboxPreviewPath } from '../preview'; +import { imageFileType } from '@fastgpt/global/common/file/constants'; +import type { ChatCompletionToolMessageContentPart } from '@fastgpt/global/core/ai/llm/type'; const SandboxGetFileUrlToolSchema = z.object({ paths: z.array(z.string()) }); +const isImageFilePath = (filePath: string) => { + const extension = `.${path.posix.extname(filePath).slice(1).toLowerCase()}`; + return ( + extension !== '.' && + imageFileType.split(',').some((item) => item.trim().toLowerCase() === extension) + ); +}; + export const sandboxGetFileUrlTool = defineTool({ zodSchema: SandboxGetFileUrlToolSchema, execute: async ({ sandboxInstance, params }) => { @@ -66,6 +76,21 @@ export const sandboxGetFileUrlTool = defineTool({ filename: path.posix.basename(relativePath) })); - return { response: JSON.stringify(result) }; + // content 必须以承载原 response 的 text part 打头（保留全部文件的 filename/fileUrl， + // 供下一轮压缩与展示使用），图片文件再以 image_url part 追加作为视觉输入。 + const content: ChatCompletionToolMessageContentPart[] = [ + { type: 'text', text: JSON.stringify(result) }, + ...result + .filter(({ filename }) => isImageFilePath(filename)) + .map(({ fileUrl }) => ({ + type: 'image_url', + image_url: { url: fileUrl } + })) + ]; + + return { + response: JSON.stringify(result), + ...(content.length ? { content } : {}) + }; } }); diff --git a/packages/service/core/ai/sandbox/application/toolCall/index.ts b/packages/service/core/ai/sandbox/application/toolCall/index.ts index da41dfd09f34..95d9c1250c4f 100644 --- a/packages/service/core/ai/sandbox/application/toolCall/index.ts +++ b/packages/service/core/ai/sandbox/application/toolCall/index.ts @@ -32,6 +32,7 @@ import { preparePackageMirrors, prepareSandbox } from '../runtime/prepare'; import type { ChatSourceTypeEnum } from '@fastgpt/global/core/chat/constants'; import { getRunningSandboxId, getSandboxUserId } from '../../utils/id'; import { createToolRunner, type SandboxToolCallValidator } from './type'; +import type { ChatCompletionToolMessageContentPart } from '@fastgpt/global/core/ai/llm/type'; const ToolMap = { [SANDBOX_EDIT_FILE_TOOL_NAME]: createToolRunner(sandboxEditFileTool), @@ -49,6 +50,8 @@ export type SandboxToolCallResult = { input: Record; response: string; durationSeconds: number; + /** 工具主动返回的结构化 content（text / image_url parts）。 */ + content?: ChatCompletionToolMessageContentPart[]; }; /** @@ -110,7 +113,8 @@ export const runSandboxTools = async ({ success: true, input: result.input, response: result.response, - durationSeconds: getDuration() + durationSeconds: getDuration(), + ...(result.content?.length ? { content: result.content } : {}) }; }; diff --git a/packages/service/core/ai/sandbox/application/toolCall/type.ts b/packages/service/core/ai/sandbox/application/toolCall/type.ts index 5f501b03c0fa..b1d07f9403e0 100644 --- a/packages/service/core/ai/sandbox/application/toolCall/type.ts +++ b/packages/service/core/ai/sandbox/application/toolCall/type.ts @@ -5,6 +5,7 @@ */ import type { z } from 'zod'; import type { SandboxClient } from '../runtime/client'; +import type { ChatCompletionToolMessageContentPart } from '@fastgpt/global/core/ai/llm/type'; /** 工具执行前的业务校验；返回拒绝原因时不访问沙箱。 */ export type SandboxToolCallValidator = (params: { @@ -21,12 +22,15 @@ type ToolExecuteContext

= { * sandbox 工具的声明式定义。 * * zodSchema 负责约束 LLM 工具参数，execute 只接收已校验的 params 和运行态 SandboxClient。 + * 工具可返回结构化 content（text / image_url parts）作为下一轮模型的视觉输入。 */ export type ToolDefinition< S extends z.ZodType> = z.ZodType> > = { zodSchema: S; - execute: (ctx: ToolExecuteContext>) => Promise<{ response: string }>; + execute: ( + ctx: ToolExecuteContext> + ) => Promise<{ response: string; content?: ChatCompletionToolMessageContentPart[] }>; }; export type ToolRunResult = @@ -35,6 +39,7 @@ export type ToolRunResult = success: true; input: Record; response: string; + content?: ChatCompletionToolMessageContentPart[]; }; /** @@ -61,6 +66,7 @@ export const createToolRunner = return { success: true, input: parsed.data, - response: result.response + response: result.response, + ...(result.content?.length ? { content: result.content } : {}) }; }; diff --git a/packages/service/core/workflow/dispatch/ai/agentLoopCore/application/runtime/workflowToolRunner.ts b/packages/service/core/workflow/dispatch/ai/agentLoopCore/application/runtime/workflowToolRunner.ts index 1f4ccad846c0..a264b87e7eca 100644 --- a/packages/service/core/workflow/dispatch/ai/agentLoopCore/application/runtime/workflowToolRunner.ts +++ b/packages/service/core/workflow/dispatch/ai/agentLoopCore/application/runtime/workflowToolRunner.ts @@ -3,7 +3,8 @@ import { chats2GPTMessages } from '@fastgpt/global/core/chat/adapt'; import type { ChatHistoryItemResType } from '@fastgpt/global/core/chat/type'; import type { ChatCompletionMessageParam, - ChatCompletionMessageToolCall + ChatCompletionMessageToolCall, + ChatCompletionToolMessageContentPart } from '@fastgpt/global/core/ai/llm/type'; import type { RuntimeEdgeItemType } from '@fastgpt/global/core/workflow/type/edge'; import type { RuntimeNodeItemType } from '@fastgpt/global/core/workflow/runtime/type'; @@ -102,6 +103,39 @@ export const formatAgentLoopCoreToolResponse = (toolResponses: any) => { return toolResponses ? String(toolResponses) : 'none'; }; +/** + * 从工具节点输出中提取结构化 tool content。 + * + * 仅接受形如 { content: [{ type: 'text' | 'image_url', ... }, ...] } 的对象输出， + * 且 content 非空、每项都是合法的 text/image_url part 时才返回，否则 undefined。 + * 工具可以通过该形状主动返回图片，作为下一轮模型的视觉输入；不在普通文本里猜测图片。 + */ +export const extractAgentLoopCoreToolContent = ( + toolResponses: any +): ChatCompletionToolMessageContentPart[] | undefined => { + if (!toolResponses || typeof toolResponses !== 'object' || Array.isArray(toolResponses)) { + return; + } + + const content = toolResponses.content; + if (!Array.isArray(content) || content.length === 0) return; + + const isValidPart = (part: any): part is ChatCompletionToolMessageContentPart => { + if (!part || typeof part !== 'object') return false; + if (part.type === 'text') return typeof part.text === 'string' && part.text.length > 0; + if (part.type === 'image_url') { + return ( + !!part.image_url && typeof part.image_url.url === 'string' && part.image_url.url.length > 0 + ); + } + return false; + }; + + return content.every(isValidPart) + ? (content as ChatCompletionToolMessageContentPart[]) + : undefined; +}; + /** * 初始化阶段直接在传入的 runtime edge/node 上标记入口状态。 * 调用方需要根据自身的状态隔离要求决定是否先创建副本。 @@ -174,6 +208,7 @@ const toToolRunResult = ( const rawToolResponses = toolRunResponse.toolResponses; const hasToolResponses = rawToolResponses !== undefined && rawToolResponses !== null && rawToolResponses !== ''; + const content = extractAgentLoopCoreToolContent(rawToolResponses); return { result: { @@ -183,6 +218,7 @@ const toToolRunResult = ( ? formatAgentLoopCoreToolResponse(rawToolResponses) : errorMessage, ...(errorMessage ? { errorMessage } : {}), + ...(content ? { content } : {}), assistantMessages: getAssistantMessages(toolRunResponse.assistantResponses), ...(toolRunResponse.assistantResponses?.length ? { assistantResponses: toolRunResponse.assistantResponses } diff --git a/packages/service/core/workflow/dispatch/ai/agentLoopCore/domain/toolProvider.ts b/packages/service/core/workflow/dispatch/ai/agentLoopCore/domain/toolProvider.ts index f8ce91fb647b..625b20118abb 100644 --- a/packages/service/core/workflow/dispatch/ai/agentLoopCore/domain/toolProvider.ts +++ b/packages/service/core/workflow/dispatch/ai/agentLoopCore/domain/toolProvider.ts @@ -1,7 +1,8 @@ import type { ChatCompletionMessageParam, ChatCompletionMessageToolCall, - ChatCompletionTool + ChatCompletionTool, + ChatCompletionToolMessageContentPart } from '@fastgpt/global/core/ai/llm/type'; import type { AIChatItemValueItemType, @@ -41,6 +42,8 @@ export type AgentLoopCoreToolRunResult = { errorMessage?: string; nodeResponse?: ChatHistoryItemResType; nodeSummary?: NodeSummary; + /** 写入 LLM tool message 的结构化 content（text / image_url parts）。 */ + content?: ChatCompletionToolMessageContentPart[]; }; export type AgentLoopCoreToolProvider = { @@ -68,7 +71,8 @@ export const normalizeAgentLoopCoreToolRunResult = interactive, stop = false, errorMessage, - nodeResponse + nodeResponse, + content }: AgentLoopCoreToolRunResult): AgentLoopToolExecutionResult => { return { response, @@ -78,6 +82,7 @@ export const normalizeAgentLoopCoreToolRunResult = interactive, stop, errorMessage, - metadata: nodeResponse + metadata: nodeResponse, + content }; }; diff --git a/packages/service/test/core/ai/llm/agentLoop/fastAgentLoop.test.ts b/packages/service/test/core/ai/llm/agentLoop/fastAgentLoop.test.ts index 7ca21497f783..7c3d779f4d7c 100644 --- a/packages/service/test/core/ai/llm/agentLoop/fastAgentLoop.test.ts +++ b/packages/service/test/core/ai/llm/agentLoop/fastAgentLoop.test.ts @@ -3,7 +3,10 @@ import { ChatCompletionRequestMessageRoleEnum, ModelTypeEnum } from '@fastgpt/global/core/ai/constants'; -import type { ChatCompletionTool } from '@fastgpt/global/core/ai/llm/type'; +import type { + ChatCompletionTool, + ChatCompletionToolMessageContentPart +} from '@fastgpt/global/core/ai/llm/type'; import { beforeEach, describe, expect, it, vi } from 'vitest'; import { mockCreateLLMResponseQueue, text, toolCall } from './_mocks/llmQueue'; @@ -61,6 +64,7 @@ import { runFastAgentMainLoop, type AgentLoopRuntime } from '@fastgpt/service/core/ai/llm/agentLoop/provider/fastAgent/loop'; +import { buildToolMessageContent } from '@fastgpt/service/core/ai/llm/agentLoop/provider/fastAgent/loop/base'; const getFinalAssistantText = (result: { assistantMessages: any[] }) => result.assistantMessages @@ -865,3 +869,33 @@ describe('runFastAgentMainLoop', () => { ); }); }); + +describe('buildToolMessageContent', () => { + it('returns the final text when no content parts are provided', () => { + expect(buildToolMessageContent({ text: 'summary' })).toBe('summary'); + expect(buildToolMessageContent({ content: [], text: 'summary' })).toBe('summary'); + }); + + it('prepends the final text and keeps image parts', () => { + const content: ChatCompletionToolMessageContentPart[] = [ + { type: 'image_url', image_url: { url: 'data:image/png;base64,aW1hZ2U=' } } + ]; + + expect(buildToolMessageContent({ content, text: 'summary' })).toEqual([ + { type: 'text', text: 'summary' }, + { type: 'image_url', image_url: { url: 'data:image/png;base64,aW1hZ2U=' } } + ]); + }); + + it('replaces raw text parts with the final text', () => { + const content: ChatCompletionToolMessageContentPart[] = [ + { type: 'text', text: 'raw uncompressed output' }, + { type: 'image_url', image_url: { url: 'data:image/png;base64,aW1hZ2U=' } } + ]; + + expect(buildToolMessageContent({ content, text: 'summary' })).toEqual([ + { type: 'text', text: 'summary' }, + { type: 'image_url', image_url: { url: 'data:image/png;base64,aW1hZ2U=' } } + ]); + }); +}); diff --git a/packages/service/test/core/ai/llm/agentLoop/piAgentMessage.test.ts b/packages/service/test/core/ai/llm/agentLoop/piAgentMessage.test.ts index fd7dd10981c8..ca12b1e20059 100644 --- a/packages/service/test/core/ai/llm/agentLoop/piAgentMessage.test.ts +++ b/packages/service/test/core/ai/llm/agentLoop/piAgentMessage.test.ts @@ -1,13 +1,23 @@ -import { describe, expect, it } from 'vitest'; +import { describe, expect, it, vi } from 'vitest'; +import { serviceEnv } from '@fastgpt/service/env'; + +const previewProxyEnv: { AGENT_SANDBOX_PREVIEW_PROXY_URL?: string | undefined } = serviceEnv; import { convertChatMessagesToPiAgentMessages, convertPiAgentMessagesToChatMessages, + convertToolResponseContentToPiContent, mapStopReason, normalizePiAgentMessages, replaceInteractiveToolResult, resolveInteractiveToolCall } from '@fastgpt/service/core/ai/llm/agentLoop/provider/piAgent/message'; +vi.mock('@fastgpt/service/common/file/image/utils', () => ({ + getImageBase64: vi.fn() +})); + +import { getImageBase64 } from '@fastgpt/service/common/file/image/utils'; + const model = { api: 'openai-completions', provider: 'openai', @@ -323,6 +333,77 @@ describe('piAgent message conversion', () => { ]); }); + it('converts tool response parts to pi content and degrades images when vision is disabled', async () => { + const parts = [ + { type: 'text', text: 'ok' }, + { type: 'image_url', image_url: { url: 'data:image/png;base64,aW1hZ2U=' } } + ]; + + const withVision = await convertToolResponseContentToPiContent(parts as any); + expect(withVision).toEqual([ + { type: 'text', text: 'ok' }, + { type: 'image', mimeType: 'image/png', data: 'aW1hZ2U=' } + ]); + + const withoutVision = await convertToolResponseContentToPiContent(parts as any, { + useVision: false + }); + expect(withoutVision).toEqual([ + { type: 'text', text: 'ok' }, + { type: 'text', text: '[Image: data:image/png;base64,aW1hZ2U=]' } + ]); + }); + + it('masks sandbox preview urls in image placeholders when vision is disabled', async () => { + const original = previewProxyEnv.AGENT_SANDBOX_PREVIEW_PROXY_URL; + previewProxyEnv.AGENT_SANDBOX_PREVIEW_PROXY_URL = 'https://sandbox-preview.example.com'; + try { + const result = await convertToolResponseContentToPiContent( + [ + { + type: 'image_url', + image_url: { + url: 'https://sandbox-preview.example.com/preview/sandbox-1/session-1/a.png' + } + } + ] as any, + { useVision: false } + ); + + expect(result).toEqual([{ type: 'text', text: '[Image: sandbox preview file]' }]); + } finally { + previewProxyEnv.AGENT_SANDBOX_PREVIEW_PROXY_URL = original; + } + }); + + it('masks sandbox preview urls when image download fails', async () => { + const original = previewProxyEnv.AGENT_SANDBOX_PREVIEW_PROXY_URL; + previewProxyEnv.AGENT_SANDBOX_PREVIEW_PROXY_URL = 'https://sandbox-preview.example.com'; + vi.mocked(getImageBase64).mockRejectedValue(new Error('download failed')); + try { + const result = await convertToolResponseContentToPiContent( + [ + { + type: 'image_url', + image_url: { + url: 'https://sandbox-preview.example.com/preview/sandbox-1/session-1/a.png' + } + } + ] as any, + { useVision: true } + ); + + expect(result).toEqual([{ type: 'text', text: '[Image: sandbox preview file]' }]); + } finally { + previewProxyEnv.AGENT_SANDBOX_PREVIEW_PROXY_URL = original; + } + }); + + it('converts string tool responses to a single text part', async () => { + const result = await convertToolResponseContentToPiContent('plain'); + expect(result).toEqual([{ type: 'text', text: 'plain' }]); + }); + it('maps pi stop reasons to the shared agent-loop finish reasons', () => { expect(mapStopReason('toolUse')).toBe('tool_calls'); expect(mapStopReason('length')).toBe('length'); diff --git a/packages/service/test/core/ai/llm/toolCall.test.ts b/packages/service/test/core/ai/llm/toolCall.test.ts index d7668459c073..303e89af5e48 100644 --- a/packages/service/test/core/ai/llm/toolCall.test.ts +++ b/packages/service/test/core/ai/llm/toolCall.test.ts @@ -7,6 +7,9 @@ import type { ChatCompletionTool } from '@fastgpt/global/core/ai/llm/type'; import { describe, expect, it } from 'vitest'; +import { serviceEnv } from '@fastgpt/service/env'; + +const previewProxyEnv: { AGENT_SANDBOX_PREVIEW_PROXY_URL?: string | undefined } = serviceEnv; describe('parsePromptToolCall function tests', () => { describe('Basic scenarios', () => { @@ -703,6 +706,90 @@ describe('promptToolCallMessageRewrite function tests', () => { '\n{"result":"success","data":[1,2,3]}\n' ); }); + + it('should handle tool message with null content without throwing', () => { + const messages: ChatCompletionMessageParam[] = [ + { role: 'user', content: 'Test' }, + { role: 'tool', tool_call_id: 'call_123', content: null as unknown as string } + ]; + const tools: ChatCompletionTool[] = []; + + const result = promptToolCallMessageRewrite(messages, tools); + + expect(result[2].role).toBe('user'); + expect(result[2].content).toBe('\n\n'); + }); + + it('should preserve image-only tool content as placeholder text', () => { + const messages: ChatCompletionMessageParam[] = [ + { role: 'user', content: 'Test' }, + { + role: 'tool', + tool_call_id: 'call_123', + content: [{ type: 'image_url', image_url: { url: 'https://files.example.com/plot.png' } }] + } + ]; + const tools: ChatCompletionTool[] = []; + + const result = promptToolCallMessageRewrite(messages, tools); + + expect(result[2].role).toBe('user'); + expect(result[2].content).toBe( + '\n[Image: https://files.example.com/plot.png]\n' + ); + }); + + it('should mask sandbox preview urls in tool image placeholders', () => { + const original = previewProxyEnv.AGENT_SANDBOX_PREVIEW_PROXY_URL; + previewProxyEnv.AGENT_SANDBOX_PREVIEW_PROXY_URL = 'https://sandbox-preview.example.com'; + try { + const messages: ChatCompletionMessageParam[] = [ + { role: 'user', content: 'Test' }, + { + role: 'tool', + tool_call_id: 'call_123', + content: [ + { + type: 'image_url', + image_url: { + url: 'https://sandbox-preview.example.com/preview/sandbox-1/session-1/a.png' + } + } + ] + } + ]; + const tools: ChatCompletionTool[] = []; + + const result = promptToolCallMessageRewrite(messages, tools); + + expect(result[2].content).toBe( + '\n[Image: sandbox preview file]\n' + ); + } finally { + previewProxyEnv.AGENT_SANDBOX_PREVIEW_PROXY_URL = original; + } + }); + + it('should join text and image parts in tool content', () => { + const messages: ChatCompletionMessageParam[] = [ + { role: 'user', content: 'Test' }, + { + role: 'tool', + tool_call_id: 'call_123', + content: [ + { type: 'text', text: 'Generated plot:' }, + { type: 'image_url', image_url: { url: 'https://files.example.com/plot.png' } } + ] + } + ]; + const tools: ChatCompletionTool[] = []; + + const result = promptToolCallMessageRewrite(messages, tools); + + expect(result[2].content).toBe( + '\nGenerated plot:\n[Image: https://files.example.com/plot.png]\n' + ); + }); }); describe('Message immutability', () => { diff --git a/packages/service/test/core/ai/llm/utils.test.ts b/packages/service/test/core/ai/llm/utils.test.ts index 66c4fc02eae9..9b5240dbaa28 100644 --- a/packages/service/test/core/ai/llm/utils.test.ts +++ b/packages/service/test/core/ai/llm/utils.test.ts @@ -1,6 +1,8 @@ import { loadRequestMessages, - filterGPTMessageByMaxContext + filterGPTMessageByMaxContext, + isSandboxPreviewFileUrl, + maskImagePlaceholderUrl } from '@fastgpt/service/core/ai/llm/utils'; import { ChatCompletionContentPartInputAudioSchema, @@ -49,12 +51,17 @@ const mockGetImageBase64 = vi.mocked(getImageBase64); const mockAxiosHead = vi.mocked(axios.head); const mockAxiosGet = vi.mocked(axios.get); const originalMultipleDataToBase64 = serviceEnv.MULTIPLE_DATA_TO_BASE64; -const multimediaEnv: { MULTIPLE_DATA_TO_BASE64: boolean } = serviceEnv; +const originalPreviewProxyUrl = serviceEnv.AGENT_SANDBOX_PREVIEW_PROXY_URL; +const multimediaEnv: { + MULTIPLE_DATA_TO_BASE64: boolean; + AGENT_SANDBOX_PREVIEW_PROXY_URL?: string | undefined; +} = serviceEnv; describe('filterGPTMessageByMaxContext function tests', () => { beforeEach(() => { vi.clearAllMocks(); multimediaEnv.MULTIPLE_DATA_TO_BASE64 = originalMultipleDataToBase64; + multimediaEnv.AGENT_SANDBOX_PREVIEW_PROXY_URL = originalPreviewProxyUrl; mockCountGptMessagesTokens.mockResolvedValue(10); }); @@ -1380,6 +1387,178 @@ describe('loadRequestMessages function tests', () => { expect(result[1].content).toBe('Tool result'); }); + it('should keep tool text and move image parts into a following user message', async () => { + multimediaEnv.MULTIPLE_DATA_TO_BASE64 = false; + const messages: ChatCompletionMessageParam[] = [ + { role: ChatCompletionRequestMessageRoleEnum.User, content: 'Hello' }, + { + role: ChatCompletionRequestMessageRoleEnum.Tool, + tool_call_id: 'call1', + content: [ + { type: 'text', text: 'Look at this image' }, + { type: 'image_url', image_url: { url: 'https://example.com/a.png' } } + ] + } + ]; + + const result = await loadRequestMessages({ messages, useVision: true }); + + expect(result).toHaveLength(3); + expect(result[1].role).toBe(ChatCompletionRequestMessageRoleEnum.Tool); + expect(result[1].content).toBe('Look at this image'); + expect(result[2].role).toBe(ChatCompletionRequestMessageRoleEnum.User); + expect(result[2].content).toEqual([ + { type: 'image_url', image_url: { url: 'https://example.com/a.png' } } + ]); + }); + + it('should convert tool image urls to base64 when configured', async () => { + multimediaEnv.MULTIPLE_DATA_TO_BASE64 = true; + mockGetImageBase64.mockResolvedValue({ + completeBase64: 'data:image/png;base64,converted', + base64: 'converted', + mime: 'image/png' + }); + + const messages: ChatCompletionMessageParam[] = [ + { role: ChatCompletionRequestMessageRoleEnum.User, content: 'Hello' }, + { + role: ChatCompletionRequestMessageRoleEnum.Tool, + tool_call_id: 'call1', + content: [{ type: 'image_url', image_url: { url: 'https://example.com/a.png' } }] + } + ]; + + const result = await loadRequestMessages({ messages, useVision: true }); + + expect(result).toHaveLength(3); + expect(result[1].role).toBe(ChatCompletionRequestMessageRoleEnum.Tool); + expect(result[1].content).toBe('none'); + expect(result[2].role).toBe(ChatCompletionRequestMessageRoleEnum.User); + expect(result[2].content).toEqual([ + { type: 'image_url', image_url: { url: 'data:image/png;base64,converted' } } + ]); + }); + + it('should inline sandbox preview image urls before sending', async () => { + multimediaEnv.MULTIPLE_DATA_TO_BASE64 = false; + multimediaEnv.AGENT_SANDBOX_PREVIEW_PROXY_URL = 'https://sandbox-preview.example.com'; + mockGetImageBase64.mockResolvedValue({ + completeBase64: 'data:image/png;base64,inlined', + base64: 'inlined', + mime: 'image/png' + }); + + const messages: ChatCompletionMessageParam[] = [ + { role: ChatCompletionRequestMessageRoleEnum.User, content: 'Hello' }, + { + role: ChatCompletionRequestMessageRoleEnum.Tool, + tool_call_id: 'call1', + content: [ + { + type: 'image_url', + image_url: { + url: 'https://sandbox-preview.example.com/preview/sandbox-1/session-1/a.png' + } + } + ] + } + ]; + + const result = await loadRequestMessages({ messages, useVision: true }); + + expect(result).toHaveLength(3); + expect(result[2].content).toEqual([ + { type: 'image_url', image_url: { url: 'data:image/png;base64,inlined' } } + ]); + expect(mockGetImageBase64).toHaveBeenCalledWith( + 'https://sandbox-preview.example.com/preview/sandbox-1/session-1/a.png' + ); + }); + + it('should drop sandbox preview images when inlining fails', async () => { + multimediaEnv.MULTIPLE_DATA_TO_BASE64 = false; + multimediaEnv.AGENT_SANDBOX_PREVIEW_PROXY_URL = 'https://sandbox-preview.example.com'; + mockGetImageBase64.mockRejectedValue(new Error('download failed')); + + const messages: ChatCompletionMessageParam[] = [ + { role: ChatCompletionRequestMessageRoleEnum.User, content: 'Hello' }, + { + role: ChatCompletionRequestMessageRoleEnum.Tool, + tool_call_id: 'call1', + content: [ + { + type: 'image_url', + image_url: { + url: 'https://sandbox-preview.example.com/preview/sandbox-1/session-1/a.png' + } + } + ] + } + ]; + + const result = await loadRequestMessages({ messages, useVision: true }); + + expect(result).toHaveLength(2); + expect(result[1].role).toBe(ChatCompletionRequestMessageRoleEnum.Tool); + expect(result[1].content).toBe('none'); + expect(mockGetImageBase64).toHaveBeenCalledTimes(1); + }); + + it('should not inline non-preview remote image urls', async () => { + multimediaEnv.MULTIPLE_DATA_TO_BASE64 = false; + multimediaEnv.AGENT_SANDBOX_PREVIEW_PROXY_URL = 'https://sandbox-preview.example.com'; + + const messages: ChatCompletionMessageParam[] = [ + { role: ChatCompletionRequestMessageRoleEnum.User, content: 'Hello' }, + { + role: ChatCompletionRequestMessageRoleEnum.Tool, + tool_call_id: 'call1', + content: [{ type: 'image_url', image_url: { url: 'https://example.com/a.png' } }] + } + ]; + + const result = await loadRequestMessages({ messages, useVision: true }); + + expect(result[2].content).toEqual([ + { type: 'image_url', image_url: { url: 'https://example.com/a.png' } } + ]); + expect(mockGetImageBase64).not.toHaveBeenCalled(); + }); + + it('should drop tool images and collapse single text part without vision', async () => { + const messages: ChatCompletionMessageParam[] = [ + { role: ChatCompletionRequestMessageRoleEnum.User, content: 'Hello' }, + { + role: ChatCompletionRequestMessageRoleEnum.Tool, + tool_call_id: 'call1', + content: [ + { type: 'text', text: 'Only text survives' }, + { type: 'image_url', image_url: { url: 'https://example.com/a.png' } } + ] + } + ]; + + const result = await loadRequestMessages({ messages }); + + expect(result[1].content).toBe('Only text survives'); + }); + + it('should normalize empty tool content parts to none', async () => { + const messages: ChatCompletionMessageParam[] = [ + { role: ChatCompletionRequestMessageRoleEnum.User, content: 'Hello' }, + { + role: ChatCompletionRequestMessageRoleEnum.Tool, + tool_call_id: 'call1', + content: [] + } + ]; + + const result = await loadRequestMessages({ messages }); + + expect(result[1].content).toBe('none'); + }); + it('should handle user message with empty content as null', async () => { const messages: ChatCompletionMessageParam[] = [ { role: ChatCompletionRequestMessageRoleEnum.User, content: '' } @@ -1488,3 +1667,42 @@ describe('loadRequestMessages function tests', () => { }); }); }); + +describe('sandbox preview url helpers', () => { + beforeEach(() => { + multimediaEnv.AGENT_SANDBOX_PREVIEW_PROXY_URL = originalPreviewProxyUrl; + }); + + it('should detect sandbox preview file urls by env prefix', () => { + multimediaEnv.AGENT_SANDBOX_PREVIEW_PROXY_URL = 'https://sandbox-preview.example.com/'; + + expect(isSandboxPreviewFileUrl('https://sandbox-preview.example.com/preview/s/a/b.png')).toBe( + true + ); + expect(isSandboxPreviewFileUrl('https://sandbox-preview.example.com/other/a.png')).toBe(false); + expect(isSandboxPreviewFileUrl('https://example.com/a.png')).toBe(false); + }); + + it('should treat every url as non-preview when the env is unset', () => { + multimediaEnv.AGENT_SANDBOX_PREVIEW_PROXY_URL = undefined; + + expect(isSandboxPreviewFileUrl('https://sandbox-preview.example.com/preview/s/a/b.png')).toBe( + false + ); + }); + + it('should mask sandbox preview urls in image placeholders', () => { + multimediaEnv.AGENT_SANDBOX_PREVIEW_PROXY_URL = 'https://sandbox-preview.example.com'; + + expect(maskImagePlaceholderUrl('https://sandbox-preview.example.com/preview/s/a/b.png')).toBe( + 'sandbox preview file' + ); + expect(maskImagePlaceholderUrl('https://example.com/a.png')).toBe('https://example.com/a.png'); + }); + + it('should leave remote urls untouched in image placeholders when the env is unset', () => { + multimediaEnv.AGENT_SANDBOX_PREVIEW_PROXY_URL = undefined; + + expect(maskImagePlaceholderUrl('https://example.com/a.png')).toBe('https://example.com/a.png'); + }); +}); diff --git a/packages/service/test/core/ai/sandbox/application/toolCall/getFileUrl.tool.test.ts b/packages/service/test/core/ai/sandbox/application/toolCall/getFileUrl.tool.test.ts index b4fc151f8c03..4631451233cb 100644 --- a/packages/service/test/core/ai/sandbox/application/toolCall/getFileUrl.tool.test.ts +++ b/packages/service/test/core/ai/sandbox/application/toolCall/getFileUrl.tool.test.ts @@ -94,6 +94,33 @@ describe('sandboxGetFileUrlTool', () => { sessionId: 'a12345678901234567890123', filePath: '/workspace/sessions/chat-1/report/data.csv' }); + // 非图片文件只保留承载原 response 的 text part + expect(result.content).toEqual([{ type: 'text', text: result.response }]); + }); + + it('adds image_url parts for image files on top of the text part', async () => { + getFileInfo.mockResolvedValue( + new Map([['/workspace/sessions/chat-1/report/chart.png', { isDirectory: false }]]) + ); + + const result = await sandboxGetFileUrlTool.execute({ + sandboxInstance: createSandboxInstance(), + params: { paths: ['report/chart.png'] } + }); + + expect(JSON.parse(result.response)).toEqual([ + { + fileUrl: 'preview:/workspace/sessions/chat-1/report/chart.png', + filename: 'chart.png' + } + ]); + expect(result.content).toEqual([ + { type: 'text', text: result.response }, + { + type: 'image_url', + image_url: { url: 'preview:/workspace/sessions/chat-1/report/chart.png' } + } + ]); }); it('returns an empty result without creating a preview session', async () => { diff --git a/packages/service/test/core/workflow/dispatch/ai/agentLoopCore/runtime/workflowToolRunner.test.ts b/packages/service/test/core/workflow/dispatch/ai/agentLoopCore/runtime/workflowToolRunner.test.ts index 9a1108439877..919910c73558 100644 --- a/packages/service/test/core/workflow/dispatch/ai/agentLoopCore/runtime/workflowToolRunner.test.ts +++ b/packages/service/test/core/workflow/dispatch/ai/agentLoopCore/runtime/workflowToolRunner.test.ts @@ -1,5 +1,8 @@ import { beforeEach, describe, expect, it, vi } from 'vitest'; -import { createAgentLoopCoreWorkflowToolRunner } from '@fastgpt/service/core/workflow/dispatch/ai/agentLoopCore/application/runtime/workflowToolRunner'; +import { + createAgentLoopCoreWorkflowToolRunner, + extractAgentLoopCoreToolContent +} from '@fastgpt/service/core/workflow/dispatch/ai/agentLoopCore/application/runtime/workflowToolRunner'; import { dispatchUpdateVariable } from '@fastgpt/service/core/workflow/dispatch/tools/runUpdateVar'; const createCall = ({ @@ -597,3 +600,46 @@ describe('createAgentLoopCoreWorkflowToolRunner', () => { ]); }); }); + +describe('extractAgentLoopCoreToolContent', () => { + it('returns structured content parts when every part is valid', () => { + const content = [ + { type: 'text', text: 'done' }, + { type: 'image_url', image_url: { url: 'https://files.example.com/plot.png' } } + ]; + + expect(extractAgentLoopCoreToolContent({ content })).toEqual(content); + }); + + it('returns undefined for non-object or missing content', () => { + expect(extractAgentLoopCoreToolContent(undefined)).toBeUndefined(); + expect(extractAgentLoopCoreToolContent(null)).toBeUndefined(); + expect(extractAgentLoopCoreToolContent('text')).toBeUndefined(); + expect(extractAgentLoopCoreToolContent([])).toBeUndefined(); + expect(extractAgentLoopCoreToolContent({})).toBeUndefined(); + }); + + it('returns undefined when content is an empty array', () => { + expect(extractAgentLoopCoreToolContent({ content: [] })).toBeUndefined(); + }); + + it('returns undefined when an image_url part has an empty url', () => { + expect( + extractAgentLoopCoreToolContent({ + content: [{ type: 'image_url', image_url: { url: '' } }] + }) + ).toBeUndefined(); + }); + + it('returns undefined when any part is invalid', () => { + expect( + extractAgentLoopCoreToolContent({ + content: [ + { type: 'text', text: 'ok' }, + { type: 'image_url', image_url: {} }, + { type: 'unknown' } + ] + }) + ).toBeUndefined(); + }); +}); From 8a02b10016df64c7c7741e683d9d6d5ca59da1ca Mon Sep 17 00:00:00 2001 From: lirenjie Date: Sun, 25 Oct 2026 22:31:58 +0800 Subject: [PATCH 2/5] fix(agent): keep tool result grouping and validate tool content parts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 同一轮多个 tool call 的图片消息统一追加在全部 tool 结果之后， 避免 assistant(A,B) 变成 tool A -> user(image) -> tool B 打断回填顺序 - Agent V2 业务工具（插件/MCP/HTTP）输出的结构化 content 逐层透传到 agent-loop - content parts 提取复用共享 schema 校验，拒绝 detail 等非法字段并回退文本 - 修正 tool content part 注释：属于 FastGPT 内部扩展，出站转成 user 图片消息 --- packages/global/core/ai/llm/type.ts | 12 +- packages/service/core/ai/llm/utils.ts | 117 ++++++++++++------ .../dispatch/ai/agent/sub/tool/index.ts | 12 +- .../workflow/dispatch/ai/agent/sub/utils.ts | 44 ++++--- .../core/workflow/dispatch/ai/agent/type.ts | 10 +- .../application/runtime/workflowToolRunner.ts | 38 +----- .../ai/agentLoopCore/domain/toolContent.ts | 41 ++++++ .../ai/agentLoopCore/interface/index.ts | 1 + .../service/test/core/ai/llm/utils.test.ts | 52 ++++++++ .../workflow/dispatch/ai/agent/utils.test.ts | 94 ++++++++++++++ .../runtime/workflowToolRunner.test.ts | 14 +++ 11 files changed, 335 insertions(+), 100 deletions(-) create mode 100644 packages/service/core/workflow/dispatch/ai/agentLoopCore/domain/toolContent.ts diff --git a/packages/global/core/ai/llm/type.ts b/packages/global/core/ai/llm/type.ts index a168cb19b2e2..46d0166a56bc 100644 --- a/packages/global/core/ai/llm/type.ts +++ b/packages/global/core/ai/llm/type.ts @@ -129,8 +129,12 @@ export type ChatCompletionUserMessageParam = z.infer; /** - * Tool message: 对齐 openai SDK 的 ChatCompletionToolMessageParam，新增可选 `name` - * SDK content 允许字符串或 text/image_url part 数组 + * Tool message: 对齐 openai SDK 的 ChatCompletionToolMessageParam，新增可选 `name`。 + * content 在内部允许带 image_url part（见上），出站时会转换；供应商只认字符串或文本 parts。 */ export const ChatCompletionToolMessageParamSchema = z.object({ role: z.literal('tool'), diff --git a/packages/service/core/ai/llm/utils.ts b/packages/service/core/ai/llm/utils.ts index 174a95535a26..6ab6dcb62aa8 100644 --- a/packages/service/core/ai/llm/utils.ts +++ b/packages/service/core/ai/llm/utils.ts @@ -715,6 +715,14 @@ export const loadRequestMessages = async ({ }, []); })(messages); + /** + * 工具消息里的图片不能直接放进 tool content（供应商只接受字符串或文本 parts）， + * 需要转换成紧随其后的 user 视觉消息。但同一个 assistant 消息可能同时发起多个工具调用， + * 这时的图片消息必须排在全部 tool 结果之后，否则会打断 tool_call 结果回填顺序。 + * 这里先给 tool 消息登记图片，拼装阶段再按“整轮 tool 结果”统一追加。 + */ + const toolImageParts = new WeakMap(); + const loadMessages = ( await Promise.all( mergeMessages.map(async (raw, i) => { @@ -781,53 +789,45 @@ export const loadRequestMessages = async ({ }; } else if (item.role === ChatCompletionRequestMessageRoleEnum.Tool) { // 供应商接口的 tool content 只接受字符串或文本 parts，图片不能放进 tool 消息。 - // 文本保留在 tool 消息里；图片先按模型能力过滤、再归一化， - // 并转换成紧随 tool 消息之后的 user 视觉消息。 - if (Array.isArray(item.content)) { - const supportedContent = item.content - .map((part) => filterSupportedUserContentPart(part as ChatCompletionContentPart)) - .filter(Boolean) as ChatCompletionContentPart[]; - - const textParts = supportedContent.filter( - (part): part is ChatCompletionContentPartText => part.type === 'text' - ); - const mediaParts = supportedContent.filter((part) => part.type !== 'text'); - - const normalizedTextParts = (await normalizeMediaContentParts( - textParts - )) as ChatCompletionContentPartText[]; - const toolContent = + // 文本保留在 tool 消息里；图片先按模型能力过滤、再归一化，登记到 toolImageParts， + // 由拼装阶段转换成这一轮 tool 结果之后的 user 视觉消息。 + if (!Array.isArray(item.content)) { + return item; + } + + const supportedContent = item.content + .map((part) => filterSupportedUserContentPart(part as ChatCompletionContentPart)) + .filter(Boolean) as ChatCompletionContentPart[]; + + const textParts = supportedContent.filter( + (part): part is ChatCompletionContentPartText => part.type === 'text' + ); + const mediaParts = supportedContent.filter((part) => part.type !== 'text'); + + const normalizedTextParts = (await normalizeMediaContentParts( + textParts + )) as ChatCompletionContentPartText[]; + const normalizedMediaParts = await normalizeMediaContentParts( + (await Promise.all(mediaParts.map(inlineSandboxPreviewImage))).filter( + (part): part is ChatCompletionContentPart => Boolean(part) + ) + ); + + const toolMessage = { + ...item, + content: normalizedTextParts.length === 0 ? 'none' : normalizedTextParts.length === 1 ? normalizedTextParts[0].text - : normalizedTextParts; - - const normalizedMediaParts = await normalizeMediaContentParts( - (await Promise.all(mediaParts.map(inlineSandboxPreviewImage))).filter( - (part): part is ChatCompletionContentPart => Boolean(part) - ) - ); - if (normalizedMediaParts.length === 0) { - return { - ...item, - content: toolContent - }; - } + : normalizedTextParts + } as ChatCompletionMessageParam; - return [ - { - ...item, - content: toolContent - }, - { - role: ChatCompletionRequestMessageRoleEnum.User, - content: normalizedMediaParts - } as ChatCompletionMessageParam - ]; + if (normalizedMediaParts.length > 0) { + toolImageParts.set(toolMessage, normalizedMediaParts); } - return item; + return toolMessage; } else { return item; } @@ -837,5 +837,40 @@ export const loadRequestMessages = async ({ .flat() .filter(Boolean) as ChatCompletionMessageParam[]; - return loadMessages; + /** + * 同一轮的全部 tool 结果必须连续回填，工具图片消息统一追加在这一轮 tool 结果之后。 + * 否则 assistant(A,B) 的回填会变成 tool A → user(image) → tool B，打断协议顺序。 + */ + const appendToolImagesAfterToolRun = (messages: ChatCompletionMessageParam[]) => { + const result: ChatCompletionMessageParam[] = []; + let pendingToolImages: ChatCompletionContentPart[] = []; + + const flushToolImages = () => { + if (pendingToolImages.length === 0) return; + result.push({ + role: ChatCompletionRequestMessageRoleEnum.User, + content: pendingToolImages + } as ChatCompletionMessageParam); + pendingToolImages = []; + }; + + messages.forEach((message) => { + if (message.role === ChatCompletionRequestMessageRoleEnum.Tool) { + result.push(message); + const images = toolImageParts.get(message); + if (images?.length) { + pendingToolImages.push(...images); + } + return; + } + + flushToolImages(); + result.push(message); + }); + flushToolImages(); + + return result; + }; + + return appendToolImagesAfterToolRun(loadMessages); }; diff --git a/packages/service/core/workflow/dispatch/ai/agent/sub/tool/index.ts b/packages/service/core/workflow/dispatch/ai/agent/sub/tool/index.ts index d1b51cd2fb6e..0fae94c5a548 100644 --- a/packages/service/core/workflow/dispatch/ai/agent/sub/tool/index.ts +++ b/packages/service/core/workflow/dispatch/ai/agent/sub/tool/index.ts @@ -41,6 +41,7 @@ import { assertTeamPluginSourceAccess, getRawPluginIdFromSystemToolId } from '../../../../../../plugin/teamPluginPolicy'; +import { extractAgentLoopCoreToolContent } from '../../../agentLoopCore/domain/toolContent'; type SystemInputConfigType = { type: SystemToolSecretInputTypeEnum; @@ -258,8 +259,13 @@ export const dispatchTool = async ({ msg: result[NodeOutputKeyEnum.systemError] }); + // 插件输出若显式声明了结构化 content，透传给 agent-loop 作为下一轮模型输入； + // response 仍是完整业务输出，两者互不替代。 + const content = extractAgentLoopCoreToolContent(result); + return { response: JSON.stringify(result), + ...(content ? { content } : {}), nodeResponse: getNodeResponse({ result }), @@ -312,8 +318,10 @@ export const dispatchTool = async ({ params }); const errorMessage = getMCPToolErrorText(result); + const content = extractAgentLoopCoreToolContent(result); return { response: JSON.stringify(result), + ...(content ? { content } : {}), ...(errorMessage !== undefined ? { errorMessage } : {}), nodeResponse: { ...getNodeResponse({ result }), @@ -376,11 +384,13 @@ export const dispatchTool = async ({ }; } + const content = extractAgentLoopCoreToolContent(data); return { nodeResponse: getNodeResponse({ result: data }), - response: typeof data === 'object' ? JSON.stringify(data) : data + response: typeof data === 'object' ? JSON.stringify(data) : data, + ...(content ? { content } : {}) }; } else { return getErrResponse("Can't find the tool"); diff --git a/packages/service/core/workflow/dispatch/ai/agent/sub/utils.ts b/packages/service/core/workflow/dispatch/ai/agent/sub/utils.ts index 4fd49da86a25..2fe6120f8bc1 100644 --- a/packages/service/core/workflow/dispatch/ai/agent/sub/utils.ts +++ b/packages/service/core/workflow/dispatch/ai/agent/sub/utils.ts @@ -3,6 +3,7 @@ import type { AgentToolType } from '@fastgpt/global/core/app/tool/type'; import type { DispatchSubAppResponse, GetSubAppInfoFnType, SubAppRuntimeType } from '../type'; import { getAgentRuntimeTools } from './tool/utils'; import type { ChatCompletionTool } from '@fastgpt/global/core/ai/llm/type'; +import type { ChatCompletionToolMessageContentPart } from '@fastgpt/global/core/ai/llm/type'; import type { ChatNodeUsageType } from '@fastgpt/global/support/wallet/bill/type'; import type { DispatchAgentModuleProps } from '..'; import { FlowNodeTypeEnum } from '@fastgpt/global/core/workflow/node/constant'; @@ -194,7 +195,8 @@ export const getExecuteTool = ({ stop = false, nodeResponse, nodeSummary, - errorMessage + errorMessage, + content } = await (async (): Promise<{ response: string; assistantMessages?: DispatchSubAppResponse['assistantMessages']; @@ -204,6 +206,7 @@ export const getExecuteTool = ({ errorMessage?: string; nodeResponse?: DispatchSubAppResponse['nodeResponse']; nodeSummary?: DispatchSubAppResponse['nodeSummary']; + content?: ChatCompletionToolMessageContentPart[]; }> => { try { // User Sub App @@ -233,30 +236,32 @@ export const getExecuteTool = ({ }); if (tool.type === 'tool') { - const { response, usages, nodeResponse, nodeSummary, errorMessage } = await dispatchTool({ - tool: { - name: tool.name, - avatar: tool.avatar, - version: tool.version, - toolConfig: tool.toolConfig - }, - params: requestParams, - runningUserInfo, - runningAppInfo, - chatId, - uid, - variableState, - runtimeNodesMap, - // Agent 统一负责外层流输出，子工具不得绕过 Agent 直接写入响应流。 - workflowStreamResponse: undefined - }); + const { response, usages, nodeResponse, nodeSummary, errorMessage, content } = + await dispatchTool({ + tool: { + name: tool.name, + avatar: tool.avatar, + version: tool.version, + toolConfig: tool.toolConfig + }, + params: requestParams, + runningUserInfo, + runningAppInfo, + chatId, + uid, + variableState, + runtimeNodesMap, + // Agent 统一负责外层流输出，子工具不得绕过 Agent 直接写入响应流。 + workflowStreamResponse: undefined + }); return { response, usages, nodeResponse, nodeSummary, - errorMessage + errorMessage, + content }; } else if (tool.type === 'workflow') { const { userChatInput, ...params } = filterAgentWorkflowRuntimeParams(requestParams); @@ -430,6 +435,7 @@ export const getExecuteTool = ({ stop, nodeSummary, ...(errorMessage ? { errorMessage } : {}), + ...(content?.length ? { content } : {}), nodeResponse: formatNodeResponse }; }; diff --git a/packages/service/core/workflow/dispatch/ai/agent/type.ts b/packages/service/core/workflow/dispatch/ai/agent/type.ts index 076a615c16c9..f3a91a612a07 100644 --- a/packages/service/core/workflow/dispatch/ai/agent/type.ts +++ b/packages/service/core/workflow/dispatch/ai/agent/type.ts @@ -3,7 +3,10 @@ import z from 'zod'; import { NodeToolConfigTypeSchema } from '@fastgpt/global/core/workflow/type/node'; import type { RuntimeNodeItemType } from '@fastgpt/global/core/workflow/runtime/type'; import type { ChatHistoryItemResType } from '@fastgpt/global/core/chat/type'; -import type { ChatCompletionMessageParam } from '@fastgpt/global/core/ai/llm/type'; +import type { + ChatCompletionMessageParam, + ChatCompletionToolMessageContentPart +} from '@fastgpt/global/core/ai/llm/type'; import type { WorkflowInteractiveResponseType } from '@fastgpt/global/core/workflow/template/system/interactive/type'; import type { NodeSummary } from '../../../types/runtime'; @@ -21,6 +24,11 @@ export type DispatchSubAppResponse = { * 工具自身的运行详情。显式 totalPoints 表示当前响应自身费用；省略时由外层按 usages 计算。 */ nodeResponse?: Omit; + /** + * 写入 LLM tool message 的结构化 content（text / image_url parts）。 + * 仅当工具显式返回合法的 parts 时才透传，缺省时按 response 字符串兜底。 + */ + content?: ChatCompletionToolMessageContentPart[]; }; export const SubAppRuntimeSchema = z.object({ diff --git a/packages/service/core/workflow/dispatch/ai/agentLoopCore/application/runtime/workflowToolRunner.ts b/packages/service/core/workflow/dispatch/ai/agentLoopCore/application/runtime/workflowToolRunner.ts index a264b87e7eca..0270c8f0bba0 100644 --- a/packages/service/core/workflow/dispatch/ai/agentLoopCore/application/runtime/workflowToolRunner.ts +++ b/packages/service/core/workflow/dispatch/ai/agentLoopCore/application/runtime/workflowToolRunner.ts @@ -3,8 +3,7 @@ import { chats2GPTMessages } from '@fastgpt/global/core/chat/adapt'; import type { ChatHistoryItemResType } from '@fastgpt/global/core/chat/type'; import type { ChatCompletionMessageParam, - ChatCompletionMessageToolCall, - ChatCompletionToolMessageContentPart + ChatCompletionMessageToolCall } from '@fastgpt/global/core/ai/llm/type'; import type { RuntimeEdgeItemType } from '@fastgpt/global/core/workflow/type/edge'; import type { RuntimeNodeItemType } from '@fastgpt/global/core/workflow/runtime/type'; @@ -16,6 +15,7 @@ import type { AgentLoopInteractiveToolExecuteParams } from '../../../../../../ai import type { AgentLoopDatasetSearchExecutor } from '../../../../../../ai/llm/agentLoop/interface'; import { parseJsonArgs } from '../../../../../../ai/utils'; import type { AgentLoopCoreToolInfo, AgentLoopCoreToolRunResult } from '../../domain/toolProvider'; +import { extractAgentLoopCoreToolContent } from '../../domain/toolContent'; import type { AgentLoopCoreToolRunFlowResponse } from '../../adapter/nodeResponse/toolRunCollector'; import { normalizeAgentLoopCoreDatasetSearchResult } from './systemToolHelpers'; import { cloneDeep } from 'lodash-es'; @@ -103,38 +103,8 @@ export const formatAgentLoopCoreToolResponse = (toolResponses: any) => { return toolResponses ? String(toolResponses) : 'none'; }; -/** - * 从工具节点输出中提取结构化 tool content。 - * - * 仅接受形如 { content: [{ type: 'text' | 'image_url', ... }, ...] } 的对象输出， - * 且 content 非空、每项都是合法的 text/image_url part 时才返回，否则 undefined。 - * 工具可以通过该形状主动返回图片，作为下一轮模型的视觉输入；不在普通文本里猜测图片。 - */ -export const extractAgentLoopCoreToolContent = ( - toolResponses: any -): ChatCompletionToolMessageContentPart[] | undefined => { - if (!toolResponses || typeof toolResponses !== 'object' || Array.isArray(toolResponses)) { - return; - } - - const content = toolResponses.content; - if (!Array.isArray(content) || content.length === 0) return; - - const isValidPart = (part: any): part is ChatCompletionToolMessageContentPart => { - if (!part || typeof part !== 'object') return false; - if (part.type === 'text') return typeof part.text === 'string' && part.text.length > 0; - if (part.type === 'image_url') { - return ( - !!part.image_url && typeof part.image_url.url === 'string' && part.image_url.url.length > 0 - ); - } - return false; - }; - - return content.every(isValidPart) - ? (content as ChatCompletionToolMessageContentPart[]) - : undefined; -}; +// 结构化 content 的提取规则由 workflow 工具与 Agent V2 业务工具共用，实现在 domain 层。 +export { extractAgentLoopCoreToolContent }; /** * 初始化阶段直接在传入的 runtime edge/node 上标记入口状态。 diff --git a/packages/service/core/workflow/dispatch/ai/agentLoopCore/domain/toolContent.ts b/packages/service/core/workflow/dispatch/ai/agentLoopCore/domain/toolContent.ts new file mode 100644 index 000000000000..f37e2ab7683a --- /dev/null +++ b/packages/service/core/workflow/dispatch/ai/agentLoopCore/domain/toolContent.ts @@ -0,0 +1,41 @@ +import { + ChatCompletionToolMessageContentPartSchema, + type ChatCompletionToolMessageContentPart +} from '@fastgpt/global/core/ai/llm/type'; + +/** + * 从工具输出中提取结构化 tool content。 + * + * 仅接受形如 `{ content: [{ type: 'text' | 'image_url', ... }, ...] }` 的对象输出： + * content 必须是非空数组，且每个 part 都能通过共享的 + * `ChatCompletionToolMessageContentPartSchema` 校验（含 `image_url.detail` 这类细分字段）。 + * + * 任一 part 不合法时整体返回 undefined，由调用方按普通文本处理：工具输出的业务数据 + * 可能只是恰好带了一个 content 字段，不能被半解析成请求字段。只有工具显式声明了合法的 + * parts，才作为下一轮模型的视觉输入；不在普通文本里猜测图片。 + */ +export const extractAgentLoopCoreToolContent = ( + toolResponses: unknown +): ChatCompletionToolMessageContentPart[] | undefined => { + if (!toolResponses || typeof toolResponses !== 'object' || Array.isArray(toolResponses)) { + return; + } + + const content = (toolResponses as { content?: unknown }).content; + if (!Array.isArray(content) || content.length === 0) return; + + const parts: ChatCompletionToolMessageContentPart[] = []; + for (const item of content) { + const parsed = ChatCompletionToolMessageContentPartSchema.safeParse(item); + if (!parsed.success) return; + + // 空文本、空链接没有传递价值，按非法处理，回退到普通文本链路。 + if (parsed.data.type === 'text' ? !parsed.data.text : !parsed.data.image_url.url) { + return; + } + + parts.push(parsed.data); + } + + return parts; +}; diff --git a/packages/service/core/workflow/dispatch/ai/agentLoopCore/interface/index.ts b/packages/service/core/workflow/dispatch/ai/agentLoopCore/interface/index.ts index 61e824e6331c..0650072fdc58 100644 --- a/packages/service/core/workflow/dispatch/ai/agentLoopCore/interface/index.ts +++ b/packages/service/core/workflow/dispatch/ai/agentLoopCore/interface/index.ts @@ -41,6 +41,7 @@ export { createAgentLoopCoreWorkflowToolRunner, type CreateAgentLoopCoreWorkflowToolRunnerParams } from '../application/runtime/workflowToolRunner'; +export { extractAgentLoopCoreToolContent } from '../domain/toolContent'; export { createAgentLoopCoreChildInteractiveParams } from '../adapter/interactive/child'; export { diff --git a/packages/service/test/core/ai/llm/utils.test.ts b/packages/service/test/core/ai/llm/utils.test.ts index 9b5240dbaa28..dde4005154fd 100644 --- a/packages/service/test/core/ai/llm/utils.test.ts +++ b/packages/service/test/core/ai/llm/utils.test.ts @@ -1412,6 +1412,58 @@ describe('loadRequestMessages function tests', () => { ]); }); + it('should append tool images after every tool result of the same assistant turn', async () => { + multimediaEnv.MULTIPLE_DATA_TO_BASE64 = false; + const messages: ChatCompletionMessageParam[] = [ + { role: ChatCompletionRequestMessageRoleEnum.User, content: 'Hello' }, + { + role: ChatCompletionRequestMessageRoleEnum.Assistant, + content: '', + tool_calls: [ + { + id: 'call_a', + type: 'function', + function: { name: 'get_file_url', arguments: '{}' } + }, + { + id: 'call_b', + type: 'function', + function: { name: 'get_file_url', arguments: '{}' } + } + ] + }, + { + role: ChatCompletionRequestMessageRoleEnum.Tool, + tool_call_id: 'call_a', + content: [ + { type: 'text', text: 'files' }, + { type: 'image_url', image_url: { url: 'https://example.com/a.png' } } + ] + }, + { + role: ChatCompletionRequestMessageRoleEnum.Tool, + tool_call_id: 'call_b', + content: 'plain result' + } + ]; + + const result = await loadRequestMessages({ messages, useVision: true }); + + // 同一轮的全部 tool 结果必须连续回填，图片消息统一追加在其后。 + expect(result.map((item) => item.role)).toEqual([ + ChatCompletionRequestMessageRoleEnum.User, + ChatCompletionRequestMessageRoleEnum.Assistant, + ChatCompletionRequestMessageRoleEnum.Tool, + ChatCompletionRequestMessageRoleEnum.Tool, + ChatCompletionRequestMessageRoleEnum.User + ]); + expect((result[2] as any).tool_call_id).toBe('call_a'); + expect((result[3] as any).tool_call_id).toBe('call_b'); + expect(result[4].content).toEqual([ + { type: 'image_url', image_url: { url: 'https://example.com/a.png' } } + ]); + }); + it('should convert tool image urls to base64 when configured', async () => { multimediaEnv.MULTIPLE_DATA_TO_BASE64 = true; mockGetImageBase64.mockResolvedValue({ diff --git a/packages/service/test/core/workflow/dispatch/ai/agent/utils.test.ts b/packages/service/test/core/workflow/dispatch/ai/agent/utils.test.ts index d1eec7c5ac2a..17f5786de5a8 100644 --- a/packages/service/test/core/workflow/dispatch/ai/agent/utils.test.ts +++ b/packages/service/test/core/workflow/dispatch/ai/agent/utils.test.ts @@ -180,4 +180,98 @@ describe('Agent read_files tool protocol', () => { ); expect(streamResponseFn).not.toHaveBeenCalled(); }); + + it('passes structured tool content from dispatchTool to the agent loop result', async () => { + const content = [ + { type: 'text', text: '{"files":["chart.png"]}' }, + { type: 'image_url', image_url: { url: 'https://files.example.com/chart.png' } } + ]; + dispatchToolMock.mockResolvedValue({ + response: '{"files":["chart.png"]}', + content + }); + + const executeTool = getExecuteTool({ + checkIsStopping: vi.fn(), + chatConfig: {}, + runningUserInfo: { teamId: 'team_1', tmbId: 'tmb_1' }, + runningAppInfo: { id: 'app_1' }, + chatId: 'chat_1', + uid: 'user_1', + variableState: {} as any, + externalProvider: {} as any, + lang: 'zh-CN', + requestOrigin: '', + mode: 'chat', + timezone: 'Asia/Shanghai', + retainDatasetCite: false, + maxRunTimes: 10, + workflowDispatchDeep: 0, + params: { model: 'gpt-4' }, + stream: true, + streamResponseFn: vi.fn(), + getSubApp: () => ({ + type: 'tool', + id: 'system-tool', + name: 'System Tool', + avatar: '', + toolConfig: { systemTool: { toolId: 'systemTool-test' } }, + params: {} + }), + completionTools: [] + } as any); + + const result = await executeTool({ + callId: 'call_plugin', + toolId: 'system-tool', + args: '{}' + }); + + // 业务输出保持原样，结构化 content 单独透传给 agent-loop。 + expect(result.response).toBe('{"files":["chart.png"]}'); + expect(result.content).toEqual(content); + }); + + it('omits content when the tool returns plain text', async () => { + dispatchToolMock.mockResolvedValue({ response: 'plain result' }); + + const executeTool = getExecuteTool({ + checkIsStopping: vi.fn(), + chatConfig: {}, + runningUserInfo: { teamId: 'team_1', tmbId: 'tmb_1' }, + runningAppInfo: { id: 'app_1' }, + chatId: 'chat_1', + uid: 'user_1', + variableState: {} as any, + externalProvider: {} as any, + lang: 'zh-CN', + requestOrigin: '', + mode: 'chat', + timezone: 'Asia/Shanghai', + retainDatasetCite: false, + maxRunTimes: 10, + workflowDispatchDeep: 0, + params: { model: 'gpt-4' }, + stream: true, + streamResponseFn: vi.fn(), + getSubApp: () => ({ + type: 'tool', + id: 'system-tool', + name: 'System Tool', + avatar: '', + toolConfig: { systemTool: { toolId: 'systemTool-test' } }, + params: {} + }), + completionTools: [] + } as any); + + const result = await executeTool({ + callId: 'call_plain', + toolId: 'system-tool', + args: '{}' + }); + + expect(result.response).toBe('plain result'); + expect(result.content).toBeUndefined(); + }); }); diff --git a/packages/service/test/core/workflow/dispatch/ai/agentLoopCore/runtime/workflowToolRunner.test.ts b/packages/service/test/core/workflow/dispatch/ai/agentLoopCore/runtime/workflowToolRunner.test.ts index 919910c73558..6f54bc21fbc2 100644 --- a/packages/service/test/core/workflow/dispatch/ai/agentLoopCore/runtime/workflowToolRunner.test.ts +++ b/packages/service/test/core/workflow/dispatch/ai/agentLoopCore/runtime/workflowToolRunner.test.ts @@ -642,4 +642,18 @@ describe('extractAgentLoopCoreToolContent', () => { }) ).toBeUndefined(); }); + + it('rejects image parts whose detail is not accepted by the shared schema', () => { + // detail 只允许 auto/low/high；手写校验放过它会让下一轮请求被供应商拒绝。 + expect( + extractAgentLoopCoreToolContent({ + content: [ + { + type: 'image_url', + image_url: { url: 'https://files.example.com/plot.png', detail: 'ultra' } + } + ] + }) + ).toBeUndefined(); + }); }); From 99a6165a4c1ac18bf264a4aea6c16e1c72756665 Mon Sep 17 00:00:00 2001 From: lirenjie Date: Thu, 29 Oct 2026 13:20:07 +0800 Subject: [PATCH 3/5] fix(agent): convert MCP images and keep tool response text MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - MCP 原生 { type: 'image', data, mimeType } 在工具边界转成 data URI image_url， 再按共享 schema 校验，避免整组提取失败后只回退文本 - piAgent 工具结果与 fastAgent 共用 buildToolMessageContent：业务 response 文本 始终保留，图片只作为附加 part，交互恢复路径同样处理 - 沙箱预览取图改走配置 origin 的受限通道，CHECK_INTERNAL_IP 开启时内网预览代理 不再被公共 SSRF 拦截，其余地址仍使用 safe axios - 移除 agentLoopCore 的转导，提取规则直接从 domain/toolContent 引入 --- packages/service/common/file/image/utils.ts | 31 ++++++- .../core/ai/llm/agentLoop/domain/tool.ts | 20 +++++ .../agentLoop/provider/fastAgent/loop/base.ts | 18 +--- .../llm/agentLoop/provider/piAgent/message.ts | 6 +- .../ai/llm/agentLoop/provider/piAgent/run.ts | 7 +- .../agentLoop/provider/piAgent/tool/build.ts | 11 ++- packages/service/core/ai/llm/utils.ts | 23 ++++- .../dispatch/ai/agent/sub/tool/index.ts | 10 ++- .../application/runtime/workflowToolRunner.ts | 1 - .../ai/agentLoopCore/domain/toolContent.ts | 47 ++++++++++ .../ai/agentLoopCore/interface/index.ts | 1 - .../common/file/image/trustedOrigin.test.ts | 68 +++++++++++++++ .../ai/llm/agentLoop/fastAgentLoop.test.ts | 2 +- .../llm/agentLoop/piAgentToolContent.test.ts | 85 +++++++++++++++++++ .../service/test/core/ai/llm/utils.test.ts | 4 +- .../runtime/workflowToolRunner.test.ts | 32 ++++++- 16 files changed, 328 insertions(+), 38 deletions(-) create mode 100644 packages/service/test/common/file/image/trustedOrigin.test.ts create mode 100644 packages/service/test/core/ai/llm/agentLoop/piAgentToolContent.test.ts diff --git a/packages/service/common/file/image/utils.ts b/packages/service/common/file/image/utils.ts index 799e1bc0072f..fbdf74823c05 100644 --- a/packages/service/common/file/image/utils.ts +++ b/packages/service/common/file/image/utils.ts @@ -1,4 +1,4 @@ -import { axios } from '../../api/axios'; +import { axios, axiosWithoutSSRF } from '../../api/axios'; import { serverRequestBaseUrl } from '../../api/serverRequest'; import { getAxiosContentType, getAxiosHeaderValue } from '@fastgpt/global/common/axios/utils'; import { getLogger, LogCategories } from '../../logger'; @@ -115,6 +115,25 @@ export const guessBase64ImageType = (str: string): string => { return BASE64_PREFIX_MAP[str.charAt(0)] || DEFAULT_IMAGE_TYPE; }; +/** + * 判断一次图片下载是否命中调用方声明的可信 origin。 + * + * 沙箱预览地址来自服务端配置（AGENT_SANDBOX_PREVIEW_PROXY_URL），可能部署在内网， + * 不能被公共 SSRF 内网拦截误杀；这类地址不是用户任意输入，因此允许对该 origin 走 + * 受限通道。origin 之外的地址仍然走 safe axios，避免把“可信”扩大成任意内网访问。 + */ +export const isTrustedOriginUrl = (url: string, trustedOrigin?: string) => { + if (!trustedOrigin) return false; + + try { + const target = new URL(url, serverRequestBaseUrl); + const trusted = new URL(trustedOrigin, serverRequestBaseUrl); + return target.origin === trusted.origin; + } catch { + return false; + } +}; + /** * 下载远程图片并返回 Buffer。 * @@ -127,6 +146,8 @@ export const getImageBuffer = async ( options: { timeoutMs?: number; maxSize?: number; + /** 可信 origin（如沙箱预览代理），命中时跳过公共内网拦截。 */ + trustedOrigin?: string; } = {} ) => { logger.debug('Load image to buffer', { url }); @@ -134,7 +155,8 @@ export const getImageBuffer = async ( try { const timeoutMs = options.timeoutMs ?? DEFAULT_IMAGE_DOWNLOAD_TIMEOUT_MS; const maxSize = options.maxSize ?? DEFAULT_IMAGE_DOWNLOAD_MAX_SIZE; - const response = await axios.get(url, { + const requestClient = isTrustedOriginUrl(url, options.trustedOrigin) ? axiosWithoutSSRF : axios; + const response = await requestClient.get(url, { baseURL: serverRequestBaseUrl, responseType: 'stream', timeout: timeoutMs, @@ -205,6 +227,8 @@ export const getImageBase64 = async ( timeoutMs?: number; maxSize?: number; maxBase64BufferSize?: number; + /** 可信 origin（如沙箱预览代理），命中时跳过公共内网拦截。 */ + trustedOrigin?: string; } = {} ) => { logger.debug('Load image to base64', { url }); @@ -212,7 +236,8 @@ export const getImageBase64 = async ( try { const { buffer, mime } = await getImageBuffer(url, { timeoutMs: options.timeoutMs, - maxSize: options.maxSize + maxSize: options.maxSize, + trustedOrigin: options.trustedOrigin }); const maxBase64BufferSize = options.maxBase64BufferSize ?? DEFAULT_IMAGE_BASE64_MAX_BUFFER_SIZE; diff --git a/packages/service/core/ai/llm/agentLoop/domain/tool.ts b/packages/service/core/ai/llm/agentLoop/domain/tool.ts index 130d8d309991..ea0a91634d11 100644 --- a/packages/service/core/ai/llm/agentLoop/domain/tool.ts +++ b/packages/service/core/ai/llm/agentLoop/domain/tool.ts @@ -15,6 +15,26 @@ export type AgentLoopToolCatalog = { batchToolSize?: number; }; +/** + * 构造写入 tool message 的 content： + * - 无结构化 parts 时直接用文本字符串； + * - 有 parts 时把最终要发送的文本作为 text part 拼在图片等 parts 前， + * 保证模型实际拿到的文本与展示/计费统计一致（文本已压缩时传压缩后的值）。 + * + * fastAgent 与 piAgent 共用这层组合规则，避免两条 provider 路径对同一个工具结果 + * 给出不同的内容：业务 response 文本永远保留，图片只作为附加 part。 + */ +export const buildToolMessageContent = ({ + content, + text +}: { + content?: ChatCompletionToolMessageContentPart[]; + text: string; +}): string | ChatCompletionToolMessageContentPart[] => { + if (!content?.length) return text; + return [{ type: 'text', text }, ...content.filter((part) => part.type !== 'text')]; +}; + /** * 工具内部产生的可持久化 assistant 展示项。 * diff --git a/packages/service/core/ai/llm/agentLoop/provider/fastAgent/loop/base.ts b/packages/service/core/ai/llm/agentLoop/provider/fastAgent/loop/base.ts index 4aaa05130897..f3cfdddbb81a 100644 --- a/packages/service/core/ai/llm/agentLoop/provider/fastAgent/loop/base.ts +++ b/packages/service/core/ai/llm/agentLoop/provider/fastAgent/loop/base.ts @@ -16,6 +16,7 @@ import { countGptMessagesTokens } from '../../../../../../../common/string/tikto import { formatModelChars2Points } from '../../../../../../../support/wallet/usage/utils'; import type { LLMModelDataType } from '@fastgpt/global/core/ai/model/schema'; import type { AgentLoopAssistantResponse } from '../../../domain/tool'; +import { buildToolMessageContent } from '../../../domain/tool'; import type { AgentLoopChildrenInteractiveParams, AgentLoopCompletionPolicy, @@ -138,23 +139,6 @@ type RunAgentResponse = { finish_reason: CompletionFinishReason | undefined; }; -/** - * 构造写入 tool message 的 content： - * - 无结构化 parts 时直接用文本字符串； - * - 有 parts 时把最终要发送的文本作为 text part 拼在图片等 parts 前， - * 保证模型实际拿到的文本与展示/计费统计一致（文本已压缩时传压缩后的值）。 - */ -export const buildToolMessageContent = ({ - content, - text -}: { - content?: ChatCompletionToolMessageContentPart[]; - text: string; -}): string | ChatCompletionToolMessageContentPart[] => { - if (!content?.length) return text; - return [{ type: 'text', text }, ...content.filter((part) => part.type !== 'text')]; -}; - /** * 上下文压缩，内部会判断是否需要压缩 */ diff --git a/packages/service/core/ai/llm/agentLoop/provider/piAgent/message.ts b/packages/service/core/ai/llm/agentLoop/provider/piAgent/message.ts index bfb88a34d8f6..b376175933ac 100644 --- a/packages/service/core/ai/llm/agentLoop/provider/piAgent/message.ts +++ b/packages/service/core/ai/llm/agentLoop/provider/piAgent/message.ts @@ -16,8 +16,7 @@ import type { ChatCompletionToolMessageContentPart, CompletionFinishReason } from '@fastgpt/global/core/ai/llm/type'; -import { getImageBase64 } from '../../../../../../common/file/image/utils'; -import { maskImagePlaceholderUrl } from '../../../utils'; +import { loadImageBase64, maskImagePlaceholderUrl } from '../../../utils'; /** 将工具参数稳定序列化；不可序列化值回退为空对象，避免中断事件链。 */ export const stringifyJson = (value: unknown) => { @@ -130,7 +129,8 @@ export const convertToolResponseContentToPiContent = async ( } try { - const { base64, mime } = await getImageBase64(url); + // 与 fastAgent 共用取图通道：沙箱预览地址允许走配置 origin 的受限下载。 + const { base64, mime } = await loadImageBase64(url); return { type: 'image', mimeType: mime, data: base64 }; } catch { return { type: 'text', text: `[Image: ${maskImagePlaceholderUrl(url)}]` }; diff --git a/packages/service/core/ai/llm/agentLoop/provider/piAgent/run.ts b/packages/service/core/ai/llm/agentLoop/provider/piAgent/run.ts index f40e252d7c47..b8bc6832c64a 100644 --- a/packages/service/core/ai/llm/agentLoop/provider/piAgent/run.ts +++ b/packages/service/core/ai/llm/agentLoop/provider/piAgent/run.ts @@ -19,6 +19,7 @@ import { AgentUsageModuleName } from '../../domain/usage'; import { askUserToolName, formatAgentAskToolResponse } from '../../domain/systemTool/ask'; import { setPlanToolName, updatePlanToolName } from '../../domain/systemTool/plan'; import { + buildToolMessageContent, normalizeAgentLoopUsages, type AgentLoopInput, type AgentLoopResult, @@ -325,7 +326,11 @@ export const runPiAgentLoop = async ({ metadata: result.metadata, seconds: +((Date.now() - startedAt) / 1000).toFixed(2) }); - const toolContent = result.content ?? normalizeToolResponseContent(result.response); + // 交互恢复与普通执行共用同一套组合规则：response 文本保留，图片追加在后。 + const toolContent = buildToolMessageContent({ + content: result.content, + text: normalizedResponse + }); const piToolContent = await convertToolResponseContentToPiContent(toolContent, { useVision }); initialPiMessages = replaceInteractiveToolResult({ messages: initialPiMessages, diff --git a/packages/service/core/ai/llm/agentLoop/provider/piAgent/tool/build.ts b/packages/service/core/ai/llm/agentLoop/provider/piAgent/tool/build.ts index b3b48cab0447..8655502d7f3b 100644 --- a/packages/service/core/ai/llm/agentLoop/provider/piAgent/tool/build.ts +++ b/packages/service/core/ai/llm/agentLoop/provider/piAgent/tool/build.ts @@ -23,6 +23,7 @@ import { patchDatasetSearchParams } from '../../../domain/systemTool/datasetSearch'; import { + buildToolMessageContent, normalizeAgentLoopUsages, type AgentLoopAssistantResponse, type AgentLoopEvent, @@ -146,10 +147,12 @@ export const buildPiAgentTools = async ({ metadata: result.metadata, seconds: +((Date.now() - startedAt) / 1000).toFixed(2) }); - // 工具返回结构化 content 时优先透传；否则用规范化后的 response 字符串兜底 - const toolContent = result.content?.length - ? result.content - : normalizeToolResponseContent(result.response); + // 与 fastAgent 保持一致：业务 response 文本始终保留，结构化 parts 只在其后追加图片， + // 避免 `{ jobId, content: [image_url] }` 这类输出把业务字段丢掉。 + const toolContent = buildToolMessageContent({ + content: result.content, + text: normalizedResponse + }); onToolResult({ call, response: normalizedResponse, assistantMessages, content: toolContent }); return { ...result, diff --git a/packages/service/core/ai/llm/utils.ts b/packages/service/core/ai/llm/utils.ts index 6ab6dcb62aa8..2e501a46115b 100644 --- a/packages/service/core/ai/llm/utils.ts +++ b/packages/service/core/ai/llm/utils.ts @@ -57,6 +57,25 @@ export const isSandboxPreviewFileUrl = (url: string) => { export const maskImagePlaceholderUrl = (url: string) => isSandboxPreviewFileUrl(url) ? 'sandbox preview file' : url; +/** + * 按来源选择图片下载通道。 + * + * 沙箱预览代理可能部署在内网，公共 SSRF 内网拦截会把配置里的合法预览地址一起拒掉， + * 导致工具图片全部丢失。这类地址的 origin 来自服务端配置而不是用户输入，因此只对 + * 该 origin 使用受限通道；其它地址继续走安全的公共下载逻辑。 + * + * fastAgent（loadRequestMessages）与 piAgent（工具 content 转换）都通过这里取图， + * 保证两条 provider 路径对沙箱预览地址的能力一致。 + */ +export const loadImageBase64 = (url: string) => { + const previewProxyBase = serviceEnv.AGENT_SANDBOX_PREVIEW_PROXY_URL; + if (!previewProxyBase || !isSandboxPreviewFileUrl(url)) { + return getImageBase64(url); + } + + return getImageBase64(url, { trustedOrigin: previewProxyBase }); +}; + /** * 判断消息是否属于系统级提示。 * 这类消息不参与普通会话窗口裁剪，避免 system/developer 指令被上下文截断。 @@ -307,7 +326,7 @@ export const loadRequestMessages = async ({ } try { - const { completeBase64: base64 } = await getImageBase64(part.image_url.url); + const { completeBase64: base64 } = await loadImageBase64(part.image_url.url); return { ...part, image_url: { @@ -410,7 +429,7 @@ export const loadRequestMessages = async ({ if (shouldLoadMediaAsBase64(imgUrl)) { try { - const { completeBase64: base64 } = await getImageBase64(imgUrl); + const { completeBase64: base64 } = await loadImageBase64(imgUrl); return { ...imageItem, diff --git a/packages/service/core/workflow/dispatch/ai/agent/sub/tool/index.ts b/packages/service/core/workflow/dispatch/ai/agent/sub/tool/index.ts index 0fae94c5a548..833fb0b41b4c 100644 --- a/packages/service/core/workflow/dispatch/ai/agent/sub/tool/index.ts +++ b/packages/service/core/workflow/dispatch/ai/agent/sub/tool/index.ts @@ -41,7 +41,10 @@ import { assertTeamPluginSourceAccess, getRawPluginIdFromSystemToolId } from '../../../../../../plugin/teamPluginPolicy'; -import { extractAgentLoopCoreToolContent } from '../../../agentLoopCore/domain/toolContent'; +import { + convertMcpToolContentToToolContent, + extractAgentLoopCoreToolContent +} from '../../../agentLoopCore/domain/toolContent'; type SystemInputConfigType = { type: SystemToolSecretInputTypeEnum; @@ -318,7 +321,10 @@ export const dispatchTool = async ({ params }); const errorMessage = getMCPToolErrorText(result); - const content = extractAgentLoopCoreToolContent(result); + // MCP 图片块是 { type: 'image', data, mimeType }，需要先转成 data URI image_url。 + const content = convertMcpToolContentToToolContent( + (result as { content?: unknown } | undefined)?.content + ); return { response: JSON.stringify(result), ...(content ? { content } : {}), diff --git a/packages/service/core/workflow/dispatch/ai/agentLoopCore/application/runtime/workflowToolRunner.ts b/packages/service/core/workflow/dispatch/ai/agentLoopCore/application/runtime/workflowToolRunner.ts index 0270c8f0bba0..0e6eb073aa89 100644 --- a/packages/service/core/workflow/dispatch/ai/agentLoopCore/application/runtime/workflowToolRunner.ts +++ b/packages/service/core/workflow/dispatch/ai/agentLoopCore/application/runtime/workflowToolRunner.ts @@ -104,7 +104,6 @@ export const formatAgentLoopCoreToolResponse = (toolResponses: any) => { }; // 结构化 content 的提取规则由 workflow 工具与 Agent V2 业务工具共用，实现在 domain 层。 -export { extractAgentLoopCoreToolContent }; /** * 初始化阶段直接在传入的 runtime edge/node 上标记入口状态。 diff --git a/packages/service/core/workflow/dispatch/ai/agentLoopCore/domain/toolContent.ts b/packages/service/core/workflow/dispatch/ai/agentLoopCore/domain/toolContent.ts index f37e2ab7683a..6656fb31658f 100644 --- a/packages/service/core/workflow/dispatch/ai/agentLoopCore/domain/toolContent.ts +++ b/packages/service/core/workflow/dispatch/ai/agentLoopCore/domain/toolContent.ts @@ -39,3 +39,50 @@ export const extractAgentLoopCoreToolContent = ( return parts; }; + +/** + * 把 MCP `CallToolResult.content` 的原生内容块转换成 tool content parts。 + * + * MCP 协议里的图片块是 `{ type: 'image', data, mimeType }`（base64），不是 `image_url`； + * 直接交给 extractAgentLoopCoreToolContent 会整组校验失败并回退成 JSON 文本，模型拿不到图片。 + * 因此在这里先转成 data URI，再走共享 schema 校验。 + * + * 只产出图片 part：文本块的内容已经包含在 response JSON 里，重复发送没有意义； + * 音频、resource 等 MCP 块保持原有文本路径，不做猜测性转换。 + */ +export const convertMcpToolContentToToolContent = ( + mcpContent: unknown +): ChatCompletionToolMessageContentPart[] | undefined => { + if (!Array.isArray(mcpContent)) return; + + const parts: ChatCompletionToolMessageContentPart[] = []; + for (const block of mcpContent) { + if (!block || typeof block !== 'object') continue; + + const { type, data, mimeType } = block as { + type?: unknown; + data?: unknown; + mimeType?: unknown; + }; + if ( + type !== 'image' || + typeof data !== 'string' || + !data || + typeof mimeType !== 'string' || + !mimeType.startsWith('image/') + ) { + continue; + } + + const parsed = ChatCompletionToolMessageContentPartSchema.safeParse({ + type: 'image_url', + image_url: { url: `data:${mimeType};base64,${data}` } + }); + // MCP 服务返回的 mimeType 不可信，转出来不合法就整块丢弃，不污染请求字段。 + if (!parsed.success) continue; + + parts.push(parsed.data); + } + + return parts.length ? parts : undefined; +}; diff --git a/packages/service/core/workflow/dispatch/ai/agentLoopCore/interface/index.ts b/packages/service/core/workflow/dispatch/ai/agentLoopCore/interface/index.ts index 0650072fdc58..61e824e6331c 100644 --- a/packages/service/core/workflow/dispatch/ai/agentLoopCore/interface/index.ts +++ b/packages/service/core/workflow/dispatch/ai/agentLoopCore/interface/index.ts @@ -41,7 +41,6 @@ export { createAgentLoopCoreWorkflowToolRunner, type CreateAgentLoopCoreWorkflowToolRunnerParams } from '../application/runtime/workflowToolRunner'; -export { extractAgentLoopCoreToolContent } from '../domain/toolContent'; export { createAgentLoopCoreChildInteractiveParams } from '../adapter/interactive/child'; export { diff --git a/packages/service/test/common/file/image/trustedOrigin.test.ts b/packages/service/test/common/file/image/trustedOrigin.test.ts new file mode 100644 index 000000000000..a34c596d8a5f --- /dev/null +++ b/packages/service/test/common/file/image/trustedOrigin.test.ts @@ -0,0 +1,68 @@ +import { Readable } from 'node:stream'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +const { axiosGetMock, axiosWithoutSSRFGetMock } = vi.hoisted(() => ({ + axiosGetMock: vi.fn(), + axiosWithoutSSRFGetMock: vi.fn() +})); + +beforeEach(() => { + vi.clearAllMocks(); +}); + +vi.mock('@fastgpt/service/common/api/axios', () => ({ + axios: { get: axiosGetMock }, + axiosWithoutSSRF: { get: axiosWithoutSSRFGetMock } +})); + +vi.mock('@fastgpt/service/common/api/serverRequest', () => ({ + serverRequestBaseUrl: 'http://localhost:3000' +})); + +const createStreamResponse = () => ({ + headers: { 'content-length': '4', 'content-type': 'image/png' }, + data: Readable.from([Buffer.from([0x89, 0x50, 0x4e, 0x47])]) +}); + +const { getImageBuffer, isTrustedOriginUrl } = + await import('@fastgpt/service/common/file/image/utils'); + +describe('isTrustedOriginUrl', () => { + it('matches only the configured origin', () => { + expect(isTrustedOriginUrl('http://10.0.0.5:8080/preview/a.png', 'http://10.0.0.5:8080')).toBe( + true + ); + expect(isTrustedOriginUrl('http://10.0.0.5:9999/preview/a.png', 'http://10.0.0.5:8080')).toBe( + false + ); + expect(isTrustedOriginUrl('https://evil.example.com/a.png', 'http://10.0.0.5:8080')).toBe( + false + ); + expect(isTrustedOriginUrl('http://10.0.0.5:8080/preview/a.png')).toBe(false); + }); +}); + +describe('getImageBuffer trusted origin', () => { + it('uses the SSRF-free client for the configured preview origin', async () => { + axiosWithoutSSRFGetMock.mockResolvedValueOnce(createStreamResponse()); + + const result = await getImageBuffer('http://10.0.0.5:8080/preview/a.png', { + trustedOrigin: 'http://10.0.0.5:8080' + }); + + expect(result.mime).toBe('image/png'); + expect(axiosWithoutSSRFGetMock).toHaveBeenCalledTimes(1); + expect(axiosGetMock).not.toHaveBeenCalled(); + }); + + it('keeps the safe client for any other URL', async () => { + axiosGetMock.mockResolvedValueOnce(createStreamResponse()); + + await getImageBuffer('https://files.example.com/a.png', { + trustedOrigin: 'http://10.0.0.5:8080' + }); + + expect(axiosGetMock).toHaveBeenCalledTimes(1); + expect(axiosWithoutSSRFGetMock).not.toHaveBeenCalled(); + }); +}); diff --git a/packages/service/test/core/ai/llm/agentLoop/fastAgentLoop.test.ts b/packages/service/test/core/ai/llm/agentLoop/fastAgentLoop.test.ts index 7c3d779f4d7c..f8ee56b44329 100644 --- a/packages/service/test/core/ai/llm/agentLoop/fastAgentLoop.test.ts +++ b/packages/service/test/core/ai/llm/agentLoop/fastAgentLoop.test.ts @@ -64,7 +64,7 @@ import { runFastAgentMainLoop, type AgentLoopRuntime } from '@fastgpt/service/core/ai/llm/agentLoop/provider/fastAgent/loop'; -import { buildToolMessageContent } from '@fastgpt/service/core/ai/llm/agentLoop/provider/fastAgent/loop/base'; +import { buildToolMessageContent } from '@fastgpt/service/core/ai/llm/agentLoop/domain/tool'; const getFinalAssistantText = (result: { assistantMessages: any[] }) => result.assistantMessages diff --git a/packages/service/test/core/ai/llm/agentLoop/piAgentToolContent.test.ts b/packages/service/test/core/ai/llm/agentLoop/piAgentToolContent.test.ts new file mode 100644 index 000000000000..8c0eb6995ab5 --- /dev/null +++ b/packages/service/test/core/ai/llm/agentLoop/piAgentToolContent.test.ts @@ -0,0 +1,85 @@ +import { describe, expect, it, vi } from 'vitest'; +import { buildPiAgentTools } from '@fastgpt/service/core/ai/llm/agentLoop/provider/piAgent/tool/build'; + +const createRuntime = ({ + useVision = true, + executeTool = vi.fn() +}: { + useVision?: boolean; + executeTool?: ReturnType; +}) => + ({ + llmParams: { + useVision, + model: { config: { vision: useVision } } + }, + toolCatalog: { + runtimeTools: [ + { + type: 'function', + function: { + name: 'get_task', + description: 'get task detail', + parameters: { type: 'object', properties: {} } + } + } + ] + }, + executeTool, + emitEvent: vi.fn(), + usagePush: vi.fn() + }) as any; + +const buildTools = (runtime: any) => + buildPiAgentTools({ + runtime, + getActivePlan: () => undefined, + setActivePlan: vi.fn(), + setPendingAsk: vi.fn(), + getMessages: () => [], + onToolCall: vi.fn(), + onToolResult: vi.fn() + }); + +describe('piAgent tool content composition', () => { + it('keeps the business response text when the tool returns an image part', async () => { + const response = '{"jobId":"task-123"}'; + const executeTool = vi.fn(async () => ({ + response, + content: [{ type: 'image_url', image_url: { url: 'data:image/png;base64,AAAA' } }], + assistantMessages: [], + usages: [], + stop: false + })); + + const tools = await buildTools(createRuntime({ executeTool })); + const tool = tools.find((item) => item.name === 'get_task'); + const result: any = await tool!.execute('call_1', {}); + + // jobId 这类业务字段必须继续发给模型，图片只作为附加 part。 + expect(result.content).toEqual([ + { type: 'text', text: response }, + { type: 'image', mimeType: 'image/png', data: 'AAAA' } + ]); + }); + + it('degrades the image part to text when the model has no vision', async () => { + const response = '{"jobId":"task-123"}'; + const executeTool = vi.fn(async () => ({ + response, + content: [{ type: 'image_url', image_url: { url: 'https://files.example.com/chart.png' } }], + assistantMessages: [], + usages: [], + stop: false + })); + + const tools = await buildTools(createRuntime({ useVision: false, executeTool })); + const tool = tools.find((item) => item.name === 'get_task'); + const result: any = await tool!.execute('call_1', {}); + + expect(result.content).toEqual([ + { type: 'text', text: response }, + { type: 'text', text: '[Image: https://files.example.com/chart.png]' } + ]); + }); +}); diff --git a/packages/service/test/core/ai/llm/utils.test.ts b/packages/service/test/core/ai/llm/utils.test.ts index dde4005154fd..977aa4bd4c40 100644 --- a/packages/service/test/core/ai/llm/utils.test.ts +++ b/packages/service/test/core/ai/llm/utils.test.ts @@ -1523,8 +1523,10 @@ describe('loadRequestMessages function tests', () => { expect(result[2].content).toEqual([ { type: 'image_url', image_url: { url: 'data:image/png;base64,inlined' } } ]); + // 沙箱预览代理可能在内网，取图时按配置 origin 走受限通道。 expect(mockGetImageBase64).toHaveBeenCalledWith( - 'https://sandbox-preview.example.com/preview/sandbox-1/session-1/a.png' + 'https://sandbox-preview.example.com/preview/sandbox-1/session-1/a.png', + { trustedOrigin: 'https://sandbox-preview.example.com' } ); }); diff --git a/packages/service/test/core/workflow/dispatch/ai/agentLoopCore/runtime/workflowToolRunner.test.ts b/packages/service/test/core/workflow/dispatch/ai/agentLoopCore/runtime/workflowToolRunner.test.ts index 6f54bc21fbc2..bb7473cd38e3 100644 --- a/packages/service/test/core/workflow/dispatch/ai/agentLoopCore/runtime/workflowToolRunner.test.ts +++ b/packages/service/test/core/workflow/dispatch/ai/agentLoopCore/runtime/workflowToolRunner.test.ts @@ -1,8 +1,9 @@ import { beforeEach, describe, expect, it, vi } from 'vitest'; +import { createAgentLoopCoreWorkflowToolRunner } from '@fastgpt/service/core/workflow/dispatch/ai/agentLoopCore/application/runtime/workflowToolRunner'; import { - createAgentLoopCoreWorkflowToolRunner, + convertMcpToolContentToToolContent, extractAgentLoopCoreToolContent -} from '@fastgpt/service/core/workflow/dispatch/ai/agentLoopCore/application/runtime/workflowToolRunner'; +} from '@fastgpt/service/core/workflow/dispatch/ai/agentLoopCore/domain/toolContent'; import { dispatchUpdateVariable } from '@fastgpt/service/core/workflow/dispatch/tools/runUpdateVar'; const createCall = ({ @@ -657,3 +658,30 @@ describe('extractAgentLoopCoreToolContent', () => { ).toBeUndefined(); }); }); + +describe('convertMcpToolContentToToolContent', () => { + it('converts MCP image blocks into data URI image parts', () => { + // MCP 原生图片块是 { type: 'image', data, mimeType }，不是 image_url。 + expect( + convertMcpToolContentToToolContent([ + { type: 'text', text: 'chart ready' }, + { type: 'image', data: 'AAAA', mimeType: 'image/png' } + ]) + ).toEqual([{ type: 'image_url', image_url: { url: 'data:image/png;base64,AAAA' } }]); + }); + + it('returns undefined when the result has no image block', () => { + expect(convertMcpToolContentToToolContent([{ type: 'text', text: 'ok' }])).toBeUndefined(); + expect(convertMcpToolContentToToolContent(undefined)).toBeUndefined(); + expect(convertMcpToolContentToToolContent([])).toBeUndefined(); + }); + + it('drops image blocks with unusable payload', () => { + expect( + convertMcpToolContentToToolContent([{ type: 'image', data: '', mimeType: 'image/png' }]) + ).toBeUndefined(); + expect( + convertMcpToolContentToToolContent([{ type: 'image', data: 'AAAA', mimeType: 'text/html' }]) + ).toBeUndefined(); + }); +}); From e0ae9c769a3b5e3404112982f8f772a011944627 Mon Sep 17 00:00:00 2001 From: lirenjie Date: Thu, 29 Oct 2026 20:33:58 +0800 Subject: [PATCH 4/5] fix(image): stop following redirects on the trusted preview channel MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 可信 origin 通道关闭自动重定向，并把非 2xx 直接当失败， 避免预览代理/网关返回的跨 origin 3xx 让后续请求绕过内网校验 - 新增真实 HTTP 服务用例：配置 origin 直接取图仍成功，跨 origin 302 不会 请求跳转目标；mock 用例补上 status 字段 --- packages/service/common/file/image/utils.ts | 18 ++++- .../common/file/image/trustedOrigin.test.ts | 1 + .../file/image/trustedOriginRedirect.test.ts | 76 +++++++++++++++++++ 3 files changed, 93 insertions(+), 2 deletions(-) create mode 100644 packages/service/test/common/file/image/trustedOriginRedirect.test.ts diff --git a/packages/service/common/file/image/utils.ts b/packages/service/common/file/image/utils.ts index fbdf74823c05..9d2a893db357 100644 --- a/packages/service/common/file/image/utils.ts +++ b/packages/service/common/file/image/utils.ts @@ -140,6 +140,9 @@ export const isTrustedOriginUrl = (url: string, trustedOrigin?: string) => { * 该函数用于文档解析链路中转存 markdown http 图片，因此默认限制为 180 秒和 10MB： * 先用 Content-Length 快速拒绝明显超限的资源，下载过程中再按累计字节数中断，避免 * 第三方图片拖住解析任务或把大响应一次性读进内存。 + * + * 可信 origin 通道关闭自动重定向：safe axios 的每一跳都会做内网校验，而无 SSRF 通道 + * 一旦跟随 3xx 就会把“只放行配置 origin”扩大成任意地址，因此这里宁可让重定向失败。 */ export const getImageBuffer = async ( url: string, @@ -155,14 +158,25 @@ export const getImageBuffer = async ( try { const timeoutMs = options.timeoutMs ?? DEFAULT_IMAGE_DOWNLOAD_TIMEOUT_MS; const maxSize = options.maxSize ?? DEFAULT_IMAGE_DOWNLOAD_MAX_SIZE; - const requestClient = isTrustedOriginUrl(url, options.trustedOrigin) ? axiosWithoutSSRF : axios; + const isTrustedOrigin = isTrustedOriginUrl(url, options.trustedOrigin); + const requestClient = isTrustedOrigin ? axiosWithoutSSRF : axios; const response = await requestClient.get(url, { baseURL: serverRequestBaseUrl, responseType: 'stream', timeout: timeoutMs, - maxContentLength: maxSize + maxContentLength: maxSize, + // 不跟随 3xx：跨 origin 跳转会绕开内网校验，同 origin 跳转也不值得放开。 + ...(isTrustedOrigin ? { maxRedirects: 0 } : {}) }); + // 可信通道不跟随重定向时，3xx 会作为普通响应返回；必须显式拒绝， + // 否则会把跳转响应当成图片内容继续读取。 + const status = response?.status ?? 0; + if (status < 200 || status >= 300) { + response.data?.destroy?.(); + throw new Error(`Unexpected image response status: ${status}`); + } + const contentLength = Number(getAxiosHeaderValue(response?.headers?.['content-length']) || 0); if (contentLength > maxSize) { response.data?.destroy?.(); diff --git a/packages/service/test/common/file/image/trustedOrigin.test.ts b/packages/service/test/common/file/image/trustedOrigin.test.ts index a34c596d8a5f..1bc6656166ae 100644 --- a/packages/service/test/common/file/image/trustedOrigin.test.ts +++ b/packages/service/test/common/file/image/trustedOrigin.test.ts @@ -20,6 +20,7 @@ vi.mock('@fastgpt/service/common/api/serverRequest', () => ({ })); const createStreamResponse = () => ({ + status: 200, headers: { 'content-length': '4', 'content-type': 'image/png' }, data: Readable.from([Buffer.from([0x89, 0x50, 0x4e, 0x47])]) }); diff --git a/packages/service/test/common/file/image/trustedOriginRedirect.test.ts b/packages/service/test/common/file/image/trustedOriginRedirect.test.ts new file mode 100644 index 000000000000..25140ca100b1 --- /dev/null +++ b/packages/service/test/common/file/image/trustedOriginRedirect.test.ts @@ -0,0 +1,76 @@ +import http from 'node:http'; +import type { AddressInfo } from 'node:net'; +import { afterAll, beforeAll, describe, expect, it } from 'vitest'; + +// 真实 HTTP 服务，不走 mock：验证可信 origin 通道不会跟随跨 origin 3xx。 +const { getImageBuffer } = await import('@fastgpt/service/common/file/image/utils'); + +const PNG = Buffer.from([0x89, 0x50, 0x4e, 0x47]); + +const listen = (handler: http.RequestListener) => + new Promise((resolve) => { + const server = http.createServer(handler); + server.listen(0, '127.0.0.1', () => resolve(server)); + }); + +const getPort = (server: http.Server) => (server.address() as AddressInfo).port; + +let trustedServer: http.Server; +let otherServer: http.Server; +let trustedPort = 0; +let otherPort = 0; +const hits = { trusted: 0, other: 0 }; + +beforeAll(async () => { + otherServer = await listen((_req, res) => { + hits.other += 1; + res.writeHead(200, { 'content-type': 'image/png' }); + res.end(PNG); + }); + otherPort = getPort(otherServer); + + trustedServer = await listen((req, res) => { + hits.trusted += 1; + if (req.url?.startsWith('/preview/redirect')) { + // 预览代理/网关把请求跳到另一个 origin（这里用不同端口模拟）。 + res.writeHead(302, { location: `http://127.0.0.1:${otherPort}/evil.png` }); + res.end(); + return; + } + res.writeHead(200, { 'content-type': 'image/png' }); + res.end(PNG); + }); + trustedPort = getPort(trustedServer); +}); + +afterAll(async () => { + await Promise.all([ + new Promise((resolve) => trustedServer.close(resolve)), + new Promise((resolve) => otherServer.close(resolve)) + ]); +}); + +describe('trusted origin image download', () => { + it('downloads a direct image from the configured origin', async () => { + const result = await getImageBuffer(`http://127.0.0.1:${trustedPort}/preview/a.png`, { + trustedOrigin: `http://127.0.0.1:${trustedPort}` + }); + + expect(result.mime).toBe('image/png'); + expect(result.buffer.length).toBe(PNG.length); + }); + + it('does not follow a cross-origin redirect on the trusted channel', async () => { + const otherHitsBefore = hits.other; + + // 302 直接作为错误返回（axios 不跟随、validateStatus 拒绝），不会去请求跳转目标。 + await expect( + getImageBuffer(`http://127.0.0.1:${trustedPort}/preview/redirect.png`, { + trustedOrigin: `http://127.0.0.1:${trustedPort}` + }) + ).rejects.toThrow(/status code 302/); + + // 关键：跳转目标一次都不能被请求，否则等于绕过内网校验。 + expect(hits.other).toBe(otherHitsBefore); + }); +}); From 3024c74af56fa66a1486140785dc41abee6a13d1 Mon Sep 17 00:00:00 2001 From: lirenjie Date: Thu, 29 Oct 2026 20:50:02 +0800 Subject: [PATCH 5/5] fix(image): tolerate responses without status and match the redirect test helper - Enforce the status check only when the response carries a numeric status, so the existing mocked cases in common/file/image/utils.test.ts keep working; real axios responses always include one. - Rewrite the trusted-origin redirect case with the same listen/closeServer/getServerPort helpers and Chinese titles used by the safe axios redirect suite in common/api/axios.test.ts. --- packages/service/common/file/image/utils.ts | 6 +- .../file/image/trustedOriginRedirect.test.ts | 60 ++++++++++--------- 2 files changed, 36 insertions(+), 30 deletions(-) diff --git a/packages/service/common/file/image/utils.ts b/packages/service/common/file/image/utils.ts index 9d2a893db357..0baecd583bac 100644 --- a/packages/service/common/file/image/utils.ts +++ b/packages/service/common/file/image/utils.ts @@ -170,9 +170,9 @@ export const getImageBuffer = async ( }); // 可信通道不跟随重定向时，3xx 会作为普通响应返回；必须显式拒绝， - // 否则会把跳转响应当成图片内容继续读取。 - const status = response?.status ?? 0; - if (status < 200 || status >= 300) { + // 否则会把跳转响应当成图片内容继续读取。真实 axios 响应一定带 status。 + const status = response?.status; + if (typeof status === 'number' && (status < 200 || status >= 300)) { response.data?.destroy?.(); throw new Error(`Unexpected image response status: ${status}`); } diff --git a/packages/service/test/common/file/image/trustedOriginRedirect.test.ts b/packages/service/test/common/file/image/trustedOriginRedirect.test.ts index 25140ca100b1..311d1f569b74 100644 --- a/packages/service/test/common/file/image/trustedOriginRedirect.test.ts +++ b/packages/service/test/common/file/image/trustedOriginRedirect.test.ts @@ -1,76 +1,82 @@ import http from 'node:http'; -import type { AddressInfo } from 'node:net'; import { afterAll, beforeAll, describe, expect, it } from 'vitest'; -// 真实 HTTP 服务，不走 mock：验证可信 origin 通道不会跟随跨 origin 3xx。 const { getImageBuffer } = await import('@fastgpt/service/common/file/image/utils'); +// 与 common/api/axios.test.ts 的 redirect 用例保持同一套写法： +// 真实 http.createServer + listen(0)，验证可信 origin 通道不跟随跨 origin 3xx。 const PNG = Buffer.from([0x89, 0x50, 0x4e, 0x47]); -const listen = (handler: http.RequestListener) => +const listen = (handler: http.RequestListener, host = '127.0.0.1') => new Promise((resolve) => { const server = http.createServer(handler); - server.listen(0, '127.0.0.1', () => resolve(server)); + server.listen(0, host, () => resolve(server)); }); -const getPort = (server: http.Server) => (server.address() as AddressInfo).port; +const closeServer = (server: http.Server) => + new Promise((resolve, reject) => { + server.close((err) => (err ? reject(err) : resolve())); + }); + +const getServerPort = (server: http.Server): number => { + const address = server.address(); + if (!address || typeof address === 'string') { + throw new Error('Invalid test server address'); + } + return address.port; +}; let trustedServer: http.Server; let otherServer: http.Server; -let trustedPort = 0; -let otherPort = 0; -const hits = { trusted: 0, other: 0 }; +let trustedBaseUrl = ''; +let otherHits = 0; beforeAll(async () => { otherServer = await listen((_req, res) => { - hits.other += 1; + otherHits += 1; res.writeHead(200, { 'content-type': 'image/png' }); res.end(PNG); }); - otherPort = getPort(otherServer); + const otherBaseUrl = `http://127.0.0.1:${getServerPort(otherServer)}`; trustedServer = await listen((req, res) => { - hits.trusted += 1; + // 预览代理/网关把请求跳到另一个 origin（这里用不同端口模拟）。 if (req.url?.startsWith('/preview/redirect')) { - // 预览代理/网关把请求跳到另一个 origin（这里用不同端口模拟）。 - res.writeHead(302, { location: `http://127.0.0.1:${otherPort}/evil.png` }); + res.writeHead(302, { location: `${otherBaseUrl}/evil.png` }); res.end(); return; } res.writeHead(200, { 'content-type': 'image/png' }); res.end(PNG); }); - trustedPort = getPort(trustedServer); + trustedBaseUrl = `http://127.0.0.1:${getServerPort(trustedServer)}`; }); afterAll(async () => { - await Promise.all([ - new Promise((resolve) => trustedServer.close(resolve)), - new Promise((resolve) => otherServer.close(resolve)) - ]); + await Promise.all([closeServer(trustedServer), closeServer(otherServer)]); }); describe('trusted origin image download', () => { - it('downloads a direct image from the configured origin', async () => { - const result = await getImageBuffer(`http://127.0.0.1:${trustedPort}/preview/a.png`, { - trustedOrigin: `http://127.0.0.1:${trustedPort}` + it('应该下载配置 origin 上的图片', async () => { + const result = await getImageBuffer(`${trustedBaseUrl}/preview/a.png`, { + trustedOrigin: trustedBaseUrl }); expect(result.mime).toBe('image/png'); expect(result.buffer.length).toBe(PNG.length); }); - it('does not follow a cross-origin redirect on the trusted channel', async () => { - const otherHitsBefore = hits.other; + it('不应该跟随可信通道上的跨 origin 重定向', async () => { + const otherHitsBefore = otherHits; - // 302 直接作为错误返回（axios 不跟随、validateStatus 拒绝），不会去请求跳转目标。 + // 302 直接作为错误返回（不跟随 + 非 2xx 拒绝），不会去请求跳转目标。 await expect( - getImageBuffer(`http://127.0.0.1:${trustedPort}/preview/redirect.png`, { - trustedOrigin: `http://127.0.0.1:${trustedPort}` + getImageBuffer(`${trustedBaseUrl}/preview/redirect.png`, { + trustedOrigin: trustedBaseUrl }) ).rejects.toThrow(/status code 302/); // 关键：跳转目标一次都不能被请求，否则等于绕过内网校验。 - expect(hits.other).toBe(otherHitsBefore); + expect(otherHits).toBe(otherHitsBefore); }); });