diff --git a/packages/global/common/file/constants.ts b/packages/global/common/file/constants.ts index 94d4720c72ad..3bbcecebcf9b 100644 --- a/packages/global/common/file/constants.ts +++ b/packages/global/common/file/constants.ts @@ -58,4 +58,21 @@ export const imageFileType = export const audioFileType = '.mp3, .wav, .ogg, .m4a, .amr, .mpga'; +/** + * 主流视觉模型可消费的栅格图片格式。 + * + * `imageFileType` 还包含 svg / psd / raw / eps 等格式：把这些当图片发给供应商大概率被拒， + * 反而丢掉原本可读的文本内容，所以视觉得到图片链路统一按这份白名单过滤。 + */ +export const providerImageFileType = '.png, .jpg, .jpeg, .gif, .webp, .bmp'; +export const providerImageMimeType = 'image/png, image/jpeg, image/gif, image/webp, image/bmp'; + +/** + * 单次工具结果最多携带多少张图片。 + * + * 工具可以一次列出很多文件，若全部转成视觉输入，后续每轮都要重复下载并内联 base64， + * 很容易把请求撑到供应商上限；超出的文件仍保留在文本链接里。 + */ +export const maxToolImagePartAmount = 4; + export const videoFileType = '.mp4, .mov, .avi, .mpeg, .webm'; diff --git a/packages/global/core/ai/llm/type.ts b/packages/global/core/ai/llm/type.ts index 432cefc6f0c0..46d0166a56bc 100644 --- a/packages/global/core/ai/llm/type.ts +++ b/packages/global/core/ai/llm/type.ts @@ -129,12 +129,28 @@ export type ChatCompletionUserMessageParam = z.infer; + +/** + * Tool message: 对齐 openai SDK 的 ChatCompletionToolMessageParam，新增可选 `name`。 + * content 在内部允许带 image_url part（见上），出站时会转换；供应商只认字符串或文本 parts。 */ export const ChatCompletionToolMessageParamSchema = z.object({ role: z.literal('tool'), - content: z.union([z.string(), z.array(ChatCompletionContentPartTextSchema)]), + content: z.union([z.string(), z.array(ChatCompletionToolMessageContentPartSchema)]), tool_call_id: z.string(), name: z.string().optional() }); diff --git a/packages/global/core/chat/adapt.ts b/packages/global/core/chat/adapt.ts index 50c4eaa670c2..54c8b679d175 100644 --- a/packages/global/core/chat/adapt.ts +++ b/packages/global/core/chat/adapt.ts @@ -12,6 +12,7 @@ import type { import { ChatFileTypeEnum, ChatRoleEnum } from '../../core/chat/constants'; import type { ChatCompletionContentPart, + ChatCompletionContentPartText, ChatCompletionFunctionMessageParam, ChatCompletionMessageFunctionCall, ChatCompletionMessageParam, @@ -842,6 +843,26 @@ export const chats2GPTMessages = ({ return results; }; +/** + * 提取工具响应的纯文本部分。 + * + * tool content 允许是字符串或 parts 数组。折回 ChatItem 时只保留 text parts: + * 工具卡片展示与历史持久化都基于文本,image_url 等富媒体内容不进入这条链路, + * 避免把数组整体 JSON 化后污染展示与后续轮次读取。 + */ +const getToolResponseText = (content: ChatCompletionMessageParam['content']): string => { + if (typeof content === 'string') return content; + if (!Array.isArray(content)) return ''; + return content + .filter( + (part): part is ChatCompletionContentPartText => + typeof part === 'object' && part !== null && part.type === 'text' + ) + .map((part) => part.text) + .filter(Boolean) + .join('\n'); +}; + /** * 将 GPT messages 转回 FastGPT ChatItem。 * @@ -969,14 +990,12 @@ export const GPTMessages2Chats = ({ const toolCalls = item.tool_calls as ChatCompletionMessageToolCall[]; const tools = toolCalls.flatMap((tool) => { - let toolResponse = - normalizedMessages.find( - (msg) => - msg.role === ChatCompletionRequestMessageRoleEnum.Tool && - msg.tool_call_id === tool.id - )?.content || ''; - toolResponse = - typeof toolResponse === 'string' ? toolResponse : JSON.stringify(toolResponse); + const toolContent = normalizedMessages.find( + (msg) => + msg.role === ChatCompletionRequestMessageRoleEnum.Tool && + msg.tool_call_id === tool.id + )?.content; + const toolResponse = getToolResponseText(toolContent); const toolInfo = getToolInfo?.(tool.function.name); diff --git a/packages/global/test/core/chat/adapt.test.ts b/packages/global/test/core/chat/adapt.test.ts index f2f09dff98de..b012ff1ae736 100644 --- a/packages/global/test/core/chat/adapt.test.ts +++ b/packages/global/test/core/chat/adapt.test.ts @@ -3145,7 +3145,7 @@ describe('GPTMessages2Chats', () => { expect(toolValue.tools[0].response).toBe('{"results": []}'); }); - it('should stringify non-string tool response content', () => { + it('should keep text parts when folding non-string tool response content', () => { const messages: ChatCompletionMessageParam[] = [ { role: ChatCompletionRequestMessageRoleEnum.Assistant, @@ -3170,9 +3170,7 @@ describe('GPTMessages2Chats', () => { const result = GPTMessages2Chats({ messages, reserveTool: true }); expect(result).toHaveLength(1); - expect((result[0].value[0] as any).tools[0].response).toBe( - '[{"type":"text","text":"chunked result"}]' - ); + expect((result[0].value[0] as any).tools[0].response).toBe('chunked result'); }); it('should keep assistant content and reasoning separate from tools after chat roundtrip', () => { @@ -3642,6 +3640,40 @@ describe('GPTMessages2Chats', () => { expect(toolValue.tools[0].toolName).toBe('Custom Tool Display Name'); expect(toolValue.tools[0].toolAvatar).toBe('http://example.com/avatar.png'); }); + + it('should keep only text parts when folding tool content parts back to chat items', () => { + const messages: ChatCompletionMessageParam[] = [ + { + role: ChatCompletionRequestMessageRoleEnum.Assistant, + content: '', + tool_calls: [ + { + id: 'call_1', + type: 'function', + function: { + name: 'custom_tool', + arguments: '{}' + } + } + ] + }, + { + role: ChatCompletionRequestMessageRoleEnum.Tool, + tool_call_id: 'call_1', + content: [ + { type: 'text', text: 'Look at these images' }, + { type: 'image_url', image_url: { url: 'https://example.com/a.png' } } + ] + } + ]; + + const result = GPTMessages2Chats({ messages, reserveTool: true }); + + const toolValue = result[0].value[0] as any; + expect(toolValue.tools).toBeDefined(); + expect(toolValue.tools).toHaveLength(1); + expect(toolValue.tools[0].response).toBe('Look at these images'); + }); }); describe('chatValue2RuntimePrompt', () => { diff --git a/packages/service/common/file/image/utils.ts b/packages/service/common/file/image/utils.ts index 799e1bc0072f..4545175986d9 100644 --- a/packages/service/common/file/image/utils.ts +++ b/packages/service/common/file/image/utils.ts @@ -1,4 +1,4 @@ -import { axios } from '../../api/axios'; +import { axios, axiosWithoutSSRF } from '../../api/axios'; import { serverRequestBaseUrl } from '../../api/serverRequest'; import { getAxiosContentType, getAxiosHeaderValue } from '@fastgpt/global/common/axios/utils'; import { getLogger, LogCategories } from '../../logger'; @@ -115,18 +115,42 @@ export const guessBase64ImageType = (str: string): string => { return BASE64_PREFIX_MAP[str.charAt(0)] || DEFAULT_IMAGE_TYPE; }; +/** + * 判断一次图片下载是否命中调用方声明的可信 origin。 + * + * 沙箱预览地址来自服务端配置（AGENT_SANDBOX_PREVIEW_PROXY_URL），可能部署在内网， + * 不能被公共 SSRF 内网拦截误杀；这类地址不是用户任意输入，因此允许对该 origin 走 + * 受限通道。origin 之外的地址仍然走 safe axios，避免把“可信”扩大成任意内网访问。 + */ +export const isTrustedOriginUrl = (url: string, trustedOrigin?: string) => { + if (!trustedOrigin) return false; + + try { + const target = new URL(url, serverRequestBaseUrl); + const trusted = new URL(trustedOrigin, serverRequestBaseUrl); + return target.origin === trusted.origin; + } catch { + return false; + } +}; + /** * 下载远程图片并返回 Buffer。 * * 该函数用于文档解析链路中转存 markdown http 图片，因此默认限制为 180 秒和 10MB： * 先用 Content-Length 快速拒绝明显超限的资源，下载过程中再按累计字节数中断，避免 * 第三方图片拖住解析任务或把大响应一次性读进内存。 + * + * 可信 origin 通道关闭自动重定向：safe axios 的每一跳都会做内网校验，而无 SSRF 通道 + * 一旦跟随 3xx 就会把“只放行配置 origin”扩大成任意地址，因此这里宁可让重定向失败。 */ export const getImageBuffer = async ( url: string, options: { timeoutMs?: number; maxSize?: number; + /** 可信 origin（如沙箱预览代理），命中时跳过公共内网拦截。 */ + trustedOrigin?: string; } = {} ) => { logger.debug('Load image to buffer', { url }); @@ -134,13 +158,25 @@ export const getImageBuffer = async ( try { const timeoutMs = options.timeoutMs ?? DEFAULT_IMAGE_DOWNLOAD_TIMEOUT_MS; const maxSize = options.maxSize ?? DEFAULT_IMAGE_DOWNLOAD_MAX_SIZE; - const response = await axios.get(url, { + const isTrustedOrigin = isTrustedOriginUrl(url, options.trustedOrigin); + const requestClient = isTrustedOrigin ? axiosWithoutSSRF : axios; + const response = await requestClient.get(url, { baseURL: serverRequestBaseUrl, responseType: 'stream', timeout: timeoutMs, - maxContentLength: maxSize + maxContentLength: maxSize, + // 不跟随 3xx：跨 origin 跳转会绕开内网校验，同 origin 跳转也不值得放开。 + ...(isTrustedOrigin ? { maxRedirects: 0 } : {}) }); + // 兜底校验：axios 默认 validateStatus 已拒绝非 2xx，这里再显式拒绝一次， + // 避免换成不抛错的自定义客户端时把跳转响应当成图片内容继续读取。 + const status = response?.status; + if (typeof status === 'number' && (status < 200 || status >= 300)) { + response.data?.destroy?.(); + throw new Error(`Unexpected image response status: ${status}`); + } + const contentLength = Number(getAxiosHeaderValue(response?.headers?.['content-length']) || 0); if (contentLength > maxSize) { response.data?.destroy?.(); @@ -205,6 +241,8 @@ export const getImageBase64 = async ( timeoutMs?: number; maxSize?: number; maxBase64BufferSize?: number; + /** 可信 origin（如沙箱预览代理），命中时跳过公共内网拦截。 */ + trustedOrigin?: string; } = {} ) => { logger.debug('Load image to base64', { url }); @@ -212,7 +250,8 @@ export const getImageBase64 = async ( try { const { buffer, mime } = await getImageBuffer(url, { timeoutMs: options.timeoutMs, - maxSize: options.maxSize + maxSize: options.maxSize, + trustedOrigin: options.trustedOrigin }); const maxBase64BufferSize = options.maxBase64BufferSize ?? DEFAULT_IMAGE_BASE64_MAX_BUFFER_SIZE; diff --git a/packages/service/core/ai/llm/agentLoop/domain/tool.ts b/packages/service/core/ai/llm/agentLoop/domain/tool.ts index 944889ba184f..f9ac63dc53f5 100644 --- a/packages/service/core/ai/llm/agentLoop/domain/tool.ts +++ b/packages/service/core/ai/llm/agentLoop/domain/tool.ts @@ -1,8 +1,10 @@ import type { ChatCompletionMessageParam, ChatCompletionMessageToolCall, - ChatCompletionTool + ChatCompletionTool, + ChatCompletionToolMessageContentPart } from '@fastgpt/global/core/ai/llm/type'; +import { maxToolImagePartAmount } from '@fastgpt/global/common/file/constants'; import type { SandboxClient } from '../../../sandbox/application/runtime/client'; import type { SandboxToolCallValidator } from '../../../sandbox/application/toolCall/type'; import type { AgentLoopDatasetSearchExecutor } from './systemTool/datasetSearch'; @@ -14,6 +16,32 @@ export type AgentLoopToolCatalog = { batchToolSize?: number; }; +/** + * 构造写入 tool message 的 content： + * - 无结构化 parts 时直接用文本字符串； + * - 有 parts 时把最终要发送的文本作为 text part 拼在图片等 parts 前， + * 保证模型实际拿到的文本与展示/计费统计一致（文本已压缩时传压缩后的值）。 + * + * fastAgent 与 piAgent 共用这层组合规则，避免两条 provider 路径对同一个工具结果 + * 给出不同的内容：业务 response 文本永远保留，图片只作为附加 part。 + * + * 图片数量按 `maxToolImagePartAmount` 截断：一次工具调用列出很多文件时，每轮请求都会重复 + * 下载并内联这些图片，必须限制单个工具结果的视觉输入量。 + */ +export const buildToolMessageContent = ({ + content, + text +}: { + content?: ChatCompletionToolMessageContentPart[]; + text: string; +}): string | ChatCompletionToolMessageContentPart[] => { + if (!content?.length) return text; + + const images = content.filter((part) => part.type !== 'text').slice(0, maxToolImagePartAmount); + + return images.length ? [{ type: 'text', text }, ...images] : text; +}; + /** * 工具内部产生的可持久化 assistant 展示项。 * @@ -53,6 +81,11 @@ export type AgentLoopToolExecutionResult = { hideInUI?: boolean; /** 由调用方透传并在 agent-loop 外部解释的工具运行元数据。 */ metadata?: unknown; + /** + * 写入 LLM tool message 的结构化 content（text / image_url parts）。 + * 工具取到图片时可返回 image_url part，作为下一轮模型的视觉输入；缺省时按 response 字符串兜底。 + */ + content?: ChatCompletionToolMessageContentPart[]; }; export type AgentLoopReadFileExecutionResult = { diff --git a/packages/service/core/ai/llm/agentLoop/provider/fastAgent/loop/base.ts b/packages/service/core/ai/llm/agentLoop/provider/fastAgent/loop/base.ts index 5fd5f4bb2f43..bed7eb2ada53 100644 --- a/packages/service/core/ai/llm/agentLoop/provider/fastAgent/loop/base.ts +++ b/packages/service/core/ai/llm/agentLoop/provider/fastAgent/loop/base.ts @@ -15,6 +15,7 @@ import { countGptMessagesTokens } from '../../../../../../../common/string/tikto import { formatModelChars2Points } from '../../../../../../../support/wallet/usage/utils'; import type { LLMModelDataType } from '@fastgpt/global/core/ai/model/schema'; import type { AgentLoopAssistantResponse } from '../../../domain/tool'; +import { buildToolMessageContent } from '../../../domain/tool'; import type { AgentLoopChildrenInteractiveParams, AgentLoopCompletionPolicy, @@ -305,7 +306,8 @@ export const runAgentLoop = async ({ interactive, stop, errorMessage, - metadata + metadata, + content: toolContent } = await (async () => { try { return await onRunInteractiveTool({ @@ -338,11 +340,16 @@ export const runAgentLoop = async ({ }); // FastAgent 只使用 ChatItem 重建的上下文，旧交互中的 memory 快照不再参与恢复。 + // 与普通工具执行路径保持一致：结构化 parts 与最终文本统一通过 buildToolMessageContent 组装。 + const interactiveToolContent = buildToolMessageContent({ + content: toolContent, + text: normalizeToolResponseContent(response) + }); requestMessages = requestMessages.map((item) => item.role === 'tool' && item.tool_call_id === childrenInteractiveParams.toolCallId ? { ...item, - content: normalizeToolResponseContent(response) + content: interactiveToolContent } : item ); @@ -352,7 +359,7 @@ export const runAgentLoop = async ({ assistantMessages.push({ role: ChatCompletionRequestMessageRoleEnum.Tool, tool_call_id: childrenInteractiveParams.toolCallId, - content: response + content: interactiveToolContent }); assistantMessages.push(...filterEmptyAssistantMessages(toolAssistantMessages)); // 相同 tool 触发了多次交互, 调用的 toolId 认为是相同的 @@ -568,7 +575,8 @@ export const runAgentLoop = async ({ skipResponseCompress, errorMessage, hideInUI, - metadata + metadata, + content: toolContent } = await (async () => { try { return await onRunTool({ @@ -638,6 +646,14 @@ export const runAgentLoop = async ({ interactive }); + // 工具返回结构化 content（如含图片 parts）时，在保留图片的前提下， + // 把展示/计费用的最终文本（可能已压缩）作为 text part 一并发送； + // 否则直接使用最终文本字符串。 + const toolMessageContent = buildToolMessageContent({ + content: toolContent, + text: toolFinalResponse + }); + return { tool, interactive, @@ -645,7 +661,7 @@ export const runAgentLoop = async ({ toolMessage: { tool_call_id: tool.id, role: ChatCompletionRequestMessageRoleEnum.Tool, - content: toolFinalResponse + content: toolMessageContent } as ChatCompletionMessageParam, toolAssistantMessages: filterEmptyAssistantMessages(toolAssistantMessages) }; diff --git a/packages/service/core/ai/llm/agentLoop/provider/fastAgent/loop/index.ts b/packages/service/core/ai/llm/agentLoop/provider/fastAgent/loop/index.ts index b4971139bb68..0562d3241d04 100644 --- a/packages/service/core/ai/llm/agentLoop/provider/fastAgent/loop/index.ts +++ b/packages/service/core/ai/llm/agentLoop/provider/fastAgent/loop/index.ts @@ -375,7 +375,8 @@ export const runFastAgentMainLoop = async ({ }); return createToolResponse(sandboxResult.response, { skipResponseCompress: true, - errorMessage: sandboxResult.success ? undefined : sandboxResult.response + errorMessage: sandboxResult.success ? undefined : sandboxResult.response, + content: sandboxResult.content }); } diff --git a/packages/service/core/ai/llm/agentLoop/provider/piAgent/message.ts b/packages/service/core/ai/llm/agentLoop/provider/piAgent/message.ts index de0f10c50eab..d08452d969f9 100644 --- a/packages/service/core/ai/llm/agentLoop/provider/piAgent/message.ts +++ b/packages/service/core/ai/llm/agentLoop/provider/piAgent/message.ts @@ -13,8 +13,10 @@ import type { ChatCompletionMessageParam, ChatCompletionMessageToolCall, ChatCompletionTool, + ChatCompletionToolMessageContentPart, CompletionFinishReason } from '@fastgpt/global/core/ai/llm/type'; +import { loadImageBase64, maskImagePlaceholderUrl } from '../../../utils'; /** 将工具参数稳定序列化；不可序列化值回退为空对象，避免中断事件链。 */ export const stringifyJson = (value: unknown) => { @@ -81,12 +83,72 @@ const getContentText = ( content: | string | null - | Array<{ type: 'text'; text: string } | { type: 'refusal'; refusal: string }> + | Array< + | { type: 'text'; text: string } + | { type: 'refusal'; refusal: string } + | { type: 'image_url'; image_url: { url: string } } + > | undefined ) => { if (!content) return ''; if (typeof content === 'string') return content; - return content.map((item) => (item.type === 'text' ? item.text : item.refusal)).join(''); + return content + .map((item) => (item.type === 'text' ? item.text : item.type === 'refusal' ? item.refusal : '')) + .join(''); +}; + +/** + * 图片 part 的文本占位。 + * + * data URI 内联了整段 base64，原样铺进 prompt 会撑爆上下文，因此只保留 mime； + * 沙箱预览地址带一次性 session 凭证，需要脱敏后再写入文本。 + */ +const getImagePlaceholder = (url: string) => { + const dataMatch = url.match(/^data:([^;,]+)[;,]/); + if (dataMatch) return `[Image: ${dataMatch[1]}]`; + return `[Image: ${maskImagePlaceholderUrl(url)}]`; +}; + +/** + * 将工具返回的 content（字符串或 text / image_url parts）转换成 pi-agent 的 + * TextContent / ImageContent。图片 URL 取回 base64 后进入 transcript； + * 取图失败时降级为占位文本，避免中断 agent loop。 + * useVision 为 false 时图片 part 统一降级为占位文本，与 loadRequestMessages + * 的能力过滤语义一致，避免把图片发给不支持视觉的模型。 + */ +export const convertToolResponseContentToPiContent = async ( + content: string | ChatCompletionToolMessageContentPart[], + options?: { useVision?: boolean } +): Promise> => { + const useVision = options?.useVision ?? true; + if (typeof content === 'string') { + return [{ type: 'text', text: content }]; + } + + return Promise.all( + content.map(async (part): Promise => { + if (part.type === 'text') { + return { type: 'text', text: part.text }; + } + + const url = part.image_url.url; + if (!useVision) { + return { type: 'text', text: getImagePlaceholder(url) }; + } + const dataMatch = url.match(/^data:([^;]+);base64,(.+)$/); + if (dataMatch) { + return { type: 'image', mimeType: dataMatch[1], data: dataMatch[2] }; + } + + try { + // 与 fastAgent 共用取图通道：沙箱预览地址允许走配置 origin 的受限下载。 + const { base64, mime } = await loadImageBase64(url); + return { type: 'image', mimeType: mime, data: base64 }; + } catch { + return { type: 'text', text: getImagePlaceholder(url) }; + } + }) + ); }; /** @@ -178,12 +240,32 @@ export const convertChatMessagesToPiAgentMessages = ({ } if (message.role === ChatCompletionRequestMessageRoleEnum.Tool) { + // 工具消息允许 text / image_url parts：data URI 直接还原为 ImageContent， + // 远程图片 URL 与 user 分支一致降级为占位文本，不阻塞转录恢复。 + const content: Array = []; + if (Array.isArray(message.content)) { + message.content.forEach((part) => { + if (part.type === 'text') { + content.push({ type: 'text', text: part.text }); + return; + } + const match = part.image_url.url.match(/^data:([^;]+);base64,(.+)$/); + if (match) { + content.push({ type: 'image', mimeType: match[1], data: match[2] }); + return; + } + content.push({ type: 'text', text: getImagePlaceholder(part.image_url.url) }); + }); + } else { + content.push({ type: 'text', text: getContentText(message.content) }); + } + return [ { role: 'toolResult', toolCallId: message.tool_call_id, toolName: message.name ?? toolNames.get(message.tool_call_id) ?? '', - content: [{ type: 'text', text: getContentText(message.content) }], + content, details: {}, isError: false, timestamp: Date.now() @@ -304,18 +386,24 @@ export const replaceInteractiveToolResult = ({ messages, call, response, - isError + isError, + content }: { messages: AgentMessage[]; call: ChatCompletionMessageToolCall; response: string; isError: boolean; + /** 工具返回的图片 content（已转成 pi ImageContent），错误时不生效。 */ + content?: Array; }): AgentMessage[] => { + const nextContent = + !isError && content?.length ? content : [{ type: 'text' as const, text: response }]; + let replaced = false; const nextMessages = messages.map((message) => { if (message.role !== 'toolResult' || message.toolCallId !== call.id || replaced) return message; replaced = true; - return { ...message, content: [{ type: 'text' as const, text: response }], isError }; + return { ...message, content: nextContent, isError }; }); if (replaced) return nextMessages; @@ -325,7 +413,7 @@ export const replaceInteractiveToolResult = ({ role: 'toolResult', toolCallId: call.id, toolName: call.function.name, - content: [{ type: 'text', text: response }], + content: nextContent, details: {}, isError, timestamp: Date.now() diff --git a/packages/service/core/ai/llm/agentLoop/provider/piAgent/run.ts b/packages/service/core/ai/llm/agentLoop/provider/piAgent/run.ts index 46560c144cc3..60452cf972bd 100644 --- a/packages/service/core/ai/llm/agentLoop/provider/piAgent/run.ts +++ b/packages/service/core/ai/llm/agentLoop/provider/piAgent/run.ts @@ -2,6 +2,7 @@ import { Agent, type AgentEvent } from '@mariozechner/pi-agent-core'; import type { ChatCompletionMessageParam, ChatCompletionMessageToolCall, + ChatCompletionToolMessageContentPart, CompletionFinishReason } from '@fastgpt/global/core/ai/llm/type'; import { ChatCompletionRequestMessageRoleEnum } from '@fastgpt/global/core/ai/constants'; @@ -17,6 +18,7 @@ import { formatModelChars2Points } from '../../../../../../support/wallet/usage/ import { AgentUsageModuleName } from '../../domain/usage'; import { askUserToolName, formatAgentAskToolResponse } from '../../domain/systemTool/ask'; import { setPlanToolName, updatePlanToolName } from '../../domain/systemTool/plan'; +import { buildToolMessageContent } from '../../domain/tool'; import { normalizeAgentLoopUsages, type AgentLoopInput, @@ -33,6 +35,7 @@ import { compressRequestMessages } from '../../../compress'; import { convertChatMessagesToPiAgentMessages, convertPiAgentMessagesToChatMessages, + convertToolResponseContentToPiContent, getPiAgentPrompt, isAssistantMessage, mapStopReason, @@ -65,6 +68,8 @@ export const runPiAgentLoop = async ({ }): Promise> => { const state = readPiAgentProviderState(input.providerState); const modelData = runtime.llmParams.model; + // 显式转布尔：useVision 缺省时与 fastAgent / loadRequestMessages 的“按纯文本处理”一致。 + const useVision = !!(runtime.llmParams.useVision && modelData.config.vision); const piModel = buildPiModel( modelData, runtime.llmParams.useVision, @@ -73,7 +78,7 @@ export const runPiAgentLoop = async ({ ); const requestMessages = await loadRequestMessages({ messages: input.messages, - useVision: runtime.llmParams.useVision && modelData.config.vision, + useVision, useAudio: runtime.llmParams.useAudio && modelData.config.audio, useVideo: runtime.llmParams.useVideo && modelData.config.video, extractFiles: runtime.llmParams.extractFiles, @@ -170,17 +175,20 @@ export const runPiAgentLoop = async ({ call, response, childAssistantMessages = [], - appendToAssistantMessages = true + appendToAssistantMessages = true, + content }: { call: ChatCompletionMessageToolCall; response: string; childAssistantMessages?: ChatCompletionMessageParam[]; appendToAssistantMessages?: boolean; + /** 工具返回的结构化 content（text / image_url parts），缺省时回退 response 字符串。 */ + content?: string | ChatCompletionToolMessageContentPart[]; }) => { const toolMessage: ChatCompletionMessageParam = { role: ChatCompletionRequestMessageRoleEnum.Tool, tool_call_id: call.id, - content: response + content: content ?? response }; const existingToolIndex = completeMessages.findIndex( (message) => message.role === 'tool' && message.tool_call_id === call.id @@ -319,11 +327,18 @@ export const runPiAgentLoop = async ({ metadata: result.metadata, seconds: +((Date.now() - startedAt) / 1000).toFixed(2) }); + // 交互恢复与普通执行共用同一套组合规则：response 文本保留，图片追加在后。 + const toolContent = buildToolMessageContent({ + content: result.content, + text: normalizedResponse + }); + const piToolContent = await convertToolResponseContentToPiContent(toolContent, { useVision }); initialPiMessages = replaceInteractiveToolResult({ messages: initialPiMessages, call, response: normalizedResponse, - isError: !!result.errorMessage + isError: !!result.errorMessage, + content: piToolContent }); const providerState: PiAgentProviderState = { @@ -334,7 +349,8 @@ export const runPiAgentLoop = async ({ recordToolResult({ call, response: normalizedResponse, - childAssistantMessages: result.assistantMessages + childAssistantMessages: result.assistantMessages, + content: toolContent }); if (result.interactive) { return { @@ -397,11 +413,12 @@ export const runPiAgentLoop = async ({ }, getMessages: () => completeMessages, onToolCall: emitOrdinaryToolExecution, - onToolResult: ({ call, response, assistantMessages }) => { + onToolResult: ({ call, response, assistantMessages, content }) => { recordToolResult({ call, response, - childAssistantMessages: assistantMessages + childAssistantMessages: assistantMessages, + content }); } }); diff --git a/packages/service/core/ai/llm/agentLoop/provider/piAgent/tool/build.ts b/packages/service/core/ai/llm/agentLoop/provider/piAgent/tool/build.ts index 51a6e543fae8..743501fd3d7d 100644 --- a/packages/service/core/ai/llm/agentLoop/provider/piAgent/tool/build.ts +++ b/packages/service/core/ai/llm/agentLoop/provider/piAgent/tool/build.ts @@ -5,7 +5,8 @@ import { AgentAskPayloadSchema, type AgentAskPayload } from '@fastgpt/global/cor import { normalizeToolResponseContent } from '@fastgpt/global/core/ai/llm/utils'; import type { ChatCompletionMessageParam, - ChatCompletionMessageToolCall + ChatCompletionMessageToolCall, + ChatCompletionToolMessageContentPart } from '@fastgpt/global/core/ai/llm/type'; import { getErrText } from '@fastgpt/global/common/error/utils'; import { createAskUserAgentTool } from '../../../domain/systemTool/ask'; @@ -21,6 +22,7 @@ import { createDatasetSearchTool, patchDatasetSearchParams } from '../../../domain/systemTool/datasetSearch'; +import { buildToolMessageContent } from '../../../domain/tool'; import { normalizeAgentLoopUsages, type AgentLoopAssistantResponse, @@ -29,7 +31,12 @@ import { type AgentLoopUsage } from '../../../domain'; import { runSandboxTools } from '../../../../../sandbox/application/toolCall'; -import { createToolCall, normalizeToolArgs, stringifyJson } from '../message'; +import { + convertToolResponseContentToPiContent, + createToolCall, + normalizeToolArgs, + stringifyJson +} from '../message'; import { getPiAgentRuntimeTools } from './catalog'; type PlanOperationEvent = Extract; @@ -85,10 +92,15 @@ export const buildPiAgentTools = async ({ call: ChatCompletionMessageToolCall; response: string; assistantMessages?: ChatCompletionMessageParam[]; + content?: string | ChatCompletionToolMessageContentPart[]; }) => void; }): Promise => { const tools: AgentTool[] = []; + // 与 loadRequestMessages 对齐：模型不支持视觉时，工具图片 part 降级为占位文本。 + // 显式转布尔，避免 useVision 缺省时被转换器默认值当成 true。 + const useVision = !!(runtime.llmParams.useVision && runtime.llmParams.model.config.vision); + /** 保证每个普通工具 start 都对应 end，并将异常变成可回填模型的工具结果。 */ const executeOrdinaryTool = async ({ call, @@ -97,6 +109,7 @@ export const buildPiAgentTools = async ({ call: ChatCompletionMessageToolCall; execute: () => Promise<{ response: string; + content?: ChatCompletionToolMessageContentPart[]; assistantMessages?: ChatCompletionMessageParam[]; assistantResponses?: AgentLoopAssistantResponse[]; usages?: AgentLoopUsage[]; @@ -135,8 +148,20 @@ export const buildPiAgentTools = async ({ metadata: result.metadata, seconds: +((Date.now() - startedAt) / 1000).toFixed(2) }); - onToolResult({ call, response: normalizedResponse, assistantMessages }); - return { ...result, response: normalizedResponse, assistantMessages, usages }; + // 与 fastAgent 保持一致：业务 response 文本始终保留，结构化 parts 只在其后追加图片， + // 避免 `{ jobId, content: [image_url] }` 这类输出把业务字段丢掉。 + const toolContent = buildToolMessageContent({ + content: result.content, + text: normalizedResponse + }); + onToolResult({ call, response: normalizedResponse, assistantMessages, content: toolContent }); + return { + ...result, + response: normalizedResponse, + assistantMessages, + usages, + content: toolContent + }; }; for (const tool of getPiAgentRuntimeTools(runtime)) { @@ -159,7 +184,13 @@ export const buildPiAgentTools = async ({ } else if (result.stop) { onToolStop?.(); } - return { content: [{ type: 'text' as const, text: result.response }], details: {} }; + return { + content: await convertToolResponseContentToPiContent( + result.content?.length ? result.content : normalizeToolResponseContent(result.response), + { useVision } + ), + details: {} + }; } }); } @@ -301,11 +332,18 @@ export const buildPiAgentTools = async ({ response: sandboxResult.response, assistantMessages: [], usages: [], - errorMessage: sandboxResult.success ? undefined : sandboxResult.response + errorMessage: sandboxResult.success ? undefined : sandboxResult.response, + content: sandboxResult.content }; } }); - return { content: [{ type: 'text' as const, text: result.response }], details: {} }; + return { + content: await convertToolResponseContentToPiContent( + result.content ?? normalizeToolResponseContent(result.response), + { useVision } + ), + details: {} + }; } }); } @@ -344,7 +382,13 @@ export const buildPiAgentTools = async ({ }; } }); - return { content: [{ type: 'text' as const, text: result.response }], details: {} }; + return { + content: await convertToolResponseContentToPiContent( + result.content?.length ? result.content : normalizeToolResponseContent(result.response), + { useVision } + ), + details: {} + }; } }); } @@ -389,7 +433,13 @@ export const buildPiAgentTools = async ({ }; } }); - return { content: [{ type: 'text' as const, text: result.response }], details: {} }; + return { + content: await convertToolResponseContentToPiContent( + result.content?.length ? result.content : normalizeToolResponseContent(result.response), + { useVision } + ), + details: {} + }; } }); } diff --git a/packages/service/core/ai/llm/promptCall/index.ts b/packages/service/core/ai/llm/promptCall/index.ts index 8bac89871c80..76fb5ee29a40 100644 --- a/packages/service/core/ai/llm/promptCall/index.ts +++ b/packages/service/core/ai/llm/promptCall/index.ts @@ -7,6 +7,7 @@ import type { ChatCompletionTool } from '@fastgpt/global/core/ai/llm/type'; import { getPromptToolCallPrompt } from './prompt'; +import { maskImagePlaceholderUrl } from '../utils'; import { cloneDeep } from 'lodash-es'; export const promptToolCallMessageRewrite = ( @@ -56,9 +57,25 @@ export const promptToolCallMessageRewrite = ( delete message.tool_calls; } } else if (message.role === 'tool') { + // 工具消息可能是字符串、text/image_url parts 或空值；纯图片时保留占位文本，避免响应静默丢失。 + const toolContent = + typeof message.content === 'string' + ? message.content + : Array.isArray(message.content) + ? message.content + .map((part) => { + if (part.type === 'text' && part.text) return part.text; + if (part.type === 'image_url' && part.image_url?.url) { + return `[Image: ${maskImagePlaceholderUrl(part.image_url.url)}]`; + } + return ''; + }) + .filter(Boolean) + .join('\n') + : String(message.content ?? ''); cloneMessages.splice(i, 1, { role: 'user', - content: `\n${message.content}\n` + content: `\n${toolContent}\n` }); } } diff --git a/packages/service/core/ai/llm/utils.ts b/packages/service/core/ai/llm/utils.ts index 83c35bbc6e37..3e3150a38175 100644 --- a/packages/service/core/ai/llm/utils.ts +++ b/packages/service/core/ai/llm/utils.ts @@ -9,7 +9,7 @@ import type { ChatCompletionContentPartText, ChatCompletionMessageParam } from '@fastgpt/global/core/ai/llm/type'; -import { audioFileType } from '@fastgpt/global/common/file/constants'; +import { audioFileType, maxToolImagePartAmount } from '@fastgpt/global/common/file/constants'; import { axios } from '../../../common/api/axios'; import { ChatCompletionRequestMessageRoleEnum } from '@fastgpt/global/core/ai/constants'; @@ -37,6 +37,56 @@ import { const logger = getLogger(LogCategories.MODULE.AI.LLM); +/** + * 判断 URL 是否为沙箱文件预览地址。 + * + * 预览地址带一次性 session 凭证且只在服务端网络拓扑内可达，不适合直接透传给模型服务商。 + * 这里按 env 里的代理前缀识别（与 sandbox 层 buildSandboxPreviewFileUrl 的 `/preview/` 契约一致）， + * 避免把 sandbox 模块及其 dal/redis 依赖拖进 llm 模块。 + */ +export const isSandboxPreviewFileUrl = (url: string) => { + const base = serviceEnv.AGENT_SANDBOX_PREVIEW_PROXY_URL; + if (!base) return false; + + try { + // 与 sandbox 层 buildSandboxPreviewFileUrl 的 URL 规范化保持一致：host 小写、去掉默认端口， + // 并清掉 base 自身可能带的 query/hash，否则生成的预览地址认不出来。 + const parsedBase = new URL(base); + parsedBase.search = ''; + parsedBase.hash = ''; + const normalizedBase = parsedBase.toString().replace(/\/+$/, ''); + return url.startsWith(`${normalizedBase}/preview/`); + } catch { + return false; + } +}; + +/** + * 图片占位文本里的 URL 可能是带凭证的预览地址，写入提示词前先脱敏， + * 避免凭证经纯文本路径外发给模型服务商。 + */ +export const maskImagePlaceholderUrl = (url: string) => + isSandboxPreviewFileUrl(url) ? 'sandbox preview file' : url; + +/** + * 按来源选择图片下载通道。 + * + * 沙箱预览代理可能部署在内网，公共 SSRF 内网拦截会把配置里的合法预览地址一起拒掉， + * 导致工具图片全部丢失。这类地址的 origin 来自服务端配置而不是用户输入，因此只对 + * 该 origin 使用受限通道；其它地址继续走安全的公共下载逻辑。 + * + * fastAgent（loadRequestMessages）与 piAgent（工具 content 转换）都通过这里取图， + * 保证两条 provider 路径对沙箱预览地址的能力一致。 + */ +export const loadImageBase64 = (url: string) => { + const previewProxyBase = serviceEnv.AGENT_SANDBOX_PREVIEW_PROXY_URL; + if (!previewProxyBase || !isSandboxPreviewFileUrl(url)) { + return getImageBase64(url); + } + + return getImageBase64(url, { trustedOrigin: previewProxyBase }); +}; + /** * 判断消息是否属于系统级提示。 * 这类消息不参与普通会话窗口裁剪，避免 system/developer 指令被上下文截断。 @@ -271,6 +321,38 @@ export const loadRequestMessages = async ({ const stripSystemReminder = (input: string) => input.replace(/[\s\S]*?<\/system-reminder>\s*/g, ''); + /** + * 预览图片在发送前由服务端下载并转成 data URI。 + * 下载失败时丢弃该图片 part，避免把带凭证的预览地址透传给模型服务商。 + */ + const inlineSandboxPreviewImage = async ( + part: ChatCompletionContentPart + ): Promise => { + if ( + part.type !== 'image_url' || + !part.image_url.url || + !isSandboxPreviewFileUrl(part.image_url.url) + ) { + return part; + } + + try { + const { completeBase64: base64 } = await loadImageBase64(part.image_url.url); + return { + ...part, + image_url: { + ...part.image_url, + url: base64 + } + }; + } catch (error) { + logger.warn('Failed to inline sandbox preview image, drop the image part', { + error: getErrText(error) + }); + return undefined; + } + }; + /** * 从短文本中提取候选 HTTP(S) 链接并做尾部标点与 Markdown 语法清洗。 * 具体是否为媒体文件交由统一分类入口 classifyUrlMediaType 判断，支持无后缀短链探测。 @@ -358,7 +440,7 @@ export const loadRequestMessages = async ({ if (shouldLoadMediaAsBase64(imgUrl)) { try { - const { completeBase64: base64 } = await getImageBase64(imgUrl); + const { completeBase64: base64 } = await loadImageBase64(imgUrl); return { ...imageItem, @@ -478,6 +560,30 @@ export const loadRequestMessages = async ({ return normalized.filter(Boolean) as ChatCompletionContentPart[]; }; + /** + * 工具图片的归一化不阻塞整轮请求。 + * + * 工具给出的图片可能已经失效（404、超限、被内网策略拒绝）；此时只丢弃这张图片， + * 保留 tool 文本继续请求，与 piAgent 的占位降级语义保持一致，避免整次 agent 运行失败。 + */ + const normalizeToolMediaContentParts = async (content: ChatCompletionContentPart[]) => { + const normalized = await Promise.all( + content.map(async (item) => { + try { + const [result] = await normalizeMediaContentParts([item]); + return result; + } catch (error) { + logger.warn('Drop tool image part after normalization failure', { + error: getErrText(error) + }); + return undefined; + } + }) + ); + + return normalized.filter(Boolean) as ChatCompletionContentPart[]; + }; + /** * 在归一化前按模型能力过滤用户输入。 * 这里处理的是能力边界，不做 URL/base64 转换，避免过滤和转换逻辑混在一起。 @@ -664,6 +770,14 @@ export const loadRequestMessages = async ({ }, []); })(messages); + /** + * 工具消息里的图片不能直接放进 tool content（供应商只接受字符串或文本 parts）， + * 需要转换成紧随其后的 user 视觉消息。但同一个 assistant 消息可能同时发起多个工具调用， + * 这时的图片消息必须排在全部 tool 结果之后，否则会打断 tool_call 结果回填顺序。 + * 这里先给 tool 消息登记图片，拼装阶段再按“整轮 tool 结果”统一追加。 + */ + const toolImageParts = new WeakMap(); + const loadMessages = ( await Promise.all( mergeMessages.map(async (raw, i) => { @@ -728,6 +842,49 @@ export const loadRequestMessages = async ({ ...formatAssistantItem(item, supportReason), content: formatContent }; + } else if (item.role === ChatCompletionRequestMessageRoleEnum.Tool) { + // 供应商接口的 tool content 只接受字符串或文本 parts，图片不能放进 tool 消息。 + // 文本保留在 tool 消息里；图片先按模型能力过滤、再归一化，登记到 toolImageParts， + // 由拼装阶段转换成这一轮 tool 结果之后的 user 视觉消息。 + if (!Array.isArray(item.content)) { + return item; + } + + const supportedContent = item.content + .map((part) => filterSupportedUserContentPart(part as ChatCompletionContentPart)) + .filter(Boolean) as ChatCompletionContentPart[]; + + const textParts = supportedContent.filter( + (part): part is ChatCompletionContentPartText => part.type === 'text' + ); + const mediaParts = supportedContent.filter((part) => part.type !== 'text'); + + const normalizedTextParts = (await normalizeMediaContentParts( + textParts + )) as ChatCompletionContentPartText[]; + const normalizedMediaParts = ( + await normalizeToolMediaContentParts( + (await Promise.all(mediaParts.map(inlineSandboxPreviewImage))).filter( + (part): part is ChatCompletionContentPart => Boolean(part) + ) + ) + ).slice(0, maxToolImagePartAmount); + + const toolMessage = { + ...item, + content: + normalizedTextParts.length === 0 + ? 'none' + : normalizedTextParts.length === 1 + ? normalizedTextParts[0].text + : normalizedTextParts + } as ChatCompletionMessageParam; + + if (normalizedMediaParts.length > 0) { + toolImageParts.set(toolMessage, normalizedMediaParts); + } + + return toolMessage; } else { return item; } @@ -735,5 +892,40 @@ export const loadRequestMessages = async ({ ) ).filter(Boolean) as ChatCompletionMessageParam[]; - return loadMessages; + /** + * 同一轮的全部 tool 结果必须连续回填，工具图片消息统一追加在这一轮 tool 结果之后。 + * 否则 assistant(A,B) 的回填会变成 tool A → user(image) → tool B，打断协议顺序。 + */ + const appendToolImagesAfterToolRun = (messages: ChatCompletionMessageParam[]) => { + const result: ChatCompletionMessageParam[] = []; + let pendingToolImages: ChatCompletionContentPart[] = []; + + const flushToolImages = () => { + if (pendingToolImages.length === 0) return; + result.push({ + role: ChatCompletionRequestMessageRoleEnum.User, + content: pendingToolImages + } as ChatCompletionMessageParam); + pendingToolImages = []; + }; + + messages.forEach((message) => { + if (message.role === ChatCompletionRequestMessageRoleEnum.Tool) { + result.push(message); + const images = toolImageParts.get(message); + if (images?.length) { + pendingToolImages.push(...images); + } + return; + } + + flushToolImages(); + result.push(message); + }); + flushToolImages(); + + return result; + }; + + return appendToolImagesAfterToolRun(loadMessages); }; diff --git a/packages/service/core/ai/sandbox/application/toolCall/getFileUrl.tool.ts b/packages/service/core/ai/sandbox/application/toolCall/getFileUrl.tool.ts index 54266dd65261..6a6e2830991b 100644 --- a/packages/service/core/ai/sandbox/application/toolCall/getFileUrl.tool.ts +++ b/packages/service/core/ai/sandbox/application/toolCall/getFileUrl.tool.ts @@ -11,11 +11,21 @@ import { createSandboxPreviewSession, resolveSandboxPreviewPath } from '../preview'; +import type { ChatCompletionToolMessageContentPart } from '@fastgpt/global/core/ai/llm/type'; +import { providerImageFileType } from '@fastgpt/global/common/file/constants'; const SandboxGetFileUrlToolSchema = z.object({ paths: z.array(z.string()) }); +/** 只有主流视觉模型能消费的栅格格式才附加 image_url part，其余文件只保留文本链接。 */ +const providerImageFileTypes = providerImageFileType.split(',').map((item) => item.trim()); + +const isProviderImageFilePath = (filePath: string) => { + const extension = `.${path.posix.extname(filePath).slice(1).toLowerCase()}`; + return extension !== '.' && providerImageFileTypes.includes(extension); +}; + export const sandboxGetFileUrlTool = defineTool({ zodSchema: SandboxGetFileUrlToolSchema, execute: async ({ sandboxInstance, params }) => { @@ -66,6 +76,21 @@ export const sandboxGetFileUrlTool = defineTool({ filename: path.posix.basename(relativePath) })); - return { response: JSON.stringify(result) }; + // content 必须以承载原 response 的 text part 打头（保留全部文件的 filename/fileUrl， + // 供下一轮压缩与展示使用），图片文件再以 image_url part 追加作为视觉输入。 + const content: ChatCompletionToolMessageContentPart[] = [ + { type: 'text', text: JSON.stringify(result) }, + ...result + .filter(({ filename }) => isProviderImageFilePath(filename)) + .map(({ fileUrl }) => ({ + type: 'image_url', + image_url: { url: fileUrl } + })) + ]; + + return { + response: JSON.stringify(result), + content + }; } }); diff --git a/packages/service/core/ai/sandbox/application/toolCall/index.ts b/packages/service/core/ai/sandbox/application/toolCall/index.ts index da41dfd09f34..95d9c1250c4f 100644 --- a/packages/service/core/ai/sandbox/application/toolCall/index.ts +++ b/packages/service/core/ai/sandbox/application/toolCall/index.ts @@ -32,6 +32,7 @@ import { preparePackageMirrors, prepareSandbox } from '../runtime/prepare'; import type { ChatSourceTypeEnum } from '@fastgpt/global/core/chat/constants'; import { getRunningSandboxId, getSandboxUserId } from '../../utils/id'; import { createToolRunner, type SandboxToolCallValidator } from './type'; +import type { ChatCompletionToolMessageContentPart } from '@fastgpt/global/core/ai/llm/type'; const ToolMap = { [SANDBOX_EDIT_FILE_TOOL_NAME]: createToolRunner(sandboxEditFileTool), @@ -49,6 +50,8 @@ export type SandboxToolCallResult = { input: Record; response: string; durationSeconds: number; + /** 工具主动返回的结构化 content（text / image_url parts）。 */ + content?: ChatCompletionToolMessageContentPart[]; }; /** @@ -110,7 +113,8 @@ export const runSandboxTools = async ({ success: true, input: result.input, response: result.response, - durationSeconds: getDuration() + durationSeconds: getDuration(), + ...(result.content?.length ? { content: result.content } : {}) }; }; diff --git a/packages/service/core/ai/sandbox/application/toolCall/type.ts b/packages/service/core/ai/sandbox/application/toolCall/type.ts index 5f501b03c0fa..b1d07f9403e0 100644 --- a/packages/service/core/ai/sandbox/application/toolCall/type.ts +++ b/packages/service/core/ai/sandbox/application/toolCall/type.ts @@ -5,6 +5,7 @@ */ import type { z } from 'zod'; import type { SandboxClient } from '../runtime/client'; +import type { ChatCompletionToolMessageContentPart } from '@fastgpt/global/core/ai/llm/type'; /** 工具执行前的业务校验；返回拒绝原因时不访问沙箱。 */ export type SandboxToolCallValidator = (params: { @@ -21,12 +22,15 @@ type ToolExecuteContext

= { * sandbox 工具的声明式定义。 * * zodSchema 负责约束 LLM 工具参数，execute 只接收已校验的 params 和运行态 SandboxClient。 + * 工具可返回结构化 content（text / image_url parts）作为下一轮模型的视觉输入。 */ export type ToolDefinition< S extends z.ZodType> = z.ZodType> > = { zodSchema: S; - execute: (ctx: ToolExecuteContext>) => Promise<{ response: string }>; + execute: ( + ctx: ToolExecuteContext> + ) => Promise<{ response: string; content?: ChatCompletionToolMessageContentPart[] }>; }; export type ToolRunResult = @@ -35,6 +39,7 @@ export type ToolRunResult = success: true; input: Record; response: string; + content?: ChatCompletionToolMessageContentPart[]; }; /** @@ -61,6 +66,7 @@ export const createToolRunner = return { success: true, input: parsed.data, - response: result.response + response: result.response, + ...(result.content?.length ? { content: result.content } : {}) }; }; diff --git a/packages/service/core/workflow/dispatch/ai/agent/sub/tool/index.ts b/packages/service/core/workflow/dispatch/ai/agent/sub/tool/index.ts index d1b51cd2fb6e..649d894754da 100644 --- a/packages/service/core/workflow/dispatch/ai/agent/sub/tool/index.ts +++ b/packages/service/core/workflow/dispatch/ai/agent/sub/tool/index.ts @@ -41,6 +41,11 @@ import { assertTeamPluginSourceAccess, getRawPluginIdFromSystemToolId } from '../../../../../../plugin/teamPluginPolicy'; +import { + convertMcpToolContentToToolContent, + extractAgentLoopCoreToolContent, + maskToolImagePayload +} from '../../../agentLoopCore/domain/toolContent'; type SystemInputConfigType = { type: SystemToolSecretInputTypeEnum; @@ -258,8 +263,16 @@ export const dispatchTool = async ({ msg: result[NodeOutputKeyEnum.systemError] }); + // 插件输出若显式声明了结构化 content，透传给 agent-loop 作为下一轮模型输入； + // response 仍是完整业务输出，两者互不替代。图片数据只保留在 content 通道里一份。 + const content = extractAgentLoopCoreToolContent(result); + const responsePayload = content + ? maskToolImagePayload(result, { maskDataUrlParts: true }) + : result; + return { - response: JSON.stringify(result), + response: JSON.stringify(responsePayload), + ...(content ? { content } : {}), nodeResponse: getNodeResponse({ result }), @@ -312,8 +325,14 @@ export const dispatchTool = async ({ params }); const errorMessage = getMCPToolErrorText(result); + // MCP 图片块是 { type: 'image', data, mimeType }，需要先转成 data URI image_url。 + const content = convertMcpToolContentToToolContent( + (result as { content?: unknown } | undefined)?.content + ); return { - response: JSON.stringify(result), + // 图片走 image_url part；文本里把 base64 换成占位，避免同一张图在上下文里出现两份。 + response: JSON.stringify(maskToolImagePayload(result)), + ...(content ? { content } : {}), ...(errorMessage !== undefined ? { errorMessage } : {}), nodeResponse: { ...getNodeResponse({ result }), @@ -376,11 +395,19 @@ export const dispatchTool = async ({ }; } + const content = extractAgentLoopCoreToolContent(data); + // 图片数据只保留在 content 通道里一份，业务文本里换成占位。 + const responseData = + content && data && typeof data === 'object' + ? maskToolImagePayload(data, { maskDataUrlParts: true }) + : data; + return { nodeResponse: getNodeResponse({ result: data }), - response: typeof data === 'object' ? JSON.stringify(data) : data + response: typeof responseData === 'object' ? JSON.stringify(responseData) : responseData, + ...(content ? { content } : {}) }; } else { return getErrResponse("Can't find the tool"); diff --git a/packages/service/core/workflow/dispatch/ai/agent/sub/utils.ts b/packages/service/core/workflow/dispatch/ai/agent/sub/utils.ts index 4fd49da86a25..a100a3ef6ad3 100644 --- a/packages/service/core/workflow/dispatch/ai/agent/sub/utils.ts +++ b/packages/service/core/workflow/dispatch/ai/agent/sub/utils.ts @@ -2,7 +2,10 @@ import type { localeType } from '@fastgpt/global/common/i18n/type'; import type { AgentToolType } from '@fastgpt/global/core/app/tool/type'; import type { DispatchSubAppResponse, GetSubAppInfoFnType, SubAppRuntimeType } from '../type'; import { getAgentRuntimeTools } from './tool/utils'; -import type { ChatCompletionTool } from '@fastgpt/global/core/ai/llm/type'; +import type { + ChatCompletionTool, + ChatCompletionToolMessageContentPart +} from '@fastgpt/global/core/ai/llm/type'; import type { ChatNodeUsageType } from '@fastgpt/global/support/wallet/bill/type'; import type { DispatchAgentModuleProps } from '..'; import { FlowNodeTypeEnum } from '@fastgpt/global/core/workflow/node/constant'; @@ -194,7 +197,8 @@ export const getExecuteTool = ({ stop = false, nodeResponse, nodeSummary, - errorMessage + errorMessage, + content } = await (async (): Promise<{ response: string; assistantMessages?: DispatchSubAppResponse['assistantMessages']; @@ -204,6 +208,7 @@ export const getExecuteTool = ({ errorMessage?: string; nodeResponse?: DispatchSubAppResponse['nodeResponse']; nodeSummary?: DispatchSubAppResponse['nodeSummary']; + content?: ChatCompletionToolMessageContentPart[]; }> => { try { // User Sub App @@ -233,30 +238,32 @@ export const getExecuteTool = ({ }); if (tool.type === 'tool') { - const { response, usages, nodeResponse, nodeSummary, errorMessage } = await dispatchTool({ - tool: { - name: tool.name, - avatar: tool.avatar, - version: tool.version, - toolConfig: tool.toolConfig - }, - params: requestParams, - runningUserInfo, - runningAppInfo, - chatId, - uid, - variableState, - runtimeNodesMap, - // Agent 统一负责外层流输出，子工具不得绕过 Agent 直接写入响应流。 - workflowStreamResponse: undefined - }); + const { response, usages, nodeResponse, nodeSummary, errorMessage, content } = + await dispatchTool({ + tool: { + name: tool.name, + avatar: tool.avatar, + version: tool.version, + toolConfig: tool.toolConfig + }, + params: requestParams, + runningUserInfo, + runningAppInfo, + chatId, + uid, + variableState, + runtimeNodesMap, + // Agent 统一负责外层流输出，子工具不得绕过 Agent 直接写入响应流。 + workflowStreamResponse: undefined + }); return { response, usages, nodeResponse, nodeSummary, - errorMessage + errorMessage, + content }; } else if (tool.type === 'workflow') { const { userChatInput, ...params } = filterAgentWorkflowRuntimeParams(requestParams); @@ -430,6 +437,7 @@ export const getExecuteTool = ({ stop, nodeSummary, ...(errorMessage ? { errorMessage } : {}), + ...(content?.length ? { content } : {}), nodeResponse: formatNodeResponse }; }; diff --git a/packages/service/core/workflow/dispatch/ai/agent/type.ts b/packages/service/core/workflow/dispatch/ai/agent/type.ts index 076a615c16c9..f3a91a612a07 100644 --- a/packages/service/core/workflow/dispatch/ai/agent/type.ts +++ b/packages/service/core/workflow/dispatch/ai/agent/type.ts @@ -3,7 +3,10 @@ import z from 'zod'; import { NodeToolConfigTypeSchema } from '@fastgpt/global/core/workflow/type/node'; import type { RuntimeNodeItemType } from '@fastgpt/global/core/workflow/runtime/type'; import type { ChatHistoryItemResType } from '@fastgpt/global/core/chat/type'; -import type { ChatCompletionMessageParam } from '@fastgpt/global/core/ai/llm/type'; +import type { + ChatCompletionMessageParam, + ChatCompletionToolMessageContentPart +} from '@fastgpt/global/core/ai/llm/type'; import type { WorkflowInteractiveResponseType } from '@fastgpt/global/core/workflow/template/system/interactive/type'; import type { NodeSummary } from '../../../types/runtime'; @@ -21,6 +24,11 @@ export type DispatchSubAppResponse = { * 工具自身的运行详情。显式 totalPoints 表示当前响应自身费用；省略时由外层按 usages 计算。 */ nodeResponse?: Omit; + /** + * 写入 LLM tool message 的结构化 content（text / image_url parts）。 + * 仅当工具显式返回合法的 parts 时才透传，缺省时按 response 字符串兜底。 + */ + content?: ChatCompletionToolMessageContentPart[]; }; export const SubAppRuntimeSchema = z.object({ diff --git a/packages/service/core/workflow/dispatch/ai/agentLoopCore/application/runtime/workflowToolRunner.ts b/packages/service/core/workflow/dispatch/ai/agentLoopCore/application/runtime/workflowToolRunner.ts index 1f4ccad846c0..48fb9b4bc4e2 100644 --- a/packages/service/core/workflow/dispatch/ai/agentLoopCore/application/runtime/workflowToolRunner.ts +++ b/packages/service/core/workflow/dispatch/ai/agentLoopCore/application/runtime/workflowToolRunner.ts @@ -15,6 +15,11 @@ import type { AgentLoopInteractiveToolExecuteParams } from '../../../../../../ai import type { AgentLoopDatasetSearchExecutor } from '../../../../../../ai/llm/agentLoop/interface'; import { parseJsonArgs } from '../../../../../../ai/utils'; import type { AgentLoopCoreToolInfo, AgentLoopCoreToolRunResult } from '../../domain/toolProvider'; +import { + convertMcpToolContentToToolContent, + extractAgentLoopCoreToolContent, + maskToolImagePayload +} from '../../domain/toolContent'; import type { AgentLoopCoreToolRunFlowResponse } from '../../adapter/nodeResponse/toolRunCollector'; import { normalizeAgentLoopCoreDatasetSearchResult } from './systemToolHelpers'; import { cloneDeep } from 'lodash-es'; @@ -174,15 +179,25 @@ const toToolRunResult = ( const rawToolResponses = toolRunResponse.toolResponses; const hasToolResponses = rawToolResponses !== undefined && rawToolResponses !== null && rawToolResponses !== ''; + // ToolCall 节点里的 MCP 工具会返回 MCP 原生图片块，先按 MCP 契约转换； + // 其余工具（插件 / HTTP / workflow）仍走共享 schema 提取。 + const content = + convertMcpToolContentToToolContent( + (rawToolResponses as { content?: unknown } | undefined)?.content + ) ?? extractAgentLoopCoreToolContent(rawToolResponses); return { result: { // 工具失败时把错误原因返回给模型；'none' 会掩盖失败原因。 + // 图片已经作为 content part 传给模型，文本里把 MCP base64 换成占位，避免重复发送。 response: hasToolResponses || !errorMessage - ? formatAgentLoopCoreToolResponse(rawToolResponses) + ? formatAgentLoopCoreToolResponse( + maskToolImagePayload(rawToolResponses, { maskDataUrlParts: Boolean(content) }) + ) : errorMessage, ...(errorMessage ? { errorMessage } : {}), + ...(content ? { content } : {}), assistantMessages: getAssistantMessages(toolRunResponse.assistantResponses), ...(toolRunResponse.assistantResponses?.length ? { assistantResponses: toolRunResponse.assistantResponses } diff --git a/packages/service/core/workflow/dispatch/ai/agentLoopCore/domain/toolContent.ts b/packages/service/core/workflow/dispatch/ai/agentLoopCore/domain/toolContent.ts new file mode 100644 index 000000000000..0c30ad7cddd7 --- /dev/null +++ b/packages/service/core/workflow/dispatch/ai/agentLoopCore/domain/toolContent.ts @@ -0,0 +1,142 @@ +import { + ChatCompletionToolMessageContentPartSchema, + type ChatCompletionToolMessageContentPart +} from '@fastgpt/global/core/ai/llm/type'; +import { providerImageMimeType } from '@fastgpt/global/common/file/constants'; + +/** + * 从工具输出中提取结构化 tool content。 + * + * 仅接受形如 `{ content: [{ type: 'text' | 'image_url', ... }, ...] }` 的对象输出： + * content 必须是非空数组，且每个 part 都能通过共享的 + * `ChatCompletionToolMessageContentPartSchema` 校验（含 `image_url.detail` 这类细分字段）。 + * + * 任一 part 不合法时整体返回 undefined，由调用方按普通文本处理：工具输出的业务数据 + * 可能只是恰好带了一个 content 字段，不能被半解析成请求字段。只有工具显式声明了合法的 + * parts，才作为下一轮模型的视觉输入；不在普通文本里猜测图片。 + */ +export const extractAgentLoopCoreToolContent = ( + toolResponses: unknown +): ChatCompletionToolMessageContentPart[] | undefined => { + if (!toolResponses || typeof toolResponses !== 'object' || Array.isArray(toolResponses)) { + return; + } + + const content = (toolResponses as { content?: unknown }).content; + if (!Array.isArray(content) || content.length === 0) return; + + const parts: ChatCompletionToolMessageContentPart[] = []; + for (const item of content) { + const parsed = ChatCompletionToolMessageContentPartSchema.safeParse(item); + if (!parsed.success) return; + + // 空文本、空链接没有传递价值，按非法处理，回退到普通文本链路。 + if (parsed.data.type === 'text' ? !parsed.data.text : !parsed.data.image_url.url) { + return; + } + + parts.push(parsed.data); + } + + return parts; +}; + +const providerImageMimeTypes = providerImageMimeType.split(',').map((item) => item.trim()); +const isProviderImageMimeType = (mimeType: string) => providerImageMimeTypes.includes(mimeType); + +/** MCP 图片块是否能转换成供应商可消费的视觉输入。 */ +const isConvertibleMcpImageBlock = (block: { data?: unknown; mimeType?: unknown }) => + typeof block.data === 'string' && + block.data.length > 0 && + typeof block.mimeType === 'string' && + isProviderImageMimeType(block.mimeType); + +/** + * 把模型文本里的图片数据替换成占位，避免同一份图片既进 tool 文本又进 vision part。 + * + * - MCP `{ type: 'image', data, mimeType }` 只替换真正会被转换的块（栅格格式、data 非空）； + * 未转换的块保持原样，否则这些内容会在两个通道里同时消失。 + * - `image_url.url` 为 data URI 的 part 只在调用方确认它已通过 content 通道发送时替换 + * （`maskDataUrlParts`），否则会把唯一一份数据丢掉。 + * + * 原始结果仍保留在节点响应里供详情展示。 + */ +export const maskToolImagePayload = ( + toolResponses: unknown, + options: { maskDataUrlParts?: boolean } = {} +): unknown => { + if (!toolResponses || typeof toolResponses !== 'object') return toolResponses; + + const content = (toolResponses as { content?: unknown }).content; + if (!Array.isArray(content)) return toolResponses; + + let changed = false; + const maskedContent = content.map((block) => { + if (!block || typeof block !== 'object') return block; + + const item = block as { + type?: unknown; + data?: unknown; + mimeType?: unknown; + image_url?: { url?: unknown }; + }; + + if (item.type === 'image') { + if (!isConvertibleMcpImageBlock(item)) return block; + changed = true; + return { type: 'image', mimeType: item.mimeType, data: '[image data omitted]' }; + } + + const url = item.image_url?.url; + if (options.maskDataUrlParts && typeof url === 'string' && url.startsWith('data:')) { + changed = true; + return { ...item, image_url: { ...item.image_url, url: '[image data omitted]' } }; + } + + return block; + }); + + return changed ? { ...toolResponses, content: maskedContent } : toolResponses; +}; + +/** + * 把 MCP `CallToolResult.content` 的原生内容块转换成 tool content parts。 + * + * MCP 协议里的图片块是 `{ type: 'image', data, mimeType }`（base64），不是 `image_url`； + * 直接交给 extractAgentLoopCoreToolContent 会整组校验失败并回退成 JSON 文本，模型拿不到图片。 + * 因此在这里先转成 data URI，再走共享 schema 校验。 + * + * 只产出图片 part：文本块的内容已经包含在 response JSON 里，重复发送没有意义； + * 音频、resource 等 MCP 块保持原有文本路径，不做猜测性转换。 + */ +export const convertMcpToolContentToToolContent = ( + mcpContent: unknown +): ChatCompletionToolMessageContentPart[] | undefined => { + if (!Array.isArray(mcpContent)) return; + + const parts: ChatCompletionToolMessageContentPart[] = []; + for (const block of mcpContent) { + if (!block || typeof block !== 'object') continue; + + const { type, data, mimeType } = block as { + type?: unknown; + data?: unknown; + mimeType?: unknown; + }; + if (type !== 'image' || !isConvertibleMcpImageBlock({ data, mimeType })) { + continue; + } + + const parsed = ChatCompletionToolMessageContentPartSchema.safeParse({ + type: 'image_url', + image_url: { url: `data:${mimeType};base64,${data}` } + }); + // 正常不会失败（url 允许任意字符串）；保留 safeParse 作为 schema 变更时的兜底， + // 真正的过滤在 isConvertibleMcpImageBlock 的白名单里。 + if (!parsed.success) continue; + + parts.push(parsed.data); + } + + return parts.length ? parts : undefined; +}; diff --git a/packages/service/core/workflow/dispatch/ai/agentLoopCore/domain/toolProvider.ts b/packages/service/core/workflow/dispatch/ai/agentLoopCore/domain/toolProvider.ts index f8ce91fb647b..625b20118abb 100644 --- a/packages/service/core/workflow/dispatch/ai/agentLoopCore/domain/toolProvider.ts +++ b/packages/service/core/workflow/dispatch/ai/agentLoopCore/domain/toolProvider.ts @@ -1,7 +1,8 @@ import type { ChatCompletionMessageParam, ChatCompletionMessageToolCall, - ChatCompletionTool + ChatCompletionTool, + ChatCompletionToolMessageContentPart } from '@fastgpt/global/core/ai/llm/type'; import type { AIChatItemValueItemType, @@ -41,6 +42,8 @@ export type AgentLoopCoreToolRunResult = { errorMessage?: string; nodeResponse?: ChatHistoryItemResType; nodeSummary?: NodeSummary; + /** 写入 LLM tool message 的结构化 content（text / image_url parts）。 */ + content?: ChatCompletionToolMessageContentPart[]; }; export type AgentLoopCoreToolProvider = { @@ -68,7 +71,8 @@ export const normalizeAgentLoopCoreToolRunResult = interactive, stop = false, errorMessage, - nodeResponse + nodeResponse, + content }: AgentLoopCoreToolRunResult): AgentLoopToolExecutionResult => { return { response, @@ -78,6 +82,7 @@ export const normalizeAgentLoopCoreToolRunResult = interactive, stop, errorMessage, - metadata: nodeResponse + metadata: nodeResponse, + content }; }; diff --git a/packages/service/test/common/file/image/trustedOrigin.test.ts b/packages/service/test/common/file/image/trustedOrigin.test.ts new file mode 100644 index 000000000000..1bc6656166ae --- /dev/null +++ b/packages/service/test/common/file/image/trustedOrigin.test.ts @@ -0,0 +1,69 @@ +import { Readable } from 'node:stream'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +const { axiosGetMock, axiosWithoutSSRFGetMock } = vi.hoisted(() => ({ + axiosGetMock: vi.fn(), + axiosWithoutSSRFGetMock: vi.fn() +})); + +beforeEach(() => { + vi.clearAllMocks(); +}); + +vi.mock('@fastgpt/service/common/api/axios', () => ({ + axios: { get: axiosGetMock }, + axiosWithoutSSRF: { get: axiosWithoutSSRFGetMock } +})); + +vi.mock('@fastgpt/service/common/api/serverRequest', () => ({ + serverRequestBaseUrl: 'http://localhost:3000' +})); + +const createStreamResponse = () => ({ + status: 200, + headers: { 'content-length': '4', 'content-type': 'image/png' }, + data: Readable.from([Buffer.from([0x89, 0x50, 0x4e, 0x47])]) +}); + +const { getImageBuffer, isTrustedOriginUrl } = + await import('@fastgpt/service/common/file/image/utils'); + +describe('isTrustedOriginUrl', () => { + it('matches only the configured origin', () => { + expect(isTrustedOriginUrl('http://10.0.0.5:8080/preview/a.png', 'http://10.0.0.5:8080')).toBe( + true + ); + expect(isTrustedOriginUrl('http://10.0.0.5:9999/preview/a.png', 'http://10.0.0.5:8080')).toBe( + false + ); + expect(isTrustedOriginUrl('https://evil.example.com/a.png', 'http://10.0.0.5:8080')).toBe( + false + ); + expect(isTrustedOriginUrl('http://10.0.0.5:8080/preview/a.png')).toBe(false); + }); +}); + +describe('getImageBuffer trusted origin', () => { + it('uses the SSRF-free client for the configured preview origin', async () => { + axiosWithoutSSRFGetMock.mockResolvedValueOnce(createStreamResponse()); + + const result = await getImageBuffer('http://10.0.0.5:8080/preview/a.png', { + trustedOrigin: 'http://10.0.0.5:8080' + }); + + expect(result.mime).toBe('image/png'); + expect(axiosWithoutSSRFGetMock).toHaveBeenCalledTimes(1); + expect(axiosGetMock).not.toHaveBeenCalled(); + }); + + it('keeps the safe client for any other URL', async () => { + axiosGetMock.mockResolvedValueOnce(createStreamResponse()); + + await getImageBuffer('https://files.example.com/a.png', { + trustedOrigin: 'http://10.0.0.5:8080' + }); + + expect(axiosGetMock).toHaveBeenCalledTimes(1); + expect(axiosWithoutSSRFGetMock).not.toHaveBeenCalled(); + }); +}); diff --git a/packages/service/test/common/file/image/trustedOriginRedirect.test.ts b/packages/service/test/common/file/image/trustedOriginRedirect.test.ts new file mode 100644 index 000000000000..311d1f569b74 --- /dev/null +++ b/packages/service/test/common/file/image/trustedOriginRedirect.test.ts @@ -0,0 +1,82 @@ +import http from 'node:http'; +import { afterAll, beforeAll, describe, expect, it } from 'vitest'; + +const { getImageBuffer } = await import('@fastgpt/service/common/file/image/utils'); + +// 与 common/api/axios.test.ts 的 redirect 用例保持同一套写法： +// 真实 http.createServer + listen(0)，验证可信 origin 通道不跟随跨 origin 3xx。 +const PNG = Buffer.from([0x89, 0x50, 0x4e, 0x47]); + +const listen = (handler: http.RequestListener, host = '127.0.0.1') => + new Promise((resolve) => { + const server = http.createServer(handler); + server.listen(0, host, () => resolve(server)); + }); + +const closeServer = (server: http.Server) => + new Promise((resolve, reject) => { + server.close((err) => (err ? reject(err) : resolve())); + }); + +const getServerPort = (server: http.Server): number => { + const address = server.address(); + if (!address || typeof address === 'string') { + throw new Error('Invalid test server address'); + } + return address.port; +}; + +let trustedServer: http.Server; +let otherServer: http.Server; +let trustedBaseUrl = ''; +let otherHits = 0; + +beforeAll(async () => { + otherServer = await listen((_req, res) => { + otherHits += 1; + res.writeHead(200, { 'content-type': 'image/png' }); + res.end(PNG); + }); + const otherBaseUrl = `http://127.0.0.1:${getServerPort(otherServer)}`; + + trustedServer = await listen((req, res) => { + // 预览代理/网关把请求跳到另一个 origin（这里用不同端口模拟）。 + if (req.url?.startsWith('/preview/redirect')) { + res.writeHead(302, { location: `${otherBaseUrl}/evil.png` }); + res.end(); + return; + } + res.writeHead(200, { 'content-type': 'image/png' }); + res.end(PNG); + }); + trustedBaseUrl = `http://127.0.0.1:${getServerPort(trustedServer)}`; +}); + +afterAll(async () => { + await Promise.all([closeServer(trustedServer), closeServer(otherServer)]); +}); + +describe('trusted origin image download', () => { + it('应该下载配置 origin 上的图片', async () => { + const result = await getImageBuffer(`${trustedBaseUrl}/preview/a.png`, { + trustedOrigin: trustedBaseUrl + }); + + expect(result.mime).toBe('image/png'); + expect(result.buffer.length).toBe(PNG.length); + }); + + it('不应该跟随可信通道上的跨 origin 重定向', async () => { + const otherHitsBefore = otherHits; + + // 302 直接作为错误返回（不跟随 + 非 2xx 拒绝），不会去请求跳转目标。 + await expect( + getImageBuffer(`${trustedBaseUrl}/preview/redirect.png`, { + trustedOrigin: trustedBaseUrl + }) + ).rejects.toThrow(/status code 302/); + + // 关键：跳转目标一次都不能被请求，否则等于绕过内网校验。 + expect(otherHits).toBe(otherHitsBefore); + }); +}); diff --git a/packages/service/test/core/ai/llm/agentLoop/fastAgentLoop.test.ts b/packages/service/test/core/ai/llm/agentLoop/fastAgentLoop.test.ts index 7ca21497f783..d4b2af787f59 100644 --- a/packages/service/test/core/ai/llm/agentLoop/fastAgentLoop.test.ts +++ b/packages/service/test/core/ai/llm/agentLoop/fastAgentLoop.test.ts @@ -3,7 +3,10 @@ import { ChatCompletionRequestMessageRoleEnum, ModelTypeEnum } from '@fastgpt/global/core/ai/constants'; -import type { ChatCompletionTool } from '@fastgpt/global/core/ai/llm/type'; +import type { + ChatCompletionTool, + ChatCompletionToolMessageContentPart +} from '@fastgpt/global/core/ai/llm/type'; import { beforeEach, describe, expect, it, vi } from 'vitest'; import { mockCreateLLMResponseQueue, text, toolCall } from './_mocks/llmQueue'; @@ -61,6 +64,7 @@ import { runFastAgentMainLoop, type AgentLoopRuntime } from '@fastgpt/service/core/ai/llm/agentLoop/provider/fastAgent/loop'; +import { buildToolMessageContent } from '@fastgpt/service/core/ai/llm/agentLoop/domain/tool'; const getFinalAssistantText = (result: { assistantMessages: any[] }) => result.assistantMessages @@ -753,6 +757,41 @@ describe('runFastAgentMainLoop', () => { expect(createLLMResponseMock).toHaveBeenCalledTimes(2); }); + it('writes the tool response parts into the next round tool message', async () => { + const imagePart = { + type: 'image_url' as const, + image_url: { url: 'data:image/png;base64,AAAA' } + }; + mockCreateLLMResponseQueue(createLLMResponseMock, [ + toolCall({ id: 'call_image', name: 'search', args: { query: 'chart' } }), + text({ requestId: 'req_after_image', content: 'done' }) + ]); + + const executeTool = vi.fn(async () => ({ + response: '{"jobId":"task-123"}', + content: [imagePart], + assistantMessages: [], + usages: [], + stop: false + })); + + const result = await runFastAgentMainLoop({ + runtime: createRuntime({ executeTool }), + input: { messages: [{ role: ChatCompletionRequestMessageRoleEnum.User, content: 'draw it' }] } + }); + + expect(result.status).toBe('done'); + // 关键接缝：结构化的 content 必须真的写进下一轮请求的 tool 消息，否则等于没实现。 + const secondRequest = createLLMResponseMock.mock.calls[1][0]; + const toolMessage = (secondRequest.body.messages as ChatCompletionMessageParam[]).find( + (message) => message.role === ChatCompletionRequestMessageRoleEnum.Tool + ); + expect(toolMessage?.content).toEqual([ + { type: 'text', text: '{"jobId":"task-123"}' }, + imagePart + ]); + }); + it('uses reconstructed messages without an Ask-specific snapshot', async () => { mockCreateLLMResponseQueue(createLLMResponseMock, [text({ content: 'continued' })]); const messages = [{ role: 'user' as const, content: 'Continue normally' }]; @@ -865,3 +904,48 @@ describe('runFastAgentMainLoop', () => { ); }); }); + +describe('buildToolMessageContent', () => { + it('returns the final text when no content parts are provided', () => { + expect(buildToolMessageContent({ text: 'summary' })).toBe('summary'); + expect(buildToolMessageContent({ content: [], text: 'summary' })).toBe('summary'); + }); + + it('prepends the final text and keeps image parts', () => { + const content: ChatCompletionToolMessageContentPart[] = [ + { type: 'image_url', image_url: { url: 'data:image/png;base64,aW1hZ2U=' } } + ]; + + expect(buildToolMessageContent({ content, text: 'summary' })).toEqual([ + { type: 'text', text: 'summary' }, + { type: 'image_url', image_url: { url: 'data:image/png;base64,aW1hZ2U=' } } + ]); + }); + + it('caps the image parts so one tool result cannot flood the request', () => { + const content: ChatCompletionToolMessageContentPart[] = Array.from( + { length: 6 }, + (_, index) => ({ + type: 'image_url' as const, + image_url: { url: `data:image/png;base64,IMG${index}` } + }) + ); + + const result = buildToolMessageContent({ content, text: 'summary' }); + + expect(Array.isArray(result)).toBe(true); + expect(result).toHaveLength(1 + 4); + }); + + it('replaces raw text parts with the final text', () => { + const content: ChatCompletionToolMessageContentPart[] = [ + { type: 'text', text: 'raw uncompressed output' }, + { type: 'image_url', image_url: { url: 'data:image/png;base64,aW1hZ2U=' } } + ]; + + expect(buildToolMessageContent({ content, text: 'summary' })).toEqual([ + { type: 'text', text: 'summary' }, + { type: 'image_url', image_url: { url: 'data:image/png;base64,aW1hZ2U=' } } + ]); + }); +}); diff --git a/packages/service/test/core/ai/llm/agentLoop/piAgentMessage.test.ts b/packages/service/test/core/ai/llm/agentLoop/piAgentMessage.test.ts index a5504d1b9cf7..fa2f1abdd9b0 100644 --- a/packages/service/test/core/ai/llm/agentLoop/piAgentMessage.test.ts +++ b/packages/service/test/core/ai/llm/agentLoop/piAgentMessage.test.ts @@ -1,14 +1,24 @@ -import { describe, expect, it } from 'vitest'; +import { describe, expect, it, vi } from 'vitest'; import { convertMessages } from '@mariozechner/pi-ai/openai-completions'; +import { serviceEnv } from '@fastgpt/service/env'; + +const previewProxyEnv: { AGENT_SANDBOX_PREVIEW_PROXY_URL?: string | undefined } = serviceEnv; import { convertChatMessagesToPiAgentMessages, convertPiAgentMessagesToChatMessages, + convertToolResponseContentToPiContent, mapStopReason, normalizePiAgentMessages, replaceInteractiveToolResult, resolveInteractiveToolCall } from '@fastgpt/service/core/ai/llm/agentLoop/provider/piAgent/message'; +vi.mock('@fastgpt/service/common/file/image/utils', () => ({ + getImageBase64: vi.fn() +})); + +import { getImageBase64 } from '@fastgpt/service/common/file/image/utils'; + const model = { api: 'openai-completions', provider: 'openai', @@ -367,6 +377,113 @@ describe('piAgent message conversion', () => { ]); }); + it('keeps tool image content when an interactive tool resumes', () => { + const call = { + id: 'call_ask', + type: 'function', + function: { name: 'ask_user', arguments: '{}' } + } as any; + + const result = replaceInteractiveToolResult({ + messages: [ + { + role: 'toolResult', + toolCallId: 'call_ask', + toolName: 'ask_user', + content: [{ type: 'text', text: 'pending' }], + details: {}, + isError: true, + timestamp: 1 + } as any + ], + call, + response: '{"jobId":"task-123"}', + isError: false, + content: [ + { type: 'text', text: '{"jobId":"task-123"}' }, + { type: 'image', mimeType: 'image/png', data: 'AAAA' } + ] + }); + + // 恢复路径必须和普通路径一样保留图片，否则 ask_user 续跑会丢视觉输入。 + expect((result[0] as any).content).toEqual([ + { type: 'text', text: '{"jobId":"task-123"}' }, + { type: 'image', mimeType: 'image/png', data: 'AAAA' } + ]); + }); + + it('converts tool response parts to pi content and degrades images when vision is disabled', async () => { + const parts = [ + { type: 'text', text: 'ok' }, + { type: 'image_url', image_url: { url: 'data:image/png;base64,aW1hZ2U=' } } + ]; + + const withVision = await convertToolResponseContentToPiContent(parts as any); + expect(withVision).toEqual([ + { type: 'text', text: 'ok' }, + { type: 'image', mimeType: 'image/png', data: 'aW1hZ2U=' } + ]); + + const withoutVision = await convertToolResponseContentToPiContent(parts as any, { + useVision: false + }); + expect(withoutVision).toEqual([ + { type: 'text', text: 'ok' }, + // data URI 不能原样铺进文本，只保留 mime 占位。 + { type: 'text', text: '[Image: image/png]' } + ]); + }); + + it('masks sandbox preview urls in image placeholders when vision is disabled', async () => { + const original = previewProxyEnv.AGENT_SANDBOX_PREVIEW_PROXY_URL; + previewProxyEnv.AGENT_SANDBOX_PREVIEW_PROXY_URL = 'https://sandbox-preview.example.com'; + try { + const result = await convertToolResponseContentToPiContent( + [ + { + type: 'image_url', + image_url: { + url: 'https://sandbox-preview.example.com/preview/sandbox-1/session-1/a.png' + } + } + ] as any, + { useVision: false } + ); + + expect(result).toEqual([{ type: 'text', text: '[Image: sandbox preview file]' }]); + } finally { + previewProxyEnv.AGENT_SANDBOX_PREVIEW_PROXY_URL = original; + } + }); + + it('masks sandbox preview urls when image download fails', async () => { + const original = previewProxyEnv.AGENT_SANDBOX_PREVIEW_PROXY_URL; + previewProxyEnv.AGENT_SANDBOX_PREVIEW_PROXY_URL = 'https://sandbox-preview.example.com'; + vi.mocked(getImageBase64).mockRejectedValue(new Error('download failed')); + try { + const result = await convertToolResponseContentToPiContent( + [ + { + type: 'image_url', + image_url: { + url: 'https://sandbox-preview.example.com/preview/sandbox-1/session-1/a.png' + } + } + ] as any, + { useVision: true } + ); + + expect(result).toEqual([{ type: 'text', text: '[Image: sandbox preview file]' }]); + } finally { + previewProxyEnv.AGENT_SANDBOX_PREVIEW_PROXY_URL = original; + } + }); + + it('converts string tool responses to a single text part', async () => { + const result = await convertToolResponseContentToPiContent('plain'); + expect(result).toEqual([{ type: 'text', text: 'plain' }]); + }); + it('maps pi stop reasons to the shared agent-loop finish reasons', () => { expect(mapStopReason('toolUse')).toBe('tool_calls'); expect(mapStopReason('length')).toBe('length'); diff --git a/packages/service/test/core/ai/llm/agentLoop/piAgentToolContent.test.ts b/packages/service/test/core/ai/llm/agentLoop/piAgentToolContent.test.ts new file mode 100644 index 000000000000..8c0eb6995ab5 --- /dev/null +++ b/packages/service/test/core/ai/llm/agentLoop/piAgentToolContent.test.ts @@ -0,0 +1,85 @@ +import { describe, expect, it, vi } from 'vitest'; +import { buildPiAgentTools } from '@fastgpt/service/core/ai/llm/agentLoop/provider/piAgent/tool/build'; + +const createRuntime = ({ + useVision = true, + executeTool = vi.fn() +}: { + useVision?: boolean; + executeTool?: ReturnType; +}) => + ({ + llmParams: { + useVision, + model: { config: { vision: useVision } } + }, + toolCatalog: { + runtimeTools: [ + { + type: 'function', + function: { + name: 'get_task', + description: 'get task detail', + parameters: { type: 'object', properties: {} } + } + } + ] + }, + executeTool, + emitEvent: vi.fn(), + usagePush: vi.fn() + }) as any; + +const buildTools = (runtime: any) => + buildPiAgentTools({ + runtime, + getActivePlan: () => undefined, + setActivePlan: vi.fn(), + setPendingAsk: vi.fn(), + getMessages: () => [], + onToolCall: vi.fn(), + onToolResult: vi.fn() + }); + +describe('piAgent tool content composition', () => { + it('keeps the business response text when the tool returns an image part', async () => { + const response = '{"jobId":"task-123"}'; + const executeTool = vi.fn(async () => ({ + response, + content: [{ type: 'image_url', image_url: { url: 'data:image/png;base64,AAAA' } }], + assistantMessages: [], + usages: [], + stop: false + })); + + const tools = await buildTools(createRuntime({ executeTool })); + const tool = tools.find((item) => item.name === 'get_task'); + const result: any = await tool!.execute('call_1', {}); + + // jobId 这类业务字段必须继续发给模型，图片只作为附加 part。 + expect(result.content).toEqual([ + { type: 'text', text: response }, + { type: 'image', mimeType: 'image/png', data: 'AAAA' } + ]); + }); + + it('degrades the image part to text when the model has no vision', async () => { + const response = '{"jobId":"task-123"}'; + const executeTool = vi.fn(async () => ({ + response, + content: [{ type: 'image_url', image_url: { url: 'https://files.example.com/chart.png' } }], + assistantMessages: [], + usages: [], + stop: false + })); + + const tools = await buildTools(createRuntime({ useVision: false, executeTool })); + const tool = tools.find((item) => item.name === 'get_task'); + const result: any = await tool!.execute('call_1', {}); + + expect(result.content).toEqual([ + { type: 'text', text: response }, + { type: 'text', text: '[Image: https://files.example.com/chart.png]' } + ]); + }); +}); diff --git a/packages/service/test/core/ai/llm/toolCall.test.ts b/packages/service/test/core/ai/llm/toolCall.test.ts index d7668459c073..303e89af5e48 100644 --- a/packages/service/test/core/ai/llm/toolCall.test.ts +++ b/packages/service/test/core/ai/llm/toolCall.test.ts @@ -7,6 +7,9 @@ import type { ChatCompletionTool } from '@fastgpt/global/core/ai/llm/type'; import { describe, expect, it } from 'vitest'; +import { serviceEnv } from '@fastgpt/service/env'; + +const previewProxyEnv: { AGENT_SANDBOX_PREVIEW_PROXY_URL?: string | undefined } = serviceEnv; describe('parsePromptToolCall function tests', () => { describe('Basic scenarios', () => { @@ -703,6 +706,90 @@ describe('promptToolCallMessageRewrite function tests', () => { '\n{"result":"success","data":[1,2,3]}\n' ); }); + + it('should handle tool message with null content without throwing', () => { + const messages: ChatCompletionMessageParam[] = [ + { role: 'user', content: 'Test' }, + { role: 'tool', tool_call_id: 'call_123', content: null as unknown as string } + ]; + const tools: ChatCompletionTool[] = []; + + const result = promptToolCallMessageRewrite(messages, tools); + + expect(result[2].role).toBe('user'); + expect(result[2].content).toBe('\n\n'); + }); + + it('should preserve image-only tool content as placeholder text', () => { + const messages: ChatCompletionMessageParam[] = [ + { role: 'user', content: 'Test' }, + { + role: 'tool', + tool_call_id: 'call_123', + content: [{ type: 'image_url', image_url: { url: 'https://files.example.com/plot.png' } }] + } + ]; + const tools: ChatCompletionTool[] = []; + + const result = promptToolCallMessageRewrite(messages, tools); + + expect(result[2].role).toBe('user'); + expect(result[2].content).toBe( + '\n[Image: https://files.example.com/plot.png]\n' + ); + }); + + it('should mask sandbox preview urls in tool image placeholders', () => { + const original = previewProxyEnv.AGENT_SANDBOX_PREVIEW_PROXY_URL; + previewProxyEnv.AGENT_SANDBOX_PREVIEW_PROXY_URL = 'https://sandbox-preview.example.com'; + try { + const messages: ChatCompletionMessageParam[] = [ + { role: 'user', content: 'Test' }, + { + role: 'tool', + tool_call_id: 'call_123', + content: [ + { + type: 'image_url', + image_url: { + url: 'https://sandbox-preview.example.com/preview/sandbox-1/session-1/a.png' + } + } + ] + } + ]; + const tools: ChatCompletionTool[] = []; + + const result = promptToolCallMessageRewrite(messages, tools); + + expect(result[2].content).toBe( + '\n[Image: sandbox preview file]\n' + ); + } finally { + previewProxyEnv.AGENT_SANDBOX_PREVIEW_PROXY_URL = original; + } + }); + + it('should join text and image parts in tool content', () => { + const messages: ChatCompletionMessageParam[] = [ + { role: 'user', content: 'Test' }, + { + role: 'tool', + tool_call_id: 'call_123', + content: [ + { type: 'text', text: 'Generated plot:' }, + { type: 'image_url', image_url: { url: 'https://files.example.com/plot.png' } } + ] + } + ]; + const tools: ChatCompletionTool[] = []; + + const result = promptToolCallMessageRewrite(messages, tools); + + expect(result[2].content).toBe( + '\nGenerated plot:\n[Image: https://files.example.com/plot.png]\n' + ); + }); }); describe('Message immutability', () => { diff --git a/packages/service/test/core/ai/llm/utils.test.ts b/packages/service/test/core/ai/llm/utils.test.ts index f670631b8bef..9b3cff056b13 100644 --- a/packages/service/test/core/ai/llm/utils.test.ts +++ b/packages/service/test/core/ai/llm/utils.test.ts @@ -1,6 +1,8 @@ import { loadRequestMessages, - filterGPTMessageByMaxContext + filterGPTMessageByMaxContext, + isSandboxPreviewFileUrl, + maskImagePlaceholderUrl } from '@fastgpt/service/core/ai/llm/utils'; import { ChatCompletionContentPartInputAudioSchema, @@ -49,12 +51,17 @@ const mockGetImageBase64 = vi.mocked(getImageBase64); const mockAxiosHead = vi.mocked(axios.head); const mockAxiosGet = vi.mocked(axios.get); const originalMultipleDataToBase64 = serviceEnv.MULTIPLE_DATA_TO_BASE64; -const multimediaEnv: { MULTIPLE_DATA_TO_BASE64: boolean } = serviceEnv; +const originalPreviewProxyUrl = serviceEnv.AGENT_SANDBOX_PREVIEW_PROXY_URL; +const multimediaEnv: { + MULTIPLE_DATA_TO_BASE64: boolean; + AGENT_SANDBOX_PREVIEW_PROXY_URL?: string | undefined; +} = serviceEnv; describe('filterGPTMessageByMaxContext function tests', () => { beforeEach(() => { vi.clearAllMocks(); multimediaEnv.MULTIPLE_DATA_TO_BASE64 = originalMultipleDataToBase64; + multimediaEnv.AGENT_SANDBOX_PREVIEW_PROXY_URL = originalPreviewProxyUrl; mockCountGptMessagesTokens.mockResolvedValue(10); }); @@ -1421,6 +1428,291 @@ describe('loadRequestMessages function tests', () => { expect(result[1].content).toBe('Tool result'); }); + it('should keep tool text and move image parts into a following user message', async () => { + multimediaEnv.MULTIPLE_DATA_TO_BASE64 = false; + const messages: ChatCompletionMessageParam[] = [ + { role: ChatCompletionRequestMessageRoleEnum.User, content: 'Hello' }, + { + role: ChatCompletionRequestMessageRoleEnum.Tool, + tool_call_id: 'call1', + content: [ + { type: 'text', text: 'Look at this image' }, + { type: 'image_url', image_url: { url: 'https://example.com/a.png' } } + ] + } + ]; + + const result = await loadRequestMessages({ messages, useVision: true }); + + expect(result).toHaveLength(3); + expect(result[1].role).toBe(ChatCompletionRequestMessageRoleEnum.Tool); + expect(result[1].content).toBe('Look at this image'); + expect(result[2].role).toBe(ChatCompletionRequestMessageRoleEnum.User); + expect(result[2].content).toEqual([ + { type: 'image_url', image_url: { url: 'https://example.com/a.png' } } + ]); + }); + + it('should drop a tool image that fails to load and keep the tool text', async () => { + multimediaEnv.MULTIPLE_DATA_TO_BASE64 = true; + mockGetImageBase64.mockRejectedValue(new Error('download failed')); + const messages: ChatCompletionMessageParam[] = [ + { role: ChatCompletionRequestMessageRoleEnum.User, content: 'Hello' }, + { + role: ChatCompletionRequestMessageRoleEnum.Tool, + tool_call_id: 'call1', + content: [ + { type: 'text', text: 'Look at this image' }, + { type: 'image_url', image_url: { url: 'https://example.com/gone.png' } } + ] + } + ]; + + // 单张工具图片失效不能让整轮请求失败：丢掉图片，保留 tool 文本。 + const result = await loadRequestMessages({ messages, useVision: true }); + + expect(result).toHaveLength(2); + expect(result[1].role).toBe(ChatCompletionRequestMessageRoleEnum.Tool); + expect(result[1].content).toBe('Look at this image'); + }); + + it('should append tool images after every tool result of the same assistant turn', async () => { + multimediaEnv.MULTIPLE_DATA_TO_BASE64 = false; + const messages: ChatCompletionMessageParam[] = [ + { role: ChatCompletionRequestMessageRoleEnum.User, content: 'Hello' }, + { + role: ChatCompletionRequestMessageRoleEnum.Assistant, + content: '', + tool_calls: [ + { + id: 'call_a', + type: 'function', + function: { name: 'get_file_url', arguments: '{}' } + }, + { + id: 'call_b', + type: 'function', + function: { name: 'get_file_url', arguments: '{}' } + } + ] + }, + { + role: ChatCompletionRequestMessageRoleEnum.Tool, + tool_call_id: 'call_a', + content: [ + { type: 'text', text: 'files' }, + { type: 'image_url', image_url: { url: 'https://example.com/a.png' } } + ] + }, + { + role: ChatCompletionRequestMessageRoleEnum.Tool, + tool_call_id: 'call_b', + content: 'plain result' + } + ]; + + const result = await loadRequestMessages({ messages, useVision: true }); + + // 同一轮的全部 tool 结果必须连续回填，图片消息统一追加在其后。 + expect(result.map((item) => item.role)).toEqual([ + ChatCompletionRequestMessageRoleEnum.User, + ChatCompletionRequestMessageRoleEnum.Assistant, + ChatCompletionRequestMessageRoleEnum.Tool, + ChatCompletionRequestMessageRoleEnum.Tool, + ChatCompletionRequestMessageRoleEnum.User + ]); + expect((result[2] as any).tool_call_id).toBe('call_a'); + expect((result[3] as any).tool_call_id).toBe('call_b'); + expect(result[4].content).toEqual([ + { type: 'image_url', image_url: { url: 'https://example.com/a.png' } } + ]); + }); + + it('should convert tool image urls to base64 when configured', async () => { + multimediaEnv.MULTIPLE_DATA_TO_BASE64 = true; + mockGetImageBase64.mockResolvedValue({ + completeBase64: 'data:image/png;base64,converted', + base64: 'converted', + mime: 'image/png' + }); + + const messages: ChatCompletionMessageParam[] = [ + { role: ChatCompletionRequestMessageRoleEnum.User, content: 'Hello' }, + { + role: ChatCompletionRequestMessageRoleEnum.Tool, + tool_call_id: 'call1', + content: [{ type: 'image_url', image_url: { url: 'https://example.com/a.png' } }] + } + ]; + + const result = await loadRequestMessages({ messages, useVision: true }); + + expect(result).toHaveLength(3); + expect(result[1].role).toBe(ChatCompletionRequestMessageRoleEnum.Tool); + expect(result[1].content).toBe('none'); + expect(result[2].role).toBe(ChatCompletionRequestMessageRoleEnum.User); + expect(result[2].content).toEqual([ + { type: 'image_url', image_url: { url: 'data:image/png;base64,converted' } } + ]); + }); + + it('should inline sandbox preview image urls before sending', async () => { + multimediaEnv.MULTIPLE_DATA_TO_BASE64 = false; + multimediaEnv.AGENT_SANDBOX_PREVIEW_PROXY_URL = 'https://sandbox-preview.example.com'; + mockGetImageBase64.mockResolvedValue({ + completeBase64: 'data:image/png;base64,inlined', + base64: 'inlined', + mime: 'image/png' + }); + + const messages: ChatCompletionMessageParam[] = [ + { role: ChatCompletionRequestMessageRoleEnum.User, content: 'Hello' }, + { + role: ChatCompletionRequestMessageRoleEnum.Tool, + tool_call_id: 'call1', + content: [ + { + type: 'image_url', + image_url: { + url: 'https://sandbox-preview.example.com/preview/sandbox-1/session-1/a.png' + } + } + ] + } + ]; + + const result = await loadRequestMessages({ messages, useVision: true }); + + expect(result).toHaveLength(3); + expect(result[2].content).toEqual([ + { type: 'image_url', image_url: { url: 'data:image/png;base64,inlined' } } + ]); + // 沙箱预览代理可能在内网，取图时按配置 origin 走受限通道。 + expect(mockGetImageBase64).toHaveBeenCalledWith( + 'https://sandbox-preview.example.com/preview/sandbox-1/session-1/a.png', + { trustedOrigin: 'https://sandbox-preview.example.com' } + ); + }); + + it('should recognize preview urls when the configured base carries query or default port', async () => { + multimediaEnv.MULTIPLE_DATA_TO_BASE64 = false; + // builder 会规范化 base：host 小写、去掉默认端口与 query；识别逻辑必须一致。 + multimediaEnv.AGENT_SANDBOX_PREVIEW_PROXY_URL = + 'https://Sandbox-Preview.Example.com:443/?v=1#x'; + mockGetImageBase64.mockResolvedValue({ + completeBase64: 'data:image/png;base64,inlined', + base64: 'inlined', + mime: 'image/png' + }); + + const messages: ChatCompletionMessageParam[] = [ + { role: ChatCompletionRequestMessageRoleEnum.User, content: 'Hello' }, + { + role: ChatCompletionRequestMessageRoleEnum.Tool, + tool_call_id: 'call1', + content: [ + { + type: 'image_url', + image_url: { + url: 'https://sandbox-preview.example.com/preview/sandbox-1/session-1/a.png' + } + } + ] + } + ]; + + const result = await loadRequestMessages({ messages, useVision: true }); + + expect(result).toHaveLength(3); + expect(mockGetImageBase64).toHaveBeenCalledWith( + 'https://sandbox-preview.example.com/preview/sandbox-1/session-1/a.png', + { trustedOrigin: 'https://Sandbox-Preview.Example.com:443/?v=1#x' } + ); + }); + + it('should drop sandbox preview images when inlining fails', async () => { + multimediaEnv.MULTIPLE_DATA_TO_BASE64 = false; + multimediaEnv.AGENT_SANDBOX_PREVIEW_PROXY_URL = 'https://sandbox-preview.example.com'; + mockGetImageBase64.mockRejectedValue(new Error('download failed')); + + const messages: ChatCompletionMessageParam[] = [ + { role: ChatCompletionRequestMessageRoleEnum.User, content: 'Hello' }, + { + role: ChatCompletionRequestMessageRoleEnum.Tool, + tool_call_id: 'call1', + content: [ + { + type: 'image_url', + image_url: { + url: 'https://sandbox-preview.example.com/preview/sandbox-1/session-1/a.png' + } + } + ] + } + ]; + + const result = await loadRequestMessages({ messages, useVision: true }); + + expect(result).toHaveLength(2); + expect(result[1].role).toBe(ChatCompletionRequestMessageRoleEnum.Tool); + expect(result[1].content).toBe('none'); + expect(mockGetImageBase64).toHaveBeenCalledTimes(1); + }); + + it('should not inline non-preview remote image urls', async () => { + multimediaEnv.MULTIPLE_DATA_TO_BASE64 = false; + multimediaEnv.AGENT_SANDBOX_PREVIEW_PROXY_URL = 'https://sandbox-preview.example.com'; + + const messages: ChatCompletionMessageParam[] = [ + { role: ChatCompletionRequestMessageRoleEnum.User, content: 'Hello' }, + { + role: ChatCompletionRequestMessageRoleEnum.Tool, + tool_call_id: 'call1', + content: [{ type: 'image_url', image_url: { url: 'https://example.com/a.png' } }] + } + ]; + + const result = await loadRequestMessages({ messages, useVision: true }); + + expect(result[2].content).toEqual([ + { type: 'image_url', image_url: { url: 'https://example.com/a.png' } } + ]); + expect(mockGetImageBase64).not.toHaveBeenCalled(); + }); + + it('should drop tool images and collapse single text part without vision', async () => { + const messages: ChatCompletionMessageParam[] = [ + { role: ChatCompletionRequestMessageRoleEnum.User, content: 'Hello' }, + { + role: ChatCompletionRequestMessageRoleEnum.Tool, + tool_call_id: 'call1', + content: [ + { type: 'text', text: 'Only text survives' }, + { type: 'image_url', image_url: { url: 'https://example.com/a.png' } } + ] + } + ]; + + const result = await loadRequestMessages({ messages }); + + expect(result[1].content).toBe('Only text survives'); + }); + + it('should normalize empty tool content parts to none', async () => { + const messages: ChatCompletionMessageParam[] = [ + { role: ChatCompletionRequestMessageRoleEnum.User, content: 'Hello' }, + { + role: ChatCompletionRequestMessageRoleEnum.Tool, + tool_call_id: 'call1', + content: [] + } + ]; + + const result = await loadRequestMessages({ messages }); + + expect(result[1].content).toBe('none'); + }); + it('should handle user message with empty content as null', async () => { const messages: ChatCompletionMessageParam[] = [ { role: ChatCompletionRequestMessageRoleEnum.User, content: '' } @@ -1529,3 +1821,42 @@ describe('loadRequestMessages function tests', () => { }); }); }); + +describe('sandbox preview url helpers', () => { + beforeEach(() => { + multimediaEnv.AGENT_SANDBOX_PREVIEW_PROXY_URL = originalPreviewProxyUrl; + }); + + it('should detect sandbox preview file urls by env prefix', () => { + multimediaEnv.AGENT_SANDBOX_PREVIEW_PROXY_URL = 'https://sandbox-preview.example.com/'; + + expect(isSandboxPreviewFileUrl('https://sandbox-preview.example.com/preview/s/a/b.png')).toBe( + true + ); + expect(isSandboxPreviewFileUrl('https://sandbox-preview.example.com/other/a.png')).toBe(false); + expect(isSandboxPreviewFileUrl('https://example.com/a.png')).toBe(false); + }); + + it('should treat every url as non-preview when the env is unset', () => { + multimediaEnv.AGENT_SANDBOX_PREVIEW_PROXY_URL = undefined; + + expect(isSandboxPreviewFileUrl('https://sandbox-preview.example.com/preview/s/a/b.png')).toBe( + false + ); + }); + + it('should mask sandbox preview urls in image placeholders', () => { + multimediaEnv.AGENT_SANDBOX_PREVIEW_PROXY_URL = 'https://sandbox-preview.example.com'; + + expect(maskImagePlaceholderUrl('https://sandbox-preview.example.com/preview/s/a/b.png')).toBe( + 'sandbox preview file' + ); + expect(maskImagePlaceholderUrl('https://example.com/a.png')).toBe('https://example.com/a.png'); + }); + + it('should leave remote urls untouched in image placeholders when the env is unset', () => { + multimediaEnv.AGENT_SANDBOX_PREVIEW_PROXY_URL = undefined; + + expect(maskImagePlaceholderUrl('https://example.com/a.png')).toBe('https://example.com/a.png'); + }); +}); diff --git a/packages/service/test/core/ai/sandbox/application/toolCall/getFileUrl.tool.test.ts b/packages/service/test/core/ai/sandbox/application/toolCall/getFileUrl.tool.test.ts index b4fc151f8c03..f9b7d703accc 100644 --- a/packages/service/test/core/ai/sandbox/application/toolCall/getFileUrl.tool.test.ts +++ b/packages/service/test/core/ai/sandbox/application/toolCall/getFileUrl.tool.test.ts @@ -94,6 +94,50 @@ describe('sandboxGetFileUrlTool', () => { sessionId: 'a12345678901234567890123', filePath: '/workspace/sessions/chat-1/report/data.csv' }); + // 非图片文件只保留承载原 response 的 text part + expect(result.content).toEqual([{ type: 'text', text: result.response }]); + }); + + it('adds image_url parts for image files on top of the text part', async () => { + getFileInfo.mockResolvedValue( + new Map([['/workspace/sessions/chat-1/report/chart.png', { isDirectory: false }]]) + ); + + const result = await sandboxGetFileUrlTool.execute({ + sandboxInstance: createSandboxInstance(), + params: { paths: ['report/chart.png'] } + }); + + expect(JSON.parse(result.response)).toEqual([ + { + fileUrl: 'preview:/workspace/sessions/chat-1/report/chart.png', + filename: 'chart.png' + } + ]); + expect(result.content).toEqual([ + { type: 'text', text: result.response }, + { + type: 'image_url', + image_url: { url: 'preview:/workspace/sessions/chat-1/report/chart.png' } + } + ]); + }); + + it('keeps non-raster image files on the text path', async () => { + getFileInfo.mockResolvedValue( + new Map([ + ['/workspace/sessions/chat-1/report/logo.svg', { isDirectory: false }], + ['/workspace/sessions/chat-1/report/design.psd', { isDirectory: false }] + ]) + ); + + const result = await sandboxGetFileUrlTool.execute({ + sandboxInstance: createSandboxInstance(), + params: { paths: ['report/logo.svg', 'report/design.psd'] } + }); + + // svg / psd 等格式视觉模型消费不了，强行塞 image_url 会连文本内容一起失效。 + expect(result.content).toEqual([{ type: 'text', text: result.response }]); }); it('returns an empty result without creating a preview session', async () => { diff --git a/packages/service/test/core/workflow/dispatch/ai/agent/sub/tool/index.test.ts b/packages/service/test/core/workflow/dispatch/ai/agent/sub/tool/index.test.ts index efcbd02ce904..d71d1e44fd31 100644 --- a/packages/service/test/core/workflow/dispatch/ai/agent/sub/tool/index.test.ts +++ b/packages/service/test/core/workflow/dispatch/ai/agent/sub/tool/index.test.ts @@ -141,6 +141,56 @@ describe('dispatchTool runtime toolset auth', () => { expect(JSON.parse(result.response)).toEqual(toolResult); }); + it('extracts structured content from plugin output and keeps inline image data out of the text', async () => { + const imagePart = { + type: 'image_url' as const, + image_url: { url: 'data:image/png;base64,AAAA' } + }; + runToolStreamMock.mockResolvedValueOnce({ + output: { + jobId: 'task-123', + content: [{ type: 'text', text: '{"jobId":"task-123"}' }, imagePart] + } + }); + + const result = await dispatchTool( + createDispatchToolProps({ systemTool: { toolId: 'systemTool-search' } }) + ); + + // content 通道承载图片，业务字段仍留在 response 文本里。 + expect(result.content).toEqual([{ type: 'text', text: '{"jobId":"task-123"}' }, imagePart]); + expect(JSON.parse(result.response).jobId).toBe('task-123'); + expect(result.response).not.toContain('AAAA'); + }); + + it('converts MCP image blocks into image parts and masks the base64 text', async () => { + const tool = { name: 'search', description: 'Search' }; + const nodes = [ + { toolConfig: { mcpToolSet: { url: 'https://mcp.example.com', toolList: [tool] } } } + ]; + authAppByTmbIdMock.mockResolvedValueOnce({ app: { _id: 'victim-toolset', modules: nodes } }); + getAppVersionByIdMock.mockResolvedValueOnce({ nodes }); + getMCPChildrenMock.mockResolvedValueOnce([tool]); + const toolResult = { + content: [ + { type: 'text', text: 'chart ready' }, + { type: 'image', data: 'AAAA', mimeType: 'image/png' } + ] + }; + mcpToolCallMock.mockResolvedValueOnce(toolResult); + + const result = await dispatchTool( + createDispatchToolProps({ mcpTool: { toolId: 'mcp-victim-toolset/search' } }) + ); + + expect(result.content).toEqual([ + { type: 'image_url', image_url: { url: 'data:image/png;base64,AAAA' } } + ]); + // 文本里不再带一份 base64；原始结果仍保留在节点响应里供详情展示。 + expect(result.response).not.toContain('AAAA'); + expect(result.nodeResponse?.toolRes).toEqual(toolResult); + }); + it.each(['mcp', 'http'] as const)( 'passes Agent %s params through without schema validation', async (source) => { diff --git a/packages/service/test/core/workflow/dispatch/ai/agent/utils.test.ts b/packages/service/test/core/workflow/dispatch/ai/agent/utils.test.ts index d1eec7c5ac2a..17f5786de5a8 100644 --- a/packages/service/test/core/workflow/dispatch/ai/agent/utils.test.ts +++ b/packages/service/test/core/workflow/dispatch/ai/agent/utils.test.ts @@ -180,4 +180,98 @@ describe('Agent read_files tool protocol', () => { ); expect(streamResponseFn).not.toHaveBeenCalled(); }); + + it('passes structured tool content from dispatchTool to the agent loop result', async () => { + const content = [ + { type: 'text', text: '{"files":["chart.png"]}' }, + { type: 'image_url', image_url: { url: 'https://files.example.com/chart.png' } } + ]; + dispatchToolMock.mockResolvedValue({ + response: '{"files":["chart.png"]}', + content + }); + + const executeTool = getExecuteTool({ + checkIsStopping: vi.fn(), + chatConfig: {}, + runningUserInfo: { teamId: 'team_1', tmbId: 'tmb_1' }, + runningAppInfo: { id: 'app_1' }, + chatId: 'chat_1', + uid: 'user_1', + variableState: {} as any, + externalProvider: {} as any, + lang: 'zh-CN', + requestOrigin: '', + mode: 'chat', + timezone: 'Asia/Shanghai', + retainDatasetCite: false, + maxRunTimes: 10, + workflowDispatchDeep: 0, + params: { model: 'gpt-4' }, + stream: true, + streamResponseFn: vi.fn(), + getSubApp: () => ({ + type: 'tool', + id: 'system-tool', + name: 'System Tool', + avatar: '', + toolConfig: { systemTool: { toolId: 'systemTool-test' } }, + params: {} + }), + completionTools: [] + } as any); + + const result = await executeTool({ + callId: 'call_plugin', + toolId: 'system-tool', + args: '{}' + }); + + // 业务输出保持原样，结构化 content 单独透传给 agent-loop。 + expect(result.response).toBe('{"files":["chart.png"]}'); + expect(result.content).toEqual(content); + }); + + it('omits content when the tool returns plain text', async () => { + dispatchToolMock.mockResolvedValue({ response: 'plain result' }); + + const executeTool = getExecuteTool({ + checkIsStopping: vi.fn(), + chatConfig: {}, + runningUserInfo: { teamId: 'team_1', tmbId: 'tmb_1' }, + runningAppInfo: { id: 'app_1' }, + chatId: 'chat_1', + uid: 'user_1', + variableState: {} as any, + externalProvider: {} as any, + lang: 'zh-CN', + requestOrigin: '', + mode: 'chat', + timezone: 'Asia/Shanghai', + retainDatasetCite: false, + maxRunTimes: 10, + workflowDispatchDeep: 0, + params: { model: 'gpt-4' }, + stream: true, + streamResponseFn: vi.fn(), + getSubApp: () => ({ + type: 'tool', + id: 'system-tool', + name: 'System Tool', + avatar: '', + toolConfig: { systemTool: { toolId: 'systemTool-test' } }, + params: {} + }), + completionTools: [] + } as any); + + const result = await executeTool({ + callId: 'call_plain', + toolId: 'system-tool', + args: '{}' + }); + + expect(result.response).toBe('plain result'); + expect(result.content).toBeUndefined(); + }); }); diff --git a/packages/service/test/core/workflow/dispatch/ai/agentLoopCore/runtime/workflowToolRunner.test.ts b/packages/service/test/core/workflow/dispatch/ai/agentLoopCore/runtime/workflowToolRunner.test.ts index 9a1108439877..6787c52cf43e 100644 --- a/packages/service/test/core/workflow/dispatch/ai/agentLoopCore/runtime/workflowToolRunner.test.ts +++ b/packages/service/test/core/workflow/dispatch/ai/agentLoopCore/runtime/workflowToolRunner.test.ts @@ -1,5 +1,10 @@ import { beforeEach, describe, expect, it, vi } from 'vitest'; import { createAgentLoopCoreWorkflowToolRunner } from '@fastgpt/service/core/workflow/dispatch/ai/agentLoopCore/application/runtime/workflowToolRunner'; +import { + convertMcpToolContentToToolContent, + extractAgentLoopCoreToolContent, + maskToolImagePayload +} from '@fastgpt/service/core/workflow/dispatch/ai/agentLoopCore/domain/toolContent'; import { dispatchUpdateVariable } from '@fastgpt/service/core/workflow/dispatch/tools/runUpdateVar'; const createCall = ({ @@ -52,6 +57,46 @@ describe('createAgentLoopCoreWorkflowToolRunner', () => { vi.clearAllMocks(); }); + it('converts MCP image blocks from a tool node into image_url content', async () => { + const toolResponses = { + content: [ + { type: 'text', text: 'chart ready' }, + { type: 'image', data: 'AAAA', mimeType: 'image/png' } + ] + }; + const runner = createRunner({ + runtimeNodes: [{ nodeId: 'search', inputs: [], outputs: [] }], + getToolInfo: () => ({ type: 'user', rawData: { nodeId: 'search' } }), + runWorkflowTool: vi.fn().mockResolvedValue({ + flowResponses: [], + flowUsages: [], + assistantResponses: [], + toolResponses + }) + }); + + const result = await runner.runTool({ call: createCall() }); + + // ToolCall 节点里的 MCP 工具同样要能把原生图片块变成视觉输入。 + expect(result.content).toEqual([ + { type: 'image_url', image_url: { url: 'data:image/png;base64,AAAA' } } + ]); + // 文本里不能再带一份 base64：图片已经由 content part 承载。 + expect(result.response).toBe( + JSON.stringify( + { + content: [ + { type: 'text', text: 'chart ready' }, + { type: 'image', mimeType: 'image/png', data: '[image data omitted]' } + ] + }, + null, + 2 + ) + ); + expect(result.response).not.toContain('AAAA'); + }); + it.each([0, 1])( 'derives tool failure from errorCount=%s without a hasError flag', async (errorCount) => { @@ -597,3 +642,134 @@ describe('createAgentLoopCoreWorkflowToolRunner', () => { ]); }); }); + +describe('extractAgentLoopCoreToolContent', () => { + it('returns structured content parts when every part is valid', () => { + const content = [ + { type: 'text', text: 'done' }, + { type: 'image_url', image_url: { url: 'https://files.example.com/plot.png' } } + ]; + + expect(extractAgentLoopCoreToolContent({ content })).toEqual(content); + }); + + it('returns undefined for non-object or missing content', () => { + expect(extractAgentLoopCoreToolContent(undefined)).toBeUndefined(); + expect(extractAgentLoopCoreToolContent(null)).toBeUndefined(); + expect(extractAgentLoopCoreToolContent('text')).toBeUndefined(); + expect(extractAgentLoopCoreToolContent([])).toBeUndefined(); + expect(extractAgentLoopCoreToolContent({})).toBeUndefined(); + }); + + it('returns undefined when content is an empty array', () => { + expect(extractAgentLoopCoreToolContent({ content: [] })).toBeUndefined(); + }); + + it('returns undefined when an image_url part has an empty url', () => { + expect( + extractAgentLoopCoreToolContent({ + content: [{ type: 'image_url', image_url: { url: '' } }] + }) + ).toBeUndefined(); + }); + + it('returns undefined when any part is invalid', () => { + expect( + extractAgentLoopCoreToolContent({ + content: [ + { type: 'text', text: 'ok' }, + { type: 'image_url', image_url: {} }, + { type: 'unknown' } + ] + }) + ).toBeUndefined(); + }); + + it('rejects image parts whose detail is not accepted by the shared schema', () => { + // detail 只允许 auto/low/high；手写校验放过它会让下一轮请求被供应商拒绝。 + expect( + extractAgentLoopCoreToolContent({ + content: [ + { + type: 'image_url', + image_url: { url: 'https://files.example.com/plot.png', detail: 'ultra' } + } + ] + }) + ).toBeUndefined(); + }); +}); + +describe('convertMcpToolContentToToolContent', () => { + it('converts MCP image blocks into data URI image parts', () => { + // MCP 原生图片块是 { type: 'image', data, mimeType }，不是 image_url。 + expect( + convertMcpToolContentToToolContent([ + { type: 'text', text: 'chart ready' }, + { type: 'image', data: 'AAAA', mimeType: 'image/png' } + ]) + ).toEqual([{ type: 'image_url', image_url: { url: 'data:image/png;base64,AAAA' } }]); + }); + + it('returns undefined when the result has no image block', () => { + expect(convertMcpToolContentToToolContent([{ type: 'text', text: 'ok' }])).toBeUndefined(); + expect(convertMcpToolContentToToolContent(undefined)).toBeUndefined(); + expect(convertMcpToolContentToToolContent([])).toBeUndefined(); + }); + + it('skips non-raster image blocks that vision providers reject', () => { + expect( + convertMcpToolContentToToolContent([ + { type: 'image', data: 'AAAA', mimeType: 'image/svg+xml' } + ]) + ).toBeUndefined(); + }); + + it('drops image blocks with unusable payload', () => { + expect( + convertMcpToolContentToToolContent([{ type: 'image', data: '', mimeType: 'image/png' }]) + ).toBeUndefined(); + expect( + convertMcpToolContentToToolContent([{ type: 'image', data: 'AAAA', mimeType: 'text/html' }]) + ).toBeUndefined(); + }); +}); + +describe('maskToolImagePayload', () => { + it('replaces only the MCP image blocks that get converted', () => { + expect( + maskToolImagePayload({ + content: [ + { type: 'text', text: 'ok' }, + { type: 'image', data: 'AAAA', mimeType: 'image/png' }, + { type: 'image', data: 'BBBB', mimeType: 'image/svg+xml' } + ] + }) + ).toEqual({ + content: [ + { type: 'text', text: 'ok' }, + { type: 'image', mimeType: 'image/png', data: '[image data omitted]' }, + // 没被转换的块不能跟着消失，否则两条通道都拿不到内容。 + { type: 'image', data: 'BBBB', mimeType: 'image/svg+xml' } + ] + }); + }); + + it('masks data URI parts only when the part is also sent as content', () => { + const input = { + content: [ + { type: 'text', text: 'ok' }, + { type: 'image_url', image_url: { url: 'data:image/png;base64,AAAA' } } + ] + }; + + expect(maskToolImagePayload(input, { maskDataUrlParts: true })).toEqual({ + content: [ + { type: 'text', text: 'ok' }, + { type: 'image_url', image_url: { url: '[image data omitted]' } } + ] + }); + // 没走 content 通道时保持原样，否则唯一一份数据会被抹掉。 + expect(maskToolImagePayload(input)).toBe(input); + }); +});