fix(agent): allow tool responses to pass image content to next-round vision input

packages/global/core/ai/llm/type.ts

@@ -129,12 +129,28 @@ export type ChatCompletionUserMessageParam = z.infer<typeof ChatCompletionUserMe
 /* ========= User end ===== */
 
 /**
- * Tool message: 对齐 openai SDK 的 ChatCompletionToolMessageParam，新增可选 `name`
- * SDK content 仅允许纯文本 part
+ * Tool message content part: FastGPT 内部扩展契约。
+ *
+ * 模型供应商的 tool message content 只接受字符串或文本 parts，openai SDK 的类型同样是
+ * `string | ChatCompletionContentPartText[]`，不接受 image_url。这里允许 image_url
+ * 是为了让工具能在内部链路里声明“取到的图片”，出站前由 loadRequestMessages 把图片
+ * 拆成这一轮 tool 结果之后的 user 视觉消息，不会把图片直接发给供应商。
+ */
+export const ChatCompletionToolMessageContentPartSchema = z.union([
+  ChatCompletionContentPartTextSchema,
+  ChatCompletionContentPartImageSchema
+]);
+export type ChatCompletionToolMessageContentPart = z.infer<
+  typeof ChatCompletionToolMessageContentPartSchema
+>;
+
+/**
+ * Tool message: 对齐 openai SDK 的 ChatCompletionToolMessageParam，新增可选 `name`。
+ * content 在内部允许带 image_url part（见上），出站时会转换；供应商只认字符串或文本 parts。
  */
 export const ChatCompletionToolMessageParamSchema = z.object({
   role: z.literal('tool'),
-  content: z.union([z.string(), z.array(ChatCompletionContentPartTextSchema)]),
+  content: z.union([z.string(), z.array(ChatCompletionToolMessageContentPartSchema)]),
   tool_call_id: z.string(),
   name: z.string().optional()
 });

packages/global/core/chat/adapt.ts

@@ -12,6 +12,7 @@ import type {
 import { ChatFileTypeEnum, ChatRoleEnum } from '../../core/chat/constants';
 import type {
   ChatCompletionContentPart,
+  ChatCompletionContentPartText,
   ChatCompletionFunctionMessageParam,
   ChatCompletionMessageFunctionCall,
   ChatCompletionMessageParam,
@@ -577,6 +578,26 @@ export const chats2GPTMessages = ({
   return results;
 };
 
+/**
+ * 提取工具响应的纯文本部分。
+ *
+ * tool content 允许是字符串或 parts 数组。折回 ChatItem 时只保留 text parts:
+ * 工具卡片展示与历史持久化都基于文本,image_url 等富媒体内容不进入这条链路,
+ * 避免把数组整体 JSON 化后污染展示与后续轮次读取。
+ */
+const getToolResponseText = (content: ChatCompletionMessageParam['content']): string => {
+  if (typeof content === 'string') return content;
+  if (!Array.isArray(content)) return '';
+  return content
+    .filter(
+      (part): part is ChatCompletionContentPartText =>
+        typeof part === 'object' && part !== null && part.type === 'text'
+    )
+    .map((part) => part.text)
+    .filter(Boolean)
+    .join('\n');
+};
+
 /**
  * 将 GPT messages 转回 FastGPT ChatItem。
  *
@@ -704,14 +725,12 @@ export const GPTMessages2Chats = ({
           const toolCalls = item.tool_calls as ChatCompletionMessageToolCall[];
 
           const tools = toolCalls.flatMap<ToolModuleResponseItemType>((tool) => {
-            let toolResponse =
-              normalizedMessages.find(
-                (msg) =>
-                  msg.role === ChatCompletionRequestMessageRoleEnum.Tool &&
-                  msg.tool_call_id === tool.id
-              )?.content || '';
-            toolResponse =
-              typeof toolResponse === 'string' ? toolResponse : JSON.stringify(toolResponse);
+            const toolContent = normalizedMessages.find(
+              (msg) =>
+                msg.role === ChatCompletionRequestMessageRoleEnum.Tool &&
+                msg.tool_call_id === tool.id
+            )?.content;
+            const toolResponse = getToolResponseText(toolContent);
 
             const toolInfo = getToolInfo?.(tool.function.name);
 

packages/global/test/core/chat/adapt.test.ts

@@ -2933,7 +2933,7 @@ describe('GPTMessages2Chats', () => {
     expect(toolValue.tools[0].response).toBe('{"results": []}');
   });
 
-  it('should stringify non-string tool response content', () => {
+  it('should keep text parts when folding non-string tool response content', () => {
     const messages: ChatCompletionMessageParam[] = [
       {
         role: ChatCompletionRequestMessageRoleEnum.Assistant,
@@ -2958,9 +2958,7 @@ describe('GPTMessages2Chats', () => {
     const result = GPTMessages2Chats({ messages, reserveTool: true });
 
     expect(result).toHaveLength(1);
-    expect((result[0].value[0] as any).tools[0].response).toBe(
-      '[{"type":"text","text":"chunked result"}]'
-    );
+    expect((result[0].value[0] as any).tools[0].response).toBe('chunked result');
   });
 
   it('should keep assistant content and reasoning separate from tools after chat roundtrip', () => {
@@ -3426,6 +3424,40 @@ describe('GPTMessages2Chats', () => {
     expect(toolValue.tools[0].toolName).toBe('Custom Tool Display Name');
     expect(toolValue.tools[0].toolAvatar).toBe('http://example.com/avatar.png');
   });
+
+  it('should keep only text parts when folding tool content parts back to chat items', () => {
+    const messages: ChatCompletionMessageParam[] = [
+      {
+        role: ChatCompletionRequestMessageRoleEnum.Assistant,
+        content: '',
+        tool_calls: [
+          {
+            id: 'call_1',
+            type: 'function',
+            function: {
+              name: 'custom_tool',
+              arguments: '{}'
+            }
+          }
+        ]
+      },
+      {
+        role: ChatCompletionRequestMessageRoleEnum.Tool,
+        tool_call_id: 'call_1',
+        content: [
+          { type: 'text', text: 'Look at these images' },
+          { type: 'image_url', image_url: { url: 'https://example.com/a.png' } }
+        ]
+      }
+    ];
+
+    const result = GPTMessages2Chats({ messages, reserveTool: true });
+
+    const toolValue = result[0].value[0] as any;
+    expect(toolValue.tools).toBeDefined();
+    expect(toolValue.tools).toHaveLength(1);
+    expect(toolValue.tools[0].response).toBe('Look at these images');
+  });
 });
 
 describe('chatValue2RuntimePrompt', () => {

packages/service/core/ai/llm/agentLoop/domain/tool.ts

@@ -1,7 +1,8 @@
 import type {
   ChatCompletionMessageParam,
   ChatCompletionMessageToolCall,
-  ChatCompletionTool
+  ChatCompletionTool,
+  ChatCompletionToolMessageContentPart
 } from '@fastgpt/global/core/ai/llm/type';
 import type { SandboxClient } from '../../../sandbox/application/runtime/client';
 import type { SandboxToolCallValidator } from '../../../sandbox/application/toolCall/type';
@@ -51,6 +52,11 @@ export type AgentLoopToolExecutionResult<TChildrenResponse = unknown> = {
   errorMessage?: string;
   /** 由调用方透传并在 agent-loop 外部解释的工具运行元数据。 */
   metadata?: unknown;
+  /**
+   * 写入 LLM tool message 的结构化 content（text / image_url parts）。
+   * 工具取到图片时可返回 image_url part，作为下一轮模型的视觉输入；缺省时按 response 字符串兜底。
+   */
+  content?: ChatCompletionToolMessageContentPart[];
 };
 
 export type AgentLoopReadFileExecutionResult = {

packages/service/core/ai/llm/agentLoop/provider/fastAgent/loop/base.ts

@@ -2,6 +2,7 @@ import type {
   ChatCompletionMessageParam,
   ChatCompletionMessageToolCall,
   ChatCompletionTool,
+  ChatCompletionToolMessageContentPart,
   CompletionFinishReason
 } from '@fastgpt/global/core/ai/llm/type';
 import { ChatCompletionRequestMessageRoleEnum } from '@fastgpt/global/core/ai/constants';
@@ -127,6 +128,23 @@ type RunAgentResponse<TChildrenResponse = unknown> = {
   finish_reason: CompletionFinishReason | undefined;
 };
 
+/**
+ * 构造写入 tool message 的 content：
+ * - 无结构化 parts 时直接用文本字符串；
+ * - 有 parts 时把最终要发送的文本作为 text part 拼在图片等 parts 前，
+ *   保证模型实际拿到的文本与展示/计费统计一致（文本已压缩时传压缩后的值）。
+ */
+export const buildToolMessageContent = ({
+  content,
+  text
+}: {
+  content?: ChatCompletionToolMessageContentPart[];
+  text: string;
+}): string | ChatCompletionToolMessageContentPart[] => {
+  if (!content?.length) return text;
+  return [{ type: 'text', text }, ...content.filter((part) => part.type !== 'text')];
+};
+
 /**
  * 上下文压缩，内部会判断是否需要压缩
  */
@@ -297,7 +315,8 @@ export const runAgentLoop = async <TChildrenResponse = unknown>({
       interactive,
       stop,
       errorMessage,
-      metadata
+      metadata,
+      content: toolContent
     } = await (async () => {
       try {
         return await onRunInteractiveTool({
@@ -331,14 +350,19 @@ export const runAgentLoop = async <TChildrenResponse = unknown>({
     // 将 requestMessages 复原成上一轮中断时的内容，并附上 tool response。
     // 新版 workflow interactive 不再持久化完整 messages 快照，调用方会从 chat history 重建；
     // 旧历史若仍带 memoryRequestMessages，则继续按旧快照恢复。
+    // 与普通工具执行路径保持一致：结构化 parts 与最终文本统一通过 buildToolMessageContent 组装。
+    const interactiveToolContent = buildToolMessageContent({
+      content: toolContent,
+      text: normalizeToolResponseContent(response)
+    });
     const resumeMessages = childrenInteractiveParams.toolParams.memoryRequestMessages?.length
       ? childrenInteractiveParams.toolParams.memoryRequestMessages
       : requestMessages;
     requestMessages = resumeMessages.map((item) =>
       item.role === 'tool' && item.tool_call_id === childrenInteractiveParams.toolParams.toolCallId
         ? {
             ...item,
-            content: normalizeToolResponseContent(response)
+            content: interactiveToolContent
           }
         : item
     );
@@ -348,7 +372,7 @@ export const runAgentLoop = async <TChildrenResponse = unknown>({
     assistantMessages.push({
       role: ChatCompletionRequestMessageRoleEnum.Tool,
       tool_call_id: childrenInteractiveParams.toolParams.toolCallId,
-      content: response
+      content: interactiveToolContent
     });
     assistantMessages.push(...filterEmptyAssistantMessages(toolAssistantMessages));
     // 相同 tool 触发了多次交互, 调用的 toolId 认为是相同的
@@ -558,7 +582,8 @@ export const runAgentLoop = async <TChildrenResponse = unknown>({
           stop: stopLoop,
           skipResponseCompress,
           errorMessage,
-          metadata
+          metadata,
+          content: toolContent
         } = await (async () => {
           try {
             return await onRunTool({
@@ -627,14 +652,22 @@ export const runAgentLoop = async <TChildrenResponse = unknown>({
           metadata
         });
 
+        // 工具返回结构化 content（如含图片 parts）时，在保留图片的前提下，
+        // 把展示/计费用的最终文本（可能已压缩）作为 text part 一并发送；
+        // 否则直接使用最终文本字符串。
+        const toolMessageContent = buildToolMessageContent({
+          content: toolContent,
+          text: toolFinalResponse
+        });
+
         return {
           tool,
           interactive,
           stopLoop,
           toolMessage: {
             tool_call_id: tool.id,
             role: ChatCompletionRequestMessageRoleEnum.Tool,
-            content: toolFinalResponse
+            content: toolMessageContent
           } as ChatCompletionMessageParam,
           toolAssistantMessages: filterEmptyAssistantMessages(toolAssistantMessages)
         };

packages/service/core/ai/llm/agentLoop/provider/fastAgent/loop/index.ts

@@ -458,7 +458,8 @@ export const runFastAgentMainLoop = async <TChildrenResponse = unknown>({
         });
         return createToolResponse(sandboxResult.response, {
           skipResponseCompress: true,
-          errorMessage: sandboxResult.success ? undefined : sandboxResult.response
+          errorMessage: sandboxResult.success ? undefined : sandboxResult.response,
+          content: sandboxResult.content
         });
       }