From 61468e9a82c3937daee431b05884d8de8e811fc2 Mon Sep 17 00:00:00 2001 From: Renjie Li Date: Thu, 1 Oct 2026 15:14:19 +0800 Subject: [PATCH] fix(agent): use tool-returned image URLs as vision input in next round --- packages/global/core/ai/llm/type.ts | 5 +- .../llm/agentLoop/provider/piAgent/message.ts | 10 ++-- .../service/core/ai/llm/promptCall/index.ts | 8 ++- packages/service/core/ai/llm/utils.ts | 17 ++++++ .../service/test/core/ai/llm/utils.test.ts | 59 +++++++++++++++++++ 5 files changed, 90 insertions(+), 9 deletions(-) diff --git a/packages/global/core/ai/llm/type.ts b/packages/global/core/ai/llm/type.ts index 7596462acbef..ab39cb9c4648 100644 --- a/packages/global/core/ai/llm/type.ts +++ b/packages/global/core/ai/llm/type.ts @@ -118,11 +118,12 @@ export type ChatCompletionUserMessageParam = z.infer { }; const getContentText = ( - content: - | string - | null - | Array<{ type: 'text'; text: string } | { type: 'refusal'; refusal: string }> - | undefined + content: string | null | Array<{ type: string; text?: string; refusal?: string }> | undefined ) => { if (!content) return ''; if (typeof content === 'string') return content; - return content.map((item) => (item.type === 'text' ? item.text : item.refusal)).join(''); + return content + .map((item) => (item.type === 'text' ? (item.text ?? '') : (item.refusal ?? ''))) + .join(''); }; /** diff --git a/packages/service/core/ai/llm/promptCall/index.ts b/packages/service/core/ai/llm/promptCall/index.ts index 8bac89871c80..dca42ce97c5a 100644 --- a/packages/service/core/ai/llm/promptCall/index.ts +++ b/packages/service/core/ai/llm/promptCall/index.ts @@ -56,9 +56,15 @@ export const promptToolCallMessageRewrite = ( delete message.tool_calls; } } else if (message.role === 'tool') { + const toolContent = + typeof message.content === 'string' + ? message.content + : Array.isArray(message.content) + ? message.content.map((item) => (item.type === 'text' ? item.text : '')).join('') + : ''; cloneMessages.splice(i, 1, { role: 'user', - content: `\n${message.content}\n` + content: `\n${toolContent}\n` }); } } diff --git a/packages/service/core/ai/llm/utils.ts b/packages/service/core/ai/llm/utils.ts index 307e4790a913..021ab29dc06b 100644 --- a/packages/service/core/ai/llm/utils.ts +++ b/packages/service/core/ai/llm/utils.ts @@ -747,6 +747,23 @@ export const loadRequestMessages = async ({ ...formatAssistantItem(item, supportReason), content: formatContent }; + } else if (item.role === ChatCompletionRequestMessageRoleEnum.Tool) { + // 工具返回的文本里可能包含图片 URL，提取为 image_url 内容块进入下一轮 + if (typeof item.content === 'string' && item.content !== '') { + const parts = parseTextUserContentParts(item.content); + // 只有提取到媒体 part（存在非 text part）时才结构化，否则保持 string 兼容 + const hasMedia = parts.some((part) => part.type !== 'text'); + if (hasMedia) { + const normalizedContent = await normalizeMediaContentParts(parts); + if (normalizedContent.length > 0) { + return { + ...item, + content: normalizedContent + }; + } + } + } + return item; } else { return item; } diff --git a/packages/service/test/core/ai/llm/utils.test.ts b/packages/service/test/core/ai/llm/utils.test.ts index 3780d74d97ff..6087903875dc 100644 --- a/packages/service/test/core/ai/llm/utils.test.ts +++ b/packages/service/test/core/ai/llm/utils.test.ts @@ -1154,6 +1154,65 @@ describe('loadRequestMessages function tests', () => { }); }); + describe('Tool message image extraction', () => { + it('should extract image URL from tool message into image_url part when useVision is true', async () => { + serviceEnv.MULTIPLE_DATA_TO_BASE64 = false; + const imageUrl = 'https://example.com/screenshot.png'; + const messages: ChatCompletionMessageParam[] = [ + { role: ChatCompletionRequestMessageRoleEnum.User, content: 'Generate a chart' }, + { + role: ChatCompletionRequestMessageRoleEnum.Tool, + tool_call_id: 'call1', + content: `Generated chart: ${imageUrl}` + } + ]; + + const result = await loadRequestMessages({ messages, useVision: true }); + + const toolMessage = result.find((m) => m.role === ChatCompletionRequestMessageRoleEnum.Tool); + expect(toolMessage).toBeDefined(); + const content = toolMessage!.content as any[]; + expect(Array.isArray(content)).toBe(true); + expect( + content.some((item: any) => item.type === 'image_url' && item.image_url.url === imageUrl) + ).toBe(true); + expect(content.some((item: any) => item.type === 'text')).toBe(true); + }); + + it('should keep tool message as string when useVision is false', async () => { + const imageUrl = 'https://example.com/screenshot.png'; + const messages: ChatCompletionMessageParam[] = [ + { role: ChatCompletionRequestMessageRoleEnum.User, content: 'Generate a chart' }, + { + role: ChatCompletionRequestMessageRoleEnum.Tool, + tool_call_id: 'call1', + content: `Generated chart: ${imageUrl}` + } + ]; + + const result = await loadRequestMessages({ messages, useVision: false }); + + const toolMessage = result.find((m) => m.role === ChatCompletionRequestMessageRoleEnum.Tool); + expect(toolMessage!.content).toBe(`Generated chart: ${imageUrl}`); + }); + + it('should keep tool message as string when it contains no image URL', async () => { + const messages: ChatCompletionMessageParam[] = [ + { role: ChatCompletionRequestMessageRoleEnum.User, content: 'Hello' }, + { + role: ChatCompletionRequestMessageRoleEnum.Tool, + tool_call_id: 'call1', + content: 'plain text result' + } + ]; + + const result = await loadRequestMessages({ messages, useVision: true }); + + const toolMessage = result.find((m) => m.role === ChatCompletionRequestMessageRoleEnum.Tool); + expect(toolMessage!.content).toBe('plain text result'); + }); + }); + describe('Complex scenarios', () => { it('should handle mixed conversation with all message types', async () => { const messages: ChatCompletionMessageParam[] = [