fix(plugin): support structured content parts in tool responses

packages/domain/src/value-objects/tool.vo.test.ts

@@ -0,0 +1,168 @@
+import { describe, expect, it } from 'vitest';
+
+import {
+  ToolHandlerReturnSchema,
+  ToolMessageContentPartSchema,
+  ToolStreamMessageSchema
+} from './tool.vo';
+
+describe('ToolMessageContentPartSchema', () => {
+  it('accepts text parts', () => {
+    expect(ToolMessageContentPartSchema.parse({ type: 'text', text: 'hi' })).toEqual({
+      type: 'text',
+      text: 'hi'
+    });
+  });
+
+  it('accepts image_url parts with optional detail', () => {
+    const part = {
+      type: 'image_url',
+      image_url: { url: 'https://example.com/a.png', detail: 'high' }
+    };
+
+    expect(ToolMessageContentPartSchema.parse(part)).toEqual(part);
+  });
+
+  it('rejects empty text parts', () => {
+    expect(() => ToolMessageContentPartSchema.parse({ type: 'text', text: '' })).toThrow();
+  });
+
+  it('rejects image_url parts without url', () => {
+    expect(() =>
+      ToolMessageContentPartSchema.parse({ type: 'image_url', image_url: {} })
+    ).toThrow();
+  });
+
+  it('rejects unknown part types', () => {
+    expect(() => ToolMessageContentPartSchema.parse({ type: 'video' })).toThrow();
+  });
+
+  it('rejects invalid detail values', () => {
+    expect(() =>
+      ToolMessageContentPartSchema.parse({
+        type: 'image_url',
+        image_url: { url: 'https://example.com/a.png', detail: 'ultra' }
+      })
+    ).toThrow();
+  });
+
+  it('strips unknown fields inside parts', () => {
+    expect(
+      ToolMessageContentPartSchema.parse({ type: 'text', text: 'hi', extra: 1 })
+    ).toEqual({ type: 'text', text: 'hi' });
+  });
+});
+
+describe('ToolHandlerReturnSchema', () => {
+  it('keeps plain tool outputs as before', () => {
+    const data = { answer: 'ok', list: [1, 2] };
+
+    expect(ToolHandlerReturnSchema.parse(data)).toEqual(data);
+  });
+
+  it('accepts empty output objects', () => {
+    expect(ToolHandlerReturnSchema.parse({})).toEqual({});
+  });
+
+  it('accepts structured content parts', () => {
+    const data = {
+      summary: 'generated an image',
+      content: [
+        { type: 'text', text: 'see below' },
+        { type: 'image_url', image_url: { url: 'https://example.com/a.png' } }
+      ]
+    };
+
+    expect(ToolHandlerReturnSchema.parse(data)).toEqual(data);
+  });
+
+  it('keeps legacy string content untouched', () => {
+    const data = { content: 'https://example.com/a.png', title: 'Result' };
+
+    expect(ToolHandlerReturnSchema.parse(data)).toEqual(data);
+  });
+
+  it('keeps legacy object and array content untouched', () => {
+    expect(
+      ToolHandlerReturnSchema.parse({ content: { text: 'ordinary document text' } })
+    ).toEqual({ content: { text: 'ordinary document text' } });
+    expect(ToolHandlerReturnSchema.parse({ content: [1, 2, 3] })).toEqual({ content: [1, 2, 3] });
+  });
+
+  it('keeps malformed content parts untouched without wire validation', () => {
+    expect(ToolHandlerReturnSchema.parse({ content: [{ type: 'text', text: '' }] })).toEqual({
+      content: [{ type: 'text', text: '' }]
+    });
+    expect(
+      ToolHandlerReturnSchema.parse({ content: [{ type: 'image_url', image_url: {} }] })
+    ).toEqual({ content: [{ type: 'image_url', image_url: {} }] });
+  });
+
+  it('accepts an empty content array', () => {
+    expect(ToolHandlerReturnSchema.parse({ content: [] })).toEqual({ content: [] });
+  });
+
+  it('keeps unknown top-level fields', () => {
+    const data = {
+      content: [{ type: 'text', text: 'hi' }],
+      extra: { anything: true },
+      count: 3
+    };
+
+    expect(ToolHandlerReturnSchema.parse(data)).toEqual(data);
+  });
+});
+
+describe('ToolStreamMessageSchema', () => {
+  it('parses a response carrying image content parts', () => {
+    const message = {
+      type: 'response',
+      data: {
+        content: [{ type: 'image_url', image_url: { url: 'https://example.com/a.png' } }]
+      }
+    };
+
+    expect(ToolStreamMessageSchema.parse(message)).toEqual(message);
+  });
+
+  it('parses a legacy response whose content is a plain string', () => {
+    const message = {
+      type: 'response',
+      data: { content: 'https://example.com/a.png' }
+    };
+
+    expect(ToolStreamMessageSchema.parse(message)).toEqual(message);
+  });
+
+  it('parses a legacy response whose content is an object', () => {
+    const message = {
+      type: 'response',
+      data: { content: { summary: 'done', items: 2 } }
+    };
+
+    expect(ToolStreamMessageSchema.parse(message)).toEqual(message);
+  });
+
+  it('parses a legacy response whose content is an array', () => {
+    const message = {
+      type: 'response',
+      data: { content: ['a.png', 'b.png'] }
+    };
+
+    expect(ToolStreamMessageSchema.parse(message)).toEqual(message);
+  });
+
+  it('parses stream and error messages', () => {
+    const streamMessage = {
+      type: 'stream',
+      data: { type: 'answer', content: 'hello' }
+    };
+    const errorMessage = {
+      type: 'error',
+      data: 'something went wrong'
+    };
+
+    expect(ToolStreamMessageSchema.parse(streamMessage)).toEqual(streamMessage);
+    expect(ToolStreamMessageSchema.parse(errorMessage)).toEqual(errorMessage);
+  });
+});

packages/domain/src/value-objects/tool.vo.ts

@@ -3,6 +3,40 @@ import z from 'zod';
 import { PluginSourceSchema } from './plugin.vo';
 import { SystemVarSchema } from './system-var.vo';
 
+/**
+ * 工具 handler 返回的结构化内容部分,与 FastGPT 主程序对齐:
+ * text 为纯文本,image_url 为图片;主程序会把合法的 content parts 作为下一轮模型的视觉输入。
+ *
+ * 注意:发给模型的 tool 文本始终是 handler 返回值的完整序列化结果,
+ * content 里的 text part 不会替换这段文本(主程序只取其中的图片 part)。
+ * 需要模型看到业务字段时,把字段放在返回对象顶层;content 主要用于携带图片。
+ *
+ * 该 schema 供插件作者在 outputSchema 中声明使用,不参与 wire 层强校验。
+ */
+export const ToolMessageContentPartSchema = z.discriminatedUnion('type', [
+  z.object({
+    type: z.literal('text'),
+    text: z.string().min(1)
+  }),
+  z.object({
+    type: z.literal('image_url'),
+    image_url: z.object({
+      url: z.string().min(1),
+      detail: z.enum(['auto', 'low', 'high']).optional()
+    })
+  })
+]);
+
+export type ToolMessageContentPartType = z.infer<typeof ToolMessageContentPartSchema>;
+
+/**
+ * 工具 handler 返回对象的 wire 契约:保持任意 JSON 透传,与历史版本一致。
+ *
+ * content 字段不做结构化强校验,插件返回的 content 可以是任意业务值
+ * (字符串、对象、数组等),旧插件输出不受影响。需要向模型传递结构化内容时,
+ * 按约定将 content 声明并返回为 ToolMessageContentPartType[](text / image_url parts);
+ * FastGPT 主程序仅在 content 形状合法时把它作为视觉输入,否则回退到普通文本路径。
+ */
 export const ToolHandlerReturnSchema = z.record(z.string(), z.unknown());
 export type ToolHandlerReturnType = z.infer<typeof ToolHandlerReturnSchema>;
 

sdk/client/src/index.ts

@@ -57,13 +57,15 @@ export type {
   ToolListItemType,
   ToolListParamsType,
   ToolListType,
+  ToolMessageContentPartType,
   ToolRunInputType,
   ToolStreamMessageType,
   TTSModelItemType,
   WorkflowListType,
   WorkflowTemplateType
 } from './types';
 export { PluginPermissionEnum, PluginPermissionEnumSchema } from './types';
+export { ToolMessageContentPartSchema } from './types';
 export { pluginTagList } from './types';
 
 // PKG 解析