From 178d4dc38d9a2d9a48ecf1af5daf26288a9b262e Mon Sep 17 00:00:00 2001 From: lirenjie Date: Thu, 8 Oct 2026 20:13:53 +0800 Subject: [PATCH 1/7] fix(plugin): support structured content parts in tool responses - add ToolMessageContentPartSchema (text / image_url) in domain tool vo, allow an optional content parts array in ToolHandlerReturnSchema - sdk factory: validate the content contract before sending the response and report readable validation errors - sdk client: re-export the part schema and type for tool authors - docs: document the content parts contract and wire-first priority in the system tool SKILL - tests: cover part validation, passthrough, unknown fields and error cases --- .../domain/src/value-objects/tool.vo.test.ts | 127 ++++++++ packages/domain/src/value-objects/tool.vo.ts | 32 +- sdk/client/src/index.ts | 2 + sdk/client/src/types.ts | 3 + .../fastgpt-system-tool-development/SKILL.md | 60 ++++ sdk/factory/src/index.ts | 7 + sdk/factory/src/tool-factory.test.ts | 307 +++++++++++++++++- sdk/factory/src/tool-factory.ts | 32 +- 8 files changed, 565 insertions(+), 5 deletions(-) create mode 100644 packages/domain/src/value-objects/tool.vo.test.ts diff --git a/packages/domain/src/value-objects/tool.vo.test.ts b/packages/domain/src/value-objects/tool.vo.test.ts new file mode 100644 index 00000000..ea46c5a5 --- /dev/null +++ b/packages/domain/src/value-objects/tool.vo.test.ts @@ -0,0 +1,127 @@ +import { describe, expect, it } from 'vitest'; + +import { + ToolHandlerReturnSchema, + ToolMessageContentPartSchema, + ToolStreamMessageSchema +} from './tool.vo'; + +describe('ToolMessageContentPartSchema', () => { + it('accepts text parts', () => { + expect(ToolMessageContentPartSchema.parse({ type: 'text', text: 'hi' })).toEqual({ + type: 'text', + text: 'hi' + }); + }); + + it('accepts image_url parts with optional detail', () => { + const part = { + type: 'image_url', + image_url: { url: 'https://example.com/a.png', detail: 'high' } + }; + + expect(ToolMessageContentPartSchema.parse(part)).toEqual(part); + }); + + it('rejects empty text parts', () => { + expect(() => ToolMessageContentPartSchema.parse({ type: 'text', text: '' })).toThrow(); + }); + + it('rejects image_url parts without url', () => { + expect(() => + ToolMessageContentPartSchema.parse({ type: 'image_url', image_url: {} }) + ).toThrow(); + }); + + it('rejects unknown part types', () => { + expect(() => ToolMessageContentPartSchema.parse({ type: 'video' })).toThrow(); + }); + + it('rejects invalid detail values', () => { + expect(() => + ToolMessageContentPartSchema.parse({ + type: 'image_url', + image_url: { url: 'https://example.com/a.png', detail: 'ultra' } + }) + ).toThrow(); + }); + + it('strips unknown fields inside parts', () => { + expect( + ToolMessageContentPartSchema.parse({ type: 'text', text: 'hi', extra: 1 }) + ).toEqual({ type: 'text', text: 'hi' }); + }); +}); + +describe('ToolHandlerReturnSchema', () => { + it('keeps plain tool outputs as before', () => { + const data = { answer: 'ok', list: [1, 2] }; + + expect(ToolHandlerReturnSchema.parse(data)).toEqual(data); + }); + + it('accepts empty output objects', () => { + expect(ToolHandlerReturnSchema.parse({})).toEqual({}); + }); + + it('accepts structured content parts', () => { + const data = { + summary: 'generated an image', + content: [ + { type: 'text', text: 'see below' }, + { type: 'image_url', image_url: { url: 'https://example.com/a.png' } } + ] + }; + + expect(ToolHandlerReturnSchema.parse(data)).toEqual(data); + }); + + it('rejects content that is not a parts array', () => { + expect(() => + ToolHandlerReturnSchema.parse({ content: 'https://example.com/a.png' }) + ).toThrow(); + }); + + it('rejects invalid content parts', () => { + expect(() => ToolHandlerReturnSchema.parse({ content: [{ type: 'text', text: '' }] })).toThrow(); + expect(() => + ToolHandlerReturnSchema.parse({ content: [{ type: 'image_url', image_url: {} }] }) + ).toThrow(); + }); + + it('accepts an empty content array', () => { + expect(ToolHandlerReturnSchema.parse({ content: [] })).toEqual({ content: [] }); + }); + + it('keeps unknown top-level fields', () => { + const data = { + content: [{ type: 'text', text: 'hi' }], + extra: { anything: true }, + count: 3 + }; + + expect(ToolHandlerReturnSchema.parse(data)).toEqual(data); + }); +}); + +describe('ToolStreamMessageSchema', () => { + it('parses a response carrying image content parts', () => { + const message = { + type: 'response', + data: { + content: [{ type: 'image_url', image_url: { url: 'https://example.com/a.png' } }] + } + }; + + expect(ToolStreamMessageSchema.parse(message)).toEqual(message); + }); + + it('rejects a response whose content is not structured', () => { + expect(() => + ToolStreamMessageSchema.parse({ + type: 'response', + data: { content: 'https://example.com/a.png' } + }) + ).toThrow(); + }); +}); diff --git a/packages/domain/src/value-objects/tool.vo.ts b/packages/domain/src/value-objects/tool.vo.ts index 774cdc75..25a241d7 100644 --- a/packages/domain/src/value-objects/tool.vo.ts +++ b/packages/domain/src/value-objects/tool.vo.ts @@ -3,7 +3,37 @@ import z from 'zod'; import { PluginSourceSchema } from './plugin.vo'; import { SystemVarSchema } from './system-var.vo'; -export const ToolHandlerReturnSchema = z.record(z.string(), z.unknown()); +/** + * 工具 handler 返回的结构化内容部分,与 FastGPT 主程序对齐: + * text 为纯文本,image_url 为图片;主程序会把合法的 content 作为下一轮模型的视觉输入。 + */ +export const ToolMessageContentPartSchema = z.discriminatedUnion('type', [ + z.object({ + type: z.literal('text'), + text: z.string().min(1) + }), + z.object({ + type: z.literal('image_url'), + image_url: z.object({ + url: z.string().min(1), + detail: z.enum(['auto', 'low', 'high']).optional() + }) + }) +]); + +export type ToolMessageContentPartType = z.infer; + +/** + * 工具 handler 返回对象的结构契约: + * 普通输出字段保持任意 JSON;当包含 content 字段时,必须是 + * ToolMessageContentPartType[](text / image_url 结构化内容), + * 不允许用 content 传字符串等非结构化内容。 + */ +export const ToolHandlerReturnSchema = z + .object({ + content: z.array(ToolMessageContentPartSchema).optional() + }) + .catchall(z.unknown()); export type ToolHandlerReturnType = z.infer; export const StreamMessageTypeSchema = z.enum(['response', 'error', 'stream']); diff --git a/sdk/client/src/index.ts b/sdk/client/src/index.ts index 86b3f6ce..80aac68d 100644 --- a/sdk/client/src/index.ts +++ b/sdk/client/src/index.ts @@ -57,6 +57,7 @@ export type { ToolListItemType, ToolListParamsType, ToolListType, + ToolMessageContentPartType, ToolRunInputType, ToolStreamMessageType, TTSModelItemType, @@ -64,6 +65,7 @@ export type { WorkflowTemplateType } from './types'; export { PluginPermissionEnum, PluginPermissionEnumSchema } from './types'; +export { ToolMessageContentPartSchema } from './types'; export { pluginTagList } from './types'; // PKG 解析 diff --git a/sdk/client/src/types.ts b/sdk/client/src/types.ts index 41c4f4b2..7d2fec78 100644 --- a/sdk/client/src/types.ts +++ b/sdk/client/src/types.ts @@ -65,6 +65,7 @@ import type { SystemVarType } from '@domain/value-objects/system-var.vo'; import type { ToolAnswerType, ToolHandlerReturnType, + ToolMessageContentPartType, ToolStreamMessageType } from '@domain/value-objects/tool.vo'; import { PluginTagsNameMap } from '@infrastructure/static-data/plugin-tag'; @@ -87,6 +88,7 @@ export type { SystemVarType, ToolAnswerType, ToolHandlerReturnType, + ToolMessageContentPartType, ToolStreamMessageType, TTSModelItemType }; @@ -165,3 +167,4 @@ export { PluginPermissionEnumSchema, type PluginPermissionEnumType } from '@domain/value-objects/permission.vo'; +export { ToolMessageContentPartSchema } from '@domain/value-objects/tool.vo'; diff --git a/sdk/factory/skills/fastgpt-system-tool-development/SKILL.md b/sdk/factory/skills/fastgpt-system-tool-development/SKILL.md index 82d7ae19..39a3fc9b 100644 --- a/sdk/factory/skills/fastgpt-system-tool-development/SKILL.md +++ b/sdk/factory/skills/fastgpt-system-tool-development/SKILL.md @@ -67,6 +67,7 @@ npx @fastgpt-plugin/cli pack --entry --dist **契约优先级（重要）**：`content` 字段的合法性由 SDK 运行时按上述 wire 契约强校验，先于 `outputSchema` 的类型声明。因此 `outputSchema` 里的 `content` 也必须声明为 `z.array(ToolMessageContentPartSchema)`（而不是 `z.string()` 之类），否则类型声明与运行时行为不一致。 +> +> 这是对旧版插件的破坏性变更：此前工具可以返回任意 JSON；现在只要顶层出现 `content` 字段且不是合法的内容部分数组，工具执行就会直接报错。存量工具若把图片地址以字符串形式放在 `content` 字段里，需要改成结构化 parts，或改用其它字段名。 + +示例： + +```ts +import { + createToolHandler, + defineTool, + ToolMessageContentPartSchema, + type ToolMessageContentPartType +} from '@fastgpt-plugin/sdk-factory'; +import z from 'zod'; + +const handler = createToolHandler({ + inputSchema: z.object({ + prompt: z.string().min(1) + }), + outputSchema: z.object({ + message: z.string(), + content: z.array(ToolMessageContentPartSchema) + }), + handler: async (input) => { + const imageUrl = 'https://example.com/result.png'; + const content: ToolMessageContentPartType[] = [ + { type: 'image_url', image_url: { url: imageUrl } } + ]; + + return { + message: `Generated image for: ${input.prompt}`, + content + }; + } +}); + +export default defineTool({ + manifest: { + pluginId: 'image-generator', + version: '1.0.0', + name: { en: 'Image Generator', 'zh-CN': '图片生成' }, + description: { en: 'Generate an image', 'zh-CN': '生成图片' }, + permission: [] + }, + handler +}); +``` + ## 验证重点 - `index.ts` 默认导出正确。 @@ -266,6 +324,8 @@ export default defineToolSet({ - 使用 `ctx.invoke` 时已在 `manifest.permission` 声明对应权限，例如上传文件声明 `file-upload:allow`。 - Zod schema 覆盖全部用户可见输入和输出；推荐由 AI 托管补充的输入字段设置 `isToolParam: true`。 - handler 成功路径返回值与 `outputSchema` 一致。 +- 返回 `content` 时 `outputSchema` 声明为 `z.array(ToolMessageContentPartSchema)`，与运行时 wire 校验保持一致（运行时优先于类型声明）。 +- 返回图片内容时使用结构化 `content` 字段（`ToolMessageContentPartSchema`），不在文本中夹带图片地址。 - 外部调用失败、空响应、超时、鉴权失败都有明确错误。 - 密钥配置只通过 `secretSchema` 和 `ctx.secrets` 处理。 - 系统工具根目录存在 `logo.`。 diff --git a/sdk/factory/src/index.ts b/sdk/factory/src/index.ts index d19b30de..d8f43fa8 100644 --- a/sdk/factory/src/index.ts +++ b/sdk/factory/src/index.ts @@ -21,6 +21,13 @@ export type I18nSchemaMetaType = { placeholder?: I18nStringType; }; +export { + ToolHandlerReturnSchema, + type ToolHandlerReturnType, + ToolMessageContentPartSchema, + type ToolMessageContentPartType +} from '@domain/value-objects/tool.vo'; + export type InputSchemaMetaType = z.GlobalMeta & { /** Localized strings for the UI. */ 'x-i18n'?: I18nSchemaMetaType; diff --git a/sdk/factory/src/tool-factory.test.ts b/sdk/factory/src/tool-factory.test.ts index 0b44012f..c2d384e1 100644 --- a/sdk/factory/src/tool-factory.test.ts +++ b/sdk/factory/src/tool-factory.test.ts @@ -18,7 +18,9 @@ import { type I18nSchemaMetaType, type InputSchemaMetaType, type OutputSchemaMetaType, - type SecretSchemaMetaType + type SecretSchemaMetaType, + ToolMessageContentPartSchema, + type ToolMessageContentPartType } from './index'; describe('ToolFactory streaming', () => { @@ -326,6 +328,309 @@ describe('ToolFactory streaming', () => { }); }); +describe('ToolFactory response content parts', () => { + const previousRuntimeMode = process.env.RUNTIME_MODE; + + afterEach(() => { + setCurrentLocalDebugRuntime(undefined); + if (previousRuntimeMode === undefined) { + delete process.env.RUNTIME_MODE; + } else { + process.env.RUNTIME_MODE = previousRuntimeMode; + } + }); + + it('streams structured content parts in the response data', async () => { + const runtime = createLocalDebugRuntime(); + setCurrentLocalDebugRuntime(runtime); + process.env.RUNTIME_MODE = 'dev'; + + defineTool({ + manifest: { + pluginId: 'content-parts-test', + name: { en: 'Content Parts Test', 'zh-CN': '内容部分测试' }, + description: { en: 'Content Parts Test', 'zh-CN': '内容部分测试' }, + version: '0.0.1', + versionDescription: { en: 'Test version', 'zh-CN': '测试版本' }, + tags: ['tools'] + }, + handler: createToolHandler({ + inputSchema: z.object({}), + outputSchema: z.object({ + content: z.array(ToolMessageContentPartSchema) + }), + handler: async () => { + const content: ToolMessageContentPartType[] = [ + { type: 'text', text: 'generated image' }, + { type: 'image_url', image_url: { url: 'https://example.com/a.png' } } + ]; + + return { content }; + } + }) + }); + + await runtime.waitUntilReady(); + + const response = await runtime.invokePlugin< + { + input: Record; + systemVar: Record; + }, + void, + never, + ToolStreamMessageType + >('run', { + input: {}, + systemVar: {} + }); + + const messages: ToolStreamMessageType[] = []; + await response.output!.stream.consume((chunk) => { + messages.push(chunk); + }); + + expect(messages).toEqual([ + { + type: 'response', + data: { + content: [ + { type: 'text', text: 'generated image' }, + { type: 'image_url', image_url: { url: 'https://example.com/a.png' } } + ] + } + } + ]); + }); + + it('sends an error event when the handler returns invalid content parts', async () => { + const runtime = createLocalDebugRuntime(); + setCurrentLocalDebugRuntime(runtime); + process.env.RUNTIME_MODE = 'dev'; + + defineTool({ + manifest: { + pluginId: 'invalid-content-parts-test', + name: { en: 'Invalid Content Parts Test', 'zh-CN': '非法内容部分测试' }, + description: { en: 'Invalid Content Parts Test', 'zh-CN': '非法内容部分测试' }, + version: '0.0.1', + versionDescription: { en: 'Test version', 'zh-CN': '测试版本' }, + tags: ['tools'] + }, + handler: createToolHandler({ + inputSchema: z.object({}), + outputSchema: z.object({ + content: z.string() + }), + handler: async () => ({ + content: 'https://example.com/a.png' + }) + }) + }); + + await runtime.waitUntilReady(); + + const response = await runtime.invokePlugin< + { + input: Record; + systemVar: Record; + }, + void, + never, + ToolStreamMessageType + >('run', { + input: {}, + systemVar: {} + }); + + const messages: ToolStreamMessageType[] = []; + await response.output!.stream.consume((chunk) => { + messages.push(chunk); + }); + + expect(messages).toEqual([ + { + type: 'error', + data: expect.stringContaining('content 字段必须是') + } + ]); + }); + + it('accepts an empty content array', async () => { + const runtime = createLocalDebugRuntime(); + setCurrentLocalDebugRuntime(runtime); + process.env.RUNTIME_MODE = 'dev'; + + defineTool({ + manifest: { + pluginId: 'empty-content-test', + name: { en: 'Empty Content Test', 'zh-CN': '空内容测试' }, + description: { en: 'Empty Content Test', 'zh-CN': '空内容测试' }, + version: '0.0.1', + versionDescription: { en: 'Test version', 'zh-CN': '测试版本' }, + tags: ['tools'] + }, + handler: createToolHandler({ + inputSchema: z.object({}), + outputSchema: z.object({ + content: z.array(ToolMessageContentPartSchema) + }), + handler: async () => ({ + content: [] + }) + }) + }); + + await runtime.waitUntilReady(); + + const response = await runtime.invokePlugin< + { + input: Record; + systemVar: Record; + }, + void, + never, + ToolStreamMessageType + >('run', { + input: {}, + systemVar: {} + }); + + const messages: ToolStreamMessageType[] = []; + await response.output!.stream.consume((chunk) => { + messages.push(chunk); + }); + + expect(messages).toEqual([ + { + type: 'response', + data: { + content: [] + } + } + ]); + }); + + it('keeps unknown fields alongside valid content parts', async () => { + const runtime = createLocalDebugRuntime(); + setCurrentLocalDebugRuntime(runtime); + process.env.RUNTIME_MODE = 'dev'; + + defineTool({ + manifest: { + pluginId: 'extra-fields-test', + name: { en: 'Extra Fields Test', 'zh-CN': '附加字段测试' }, + description: { en: 'Extra Fields Test', 'zh-CN': '附加字段测试' }, + version: '0.0.1', + versionDescription: { en: 'Test version', 'zh-CN': '测试版本' }, + tags: ['tools'] + }, + handler: createToolHandler({ + inputSchema: z.object({}), + outputSchema: z.object({ + content: z.array(ToolMessageContentPartSchema), + count: z.number(), + summary: z.string() + }), + handler: async () => { + const content: ToolMessageContentPartType[] = [{ type: 'text', text: 'see below' }]; + + return { content, count: 3, summary: 'done' }; + } + }) + }); + + await runtime.waitUntilReady(); + + const response = await runtime.invokePlugin< + { + input: Record; + systemVar: Record; + }, + void, + never, + ToolStreamMessageType + >('run', { + input: {}, + systemVar: {} + }); + + const messages: ToolStreamMessageType[] = []; + await response.output!.stream.consume((chunk) => { + messages.push(chunk); + }); + + expect(messages).toEqual([ + { + type: 'response', + data: { + content: [{ type: 'text', text: 'see below' }], + count: 3, + summary: 'done' + } + } + ]); + }); + + it('sends a readable error for invalid content part details', async () => { + const runtime = createLocalDebugRuntime(); + setCurrentLocalDebugRuntime(runtime); + process.env.RUNTIME_MODE = 'dev'; + + defineTool({ + manifest: { + pluginId: 'invalid-detail-test', + name: { en: 'Invalid Detail Test', 'zh-CN': '非法详情测试' }, + description: { en: 'Invalid Detail Test', 'zh-CN': '非法详情测试' }, + version: '0.0.1', + versionDescription: { en: 'Test version', 'zh-CN': '测试版本' }, + tags: ['tools'] + }, + handler: createToolHandler({ + inputSchema: z.object({}), + outputSchema: z.object({ + content: z.array(ToolMessageContentPartSchema) + }), + handler: async () => ({ + content: [ + { + type: 'image_url', + image_url: { url: 'https://example.com/a.png', detail: 'ultra' } + } as unknown as ToolMessageContentPartType + ] + }) + }) + }); + + await runtime.waitUntilReady(); + + const response = await runtime.invokePlugin< + { + input: Record; + systemVar: Record; + }, + void, + never, + ToolStreamMessageType + >('run', { + input: {}, + systemVar: {} + }); + + const messages: ToolStreamMessageType[] = []; + await response.output!.stream.consume((chunk) => { + messages.push(chunk); + }); + + expect(messages).toEqual([ + { + type: 'error', + data: expect.stringContaining('content.0.image_url.detail') + } + ]); + }); +}); + describe('ToolFactory schema metadata', () => { const localized = { en: 'Localized text', diff --git a/sdk/factory/src/tool-factory.ts b/sdk/factory/src/tool-factory.ts index 850c48d6..f5ffc45b 100644 --- a/sdk/factory/src/tool-factory.ts +++ b/sdk/factory/src/tool-factory.ts @@ -3,7 +3,11 @@ import z from 'zod'; import type { InvokePort } from '@domain/ports/invoke.port'; import { StreamData } from '@domain/value-objects/stream.vo'; import type { SystemVarType } from '@domain/value-objects/system-var.vo'; -import type { ToolAnswerType, ToolStreamMessageType } from '@domain/value-objects/tool.vo'; +import { + type ToolAnswerType, + ToolHandlerReturnSchema, + type ToolStreamMessageType +} from '@domain/value-objects/tool.vo'; import { PluginChannelHostMethod } from '@infrastructure/plugin/plugin-runtime/ports/channel'; import type { PluginToolRunPayloadType } from '@infrastructure/plugin/tool.impl'; import { getErrText } from '@shared/utils/err'; @@ -11,6 +15,24 @@ import { getErrText } from '@shared/utils/err'; import { InvokeClient } from './invoke.client'; import type { UserToolManifestType } from './manifest.type'; import { PluginFactory } from './plugin-factory'; + +/** + * 把 zod 校验错误整理成人类可读文本,避免默认的 JSON issues 大段回显。 + * 当错误发生在 content 字段时附上结构化内容契约提示,方便插件作者定位。 + */ +const formatZodError = (error: z.ZodError): string => { + const details = error.issues + .map((issue) => { + const path = issue.path.length > 0 ? `${issue.path.join('.')}: ` : ''; + return `${path}${issue.message}`; + }) + .join('; '); + const contentHint = error.issues.some((issue) => issue.path[0] === 'content') + ? '工具返回的 content 字段必须是结构化内容数组({type:"text",text} 或 {type:"image_url",image_url:{url}}),不能是字符串或其他结构。' + : ''; + + return contentHint ? `${contentHint} ${details}` : details; +}; export type ToolContextType> = { systemVar: SystemVarType; secrets: TSecret; @@ -111,13 +133,17 @@ export class ToolFactory extends PluginFactory { } }); + // 校验 handler 返回值:content 字段(若存在)必须是合法的 content parts,否则视为工具错误 output.send({ type: 'response', - data: result + data: ToolHandlerReturnSchema.parse(result) }); } catch (err) { output.send({ - data: getErrText(err, 'Unknown error during tool execution'), + data: getErrText( + err instanceof z.ZodError ? formatZodError(err) : err, + 'Unknown error during tool execution' + ), type: 'error' }); } finally { From c6205149e564f015f64f27d30ed4c0c62010350c Mon Sep 17 00:00:00 2001 From: Renjie Li Date: Tue, 13 Oct 2026 10:37:41 +0800 Subject: [PATCH 2/7] fix(plugin): keep legacy tool response content passthrough on the wire --- .../domain/src/value-objects/tool.vo.test.ts | 39 ++++--- packages/domain/src/value-objects/tool.vo.ts | 19 ++-- .../fastgpt-system-tool-development/SKILL.md | 10 +- sdk/factory/src/tool-factory.test.ts | 105 +++++++++++++++--- sdk/factory/src/tool-factory.ts | 33 +----- 5 files changed, 131 insertions(+), 75 deletions(-) diff --git a/packages/domain/src/value-objects/tool.vo.test.ts b/packages/domain/src/value-objects/tool.vo.test.ts index ea46c5a5..f2e9a5e4 100644 --- a/packages/domain/src/value-objects/tool.vo.test.ts +++ b/packages/domain/src/value-objects/tool.vo.test.ts @@ -76,17 +76,26 @@ describe('ToolHandlerReturnSchema', () => { expect(ToolHandlerReturnSchema.parse(data)).toEqual(data); }); - it('rejects content that is not a parts array', () => { - expect(() => - ToolHandlerReturnSchema.parse({ content: 'https://example.com/a.png' }) - ).toThrow(); + it('keeps legacy string content untouched', () => { + const data = { content: 'https://example.com/a.png', title: 'Result' }; + + expect(ToolHandlerReturnSchema.parse(data)).toEqual(data); }); - it('rejects invalid content parts', () => { - expect(() => ToolHandlerReturnSchema.parse({ content: [{ type: 'text', text: '' }] })).toThrow(); - expect(() => + it('keeps legacy object and array content untouched', () => { + expect( + ToolHandlerReturnSchema.parse({ content: { text: 'ordinary document text' } }) + ).toEqual({ content: { text: 'ordinary document text' } }); + expect(ToolHandlerReturnSchema.parse({ content: [1, 2, 3] })).toEqual({ content: [1, 2, 3] }); + }); + + it('keeps malformed content parts untouched without wire validation', () => { + expect(ToolHandlerReturnSchema.parse({ content: [{ type: 'text', text: '' }] })).toEqual({ + content: [{ type: 'text', text: '' }] + }); + expect( ToolHandlerReturnSchema.parse({ content: [{ type: 'image_url', image_url: {} }] }) - ).toThrow(); + ).toEqual({ content: [{ type: 'image_url', image_url: {} }] }); }); it('accepts an empty content array', () => { @@ -116,12 +125,12 @@ describe('ToolStreamMessageSchema', () => { expect(ToolStreamMessageSchema.parse(message)).toEqual(message); }); - it('rejects a response whose content is not structured', () => { - expect(() => - ToolStreamMessageSchema.parse({ - type: 'response', - data: { content: 'https://example.com/a.png' } - }) - ).toThrow(); + it('parses a legacy response whose content is a plain string', () => { + const message = { + type: 'response', + data: { content: 'https://example.com/a.png' } + }; + + expect(ToolStreamMessageSchema.parse(message)).toEqual(message); }); }); diff --git a/packages/domain/src/value-objects/tool.vo.ts b/packages/domain/src/value-objects/tool.vo.ts index 25a241d7..b8f647d8 100644 --- a/packages/domain/src/value-objects/tool.vo.ts +++ b/packages/domain/src/value-objects/tool.vo.ts @@ -5,7 +5,8 @@ import { SystemVarSchema } from './system-var.vo'; /** * 工具 handler 返回的结构化内容部分,与 FastGPT 主程序对齐: - * text 为纯文本,image_url 为图片;主程序会把合法的 content 作为下一轮模型的视觉输入。 + * text 为纯文本,image_url 为图片;主程序会把合法的 content parts 作为下一轮模型的视觉输入。 + * 该 schema 供插件作者在 outputSchema 中声明使用,不参与 wire 层强校验。 */ export const ToolMessageContentPartSchema = z.discriminatedUnion('type', [ z.object({ @@ -24,16 +25,14 @@ export const ToolMessageContentPartSchema = z.discriminatedUnion('type', [ export type ToolMessageContentPartType = z.infer; /** - * 工具 handler 返回对象的结构契约: - * 普通输出字段保持任意 JSON;当包含 content 字段时,必须是 - * ToolMessageContentPartType[](text / image_url 结构化内容), - * 不允许用 content 传字符串等非结构化内容。 + * 工具 handler 返回对象的 wire 契约:保持任意 JSON 透传,与历史版本一致。 + * + * content 字段不做结构化强校验,插件返回的 content 可以是任意业务值 + * (字符串、对象、数组等),旧插件输出不受影响。需要向模型传递结构化内容时, + * 按约定将 content 声明并返回为 ToolMessageContentPartType[](text / image_url parts); + * FastGPT 主程序仅在 content 形状合法时把它作为视觉输入,否则回退到普通文本路径。 */ -export const ToolHandlerReturnSchema = z - .object({ - content: z.array(ToolMessageContentPartSchema).optional() - }) - .catchall(z.unknown()); +export const ToolHandlerReturnSchema = z.record(z.string(), z.unknown()); export type ToolHandlerReturnType = z.infer; export const StreamMessageTypeSchema = z.enum(['response', 'error', 'stream']); diff --git a/sdk/factory/skills/fastgpt-system-tool-development/SKILL.md b/sdk/factory/skills/fastgpt-system-tool-development/SKILL.md index 39a3fc9b..96187d4d 100644 --- a/sdk/factory/skills/fastgpt-system-tool-development/SKILL.md +++ b/sdk/factory/skills/fastgpt-system-tool-development/SKILL.md @@ -261,16 +261,14 @@ export default defineToolSet({ ## 返回图片内容（可选） -工具 handler 的返回对象里可以包含 `content` 字段，向 FastGPT 主程序返回结构化图片内容，作为下一轮模型的视觉输入。`content` 必须是内容部分数组，每项只能是以下两种之一： +工具 handler 的返回对象里可以包含 `content` 字段，向 FastGPT 主程序返回结构化图片内容，作为下一轮模型的视觉输入。此时 `content` 应为内容部分数组，每项只能是以下两种之一： - `{ type: 'text', text: string }`：文本内容。 - `{ type: 'image_url', image_url: { url: string, detail?: 'auto' | 'low' | 'high' } }`：图片内容，`url` 为图片地址。 -`content` 必须整体合法（是数组、每项符合上述结构），否则工具执行会报错；不要在普通字符串文本里猜测图片地址。普通文本结果仍放在其它输出字段中。 - -> **契约优先级（重要）**：`content` 字段的合法性由 SDK 运行时按上述 wire 契约强校验，先于 `outputSchema` 的类型声明。因此 `outputSchema` 里的 `content` 也必须声明为 `z.array(ToolMessageContentPartSchema)`（而不是 `z.string()` 之类），否则类型声明与运行时行为不一致。 +> **向后兼容（重要）**：`content` 是普通业务字段，SDK 的 wire 层保持任意 JSON 透传，不做强校验。旧插件把字符串、对象或普通数组放在 `content` 里仍可正常工作，主程序会把这些值当作普通文本结果处理。 > -> 这是对旧版插件的破坏性变更：此前工具可以返回任意 JSON；现在只要顶层出现 `content` 字段且不是合法的内容部分数组，工具执行就会直接报错。存量工具若把图片地址以字符串形式放在 `content` 字段里，需要改成结构化 parts，或改用其它字段名。 +> 只有当你希望图片作为下一轮模型的视觉输入时，才按上述 parts 形状返回 `content`。主程序仅在 `content` 形状合法时把它作为视觉输入，否则回退到普通文本路径；因此不要只放一个孤立的图片链接字符串，也不要在普通文本里夹带图片地址。 示例： @@ -324,7 +322,7 @@ export default defineTool({ - 使用 `ctx.invoke` 时已在 `manifest.permission` 声明对应权限，例如上传文件声明 `file-upload:allow`。 - Zod schema 覆盖全部用户可见输入和输出；推荐由 AI 托管补充的输入字段设置 `isToolParam: true`。 - handler 成功路径返回值与 `outputSchema` 一致。 -- 返回 `content` 时 `outputSchema` 声明为 `z.array(ToolMessageContentPartSchema)`，与运行时 wire 校验保持一致（运行时优先于类型声明）。 +- 返回结构化图片内容时 `outputSchema` 里的 `content` 声明为 `z.array(ToolMessageContentPartSchema)`；旧插件把 `content` 当普通业务字段（字符串/对象/数组）使用仍兼容，无需迁移。 - 返回图片内容时使用结构化 `content` 字段（`ToolMessageContentPartSchema`），不在文本中夹带图片地址。 - 外部调用失败、空响应、超时、鉴权失败都有明确错误。 - 密钥配置只通过 `secretSchema` 和 `ctx.secrets` 处理。 diff --git a/sdk/factory/src/tool-factory.test.ts b/sdk/factory/src/tool-factory.test.ts index c2d384e1..ebb75f5a 100644 --- a/sdk/factory/src/tool-factory.test.ts +++ b/sdk/factory/src/tool-factory.test.ts @@ -403,16 +403,16 @@ describe('ToolFactory response content parts', () => { ]); }); - it('sends an error event when the handler returns invalid content parts', async () => { + it('passes legacy string content through the response data', async () => { const runtime = createLocalDebugRuntime(); setCurrentLocalDebugRuntime(runtime); process.env.RUNTIME_MODE = 'dev'; defineTool({ manifest: { - pluginId: 'invalid-content-parts-test', - name: { en: 'Invalid Content Parts Test', 'zh-CN': '非法内容部分测试' }, - description: { en: 'Invalid Content Parts Test', 'zh-CN': '非法内容部分测试' }, + pluginId: 'legacy-string-content-test', + name: { en: 'Legacy String Content Test', 'zh-CN': '旧字符串内容测试' }, + description: { en: 'Legacy String Content Test', 'zh-CN': '旧字符串内容测试' }, version: '0.0.1', versionDescription: { en: 'Test version', 'zh-CN': '测试版本' }, tags: ['tools'] @@ -420,10 +420,12 @@ describe('ToolFactory response content parts', () => { handler: createToolHandler({ inputSchema: z.object({}), outputSchema: z.object({ - content: z.string() + content: z.string(), + title: z.string() }), handler: async () => ({ - content: 'https://example.com/a.png' + content: 'ordinary document text', + title: 'Result' }) }) }); @@ -450,8 +452,11 @@ describe('ToolFactory response content parts', () => { expect(messages).toEqual([ { - type: 'error', - data: expect.stringContaining('content 字段必须是') + type: 'response', + data: { + content: 'ordinary document text', + title: 'Result' + } } ]); }); @@ -572,16 +577,16 @@ describe('ToolFactory response content parts', () => { ]); }); - it('sends a readable error for invalid content part details', async () => { + it('passes malformed content parts through without wire validation', async () => { const runtime = createLocalDebugRuntime(); setCurrentLocalDebugRuntime(runtime); process.env.RUNTIME_MODE = 'dev'; defineTool({ manifest: { - pluginId: 'invalid-detail-test', - name: { en: 'Invalid Detail Test', 'zh-CN': '非法详情测试' }, - description: { en: 'Invalid Detail Test', 'zh-CN': '非法详情测试' }, + pluginId: 'malformed-parts-test', + name: { en: 'Malformed Parts Test', 'zh-CN': '非法部分透传测试' }, + description: { en: 'Malformed Parts Test', 'zh-CN': '非法部分透传测试' }, version: '0.0.1', versionDescription: { en: 'Test version', 'zh-CN': '测试版本' }, tags: ['tools'] @@ -591,13 +596,78 @@ describe('ToolFactory response content parts', () => { outputSchema: z.object({ content: z.array(ToolMessageContentPartSchema) }), - handler: async () => ({ + handler: async () => + ({ + content: [ + { + type: 'image_url', + image_url: { url: 'https://example.com/a.png', detail: 'ultra' } + } + ] + }) as unknown as { + content: ToolMessageContentPartType[]; + } + }) + }); + + await runtime.waitUntilReady(); + + const response = await runtime.invokePlugin< + { + input: Record; + systemVar: Record; + }, + void, + never, + ToolStreamMessageType + >('run', { + input: {}, + systemVar: {} + }); + + const messages: ToolStreamMessageType[] = []; + await response.output!.stream.consume((chunk) => { + messages.push(chunk); + }); + + expect(messages).toEqual([ + { + type: 'response', + data: { content: [ { type: 'image_url', image_url: { url: 'https://example.com/a.png', detail: 'ultra' } - } as unknown as ToolMessageContentPartType + } ] + } + } + ]); + }); + + it('passes legacy object and array content through the response data', async () => { + const runtime = createLocalDebugRuntime(); + setCurrentLocalDebugRuntime(runtime); + process.env.RUNTIME_MODE = 'dev'; + + defineTool({ + manifest: { + pluginId: 'legacy-structured-content-test', + name: { en: 'Legacy Structured Content Test', 'zh-CN': '旧结构化内容测试' }, + description: { en: 'Legacy Structured Content Test', 'zh-CN': '旧结构化内容测试' }, + version: '0.0.1', + versionDescription: { en: 'Test version', 'zh-CN': '测试版本' }, + tags: ['tools'] + }, + handler: createToolHandler({ + inputSchema: z.object({}), + outputSchema: z.object({ + content: z.unknown(), + items: z.unknown() + }), + handler: async () => ({ + content: { text: 'business payload' }, + items: [1, 'two', { three: true }] }) }) }); @@ -624,8 +694,11 @@ describe('ToolFactory response content parts', () => { expect(messages).toEqual([ { - type: 'error', - data: expect.stringContaining('content.0.image_url.detail') + type: 'response', + data: { + content: { text: 'business payload' }, + items: [1, 'two', { three: true }] + } } ]); }); diff --git a/sdk/factory/src/tool-factory.ts b/sdk/factory/src/tool-factory.ts index f5ffc45b..3eadde88 100644 --- a/sdk/factory/src/tool-factory.ts +++ b/sdk/factory/src/tool-factory.ts @@ -3,11 +3,7 @@ import z from 'zod'; import type { InvokePort } from '@domain/ports/invoke.port'; import { StreamData } from '@domain/value-objects/stream.vo'; import type { SystemVarType } from '@domain/value-objects/system-var.vo'; -import { - type ToolAnswerType, - ToolHandlerReturnSchema, - type ToolStreamMessageType -} from '@domain/value-objects/tool.vo'; +import type { ToolAnswerType, ToolStreamMessageType } from '@domain/value-objects/tool.vo'; import { PluginChannelHostMethod } from '@infrastructure/plugin/plugin-runtime/ports/channel'; import type { PluginToolRunPayloadType } from '@infrastructure/plugin/tool.impl'; import { getErrText } from '@shared/utils/err'; @@ -16,23 +12,6 @@ import { InvokeClient } from './invoke.client'; import type { UserToolManifestType } from './manifest.type'; import { PluginFactory } from './plugin-factory'; -/** - * 把 zod 校验错误整理成人类可读文本,避免默认的 JSON issues 大段回显。 - * 当错误发生在 content 字段时附上结构化内容契约提示,方便插件作者定位。 - */ -const formatZodError = (error: z.ZodError): string => { - const details = error.issues - .map((issue) => { - const path = issue.path.length > 0 ? `${issue.path.join('.')}: ` : ''; - return `${path}${issue.message}`; - }) - .join('; '); - const contentHint = error.issues.some((issue) => issue.path[0] === 'content') - ? '工具返回的 content 字段必须是结构化内容数组({type:"text",text} 或 {type:"image_url",image_url:{url}}),不能是字符串或其他结构。' - : ''; - - return contentHint ? `${contentHint} ${details}` : details; -}; export type ToolContextType> = { systemVar: SystemVarType; secrets: TSecret; @@ -133,17 +112,15 @@ export class ToolFactory extends PluginFactory { } }); - // 校验 handler 返回值:content 字段(若存在)必须是合法的 content parts,否则视为工具错误 + // handler 返回值原样透传:content 等业务字段保持旧插件的任意 JSON 语义, + // 结构化 content parts 由 outputSchema 声明约定,不在 wire 层强校验。 output.send({ type: 'response', - data: ToolHandlerReturnSchema.parse(result) + data: result }); } catch (err) { output.send({ - data: getErrText( - err instanceof z.ZodError ? formatZodError(err) : err, - 'Unknown error during tool execution' - ), + data: getErrText(err, 'Unknown error during tool execution'), type: 'error' }); } finally { From 7ce6c925c1886b5dcb252a72394ecb367c16874d Mon Sep 17 00:00:00 2001 From: Renjie Li Date: Tue, 13 Oct 2026 11:25:41 +0800 Subject: [PATCH 3/7] test(plugin): cover legacy tool response content on the client stream path --- .../domain/src/value-objects/tool.vo.test.ts | 32 ++++ sdk/client/src/tool-stream.test.ts | 166 ++++++++++++++++++ 2 files changed, 198 insertions(+) create mode 100644 sdk/client/src/tool-stream.test.ts diff --git a/packages/domain/src/value-objects/tool.vo.test.ts b/packages/domain/src/value-objects/tool.vo.test.ts index f2e9a5e4..7da2a636 100644 --- a/packages/domain/src/value-objects/tool.vo.test.ts +++ b/packages/domain/src/value-objects/tool.vo.test.ts @@ -133,4 +133,36 @@ describe('ToolStreamMessageSchema', () => { expect(ToolStreamMessageSchema.parse(message)).toEqual(message); }); + + it('parses a legacy response whose content is an object', () => { + const message = { + type: 'response', + data: { content: { summary: 'done', items: 2 } } + }; + + expect(ToolStreamMessageSchema.parse(message)).toEqual(message); + }); + + it('parses a legacy response whose content is an array', () => { + const message = { + type: 'response', + data: { content: ['a.png', 'b.png'] } + }; + + expect(ToolStreamMessageSchema.parse(message)).toEqual(message); + }); + + it('parses stream and error messages', () => { + const streamMessage = { + type: 'stream', + data: { type: 'answer', content: 'hello' } + }; + const errorMessage = { + type: 'error', + data: 'something went wrong' + }; + + expect(ToolStreamMessageSchema.parse(streamMessage)).toEqual(streamMessage); + expect(ToolStreamMessageSchema.parse(errorMessage)).toEqual(errorMessage); + }); }); diff --git a/sdk/client/src/tool-stream.test.ts b/sdk/client/src/tool-stream.test.ts new file mode 100644 index 00000000..d36e5400 --- /dev/null +++ b/sdk/client/src/tool-stream.test.ts @@ -0,0 +1,166 @@ +import { describe, expect, it, vi } from 'vitest'; + +import { FastGPTPluginClient } from './client'; + +type FetchMock = ReturnType; + +const createSseResponse = (chunks: string[]) => { + const encoder = new TextEncoder(); + let index = 0; + const stream = new ReadableStream({ + async pull(controller) { + if (index >= chunks.length) { + controller.close(); + return; + } + + const chunk = chunks[index]; + index += 1; + controller.enqueue(encoder.encode(chunk)); + } + }); + + return new Response(stream, { + status: 200, + headers: { 'Content-Type': 'text/event-stream' } + }); +}; + +const createClient = (fetchMock: FetchMock) => + new FastGPTPluginClient({ + baseUrl: 'https://plugin.local', + token: 'token', + fetch: fetchMock as unknown as typeof globalThis.fetch + }); + +const runParams = (overrides: Record = {}) => ({ + input: {}, + pluginId: 'getTime', + systemVar: { + time: '2024-01-01T00:00:00.000Z', + app: { id: 'test-app', name: 'Test App' }, + chat: { chatId: 'test-chat', uid: 'test-user' }, + invokeToken: '' + }, + ...overrides +}); + +const responseEvent = (data: unknown) => + `data: ${JSON.stringify({ type: 'response', data })}\n\n`; + +describe('FastGPTPluginClient tool stream', () => { + it('passes legacy string content through the stream response', async () => { + const fetchMock = vi.fn().mockResolvedValue( + createSseResponse([responseEvent({ content: 'ordinary text', title: 'Result' })]) + ); + const client = createClient(fetchMock); + + const result = await client.runToolStream(runParams()); + + expect(result.error).toBeUndefined(); + expect(result.output).toEqual({ content: 'ordinary text', title: 'Result' }); + }); + + it('passes legacy object content through the stream response', async () => { + const data = { content: { summary: 'done', items: 2 }, title: 'Result' }; + const fetchMock = vi.fn().mockResolvedValue(createSseResponse([responseEvent(data)])); + const client = createClient(fetchMock); + + const result = await client.runToolStream(runParams()); + + expect(result.error).toBeUndefined(); + expect(result.output).toEqual(data); + }); + + it('passes legacy array content through the stream response', async () => { + const data = { content: ['a.png', 'b.png'], title: 'Result' }; + const fetchMock = vi.fn().mockResolvedValue(createSseResponse([responseEvent(data)])); + const client = createClient(fetchMock); + + const result = await client.runToolStream(runParams()); + + expect(result.error).toBeUndefined(); + expect(result.output).toEqual(data); + }); + + it('passes structured content parts through the stream response', async () => { + const data = { + content: [ + { type: 'text', text: 'Look at this image' }, + { type: 'image_url', image_url: { url: 'https://example.com/a.png' } } + ] + }; + const fetchMock = vi.fn().mockResolvedValue(createSseResponse([responseEvent(data)])); + const client = createClient(fetchMock); + + const result = await client.runToolStream(runParams()); + + expect(result.error).toBeUndefined(); + expect(result.output).toEqual(data); + }); + + it('reassembles response events split across chunks', async () => { + const payload = JSON.stringify({ type: 'response', data: { content: 'split text' } }); + const fetchMock = vi.fn().mockResolvedValue( + createSseResponse([ + `data: ${payload.slice(0, 18)}`, + `${payload.slice(18)}\n\n` + ]) + ); + const client = createClient(fetchMock); + + const result = await client.runToolStream(runParams()); + + expect(result.error).toBeUndefined(); + expect(result.output).toEqual({ content: 'split text' }); + }); + + it('emits stream messages through onMessage', async () => { + const fetchMock = vi.fn().mockResolvedValue( + createSseResponse([ + 'data: {"type":"stream","data":{"type":"answer","content":"hello"}}\n\n', + 'data: {"type":"stream","data":{"type":"answer","content":" world"}}\n\n', + responseEvent({ content: 'final text' }) + ]) + ); + const client = createClient(fetchMock); + const onMessage = vi.fn(); + + const result = await client.runToolStream(runParams({ onMessage })); + + expect(result.error).toBeUndefined(); + expect(result.output).toEqual({ content: 'final text' }); + expect(onMessage).toHaveBeenCalledTimes(2); + expect(onMessage).toHaveBeenNthCalledWith(1, { type: 'answer', content: 'hello' }); + expect(onMessage).toHaveBeenNthCalledWith(2, { type: 'answer', content: ' world' }); + }); + + it('skips malformed frames and keeps reading the stream', async () => { + const fetchMock = vi.fn().mockResolvedValue( + createSseResponse(['data: not-json\n\n', responseEvent({ content: 'still works' })]) + ); + const client = createClient(fetchMock); + + const result = await client.runToolStream(runParams()); + + expect(result.error).toBeUndefined(); + expect(result.output).toEqual({ content: 'still works' }); + }); + + it('returns an error when the stream reports one', async () => { + const fetchMock = vi.fn().mockResolvedValue( + createSseResponse(['data: {"type":"error","data":"something went wrong"}\n\n']) + ); + const client = createClient(fetchMock); + + // runToolStream 的公开声明类型是 ToolHandlerReturnType, + // 运行时实际返回 { output?, error? },这里按运行形状读取。 + const result = (await client.runToolStream(runParams())) as unknown as { + output?: Record; + error?: Error; + }; + + expect(result.output).toBeUndefined(); + expect(result.error?.message).toBe('something went wrong'); + }); +}); From 1379863eca2e6bf941885ccaed95ccd1f1d34aa0 Mon Sep 17 00:00:00 2001 From: Renjie Li Date: Tue, 13 Oct 2026 11:51:21 +0800 Subject: [PATCH 4/7] test(plugin): cover frames that fail schema parsing on the tool stream --- sdk/client/src/tool-stream.test.ts | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/sdk/client/src/tool-stream.test.ts b/sdk/client/src/tool-stream.test.ts index d36e5400..a9c8fe6f 100644 --- a/sdk/client/src/tool-stream.test.ts +++ b/sdk/client/src/tool-stream.test.ts @@ -147,6 +147,21 @@ describe('FastGPTPluginClient tool stream', () => { expect(result.output).toEqual({ content: 'still works' }); }); + it('skips frames that fail schema parsing and keeps reading the stream', async () => { + const fetchMock = vi.fn().mockResolvedValue( + createSseResponse([ + 'data: {"type":"nope","data":{}}\n\n', + responseEvent({ content: 'still works' }) + ]) + ); + const client = createClient(fetchMock); + + const result = await client.runToolStream(runParams()); + + expect(result.error).toBeUndefined(); + expect(result.output).toEqual({ content: 'still works' }); + }); + it('returns an error when the stream reports one', async () => { const fetchMock = vi.fn().mockResolvedValue( createSseResponse(['data: {"type":"error","data":"something went wrong"}\n\n']) From 108dae3e40628cedba33b6f810f221e930a7a702 Mon Sep 17 00:00:00 2001 From: lirenjie Date: Thu, 29 Oct 2026 13:35:30 +0800 Subject: [PATCH 5/7] docs(plugin): clarify how FastGPT consumes tool response content parts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 说明发给模型的 tool 文本始终是 handler 返回值的完整序列化结果， content 里的 text part 不替换它，业务字段应放在返回对象顶层 - 说明主程序按整组校验 content：detail 只能是 auto/low/high， 任一项不合法会让整组图片失效并回退文本 --- packages/domain/src/value-objects/tool.vo.ts | 5 +++++ .../skills/fastgpt-system-tool-development/SKILL.md | 10 +++++++--- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/packages/domain/src/value-objects/tool.vo.ts b/packages/domain/src/value-objects/tool.vo.ts index b8f647d8..ce40197a 100644 --- a/packages/domain/src/value-objects/tool.vo.ts +++ b/packages/domain/src/value-objects/tool.vo.ts @@ -6,6 +6,11 @@ import { SystemVarSchema } from './system-var.vo'; /** * 工具 handler 返回的结构化内容部分,与 FastGPT 主程序对齐: * text 为纯文本,image_url 为图片;主程序会把合法的 content parts 作为下一轮模型的视觉输入。 + * + * 注意:发给模型的 tool 文本始终是 handler 返回值的完整序列化结果, + * content 里的 text part 不会替换这段文本(主程序只取其中的图片 part)。 + * 需要模型看到业务字段时,把字段放在返回对象顶层;content 主要用于携带图片。 + * * 该 schema 供插件作者在 outputSchema 中声明使用,不参与 wire 层强校验。 */ export const ToolMessageContentPartSchema = z.discriminatedUnion('type', [ diff --git a/sdk/factory/skills/fastgpt-system-tool-development/SKILL.md b/sdk/factory/skills/fastgpt-system-tool-development/SKILL.md index 96187d4d..1098bc0e 100644 --- a/sdk/factory/skills/fastgpt-system-tool-development/SKILL.md +++ b/sdk/factory/skills/fastgpt-system-tool-development/SKILL.md @@ -261,14 +261,17 @@ export default defineToolSet({ ## 返回图片内容（可选） -工具 handler 的返回对象里可以包含 `content` 字段，向 FastGPT 主程序返回结构化图片内容，作为下一轮模型的视觉输入。此时 `content` 应为内容部分数组，每项只能是以下两种之一： +工具 handler 的返回对象里可以包含 `content` 字段，向 FastGPT 主程序返回结构化图片内容，作为下一轮模型的视觉输入。此时 `content` 应为非空内容部分数组，每项只能是以下两种之一： - `{ type: 'text', text: string }`：文本内容。 - `{ type: 'image_url', image_url: { url: string, detail?: 'auto' | 'low' | 'high' } }`：图片内容，`url` 为图片地址。 -> **向后兼容（重要）**：`content` 是普通业务字段，SDK 的 wire 层保持任意 JSON 透传，不做强校验。旧插件把字符串、对象或普通数组放在 `content` 里仍可正常工作，主程序会把这些值当作普通文本结果处理。 +> **主程序如何使用 `content`（重要）**：发给模型的 tool 文本始终是 handler 返回值的完整序列化结果，`content` 里的 `text` part **不会**替换这段文本（FastGPT 只取其中的图片 part）。所以业务字段直接放在返回对象顶层即可，不要指望靠 `text` part 给模型讲一句话；`content` 的用途就是携带图片。 > -> 只有当你希望图片作为下一轮模型的视觉输入时，才按上述 parts 形状返回 `content`。主程序仅在 `content` 形状合法时把它作为视觉输入，否则回退到普通文本路径；因此不要只放一个孤立的图片链接字符串，也不要在普通文本里夹带图片地址。 +> **向后兼容**：`content` 是普通业务字段，SDK 的 wire 层保持任意 JSON 透传，不做强校验。旧插件把字符串、对象或普通数组放在 `content` 里仍可正常工作，主程序会把这些值当作普通文本结果处理。 +> +> 只有当你希望图片作为下一轮模型的视觉输入时，才按上述 parts 形状返回 `content`。主程序仅在 `content` 是非空数组且每项都合法时把它作为视觉输入（`detail` 只能是 `auto` / `low` / `high`），否则整组回退到普通文本路径；因此不要只放一个孤立的图片链接字符串，也不要在普通文本里夹带图片地址，更不要混入无法解析的 part（例如自定义对象）——那会让整组图片一起失效。 + 示例： @@ -324,6 +327,7 @@ export default defineTool({ - handler 成功路径返回值与 `outputSchema` 一致。 - 返回结构化图片内容时 `outputSchema` 里的 `content` 声明为 `z.array(ToolMessageContentPartSchema)`；旧插件把 `content` 当普通业务字段（字符串/对象/数组）使用仍兼容，无需迁移。 - 返回图片内容时使用结构化 `content` 字段（`ToolMessageContentPartSchema`），不在文本中夹带图片地址。 +- `content` 数组里不要混入非 text / image_url 的项：主程序按整组校验，任一项不合法会让这组图片一起失效，只回退成文本。 - 外部调用失败、空响应、超时、鉴权失败都有明确错误。 - 密钥配置只通过 `secretSchema` 和 `ctx.secrets` 处理。 - 系统工具根目录存在 `logo.`。 From 6a10d4574dd415bf4fa5a1c9075d16a405f6cf65 Mon Sep 17 00:00:00 2001 From: lirenjie Date: Mon, 2 Nov 2026 15:04:02 +0800 Subject: [PATCH 6/7] docs(plugin): document image count, format and masking rules - State that only the first 4 images per tool result become vision input - Recommend raster formats so providers do not reject the whole round - Explain that inline data URI payloads are masked in the model-facing text --- packages/domain/src/value-objects/tool.vo.ts | 4 ++++ sdk/factory/skills/fastgpt-system-tool-development/SKILL.md | 5 +++++ 2 files changed, 9 insertions(+) diff --git a/packages/domain/src/value-objects/tool.vo.ts b/packages/domain/src/value-objects/tool.vo.ts index ce40197a..799f22f0 100644 --- a/packages/domain/src/value-objects/tool.vo.ts +++ b/packages/domain/src/value-objects/tool.vo.ts @@ -11,6 +11,10 @@ import { SystemVarSchema } from './system-var.vo'; * content 里的 text part 不会替换这段文本(主程序只取其中的图片 part)。 * 需要模型看到业务字段时,把字段放在返回对象顶层;content 主要用于携带图片。 * + * 主程序消费规则:单次结果最多取前 4 张图片作为视觉输入,超出部分仅保留在文本里; + * 建议只使用 png/jpg/jpeg/gif/webp/bmp 等主流栅格格式,其它格式供应商通常会拒绝; + * `data:` 内联图片在把文本发给模型时会被主程序替换成占位,图片本身仍照常发送。 + * * 该 schema 供插件作者在 outputSchema 中声明使用,不参与 wire 层强校验。 */ export const ToolMessageContentPartSchema = z.discriminatedUnion('type', [ diff --git a/sdk/factory/skills/fastgpt-system-tool-development/SKILL.md b/sdk/factory/skills/fastgpt-system-tool-development/SKILL.md index 1098bc0e..c9ecbb99 100644 --- a/sdk/factory/skills/fastgpt-system-tool-development/SKILL.md +++ b/sdk/factory/skills/fastgpt-system-tool-development/SKILL.md @@ -268,6 +268,10 @@ export default defineToolSet({ > **主程序如何使用 `content`（重要）**：发给模型的 tool 文本始终是 handler 返回值的完整序列化结果，`content` 里的 `text` part **不会**替换这段文本（FastGPT 只取其中的图片 part）。所以业务字段直接放在返回对象顶层即可，不要指望靠 `text` part 给模型讲一句话；`content` 的用途就是携带图片。 > +> **图片数量与格式**：单次工具结果只有前 4 张图片会作为视觉输入，超出的部分仍保留在文本里；建议使用 `png` / `jpg` / `jpeg` / `gif` / `webp` / `bmp` 等主流栅格格式，`svg`、`psd` 等格式供应商通常会拒绝，反而让整轮请求失败。 +> +> **内联 data URI**：如果 `image_url.url` 是 `data:;base64,...`，主程序在把文本发给模型时会把这段 base64 换成占位（图片本身仍通过视觉通道发送），因此不必担心同一张图在上下文里出现两份。 +> > **向后兼容**：`content` 是普通业务字段，SDK 的 wire 层保持任意 JSON 透传，不做强校验。旧插件把字符串、对象或普通数组放在 `content` 里仍可正常工作，主程序会把这些值当作普通文本结果处理。 > > 只有当你希望图片作为下一轮模型的视觉输入时，才按上述 parts 形状返回 `content`。主程序仅在 `content` 是非空数组且每项都合法时把它作为视觉输入（`detail` 只能是 `auto` / `low` / `high`），否则整组回退到普通文本路径；因此不要只放一个孤立的图片链接字符串，也不要在普通文本里夹带图片地址，更不要混入无法解析的 part（例如自定义对象）——那会让整组图片一起失效。 @@ -328,6 +332,7 @@ export default defineTool({ - 返回结构化图片内容时 `outputSchema` 里的 `content` 声明为 `z.array(ToolMessageContentPartSchema)`；旧插件把 `content` 当普通业务字段（字符串/对象/数组）使用仍兼容，无需迁移。 - 返回图片内容时使用结构化 `content` 字段（`ToolMessageContentPartSchema`），不在文本中夹带图片地址。 - `content` 数组里不要混入非 text / image_url 的项：主程序按整组校验，任一项不合法会让这组图片一起失效，只回退成文本。 +- `content` 里的图片不超过 4 张（超出部分只留在文本里），并使用主流栅格格式（png/jpg/jpeg/gif/webp/bmp）；svg / psd 等格式会让供应商拒绝整轮请求。 - 外部调用失败、空响应、超时、鉴权失败都有明确错误。 - 密钥配置只通过 `secretSchema` 和 `ctx.secrets` 处理。 - 系统工具根目录存在 `logo.`。 From 505b406f264528ce41f951e4a7000ddd445841b2 Mon Sep 17 00:00:00 2001 From: lirenjie Date: Mon, 2 Nov 2026 16:12:15 +0800 Subject: [PATCH 7/7] docs(plugin): restrict supported image formats to png/jpeg/gif/webp --- packages/domain/src/value-objects/tool.vo.ts | 3 ++- sdk/factory/skills/fastgpt-system-tool-development/SKILL.md | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/packages/domain/src/value-objects/tool.vo.ts b/packages/domain/src/value-objects/tool.vo.ts index 799f22f0..4a5bbecb 100644 --- a/packages/domain/src/value-objects/tool.vo.ts +++ b/packages/domain/src/value-objects/tool.vo.ts @@ -12,7 +12,8 @@ import { SystemVarSchema } from './system-var.vo'; * 需要模型看到业务字段时,把字段放在返回对象顶层;content 主要用于携带图片。 * * 主程序消费规则:单次结果最多取前 4 张图片作为视觉输入,超出部分仅保留在文本里; - * 建议只使用 png/jpg/jpeg/gif/webp/bmp 等主流栅格格式,其它格式供应商通常会拒绝; + * 只支持 png/jpg/jpeg/gif/webp(OpenAI 与 Claude 的图片契约),svg/psd/bmp 等格式 + * 会被供应商拒绝,需要插件自己先转码; * `data:` 内联图片在把文本发给模型时会被主程序替换成占位,图片本身仍照常发送。 * * 该 schema 供插件作者在 outputSchema 中声明使用,不参与 wire 层强校验。 diff --git a/sdk/factory/skills/fastgpt-system-tool-development/SKILL.md b/sdk/factory/skills/fastgpt-system-tool-development/SKILL.md index c9ecbb99..70d5caee 100644 --- a/sdk/factory/skills/fastgpt-system-tool-development/SKILL.md +++ b/sdk/factory/skills/fastgpt-system-tool-development/SKILL.md @@ -268,7 +268,7 @@ export default defineToolSet({ > **主程序如何使用 `content`（重要）**：发给模型的 tool 文本始终是 handler 返回值的完整序列化结果，`content` 里的 `text` part **不会**替换这段文本（FastGPT 只取其中的图片 part）。所以业务字段直接放在返回对象顶层即可，不要指望靠 `text` part 给模型讲一句话；`content` 的用途就是携带图片。 > -> **图片数量与格式**：单次工具结果只有前 4 张图片会作为视觉输入，超出的部分仍保留在文本里；建议使用 `png` / `jpg` / `jpeg` / `gif` / `webp` / `bmp` 等主流栅格格式，`svg`、`psd` 等格式供应商通常会拒绝，反而让整轮请求失败。 +> **图片数量与格式**：单次工具结果只有前 4 张图片会作为视觉输入，超出的部分仍保留在文本里；只支持 `png` / `jpg` / `jpeg` / `gif` / `webp`（OpenAI 与 Claude 的图片契约），`svg`、`psd`、`bmp` 等格式会被供应商拒绝，反而让整轮请求失败，请先自行转成上面的格式。 > > **内联 data URI**：如果 `image_url.url` 是 `data:;base64,...`，主程序在把文本发给模型时会把这段 base64 换成占位（图片本身仍通过视觉通道发送），因此不必担心同一张图在上下文里出现两份。 > @@ -332,7 +332,7 @@ export default defineTool({ - 返回结构化图片内容时 `outputSchema` 里的 `content` 声明为 `z.array(ToolMessageContentPartSchema)`；旧插件把 `content` 当普通业务字段（字符串/对象/数组）使用仍兼容，无需迁移。 - 返回图片内容时使用结构化 `content` 字段（`ToolMessageContentPartSchema`），不在文本中夹带图片地址。 - `content` 数组里不要混入非 text / image_url 的项：主程序按整组校验，任一项不合法会让这组图片一起失效，只回退成文本。 -- `content` 里的图片不超过 4 张（超出部分只留在文本里），并使用主流栅格格式（png/jpg/jpeg/gif/webp/bmp）；svg / psd 等格式会让供应商拒绝整轮请求。 +- `content` 里的图片不超过 4 张（超出部分只留在文本里），且只支持 png/jpg/jpeg/gif/webp；svg / psd / bmp 等格式会让供应商拒绝整轮请求。 - 外部调用失败、空响应、超时、鉴权失败都有明确错误。 - 密钥配置只通过 `secretSchema` 和 `ctx.secrets` 处理。 - 系统工具根目录存在 `logo.`。