diff --git a/packages/codingcode/src/agent/agent.ts b/packages/codingcode/src/agent/agent.ts index a9cfc25..4c1c700 100644 --- a/packages/codingcode/src/agent/agent.ts +++ b/packages/codingcode/src/agent/agent.ts @@ -29,7 +29,10 @@ import { isTurnEnd } from '../sink/types.js'; import type { SessionRef } from '../session/types.js'; import type { PermissionMode } from '../util/enums.js'; import type { ToolCatalog, ToolResult } from '../tools/types.js'; +import { mediaKindOf, textOf, textPart, type IncomingPart } from '../llm/types.js'; import type { ToolCall } from '../llm/types.js'; +import { capabilitiesOf } from '../infra/models.js'; +import { sniffMediaMime } from '../util/media.js'; import { loadConfig } from '../infra/config.js'; import { createLogger } from '../infra/logger.js'; import { normalizePath } from '../util/path.js'; @@ -42,6 +45,35 @@ function toolOutcomeOf(result: ToolResult): ToolOutcome { : { status: result.status, output: result.output }; } +function assertMediaAllowed( + input: readonly IncomingPart[], + model: string +): Effect.Effect { + return Effect.gen(function* () { + let needsVision = false; + let needsAudio = false; + for (const p of input) { + if (p.type !== 'media') continue; + const mimeType = sniffMediaMime(p.bytes) ?? p.declaredMimeType ?? ''; + if (mediaKindOf(mimeType) === 'audio') needsAudio = true; + else needsVision = true; + } + if (!needsVision && !needsAudio) return; + + // capabilitiesOf 是同步查询,抛出的 AgentError 原样进错误通道 + const caps = yield* Effect.try({ + try: () => capabilitiesOf(model), + catch: (e) => (e instanceof AgentError ? e : AgentError.invalidInput(String(e))), + }); + if (needsVision && !caps.vision) { + return yield* Effect.fail(AgentError.invalidInput('model does not accept image or PDF input')); + } + if (needsAudio && !caps.audio) { + return yield* Effect.fail(AgentError.invalidInput('model does not accept audio input')); + } + }); +} + const logger = createLogger(); function toFrameError(e: AgentError): FrameError { @@ -81,7 +113,7 @@ export const AgentLayer = Layer.effect( ) ); - const runTurn = (input: string, opts: RunTurnOptions) => + const runTurn = (input: IncomingPart[], opts: RunTurnOptions) => Effect.gen(function* () { const normalizedCwd = normalizePath(opts.cwd); @@ -106,7 +138,7 @@ export const AgentLayer = Layer.effect( normalizedCwd, { model, - title: input, + title: textOf(input) || 'New session', activeProfile: opts.activeProfile, permissionMode: opts.permissionMode, }, @@ -148,7 +180,9 @@ export const AgentLayer = Layer.effect( const toolEnv = yield* toolEnvPort.getToolEnv(); - const turnId = (yield* session.recordUser(state, input)).turnId; + yield* assertMediaAllowed(input, model); + const parts = yield* session.materializeInput(state, input); + const turnId = (yield* session.recordUser(state, parts)).turnId; // 用户显式 @ 的 skill:按 path 回查权威数据,正文拼块后作为同回合的第二条 user 事件 if (opts.skills?.length) { @@ -158,7 +192,7 @@ export const AgentLayer = Layer.effect( const entries = yield* Effect.forEach(chosen, (s) => skills.readContent(s.skillPath).pipe(Effect.map((body) => ({ skill: s, body }))) ); - yield* session.recordSystem(state, renderSkillBlock(entries)); + yield* session.recordSystem(state, [textPart(renderSkillBlock(entries))]); } } @@ -347,7 +381,12 @@ export const AgentLayer = Layer.effect( Effect.tryPromise({ try: async () => { for await (const part of llm.completeStream( - { messages: llmMessages, system, tools, maxSteps: 1 }, + { + messages: llmMessages, + system, + tools, + maxSteps: 1, + }, model, abortSignal )) { @@ -436,7 +475,7 @@ export const AgentLayer = Layer.effect( } stopContinuations++; const injection = stopDecision.injection ?? '(continue)'; - const systemEv = yield* session.recordSystem(state, injection); + const systemEv = yield* session.recordSystem(state, [textPart(injection)]); yield* context.absorb(sessionRef, [systemEv]); continue; } diff --git a/packages/codingcode/src/agent/port.ts b/packages/codingcode/src/agent/port.ts index e12247f..924a36c 100644 --- a/packages/codingcode/src/agent/port.ts +++ b/packages/codingcode/src/agent/port.ts @@ -2,6 +2,7 @@ import { Context } from 'effect'; import type { Effect } from 'effect'; import type { FrameBody } from '../sink/types.js'; import type { ProfileName, PermissionMode } from '../util/enums.js'; +import type { IncomingPart } from '../llm/types.js'; import type { AgentError } from '../util/error.js'; @@ -20,7 +21,7 @@ export interface RunTurnOptions { export interface AgentShape { runTurn( - input: string, + input: IncomingPart[], opts: RunTurnOptions ): Effect.Effect< { diff --git a/packages/codingcode/src/context/context.ts b/packages/codingcode/src/context/context.ts index f8d1948..02b9b97 100644 --- a/packages/codingcode/src/context/context.ts +++ b/packages/codingcode/src/context/context.ts @@ -2,7 +2,8 @@ import { Layer, Effect } from 'effect'; import { randomUUID } from 'crypto'; import { loadConfig } from '../infra/config.js'; import { transcriptPathOf } from '../session/paths.js'; -import type { Message } from '../llm/types.js'; +import type { Message, ResolvedMessage, ResolvedContentPart, ResolvedMediaPart } from '../llm/types.js'; +import { textOf, textPart } from '../llm/types.js'; import type { SessionEvent, AssistantEvent, @@ -111,7 +112,7 @@ export function buildContextMessages( break; case 'assistant': { const ev = event as AssistantEvent; - const msg: Message = { role: 'assistant', content: event.content }; + const msg: Message = { role: 'assistant', content: [textPart(event.content)] }; if (event.toolCalls && event.toolCalls.length > 0) { msg.tool_calls = event.toolCalls.map((tc) => ({ id: tc.id, @@ -135,17 +136,21 @@ export function buildContextMessages( resolvedIds.add(event.toolCallId); messages.push({ role: 'tool', - content: output, + content: [textPart(output)], tool_call_id: event.toolCallId, tool_name: event.toolName, }); break; } case 'summary': - messages.push({ role: 'system', name: 'compacted_history', content: event.summaryText }); + messages.push({ + role: 'system', + name: 'compacted_history', + content: [textPart(event.summaryText)], + }); break; case 'subagent_result': - messages.push({ role: 'user', content: event.content }); + messages.push({ role: 'user', content: [textPart(event.content)] }); break; } } @@ -180,7 +185,8 @@ export function buildContextMessages( if (curr.role === prev.role && curr.role !== 'system') { if (curr.role === 'tool') continue; if (curr.role === 'assistant' && curr.tool_calls && curr.tool_calls.length > 0) continue; - prev.content += '\n\n' + curr.content; + // parts 数组本身就是分隔符,providers 侧按 part 逐个送出 + prev.content = [...prev.content, ...curr.content]; filtered.splice(i, 1); } } @@ -322,7 +328,11 @@ export const ContextLayer = Layer.effect( ); buf.events.push(summaryEvent); - const summaryMsg: Message = { role: 'system', name: 'compacted_history', content: summary }; + const summaryMsg: Message = { + role: 'system', + name: 'compacted_history', + content: [textPart(summary)], + }; return Math.max(0, totalTokens - estimateMessageTokens(summaryMsg)); }); @@ -361,15 +371,20 @@ export const ContextLayer = Layer.effect( const transcriptText = transcript .map( (m) => - `[${m.role}${(m as any).tool_name ? ':' + (m as any).tool_name : ''}]\n${m.content}` + `[${m.role}${(m as any).tool_name ? ':' + (m as any).tool_name : ''}]\n${textOf(m.content)}` ) .join('\n\n'); const system = COMPACTION_SYSTEM_PROMPT; - const userMsg: Message = { + // 压缩走纯文本投影 + const userMsg: ResolvedMessage = { role: 'user', - content: `Compact the following conversation transcript into the sections above:\n\n${transcriptText}`, + content: [ + textPart( + `Compact the following conversation transcript into the sections above:\n\n${transcriptText}` + ), + ], }; const result = yield* llm @@ -383,7 +398,32 @@ export const ContextLayer = Layer.effect( return raw.trim(); } - const getHistory = (ref: SessionRef, model: string): Effect.Effect => + const resolveMediaParts = ( + msgs: Message[], + cwd: string + ): Effect.Effect => + Effect.gen(function* () { + const assets = new Set(); + for (const m of msgs) { + for (const p of m.content) if (p.type === 'media') assets.add(p.asset); + } + if (assets.size === 0) return msgs as ResolvedMessage[]; + + const resolved = yield* session.resolveAssets(cwd, [...assets]); + return msgs.map((m) => ({ + ...m, + content: m.content.map((p): ResolvedContentPart => { + if (p.type === 'text') return p; + const dataUrl = resolved.get(p.asset); + if (!dataUrl) return { type: 'text', text: `[media missing: ${p.asset}]` }; + const part: ResolvedMediaPart = { type: 'media', dataUrl, mimeType: p.mimeType }; + if (p.filename) part.filename = p.filename; + return part; + }), + })); + }); + + const getHistory = (ref: SessionRef, model: string): Effect.Effect => Effect.gen(function* () { const buf = yield* ensureBuffer(ref); const contextWindow = contextWindowOf(model); @@ -397,7 +437,7 @@ export const ContextLayer = Layer.effect( transition: { to: 'executing' }, }); } - return buildContextMessages(buf.events, buf.compactedTurnIds); + return yield* resolveMediaParts(buildContextMessages(buf.events, buf.compactedTurnIds), ref.cwd); }); const absorb = (ref: SessionRef, events: readonly SessionEvent[]): Effect.Effect => diff --git a/packages/codingcode/src/context/port.ts b/packages/codingcode/src/context/port.ts index e472cd8..d6a2964 100644 --- a/packages/codingcode/src/context/port.ts +++ b/packages/codingcode/src/context/port.ts @@ -1,6 +1,6 @@ import { Context } from 'effect'; import type { Effect } from 'effect'; -import type { Message } from '../llm/types.js'; +import type { ResolvedMessage } from '../llm/types.js'; import type { SessionRef } from '../session/types.js'; import type { SessionEvent } from '../session/types.js'; @@ -13,8 +13,8 @@ export interface CompressResult { } export interface ContextShape { - /** 取该会话当前给模型的 history。回合内首次调用读一次盘,之后走内存;换回合自动重建 */ - getHistory(ref: SessionRef, model: string): Effect.Effect; + /** 取该会话当前给模型的完整载荷:回合内首次调用读一次盘,之后走内存;换回合自动重建。媒体字节在此装配完成 */ + getHistory(ref: SessionRef, model: string): Effect.Effect; /** 把本回合自己写进 transcript 的事件并入内存态(零 IO) */ absorb(ref: SessionRef, events: readonly SessionEvent[]): Effect.Effect; /** 手动压缩入口(HTTP /compact),不受阈值限制 */ diff --git a/packages/codingcode/src/context/tokens.ts b/packages/codingcode/src/context/tokens.ts index cb8373f..7969b88 100644 --- a/packages/codingcode/src/context/tokens.ts +++ b/packages/codingcode/src/context/tokens.ts @@ -1,7 +1,42 @@ -import type { Message } from '../llm/types.js'; +import { mediaKindOf, type MediaPart, type Message, type TextPart } from '../llm/types.js'; +import type { StoredMediaPart } from '../session/types.js'; + +/** + * 音频每秒 token 数:取各家公开值里的上界(Gemini 32 / 秒,Qwen-Audio 25 / 秒, + * OpenAI 未公布固定值),让压缩早触发。 + */ +const AUDIO_TOKENS_PER_SECOND = 32; +/** PDF 按体积折算:1MB ≈ 5 万 token。 */ +const PDF_TOKENS_PER_MB = 50_000; + +/** + * 可估算的内容块:瘦块(llm 层)与落盘富块(session 层)都收。 + * + * 元数据存在就精算,不存在就取保守回退——估算只做压缩决策,不追求精确。 + */ +export type EstimablePart = TextPart | MediaPart | StoredMediaPart; + +export function estimateTokensForPart(p: EstimablePart): number { + if (p.type === 'text') return estimateTokensForContent(p.text); + switch (mediaKindOf(p.mimeType)) { + case 'image': + // tile 计费近似:基准 85 + 每 512×512 tile 170 + return ( + 85 + + 170 * + Math.ceil((('width' in p ? p.width : undefined) ?? 512) / 512) * + Math.ceil((('height' in p ? p.height : undefined) ?? 512) / 512) + ); + case 'audio': + return Math.ceil((('durationSec' in p ? p.durationSec : undefined) ?? 1) * AUDIO_TOKENS_PER_SECOND); + case 'file': + return Math.ceil((('bytes' in p ? p.bytes : 0) / (1024 * 1024)) * PDF_TOKENS_PER_MB); + } +} export function estimateMessageTokens(m: Message): number { - let tokens = estimateTokensForContent(m.content ?? ''); + let tokens = 0; + for (const p of m.content) tokens += estimateTokensForPart(p); tokens += estimateTokensForContent(m.role); if (m.name) tokens += estimateTokensForContent(m.name); if (m.tool_call_id) tokens += estimateTokensForContent(m.tool_call_id); diff --git a/packages/codingcode/src/infra/models.ts b/packages/codingcode/src/infra/models.ts index a2337e5..2436aac 100644 --- a/packages/codingcode/src/infra/models.ts +++ b/packages/codingcode/src/infra/models.ts @@ -3,6 +3,18 @@ import { join } from 'path'; import { AgentError } from '../util/error.js'; import { loadConfig, updateActiveModel } from './config.js'; +/** config/models.json 里每个模型的 capabilities 声明。 */ +export interface ModelCapabilitiesEntry { + vision?: 'supported' | 'unsupported'; + audio?: 'supported' | 'unsupported'; +} + +/** 准入要看的能力位。 */ +export interface ModelCapabilities { + vision: boolean; + audio: boolean; +} + /** 模型清单里一条可被会话选中的模型。 */ export interface SelectableModel { id: string; @@ -13,12 +25,14 @@ export interface SelectableModel { base_url: string; api_key_env: string; context_window: number; + capabilities: ModelCapabilities; } export interface ModelDescriptor { id: string; name: string; context_window?: number; + capabilities?: ModelCapabilitiesEntry; } export interface ProviderEntry { @@ -70,6 +84,10 @@ export function flattenModels(cat: ProviderCatalog): SelectableModel[] { base_url: p.base_url, api_key_env: p.api_key_env, context_window: m.context_window ?? DEFAULT_CONTEXT_WINDOW, + capabilities: { + vision: m.capabilities?.vision === 'supported', + audio: m.capabilities?.audio === 'supported', + }, }); } } @@ -115,6 +133,19 @@ export function contextWindowOf(model: string): number { return entry?.context_window ?? DEFAULT_CONTEXT_WINDOW; } +/** 指定模型的能力位;模型不存在时报 CONFIG_INVALID。 */ +export function capabilitiesOf(model: string): ModelCapabilities { + const target = model?.trim(); + if (!target) { + const entry = activeModel(); + if (!entry) throw new AgentError('CONFIG_INVALID', activeModelError()); + return entry.capabilities; + } + const found = findModel(target); + if (!found) throw new AgentError('CONFIG_INVALID', `Model "${target}" not found in models.json`); + return found.capabilities; +} + export function setGlobalActive(model: string): void { const found = findModel(model.trim()); if (!found) { diff --git a/packages/codingcode/src/llm/providers/shared.ts b/packages/codingcode/src/llm/providers/shared.ts index 834ed88..550f776 100644 --- a/packages/codingcode/src/llm/providers/shared.ts +++ b/packages/codingcode/src/llm/providers/shared.ts @@ -1,12 +1,10 @@ import { jsonSchema, type LanguageModelUsage, type ModelMessage } from 'ai'; -import type { TokenUsage } from '../types.js'; +import { textOf, type ResolvedMessage, type TokenUsage } from '../types.js'; -export function convertMessages( - messages: Array<{ role: string; content: string; tool_calls?: unknown[]; tool_call_id?: string }> -): ModelMessage[] { +export function convertMessages(messages: ResolvedMessage[]): ModelMessage[] { return messages.map((m) => { if (m.role === 'assistant' && m.tool_calls && Array.isArray(m.tool_calls)) { - const content: any[] = [{ type: 'text', text: m.content }]; + const content: any[] = [{ type: 'text', text: textOf(m.content) }]; for (const tc of m.tool_calls) { content.push({ type: 'tool-call', @@ -25,12 +23,32 @@ export function convertMessages( type: 'tool-result', toolCallId: m.tool_call_id, toolName: (m as any).tool_name || '', - output: { type: 'text', value: m.content }, + output: { type: 'text', value: textOf(m.content) }, }, ], } as unknown as ModelMessage; } - return { role: m.role as any, content: m.content } as ModelMessage; + + if (m.role === 'user' || m.role === 'system') { + const parts: any[] = []; + for (const p of m.content) { + if (p.type === 'text') { + parts.push({ type: 'text', text: p.text }); + continue; + } + // 图片、音频、PDF 一律发 file 部件:协议层 user 消息只有 text | file, + // 驱动按 mediaType 分派到 image_url / input_audio / file + parts.push({ + type: 'file', + data: p.dataUrl, + mediaType: p.mimeType, + filename: p.filename, + }); + } + return { role: m.role, content: parts } as unknown as ModelMessage; + } + + return { role: m.role as any, content: textOf(m.content) } as ModelMessage; }); } diff --git a/packages/codingcode/src/llm/types.ts b/packages/codingcode/src/llm/types.ts index 963b466..7cae404 100644 --- a/packages/codingcode/src/llm/types.ts +++ b/packages/codingcode/src/llm/types.ts @@ -15,9 +15,82 @@ export interface ToolCall { arguments: Record; } +/** 文本块。 */ +export interface TextPart { + type: 'text'; + text: string; +} +/** 媒体块的引用形态:装配、估算、压缩共用的轻量形状。落盘元数据(体积 / 宽高 / 时长)归 session 层的 StoredMediaPart。 */ +export interface MediaPart { + type: 'media'; + /** 项目 assets 目录下的文件名,内容寻址、不可变 */ + asset: string; + /** 服务端嗅探结果,不采信客户端声明;种类也由它判定 */ + mimeType: string; + /** 客户端原始文件名,出网作 file part 的 filename;PDF 必填 */ + filename?: string; +} + +/** 媒体块的出网形态:装配层已把 asset 解析成 data URL,驱动层只做序列化。 */ +export interface ResolvedMediaPart { + type: 'media'; + dataUrl: string; + mimeType: string; + filename?: string; +} + +export type ContentPart = TextPart | MediaPart; + +/** 出网内容块:媒体字节已在装配层内联。 */ +export type ResolvedContentPart = TextPart | ResolvedMediaPart; + +/** 入口形态:媒体携带原始字节。只出现在请求边界与 agent 入参,落盘即消失。 */ +export interface IncomingMedia { + type: 'media'; + bytes: Uint8Array; + filename?: string; + declaredMimeType?: string; +} + +export type IncomingPart = TextPart | IncomingMedia; + +/** 媒体按 mime 主类型分档:图片 / 音频 / 其它文件。全链路唯一的判定口径。 */ +export type MediaKind = 'image' | 'audio' | 'file'; + +export function mediaKindOf(mimeType: string): MediaKind { + const mime = (mimeType ?? '').toLowerCase(); + if (mime.startsWith('image/')) return 'image'; + if (mime.startsWith('audio/')) return 'audio'; + return 'file'; +} + +/** 纯文本构造的短助手,给只有文本的构造点用。 */ +export const textPart = (text: string): TextPart => ({ type: 'text', text }); + +const MEDIA_MARKER: Record = { + image: '[image]', + audio: '[audio]', + file: '[file]', +}; + +/** + * 文本投影:标题 / 压缩模板 / 记忆摘要 / token 估算共用的唯一口径。 + * + * 落盘形态与入口形态都收:入口只在 agent 首帧定标题时用一次。 + */ +export function textOf(parts: readonly (ContentPart | ResolvedContentPart | IncomingPart)[]): string { + return parts + .map((p) => { + if (p.type === 'text') return p.text; + const mime = 'mimeType' in p ? p.mimeType : (p.declaredMimeType ?? ''); + return MEDIA_MARKER[mediaKindOf(mime)]; + }) + .join('\n'); +} + export interface Message { role: MessageRole; - content: string; + content: ContentPart[]; tool_calls?: ToolCall[]; tool_call_id?: string; tool_name?: string; @@ -25,6 +98,11 @@ export interface Message { usage?: TokenUsage; } +/** 出网消息:交给驱动的完整载荷,媒体字节已由装配层内联,驱动不再需要外部解析。 */ +export interface ResolvedMessage extends Omit { + content: ResolvedContentPart[]; +} + /** 请求里"给模型看的"工具描述。 */ export interface ToolDescription { name: string; @@ -33,7 +111,7 @@ export interface ToolDescription { } export interface LLMRequest { - messages: Message[]; + messages: ResolvedMessage[]; system?: string; tools?: ToolDescription[]; maxSteps?: number; diff --git a/packages/codingcode/src/memory/extractor.ts b/packages/codingcode/src/memory/extractor.ts index 51e0b5d..78bc9fa 100644 --- a/packages/codingcode/src/memory/extractor.ts +++ b/packages/codingcode/src/memory/extractor.ts @@ -1,5 +1,6 @@ import { Effect } from 'effect'; import type { LLMShape } from '../llm/port.js'; +import { textPart } from '../llm/types.js'; const SYSTEM_PROMPT = `你是记忆整理器。基于"已有记忆"和"会话记录",输出整份最新版长期记忆。 @@ -35,7 +36,7 @@ ${transcript || '(空)'}`; return llm .complete( { - messages: [{ role: 'user', content: userMessage }], + messages: [{ role: 'user', content: [textPart(userMessage)] }], system: SYSTEM_PROMPT, }, model diff --git a/packages/codingcode/src/memory/memory.ts b/packages/codingcode/src/memory/memory.ts index 35bfb40..86d3389 100644 --- a/packages/codingcode/src/memory/memory.ts +++ b/packages/codingcode/src/memory/memory.ts @@ -1,6 +1,7 @@ import { Layer, Effect } from 'effect'; import { readTranscript } from '../session/file-ops.js'; import type { SessionEvent } from '../session/types.js'; +import { textOf } from '../llm/types.js'; import { readMemoryFile, resolveMemoryPath, @@ -65,7 +66,7 @@ export const MemoryLayer = Layer.effect( for (const event of events) { switch (event.type) { case 'user': - lines.push(`[user] ${event.content}`); + lines.push(`[user] ${textOf(event.content)}`); break; case 'assistant': lines.push(`[assistant] ${event.content}`); diff --git a/packages/codingcode/src/scheduler/scheduler.ts b/packages/codingcode/src/scheduler/scheduler.ts index 34b83aa..15dd6b3 100644 --- a/packages/codingcode/src/scheduler/scheduler.ts +++ b/packages/codingcode/src/scheduler/scheduler.ts @@ -10,6 +10,7 @@ import type { import { readAutomations, writeAutomations } from './store.js'; import { AgentService } from '../agent/port.js'; import { activeModelId } from '../infra/models.js'; +import { textPart } from '../llm/types.js'; import { SchedulerService } from './port.js'; import { BYPASS_PERMISSION_MODE, BUILD_PROFILE_NAME } from '../util/enums.js'; @@ -50,7 +51,7 @@ export const SchedulerLayer = Layer.effect( const { stream, sessionId } = await _rt.runPromise( Effect.gen(function* () { const agent = yield* AgentService; - return yield* agent.runTurn(auto.description, { + return yield* agent.runTurn([textPart(auto.description)], { cwd: auto.projectCwd, signal: controller.signal, // 自动化没有独立的模型配置,统一用 config.yaml 的活动模型 @@ -183,7 +184,7 @@ export const SchedulerLayer = Layer.effect( const { stream, sessionId } = await _rt.runPromise( Effect.gen(function* () { const agent = yield* AgentService; - return yield* agent.runTurn(auto.description, { + return yield* agent.runTurn([textPart(auto.description)], { cwd: auto.projectCwd, signal: controller.signal, model: activeModelId(), diff --git a/packages/codingcode/src/server/app.ts b/packages/codingcode/src/server/app.ts index 35318e0..8a423e5 100644 --- a/packages/codingcode/src/server/app.ts +++ b/packages/codingcode/src/server/app.ts @@ -5,6 +5,7 @@ import { toHttpServerResponse } from './http-error.js'; import { json, type Handler } from './handler.js'; import { addSessionsRoutes } from './routes/sessions.js'; import { addMessagesRoutes } from './routes/messages.js'; +import { addAssetsRoutes } from './routes/assets.js'; import { addModelsRoutes } from './routes/models.js'; import { addApprovalRoutes } from './routes/approval.js'; import { addSettingsRoutes } from './routes/settings.js'; @@ -24,6 +25,7 @@ export const buildRouter = (): HttpRouter.HttpRouter => HttpRouter.get('/api/health', health), addSessionsRoutes, addMessagesRoutes, + addAssetsRoutes, addModelsRoutes, addApprovalRoutes, addSettingsRoutes, diff --git a/packages/codingcode/src/server/http-error.ts b/packages/codingcode/src/server/http-error.ts index 03d93de..c6cbacf 100644 --- a/packages/codingcode/src/server/http-error.ts +++ b/packages/codingcode/src/server/http-error.ts @@ -20,6 +20,7 @@ const DOMAIN_STATUS: Record = { AGENT_LOOP_DETECTED: 500, EMPTY_RESPONSE: 500, SESSION_IO_ERROR: 500, + INVALID_INPUT: 400, }; export class NotFoundError extends Error { diff --git a/packages/codingcode/src/server/routes/assets.ts b/packages/codingcode/src/server/routes/assets.ts new file mode 100644 index 0000000..0631fdb --- /dev/null +++ b/packages/codingcode/src/server/routes/assets.ts @@ -0,0 +1,34 @@ +import * as HttpRouter from '@effect/platform/HttpRouter'; +import * as HttpServerResponse from '@effect/platform/HttpServerResponse'; +import { Effect } from 'effect'; +import { AgentError } from '../../util/error.js'; +import { assertAssetName, assetsDirOf } from '../../session/paths.js'; +import { mimeTypeFromAssetName, readAsset } from '../../session/assets.js'; +import { resolveCwd } from '../cwd.js'; +import { pathParams, query, type Handler, type Router } from '../handler.js'; + +const getAsset: Handler = Effect.gen(function* () { + const { name } = yield* pathParams; + const { cwd } = yield* query; + const assetName = name ?? ''; + const dir = assetsDirOf(resolveCwd(cwd)); + + // 名字会直接成为文件名:只接受内容寻址形状(拦掉 .. 与路径分隔符) + yield* Effect.try({ + try: () => assertAssetName(assetName), + catch: (e) => (e instanceof AgentError ? e : AgentError.invalidInput(String(e))), + }); + + const bytes = yield* Effect.try({ + try: () => readAsset(dir, assetName), + catch: (e) => new AgentError('SESSION_IO_ERROR', `Failed to read asset: ${String(e)}`), + }); + + return HttpServerResponse.uint8Array(bytes, { + contentType: mimeTypeFromAssetName(assetName), + headers: { 'cache-control': 'public, max-age=31536000, immutable' }, + }); +}); + +export const addAssetsRoutes = (router: Router): Router => + router.pipe(HttpRouter.get('/api/assets/:name', getAsset)); diff --git a/packages/codingcode/src/server/routes/messages.ts b/packages/codingcode/src/server/routes/messages.ts index 776492d..c708611 100644 --- a/packages/codingcode/src/server/routes/messages.ts +++ b/packages/codingcode/src/server/routes/messages.ts @@ -7,6 +7,7 @@ import { isPermissionMode } from '../../approval/types.js'; import { loadConfig } from '../../infra/config.js'; import { AgentError } from '../../util/error.js'; import { ASK_BEFORE_EXEC_PERMISSION_MODE, BUILD_PROFILE_NAME } from '../../util/enums.js'; +import type { IncomingMedia, IncomingPart } from '../../llm/types.js'; import { frameStream, pathParams, @@ -17,14 +18,60 @@ import { type Router, } from '../handler.js'; +/** 线上内容部件的形状。 */ +type WirePart = + | { type: 'text'; text: string } + | { type: 'media'; dataUrl: string; filename?: string }; + /** POST body 的线上形状:模型是回合的必要输入,缺失即拒绝,不在 agent 层兜底成空串。 */ type MessageBody = { - input: string; + input: WirePart[]; cwd: string; model?: string; skills?: Array<{ name: string; path: string }>; }; +const DATA_URL_RE = /^data:([^;,]+);base64,(.*)$/s; +const BASE64_RE = /^[A-Za-z0-9+/]*={0,2}$/; + +export function toIncomingParts(input: unknown): IncomingPart[] { + if (!Array.isArray(input)) { + throw AgentError.invalidInput('input must be an array of content parts'); + } + const parts: IncomingPart[] = []; + for (const raw of input) { + if (typeof raw !== 'object' || raw === null) { + throw AgentError.invalidInput('input contains a non-object content part'); + } + const p = raw as Record; + if (p.type === 'text') { + if (typeof p.text !== 'string') { + throw AgentError.invalidInput('text part requires a string "text"'); + } + parts.push({ type: 'text', text: p.text }); + continue; + } + if (p.type === 'media') { + const dataUrl = typeof p.dataUrl === 'string' ? p.dataUrl : ''; + const matched = DATA_URL_RE.exec(dataUrl); + const payload = matched?.[2] ?? ''; + if (!matched || !BASE64_RE.test(payload) || payload.length % 4 === 1) { + throw AgentError.invalidInput('media part requires a base64 data URL'); + } + const media: IncomingMedia = { + type: 'media', + bytes: new Uint8Array(Buffer.from(payload, 'base64')), + ...(matched[1] ? { declaredMimeType: matched[1] } : {}), + }; + if (typeof p.filename === 'string' && p.filename) media.filename = p.filename; + parts.push(media); + continue; + } + throw AgentError.invalidInput(`unknown content part type: ${String(p.type)}`); + } + return parts; +} + const sendMessage: Handler = Effect.gen(function* () { const { id } = yield* pathParams; const web = yield* webRequest; @@ -36,6 +83,11 @@ const sendMessage: Handler = Effect.gen(function* () { // 工作区目录必须存在,否则拒绝(不允许带着不存在的路径开回合) const cwd = resolveWorkspaceCwd(body.cwd); + const input = yield* Effect.try({ + try: () => toIncomingParts(body.input), + catch: (e) => (e instanceof AgentError ? e : AgentError.invalidInput(String(e))), + }); + const sessionId = id ?? ''; const isNew = sessionId === '_' || !sessionId; const runOpts: Record = { @@ -55,7 +107,7 @@ const sendMessage: Handler = Effect.gen(function* () { } const agent = yield* AgentService; - const { stream, sessionId: actualSid } = yield* agent.runTurn(body.input, { + const { stream, sessionId: actualSid } = yield* agent.runTurn(input, { sessionId: isNew ? undefined : sessionId, ...runOpts, } as never); diff --git a/packages/codingcode/src/session/assets.ts b/packages/codingcode/src/session/assets.ts new file mode 100644 index 0000000..df63f02 --- /dev/null +++ b/packages/codingcode/src/session/assets.ts @@ -0,0 +1,257 @@ +import { createHash, randomUUID } from 'crypto'; +import { existsSync, mkdirSync, readFileSync, renameSync, unlinkSync, writeFileSync } from 'fs'; +import { join } from 'path'; +import { ascii } from '../util/media.js'; + +/** 单份媒体的字节上限。 */ +export const MAX_MEDIA_BYTES = 10 * 1024 * 1024; +/** 单回合的媒体份数上限。 */ +export const MAX_MEDIA_PER_TURN = 8; +/** 单条输入文本的字符上限。 */ +export const MAX_TEXT_CHARS = 1 << 20; + +/** mime → 扩展名。白名单即此表,顺序无关。 */ +const EXT_BY_MIME: Record = { + 'image/png': 'png', + 'image/jpeg': 'jpg', + 'image/gif': 'gif', + 'image/webp': 'webp', + 'audio/wav': 'wav', + 'audio/mpeg': 'mp3', + 'application/pdf': 'pdf', +}; + +const MIME_BY_EXT: Record = { + png: 'image/png', + jpg: 'image/jpeg', + gif: 'image/gif', + webp: 'image/webp', + wav: 'audio/wav', + mp3: 'audio/mpeg', + pdf: 'application/pdf', +}; + +function u16le(bytes: Uint8Array, at: number): number { + return (bytes[at] ?? 0) | ((bytes[at + 1] ?? 0) << 8); +} + +function u32be(bytes: Uint8Array, at: number): number { + return ( + ((bytes[at] ?? 0) << 24) | + ((bytes[at + 1] ?? 0) << 16) | + ((bytes[at + 2] ?? 0) << 8) | + (bytes[at + 3] ?? 0) + ) >>> 0; +} + +function u32le(bytes: Uint8Array, at: number): number { + return ( + ((bytes[at] ?? 0) | + ((bytes[at + 1] ?? 0) << 8) | + ((bytes[at + 2] ?? 0) << 16) | + ((bytes[at + 3] ?? 0) << 24)) >>> + 0 + ); +} + +/** 图片宽高,从文件头解析,不解码像素。 */ +export function readImageSize( + bytes: Uint8Array, + mimeType: string +): { width: number; height: number } | null { + try { + switch (mimeType) { + case 'image/png': { + // IHDR 紧随 8 字节签名 + 4 字节长度 + 4 字节类型 + if (bytes.length < 24) return null; + if (ascii(bytes, 12, 4) !== 'IHDR') return null; + return { width: u32be(bytes, 16), height: u32be(bytes, 20) }; + } + case 'image/gif': { + // 逻辑屏幕描述符:offset 6 / 8 + if (bytes.length < 10) return null; + return { width: u16le(bytes, 6), height: u16le(bytes, 8) }; + } + case 'image/jpeg': { + let at = 2; + while (at + 9 < bytes.length) { + if (bytes[at] !== 0xff) { + at++; + continue; + } + const marker = bytes[at + 1] ?? 0; + // 填充字节 + if (marker === 0xff) { + at++; + continue; + } + // 无载荷的标记 + if (marker === 0xd8 || marker === 0x01 || (marker >= 0xd0 && marker <= 0xd7)) { + at += 2; + continue; + } + const length = ((bytes[at + 2] ?? 0) << 8) | (bytes[at + 3] ?? 0); + // SOF0..SOF15,排除 DHT(C4) / JPG(C8) / DAC(CC) + const isSof = + marker >= 0xc0 && marker <= 0xcf && marker !== 0xc4 && marker !== 0xc8 && marker !== 0xcc; + if (isSof) { + return { + height: ((bytes[at + 5] ?? 0) << 8) | (bytes[at + 6] ?? 0), + width: ((bytes[at + 7] ?? 0) << 8) | (bytes[at + 8] ?? 0), + }; + } + // SOS 之后是熵编码数据,不再有尺寸信息 + if (marker === 0xda) return null; + if (length < 2) return null; + at += 2 + length; + } + return null; + } + case 'image/webp': { + if (bytes.length < 30) return null; + const chunk = ascii(bytes, 12, 4); + if (chunk === 'VP8 ') { + // 有损:帧标签 3 字节 + 同步码 3 字节,随后 14 位宽 / 14 位高 + if (bytes[23] !== 0x9d || bytes[24] !== 0x01 || bytes[25] !== 0x2a) return null; + return { width: u16le(bytes, 26) & 0x3fff, height: u16le(bytes, 28) & 0x3fff }; + } + if (chunk === 'VP8L') { + if (bytes[20] !== 0x2f) return null; + const bits = u32le(bytes, 21); + return { width: (bits & 0x3fff) + 1, height: ((bits >> 14) & 0x3fff) + 1 }; + } + if (chunk === 'VP8X') { + const width = ((bytes[24] ?? 0) | ((bytes[25] ?? 0) << 8) | ((bytes[26] ?? 0) << 16)) + 1; + const height = ((bytes[27] ?? 0) | ((bytes[28] ?? 0) << 8) | ((bytes[29] ?? 0) << 16)) + 1; + return { width, height }; + } + return null; + } + default: + return null; + } + } catch { + return null; + } +} + +/** MPEG 音频比特率表(kbps);索引即帧头的 bitrate_index。 */ +const BITRATES: Record = { + '1-1': [0, 32, 64, 96, 128, 160, 192, 224, 256, 288, 320, 352, 384, 416, 448], + '1-2': [0, 32, 48, 56, 64, 80, 96, 112, 128, 160, 192, 224, 256, 320, 384], + '1-3': [0, 32, 40, 48, 56, 64, 80, 96, 112, 128, 160, 192, 224, 256, 320], + '2-1': [0, 32, 48, 56, 64, 80, 96, 112, 128, 144, 160, 176, 192, 224, 256], + '2-2': [0, 8, 16, 24, 32, 40, 48, 56, 64, 80, 96, 112, 128, 144, 160], +}; +BITRATES['2-3'] = BITRATES['2-2']!; + +function id3v2Size(bytes: Uint8Array): number { + if (ascii(bytes, 0, 3) !== 'ID3' || bytes.length < 10) return 0; + const size = + (((bytes[6] ?? 0) & 0x7f) << 21) | + (((bytes[7] ?? 0) & 0x7f) << 14) | + (((bytes[8] ?? 0) & 0x7f) << 7) | + ((bytes[9] ?? 0) & 0x7f); + const footer = ((bytes[5] ?? 0) & 0x10) !== 0 ? 10 : 0; + return 10 + size + footer; +} + +function readMp3BitrateKbps(bytes: Uint8Array, start: number): number | null { + for (let at = start; at + 4 <= bytes.length; at++) { + if (bytes[at] !== 0xff || ((bytes[at + 1] ?? 0) & 0xe0) !== 0xe0) continue; + const versionBits = ((bytes[at + 1] ?? 0) >> 3) & 0x03; // 3=MPEG1, 2=MPEG2, 0=MPEG2.5 + const layerBits = ((bytes[at + 1] ?? 0) >> 1) & 0x03; // 3=Layer1, 2=Layer2, 1=Layer3 + const bitrateIndex = ((bytes[at + 2] ?? 0) >> 4) & 0x0f; + if (versionBits === 1 || layerBits === 0 || bitrateIndex === 0 || bitrateIndex === 15) continue; + const version = versionBits === 3 ? '1' : '2'; + const layer = layerBits === 3 ? '1' : layerBits === 2 ? '2' : '3'; + const table = BITRATES[`${version}-${layer}`]; + const kbps = table?.[bitrateIndex]; + if (kbps) return kbps; + } + return null; +} + +/** + * 音频时长(秒),从容器头解析,不做解码。 + * + * MP3 按首帧比特率折算(VBR 文件会偏大),只用于 token 估算,不参与出网。 + */ +export function readAudioDurationSec(bytes: Uint8Array, mimeType: string): number | null { + try { + if (mimeType === 'audio/wav') return readWavDurationSec(bytes); + if (mimeType === 'audio/mpeg') return readMp3DurationSec(bytes); + return null; + } catch { + return null; + } +} + +function readWavDurationSec(bytes: Uint8Array): number | null { + if (bytes.length < 12 || ascii(bytes, 8, 4) !== 'WAVE') return null; + let at = 12; + let byteRate = 0; + let dataSize = -1; + while (at + 8 <= bytes.length) { + const id = ascii(bytes, at, 4); + const size = u32le(bytes, at + 4); + const body = at + 8; + if (id === 'fmt ' && body + 16 <= bytes.length) { + byteRate = u32le(bytes, body + 8); + } else if (id === 'data') { + dataSize = size; + // 两个块的顺序不固定:两个都拿到就可以算 + if (byteRate > 0) break; + } + at = body + size + (size % 2); + } + if (byteRate <= 0 || dataSize < 0) return null; + return dataSize / byteRate; +} + +function readMp3DurationSec(bytes: Uint8Array): number | null { + const id3 = id3v2Size(bytes); + const kbps = readMp3BitrateKbps(bytes, id3); + if (!kbps) return null; + const audioBytes = bytes.length - id3; + if (audioBytes <= 0) return null; + return (audioBytes * 8) / (kbps * 1000); +} + +/** 内容寻址:sha256(bytes).slice(0, 32) + 扩展名。 */ +export function assetNameFor(bytes: Uint8Array, mimeType: string): string { + const ext = EXT_BY_MIME[mimeType]; + if (!ext) throw new Error(`Unsupported media type: ${mimeType}`); + const digest = createHash('sha256').update(bytes).digest('hex').slice(0, 32); + return `${digest}.${ext}`; +} + +/** 资产名 → mime。扩展名由嗅探结果生成,反向查表即权威。 */ +export function mimeTypeFromAssetName(name: string): string { + const ext = name.slice(name.lastIndexOf('.') + 1).toLowerCase(); + return MIME_BY_EXT[ext] ?? 'application/octet-stream'; +} + +/** 目标已存在则直接返回;否则原子就位。 */ +export function writeAsset(dir: string, name: string, bytes: Uint8Array): void { + const target = join(dir, name); + if (existsSync(target)) return; + mkdirSync(dir, { recursive: true }); + // 同目录重命名是原子的:项目级共享下同一张图可能被多个会话并发写入 + const tmp = join(dir, `${name}.${randomUUID()}.tmp`); + try { + writeFileSync(tmp, bytes); + renameSync(tmp, target); + } catch (e) { + try { + if (existsSync(tmp)) unlinkSync(tmp); + } catch { + /* 清理失败不掩盖原错误 */ + } + throw e; + } +} + +export function readAsset(dir: string, name: string): Uint8Array { + return new Uint8Array(readFileSync(join(dir, name))); +} diff --git a/packages/codingcode/src/session/paths.ts b/packages/codingcode/src/session/paths.ts index c8838c5..dc68729 100644 --- a/packages/codingcode/src/session/paths.ts +++ b/packages/codingcode/src/session/paths.ts @@ -1,12 +1,18 @@ import { join } from 'path'; import { normalizePath, encodeProjectPath, projectDataDir } from '../util/path.js'; +import { AgentError } from '../util/error.js'; /** 会话转录在项目数据目录下的布局。 */ export const SESSIONS_DIRNAME = 'sessions'; +/** 项目数据目录下存放输入附件(图片 / 音频 / PDF)的目录名。 */ +export const ASSETS_DIRNAME = 'assets'; + const SUBAGENTS_DIRNAME = 'subagents'; const TRANSCRIPT_SUFFIX = '.jsonl'; +const ASSET_RE = /^[0-9a-f]{32}\.(png|jpg|webp|gif|wav|mp3|pdf)$/; + export interface SessionPaths { sessionId: string; cwd: string; @@ -18,6 +24,18 @@ export function projectSessionsDir(cwd: string): string { return join(projectDataDir(cwd), SESSIONS_DIRNAME); } +/** 项目级资产目录:本项目的全部会话共享,与 sessions/ 并列。 */ +export function assetsDirOf(cwd: string): string { + return join(projectDataDir(cwd), ASSETS_DIRNAME); +} + +/** 资产名会直接成为文件名,只接受内容寻址形状。 */ +export function assertAssetName(name: string): void { + if (!ASSET_RE.test(name)) { + throw new AgentError('INVALID_INPUT', `Invalid asset name: ${name}`); + } +} + export function transcriptPathOf(cwd: string, sessionId: string, parentSessionId?: string): string { const sessionsDir = projectSessionsDir(cwd); return parentSessionId diff --git a/packages/codingcode/src/session/port.ts b/packages/codingcode/src/session/port.ts index b99c32e..b656d34 100644 --- a/packages/codingcode/src/session/port.ts +++ b/packages/codingcode/src/session/port.ts @@ -16,7 +16,8 @@ import type { SessionStoreState, } from './types.js'; import type { UITurn } from './types.js'; -import type { TokenUsage } from '../llm/types.js'; +import type { StoredPart } from './types.js'; +import type { IncomingPart, TokenUsage } from '../llm/types.js'; import type { ProfileName, PermissionMode } from '../util/enums.js'; export interface SessionShape { @@ -35,8 +36,22 @@ export interface SessionShape { renameSession(state: SessionStoreState, text: string): Effect.Effect; listSessions(cwd?: string): Effect.Effect; readHistory(state: SessionStoreState): Effect.Effect; - recordUser(state: SessionStoreState, content: string): Effect.Effect; - recordSystem(state: SessionStoreState, content: string): Effect.Effect; + /** 落盘入口媒体:嗅探 → 校验 → 写盘,返回落盘形态(文本原样通过,顺序保留)。 */ + materializeInput( + state: SessionStoreState, + parts: readonly IncomingPart[] + ): Effect.Effect; + /** 解析资产为 data URL,进程内按 (assetsDir, asset) 缓存。 */ + resolveAssets( + cwd: string, + assets: readonly string[] + ): Effect.Effect, AgentError>; + + recordUser(state: SessionStoreState, content: StoredPart[]): Effect.Effect; + recordSystem( + state: SessionStoreState, + content: StoredPart[] + ): Effect.Effect; recordAssistant( state: SessionStoreState, content: string, diff --git a/packages/codingcode/src/session/session.ts b/packages/codingcode/src/session/session.ts index 44350a6..c6bef86 100644 --- a/packages/codingcode/src/session/session.ts +++ b/packages/codingcode/src/session/session.ts @@ -4,7 +4,19 @@ import { existsSync } from 'fs'; import { join, dirname } from 'path'; import { AgentError } from '../util/error.js'; import { encodeProjectPath } from '../util/path.js'; -import { computePaths } from './paths.js'; +import { assetsDirOf, assertAssetName, computePaths } from './paths.js'; +import { + MAX_MEDIA_BYTES, + MAX_MEDIA_PER_TURN, + MAX_TEXT_CHARS, + assetNameFor, + mimeTypeFromAssetName, + readAsset, + readAudioDurationSec, + readImageSize, + writeAsset, +} from './assets.js'; +import { sniffMediaMime } from '../util/media.js'; import type { SessionMetaEvent, UserEvent, @@ -15,10 +27,19 @@ import type { RollbackEvent, SessionEvent, CompactEvent, + StoredPart, + StoredMediaPart, } from './types.js'; import type { SessionStoreState, SessionSummary } from './types.js'; import type { UITurn } from './types.js'; -import type { TokenUsage } from '../llm/types.js'; +import { + mediaKindOf, + textOf, + textPart, + type IncomingMedia, + type IncomingPart, + type TokenUsage, +} from '../llm/types.js'; import type { ProfileName, PermissionMode } from '../util/enums.js'; import { SessionService } from './port.js'; @@ -122,7 +143,7 @@ export function sessionEventsToTurns(events: SessionEvent[]): UITurn[] { id: nextId('user', event.turnId), type: 'message', role: 'user', - content: event.content, + parts: event.content, }); break; case 'assistant': @@ -131,7 +152,7 @@ export function sessionEventsToTurns(events: SessionEvent[]): UITurn[] { id: nextId('assistant', event.turnId), type: 'message', role: 'assistant', - content: event.content, + parts: [textPart(event.content)], }); } for (const tc of event.toolCalls ?? []) { @@ -171,6 +192,8 @@ function readUIHistory(sessionId: string, cwd: string): UITurn[] { export const SessionLayer = Layer.effect( SessionService, Effect.gen(function* () { + const assetCache = new Map(); + const create = ( cwd: string, options: { @@ -237,7 +260,7 @@ export const SessionLayer = Layer.effect( const recordUser = ( state: SessionStoreState, - content: string + content: StoredPart[] ): Effect.Effect => Effect.try({ try: () => { @@ -251,7 +274,7 @@ export const SessionLayer = Layer.effect( const transcriptPath = pathsFromState(state).transcriptPath; appendLine(transcriptPath, event); if (!state.title) { - const derived = truncateTitle(content); + const derived = truncateTitle(textOf(content)); if (derived) { rewriteSessionMeta(transcriptPath, { title: derived }); state.title = derived; @@ -267,7 +290,7 @@ export const SessionLayer = Layer.effect( const recordSystem = ( state: SessionStoreState, - content: string + content: StoredPart[] ): Effect.Effect => Effect.try({ try: () => { @@ -286,6 +309,109 @@ export const SessionLayer = Layer.effect( : new AgentError('SESSION_IO_ERROR', `Session write failed: ${String(e)}`, e), }); + const materializeInput = ( + state: SessionStoreState, + parts: readonly IncomingPart[] + ): Effect.Effect => + Effect.try({ + try: () => { + const media = parts.filter((p): p is IncomingMedia => p.type === 'media'); + if (media.length > MAX_MEDIA_PER_TURN) { + throw AgentError.invalidInput( + `Too many media attachments in one turn: ${media.length} > ${MAX_MEDIA_PER_TURN}` + ); + } + + const dir = media.length > 0 ? assetsDirOf(state.cwd) : ''; + const out: StoredPart[] = []; + for (const p of parts) { + if (p.type === 'text') { + if (p.text.length > MAX_TEXT_CHARS) { + throw AgentError.invalidInput( + `Input text too long: ${p.text.length} > ${MAX_TEXT_CHARS}` + ); + } + out.push({ type: 'text', text: p.text }); + continue; + } + + if (p.bytes.byteLength > MAX_MEDIA_BYTES) { + throw AgentError.invalidInput( + `Media too large: ${p.bytes.byteLength} > ${MAX_MEDIA_BYTES}` + ); + } + const mimeType = sniffMediaMime(p.bytes); + if (!mimeType) { + throw AgentError.invalidInput('Unsupported media: bytes are not in the whitelist'); + } + + const asset = assetNameFor(p.bytes, mimeType); + writeAsset(dir, asset, p.bytes); + + const kind = mediaKindOf(mimeType); + const part: StoredMediaPart = { + type: 'media', + asset, + mimeType, + bytes: p.bytes.byteLength, + }; + // PDF 的 filename 必填:驱动在缺省时会生成 part-N.pdf 这种无意义名字 + if (p.filename) part.filename = p.filename; + else if (kind === 'file') part.filename = asset; + if (kind === 'image') { + const size = readImageSize(p.bytes, mimeType); + if (size) { + part.width = size.width; + part.height = size.height; + } + } else if (kind === 'audio') { + const durationSec = readAudioDurationSec(p.bytes, mimeType); + if (durationSec !== null) part.durationSec = durationSec; + } + out.push(part); + } + return out; + }, + catch: (e) => + e instanceof AgentError + ? e + : new AgentError( + 'SESSION_IO_ERROR', + `Failed to store input media: ${String(e)}`, + e + ), + }); + + /** + * 读盘并转 data URL。内容寻址让缓存永不失效,键为 (assetsDir, asset)。 + */ + const resolveAssets = ( + cwd: string, + assets: readonly string[] + ): Effect.Effect, AgentError> => + Effect.sync(() => { + const dir = assetsDirOf(cwd); + const resolved = new Map(); + for (const asset of assets) { + const cacheKey = `${dir}\u0000${asset}`; + const cached = assetCache.get(cacheKey); + if (cached !== undefined) { + resolved.set(asset, cached); + continue; + } + try { + assertAssetName(asset); + const bytes = readAsset(dir, asset); + const url = `data:${mimeTypeFromAssetName(asset)};base64,${Buffer.from(bytes).toString('base64')}`; + assetCache.set(cacheKey, url); + resolved.set(asset, url); + } catch { + /* 资产缺失或不可读:跳过,出网时降级成文本标记 */ + } + } + return resolved; + }); + const recordAssistant = ( state: SessionStoreState, content: string, @@ -472,6 +598,8 @@ export const SessionLayer = Layer.effect( listSessions: listSessionsFromCwd, readHistory: readHistoryFromState, + materializeInput, + resolveAssets, recordUser, recordSystem, recordAssistant, diff --git a/packages/codingcode/src/session/types.ts b/packages/codingcode/src/session/types.ts index 726d6f1..8b81513 100644 --- a/packages/codingcode/src/session/types.ts +++ b/packages/codingcode/src/session/types.ts @@ -1,6 +1,23 @@ -import type { TokenUsage, ToolCall } from '../llm/types.js'; +import type { MediaPart, TextPart, TokenUsage, ToolCall } from '../llm/types.js'; import type { ProfileName, PermissionMode } from '../util/enums.js'; +/** + * 落盘形态的媒体块:MediaPart 之上附物化时解析出的元数据。 + * + */ +export interface StoredMediaPart extends MediaPart { + /** 原始字节数;UI 展示体积、PDF 的 token 估算用 */ + bytes: number; + /** 仅图片:服务端从文件头读出 */ + width?: number; + height?: number; + /** 仅音频:服务端从容器头读出,token 估算用 */ + durationSec?: number; +} + +/** 落盘形态的内容块:转录事件与 UI 转通用。 */ +export type StoredPart = TextPart | StoredMediaPart; + export interface SessionMetaEvent { type: 'session_meta'; sessionId: string; @@ -17,7 +34,7 @@ export interface SessionMetaEvent { export interface UserEvent { type: 'user'; turnId: number; - content: string; + content: StoredPart[]; source?: 'user' | 'system'; } @@ -103,7 +120,13 @@ export interface SessionRef { } export type UITurnItem = - | { id: string; type: 'message'; role: 'user' | 'assistant'; content: string; partial?: boolean } + | { + id: string; + type: 'message'; + role: 'user' | 'assistant'; + parts: StoredPart[]; + partial?: boolean; + } | { id: string; type: 'tool_call'; diff --git a/packages/codingcode/src/subagent/port.ts b/packages/codingcode/src/subagent/port.ts index 59ea348..b5eda2b 100644 --- a/packages/codingcode/src/subagent/port.ts +++ b/packages/codingcode/src/subagent/port.ts @@ -4,6 +4,7 @@ import type { FrameBody } from '../sink/types.js'; import type { AgentError } from '../util/error.js'; import type { Result } from '../util/result.js'; import type { ProfileName, PermissionMode } from '../util/enums.js'; +import type { IncomingPart } from '../llm/types.js'; export interface RunSubagentOptions { sessionId?: string; @@ -19,7 +20,7 @@ export interface RunSubagentOptions { export interface SubagentRunnerShape { runSubagent( - input: string, + input: IncomingPart[], opts: RunSubagentOptions ): Effect.Effect< { diff --git a/packages/codingcode/src/subagent/registry.ts b/packages/codingcode/src/subagent/registry.ts index 0dc383b..864b501 100644 --- a/packages/codingcode/src/subagent/registry.ts +++ b/packages/codingcode/src/subagent/registry.ts @@ -5,6 +5,7 @@ import type { EndTransition, FrameBody } from '../sink/types.js'; import { BYPASS_PERMISSION_MODE, type ProfileName } from '../util/enums.js'; import { SUBAGENT_RESULT_PREFIX } from '../session/types.js'; import { estimateTokensForContent } from '../context/tokens.js'; +import { textPart } from '../llm/types.js'; import { loadConfig } from '../infra/config.js'; import { MailboxService } from '../session/mailbox.js'; @@ -195,7 +196,7 @@ export const SubagentRunRegistryLayer = Layer.scoped( } const abort = new AbortController(); - const { stream, sessionId } = yield* runner.runSubagent(opts.prompt, { + const { stream, sessionId } = yield* runner.runSubagent([textPart(opts.prompt)], { cwd: opts.parentCwd, signal: abort.signal, // 子代理自己的 signal,与父回合无关 activeProfile: opts.parentProfile, diff --git a/packages/codingcode/src/subagent/subagent.ts b/packages/codingcode/src/subagent/subagent.ts index ea2b250..63505ee 100644 --- a/packages/codingcode/src/subagent/subagent.ts +++ b/packages/codingcode/src/subagent/subagent.ts @@ -5,13 +5,14 @@ import { AgentService } from '../agent/port.js'; import type { FrameBody } from '../sink/types.js'; import type { Result } from '../util/result.js'; import { BYPASS_PERMISSION_MODE } from '../util/enums.js'; +import type { IncomingPart } from '../llm/types.js'; export const SubagentRunnerLayer = Layer.effect( SubagentRunnerService, Effect.gen(function* () { const agent = yield* AgentService; - const runSubagent = (input: string, opts: RunSubagentOptions) => + const runSubagent = (input: IncomingPart[], opts: RunSubagentOptions) => Effect.gen(function* () { const result = yield* agent.runTurn(input, { sessionId: opts.sessionId, diff --git a/packages/codingcode/src/util/error.ts b/packages/codingcode/src/util/error.ts index d01aa45..1418241 100644 --- a/packages/codingcode/src/util/error.ts +++ b/packages/codingcode/src/util/error.ts @@ -15,7 +15,8 @@ export type ErrorCode = | 'AGENT_ABORTED' | 'AGENT_LOOP_DETECTED' | 'EMPTY_RESPONSE' - | 'SESSION_IO_ERROR'; + | 'SESSION_IO_ERROR' + | 'INVALID_INPUT'; export class AgentError extends Error { readonly _tag = 'AgentError'; @@ -54,6 +55,9 @@ export class AgentError extends Error { static configMissing(msg: string) { return new AgentError('CONFIG_MISSING', msg); } + static invalidInput(msg: string) { + return new AgentError('INVALID_INPUT', msg); + } static sessionNotFound(sessionId: string) { return new AgentError('SESSION_NOT_FOUND', `Session "${sessionId}" not found`); } diff --git a/packages/codingcode/src/util/media.ts b/packages/codingcode/src/util/media.ts new file mode 100644 index 0000000..4be3167 --- /dev/null +++ b/packages/codingcode/src/util/media.ts @@ -0,0 +1,47 @@ +export function ascii(bytes: Uint8Array, at: number, length: number): string { + let out = ''; + for (let i = 0; i < length; i++) out += String.fromCharCode(bytes[at + i] ?? 0); + return out; +} + +/** 魔数嗅探 mime;识别不了的返回 null,调用方决定降级策略。 */ +export function sniffMediaMime(bytes: Uint8Array): string | null { + if (bytes.length < 12) return null; + + // PNG: 89 50 4E 47 0D 0A 1A 0A + if ( + bytes[0] === 0x89 && + bytes[1] === 0x50 && + bytes[2] === 0x4e && + bytes[3] === 0x47 && + bytes[4] === 0x0d && + bytes[5] === 0x0a && + bytes[6] === 0x1a && + bytes[7] === 0x0a + ) { + return 'image/png'; + } + + // JPEG: FF D8 FF + if (bytes[0] === 0xff && bytes[1] === 0xd8 && bytes[2] === 0xff) return 'image/jpeg'; + + // GIF: GIF87a / GIF89a + if (ascii(bytes, 0, 4) === 'GIF8') return 'image/gif'; + + // RIFF 容器:偏移 8 处区分 WebP / WAV + if (ascii(bytes, 0, 4) === 'RIFF') { + const form = ascii(bytes, 8, 4); + if (form === 'WEBP') return 'image/webp'; + if (form === 'WAVE') return 'audio/wav'; + return null; + } + + // PDF: %PDF- + if (ascii(bytes, 0, 5) === '%PDF-') return 'application/pdf'; + + // MP3: ID3v2 标签 或 帧同步字(11 个 1) + if (ascii(bytes, 0, 3) === 'ID3') return 'audio/mpeg'; + if (bytes[0] === 0xff && ((bytes[1] ?? 0) & 0xe0) === 0xe0) return 'audio/mpeg'; + + return null; +} diff --git a/packages/codingcode/test/agent/skill-mention-persist.test.ts b/packages/codingcode/test/agent/skill-mention-persist.test.ts index 9b9914c..8188ae3 100644 --- a/packages/codingcode/test/agent/skill-mention-persist.test.ts +++ b/packages/codingcode/test/agent/skill-mention-persist.test.ts @@ -13,6 +13,8 @@ import { readHistory } from '../../src/session/file-ops.js'; import type { UserEvent } from '../../src/session/types.js'; import { runAgentTurn, llmStream, pText, pEnd } from '../helpers/agent-harness.js'; import { useTempProjectBase } from '../helpers/project-base.js'; +import { incomingText } from '../helpers/parts.js'; +import { textOf } from '../../src/llm/types.js'; const base = useTempProjectBase(); @@ -55,7 +57,7 @@ describe('agent runTurn with an explicit @ skill', () => { /** 真实 Session + 真实 Skill、mock LLM,跑一轮带 @ skill 的回合。 */ async function runTurnWithSkill() { - const input = `please @${SKILL_NAME} before shipping`; + const input = incomingText(`please @${SKILL_NAME} before shipping`); const skillPath = await canonicalSkillPath(projectDir); const { sessionId } = await runAgentTurn( @@ -86,15 +88,15 @@ describe('agent runTurn with an explicit @ skill', () => { const userEvents = persisted.filter((e): e is UserEvent => e.type === 'user'); expect(userEvents).toHaveLength(2); - expect(userEvents[0]!.content).toBe(input); + expect(userEvents[0]!.content).toEqual(input); expect(userEvents[0]!.source).toBe('user'); const block = userEvents[1]!; expect(block.source).toBe('system'); expect(block.turnId).toBe(userEvents[0]!.turnId); - expect(block.content).toContain(SKILL_NAME); - expect(block.content).toContain(skillPath); - expect(block.content).toContain('Collect merged PRs'); + expect(textOf(block.content)).toContain(SKILL_NAME); + expect(textOf(block.content)).toContain(skillPath); + expect(textOf(block.content)).toContain('Collect merged PRs'); }); it('does not surface the skill block in the UI history', async () => { @@ -112,7 +114,7 @@ describe('agent runTurn with an explicit @ skill', () => { const userMessages = turns .flatMap((t) => t.items) - .flatMap((i) => (i.type === 'message' && i.role === 'user' ? [i.content] : [])); + .flatMap((i) => (i.type === 'message' && i.role === 'user' ? [i.parts] : [])); expect(userMessages).toEqual([input]); }); diff --git a/packages/codingcode/test/agent/stop-hook.test.ts b/packages/codingcode/test/agent/stop-hook.test.ts index 4b217c6..34a37a6 100644 --- a/packages/codingcode/test/agent/stop-hook.test.ts +++ b/packages/codingcode/test/agent/stop-hook.test.ts @@ -1,4 +1,5 @@ import { expect, it, describe, vi } from 'vitest'; +import { textOf } from '../../src/llm/types.js'; import { Effect } from 'effect'; import { makeState, @@ -123,7 +124,7 @@ describe('agent runTurn stop hook', () => { { sessionId: 'test-sid', cwd: '/tmp' } ); - const contents = recordSystem.mock.calls.map((c: any) => c[1] as string); + const contents = recordSystem.mock.calls.map((c: any) => textOf(c[1])); expect(contents.some((c) => c === 'Custom injection message')).toBe(true); }); @@ -138,7 +139,7 @@ describe('agent runTurn stop hook', () => { { sessionId: 'test-sid', cwd: '/tmp' } ); - const contents = recordSystem.mock.calls.map((c: any) => c[1] as string); + const contents = recordSystem.mock.calls.map((c: any) => textOf(c[1])); expect(contents.some((c) => c === '(continue)')).toBe(true); }); }); diff --git a/packages/codingcode/test/context/budget-integration.test.ts b/packages/codingcode/test/context/budget-integration.test.ts index 917f15b..f9569af 100644 --- a/packages/codingcode/test/context/budget-integration.test.ts +++ b/packages/codingcode/test/context/budget-integration.test.ts @@ -53,7 +53,7 @@ describe('getHistory integration', () => { activeProfile: 'build', permissionMode: 'askBeforeExec', }, - { type: 'user', turnId: 1, content: 'q1' }, + { type: 'user', turnId: 1, content: [{ type: 'text', text: 'q1' }] }, { type: 'assistant', turnId: 1, diff --git a/packages/codingcode/test/context/compressor/behavior.test.ts b/packages/codingcode/test/context/compressor/behavior.test.ts index c95f209..280c68a 100644 --- a/packages/codingcode/test/context/compressor/behavior.test.ts +++ b/packages/codingcode/test/context/compressor/behavior.test.ts @@ -61,7 +61,7 @@ function makeFixture(opts: FixtureOptions) { lines.push({ type: 'user', turnId: turn, - content: `q${turn}`, + content: [{ type: 'text', text: `q${turn}` }], }); lines.push({ type: 'assistant', diff --git a/packages/codingcode/test/context/get-history-media.test.ts b/packages/codingcode/test/context/get-history-media.test.ts new file mode 100644 index 0000000..60967ed --- /dev/null +++ b/packages/codingcode/test/context/get-history-media.test.ts @@ -0,0 +1,119 @@ +import { describe, it, expect, beforeEach, afterEach } from 'vitest'; +import { mkdirSync, writeFileSync, rmSync, existsSync } from 'fs'; +import { dirname } from 'path'; +import { randomUUID } from 'crypto'; +import { Effect, Layer } from 'effect'; +import { ContextService } from '../../src/context/port.js'; +import type { ContextShape } from '../../src/context/port.js'; +import { SessionLayer } from '../../src/session/session.js'; +import { LLMService } from '../../src/llm/port.js'; +import type { SessionRef } from '../../src/session/types.js'; +import { assetsDirOf } from '../../src/session/paths.js'; +import { assetNameFor, writeAsset } from '../../src/session/assets.js'; +import { useTempProjectBase } from '../helpers/project-base.js'; +import { ContextLayer, transcriptPathFor } from '../../src/context/context.js'; +import { EventSinkLayer } from '../../src/sink/sink.js'; + +useTempProjectBase(); + +const TestLayer = Layer.mergeAll( + SessionLayer, + Layer.succeed(LLMService, { + complete: () => Effect.fail(new Error('no llm')), + completeStream: () => (async function* () {})(), + } as any), + EventSinkLayer +); + +const CWD = '/tmp/test'; +const PNG = new Uint8Array([ + 0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0x00, 0x00, 0x00, 0x0d, 0x49, 0x48, 0x44, 0x52, +]); + +function getCtxService(): Promise { + return Effect.runPromise( + Effect.gen(function* () { + return yield* ContextService; + }).pipe(Effect.provide(ContextLayer), Effect.provide(TestLayer)) + ); +} + +/** 转录里引用一个媒体块;`onDisk` 决定资产是否真的落在 assets 下。 */ +function writeTranscript(ref: SessionRef, asset: string, onDisk: boolean): string { + const path = transcriptPathFor(ref); + mkdirSync(dirname(path), { recursive: true }); + const lines = [ + { + type: 'session_meta', + sessionId: ref.sessionId, + cwd: CWD, + createdAt: new Date().toISOString(), + model: 'test-model', + title: 'media-fixture', + activeProfile: 'build', + permissionMode: 'askBeforeExec', + }, + { + type: 'user', + turnId: 1, + content: [ + { type: 'text', text: '看这张图' }, + { type: 'media', asset, mimeType: 'image/png', bytes: PNG.byteLength }, + ], + }, + ]; + writeFileSync(path, lines.map((l) => JSON.stringify(l)).join('\n') + '\n', 'utf8'); + + const assetDir = assetsDirOf(CWD); + mkdirSync(assetDir, { recursive: true }); + if (onDisk) writeAsset(assetDir, asset, PNG); + return path; +} + +describe('getHistory 的媒体装配', () => { + let ref: SessionRef; + let transcriptPath: string; + let assetDir: string; + + beforeEach(() => { + ref = { cwd: CWD, sessionId: randomUUID(), currentTurnId: 1 }; + assetDir = assetsDirOf(CWD); + }); + + afterEach(() => { + const dir = dirname(transcriptPath); + if (existsSync(dir)) rmSync(dir, { recursive: true, force: true }); + if (existsSync(assetDir)) rmSync(assetDir, { recursive: true, force: true }); + }); + + it('资产在位时解析成 data URL,交给驱动的载荷自带字节', async () => { + const asset = assetNameFor(PNG, 'image/png'); + transcriptPath = writeTranscript(ref, asset, true); + + const ctx = await getCtxService(); + const messages = await Effect.runPromise(ctx.getHistory(ref, 'test-model')); + + const user = messages.find((m) => m.role === 'user')!; + expect(user.content[0]).toEqual({ type: 'text', text: '看这张图' }); + expect(user.content[1]).toMatchObject({ + type: 'media', + mimeType: 'image/png', + dataUrl: `data:image/png;base64,${Buffer.from(PNG).toString('base64')}`, + }); + // 引用形态不该漏到出网载荷里 + expect('asset' in (user.content[1] as object)).toBe(false); + }); + + it('资产缺失时降级成文本标记,不在驱动层抛错', async () => { + transcriptPath = writeTranscript(ref, 'deadbeefdeadbeef.png', false); + + const ctx = await getCtxService(); + const messages = await Effect.runPromise(ctx.getHistory(ref, 'test-model')); + + const user = messages.find((m) => m.role === 'user')!; + expect(user.content[1]).toEqual({ + type: 'text', + text: '[media missing: deadbeefdeadbeef.png]', + }); + }); +}); diff --git a/packages/codingcode/test/context/memory-buffer.test.ts b/packages/codingcode/test/context/memory-buffer.test.ts index 8e840ee..f1768d4 100644 --- a/packages/codingcode/test/context/memory-buffer.test.ts +++ b/packages/codingcode/test/context/memory-buffer.test.ts @@ -8,6 +8,8 @@ import { LLMService } from '../../src/llm/port.js'; import { EventSinkLayer } from '../../src/sink/sink.js'; import type { SessionEvent } from '../../src/session/types.js'; import type { SessionRef } from '../../src/session/types.js'; +import { text } from '../helpers/parts.js'; +import { textOf } from '../../src/llm/types.js'; // 上下文窗口钉死成可控值(与其余 context 用例一致) const windowState = vi.hoisted(() => ({ value: 128000 })); @@ -67,7 +69,7 @@ function seedEvents(): SessionEvent[] { activeProfile: 'build', permissionMode: 'askBeforeExec', }, - { type: 'user', turnId: 1, content: 'q1' }, + { type: 'user', turnId: 1, content: text('q1') }, ]; } @@ -103,7 +105,7 @@ describe('context memory buffer', () => { ); const messages = await Effect.runPromise(ctx.getHistory(REF, 'test-model')); - expect(messages.some((m) => m.content === 'r1')).toBe(true); + expect(messages.some((m) => textOf(m.content) === 'r1')).toBe(true); expect(counting.reads.count).toBe(1); }); @@ -111,11 +113,11 @@ describe('context memory buffer', () => { const counting = makeCountingSession(seedEvents()); const ctx = await getCtx(counting); - await Effect.runPromise(ctx.absorb(REF, [{ type: 'user', turnId: 1, content: 'early' }])); + await Effect.runPromise(ctx.absorb(REF, [{ type: 'user', turnId: 1, content: text('early') }])); const messages = await Effect.runPromise(ctx.getHistory(REF, 'test-model')); expect(counting.reads.count).toBe(1); - expect(messages.some((m) => m.content === 'q1')).toBe(true); + expect(messages.some((m) => textOf(m.content) === 'q1')).toBe(true); }); it('压缩轮不再额外读盘', async () => { diff --git a/packages/codingcode/test/context/subagent-result-role.test.ts b/packages/codingcode/test/context/subagent-result-role.test.ts index c05acd3..fb550e1 100644 --- a/packages/codingcode/test/context/subagent-result-role.test.ts +++ b/packages/codingcode/test/context/subagent-result-role.test.ts @@ -1,7 +1,8 @@ import { describe, it, expect } from 'vitest'; import { buildContextMessages } from '../../src/context/context.js'; import { SUBAGENT_RESULT_PREFIX, type SessionEvent } from '../../src/session/types.js'; -import type { Message } from '../../src/llm/types.js'; +import { textOf, type Message } from '../../src/llm/types.js'; +import { text } from '../helpers/parts.js'; function build(events: unknown[]): Message[] { return buildContextMessages(events as SessionEvent[]); @@ -13,7 +14,7 @@ const resultContent = (payload: string) => describe('subagent_result 的入模形态', () => { it('以 user 角色注入,且保留正文首行的判别标记', () => { const messages = build([ - { type: 'user', turnId: 1, content: 'delegate this' }, + { type: 'user', turnId: 1, content: text('delegate this') }, { type: 'assistant', turnId: 1, @@ -37,12 +38,12 @@ describe('subagent_result 的入模形态', () => { const last = messages[messages.length - 1]!; expect(last.role).toBe('user'); - expect(last.content.startsWith(SUBAGENT_RESULT_PREFIX)).toBe(true); + expect(textOf(last.content).startsWith(SUBAGENT_RESULT_PREFIX)).toBe(true); }); it('绝不让 prompt 以 assistant 结尾(回归:模型空回复导致回合静默结束)', () => { const messages = build([ - { type: 'user', turnId: 1, content: 'delegate this' }, + { type: 'user', turnId: 1, content: text('delegate this') }, { type: 'assistant', turnId: 1, diff --git a/packages/codingcode/test/context/tokens-media.test.ts b/packages/codingcode/test/context/tokens-media.test.ts new file mode 100644 index 0000000..801d5db --- /dev/null +++ b/packages/codingcode/test/context/tokens-media.test.ts @@ -0,0 +1,75 @@ +import { describe, it, expect } from 'vitest'; +import { + estimateMessageTokens, + estimateTokensForPart, +} from '../../src/context/tokens.js'; +import type { StoredMediaPart } from '../../src/session/types.js'; + +function image(extra: Partial = {}): StoredMediaPart { + return { type: 'media', asset: 'a.png', mimeType: 'image/png', bytes: 1024, ...extra }; +} + +describe('estimateTokensForPart', () => { + it('图片按 tile 折算,显著高于同长度纯文本', () => { + const textTokens = estimateTokensForPart({ type: 'text', text: 'a'.repeat(20) }); + const imageTokens = estimateTokensForPart(image({ width: 512, height: 512 })); + expect(textTokens).toBeLessThan(20); + expect(imageTokens).toBe(85 + 170); + expect(imageTokens).toBeGreaterThan(textTokens * 10); + }); + + it('图片按 tile 数递增,缺宽高时按 1 个 tile 兜底', () => { + expect(estimateTokensForPart(image({ width: 512, height: 512 }))).toBe(255); + expect(estimateTokensForPart(image({ width: 1024, height: 1024 }))).toBe(85 + 170 * 4); + expect(estimateTokensForPart(image())).toBe(255); + }); + + it('音频随 durationSec 线性变化', () => { + const one = estimateTokensForPart({ + type: 'media', + asset: 'a.wav', + mimeType: 'audio/wav', + bytes: 8000, + durationSec: 1, + }); + const ten = estimateTokensForPart({ + type: 'media', + asset: 'a.wav', + mimeType: 'audio/wav', + bytes: 80000, + durationSec: 10, + }); + expect(one).toBe(32); + expect(ten).toBe(320); + expect(ten).toBe(one * 10); + }); + + it('file 档(PDF)随字节数线性变化', () => { + const oneMb = estimateTokensForPart({ + type: 'media', + asset: 'a.pdf', + mimeType: 'application/pdf', + bytes: 1024 * 1024, + }); + const twoMb = estimateTokensForPart({ + type: 'media', + asset: 'a.pdf', + mimeType: 'application/pdf', + bytes: 2 * 1024 * 1024, + }); + expect(oneMb).toBe(50_000); + expect(twoMb).toBe(100_000); + expect(twoMb).toBe(oneMb * 2); + }); +}); + +describe('estimateMessageTokens', () => { + it('把每个 part 的估算与消息固定开销相加', () => { + const withImage = estimateMessageTokens({ + role: 'user', + content: [image({ width: 512, height: 512 })], + }); + const empty = estimateMessageTokens({ role: 'user', content: [] }); + expect(withImage).toBe(empty + 255); + }); +}); diff --git a/packages/codingcode/test/context/tokens.test.ts b/packages/codingcode/test/context/tokens.test.ts index 0ffadb2..83634b3 100644 --- a/packages/codingcode/test/context/tokens.test.ts +++ b/packages/codingcode/test/context/tokens.test.ts @@ -1,9 +1,5 @@ import { describe, it, expect } from 'vitest'; -import { - estimateTokensForContent, - estimateTokens, - estimateMessageTokens, -} from '../../src/context/tokens.js'; +import { estimateTokensForContent } from '../../src/context/tokens.js'; describe('token estimation', () => { it('empty content returns 0', () => { @@ -25,41 +21,3 @@ describe('token estimation', () => { expect(estimateTokensForContent('hello世界')).toBe(4); }); }); - -describe('estimateMessageTokens', () => { - it('counts content + role + fixed structure overhead', () => { - const msg = { role: 'user', content: 'hello' } as any; - // content: ceil(5/3.5)=2, role: ceil(4/3.5)=2, structure: 4 => 8 - expect(estimateMessageTokens(msg)).toBe(8); - }); - - it('includes tool_call_id and tool_name for tool messages', () => { - const msg = { - role: 'tool', - content: 'result', - tool_call_id: 'tc123', - tool_name: 'bash', - } as any; - // content: ceil(6/3.5)=2, role: ceil(4/3.5)=2, tool_call_id: ceil(5/3.5)=2, - // tool_name: ceil(4/3.5)=2, structure: 4 => 12 - expect(estimateMessageTokens(msg)).toBe(12); - }); - - it('includes name for system/assistant messages', () => { - const msg = { role: 'system', name: 'compacted_history', content: 'summary' } as any; - // content: ceil(7/3.5)=2, role: ceil(6/3.5)=2, name: ceil(17/3.5)=5, structure: 4 => 13 - expect(estimateMessageTokens(msg)).toBe(13); - }); -}); - -describe('estimateTokens', () => { - it('aggregates full message tokens across array', () => { - const messages = [ - { role: 'user', content: 'hello' }, - { role: 'assistant', content: '你好' }, - ] as any; - // user: content(2) + role(2) + structure(4) = 8 - // assistant: content(2) + role(3) + structure(4) = 9 - expect(estimateTokens(messages)).toBe(17); - }); -}); diff --git a/packages/codingcode/test/helpers/agent-harness.ts b/packages/codingcode/test/helpers/agent-harness.ts index 5e550e4..1ef7f43 100644 --- a/packages/codingcode/test/helpers/agent-harness.ts +++ b/packages/codingcode/test/helpers/agent-harness.ts @@ -22,6 +22,8 @@ import { ToolExecutorService } from '../../src/tools/port.js'; import type { FrameBody, RuntimeEvent, Transition } from '../../src/sink/types.js'; import type { TokenUsage, LLMStreamPart } from '../../src/llm/types.js'; import type { SessionStoreState, SessionRef } from '../../src/session/types.js'; +import type { IncomingPart } from '../../src/llm/types.js'; +import { incomingText } from './parts.js'; // ---- LLM 部件构造器 ---- @@ -146,9 +148,11 @@ export interface HarnessMocks { session?: Partial<{ load: (cwd: string, sid: string) => any; create: (cwd: string, opts: any, extra?: any) => any; - recordUser: (state: any, content: string) => any; - recordSystem: (state: any, content: string) => any; - recordAssistant: (state: any, content: string, toolCalls: any[], usage?: any) => any; + materializeInput: (state: any, input: any) => any; + resolveAssets: (cwd: string, assets: readonly string[]) => any; + recordUser: (state: any, content: any) => any; + recordSystem: (state: any, content: any) => any; + recordAssistant: (state: any, content: any, toolCalls: any[], usage?: any) => any; recordToolResult: (state: any, name: string, id: string, output: string) => any; setPermissionMode: (cwd: string, sid: string, mode: any) => any; setActiveProfile: (cwd: string, sid: string, profile: any) => any; @@ -200,7 +204,7 @@ export function makeHarnessMocks(overrides: Partial = {}): Harness } export interface RunAgentOptions { - input?: string; + input?: IncomingPart[]; sessionId?: string; cwd?: string; signal?: AbortSignal; @@ -247,6 +251,21 @@ export function makeAgentLayer(mocks: HarnessMocks): Layer.Layer { sessionId: opts.sessionId ?? 'created-sid', activeProfile: opts.activeProfile ?? 'build', }), + materializeInput: (_state: any, input: any[]) => + Effect.succeed( + input.map((p: any) => + p.type === 'text' + ? { type: 'text' as const, text: p.text } + : { + type: 'media' as const, + asset: 'a'.repeat(32) + '.png', + mimeType: p.declaredMimeType ?? 'image/png', + bytes: p.bytes?.length ?? 0, + ...(p.filename ? { filename: p.filename } : {}), + } + ) + ), + resolveAssets: () => Effect.succeed(new Map()), recordUser: () => Effect.succeed({ turnId: 1 }), recordSystem: () => Effect.succeed({}), recordAssistant: () => Effect.succeed({}), @@ -382,7 +401,7 @@ export async function runAgentTurn( if (opts.activeProfile) runOpts.activeProfile = opts.activeProfile; if (opts.permissionMode) runOpts.permissionMode = opts.permissionMode; if (opts.skills) runOpts.skills = opts.skills; - return yield* agent.runTurn(opts.input ?? 'test', runOpts); + return yield* agent.runTurn(opts.input ?? incomingText('test'), runOpts); }); let runRes: { stream: AsyncGenerator; sessionId: string }; try { diff --git a/packages/codingcode/test/helpers/parts.ts b/packages/codingcode/test/helpers/parts.ts new file mode 100644 index 0000000..39ef7d2 --- /dev/null +++ b/packages/codingcode/test/helpers/parts.ts @@ -0,0 +1,22 @@ +import type { ContentPart, IncomingPart } from '../../src/llm/types.js'; +import type { StoredPart } from '../../src/session/types.js'; + +/** + * 测试里的纯文本内容块。 + * + * 消息内容统一是 parts 数组,用例只关心文本时用这个helper包一层, + * 免得每个断言点都写一遍 `[{ type: 'text', text }]`。 + */ +export function text(content: string): StoredPart[] { + return [{ type: 'text', text: content }]; +} + +/** 入口一侧的纯文本块(`IncomingPart[]`),程序化发起一轮输入时用。 */ +export function incomingText(content: string): IncomingPart[] { + return [{ type: 'text', text: content }]; +} + +/** 从 parts 里把文本拼回来,等价于生产侧的 `textOf`。 */ +export function textOfParts(parts: readonly ContentPart[]): string { + return parts.map((p) => (p.type === 'text' ? p.text : `[${p.type}]`)).join('\n'); +} diff --git a/packages/codingcode/test/llm/convert-messages.test.ts b/packages/codingcode/test/llm/convert-messages.test.ts new file mode 100644 index 0000000..587fc28 --- /dev/null +++ b/packages/codingcode/test/llm/convert-messages.test.ts @@ -0,0 +1,97 @@ +import { describe, it, expect } from 'vitest'; +import { convertMessages } from '../../src/llm/providers/shared.js'; +import type { ResolvedContentPart, ResolvedMessage } from '../../src/llm/types.js'; + +const t = (text: string): ResolvedContentPart => ({ type: 'text', text }); + +function userWith(content: ResolvedContentPart[]): ResolvedMessage[] { + return [{ role: 'user', content }]; +} + +describe('convertMessages', () => { + it('纯文本 user 消息产出一个 text part', () => { + expect(convertMessages(userWith([t('hello')]))).toEqual([ + { role: 'user', content: [{ type: 'text', text: 'hello' }] }, + ]); + }); + + it('图片 / 音频 / PDF 都产出 file part,不按种类分派', () => { + const out = convertMessages( + userWith([ + t('看看这些'), + { type: 'media', dataUrl: 'data:image/png;base64,AAA', mimeType: 'image/png' }, + { type: 'media', dataUrl: 'data:audio/wav;base64,BBB', mimeType: 'audio/wav' }, + { + type: 'media', + dataUrl: 'data:application/pdf;base64,CCC', + mimeType: 'application/pdf', + filename: 'doc.pdf', + }, + ]) + ); + + expect(out[0]!.content).toEqual([ + { type: 'text', text: '看看这些' }, + { + type: 'file', + data: 'data:image/png;base64,AAA', + mediaType: 'image/png', + filename: undefined, + }, + { + type: 'file', + data: 'data:audio/wav;base64,BBB', + mediaType: 'audio/wav', + filename: undefined, + }, + { + type: 'file', + data: 'data:application/pdf;base64,CCC', + mediaType: 'application/pdf', + filename: 'doc.pdf', + }, + ]); + }); + + it('请求体里不出现 detail 字段(不把 OpenAI 的图片档位带进协议层)', () => { + const out = convertMessages( + userWith([{ type: 'media', dataUrl: 'data:image/png;base64,AAA', mimeType: 'image/png' }]) + ); + const serialized = JSON.stringify(out); + expect(serialized).not.toContain('detail'); + }); + + it('assistant 带 tool_calls 时产出 text + tool-call part', () => { + const out = convertMessages([ + { + role: 'assistant', + content: [t('thinking')], + tool_calls: [{ id: 'tc1', name: 'read_file', arguments: { path: 'a.ts' } }], + }, + ]); + expect(out[0]!.content).toEqual([ + { type: 'text', text: 'thinking' }, + { type: 'tool-call', toolCallId: 'tc1', toolName: 'read_file', input: { path: 'a.ts' } }, + ]); + }); + + it('tool 消息产出 tool-result part,输出取自文本投影', () => { + const out = convertMessages([ + { role: 'tool', content: [t('output')], tool_call_id: 'tc1', tool_name: 'read_file' }, + ]); + expect(out[0]!.content).toEqual([ + { + type: 'tool-result', + toolCallId: 'tc1', + toolName: 'read_file', + output: { type: 'text', value: 'output' }, + }, + ]); + }); + + it('system 消息按 parts 映射', () => { + expect(convertMessages([{ role: 'system', content: [t('rules')] }])).toEqual([ + { role: 'system', content: [{ type: 'text', text: 'rules' }] }, + ]); + }); +}); diff --git a/packages/codingcode/test/llm/deepseek-provider.test.ts b/packages/codingcode/test/llm/deepseek-provider.test.ts index 3845cae..b127819 100644 --- a/packages/codingcode/test/llm/deepseek-provider.test.ts +++ b/packages/codingcode/test/llm/deepseek-provider.test.ts @@ -1,5 +1,6 @@ import { describe, it, expect, vi, beforeEach } from 'vitest'; import type { LLMStreamPart } from '../../src/llm/types.js'; +import { text } from '../helpers/parts.js'; const streamText = vi.fn(); const stepCountIs = vi.fn((count: number) => ({ count })); @@ -32,13 +33,14 @@ function entry() { base_url: 'https://api.deepseek.com/v1', api_key_env: 'DEEPSEEK_API_KEY', context_window: 64000, + capabilities: { vision: false, audio: false }, }; } function request() { return { system: 'system', - messages: [{ role: 'user', content: 'hello' }], + messages: [{ role: 'user', content: text('hello') }], tools: undefined, maxSteps: 1, }; diff --git a/packages/codingcode/test/llm/openai-provider.test.ts b/packages/codingcode/test/llm/openai-provider.test.ts index 8c6e8b8..740fb4f 100644 --- a/packages/codingcode/test/llm/openai-provider.test.ts +++ b/packages/codingcode/test/llm/openai-provider.test.ts @@ -1,5 +1,6 @@ import { describe, it, expect, vi, beforeEach } from 'vitest'; import type { LLMStreamPart } from '../../src/llm/types.js'; +import { text } from '../helpers/parts.js'; const generateText = vi.fn(); const streamText = vi.fn(); @@ -34,13 +35,14 @@ function entry(provider: string) { base_url: 'https://example.com/v1', api_key_env: 'API_KEY', context_window: 128000, + capabilities: { vision: false, audio: false }, }; } function request(withTools: boolean) { return { system: 'system', - messages: [{ role: 'user', content: 'hello' }], + messages: [{ role: 'user', content: text('hello') }], tools: withTools ? [{ name: 'read_file', description: 'Read file', parameters: { type: 'object' } }] : undefined, diff --git a/packages/codingcode/test/memory/extractor.test.ts b/packages/codingcode/test/memory/extractor.test.ts index 3ebbc2c..4fdb19c 100644 --- a/packages/codingcode/test/memory/extractor.test.ts +++ b/packages/codingcode/test/memory/extractor.test.ts @@ -1,6 +1,7 @@ import { describe, it, expect, vi } from 'vitest'; import { Effect } from 'effect'; import { extractMemory } from '../../src/memory/extractor.js'; +import { textOf } from '../../src/llm/types.js'; import { AgentError } from '../../src/util/error.js'; import type { LLMShape } from '../../src/llm/port.js'; import type { LLMRequest } from '../../src/llm/types.js'; @@ -65,9 +66,10 @@ describe('Memory Extractor', () => { await extract(mockLlm, '### project\n- 旧信息', '[user] 新对话'); const callArgs = mockLlm.complete.mock.calls[0]?.[0]; - expect(callArgs?.messages[0]?.content).toContain('已有记忆'); - expect(callArgs?.messages[0]?.content).toContain('旧信息'); - expect(callArgs?.messages[0]?.content).toContain('新对话'); + const prompt = textOf(callArgs?.messages[0]?.content ?? []); + expect(prompt).toContain('已有记忆'); + expect(prompt).toContain('旧信息'); + expect(prompt).toContain('新对话'); }); it('keeps instructions in system and transcript data in messages', async () => { @@ -76,11 +78,12 @@ describe('Memory Extractor', () => { await extract(mockLlm, '### project\n- Likes TypeScript', '[user] I use Python'); const callArgs = mockLlm.complete.mock.calls[0]?.[0]; + const prompt = textOf(callArgs?.messages[0]?.content ?? []); expect(callArgs?.system).toContain('规则'); expect(callArgs?.system).toContain('整份'); expect(callArgs?.system).not.toContain('I use Python'); - expect(callArgs?.messages[0]?.content).toContain('I use Python'); - expect(callArgs?.messages[0]?.content).toContain('Likes TypeScript'); + expect(prompt).toContain('I use Python'); + expect(prompt).toContain('Likes TypeScript'); }); it('passes the target model to the non-streaming channel', async () => { diff --git a/packages/codingcode/test/memory/index.test.ts b/packages/codingcode/test/memory/index.test.ts index bbce3a8..18535de 100644 --- a/packages/codingcode/test/memory/index.test.ts +++ b/packages/codingcode/test/memory/index.test.ts @@ -155,7 +155,9 @@ describe('flushSessionToMemory', () => { it('gracefully handles an LLM failure', async () => { await enableConfig(); const { readTranscript } = await import('../../src/session/file-ops.js'); - vi.mocked(readTranscript).mockImplementation(() => [{ type: 'user', content: 'hello' }] as any); + vi.mocked(readTranscript).mockImplementation( + () => [{ type: 'user', content: [{ type: 'text', text: 'hello' }] }] as any + ); mockLlm.complete.mockImplementation(() => Effect.fail(new AgentError('LLM_FAILED', 'llm unavailable')) ); @@ -170,7 +172,7 @@ describe('flushSessionToMemory', () => { vi.mocked(readTranscript).mockImplementation( () => [ - { type: 'user', content: '记住新架构决策' }, + { type: 'user', content: [{ type: 'text', text: '记住新架构决策' }] }, { type: 'assistant', content: '好的' }, ] as any ); @@ -188,7 +190,9 @@ describe('flushSessionToMemory', () => { await enableConfig(); writeMemory('### 旧主题\n- 旧内容'); const { readTranscript } = await import('../../src/session/file-ops.js'); - vi.mocked(readTranscript).mockImplementation(() => [{ type: 'user', content: 'hello' }] as any); + vi.mocked(readTranscript).mockImplementation( + () => [{ type: 'user', content: [{ type: 'text', text: 'hello' }] }] as any + ); setLlmResponse(''); const result = await run(service.flushSessionToMemory('session', TEST_MODEL, tmpDir)); @@ -202,7 +206,7 @@ describe('flushSessionToMemory', () => { writeMemory('### 主题\n- 不变的内容'); const { readTranscript } = await import('../../src/session/file-ops.js'); vi.mocked(readTranscript).mockImplementation( - () => [{ type: 'user', content: '无新信息' }] as any + () => [{ type: 'user', content: [{ type: 'text', text: '无新信息' }] }] as any ); setLlmResponse('### 主题\n- 不变的内容'); @@ -215,7 +219,9 @@ describe('flushSessionToMemory', () => { await enableConfig(); writeMemory('### 旧主题\n- 旧内容'); const { readTranscript } = await import('../../src/session/file-ops.js'); - vi.mocked(readTranscript).mockImplementation(() => [{ type: 'user', content: 'hello' }] as any); + vi.mocked(readTranscript).mockImplementation( + () => [{ type: 'user', content: [{ type: 'text', text: 'hello' }] }] as any + ); setLlmResponse('### 自动\n- 新记忆', () => { writeMemory('### 手动\n- 用户并发编辑'); }); diff --git a/packages/codingcode/test/server/messages-fork-permission-mode.test.ts b/packages/codingcode/test/server/messages-fork-permission-mode.test.ts index b17842f..b215b82 100644 --- a/packages/codingcode/test/server/messages-fork-permission-mode.test.ts +++ b/packages/codingcode/test/server/messages-fork-permission-mode.test.ts @@ -10,6 +10,7 @@ import { HookService } from '../../src/hooks/port.js'; import { ApprovalWaitService } from '../../src/approval/wait-port.js'; import { AgentService } from '../../src/agent/port.js'; import { useTempProjectBase } from '../helpers/project-base.js'; +import type { IncomingPart } from '../../src/llm/types.js'; useTempProjectBase(); @@ -32,7 +33,7 @@ const mockApprovalWaitService = { const loadedPermissionModes: string[] = []; const mockAgentService = { - runTurn: (_input: string, opts: any) => + runTurn: (_input: IncomingPart[], opts: any) => Effect.gen(function* () { const session = yield* SessionService; const state = yield* session.load(opts.cwd, opts.sessionId); @@ -94,7 +95,7 @@ describe('POST /api/sessions/:id/messages — reads permissionMode from disk', ( const res = await app.request('/api/sessions/' + sessionId + '/messages', { method: 'POST', headers: { 'content-type': 'application/json' }, - body: JSON.stringify({ input: 'hello', cwd, model: 'm' }), + body: JSON.stringify({ input: [{ type: 'text', text: 'hello' }], cwd, model: 'm' }), }); expect(res.status).not.toBe(404); expect(loadedPermissionModes[0]).toBe('bypass'); diff --git a/packages/codingcode/test/server/messages-input.test.ts b/packages/codingcode/test/server/messages-input.test.ts new file mode 100644 index 0000000..5dc156e --- /dev/null +++ b/packages/codingcode/test/server/messages-input.test.ts @@ -0,0 +1,71 @@ +import { describe, it, expect } from 'vitest'; +import { toIncomingParts } from '../../src/server/routes/messages.js'; +import type { AgentError } from '../../src/util/error.js'; + +function codeOf(fn: () => unknown): string { + try { + fn(); + } catch (e) { + return (e as AgentError).code; + } + return 'no-throw'; +} + +describe('toIncomingParts', () => { + it('文本块原样通过', () => { + expect(toIncomingParts([{ type: 'text', text: 'hi' }])).toEqual([{ type: 'text', text: 'hi' }]); + }); + + it('媒体块解出字节、声明 mime 与文件名', () => { + const parts = toIncomingParts([ + { + type: 'media', + dataUrl: 'data:image/png;base64,aGVsbG8=', + filename: 'x.png', + }, + ]); + expect(parts).toHaveLength(1); + const media = parts[0] as any; + expect(media.type).toBe('media'); + expect(Buffer.from(media.bytes).toString('utf8')).toBe('hello'); + expect(media.declaredMimeType).toBe('image/png'); + expect(media.filename).toBe('x.png'); + }); + + it('没有文件名也能通过(PDF 之外不强制)', () => { + const parts = toIncomingParts([ + { type: 'media', dataUrl: 'data:audio/wav;base64,aGVsbG8=' }, + ]); + expect((parts[0] as any).filename).toBeUndefined(); + }); + + it('非数组 input 被拒', () => { + expect(codeOf(() => toIncomingParts('hi'))).toBe('INVALID_INPUT'); + expect(codeOf(() => toIncomingParts(undefined))).toBe('INVALID_INPUT'); + expect(codeOf(() => toIncomingParts([null]))).toBe('INVALID_INPUT'); + }); + + it('shape 不对的部件被拒', () => { + expect(codeOf(() => toIncomingParts([{ type: 'text' }]))).toBe('INVALID_INPUT'); + expect(codeOf(() => toIncomingParts([{ type: 'media' }]))).toBe('INVALID_INPUT'); + expect(codeOf(() => toIncomingParts([{ type: 'image', dataUrl: 'x' }]))).toBe('INVALID_INPUT'); + }); + + it('非 base64 的 dataUrl 被拒', () => { + // 明文 data URL:没有 ;base64 段 + expect(codeOf(() => toIncomingParts([{ type: 'media', dataUrl: 'data:text/plain,hi' }]))).toBe( + 'INVALID_INPUT' + ); + // base64 载荷里出现非法字符 + expect( + codeOf(() => toIncomingParts([{ type: 'media', dataUrl: 'data:image/png;base64,!!!!' }])) + ).toBe('INVALID_INPUT'); + // 长度模 4 余 1:绝无可能是合法 base64 + expect( + codeOf(() => toIncomingParts([{ type: 'media', dataUrl: 'data:image/png;base64,aGVsbG8=' }])) + ).not.toBe('INVALID_INPUT'); + expect( + codeOf(() => toIncomingParts([{ type: 'media', dataUrl: 'data:image/png;base64,aGVsb' }])) + ).toBe('INVALID_INPUT'); + }); +}); diff --git a/packages/codingcode/test/server/session-title-route.test.ts b/packages/codingcode/test/server/session-title-route.test.ts index cc6e176..099efbe 100644 --- a/packages/codingcode/test/server/session-title-route.test.ts +++ b/packages/codingcode/test/server/session-title-route.test.ts @@ -8,6 +8,7 @@ import { createServer, type ServerApp } from '../../src/server/index.js'; import { computePaths } from '../../src/session/paths.js'; import { readSessionMeta } from '../../src/session/file-ops.js'; import { useTempProjectBase } from '../helpers/project-base.js'; +import { text } from '../helpers/parts.js'; const base = useTempProjectBase(); @@ -73,7 +74,7 @@ describe('PUT /api/sessions/:id/title', () => { Effect.gen(function* () { const session = yield* SessionService; const state = yield* session.load(cwd, sessionId); - yield* session.recordUser(state, 'first message'); + yield* session.recordUser(state, text('first message')); }) ); expect(await headTitle(sessionId)).toBe('first message'); diff --git a/packages/codingcode/test/server/sse.test.ts b/packages/codingcode/test/server/sse.test.ts index 27cbd6c..c788c23 100644 --- a/packages/codingcode/test/server/sse.test.ts +++ b/packages/codingcode/test/server/sse.test.ts @@ -14,6 +14,7 @@ import { AgentService } from '../../src/agent/port.js'; import { ApprovalWaitService } from '../../src/approval/wait-port.js'; import { AgentError } from '../../src/util/error.js'; import type { FrameBody } from '../../src/sink/types.js'; +import type { IncomingPart } from '../../src/llm/types.js'; const TEXT_FRAME: FrameBody = { family: 'event', event: { type: 'text_delta', text: 'hello' } }; @@ -23,7 +24,7 @@ let cancelled: string[] = []; function makeServer(frames: () => AsyncGenerator): Promise { const layer = Layer.mergeAll( Layer.succeed(AgentService, { - runTurn: (_input: string, opts: { sessionId?: string }) => + runTurn: (_input: IncomingPart[], opts: { sessionId?: string }) => Effect.succeed({ stream: frames(), sessionId: opts.sessionId ?? 's1', @@ -46,7 +47,7 @@ const send = (app: ServerApp, sessionId = 's1') => app.request(`/api/sessions/${sessionId}/messages`, { method: 'POST', headers: { 'content-type': 'application/json' }, - body: JSON.stringify({ input: 'hi', cwd, model: 'm' }), + body: JSON.stringify({ input: [{ type: 'text', text: 'hi' }], cwd, model: 'm' }), }); /** 把 SSE body 拆成逐帧的 JSON。 */ diff --git a/packages/codingcode/test/session/assets.test.ts b/packages/codingcode/test/session/assets.test.ts new file mode 100644 index 0000000..9e45e8b --- /dev/null +++ b/packages/codingcode/test/session/assets.test.ts @@ -0,0 +1,295 @@ +import { describe, it, expect } from 'vitest'; +import { mkdtempSync, readdirSync, rmSync } from 'fs'; +import { join } from 'path'; +import { tmpdir } from 'os'; +import { + assetNameFor, + mimeTypeFromAssetName, + readAudioDurationSec, + readImageSize, + writeAsset, +} from '../../src/session/assets.js'; +import { sniffMediaMime } from '../../src/util/media.js'; + +function ascii(b: Uint8Array, at: number, s: string): void { + for (let i = 0; i < s.length; i++) b[at + i] = s.charCodeAt(i); +} + +function u16le(b: Uint8Array, at: number, v: number): void { + b[at] = v & 0xff; + b[at + 1] = (v >> 8) & 0xff; +} + +function u16be(b: Uint8Array, at: number, v: number): void { + b[at] = (v >> 8) & 0xff; + b[at + 1] = v & 0xff; +} + +function u32le(b: Uint8Array, at: number, v: number): void { + b[at] = v & 0xff; + b[at + 1] = (v >> 8) & 0xff; + b[at + 2] = (v >> 16) & 0xff; + b[at + 3] = (v >>> 24) & 0xff; +} + +function u32be(b: Uint8Array, at: number, v: number): void { + b[at] = (v >>> 24) & 0xff; + b[at + 1] = (v >> 16) & 0xff; + b[at + 2] = (v >> 8) & 0xff; + b[at + 3] = v & 0xff; +} + +/** 只搭出嗅探与尺寸解析需要的那几个字节,不做真解码。 */ +function pngBytes(width: number, height: number): Uint8Array { + const b = new Uint8Array(33); + b.set([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a], 0); + u32be(b, 8, 13); + ascii(b, 12, 'IHDR'); + u32be(b, 16, width); + u32be(b, 20, height); + return b; +} + +function jpegBytes(width: number, height: number): Uint8Array { + const b = new Uint8Array(64); + b[0] = 0xff; + b[1] = 0xd8; + b[2] = 0xff; + // APP0 段:长度 16 ⇒ 下一个标记落在偏移 20 + b[3] = 0xe0; + u16be(b, 4, 16); + b[20] = 0xff; + b[21] = 0xc0; // SOF0 + u16be(b, 22, 17); + b[24] = 0x08; + u16be(b, 25, height); + u16be(b, 27, width); + return b; +} + +function gifBytes(width: number, height: number): Uint8Array { + const b = new Uint8Array(16); + ascii(b, 0, 'GIF89a'); + u16le(b, 6, width); + u16le(b, 8, height); + return b; +} + +function webpLossyBytes(width: number, height: number): Uint8Array { + const b = new Uint8Array(32); + ascii(b, 0, 'RIFF'); + u32le(b, 4, b.length - 8); + ascii(b, 8, 'WEBP'); + ascii(b, 12, 'VP8 '); + b[23] = 0x9d; + b[24] = 0x01; + b[25] = 0x2a; + u16le(b, 26, width); + u16le(b, 28, height); + return b; +} + +function webpLosslessBytes(width: number, height: number): Uint8Array { + const b = new Uint8Array(32); + ascii(b, 0, 'RIFF'); + u32le(b, 4, b.length - 8); + ascii(b, 8, 'WEBP'); + ascii(b, 12, 'VP8L'); + b[20] = 0x2f; + u32le(b, 21, (width - 1) | ((height - 1) << 14)); + return b; +} + +function webpExtendedBytes(width: number, height: number): Uint8Array { + const b = new Uint8Array(40); + ascii(b, 0, 'RIFF'); + u32le(b, 4, b.length - 8); + ascii(b, 8, 'WEBP'); + ascii(b, 12, 'VP8X'); + const w = width - 1; + const h = height - 1; + b[24] = w & 0xff; + b[25] = (w >> 8) & 0xff; + b[26] = (w >> 16) & 0xff; + b[27] = h & 0xff; + b[28] = (h >> 8) & 0xff; + b[29] = (h >> 16) & 0xff; + return b; +} + +function wavBytes(durationSec: number, byteRate: number): Uint8Array { + const dataSize = Math.round(durationSec * byteRate); + const b = new Uint8Array(44); + ascii(b, 0, 'RIFF'); + u32le(b, 4, 36); + ascii(b, 8, 'WAVE'); + ascii(b, 12, 'fmt '); + u32le(b, 16, 16); + u16le(b, 20, 1); + u16le(b, 22, 1); + u32le(b, 24, byteRate); + u32le(b, 28, byteRate); + u16le(b, 32, 1); + u16le(b, 34, 8); + ascii(b, 36, 'data'); + u32le(b, 40, dataSize); + return b; +} + +/** MPEG1 Layer3、比特率索引 9(128kbps)的首帧头。 */ +function mp3Bytes(totalBytes: number, id3TagBytes = 0): Uint8Array { + const b = new Uint8Array(totalBytes); + if (id3TagBytes > 0) { + ascii(b, 0, 'ID3'); + b[3] = 3; + b[5] = 0; + const size = id3TagBytes - 10; + b[6] = (size >> 21) & 0x7f; + b[7] = (size >> 14) & 0x7f; + b[8] = (size >> 7) & 0x7f; + b[9] = size & 0x7f; + } + b[id3TagBytes] = 0xff; + b[id3TagBytes + 1] = 0xfb; + b[id3TagBytes + 2] = 0x90; + return b; +} + +function pdfBytes(): Uint8Array { + const b = new Uint8Array(16); + ascii(b, 0, '%PDF-1.7'); + return b; +} + +describe('sniffMediaMime', () => { + it('识别白名单里的七种格式', () => { + expect(sniffMediaMime(pngBytes(4, 4))).toBe('image/png'); + expect(sniffMediaMime(jpegBytes(4, 4))).toBe('image/jpeg'); + expect(sniffMediaMime(gifBytes(4, 4))).toBe('image/gif'); + expect(sniffMediaMime(webpLossyBytes(4, 4))).toBe('image/webp'); + expect(sniffMediaMime(webpLosslessBytes(4, 4))).toBe('image/webp'); + expect(sniffMediaMime(wavBytes(1, 8000))).toBe('audio/wav'); + expect(sniffMediaMime(mp3Bytes(1024))).toBe('audio/mpeg'); + expect(sniffMediaMime(pdfBytes())).toBe('application/pdf'); + }); + + it('RIFF 容器按偏移 8 的 form 区分 WebP 与 WAV', () => { + const avi = new Uint8Array(16); + ascii(avi, 0, 'RIFF'); + ascii(avi, 8, 'AVI '); + expect(sniffMediaMime(avi)).toBeNull(); + }); + + it('伪造扩展名的非媒体文件一律返回 null', () => { + const text = new Uint8Array(64); + ascii(text, 0, 'this is not media at all'); + expect(sniffMediaMime(text)).toBeNull(); + expect(sniffMediaMime(new Uint8Array(4))).toBeNull(); + }); +}); + +describe('readImageSize', () => { + it('PNG 从 IHDR 读宽高', () => { + expect(readImageSize(pngBytes(640, 480), 'image/png')).toEqual({ width: 640, height: 480 }); + }); + + it('JPEG 扫到 SOF0 才读宽高', () => { + expect(readImageSize(jpegBytes(320, 200), 'image/jpeg')).toEqual({ width: 320, height: 200 }); + }); + + it('GIF 从逻辑屏幕描述符读宽高', () => { + expect(readImageSize(gifBytes(120, 90), 'image/gif')).toEqual({ width: 120, height: 90 }); + }); + + it('WebP 三种变体都能读宽高', () => { + expect(readImageSize(webpLossyBytes(800, 600), 'image/webp')).toEqual({ + width: 800, + height: 600, + }); + expect(readImageSize(webpLosslessBytes(333, 222), 'image/webp')).toEqual({ + width: 333, + height: 222, + }); + expect(readImageSize(webpExtendedBytes(4096, 2048), 'image/webp')).toEqual({ + width: 4096, + height: 2048, + }); + }); + + it('头不完整时返回 null 而不是抛错', () => { + expect(readImageSize(new Uint8Array(4), 'image/png')).toBeNull(); + expect(readImageSize(new Uint8Array(4), 'image/jpeg')).toBeNull(); + expect(readImageSize(new Uint8Array(4), 'image/webp')).toBeNull(); + }); +}); + +describe('readAudioDurationSec', () => { + it('WAV 按 data 块体积与 byteRate 折算', () => { + expect(readAudioDurationSec(wavBytes(2, 8000), 'audio/wav')).toBe(2); + expect(readAudioDurationSec(wavBytes(3, 44100), 'audio/wav')).toBe(3); + }); + + it('MP3 按首帧比特率折算,并跳过 ID3v2 标签', () => { + // 32000 字节 / 128kbps = 2 秒 + expect(readAudioDurationSec(mp3Bytes(32000), 'audio/mpeg')).toBe(2); + // ID3 标签不计入音频体积:多出来的 1000 字节不改变时长 + expect(readAudioDurationSec(mp3Bytes(33000, 1000), 'audio/mpeg')).toBe(2); + }); + + it('非音频 mime 返回 null', () => { + expect(readAudioDurationSec(pngBytes(4, 4), 'image/png')).toBeNull(); + }); +}); + +describe('assetNameFor / mimeTypeFromAssetName', () => { + it('同字节同名、异字节异名,且扩展名跟随嗅探结果', () => { + const a = pngBytes(4, 4); + const b = pngBytes(8, 8); + expect(assetNameFor(a, 'image/png')).toBe(assetNameFor(a, 'image/png')); + expect(assetNameFor(a, 'image/png')).not.toBe(assetNameFor(b, 'image/png')); + expect(assetNameFor(a, 'image/png')).toMatch(/^[0-9a-f]{32}\.png$/); + expect(assetNameFor(mp3Bytes(64), 'audio/mpeg')).toMatch(/^[0-9a-f]{32}\.mp3$/); + }); + + it('扩展名与白名单反向一致,未知扩展名退回 octet-stream', () => { + expect(mimeTypeFromAssetName('a'.repeat(32) + '.png')).toBe('image/png'); + expect(mimeTypeFromAssetName('a'.repeat(32) + '.jpg')).toBe('image/jpeg'); + expect(mimeTypeFromAssetName('a'.repeat(32) + '.gif')).toBe('image/gif'); + expect(mimeTypeFromAssetName('a'.repeat(32) + '.webp')).toBe('image/webp'); + expect(mimeTypeFromAssetName('a'.repeat(32) + '.wav')).toBe('audio/wav'); + expect(mimeTypeFromAssetName('a'.repeat(32) + '.mp3')).toBe('audio/mpeg'); + expect(mimeTypeFromAssetName('a'.repeat(32) + '.pdf')).toBe('application/pdf'); + expect(mimeTypeFromAssetName('a'.repeat(32) + '.exe')).toBe('application/octet-stream'); + }); +}); + +describe('writeAsset', () => { + it('内容寻址 ⇒ 写两次目录里仍只有一份,且不留 .tmp', () => { + const dir = mkdtempSync(join(tmpdir(), 'codingcode-assets-')); + try { + const bytes = pngBytes(16, 16); + const name = assetNameFor(bytes, 'image/png'); + writeAsset(dir, name, bytes); + writeAsset(dir, name, bytes); + + const entries = readdirSync(dir); + expect(entries).toEqual([name]); + expect(entries.some((e) => e.endsWith('.tmp'))).toBe(false); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it('目录不存在时自建(项目级 assets 与 sessions 并列,可能还没被创建)', () => { + const root = mkdtempSync(join(tmpdir(), 'codingcode-assets-')); + const dir = join(root, 'assets'); + try { + const bytes = pdfBytes(); + const name = assetNameFor(bytes, 'application/pdf'); + writeAsset(dir, name, bytes); + expect(readdirSync(dir)).toEqual([name]); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +}); diff --git a/packages/codingcode/test/session/create-active-profile.test.ts b/packages/codingcode/test/session/create-active-profile.test.ts index 1be5653..89d708f 100644 --- a/packages/codingcode/test/session/create-active-profile.test.ts +++ b/packages/codingcode/test/session/create-active-profile.test.ts @@ -5,6 +5,7 @@ import { SessionService } from '../../src/session/port.js'; import { SessionLayer } from '../../src/session/session.js'; import { readSessionMeta } from '../../src/session/file-ops.js'; import { useTempProjectBase } from '../helpers/project-base.js'; +import { text } from '../helpers/parts.js'; useTempProjectBase(); @@ -53,7 +54,7 @@ describe('session activeProfile persistence', () => { const session = yield* SessionService; yield* session.setActiveProfile(state.cwd, state.sessionId, 'plan'); const reloaded = yield* session.load(state.cwd, state.sessionId); - yield* session.recordUser(reloaded, 'hello'); + yield* session.recordUser(reloaded, text('hello')); }) ); diff --git a/packages/codingcode/test/session/filter-ui.test.ts b/packages/codingcode/test/session/filter-ui.test.ts index 99d98c4..901e343 100644 --- a/packages/codingcode/test/session/filter-ui.test.ts +++ b/packages/codingcode/test/session/filter-ui.test.ts @@ -1,6 +1,7 @@ import { describe, it, expect } from 'vitest'; import type { SessionEvent } from '../../src/session/types.js'; import { filterForUI, sessionEventsToTurns } from '../../src/session/session.js'; +import { text } from '../helpers/parts.js'; function makeBaseEvents(extra: SessionEvent[] = []): SessionEvent[] { const base: SessionEvent[] = [ @@ -14,11 +15,11 @@ function makeBaseEvents(extra: SessionEvent[] = []): SessionEvent[] { activeProfile: 'build', permissionMode: 'askBeforeExec', }, - { type: 'user', turnId: 1, content: 'hello' }, + { type: 'user', turnId: 1, content: text('hello') }, { type: 'assistant', turnId: 1, content: 'hi', toolCalls: [] }, - { type: 'user', turnId: 2, content: 'do stuff' }, + { type: 'user', turnId: 2, content: text('do stuff') }, { type: 'assistant', turnId: 2, content: 'ok', toolCalls: [] }, - { type: 'user', turnId: 3, content: 'done' }, + { type: 'user', turnId: 3, content: text('done') }, { type: 'assistant', turnId: 3, content: 'great', toolCalls: [] }, ]; return [...base, ...extra]; @@ -133,9 +134,9 @@ describe('sessionEventsToTurns with summary', () => { activeProfile: 'build', permissionMode: 'askBeforeExec', }, - { type: 'user', turnId: 1, content: 'hello' }, + { type: 'user', turnId: 1, content: text('hello') }, { type: 'assistant', turnId: 1, content: 'hi', toolCalls: [] }, - { type: 'user', turnId: 2, content: 'more' }, + { type: 'user', turnId: 2, content: text('more') }, { type: 'assistant', turnId: 2, content: 'ok', toolCalls: [] }, { type: 'summary', diff --git a/packages/codingcode/test/session/io-error.test.ts b/packages/codingcode/test/session/io-error.test.ts index 8976165..5cb8728 100644 --- a/packages/codingcode/test/session/io-error.test.ts +++ b/packages/codingcode/test/session/io-error.test.ts @@ -4,6 +4,7 @@ import { SessionService } from '../../src/session/port.js'; import { SessionLayer } from '../../src/session/session.js'; import { AgentError } from '../../src/util/error.js'; import * as fs from 'fs'; +import { text } from '../helpers/parts.js'; vi.mock('fs', async (importOriginal) => ({ ...(await importOriginal()), @@ -31,7 +32,7 @@ describe('SessionService — SESSION_IO_ERROR', () => { const exit = await Effect.runPromiseExit( Effect.gen(function* () { const svc = yield* SessionService; - return yield* svc.recordUser(state, 'hello'); + return yield* svc.recordUser(state, text('hello')); }).pipe(Effect.provide(SessionLayer)) ); @@ -89,7 +90,7 @@ describe('SessionService — SESSION_IO_ERROR', () => { const program = Effect.gen(function* () { const session = yield* SessionService; - return yield* session.recordUser(state, 'hello'); + return yield* session.recordUser(state, text('hello')); }).pipe(Effect.provide(SessionLayer)); const exit = await Effect.runPromiseExit(program); diff --git a/packages/codingcode/test/session/load-create.test.ts b/packages/codingcode/test/session/load-create.test.ts index 726cdfa..30115fa 100644 --- a/packages/codingcode/test/session/load-create.test.ts +++ b/packages/codingcode/test/session/load-create.test.ts @@ -9,6 +9,7 @@ import { encodeProjectPath } from '../../src/util/path.js'; import { computePaths } from '../../src/session/paths.js'; import { readSessionMeta } from '../../src/session/file-ops.js'; import { useTempProjectBase } from '../helpers/project-base.js'; +import { text } from '../helpers/parts.js'; const base = useTempProjectBase(); @@ -80,7 +81,7 @@ describe('load — keeps the persisted model untouched', () => { Effect.gen(function* () { const svc = yield* SessionService; const state = yield* svc.load(dir, sid); - yield* svc.recordUser(state, 'first message'); + yield* svc.recordUser(state, text('first message')); }) ); @@ -267,14 +268,14 @@ describe('load restores persisted fields', () => { Effect.gen(function* () { const svc = yield* SessionService; const state = yield* svc.load(dir, sid); - yield* svc.recordUser(state, 'first'); + yield* svc.recordUser(state, text('first')); }) ); await run( Effect.gen(function* () { const svc = yield* SessionService; const state = yield* svc.load(dir, sid); - yield* svc.recordUser(state, 'second'); + yield* svc.recordUser(state, text('second')); }) ); @@ -313,7 +314,7 @@ describe('load restores persisted fields', () => { Effect.gen(function* () { const svc = yield* SessionService; const state = yield* svc.load(dir, sid); - yield* svc.recordUser(state, 'hello'); + yield* svc.recordUser(state, text('hello')); yield* svc.recordAssistant(state, 'world', [ { id: 'tc1', name: 'bash', arguments: { cmd: 'echo' } }, ]); diff --git a/packages/codingcode/test/session/materialize-input.test.ts b/packages/codingcode/test/session/materialize-input.test.ts new file mode 100644 index 0000000..2250220 --- /dev/null +++ b/packages/codingcode/test/session/materialize-input.test.ts @@ -0,0 +1,299 @@ +import { describe, it, expect } from 'vitest'; +import { existsSync, mkdirSync, readdirSync, rmSync, unlinkSync } from 'fs'; +import { join } from 'path'; +import { randomUUID } from 'crypto'; +import { Effect, Either } from 'effect'; +import { SessionService } from '../../src/session/port.js'; +import { SessionLayer } from '../../src/session/session.js'; +import type { SessionStoreState } from '../../src/session/types.js'; +import { assetsDirOf } from '../../src/session/paths.js'; +import { encodeProjectPath } from '../../src/util/path.js'; +import { assetNameFor, MAX_MEDIA_PER_TURN } from '../../src/session/assets.js'; +import { useTempProjectBase } from '../helpers/project-base.js'; +import { incomingText } from '../helpers/parts.js'; + +const base = useTempProjectBase(); + +function run(eff: Effect.Effect): Promise { + return Effect.runPromise(eff.pipe(Effect.provide(SessionLayer) as any)); +} + +function either(eff: Effect.Effect): Promise> { + return Effect.runPromise(Effect.either(eff.pipe(Effect.provide(SessionLayer)) as any)); +} + +function newDir(): string { + const dir = join(base.dir, randomUUID()); + mkdirSync(dir, { recursive: true }); + return dir; +} + +function cleanup(dir: string): void { + rmSync(join(base.dir, encodeProjectPath(dir)), { recursive: true, force: true }); + rmSync(dir, { recursive: true, force: true }); +} + +async function newSession( + dir: string, + opts?: { parentSessionId?: string; agentName?: string } +): Promise { + return run( + Effect.gen(function* () { + const svc = yield* SessionService; + return yield* svc.create( + dir, + { model: 'test-model', activeProfile: 'build', permissionMode: 'askBeforeExec' }, + opts + ); + }) + ); +} + +/** 只搭出嗅探需要的头十来个字节。 */ +function mediaBytes(kind: 'png' | 'wav' | 'txt'): Uint8Array { + if (kind === 'png') { + const b = new Uint8Array(33); + b.set([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a], 0); + b[15] = 13; + b.set([0x49, 0x48, 0x44, 0x52], 12); // IHDR + b[18] = 0x01; + b[19] = 0x40; // width 320(big-endian) + b[23] = 0xf0; // height 240 + return b; + } + if (kind === 'wav') { + const b = new Uint8Array(44); + b.set([0x52, 0x49, 0x46, 0x46], 0); // RIFF + b.set([0x57, 0x41, 0x56, 0x45], 8); // WAVE + b.set([0x66, 0x6d, 0x74, 0x20], 12); // fmt + b[16] = 16; + b[24] = 0x40; + b[25] = 0x1f; // sampleRate 8000 + b[28] = 0x40; + b[29] = 0x1f; // byteRate 8000 + b[36] = 0x64; + b[37] = 0x61; + b[38] = 0x74; // 'dat' + b[39] = 0x61; // 'a' ⇒ 'data' + b[40] = 0x40; + b[41] = 0x1f; // dataSize 8000 ⇒ 1 秒 + return b; + } + const b = new Uint8Array(64); + for (let i = 0; i < b.length; i++) b[i] = 0x20 + (i % 60); + return b; +} + +describe('SessionService.materializeInput', () => { + it('纯文本原样通过且顺序保留', async () => { + const dir = newDir(); + try { + const state = await newSession(dir); + const parts = await run( + Effect.gen(function* () { + const svc = yield* SessionService; + return yield* svc.materializeInput(state, [incomingText('a')[0]!, incomingText('b')[0]!]); + }) + ); + expect(parts).toEqual([ + { type: 'text', text: 'a' }, + { type: 'text', text: 'b' }, + ]); + } finally { + cleanup(dir); + } + }); + + it('媒体落进项目 assets 目录,mime 取嗅探结果并带上宽高 / 时长', async () => { + const dir = newDir(); + try { + const state = await newSession(dir); + const png = mediaBytes('png'); + const wav = mediaBytes('wav'); + const parts = await run( + Effect.gen(function* () { + const svc = yield* SessionService; + return yield* svc.materializeInput(state, [ + { type: 'text', text: '看这张图' }, + // declaredMimeType 与真实字节不符:准入只看嗅探 + { type: 'media', bytes: png, declaredMimeType: 'audio/mpeg', filename: 'x.wav' }, + { type: 'media', bytes: wav, filename: 'sound.wav' }, + ]); + }) + ); + + const assetDir = assetsDirOf(state.cwd); + expect(parts).toHaveLength(3); + expect(parts[0]).toEqual({ type: 'text', text: '看这张图' }); + expect(parts[1]).toMatchObject({ + type: 'media', + mimeType: 'image/png', + bytes: png.byteLength, + filename: 'x.wav', + width: 320, + height: 240, + }); + expect(parts[2]).toMatchObject({ + type: 'media', + mimeType: 'audio/wav', + bytes: wav.byteLength, + durationSec: 1, + }); + + const names = readdirSync(assetDir).sort(); + expect(names).toEqual([ + assetNameFor(png, 'image/png'), + assetNameFor(wav, 'audio/wav'), + ].sort()); + // 转录目录下不落媒体:assets 与 sessions 并列 + expect(existsSync(join(assetDir, '..', 'sessions', assetNameFor(png, 'image/png')))).toBe( + false + ); + } finally { + cleanup(dir); + } + }); + + it('同一份字节在两个会话里各落一次,assets 下仍只有一份', async () => { + const dir = newDir(); + try { + const png = mediaBytes('png'); + const first = await newSession(dir); + const second = await newSession(dir); + for (const state of [first, second]) { + await run( + Effect.gen(function* () { + const svc = yield* SessionService; + return yield* svc.materializeInput(state, [{ type: 'media', bytes: png }]); + }) + ); + } + expect(readdirSync(assetsDirOf(dir))).toEqual([assetNameFor(png, 'image/png')]); + } finally { + cleanup(dir); + } + }); + + it('白名单外的字节 → INVALID_INPUT,且不落任何文件', async () => { + const dir = newDir(); + try { + const state = await newSession(dir); + const res = await either( + Effect.gen(function* () { + const svc = yield* SessionService; + return yield* svc.materializeInput(state, [ + { type: 'media', bytes: mediaBytes('txt'), declaredMimeType: 'image/png' }, + ]); + }) + ); + expect(Either.isLeft(res)).toBe(true); + if (Either.isLeft(res)) expect((res.left as any).code).toBe('INVALID_INPUT'); + expect(existsSync(assetsDirOf(dir))).toBe(false); + } finally { + cleanup(dir); + } + }); + + it('单回合媒体数量超上限 → INVALID_INPUT', async () => { + const dir = newDir(); + try { + const state = await newSession(dir); + const png = mediaBytes('png'); + const tooMany = Array.from({ length: MAX_MEDIA_PER_TURN + 1 }, () => ({ + type: 'media' as const, + bytes: png, + })); + const res = await either( + Effect.gen(function* () { + const svc = yield* SessionService; + return yield* svc.materializeInput(state, tooMany); + }) + ); + expect(Either.isLeft(res)).toBe(true); + if (Either.isLeft(res)) expect((res.left as any).code).toBe('INVALID_INPUT'); + } finally { + cleanup(dir); + } + }); + + it('子会话媒体与主会话走同一条物化路径', async () => { + const dir = newDir(); + try { + const child = await newSession(dir, { parentSessionId: 'parent-1', agentName: 'reviewer' }); + const png = mediaBytes('png'); + const parts = await run( + Effect.gen(function* () { + const svc = yield* SessionService; + return yield* svc.materializeInput(child, [{ type: 'media', bytes: png }]); + }) + ); + expect(parts).toHaveLength(1); + expect(parts[0]).toMatchObject({ + type: 'media', + mimeType: 'image/png', + bytes: png.byteLength, + }); + const assetDir = assetsDirOf(child.cwd); + expect(readdirSync(assetDir)).toEqual([assetNameFor(png, 'image/png')]); + } finally { + cleanup(dir); + } + }); +}); + +describe('SessionService.resolveAssets', () => { + it('命中资产返回 data URL,未命中的资产直接跳过', async () => { + const dir = newDir(); + try { + const state = await newSession(dir); + const png = mediaBytes('png'); + const parts = await run( + Effect.gen(function* () { + const svc = yield* SessionService; + return yield* svc.materializeInput(state, [{ type: 'media', bytes: png }]); + }) + ); + const asset = (parts[0] as any).asset as string; + + const resolved = await run( + Effect.gen(function* () { + const svc = yield* SessionService; + return yield* svc.resolveAssets(state.cwd, [asset, 'deadbeef'.repeat(4) + '.png']); + }) + ); + + expect(resolved.get(asset)).toBe(`data:image/png;base64,${Buffer.from(png).toString('base64')}`); + expect(resolved.has('deadbeef'.repeat(4) + '.png')).toBe(false); + } finally { + cleanup(dir); + } + }); + + it('同一资产第二次解析走缓存:盘上文件被删掉也仍能解析', async () => { + const dir = newDir(); + try { + // assetCache 挂在 SessionLayer 内部:三次调用必须共用同一层实例 + const { first, second } = await run( + Effect.gen(function* () { + const svc = yield* SessionService; + const state = yield* svc.create( + dir, + { model: 'test-model', activeProfile: 'build', permissionMode: 'askBeforeExec' } + ); + const png = mediaBytes('png'); + const parts = yield* svc.materializeInput(state, [{ type: 'media', bytes: png }]); + const asset = (parts[0] as any).asset as string; + + const first = yield* svc.resolveAssets(state.cwd, [asset]); + unlinkSync(join(assetsDirOf(state.cwd), asset)); + const second = yield* svc.resolveAssets(state.cwd, [asset]); + return { first: first.get(asset), second: second.get(asset) }; + }) + ); + + expect(second).toBe(first); + } finally { + cleanup(dir); + } + }); +}); diff --git a/packages/codingcode/test/session/prompt-estimate.test.ts b/packages/codingcode/test/session/prompt-estimate.test.ts index 398159a..2cd7d10 100644 --- a/packages/codingcode/test/session/prompt-estimate.test.ts +++ b/packages/codingcode/test/session/prompt-estimate.test.ts @@ -13,6 +13,7 @@ import { encodeProjectPath } from '../../src/util/path.js'; import { computePaths } from '../../src/session/paths.js'; import type { SessionStoreState } from '../../src/session/types.js'; import { useTempProjectBase } from '../helpers/project-base.js'; +import { text } from '../helpers/parts.js'; const base = useTempProjectBase(); @@ -41,7 +42,7 @@ function makeFixture( { type: 'user', turnId: 1, - content: 'hello world', + content: text('hello world'), }, { type: 'assistant', @@ -53,7 +54,7 @@ function makeFixture( { type: 'user', turnId: 2, - content: 'do stuff', + content: text('do stuff'), }, { type: 'assistant', diff --git a/packages/codingcode/test/session/record-user-title.test.ts b/packages/codingcode/test/session/record-user-title.test.ts index 89ae8d5..7a6fdc4 100644 --- a/packages/codingcode/test/session/record-user-title.test.ts +++ b/packages/codingcode/test/session/record-user-title.test.ts @@ -9,6 +9,7 @@ import { encodeProjectPath } from '../../src/util/path.js'; import { computePaths } from '../../src/session/paths.js'; import { readSessionMeta } from '../../src/session/file-ops.js'; import { useTempProjectBase } from '../helpers/project-base.js'; +import { text } from '../helpers/parts.js'; const base = useTempProjectBase(); @@ -71,7 +72,7 @@ describe('title persistence — backfilled from the first user message', () => { Effect.gen(function* () { const svc = yield* SessionService; const state = yield* svc.load(dir, created.sessionId); - yield* svc.recordUser(state, 'fix the login bug'); + yield* svc.recordUser(state, text('fix the login bug')); }) ); @@ -99,7 +100,7 @@ describe('title persistence — backfilled from the first user message', () => { Effect.gen(function* () { const svc = yield* SessionService; const state = yield* svc.load(dir, created.sessionId); - yield* svc.recordUser(state, `${'a'.repeat(40)}\nsecond line`); + yield* svc.recordUser(state, text(`${'a'.repeat(40)}\nsecond line`)); }) ); @@ -127,14 +128,14 @@ describe('title persistence — backfilled from the first user message', () => { Effect.gen(function* () { const svc = yield* SessionService; const state = yield* svc.load(dir, created.sessionId); - yield* svc.recordUser(state, 'first message'); + yield* svc.recordUser(state, text('first message')); }) ); await run( Effect.gen(function* () { const svc = yield* SessionService; const state = yield* svc.load(dir, created.sessionId); - yield* svc.recordUser(state, 'second message should not win'); + yield* svc.recordUser(state, text('second message should not win')); }) ); @@ -163,7 +164,7 @@ describe('title persistence — backfilled from the first user message', () => { Effect.gen(function* () { const svc = yield* SessionService; const state = yield* svc.load(dir, created.sessionId); - yield* svc.recordUser(state, 'a totally different message'); + yield* svc.recordUser(state, text('a totally different message')); }) ); @@ -192,7 +193,7 @@ describe('title persistence — backfilled from the first user message', () => { const svc = yield* SessionService; const state = yield* svc.load(dir, created.sessionId); yield* svc.renameSession(state, 'my custom name'); - yield* svc.recordUser(state, 'first message'); + yield* svc.recordUser(state, text('first message')); }) ); diff --git a/packages/codingcode/test/session/store-diff-rebuild.test.ts b/packages/codingcode/test/session/store-diff-rebuild.test.ts index 09c9678..13e2c82 100644 --- a/packages/codingcode/test/session/store-diff-rebuild.test.ts +++ b/packages/codingcode/test/session/store-diff-rebuild.test.ts @@ -1,6 +1,7 @@ import { describe, it, expect } from 'vitest'; import type { SessionEvent } from '../../src/session/types.js'; import { sessionEventsToTurns } from '../../src/session/session.js'; +import { text } from '../helpers/parts.js'; describe('sessionEventsToTurns', () => { it('parses edit_file tool_result without diff (diff is computed on frontend)', () => { @@ -18,7 +19,7 @@ describe('sessionEventsToTurns', () => { { type: 'user', turnId: 1, - content: 'edit file', + content: text('edit file'), }, { type: 'assistant', @@ -70,7 +71,7 @@ describe('sessionEventsToTurns', () => { { type: 'user', turnId: 1, - content: 'write file', + content: text('write file'), }, { type: 'assistant', @@ -120,7 +121,7 @@ describe('sessionEventsToTurns', () => { { type: 'user', turnId: 1, - content: 'run command', + content: text('run command'), }, { type: 'assistant', diff --git a/packages/codingcode/test/session/ui-history-rollback.test.ts b/packages/codingcode/test/session/ui-history-rollback.test.ts index 37ebd8c..4a35e9b 100644 --- a/packages/codingcode/test/session/ui-history-rollback.test.ts +++ b/packages/codingcode/test/session/ui-history-rollback.test.ts @@ -7,6 +7,7 @@ import { readHistory } from '../../src/session/file-ops.js'; import { filterForUI } from '../../src/session/session.js'; import type { SessionEvent } from '../../src/session/types.js'; import { useTempProjectBase } from '../helpers/project-base.js'; +import { text } from '../helpers/parts.js'; const base = useTempProjectBase(); @@ -68,9 +69,9 @@ describe('filterForContext', () => { activeProfile: 'build', permissionMode: 'askBeforeExec', }, - { type: 'user', turnId: 1, content: 'hello' }, + { type: 'user', turnId: 1, content: text('hello') }, { type: 'assistant', turnId: 1, content: 'hi', toolCalls: [] }, - { type: 'user' as const, turnId: 2, content: 'bye' }, + { type: 'user' as const, turnId: 2, content: text('bye') }, { type: 'assistant' as const, turnId: 2, content: 'bye', toolCalls: [] }, { type: 'rollback' as const, throughTurnId: 1, reason: 'test' }, ]; diff --git a/packages/codingcode/test/session/view-assembly.test.ts b/packages/codingcode/test/session/view-assembly.test.ts index f7d9f39..9413195 100644 --- a/packages/codingcode/test/session/view-assembly.test.ts +++ b/packages/codingcode/test/session/view-assembly.test.ts @@ -1,6 +1,12 @@ import { describe, it, expect } from 'vitest'; import { filterForContext, buildContextMessages } from '../../src/context/context.js'; import type { SessionEvent } from '../../src/session/types.js'; +import { text } from '../helpers/parts.js'; + +/** 合并后的 parts 拍平成文本,断言里只关心文本内容。 */ +function flat(parts: Array<{ text?: string }>): string { + return parts.map((p) => p.text ?? '').join(''); +} function toMessages(events: SessionEvent[]) { const { visible, compactedTurnIds } = filterForContext(events); @@ -19,7 +25,7 @@ function makeEvents(extra: SessionEvent[] = []): SessionEvent[] { activeProfile: 'build', permissionMode: 'askBeforeExec', }, - { type: 'user', turnId: 1, content: 'hello' }, + { type: 'user', turnId: 1, content: text('hello') }, { type: 'assistant', turnId: 1, @@ -29,7 +35,7 @@ function makeEvents(extra: SessionEvent[] = []): SessionEvent[] { { type: 'user', turnId: 2, - content: 'run a command', + content: text('run a command'), }, { type: 'assistant', @@ -44,7 +50,7 @@ function makeEvents(extra: SessionEvent[] = []): SessionEvent[] { toolCallId: 'tc1', output: 'output line 1\nline 2', }, - { type: 'user', turnId: 3, content: 'thanks' }, + { type: 'user', turnId: 3, content: text('thanks') }, { type: 'assistant', turnId: 3, @@ -60,14 +66,14 @@ describe('buildContextMessages', () => { const events = makeEvents(); const messages = toMessages(events); expect(messages).toHaveLength(7); - expect(messages[0]).toEqual({ role: 'user', content: 'hello' }); - expect(messages[1]).toEqual({ role: 'assistant', content: 'hi there' }); - expect(messages[2]).toEqual({ role: 'user', content: 'run a command' }); + expect(messages[0]).toEqual({ role: 'user', content: text('hello') }); + expect(messages[1]).toEqual({ role: 'assistant', content: text('hi there') }); + expect(messages[2]).toEqual({ role: 'user', content: text('run a command') }); expect(messages[3]?.role).toBe('assistant'); expect((messages[3] as any).tool_calls).toHaveLength(1); expect(messages[4]?.role).toBe('tool'); - expect(messages[5]).toEqual({ role: 'user', content: 'thanks' }); - expect(messages[6]).toEqual({ role: 'assistant', content: 'welcome' }); + expect(messages[5]).toEqual({ role: 'user', content: text('thanks') }); + expect(messages[6]).toEqual({ role: 'assistant', content: text('welcome') }); }); it('summary events hide replaced events and emit as system message', () => { @@ -85,7 +91,7 @@ describe('buildContextMessages', () => { expect(toolMessages).toHaveLength(0); const summaryMessages = messages.filter((m) => m.role === 'system'); expect(summaryMessages).toHaveLength(1); - expect(summaryMessages[0]?.content).toBe('[compacted]'); + expect(summaryMessages[0]?.content).toEqual(text('[compacted]')); }); it('rollback removes all events from the given turn onwards', () => { @@ -118,7 +124,7 @@ describe('buildContextMessages', () => { { type: 'user', turnId: 1, - content: 'do something', + content: text('do something'), }, { type: 'assistant', @@ -129,7 +135,7 @@ describe('buildContextMessages', () => { ]; const messages = toMessages(events); expect(messages).toHaveLength(1); - expect(messages[0]).toEqual({ role: 'user', content: 'do something' }); + expect(messages[0]).toEqual({ role: 'user', content: text('do something') }); }); it('filters assistant with partially resolved tool_calls and their orphaned tool results', () => { @@ -147,7 +153,7 @@ describe('buildContextMessages', () => { { type: 'user', turnId: 1, - content: 'step 1', + content: text('step 1'), }, { type: 'assistant', @@ -168,7 +174,7 @@ describe('buildContextMessages', () => { { type: 'user', turnId: 2, - content: 'step 2', + content: text('step 2'), }, { type: 'assistant', @@ -179,7 +185,7 @@ describe('buildContextMessages', () => { ]; const messages = toMessages(events); expect(messages.filter((m) => m.role === 'assistant')).toHaveLength(1); - expect((messages.find((m) => m.role === 'assistant') as any).content).toBe('done'); + expect((messages.find((m) => m.role === 'assistant') as any).content).toEqual(text('done')); expect(messages.filter((m) => m.role === 'tool')).toHaveLength(0); }); @@ -198,7 +204,7 @@ describe('buildContextMessages', () => { { type: 'user', turnId: 1, - content: 'do something', + content: text('do something'), }, { type: 'assistant', @@ -220,7 +226,7 @@ describe('buildContextMessages', () => { endTurnId: 1, summaryText: '[compacted]', }, - { type: 'user', turnId: 2, content: 'next' }, + { type: 'user', turnId: 2, content: text('next') }, { type: 'assistant', turnId: 2, @@ -232,10 +238,10 @@ describe('buildContextMessages', () => { const assistantContents = messages .filter((m) => m.role === 'assistant') .map((m) => (m as any).content); - expect(assistantContents).toEqual(['done']); + expect(assistantContents).toEqual([text('done')]); expect(messages.filter((m) => m.role === 'tool')).toHaveLength(0); - expect(messages.filter((m) => m.role === 'system').map((m) => m.content)).toContain( - '[compacted]' + expect(messages.filter((m) => m.role === 'system').map((m) => m.content)).toContainEqual( + text('[compacted]') ); }); @@ -254,7 +260,7 @@ describe('buildContextMessages', () => { { type: 'user', turnId: 1, - content: 'first', + content: text('first'), }, { type: 'assistant', @@ -275,14 +281,14 @@ describe('buildContextMessages', () => { { type: 'user', turnId: 2, - content: 'second', + content: text('second'), }, ]; const messages = toMessages(events); const userMsgs = messages.filter((m) => m.role === 'user'); expect(userMsgs).toHaveLength(1); - expect((userMsgs[0] as any).content).toContain('first'); - expect((userMsgs[0] as any).content).toContain('second'); + expect(flat((userMsgs[0] as any).content)).toContain('first'); + expect(flat((userMsgs[0] as any).content)).toContain('second'); }); it('does not merge adjacent tool messages', () => { @@ -300,7 +306,7 @@ describe('buildContextMessages', () => { { type: 'user', turnId: 1, - content: 'do something', + content: text('do something'), }, { type: 'assistant', @@ -342,7 +348,7 @@ describe('buildContextMessages', () => { activeProfile: 'build', permissionMode: 'askBeforeExec', }, - { type: 'user', turnId: 1, content: 'q1' }, + { type: 'user', turnId: 1, content: text('q1') }, { type: 'assistant', turnId: 1, @@ -359,8 +365,8 @@ describe('buildContextMessages', () => { const messages = toMessages(events); const assistantMsgs = messages.filter((m) => m.role === 'assistant'); expect(assistantMsgs).toHaveLength(1); - expect((assistantMsgs[0] as any).content).toContain('reply1'); - expect((assistantMsgs[0] as any).content).toContain('reply2'); + expect(flat((assistantMsgs[0] as any).content)).toContain('reply1'); + expect(flat((assistantMsgs[0] as any).content)).toContain('reply2'); }); it('handles empty events list', () => { diff --git a/packages/codingcode/test/subagent/runner-service.test.ts b/packages/codingcode/test/subagent/runner-service.test.ts index b7e5717..c1aa44f 100644 --- a/packages/codingcode/test/subagent/runner-service.test.ts +++ b/packages/codingcode/test/subagent/runner-service.test.ts @@ -1,6 +1,8 @@ import { expect, it, describe } from 'vitest'; import { Effect, Layer } from 'effect'; import { SubagentRunnerService } from '../../src/subagent/port.js'; +import type { IncomingPart } from '../../src/llm/types.js'; +import { incomingText } from '../helpers/parts.js'; const SAMPLE_FRAME = { family: 'event', @@ -13,7 +15,7 @@ describe('SubagentRunnerService', () => { }); it('should allow creating a Layer with a custom runSubagent implementation', async () => { - const mockRunSubagent = (_input: string, _opts: { cwd: string }) => + const mockRunSubagent = (_input: IncomingPart[], _opts: { cwd: string }) => Effect.succeed({ stream: (async function* () { yield SAMPLE_FRAME; @@ -37,7 +39,7 @@ describe('SubagentRunnerService', () => { it('should allow runSubagent to be called and produce events', async () => { const events: any[] = []; - const mockRunSubagent = (_input: string, _opts: { cwd: string }) => + const mockRunSubagent = (_input: IncomingPart[], _opts: { cwd: string }) => Effect.succeed({ stream: (async function* () { yield SAMPLE_FRAME; @@ -51,7 +53,7 @@ describe('SubagentRunnerService', () => { ( Effect.gen(function* () { const runner = yield* SubagentRunnerService; - const { stream, sessionId } = yield* runner.runSubagent('go', { + const { stream, sessionId } = yield* runner.runSubagent(incomingText('go'), { cwd: '/test', model: 'm', }); diff --git a/packages/codingcode/test/subagent/runner-wiring.test.ts b/packages/codingcode/test/subagent/runner-wiring.test.ts index 34108e8..6c788ff 100644 --- a/packages/codingcode/test/subagent/runner-wiring.test.ts +++ b/packages/codingcode/test/subagent/runner-wiring.test.ts @@ -23,6 +23,8 @@ import { SessionLayer } from '../../src/session/session.js'; import { SessionService } from '../../src/session/port.js'; import { HookService } from '../../src/hooks/port.js'; import { McpService } from '../../src/mcp/port.js'; +import { textPart } from '../../src/llm/types.js'; +import { incomingText } from '../helpers/parts.js'; import { TodoService } from '../../src/todo/port.js'; import { readHistory } from '../../src/session/file-ops.js'; import { encodeProjectPath, normalizePath } from '../../src/util/path.js'; @@ -65,12 +67,14 @@ function readMessages(transcriptPath: string): Message[] { return readHistory(transcriptPath).flatMap((e) => { if (e.type === 'user') return [{ role: 'user', content: e.content }] as Message[]; if (e.type === 'assistant') - return [{ role: 'assistant', content: e.content, tool_calls: e.toolCalls }] as Message[]; + return [ + { role: 'assistant', content: [textPart(e.content)], tool_calls: e.toolCalls }, + ] as Message[]; if (e.type === 'tool_result') return [ { role: 'tool', - content: e.output ?? '', + content: [textPart(e.output ?? '')], tool_call_id: e.toolCallId, tool_name: e.toolName, } as Message, @@ -207,7 +211,7 @@ describe('subagent runner wiring (child session mounts under the parent)', () => activeProfile: 'build', permissionMode: 'askBeforeExec', }); - const { stream, sessionId } = yield* runner.runSubagent('do the thing', { + const { stream, sessionId } = yield* runner.runSubagent(incomingText('do the thing'), { cwd, activeProfile: 'build', parentSessionId: parent.sessionId, diff --git a/packages/codingcode/test/types/type-collapse.test.ts b/packages/codingcode/test/types/type-collapse.test.ts index 2ea1afc..54bdb59 100644 --- a/packages/codingcode/test/types/type-collapse.test.ts +++ b/packages/codingcode/test/types/type-collapse.test.ts @@ -12,7 +12,7 @@ describe('类型收口', () => { it('UITurnItem 覆盖 session 产出的全部变体', () => { const items: UITurnItem[] = [ - { id: 'a', type: 'message', role: 'user', content: 'hi' }, + { id: 'a', type: 'message', role: 'user', parts: [{ type: 'text', text: 'hi' }] }, { id: 'b', type: 'tool_call', name: 'read_file', args: {}, status: 'approved' }, { id: 'c', type: 'tool_result', callId: 'b', name: 'read_file', output: 'ok' }, { id: 'd', type: 'summary', content: 's', startTurnId: 1, endTurnId: 2 }, diff --git a/packages/desktop/shared/parts.ts b/packages/desktop/shared/parts.ts new file mode 100644 index 0000000..3156009 --- /dev/null +++ b/packages/desktop/shared/parts.ts @@ -0,0 +1,119 @@ +import type { InputPart } from '@codingcode/sdk'; + +/** 媒体按 mime 主类型分档:与 `packages/codingcode/src/llm/types.ts` 同一口径。 */ +export type MediaKind = 'image' | 'audio' | 'file'; + +export function mediaKindOf(mimeType: string): MediaKind { + const mime = (mimeType ?? '').toLowerCase(); + if (mime.startsWith('image/')) return 'image'; + if (mime.startsWith('audio/')) return 'audio'; + return 'file'; +} + +/** 服务端回吐的落盘形态:只有资产名与元数据。 */ +export interface StoredMediaPart { + type: 'media'; + asset: string; + mimeType: string; + bytes: number; + filename?: string; + width?: number; + height?: number; + durationSec?: number; +} + +/** 本地乐观条目:还没拿到服务端资产名,先带 data URL。 */ +export interface LocalMediaPart { + type: 'media'; + dataUrl: string; + mimeType: string; + filename?: string; +} + +export type ContentPart = + | { type: 'text'; text: string } + | StoredMediaPart + | LocalMediaPart; + +export function isStoredMedia(p: ContentPart): p is StoredMediaPart { + return p.type === 'media' && 'asset' in p; +} + +export function isLocalMedia(p: ContentPart): p is LocalMediaPart { + return p.type === 'media' && 'dataUrl' in p; +} + +/** 纯文本块:程序化发起的一轮输入(如 plan 决策)用得上。 */ +export function textPart(text: string): ContentPart { + return { type: 'text', text }; +} + +const MEDIA_MARKER: Record = { + image: '[image]', + audio: '[audio]', + file: '[file]', +}; + +/** 文本投影:标题、回滚回填、assistant 合并共用的口径。 */ +export function textOf(parts: readonly ContentPart[]): string { + return parts + .map((p) => (p.type === 'text' ? p.text : MEDIA_MARKER[mediaKindOf(p.mimeType)])) + .join('\n'); +} + +/** 待发送的 parts → 线上部件。落盘形态(asset)无法再次上传,直接跳过。 */ +export function toWireParts(parts: readonly ContentPart[]): InputPart[] { + const wire: InputPart[] = []; + for (const p of parts) { + if (p.type === 'text') { + wire.push({ type: 'text', text: p.text }); + continue; + } + if (isLocalMedia(p)) { + wire.push({ + type: 'media', + dataUrl: p.dataUrl, + ...(p.filename ? { filename: p.filename } : {}), + }); + } + } + return wire; +} + +/** 资产地址:媒体不经过 SSE,直接指向服务端的资产路由。 */ +export function assetUrl(asset: string, cwd: string, apiBase: string): string { + return `${apiBase}/api/assets/${asset}?cwd=${encodeURIComponent(cwd)}`; +} + +/** 渲染用的媒体地址:落盘条目走后端路由,本地条目直接用 data URL。 */ +export function srcOf(p: ContentPart, cwd: string, apiBase: string): string { + return isStoredMedia(p) ? assetUrl(p.asset, cwd, apiBase) : (p as LocalMediaPart).dataUrl; +} + +/** 前端白名单提示,与服务端 `sniffMediaMime` 同表。最终准入以服务端嗅探结果为准。 */ +export const ACCEPTED_MIME: readonly string[] = [ + 'image/png', + 'image/jpeg', + 'image/gif', + 'image/webp', + 'audio/wav', + 'audio/mpeg', + 'application/pdf', +]; + +/** 当前模型可达的格式:vision 覆盖图片与 PDF,audio 覆盖音频输入。 */ +export function reachableMimes(caps?: { vision: boolean; audio: boolean }): string[] { + if (!caps) return [...ACCEPTED_MIME]; + return ACCEPTED_MIME.filter((m) => + mediaKindOf(m) === 'audio' ? caps.audio : caps.vision + ); +} + +/** 模型拒绝某类附件时的提示文案;都能收时为 null。 */ +export function attachmentHint(caps?: { vision: boolean; audio: boolean }): string | null { + if (!caps) return null; + if (!caps.vision && !caps.audio) return '当前模型不支持图片、PDF 与音频输入'; + if (!caps.vision) return '当前模型不支持图片与 PDF 输入'; + if (!caps.audio) return '当前模型不支持音频输入'; + return null; +} diff --git a/packages/desktop/shared/types.ts b/packages/desktop/shared/types.ts index 7f82817..52d8194 100644 --- a/packages/desktop/shared/types.ts +++ b/packages/desktop/shared/types.ts @@ -1,3 +1,7 @@ +import type { ContentPart } from './parts'; + +export type { ContentPart } from './parts'; + export type ItemType = | 'message' | 'reasoning' @@ -7,7 +11,7 @@ export type ItemType = | 'subagent'; export type Item = - | { id: string; type: 'message'; role: 'user' | 'assistant'; content: string; partial?: boolean } + | { id: string; type: 'message'; role: 'user' | 'assistant'; parts: ContentPart[]; partial?: boolean } | { id: string; type: 'reasoning'; content: string; isVisible: boolean } | { id: string; diff --git a/packages/desktop/src/agent/AgentWorkspace.tsx b/packages/desktop/src/agent/AgentWorkspace.tsx index 336a340..b3bfc65 100644 --- a/packages/desktop/src/agent/AgentWorkspace.tsx +++ b/packages/desktop/src/agent/AgentWorkspace.tsx @@ -1,6 +1,6 @@ import { useState, useRef, useCallback, useLayoutEffect, useEffect } from 'react'; import { createPortal } from 'react-dom'; -import { Send, Square, ShieldCheck, Shield, FileText } from 'lucide-react'; +import { Send, Square, ShieldCheck, Shield, FileText, Paperclip, X } from 'lucide-react'; import { useAgentStore } from '../stores/agent.store'; import { useWorkspaceStore } from '../stores/workspace.store'; import { @@ -16,7 +16,23 @@ import TodoPanel from './TodoPanel'; import ApprovalPanel from './ApprovalPanel'; import ProfileIndicator from './ProfileIndicator'; import PlanPanel from '../shared/PlanPanel'; -import type { PermissionMode } from '@codingcode/sdk'; +import MediaView from '../shared/MediaView'; +import { reachableMimes, attachmentHint, type ContentPart } from '@shared/parts'; +import { MAX_MEDIA_BYTES, type PermissionMode } from '@codingcode/sdk'; + +/** 附件条上限:单文件字节上限用 sdk 的 `MAX_MEDIA_BYTES`,与服务端闸门同值,前端只做即时提示。 */ +const MAX_ATTACHMENTS = 8; + +interface Attachment { + id: string; + dataUrl: string; + mimeType: string; + filename: string; +} + +function newId(): string { + return crypto.randomUUID ? crypto.randomUUID() : Math.random().toString(36).slice(2, 11); +} const MODE_LABELS: Record = { askBeforeExec: '执行前询问', @@ -235,13 +251,22 @@ function InputBox({ onOpenPlanPanel, }: { centered?: boolean; - sendMessage: (content: string, cwd?: string, skills?: SkillRef[]) => Promise; + sendMessage: (parts: ContentPart[], cwd?: string, skills?: SkillRef[]) => Promise; abort: () => void; onOpenPlanPanel?: () => void; }) { const [text, setText] = useState(''); + const [attachments, setAttachments] = useState([]); + const [notice, setNotice] = useState(null); + const [dragging, setDragging] = useState(false); + const fileInputRef = useRef(null); const textareaRef = useRef(null); const currentThreadId = useAgentStore((s) => s.currentThreadId); + /** 当前模型的输入侧能力位:决定 accept 与附件按钮是否可用 */ + const caps = useAgentStore((s) => s.models.find((m) => m.id === s.model)?.capabilities); + const accept = reachableMimes(caps).join(','); + const capsHint = attachmentHint(caps); + const attachDisabled = !!(caps && !caps.vision && !caps.audio); const isStreaming = useAgentStore((s) => { const tid = s.currentThreadId; if (!tid) return false; @@ -351,9 +376,79 @@ function InputBox({ textareaRef.current?.focus(); }; + /** + * 粘贴、拖入、选文件共用同一条入口:先按当前模型可达格式筛一遍,被拒的 + * 就地提示、不入附件条。前端判定只是提示,最终准入在服务端(能力位 + 嗅探)。 + */ + const addFiles = useCallback( + (files: FileList | File[]) => { + const reachable = new Set(reachableMimes(caps)); + const room = MAX_ATTACHMENTS - attachments.length; + const rejected: string[] = []; + let overflow = 0; + for (const f of Array.from(files)) { + if (!reachable.has(f.type) || f.size > MAX_MEDIA_BYTES) { + rejected.push(f.name || f.type || '未知文件'); + continue; + } + if (overflow >= room) { + overflow += 1; + continue; + } + overflow += 1; + const reader = new FileReader(); + reader.onload = () => + setAttachments((prev) => + prev.length >= MAX_ATTACHMENTS + ? prev + : [ + ...prev, + { + id: newId(), + dataUrl: String(reader.result), + mimeType: f.type, + filename: f.name, + }, + ] + ); + reader.readAsDataURL(f); + } + if (rejected.length > 0) { + setNotice(capsHint ?? `已忽略不支持的附件:${rejected.join('、')}`); + } else if (overflow > 0) { + setNotice(`一轮最多 ${MAX_ATTACHMENTS} 个附件`); + } else { + setNotice(null); + } + }, + [caps, capsHint, attachments.length] + ); + + const onPaste = useCallback( + (e: React.ClipboardEvent) => { + const files = Array.from(e.clipboardData.items) + .filter((it) => it.kind === 'file') + .map((it) => it.getAsFile()) + .filter((f): f is File => !!f); + if (files.length === 0) return; + e.preventDefault(); + addFiles(files); + }, + [addFiles] + ); + + const onDrop = useCallback( + (e: React.DragEvent) => { + e.preventDefault(); + setDragging(false); + if (e.dataTransfer.files.length > 0) addFiles(e.dataTransfer.files); + }, + [addFiles] + ); + const handleSend = useCallback(() => { const trimmed = text.trim(); - if (!trimmed || isStreaming) return; + if ((!trimmed && attachments.length === 0) || isStreaming) return; const skills: SkillRef[] = []; const seen = new Set(); for (const m of trimmed.matchAll(/@([a-zA-Z0-9-]+)/g)) { @@ -365,17 +460,75 @@ function InputBox({ const path = pickedPaths[name] ?? (hits.length === 1 ? hits[0]!.skillPath : undefined); if (path) skills.push({ name, path }); } + const parts: ContentPart[] = attachments.map((a) => ({ + type: 'media', + dataUrl: a.dataUrl, + mimeType: a.mimeType, + filename: a.filename, + })); + parts.push({ type: 'text', text: trimmed }); setText(''); + setAttachments([]); + setNotice(null); setPickedPaths({}); setSkillMenu(null); - sendMessage(trimmed, workspace.rootPath || undefined, skills); - }, [text, isStreaming, sendMessage, workspace.rootPath, pickedPaths, skillOptions]); + sendMessage(parts, workspace.rootPath || undefined, skills); + }, [ + text, + attachments, + isStreaming, + sendMessage, + workspace.rootPath, + pickedPaths, + skillOptions, + ]); return (
-
- {/* Row 1: textarea + send button side by side */} +
{ + e.preventDefault(); + setDragging(true); + }} + onDragLeave={() => setDragging(false)} + onDrop={onDrop} + className={`rounded-2xl border bg-[var(--bg-card)] transition-colors shadow-xl overflow-hidden ${ + dragging + ? 'border-[var(--accent-primary)]' + : 'border-[var(--border-card)] hover:border-[var(--border-hover)] focus-within:border-[var(--accent-primary)]' + }`} + > + {/* Row 0: 附件条 + 提示 */} + {(attachments.length > 0 || notice) && ( +
+ {attachments.map((a) => ( +
+ + +
+ ))} + {notice && {notice}} +
+ )} + {/* Row 1: textarea + attach + send button side by side */}