diff --git a/.env.local.example b/.env.local.example index 097692f..525e9c9 100644 --- a/.env.local.example +++ b/.env.local.example @@ -17,8 +17,8 @@ KNOWHERE_BASE_URL=https://api.knowhereto.ai # Vercel AI Gateway key; AI SDK picks it up automatically AI_GATEWAY_API_KEY=vck_your_key_here -# Optional override — defaults to deepseek/deepseek-v4-flash -# CHAT_MODEL=deepseek/deepseek-v4-flash +# Optional override — defaults to DeepSeek V4.1 Flash (native multimodal) +# CHAT_MODEL=deepseek/deepseek-v4.1-flash # --- Auth (server-side only) --- # diff --git a/src/agent-harness/answer-context.test.ts b/src/agent-harness/answer-context.test.ts index a21e92c..76cf05d 100644 --- a/src/agent-harness/answer-context.test.ts +++ b/src/agent-harness/answer-context.test.ts @@ -1,13 +1,23 @@ import { describe, expect, it, vi } from "vitest" +import { logger } from "@/lib/logger" + import { composeAnswerContext } from "./answer-context" import type { EvidenceLedgerSnapshot } from "./types" +const revenuePng = new Uint8Array([1, 2, 3]) +const marginPng = new Uint8Array([4, 5, 6]) + describe("composeAnswerContext", () => { - it("composes retained chunks, full table HTML, images, memory, and the original question", async () => { + it("composes retained chunks, full table HTML, images at placeholders, memory, and the original question", async () => { const readTableHtml = vi.fn().mockResolvedValue( "
Q424.9
", ) + const readImage = vi.fn(async (assetUrl: string) => + assetUrl.includes("revenue.png") + ? { body: revenuePng, mediaType: "image/png" } + : { body: marginPng, mediaType: "image/png" }, + ) const message = await composeAnswerContext({ ledger: makeLedger(), memoryItems: [ @@ -20,11 +30,18 @@ describe("composeAnswerContext", () => { ], userText: "Q4 的收入是多少？", readTableHtml, + readImage, }) expect(readTableHtml).toHaveBeenCalledWith( "https://assets.example/tables/revenue.html?signature=valid", ) + expect(readImage).toHaveBeenCalledWith( + "https://assets.example/images/revenue.png?signature=valid", + ) + expect(readImage).toHaveBeenCalledWith( + "https://assets.example/images/margin.png?signature=valid", + ) expect(message).toEqual({ role: "user", content: [ @@ -38,24 +55,29 @@ describe("composeAnswerContext", () => { "Revenue increased in Q4.", "
Q424.9
", "", - '[image ref="asset:r1:result:1:image_chunk_1"]', + "", ].join("\n"), }, { type: "image", - image: new URL("https://assets.example/images/revenue.png?signature=valid"), + image: revenuePng, + mediaType: "image/png", }, { type: "text", - text: '[image ref="asset:r1:result:1:image_chunk_2"]', + text: "\n\n\n", }, { type: "image", - image: new URL("https://assets.example/images/margin.png?signature=valid"), + image: marginPng, + mediaType: "image/png", }, { type: "text", text: [ + "", + "", + "", "## Fluid Memory", "", '[memory ref="mem:1" kind="stance"]', @@ -69,6 +91,90 @@ describe("composeAnswerContext", () => { }) }) + it("drops image placeholders and warns when the image URL cannot be read", async () => { + const warn = vi.spyOn(logger, "warn").mockImplementation(() => {}) + const message = await composeAnswerContext({ + ledger: makeLedger({ + content: + "Before [images/revenue.png] after [images/margin.png] end.", + assets: makeLedger().assets.filter((asset) => asset.type === "image"), + }), + memoryItems: [], + userText: "图在哪？", + readImage: vi.fn().mockRejectedValue(new Error("Unable to read image: 403")), + }) + + const parts = userContentParts(message) + const text = parts + .filter((part) => part.type === "text") + .map((part) => part.text) + .join("") + expect(text).toContain("Before after end.") + expect(text).not.toContain("[images/revenue.png]") + expect(text).not.toContain("[images/margin.png]") + expect(parts.some((part) => part.type === "image")).toBe(false) + expect(warn).toHaveBeenCalledWith( + "chat: skipped unreachable evidence asset", + expect.objectContaining({ + type: "image", + reason: "Unable to read image: 403", + }), + ) + warn.mockRestore() + }) + + it("drops table placeholders and warns when the table URL cannot be read", async () => { + const warn = vi.spyOn(logger, "warn").mockImplementation(() => {}) + const message = await composeAnswerContext({ + ledger: makeLedger({ + content: "Revenue [Table: tables/revenue.html] rose.", + assets: makeLedger().assets.filter((asset) => asset.type === "table"), + }), + memoryItems: [], + userText: "收入多少？", + readTableHtml: vi.fn().mockRejectedValue(new Error("Unable to read table HTML: 404")), + }) + + const text = userContentParts(message) + .filter((part) => part.type === "text") + .map((part) => part.text) + .join("") + expect(text).toContain("Revenue rose.") + expect(text).not.toContain("[Table: tables/revenue.html]") + expect(text).not.toContain("") + expect(warn).toHaveBeenCalledWith( + "chat: skipped unreachable evidence asset", + expect.objectContaining({ + type: "table", + reason: "Unable to read table HTML: 404", + }), + ) + warn.mockRestore() + }) + + it("throws when a connected image placeholder is missing", async () => { + await expect( + composeAnswerContext({ + ledger: { + ...makeLedger(), + chunks: [ + { + ...makeLedger().chunks[0]!, + content: "Revenue increased in Q4.\n[Table: tables/revenue.html]", + }, + ], + }, + memoryItems: [], + userText: "Q4 的收入是多少？", + readTableHtml: vi.fn().mockResolvedValue("
"), + readImage: vi.fn().mockResolvedValue({ + body: revenuePng, + mediaType: "image/png", + }), + }), + ).rejects.toThrow("Image placeholder") + }) + it("omits knowledge and memory sections when neither search ran", async () => { const message = await composeAnswerContext({ ledger: { ...makeLedger(), retrievalCount: 0, retainedPicks: [] }, @@ -132,14 +238,31 @@ describe("composeAnswerContext", () => { }) }) -function makeLedger(): EvidenceLedgerSnapshot { +function userContentParts(message: { readonly content: unknown }): readonly { + readonly type: string + readonly text?: string +}[] { + if (!Array.isArray(message.content)) { + throw new Error("expected composed message content to be an array") + } + return message.content +} + +function makeLedger( + overrides: { + readonly content?: string + readonly assets?: EvidenceLedgerSnapshot["assets"] + } = {}, +): EvidenceLedgerSnapshot { return { retrievalCount: 1, chunks: [ { ref: "r1:result:1", kind: "result", - content: "Revenue increased in Q4.\n[Table: tables/revenue.html]", + content: + overrides.content ?? + "Revenue increased in Q4.\n[Table: tables/revenue.html]\n[images/revenue.png]\n[images/margin.png]", contentPreview: "Revenue increased in Q4. [Table: tables/revenue.html]", chunkType: "text", score: 0.9, @@ -150,7 +273,7 @@ function makeLedger(): EvidenceLedgerSnapshot { }, }, ], - assets: [ + assets: overrides.assets ?? [ { ref: "asset:r1:result:1:table_chunk", chunkRef: "r1:result:1", diff --git a/src/agent-harness/answer-context.ts b/src/agent-harness/answer-context.ts index 60fea5c..d99f9d8 100644 --- a/src/agent-harness/answer-context.ts +++ b/src/agent-harness/answer-context.ts @@ -1,5 +1,8 @@ import type { ModelMessage } from "ai" +import { summarizeUnknownError } from "@/lib/format-log-value" +import { logger } from "@/lib/logger" + import type { EvidenceAsset, EvidenceChunk, @@ -8,10 +11,19 @@ import type { } from "./types" export type ReadTableHtml = (assetUrl: string) => Promise +export type ReadImage = (assetUrl: string) => Promise<{ + readonly body: Uint8Array + readonly mediaType: string +}> type UserContentParts = Exclude< Extract["content"], string > +type ImagePart = { + readonly type: "image" + readonly image: Uint8Array + readonly mediaType: string +} export async function composeAnswerContext(input: { readonly ledger: EvidenceLedgerSnapshot @@ -19,6 +31,7 @@ export async function composeAnswerContext(input: { readonly memorySearchAttempted?: boolean readonly userText: string readonly readTableHtml?: ReadTableHtml + readonly readImage?: ReadImage }): Promise { const content: UserContentParts = [] const retainedPicks = new Set(input.ledger.retainedPicks) @@ -49,27 +62,13 @@ export async function composeAnswerContext(input: { }) } - appendText( + await appendChunkEvidence({ content, - [ - formatChunkLabel(pick, chunk), - formatTableArtifactRefs(assets), - chunkText.trim(), - ] - .filter((part) => part.length > 0) - .join("\n"), - ) - - for (const image of assets.filter((asset) => asset.type === "image")) { - if (!image.assetUrl) { - throw new Error(`Retained image asset ${image.ref} has no signed URL.`) - } - appendText(content, `[image ref="${image.ref}"]`) - content.push({ - type: "image", - image: new URL(image.assetUrl), - }) - } + pick, + chunk: { ...chunk, content: chunkText }, + assets, + readImage: input.readImage, + }) } } @@ -93,6 +92,120 @@ export async function composeAnswerContext(input: { return { role: "user", content } } +async function appendChunkEvidence(input: { + readonly content: UserContentParts + readonly pick: number + readonly chunk: EvidenceChunk + readonly assets: readonly EvidenceAsset[] + readonly readImage?: ReadImage +}): Promise { + const images = input.assets.filter((asset) => asset.type === "image") + const prefix = [ + formatChunkLabel(input.pick, input.chunk), + formatTableArtifactRefs(input.assets), + ] + .filter((part) => part.length > 0) + .join("\n") + + if (images.length === 0) { + appendText( + input.content, + [prefix, input.chunk.content.trim()].filter((part) => part.length > 0).join("\n"), + ) + return + } + + if (prefix) appendRawText(input.content, `\n\n${prefix}\n`) + + for (const segment of splitChunkTextAtImages(input.chunk, images)) { + if (segment.type === "text") { + appendRawText(input.content, segment.text) + continue + } + + const image = await tryReadImage(segment.asset, input.readImage) + if (!image) continue + + appendRawText(input.content, "\n") + input.content.push(toImagePart(image)) + appendRawText(input.content, "\n") + } +} + +function splitChunkTextAtImages( + chunk: EvidenceChunk, + images: readonly EvidenceAsset[], +): Array< + | { readonly type: "text"; readonly text: string } + | { readonly type: "image"; readonly asset: EvidenceAsset } +> { + if (!chunk.content.trim()) { + return images.map((asset) => ({ type: "image" as const, asset })) + } + + let remaining = chunk.content + const unused = [...images] + const segments: Array< + | { readonly type: "text"; readonly text: string } + | { readonly type: "image"; readonly asset: EvidenceAsset } + > = [] + + while (unused.length > 0) { + const match = findEarliestImagePlaceholder(remaining, chunk, unused) + if (!match) { + throw new Error( + `Image placeholder for ${unused[0]?.ref} was not found in its chunk content.`, + ) + } + if (match.before.length > 0) { + segments.push({ type: "text", text: match.before }) + } + segments.push({ type: "image", asset: match.asset }) + remaining = match.after + unused.splice(unused.indexOf(match.asset), 1) + } + + if (remaining.length > 0) { + segments.push({ type: "text", text: remaining }) + } + return segments +} + +function findEarliestImagePlaceholder( + text: string, + chunk: EvidenceChunk, + images: readonly EvidenceAsset[], +): { + readonly asset: EvidenceAsset + readonly before: string + readonly after: string +} | null { + let best: { + readonly asset: EvidenceAsset + readonly index: number + readonly length: number + } | null = null + + for (const asset of images) { + const match = findFirstPlaceholder(text, chunk, asset, "image") + if (!match) continue + if ( + !best || + match.index < best.index || + (match.index === best.index && match.length > best.length) + ) { + best = { asset, ...match } + } + } + + if (!best) return null + return { + asset: best.asset, + before: text.slice(0, best.index), + after: text.slice(best.index + best.length), + } +} + function groupAssetsByChunkRef( assets: readonly EvidenceAsset[], ): Map { @@ -126,6 +239,27 @@ function appendText( content.push({ type: "text", text: trimmed }) } +function appendRawText(content: UserContentParts, text: string): void { + if (text.length === 0) return + const previous = content.at(-1) + if (previous?.type === "text") { + previous.text += text + return + } + content.push({ type: "text", text }) +} + +function toImagePart(image: { + readonly body: Uint8Array + readonly mediaType: string +}): ImagePart { + return { + type: "image", + image: image.body, + mediaType: image.mediaType.split(";")[0]?.trim() || "application/octet-stream", + } +} + function formatChunkLabel(pick: number, chunk: EvidenceChunk): string { const location = [chunk.source.sourceFileName, chunk.source.sectionPath] .map((part) => part?.trim()) @@ -148,31 +282,16 @@ async function replaceTableWithHtml(input: { readonly asset: EvidenceAsset readonly readTableHtml?: ReadTableHtml }): Promise { - if (!input.asset.assetUrl) { - throw new Error(`Retained table asset ${input.asset.ref} has no signed URL.`) - } - if (!input.readTableHtml) { - throw new Error("No table HTML reader was provided for retained table evidence.") - } - - const html = await input.readTableHtml(input.asset.assetUrl) + const html = await tryReadTableHtml(input.asset, input.readTableHtml) const content = input.chunk.content - if (!content.trim()) return html + if (!content.trim()) return html ?? "" - const placeholders = getTablePlaceholderCandidates(input.chunk, input.asset) - const placeholderPattern = placeholders - .map(escapeRegExp) - .sort((left, right) => right.length - left.length) - .map((placeholder) => - `\\[(?:Table\\s*:\\s*)?${placeholder}\\]|${placeholder}`, - ) - .join("|") - const replaced = content.replace( - new RegExp(placeholderPattern, "gi"), - () => html, + const pattern = placeholderPattern( + getAssetPlaceholderCandidates(input.chunk, input.asset), + "table", ) - - if (replaced === content) { + const replaced = content.replace(new RegExp(pattern, "gi"), () => html ?? "") + if (html !== null && replaced === content) { throw new Error( `Table placeholder for ${input.asset.ref} was not found in its chunk content.`, ) @@ -180,7 +299,68 @@ async function replaceTableWithHtml(input: { return replaced } -function getTablePlaceholderCandidates( +async function tryReadTableHtml( + asset: EvidenceAsset, + readTableHtml?: ReadTableHtml, +): Promise { + if (!asset.assetUrl) { + warnUnreachableAsset(asset, "missing signed URL") + return null + } + if (!readTableHtml) { + throw new Error("No table HTML reader was provided for retained table evidence.") + } + try { + return await readTableHtml(asset.assetUrl) + } catch (error) { + warnUnreachableAsset(asset, summarizeUnknownError(error)) + return null + } +} + +async function tryReadImage( + asset: EvidenceAsset, + readImage?: ReadImage, +): Promise<{ readonly body: Uint8Array; readonly mediaType: string } | null> { + if (!asset.assetUrl) { + warnUnreachableAsset(asset, "missing signed URL") + return null + } + if (!readImage) { + throw new Error("No image reader was provided for retained image evidence.") + } + try { + return await readImage(asset.assetUrl) + } catch (error) { + warnUnreachableAsset(asset, summarizeUnknownError(error)) + return null + } +} + +function warnUnreachableAsset(asset: EvidenceAsset, reason: string): void { + logger.warn("chat: skipped unreachable evidence asset", { + type: asset.type, + ref: asset.ref, + assetUrl: asset.assetUrl ?? null, + sourcePath: asset.sourcePath ?? null, + reason, + }) +} + +function findFirstPlaceholder( + text: string, + chunk: EvidenceChunk, + asset: EvidenceAsset, + kind: "table" | "image", +): { readonly index: number; readonly length: number } | null { + const pattern = placeholderPattern(getAssetPlaceholderCandidates(chunk, asset), kind) + if (!pattern) return null + const match = new RegExp(pattern, "gi").exec(text) + if (!match) return null + return { index: match.index, length: match[0].length } +} + +function getAssetPlaceholderCandidates( chunk: EvidenceChunk, asset: EvidenceAsset, ): string[] { @@ -199,6 +379,20 @@ function getTablePlaceholderCandidates( return unique } +function placeholderPattern( + candidates: readonly string[], + kind: "table" | "image", +): string { + const label = kind === "table" ? "Table" : "Image" + return candidates + .map(escapeRegExp) + .sort((left, right) => right.length - left.length) + .map((placeholder) => + `\\[(?:${label}\\s*:\\s*)?${placeholder}\\]|${placeholder}`, + ) + .join("|") +} + function escapeRegExp(value: string): string { return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&") } diff --git a/src/agent-harness/runtime.test.ts b/src/agent-harness/runtime.test.ts index 7d4c714..60f3d4a 100644 --- a/src/agent-harness/runtime.test.ts +++ b/src/agent-harness/runtime.test.ts @@ -117,6 +117,11 @@ describe("agent harness runtime", () => { const readTableHtml = vi .fn() .mockResolvedValue("
comparison
") + const comparisonJpg = new Uint8Array([9, 8, 7]) + const readImage = vi.fn().mockResolvedValue({ + body: comparisonJpg, + mediaType: "image/jpeg", + }) const result = await runAgentHarness({ model, @@ -127,6 +132,7 @@ describe("agent harness runtime", () => { memoryTools: makeMemoryTools(), resolveConnectedAssets, readTableHtml, + readImage, }) expect(model.doGenerateCalls).toHaveLength(5) @@ -137,6 +143,7 @@ describe("agent harness runtime", () => { expect(readTableHtml).toHaveBeenCalledWith( "https://assets.example/table_chunk", ) + expect(readImage).toHaveBeenCalledWith("https://assets.example/image_chunk") expect(JSON.stringify(model.doGenerateCalls[4]?.prompt)).toContain( "
comparison
", ) @@ -1208,7 +1215,7 @@ function makeConnectedRetrievalResponse(): RetrievalQueryResponse { results: [ { chunkId: "text_chunk", - content: "Comparison [tables/comparison.html]", + content: "Comparison [tables/comparison.html] [images/comparison.jpg]", chunkType: "text", score: 0.9, metadata: { diff --git a/src/agent-harness/runtime.ts b/src/agent-harness/runtime.ts index 4e69c07..e370ae9 100644 --- a/src/agent-harness/runtime.ts +++ b/src/agent-harness/runtime.ts @@ -9,7 +9,11 @@ import { z } from "zod" import { logger } from "@/lib/logger" -import { composeAnswerContext, type ReadTableHtml } from "./answer-context" +import { + composeAnswerContext, + type ReadImage, + type ReadTableHtml, +} from "./answer-context" import { createEvidenceLedger } from "./ledger" import { knowhereToolText } from "./knowhere-text" import { memoryToolText } from "./memory-text" @@ -49,6 +53,7 @@ export type RunAgentHarnessInput = { readonly memoryTools: MemoryToolRuntime readonly resolveConnectedAssets?: ResolveConnectedAssets readonly readTableHtml?: ReadTableHtml + readonly readImage?: ReadImage readonly maxSteps?: number } @@ -224,6 +229,7 @@ export async function runAgentHarness( memorySearchAttempted: state.memorySearchAttempted === true, userText: input.turn.userText, readTableHtml: input.readTableHtml, + readImage: input.readImage, }) } return prepareHarnessStep({ diff --git a/src/domains/chat/contracts.ts b/src/domains/chat/contracts.ts index 784e3de..68e233e 100644 --- a/src/domains/chat/contracts.ts +++ b/src/domains/chat/contracts.ts @@ -9,6 +9,7 @@ import type { HarnessRunResult, KnowhereToolRuntime, KnowhereSearchRequest, + ReadImage, ReadTableHtml, ResolveConnectedAssets, } from "@/agent-harness" @@ -63,6 +64,7 @@ export type GenerateAnswer = (input: { knowhereTools?: KnowhereToolRuntime resolveConnectedAssets?: ResolveConnectedAssets readTableHtml?: ReadTableHtml + readImage?: ReadImage }) => Promise export type AnswerQuestionInput = { @@ -80,6 +82,7 @@ export type AnswerQuestionInput = { hardenMediaAssetUrls?: HardenMediaAssetUrls resolveConnectedAssets?: ResolveConnectedAssets readTableHtml?: ReadTableHtml + readImage?: ReadImage messages: readonly ChatHistoryMessage[] } diff --git a/src/domains/chat/index.ts b/src/domains/chat/index.ts index 58ef064..a9ff504 100644 --- a/src/domains/chat/index.ts +++ b/src/domains/chat/index.ts @@ -263,6 +263,7 @@ export const answerQuestionWithRetrieval = ( ? { resolveConnectedAssets: input.resolveConnectedAssets } : {}), ...(input.readTableHtml ? { readTableHtml: input.readTableHtml } : {}), + ...(input.readImage ? { readImage: input.readImage } : {}), }), ) diff --git a/src/domains/chat/prompt.ts b/src/domains/chat/prompt.ts index fa2f733..e501899 100644 --- a/src/domains/chat/prompt.ts +++ b/src/domains/chat/prompt.ts @@ -10,6 +10,7 @@ import { type AgentTurnInput, type HarnessRunResult, type KnowhereToolRuntime, + type ReadImage, type ReadTableHtml, type ResolveConnectedAssets, } from "@/agent-harness" @@ -37,6 +38,7 @@ type GenerateAgenticOutputManifestInput = { knowhereTools?: KnowhereToolRuntime resolveConnectedAssets?: ResolveConnectedAssets readTableHtml?: ReadTableHtml + readImage?: ReadImage folderScopeSourceIds?: readonly string[] } @@ -79,6 +81,7 @@ export const generateAgenticOutputManifestEffect = ( ? { resolveConnectedAssets: input.resolveConnectedAssets } : {}), ...(input.readTableHtml ? { readTableHtml: input.readTableHtml } : {}), + ...(input.readImage ? { readImage: input.readImage } : {}), }), ) diff --git a/src/domains/chat/route-answer.ts b/src/domains/chat/route-answer.ts index 5f76eb9..7d64ed3 100644 --- a/src/domains/chat/route-answer.ts +++ b/src/domains/chat/route-answer.ts @@ -92,6 +92,7 @@ const answerChatEffect = (input: AnswerChatInput) => }) const readTableHtml = (assetUrl: string): Promise => readSignedTableHtml(assetUrl) + const readImage = (assetUrl: string) => readSignedImage(assetUrl) const resolveConnectedAssets = createConnectedAssetResolver( knowhereResources.client.documents, ) @@ -121,6 +122,7 @@ const answerChatEffect = (input: AnswerChatInput) => }), resolveConnectedAssets, readTableHtml, + readImage, repository: chatTurnPersistence.createRepository(), }), ).pipe( @@ -186,6 +188,20 @@ async function readSignedTableHtml(assetUrl: string): Promise { return response.text() } +async function readSignedImage(assetUrl: string): Promise<{ + readonly body: Uint8Array + readonly mediaType: string +}> { + const fetched = await fetchChatAsset({ + assetUrl, + fallbackContentType: inferContentTypeFromPath(assetUrl), + }) + return { + body: fetched.body, + mediaType: fetched.contentType, + } +} + async function hardenSingleChatAsset(input: { readonly workspaceId: string readonly parsedStorage: BlobParsedDocumentStorage diff --git a/src/domains/chat/route-service.test.ts b/src/domains/chat/route-service.test.ts index 13308a7..0ca511c 100644 --- a/src/domains/chat/route-service.test.ts +++ b/src/domains/chat/route-service.test.ts @@ -204,6 +204,7 @@ describe("chat route services", () => { hardenChatAssetUrl: expect.any(Function), resolveConnectedAssets: expect.any(Function), readTableHtml: expect.any(Function), + readImage: expect.any(Function), repository: expect.objectContaining({ appendMessageToThread: expect.any(Function), ensureDefaultChatThread: expect.any(Function), @@ -250,6 +251,53 @@ describe("chat route services", () => { expect(fetch).toHaveBeenCalledWith(signedUrl) }) + it("injects a reader that returns signed image bytes", async () => { + const workspace = makeWorkspace() + const client = { retrieval: { query: vi.fn() } } + const signedUrl = "https://knowhere-storage.example/images/chart.png?signature=valid" + const imageBytes = new Uint8Array([137, 80, 78, 71]) + vi.stubGlobal( + "fetch", + vi.fn(async () => new Response(imageBytes, { + headers: { "content-type": "image/png" }, + })), + ) + mocks.getAuthenticatedWithClient.mockResolvedValue({ + user: { id: "user_1" }, + workspace, + apiKey: "jwt_123", + client, + }) + mocks.listSourcesForWorkspace.mockResolvedValue([makeSource()]) + mocks.handleChatTurn.mockImplementation( + async (input: { + readonly readImage?: (assetUrl: string) => Promise<{ + readonly body: Uint8Array + readonly mediaType: string + }> + }) => { + expect(await input.readImage?.(signedUrl)).toEqual({ + body: imageBytes, + mediaType: "image/png", + }) + return Either.right({ + threadId: "thread_1", + messages: [ + { id: "message_user", role: "user", content: "Read the chart" }, + { id: "message_assistant", role: "assistant", content: "Answer" }, + ], + }) + }, + ) + + const result = await chatAnswerRouteService.answerChat({ + body: { message: "Read the chart" }, + }) + + expect(result.status).toBe(200) + expect(fetch).toHaveBeenCalledWith(signedUrl) + }) + it("injects a resolver that matches connected assets by parser chunk id", async () => { const workspace = makeWorkspace() const client = { retrieval: { query: vi.fn() } } diff --git a/src/domains/chat/service.ts b/src/domains/chat/service.ts index d300d0c..863969a 100644 --- a/src/domains/chat/service.ts +++ b/src/domains/chat/service.ts @@ -80,6 +80,7 @@ type ChatTurnInput = { hardenMediaAssetUrls?: AnswerQuestionInput["hardenMediaAssetUrls"] resolveConnectedAssets?: AnswerQuestionInput["resolveConnectedAssets"] readTableHtml?: AnswerQuestionInput["readTableHtml"] + readImage?: AnswerQuestionInput["readImage"] repository: ChatRepository } @@ -148,6 +149,7 @@ export const handleChatTurnEffect = (input: ChatTurnInput) => ? { resolveConnectedAssets: input.resolveConnectedAssets } : {}), readTableHtml: input.readTableHtml, + readImage: input.readImage, messages: chatHistoryMessages, }).pipe(Effect.catchAllCause(Effect.die))