diff --git a/src/integrations/knowhere-demo.test.ts b/src/integrations/knowhere-demo.test.ts index aa56041..69101be 100644 --- a/src/integrations/knowhere-demo.test.ts +++ b/src/integrations/knowhere-demo.test.ts @@ -316,6 +316,160 @@ describe("knowhereDemoApi", () => { "/api/demo-sources/demo-tsla-q4-2025/assets/page_citation_assets/page-12.png", }) }) + + it("maps shared-corpus demo citations without legacy nested fields", async () => { + globalThis.fetch = vi.fn().mockResolvedValue( + new Response( + JSON.stringify({ + sources: [ + { + demo_source_id: "demo-tsla-q4-2025", + canonical_document_id: "ddoc_d55868b790cd", + title: "TSLA-Q4-2025-Update.pdf", + mime_type: "application/pdf", + size_bytes: 5648867, + import_status: "ready", + chunk_count: 71, + original_file: { + url: "/api/v1/demo/sources/demo-tsla-q4-2025/original", + mime_type: "application/pdf", + size_bytes: 5648867, + can_download: true, + }, + examples: [ + { + id: "demo-tsla-q4-2025-xai", + question: "What does the document say about Tesla's xAI investment?", + answer: "Tesla invested in xAI. [[cite:1]]", + citations: [ + { + description: "xAI investment", + document_id: "ddoc_d55868b790cd", + canonical_document_id: "ddoc_d55868b790cd", + job_result_id: "afc92902-5846-4d11-b658-2ccaf4f04242", + demo_source_id: "demo-tsla-q4-2025", + id: "dchk_1236885e2eda", + demo_chunk_id: "dchk_1236885e2eda", + chunk_id: "node_5ead044d-60cc-5b64-a066-fa6c0a81bbd6", + section_path: "Q4 and FY 2025 Update / OTHER UPDATES", + content: + "On January 16, 2026, Tesla entered into an agreement to invest approximately", + page_number: 12, + }, + ], + }, + ], + }, + ], + official_library: { categories: [], sources: [] }, + }), + { status: 200, headers: { "content-type": "application/json" } }, + ), + ) + + const catalog = await knowhereDemoApi.fetchCatalog() + const citation = catalog.sources[0]?.examples[0]?.citations[0] + + expect(citation).toMatchObject({ + demoSourceId: "demo-tsla-q4-2025", + canonicalDocumentId: "ddoc_d55868b790cd", + canonicalChunkId: "dchk_1236885e2eda", + chunkId: "node_5ead044d-60cc-5b64-a066-fa6c0a81bbd6", + chunkType: "text", + content: + "On January 16, 2026, Tesla entered into an agreement to invest approximately", + pageCitationPageNumber: 12, + pageNums: [12], + source: { + documentId: "ddoc_d55868b790cd", + sourceFileName: "TSLA-Q4-2025-Update.pdf", + sectionPath: "Q4 and FY 2025 Update / OTHER UPDATES", + }, + }) + }) + + it("normalizes shared-corpus chunk pages with catalog metadata fallback", async () => { + globalThis.fetch = vi + .fn() + .mockResolvedValueOnce( + new Response( + JSON.stringify({ + document_id: "ddoc_d55868b790cd", + namespace: "__knowhere_demo__", + job_result_id: "afc92902-5846-4d11-b658-2ccaf4f04242", + chunks: [ + { + id: "dchk_1236885e2eda", + chunk_id: "node_5ead044d-60cc-5b64-a066-fa6c0a81bbd6", + chunk_type: "text", + content: "Tesla entered into an agreement to invest approximately", + section_path: "Q4 and FY 2025 Update / OTHER UPDATES", + source_chunk_path: + "Q4 and FY 2025 Update / OTHER UPDATES", + file_path: null, + sort_order: 0, + metadata: { page_nums: [12] }, + asset_url: null, + }, + ], + pagination: { page: 1, page_size: 50, total: 1, total_pages: 1 }, + demo_source_id: undefined, + }), + { status: 200, headers: { "content-type": "application/json" } }, + ), + ) + .mockResolvedValueOnce( + new Response( + JSON.stringify({ + sources: [ + { + demo_source_id: "demo-tsla-q4-2025", + canonical_document_id: "ddoc_d55868b790cd", + title: "TSLA-Q4-2025-Update.pdf", + mime_type: "application/pdf", + size_bytes: 5648867, + import_status: "ready", + chunk_count: 71, + original_file: { + url: "/api/v1/demo/sources/demo-tsla-q4-2025/original", + mime_type: "application/pdf", + size_bytes: 5648867, + can_download: true, + }, + examples: [], + }, + ], + }), + { status: 200, headers: { "content-type": "application/json" } }, + ), + ) + + const page = await knowhereDemoApi.fetchChunkPage({ + demoSourceId: "demo-tsla-q4-2025", + page: 1, + pageSize: 50, + }) + + expect(page).toMatchObject({ + demoSourceId: "demo-tsla-q4-2025", + canonicalDocumentId: "ddoc_d55868b790cd", + title: "TSLA-Q4-2025-Update.pdf", + mimeType: "application/pdf", + }) + expect(page.chunks[0]).toMatchObject({ + id: "dchk_1236885e2eda", + chunkId: "node_5ead044d-60cc-5b64-a066-fa6c0a81bbd6", + sectionPath: "Q4 and FY 2025 Update / OTHER UPDATES", + }) + expect(globalThis.fetch).toHaveBeenNthCalledWith( + 1, + "https://api.knowhereto.ai/api/v1/demo/sources/demo-tsla-q4-2025/chunks?page=1&page_size=50", + ) + expect(globalThis.fetch).toHaveBeenNthCalledWith( + 2, + "https://api.knowhereto.ai/api/v1/demo/catalog", + ) + }) }) function restoreEnv(key: string, value: string | undefined): void { diff --git a/src/integrations/knowhere-demo.ts b/src/integrations/knowhere-demo.ts index 83c56ee..33ba6ab 100644 --- a/src/integrations/knowhere-demo.ts +++ b/src/integrations/knowhere-demo.ts @@ -126,6 +126,7 @@ type DemoCatalogResponse = { type DemoSourceResponse = { readonly demo_source_id?: unknown readonly canonical_document_id?: unknown + readonly job_result_id?: unknown readonly title?: unknown readonly mime_type?: unknown readonly size_bytes?: unknown @@ -151,13 +152,19 @@ type DemoExampleResponse = { } type DemoCitationResponse = { + readonly id?: unknown + readonly demo_chunk_id?: unknown + readonly document_id?: unknown readonly demo_source_id?: unknown readonly canonical_document_id?: unknown readonly canonical_chunk_id?: unknown readonly chunk_id?: unknown readonly chunk_type?: unknown + readonly type?: unknown readonly content?: unknown readonly description?: unknown + readonly section_path?: unknown + readonly page_number?: unknown readonly page_citation_page_number?: unknown readonly page_citation_asset_url?: unknown readonly page_nums?: unknown @@ -169,8 +176,10 @@ type DemoCitationResponse = { } type DemoChunkPageResponse = { + readonly document_id?: unknown readonly demo_source_id?: unknown readonly canonical_document_id?: unknown + readonly job_result_id?: unknown readonly title?: unknown readonly mime_type?: unknown readonly chunks?: readonly DemoChunkResponse[] @@ -279,14 +288,54 @@ const fetchChunkPageEffect = Effect.fn("knowhereDemo.fetchChunkPage")( ) yield* assertOkEffect(response) - return toDemoChunkPage( - (yield* Effect.tryPromise(() => - response.json(), - )) as DemoChunkPageResponse, - ) + const body = (yield* Effect.tryPromise(() => + response.json(), + )) as DemoChunkPageResponse + const source = yield* readDemoSourceFallback(input.demoSourceId, body) + + return toDemoChunkPage(body, source) }, ) +function readDemoSourceFallback( + demoSourceId: string, + response: DemoChunkPageResponse, +): Effect.Effect<{ + readonly demoSourceId: string + readonly canonicalDocumentId?: string + readonly title?: string + readonly mimeType?: string +}, never> { + const hasCompleteMetadata = + optionalString(response.demo_source_id) !== undefined && + (optionalString(response.canonical_document_id) ?? + optionalString(response.document_id)) !== undefined && + optionalString(response.title) !== undefined && + optionalString(response.mime_type) !== undefined + if (hasCompleteMetadata) { + return Effect.succeed({ demoSourceId }) + } + + return fetchCatalogEffect().pipe( + Effect.map((catalog) => { + const source = catalog.sources.find( + (candidate) => candidate.demoSourceId === demoSourceId, + ) + return { + demoSourceId, + ...(source + ? { + canonicalDocumentId: source.canonicalDocumentId, + title: source.title, + mimeType: source.mimeType, + } + : {}), + } + }), + Effect.catchAll(() => Effect.succeed({ demoSourceId })), + ) +} + const materializeSourcesEffect = Effect.fn("knowhereDemo.materializeSources")( function* (input: { readonly apiKey: string @@ -419,48 +468,74 @@ function assertOkEffect( } function toDemoSource(source: DemoSourceResponse): DemoSource { + const demoSourceId = requireString(source.demo_source_id) + const title = requireString(source.title) const officialLibrary = source.official_library ? toOfficialLibrarySource(source.official_library) : undefined return { - demoSourceId: requireString(source.demo_source_id), + demoSourceId, canonicalDocumentId: requireString(source.canonical_document_id), - title: requireString(source.title), + title, mimeType: requireString(source.mime_type), sizeBytes: requireNumber(source.size_bytes), status: "ready", chunkCount: requireNumber(source.chunk_count), originalFile: toOriginalFile(source.original_file), ...(officialLibrary ? { officialLibrary } : {}), - examples: (source.examples ?? []).map(toDemoExample), + examples: (source.examples ?? []).map((example) => + toDemoExample(example, title, demoSourceId), + ), } } -function toDemoExample(example: DemoExampleResponse): DemoExample { +function toDemoExample( + example: DemoExampleResponse, + sourceTitle: string, + demoSourceId: string, +): DemoExample { return { id: requireString(example.id), question: requireString(example.question), answer: requireString(example.answer), - citations: (example.citations ?? []).map(toDemoCitation), + citations: (example.citations ?? []).map((citation) => + toDemoCitation(citation, sourceTitle, demoSourceId), + ), } } -function toDemoCitation(citation: DemoCitationResponse): DemoCitation { +function toDemoCitation( + citation: DemoCitationResponse, + sourceTitle: string, + demoSourceId: string, +): DemoCitation { const source = citation.source ?? {} const description = optionalString(citation.description) - const pageNums = toPositiveIntegers(citation.page_nums) + const canonicalDocumentId = requireString( + citation.canonical_document_id ?? citation.document_id, + ) + const pageNums = toPositiveIntegers( + citation.page_nums, + citation.page_number, + ) const pageCitationPageNumber = - optionalPositiveInteger(citation.page_citation_page_number) ?? pageNums[0] + optionalPositiveInteger(citation.page_citation_page_number) ?? + optionalPositiveInteger(citation.page_number) ?? + pageNums[0] const pageCitationAssetUrl = toDemoAssetUrl( - requireString(citation.demo_source_id), + optionalString(citation.demo_source_id) ?? demoSourceId, optionalString(citation.page_citation_asset_url), ) return { - demoSourceId: requireString(citation.demo_source_id), - canonicalDocumentId: requireString(citation.canonical_document_id), - canonicalChunkId: requireString(citation.canonical_chunk_id), + demoSourceId: optionalString(citation.demo_source_id) ?? demoSourceId, + canonicalDocumentId, + canonicalChunkId: requireString( + citation.canonical_chunk_id ?? citation.id ?? citation.demo_chunk_id, + ), chunkId: requireString(citation.chunk_id), - chunkType: requireString(citation.chunk_type), + chunkType: optionalString(citation.chunk_type) ?? + optionalString(citation.type) ?? + "text", content: requireString(citation.content), ...(description ? { description } : {}), ...(pageCitationPageNumber !== undefined @@ -469,22 +544,39 @@ function toDemoCitation(citation: DemoCitationResponse): DemoCitation { ...(pageCitationAssetUrl ? { pageCitationAssetUrl } : {}), ...(pageNums.length > 0 ? { pageNums } : {}), source: { - documentId: requireString(source.document_id), - sourceFileName: requireString(source.source_file_name), - sectionPath: requireString(source.section_path), + documentId: requireString( + source.document_id ?? citation.document_id ?? canonicalDocumentId, + ), + sourceFileName: optionalString(source.source_file_name) ?? sourceTitle, + sectionPath: requireString(source.section_path ?? citation.section_path), }, } } -function toDemoChunkPage(response: DemoChunkPageResponse): DemoChunkPage { +function toDemoChunkPage( + response: DemoChunkPageResponse, + fallback: { + readonly demoSourceId: string + readonly canonicalDocumentId?: string + readonly title?: string + readonly mimeType?: string + }, +): DemoChunkPage { const pagination = response.pagination ?? {} return { - demoSourceId: requireString(response.demo_source_id), - canonicalDocumentId: requireString(response.canonical_document_id), - title: requireString(response.title), - mimeType: requireString(response.mime_type), + demoSourceId: requireString(response.demo_source_id ?? fallback.demoSourceId), + canonicalDocumentId: requireString( + response.canonical_document_id ?? + response.document_id ?? + fallback.canonicalDocumentId, + ), + title: requireString(response.title ?? fallback.title), + mimeType: requireString(response.mime_type ?? fallback.mimeType), chunks: (response.chunks ?? []).map((chunk) => - toDemoChunk(requireString(response.demo_source_id), chunk), + toDemoChunk( + requireString(response.demo_source_id ?? fallback.demoSourceId), + chunk, + ), ), pagination: { page: requireNumber(pagination.page), @@ -697,12 +789,19 @@ function optionalPositiveInteger(value: unknown): number | undefined { : undefined } -function toPositiveIntegers(value: unknown): readonly number[] { - if (!Array.isArray(value)) return [] - return value.flatMap((item) => { +function toPositiveIntegers( + value: unknown, + scalarValue?: unknown, +): readonly number[] { + const values = Array.isArray(value) ? value : [] + const pages = values.flatMap((item) => { const page = optionalPositiveInteger(item) return page === undefined ? [] : [page] }) + const scalarPage = optionalPositiveInteger(scalarValue) + return scalarPage !== undefined && !pages.includes(scalarPage) + ? [...pages, scalarPage] + : pages } function toDemoAssetUrl(