diff --git a/.env.example b/.env.example index 9e75431698..36c4aafd53 100644 --- a/.env.example +++ b/.env.example @@ -739,6 +739,11 @@ OPENRAG_RUN_MODE=oss # hidden/disabled. # OPENRAG_WORKSPACE_OAUTH_OVERRIDES_ENABLED=false +# Feature flag for the managed public-website connector (default: false = +# disabled). When true, users can add website sources from Knowledge and the +# Connectors settings page. This is also enforced by the backend URL APIs. +# OPENRAG_URL_CONNECTOR_ENABLED=false + # Permission/identity cache backend. Only "memory" is currently wired. # When CACHE_BACKEND=memory the RBAC permission cache and OAuth-subject # cache are per-process, so OpenRAG must run with UVICORN_WORKERS=1 (and diff --git a/.github/workflows/test-ci.yml b/.github/workflows/test-ci.yml index 5ccd25178a..82bbbd119c 100644 --- a/.github/workflows/test-ci.yml +++ b/.github/workflows/test-ci.yml @@ -454,7 +454,7 @@ jobs: - name: Setup E2E infrastructure run: | chmod +x scripts/setup-e2e.sh - OPENRAG_VERSION="${CI_IMAGE_TAG}" ./scripts/setup-e2e.sh + OPENRAG_VERSION="${CI_IMAGE_TAG}" OPENRAG_URL_CONNECTOR_ENABLED=true ./scripts/setup-e2e.sh - name: Run Playwright tests (shard ${{ matrix.shard }}/${{ matrix.total_shards }}) working-directory: frontend diff --git a/docker-compose.yml b/docker-compose.yml index bd702ce9d8..e961faf497 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -138,6 +138,7 @@ services: - MAX_WORKERS=${MAX_WORKERS:-4} - IBM_AUTH_ENABLED=${IBM_AUTH_ENABLED:-false} - OPENRAG_AZURE_BLOB_ENABLED=${OPENRAG_AZURE_BLOB_ENABLED:-true} + - OPENRAG_URL_CONNECTOR_ENABLED=${OPENRAG_URL_CONNECTOR_ENABLED:-false} - PLATFORM_AUTH_DEV_MODE=${PLATFORM_AUTH_DEV_MODE:-false} - DOCLING_SERVE_URL=${DOCLING_SERVE_URL:-http://host.docker.internal:5001} - DOCLING_SERVE_VERIFY_SSL=${DOCLING_SERVE_VERIFY_SSL:-true} diff --git a/frontend/app/api/mutations/useWebsiteSourceMutation.test.tsx b/frontend/app/api/mutations/useWebsiteSourceMutation.test.tsx new file mode 100644 index 0000000000..6abfca4948 --- /dev/null +++ b/frontend/app/api/mutations/useWebsiteSourceMutation.test.tsx @@ -0,0 +1,50 @@ +import { act, renderHook, waitFor } from "@testing-library/react"; +import { HttpResponse, http } from "msw"; +import { describe, expect, it } from "vitest"; +import { useGetTasksQuery } from "@/app/api/queries/useGetTasksQuery"; +import { server } from "@/test-utils/msw/server"; +import { createQueryWrapper } from "@/test-utils/render"; +import { useSyncWebsiteSourceMutation } from "./useWebsiteSourceMutation"; + +describe("useSyncWebsiteSourceMutation", () => { + it("syncs a child page and automatically refreshes active task queries", async () => { + let syncCalls = 0; + let taskRequests = 0; + server.use( + http.post( + "/api/connectors/url/sources/source-1/pages/page-1/sync", + () => { + syncCalls += 1; + return HttpResponse.json({ task_id: "task-1" }); + }, + ), + http.get("/api/tasks/enhanced", () => { + taskRequests += 1; + return HttpResponse.json({ tasks: [] }); + }), + ); + + const { result } = renderHook( + () => { + useGetTasksQuery(); + return useSyncWebsiteSourceMutation(); + }, + { wrapper: createQueryWrapper() }, + ); + + await waitFor(() => expect(taskRequests).toBeGreaterThan(0)); + const taskRequestsBeforeSync = taskRequests; + + await act(async () => { + await result.current.mutateAsync({ + sourceId: "source-1", + pageId: "page-1", + }); + }); + + expect(syncCalls).toBe(1); + await waitFor(() => + expect(taskRequests).toBeGreaterThan(taskRequestsBeforeSync), + ); + }); +}); diff --git a/frontend/app/api/mutations/useWebsiteSourceMutation.ts b/frontend/app/api/mutations/useWebsiteSourceMutation.ts new file mode 100644 index 0000000000..b95d244d1e --- /dev/null +++ b/frontend/app/api/mutations/useWebsiteSourceMutation.ts @@ -0,0 +1,135 @@ +import { + type QueryClient, + useMutation, + useQueryClient, +} from "@tanstack/react-query"; +import { TASKS_QUERY_KEY } from "@/app/api/queries/useGetTasksQuery"; +import { websiteSourceQueryKey } from "@/app/api/queries/useGetWebsiteSourceQuery"; +import { websiteSourcesQueryKey } from "@/app/api/queries/useGetWebsiteSourcesQuery"; +import type { WebsiteSource } from "@/components/website-pages/types"; + +export interface CreateWebsiteSourceRequest { + name: string; + starting_url: string; + scope: "path" | "page" | "site"; + allow_subdomains: boolean; + additional_hosts: string[]; + include_paths: string[]; + exclude_paths: string[]; + max_pages: number; + max_depth: number; + max_downloaded_mb: number; + max_crawl_minutes: number; + change_detection: "normalized_content_hash" | "always_reingest"; + resync_behavior: "full" | "root"; + removed_page_behavior: "retain" | "delete"; +} + +export interface WebsiteSourceTaskResponse { + last_task_id?: string; + task_id?: string; +} + +export interface WebsiteSourcePageRequest { + sourceId: string; + pageId?: string; +} + +async function readError(response: Response, fallback: string) { + const data = (await response.json().catch(() => ({}))) as { + detail?: string; + error?: string; + }; + return data.detail || data.error || fallback; +} + +async function createWebsiteSource( + request: CreateWebsiteSourceRequest, +): Promise { + const response = await fetch("/api/connectors/url/sources", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(request), + }); + if (!response.ok) { + throw new Error( + await readError(response, "Could not create website source"), + ); + } + return response.json(); +} + +async function syncWebsiteSource({ + sourceId, + pageId, +}: WebsiteSourcePageRequest): Promise { + const endpoint = pageId + ? `/api/connectors/url/sources/${sourceId}/pages/${pageId}/sync` + : `/api/connectors/url/sources/${sourceId}/sync`; + const response = await fetch(endpoint, { method: "POST" }); + if (!response.ok) { + throw new Error(await readError(response, "Could not start website sync")); + } + return response.json(); +} + +async function deleteWebsiteSourcePage({ + sourceId, + pageId, +}: WebsiteSourcePageRequest): Promise { + const endpoint = pageId + ? `/api/connectors/url/sources/${sourceId}/pages/${pageId}` + : `/api/connectors/url/sources/${sourceId}`; + const response = await fetch(endpoint, { method: "DELETE" }); + if (!response.ok) { + throw new Error( + await readError(response, "Could not delete website source"), + ); + } +} + +function invalidateWebsiteSourceQueries( + queryClient: QueryClient, + sourceId: string, +) { + queryClient.invalidateQueries({ queryKey: websiteSourceQueryKey(sourceId) }); + queryClient.invalidateQueries({ queryKey: websiteSourcesQueryKey }); + queryClient.invalidateQueries({ queryKey: ["search"] }); + queryClient.invalidateQueries({ queryKey: [...TASKS_QUERY_KEY] }); +} + +export function useCreateWebsiteSourceMutation() { + const queryClient = useQueryClient(); + + return useMutation({ + mutationFn: createWebsiteSource, + onSuccess: () => { + queryClient.invalidateQueries({ queryKey: ["search"] }); + queryClient.invalidateQueries({ queryKey: ["listFiles"] }); + queryClient.invalidateQueries({ queryKey: websiteSourcesQueryKey }); + queryClient.invalidateQueries({ queryKey: [...TASKS_QUERY_KEY] }); + }, + }); +} + +export function useSyncWebsiteSourceMutation() { + const queryClient = useQueryClient(); + + return useMutation({ + mutationFn: syncWebsiteSource, + onSuccess: (_data, { sourceId }) => { + invalidateWebsiteSourceQueries(queryClient, sourceId); + }, + }); +} + +export function useDeleteWebsiteSourcePageMutation() { + const queryClient = useQueryClient(); + + return useMutation({ + mutationFn: deleteWebsiteSourcePage, + onSuccess: (_data, { sourceId }) => { + invalidateWebsiteSourceQueries(queryClient, sourceId); + }, + }); +} diff --git a/frontend/app/api/queries/useFileScopedChunksQuery.test.tsx b/frontend/app/api/queries/useFileScopedChunksQuery.test.tsx index f748ecf200..e67607bec6 100644 --- a/frontend/app/api/queries/useFileScopedChunksQuery.test.tsx +++ b/frontend/app/api/queries/useFileScopedChunksQuery.test.tsx @@ -2,7 +2,10 @@ import { HttpResponse, http } from "msw"; import { describe, expect, it } from "vitest"; import { server } from "@/test-utils/msw/server"; import { createQueryWrapper, renderHook, waitFor } from "@/test-utils/render"; -import { useFileScopedChunksQuery } from "./useFileScopedChunksQuery"; +import { + useDocumentScopedChunksQuery, + useFileScopedChunksQuery, +} from "./useFileScopedChunksQuery"; import type { ChunkResult } from "./useGetSearchQuery"; function chunk(overrides: Partial = {}): ChunkResult { @@ -125,6 +128,51 @@ describe("useFileScopedChunksQuery", () => { expect(result.current.file?.chunks?.[0]?.highlights).toBeDefined(); }); + it("merges highlights without dropping non-matching chunks", async () => { + server.use( + http.post("/api/search", async ({ request }) => { + const body = (await request.json()) as { query: string }; + return HttpResponse.json({ + results: + body.query === "*" + ? [ + chunk({ + filename: "test.pdf", + chunk_id: "one", + text: "match", + }), + chunk({ + filename: "test.pdf", + chunk_id: "two", + text: "other", + }), + ] + : [ + chunk({ + filename: "test.pdf", + chunk_id: "one", + text: "match", + highlights: ["match"], + }), + ], + warnings: [], + }); + }), + ); + + const { result } = renderHook( + () => useFileScopedChunksQuery("test.pdf", "match"), + { wrapper: createQueryWrapper() }, + ); + + await waitFor(() => expect(result.current.isFetching).toBe(false)); + expect(result.current.file?.chunks).toHaveLength(2); + expect(result.current.file?.chunks?.[0]?.highlights).toEqual([ + "match", + ]); + expect(result.current.file?.chunks?.[1]?.highlights).toEqual([]); + }); + it("finds the matching file from multiple files in response", async () => { serveSearch([ chunk({ filename: "other.pdf", text: "other chunk" }), @@ -171,3 +219,63 @@ describe("useFileScopedChunksQuery", () => { expect(result.current.file?.filename).toBe("test.pdf"); }); }); + +describe("useDocumentScopedChunksQuery", () => { + it("merges highlights by stable document id without removing other chunks", async () => { + const requests: { query: string; documentIds?: string[] }[] = []; + server.use( + http.post("/api/search", async ({ request }) => { + const body = (await request.json()) as { + query: string; + filters?: { document_ids?: string[] }; + }; + requests.push({ + query: body.query, + documentIds: body.filters?.document_ids, + }); + return HttpResponse.json({ + results: + body.query === "*" + ? [ + chunk({ + document_id: "page-1", + chunk_id: "one", + text: "match", + }), + chunk({ + document_id: "page-1", + chunk_id: "two", + text: "other", + }), + ] + : [ + chunk({ + document_id: "page-1", + chunk_id: "one", + text: "match", + highlights: ["match"], + }), + ], + warnings: [], + }); + }), + ); + + const { result } = renderHook( + () => useDocumentScopedChunksQuery("page-1", "match"), + { wrapper: createQueryWrapper() }, + ); + + await waitFor(() => expect(result.current.isFetching).toBe(false)); + expect(requests).toEqual( + expect.arrayContaining([ + expect.objectContaining({ query: "*", documentIds: ["page-1"] }), + expect.objectContaining({ query: "match", documentIds: ["page-1"] }), + ]), + ); + expect(result.current.file?.chunks).toHaveLength(2); + expect(result.current.file?.chunks?.[0]?.highlights).toEqual([ + "match", + ]); + }); +}); diff --git a/frontend/app/api/queries/useFileScopedChunksQuery.ts b/frontend/app/api/queries/useFileScopedChunksQuery.ts index 99d5384941..3adc8d0ee0 100644 --- a/frontend/app/api/queries/useFileScopedChunksQuery.ts +++ b/frontend/app/api/queries/useFileScopedChunksQuery.ts @@ -6,7 +6,10 @@ import { type SearchResult, useGetSearchQuery, } from "@/app/api/queries/useGetSearchQuery"; -import { fileScopedSearchQueryData } from "@/lib/file-chunks"; +import { + documentScopedSearchQueryData, + fileScopedSearchQueryData, +} from "@/lib/file-chunks"; /** * Loads every chunk for one filename (shared by chunks page + FileChunksPanel). @@ -79,3 +82,53 @@ export function useFileScopedChunksQuery( return { file, isFetching: isFetchingAll || isFetchingHl }; } + +export function useDocumentScopedChunksQuery( + documentId: string | null | undefined, + searchQuery?: string, +) { + const queryData = documentId + ? documentScopedSearchQueryData(documentId) + : null; + const isRealQuery = + Boolean(searchQuery) && + searchQuery!.trim() !== "*" && + searchQuery!.trim() !== ""; + const { data: allData = EMPTY_SEARCH_RESULT, isFetching: isFetchingAll } = + useGetSearchQuery( + "*", + queryData, + { enabled: Boolean(documentId) }, + { groupBy: "document_id" }, + ); + const { data: hlData = EMPTY_SEARCH_RESULT, isFetching: isFetchingHl } = + useGetSearchQuery( + isRealQuery ? searchQuery! : "*", + queryData, + { + enabled: Boolean(documentId) && isRealQuery, + placeholderData: undefined, + }, + { groupBy: "document_id" }, + ); + const file = useMemo(() => { + const allFile = (allData as SearchResult).files[0]; + if (!allFile || !isRealQuery) return allFile; + const highlightsByChunk = new Map(); + for (const chunk of (hlData as SearchResult).files[0]?.chunks ?? []) { + const key = chunk.chunk_id ?? chunk.id; + if (key && chunk.highlights?.length) { + highlightsByChunk.set(key, chunk.highlights); + } + } + return { + ...allFile, + chunks: (allFile.chunks ?? []).map((chunk) => { + const key = chunk.chunk_id ?? chunk.id; + const highlights = key ? (highlightsByChunk.get(key) ?? []) : []; + return highlights.length ? { ...chunk, highlights } : chunk; + }), + }; + }, [allData, hlData, isRealQuery]); + return { file, isFetching: isFetchingAll || isFetchingHl }; +} diff --git a/frontend/app/api/queries/useGetConnectorsQuery.test.ts b/frontend/app/api/queries/useGetConnectorsQuery.test.ts index 0c323f5128..1865adca56 100644 --- a/frontend/app/api/queries/useGetConnectorsQuery.test.ts +++ b/frontend/app/api/queries/useGetConnectorsQuery.test.ts @@ -129,6 +129,45 @@ describe("useGetConnectorsQuery", () => { expect(result.current.data?.[0].status).toBe("not_connected"); }); + it("marks an always-connected managed connector connected without OAuth", async () => { + let statusRequests = 0; + server.use( + http.get("/api/connectors", () => + HttpResponse.json( + connectorsPayload({ + url: { + name: "URL", + description: "Crawl websites", + icon: "url", + kind: "managed", + always_connected: true, + available: true, + }, + }), + ), + ), + http.get("/api/connectors/url/status", () => { + statusRequests += 1; + return HttpResponse.json({ connections: [] }); + }), + ); + + const { result } = renderHook(() => useGetConnectorsQuery(), { + wrapper: createQueryWrapper({ providers: ["brand"] }), + }); + + await waitFor(() => expect(result.current.isSuccess).toBe(true)); + expect(result.current.data).toEqual([ + expect.objectContaining({ + id: "url", + status: "connected", + requiresOAuth: false, + alwaysConnected: true, + }), + ]); + expect(statusRequests).toBe(0); + }); + it("filters out aws_s3 and ibm_cos for a non-IBM oss deployment", async () => { server.use( http.get("/api/connectors", () => diff --git a/frontend/app/api/queries/useGetConnectorsQuery.ts b/frontend/app/api/queries/useGetConnectorsQuery.ts index 35f275ca21..758f27c4db 100644 --- a/frontend/app/api/queries/useGetConnectorsQuery.ts +++ b/frontend/app/api/queries/useGetConnectorsQuery.ts @@ -143,6 +143,7 @@ export interface Connector { selectedFiles?: GoogleDriveFile[] | OneDriveFile[]; available?: boolean; requiresOAuth?: boolean; + alwaysConnected?: boolean; } interface Connection { @@ -184,6 +185,7 @@ export const useGetConnectorsQuery = ( description: string; icon: string; available?: boolean; + always_connected?: boolean; } >; }>("/connectors"); @@ -197,18 +199,33 @@ export const useGetConnectorsQuery = ( const connectorsWithStatus = await Promise.all( connectorTypes.map(async (type) => { const connectorData = connectorsMap[type]; + // Determine if this connector requires OAuth based on connector kind + // "oauth" connectors require OAuth flow (Google Drive, OneDrive, SharePoint) + // "bucket" connectors use credential-based auth (Azure Blob, S3, IBM COS) + const requiresOAuth = connectorData.kind === "oauth"; + const alwaysConnected = connectorData.always_connected === true; + + if (alwaysConnected) { + return { + id: type, + name: connectorData.name, + description: connectorData.description, + icon: connectorData.icon, + status: "connected", + type, + connectionId: type, + available: connectorData.available, + requiresOAuth, + alwaysConnected, + } as Connector; + } + const statusResponse = await apiClient.get( `/connectors/${type}/status`, ); - let status: Connector["status"] = "not_connected"; let connectionId: string | undefined; - // Determine if this connector requires OAuth based on connector kind - // "oauth" connectors require OAuth flow (Google Drive, OneDrive, SharePoint) - // "bucket" connectors use credential-based auth (Azure Blob, S3, IBM COS) - const requiresOAuth = connectorData.kind === "oauth"; - if (statusResponse.status >= 200 && statusResponse.status < 300) { const statusData = statusResponse.data; const connections = statusData.connections || []; diff --git a/frontend/app/api/queries/useGetSearchQuery.test.tsx b/frontend/app/api/queries/useGetSearchQuery.test.tsx index d29fadb5b0..0d5fdfb3a3 100644 --- a/frontend/app/api/queries/useGetSearchQuery.test.tsx +++ b/frontend/app/api/queries/useGetSearchQuery.test.tsx @@ -6,6 +6,7 @@ import { createQueryWrapper, renderHook, waitFor } from "@/test-utils/render"; import { type ChunkResult, type SearchPayload, + type SearchResultDisplayOptions, useGetSearchQuery, } from "./useGetSearchQuery"; @@ -61,10 +62,17 @@ function serveSearch(body: Record, status = 200) { return seen; } -async function runSearch(query: string, data?: ParsedQueryData | null) { - const rendered = renderHook(() => useGetSearchQuery(query, data), { - wrapper: createQueryWrapper(), - }); +async function runSearch( + query: string, + data?: ParsedQueryData | null, + displayOptions?: SearchResultDisplayOptions, +) { + const rendered = renderHook( + () => useGetSearchQuery(query, data, undefined, displayOptions), + { + wrapper: createQueryWrapper(), + }, + ); await waitFor(() => expect( rendered.result.current.isSuccess || rendered.result.current.isError, @@ -171,6 +179,30 @@ describe("useGetSearchQuery", () => { }); }); + it("preserves a URL source scope and website-page result mode", async () => { + const seen = serveSearch({ results: [] }); + + await runSearch( + "", + queryData({ + filters: { + data_sources: [], + document_types: [], + owners: [], + connector_types: [], + web_source_ids: ["source-123"], + }, + }), + { groupBy: "document_id", resultMode: "website_pages" }, + ); + + expect(seen.payload).toMatchObject({ + query: "*", + resultMode: "website_pages", + filters: { web_source_ids: ["source-123"] }, + }); + }); + it("treats a '*' dimension as unfiltered", async () => { const seen = serveSearch({ results: [] }); @@ -191,6 +223,26 @@ describe("useGetSearchQuery", () => { }); describe("grouping chunks into files", () => { + it("keeps URL pages with duplicate titles separate by document id", async () => { + serveSearch({ + results: [ + chunk({ filename: "Guide", document_id: "page-a" }), + chunk({ filename: "Guide", document_id: "page-b" }), + ], + }); + + const { result } = await runSearch("", queryData(), { + groupBy: "document_id", + }); + + expect(result.current.data?.files).toEqual( + expect.arrayContaining([ + expect.objectContaining({ filename: "Guide", document_id: "page-a" }), + expect.objectContaining({ filename: "Guide", document_id: "page-b" }), + ]), + ); + }); + it("groups chunks by filename and averages their scores", async () => { serveSearch({ results: [ @@ -212,6 +264,33 @@ describe("useGetSearchQuery", () => { expect(files[1]).toMatchObject({ filename: "b.pdf", chunkCount: 1 }); }); + it("uses the backend chunk count for a website page", async () => { + serveSearch({ + results: [ + chunk({ + filename: "Guide", + document_id: "page-a", + chunk_count: 12, + }), + chunk({ + filename: "Guide", + document_id: "page-a", + chunk_count: 12, + }), + ], + }); + + const { result } = await runSearch("", queryData(), { + groupBy: "document_id", + resultMode: "website_pages", + }); + + expect(result.current.data?.files[0]).toMatchObject({ + document_id: "page-a", + chunkCount: 12, + }); + }); + it("falls back to source_url, then to Untitled source (#1609)", async () => { serveSearch({ results: [ diff --git a/frontend/app/api/queries/useGetSearchQuery.ts b/frontend/app/api/queries/useGetSearchQuery.ts index da4a0dd2e7..33785cb960 100644 --- a/frontend/app/api/queries/useGetSearchQuery.ts +++ b/frontend/app/api/queries/useGetSearchQuery.ts @@ -6,18 +6,17 @@ import { import type { ParsedQueryData } from "@/contexts/knowledge-filter-context"; import { apiClient, isRequestCancelled } from "@/lib/api-client"; import { SEARCH_CONSTANTS } from "@/lib/constants"; -import { buildSearchPayloadFilters } from "@/lib/filter-normalization"; +import { + buildSearchPayloadFilters, + type FilterInput, +} from "@/lib/filter-normalization"; export interface SearchPayload { query: string; limit: number; scoreThreshold: number; - filters?: { - data_sources?: string[]; - document_types?: string[]; - owners?: string[]; - connector_types?: string[]; - }; + filters?: FilterInput; + resultMode?: "chunks" | "knowledge_sources" | "website_pages"; } export interface ChunkResult { @@ -46,6 +45,15 @@ export interface ChunkResult { index?: number; allowed_users?: string[]; allowed_groups?: string[]; + document_id?: string; + web_source_id?: string; + web_page_id?: string; + web_page_depth?: number; + web_child_count?: number; + canonical_url?: string; + status?: File["status"]; + error?: string; + chunk_count?: number; } export interface File { @@ -69,7 +77,8 @@ export interface File { | "cancelled" | "skipped" | "hidden" - | "sync"; + | "sync" + | "disabled"; error?: string; /** * Skip reason forwarded from result.reason (e.g. "duplicate_content", @@ -82,6 +91,11 @@ export interface File { chunks?: ChunkResult[]; allowed_users?: string[]; allowed_groups?: string[]; + document_id?: string; + web_source_id?: string; + web_page_id?: string; + web_page_depth?: number; + web_child_count?: number; } // Non-fatal signal from the backend — e.g. an embedding provider was removed @@ -99,11 +113,22 @@ export interface SearchResult { warnings: SearchWarning[]; } +export interface SearchResultDisplayOptions { + groupBy?: "filename" | "document_id"; + resultMode?: "chunks" | "knowledge_sources" | "website_pages"; +} + const EMPTY_SEARCH_RESULT: SearchResult = { files: [], warnings: [] }; export { EMPTY_SEARCH_RESULT }; -const getFileIdentity = (chunk: ChunkResult): string => { +const getFileIdentity = ( + chunk: ChunkResult, + groupBy: SearchResultDisplayOptions["groupBy"], +): string => { + if (groupBy === "document_id" && chunk.document_id) { + return chunk.document_id; + } const normalizedFilename = chunk.filename?.trim(); if (normalizedFilename) { return normalizedFilename; @@ -124,6 +149,7 @@ export const useGetSearchQuery = ( UseQueryOptions, "queryKey" | "queryFn" >, + displayOptions: SearchResultDisplayOptions = {}, ) => { const queryClient = useQueryClient(); @@ -156,6 +182,7 @@ export const useGetSearchQuery = ( query: effectiveQuery, limit: searchLimit, scoreThreshold: dynamicScoreThreshold, + resultMode: displayOptions.resultMode, }; if (queryData?.filters) { searchPayload.filters = @@ -193,11 +220,20 @@ export const useGetSearchQuery = ( embedding_dimensions?: number; allowed_users?: string[]; allowed_groups?: string[]; + document_id?: string; + web_source_id?: string; + web_page_id?: string; + web_page_depth?: number; + web_child_count?: number; + canonical_url?: string; + status?: File["status"]; + error?: string; + chunk_count?: number; } >(); (data.results || []).forEach((chunk: ChunkResult) => { - const fileIdentity = getFileIdentity(chunk); + const fileIdentity = getFileIdentity(chunk, displayOptions.groupBy); // Preserve highlights on the chunk object itself — they are per-chunk, // not per-file, so we carry them through rather than aggregating. const chunkWithHighlights: ChunkResult = { @@ -217,9 +253,18 @@ export const useGetSearchQuery = ( ) { existing.embedding_dimensions = chunk.embedding_dimensions; } + if (typeof chunk.chunk_count === "number") { + existing.chunk_count = chunk.chunk_count; + } + if (typeof chunk.web_child_count === "number") { + existing.web_child_count = chunk.web_child_count; + } } else { fileMap.set(fileIdentity, { - filename: fileIdentity, + filename: + displayOptions.groupBy === "document_id" + ? chunk.filename || fileIdentity + : fileIdentity, mimetype: chunk.mimetype, chunks: [chunkWithHighlights], totalScore: chunk.score, @@ -233,6 +278,15 @@ export const useGetSearchQuery = ( embedding_dimensions: chunk.embedding_dimensions, allowed_users: chunk.allowed_users || [], allowed_groups: chunk.allowed_groups || [], + document_id: chunk.document_id, + web_source_id: chunk.web_source_id, + web_page_id: chunk.web_page_id, + web_page_depth: chunk.web_page_depth, + web_child_count: chunk.web_child_count, + canonical_url: chunk.canonical_url, + status: chunk.status, + error: chunk.error, + chunk_count: chunk.chunk_count, }); } }); @@ -240,7 +294,7 @@ export const useGetSearchQuery = ( const files: File[] = Array.from(fileMap.values()).map((file) => ({ filename: file.filename, mimetype: file.mimetype, - chunkCount: file.chunks.length, + chunkCount: file.chunk_count ?? file.chunks.length, avgScore: file.totalScore / file.chunks.length, source_url: file.source_url || "", owner: file.owner || "", @@ -253,6 +307,13 @@ export const useGetSearchQuery = ( chunks: file.chunks, allowed_users: file.allowed_users || [], allowed_groups: file.allowed_groups || [], + document_id: file.document_id, + web_source_id: file.web_source_id, + web_page_id: file.web_page_id, + web_page_depth: file.web_page_depth, + web_child_count: file.web_child_count, + status: file.status, + error: file.error, })); const warnings: SearchWarning[] = Array.isArray(data.warnings) @@ -272,7 +333,7 @@ export const useGetSearchQuery = ( return useQuery( { - queryKey: ["search", queryData, query], + queryKey: ["search", queryData, query, displayOptions], placeholderData: (prev) => prev, staleTime: 0, queryFn: getFiles, diff --git a/frontend/app/api/queries/useGetSettingsQuery.ts b/frontend/app/api/queries/useGetSettingsQuery.ts index 7c73bddde4..3b54fff3db 100644 --- a/frontend/app/api/queries/useGetSettingsQuery.ts +++ b/frontend/app/api/queries/useGetSettingsQuery.ts @@ -106,6 +106,7 @@ export interface Settings { show_vlm_settings?: boolean; local_vlm_models?: string[]; show_shared_upload_toggle?: boolean; + show_url_connector?: boolean; show_workspace_oauth_overrides?: boolean; segment_write_key?: string; environment?: string; diff --git a/frontend/app/api/queries/useGetWebsiteSourceQuery.ts b/frontend/app/api/queries/useGetWebsiteSourceQuery.ts new file mode 100644 index 0000000000..8523fe034b --- /dev/null +++ b/frontend/app/api/queries/useGetWebsiteSourceQuery.ts @@ -0,0 +1,47 @@ +import { + type UseQueryOptions, + useQuery, + useQueryClient, +} from "@tanstack/react-query"; +import type { WebsiteSource } from "@/components/website-pages/types"; + +const PROCESSING_SOURCE_POLL_INTERVAL_MS = 2_000; + +export const websiteSourceQueryKey = (sourceId: string) => + ["websiteSource", sourceId] as const; + +async function getWebsiteSource(sourceId: string): Promise { + const response = await fetch(`/api/connectors/url/sources/${sourceId}`); + if (!response.ok) { + const data = (await response.json().catch(() => ({}))) as { + detail?: string; + error?: string; + }; + throw new Error( + data.detail || data.error || "Website source was not found", + ); + } + return response.json(); +} + +export function useGetWebsiteSourceQuery( + sourceId: string, + options?: Omit, "queryKey" | "queryFn">, +) { + const queryClient = useQueryClient(); + + return useQuery( + { + queryKey: websiteSourceQueryKey(sourceId), + queryFn: () => getWebsiteSource(sourceId), + refetchInterval: (query) => + query.state.data?.status === "processing" + ? PROCESSING_SOURCE_POLL_INTERVAL_MS + : false, + refetchIntervalInBackground: true, + retry: false, + ...options, + }, + queryClient, + ); +} diff --git a/frontend/app/api/queries/useGetWebsiteSourcesQuery.ts b/frontend/app/api/queries/useGetWebsiteSourcesQuery.ts new file mode 100644 index 0000000000..a1804514b0 --- /dev/null +++ b/frontend/app/api/queries/useGetWebsiteSourcesQuery.ts @@ -0,0 +1,22 @@ +import { useQuery } from "@tanstack/react-query"; +import type { WebsiteSource } from "@/components/website-pages/types"; + +export const websiteSourcesQueryKey = ["websiteSources"] as const; + +async function getWebsiteSources(): Promise { + const response = await fetch("/api/connectors/url/sources"); + if (!response.ok) { + throw new Error("Website sources could not be loaded"); + } + const data = (await response.json()) as { sources?: WebsiteSource[] }; + return data.sources ?? []; +} + +export function useGetWebsiteSourcesQuery(enabled: boolean) { + return useQuery({ + queryKey: websiteSourcesQueryKey, + queryFn: getWebsiteSources, + enabled, + retry: false, + }); +} diff --git a/frontend/app/api/queries/useListFiles.ts b/frontend/app/api/queries/useListFiles.ts index a14306a104..909325fae7 100644 --- a/frontend/app/api/queries/useListFiles.ts +++ b/frontend/app/api/queries/useListFiles.ts @@ -79,7 +79,14 @@ export const useListFiles = ( embedding_dimensions: f.embedding_dimensions as number | undefined, allowed_users: (f.allowed_users as string[]) || [], allowed_groups: (f.allowed_groups as string[]) || [], - status: "active" as const, + status: (f.status as File["status"]) || "active", + error: f.error as string | undefined, + document_id: f.document_id as string | undefined, + web_source_id: f.web_source_id as string | undefined, + web_page_id: f.web_page_id as string | undefined, + web_page_depth: + typeof f.web_page_depth === "number" ? f.web_page_depth : 0, + web_child_count: f.web_child_count as number | undefined, }), ); diff --git a/frontend/app/knowledge/chunks/page.test.tsx b/frontend/app/knowledge/chunks/page.test.tsx index 153d0bfdaf..0db63e546d 100644 --- a/frontend/app/knowledge/chunks/page.test.tsx +++ b/frontend/app/knowledge/chunks/page.test.tsx @@ -9,13 +9,14 @@ */ import { screen } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; import { HttpResponse, http } from "msw"; import { describe, expect, it } from "vitest"; import type { ChunkResult } from "@/app/api/queries/useGetSearchQuery"; import { authPresets } from "@/test-utils/fixtures/auth"; import { server } from "@/test-utils/msw/server"; import { renderWithProviders, waitFor } from "@/test-utils/render"; -import { setMockLocation } from "@/test-utils/router"; +import { mockRouter, setMockLocation } from "@/test-utils/router"; import ProtectedChunksPage from "./page"; function chunk(overrides: Partial = {}): ChunkResult { @@ -57,7 +58,7 @@ describe("ChunksPage — highlight wiring", () => { }); await waitFor(() => expect(calls.length).toBeGreaterThanOrEqual(1)); - expect(calls.every((c) => c.query === "*")).toBe(true); + expect(calls.every((call) => call.query === "*")).toBe(true); }); it("fires a second request with the search query when ?q= is present", async () => { @@ -74,10 +75,9 @@ describe("ChunksPage — highlight wiring", () => { auth: authPresets.admin, }); - // Two requests: wildcard (all chunks) + the search query (highlights). await waitFor(() => expect(calls.length).toBeGreaterThanOrEqual(2)); - expect(calls.some((c) => c.query === "*")).toBe(true); - expect(calls.some((c) => c.query === "brown fox")).toBe(true); + expect(calls.some((call) => call.query === "*")).toBe(true); + expect(calls.some((call) => call.query === "brown fox")).toBe(true); }); it("fires both wildcard and search requests when only some chunks match the search query", async () => { @@ -100,7 +100,6 @@ describe("ChunksPage — highlight wiring", () => { warnings: [], }); } - // Only c1 matches the search query — c2 must still survive in the merged result. return HttpResponse.json({ results: [ chunk({ @@ -119,7 +118,6 @@ describe("ChunksPage — highlight wiring", () => { auth: authPresets.admin, }); - // Two requests must be fired: wildcard (full chunk list) + "fox" (highlights only). await waitFor(() => expect(queriesSeen.length).toBeGreaterThanOrEqual(2)); expect(queriesSeen).toContain("*"); expect(queriesSeen).toContain("fox"); @@ -165,6 +163,60 @@ describe("ChunksPage — highlight wiring", () => { }); }); +describe("ChunksPage — URL child pages", () => { + it("loads URL child chunks by document id and keeps the debounced query in the URL", async () => { + const user = userEvent.setup(); + setMockLocation({ + pathname: "/knowledge/chunks", + searchParams: { document_id: "web-page-1", web_source_id: "source-1" }, + }); + + renderWithProviders(, { + providers: ["auth", "knowledgeFilter"], + auth: authPresets.noAuthMode, + handlers: [ + http.post("/api/search", () => + HttpResponse.json({ + results: [ + { + filename: "A website page", + document_id: "web-page-1", + mimetype: "text/html", + text: "The crawled page content", + chunk_id: "chunk-1", + score: 1, + page: 1, + file_size: 123, + owner: "anonymous", + }, + ], + warnings: [], + }), + ), + ], + }); + + expect(await screen.findByText("The crawled page content")).toBeVisible(); + expect( + screen.getByRole("heading", { name: "A website page" }), + ).toBeVisible(); + expect(screen.getByText("Total chunks")).toBeVisible(); + + await user.type(screen.getByPlaceholderText("Search chunks…"), "crawled"); + await waitFor( + () => + expect(mockRouter.replace).toHaveBeenCalledWith( + "/knowledge/chunks?document_id=web-page-1&web_source_id=source-1&q=crawled", + { scroll: false }, + ), + { timeout: 1_000 }, + ); + + await user.click(screen.getAllByRole("button")[0]); + expect(mockRouter.back).toHaveBeenCalledOnce(); + }); +}); + /** * Serve a minimal /api/search response carrying per-chunk access-control * fields. The query aggregates these onto the File object, making them visible diff --git a/frontend/app/knowledge/chunks/page.tsx b/frontend/app/knowledge/chunks/page.tsx index bfe641c0ce..0283bd10ff 100644 --- a/frontend/app/knowledge/chunks/page.tsx +++ b/frontend/app/knowledge/chunks/page.tsx @@ -3,7 +3,10 @@ import { ArrowLeft, Loader2, Search } from "lucide-react"; import { useRouter, useSearchParams } from "next/navigation"; import { Suspense, useCallback, useEffect, useRef, useState } from "react"; -import { useFileScopedChunksQuery } from "@/app/api/queries/useFileScopedChunksQuery"; +import { + useDocumentScopedChunksQuery, + useFileScopedChunksQuery, +} from "@/app/api/queries/useFileScopedChunksQuery"; import { FileChunksPanel } from "@/components/file-chunks-panel"; import { ProtectedRoute } from "@/components/protected-route"; import { Button } from "@/components/ui/button"; @@ -21,6 +24,8 @@ function ChunksPageContent() { const router = useRouter(); const searchParams = useSearchParams(); const filename = searchParams.get("filename"); + const documentId = searchParams.get("document_id"); + const webSourceId = searchParams.get("web_source_id"); // initialQuery comes from ?q= (set when navigating from the knowledge page). const initialQuery = searchParams.get("q") ?? ""; @@ -42,14 +47,17 @@ function ChunksPageContent() { debounceRef.current = setTimeout(() => { setSearchQuery(value); // Keep the URL in sync so the user can share/refresh the page. - const params = new URLSearchParams({ filename: filename ?? "" }); + const params = new URLSearchParams(); + if (filename) params.set("filename", filename); + if (documentId) params.set("document_id", documentId); + if (webSourceId) params.set("web_source_id", webSourceId); if (value.trim() && value.trim() !== "*") params.set("q", value.trim()); router.replace(`/knowledge/chunks?${params.toString()}`, { scroll: false, }); }, HIGHLIGHT_DEBOUNCE_MS); }, - [filename, router], + [documentId, filename, router, webSourceId], ); // Clean up any pending debounce on unmount. @@ -60,12 +68,17 @@ function ChunksPageContent() { [], ); - const { file: fileData } = useFileScopedChunksQuery( + const filenameQuery = useFileScopedChunksQuery( filename, searchQuery || undefined, ); + const documentQuery = useDocumentScopedChunksQuery( + documentId, + searchQuery || undefined, + ); + const { file: fileData } = documentId ? documentQuery : filenameQuery; - if (!filename) { + if (!filename && !documentId) { return (
@@ -97,14 +110,17 @@ function ChunksPageContent() {

- {filename.replace(/\.[^/.]+$/, "")} + {(filename || fileData?.filename || "Website page").replace( + /\.[^/.]+$/, + "", + )}

@@ -112,7 +128,8 @@ function ChunksPageContent() {
{ }); }); +describe("buildWebsitePagesUrl", () => { + it("carries a non-wildcard knowledge search into the website page view", () => { + expect(buildWebsitePagesUrl("source-1", "baseball news")).toBe( + "/knowledge?website=source-1&q=baseball+news", + ); + }); + + it("omits empty and wildcard search terms", () => { + expect(buildWebsitePagesUrl("source-1", "")).toBe( + "/knowledge?website=source-1", + ); + expect(buildWebsitePagesUrl("source-1", "*")).toBe( + "/knowledge?website=source-1", + ); + }); +}); + describe("getOwnerLabel", () => { // Minimal File-shaped stubs — only the fields getOwnerLabel reads. const f = (partial: Partial) => partial as File; diff --git a/frontend/app/knowledge/page.tsx b/frontend/app/knowledge/page.tsx index 624db4f1bc..18b57e80a1 100644 --- a/frontend/app/knowledge/page.tsx +++ b/frontend/app/knowledge/page.tsx @@ -5,23 +5,33 @@ import { type CheckboxSelectionCallbackParams, type ColDef, type ColumnState, - type GetRowIdParams, - themeQuartz, type ValueFormatterParams, type ValueGetterParams, } from "ag-grid-community"; import { AgGridReact, type CustomCellRendererProps } from "ag-grid-react"; -import { AlertTriangle, Cloud, FileIcon, Globe, RefreshCw } from "lucide-react"; -import { useRouter } from "next/navigation"; -import { useCallback, useEffect, useMemo, useRef, useState } from "react"; -import { KnowledgeDropdown } from "@/components/knowledge-dropdown"; +import { + AlertTriangle, + ChevronRight, + Cloud, + FileIcon, + RefreshCw, +} from "lucide-react"; +import { useRouter, useSearchParams } from "next/navigation"; +import { + Suspense, + useCallback, + useEffect, + useMemo, + useRef, + useState, +} from "react"; +import { KnowledgeDataTable } from "@/components/knowledge-data-table"; +import { KnowledgeUrlIcon } from "@/components/knowledge-url-icon"; import { ProtectedRoute } from "@/components/protected-route"; import { Banner, BannerIcon, BannerTitle } from "@/components/ui/banner"; -import { Button } from "@/components/ui/button"; import { useOpenTaskMenu } from "@/contexts/console-status-context"; import { useKnowledgeFilter } from "@/contexts/knowledge-filter-context"; import { useTask } from "@/contexts/task-context"; -import { trackButton } from "@/lib/analytics"; import { isFileCancelled } from "@/lib/task-error-display"; import { EMPTY_SEARCH_RESULT, @@ -39,13 +49,13 @@ import { KnowledgeBatchActionsBar } from "@/components/knowledge-batch-actions-b import { KnowledgePaginationFooter } from "@/components/knowledge-pagination-footer"; import { KnowledgeSearchBar } from "@/components/knowledge-search-bar"; import { KnowledgeSearchInput } from "@/components/knowledge-search-input"; -import { RequirePermission } from "@/components/require-permission"; import { StatusBadge } from "@/components/ui/status-badge"; import { Tooltip, TooltipContent, TooltipTrigger, } from "@/components/ui/tooltip"; +import { WebsitePagesView } from "@/components/website-pages-view"; import { useIsCloudBrand } from "@/contexts/brand-context"; import { getConnectorDescriptor } from "@/lib/connectors/registry"; import { formatFileSize } from "@/lib/file-format"; @@ -60,14 +70,8 @@ import { DeleteConfirmationDialog, formatFilesToDelete, } from "../../components/delete-confirmation-dialog"; -import { SyncConfirmDialog } from "../../components/sync-confirm-dialog"; import { useDeleteDocument } from "../api/mutations/useDeleteDocument"; -import { useRefreshOpenragDocs } from "../api/mutations/useRefreshOpenragDocs"; -import { - type SyncAllPreviewResponse, - useSyncAllConnectors, - useSyncAllConnectorsPreview, -} from "../api/mutations/useSyncConnector"; +import { useDeleteWebsiteSourcePageMutation } from "../api/mutations/useWebsiteSourceMutation"; function sameFileSelection(a: File[], b: File[]): boolean { if (a.length !== b.length) { @@ -107,14 +111,14 @@ function pruneNonDeletableGridSelection( // Function to get the appropriate icon for a connector type export function getSourceIcon(connectorType?: string) { + if (connectorType === "openrag_docs" || connectorType === "url") { + return ; + } if (connectorType) { const Icon = getConnectorDescriptor(connectorType)?.Icon; if (Icon) return ; } switch (connectorType) { - case "openrag_docs": - case "url": - return ; case "s3": return ; default: @@ -266,6 +270,18 @@ export function buildChunksUrl( return `/knowledge/chunks?${params.toString()}`; } +export function buildWebsitePagesUrl( + sourceId: string, + effectiveSearchText: string, +): string { + const params = new URLSearchParams({ website: sourceId }); + const trimmed = effectiveSearchText.trim(); + if (trimmed && trimmed !== "*") { + params.set("q", trimmed); + } + return `/knowledge?${params.toString()}`; +} + const AVG_SCORE_TOOLTIP = "Average relevance score across the matched chunks of this file. Higher means a stronger match — sort by this column to rank results by relevance."; @@ -442,14 +458,7 @@ function SearchPage() { const seenFailedFileKeysRef = useRef>(new Set()); const deleteDocumentMutation = useDeleteDocument(); - const syncAllConnectorsMutation = useSyncAllConnectors(); - const syncAllPreviewMutation = useSyncAllConnectorsPreview(); - const refreshOpenragDocsMutation = useRefreshOpenragDocs(); - const [syncDialogOpen, setSyncDialogOpen] = useState(false); - const [syncPreview, setSyncPreview] = useState( - null, - ); - + const deleteWebsiteSourceMutation = useDeleteWebsiteSourcePageMutation(); const [currentPage, setCurrentPage] = useState(1); const [currentPageSize, setCurrentPageSize] = useState(25); @@ -460,45 +469,6 @@ function SearchPage() { cursorCacheRef.current = new Map(); } - const handleOpenSyncDialog = useCallback(async () => { - setSyncPreview(null); - setSyncDialogOpen(true); - try { - const preview = await syncAllPreviewMutation.mutateAsync(); - setSyncPreview(preview); - } catch (error) { - setSyncDialogOpen(false); - toast.error( - error instanceof Error ? error.message : "Failed to preview sync", - ); - } - }, [syncAllPreviewMutation]); - - const handleConfirmSync = useCallback(async () => { - try { - const result = await syncAllConnectorsMutation.mutateAsync(); - if (result.status === "no_files") { - toast.info( - result.message || - "No cloud files to sync. Add files from cloud connectors first.", - ); - } else if ( - result.synced_connectors && - result.synced_connectors.length > 0 - ) { - toast.success( - `Sync started for ${result.synced_connectors.join(", ")}. Check task notifications for progress.`, - ); - } else if (result.errors && result.errors.length > 0) { - toast.error("Some connectors failed to sync"); - } - } catch (error) { - toast.error( - error instanceof Error ? error.message : "Failed to sync connectors", - ); - } - }, [syncAllConnectorsMutation]); - useEffect(() => { refreshTasks(); }, [refreshTasks]); @@ -650,9 +620,12 @@ function SearchPage() { isLoading: isSearchLoading, error: searchError, isError: isSearchError, - } = useGetSearchQuery(queryOverride, parsedFilterData, { - enabled: !isWildcardQuery, - }); + } = useGetSearchQuery( + queryOverride, + parsedFilterData, + { enabled: !isWildcardQuery }, + { resultMode: "knowledge_sources" }, + ); const { files: searchFiles, warnings: searchWarnings } = searchData as SearchResult; @@ -712,8 +685,7 @@ function SearchPage() { ); }); }); - const hasOpenragRefreshCue = - refreshOpenragDocsMutation.isPending || hasOpenragRefreshCueFromTasks; + const hasOpenragRefreshCue = hasOpenragRefreshCueFromTasks; // Show toast notification for search errors useEffect(() => { @@ -841,6 +813,10 @@ function SearchPage() { (typeof value === "string" ? value.split("::")[0] : ""); const status = data?.status || "active"; const isActive = status === "active"; + const isParentWebsiteSource = + data?.connector_type === "url" && + Boolean(data.web_source_id) && + !data.web_page_id; const showOpenragSourceAnimation = isOpenragDocsRow(data) && hasOpenragRefreshCue; return ( @@ -862,6 +838,15 @@ function SearchPage() { )} onClick={() => { if (!isActive) return; + if (data?.connector_type === "url" && data.web_source_id) { + router.push( + buildWebsitePagesUrl( + data.web_source_id, + effectiveSearchText, + ), + ); + return; + } router.push(buildChunksUrl(filename, effectiveSearchText)); }} > @@ -883,6 +868,12 @@ function SearchPage() { {filename} + {isParentWebsiteSource && ( +
); @@ -1006,6 +997,10 @@ function SearchPage() { ); } @@ -1062,9 +1057,13 @@ function SearchPage() { try { const deleteResults = await Promise.allSettled( rowsToDelete.map((row) => - deleteDocumentMutation.mutateAsync({ - filename: resolveDeleteFilename(row), - }), + row.web_source_id + ? deleteWebsiteSourceMutation.mutateAsync({ + sourceId: row.web_source_id, + }) + : deleteDocumentMutation.mutateAsync({ + filename: resolveDeleteFilename(row), + }), ), ); @@ -1077,18 +1076,7 @@ function SearchPage() { ]); const deleted = deleteResults.filter( - ( - result, - ): result is PromiseFulfilledResult< - Awaited> - > => - result.status === "fulfilled" && - (result.value.deleted_chunks || 0) > 0, - ); - const noChunks = deleteResults.filter( - (result) => - result.status === "fulfilled" && - (result.value.deleted_chunks || 0) === 0, + (result) => result.status === "fulfilled", ); const failed = deleteResults.filter( (result): result is PromiseRejectedResult => @@ -1099,16 +1087,6 @@ function SearchPage() { toast.success( `Deleted ${deleted.length} document${deleted.length > 1 ? "s" : ""}`, ); - } else if (failed.length === 0) { - toast.warning( - "No document chunks were deleted. Files may be missing or not deletable in your current context.", - ); - } - - if (noChunks.length > 0 && deleted.length > 0) { - toast.warning( - `${noChunks.length} selected file${noChunks.length > 1 ? "s had" : " had"} no matching chunks.`, - ); } if (failed.length > 0) { @@ -1187,82 +1165,10 @@ function SearchPage() {
) : ( - /* Search Input Area */ -
- {!isNarrow && ( - - )} - - - - - - {selectedRows.length > 0 && ( - - )} -
- -
-
+ setShowBulkDeleteDialog(true)} + /> )} {!isWildcardQuery && searchWarnings.length > 0 && (
@@ -1301,75 +1207,25 @@ function SearchPage() { })}
)} - {isCloudBrand ? ( -
- []} - defaultColDef={defaultColDef} - loading={isLoading || deleteDocumentMutation.isPending} - ref={gridRef} - theme={themeQuartz.withParams({ browserColorScheme: "inherit" })} - rowData={gridRows} - rowSelection="multiple" - getRowId={(params: GetRowIdParams) => - getFileIdentity(params.data) - } - isRowSelectable={(params) => isDeletableKnowledgeRow(params.data)} - domLayout="normal" - onGridReady={handleGridReady} - onGridPreDestroyed={handleGridPreDestroyed} - onSelectionChanged={onSelectionChanged} - onSortChanged={onSortChanged} - headerHeight={64} - rowHeight={64} - noRowsOverlayComponent={() => ( -
-
- No knowledge -
-
- Add files from local or your preferred cloud. -
-
- )} - /> -
- ) : ( -
- []} - defaultColDef={defaultColDef} - loading={isLoading || deleteDocumentMutation.isPending} - ref={gridRef} - theme={themeQuartz.withParams({ browserColorScheme: "inherit" })} - rowData={gridRows} - rowSelection="multiple" - rowMultiSelectWithClick={false} - suppressRowClickSelection={true} - getRowId={(params: GetRowIdParams) => - getFileIdentity(params.data) - } - isRowSelectable={(params) => isDeletableKnowledgeRow(params.data)} - domLayout="normal" - onGridReady={handleGridReady} - onGridPreDestroyed={handleGridPreDestroyed} - onSelectionChanged={onSelectionChanged} - onSortChanged={onSortChanged} - noRowsOverlayComponent={() => ( -
-
- No knowledge -
-
- Add files from local or your preferred cloud. -
-
- )} - /> -
- )} + getFileIdentity(params.data)} + isRowSelectable={(params) => isDeletableKnowledgeRow(params.data)} + onGridReady={handleGridReady} + onGridPreDestroyed={handleGridPreDestroyed} + onSelectionChanged={onSelectionChanged} + onSortChanged={onSortChanged} + /> 1 ? "s" : ""}?`} confirmText={selectedRows.length > 1 ? "Delete all" : "Delete"} onConfirm={handleBulkDelete} - isLoading={deleteDocumentMutation.isPending} + isLoading={ + deleteDocumentMutation.isPending || + deleteWebsiteSourceMutation.isPending + } >

This will remove all chunks and data associated with these documents. @@ -1400,28 +1259,33 @@ function SearchPage() {

Documents to be deleted:

{formatFilesToDelete(selectedRows)} - - ); } -export default function ProtectedSearchPage() { +function ProtectedSearchPageContent() { + const searchParams = useSearchParams(); + const websiteId = searchParams.get("website"); + const websiteSearch = searchParams.get("q") ?? ""; return ( - + {websiteId ? ( + + ) : ( + + )} ); } + +export default function ProtectedSearchPage() { + return ( + + + + ); +} diff --git a/frontend/app/settings/_components/connector-card.tsx b/frontend/app/settings/_components/connector-card.tsx index 1b2ed125fc..96c2cc47df 100644 --- a/frontend/app/settings/_components/connector-card.tsx +++ b/frontend/app/settings/_components/connector-card.tsx @@ -25,6 +25,7 @@ export interface Connector { status?: string; connectionId?: string; requiresOAuth?: boolean; + alwaysConnected?: boolean; } interface ConnectorCardProps { @@ -33,7 +34,7 @@ interface ConnectorCardProps { isDisconnecting: boolean; onConnect: (connector: Connector) => void; onDisconnect: (connector: Connector) => void; - onNavigateToKnowledge: (connector: Connector) => void; + onAddKnowledge: (connector: Connector) => void; /** Optional: open a connector-specific settings/edit dialog */ onConfigure?: (connector: Connector) => void; } @@ -44,7 +45,7 @@ export default function ConnectorCard({ isDisconnecting, onConnect, onDisconnect, - onNavigateToKnowledge, + onAddKnowledge, onConfigure, }: ConnectorCardProps) { const isCloudBrand = useIsCloudBrand(); @@ -56,7 +57,8 @@ export default function ConnectorCard({ ]); const canUpload = can("knowledge:upload"); const isConnected = - connector.status === "connected" && connector.connectionId; + connector.status === "connected" && + (connector.connectionId || connector.alwaysConnected); const isConfigured = connector.status === "configured"; return ( @@ -123,7 +125,7 @@ export default function ConnectorCard({
- {canCreate && ( + {canCreate && !connector.alwaysConnected && ( )} - {canDisconnect && ( + {canDisconnect && !connector.alwaysConnected && (
)} + {/* Credential-Based Connectors Section */} + {managedConnectors.length > 0 && ( +
+

Built-in Connectors

+
+ {managedConnectors.map((connector) => ( + + ))} +
+
+ )} + {/* Credential-Based Connectors Section */} {(connectorsLoading || credentialConnectors.length > 0) && (
@@ -185,7 +223,7 @@ export default function ConnectorCards() { } onConnect={handleConnect} onDisconnect={handleDisconnect} - onNavigateToKnowledge={navigateToKnowledgePage} + onAddKnowledge={handleAddKnowledge} onConfigure={getConfigureHandler(connector)} /> )) @@ -215,6 +253,10 @@ export default function ConnectorCards() { ); })} + {showUrlConnector && ( + + )} + { diff --git a/frontend/components/file-chunks-panel.tsx b/frontend/components/file-chunks-panel.tsx index 6d075aaeb1..2b8e341485 100644 --- a/frontend/components/file-chunks-panel.tsx +++ b/frontend/components/file-chunks-panel.tsx @@ -2,7 +2,10 @@ import { Check, Copy, Loader2 } from "lucide-react"; import { useDeferredValue, useEffect, useMemo, useRef, useState } from "react"; -import { useFileScopedChunksQuery } from "@/app/api/queries/useFileScopedChunksQuery"; +import { + useDocumentScopedChunksQuery, + useFileScopedChunksQuery, +} from "@/app/api/queries/useFileScopedChunksQuery"; import type { ChunkResult } from "@/app/api/queries/useGetSearchQuery"; import { HighlightedText } from "@/components/highlighted-text"; import { KnowledgeSearchInput } from "@/components/knowledge-search-input"; @@ -21,6 +24,7 @@ function compareChunksByDocumentOrder(a: ChunkResult, b: ChunkResult): number { export interface FileChunksPanelProps { filename: string; + documentId?: string | null; /** Compact layout for dialogs (e.g. ingest review). */ compact?: boolean; /** When false, show metadata only (no chunk body). Default true. */ @@ -160,6 +164,7 @@ function FileChunkCard({ */ export function FileChunksPanel({ filename, + documentId, compact = false, showContents = true, selectedPage, @@ -172,7 +177,9 @@ export function FileChunksPanel({ fillHeight = false, searchQuery, }: FileChunksPanelProps) { - const { file, isFetching } = useFileScopedChunksQuery(filename, searchQuery); + const filenameQuery = useFileScopedChunksQuery(filename, searchQuery); + const documentQuery = useDocumentScopedChunksQuery(documentId, searchQuery); + const { file, isFetching } = documentId ? documentQuery : filenameQuery; const allChunks = useMemo(() => { const sorted = [...(file?.chunks ?? [])].sort(compareChunksByDocumentOrder); return sorted.map((chunk, i) => ({ diff --git a/frontend/components/knowledge-actions-dropdown.test.tsx b/frontend/components/knowledge-actions-dropdown.test.tsx new file mode 100644 index 0000000000..2d4f02dd62 --- /dev/null +++ b/frontend/components/knowledge-actions-dropdown.test.tsx @@ -0,0 +1,113 @@ +import { screen } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { HttpResponse, http } from "msw"; +import { toast } from "sonner"; +import { describe, expect, it, vi } from "vitest"; +import { authPresets } from "@/test-utils/fixtures/auth"; +import { renderWithProviders, waitFor } from "@/test-utils/render"; +import { mockRouter } from "@/test-utils/router"; +import { KnowledgeActionsDropdown } from "./knowledge-actions-dropdown"; + +describe("KnowledgeActionsDropdown", () => { + it("syncs a website source and removes it after confirmation", async () => { + const user = userEvent.setup(); + let syncCalls = 0; + let deleteCalls = 0; + renderWithProviders( + , + { + providers: ["task"], + auth: authPresets.noAuthMode, + handlers: [ + http.post("/api/connectors/url/sources/source-1/sync", () => { + syncCalls += 1; + return HttpResponse.json({ run_id: "run-1" }); + }), + http.delete("/api/connectors/url/sources/source-1", () => { + deleteCalls += 1; + return HttpResponse.json({ success: true }); + }), + ], + }, + ); + + await user.click(screen.getByRole("button")); + await user.click( + await screen.findByRole("menuitem", { name: "View pages" }), + ); + expect(mockRouter.push).toHaveBeenCalledWith( + "/knowledge?website=source-1&q=baseball", + ); + + await user.click(screen.getByRole("button")); + await user.click(await screen.findByRole("menuitem", { name: "Sync" })); + await waitFor(() => expect(syncCalls).toBe(1)); + + await user.click(screen.getByRole("button")); + await user.click(await screen.findByRole("menuitem", { name: "Delete" })); + expect(await screen.findByText("Delete website source")).toBeVisible(); + expect(screen.getByText(/3 child pages/)).toBeVisible(); + + await user.click(screen.getByRole("button", { name: "Delete" })); + await waitFor(() => expect(deleteCalls).toBe(1)); + }); + + it("opens chunks for a regular document from the same action menu", async () => { + const user = userEvent.setup(); + renderWithProviders( + , + { providers: ["task"], auth: authPresets.noAuthMode }, + ); + + await user.click(screen.getByRole("button")); + await user.click( + await screen.findByRole("menuitem", { name: "View chunks" }), + ); + + expect(mockRouter.push).toHaveBeenCalledWith( + "/knowledge/chunks?filename=Product%20docs.pdf", + ); + }); + + it("shows the deletion failure instead of silently closing the website dialog", async () => { + const user = userEvent.setup(); + const toastError = vi.spyOn(toast, "error").mockImplementation(vi.fn()); + try { + renderWithProviders( + , + { + providers: ["task"], + auth: authPresets.noAuthMode, + handlers: [ + http.delete("/api/connectors/url/sources/source-1", () => + HttpResponse.json( + { detail: "Website is still syncing" }, + { status: 409 }, + ), + ), + ], + }, + ); + + await user.click(screen.getByRole("button")); + await user.click(await screen.findByRole("menuitem", { name: "Delete" })); + await user.click(screen.getByRole("button", { name: "Delete" })); + + await waitFor(() => + expect(toastError).toHaveBeenCalledWith("Website is still syncing"), + ); + } finally { + toastError.mockRestore(); + } + }); +}); diff --git a/frontend/components/knowledge-actions-dropdown.tsx b/frontend/components/knowledge-actions-dropdown.tsx index 91ea6a36a0..11e67b6220 100644 --- a/frontend/components/knowledge-actions-dropdown.tsx +++ b/frontend/components/knowledge-actions-dropdown.tsx @@ -10,6 +10,10 @@ import { useSyncConnector, useSyncConnectorPreview, } from "@/app/api/mutations/useSyncConnector"; +import { + useDeleteWebsiteSourcePageMutation, + useSyncWebsiteSourceMutation, +} from "@/app/api/mutations/useWebsiteSourceMutation"; import { useGetConnectorsQuery } from "@/app/api/queries/useGetConnectorsQuery"; import { DropdownMenu, @@ -35,6 +39,10 @@ import { Button } from "./ui/button"; interface KnowledgeActionsDropdownProps { filename: string; connectorType?: string; + webSourceId?: string; + webPageId?: string; + webChildCount?: number; + searchQuery?: string; } // Cloud connector types that support sync @@ -47,10 +55,16 @@ const CLOUD_CONNECTOR_TYPES = new Set([ export const KnowledgeActionsDropdown = ({ filename, connectorType, + webSourceId, + webPageId, + webChildCount, + searchQuery, }: KnowledgeActionsDropdownProps) => { const { refreshTasks } = useTask(); const [showDeleteDialog, setShowDeleteDialog] = useState(false); const deleteDocumentMutation = useDeleteDocument(); + const syncWebsiteSourceMutation = useSyncWebsiteSourceMutation(); + const deleteWebsiteSourceMutation = useDeleteWebsiteSourcePageMutation(); const syncConnectorMutation = useSyncConnector(); const syncPreviewMutation = useSyncConnectorPreview(); const [syncDialogOpen, setSyncDialogOpen] = useState(false); @@ -127,6 +141,32 @@ export const KnowledgeActionsDropdown = ({ } }; + const syncWebsite = async () => { + if (!webSourceId) return; + try { + await syncWebsiteSourceMutation.mutateAsync({ sourceId: webSourceId }); + toast.success("Website sync started"); + } catch (error) { + toast.error( + error instanceof Error ? error.message : "Could not start website sync", + ); + } + }; + + const deleteWebsite = async () => { + if (!webSourceId) return; + try { + await deleteWebsiteSourceMutation.mutateAsync({ sourceId: webSourceId }); + toast.success("Website source deleted"); + } catch (error) { + toast.error( + error instanceof Error + ? error.message + : "Could not delete website source", + ); + } + }; + return ( <> @@ -136,112 +176,174 @@ export const KnowledgeActionsDropdown = ({ - { - trackButton({ - CTA: "View Chunks", - elementId: "view-chunks-button", - namespace: "knowledge", - }); - router.push( - `/knowledge/chunks?filename=${encodeURIComponent(filename)}`, - ); - }} - > - View chunks - - {isCloudFile && ( - - - -
- { - if (!isConnected) { - e.preventDefault(); - return; - } - trackButton({ - CTA: "Sync File", - elementId: "sync-file-button", - namespace: "knowledge", - payload: { connector_type: connectorType }, - }); - handleOpenSyncDialog(); - }} - > - {syncConnectorMutation.isPending || - syncPreviewMutation.isPending ? ( - <> - - Syncing... - - ) : ( - <> - - Sync - - )} - {!isConnected && ( - - )} - -
- - {!isConnected && ( - -

- {connectorType.charAt(0).toUpperCase() + - connectorType.slice(1)}{" "} - is not connected. Connect it in Settings to enable sync. -

- - )} - - + {webSourceId ? ( + <> + {!webPageId && ( + { + trackButton({ + CTA: "View Pages", + elementId: "view-website-pages-button", + namespace: "knowledge", + }); + const params = new URLSearchParams({ + website: webSourceId, + }); + const query = searchQuery?.trim(); + if (query && query !== "*") { + params.set("q", query); + } + router.push(`/knowledge?${params.toString()}`); + }} + > + View pages + + )} + + + Sync + + setShowDeleteDialog(true)} + > + Delete + + + ) : ( + <> + { + trackButton({ + CTA: "View Chunks", + elementId: "view-chunks-button", + namespace: "knowledge", + }); + router.push( + `/knowledge/chunks?filename=${encodeURIComponent(filename)}`, + ); + }} + > + View chunks + + {isCloudFile && ( + + + +
+ { + if (!isConnected) { + e.preventDefault(); + return; + } + trackButton({ + CTA: "Sync File", + elementId: "sync-file-button", + namespace: "knowledge", + payload: { connector_type: connectorType }, + }); + handleOpenSyncDialog(); + }} + > + {syncConnectorMutation.isPending || + syncPreviewMutation.isPending ? ( + <> + + Syncing... + + ) : ( + <> + + Sync + + )} + {!isConnected && ( + + )} + +
+ + {!isConnected && ( + +

+ {connectorType.charAt(0).toUpperCase() + + connectorType.slice(1)}{" "} + is not connected. Connect it in Settings to enable + sync. +

+ + )} + + + )} + + { + trackButton({ + CTA: "Delete Document", + elementId: "delete-document-button", + namespace: "knowledge", + }); + setShowDeleteDialog(true); + }} + > + Delete + + + )} - - { - trackButton({ - CTA: "Delete Document", - elementId: "delete-document-button", - namespace: "knowledge", - }); - setShowDeleteDialog(true); - }} - > - Delete - - -

- This will remove all chunks and data associated with this document. - This action cannot be undone. -

-

Document to be deleted:

- {formatFilesToDelete([{ filename }])} + {!webSourceId && ( +

+ This will remove all chunks and data associated with this document. + This action cannot be undone. +

+ )} + {webSourceId ? ( +

+ All of the website's indexed child pages and chunks will be + removed. +

+ ) : ( + <> +

Document to be deleted:

+ {formatFilesToDelete([{ filename }])} + + )} { + rows: T[]; + columnDefs: ColDef[]; + defaultColDef: ColDef; + gridRef?: RefObject | null>; + loading?: boolean; + isCloudBrand?: boolean; + rowSelection?: "single" | "multiple"; + getRowId: (params: GetRowIdParams) => string; + isRowSelectable?: (params: { data?: T }) => boolean; + onGridReady?: () => void; + onGridPreDestroyed?: () => void; + onSelectionChanged?: () => void; + onSortChanged?: () => void; + postSortRows?: PostSortRows; + emptyTitle?: string; + emptyDescription?: string; +} + +/** + * Shared Knowledge AG Grid shell. Column factories stay with their owning + * surface; grid sizing, theme, loading, focus behaviour and empty states do not. + */ +export function KnowledgeDataTable({ + rows, + columnDefs, + defaultColDef, + gridRef, + loading = false, + isCloudBrand = false, + rowSelection, + getRowId, + isRowSelectable, + onGridReady, + onGridPreDestroyed, + onSelectionChanged, + onSortChanged, + postSortRows, + emptyTitle = "No knowledge", + emptyDescription = "Add files from local or your preferred cloud.", +}: KnowledgeDataTableProps) { + return ( +
+ + className={cn("h-full w-full", isCloudBrand && "border")} + columnDefs={columnDefs} + defaultColDef={defaultColDef} + loading={loading} + ref={gridRef} + theme={themeQuartz.withParams({ browserColorScheme: "inherit" })} + rowData={rows} + rowSelection={rowSelection} + rowMultiSelectWithClick={false} + suppressRowClickSelection={rowSelection === "multiple"} + getRowId={getRowId} + isRowSelectable={isRowSelectable} + domLayout="normal" + onGridReady={onGridReady} + onGridPreDestroyed={onGridPreDestroyed} + onSelectionChanged={onSelectionChanged} + onSortChanged={onSortChanged} + postSortRows={postSortRows} + {...(isCloudBrand ? { headerHeight: 64, rowHeight: 64 } : {})} + noRowsOverlayComponent={() => ( +
+
+ {emptyTitle} +
+
+ {emptyDescription} +
+
+ )} + /> +
+ ); +} diff --git a/frontend/components/knowledge-dropdown.tsx b/frontend/components/knowledge-dropdown.tsx index c135cbb2cb..0f894dd5c5 100644 --- a/frontend/components/knowledge-dropdown.tsx +++ b/frontend/components/knowledge-dropdown.tsx @@ -13,9 +13,11 @@ import { useRouter } from "next/navigation"; import { useEffect, useRef, useState } from "react"; import { toast } from "sonner"; import type { File as SearchFile } from "@/app/api/queries/useGetSearchQuery"; +import { useGetSettingsQuery } from "@/app/api/queries/useGetSettingsQuery"; import { useGetTasksQuery } from "@/app/api/queries/useGetTasksQuery"; import { DuplicateHandlingDialog } from "@/components/duplicate-handling-dialog"; import { IngestReviewDialog } from "@/components/ingest-review"; +import { KnowledgeUrlIcon } from "@/components/knowledge-url-icon"; import { Button } from "@/components/ui/button"; import { Dialog, @@ -32,6 +34,7 @@ import { } from "@/components/ui/dropdown-menu"; import { Input } from "@/components/ui/input"; import { Label } from "@/components/ui/label"; +import { UrlSourceDialog } from "@/components/url-source-dialog"; import { useAuth } from "@/contexts/auth-context"; import { useIsCloudBrand } from "@/contexts/brand-context"; import { useTask } from "@/contexts/task-context"; @@ -105,7 +108,11 @@ const FolderIconWithColor = ({ className }: { className?: string }) => ( ); export function KnowledgeDropdown() { - const { runMode } = useAuth(); + const { runMode, isAuthenticated, isNoAuthMode } = useAuth(); + const { data: apiSettings } = useGetSettingsQuery({ + enabled: isAuthenticated || isNoAuthMode, + }); + const showUrlConnector = apiSettings?.show_url_connector ?? false; const { supportedExtensions, supportedExtensionSet } = useSupportedFileTypes(); const { can } = usePermissions(); @@ -120,6 +127,7 @@ export function KnowledgeDropdown() { }); const [mounted, setMounted] = useState(false); const [isMenuOpen, setIsMenuOpen] = useState(false); + const [urlDialogOpen, setUrlDialogOpen] = useState(false); const [showFolderDialog, setShowFolderDialog] = useState(false); const [showDuplicateDialog, setShowDuplicateDialog] = useState(false); const [uploadBatchSize, setUploadBatchSize] = useState(25); @@ -301,6 +309,15 @@ export function KnowledgeDropdown() { setMounted(true); }, []); + useEffect(() => { + if ( + showUrlConnector && + new URLSearchParams(window.location.search).get("add") === "url" + ) { + setUrlDialogOpen(true); + } + }, [showUrlConnector]); + const handleFileUpload = () => { fileInputRef.current?.click(); }; @@ -925,6 +942,15 @@ export function KnowledgeDropdown() { icon: FolderIconWithColor, onClick: () => folderInputRef.current?.click(), }, + ...(showUrlConnector + ? [ + { + label: "URL", + icon: KnowledgeUrlIcon, + onClick: () => setUrlDialogOpen(true), + }, + ] + : []), ...bucketConnectorItems, ...cloudConnectorItems, ]; @@ -1115,6 +1141,16 @@ export function KnowledgeDropdown() { taskIds={preview.taskIds} previewFiles={preview.files} /> + {showUrlConnector && ( + { + if (taskId) addTask(taskId, { connectorType: "url" }); + void queryClient.invalidateQueries({ queryKey: ["listFiles"] }); + }} + /> + )} ); } diff --git a/frontend/components/knowledge-search-bar.test.tsx b/frontend/components/knowledge-search-bar.test.tsx new file mode 100644 index 0000000000..a07e7a9889 --- /dev/null +++ b/frontend/components/knowledge-search-bar.test.tsx @@ -0,0 +1,78 @@ +import { screen } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { HttpResponse, http } from "msw"; +import { useState } from "react"; +import { describe, expect, it, vi } from "vitest"; +import { authPresets } from "@/test-utils/fixtures/auth"; +import { renderWithProviders } from "@/test-utils/render"; +import { KnowledgeSearchBar } from "./knowledge-search-bar"; + +function ControlledSearch({ + onSearch, + onClear, + onDelete, +}: { + onSearch(value: string): void; + onClear(): void; + onDelete(): void; +}) { + const [value, setValue] = useState("initial"); + return ( + { + setValue(next); + onSearch(next); + }} + onClear={onClear} + selectedCount={1} + onDeleteSelected={onDelete} + /> + ); +} + +describe("KnowledgeSearchBar", () => { + it("submits and clears a controlled search while retaining its standard actions", async () => { + const user = userEvent.setup(); + const onSearch = vi.fn(); + const onClear = vi.fn(); + const onDelete = vi.fn(); + renderWithProviders( + , + { + providers: ["brand", "knowledgeFilter", "task"], + auth: authPresets.admin, + brand: "oss", + handlers: [ + http.get("/api/upload_options", () => HttpResponse.json({})), + http.get("/api/connectors/aws_s3/defaults", () => + HttpResponse.json({}), + ), + http.get("/api/connectors/ibm_cos/defaults", () => + HttpResponse.json({}), + ), + http.get("/api/connectors/azure_blob/defaults", () => + HttpResponse.json({}), + ), + ], + }, + ); + + const input = screen.getByPlaceholderText("Search your documents..."); + await user.clear(input); + await user.type(input, "docs"); + await user.keyboard("{Enter}"); + expect(onSearch).toHaveBeenLastCalledWith("docs"); + + await user.click(screen.getByRole("button", { name: "Delete" })); + expect(onDelete).toHaveBeenCalledOnce(); + + await user.click(screen.getByRole("button", { name: "Clear search" })); + expect(onClear).toHaveBeenCalledOnce(); + expect(onSearch).toHaveBeenLastCalledWith(""); + }); +}); diff --git a/frontend/components/knowledge-search-bar.tsx b/frontend/components/knowledge-search-bar.tsx index ca8cc02c7a..358fd2d020 100644 --- a/frontend/components/knowledge-search-bar.tsx +++ b/frontend/components/knowledge-search-bar.tsx @@ -1,5 +1,5 @@ -import { ArrowRight, Download, RefreshCw, Search, X } from "lucide-react"; -import { type ChangeEvent, type FormEvent, useCallback, useState } from "react"; +import { Download, RefreshCw, X } from "lucide-react"; +import { type ReactNode, useCallback, useState } from "react"; import { toast } from "sonner"; import { useRefreshOpenragDocs } from "@/app/api/mutations/useRefreshOpenragDocs"; import { @@ -7,15 +7,41 @@ import { useSyncAllConnectors, useSyncAllConnectorsPreview, } from "@/app/api/mutations/useSyncConnector"; +import { KnowledgeDropdown } from "@/components/knowledge-dropdown"; +import { KnowledgeSearchInput } from "@/components/knowledge-search-input"; +import { KnowledgeSearchToolbar } from "@/components/knowledge-search-toolbar"; import { RequirePermission } from "@/components/require-permission"; import { Button } from "@/components/ui/button"; +import { useIsCloudBrand } from "@/contexts/brand-context"; import { useKnowledgeFilter } from "@/contexts/knowledge-filter-context"; +import { trackButton } from "@/lib/analytics"; import { cn } from "@/lib/utils"; -import { KnowledgeDropdown } from "./knowledge-dropdown"; import { filterAccentClasses } from "./knowledge-filter-panel"; import { SyncConfirmDialog } from "./sync-confirm-dialog"; -export const KnowledgeSearchBar = () => { +type KnowledgeSearchBarProps = { + value?: string; + onSearch?: (query: string) => void; + onClear?: () => void; + placeholder?: string; + nonCloudPlaceholder?: string; + rightActions?: ReactNode; + selectedCount?: number; + onDeleteSelected?: () => void; +}; + +export const KnowledgeSearchBar = ({ + value, + onSearch, + onClear, + placeholder = "Search knowledge", + nonCloudPlaceholder, + rightActions, + selectedCount = 0, + onDeleteSelected, +}: KnowledgeSearchBarProps) => { + const isCloudBrand = useIsCloudBrand(); + const controlled = onSearch != null; const { selectedFilter, setSelectedFilter, @@ -26,23 +52,31 @@ export const KnowledgeSearchBar = () => { const [searchQueryInput, setSearchQueryInput] = useState(queryOverride || ""); const [prevQueryOverride, setPrevQueryOverride] = useState(queryOverride); - if (queryOverride !== prevQueryOverride) { + if (!controlled && queryOverride !== prevQueryOverride) { setPrevQueryOverride(queryOverride); setSearchQueryInput(queryOverride); } - const handleSearch = useCallback( - (e?: FormEvent) => { - if (e) e.preventDefault(); - setQueryOverride(searchQueryInput.trim()); - }, - [searchQueryInput, setQueryOverride], - ); + const displayedValue = controlled ? (value ?? "") : searchQueryInput; + + const handleSearch = useCallback(() => { + const next = displayedValue.trim(); + if (controlled) { + onSearch(next); + } else { + setQueryOverride(next); + } + }, [controlled, displayedValue, onSearch, setQueryOverride]); const handleReset = useCallback(() => { - setSearchQueryInput(""); - setQueryOverride(""); - }, [setQueryOverride]); + if (controlled) { + onClear?.(); + onSearch(""); + } else { + setSearchQueryInput(""); + setQueryOverride(""); + } + }, [controlled, onClear, onSearch, setQueryOverride]); const syncAllConnectorsMutation = useSyncAllConnectors(); const syncAllPreviewMutation = useSyncAllConnectorsPreview(); @@ -91,135 +125,191 @@ export const KnowledgeSearchBar = () => { } }, [syncAllConnectorsMutation]); - return ( -
-
- {!!selectedFilter?.name && ( -
- - {selectedFilter.name} - - -
- )} - -
- -
- -
- ) => - setSearchQueryInput(e.target.value) - } - className="h-full w-full bg-transparent text-sm text-foreground placeholder:text-[hsl(var(--placeholder))] focus:outline-hidden focus:ring-0" - /> - {searchQueryInput && ( - - )} - -
+ const defaultRightActions = isCloudBrand ? ( + <> + + - - + + - -
- -
+ }} + > + {refreshOpenragDocsMutation.isPending + ? "Refreshing docs..." + : "Fetch latest docs"} + + + {selectedCount > 0 && ( + + )} +
+
- + ); + + return ( + <> + + + {selectedFilter.name} + + +
+ ) : undefined + } + rightActions={rightActions ?? defaultRightActions} + nonCloudSearch={ + controlled ? ( + + ) : ( + + ) + } /> - + {!controlled && ( + + )} + ); }; diff --git a/frontend/components/knowledge-search-input.tsx b/frontend/components/knowledge-search-input.tsx index 08133a6c6b..4cef1f9513 100644 --- a/frontend/components/knowledge-search-input.tsx +++ b/frontend/components/knowledge-search-input.tsx @@ -142,6 +142,7 @@ export function KnowledgeSearchInput({ variant="ghost" className="h-full rounded-sm px-1.5! py-0!" type="button" + aria-label="Clear search" onClick={handleClear} > diff --git a/frontend/components/knowledge-search-toolbar.test.tsx b/frontend/components/knowledge-search-toolbar.test.tsx new file mode 100644 index 0000000000..1e3f69a24f --- /dev/null +++ b/frontend/components/knowledge-search-toolbar.test.tsx @@ -0,0 +1,45 @@ +import { screen } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { useState } from "react"; +import { describe, expect, it, vi } from "vitest"; +import { authPresets } from "@/test-utils/fixtures/auth"; +import { renderWithProviders } from "@/test-utils/render"; +import { KnowledgeSearchToolbar } from "./knowledge-search-toolbar"; + +function ControlledToolbar({ onSubmit }: { onSubmit(value: string): void }) { + const [value, setValue] = useState("existing"); + return ( + onSubmit(value)} + onClear={() => setValue("")} + filter={Active filter} + rightActions={} + nonCloudSearch={Fallback search} + /> + ); +} + +describe("KnowledgeSearchToolbar", () => { + it("uses the shared cloud search surface to edit, submit, and clear a search", async () => { + const user = userEvent.setup(); + const onSubmit = vi.fn(); + renderWithProviders(, { + providers: ["brand"], + auth: authPresets.ibmAuthMode, + }); + + const input = await screen.findByRole("textbox"); + expect(screen.getByText("Active filter")).toBeVisible(); + expect(screen.getByRole("button", { name: "Refresh" })).toBeVisible(); + + await user.clear(input); + await user.type(input, "website docs"); + await user.keyboard("{Enter}"); + expect(onSubmit).toHaveBeenCalledWith("website docs"); + + await user.click(screen.getByRole("button", { name: "Clear search" })); + expect(input).toHaveValue(""); + }); +}); diff --git a/frontend/components/knowledge-search-toolbar.tsx b/frontend/components/knowledge-search-toolbar.tsx new file mode 100644 index 0000000000..2ab272969b --- /dev/null +++ b/frontend/components/knowledge-search-toolbar.tsx @@ -0,0 +1,95 @@ +import { ArrowRight, Search, X } from "lucide-react"; +import type { ChangeEvent, FormEvent, ReactNode } from "react"; +import { Button } from "@/components/ui/button"; +import { useIsCloudBrand } from "@/contexts/brand-context"; +import { cn } from "@/lib/utils"; + +type KnowledgeSearchToolbarProps = { + value: string; + onValueChange: (value: string) => void; + onSubmit: () => void; + onClear: () => void; + placeholder?: string; + filter?: ReactNode; + rightActions?: ReactNode; + nonCloudSearch: ReactNode; +}; + +/** + * Shared Knowledge search surface. The input body and outer layout are fixed; + * each page supplies only the actions that belong on the right edge. + */ +export function KnowledgeSearchToolbar({ + value, + onValueChange, + onSubmit, + onClear, + placeholder = "Search knowledge", + filter, + rightActions, + nonCloudSearch, +}: KnowledgeSearchToolbarProps) { + const isCloudBrand = useIsCloudBrand(); + + if (!isCloudBrand) { + return ( +
+ {nonCloudSearch} + {rightActions} +
+ ); + } + + const handleSubmit = (event: FormEvent) => { + event.preventDefault(); + onSubmit(); + }; + + return ( +
+
+ {filter} +
+ +
+
+ ) => + onValueChange(event.target.value) + } + className="h-full w-full bg-transparent text-sm text-foreground placeholder:text-[hsl(var(--placeholder))] focus:outline-none focus:ring-0" + /> + {value && ( + + )} + +
+ {rightActions} +
+
+ ); +} diff --git a/frontend/components/knowledge-url-icon.tsx b/frontend/components/knowledge-url-icon.tsx new file mode 100644 index 0000000000..f62633c9c2 --- /dev/null +++ b/frontend/components/knowledge-url-icon.tsx @@ -0,0 +1,7 @@ +import { Globe } from "lucide-react"; +import { cn } from "@/lib/utils"; + +/** The one URL glyph used in the source table, connector menu, and child view. */ +export function KnowledgeUrlIcon({ className }: { className?: string }) { + return ; +} diff --git a/frontend/components/ui/status-badge.tsx b/frontend/components/ui/status-badge.tsx index cedb1472ef..ca2c4d0903 100644 --- a/frontend/components/ui/status-badge.tsx +++ b/frontend/components/ui/status-badge.tsx @@ -11,7 +11,8 @@ export type Status = | "skipped" | "ready" | "fallback" - | "not-configured"; + | "not-configured" + | "disabled"; interface StatusBadgeProps { /** @@ -68,6 +69,10 @@ const statusConfig: Record = { label: "Not configured", className: "text-muted-foreground ", }, + disabled: { + label: "Disabled", + className: "text-muted-foreground ", + }, }; /** Title-cases a status this badge has no entry for: "in_progress" → "In progress". */ diff --git a/frontend/components/url-source-dialog.test.tsx b/frontend/components/url-source-dialog.test.tsx new file mode 100644 index 0000000000..e4e05a42e1 --- /dev/null +++ b/frontend/components/url-source-dialog.test.tsx @@ -0,0 +1,149 @@ +import { screen, waitFor } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { HttpResponse, http } from "msw"; +import { describe, expect, it, vi } from "vitest"; +import { server } from "@/test-utils/msw/server"; +import { renderWithProviders } from "@/test-utils/render"; +import { UrlSourceDialog } from "./url-source-dialog"; + +describe("UrlSourceDialog", () => { + it("switches between source and re-sync steps from the dialog header", async () => { + const user = userEvent.setup(); + server.use( + http.get("/api/connectors/url/sources", () => + HttpResponse.json({ sources: [] }), + ), + ); + + renderWithProviders(); + + const sourceTab = screen.getByRole("button", { name: "Source & scope" }); + const resyncTab = screen.getByRole("button", { name: "Re-sync behavior" }); + + expect(sourceTab).toHaveAttribute("aria-current", "step"); + expect( + screen.getByRole("heading", { name: "Where should we crawl?" }), + ).toBeInTheDocument(); + + await user.click(resyncTab); + + expect(resyncTab).toHaveAttribute("aria-current", "step"); + expect( + screen.getByRole("heading", { name: "How should re-syncs work?" }), + ).toBeInTheDocument(); + expect( + screen.queryByRole("button", { name: "Back" }), + ).not.toBeInTheDocument(); + + await user.click(sourceTab); + + expect(sourceTab).toHaveAttribute("aria-current", "step"); + expect( + screen.getByRole("heading", { name: "Where should we crawl?" }), + ).toBeInTheDocument(); + }); + + it("submits normalized settings after configuring the source and re-sync behavior", async () => { + const user = userEvent.setup(); + const onCreated = vi.fn(); + let requestBody: Record | undefined; + server.use( + http.get("/api/connectors/url/sources", () => + HttpResponse.json({ sources: [] }), + ), + http.post("/api/connectors/url/sources", async ({ request }) => { + requestBody = (await request.json()) as Record; + return HttpResponse.json({ last_task_id: "website-task" }); + }), + ); + + renderWithProviders( + , + ); + + const continueButton = screen.getByRole("button", { name: "Continue" }); + expect(continueButton).toBeDisabled(); + + await user.type(screen.getByLabelText("Connection name"), "Docs"); + await user.type( + screen.getByLabelText("Starting URL"), + "https://docs.example.com/guides", + ); + await user.click(screen.getByRole("button", { name: /This page only/ })); + await user.click( + screen.getByRole("button", { name: "Advanced crawl settings" }), + ); + await user.type( + screen.getByPlaceholderText("assets.example.com (one per line)"), + "assets.example.com\ncdn.example.com", + ); + await user.type( + screen.getByPlaceholderText("/docs/ (one per line)"), + "/guides", + ); + await user.type( + screen.getByPlaceholderText("/archive/ (one per line)"), + "/archive", + ); + await user.click(screen.getByLabelText("Change detection")); + await user.click(screen.getByRole("option", { name: "Always re-ingest" })); + await user.click(screen.getByRole("checkbox")); + + expect(continueButton).toBeEnabled(); + const limits = screen.getAllByRole("spinbutton"); + expect(limits[0]).toBeDisabled(); + expect(limits[1]).toBeDisabled(); + + await user.click(continueButton); + await user.click( + screen.getByRole("button", { name: /Ingest only root page/ }), + ); + await user.click( + screen.getByRole("button", { name: /Full crawl; ingest returned pages/ }), + ); + await user.click(screen.getByRole("button", { name: /Retain in corpus/ })); + await user.click(screen.getByRole("button", { name: /Delete/ })); + await user.click(screen.getByRole("button", { name: /Ingest URL/i })); + + await waitFor(() => expect(onCreated).toHaveBeenCalledWith("website-task")); + expect(requestBody).toMatchObject({ + name: "Docs", + scope: "page", + additional_hosts: ["assets.example.com", "cdn.example.com"], + include_paths: ["/guides"], + change_detection: "always_reingest", + max_pages: 1, + max_depth: 0, + removed_page_behavior: "delete", + }); + }); + + it("reports an existing website connection name before submission", async () => { + const user = userEvent.setup(); + server.use( + http.get("/api/connectors/url/sources", () => + HttpResponse.json({ + sources: [ + { + id: "source-1", + name: "Docs", + starting_url: "https://docs.example.com", + status: "active", + }, + ], + }), + ), + ); + + renderWithProviders(); + + await user.type(screen.getByLabelText("Connection name"), " docs "); + + expect( + await screen.findByText( + "A website connection with this name already exists.", + ), + ).toBeInTheDocument(); + expect(screen.getByRole("button", { name: "Continue" })).toBeDisabled(); + }); +}); diff --git a/frontend/components/url-source-dialog.tsx b/frontend/components/url-source-dialog.tsx new file mode 100644 index 0000000000..ee5eb2180c --- /dev/null +++ b/frontend/components/url-source-dialog.tsx @@ -0,0 +1,135 @@ +"use client"; + +import { useState } from "react"; +import { toast } from "sonner"; +import { useCreateWebsiteSourceMutation } from "@/app/api/mutations/useWebsiteSourceMutation"; +import { useGetWebsiteSourcesQuery } from "@/app/api/queries/useGetWebsiteSourcesQuery"; +import { Button } from "@/components/ui/button"; +import { + Dialog, + DialogContent, + DialogDescription, + DialogHeader, + DialogTitle, +} from "@/components/ui/dialog"; +import { useIsCloudBrand } from "@/contexts/brand-context"; +import { + createUrlSourcePayload, + INITIAL_URL_SOURCE_FORM, + isUrlSourceFormValid, + isWebsiteSourceNameAvailable, + type UrlSourceForm, +} from "./url-source-dialog/form"; +import { ResyncBehaviorStep } from "./url-source-dialog/resync-behavior-step"; +import { SourceScopeStep } from "./url-source-dialog/source-scope-step"; +import { + type UrlSourceDialogStep, + UrlSourceDialogTabs, +} from "./url-source-dialog/tabs"; + +export function UrlSourceDialog({ + open, + onOpenChange, + onCreated, +}: { + open: boolean; + onOpenChange(open: boolean): void; + onCreated?(taskId?: string): void; +}) { + const [step, setStep] = useState(1); + const [advanced, setAdvanced] = useState(false); + const [form, setForm] = useState(INITIAL_URL_SOURCE_FORM); + const isCloudBrand = useIsCloudBrand(); + const createWebsiteSourceMutation = useCreateWebsiteSourceMutation(); + const { data: sources = [] } = useGetWebsiteSourcesQuery(open); + + const update = ( + key: K, + value: UrlSourceForm[K], + ) => setForm((current) => ({ ...current, [key]: value })); + const existingNames = sources.map((source) => source.name); + const nameAvailable = isWebsiteSourceNameAvailable(form.name, existingNames); + const valid = isUrlSourceFormValid(form, existingNames); + + const close = (next: boolean) => { + if (!next) { + setStep(1); + setAdvanced(false); + setForm(INITIAL_URL_SOURCE_FORM); + } + onOpenChange(next); + }; + + const submit = async () => { + try { + const data = await createWebsiteSourceMutation.mutateAsync( + createUrlSourcePayload(form), + ); + onCreated?.(data.last_task_id); + close(false); + toast.success("Website crawl started"); + } catch (error) { + toast.error("Could not start website crawl", { + description: error instanceof Error ? error.message : "Unknown error", + }); + } + }; + + return ( + + + + + Add a Website connection + + + Crawl public website content into OpenRAG. + + + +
+ {step === 1 ? ( + + ) : ( + + )} +
+
+ + +
+ + + ); +} diff --git a/frontend/components/url-source-dialog/advanced-crawl-settings.tsx b/frontend/components/url-source-dialog/advanced-crawl-settings.tsx new file mode 100644 index 0000000000..733163a658 --- /dev/null +++ b/frontend/components/url-source-dialog/advanced-crawl-settings.tsx @@ -0,0 +1,147 @@ +import { Info } from "lucide-react"; +import { Input } from "@/components/ui/input"; +import { Label } from "@/components/ui/label"; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from "@/components/ui/select"; +import { + CRAWL_LIMITS, + type UpdateUrlSourceForm, + type UrlSourceForm, +} from "./form"; + +export function AdvancedCrawlSettings({ + form, + onUpdate, +}: { + form: UrlSourceForm; + onUpdate: UpdateUrlSourceForm; +}) { + const pageScope = form.scope === "page"; + + return ( +
+
+

Fine-tune the crawler

+

+ Optional. Leave defaults unless you need precise control over hosts, + paths, or limits. +

+
+
+