From 50a188ba77d652b1330f43f631680c5fb9faa6d6 Mon Sep 17 00:00:00 2001 From: Lucas Oliveira Date: Thu, 24 Sep 2026 15:59:02 -0300 Subject: [PATCH 01/34] v1 of URL website crawler connector --- alembic/versions/0008_website_sources.py | 56 ++ .../api/queries/useFileScopedChunksQuery.ts | 19 +- .../app/api/queries/useGetConnectorsQuery.ts | 17 + .../api/queries/useGetSearchQuery.test.tsx | 60 ++- frontend/app/api/queries/useGetSearchQuery.ts | 80 ++- frontend/app/api/queries/useListFiles.ts | 9 +- frontend/app/knowledge/chunks/page.tsx | 24 +- frontend/app/knowledge/page.tsx | 259 ++------- .../settings/_components/connector-card.tsx | 8 +- .../settings/_components/connector-cards.tsx | 32 +- frontend/app/upload/[provider]/page.tsx | 8 +- frontend/components/file-chunks-panel.tsx | 11 +- .../components/knowledge-actions-dropdown.tsx | 255 +++++---- frontend/components/knowledge-data-table.tsx | 87 +++ frontend/components/knowledge-dropdown.tsx | 20 + frontend/components/knowledge-search-bar.tsx | 362 ++++++++----- .../components/knowledge-search-toolbar.tsx | 95 ++++ frontend/components/knowledge-url-icon.tsx | 7 + frontend/components/ui/status-badge.tsx | 7 +- .../components/url-source-dialog.test.tsx | 37 ++ frontend/components/url-source-dialog.tsx | 505 ++++++++++++++++++ frontend/components/website-pages-view.tsx | 491 +++++++++++++++++ .../contexts/knowledge-filter-context.tsx | 2 + frontend/lib/connectors/registry.ts | 8 + frontend/lib/connectors/types.ts | 2 +- frontend/lib/file-chunks.ts | 16 + frontend/lib/filter-normalization.ts | 12 + src/api/search.py | 112 +++- src/config/settings.py | 9 + src/connectors/base.py | 3 + src/connectors/connection_manager.py | 7 +- src/connectors/registry.py | 11 + src/connectors/url/__init__.py | 5 + src/connectors/url/api.py | 225 ++++++++ src/connectors/url/connector.py | 73 +++ src/connectors/url/crawler.py | 139 +++++ src/connectors/url/document.py | 78 +++ src/connectors/url/policy.py | 149 ++++++ src/connectors/url/processor.py | 211 ++++++++ src/connectors/url/projection.py | 79 +++ src/db/models/__init__.py | 4 + src/db/models/website_source.py | 62 +++ src/models/processors.py | 37 +- src/services/document_index_writer.py | 17 + src/services/file_service_v2.py | 40 +- src/services/search_service.py | 110 ++-- src/utils/embeddings.py | 9 + tests/unit/api/test_search_website_pages.py | 102 ++++ .../test_search_service_embedding_spaces.py | 1 + 49 files changed, 3411 insertions(+), 561 deletions(-) create mode 100644 alembic/versions/0008_website_sources.py create mode 100644 frontend/components/knowledge-data-table.tsx create mode 100644 frontend/components/knowledge-search-toolbar.tsx create mode 100644 frontend/components/knowledge-url-icon.tsx create mode 100644 frontend/components/url-source-dialog.test.tsx create mode 100644 frontend/components/url-source-dialog.tsx create mode 100644 frontend/components/website-pages-view.tsx create mode 100644 src/connectors/url/__init__.py create mode 100644 src/connectors/url/api.py create mode 100644 src/connectors/url/connector.py create mode 100644 src/connectors/url/crawler.py create mode 100644 src/connectors/url/document.py create mode 100644 src/connectors/url/policy.py create mode 100644 src/connectors/url/processor.py create mode 100644 src/connectors/url/projection.py create mode 100644 src/db/models/website_source.py create mode 100644 tests/unit/api/test_search_website_pages.py diff --git a/alembic/versions/0008_website_sources.py b/alembic/versions/0008_website_sources.py new file mode 100644 index 0000000000..a0088a33ad --- /dev/null +++ b/alembic/versions/0008_website_sources.py @@ -0,0 +1,56 @@ +"""add managed URL website source tables + +Revision ID: 0008_website_sources +Revises: 0007_add_knowledge_delete_anonymous +""" + +from collections.abc import Sequence + +import sqlalchemy as sa +from alembic import op + +revision: str = "0008_website_sources" +down_revision: str | Sequence[str] | None = "0007_add_knowledge_delete_anonymous" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.create_table("website_sources", + sa.Column("id", sa.String(64), primary_key=True), sa.Column("owner_id", sa.String(64), nullable=False), + sa.Column("name", sa.String(256), nullable=False), sa.Column("starting_url", sa.String(2048), nullable=False), + sa.Column("crawl_settings", sa.JSON(), nullable=False), sa.Column("resync_behavior", sa.String(32), nullable=False), + sa.Column("removed_page_behavior", sa.String(32), nullable=False), sa.Column("status", sa.String(32), nullable=False), + sa.Column("last_error", sa.String(2048)), sa.Column("last_task_id", sa.String(64)), + sa.Column("last_successful_sync_at", sa.DateTime()), sa.Column("created_at", sa.DateTime(), nullable=False), sa.Column("updated_at", sa.DateTime(), nullable=False), + ) + op.create_index("ix_website_sources_owner_id", "website_sources", ["owner_id"]) + op.create_index("ix_website_sources_status", "website_sources", ["status"]) + op.create_index("ix_website_sources_last_task_id", "website_sources", ["last_task_id"]) + op.create_table("website_pages", + sa.Column("id", sa.String(64), primary_key=True), sa.Column("web_source_id", sa.String(64), nullable=False), + sa.Column("canonical_url", sa.String(2048), nullable=False), sa.Column("final_url", sa.String(2048)), sa.Column("title", sa.String(512), nullable=False), + sa.Column("document_id", sa.String(64), nullable=False), sa.Column("content_hash", sa.String(64)), sa.Column("depth", sa.Integer(), nullable=False), + sa.Column("content_type", sa.String(128)), sa.Column("byte_size", sa.Integer(), nullable=False), sa.Column("chunk_count", sa.Integer(), nullable=False), + sa.Column("status", sa.String(32), nullable=False), sa.Column("suppressed_by_user", sa.Boolean(), nullable=False), + sa.Column("last_seen_at", sa.DateTime()), sa.Column("last_ingested_at", sa.DateTime()), sa.Column("last_error", sa.String(2048)), + sa.Column("created_at", sa.DateTime(), nullable=False), sa.Column("updated_at", sa.DateTime(), nullable=False), + sa.ForeignKeyConstraint(["web_source_id"], ["website_sources.id"], ondelete="CASCADE"), + sa.UniqueConstraint("web_source_id", "canonical_url", name="uq_website_page_source_url"), sa.UniqueConstraint("document_id"), + ) + op.create_index("ix_website_pages_source_status", "website_pages", ["web_source_id", "status"]) + op.create_index("ix_website_pages_web_source_id", "website_pages", ["web_source_id"]) + op.create_table("website_crawl_runs", + sa.Column("id", sa.String(64), primary_key=True), sa.Column("web_source_id", sa.String(64), nullable=False), sa.Column("task_id", sa.String(64)), + sa.Column("settings_snapshot", sa.JSON(), nullable=False), sa.Column("completed", sa.Boolean(), nullable=False), sa.Column("capped", sa.Boolean(), nullable=False), + sa.Column("error", sa.String(2048)), sa.Column("created_at", sa.DateTime(), nullable=False), sa.Column("finished_at", sa.DateTime()), + sa.ForeignKeyConstraint(["web_source_id"], ["website_sources.id"], ondelete="CASCADE"), + ) + op.create_index("ix_website_crawl_runs_web_source_id", "website_crawl_runs", ["web_source_id"]) + op.create_index("ix_website_crawl_runs_task_id", "website_crawl_runs", ["task_id"]) + + +def downgrade() -> None: + op.drop_table("website_crawl_runs") + op.drop_table("website_pages") + op.drop_table("website_sources") diff --git a/frontend/app/api/queries/useFileScopedChunksQuery.ts b/frontend/app/api/queries/useFileScopedChunksQuery.ts index ec2389bef6..e21917ad48 100644 --- a/frontend/app/api/queries/useFileScopedChunksQuery.ts +++ b/frontend/app/api/queries/useFileScopedChunksQuery.ts @@ -4,7 +4,10 @@ import { type SearchResult, useGetSearchQuery, } from "@/app/api/queries/useGetSearchQuery"; -import { fileScopedSearchQueryData } from "@/lib/file-chunks"; +import { + documentScopedSearchQueryData, + fileScopedSearchQueryData, +} from "@/lib/file-chunks"; /** Loads every chunk for one filename (shared by chunks page + FileChunksPanel). */ export function useFileScopedChunksQuery(filename: string | null | undefined) { @@ -19,3 +22,17 @@ export function useFileScopedChunksQuery(filename: string | null | undefined) { ); return { file, isFetching }; } + +export function useDocumentScopedChunksQuery( + documentId: string | null | undefined, +) { + const queryData = documentId + ? documentScopedSearchQueryData(documentId) + : null; + const { data = EMPTY_SEARCH_RESULT, isFetching } = useGetSearchQuery( + "*", + queryData, + { enabled: Boolean(documentId) }, + ); + return { file: (data as SearchResult).files[0], isFetching }; +} diff --git a/frontend/app/api/queries/useGetConnectorsQuery.ts b/frontend/app/api/queries/useGetConnectorsQuery.ts index 152d37b407..5de0362739 100644 --- a/frontend/app/api/queries/useGetConnectorsQuery.ts +++ b/frontend/app/api/queries/useGetConnectorsQuery.ts @@ -142,6 +142,7 @@ export interface Connector { selectedFiles?: GoogleDriveFile[] | OneDriveFile[]; available?: boolean; requiresOAuth?: boolean; + alwaysConnected?: boolean; } interface Connection { @@ -194,6 +195,22 @@ export const useGetConnectorsQuery = ( // "oauth" connectors require OAuth flow (Google Drive, OneDrive, SharePoint) // "bucket" connectors use credential-based auth (Azure Blob, S3, IBM COS) const requiresOAuth = connectorData.kind === "oauth"; + const alwaysConnected = connectorData.always_connected === true; + + if (alwaysConnected) { + return { + id: type, + name: connectorData.name, + description: connectorData.description, + icon: connectorData.icon, + status: "connected", + type, + connectionId: type, + available: connectorData.available, + requiresOAuth, + alwaysConnected, + } as Connector; + } if (statusResponse.ok) { const statusData = await statusResponse.json(); diff --git a/frontend/app/api/queries/useGetSearchQuery.test.tsx b/frontend/app/api/queries/useGetSearchQuery.test.tsx index d29fadb5b0..046d622e61 100644 --- a/frontend/app/api/queries/useGetSearchQuery.test.tsx +++ b/frontend/app/api/queries/useGetSearchQuery.test.tsx @@ -6,6 +6,7 @@ import { createQueryWrapper, renderHook, waitFor } from "@/test-utils/render"; import { type ChunkResult, type SearchPayload, + type SearchResultDisplayOptions, useGetSearchQuery, } from "./useGetSearchQuery"; @@ -61,10 +62,17 @@ function serveSearch(body: Record, status = 200) { return seen; } -async function runSearch(query: string, data?: ParsedQueryData | null) { - const rendered = renderHook(() => useGetSearchQuery(query, data), { - wrapper: createQueryWrapper(), - }); +async function runSearch( + query: string, + data?: ParsedQueryData | null, + displayOptions?: SearchResultDisplayOptions, +) { + const rendered = renderHook( + () => useGetSearchQuery(query, data, undefined, displayOptions), + { + wrapper: createQueryWrapper(), + }, + ); await waitFor(() => expect( rendered.result.current.isSuccess || rendered.result.current.isError, @@ -171,6 +179,30 @@ describe("useGetSearchQuery", () => { }); }); + it("preserves a URL source scope and website-page result mode", async () => { + const seen = serveSearch({ results: [] }); + + await runSearch( + "", + queryData({ + filters: { + data_sources: [], + document_types: [], + owners: [], + connector_types: [], + web_source_ids: ["source-123"], + }, + }), + { groupBy: "document_id", resultMode: "website_pages" }, + ); + + expect(seen.payload).toMatchObject({ + query: "*", + resultMode: "website_pages", + filters: { web_source_ids: ["source-123"] }, + }); + }); + it("treats a '*' dimension as unfiltered", async () => { const seen = serveSearch({ results: [] }); @@ -191,6 +223,26 @@ describe("useGetSearchQuery", () => { }); describe("grouping chunks into files", () => { + it("keeps URL pages with duplicate titles separate by document id", async () => { + serveSearch({ + results: [ + chunk({ filename: "Guide", document_id: "page-a" }), + chunk({ filename: "Guide", document_id: "page-b" }), + ], + }); + + const { result } = await runSearch("", queryData(), { + groupBy: "document_id", + }); + + expect(result.current.data?.files).toEqual( + expect.arrayContaining([ + expect.objectContaining({ filename: "Guide", document_id: "page-a" }), + expect.objectContaining({ filename: "Guide", document_id: "page-b" }), + ]), + ); + }); + it("groups chunks by filename and averages their scores", async () => { serveSearch({ results: [ diff --git a/frontend/app/api/queries/useGetSearchQuery.ts b/frontend/app/api/queries/useGetSearchQuery.ts index e7e44c023e..c29b51af09 100644 --- a/frontend/app/api/queries/useGetSearchQuery.ts +++ b/frontend/app/api/queries/useGetSearchQuery.ts @@ -5,18 +5,17 @@ import { } from "@tanstack/react-query"; import type { ParsedQueryData } from "@/contexts/knowledge-filter-context"; import { SEARCH_CONSTANTS } from "@/lib/constants"; -import { buildSearchPayloadFilters } from "@/lib/filter-normalization"; +import { + buildSearchPayloadFilters, + type FilterInput, +} from "@/lib/filter-normalization"; export interface SearchPayload { query: string; limit: number; scoreThreshold: number; - filters?: { - data_sources?: string[]; - document_types?: string[]; - owners?: string[]; - connector_types?: string[]; - }; + filters?: FilterInput; + resultMode?: "chunks" | "website_pages"; } export interface ChunkResult { @@ -41,6 +40,14 @@ export interface ChunkResult { index?: number; allowed_users?: string[]; allowed_groups?: string[]; + document_id?: string; + web_source_id?: string; + web_page_id?: string; + web_page_depth?: number; + canonical_url?: string; + status?: File["status"]; + error?: string; + chunk_count?: number; } export interface File { @@ -63,12 +70,18 @@ export interface File { | "failed" | "cancelled" | "hidden" - | "sync"; + | "sync" + | "disabled"; error?: string; task_id?: string; // Task ID for file-level cancellation chunks?: ChunkResult[]; allowed_users?: string[]; allowed_groups?: string[]; + document_id?: string; + web_source_id?: string; + web_page_id?: string; + web_page_depth?: number; + web_child_count?: number; } // Non-fatal signal from the backend — e.g. an embedding provider was removed @@ -86,11 +99,22 @@ export interface SearchResult { warnings: SearchWarning[]; } +export interface SearchResultDisplayOptions { + groupBy?: "filename" | "document_id"; + resultMode?: "chunks" | "website_pages"; +} + const EMPTY_SEARCH_RESULT: SearchResult = { files: [], warnings: [] }; export { EMPTY_SEARCH_RESULT }; -const getFileIdentity = (chunk: ChunkResult): string => { +const getFileIdentity = ( + chunk: ChunkResult, + groupBy: SearchResultDisplayOptions["groupBy"], +): string => { + if (groupBy === "document_id" && chunk.document_id) { + return chunk.document_id; + } const normalizedFilename = chunk.filename?.trim(); if (normalizedFilename) { return normalizedFilename; @@ -111,6 +135,7 @@ export const useGetSearchQuery = ( UseQueryOptions, "queryKey" | "queryFn" >, + displayOptions: SearchResultDisplayOptions = {}, ) => { const queryClient = useQueryClient(); @@ -143,6 +168,7 @@ export const useGetSearchQuery = ( query: effectiveQuery, limit: searchLimit, scoreThreshold: dynamicScoreThreshold, + resultMode: displayOptions.resultMode, }; if (queryData?.filters) { searchPayload.filters = @@ -185,11 +211,19 @@ export const useGetSearchQuery = ( embedding_dimensions?: number; allowed_users?: string[]; allowed_groups?: string[]; + document_id?: string; + web_source_id?: string; + web_page_id?: string; + web_page_depth?: number; + canonical_url?: string; + status?: File["status"]; + error?: string; + chunk_count?: number; } >(); (data.results || []).forEach((chunk: ChunkResult) => { - const fileIdentity = getFileIdentity(chunk); + const fileIdentity = getFileIdentity(chunk, displayOptions.groupBy); const existing = fileMap.get(fileIdentity); if (existing) { existing.chunks.push(chunk); @@ -203,9 +237,15 @@ export const useGetSearchQuery = ( ) { existing.embedding_dimensions = chunk.embedding_dimensions; } + if (typeof chunk.chunk_count === "number") { + existing.chunk_count = chunk.chunk_count; + } } else { fileMap.set(fileIdentity, { - filename: fileIdentity, + filename: + displayOptions.groupBy === "document_id" + ? chunk.filename || fileIdentity + : fileIdentity, mimetype: chunk.mimetype, chunks: [chunk], totalScore: chunk.score, @@ -219,6 +259,14 @@ export const useGetSearchQuery = ( embedding_dimensions: chunk.embedding_dimensions, allowed_users: chunk.allowed_users || [], allowed_groups: chunk.allowed_groups || [], + document_id: chunk.document_id, + web_source_id: chunk.web_source_id, + web_page_id: chunk.web_page_id, + web_page_depth: chunk.web_page_depth, + canonical_url: chunk.canonical_url, + status: chunk.status, + error: chunk.error, + chunk_count: chunk.chunk_count, }); } }); @@ -226,7 +274,7 @@ export const useGetSearchQuery = ( const files: File[] = Array.from(fileMap.values()).map((file) => ({ filename: file.filename, mimetype: file.mimetype, - chunkCount: file.chunks.length, + chunkCount: file.chunk_count ?? file.chunks.length, avgScore: file.totalScore / file.chunks.length, source_url: file.source_url || "", owner: file.owner || "", @@ -239,6 +287,12 @@ export const useGetSearchQuery = ( chunks: file.chunks, allowed_users: file.allowed_users || [], allowed_groups: file.allowed_groups || [], + document_id: file.document_id, + web_source_id: file.web_source_id, + web_page_id: file.web_page_id, + web_page_depth: file.web_page_depth, + status: file.status, + error: file.error, })); const warnings: SearchWarning[] = Array.isArray(data.warnings) @@ -255,7 +309,7 @@ export const useGetSearchQuery = ( return useQuery( { - queryKey: ["search", queryData, query], + queryKey: ["search", queryData, query, displayOptions], placeholderData: (prev) => prev, staleTime: 0, queryFn: getFiles, diff --git a/frontend/app/api/queries/useListFiles.ts b/frontend/app/api/queries/useListFiles.ts index 3969750226..fa1652e5dc 100644 --- a/frontend/app/api/queries/useListFiles.ts +++ b/frontend/app/api/queries/useListFiles.ts @@ -80,7 +80,14 @@ export const useListFiles = ( embedding_dimensions: f.embedding_dimensions as number | undefined, allowed_users: (f.allowed_users as string[]) || [], allowed_groups: (f.allowed_groups as string[]) || [], - status: "active" as const, + status: (f.status as File["status"]) || "active", + error: f.error as string | undefined, + document_id: f.document_id as string | undefined, + web_source_id: f.web_source_id as string | undefined, + web_page_id: f.web_page_id as string | undefined, + web_page_depth: + typeof f.web_page_depth === "number" ? f.web_page_depth : 0, + web_child_count: f.web_child_count as number | undefined, }), ); diff --git a/frontend/app/knowledge/chunks/page.tsx b/frontend/app/knowledge/chunks/page.tsx index 5eb3f3ef8b..2835a3f659 100644 --- a/frontend/app/knowledge/chunks/page.tsx +++ b/frontend/app/knowledge/chunks/page.tsx @@ -3,7 +3,10 @@ import { ArrowLeft, Loader2, Search } from "lucide-react"; import { useRouter, useSearchParams } from "next/navigation"; import { Suspense } from "react"; -import { useFileScopedChunksQuery } from "@/app/api/queries/useFileScopedChunksQuery"; +import { + useDocumentScopedChunksQuery, + useFileScopedChunksQuery, +} from "@/app/api/queries/useFileScopedChunksQuery"; import { FileChunksPanel } from "@/components/file-chunks-panel"; import { ProtectedRoute } from "@/components/protected-route"; import { Button } from "@/components/ui/button"; @@ -18,9 +21,12 @@ function ChunksPageContent() { const router = useRouter(); const searchParams = useSearchParams(); const filename = searchParams.get("filename"); - const { file: fileData } = useFileScopedChunksQuery(filename); + const documentId = searchParams.get("document_id"); + const filenameQuery = useFileScopedChunksQuery(filename); + const documentQuery = useDocumentScopedChunksQuery(documentId); + const { file: fileData } = documentId ? documentQuery : filenameQuery; - if (!filename) { + if (!filename && !documentId) { return (
@@ -52,21 +58,27 @@ function ChunksPageContent() {

- {filename.replace(/\.[^/.]+$/, "")} + {(filename || fileData?.filename || "Website page").replace( + /\.[^/.]+$/, + "", + )}

- +
{chunkCount > 0 && ( diff --git a/frontend/app/knowledge/page.tsx b/frontend/app/knowledge/page.tsx index 92c4309ecf..c8610fa1ca 100644 --- a/frontend/app/knowledge/page.tsx +++ b/frontend/app/knowledge/page.tsx @@ -5,23 +5,22 @@ import { type CheckboxSelectionCallbackParams, type ColDef, type ColumnState, - type GetRowIdParams, - themeQuartz, type ValueFormatterParams, type ValueGetterParams, } from "ag-grid-community"; import { AgGridReact, type CustomCellRendererProps } from "ag-grid-react"; -import { AlertTriangle, Cloud, FileIcon, Globe, RefreshCw } from "lucide-react"; -import { useRouter } from "next/navigation"; +import { AlertTriangle, Cloud, FileIcon, RefreshCw } from "lucide-react"; +import { useRouter, useSearchParams } from "next/navigation"; import { useCallback, useEffect, useMemo, useRef, useState } from "react"; +import { KnowledgeDataTable } from "@/components/knowledge-data-table"; import { KnowledgeDropdown } from "@/components/knowledge-dropdown"; +import { KnowledgeUrlIcon } from "@/components/knowledge-url-icon"; import { ProtectedRoute } from "@/components/protected-route"; import { Banner, BannerIcon, BannerTitle } from "@/components/ui/banner"; import { Button } from "@/components/ui/button"; import { useOpenTaskMenu } from "@/contexts/console-status-context"; import { useKnowledgeFilter } from "@/contexts/knowledge-filter-context"; import { useTask } from "@/contexts/task-context"; -import { trackButton } from "@/lib/analytics"; import { isFileCancelled } from "@/lib/task-error-display"; import { EMPTY_SEARCH_RESULT, @@ -38,14 +37,13 @@ import { KnowledgeActionsDropdown } from "@/components/knowledge-actions-dropdow import { KnowledgeBatchActionsBar } from "@/components/knowledge-batch-actions-bar"; import { KnowledgePaginationFooter } from "@/components/knowledge-pagination-footer"; import { KnowledgeSearchBar } from "@/components/knowledge-search-bar"; -import { KnowledgeSearchInput } from "@/components/knowledge-search-input"; -import { RequirePermission } from "@/components/require-permission"; import { StatusBadge } from "@/components/ui/status-badge"; import { Tooltip, TooltipContent, TooltipTrigger, } from "@/components/ui/tooltip"; +import { WebsitePagesView } from "@/components/website-pages-view"; import { useIsCloudBrand } from "@/contexts/brand-context"; import { getConnectorDescriptor } from "@/lib/connectors/registry"; import { formatFileSize } from "@/lib/file-format"; @@ -60,14 +58,7 @@ import { DeleteConfirmationDialog, formatFilesToDelete, } from "../../components/delete-confirmation-dialog"; -import { SyncConfirmDialog } from "../../components/sync-confirm-dialog"; import { useDeleteDocument } from "../api/mutations/useDeleteDocument"; -import { useRefreshOpenragDocs } from "../api/mutations/useRefreshOpenragDocs"; -import { - type SyncAllPreviewResponse, - useSyncAllConnectors, - useSyncAllConnectorsPreview, -} from "../api/mutations/useSyncConnector"; function sameFileSelection(a: File[], b: File[]): boolean { if (a.length !== b.length) { @@ -114,7 +105,7 @@ function getSourceIcon(connectorType?: string) { switch (connectorType) { case "openrag_docs": case "url": - return ; + return ; case "s3": return ; default: @@ -192,14 +183,6 @@ function SearchPage() { const seenFailedFileKeysRef = useRef>(new Set()); const deleteDocumentMutation = useDeleteDocument(); - const syncAllConnectorsMutation = useSyncAllConnectors(); - const syncAllPreviewMutation = useSyncAllConnectorsPreview(); - const refreshOpenragDocsMutation = useRefreshOpenragDocs(); - const [syncDialogOpen, setSyncDialogOpen] = useState(false); - const [syncPreview, setSyncPreview] = useState( - null, - ); - const [currentPage, setCurrentPage] = useState(1); const [currentPageSize, setCurrentPageSize] = useState(25); @@ -210,45 +193,6 @@ function SearchPage() { cursorCacheRef.current = new Map(); } - const handleOpenSyncDialog = useCallback(async () => { - setSyncPreview(null); - setSyncDialogOpen(true); - try { - const preview = await syncAllPreviewMutation.mutateAsync(); - setSyncPreview(preview); - } catch (error) { - setSyncDialogOpen(false); - toast.error( - error instanceof Error ? error.message : "Failed to preview sync", - ); - } - }, [syncAllPreviewMutation]); - - const handleConfirmSync = useCallback(async () => { - try { - const result = await syncAllConnectorsMutation.mutateAsync(); - if (result.status === "no_files") { - toast.info( - result.message || - "No cloud files to sync. Add files from cloud connectors first.", - ); - } else if ( - result.synced_connectors && - result.synced_connectors.length > 0 - ) { - toast.success( - `Sync started for ${result.synced_connectors.join(", ")}. Check task notifications for progress.`, - ); - } else if (result.errors && result.errors.length > 0) { - toast.error("Some connectors failed to sync"); - } - } catch (error) { - toast.error( - error instanceof Error ? error.message : "Failed to sync connectors", - ); - } - }, [syncAllConnectorsMutation]); - useEffect(() => { refreshTasks(); }, [refreshTasks]); @@ -487,8 +431,7 @@ function SearchPage() { ); }); }); - const hasOpenragRefreshCue = - refreshOpenragDocsMutation.isPending || hasOpenragRefreshCueFromTasks; + const hasOpenragRefreshCue = hasOpenragRefreshCueFromTasks; // Show toast notification for search errors useEffect(() => { @@ -627,6 +570,12 @@ function SearchPage() { )} onClick={() => { if (!isActive) return; + if (data?.connector_type === "url" && data.web_source_id) { + router.push( + `/knowledge?website=${encodeURIComponent(data.web_source_id)}`, + ); + return; + } router.push( `/knowledge/chunks?filename=${encodeURIComponent( data?.filename ?? "", @@ -866,6 +815,8 @@ function SearchPage() { ); }, @@ -1026,80 +977,10 @@ function SearchPage() {
) : ( - /* Search Input Area */ -
- - - - - - - {selectedRows.length > 0 && ( - - )} -
- -
-
+ setShowBulkDeleteDialog(true)} + /> )} {!isWildcardQuery && searchWarnings.length > 0 && (
@@ -1138,75 +1019,21 @@ function SearchPage() { })}
)} - {isCloudBrand ? ( -
- []} - defaultColDef={defaultColDef} - loading={isLoading || deleteDocumentMutation.isPending} - ref={gridRef} - theme={themeQuartz.withParams({ browserColorScheme: "inherit" })} - rowData={gridRows} - rowSelection="multiple" - getRowId={(params: GetRowIdParams) => - getFileIdentity(params.data) - } - isRowSelectable={(params) => isDeletableKnowledgeRow(params.data)} - domLayout="normal" - onGridReady={handleGridReady} - onGridPreDestroyed={handleGridPreDestroyed} - onSelectionChanged={onSelectionChanged} - onSortChanged={onSortChanged} - headerHeight={64} - rowHeight={64} - noRowsOverlayComponent={() => ( -
-
- No knowledge -
-
- Add files from local or your preferred cloud. -
-
- )} - /> -
- ) : ( -
- []} - defaultColDef={defaultColDef} - loading={isLoading || deleteDocumentMutation.isPending} - ref={gridRef} - theme={themeQuartz.withParams({ browserColorScheme: "inherit" })} - rowData={gridRows} - rowSelection="multiple" - rowMultiSelectWithClick={false} - suppressRowClickSelection={true} - getRowId={(params: GetRowIdParams) => - getFileIdentity(params.data) - } - isRowSelectable={(params) => isDeletableKnowledgeRow(params.data)} - domLayout="normal" - onGridReady={handleGridReady} - onGridPreDestroyed={handleGridPreDestroyed} - onSelectionChanged={onSelectionChanged} - onSortChanged={onSortChanged} - noRowsOverlayComponent={() => ( -
-
- No knowledge -
-
- Add files from local or your preferred cloud. -
-
- )} - /> -
- )} + getFileIdentity(params.data)} + isRowSelectable={(params) => isDeletableKnowledgeRow(params.data)} + onGridReady={handleGridReady} + onGridPreDestroyed={handleGridPreDestroyed} + onSelectionChanged={onSelectionChanged} + onSortChanged={onSortChanged} + /> Documents to be deleted:

{formatFilesToDelete(selectedRows)} - - ); } export default function ProtectedSearchPage() { + const searchParams = useSearchParams(); + const websiteId = searchParams.get("website"); return ( - + {websiteId ? : } ); } diff --git a/frontend/app/settings/_components/connector-card.tsx b/frontend/app/settings/_components/connector-card.tsx index 940bf65c21..97760b30f8 100644 --- a/frontend/app/settings/_components/connector-card.tsx +++ b/frontend/app/settings/_components/connector-card.tsx @@ -25,6 +25,7 @@ export interface Connector { status?: string; connectionId?: string; requiresOAuth?: boolean; + alwaysConnected?: boolean; } interface ConnectorCardProps { @@ -56,7 +57,8 @@ export default function ConnectorCard({ ]); const canUpload = can("knowledge:upload"); const isConnected = - connector.status === "connected" && connector.connectionId; + connector.status === "connected" && + (connector.connectionId || connector.alwaysConnected); const isConfigured = connector.status === "configured"; return ( @@ -139,7 +141,7 @@ export default function ConnectorCard({ Add Knowledge - {canCreate && ( + {canCreate && !connector.alwaysConnected && ( )} - {canDisconnect && ( + {canDisconnect && !connector.alwaysConnected && ( - { - trackButton({ - CTA: "View Chunks", - elementId: "view-chunks-button", - namespace: "knowledge", - }); - router.push( - `/knowledge/chunks?filename=${encodeURIComponent(filename)}`, - ); - }} - > - View chunks - - {isCloudFile && ( - - - -
- { - if (!isConnected) { - e.preventDefault(); - return; - } - trackButton({ - CTA: "Sync File", - elementId: "sync-file-button", - namespace: "knowledge", - payload: { connector_type: connectorType }, - }); - handleOpenSyncDialog(); - }} - > - {syncConnectorMutation.isPending || - syncPreviewMutation.isPending ? ( - <> - - Syncing... - - ) : ( - <> - - Sync - - )} - {!isConnected && ( - - )} - -
- - {!isConnected && ( - -

- {connectorType.charAt(0).toUpperCase() + - connectorType.slice(1)}{" "} - is not connected. Connect it in Settings to enable sync. -

- - )} - - + {webSourceId ? ( + <> + + + Sync + + setShowDeleteDialog(true)} + > + Delete + + + ) : ( + <> + { + trackButton({ + CTA: "View Chunks", + elementId: "view-chunks-button", + namespace: "knowledge", + }); + router.push( + `/knowledge/chunks?filename=${encodeURIComponent(filename)}`, + ); + }} + > + View chunks + + {isCloudFile && ( + + + +
+ { + if (!isConnected) { + e.preventDefault(); + return; + } + trackButton({ + CTA: "Sync File", + elementId: "sync-file-button", + namespace: "knowledge", + payload: { connector_type: connectorType }, + }); + handleOpenSyncDialog(); + }} + > + {syncConnectorMutation.isPending || + syncPreviewMutation.isPending ? ( + <> + + Syncing... + + ) : ( + <> + + Sync + + )} + {!isConnected && ( + + )} + +
+ + {!isConnected && ( + +

+ {connectorType.charAt(0).toUpperCase() + + connectorType.slice(1)}{" "} + is not connected. Connect it in Settings to enable + sync. +

+ + )} + + + )} + + { + trackButton({ + CTA: "Delete Document", + elementId: "delete-document-button", + namespace: "knowledge", + }); + setShowDeleteDialog(true); + }} + > + Delete + + + )} - - { - trackButton({ - CTA: "Delete Document", - elementId: "delete-document-button", - namespace: "knowledge", - }); - setShowDeleteDialog(true); - }} - > - Delete - - -

- This will remove all chunks and data associated with this document. - This action cannot be undone. -

-

Document to be deleted:

- {formatFilesToDelete([{ filename }])} + {!webSourceId && ( +

+ This will remove all chunks and data associated with this document. + This action cannot be undone. +

+ )} + {webSourceId ? ( +

+ All of the website's indexed child pages and chunks will be + removed. +

+ ) : ( + <> +

Document to be deleted:

+ {formatFilesToDelete([{ filename }])} + + )} { + rows: T[]; + columnDefs: ColDef[]; + defaultColDef: ColDef; + gridRef?: RefObject | null>; + loading?: boolean; + isCloudBrand?: boolean; + rowSelection?: "single" | "multiple"; + getRowId: (params: GetRowIdParams) => string; + isRowSelectable?: (params: { data?: T }) => boolean; + onGridReady?: () => void; + onGridPreDestroyed?: () => void; + onSelectionChanged?: () => void; + onSortChanged?: () => void; + emptyTitle?: string; + emptyDescription?: string; +} + +/** + * Shared Knowledge AG Grid shell. Column factories stay with their owning + * surface; grid sizing, theme, loading, focus behaviour and empty states do not. + */ +export function KnowledgeDataTable({ + rows, + columnDefs, + defaultColDef, + gridRef, + loading = false, + isCloudBrand = false, + rowSelection, + getRowId, + isRowSelectable, + onGridReady, + onGridPreDestroyed, + onSelectionChanged, + onSortChanged, + emptyTitle = "No knowledge", + emptyDescription = "Add files from local or your preferred cloud.", +}: KnowledgeDataTableProps) { + return ( +
+ + className={cn("h-full w-full", isCloudBrand && "border")} + columnDefs={columnDefs} + defaultColDef={defaultColDef} + loading={loading} + ref={gridRef} + theme={themeQuartz.withParams({ browserColorScheme: "inherit" })} + rowData={rows} + rowSelection={rowSelection} + rowMultiSelectWithClick={false} + suppressRowClickSelection={rowSelection === "multiple"} + getRowId={getRowId} + isRowSelectable={isRowSelectable} + domLayout="normal" + onGridReady={onGridReady} + onGridPreDestroyed={onGridPreDestroyed} + onSelectionChanged={onSelectionChanged} + onSortChanged={onSortChanged} + {...(isCloudBrand ? { headerHeight: 64, rowHeight: 64 } : {})} + noRowsOverlayComponent={() => ( +
+
+ {emptyTitle} +
+
+ {emptyDescription} +
+
+ )} + /> +
+ ); +} diff --git a/frontend/components/knowledge-dropdown.tsx b/frontend/components/knowledge-dropdown.tsx index e90ccf6c93..b3e4cd4de6 100644 --- a/frontend/components/knowledge-dropdown.tsx +++ b/frontend/components/knowledge-dropdown.tsx @@ -16,6 +16,7 @@ import type { File as SearchFile } from "@/app/api/queries/useGetSearchQuery"; import { useGetTasksQuery } from "@/app/api/queries/useGetTasksQuery"; import { DuplicateHandlingDialog } from "@/components/duplicate-handling-dialog"; import { IngestReviewDialog } from "@/components/ingest-review"; +import { KnowledgeUrlIcon } from "@/components/knowledge-url-icon"; import { Button } from "@/components/ui/button"; import { Dialog, @@ -32,6 +33,7 @@ import { } from "@/components/ui/dropdown-menu"; import { Input } from "@/components/ui/input"; import { Label } from "@/components/ui/label"; +import { UrlSourceDialog } from "@/components/url-source-dialog"; import { useAuth } from "@/contexts/auth-context"; import { useIsCloudBrand } from "@/contexts/brand-context"; import { useTask } from "@/contexts/task-context"; @@ -114,6 +116,7 @@ export function KnowledgeDropdown() { }); const [mounted, setMounted] = useState(false); const [isMenuOpen, setIsMenuOpen] = useState(false); + const [urlDialogOpen, setUrlDialogOpen] = useState(false); const [showFolderDialog, setShowFolderDialog] = useState(false); const [showDuplicateDialog, setShowDuplicateDialog] = useState(false); const [uploadBatchSize, setUploadBatchSize] = useState(25); @@ -288,6 +291,9 @@ export function KnowledgeDropdown() { useEffect(() => { setMounted(true); + if (new URLSearchParams(window.location.search).get("add") === "url") { + setUrlDialogOpen(true); + } }, []); const handleFileUpload = () => { @@ -853,6 +859,11 @@ export function KnowledgeDropdown() { icon: FolderIconWithColor, onClick: () => folderInputRef.current?.click(), }, + { + label: "URL", + icon: KnowledgeUrlIcon, + onClick: () => setUrlDialogOpen(true), + }, ...bucketConnectorItems, ...cloudConnectorItems, ]; @@ -1043,6 +1054,15 @@ export function KnowledgeDropdown() { taskIds={preview.taskIds} previewFiles={preview.files} /> + { + if (taskId) addTask(taskId, { connectorType: "url" }); + void queryClient.invalidateQueries({ queryKey: ["listFiles"] }); + void refetchTasks(); + }} + /> ); } diff --git a/frontend/components/knowledge-search-bar.tsx b/frontend/components/knowledge-search-bar.tsx index ea945a45d7..358fd2d020 100644 --- a/frontend/components/knowledge-search-bar.tsx +++ b/frontend/components/knowledge-search-bar.tsx @@ -1,5 +1,5 @@ -import { ArrowRight, Download, RefreshCw, Search, X } from "lucide-react"; -import { type ChangeEvent, type FormEvent, useCallback, useState } from "react"; +import { Download, RefreshCw, X } from "lucide-react"; +import { type ReactNode, useCallback, useState } from "react"; import { toast } from "sonner"; import { useRefreshOpenragDocs } from "@/app/api/mutations/useRefreshOpenragDocs"; import { @@ -7,15 +7,41 @@ import { useSyncAllConnectors, useSyncAllConnectorsPreview, } from "@/app/api/mutations/useSyncConnector"; +import { KnowledgeDropdown } from "@/components/knowledge-dropdown"; +import { KnowledgeSearchInput } from "@/components/knowledge-search-input"; +import { KnowledgeSearchToolbar } from "@/components/knowledge-search-toolbar"; import { RequirePermission } from "@/components/require-permission"; import { Button } from "@/components/ui/button"; +import { useIsCloudBrand } from "@/contexts/brand-context"; import { useKnowledgeFilter } from "@/contexts/knowledge-filter-context"; +import { trackButton } from "@/lib/analytics"; import { cn } from "@/lib/utils"; -import { KnowledgeDropdown } from "./knowledge-dropdown"; import { filterAccentClasses } from "./knowledge-filter-panel"; import { SyncConfirmDialog } from "./sync-confirm-dialog"; -export const KnowledgeSearchBar = () => { +type KnowledgeSearchBarProps = { + value?: string; + onSearch?: (query: string) => void; + onClear?: () => void; + placeholder?: string; + nonCloudPlaceholder?: string; + rightActions?: ReactNode; + selectedCount?: number; + onDeleteSelected?: () => void; +}; + +export const KnowledgeSearchBar = ({ + value, + onSearch, + onClear, + placeholder = "Search knowledge", + nonCloudPlaceholder, + rightActions, + selectedCount = 0, + onDeleteSelected, +}: KnowledgeSearchBarProps) => { + const isCloudBrand = useIsCloudBrand(); + const controlled = onSearch != null; const { selectedFilter, setSelectedFilter, @@ -26,23 +52,31 @@ export const KnowledgeSearchBar = () => { const [searchQueryInput, setSearchQueryInput] = useState(queryOverride || ""); const [prevQueryOverride, setPrevQueryOverride] = useState(queryOverride); - if (queryOverride !== prevQueryOverride) { + if (!controlled && queryOverride !== prevQueryOverride) { setPrevQueryOverride(queryOverride); setSearchQueryInput(queryOverride); } - const handleSearch = useCallback( - (e?: FormEvent) => { - if (e) e.preventDefault(); - setQueryOverride(searchQueryInput.trim()); - }, - [searchQueryInput, setQueryOverride], - ); + const displayedValue = controlled ? (value ?? "") : searchQueryInput; + + const handleSearch = useCallback(() => { + const next = displayedValue.trim(); + if (controlled) { + onSearch(next); + } else { + setQueryOverride(next); + } + }, [controlled, displayedValue, onSearch, setQueryOverride]); const handleReset = useCallback(() => { - setSearchQueryInput(""); - setQueryOverride(""); - }, [setQueryOverride]); + if (controlled) { + onClear?.(); + onSearch(""); + } else { + setSearchQueryInput(""); + setQueryOverride(""); + } + }, [controlled, onClear, onSearch, setQueryOverride]); const syncAllConnectorsMutation = useSyncAllConnectors(); const syncAllPreviewMutation = useSyncAllConnectorsPreview(); @@ -91,135 +125,191 @@ export const KnowledgeSearchBar = () => { } }, [syncAllConnectorsMutation]); - return ( -
-
- {!!selectedFilter?.name && ( -
- - {selectedFilter.name} - - -
- )} - -
- -
- -
- ) => - setSearchQueryInput(e.target.value) - } - className="h-full w-full bg-transparent text-sm text-foreground placeholder:text-[hsl(var(--placeholder))] focus:outline-none focus:ring-0" - /> - {searchQueryInput && ( - - )} - -
+ const defaultRightActions = isCloudBrand ? ( + <> + + - - + + - -
- -
+ }} + > + {refreshOpenragDocsMutation.isPending + ? "Refreshing docs..." + : "Fetch latest docs"} + + + {selectedCount > 0 && ( + + )} +
+
- + ); + + return ( + <> + + + {selectedFilter.name} + + +
+ ) : undefined + } + rightActions={rightActions ?? defaultRightActions} + nonCloudSearch={ + controlled ? ( + + ) : ( + + ) + } /> - + {!controlled && ( + + )} + ); }; diff --git a/frontend/components/knowledge-search-toolbar.tsx b/frontend/components/knowledge-search-toolbar.tsx new file mode 100644 index 0000000000..2ab272969b --- /dev/null +++ b/frontend/components/knowledge-search-toolbar.tsx @@ -0,0 +1,95 @@ +import { ArrowRight, Search, X } from "lucide-react"; +import type { ChangeEvent, FormEvent, ReactNode } from "react"; +import { Button } from "@/components/ui/button"; +import { useIsCloudBrand } from "@/contexts/brand-context"; +import { cn } from "@/lib/utils"; + +type KnowledgeSearchToolbarProps = { + value: string; + onValueChange: (value: string) => void; + onSubmit: () => void; + onClear: () => void; + placeholder?: string; + filter?: ReactNode; + rightActions?: ReactNode; + nonCloudSearch: ReactNode; +}; + +/** + * Shared Knowledge search surface. The input body and outer layout are fixed; + * each page supplies only the actions that belong on the right edge. + */ +export function KnowledgeSearchToolbar({ + value, + onValueChange, + onSubmit, + onClear, + placeholder = "Search knowledge", + filter, + rightActions, + nonCloudSearch, +}: KnowledgeSearchToolbarProps) { + const isCloudBrand = useIsCloudBrand(); + + if (!isCloudBrand) { + return ( +
+ {nonCloudSearch} + {rightActions} +
+ ); + } + + const handleSubmit = (event: FormEvent) => { + event.preventDefault(); + onSubmit(); + }; + + return ( +
+
+ {filter} +
+ +
+
+ ) => + onValueChange(event.target.value) + } + className="h-full w-full bg-transparent text-sm text-foreground placeholder:text-[hsl(var(--placeholder))] focus:outline-none focus:ring-0" + /> + {value && ( + + )} + +
+ {rightActions} +
+
+ ); +} diff --git a/frontend/components/knowledge-url-icon.tsx b/frontend/components/knowledge-url-icon.tsx new file mode 100644 index 0000000000..f62633c9c2 --- /dev/null +++ b/frontend/components/knowledge-url-icon.tsx @@ -0,0 +1,7 @@ +import { Globe } from "lucide-react"; +import { cn } from "@/lib/utils"; + +/** The one URL glyph used in the source table, connector menu, and child view. */ +export function KnowledgeUrlIcon({ className }: { className?: string }) { + return ; +} diff --git a/frontend/components/ui/status-badge.tsx b/frontend/components/ui/status-badge.tsx index 356194ac9f..f2ac27a0ca 100644 --- a/frontend/components/ui/status-badge.tsx +++ b/frontend/components/ui/status-badge.tsx @@ -10,7 +10,8 @@ export type Status = | "cancelled" | "ready" | "fallback" - | "not-configured"; + | "not-configured" + | "disabled"; interface StatusBadgeProps { status: Status; @@ -58,6 +59,10 @@ const statusConfig = { label: "Not configured", className: "text-muted-foreground ", }, + disabled: { + label: "Disabled", + className: "text-muted-foreground ", + }, }; export const StatusBadge = ({ status, className }: StatusBadgeProps) => { diff --git a/frontend/components/url-source-dialog.test.tsx b/frontend/components/url-source-dialog.test.tsx new file mode 100644 index 0000000000..fd285064e4 --- /dev/null +++ b/frontend/components/url-source-dialog.test.tsx @@ -0,0 +1,37 @@ +import { render, screen } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { describe, expect, it, vi } from "vitest"; +import { UrlSourceDialog } from "./url-source-dialog"; + +describe("UrlSourceDialog", () => { + it("switches between source and re-sync steps from the dialog header", async () => { + const user = userEvent.setup(); + + render(); + + const sourceTab = screen.getByRole("button", { name: "Source & scope" }); + const resyncTab = screen.getByRole("button", { name: "Re-sync behavior" }); + + expect(sourceTab).toHaveAttribute("aria-current", "step"); + expect( + screen.getByRole("heading", { name: "Where should we crawl?" }), + ).toBeInTheDocument(); + + await user.click(resyncTab); + + expect(resyncTab).toHaveAttribute("aria-current", "step"); + expect( + screen.getByRole("heading", { name: "How should re-syncs work?" }), + ).toBeInTheDocument(); + expect( + screen.queryByRole("button", { name: "Back" }), + ).not.toBeInTheDocument(); + + await user.click(sourceTab); + + expect(sourceTab).toHaveAttribute("aria-current", "step"); + expect( + screen.getByRole("heading", { name: "Where should we crawl?" }), + ).toBeInTheDocument(); + }); +}); diff --git a/frontend/components/url-source-dialog.tsx b/frontend/components/url-source-dialog.tsx new file mode 100644 index 0000000000..9e11013245 --- /dev/null +++ b/frontend/components/url-source-dialog.tsx @@ -0,0 +1,505 @@ +"use client"; + +import { AlertTriangle, Check, ChevronDown, RefreshCw } from "lucide-react"; +import { useState } from "react"; +import { toast } from "sonner"; +import { KnowledgeUrlIcon } from "@/components/knowledge-url-icon"; +import { Button } from "@/components/ui/button"; +import { + Dialog, + DialogContent, + DialogDescription, + DialogHeader, + DialogTitle, +} from "@/components/ui/dialog"; +import { Input } from "@/components/ui/input"; +import { Label } from "@/components/ui/label"; +import { cn } from "@/lib/utils"; + +type Scope = "path" | "page" | "site"; +type Form = { + name: string; + starting_url: string; + scope: Scope; + allow_subdomains: boolean; + additional_hosts: string; + include_paths: string; + exclude_paths: string; + max_pages: number; + max_depth: number; + max_downloaded_mb: number; + max_crawl_minutes: number; + resync_behavior: "full" | "root"; + removed_page_behavior: "retain" | "delete"; +}; +const INITIAL: Form = { + name: "", + starting_url: "", + scope: "path", + allow_subdomains: false, + additional_hosts: "", + include_paths: "", + exclude_paths: "", + max_pages: 250, + max_depth: 4, + max_downloaded_mb: 128, + max_crawl_minutes: 15, + resync_behavior: "full", + removed_page_behavior: "retain", +}; +const lines = (value: string) => + value + .split("\n") + .map((x) => x.trim()) + .filter(Boolean); + +function validUrl(value: string) { + try { + const url = new URL(value); + return ( + ["http:", "https:"].includes(url.protocol) && + !url.username && + !url.password && + (!url.port || url.port === "80" || url.port === "443") && + !/^\d{1,3}(\.\d{1,3}){3}$/.test(url.hostname) && + !url.hostname.includes(":") + ); + } catch { + return false; + } +} + +function Choice({ + selected, + onClick, + title, + detail, +}: { + selected: boolean; + onClick(): void; + title: string; + detail: string; +}) { + return ( + + ); +} + +export function UrlSourceDialog({ + open, + onOpenChange, + onCreated, +}: { + open: boolean; + onOpenChange(open: boolean): void; + onCreated?(taskId?: string): void; +}) { + const [step, setStep] = useState(1); + const [advanced, setAdvanced] = useState(false); + const [pending, setPending] = useState(false); + const [form, setForm] = useState
(INITIAL); + const update = (key: K, value: Form[K]) => + setForm((current) => ({ ...current, [key]: value })); + const pageScope = form.scope === "page"; + const valid = Boolean( + form.name.trim() && + validUrl(form.starting_url) && + lines(form.additional_hosts).every( + (host) => !host.includes("/") && !/^\d{1,3}(\.\d{1,3}){3}$/.test(host), + ) && + [...lines(form.include_paths), ...lines(form.exclude_paths)].every( + (path) => path.startsWith("/"), + ), + ); + const close = (next: boolean) => { + if (!next) { + setStep(1); + setAdvanced(false); + setForm(INITIAL); + } + onOpenChange(next); + }; + const submit = async () => { + setPending(true); + try { + const response = await fetch("/api/connectors/url/sources", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + ...form, + additional_hosts: lines(form.additional_hosts), + include_paths: lines(form.include_paths), + exclude_paths: lines(form.exclude_paths), + max_pages: pageScope ? 1 : form.max_pages, + max_depth: pageScope ? 0 : form.max_depth, + }), + }); + const data = await response.json(); + if (!response.ok) + throw new Error( + data.detail || data.error || "Could not create website source", + ); + onCreated?.(data.last_task_id); + close(false); + toast.success("Website crawl started"); + } catch (error) { + toast.error("Could not start website crawl", { + description: error instanceof Error ? error.message : "Unknown error", + }); + } finally { + setPending(false); + } + }; + return ( + + + + + Add a Website connection + + + Crawl public website content into OpenRAG. + + +
+ + +
+
+ {step === 1 ? ( +
+
+ +

+ URL ingestion always uses OpenRAG's native ingestion + pipeline, even when Langflow ingestion is enabled for other + sources. +

+
+
+

+ Where should we crawl? +

+

+ Give this connection a name and tell OpenRAG where to start. +

+
+
+
+ + update("name", e.target.value)} + /> +
+
+ + update("starting_url", e.target.value)} + /> +
+
+
+
+ +

+ How far should the crawler follow links from the starting + URL? +

+
+
+ update("scope", "path")} + title="Path and children" + detail="Follow approved pages beneath the starting path." + /> + update("scope", "page")} + title="This page only" + detail="Ingest the starting page without following links." + /> + update("scope", "site")} + title="Whole site" + detail="Follow approved pages anywhere on the site." + /> +
+
+ + + {advanced && ( +
+
+

+ Fine-tune the crawler +

+

+ Optional. Leave defaults unless you need precise control + over hosts, paths, or limits. +

+
+
+