feat: add URL connector

.env.example

@@ -739,6 +739,11 @@ OPENRAG_RUN_MODE=oss
 # hidden/disabled.
 # OPENRAG_WORKSPACE_OAUTH_OVERRIDES_ENABLED=false
 
+# Feature flag for the managed public-website connector (default: false =
+# disabled). When true, users can add website sources from Knowledge and the
+# Connectors settings page. This is also enforced by the backend URL APIs.
+# OPENRAG_URL_CONNECTOR_ENABLED=false
+
 # Permission/identity cache backend. Only "memory" is currently wired.
 # When CACHE_BACKEND=memory the RBAC permission cache and OAuth-subject
 # cache are per-process, so OpenRAG must run with UVICORN_WORKERS=1 (and

.github/workflows/test-ci.yml

@@ -454,7 +454,7 @@ jobs:
       - name: Setup E2E infrastructure
         run: |
           chmod +x scripts/setup-e2e.sh
-          OPENRAG_VERSION="${CI_IMAGE_TAG}" ./scripts/setup-e2e.sh
+          OPENRAG_VERSION="${CI_IMAGE_TAG}" OPENRAG_URL_CONNECTOR_ENABLED=true ./scripts/setup-e2e.sh
 
       - name: Run Playwright tests (shard ${{ matrix.shard }}/${{ matrix.total_shards }})
         working-directory: frontend

docker-compose.yml

@@ -138,6 +138,7 @@ services:
       - MAX_WORKERS=${MAX_WORKERS:-4}
       - IBM_AUTH_ENABLED=${IBM_AUTH_ENABLED:-false}
       - OPENRAG_AZURE_BLOB_ENABLED=${OPENRAG_AZURE_BLOB_ENABLED:-true}
+      - OPENRAG_URL_CONNECTOR_ENABLED=${OPENRAG_URL_CONNECTOR_ENABLED:-false}
       - PLATFORM_AUTH_DEV_MODE=${PLATFORM_AUTH_DEV_MODE:-false}
       - DOCLING_SERVE_URL=${DOCLING_SERVE_URL:-http://host.docker.internal:5001}
       - DOCLING_SERVE_VERIFY_SSL=${DOCLING_SERVE_VERIFY_SSL:-true}

frontend/app/api/mutations/useWebsiteSourceMutation.test.tsx

@@ -0,0 +1,50 @@
+import { act, renderHook, waitFor } from "@testing-library/react";
+import { HttpResponse, http } from "msw";
+import { describe, expect, it } from "vitest";
+import { useGetTasksQuery } from "@/app/api/queries/useGetTasksQuery";
+import { server } from "@/test-utils/msw/server";
+import { createQueryWrapper } from "@/test-utils/render";
+import { useSyncWebsiteSourceMutation } from "./useWebsiteSourceMutation";
+
+describe("useSyncWebsiteSourceMutation", () => {
+  it("syncs a child page and automatically refreshes active task queries", async () => {
+    let syncCalls = 0;
+    let taskRequests = 0;
+    server.use(
+      http.post(
+        "/api/connectors/url/sources/source-1/pages/page-1/sync",
+        () => {
+          syncCalls += 1;
+          return HttpResponse.json({ task_id: "task-1" });
+        },
+      ),
+      http.get("/api/tasks/enhanced", () => {
+        taskRequests += 1;
+        return HttpResponse.json({ tasks: [] });
+      }),
+    );
+
+    const { result } = renderHook(
+      () => {
+        useGetTasksQuery();
+        return useSyncWebsiteSourceMutation();
+      },
+      { wrapper: createQueryWrapper() },
+    );
+
+    await waitFor(() => expect(taskRequests).toBeGreaterThan(0));
+    const taskRequestsBeforeSync = taskRequests;
+
+    await act(async () => {
+      await result.current.mutateAsync({
+        sourceId: "source-1",
+        pageId: "page-1",
+      });
+    });
+
+    expect(syncCalls).toBe(1);
+    await waitFor(() =>
+      expect(taskRequests).toBeGreaterThan(taskRequestsBeforeSync),
+    );
+  });
+});

frontend/app/api/mutations/useWebsiteSourceMutation.ts

@@ -0,0 +1,135 @@
+import {
+  type QueryClient,
+  useMutation,
+  useQueryClient,
+} from "@tanstack/react-query";
+import { TASKS_QUERY_KEY } from "@/app/api/queries/useGetTasksQuery";
+import { websiteSourceQueryKey } from "@/app/api/queries/useGetWebsiteSourceQuery";
+import { websiteSourcesQueryKey } from "@/app/api/queries/useGetWebsiteSourcesQuery";
+import type { WebsiteSource } from "@/components/website-pages/types";
+
+export interface CreateWebsiteSourceRequest {
+  name: string;
+  starting_url: string;
+  scope: "path" | "page" | "site";
+  allow_subdomains: boolean;
+  additional_hosts: string[];
+  include_paths: string[];
+  exclude_paths: string[];
+  max_pages: number;
+  max_depth: number;
+  max_downloaded_mb: number;
+  max_crawl_minutes: number;
+  change_detection: "normalized_content_hash" | "always_reingest";
+  resync_behavior: "full" | "root";
+  removed_page_behavior: "retain" | "delete";
+}
+
+export interface WebsiteSourceTaskResponse {
+  last_task_id?: string;
+  task_id?: string;
+}
+
+export interface WebsiteSourcePageRequest {
+  sourceId: string;
+  pageId?: string;
+}
+
+async function readError(response: Response, fallback: string) {
+  const data = (await response.json().catch(() => ({}))) as {
+    detail?: string;
+    error?: string;
+  };
+  return data.detail || data.error || fallback;
+}
+
+async function createWebsiteSource(
+  request: CreateWebsiteSourceRequest,
+): Promise<WebsiteSource & WebsiteSourceTaskResponse> {
+  const response = await fetch("/api/connectors/url/sources", {
+    method: "POST",
+    headers: { "Content-Type": "application/json" },
+    body: JSON.stringify(request),
+  });
+  if (!response.ok) {
+    throw new Error(
+      await readError(response, "Could not create website source"),
+    );
+  }
+  return response.json();
+}
+
+async function syncWebsiteSource({
+  sourceId,
+  pageId,
+}: WebsiteSourcePageRequest): Promise<WebsiteSourceTaskResponse> {
+  const endpoint = pageId
+    ? `/api/connectors/url/sources/${sourceId}/pages/${pageId}/sync`
+    : `/api/connectors/url/sources/${sourceId}/sync`;
+  const response = await fetch(endpoint, { method: "POST" });
+  if (!response.ok) {
+    throw new Error(await readError(response, "Could not start website sync"));
+  }
+  return response.json();
+}
+
+async function deleteWebsiteSourcePage({
+  sourceId,
+  pageId,
+}: WebsiteSourcePageRequest): Promise<void> {
+  const endpoint = pageId
+    ? `/api/connectors/url/sources/${sourceId}/pages/${pageId}`
+    : `/api/connectors/url/sources/${sourceId}`;
+  const response = await fetch(endpoint, { method: "DELETE" });
+  if (!response.ok) {
+    throw new Error(
+      await readError(response, "Could not delete website source"),
+    );
+  }
+}
+
+function invalidateWebsiteSourceQueries(
+  queryClient: QueryClient,
+  sourceId: string,
+) {
+  queryClient.invalidateQueries({ queryKey: websiteSourceQueryKey(sourceId) });
+  queryClient.invalidateQueries({ queryKey: websiteSourcesQueryKey });
+  queryClient.invalidateQueries({ queryKey: ["search"] });
+  queryClient.invalidateQueries({ queryKey: [...TASKS_QUERY_KEY] });
+}
+
+export function useCreateWebsiteSourceMutation() {
+  const queryClient = useQueryClient();
+
+  return useMutation({
+    mutationFn: createWebsiteSource,
+    onSuccess: () => {
+      queryClient.invalidateQueries({ queryKey: ["search"] });
+      queryClient.invalidateQueries({ queryKey: ["listFiles"] });
+      queryClient.invalidateQueries({ queryKey: websiteSourcesQueryKey });
+      queryClient.invalidateQueries({ queryKey: [...TASKS_QUERY_KEY] });
+    },
+  });
+}
+
+export function useSyncWebsiteSourceMutation() {
+  const queryClient = useQueryClient();
+
+  return useMutation({
+    mutationFn: syncWebsiteSource,
+    onSuccess: (_data, { sourceId }) => {
+      invalidateWebsiteSourceQueries(queryClient, sourceId);
+    },
+  });
+}
+
+export function useDeleteWebsiteSourcePageMutation() {
+  const queryClient = useQueryClient();
+
+  return useMutation({
+    mutationFn: deleteWebsiteSourcePage,
+    onSuccess: (_data, { sourceId }) => {
+      invalidateWebsiteSourceQueries(queryClient, sourceId);
+    },
+  });
+}

frontend/app/api/queries/useFileScopedChunksQuery.test.tsx

@@ -2,7 +2,10 @@ import { HttpResponse, http } from "msw";
 import { describe, expect, it } from "vitest";
 import { server } from "@/test-utils/msw/server";
 import { createQueryWrapper, renderHook, waitFor } from "@/test-utils/render";
-import { useFileScopedChunksQuery } from "./useFileScopedChunksQuery";
+import {
+  useDocumentScopedChunksQuery,
+  useFileScopedChunksQuery,
+} from "./useFileScopedChunksQuery";
 import type { ChunkResult } from "./useGetSearchQuery";
 
 function chunk(overrides: Partial<ChunkResult> = {}): ChunkResult {
@@ -125,6 +128,51 @@ describe("useFileScopedChunksQuery", () => {
     expect(result.current.file?.chunks?.[0]?.highlights).toBeDefined();
   });
 
+  it("merges highlights without dropping non-matching chunks", async () => {
+    server.use(
+      http.post("/api/search", async ({ request }) => {
+        const body = (await request.json()) as { query: string };
+        return HttpResponse.json({
+          results:
+            body.query === "*"
+              ? [
+                  chunk({
+                    filename: "test.pdf",
+                    chunk_id: "one",
+                    text: "match",
+                  }),
+                  chunk({
+                    filename: "test.pdf",
+                    chunk_id: "two",
+                    text: "other",
+                  }),
+                ]
+              : [
+                  chunk({
+                    filename: "test.pdf",
+                    chunk_id: "one",
+                    text: "match",
+                    highlights: ["<mark>match</mark>"],
+                  }),
+                ],
+          warnings: [],
+        });
+      }),
+    );
+
+    const { result } = renderHook(
+      () => useFileScopedChunksQuery("test.pdf", "match"),
+      { wrapper: createQueryWrapper() },
+    );
+
+    await waitFor(() => expect(result.current.isFetching).toBe(false));
+    expect(result.current.file?.chunks).toHaveLength(2);
+    expect(result.current.file?.chunks?.[0]?.highlights).toEqual([
+      "<mark>match</mark>",
+    ]);
+    expect(result.current.file?.chunks?.[1]?.highlights).toEqual([]);
+  });
+
   it("finds the matching file from multiple files in response", async () => {
     serveSearch([
       chunk({ filename: "other.pdf", text: "other chunk" }),
@@ -171,3 +219,63 @@ describe("useFileScopedChunksQuery", () => {
     expect(result.current.file?.filename).toBe("test.pdf");
   });
 });
+
+describe("useDocumentScopedChunksQuery", () => {
+  it("merges highlights by stable document id without removing other chunks", async () => {
+    const requests: { query: string; documentIds?: string[] }[] = [];
+    server.use(
+      http.post("/api/search", async ({ request }) => {
+        const body = (await request.json()) as {
+          query: string;
+          filters?: { document_ids?: string[] };
+        };
+        requests.push({
+          query: body.query,
+          documentIds: body.filters?.document_ids,
+        });
+        return HttpResponse.json({
+          results:
+            body.query === "*"
+              ? [
+                  chunk({
+                    document_id: "page-1",
+                    chunk_id: "one",
+                    text: "match",
+                  }),
+                  chunk({
+                    document_id: "page-1",
+                    chunk_id: "two",
+                    text: "other",
+                  }),
+                ]
+              : [
+                  chunk({
+                    document_id: "page-1",
+                    chunk_id: "one",
+                    text: "match",
+                    highlights: ["<mark>match</mark>"],
+                  }),
+                ],
+          warnings: [],
+        });
+      }),
+    );
+
+    const { result } = renderHook(
+      () => useDocumentScopedChunksQuery("page-1", "match"),
+      { wrapper: createQueryWrapper() },
+    );
+
+    await waitFor(() => expect(result.current.isFetching).toBe(false));
+    expect(requests).toEqual(
+      expect.arrayContaining([
+        expect.objectContaining({ query: "*", documentIds: ["page-1"] }),
+        expect.objectContaining({ query: "match", documentIds: ["page-1"] }),
+      ]),
+    );
+    expect(result.current.file?.chunks).toHaveLength(2);
+    expect(result.current.file?.chunks?.[0]?.highlights).toEqual([
+      "<mark>match</mark>",
+    ]);
+  });
+});