chore: promote main to staging (query-mode-router + pending)

.github/workflows/build-images.yml

@@ -774,6 +774,7 @@ jobs:
           API_WEBHOOK_ENDPOINT: https://api.knowhereto.ai/v1/internal/s3-events
           SNS_TOPIC_ARN: arn:aws:sns:us-east-1:107424103509:knowhere-prod-s3-events
           QSTASH_CALLBACK_BASE_URL: https://api.knowhereto.ai/api/v1
+          QSTASH_BASE_URL: https://qstash-us-east-1.upstash.io
           API_CPU: "512"
           API_MEMORY: "2048"
           WORKER_CPU: "2048"

AGENTS.md

@@ -102,8 +102,8 @@ flowchart TB
 
     subgraph RETRIEVE["⑤ Retrieval (shared)"]
         Query["POST /v1|/v2 retrieval/query"] --> Pipeline["run_retrieval_query"]
-        Pipeline --> Classic["classic_topk / small_corpus (use_agentic=False)"]
-        Pipeline --> Explore["agent_explore (use_agentic unset/true)"]
+        Pipeline --> Classic["classic_topk / small_corpus"]
+        Pipeline --> Explore["agent_explore"]
         Explore --> Cursor["AGENT_EXPLORE_HARNESS=cursor_sdk"]
         Explore --> Own["AGENT_EXPLORE_HARNESS=openai"]
         Classic --> Channels["map_unit_discovery: path+content BM25 -> RRF"]
@@ -129,8 +129,10 @@ This is the typed `ParseOutput` entry for all file types. The parser flow:
 
 1. **Profiles** the document via `profiling.doc_profiler.profile_document()`. PDF
    profiling uses `document_agent` as the single PyMuPDF feature source, then runs
-   VLM coarse classification with two fields: open semantic `category` (for example,
-   `Financial Prospectus`) and routing-only `routing_category`
+   VLM coarse classification with three fields: `category` (IPTC Media Topics
+   top-level codes from the numbered list in `profiling/domain_taxonomy.py`,
+   multiple allowed), `category_label` (open short phrase, for example
+   `Financial Prospectus`), and routing-only `routing_category`
    (`atlas/scanned/slides/generic`). Oversized non-atlas PDFs additionally run the
    structural anatomy stage once at the entry point and pass the resulting shard
    plan to PDF parsing.
@@ -237,7 +239,6 @@ DataFrame is an internal detail. Below is the complete schema.
 ```text
 ~/.knowhere/{corpus_name}/
 ├── knowledge_graph.json           # corpus-wide graph: file metadata + cross-doc edges
-├── chunk_stats.json               # Per-chunk retrieval hit analytics {chunk_id → stats}
 ├── {source_file_name}/            # One directory per ingested document
 │   ├── chunks.json                # All parsed chunks for this document
 │   ├── doc_nav.json               # Hierarchical navigation tree for agentic retrieval
@@ -264,7 +265,6 @@ DataFrame is an internal detail. Below is the complete schema.
       "types": { "image": 13, "table": 1, "text": 141 },
       "top_keywords": ["model", "security", "ai", "operations", "artificial_intelligence"],
       "top_summary": "This document includes: Legal Notice, Foreword, 1. Overview, ...",
-      "importance": 0.3,
       "created_at": "2026-05-09T09:14:12.422208+00:00"
     }
   },
@@ -276,25 +276,8 @@ DataFrame is an internal detail. Below is the complete schema.
 |:---|:---|
 | `files.{name}.top_keywords` | TF-IDF top keywords across all chunks (used for cross-doc edge scoring) |
 | `files.{name}.top_summary` | Auto-generated outline of top-level headings (injected by `load_nav_top_summary()`) |
-| `files.{name}.importance` | Base importance score (feeds `compute_importance_score()` in ranking) |
 | `edges[]` | Cross-document edges with `{source, target, weight, shared_keywords}` when keyword overlap ≥ 0.8 |
 
-### `chunk_stats.json` — Retrieval Hit Analytics
-
-```json
-{
-  "2e2beffc-90b2-5429-8ee7-3c49260a1204": {
-    "hit_count": 0,
-    "first_hit": null,
-    "last_hit": null,
-    "created_at": "2026-05-09T09:14:12.423011+00:00"
-  }
-}
-```
-
-Keyed by `chunk_id`. `hit_count` and `last_hit` feed into `importance_norm_score`
-for retrieval ranking boost.
-
 ### `chunks.json` — Per-Document Chunk Records
 
 The core persisted data. Contains `{"chunks": [...]}` — an ordered array of
@@ -340,7 +323,10 @@ chunk objects. Three chunk types exist:
     "page_nums": [],
     "file_path": "images/image-1 ai_model.png",
     "keywords": [],
-    "tokens": []
+    "tokens": [],
+    "connect_to": [
+      { "target": "d88e4c47-3c48-5bdf-b849-693c00453021", "relation": "embedded_by" }
+    ]
   }
 }
 ```
@@ -374,13 +360,14 @@ chunk objects. Three chunk types exist:
 | `path` | `str` | Hierarchical path: `{kb}/{file}/{section1}/{section2}/...` for text; `images/...` or `tables/...` for assets |
 | `metadata.length` | `int` | Character count of content |
 | `metadata.summary` | `str` | LLM summary (images/tables: `"image-N\n{description}"`) |
-| `metadata.tokens` | `list[str]` | Pre-tokenized Chinese terms for BM25 retrieval |
+| `metadata.entities` | `list[{text, type}]` | Typed entities extracted from content or asset VLM/LLM analysis |
 | `metadata.keywords` | `list[str]` | LLM-extracted keywords (semicolon-split from DataFrame) |
 | `metadata.page_nums` | `list[int]` | Source page numbers (PDF only) |
 | `metadata.file_path` | `str` | Relative asset path for images/tables |
-| `metadata.connect_to` | `list[ConnectionValue]` | Cross-chunk references (text→image/table embeddings) |
+| `metadata.connect_to` | `list[ConnectionValue]` | Cross-chunk relations (body↔image/table embeddings) |
 | `metadata.connect_to[].target` | `str` | Target chunk_id |
-| `metadata.connect_to[].relation` | `str` | `"embeds"` (inline asset) or `"related"` |
+| `metadata.connect_to[].target_document_id` | `str` | Reserved for cross-document targets; omitted for same-document edges |
+| `metadata.connect_to[].relation` | `str` | `"embeds"` (body → inline asset), `"embedded_by"` (asset → owning body), `"related"` (same-page body → asset); `"cross_doc_entity"` / `"intra_doc_ref"` are reserved and not generated yet |
 | `metadata.connect_to[].ref` | `str` | Original reference string: `"[images/image-1.png]"` |
 | `metadata.connect_to[].position` | `{start, end}` | Character offset of the reference in content |
 
@@ -555,7 +542,7 @@ Core retrieval internals are grouped by ownership:
 
 ### Retrieval Switch and Agent Modes
 
-Per-request `use_agentic` only turns agent retrieval on or off: `False` → classic top-K (map-unit BM25); `None`/`True` → agent_explore. When agent retrieval is on, `AGENT_EXPLORE_HARNESS` selects Cursor (`cursor_sdk`) or Knowhere's own harness (`openai`).
+Per-request `use_agentic`: `False` → classic top-K (map-unit BM25); `True` → agent_explore; unset (`None`) → a query router chooses classic or agent. When agent retrieval is on, `AGENT_EXPLORE_HARNESS` selects Cursor (`cursor_sdk`) or Knowhere's own harness (`openai`).
 
 #### Classic Mode (map-unit BM25)
 

CONTEXT.md

@@ -59,6 +59,18 @@ The hierarchical navigation node derived from parsed headings and section paths.
 
 The retrieval-visible text, image, or table row attached to a Document Section.
 
+### Document Root
+
+The existing document-level container above its chapters. It can own opening
+content without introducing another chapter.
+
+### Document Prefix
+
+All physical pages before the first recognized chapter, including covers,
+opening text, and printed table-of-contents pages. These pages belong directly
+to the existing document Root; they are not a separate chapter or part of the
+first chapter.
+
 ### Document Ingestion
 
 The workflow that creates a Job, accepts a file or URL source, confirms upload
@@ -69,6 +81,18 @@ state, and starts parsing work.
 The API-side handoff that advances an uploaded file Job to pending state and
 enqueues the worker parse task with the user-aware Celery queue policy.
 
+### Source Content Admission
+
+The check that rejects identical original file bytes within one User and
+Namespace before parsing or billing, including unchanged Document updates.
+Filenames do not determine duplicates. Only current active Document revisions
+and in-progress Jobs participate; failed Jobs and archived or superseded
+revisions do not block retries. The check happens after source upload or URL
+download, so a duplicate is reported through the normal Job failure contract.
+Demo Corpus publication retains its separate admission policy.
+Existing sources without recorded fingerprints do not participate, and their
+original files are not fetched for comparison.
+
 ### Worker Document Parsing
 
 The worker-side workflow that turns a source file into parsed DataFrame rows,

apps/api/.env.example

@@ -99,13 +99,17 @@ ARK_API_KEY=
 # ASSET_MODEL=
 
 # Optional retrieval overrides have code defaults. Retrieval is evidence-only:
-# evidence_text is the primary output and answer_text is always empty.
+# evidence is the primary output; evidence_text is deprecated and always empty;
+# answer_text is always empty.
 # use_agentic on the request turns agent retrieval on or off:
 #   false     → no agent; classic map-unit BM25
-#   omit/true → agent on; AGENT_EXPLORE_HARNESS selects the mode
+#   true      → agent on; AGENT_EXPLORE_HARNESS selects the mode
+#   omit      → query router chooses classic or agent
 #     cursor_sdk → Cursor (needs CURSOR_API_KEY)
 #     openai    → Knowhere's own harness (DS_KEY / ALI_API_KEYS)
 # AGENT_EXPLORE_HARNESS=cursor_sdk
+# RETRIEVAL_QUERY_ROUTER_ENABLED=true
+# RETRIEVAL_QUERY_ROUTER_MODEL=
 # CURSOR_API_KEY=   # only when harness is cursor_sdk
 # AGENT_EXPLORE_CURSOR_MODEL=composer-2.5
 # Classic BM25 may use Postgres FTS prefilter:

apps/api/alembic/versions/5e6f7a8b9c0d_add_source_content_hash_index.py

@@ -0,0 +1,71 @@
+"""Index future source content fingerprints without backfilling existing Jobs."""
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+
+from alembic import op
+from sqlalchemy import text
+
+revision: str = "5e6f7a8b9c0d"
+down_revision: str = "4d5e6f7a8b9c"
+branch_labels: Sequence[str] | None = None
+depends_on: Sequence[str] | None = None
+
+
+def _create_index(*, concurrently: bool) -> None:
+    existing_index: tuple[bool, str, str] | None = (
+        op.get_bind()
+        .execute(
+            text(
+                "SELECT indexes.indisvalid AND indexes.indisready, "
+                "tables.relname, pg_get_indexdef(indexes.indexrelid) "
+                "FROM pg_index AS indexes "
+                "JOIN pg_class AS classes ON classes.oid = indexes.indexrelid "
+                "JOIN pg_class AS tables ON tables.oid = indexes.indrelid "
+                "JOIN pg_namespace AS namespaces ON namespaces.oid = classes.relnamespace "
+                "WHERE namespaces.nspname = current_schema() "
+                "AND classes.relname = 'idx_jobs_user_source_content_hash'"
+            )
+        )
+        .tuples()
+        .first()
+    )
+    if existing_index is not None:
+        is_ready, table_name, definition = existing_index
+        normalized_definition: str = " ".join(definition.split())
+        intended_definition: str = (
+            "USING btree (user_id, ((job_metadata ->> 'source_content_sha256'::text))) "
+            "WHERE ((job_metadata ->> 'source_content_sha256'::text) IS NOT NULL)"
+        )
+        if (
+            is_ready
+            and table_name == "jobs"
+            and normalized_definition.startswith("CREATE INDEX ")
+            and normalized_definition.endswith(intended_definition)
+        ):
+            return
+    concurrent_clause: str = "CONCURRENTLY " if concurrently else ""
+    op.execute(
+        f"DROP INDEX {concurrent_clause}IF EXISTS idx_jobs_user_source_content_hash"
+    )
+    op.execute(
+        f"CREATE INDEX {concurrent_clause}IF NOT EXISTS idx_jobs_user_source_content_hash "
+        "ON jobs (user_id, (job_metadata ->> 'source_content_sha256')) "
+        "WHERE job_metadata ->> 'source_content_sha256' IS NOT NULL"
+    )
+
+
+def upgrade() -> None:
+    uses_external_transaction: bool = bool(
+        op.get_context().opts.get("knowhere_external_transaction", False)
+    )
+    if uses_external_transaction:
+        _create_index(concurrently=False)
+        return
+    with op.get_context().autocommit_block():
+        _create_index(concurrently=True)
+
+
+def downgrade() -> None:
+    op.execute("DROP INDEX IF EXISTS idx_jobs_user_source_content_hash")

apps/api/app/api/v1/routes/demo.py

@@ -31,6 +31,19 @@ async def list_demo_source_chunks(demo_source_id: str, job_result_id: str | None
     return {**(response or {}), "demo_source_id": demo_source_id, "canonical_document_id": document.document_id}
 
 
+@router.get("/sources/{demo_source_id}/structure")
+async def get_demo_source_structure(
+    demo_source_id: str,
+    response: Response,
+    job_result_id: str | None = Query(None, max_length=36),
+    db: AsyncSession = Depends(get_db),
+) -> dict[str, Any]:
+    response.headers["Cache-Control"] = "no-store"
+    return await _demo_service.get_structure(
+        db, source_id=demo_source_id, job_result_id=job_result_id,
+    )
+
+
 @router.get("/sources/{demo_source_id}/chunks/{demo_chunk_id}")
 async def get_demo_source_chunk(demo_source_id: str, demo_chunk_id: str, job_result_id: str | None = Query(None, max_length=36), db: AsyncSession = Depends(get_db)) -> dict[str, Any] | None:
     document, revision = await _demo_service.resolve_source(db, source_id=demo_source_id, job_result_id=job_result_id)

apps/api/app/api/v1/routes/retrieval.py

@@ -73,8 +73,9 @@ class RetrievalQueryRequest(BaseModel):
         None,
         description=(
             "Turns agent retrieval on or off. False uses classic map-unit "
-            "BM25. Unset or true turns agent retrieval on; "
-            "AGENT_EXPLORE_HARNESS then selects Cursor (cursor_sdk) or "
+            "BM25. True turns agent retrieval on. Unset (None) lets the "
+            "query router choose classic or agent. When agent retrieval is "
+            "on, AGENT_EXPLORE_HARNESS then selects Cursor (cursor_sdk) or "
             "Knowhere's own harness (openai)."
         ),
     )
@@ -121,17 +122,25 @@ class RetrievalQueryResponse(BaseModel):
     router_used: str
     evidence: list[dict] = Field(
         default_factory=list,
-        description="Composed evidence parts (text and inline images) for downstream agents.",
+        description=(
+            "Grouped evidence blocks. Each group starts with "
+            "[E1] [§ source_file_name / parent_path]. Text and table HTML "
+            "stay as text parts; images stay as image parts at their "
+            "placeholder positions."
+        ),
     )
     evidence_text: str = Field(
         default="",
-        description="Text projection of evidence. Tables stay as HTML; images are data URLs.",
+        description=(
+            "DEPRECATED. Always empty. Knowhere no longer projects evidence "
+            "into a string. Use evidence."
+        ),
     )
     answer_text: str = Field(
         default="",
         description=(
             "DEPRECATED. Always empty; KNOWHERE no longer generates answers. "
-            "Use evidence_text and synthesize answers downstream."
+            "Use evidence and synthesize answers downstream."
         ),
     )
     referenced_chunks: list[dict] = Field(default_factory=list)
@@ -148,6 +157,14 @@ class RetrievalQueryResponse(BaseModel):
             "observation/decision/result schema."
         ),
     )
+    router_decision: dict[str, Any] | None = Field(
+        default=None,
+        description=(
+            "Present when use_agentic was unset and the query router chose "
+            "agent or classic. Fields: mode, reason, source, model_name, "
+            "tokens_used."
+        ),
+    )
 
 
 async def execute_retrieval_query(

apps/api/app/core/exception_handlers.py

@@ -126,7 +126,7 @@ async def knowhere_exception_handler(
     # responses and ``WWW-Authenticate`` for authentication challenges.
     headers = dict(response_headers or {})
     headers["X-Request-ID"] = request_id
-    retry_after = exc.details.get("retry_after")
+    retry_after = (exc.details or {}).get("retry_after")
     if retry_after:
         headers["Retry-After"] = str(retry_after)