Staging

AGENTS.md

@@ -576,9 +576,9 @@ flowchart LR
 **Channel weights** (default): path=1.0, content=2.0. **RRF formula**:
 `score = weight / (k + rank + 1)` per channel, summed across channels, `k=60`.
 
-There is no scored term channel in this primary path. `term_search_text` /
-`term_search_text_lower` are persisted at publish time and are read by
-`corpus.recall`'s term channel, not by classic discovery.
+There is no scored term channel in this primary path. Chunk
+`term_search_text` is persisted at publish time and is read by
+`corpus.grep`, not by classic discovery.
 
 #### Agent-explore Mode (default)
 

CONTEXT.md

@@ -343,7 +343,7 @@ The query workflow that returns cited evidence from published documents.
 ### Retrieval Query
 
 The typed retrieval request that owns cache-shaping fields and route policy:
-scope, filters, data type, channels, ranking options, and agentic toggle.
+scope, filters, data type, ranking options, and agentic toggle.
 
 ### Retrieval Run
 

apps/api/.env.example

@@ -136,6 +136,12 @@ MOESIF_APPLICATION_ID=
 FREE_PLAN_INITIAL_CREDITS=5
 FRONTEND_URL=http://localhost:3000
 
+# Reuse canonical demo bundles; disable to restore per-job uploads while keeping
+# existing canonical artifact URLs readable. Does not change publication strategy.
+DEMO_CANONICAL_BUNDLE_ENABLED=false
+# Optional per-process tokenization/frequency reuse for versioned demo sources.
+DEMO_PUBLICATION_PREPARATION_CACHE_ENABLED=false
+
 # Required for specific features: parsing providers
 MINERU_API_KEYS=
 ILOVEAPI_PUBLIC_KEY=

apps/api/alembic/versions/2b3c4d5e6f7a_drop_map_unit_term_search_text.py

@@ -0,0 +1,55 @@
+"""Drop unused map-unit term-search leftovers.
+
+Classic scoring is path + content only. ``term_search_text_lower`` on
+``document_map_units`` and its trigram index had no reader. Chunk
+``term_search_text`` and ``idx_document_chunks_term_trgm`` stay: grep uses them.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+
+from alembic import op
+import sqlalchemy as sa
+
+
+revision: str = "2b3c4d5e6f7a"
+down_revision: str | None = "2f3g4h5i6j7"
+branch_labels: Sequence[str] | None = None
+depends_on: Sequence[str] | None = None
+
+__all__ = [
+    "revision",
+    "down_revision",
+    "branch_labels",
+    "depends_on",
+    "upgrade",
+    "downgrade",
+]
+
+_MAP_UNIT_INDEX = "idx_document_map_units_term_trgm"
+
+
+def upgrade() -> None:
+    op.execute(f"DROP INDEX IF EXISTS {_MAP_UNIT_INDEX}")
+    inspector = sa.inspect(op.get_bind())
+    columns = {col["name"] for col in inspector.get_columns("document_map_units")}
+    if "term_search_text_lower" in columns:
+        op.drop_column("document_map_units", "term_search_text_lower")
+
+
+def downgrade() -> None:
+    inspector = sa.inspect(op.get_bind())
+    columns = {col["name"] for col in inspector.get_columns("document_map_units")}
+    if "term_search_text_lower" not in columns:
+        op.add_column(
+            "document_map_units",
+            sa.Column("term_search_text_lower", sa.Text(), nullable=False, server_default=""),
+        )
+        op.alter_column("document_map_units", "term_search_text_lower", server_default=None)
+    op.execute("CREATE EXTENSION IF NOT EXISTS pg_trgm")
+    op.execute(
+        f"CREATE INDEX IF NOT EXISTS {_MAP_UNIT_INDEX} "
+        "ON document_map_units USING gin "
+        "(term_search_text_lower gin_trgm_ops)"
+    )

apps/api/alembic/versions/3c4d5e6f7a8b_gate_legacy_token_lookup_retirement.py

@@ -0,0 +1,56 @@
+"""Opt in to retiring the legacy token lookup after replacement validation."""
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+
+from alembic import context, op
+
+from shared.services.retrieval.maintenance.legacy_token_lookup import (
+    LegacyTokenLookupMaintenance,
+)
+
+revision: str = "3c4d5e6f7a8b"
+down_revision: str | None = "2b3c4d5e6f7a"
+branch_labels: Sequence[str] | None = None
+depends_on: Sequence[str] | None = None
+
+
+def _run_maintenance(*, is_restore: bool) -> None:
+    if context.is_offline_mode():
+        raise RuntimeError("Legacy token index maintenance requires live catalog checks")
+    is_external_transaction: bool = bool(
+        op.get_context().opts.get("knowhere_external_transaction", False)
+    )
+
+    def execute() -> None:
+        maintenance: LegacyTokenLookupMaintenance = LegacyTokenLookupMaintenance(
+            op.get_bind(), is_concurrent=not is_external_transaction
+        )
+        if is_restore:
+            maintenance.restore_index()
+        else:
+            maintenance.apply_retirement()
+
+    if is_external_transaction:
+        execute()
+        return
+    with op.get_context().autocommit_block():
+        execute()
+
+
+def upgrade() -> None:
+    """Default deployment records the revision and retains the legacy index."""
+    gate: str | None = context.get_x_argument(as_dictionary=True).get(
+        "retire_legacy_token_lookup"
+    )
+    if gate is None or gate == "false":
+        return
+    if gate != "true":
+        raise ValueError("retire_legacy_token_lookup must be exactly true or false")
+    _run_maintenance(is_restore=False)
+
+
+def downgrade() -> None:
+    """Restore the exact legacy definition, including after operator retirement."""
+    _run_maintenance(is_restore=True)

apps/api/app/api/v1/routes/retrieval.py

@@ -7,7 +7,7 @@
 from app.api.dependencies.current_user import with_current_user
 from app.services.rate_limit.data_structures import CurrentUser
 from fastapi import APIRouter, Depends
-from pydantic import BaseModel, Field, field_validator, model_validator
+from pydantic import BaseModel, Field, field_validator
 from sqlalchemy.ext.asyncio import AsyncSession
 
 from shared.core.database import get_db
@@ -64,20 +64,6 @@ class RetrievalQueryRequest(BaseModel):
     filter_mode: Literal["delete", "keep"] = Field(
         "delete", description="Signal path filter mode"
     )
-    channels: list[str] = Field(
-        default_factory=list,
-        description=(
-            "Deprecated and unsupported by the persisted map-unit route. "
-            "Leave empty; explicit channel selection is rejected."
-        ),
-    )
-    channel_weights: dict[str, float] = Field(
-        default_factory=dict,
-        description=(
-            "Deprecated and unsupported by the persisted map-unit route. "
-            "Leave empty; explicit overrides are rejected."
-        ),
-    )
     rerank: bool = Field(False, description="Enable LLM reranking after RRF fusion")
     threshold: float = Field(0.0, ge=0.0, description="Minimum RRF score threshold")
     internal_recall_k: int | None = Field(
@@ -111,15 +97,6 @@ class RetrievalQueryRequest(BaseModel):
         ),
     )
 
-    @field_validator("channels")
-    @classmethod
-    def validate_channels(cls, v: list[str]) -> list[str]:
-        valid = {"path", "content", "term"}
-        for ch in v:
-            if ch not in valid:
-                raise ValueError(f"Invalid channel: {ch}. Must be one of {valid}")
-        return v
-
     @field_validator("chunk_types")
     @classmethod
     def validate_chunk_types(cls, v: list[str] | None) -> list[str] | None:
@@ -137,20 +114,6 @@ def validate_chunk_types(cls, v: list[str] | None) -> list[str] | None:
     def normalize_namespace(cls, namespace: str | None) -> str:
         return normalize_retrieval_namespace(namespace)
 
-    @model_validator(mode="after")
-    def reject_unsupported_channel_controls(self) -> "RetrievalQueryRequest":
-        if self.channels:
-            raise ValueError(
-                "channels is deprecated and unsupported; omit it and use the "
-                "persisted path/content map-unit scorer"
-            )
-        if self.channel_weights:
-            raise ValueError(
-                "channel_weights is deprecated and unsupported; omit it and use "
-                "the persisted path/content map-unit scorer"
-            )
-        return self
-
 
 class RetrievalQueryResponse(BaseModel):
     namespace: str
@@ -209,20 +172,32 @@ async def execute_retrieval_query(
     else:
         resolved_chunk_types = None
 
+    query = str(payload.query or "").strip()
+    if not query:
+        return {
+            "namespace": normalize_retrieval_namespace(payload.namespace),
+            "query": query,
+            "router_used": "empty_query_filtered",
+            "failure_reason": "empty query — retrieval was not run",
+            "evidence": [],
+            "evidence_text": "",
+            "answer_text": "",
+            "referenced_chunks": [],
+            "results": [],
+        }
+
     return await run_retrieval_query(
         db=db,
         user_id=current_user.user_id,
         namespace=normalize_retrieval_namespace(payload.namespace),
-        query=payload.query,
+        query=query,
         top_k=payload.top_k,
         include_document_ids=payload.include_document_ids,
         exclude_document_ids=payload.exclude_document_ids,
         exclude_sections=[item.model_dump() for item in payload.exclude_sections],
         chunk_types=resolved_chunk_types,
         signal_paths=payload.signal_paths or None,
         filter_mode=payload.filter_mode,
-        channels=payload.channels or None,
-        channel_weights=payload.channel_weights or None,
         rerank=payload.rerank,
         threshold=payload.threshold,
         internal_recall_k=payload.internal_recall_k,

apps/api/app/mcp/dynamic_tools.py

@@ -20,30 +20,73 @@
 from mcp.server.fastmcp import Context, FastMCP
 from mcp.server.fastmcp.tools.base import Tool
 from mcp.server.fastmcp.utilities.func_metadata import ArgModelBase, FuncMetadata
-from pydantic import create_model
+from pydantic import ConfigDict, create_model
 from sqlalchemy.ext.asyncio import AsyncSession
 
 from shared.services.retrieval.agent_tools import REGISTRY, ToolContext, ToolSpec
 
 DbFactory = Callable[[], AsyncContextManager[AsyncSession]]
 
 
+class _AllowExtraArgModel(ArgModelBase):
+    """``ArgModelBase`` that forwards only actually-supplied keys, plus any
+    unrecognized ones.
+
+    ``FuncMetadata.call_fn_with_arg_validation`` builds ``_dispatch_tool``'s
+    kwargs by calling ``model_dump_one_level()``. The base implementation
+    (verified by reading ``mcp.server.fastmcp.utilities.func_metadata``
+    directly) iterates every *declared* ``model_fields`` name unconditionally
+    — since every property is declared here as ``(Any, None)`` (see
+    ``_lenient_arg_model`` below), an MCP client that simply omits an
+    optional argument still gets that argument dumped as an explicit
+    ``None`` into ``REGISTRY.dispatch``'s args dict. ``validate_tool_args``
+    then fails a perfectly legal call, e.g. ``corpus.grep(pattern="x")``
+    reaching dispatch as ``{"pattern": "x", "chunk_types": None, ...}``,
+    which trips ``chunk_types``'s ``"type": "array"`` schema check. This was
+    a real, reproduced MCP-entry contract break, not a hypothetical one.
+
+    Fix: dump only ``self.model_fields_set`` (pydantic's own record of which
+    declared fields were actually present in the validated input — verified
+    live that omitted fields are absent from this set even when every field
+    has a ``None`` default) plus ``model_extra`` for genuinely unknown keys.
+    Declaring ``extra="allow"`` alone is not enough for the *unknown-key*
+    half: pydantic stores those on the instance (``model_extra``) but the
+    base implementation never reads them; merging them in here is what gets
+    an unknown key from an MCP client into ``dispatch`` -> ``validate_tool_args``'s
+    unified unknown-argument error, instead of the previous fully-permissive
+    model silently dropping it before dispatch ever saw it. Together this
+    makes an MCP call's dumped kwargs exactly the set of keys the client
+    actually sent — omitted optional keys absent, unknown keys present (and
+    then rejected by schema validation), never a fabricated ``None``.
+    """
+
+    model_config = ConfigDict(arbitrary_types_allowed=True, extra="allow")
+
+    def model_dump_one_level(self) -> dict[str, Any]:
+        dumped = {name: getattr(self, name) for name in self.model_fields_set}
+        dumped.update(self.model_extra or {})
+        return dumped
+
+
 def _lenient_arg_model(spec: ToolSpec) -> type[ArgModelBase]:
-    """Build a permissive pydantic arg model for FastMCP's internal validation.
+    """Build a pydantic arg model for FastMCP's internal validation.
 
     The schema actually exposed to MCP clients is ``spec.json_schema``
     (``Tool.parameters``, returned verbatim in ``tools/list`` — see
     ``FastMCP.list_tools``); this model only has to satisfy
     ``FuncMetadata.call_fn_with_arg_validation`` well enough to forward
-    whatever the client sent through to ``_dispatch_tool``'s ``**kwargs``.
-    Every property is optional/``Any`` — each tool already validates its own
-    required args and reports a caller-facing ``ToolResult.error`` for a
-    missing one, so duplicating "required" enforcement here would just
-    produce a less informative MCP-level error instead.
+    whatever the client sent through to ``_dispatch_tool``'s ``**kwargs``,
+    unknown keys included (``_AllowExtraArgModel`` above) — real
+    argument-shape enforcement happens once in
+    ``agent_tools.registry.validate_tool_args``, not duplicated here.
+    Every declared property is optional/``Any``: each tool already
+    validates its own required args through that same unified path.
     """
     properties = spec.json_schema.get("properties", {})
     fields: dict[str, Any] = {key: (Any, None) for key in properties}
-    return create_model(f"{spec.name.replace('.', '_')}_Args", __base__=ArgModelBase, **fields)
+    return create_model(
+        f"{spec.name.replace('.', '_')}_Args", __base__=_AllowExtraArgModel, **fields
+    )
 
 
 def _make_tool(spec: ToolSpec, *, db_factory: DbFactory) -> Tool:

apps/api/app/mcp/retrieval_server.py

@@ -65,6 +65,8 @@ def to_mcp_query_response(response: dict[str, Any]) -> dict[str, Any]:
     """
     return {
         "query": response.get("query"),
+        "router_used": response.get("router_used"),
+        "failure_reason": response.get("failure_reason"),
         "evidence": response.get("evidence") or [],
         "evidence_text": response.get("evidence_text") or "",
         "results": response.get("results") or [],
@@ -155,8 +157,17 @@ async def query_documents(
         # automatically before calling run_retrieval_query.
         # See: shared/services/retrieval/intent/ (to be created)
         namespace = resolve_mcp_namespace(ctx=ctx)
+        query = str(query or "").strip()
         async with db_factory() as db:
             user_id = await resolve_mcp_user_id(ctx=ctx, db=db)
+            if not query:
+                return to_mcp_query_response(
+                    {
+                        "query": query,
+                        "router_used": "empty_query_filtered",
+                        "failure_reason": "empty query — retrieval was not run",
+                    }
+                )
             response = await run_retrieval_query(
                 db=db,
                 user_id=user_id,