From cfcd9e58faeca07932a8d21ae68659313b8fd69f Mon Sep 17 00:00:00 2001 From: Yannick Bazan Date: Tue, 20 Oct 2026 16:14:54 -0400 Subject: [PATCH] fix(retrieval): OR the keyword query and let strong keyword matches past the cosine floor (spec 019) websearch_to_tsquery ANDs every word, so a natural-language question matched almost no passage and hybrid search equalled vector search (baseline variants: R@5 0.86 / MRR 0.84 for both). The question's lexemes are now OR-ed into one tsquery, quoted as they are so nothing stems them twice. While cosine is the gate (reranking off or skipped), the floor also dropped every keyword-only hit below MinScore, the exact-term matches hybrid search exists for. A keyword match now passes the floor when the passage holds at least Hybrid:MinKeywordCoverage (default 0.5) of the question's distinct lexemes; one common word is not evidence. ConfidentScore still applies, so keyword-only evidence is low-confidence. Coverage is a new stage field on the diagnostic endpoint; the setting is shown in the admin configuration and recorded in harness reports. Co-Authored-By: Claude Opus 5.5 --- CLAUDE.md | 2 +- docs/specs/019-hybrid-search-and-reranking.md | 13 +++-- .../DocumentsModule.cs | 2 +- .../Features/Admin/AdminDocumentsModels.cs | 3 +- .../Admin/GetDocumentsConfiguration.cs | 3 +- .../Features/SearchPassages/SearchPassages.cs | 9 +++- .../SearchPassages/SearchPassagesEndpoint.cs | 6 ++- .../Options/RetrievalOptions.cs | 7 +++ .../Retrieval/EvidenceRules.cs | 9 +++- .../Retrieval/HybridRanking.cs | 9 ++-- .../Retrieval/PassageCandidates.cs | 23 ++++++--- .../Contracts/Documents/SearchResult.cs | 4 +- .../Admin/AdminModels.cs | 3 +- .../Admin/ConfigurationTab.razor | 2 +- .../HybridRankingTests.cs | 19 ++++++- .../Documents/HybridSearchTests.cs | 51 +++++++++++++++++++ .../Run/ReportWriter.cs | 2 +- .../RagAssistant.Evaluation/Run/RunCommand.cs | 3 +- .../RagAssistant.Evaluation/Run/RunReport.cs | 3 +- 19 files changed, 142 insertions(+), 31 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 7089477..4f9445a 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -12,7 +12,7 @@ Stack decisions are recorded in [ADR 0001](docs/adr/0001-ui-ai-streaming-stack.m | Module | Owns | Schema | |--------|------|--------| -| `Documents` | Upload/ingest documents, text extraction, chunking, embedding generation, vector storage, similarity search. Exists: data model, `POST /api/documents`, `GET /api/documents/{id}` ([spec 006](docs/specs/006-documents-data-model-and-import.md)); background ingestion (`IngestionWorker`, chunker, batched embedding), `POST /api/documents/{id}/ingestion/retry` and the `GET …/ingestion/events` SSE stream ([spec 007](docs/specs/007-background-ingestion-and-progress.md)); listing with filters and paging, facets, versions and passages of a version (`GET /api/documents`, `/facets`, `/{id}/versions`, `/{id}/versions/{n}/passages`) and the Documents UI ([spec 008](docs/specs/008-browse-corpus-and-documents-ui.md)); edit metadata (`PUT`, `If-Match`/`ETag`), new version, reprocess, archive and restore, with the matching detail-page actions ([spec 009](docs/specs/009-document-lifecycle.md)); PDF import up to 100 MiB (streamed upload into a content-addressed file store, text-layer extraction with page numbers on passages, scanned PDFs rejected at upload), `GET …/versions/{n}/file` for the original and `progress` events inside long ingestion stages ([spec 010](docs/specs/010-pdf-import.md)); retrieval through `IDocumentSearch` (vector search over active, indexed versions with language and tag filters, evidence thresholds, near-duplicate reduction, cached query embeddings) and the maintainers' diagnostic `GET /api/documents/search` ([spec 011](docs/specs/011-retrieval.md)); private and shared documents: `Visibility` on `Document`, owner = `CreatedBy`, one `ReadableBy(viewerSub)` rule applied on every read path, so a document the caller cannot read is a 404 and `IDocumentSearch`/`IDocumentCatalog` take a required `viewerSub` ([spec 016](docs/specs/016-private-and-shared-documents.md)); maintainers' read side under `/api/admin/documents` (corpus health, currently failed versions, a metadata-only list of private documents that deliberately bypasses `ReadableBy`, configuration) and `IDocumentTitles`, which returns titles of shared documents only ([spec 017](docs/specs/017-diagnose-traces-admin-and-metrics.md)); the document title leads every embedding input (`EmbeddingInput`), and each version records `EmbeddingInputVersion`: a version embedded the old way stays searchable, counts as `outdatedEmbeddings` in admin health and is reprocessed by the harness `seed` ([spec 021](docs/specs/021-context-budget-title-embeddings-nightly-evaluation.md)); raise `EmbeddingInput.Version` whenever the input changes; hybrid search and reranking behind switches ([spec 019](docs/specs/019-hybrid-search-and-reranking.md)): a keyword pool (`passages.search_vector`, generated from `search_config`, which database triggers keep equal to the document language: `english`, `documents.french_unaccent` or `simple`) fused with the vector pool by reciprocal rank fusion (`HybridRanking`), then optionally a cross-encoder through `IPassageReranker` (`PassageReranking`: one call, its own timeout, any failure means `rerank: skipped` and cosine gating); every passage carries `PassageStages`, the trace and the admin inspector show them, and `GET /api/documents/search` takes `mode`, `rerank` and `queryLanguage` overrides; first-release polish ([spec 020](docs/specs/020-first-release-polish.md)): after indexing, `SuggestionGenerator` asks the chat model (`IChatClient`, structured output) for 3 questions and keeps those retrieval answers confidently as the owner, on the version (`SuggestedQuestions`, `SuggestionsStatus` pending/ready/failed/skipped/edited); it never fails the run, is off in demo mode (`Documents:Suggestions:Enabled`, turned off by the API) and records a `suggestions` stage event after `completed`; the owner edits them with `PUT …/{id}/suggestions` (`If-Match`), edited ones survive reprocess; `GET /api/documents/suggestions` and `/summary` feed the home page; every passage stores `TextHash` (`PassageTextHash` in the kernel, also written out in the migrations' SQL) and `IDocumentSources` gives each cited passage a source state (current, newer-unchanged, newer-changed, archived, unavailable) | `documents` | +| `Documents` | Upload/ingest documents, text extraction, chunking, embedding generation, vector storage, similarity search. Exists: data model, `POST /api/documents`, `GET /api/documents/{id}` ([spec 006](docs/specs/006-documents-data-model-and-import.md)); background ingestion (`IngestionWorker`, chunker, batched embedding), `POST /api/documents/{id}/ingestion/retry` and the `GET …/ingestion/events` SSE stream ([spec 007](docs/specs/007-background-ingestion-and-progress.md)); listing with filters and paging, facets, versions and passages of a version (`GET /api/documents`, `/facets`, `/{id}/versions`, `/{id}/versions/{n}/passages`) and the Documents UI ([spec 008](docs/specs/008-browse-corpus-and-documents-ui.md)); edit metadata (`PUT`, `If-Match`/`ETag`), new version, reprocess, archive and restore, with the matching detail-page actions ([spec 009](docs/specs/009-document-lifecycle.md)); PDF import up to 100 MiB (streamed upload into a content-addressed file store, text-layer extraction with page numbers on passages, scanned PDFs rejected at upload), `GET …/versions/{n}/file` for the original and `progress` events inside long ingestion stages ([spec 010](docs/specs/010-pdf-import.md)); retrieval through `IDocumentSearch` (vector search over active, indexed versions with language and tag filters, evidence thresholds, near-duplicate reduction, cached query embeddings) and the maintainers' diagnostic `GET /api/documents/search` ([spec 011](docs/specs/011-retrieval.md)); private and shared documents: `Visibility` on `Document`, owner = `CreatedBy`, one `ReadableBy(viewerSub)` rule applied on every read path, so a document the caller cannot read is a 404 and `IDocumentSearch`/`IDocumentCatalog` take a required `viewerSub` ([spec 016](docs/specs/016-private-and-shared-documents.md)); maintainers' read side under `/api/admin/documents` (corpus health, currently failed versions, a metadata-only list of private documents that deliberately bypasses `ReadableBy`, configuration) and `IDocumentTitles`, which returns titles of shared documents only ([spec 017](docs/specs/017-diagnose-traces-admin-and-metrics.md)); the document title leads every embedding input (`EmbeddingInput`), and each version records `EmbeddingInputVersion`: a version embedded the old way stays searchable, counts as `outdatedEmbeddings` in admin health and is reprocessed by the harness `seed` ([spec 021](docs/specs/021-context-budget-title-embeddings-nightly-evaluation.md)); raise `EmbeddingInput.Version` whenever the input changes; hybrid search and reranking behind switches ([spec 019](docs/specs/019-hybrid-search-and-reranking.md)): a keyword pool (`passages.search_vector`, generated from `search_config`, which database triggers keep equal to the document language: `english`, `documents.french_unaccent` or `simple`; the question's lexemes are OR-ed, and a match holding at least `Hybrid:MinKeywordCoverage` of them passes the cosine floor) fused with the vector pool by reciprocal rank fusion (`HybridRanking`), then optionally a cross-encoder through `IPassageReranker` (`PassageReranking`: one call, its own timeout, any failure means `rerank: skipped` and cosine gating); every passage carries `PassageStages`, the trace and the admin inspector show them, and `GET /api/documents/search` takes `mode`, `rerank` and `queryLanguage` overrides; first-release polish ([spec 020](docs/specs/020-first-release-polish.md)): after indexing, `SuggestionGenerator` asks the chat model (`IChatClient`, structured output) for 3 questions and keeps those retrieval answers confidently as the owner, on the version (`SuggestedQuestions`, `SuggestionsStatus` pending/ready/failed/skipped/edited); it never fails the run, is off in demo mode (`Documents:Suggestions:Enabled`, turned off by the API) and records a `suggestions` stage event after `completed`; the owner edits them with `PUT …/{id}/suggestions` (`If-Match`), edited ones survive reprocess; `GET /api/documents/suggestions` and `/summary` feed the home page; every passage stores `TextHash` (`PassageTextHash` in the kernel, also written out in the migrations' SQL) and `IDocumentSources` gives each cited passage a source state (current, newer-unchanged, newer-changed, archived, unavailable) | `documents` | | `Chat` | Conversations and messages, prompt building, retrieval orchestration, LLM answer generation with citations. Exists: owned conversations (`POST/GET /api/chat/conversations`, `GET …/{id}`), the SSE answer stream `POST …/{id}/messages` (refusal without a model call on `NoEvidence`, one prompt builder, `[n]` citations validated in the stream, sentinel veto, complete/partial status, citation snapshots), `POST …/messages/{id}/retry`, the stale-answer sweep and the write side of request traces ([spec 012](docs/specs/012-chat-conversations-and-grounded-answers.md)); follow-up questions (a bounded history window feeds a `GetResponseAsync` rewrite into a standalone retrieval query, with fallback to the original question), ratings and issue reports on finished answers, and hard deletion of a conversation with its traces ([spec 014](docs/specs/014-follow-ups-feedback-and-deletion.md)); a per-conversation scope of documents and tags (`PUT …/scope`, narrowing retrieval through `SearchFilters.DocumentIds`) and the stored activity of each answer (`GET …/messages/{id}/activity`) ([spec 015](docs/specs/015-chat-scope-activity-and-layout.md)); maintainers' read side under `/api/admin/chat` (requests and the request inspector's data, summary with SQL percentiles, feedback list, configuration) and the `RagAssistant.Chat` meter, recorded from the finished trace ([spec 017](docs/specs/017-diagnose-traces-admin-and-metrics.md)); a context budget (`PromptBudget`, `Chat:Generation:ContextWindowTokens`, sent to Ollama as `num_ctx` by the API's `OllamaRequestOptions`) drops the lowest-ranked passages before `retrieval.completed` and records the trim and the prompt estimate in the trace ([spec 021](docs/specs/021-context-budget-title-embeddings-nightly-evaluation.md)); conversations can be renamed (`PATCH …/{id}`, `TitleEdited`; an empty title reverts to the first question), searched (`q`, `unaccent` + `ILIKE` over titles and questions) and exported as Markdown (`GET …/{id}/export`, the source lines of `SourceList` match the client's "Copy answer with sources" in `SourceListText`), and citations carry a `sourceState` computed from `CitationSnapshot.TextHash` when the conversation is read ([spec 020](docs/specs/020-first-release-polish.md)); load the `rag-pipeline` skill before changing it | `chat` | | `Evaluation` | Harness reports uploaded for maintainers ([spec 020](docs/specs/020-first-release-polish.md), roadmap 5.3): `POST /api/admin/evaluation/runs` (report JSON ≤ 10 MiB, schema 1, `sample` scope only, idempotent by run id, an optional `judge` property replaces the judged metrics), list, detail, compare and delete under `/api/admin/evaluation`, Maintainer only. It reads the report as JSON (`Common/Json.cs`) and binds none of the harness's types; no contract with other modules | `evaluation` | diff --git a/docs/specs/019-hybrid-search-and-reranking.md b/docs/specs/019-hybrid-search-and-reranking.md index ec422e6..523765f 100644 --- a/docs/specs/019-hybrid-search-and-reranking.md +++ b/docs/specs/019-hybrid-search-and-reranking.md @@ -23,7 +23,7 @@ Success: the harness compares vector, hybrid, and hybrid + rerank on the same ru - `search_vector` (`tsvector`), a stored generated column `to_tsvector(search_config, text)` with a GIN index. The `french_unaccent` configuration (French stemmer plus `unaccent`) and the `unaccent` extension are created by the Documents migration. The generated column backfills existing passages. - **The passage's config follows its document's language.** When a document's language changes (edit, detection, spec 018), its passages' `search_config` is updated in the same transaction. -- **Keyword query in the question's written language.** The config follows the detected question language from spec 018 (with its conversation fallback), never the requested answer language. `websearch_to_tsquery(config, query)`, ranked with `ts_rank_cd`. `IDocumentSearch` receives the language through `SearchFilters.QueryLanguage` (nullable → `simple`). French questions therefore get keyword matches from French passages only. Vector search still covers every language. +- **Keyword query in the question's written language.** The config follows the detected question language from spec 018 (with its conversation fallback), never the requested answer language. The question's distinct lexemes (`to_tsvector(config, query)`) are OR-ed into one `tsquery`, ranked with `ts_rank_cd`. *Revised after the first variants run:* the original `websearch_to_tsquery` ANDs every word, so a natural-language question matched almost no passage and hybrid equalled vector. Each match also records its **coverage**: the share of the question's distinct lexemes the passage contains. `IDocumentSearch` receives the language through `SearchFilters.QueryLanguage` (nullable → `simple`). French questions therefore get keyword matches from French passages only. Vector search still covers every language. - **Reciprocal rank fusion.** The vector pool and the keyword pool (same pool size, same filters and `ReadableBy`) are merged with `score = Σ 1 / (k + rank)`, where `k = Documents:Retrieval:Hybrid:RrfK` (default 60). - **Switch:** `Documents:Retrieval:Hybrid:Enabled`. @@ -37,7 +37,7 @@ Success: the harness compares vector, hybrid, and hybrid + rerank on the same ru 3. near-duplicate reduction and `topK` apply after reranking. - **Evidence gate:** - **Reranking on:** `MinScore` and `ConfidentScore` apply to the rerank score. Their settings are `Documents:Retrieval:Reranking:MinScore` and `…:ConfidentScore`, calibrated by the sweep. - - **Reranking off or skipped:** cosine is the gate, as today. A passage found only by keywords is admitted when its cosine is ≥ `MinScore`. + - **Reranking off or skipped:** cosine is the gate, as today, except for strong keyword matches: a passage whose keyword coverage is ≥ `Documents:Retrieval:Hybrid:MinKeywordCoverage` (default 0.5) passes the cosine floor, whichever pool found it. `ConfidentScore` still applies to its cosine, so evidence found by keywords alone is low-confidence. *Revised:* the floor used to drop every keyword-only hit below `MinScore`, which are the exact-term matches hybrid search exists for. One common word ("office") is not evidence, hence the coverage condition. - **Fallback:** on timeout (`Documents:Retrieval:Reranking:Timeout`, default 2 s), an error or an unreachable TEI, retrieval continues without reranking and is gated by cosine. The trace, activity and metrics record `rerank: skipped (unavailable)`. A question never fails because of the reranker. - **Switch:** `Documents:Retrieval:Reranking:Enabled`. The AppHost adds the TEI resource only with the launch profile `https-rerank` (or the parameter `reranking=true`), which also sets the switch. The default clone-and-run is unchanged. @@ -77,7 +77,7 @@ Success: the harness compares vector, hybrid, and hybrid + rerank on the same ru | `Passage` (`documents.passages`) | `search_config regconfig NOT NULL` (from the document language), `search_vector tsvector` generated stored, GIN index. | | Migration | `CREATE EXTENSION unaccent`; `CREATE TEXT SEARCH CONFIGURATION french_unaccent (COPY = french)` with `unaccent, french_stem` for word tokens; backfill `search_config` from each passage's document language. | | `SearchFilters` (contract) | `QueryLanguage` (two letters, nullable). | -| `RetrievedChunk` (contract) | `Stages` record: `VectorRank`, `Cosine`, `KeywordRank`, `KeywordScore`, `FusedScore`, `RerankScore` (all nullable), `AdmittedBy`. `Score` stays the gating score (rerank score when reranked, else cosine). | +| `RetrievedChunk` (contract) | `Stages` record: `VectorRank`, `Cosine`, `KeywordRank`, `KeywordScore`, `FusedScore`, `RerankScore`, `KeywordCoverage` (all nullable), `AdmittedBy`. `Score` stays the gating score (rerank score when reranked, else cosine). | | `SearchResult` (contract) | `Mode` (`vector` \| `hybrid`), `Rerank` (`applied` \| `off` \| `skipped`). | | `RequestTrace` / trace passages (`chat`) | Stage fields per passage; `RetrievalMode`, `RerankStatus`, `RerankMs`, `RerankModel`. | @@ -104,7 +104,9 @@ There are no new policies. Keyword and fused pools apply the same `ReadableBy(vi |---|---| | Query with only stop-words or punctuation (empty tsquery) | Keyword pool is empty; hybrid equals vector. | | French question, only English documents | No keyword hits; vector results only; `admittedBy = vector`. | -| Keyword-only hit with cosine < `MinScore`, rerank off | Not admitted. | +| Keyword match with cosine < `MinScore`, rerank off, coverage ≥ `MinKeywordCoverage` | Admitted; the outcome is `LowConfidence` unless another kept passage reaches `ConfidentScore`. | +| Keyword match with cosine < `MinScore`, rerank off, coverage below it | Not admitted (`minScore`). | +| Question sharing only some words with a passage | Still a keyword match (OR), ranked lower by `ts_rank_cd`; its coverage tells how much it shares. | | Keyword-only hit, rerank on, rerank score ≥ rerank `MinScore` | Admitted. | | TEI timeout, 5xx or connection refused | Fallback to cosine gating, `rerank: skipped (unavailable)`, a `documents.rerank.skipped` counter, a warning log without query text. | | TEI returns a score count different from the texts sent | Treated as unavailable (fallback). | @@ -151,6 +153,9 @@ There are no new policies. Keyword and fused pools apply the same `ReadableBy(vi - **AC-7** Given a query of only stop-words, then hybrid returns the vector result and no error. - **AC-8** Given filters `DocumentIds`, tags and language, then the keyword pool applies the same filters. - **AC-9** Given `k = 60` and known ranks, then the fused score equals `1/(60+rv) + 1/(60+rk)` (unit test). +- **AC-23** Given a passage holding three of the four lexemes of "budget BX-204 for travel" (`budget`, `bx`, `-204`, `travel`), when hybrid search runs, then it is a keyword match with coverage 0.75. +- **AC-24** Given a passage containing every term of "budget BX-204 approval" with cosine 0.3 < `MinScore`, then hybrid search keeps it with outcome `LowConfidence`, and vector search returns `NoEvidence`. +- **AC-25** Given a passage sharing only "budget" with that query (coverage 0.25) and cosine below `MinScore`, then it is dropped by the floor and the outcome is `NoEvidence`. ### Reranking diff --git a/src/Modules/Documents/RagAssistant.Modules.Documents/DocumentsModule.cs b/src/Modules/Documents/RagAssistant.Modules.Documents/DocumentsModule.cs index e55da56..20707a1 100644 --- a/src/Modules/Documents/RagAssistant.Modules.Documents/DocumentsModule.cs +++ b/src/Modules/Documents/RagAssistant.Modules.Documents/DocumentsModule.cs @@ -87,7 +87,7 @@ public static IServiceCollection AddDocumentsModule(this IServiceCollection serv .Validate( o => o.IsConsistent, "Documents:Retrieval: DefaultTopK must not exceed MaxTopK, MaxCandidates must be at least MaxTopK, " - + "ConfidentScore must be at least MinScore, and both durations must be greater than zero; Hybrid:RrfK must be at least 1; " + + "ConfidentScore must be at least MinScore, and both durations must be greater than zero; Hybrid:RrfK must be at least 1 and Hybrid:MinKeywordCoverage between 0 and 1; " + "Reranking:Candidates must be 1 to 100, its scores between 0 and 1 with ConfidentScore at least MinScore, and its " + "Timeout greater than zero.") .ValidateOnStart(); diff --git a/src/Modules/Documents/RagAssistant.Modules.Documents/Features/Admin/AdminDocumentsModels.cs b/src/Modules/Documents/RagAssistant.Modules.Documents/Features/Admin/AdminDocumentsModels.cs index 35db08b..ddf7106 100644 --- a/src/Modules/Documents/RagAssistant.Modules.Documents/Features/Admin/AdminDocumentsModels.cs +++ b/src/Modules/Documents/RagAssistant.Modules.Documents/Features/Admin/AdminDocumentsModels.cs @@ -70,4 +70,5 @@ internal sealed record RetrievalStagesConfiguration( int RerankCandidates, double RerankMinScore, double RerankConfidentScore, - double RerankTimeoutSeconds); + double RerankTimeoutSeconds, + double MinKeywordCoverage); diff --git a/src/Modules/Documents/RagAssistant.Modules.Documents/Features/Admin/GetDocumentsConfiguration.cs b/src/Modules/Documents/RagAssistant.Modules.Documents/Features/Admin/GetDocumentsConfiguration.cs index ceb8f72..03a5151 100644 --- a/src/Modules/Documents/RagAssistant.Modules.Documents/Features/Admin/GetDocumentsConfiguration.cs +++ b/src/Modules/Documents/RagAssistant.Modules.Documents/Features/Admin/GetDocumentsConfiguration.cs @@ -53,6 +53,7 @@ public ValueTask> Handle(Query query, Cancellatio retrieval.Value.Reranking.Candidates, retrieval.Value.Reranking.MinScore, retrieval.Value.Reranking.ConfidentScore, - retrieval.Value.Reranking.Timeout.TotalSeconds))); + retrieval.Value.Reranking.Timeout.TotalSeconds, + retrieval.Value.Hybrid.MinKeywordCoverage))); } } diff --git a/src/Modules/Documents/RagAssistant.Modules.Documents/Features/SearchPassages/SearchPassages.cs b/src/Modules/Documents/RagAssistant.Modules.Documents/Features/SearchPassages/SearchPassages.cs index 085824f..4408667 100644 --- a/src/Modules/Documents/RagAssistant.Modules.Documents/Features/SearchPassages/SearchPassages.cs +++ b/src/Modules/Documents/RagAssistant.Modules.Documents/Features/SearchPassages/SearchPassages.cs @@ -171,10 +171,10 @@ public async ValueTask> Handle(Query query, CancellationToken var vector = embedded.Value.Vector; var vectorPool = await candidates.FetchAsync(vector, model, filter, poolSize, cancellationToken); + var keywordHits = await keywords; List ordered; if (mode == RetrievalMode.Hybrid) { - var keywordHits = await keywords; var inVectorPool = vectorPool.Select(p => p.PassageId).ToHashSet(); Guid[] missing = [.. keywordHits.Select(h => h.PassageId).Where(id => !inVectorPool.Contains(id))]; var keywordOnly = missing.Length == 0 ? [] : await candidates.FetchByIdsAsync(vector, model, filter, missing, cancellationToken); @@ -210,7 +210,12 @@ public async ValueTask> Handle(Query query, CancellationToken } } - var decision = EvidenceRules.ApplyOrdered(ordered, topK, thresholds); + // Spec 019: while cosine is the gate, a keyword match that covers enough of the question passes the cosine floor, which + // would otherwise drop the exact-term matches dense vectors miss. A rerank score already judged every candidate. + IReadOnlySet? pastFloor = rerankStatus == RerankStatus.Applied + ? null + : keywordHits.Where(h => h.Coverage >= settings.Hybrid.MinKeywordCoverage).Select(h => h.PassageId).ToHashSet(); + var decision = EvidenceRules.ApplyOrdered(ordered, topK, thresholds, pastFloor); int? skipped = query.IncludeDiagnostics ? await candidates.CountModelMismatchesAsync(model, filter, cancellationToken) : null; diff --git a/src/Modules/Documents/RagAssistant.Modules.Documents/Features/SearchPassages/SearchPassagesEndpoint.cs b/src/Modules/Documents/RagAssistant.Modules.Documents/Features/SearchPassages/SearchPassagesEndpoint.cs index 66e6c2d..dbd1a6e 100644 --- a/src/Modules/Documents/RagAssistant.Modules.Documents/Features/SearchPassages/SearchPassagesEndpoint.cs +++ b/src/Modules/Documents/RagAssistant.Modules.Documents/Features/SearchPassages/SearchPassagesEndpoint.cs @@ -111,7 +111,7 @@ private static List RankedPool(SearchPassages.SearchRun run) internal static SearchStages? StagesOf(RetrievedChunk chunk) => chunk.Stages is not { } s ? null : new SearchStages( s.VectorRank, Round(s.Cosine), s.KeywordRank, s.KeywordScore is { } k ? Round(k) : null, s.FusedScore is { } f ? Round(f, 6) : null, - s.RerankScore is { } r ? Round(r) : null, SearchPassages.AdmittedName(s.AdmittedBy)); + s.RerankScore is { } r ? Round(r) : null, SearchPassages.AdmittedName(s.AdmittedBy), s.KeywordCoverage is { } c ? Round(c) : null); private static double Round(double value, int decimals = ScoreDecimals) => Math.Round(value, decimals, MidpointRounding.AwayFromZero); } @@ -120,6 +120,7 @@ public sealed record SearchThresholds(double MinScore, double ConfidentScore, do /// What each retrieval stage said about a passage (spec 019); a stage that did not see it leaves its fields null. /// vector, keyword or both. +/// The share of the question's terms in the passage; at least Hybrid:MinKeywordCoverage passes the floor. public sealed record SearchStages( int? VectorRank, double Cosine, @@ -127,7 +128,8 @@ public sealed record SearchStages( double? KeywordScore, double? FusedScore, double? RerankScore, - string AdmittedBy); + string AdmittedBy, + double? KeywordCoverage = null); public sealed record RankedPassage( int Rank, diff --git a/src/Modules/Documents/RagAssistant.Modules.Documents/Options/RetrievalOptions.cs b/src/Modules/Documents/RagAssistant.Modules.Documents/Options/RetrievalOptions.cs index 902fd3b..bbff668 100644 --- a/src/Modules/Documents/RagAssistant.Modules.Documents/Options/RetrievalOptions.cs +++ b/src/Modules/Documents/RagAssistant.Modules.Documents/Options/RetrievalOptions.cs @@ -48,6 +48,7 @@ internal sealed class RetrievalOptions && QueryEmbeddingTimeout > TimeSpan.Zero && QueryEmbeddingCacheDuration > TimeSpan.Zero && Hybrid.RrfK >= 1 + && Hybrid.MinKeywordCoverage is >= 0 and <= 1 && Reranking.Candidates is >= 1 and <= RerankingOptions.MaxCandidates && Reranking.MinScore is >= 0 and <= 1 && Reranking.ConfidentScore >= Reranking.MinScore && Reranking.ConfidentScore <= 1 @@ -63,6 +64,12 @@ internal sealed class HybridOptions /// The k of reciprocal rank fusion: score = Σ 1 / (k + rank). public int RrfK { get; set; } = 60; + + /// + /// While cosine is the gate, a keyword match passes the cosine floor when the passage holds at least this share of the question's + /// distinct terms (spec 019): one common word such as "office" is not evidence. 0 lets every keyword match through. + /// + public double MinKeywordCoverage { get; set; } = 0.5; } /// A cross-encoder reorders and gates the best candidates (spec 019). Off until the harness shows a gain. diff --git a/src/Modules/Documents/RagAssistant.Modules.Documents/Retrieval/EvidenceRules.cs b/src/Modules/Documents/RagAssistant.Modules.Documents/Retrieval/EvidenceRules.cs index 457a20a..56122e7 100644 --- a/src/Modules/Documents/RagAssistant.Modules.Documents/Retrieval/EvidenceRules.cs +++ b/src/Modules/Documents/RagAssistant.Modules.Documents/Retrieval/EvidenceRules.cs @@ -31,13 +31,18 @@ public static EvidenceDecision Apply(IEnumerable pool, int topK, /// The same steps on a pool already in its final order (spec 019): fused order in hybrid mode, rerank order when reranked. /// The floor and the confident score always apply to , the gating score. /// - public static EvidenceDecision ApplyOrdered(IReadOnlyList ranked, int topK, EvidenceThresholds thresholds) + /// + /// Passages admitted whatever their score: strong keyword matches while cosine is the gate. The confident score still applies + /// to them, so evidence found by keywords alone stays low-confidence. + /// + public static EvidenceDecision ApplyOrdered( + IReadOnlyList ranked, int topK, EvidenceThresholds thresholds, IReadOnlySet? pastFloor = null) { var discarded = new List(); var kept = new List<(RetrievedChunk Passage, HashSet Shingles)>(); foreach (var candidate in ranked) { - if (candidate.Score < thresholds.MinScore) + if (candidate.Score < thresholds.MinScore && pastFloor?.Contains(candidate.PassageId) != true) { discarded.Add(new DiscardedPassage(candidate, DiscardReason.BelowMinScore)); continue; diff --git a/src/Modules/Documents/RagAssistant.Modules.Documents/Retrieval/HybridRanking.cs b/src/Modules/Documents/RagAssistant.Modules.Documents/Retrieval/HybridRanking.cs index b382878..f10860c 100644 --- a/src/Modules/Documents/RagAssistant.Modules.Documents/Retrieval/HybridRanking.cs +++ b/src/Modules/Documents/RagAssistant.Modules.Documents/Retrieval/HybridRanking.cs @@ -26,7 +26,7 @@ public static List Hybrid( { var ranked = EvidenceRules.Rank(vectorPool); var vectorRanks = ranked.Select((c, i) => (c.PassageId, Rank: i + 1)).ToDictionary(x => x.PassageId, x => x.Rank); - var keyword = keywordHits.Select((h, i) => (h, Rank: i + 1)).ToDictionary(x => x.h.PassageId, x => (x.Rank, x.h.Score)); + var keyword = keywordHits.Select((h, i) => (h, Rank: i + 1)).ToDictionary(x => x.h.PassageId, x => (x.Rank, x.h.Score, x.h.Coverage)); var merged = ranked.Concat(keywordOnly.Where(c => !vectorRanks.ContainsKey(c.PassageId))); return @@ -35,10 +35,13 @@ .. merged .Select(c => { int? vectorRank = vectorRanks.TryGetValue(c.PassageId, out var v) ? v : null; - (int Rank, double Score)? match = keyword.TryGetValue(c.PassageId, out var w) ? w : null; + (int Rank, double Score, double Coverage)? match = keyword.TryGetValue(c.PassageId, out var w) ? w : null; var admitted = vectorRank is null ? AdmittedBy.Keyword : match is null ? AdmittedBy.Vector : AdmittedBy.Both; var fused = Fuse(k, vectorRank, match?.Rank); - return c with { Stages = new PassageStages(vectorRank, c.Score, match?.Rank, match?.Score, fused, null, admitted) }; + return c with + { + Stages = new PassageStages(vectorRank, c.Score, match?.Rank, match?.Score, fused, null, admitted, match?.Coverage), + }; }) .OrderByDescending(c => c.Stages!.FusedScore) .ThenByDescending(c => c.Score) diff --git a/src/Modules/Documents/RagAssistant.Modules.Documents/Retrieval/PassageCandidates.cs b/src/Modules/Documents/RagAssistant.Modules.Documents/Retrieval/PassageCandidates.cs index 7fde900..32dd6e4 100644 --- a/src/Modules/Documents/RagAssistant.Modules.Documents/Retrieval/PassageCandidates.cs +++ b/src/Modules/Documents/RagAssistant.Modules.Documents/Retrieval/PassageCandidates.cs @@ -19,7 +19,8 @@ internal sealed record CandidateFilter(string? Language, IReadOnlyList T } /// A keyword match: ts_rank_cd of the passage for the query (spec 019). -internal sealed record KeywordHit(Guid PassageId, double Score); +/// The share of the question's distinct terms the passage contains, in (0, 1]. +internal sealed record KeywordHit(Guid PassageId, double Score, double Coverage); /// The eligible-passage queries of retrieval (spec 011, eligibility and pipeline step 3; spec 019, keyword pool). internal sealed class PassageCandidates(DocumentsDbContext db) @@ -48,17 +49,25 @@ await db.Database.ExecuteSqlAsync( /// /// The eligible passages that best match as keywords (spec 019), best first: - /// websearch_to_tsquery in the configuration of the query's language, ranked with ts_rank_cd. Same eligibility and + /// any of the question's lexemes in the configuration of the query's language, ranked with ts_rank_cd. Same eligibility and /// filters as the vector pool. A query without lexemes (only stop-words or punctuation) matches nothing. /// public async Task> KeywordAsync( string query, string? queryLanguage, string embeddingModel, CandidateFilter filter, int poolSize, CancellationToken cancellationToken) { // The configuration comes from the same SQL function that sets each passage's own (migration HybridSearch). + // The lexemes are OR-ed: websearch_to_tsquery would AND them, and a passage rarely holds every word of a question. They are + // quoted into the tsquery as they are (quotes and backslashes doubled), so nothing parses or stems them a second time. var language = queryLanguage ?? string.Empty; var matches = db.Database.SqlQuery($""" - SELECT p.id AS "PassageId", ts_rank_cd(p.search_vector, q.query)::double precision AS "Score" - FROM documents.passages p, websearch_to_tsquery(documents.search_config_for({language}), {query}) AS q(query) + SELECT p.id AS "PassageId", + ts_rank_cd(p.search_vector, q.query)::double precision AS "Score", + (SELECT count(*) FROM unnest(q.lexemes) AS l WHERE l = ANY (tsvector_to_array(p.search_vector)))::double precision + / cardinality(q.lexemes) AS "Coverage" + FROM documents.passages p, + (SELECT array_agg(l) AS lexemes, + string_agg('''' || replace(replace(l, '\', '\\'), '''', '''''') || '''', ' | ')::tsquery AS query + FROM unnest(tsvector_to_array(to_tsvector(documents.search_config_for({language}), {query}))) AS l) AS q WHERE p.search_vector @@ q.query """); @@ -66,10 +75,10 @@ WHERE p.search_vector @@ q.query from m in matches join e in Eligible(embeddingModel, filter) on m.PassageId equals e.Passage.Id orderby m.Score descending, m.PassageId - select new { m.PassageId, m.Score }) + select new { m.PassageId, m.Score, m.Coverage }) .Take(poolSize) .ToListAsync(cancellationToken); - return [.. rows.Select(r => new KeywordHit(r.PassageId, r.Score))]; + return [.. rows.Select(r => new KeywordHit(r.PassageId, r.Score, r.Coverage))]; } /// The given eligible passages with their cosine to the query, in no guaranteed order: keyword hits the vector pool missed. @@ -152,6 +161,8 @@ private sealed class KeywordRow public Guid PassageId { get; init; } public double Score { get; init; } + + public double Coverage { get; init; } } // A member-init projection, so EF can keep composing on it (a constructor projection cannot be ordered by). diff --git a/src/RagAssistant.SharedKernel/Contracts/Documents/SearchResult.cs b/src/RagAssistant.SharedKernel/Contracts/Documents/SearchResult.cs index 18031bf..920c32c 100644 --- a/src/RagAssistant.SharedKernel/Contracts/Documents/SearchResult.cs +++ b/src/RagAssistant.SharedKernel/Contracts/Documents/SearchResult.cs @@ -95,6 +95,7 @@ public sealed record RetrievedChunk( /// ts_rank_cd of the passage for the keyword query. /// Reciprocal rank fusion score; null in vector mode. /// The cross-encoder's score; null when the passage was not reranked. +/// The share of the question's distinct terms the passage contains; null without a keyword match. public sealed record PassageStages( int? VectorRank, double Cosine, @@ -102,4 +103,5 @@ public sealed record PassageStages( double? KeywordScore, double? FusedScore, double? RerankScore, - AdmittedBy AdmittedBy); + AdmittedBy AdmittedBy, + double? KeywordCoverage = null); diff --git a/src/RagAssistant.Web.Client/Admin/AdminModels.cs b/src/RagAssistant.Web.Client/Admin/AdminModels.cs index bde680b..16b067d 100644 --- a/src/RagAssistant.Web.Client/Admin/AdminModels.cs +++ b/src/RagAssistant.Web.Client/Admin/AdminModels.cs @@ -200,4 +200,5 @@ public sealed record RetrievalStagesConfiguration( int RerankCandidates, double RerankMinScore, double RerankConfidentScore, - double RerankTimeoutSeconds); + double RerankTimeoutSeconds, + double MinKeywordCoverage); diff --git a/src/RagAssistant.Web.Client/Admin/ConfigurationTab.razor b/src/RagAssistant.Web.Client/Admin/ConfigurationTab.razor index 438d84d..e2c7f24 100644 --- a/src/RagAssistant.Web.Client/Admin/ConfigurationTab.razor +++ b/src/RagAssistant.Web.Client/Admin/ConfigurationTab.razor @@ -40,7 +40,7 @@ minimum @c.MinScore, confident @c.ConfidentScore, duplicate overlap @c.DuplicateOverlap @if (c.Stages is { } stages) { - @(stages.HybridEnabled ? $"on, reciprocal rank fusion k = {stages.RrfK}" : "off") + @(stages.HybridEnabled ? $"on, reciprocal rank fusion k = {stages.RrfK}, keyword matches covering at least {stages.MinKeywordCoverage:0.##} of the question pass the floor" : "off") @RerankingText(stages) } text @Formatting.Size(c.MaxFileSizeBytes), PDF @Formatting.Size(c.MaxPdfSizeBytes), @c.PdfMaxPages pages diff --git a/tests/Modules/RagAssistant.Modules.Documents.Tests/HybridRankingTests.cs b/tests/Modules/RagAssistant.Modules.Documents.Tests/HybridRankingTests.cs index ae605ab..a195c33 100644 --- a/tests/Modules/RagAssistant.Modules.Documents.Tests/HybridRankingTests.cs +++ b/tests/Modules/RagAssistant.Modules.Documents.Tests/HybridRankingTests.cs @@ -48,17 +48,18 @@ public async Task Hybrid_keywordHitLowInTheVectorPool_movesUpAndIsAdmittedByBoth var third = Chunk(0.8, "third"); var code = Chunk(0.6, "code BX-204"); var keywordOnly = Chunk(0.4, "only keywords"); - KeywordHit[] hits = [new(code.PassageId, 0.5), new(keywordOnly.PassageId, 0.2)]; + KeywordHit[] hits = [new(code.PassageId, 0.5, 1.0), new(keywordOnly.PassageId, 0.2, 0.25)]; var ordered = HybridRanking.Hybrid([first, second, third, code], hits, [keywordOnly], k: 60); await Assert.That(ordered[0].PassageId).IsEqualTo(code.PassageId); await Assert.That(ordered[0].Stages).IsEqualTo( - new PassageStages(4, 0.6, 1, 0.5, HybridRanking.Fuse(60, 4, 1), null, AdmittedBy.Both)); + new PassageStages(4, 0.6, 1, 0.5, HybridRanking.Fuse(60, 4, 1), null, AdmittedBy.Both, KeywordCoverage: 1.0)); var onlyKeywords = ordered.Single(c => c.PassageId == keywordOnly.PassageId); await Assert.That(onlyKeywords.Stages!.AdmittedBy).IsEqualTo(AdmittedBy.Keyword); await Assert.That(onlyKeywords.Stages!.VectorRank).IsNull(); await Assert.That(onlyKeywords.Score).IsEqualTo(0.4); + await Assert.That(onlyKeywords.Stages!.KeywordCoverage).IsEqualTo(0.25); await Assert.That(ordered.Single(c => c.PassageId == first.PassageId).Stages!.AdmittedBy).IsEqualTo(AdmittedBy.Vector); await Assert.That(ordered.Count).IsEqualTo(5); } @@ -100,4 +101,18 @@ public async Task ApplyOrdered_keepsTheGivenOrder_andTheOutcomeUsesTheBestKeptSc await Assert.That(decision.Kept.SequenceEqual([fusedFirst, best])).IsTrue(); await Assert.That(decision.Outcome).IsEqualTo(EvidenceOutcome.Sufficient); } + + [Test] + public async Task ApplyOrdered_passageAdmittedPastTheFloor_isKeptButLeavesTheEvidenceLowConfidence() + { + var thresholds = new EvidenceThresholds(MinScore: 0.5, ConfidentScore: 0.7, DuplicateOverlap: 0.8); + var keywordMatch = Chunk(0.3, "budget code BX-204 approval"); + var weak = Chunk(0.3, "nothing in common"); + + var decision = EvidenceRules.ApplyOrdered([keywordMatch, weak], topK: 5, thresholds, pastFloor: new HashSet { keywordMatch.PassageId }); + + await Assert.That(decision.Kept.SequenceEqual([keywordMatch])).IsTrue(); + await Assert.That(decision.Discarded.Single().Reason).IsEqualTo(DiscardReason.BelowMinScore); + await Assert.That(decision.Outcome).IsEqualTo(EvidenceOutcome.LowConfidence); + } } diff --git a/tests/RagAssistant.Integration.Tests/Documents/HybridSearchTests.cs b/tests/RagAssistant.Integration.Tests/Documents/HybridSearchTests.cs index 948cf7b..e25f9d7 100644 --- a/tests/RagAssistant.Integration.Tests/Documents/HybridSearchTests.cs +++ b/tests/RagAssistant.Integration.Tests/Documents/HybridSearchTests.cs @@ -170,6 +170,57 @@ public async Task Search_filters_applyToTheKeywordPoolToo() await Assert.That(finance.DocumentId).IsNotEqualTo(hr.DocumentId); } + // AC-23: any of the question's terms is enough to match; how many of them the passage holds is its coverage. + [Test] + public async Task Search_passageWithoutEveryTermOfTheQuestion_isStillAKeywordMatch() + { + await using var bed = await StartAsync(postgres, Hybrid); + var (_, passages) = await bed.AddAsync("Budget Exception Policy", P(0.6, CodeText)); + + var result = await bed.SearchAsync("budget BX-204 for travel", new SearchFilters(QueryLanguage: "en")); + + var stages = result.Value.Chunks.Single(c => c.PassageId == passages[0]).Stages!; + await Assert.That(stages.KeywordRank).IsEqualTo(1); + await Assert.That(stages.AdmittedBy).IsEqualTo(AdmittedBy.Both); + // budget, bx and -204 (the parser's split of "BX-204") of those and travel. + await Assert.That(stages.KeywordCoverage).IsEqualTo(0.75); + } + + // AC-24 + [Test] + public async Task Search_keywordMatchBelowTheFloor_isAdmittedInHybridModeOnly() + { + await using var bed = await StartAsync(postgres, Hybrid); + var (_, passages) = await bed.AddAsync("Budget Exception Policy", P(0.3, CodeText)); + + var hybrid = await bed.SearchAsync(CodeQuery, new SearchFilters(QueryLanguage: "en")); + var (_, vector) = await SearchEndpointAsync(bed, $"q={Uri.EscapeDataString(CodeQuery)}&mode=vector"); + + var chunk = hybrid.Value.Chunks.Single(); + await Assert.That(chunk.PassageId).IsEqualTo(passages[0]); + await Assert.That(chunk.Score).IsLessThan(0.5); + await Assert.That(chunk.Stages!.KeywordCoverage).IsEqualTo(1.0); + await Assert.That(hybrid.Value.Outcome).IsEqualTo(EvidenceOutcome.LowConfidence); + await Assert.That(vector.GetProperty("outcome").GetString()).IsEqualTo("noEvidence"); + } + + // AC-25 + [Test] + public async Task Search_keywordMatchOnOneCommonWord_staysBelowTheFloor() + { + await using var bed = await StartAsync(postgres, Hybrid); + await bed.AddAsync("Budget Exception Policy", P(0.3, "The budget is reviewed every spring.")); + + var result = await bed.SearchAsync(CodeQuery, new SearchFilters(QueryLanguage: "en")); + var (_, diagnostics) = await SearchEndpointAsync(bed, $"q={Uri.EscapeDataString(CodeQuery)}&queryLanguage=en&candidates=true"); + + await Assert.That(result.Value.Outcome).IsEqualTo(EvidenceOutcome.NoEvidence); + var candidate = diagnostics.GetProperty("candidates")[0]; + await Assert.That(candidate.GetProperty("dropped").GetString()).IsEqualTo("minScore"); + // budget alone, of budget, bx, -204 and approv. + await Assert.That(candidate.GetProperty("stages").GetProperty("keywordCoverage").GetDouble()).IsEqualTo(0.25); + } + // AC-15 [Test] public async Task Endpoint_overrides_reportModeRerankAndStagesWhateverTheConfiguration() diff --git a/tools/RagAssistant.Evaluation/Run/ReportWriter.cs b/tools/RagAssistant.Evaluation/Run/ReportWriter.cs index 1f1d7ff..9760da8 100644 --- a/tools/RagAssistant.Evaluation/Run/ReportWriter.cs +++ b/tools/RagAssistant.Evaluation/Run/ReportWriter.cs @@ -140,7 +140,7 @@ private static void AppendTable(StringBuilder md, IEnumerable<(string Name, Aggr /// Spec 019: the retrieval stages of the configuration; older reports have none. private static string Stages(RunConfiguration c) => c.RetrievalMode is null ? "—" : FormattableString.Invariant( - $"{c.RetrievalMode} (RrfK {c.RrfK}) · reranking {(c.Reranking == true ? $"on, {c.RerankerModel ?? "no reranker configured"}, {c.RerankCandidates} candidates, MinScore {c.RerankMinScore}, ConfidentScore {c.RerankConfidentScore}" : "off")}"); + $"{c.RetrievalMode} (RrfK {c.RrfK}{(c.MinKeywordCoverage is { } coverage ? FormattableString.Invariant($", MinKeywordCoverage {coverage}") : "")}) · reranking {(c.Reranking == true ? $"on, {c.RerankerModel ?? "no reranker configured"}, {c.RerankCandidates} candidates, MinScore {c.RerankMinScore}, ConfidentScore {c.RerankConfidentScore}" : "off")}"); private static string Ms(double? value) => value is { } ms ? FormattableString.Invariant($"{ms:0} ms") : "—"; diff --git a/tools/RagAssistant.Evaluation/Run/RunCommand.cs b/tools/RagAssistant.Evaluation/Run/RunCommand.cs index adc0fae..a50517b 100644 --- a/tools/RagAssistant.Evaluation/Run/RunCommand.cs +++ b/tools/RagAssistant.Evaluation/Run/RunCommand.cs @@ -236,7 +236,8 @@ private async Task ConfigurationAsync(string user, string corp stages?.GetProperty("rerankCandidates").GetInt32(), stages is { } r && r.GetProperty("rerankerModel").ValueKind == JsonValueKind.String ? r.GetProperty("rerankerModel").GetString() : null, stages?.GetProperty("rerankMinScore").GetDouble(), - stages?.GetProperty("rerankConfidentScore").GetDouble()); + stages?.GetProperty("rerankConfidentScore").GetDouble(), + stages is { } k && k.TryGetProperty("minKeywordCoverage", out var coverage) ? coverage.GetDouble() : null); } private static string CorpusHash(string corpusPath, CorpusManifest manifest) diff --git a/tools/RagAssistant.Evaluation/Run/RunReport.cs b/tools/RagAssistant.Evaluation/Run/RunReport.cs index d9f0665..dc25c9c 100644 --- a/tools/RagAssistant.Evaluation/Run/RunReport.cs +++ b/tools/RagAssistant.Evaluation/Run/RunReport.cs @@ -48,7 +48,8 @@ internal sealed record RunConfiguration( int? RerankCandidates = null, string? RerankerModel = null, double? RerankMinScore = null, - double? RerankConfidentScore = null); + double? RerankConfidentScore = null, + double? MinKeywordCoverage = null); /// A passage of the seeded corpus; the sweep needs the text to redo near-duplicate reduction offline. internal sealed record CorpusPassage(string Document, string Text);