diff --git a/.env.example b/.env.example index 2883636..47b7d97 100644 --- a/.env.example +++ b/.env.example @@ -35,6 +35,9 @@ STARLLY_API_KEY_ID=2 # SERVY_RERANKER_EXPECTED_SHA256=b1cf23ba0806f0057d73a5ab93fde88d634fc278f71b27f3a2a915afd8f8bcda # SERVY_RERANKER_DEVICE=cpu # SERVY_RERANKER_POOL_SIZE=5 +# Research-only opt-in: set to 10 only after paired V5 offline pool-size +# evaluation and live CPU/GPU latency/memory checks. Default/serving stays 5. +# SERVY_RERANKER_POOL_SIZE=10 # Note: Qwen3 dense retrieval is a SEPARATE integration, not enabled here. # EXPERIMENTAL ONLY — Qwen3 embeddings are separate from the V5 reranker. diff --git a/V5_LOCAL_RERANKER_SETUP.md b/V5_LOCAL_RERANKER_SETUP.md index e424b54..e5a4b8b 100644 --- a/V5_LOCAL_RERANKER_SETUP.md +++ b/V5_LOCAL_RERANKER_SETUP.md @@ -92,6 +92,12 @@ not yet reconciled. Neither is an independent live-UI accuracy guarantee. that case. Review backend logs if model verification fails. - V5 reranks at most 5 authorized technical documents with the example configuration; it cannot retrieve missing passages or change access scope. + The implementation accepts an explicitly configured pool size of **up to 10**. + Test 5 vs 8 vs 10 against the same frozen authorized candidates, including + answer correctness, latency, memory and cite integrity, before deploying a + larger size. The research ceiling is not a measured production improvement. + A configured SHA-256 without a local checkpoint path now fails closed, rather + than silently loading the legacy MS MARCO model. Historical service resolution records are not sent to V5 for scoring. - Changing `SERVY_RERANKER_MODEL_PATH` does **not** replace the Qwen3 embedding model, build a Qwen3 Chroma index, or reproduce the 88.10% research conditions. diff --git a/core/ml/reranker.py b/core/ml/reranker.py index 501ed98..3f87612 100644 --- a/core/ml/reranker.py +++ b/core/ml/reranker.py @@ -77,7 +77,14 @@ def get_reranker(): local_files_only=True, ) else: - # Legacy opt-in mode is unchanged when no V5 checkpoint is configured. + # An expected local digest is a strict local-only intent. + # Do not silently use the legacy network model if the V5 + # checkpoint path was omitted or misconfigured. + if expected_sha: + raise ValueError( + "SERVY_RERANKER_MODEL_PATH_REQUIRED_FOR_PINNED_CHECKPOINT" + ) + # Legacy opt-in mode remains available only with no pin. model = CrossEncoder(RERANKER_MODEL_ID, device=device) _cross_encoder = model _loaded_config = key @@ -132,13 +139,22 @@ def rerank_candidates(query: str, candidates: List, top_k: int = 8) -> List: docs = [c for c in candidates if c.source_kind == "knowledge_document"] others = [c for c in candidates if c.source_kind != "knowledge_document"] pool_size = int(getattr(settings, "SERVY_RERANKER_POOL_SIZE", 8)) - if not 1 <= pool_size <= 8: - raise ValueError("SERVY_RERANKER_POOL_SIZE must be 1..8") + # The ten-candidate setting is an explicit accuracy/latency + # experiment, not an automatic production promotion. Default unchanged. + if not 1 <= pool_size <= 10: + raise ValueError("SERVY_RERANKER_POOL_SIZE must be 1..10") pool = docs[:pool_size] # Validate the full prediction BEFORE mutating any result objects. if pool: pairs = [[query, c.text] for c in pool] - raw = model.predict(pairs) + # Explicit identity activation matches the offline research scorer. + # For one-label CrossEncoders, predict() commonly defaults to + # sigmoid; applying calibrated_sigmoid_score again would distort + # the diagnostic score scale even though rank order is monotone. + import torch + raw = model.predict( + pairs, activation_fn=torch.nn.Identity(), show_progress_bar=False + ) scores = [float(x) for x in raw] if len(scores) != len(pool) or not all(math.isfinite(x) for x in scores): raise ValueError("Invalid or non-finite CrossEncoder scores") diff --git a/core/tests_v5_reranker.py b/core/tests_v5_reranker.py index 98c5c59..f7e340e 100644 --- a/core/tests_v5_reranker.py +++ b/core/tests_v5_reranker.py @@ -91,9 +91,13 @@ def test_reranks_authorized_pool_without_scoring_history(self): with mock.patch.object(reranker, "get_reranker", return_value=fake): result = reranker.rerank_candidates("repair", docs + [history], top_k=4) self.assertEqual([x.text for x in result], ["B", "C", "A", "H"]) - fake.predict.assert_called_once_with([ + fake.predict.assert_called_once() + self.assertEqual(fake.predict.call_args.args[0], [ ["repair", "A"], ["repair", "B"], ["repair", "C"] ]) + import torch + self.assertIsInstance(fake.predict.call_args.kwargs["activation_fn"], torch.nn.Identity) + self.assertIs(fake.predict.call_args.kwargs["show_progress_bar"], False) self.assertFalse(hasattr(history, "rerank_score")) self.assertAlmostEqual(docs[1].reranker_score, reranker.calibrated_sigmoid_score(4.0)) @@ -118,6 +122,40 @@ def test_invalid_scores_restore_order_without_partial_mutation(self): self.assertEqual([x.text for x in result], ["A", "B"]) self.assertFalse(hasattr(docs[0], "reranker_score")) + @override_settings(SERVY_RERANKER_ENABLED=True, SERVY_RERANKER_POOL_SIZE=10) + def test_explicit_ten_candidate_pool_can_promote_tenth_authorized_chunk(self): + docs = [chunk(f"passage-{i}") for i in range(10)] + model = mock.Mock() + model.predict.return_value = [float(i) for i in range(10)] + with mock.patch.object(reranker, "get_reranker", return_value=model): + ranked = reranker.rerank_candidates("diagnose", docs, top_k=10) + self.assertEqual(ranked[0].text, "passage-9") + self.assertEqual(len(model.predict.call_args.args[0]), 10) + self.assertEqual(len(ranked), 10) + + @override_settings(SERVY_RERANKER_ENABLED=True, SERVY_RERANKER_POOL_SIZE=5) + def test_default_five_candidate_pool_does_not_promote_tenth_chunk(self): + docs = [chunk(f"passage-{i}") for i in range(10)] + model = mock.Mock() + model.predict.return_value = [float(i) for i in range(5)] + with mock.patch.object(reranker, "get_reranker", return_value=model): + ranked = reranker.rerank_candidates("diagnose", docs, top_k=10) + self.assertEqual(ranked[0].text, "passage-4") + self.assertEqual(ranked[-1].text, "passage-9") + self.assertEqual(len(model.predict.call_args.args[0]), 5) + + def test_pinned_v5_digest_requires_local_model_path(self): + with override_settings( + SERVY_RERANKER_ENABLED=True, + SERVY_RERANKER_MODEL_PATH="", + SERVY_RERANKER_EXPECTED_SHA256=reranker.SERVY_V5_WEIGHTS_SHA256, + ), mock.patch.dict( + "sys.modules", {"sentence_transformers": SimpleNamespace(CrossEncoder=mock.Mock())} + ) as _: + with mock.patch("sentence_transformers.CrossEncoder") as loader: + self.assertIsNone(reranker.get_reranker()) + loader.assert_not_called() + @override_settings(SERVY_RERANKER_ENABLED=True, SERVY_RERANKER_POOL_SIZE=99) def test_invalid_pool_size_falls_back(self): docs = [chunk("A")]