diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index da6cb2e..102951b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -53,6 +53,9 @@ jobs: - name: V5 Private Artifact Structural Audit Tests run: python -m unittest discover -s tools -p "test_servy_v5_artifact_audit.py" -v + - name: V5 Checkpoint Installer Safety Tests + run: python -m unittest tools.test_install_servy_v5_checkpoint -v + - name: V6 Continuation and Permission Safety Tests run: python -m unittest discover -s tools -p "test_servy_v6_continuation.py" -v diff --git a/V5_LOCAL_RERANKER_SETUP.md b/V5_LOCAL_RERANKER_SETUP.md index af357c3..e424b54 100644 --- a/V5_LOCAL_RERANKER_SETUP.md +++ b/V5_LOCAL_RERANKER_SETUP.md @@ -2,9 +2,10 @@ This integrates the **V5 BGE cross-encoder reranker** into the already-authorized Django retrieval pipeline. It does NOT install Qwen3 as the serving embedding -retriever; existing MiniLM/Chroma retrieval remains the default. The research -**88.10% Hit@1** was measured with frozen Qwen3 candidates on 42 reused, -provisional development questions and is **not a live-UI accuracy guarantee**. +retriever; existing MiniLM/Chroma retrieval remains the default. A historical V5 diagnostic reported **88.10% Hit@1**, while the latest +V5/V6 comparison reported **83.33% Hit@1 (35/42)** and **88.10% Hit@5 (37/42)** +on the reused development set. The difference in evaluation configurations is +not yet reconciled. Neither is an independent live-UI accuracy guarantee. ## Windows (no Docker) @@ -25,6 +26,22 @@ provisional development questions and is **not a live-UI accuracy guarantee**. use an explicit absolute path or `../ml_models/.../best_model`. Relative paths resolve from the Django repository root. + For a new install, use the offline verified installer instead of manually + copying a checkpoint into the active location. Once you have the complete + V5 `best_model` directory on your Windows machine, run from `prototype`: + + ```powershell + python tools/install_servy_v5_checkpoint.py --source "C:\\path\\to\\verified\\best_model" + ``` + + The installer copies into `ml_models/checkpoints/servy_v5/best_model` by + default, verifies the **exact V5 weights SHA-256 before and after copying**, + rejects symlinks, and never overwrites an existing installation. It is + offline and does not upload checkpoints or labels to GitHub. If the V5 folder + is currently only in Colab, first transfer that same verified folder from + your approved Drive storage to the serving machine; the installer cannot + access your private Colab filesystem. **Do not install the V6 checkpoint.** + 3. In your local `prototype/.env` (not tracked in Git), configure: ```ini @@ -88,6 +105,8 @@ provisional development questions and is **not a live-UI accuracy guarantee**. ```powershell python manage.py test core.tests_v5_reranker -v 2 +python manage.py test core.tests_v5_activation -v 2 +python -m unittest tools.test_install_servy_v5_checkpoint -v ``` These tests mock the model and do not require the 2.27 GB weights. They are diff --git a/core/management/commands/verify_v5_reranker.py b/core/management/commands/verify_v5_reranker.py index 453816c..d70f542 100644 --- a/core/management/commands/verify_v5_reranker.py +++ b/core/management/commands/verify_v5_reranker.py @@ -2,7 +2,7 @@ from django.conf import settings from django.core.management.base import BaseCommand, CommandError -from core.ml.reranker import get_reranker, is_reranker_enabled, _local_checkpoint +from core.ml.reranker import (SERVY_V5_WEIGHTS_SHA256, get_reranker, is_reranker_enabled, _local_checkpoint) class Command(BaseCommand): @@ -14,8 +14,9 @@ def handle(self, *args, **options): checkpoint = _local_checkpoint() if checkpoint is None: raise CommandError("Set SERVY_RERANKER_MODEL_PATH to your full best_model folder.") - if not settings.SERVY_RERANKER_EXPECTED_SHA256: - raise CommandError("Set SERVY_RERANKER_EXPECTED_SHA256 to the verified V5 weights digest.") + expected = str(getattr(settings, "SERVY_RERANKER_EXPECTED_SHA256", "") or "").strip().lower() + if expected != SERVY_V5_WEIGHTS_SHA256: + raise CommandError("SERVY_RERANKER_EXPECTED_SHA256 must equal the approved V5 weights digest.") model = get_reranker() if model is None: raise CommandError("V5 reranker unavailable: inspect Django logs for the exact cause.") diff --git a/core/tests_v5_activation.py b/core/tests_v5_activation.py new file mode 100644 index 0000000..a3a4dde --- /dev/null +++ b/core/tests_v5_activation.py @@ -0,0 +1,43 @@ +"""The V5 verification command must never certify other checkpoint identities.""" +from io import StringIO +from pathlib import Path +from unittest import mock + +from django.core.management import call_command +from django.core.management.base import CommandError +from django.test import SimpleTestCase, override_settings + +from core.ml.reranker import SERVY_V5_WEIGHTS_SHA256 + + +class V5ActivationCommandTests(SimpleTestCase): + @override_settings( + SERVY_RERANKER_ENABLED=True, + SERVY_RERANKER_EXPECTED_SHA256="0" * 64, + ) + def test_rejects_non_v5_digest_before_loading(self): + with mock.patch("core.management.commands.verify_v5_reranker._local_checkpoint", + return_value=Path("/synthetic/best_model")), mock.patch( + "core.management.commands.verify_v5_reranker.get_reranker" + ) as load: + with self.assertRaisesRegex(CommandError, "approved V5"): + call_command("verify_v5_reranker") + load.assert_not_called() + + @override_settings( + SERVY_RERANKER_ENABLED=True, + SERVY_RERANKER_EXPECTED_SHA256=SERVY_V5_WEIGHTS_SHA256, + SERVY_RERANKER_DEVICE="cpu", + ) + def test_v5_digest_and_finite_prediction_succeed(self): + model = mock.Mock() + model.predict.return_value = [1.5] + output = StringIO() + with mock.patch("core.management.commands.verify_v5_reranker._local_checkpoint", + return_value=Path("/synthetic/best_model")), mock.patch( + "core.management.commands.verify_v5_reranker.get_reranker", + return_value=model, + ): + call_command("verify_v5_reranker", stdout=output) + self.assertIn("SERVY V5 RERANKER VERIFIED", output.getvalue()) + model.predict.assert_called_once_with([["machine fault", "machine fault procedure"]]) diff --git a/tools/install_servy_v5_checkpoint.py b/tools/install_servy_v5_checkpoint.py new file mode 100644 index 0000000..f5fb1a6 --- /dev/null +++ b/tools/install_servy_v5_checkpoint.py @@ -0,0 +1,73 @@ +"""Install an already-approved Servy V5 CrossEncoder checkpoint without Git or network access. + +The installer verifies the fixed V5 weight identity, refuses symlinks and +existing targets, and stages a complete local copy before publishing it. +It does not enable production serving or attest independent accuracy. +""" +from __future__ import annotations + +import argparse +import hashlib +import os +from pathlib import Path +import shutil +import tempfile + +V5_WEIGHTS_SHA256 = "b1cf23ba0806f0057d73a5ab93fde88d634fc278f71b27f3a2a915afd8f8bcda" +DEFAULT_TARGET = Path(__file__).resolve().parent.parent / "ml_models" / "checkpoints" / "servy_v5" / "best_model" + + +def file_sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for block in iter(lambda: stream.read(8 * 1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + +def validate_source(source: Path, expected_sha256: str = V5_WEIGHTS_SHA256) -> None: + if source.is_symlink() or not source.is_dir(): + raise ValueError("Source must be a real directory, not a symlink") + for root, directories, files in os.walk(source, followlinks=False): + for name in directories + files: + if (Path(root) / name).is_symlink(): + raise ValueError("Checkpoint contains symlinks; refusing installation") + for filename in ("model.safetensors", "config.json"): + if not (source / filename).is_file(): + raise ValueError(f"V5 checkpoint missing {filename}") + if file_sha256(source / "model.safetensors") != expected_sha256: + raise ValueError("Checkpoint weights do not match the approved V5 SHA-256") + + +def install_checkpoint(source: Path, target: Path = DEFAULT_TARGET, + expected_sha256: str = V5_WEIGHTS_SHA256) -> Path: + source = Path(source).expanduser().absolute() + target = Path(target).expanduser().absolute() + if source == target or source in target.parents or target in source.parents: + raise ValueError("Source and target directories must not overlap") + if target.exists() or target.is_symlink(): + raise FileExistsError(f"Refusing to overwrite existing checkpoint: {target}") + validate_source(source, expected_sha256) + target.parent.mkdir(parents=True, exist_ok=True) + with tempfile.TemporaryDirectory(prefix=".v5-install-", dir=target.parent) as stage: + staged = Path(stage) / "best_model" + shutil.copytree(source, staged, symlinks=False) + validate_source(staged, expected_sha256) + if target.exists() or target.is_symlink(): + raise FileExistsError(f"Checkpoint destination appeared during installation: {target}") + staged.rename(target) + return target + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--source", type=Path, required=True, help="Local V5 best_model directory") + parser.add_argument("--target", type=Path, default=DEFAULT_TARGET) + args = parser.parse_args() + location = install_checkpoint(args.source, args.target) + print(f"SERVY V5 CHECKPOINT INSTALLED AND SHA256 VERIFIED: {location}") + print("Activation requires explicit local configuration and a successful Django inference smoke check.") + + +if __name__ == "__main__": + main() diff --git a/tools/test_install_servy_v5_checkpoint.py b/tools/test_install_servy_v5_checkpoint.py new file mode 100644 index 0000000..433a195 --- /dev/null +++ b/tools/test_install_servy_v5_checkpoint.py @@ -0,0 +1,64 @@ +"""Synthetic-only tests for the atomic, offline V5 checkpoint installer.""" +import hashlib +from pathlib import Path +import tempfile +import unittest + +from tools.install_servy_v5_checkpoint import install_checkpoint + + +class InstallV5CheckpointTests(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.addCleanup(self.tmp.cleanup) + self.root = Path(self.tmp.name) + self.source = self.root / "src" / "best_model" + self.source.mkdir(parents=True) + self.weights = b"synthetic-only-v5-checkpoint-weights" + (self.source / "model.safetensors").write_bytes(self.weights) + (self.source / "config.json").write_text("{}", encoding="utf-8") + (self.source / "tokenizer_config.json").write_text("{}", encoding="utf-8") + self.expected = hashlib.sha256(self.weights).hexdigest() + self.target = self.root / "installed" / "best_model" + + def test_verified_copy_and_preserves_source(self): + output = install_checkpoint(self.source, self.target, self.expected) + self.assertEqual(output, self.target) + self.assertEqual((self.target / "model.safetensors").read_bytes(), self.weights) + self.assertEqual((self.target / "tokenizer_config.json").read_text(), "{}") + self.assertTrue((self.source / "model.safetensors").exists()) + self.assertEqual(list(self.target.parent.glob(".v5-install-*")), []) + + def test_wrong_weights_cannot_publish_checkpoint(self): + with self.assertRaisesRegex(ValueError, "SHA-256"): + install_checkpoint(self.source, self.target, "0" * 64) + self.assertFalse(self.target.exists()) + + def test_existing_install_is_not_overwritten(self): + self.target.mkdir(parents=True) + (self.target / "sentinel").write_text("existing") + with self.assertRaises(FileExistsError): + install_checkpoint(self.source, self.target, self.expected) + self.assertEqual((self.target / "sentinel").read_text(), "existing") + + def test_refuses_source_inside_target(self): + with self.assertRaisesRegex(ValueError, "overlap"): + install_checkpoint(self.source, self.source / "nested", self.expected) + + def test_rejects_missing_config(self): + (self.source / "config.json").unlink() + with self.assertRaisesRegex(ValueError, "missing config.json"): + install_checkpoint(self.source, self.target, self.expected) + + def test_rejects_symlink(self): + link = self.source / "linked.bin" + try: + link.symlink_to(self.source / "config.json") + except (NotImplementedError, OSError): + self.skipTest("Symlink creation unavailable") + with self.assertRaisesRegex(ValueError, "symlinks"): + install_checkpoint(self.source, self.target, self.expected) + + +if __name__ == "__main__": + unittest.main()