From eaa654af74ee265258b09e10822864029be1b524 Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Mon, 14 Sep 2026 23:08:54 -0700 Subject: [PATCH 01/44] feat(ml): store a feature vector for every detection the processing service sends Pipeline results can now carry an `embeddings` list on each detection: one 2048-float vector per algorithm, including detections the moth/non-moth filter rejected. They are stored in a new DetectionEmbedding table, one row per (detection, algorithm), rather than as an extra classification, because an extra species classification on a rejected crop changes its occurrence's determination. - Re-saving the same results replaces the stored vector, so redelivery and reprocessing are idempotent. - Responses are matched to detections by image and box, because create_detections returns existing detections ahead of new ones. - An algorithm key the pipeline has not registered raises PipelineNotConfigured at the point an unregistered classification algorithm would. - Migration 0101 only creates the table, so it is safe on a populated database. Refs #1417 Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01C7Xf6VPbwWtTumhjjF15g8 --- .../migrations/0102_detection_embedding.py | 55 +++++ ami/main/models.py | 31 +++ ami/ml/models/pipeline.py | 69 ++++++ ami/ml/schemas.py | 26 +++ ami/ml/tests.py | 200 +++++++++++++++++- 5 files changed, 379 insertions(+), 2 deletions(-) create mode 100644 ami/main/migrations/0102_detection_embedding.py diff --git a/ami/main/migrations/0102_detection_embedding.py b/ami/main/migrations/0102_detection_embedding.py new file mode 100644 index 000000000..17a843648 --- /dev/null +++ b/ami/main/migrations/0102_detection_embedding.py @@ -0,0 +1,55 @@ +# Generated by Django 4.2.10 on 2026-09-15 02:01 +# +# Additive: creates an empty table. The unique constraint's index, which leads with +# detection_id, also serves reads of the vectors for a set of detections. See #1417. + +from django.db import migrations, models +import django.db.models.deletion +import pgvector.django.vector + + +class Migration(migrations.Migration): + dependencies = [ + ("ml", "0028_normalize_empty_endpoint_url_to_null"), + ("main", "0101_grant_run_post_processing_to_ml_data_manager"), + ] + + operations = [ + migrations.CreateModel( + name="DetectionEmbedding", + fields=[ + ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), + ("created_at", models.DateTimeField(auto_now_add=True)), + ("updated_at", models.DateTimeField(auto_now=True)), + ( + "features_2048", + pgvector.django.vector.VectorField( + dimensions=2048, help_text="Feature embedding from the model backbone" + ), + ), + ( + "algorithm", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="detection_embeddings", + to="ml.algorithm", + ), + ), + ( + "detection", + models.ForeignKey( + db_index=False, + on_delete=django.db.models.deletion.CASCADE, + related_name="embeddings", + to="main.detection", + ), + ), + ], + ), + migrations.AddConstraint( + model_name="detectionembedding", + constraint=models.UniqueConstraint( + fields=("detection", "algorithm"), name="unique_detection_embedding_per_algorithm" + ), + ), + ] diff --git a/ami/main/models.py b/ami/main/models.py index 117e0738b..506ee21bb 100644 --- a/ami/main/models.py +++ b/ami/main/models.py @@ -3339,6 +3339,7 @@ class Detection(BaseModel): # For type hints classifications: models.QuerySet["Classification"] + embeddings: models.QuerySet["DetectionEmbedding"] source_image_id: int detection_algorithm_id: int @@ -3481,6 +3482,36 @@ def __str__(self) -> str: return f"#{self.pk} from SourceImage #{self.source_image_id} with Algorithm #{self.detection_algorithm_id}" +@final +class DetectionEmbedding(BaseModel): + """A feature vector for one detection from one algorithm, used to compare detections by appearance. + + Kept apart from classifications so every detection can have one, including those the + moth/non-moth filter rejected, without adding a prediction that could change a + determination. Vectors are only comparable within one algorithm. See #1417. + """ + + project_accessor = "detection__source_image__project" + + # No separate index: the unique constraint's index leads with detection_id. + detection = models.ForeignKey(Detection, on_delete=models.CASCADE, related_name="embeddings", db_index=False) + algorithm = models.ForeignKey("ml.Algorithm", on_delete=models.CASCADE, related_name="detection_embeddings") + features_2048 = pgvector.django.VectorField( + dimensions=2048, + help_text="Feature embedding from the model backbone", + ) + + class Meta: + constraints = [ + models.UniqueConstraint( + fields=["detection", "algorithm"], name="unique_detection_embedding_per_algorithm" + ), + ] + + def __str__(self) -> str: + return f"#{self.pk} vector for Detection #{self.detection_id} from Algorithm #{self.algorithm_id}" + + class OccurrenceQuerySet(BaseQuerySet): def valid(self): """ diff --git a/ami/ml/models/pipeline.py b/ami/ml/models/pipeline.py index be739d84d..302be4091 100644 --- a/ami/ml/models/pipeline.py +++ b/ami/ml/models/pipeline.py @@ -28,6 +28,7 @@ Classification, Deployment, Detection, + DetectionEmbedding, Occurrence, Project, SourceImage, @@ -680,6 +681,65 @@ def create_detections( return existing_detections + new_detections +# A vector is roughly 40 KB of SQL text (estimate), so this keeps each INSERT to a few MB. +EMBEDDING_BATCH_SIZE = 200 + + +def create_detection_embeddings( + detections: list[Detection], + detection_responses: list[DetectionResponse], + algorithms_known: dict[str, Algorithm], + logger: logging.Logger = logger, +) -> list[DetectionEmbedding]: + """ + Store the feature vectors sent with each detection, one row per (detection, algorithm). + + A vector already stored for the pair is replaced, so saving the same results twice + changes nothing. Only ``DetectionEmbedding`` rows are written, never a classification, + so no determination can change. + + Responses are matched to detections by image and box, the key ``get_or_create_detection`` + reuses detections by, because ``create_detections`` does not return them in response order. + An algorithm key the pipeline has not registered raises ``PipelineNotConfigured``, as it + does for classifications. + """ + by_box = { + (str(detection.source_image_id), tuple(detection.bbox)): detection + for detection in detections + if detection.bbox is not None + } + embeddings: dict[tuple[int, int], DetectionEmbedding] = {} + for detection_resp in detection_responses: + if not detection_resp.embeddings or detection_resp.bbox is None: + continue + detection = by_box.get((detection_resp.source_image_id, tuple(detection_resp.bbox.dict().values()))) + if detection is None: + # Its source image was not found; create_detections has logged that. + continue + for embedding_resp in detection_resp.embeddings: + try: + algorithm = algorithms_known[embedding_resp.algorithm.key] + except KeyError as err: + raise PipelineNotConfigured( + f"Embedding algorithm {embedding_resp.algorithm.key} is not a known algorithm. " + "The processing service must declare it in the /info endpoint. " + f"Known algorithms: {list(algorithms_known.keys())}" + ) from err + embeddings[(detection.pk, algorithm.pk)] = DetectionEmbedding( + detection=detection, algorithm=algorithm, features_2048=embedding_resp.features + ) + + DetectionEmbedding.objects.bulk_create( + list(embeddings.values()), + update_conflicts=True, + unique_fields=["detection", "algorithm"], + update_fields=["features_2048", "updated_at"], + batch_size=EMBEDDING_BATCH_SIZE, + ) + logger.info(f"Stored {len(embeddings)} detection embeddings for {len(detections)} detections.") + return list(embeddings.values()) + + def create_category_map_for_classification( classification_resp: ClassificationResponse, logger: logging.Logger = logger, @@ -1082,6 +1142,15 @@ def save_results( logger=job_logger, ) + # Before classifications, so an unregistered embedding algorithm stops the batch at the + # same point an unregistered classification algorithm does. + create_detection_embeddings( + detections=detections, + detection_responses=results.detections, + algorithms_known=algorithms_known, + logger=job_logger, + ) + classifications = create_classifications( detections=detections, detection_responses=results.detections, diff --git a/ami/ml/schemas.py b/ami/ml/schemas.py index d03643f71..0224dc3f9 100644 --- a/ami/ml/schemas.py +++ b/ami/ml/schemas.py @@ -182,6 +182,25 @@ class DetectionRequest(pydantic.BaseModel): algorithm: AlgorithmReference +class EmbeddingResponse(pydantic.BaseModel): + """A feature vector for one detection and the algorithm whose backbone produced it. + + Carried on the detection rather than on a classification, so storing it can never + add a prediction that competes for the occurrence's determination. + """ + + features: list[float] = pydantic.Field( + description="The feature vector. Must be exactly 2048 floats.", + ) + algorithm: AlgorithmReference + + @pydantic.validator("features") + def _features_length(cls, v): + if len(v) != 2048: + raise ValueError(f"features must be length 2048, got {len(v)}") + return v + + class DetectionResponse(pydantic.BaseModel): source_image_id: str bbox: BoundingBox | None = None @@ -190,6 +209,13 @@ class DetectionResponse(pydantic.BaseModel): timestamp: datetime.datetime crop_image_url: str | None = None classifications: list[ClassificationResponse] = [] + embeddings: list[EmbeddingResponse] | None = pydantic.Field( + default=None, + description=( + "Feature vectors for this detection, at most one per algorithm, including detections " + "the moth/non-moth filter rejected. Only vectors from the same algorithm are comparable." + ), + ) class PipelineRequestConfigParameters(pydantic.BaseModel): diff --git a/ami/ml/tests.py b/ami/ml/tests.py index 70bb2e16b..d2cbd83d0 100644 --- a/ami/ml/tests.py +++ b/ami/ml/tests.py @@ -4,7 +4,8 @@ import unittest import uuid -from django.test import TestCase +import pydantic +from django.test import SimpleTestCase, TestCase from rest_framework.test import APIRequestFactory, APITestCase from ami.base.serializers import reverse_with_params @@ -12,6 +13,7 @@ Classification, Deployment, Detection, + DetectionEmbedding, Event, Identification, Occurrence, @@ -22,8 +24,14 @@ TaxonRank, group_images_into_events, ) +from ami.ml.exceptions import PipelineNotConfigured from ami.ml.models import Algorithm, Pipeline, ProcessingService -from ami.ml.models.pipeline import collect_images, get_or_create_algorithm_and_category_map, save_results +from ami.ml.models.pipeline import ( + collect_images, + create_detection_embeddings, + get_or_create_algorithm_and_category_map, + save_results, +) from ami.ml.post_processing.small_size_filter import SmallSizeFilterTask from ami.ml.schemas import ( AlgorithmConfigResponse, @@ -2340,3 +2348,191 @@ def test_used_lookup_is_deduplicated_in_the_database(self): len(list(lookup.order_by().distinct())), 1, "Deduplicating collapses them to the one algorithm" ) self.assertIn("Chatty Masked Classifier", self._choice_names(self.project.pk)) + + +EMBEDDING_DETECTOR = ALGORITHM_CHOICES["random-detector"] +EMBEDDING_BINARY = ALGORITHM_CHOICES["random-binary-classifier"] # labels: "Moth", "Not a moth" +EMBEDDING_SPECIES = ALGORITHM_CHOICES["random-species-classifier"] + + +def _embedding_payload(vector: list[float], algorithm=EMBEDDING_SPECIES) -> list[dict]: + return [{"algorithm": {"name": algorithm.name, "key": algorithm.key}, "features": vector}] + + +class TestEmbeddingSchema(SimpleTestCase): + """The per-detection ``embeddings`` field of the processing-service results schema.""" + + def _detection(self, **extra) -> dict: + return { + "source_image_id": "1", + "bbox": {"x1": 0.0, "y1": 0.0, "x2": 10.0, "y2": 10.0}, + "algorithm": {"name": EMBEDDING_DETECTOR.name, "key": EMBEDDING_DETECTOR.key}, + "timestamp": datetime.datetime.now().isoformat(), + **extra, + } + + def test_a_detection_carries_each_vector_with_its_algorithm(self): + """The field is optional, so a service that sends no embeddings still parses.""" + parsed = DetectionResponse.parse_obj(self._detection(embeddings=_embedding_payload([0.5] * 2048))) + self.assertEqual( + [(e.algorithm.key, len(e.features)) for e in parsed.embeddings or []], [(EMBEDDING_SPECIES.key, 2048)] + ) + self.assertIsNone(DetectionResponse.parse_obj(self._detection()).embeddings) + + def test_a_vector_of_another_length_is_refused(self): + """The column holds 2048 floats, so a shorter vector must fail validation rather than the insert.""" + with self.assertRaises(pydantic.ValidationError): + DetectionResponse.parse_obj(self._detection(embeddings=_embedding_payload([0.5] * 512))) + + +class TestDetectionEmbeddings(TestCase): + """Storing the feature vector a processing service sends with each detection. + + The vector is what tracking and merge ranking compare, and it arrives for every + detection, including those the moth/non-moth filter rejected. What these pin is that + it is stored once per detection and algorithm, on the detection it was sent with, and + that storing it never adds a classification or moves a determination. + """ + + LOW = [0.25] * 2048 + HIGH = [0.75] * 2048 + + def setUp(self) -> None: + self.project = Project.objects.create(name="Detection embeddings") + self.pipeline = Pipeline.objects.create(name="Embedding test pipeline") + self.pipeline.algorithms.set( + [ + get_or_create_algorithm_and_category_map(algorithm) + for algorithm in (EMBEDDING_DETECTOR, EMBEDDING_BINARY, EMBEDDING_SPECIES) + ] + ) + self.images = 0 + + def _image(self) -> SourceImage: + self.images += 1 + return SourceImage.objects.create( + path=f"emb-{self.images}-20240101000{self.images}00.jpg", project=self.project + ) + + @staticmethod + def _classification(algorithm, label: str, score: float, terminal: bool) -> dict: + return { + "classification": label, + "scores": [score], + "algorithm": {"name": algorithm.name, "key": algorithm.key}, + "terminal": terminal, + "timestamp": datetime.datetime.now().isoformat(), + } + + def _detection(self, image: SourceImage, classifications: list[dict], embeddings=None, box: float = 0.0) -> dict: + payload = { + "source_image_id": str(image.pk), + "bbox": {"x1": box, "y1": box, "x2": box + 10.0, "y2": box + 10.0}, + "algorithm": {"name": EMBEDDING_DETECTOR.name, "key": EMBEDDING_DETECTOR.key}, + "timestamp": datetime.datetime.now().isoformat(), + "classifications": classifications, + } + if embeddings is not None: + payload["embeddings"] = embeddings + return payload + + def _rejected(self, image: SourceImage, embeddings=None, box: float = 0.0) -> dict: + """A crop the moth/non-moth filter rejected, labelled as the service labels it: non-terminal.""" + label = self._classification(EMBEDDING_BINARY, "Not a moth", 0.8, terminal=False) + return self._detection(image, [label], embeddings, box) + + def _moth(self, image: SourceImage, embeddings=None, box: float = 0.0) -> dict: + labels = [ + self._classification(EMBEDDING_BINARY, "Moth", 0.95, terminal=False), + self._classification(EMBEDDING_SPECIES, "Vanessa cardui", 0.6, terminal=True), + ] + return self._detection(image, labels, embeddings, box) + + def _save(self, *detections: dict) -> None: + image_ids = list(dict.fromkeys(d["source_image_id"] for d in detections)) + payload = { + "pipeline": self.pipeline.slug, + "total_time": 0.01, + "source_images": [{"id": image_id, "url": f"test/{image_id}.jpg"} for image_id in image_ids], + "detections": list(detections), + } + save_results(PipelineResultsResponse.parse_obj(payload)) + + @staticmethod + def _stored(image: SourceImage) -> dict[tuple[float, str], list[float]]: + """{(box corner, algorithm key): vector} for the image's stored embeddings.""" + rows = DetectionEmbedding.objects.filter(detection__source_image=image).select_related( + "detection", "algorithm" + ) + return {(row.detection.bbox[0], row.algorithm.key): row.features_2048.tolist() for row in rows} + + def test_every_detection_stores_one_vector_per_algorithm(self): + """Including the rejected crop, which has no species classification that could carry one.""" + image = self._image() + self._save( + self._moth(image, _embedding_payload(self.LOW)), + self._rejected(image, _embedding_payload(self.HIGH), box=100.0), + ) + self.assertEqual( + self._stored(image), + {(0.0, EMBEDDING_SPECIES.key): self.LOW, (100.0, EMBEDDING_SPECIES.key): self.HIGH}, + ) + + def test_a_vector_adds_no_classification_and_moves_no_determination(self): + """The rejected crop's binary label is non-terminal, so a species classification sent in + the vector's place would become its determination. An embedding must not.""" + control, treated = self._image(), self._image() + self._save(self._rejected(control)) + self._save(self._rejected(treated, _embedding_payload(self.HIGH))) + + for image in (control, treated): + detection = Detection.objects.select_related("occurrence__determination").get(source_image=image) + self.assertEqual(detection.occurrence.determination.name, "Not a moth") + self.assertEqual(detection.classifications.count(), 1) + self.assertEqual(DetectionEmbedding.objects.filter(detection__source_image=treated).count(), 1) + + def test_saving_again_keeps_one_row_per_detection_and_algorithm(self): + """Results are re-delivered and images reprocessed; the one row holds the latest vector.""" + image = self._image() + self._save(self._rejected(image, _embedding_payload(self.LOW))) + self._save(self._rejected(image, _embedding_payload(self.LOW))) + self.assertEqual(DetectionEmbedding.objects.filter(detection__source_image=image).count(), 1) + + self._save(self._rejected(image, _embedding_payload(self.HIGH))) + self.assertEqual(self._stored(image), {(0.0, EMBEDDING_SPECIES.key): self.HIGH}) + + def test_a_vector_lands_on_its_own_detection_when_some_detections_already_exist(self): + """Detection creation returns existing detections ahead of new ones, so pairing responses + with detections by position would swap these two vectors.""" + image = self._image() + self._save(self._rejected(image, box=0.0)) + self._save( + self._rejected(image, _embedding_payload(self.HIGH), box=100.0), + self._rejected(image, _embedding_payload(self.LOW), box=0.0), + ) + self.assertEqual( + self._stored(image), + {(0.0, EMBEDDING_SPECIES.key): self.LOW, (100.0, EMBEDDING_SPECIES.key): self.HIGH}, + ) + + def test_a_vector_from_an_unregistered_algorithm_stops_the_batch_like_a_classification(self): + """It raises where an unregistered classification algorithm does: after detections are + saved and before any classification is.""" + image = self._image() + unregistered = {"algorithm": {"name": "Unregistered", "key": "unregistered-embedder"}, "features": self.LOW} + with self.assertRaises(PipelineNotConfigured): + self._save(self._rejected(image, [unregistered])) + self.assertFalse(DetectionEmbedding.objects.exists()) + self.assertFalse(Classification.objects.filter(detection__source_image=image).exists()) + + def test_storing_vectors_takes_one_query_however_many_detections(self): + image = self._image() + responses = [self._rejected(image, _embedding_payload(self.LOW), box=float(20 * i)) for i in range(5)] + self._save(*responses) + detections = list(Detection.objects.filter(source_image=image)) + parsed = [DetectionResponse.parse_obj(response) for response in responses] + algorithms_known = {algorithm.key: algorithm for algorithm in self.pipeline.algorithms.all()} + + with self.assertNumQueries(1): + stored = create_detection_embeddings(detections, parsed, algorithms_known) + self.assertEqual(len(stored), 5) From 95e2bdb6465d11bd767df4b05ebadec62e8cdafc Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Mon, 28 Sep 2026 12:16:40 -0700 Subject: [PATCH 02/44] feat(tracking): compare detections by their stored embeddings in tracking and merge ranking Tracking, the merge-candidate ranking, the extend-mode preview and the "has a vector" counts read a detection's vector from its DetectionEmbedding when there is one, and otherwise from the most recent classification vector of the same algorithm. Detections the moth/non-moth filter rejected are then compared by appearance too, and older data keeps working. - One reader (models_future/embeddings.py) reads both stores in one UNION ALL query, always keyed by algorithm. - Merge ranking scores every candidate with one algorithm, the one with vectors on the most of the occurrence's scored frames, the lowest id on a tie. - Tracking loads the vectors for a pair of captures in one query instead of one per detection, through the shared greedy matcher. - has_features is true when the classification's own algorithm stored a vector for its detection; frames_with_vectors and detections_with_features count a detection with a vector from any algorithm. - A test pins that a box whose only vector is a stored embedding is scored and linked in extend mode. Refs #1417 Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01C7Xf6VPbwWtTumhjjF15g8 --- ami/main/api/serializers.py | 4 +- ami/main/models.py | 58 ++++-- ami/main/models_future/embeddings.py | 87 +++++++++ ami/main/models_future/merge_candidates.py | 68 +++---- ami/main/tests.py | 172 +++++++++++++++++- ami/ml/post_processing/admin_forms.py | 20 +- .../tests/test_tracking_task.py | 76 ++++++++ ami/ml/post_processing/tracking_task.py | 55 ++---- 8 files changed, 417 insertions(+), 123 deletions(-) create mode 100644 ami/main/models_future/embeddings.py diff --git a/ami/main/api/serializers.py b/ami/main/api/serializers.py index 27cf79253..bf2de2dc8 100644 --- a/ami/main/api/serializers.py +++ b/ami/main/api/serializers.py @@ -1181,7 +1181,7 @@ class ClassificationNestedSerializer(ClassificationSerializer): has_features = serializers.BooleanField( read_only=True, allow_null=True, - help_text="Whether a feature embedding was stored for this classification.", + help_text="Whether this classification's algorithm stored a feature vector for its detection.", ) def get_permissions(self, instance, instance_data): @@ -1358,7 +1358,7 @@ class SourceImageSerializer(SourceImageListSerializer): ) detections_with_features = serializers.IntegerField( read_only=True, - help_text="Valid detections with at least one classification that stored a feature embedding.", + help_text="Valid detections with a stored feature vector, as an embedding or on a classification.", ) # file = serializers.ImageField(allow_empty_file=False, use_url=True) diff --git a/ami/main/models.py b/ami/main/models.py index 506ee21bb..534fcec9a 100644 --- a/ami/main/models.py +++ b/ami/main/models.py @@ -2202,16 +2202,16 @@ def with_was_processed(self): def with_detections_with_features(self): """Annotate ``detections_with_features`` and the ``detections_valid`` it is out of: - valid detections on the capture, and how many of them have a classification that - stored a feature embedding. Counted in SQL so the vectors themselves are never - loaded. Both come from the same population, so a caller can show one as a share of - the other; the cached ``detections_count`` is a different, default-filtered count. + valid detections on the capture, and how many of them carry a stored feature vector + from any algorithm (see ``DetectionQuerySet.has_vector``). Counted in SQL so the + vectors themselves are never loaded. Both come from the same population, so a caller + can show one as a share of the other; the cached ``detections_count`` is a different, + default-filtered count. """ - def count_valid(**extra): + def count_valid(detections): return models.Subquery( - Detection.objects.valid() - .filter(source_image_id=models.OuterRef("pk"), **extra) + detections.filter(source_image_id=models.OuterRef("pk")) .order_by() .values("source_image_id") .annotate(count=models.Count("id", distinct=True)) @@ -2220,8 +2220,8 @@ def count_valid(**extra): ) return self.annotate( - detections_valid=Coalesce(count_valid(), 0), - detections_with_features=Coalesce(count_valid(classifications__features_2048__isnull=False), 0), + detections_valid=Coalesce(count_valid(Detection.objects.valid()), 0), + detections_with_features=Coalesce(count_valid(Detection.objects.valid().has_vector()), 0), ) def with_thumbnails(self): @@ -3061,12 +3061,21 @@ class ClassificationQuerySet(BaseQuerySet): def with_has_features(self): """Annotate ``has_features`` and defer the embedding itself. - Read paths only need to know whether a feature vector was stored; deferring the - 2048-float column keeps it out of the row's SELECT. A select_related self-join - (``applied_to``) needs its own ``defer("applied_to__features_2048")``. + ``has_features`` is true when the classification's algorithm stored a vector for + its detection, on the classification or as a ``DetectionEmbedding``. Read paths + only need to know that; deferring the 2048-float column keeps it out of the row's + SELECT. A select_related self-join (``applied_to``) needs its own + ``defer("applied_to__features_2048")``. """ + embedded = Exists( + DetectionEmbedding.objects.filter( + detection_id=OuterRef("detection_id"), algorithm_id=OuterRef("algorithm_id") + ) + ) return self.defer("features_2048").annotate( - has_features=models.ExpressionWrapper(Q(features_2048__isnull=False), output_field=models.BooleanField()) + has_features=models.ExpressionWrapper( + Q(features_2048__isnull=False) | Q(embedded), output_field=models.BooleanField() + ) ) def find_duplicates(self, project_id: int | None = None) -> models.QuerySet: @@ -3260,6 +3269,18 @@ def null_markers(self): """ return self.filter(NULL_DETECTIONS_FILTER) + def has_vector(self, algorithm=None): + """Detections with a stored feature vector: a ``DetectionEmbedding``, or a + classification's ``features_2048``. Pass ``algorithm`` to count only that + algorithm's vectors. Tested with EXISTS, so no vector is ever loaded. + """ + embeddings = DetectionEmbedding.objects.filter(detection_id=OuterRef("pk")) + classifications = Classification.objects.filter(detection_id=OuterRef("pk"), features_2048__isnull=False) + if algorithm is not None: + embeddings = embeddings.filter(algorithm=algorithm) + classifications = classifications.filter(algorithm=algorithm) + return self.filter(Exists(embeddings) | Exists(classifications)) + class DetectionManager(models.Manager.from_queryset(DetectionQuerySet)): pass @@ -3548,15 +3569,16 @@ def with_detections_count(self): return self.annotate(detections_count=models.Count("detections", distinct=True)) def with_frames_with_vectors(self): - """Annotate ``frames_with_vectors``: detections in the occurrence with at least one - classification that stored a feature embedding. Counted in SQL so the vectors - themselves are never loaded. + """Annotate ``frames_with_vectors``: detections in the occurrence with a stored + feature vector from any algorithm (see ``DetectionQuerySet.has_vector``). Counted + in SQL so the vectors themselves are never loaded. """ subquery = ( - Detection.objects.filter(occurrence_id=OuterRef("pk"), classifications__features_2048__isnull=False) + Detection.objects.has_vector() + .filter(occurrence_id=OuterRef("pk")) .order_by() .values("occurrence_id") - .annotate(count=models.Count("id", distinct=True)) + .annotate(count=models.Count("id")) .values("count") ) return self.annotate( diff --git a/ami/main/models_future/embeddings.py b/ami/main/models_future/embeddings.py new file mode 100644 index 000000000..e9965c658 --- /dev/null +++ b/ami/main/models_future/embeddings.py @@ -0,0 +1,87 @@ +"""Feature vectors for detections, read one algorithm at a time. + +A detection's vector is stored in one of two places: a ``DetectionEmbedding`` row, which a +processing service can send for every detection, or the ``features_2048`` of one of its +classifications, which is all that data processed before embeddings existed has. Readers +take the embedding when there is one and otherwise the most recent classification vector +from the same algorithm. Vectors from different algorithms are not comparable, so every +reader here is keyed by algorithm. See #1417. +""" + +from __future__ import annotations + +from collections.abc import Iterable +from typing import Any + +from django.db.models import F, IntegerField, QuerySet, Value + +_PREFER_EMBEDDING = 0 +_PREFER_CLASSIFICATION = 1 + + +def _vector_rows(detection_ids: Iterable[int], algorithm_ids: Iterable[int] | None) -> QuerySet: + """(detection_id, algorithm_id, vector, ...) rows from both stores in one query, preferred rows first.""" + from ami.main.models import Classification, DetectionEmbedding + + detection_ids = list(detection_ids) + embeddings = DetectionEmbedding.objects.filter(detection_id__in=detection_ids) + classifications = Classification.objects.filter( + detection_id__in=detection_ids, algorithm_id__isnull=False, features_2048__isnull=False + ) + if algorithm_ids is not None: + algorithm_ids = list(algorithm_ids) + embeddings = embeddings.filter(algorithm_id__in=algorithm_ids) + classifications = classifications.filter(algorithm_id__in=algorithm_ids) + + # Model fields first, then the annotations in the same order, so both SELECT lists + # line up column for column. UNION ALL: de-duplicating would sort the vectors. + columns = ("detection_id", "algorithm_id", "features_2048", "id", "preference", "recorded_at") + embeddings = ( + embeddings.order_by() + .annotate(preference=Value(_PREFER_EMBEDDING, output_field=IntegerField()), recorded_at=F("updated_at")) + .values_list(*columns) + ) + classifications = ( + classifications.order_by() + .annotate(preference=Value(_PREFER_CLASSIFICATION, output_field=IntegerField()), recorded_at=F("timestamp")) + .values_list(*columns) + ) + return embeddings.union(classifications, all=True).order_by("preference", "-recorded_at", "-id") + + +def latest_vectors( + detection_ids: Iterable[int], algorithm_ids: Iterable[int] | None = None +) -> dict[tuple[int, int], Any]: + """The vector for each (detection, algorithm) pair that has one, in one query. + + Pass ``algorithm_ids`` to read only those algorithms. The caller must still compare + vectors from one algorithm only. + """ + vectors: dict[tuple[int, int], Any] = {} + for detection_id, algorithm_id, vector, *_ in _vector_rows(detection_ids, algorithm_ids): + vectors.setdefault((detection_id, algorithm_id), vector) + return vectors + + +def vectors_for_detections(detection_ids: Iterable[int], algorithm_id: int) -> dict[int, Any]: + """Each detection's vector from one algorithm, in one query. Detections without one are absent.""" + return { + detection_id: vector for (detection_id, _), vector in latest_vectors(detection_ids, [algorithm_id]).items() + } + + +def algorithm_ids_with_vectors(**detection_lookups: Any) -> set[int]: + """Algorithms that stored a vector for any detection matching the lookups, in one query. + + Lookups are relative to the detection, e.g. ``source_image__event=event``. + """ + from ami.main.models import Classification, DetectionEmbedding + + lookups = {f"detection__{key}": value for key, value in detection_lookups.items()} + embedded = DetectionEmbedding.objects.filter(**lookups).order_by().values_list("algorithm_id", flat=True) + classified = ( + Classification.objects.filter(**lookups, features_2048__isnull=False, algorithm_id__isnull=False) + .order_by() + .values_list("algorithm_id", flat=True) + ) + return set(embedded.union(classified)) diff --git a/ami/main/models_future/merge_candidates.py b/ami/main/models_future/merge_candidates.py index 7cef580ac..cfd587440 100644 --- a/ami/main/models_future/merge_candidates.py +++ b/ami/main/models_future/merge_candidates.py @@ -25,8 +25,11 @@ positive for ``after`` and zero for ``gap``. - ``distance``: centre-to-centre distance of the nearest pair of boxes as a fraction of the frame diagonal. -- ``similarity``: cosine similarity of the pair's feature vectors, from the same - algorithm that produced the occurrence's own vector. Null when either has none. +- ``similarity``: cosine similarity of the pair's feature vectors. Vectors from different + algorithms are not comparable, so every candidate is scored with one algorithm: the one + with vectors on the most of the occurrence's scored frames. A frame's vector is its + embedding, or failing that its classification vector from that algorithm (see + ``embeddings``). Null when either frame has no vector from it. - ``cost``: the tracking method's matching cost for the pair. Geometry only when similarity is null, which lowers the total, so a candidate without a vector can outrank one with a poor vector match. Null when either box is malformed. @@ -55,11 +58,13 @@ from __future__ import annotations +import collections import datetime from typing import TYPE_CHECKING, Any from django.db.models import Q, QuerySet +from ami.main.models_future.embeddings import algorithm_ids_with_vectors, latest_vectors, vectors_for_detections from ami.main.models_future.track_stats import bbox_corners, frame_diagonal if TYPE_CHECKING: @@ -152,13 +157,11 @@ def _pair_diagonal(track_frame: dict, frame: dict, corners_a, corners_b) -> floa return frame_diagonal(None, None, max(corners_a[2], corners_b[2]), max(corners_a[3], corners_b[3])) -def _latest_vectors(classifications) -> dict[tuple[int, int], Any]: - """Most recent feature vector per (detection, algorithm) among the rows given.""" - vectors: dict[tuple[int, int], Any] = {} - rows = classifications.order_by("-timestamp", "-pk").values_list("detection_id", "algorithm_id", "features_2048") - for detection_id, algorithm_id, vector in rows: - vectors.setdefault((detection_id, algorithm_id), vector) - return vectors +def _comparison_algorithm(track_vectors: dict[tuple[int, int], Any]) -> int | None: + """The algorithm every candidate is scored with: the one with vectors on the most of the + occurrence's scored frames, the lowest id on a tie so the choice is stable.""" + counts = collections.Counter(algorithm_id for _, algorithm_id in track_vectors) + return min(counts, key=lambda algorithm_id: (-counts[algorithm_id], algorithm_id)) if counts else None def _score_pair(track_frame: dict, frame: dict, track_vector, frame_vector) -> dict[str, float | None]: @@ -219,7 +222,7 @@ def rank_merge_candidates( place of the track's frames. Raises ``DetectionNotInOccurrence`` when it is not one of the occurrence's frames. """ - from ami.main.models import Classification, Detection, get_media_url + from ami.main.models import Detection, get_media_url config = tracking_config_for(occurrence) track = Detection.objects.valid().filter(occurrence_id=occurrence.pk) @@ -279,31 +282,17 @@ def rank_merge_candidates( for candidate, relation, _ in scored } - track_vectors = _latest_vectors( - Classification.objects.filter( - detection_id__in={track_frame["pk"] for track_frame, _ in pairs.values()}, - algorithm_id__isnull=False, - features_2048__isnull=False, - ) - ) - frame_vectors: dict[tuple[int, int], Any] = {} - if track_vectors: - frame_vectors = _latest_vectors( - Classification.objects.filter( - detection_id__in=[frame["pk"] for _, frame in pairs.values()], - algorithm_id__in={algorithm_id for _, algorithm_id in track_vectors}, - features_2048__isnull=False, - ) - ) - vector_by_track_frame = { - detection_id: (algorithm_id, vector) for (detection_id, algorithm_id), vector in track_vectors.items() - } + track_vectors = latest_vectors({track_frame["pk"] for track_frame, _ in pairs.values()}) + algorithm_id = _comparison_algorithm(track_vectors) + frame_vectors: dict[int, Any] = {} + if algorithm_id is not None: + frame_vectors = vectors_for_detections([frame["pk"] for _, frame in pairs.values()], algorithm_id) rows: list[dict[str, Any]] = [] for candidate, relation, offset in scored: track_frame, frame = pairs[candidate.pk] - algorithm_id, track_vector = vector_by_track_frame.get(track_frame["pk"], (None, None)) - frame_vector = frame_vectors.get((frame["pk"], algorithm_id)) if algorithm_id is not None else None + track_vector = track_vectors.get((track_frame["pk"], algorithm_id)) + frame_vector = frame_vectors.get(frame["pk"]) scores = _score_pair(track_frame, frame, track_vector, frame_vector) crop = frame["path"] or next((f["path"] for f in frames_by_occurrence[candidate.pk] if f["path"]), None) rows.append( @@ -407,11 +396,10 @@ def match_capture_detections(occurrence: Occurrence, capture: SourceImage) -> di such a pair of captures. The query count is fixed, whatever the track's length or the number of boxes. """ - from ami.main.models import Classification, Detection, SourceImage + from ami.main.models import Detection, SourceImage from ami.ml.models import Algorithm from ami.ml.post_processing.tracking_task import ( image_diagonal, - latest_feature_vectors, resolve_feature_algorithm, select_links, ) @@ -429,21 +417,13 @@ def match_capture_detections(occurrence: Occurrence, capture: SourceImage) -> di if reference is not None and any(box.occurrence_id == occurrence.pk for box in boxes): relation = RELATION_SAME - # Tracking takes the one extractor with embeddings anywhere in the session, which scans + # Tracking takes the one extractor with vectors anywhere in the session, which scans # every classification (about 40 ms). The two captures being paired give the same answer # unless a session mixes extractors. detection_ids = [detection.pk for detection in detections] - extractor_ids = set( - Classification.objects.filter( - detection_id__in=detection_ids, features_2048__isnull=False, algorithm_id__isnull=False - ) - .order_by() - .values_list("algorithm_id", flat=True) - .distinct() - ) - extractors = list(Algorithm.objects.filter(pk__in=extractor_ids)) + extractors = list(Algorithm.objects.filter(pk__in=algorithm_ids_with_vectors(pk__in=detection_ids))) algorithm, _, _ = resolve_feature_algorithm(occurrence.event, config, candidates=extractors) - vectors = latest_feature_vectors(detection_ids, algorithm.pk) if algorithm is not None else {} + vectors = vectors_for_detections(detection_ids, algorithm.pk) if algorithm is not None else {} def skipped(detection_id: int) -> str | None: return SKIPPED_NO_VECTOR if config.require_features and detection_id not in vectors else None diff --git a/ami/main/tests.py b/ami/main/tests.py index b12a016fd..ed52eec04 100644 --- a/ami/main/tests.py +++ b/ami/main/tests.py @@ -29,6 +29,7 @@ Classification, Deployment, Detection, + DetectionEmbedding, Device, Event, Identification, @@ -10247,18 +10248,58 @@ def test_track_edits_ignore_the_project_default_filters(self): self.assertEqual(merged.data["detections_count"], len(self.detections) + 2) self.assertFalse(Occurrence.objects.filter(pk__in=[low_score.pk, undetermined.pk]).exists()) + def test_a_candidate_whose_only_vector_is_an_embedding_is_compared_by_appearance(self): + """A crop the moth/non-moth filter rejected has no classification vector, only an embedding.""" + extractor = Algorithm.objects.create(name="Feature extractor", key="feature-extractor") + vector = [1.0] + [0.0] * 2047 + self._give_target_vectors(vector, extractor) + candidate = self._make_occurrence([self.after_capture], bbox=[12, 12, 42, 42]) + DetectionEmbedding.objects.create( + detection=candidate.detections.get(), algorithm=extractor, features_2048=vector + ) + + response = self.get_candidates() + self.assertEqual(response.status_code, 200, response.data) + similarity = {row["id"]: row["similarity"] for row in response.data["candidates"]} + self.assertEqual(similarity[candidate.pk], 1.0) + + def test_similarity_is_never_taken_between_two_algorithms(self): + """Vectors from two models are not comparable, however alike their numbers.""" + extractor = Algorithm.objects.create(name="Feature extractor", key="feature-extractor") + other = Algorithm.objects.create(name="Other feature extractor", key="other-feature-extractor") + vector = [1.0] + [0.0] * 2047 + self._give_target_vectors(vector, extractor) + same = self._make_occurrence([self.after_capture], bbox=[12, 12, 42, 42]) + different = self._make_occurrence([self.after_capture], bbox=[500, 500, 530, 530]) + DetectionEmbedding.objects.create(detection=same.detections.get(), algorithm=extractor, features_2048=vector) + DetectionEmbedding.objects.create(detection=different.detections.get(), algorithm=other, features_2048=vector) + + response = self.get_candidates() + self.assertEqual(response.status_code, 200, response.data) + similarity = {row["id"]: row["similarity"] for row in response.data["candidates"]} + self.assertEqual(similarity, {same.pk: 1.0, different.pk: None}) + def test_the_candidate_count_does_not_change_the_query_count(self): extractor = Algorithm.objects.create(name="Feature extractor", key="feature-extractor") vector = [1.0] + [0.0] * 2047 self._give_target_vectors(vector, extractor) for offset in range(3): - self._make_occurrence( - [self.after_capture], bbox=[10 + offset, 10, 40 + offset, 40], vector=vector, algorithm=extractor + # Alternate the two places a vector is stored. + candidate = self._make_occurrence( + [self.after_capture], + bbox=[10 + offset, 10, 40 + offset, 40], + vector=vector if offset % 2 else None, + algorithm=extractor, ) + if not offset % 2: + DetectionEmbedding.objects.create( + detection=candidate.detections.get(), algorithm=extractor, features_2048=vector + ) # The savepoint pair, the object lookup with its permission checks, then the # seven ranking queries: the track's frames, the capture ids before and after - # it, the frames in those captures, the candidates, and the two vector sides. + # it, the frames in those captures, the candidates, and the two vector sides, + # each reading embeddings and classification vectors together. with self.assertNumQueries(13): response = self.get_candidates() @@ -10513,6 +10554,20 @@ def test_a_box_without_an_embedding_is_skipped_while_features_are_required(self) self.assertAlmostEqual(row["cost"], geometry, places=4) self.assertAlmostEqual(row["likelihood"], 1 - geometry / 3, places=4) + def test_a_box_whose_only_vector_is_an_embedding_is_compared_and_linked(self): + """A processing service can store a vector for a box that received no classification + vector. The preview reads the embedding store too, so such a box is compared by + appearance and linked like one whose vector came with a classification.""" + track = self._track(self.captures[1:3], vector=self.VECTOR) + box = self._box(self.captures[3], self.NEAR_BOX) + DetectionEmbedding.objects.create(detection=box, algorithm=self.extractor, features_2048=self.VECTOR) + + data = self.get_matches(track, self.captures[3].pk).data + + self.assertEqual(data["feature_algorithm_id"], self.extractor.pk) + row = data["detections"][0] + self.assertEqual((row["skipped_reason"], row["would_link"], row["similarity"]), (None, True, 1.0)) + def test_a_detector_only_session_links_nothing(self): """With no embeddings at all, tracking has no extractor to compare and skips the captures while it requires features, so every box and the reference frame are marked skipped.""" @@ -11108,6 +11163,117 @@ def make_detection(vectors: list[bool], bbox: list[int] | None = [10, 10, 40, 40 listed, _ = self._get(f"/api/v2/captures/?project_id={self.project.pk}") self.assertNotIn("detections_with_features", listed.data["results"][0], "Counted on the detail only") + def test_a_frame_whose_only_vector_is_an_embedding_counts_as_having_one(self): + """A crop the moth/non-moth filter rejected carries an embedding and no classification vector.""" + extractor = Algorithm.objects.create(name="Embedding model", key="embedding-model") + occurrence = self._make_occurrence([True, False, False]) + embedded = occurrence.detections.get(source_image=self.captures[1]) + DetectionEmbedding.objects.create(detection=embedded, algorithm=extractor, features_2048=self.vector) + + response, queries = self._get(self._occurrence_url(occurrence)) + self.assertEqual(response.data["grouping_summary"]["frames_with_vectors"], 2) + self.assertFalse(self._reads_the_vector(queries)) + + capture, queries = self._get(f"/api/v2/captures/{self.captures[1].pk}/?project_id={self.project.pk}") + self.assertEqual(capture.data["detections_with_features"], 1) + self.assertFalse(self._reads_the_vector(queries)) + + def test_has_features_counts_an_embedding_only_for_the_classifications_own_algorithm(self): + """A vector from one model says nothing about another model's classification of the crop.""" + embedder = Algorithm.objects.create(name="Species classifier", key="species-classifier") + moth_filter = Algorithm.objects.create(name="Moth filter", key="moth-filter") + capture = self.captures[0] + detection = Detection.objects.create(source_image=capture, timestamp=capture.timestamp, bbox=[10, 10, 40, 40]) + for algorithm in (embedder, moth_filter): + detection.classifications.create( + taxon=self.taxon, score=0.9, timestamp=capture.timestamp, algorithm=algorithm + ) + DetectionEmbedding.objects.create(detection=detection, algorithm=embedder, features_2048=self.vector) + + flags = dict( + Classification.objects.filter(detection=detection) + .with_has_features() + .values_list("algorithm_id", "has_features") + ) + self.assertEqual(flags, {embedder.pk: True, moth_filter.pk: False}) + + +class DetectionVectorReadTestCase(TestCase): + """Reading a detection's feature vector from whichever of the two stores holds it. + + A vector is a detection embedding or, on data processed before embeddings existed, a + classification's ``features_2048``. What these pin is that a reader returns only the + requested algorithm's vectors, takes the embedding when both exist, and reads any + number of detections in one query. + """ + + def setUp(self) -> None: + self.project, self.deployment = setup_test_project(reuse=False) + self.capture = SourceImage.objects.create( + deployment=self.deployment, + project=self.project, + timestamp=datetime.datetime(2024, 1, 1, 22, 0), + path="test/vectors.jpg", + ) + self.extractor = Algorithm.objects.create(name="Embedding model", key="embedding-model") + self.other = Algorithm.objects.create(name="Other embedding model", key="other-embedding-model") + + def _detection(self, x: int) -> Detection: + return Detection.objects.create( + source_image=self.capture, timestamp=self.capture.timestamp, bbox=[x, 10, x + 30, 40] + ) + + def _classification_vector(self, detection: Detection, algorithm: Algorithm, value: float) -> None: + detection.classifications.create( + score=0.9, timestamp=self.capture.timestamp, algorithm=algorithm, features_2048=[value] * 2048 + ) + + def _embedding(self, detection: Detection, algorithm: Algorithm, value: float) -> None: + DetectionEmbedding.objects.create(detection=detection, algorithm=algorithm, features_2048=[value] * 2048) + + @staticmethod + def _read(detections: list[Detection], algorithm: Algorithm) -> dict[int, float]: + """{detection id: first component of its vector}; the test vectors are constant.""" + from ami.main.models_future.embeddings import vectors_for_detections + + vectors = vectors_for_detections([d.pk for d in detections], algorithm.pk) + return {detection_id: float(vector[0]) for detection_id, vector in vectors.items()} + + def test_each_store_supplies_the_vectors_it_holds(self): + """Older detections have only a classification vector and newer ones an embedding.""" + old, new = self._detection(0), self._detection(100) + self._classification_vector(old, self.extractor, 0.25) + self._embedding(new, self.extractor, 0.75) + self.assertEqual(self._read([old, new], self.extractor), {old.pk: 0.25, new.pk: 0.75}) + + def test_an_embedding_is_preferred_to_a_classification_vector(self): + detection = self._detection(0) + self._classification_vector(detection, self.extractor, 0.25) + self._embedding(detection, self.extractor, 0.75) + self.assertEqual(self._read([detection], self.extractor), {detection.pk: 0.75}) + + def test_another_algorithms_vector_is_never_returned(self): + first, second = self._detection(0), self._detection(100) + self._embedding(first, self.extractor, 0.25) + self._embedding(second, self.other, 0.75) + self._classification_vector(second, self.other, 0.5) + self.assertEqual(self._read([first, second], self.extractor), {first.pk: 0.25}) + self.assertEqual(self._read([first, second], self.other), {second.pk: 0.75}) + + def test_any_number_of_detections_is_read_in_one_query(self): + from cachalot.api import cachalot_disabled + + from ami.main.models_future.embeddings import vectors_for_detections + + detections = [self._detection(x) for x in range(0, 500, 100)] + for index, detection in enumerate(detections): + store = self._embedding if index % 2 else self._classification_vector + store(detection, self.extractor, 0.5) + + with cachalot_disabled(), self.assertNumQueries(1): + vectors = vectors_for_detections([d.pk for d in detections], self.extractor.pk) + self.assertEqual(len(vectors), len(detections)) + def test_the_capture_counts_ride_on_the_capture_row(self): """Both counts are subqueries on the capture's own SELECT: five detections, one query.""" from cachalot.api import cachalot_disabled diff --git a/ami/ml/post_processing/admin_forms.py b/ami/ml/post_processing/admin_forms.py index 41bac52b3..72494365d 100644 --- a/ami/ml/post_processing/admin_forms.py +++ b/ami/ml/post_processing/admin_forms.py @@ -13,29 +13,21 @@ from django import forms from django.db.models import QuerySet -from ami.main.models import Classification, Event +from ami.main.models import Event +from ami.main.models_future.embeddings import algorithm_ids_with_vectors from ami.ml.models import Algorithm from ami.ml.post_processing.tracking_task import DEFAULT_TRACKING_PARAMS def _feature_algorithm_choices_for_events(events: QuerySet[Event]) -> list[tuple[int, str]]: - """Algorithms that produced ``features_2048`` on the given events. + """Algorithms that stored a feature vector, as an embedding or on a classification, + for detections in the given events. Scoped to the operator's selection so the dropdown stays bounded on production-sized DBs and never reveals algorithms from other projects. """ - algorithm_ids = ( - Classification.objects.filter( - detection__source_image__event__in=events, - features_2048__isnull=False, - algorithm_id__isnull=False, - ) - .values_list("algorithm_id", flat=True) - .distinct() - ) - return [ - (a.pk, f"{a.name} (#{a.pk})") for a in Algorithm.objects.filter(pk__in=list(algorithm_ids)).order_by("name") - ] + algorithm_ids = algorithm_ids_with_vectors(source_image__event__in=events) + return [(a.pk, f"{a.name} (#{a.pk})") for a in Algorithm.objects.filter(pk__in=algorithm_ids).order_by("name")] class TrackingActionForm(forms.Form): diff --git a/ami/ml/post_processing/tests/test_tracking_task.py b/ami/ml/post_processing/tests/test_tracking_task.py index daad78656..ee93062ae 100644 --- a/ami/ml/post_processing/tests/test_tracking_task.py +++ b/ami/ml/post_processing/tests/test_tracking_task.py @@ -11,6 +11,7 @@ from ami.main.models import ( Classification, Detection, + DetectionEmbedding, Event, Identification, Occurrence, @@ -25,6 +26,7 @@ assign_occurrences_by_tracking_images, assign_occurrences_from_detection_chains, event_is_fresh, + pair_detections, ) from ami.tests.fixtures.images import generate_moth_series from ami.tests.fixtures.main import create_captures, create_occurrences, create_taxa, setup_test_project @@ -320,6 +322,80 @@ def test_requiring_features_leaves_data_untouched(self): ) +class TestTrackingWithEmbeddings(TestCase): + """Tracking reads a detection's vector from its embedding when it has one. + + Every detection can carry an embedding, including those the moth/non-moth filter + rejected, which have no classification vector. What these pin is that tracking finds + and uses those vectors, that it still pairs them with the classification vectors older + data has, and that it never compares vectors from two algorithms. + """ + + def setUp(self) -> None: + self.project, self.deployment = setup_test_project(reuse=False) + create_captures(deployment=self.deployment, num_nights=1, images_per_night=5, interval_minutes=1) + create_taxa(self.project) + # One detection per capture, all with the same box, so geometry never separates them. + create_occurrences(deployment=self.deployment, num=5) + + self.event = self.project.events.first() + assert self.event is not None + self.source_images = list(self.event.captures.order_by("timestamp")) + _give_captures_dimensions(self.source_images) + self.extractor = Algorithm.objects.create(name="Embedding model", key="embedding-model") + self.vector = [1.0] + [0.0] * 2047 + + def _detections(self, index: int) -> list[Detection]: + return list(self.source_images[index].detections.valid()) + + def _embed(self, detections, algorithm: Algorithm) -> None: + DetectionEmbedding.objects.bulk_create( + [DetectionEmbedding(detection=d, algorithm=algorithm, features_2048=self.vector) for d in detections] + ) + + def _pair(self, current: list[Detection], following: list[Detection]) -> int: + """Links made between two captures when both sides need a vector from the extractor.""" + return pair_detections( + current, + following, + 4096, + 2160, + cost_threshold=0.5, + algorithm=self.extractor, + logger=logger, + require_features=True, + ) + + def test_detections_whose_only_vector_is_an_embedding_are_tracked(self): + """The run finds the embedding model by itself and, requiring vectors, links a detection + only when it read one for it.""" + self._embed(Detection.objects.valid().filter(source_image__event=self.event), self.extractor) + + TrackingTask(logger=logger, event_ids=[self.event.pk], require_features=True, cost_threshold=0.5).run() + + linked = Detection.objects.filter(source_image__event=self.event, next_detection__isnull=False).count() + self.assertEqual(linked, len(self.source_images) - 1) + + def test_vectors_from_another_algorithm_are_never_compared(self): + other = Algorithm.objects.create(name="Other embedding model", key="other-embedding-model") + current, following = self._detections(0), self._detections(1) + self._embed(current, self.extractor) + self._embed(following, other) + self.assertEqual(self._pair(current, following), 0) + + self._embed(following, self.extractor) + self.assertEqual(self._pair(current, following), 1) + + def test_an_embedding_is_compared_with_a_classification_vector_from_the_same_algorithm(self): + """A session processed partly before embeddings existed still links across the boundary.""" + current, following = self._detections(0), self._detections(1) + Classification.objects.filter(detection__in=current).update( + algorithm=self.extractor, features_2048=self.vector + ) + self._embed(following, self.extractor) + self.assertEqual(self._pair(current, following), 1) + + class TestFreshEventGuard(TestCase): """The guard refuses already-grouped sessions, and only those. diff --git a/ami/ml/post_processing/tracking_task.py b/ami/ml/post_processing/tracking_task.py index 92b243fab..557102936 100644 --- a/ami/ml/post_processing/tracking_task.py +++ b/ami/ml/post_processing/tracking_task.py @@ -7,7 +7,7 @@ import numpy as np import pydantic from django.db import transaction -from django.db.models import Count +from django.db.models import Count, Exists, OuterRef from django.utils import timezone from ami.main.models import ( @@ -20,6 +20,7 @@ SourceImageCollection, update_calculated_fields_for_sessions_and_stations, ) +from ami.main.models_future.embeddings import algorithm_ids_with_vectors, vectors_for_detections from ami.main.models_future.track_stats import refresh_track_stats_for_ids from ami.main.models_future.tracks import clear_grouping_verification from ami.ml.models import Algorithm @@ -129,21 +130,14 @@ def get_unique_feature_algorithm_for_event(event: Event) -> tuple[Algorithm | No """ Return ``(unique_algorithm, all_candidates)``. - If exactly one feature-extraction algorithm produced ``features_2048`` for this - event, returns that algorithm and a single-element list. Otherwise returns - ``(None, candidates)`` so the caller can either skip with a warning or require - the operator to pass an explicit ``feature_extraction_algorithm_id``. + If exactly one feature-extraction algorithm stored vectors (embeddings or + classification ``features_2048``) for this event, returns that algorithm and a + single-element list. Otherwise returns ``(None, candidates)`` so the caller can + either skip with a warning or require the operator to pass an explicit + ``feature_extraction_algorithm_id``. """ - algo_ids = ( - Classification.objects.filter( - detection__source_image__event=event, - features_2048__isnull=False, - algorithm_id__isnull=False, - ) - .values_list("algorithm_id", flat=True) - .distinct() - ) - candidates = list(Algorithm.objects.filter(pk__in=list(algo_ids))) + algo_ids = algorithm_ids_with_vectors(source_image__event=event) + candidates = list(Algorithm.objects.filter(pk__in=algo_ids)) if len(candidates) == 1: return candidates[0], candidates return None, candidates @@ -218,14 +212,9 @@ def event_is_fresh(event: Event) -> tuple[bool, str]: def event_fully_processed(event: Event, logger: logging.Logger, algorithm: Algorithm) -> bool: total = event.captures.count() - processed = ( - event.captures.filter( - detections__classifications__features_2048__isnull=False, - detections__classifications__algorithm=algorithm, - ) - .distinct() - .count() - ) + processed = event.captures.filter( + Exists(Detection.objects.has_vector(algorithm).filter(source_image_id=OuterRef("pk"))) + ).count() if processed < total: logger.info(f"Event {event.pk} not fully processed: {processed}/{total} captures") return False @@ -420,24 +409,6 @@ def nothing_tracked_summary(skip_reasons: collections.Counter[str]) -> str: return f"Nothing was tracked: {total} session(s) skipped ({reasons})." -def latest_feature_vectors(detection_ids: Iterable[int], algorithm_id: int) -> dict[int, typing.Any]: - """The most recent embedding from one algorithm for each detection given, by detection id. - - Detections without one are left out. One query for the whole batch. - """ - vectors: dict[int, typing.Any] = {} - rows = ( - Classification.objects.filter( - detection_id__in=list(detection_ids), algorithm_id=algorithm_id, features_2048__isnull=False - ) - .order_by("-timestamp", "-pk") - .values_list("detection_id", "features_2048") - ) - for detection_id, vector in rows: - vectors.setdefault(detection_id, vector) - return vectors - - def select_links( current_detections: Sequence[Detection], next_detections: Sequence[Detection], @@ -493,7 +464,7 @@ def select_transition_links( """The links tracking makes between two adjacent captures, reading embeddings but saving nothing.""" vectors: dict[int, typing.Any] = {} if algorithm is not None: - vectors = latest_feature_vectors([det.pk for det in [*current_detections, *next_detections]], algorithm.pk) + vectors = vectors_for_detections([det.pk for det in [*current_detections, *next_detections]], algorithm.pk) return select_links( current_detections, next_detections, From 33f4144b9ec5503a407b40e0d38338f571261232 Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Wed, 23 Sep 2026 01:18:14 -0700 Subject: [PATCH 03/44] fix(exports): count a detection embedding as a feature vector in the tracks CSV The has_feature_vector column only looked at classification vectors, so a detection whose only vector is an embedding, such as a crop the moth filter rejected, was reported as having none. It now uses the same rule as Detection.objects.has_vector(). Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01C7Xf6VPbwWtTumhjjF15g8 --- ami/exports/tests.py | 14 ++++++++++++++ ami/exports/tracks.py | 12 +++++++----- 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/ami/exports/tests.py b/ami/exports/tests.py index 92ae87a71..2272c2f08 100644 --- a/ami/exports/tests.py +++ b/ami/exports/tests.py @@ -687,6 +687,20 @@ def test_feature_vector_and_next_detection(self): if pgvector_is_available(): self.assertEqual(rows[detections[0].pk]["has_feature_vector"], "true") + def test_an_embedding_alone_counts_as_a_feature_vector(self): + from ami.main.models import DetectionEmbedding + from ami.ml.models import Algorithm + from ami.tests.fixtures.tracking import pgvector_is_available + + if not pgvector_is_available(): + self.skipTest("This database cannot store embeddings.") + detection = self.occurrences[0].detections.order_by("source_image__timestamp").last() + algorithm = Algorithm.objects.create(name="Embedding model", key="embedding-model") + DetectionEmbedding.objects.create(detection=detection, algorithm=algorithm, features_2048=[0.1] * 2048) + + rows = {int(row["detection_id"]): row for row in self._rows()} + self.assertEqual(rows[detection.pk]["has_feature_vector"], "true") + def test_query_count_is_one_pair_per_chunk(self): from django.db import connection from django.test.utils import CaptureQueriesContext diff --git a/ami/exports/tracks.py b/ami/exports/tracks.py index 76150c2ef..2d36c1ff1 100644 --- a/ami/exports/tracks.py +++ b/ami/exports/tracks.py @@ -12,9 +12,9 @@ from collections.abc import Callable, Iterator from django.db import models -from django.db.models import Exists, OuterRef, Subquery +from django.db.models import Exists, ExpressionWrapper, OuterRef, Subquery -from ami.main.models import BEST_MACHINE_PREDICTION_ORDER, Classification, Detection, Occurrence +from ami.main.models import BEST_MACHINE_PREDICTION_ORDER, Classification, Detection, DetectionEmbedding, Occurrence from ami.main.models_future.tracks import CAPTURE_ORDER TRACKS_CSV_COLUMNS: typing.Final = ( @@ -67,9 +67,11 @@ def _detections_for(occurrence_ids: list[int]) -> models.QuerySet: .annotate( label=Subquery(best_classification.values("taxon__name")[:1]), label_score=Subquery(best_classification.values("score")[:1]), - # Same notion as ClassificationQuerySet.with_has_features(), per detection. - has_feature_vector=Exists( - Classification.objects.filter(detection=OuterRef("pk"), features_2048__isnull=False) + # Same notion as DetectionQuerySet.has_vector(): an embedding or a classification vector. + has_feature_vector=ExpressionWrapper( + Exists(DetectionEmbedding.objects.filter(detection=OuterRef("pk"))) + | Exists(Classification.objects.filter(detection=OuterRef("pk"), features_2048__isnull=False)), + output_field=models.BooleanField(), ), ) .order_by("occurrence_id", *CAPTURE_ORDER) From 4eae25ed88a84802813d39bfcdc4b4a1153930c5 Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Wed, 23 Sep 2026 01:21:28 -0700 Subject: [PATCH 04/44] feat(embeddings): name the stored vector "vector" and record the job that stored it DetectionEmbedding.features_2048 is renamed to vector, so the embedding table no longer shares a column name with Classification.features_2048, which stays as it is. Each embedding now points at the job whose results stored it; saving the same detection again from another job moves the pointer, and deleting the job clears it without removing the vector. The change is folded into the 0102 migration, which has not been applied anywhere shared. See #1431. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01C7Xf6VPbwWtTumhjjF15g8 --- ami/exports/tests.py | 2 +- .../migrations/0102_detection_embedding.py | 13 ++++++++++- ami/main/models.py | 4 +++- ami/main/models_future/embeddings.py | 9 ++++---- ami/main/tests.py | 18 +++++++-------- ami/ml/models/pipeline.py | 8 ++++--- .../tests/test_tracking_task.py | 2 +- ami/ml/tests.py | 22 ++++++++++++++++--- 8 files changed, 53 insertions(+), 25 deletions(-) diff --git a/ami/exports/tests.py b/ami/exports/tests.py index 2272c2f08..509c39b83 100644 --- a/ami/exports/tests.py +++ b/ami/exports/tests.py @@ -696,7 +696,7 @@ def test_an_embedding_alone_counts_as_a_feature_vector(self): self.skipTest("This database cannot store embeddings.") detection = self.occurrences[0].detections.order_by("source_image__timestamp").last() algorithm = Algorithm.objects.create(name="Embedding model", key="embedding-model") - DetectionEmbedding.objects.create(detection=detection, algorithm=algorithm, features_2048=[0.1] * 2048) + DetectionEmbedding.objects.create(detection=detection, algorithm=algorithm, vector=[0.1] * 2048) rows = {int(row["detection_id"]): row for row in self._rows()} self.assertEqual(rows[detection.pk]["has_feature_vector"], "true") diff --git a/ami/main/migrations/0102_detection_embedding.py b/ami/main/migrations/0102_detection_embedding.py index 17a843648..ba95853a5 100644 --- a/ami/main/migrations/0102_detection_embedding.py +++ b/ami/main/migrations/0102_detection_embedding.py @@ -10,6 +10,7 @@ class Migration(migrations.Migration): dependencies = [ + ("jobs", "0023_alter_job_job_type_key"), ("ml", "0028_normalize_empty_endpoint_url_to_null"), ("main", "0101_grant_run_post_processing_to_ml_data_manager"), ] @@ -22,7 +23,7 @@ class Migration(migrations.Migration): ("created_at", models.DateTimeField(auto_now_add=True)), ("updated_at", models.DateTimeField(auto_now=True)), ( - "features_2048", + "vector", pgvector.django.vector.VectorField( dimensions=2048, help_text="Feature embedding from the model backbone" ), @@ -44,6 +45,16 @@ class Migration(migrations.Migration): to="main.detection", ), ), + ( + "job", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="+", + to="jobs.job", + ), + ), ], ), migrations.AddConstraint( diff --git a/ami/main/models.py b/ami/main/models.py index 534fcec9a..5781d29aa 100644 --- a/ami/main/models.py +++ b/ami/main/models.py @@ -3517,10 +3517,12 @@ class DetectionEmbedding(BaseModel): # No separate index: the unique constraint's index leads with detection_id. detection = models.ForeignKey(Detection, on_delete=models.CASCADE, related_name="embeddings", db_index=False) algorithm = models.ForeignKey("ml.Algorithm", on_delete=models.CASCADE, related_name="detection_embeddings") - features_2048 = pgvector.django.VectorField( + vector = pgvector.django.VectorField( dimensions=2048, help_text="Feature embedding from the model backbone", ) + # The job whose results stored this vector; kept when the job is deleted, since the vector stays valid. + job = models.ForeignKey("jobs.Job", on_delete=models.SET_NULL, null=True, blank=True, related_name="+") class Meta: constraints = [ diff --git a/ami/main/models_future/embeddings.py b/ami/main/models_future/embeddings.py index e9965c658..d82991e2f 100644 --- a/ami/main/models_future/embeddings.py +++ b/ami/main/models_future/embeddings.py @@ -33,18 +33,17 @@ def _vector_rows(detection_ids: Iterable[int], algorithm_ids: Iterable[int] | No embeddings = embeddings.filter(algorithm_id__in=algorithm_ids) classifications = classifications.filter(algorithm_id__in=algorithm_ids) - # Model fields first, then the annotations in the same order, so both SELECT lists - # line up column for column. UNION ALL: de-duplicating would sort the vectors. - columns = ("detection_id", "algorithm_id", "features_2048", "id", "preference", "recorded_at") + # Model fields first, then the annotations in the same order, so both SELECT lists line up + # column for column (the vector columns differ in name). UNION ALL: de-duplicating would sort the vectors. embeddings = ( embeddings.order_by() .annotate(preference=Value(_PREFER_EMBEDDING, output_field=IntegerField()), recorded_at=F("updated_at")) - .values_list(*columns) + .values_list("detection_id", "algorithm_id", "vector", "id", "preference", "recorded_at") ) classifications = ( classifications.order_by() .annotate(preference=Value(_PREFER_CLASSIFICATION, output_field=IntegerField()), recorded_at=F("timestamp")) - .values_list(*columns) + .values_list("detection_id", "algorithm_id", "features_2048", "id", "preference", "recorded_at") ) return embeddings.union(classifications, all=True).order_by("preference", "-recorded_at", "-id") diff --git a/ami/main/tests.py b/ami/main/tests.py index ed52eec04..ac5422d98 100644 --- a/ami/main/tests.py +++ b/ami/main/tests.py @@ -10254,9 +10254,7 @@ def test_a_candidate_whose_only_vector_is_an_embedding_is_compared_by_appearance vector = [1.0] + [0.0] * 2047 self._give_target_vectors(vector, extractor) candidate = self._make_occurrence([self.after_capture], bbox=[12, 12, 42, 42]) - DetectionEmbedding.objects.create( - detection=candidate.detections.get(), algorithm=extractor, features_2048=vector - ) + DetectionEmbedding.objects.create(detection=candidate.detections.get(), algorithm=extractor, vector=vector) response = self.get_candidates() self.assertEqual(response.status_code, 200, response.data) @@ -10271,8 +10269,8 @@ def test_similarity_is_never_taken_between_two_algorithms(self): self._give_target_vectors(vector, extractor) same = self._make_occurrence([self.after_capture], bbox=[12, 12, 42, 42]) different = self._make_occurrence([self.after_capture], bbox=[500, 500, 530, 530]) - DetectionEmbedding.objects.create(detection=same.detections.get(), algorithm=extractor, features_2048=vector) - DetectionEmbedding.objects.create(detection=different.detections.get(), algorithm=other, features_2048=vector) + DetectionEmbedding.objects.create(detection=same.detections.get(), algorithm=extractor, vector=vector) + DetectionEmbedding.objects.create(detection=different.detections.get(), algorithm=other, vector=vector) response = self.get_candidates() self.assertEqual(response.status_code, 200, response.data) @@ -10293,7 +10291,7 @@ def test_the_candidate_count_does_not_change_the_query_count(self): ) if not offset % 2: DetectionEmbedding.objects.create( - detection=candidate.detections.get(), algorithm=extractor, features_2048=vector + detection=candidate.detections.get(), algorithm=extractor, vector=vector ) # The savepoint pair, the object lookup with its permission checks, then the @@ -10560,7 +10558,7 @@ def test_a_box_whose_only_vector_is_an_embedding_is_compared_and_linked(self): appearance and linked like one whose vector came with a classification.""" track = self._track(self.captures[1:3], vector=self.VECTOR) box = self._box(self.captures[3], self.NEAR_BOX) - DetectionEmbedding.objects.create(detection=box, algorithm=self.extractor, features_2048=self.VECTOR) + DetectionEmbedding.objects.create(detection=box, algorithm=self.extractor, vector=self.VECTOR) data = self.get_matches(track, self.captures[3].pk).data @@ -11168,7 +11166,7 @@ def test_a_frame_whose_only_vector_is_an_embedding_counts_as_having_one(self): extractor = Algorithm.objects.create(name="Embedding model", key="embedding-model") occurrence = self._make_occurrence([True, False, False]) embedded = occurrence.detections.get(source_image=self.captures[1]) - DetectionEmbedding.objects.create(detection=embedded, algorithm=extractor, features_2048=self.vector) + DetectionEmbedding.objects.create(detection=embedded, algorithm=extractor, vector=self.vector) response, queries = self._get(self._occurrence_url(occurrence)) self.assertEqual(response.data["grouping_summary"]["frames_with_vectors"], 2) @@ -11188,7 +11186,7 @@ def test_has_features_counts_an_embedding_only_for_the_classifications_own_algor detection.classifications.create( taxon=self.taxon, score=0.9, timestamp=capture.timestamp, algorithm=algorithm ) - DetectionEmbedding.objects.create(detection=detection, algorithm=embedder, features_2048=self.vector) + DetectionEmbedding.objects.create(detection=detection, algorithm=embedder, vector=self.vector) flags = dict( Classification.objects.filter(detection=detection) @@ -11229,7 +11227,7 @@ def _classification_vector(self, detection: Detection, algorithm: Algorithm, val ) def _embedding(self, detection: Detection, algorithm: Algorithm, value: float) -> None: - DetectionEmbedding.objects.create(detection=detection, algorithm=algorithm, features_2048=[value] * 2048) + DetectionEmbedding.objects.create(detection=detection, algorithm=algorithm, vector=[value] * 2048) @staticmethod def _read(detections: list[Detection], algorithm: Algorithm) -> dict[int, float]: diff --git a/ami/ml/models/pipeline.py b/ami/ml/models/pipeline.py index 302be4091..d4c0aad0e 100644 --- a/ami/ml/models/pipeline.py +++ b/ami/ml/models/pipeline.py @@ -690,6 +690,7 @@ def create_detection_embeddings( detection_responses: list[DetectionResponse], algorithms_known: dict[str, Algorithm], logger: logging.Logger = logger, + job_id: int | None = None, ) -> list[DetectionEmbedding]: """ Store the feature vectors sent with each detection, one row per (detection, algorithm). @@ -701,7 +702,7 @@ def create_detection_embeddings( Responses are matched to detections by image and box, the key ``get_or_create_detection`` reuses detections by, because ``create_detections`` does not return them in response order. An algorithm key the pipeline has not registered raises ``PipelineNotConfigured``, as it - does for classifications. + does for classifications. ``job_id`` records the job whose results stored each vector. """ by_box = { (str(detection.source_image_id), tuple(detection.bbox)): detection @@ -726,14 +727,14 @@ def create_detection_embeddings( f"Known algorithms: {list(algorithms_known.keys())}" ) from err embeddings[(detection.pk, algorithm.pk)] = DetectionEmbedding( - detection=detection, algorithm=algorithm, features_2048=embedding_resp.features + detection=detection, algorithm=algorithm, vector=embedding_resp.features, job_id=job_id ) DetectionEmbedding.objects.bulk_create( list(embeddings.values()), update_conflicts=True, unique_fields=["detection", "algorithm"], - update_fields=["features_2048", "updated_at"], + update_fields=["vector", "job", "updated_at"], batch_size=EMBEDDING_BATCH_SIZE, ) logger.info(f"Stored {len(embeddings)} detection embeddings for {len(detections)} detections.") @@ -1149,6 +1150,7 @@ def save_results( detection_responses=results.detections, algorithms_known=algorithms_known, logger=job_logger, + job_id=job.pk if job else None, ) classifications = create_classifications( diff --git a/ami/ml/post_processing/tests/test_tracking_task.py b/ami/ml/post_processing/tests/test_tracking_task.py index ee93062ae..78c77c6d7 100644 --- a/ami/ml/post_processing/tests/test_tracking_task.py +++ b/ami/ml/post_processing/tests/test_tracking_task.py @@ -350,7 +350,7 @@ def _detections(self, index: int) -> list[Detection]: def _embed(self, detections, algorithm: Algorithm) -> None: DetectionEmbedding.objects.bulk_create( - [DetectionEmbedding(detection=d, algorithm=algorithm, features_2048=self.vector) for d in detections] + [DetectionEmbedding(detection=d, algorithm=algorithm, vector=self.vector) for d in detections] ) def _pair(self, current: list[Detection], following: list[Detection]) -> int: diff --git a/ami/ml/tests.py b/ami/ml/tests.py index d2cbd83d0..aad4786dd 100644 --- a/ami/ml/tests.py +++ b/ami/ml/tests.py @@ -2448,7 +2448,7 @@ def _moth(self, image: SourceImage, embeddings=None, box: float = 0.0) -> dict: ] return self._detection(image, labels, embeddings, box) - def _save(self, *detections: dict) -> None: + def _save(self, *detections: dict, job_id: int | None = None) -> None: image_ids = list(dict.fromkeys(d["source_image_id"] for d in detections)) payload = { "pipeline": self.pipeline.slug, @@ -2456,7 +2456,7 @@ def _save(self, *detections: dict) -> None: "source_images": [{"id": image_id, "url": f"test/{image_id}.jpg"} for image_id in image_ids], "detections": list(detections), } - save_results(PipelineResultsResponse.parse_obj(payload)) + save_results(PipelineResultsResponse.parse_obj(payload), job_id=job_id) @staticmethod def _stored(image: SourceImage) -> dict[tuple[float, str], list[float]]: @@ -2464,7 +2464,7 @@ def _stored(image: SourceImage) -> dict[tuple[float, str], list[float]]: rows = DetectionEmbedding.objects.filter(detection__source_image=image).select_related( "detection", "algorithm" ) - return {(row.detection.bbox[0], row.algorithm.key): row.features_2048.tolist() for row in rows} + return {(row.detection.bbox[0], row.algorithm.key): row.vector.tolist() for row in rows} def test_every_detection_stores_one_vector_per_algorithm(self): """Including the rejected crop, which has no species classification that could carry one.""" @@ -2501,6 +2501,22 @@ def test_saving_again_keeps_one_row_per_detection_and_algorithm(self): self._save(self._rejected(image, _embedding_payload(self.HIGH))) self.assertEqual(self._stored(image), {(0.0, EMBEDDING_SPECIES.key): self.HIGH}) + def test_each_vector_records_the_job_that_last_stored_it_and_outlives_that_job(self): + from ami.jobs.models import Job + + image = self._image() + first, second = ( + Job.objects.create(project=self.project, name=f"Embedding job {n}", pipeline=self.pipeline) for n in (1, 2) + ) + self._save(self._rejected(image, _embedding_payload(self.LOW)), job_id=first.pk) + self._save(self._rejected(image, _embedding_payload(self.HIGH)), job_id=second.pk) + self.assertEqual(DetectionEmbedding.objects.get(detection__source_image=image).job_id, second.pk) + + second.delete() + embedding = DetectionEmbedding.objects.get(detection__source_image=image) + self.assertIsNone(embedding.job_id) + self.assertEqual(embedding.vector.tolist(), self.HIGH) + def test_a_vector_lands_on_its_own_detection_when_some_detections_already_exist(self): """Detection creation returns existing detections ahead of new ones, so pairing responses with detections by position would swap these two vectors.""" From e33406bea292388e93ef69dd6dd61d7c925ab21e Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Wed, 23 Sep 2026 01:41:38 -0700 Subject: [PATCH 05/44] test: move the capture feature-count test back into its own test case The merge appended the test to the vector-reader test case, which has no captures fixture, so it failed with an AttributeError. It belongs with the other feature-presence tests that share its setup. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01C7Xf6VPbwWtTumhjjF15g8 --- ami/main/tests.py | 44 ++++++++++++++++++++++---------------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/ami/main/tests.py b/ami/main/tests.py index ac5422d98..2181cce06 100644 --- a/ami/main/tests.py +++ b/ami/main/tests.py @@ -11195,6 +11195,28 @@ def test_has_features_counts_an_embedding_only_for_the_classifications_own_algor ) self.assertEqual(flags, {embedder.pk: True, moth_filter.pk: False}) + def test_the_capture_counts_ride_on_the_capture_row(self): + """Both counts are subqueries on the capture's own SELECT: five detections, one query.""" + from cachalot.api import cachalot_disabled + + capture = self.captures[0] + occurrence = Occurrence.objects.create(event=self.event, deployment=self.deployment, project=self.project) + for with_vector in [True, False, True, False, True]: + detection = Detection.objects.create( + source_image=capture, timestamp=capture.timestamp, bbox=[10, 10, 40, 40], occurrence=occurrence + ) + detection.classifications.create( + taxon=self.taxon, + score=0.9, + timestamp=capture.timestamp, + features_2048=self.vector if with_vector else None, + ) + + with cachalot_disabled(), self.assertNumQueries(1): + annotated = SourceImage.objects.filter(pk=capture.pk).with_detections_with_features().get() + self.assertEqual(annotated.detections_valid, 5) # type: ignore[attr-defined] + self.assertEqual(annotated.detections_with_features, 3) # type: ignore[attr-defined] + class DetectionVectorReadTestCase(TestCase): """Reading a detection's feature vector from whichever of the two stores holds it. @@ -11271,25 +11293,3 @@ def test_any_number_of_detections_is_read_in_one_query(self): with cachalot_disabled(), self.assertNumQueries(1): vectors = vectors_for_detections([d.pk for d in detections], self.extractor.pk) self.assertEqual(len(vectors), len(detections)) - - def test_the_capture_counts_ride_on_the_capture_row(self): - """Both counts are subqueries on the capture's own SELECT: five detections, one query.""" - from cachalot.api import cachalot_disabled - - capture = self.captures[0] - occurrence = Occurrence.objects.create(event=self.event, deployment=self.deployment, project=self.project) - for with_vector in [True, False, True, False, True]: - detection = Detection.objects.create( - source_image=capture, timestamp=capture.timestamp, bbox=[10, 10, 40, 40], occurrence=occurrence - ) - detection.classifications.create( - taxon=self.taxon, - score=0.9, - timestamp=capture.timestamp, - features_2048=self.vector if with_vector else None, - ) - - with cachalot_disabled(), self.assertNumQueries(1): - annotated = SourceImage.objects.filter(pk=capture.pk).with_detections_with_features().get() - self.assertEqual(annotated.detections_valid, 5) # type: ignore[attr-defined] - self.assertEqual(annotated.detections_with_features, 3) # type: ignore[attr-defined] From 91dbe9a8dbbb571108c53bf250b72d3c1fc3e65b Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Wed, 23 Sep 2026 01:58:52 -0700 Subject: [PATCH 06/44] feat(occurrences): store an occurrence's algorithm results and reviews as history records Add OccurrenceHistoryRecord, one dated entry in an occurrence's history: either the result of an algorithm run or a person's review. Each record carries a kind, a subtype, the job, algorithm and user behind it, and a JSON payload that is validated against a pydantic schema chosen by kind and subtype, both when saved and when built for a bulk insert. Identifications and predictions keep their own tables. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01C7Xf6VPbwWtTumhjjF15g8 --- .../migrations/0103_occurrence_history.py | 73 ++++++++++++++++++ ami/main/models.py | 67 +++++++++++++++++ ami/main/schemas.py | 74 +++++++++++++++++++ ami/main/test_occurrence_history.py | 54 ++++++++++++++ 4 files changed, 268 insertions(+) create mode 100644 ami/main/migrations/0103_occurrence_history.py create mode 100644 ami/main/schemas.py create mode 100644 ami/main/test_occurrence_history.py diff --git a/ami/main/migrations/0103_occurrence_history.py b/ami/main/migrations/0103_occurrence_history.py new file mode 100644 index 000000000..a71c3d66d --- /dev/null +++ b/ami/main/migrations/0103_occurrence_history.py @@ -0,0 +1,73 @@ +# Generated by Django 4.2.10 on 2026-09-23 04:57 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + dependencies = [ + ("ml", "0028_normalize_empty_endpoint_url_to_null"), + ("jobs", "0023_alter_job_job_type_key"), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ("main", "0102_detection_embedding"), + ] + + operations = [ + migrations.CreateModel( + name="OccurrenceHistoryRecord", + fields=[ + ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), + ("created_at", models.DateTimeField(auto_now_add=True)), + ("updated_at", models.DateTimeField(auto_now=True)), + ( + "kind", + models.CharField( + choices=[("algorithm_result", "Algorithm Result"), ("review", "Review")], max_length=32 + ), + ), + ("subtype", models.CharField(max_length=64)), + ("timestamp", models.DateTimeField()), + ("payload", models.JSONField(default=dict)), + ( + "algorithm", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="+", + to="ml.algorithm", + ), + ), + ( + "job", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="+", + to="jobs.job", + ), + ), + ( + "occurrence", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, related_name="history", to="main.occurrence" + ), + ), + ( + "user", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="+", + to=settings.AUTH_USER_MODEL, + ), + ), + ], + options={ + "indexes": [models.Index(fields=["occurrence", "-timestamp"], name="occur_history_occ_time_idx")], + }, + ), + ] diff --git a/ami/main/models.py b/ami/main/models.py index 5781d29aa..26eaa352a 100644 --- a/ami/main/models.py +++ b/ami/main/models.py @@ -4126,6 +4126,73 @@ class Meta: ] +@final +class OccurrenceHistoryRecord(BaseModel): + """One dated entry in an occurrence's history: an algorithm's result or a person's review. + + Identifications and predictions keep their own tables; the history endpoint merges all + three. An algorithm result stands for the prediction change a run made, and a run writes + at most one record per occurrence, none when it changed nothing about it. Reviews are + append-only; ``Occurrence.grouping_verified_at`` and ``_by`` cache the latest one. + The payload is validated against the schema for its kind and subtype (ami/main/schemas.py). + """ + + project_accessor = "occurrence__project" + + class Kind(models.TextChoices): + ALGORITHM_RESULT = "algorithm_result" + REVIEW = "review" + + occurrence = models.ForeignKey(Occurrence, on_delete=models.CASCADE, related_name="history") + kind = models.CharField(max_length=32, choices=Kind.choices) + # "tracking", "class_masking", "size_filter" or "track_complete"; see HISTORY_PAYLOAD_SCHEMAS. + subtype = models.CharField(max_length=64) + job = models.ForeignKey("jobs.Job", on_delete=models.SET_NULL, null=True, blank=True, related_name="+") + algorithm = models.ForeignKey("ml.Algorithm", on_delete=models.SET_NULL, null=True, blank=True, related_name="+") + user = models.ForeignKey(User, on_delete=models.SET_NULL, null=True, blank=True, related_name="+") + timestamp = models.DateTimeField() + payload = models.JSONField(default=dict) + + class Meta: + indexes = [models.Index(fields=["occurrence", "-timestamp"], name="occur_history_occ_time_idx")] + + def __str__(self) -> str: + return f"#{self.pk} {self.kind}/{self.subtype} for Occurrence #{self.occurrence_id}" + + @classmethod + def build( + cls, + *, + occurrence_id: int, + kind: str, + subtype: str, + payload, + timestamp: datetime.datetime | None = None, + job=None, + algorithm=None, + user: User | None = None, + ) -> "OccurrenceHistoryRecord": + """An unsaved record with a validated payload, for writers that bulk_create, which skips save().""" + from ami.main.schemas import validate_history_payload + + return cls( + occurrence_id=occurrence_id, + kind=kind, + subtype=subtype, + payload=validate_history_payload(kind, subtype, payload), + timestamp=timestamp or timezone.now(), + job=job, + algorithm=algorithm, + user=user, + ) + + def save(self, *args, **kwargs): + from ami.main.schemas import validate_history_payload + + self.payload = validate_history_payload(self.kind, self.subtype, self.payload) + super().save(*args, **kwargs) + + def update_occurrence_determination( occurrence: Occurrence, current_determination: typing.Optional["Taxon"] = None, save=True ) -> bool: diff --git a/ami/main/schemas.py b/ami/main/schemas.py new file mode 100644 index 000000000..27e843a3a --- /dev/null +++ b/ami/main/schemas.py @@ -0,0 +1,74 @@ +"""Payload schemas for an occurrence's history records, one per subtype. + +A record's ``payload`` is JSON, so these schemas are what keeps each subtype's shape +fixed: every write validates against the schema for its kind and subtype. +""" + +import datetime + +import pydantic + + +class HistoryPayload(pydantic.BaseModel): + class Config: + extra = "forbid" + + +class DeterminationChangePayload(HistoryPayload): + # The occurrence's determination before and after the run, as taxon ids. Equal + # when the run changed predictions without moving the determination. + taxon_before_id: int | None = None + taxon_after_id: int | None = None + + +class TrackingResultPayload(DeterminationChangePayload): + # The tunables the run used, as the tracking config holds them, without its scope. + settings: dict = {} + feature_algorithm_id: int | None = None + detections_count: int + frames_linked: int + occurrences_merged: list[int] = [] + cost_mean: float | None = None + cost_max: float | None = None + + +class ClassMaskingResultPayload(DeterminationChangePayload): + taxa_list_id: int + source_algorithm_id: int + detection_ids: list[int] + + +class SizeFilterResultPayload(DeterminationChangePayload): + size_threshold: float + detection_ids: list[int] + + +class TrackCompleteReviewPayload(HistoryPayload): + detection_ids: list[int] + frames_count: int + first_timestamp: datetime.datetime | None = None + last_timestamp: datetime.datetime | None = None + # Compared with the previous track_complete review; the first review lists none. + detections_added: list[int] = [] + detections_removed: list[int] = [] + + +# Keyed by (kind, subtype). A subtype missing here cannot be written. +HISTORY_PAYLOAD_SCHEMAS: dict[tuple[str, str], type[HistoryPayload]] = { + ("algorithm_result", "tracking"): TrackingResultPayload, + ("algorithm_result", "class_masking"): ClassMaskingResultPayload, + ("algorithm_result", "size_filter"): SizeFilterResultPayload, + ("review", "track_complete"): TrackCompleteReviewPayload, +} + + +def validate_history_payload(kind: str, subtype: str, payload: dict | HistoryPayload) -> dict: + """The payload as JSON-ready data, or a ValueError when it does not fit its subtype's schema.""" + schema = HISTORY_PAYLOAD_SCHEMAS.get((kind, subtype)) + if schema is None: + raise ValueError(f"No history payload schema for kind={kind!r} subtype={subtype!r}") + if isinstance(payload, HistoryPayload) and not isinstance(payload, schema): + raise ValueError(f"{type(payload).__name__} is not the payload schema for {kind}/{subtype}") + data = payload.dict() if isinstance(payload, HistoryPayload) else payload + # Round-trip through JSON so datetimes are stored as ISO strings. + return pydantic.parse_raw_as(dict, schema.parse_obj(data).json()) diff --git a/ami/main/test_occurrence_history.py b/ami/main/test_occurrence_history.py new file mode 100644 index 000000000..48b5830f4 --- /dev/null +++ b/ami/main/test_occurrence_history.py @@ -0,0 +1,54 @@ +"""An occurrence's history: algorithm results and reviews, and the endpoint that reads them.""" + +from django.test import TestCase + +from ami.main.models import Occurrence, OccurrenceHistoryRecord +from ami.tests.fixtures.main import setup_test_project + + +class OccurrenceHistoryPayloadTestCase(TestCase): + """A history payload always fits the schema for its kind and subtype, however it is written.""" + + def setUp(self) -> None: + self.project, self.deployment = setup_test_project(reuse=False) + self.occurrence = Occurrence.objects.create(project=self.project, deployment=self.deployment) + + def _size_filter(self, payload: dict) -> OccurrenceHistoryRecord: + return OccurrenceHistoryRecord.build( + occurrence_id=self.occurrence.pk, + kind=OccurrenceHistoryRecord.Kind.ALGORITHM_RESULT, + subtype="size_filter", + payload=payload, + ) + + def test_a_valid_payload_is_stored_as_given(self): + record = self._size_filter({"size_threshold": 0.001, "detection_ids": [3, 4]}) + record.save() + record.refresh_from_db() + self.assertEqual(record.payload["detection_ids"], [3, 4]) + self.assertIsNone(record.payload["taxon_after_id"]) + + def test_a_payload_that_does_not_fit_its_schema_is_refused(self): + for label, payload in ( + ("missing field", {"size_threshold": 0.001}), + ("wrong type", {"size_threshold": "small", "detection_ids": []}), + ("unknown field", {"size_threshold": 0.001, "detection_ids": [], "note": "x"}), + ): + with self.subTest(label), self.assertRaises(ValueError): + self._size_filter(payload) + + def test_save_validates_too_and_an_unknown_subtype_is_refused(self): + record = OccurrenceHistoryRecord( + occurrence=self.occurrence, + kind=OccurrenceHistoryRecord.Kind.ALGORITHM_RESULT, + subtype="size_filter", + payload={"size_threshold": 0.001}, + timestamp=self.occurrence.created_at, + ) + with self.assertRaises(ValueError): + record.save() + record.subtype = "not_a_subtype" + record.payload = {} + with self.assertRaises(ValueError): + record.save() + self.assertFalse(OccurrenceHistoryRecord.objects.exists()) From 4cd694253fb9c68239017627cd7c79421ed0a4c7 Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Wed, 23 Sep 2026 02:16:19 -0700 Subject: [PATCH 07/44] feat(occurrences): record tracking, class masking, the size filter and track reviews in occurrence history Tracking, class masking and the small size filter now leave one algorithm-result history record on each occurrence a run changes, bulk-created at the end of the run. A run that changes nothing about an occurrence writes nothing for it: a chain that is already one occurrence, a classification masking leaves unchanged, or an occurrence with no flagged detection. Tracking records the settings, feature extractor, frames linked, occurrences merged in and the link costs; the post-processing tasks record the determination before and after and the detections they changed. Confirming a track's grouping posts a track_complete review with the detection ids, frame count, time span and the detections added or removed since the previous review, but only when the set differs from the last confirmed one. The confirmation fields on the occurrence stay as a cache of the latest review. Merging occurrences, by hand or by tracking, moves their history to the surviving occurrence. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01C7Xf6VPbwWtTumhjjF15g8 --- ami/main/models_future/history.py | 68 ++++++++++ ami/main/models_future/tracks.py | 15 ++- ami/main/test_occurrence_history.py | 55 ++++++++ ami/ml/post_processing/class_masking.py | 39 +++++- ami/ml/post_processing/small_size_filter.py | 36 ++++- .../tests/test_class_masking.py | 31 +++++ .../tests/test_occurrence_history_writers.py | 126 ++++++++++++++++++ ami/ml/post_processing/tracking_task.py | 68 +++++++++- 8 files changed, 428 insertions(+), 10 deletions(-) create mode 100644 ami/main/models_future/history.py create mode 100644 ami/ml/post_processing/tests/test_occurrence_history_writers.py diff --git a/ami/main/models_future/history.py b/ami/main/models_future/history.py new file mode 100644 index 000000000..62944794e --- /dev/null +++ b/ami/main/models_future/history.py @@ -0,0 +1,68 @@ +"""An occurrence's history: what algorithms and people did to it, newest first. + +Algorithm results and reviews are ``OccurrenceHistoryRecord`` rows. Identifications and +predictions keep their own tables, and ``occurrence_timeline`` merges all of them into one +list for the history endpoint. +""" + +from __future__ import annotations + +import datetime + +from django.utils import timezone + +from ami.main.models import Detection, Occurrence, OccurrenceHistoryRecord, User +from ami.main.schemas import TrackCompleteReviewPayload + +TRACK_COMPLETE = "track_complete" + + +def latest_track_complete_review(occurrence: Occurrence) -> OccurrenceHistoryRecord | None: + return ( + OccurrenceHistoryRecord.objects.filter( + occurrence=occurrence, kind=OccurrenceHistoryRecord.Kind.REVIEW, subtype=TRACK_COMPLETE + ) + .order_by("-timestamp", "-pk") + .first() + ) + + +def record_track_complete_review( + occurrence: Occurrence, user: User, timestamp: datetime.datetime | None = None +) -> OccurrenceHistoryRecord | None: + """Record that ``user`` confirmed this occurrence's detections, unless they are the set last confirmed. + + Re-confirming an unchanged track adds nothing to the history, so the review list shows + only the sets a person actually looked at. The first review always posts. + """ + frames = list( + Detection.objects.valid() + .filter(occurrence=occurrence) + .order_by("source_image__timestamp", "pk") + .values_list("pk", "source_image_id", "source_image__timestamp") + ) + detection_ids = sorted(pk for pk, _, _ in frames) + previous = latest_track_complete_review(occurrence) + previous_ids = sorted(previous.payload.get("detection_ids", [])) if previous else None + if previous_ids == detection_ids: + return None + + capture_times = [captured for _, _, captured in frames if captured is not None] + payload = TrackCompleteReviewPayload( + detection_ids=detection_ids, + frames_count=len({capture_id for _, capture_id, _ in frames}), + first_timestamp=min(capture_times, default=None), + last_timestamp=max(capture_times, default=None), + detections_added=sorted(set(detection_ids) - set(previous_ids)) if previous_ids is not None else [], + detections_removed=sorted(set(previous_ids) - set(detection_ids)) if previous_ids is not None else [], + ) + record = OccurrenceHistoryRecord.build( + occurrence_id=occurrence.pk, + kind=OccurrenceHistoryRecord.Kind.REVIEW, + subtype=TRACK_COMPLETE, + payload=payload, + timestamp=timestamp or timezone.now(), + user=user, + ) + record.save() + return record diff --git a/ami/main/models_future/tracks.py b/ami/main/models_future/tracks.py index 77edcec74..96815fd3d 100644 --- a/ami/main/models_future/tracks.py +++ b/ami/main/models_future/tracks.py @@ -46,11 +46,13 @@ Detection, Identification, Occurrence, + OccurrenceHistoryRecord, SourceImage, User, update_calculated_fields_for_sessions_and_stations, update_occurrence_determination, ) +from ami.main.models_future.history import record_track_complete_review from ami.main.models_future.track_stats import refresh_track_stats # Frame order within a track: capture time, then capture, then detection. The edits, @@ -387,16 +389,16 @@ def clear_grouping_verification(*occurrences: Occurrence) -> None: def _absorb(target: Occurrence, sources: list[Occurrence]) -> None: - """Move every detection and identification off ``sources`` and delete them. + """Move every detection, identification and history record off ``sources`` and delete them. - Identifications move first. ``Identification.occurrence`` cascades on delete, so - a source removed before its identifications were reassigned would take a - person's work with it. + Identifications and history move first. Both cascade on delete, so a source removed + before they were reassigned would take a person's work and its record with it. """ source_pks = [o.pk for o in sources] if not source_pks: return Identification.objects.filter(occurrence_id__in=source_pks).update(occurrence=target) + OccurrenceHistoryRecord.objects.filter(occurrence_id__in=source_pks).update(occurrence=target) Detection.objects.filter(occurrence_id__in=source_pks).update(occurrence=target) Occurrence.objects.filter(pk__in=source_pks).delete() @@ -533,15 +535,18 @@ def add_detections(target: Occurrence, detections: Iterable[Detection]) -> Occur return target +@transaction.atomic def verify_grouping(occurrence: Occurrence, user: User) -> Occurrence: """Record that a person confirmed this occurrence holds the right detections. This is the label the tracking methods are scored against, so it is deliberately - an explicit act — no operation in this module sets it as a side effect. + an explicit act — no operation in this module sets it as a side effect. The review + goes into the occurrence's history; the two fields here cache the latest one. """ occurrence.grouping_verified_at = timezone.now() occurrence.grouping_verified_by = user occurrence.save(update_fields=["grouping_verified_at", "grouping_verified_by"]) + record_track_complete_review(occurrence, user, timestamp=occurrence.grouping_verified_at) return occurrence diff --git a/ami/main/test_occurrence_history.py b/ami/main/test_occurrence_history.py index 48b5830f4..aa68e7d34 100644 --- a/ami/main/test_occurrence_history.py +++ b/ami/main/test_occurrence_history.py @@ -2,6 +2,7 @@ from django.test import TestCase +from ami.main import tests as main_tests from ami.main.models import Occurrence, OccurrenceHistoryRecord from ami.tests.fixtures.main import setup_test_project @@ -52,3 +53,57 @@ def test_save_validates_too_and_an_unknown_subtype_is_refused(self): with self.assertRaises(ValueError): record.save() self.assertFalse(OccurrenceHistoryRecord.objects.exists()) + + +class TrackCompleteReviewTestCase(main_tests.TrackFixtureTestCase): + """Confirming a track posts a review only when the confirmed set of detections differs from the last one.""" + + def verify(self, occurrence: Occurrence | None = None): + self.client.force_authenticate(user=self.curator) + occurrence = occurrence or self.occurrence + response = self.client.post(f"/api/v2/occurrences/{occurrence.pk}/verify-grouping/", format="json") + self.assertEqual(response.status_code, 200, response.data) + + def reviews(self, occurrence: Occurrence | None = None): + return OccurrenceHistoryRecord.objects.filter( + occurrence=occurrence or self.occurrence, + kind=OccurrenceHistoryRecord.Kind.REVIEW, + subtype="track_complete", + ).order_by("timestamp", "pk") + + def test_the_first_confirmation_posts_a_review_and_an_unchanged_one_does_not(self): + self.verify() + self.verify() + + review = self.reviews().get() + self.assertEqual(review.user, self.curator) + self.assertEqual(review.payload["detection_ids"], sorted(d.pk for d in self.detections)) + self.assertEqual(review.payload["frames_count"], len(self.captures)) + self.assertEqual(review.payload["first_timestamp"], self.captures[0].timestamp.isoformat()) + self.assertEqual(review.payload["last_timestamp"], self.captures[-1].timestamp.isoformat()) + self.assertEqual((review.payload["detections_added"], review.payload["detections_removed"]), ([], [])) + self.occurrence.refresh_from_db() + self.assertEqual(self.occurrence.grouping_verified_by, self.curator) + self.assertGreaterEqual(self.occurrence.grouping_verified_at, review.timestamp) + + def test_a_confirmation_after_an_edit_records_what_changed(self): + self.verify() + response = self.post("remove-detection", self.detections[-1], user=self.curator) + self.assertEqual(response.status_code, 200, response.data) + self.verify() + + first, second = self.reviews() + self.assertEqual(second.payload["detections_removed"], [self.detections[-1].pk]) + self.assertEqual(second.payload["detections_added"], []) + self.assertEqual(second.payload["frames_count"], len(self.captures) - 1) + self.assertEqual(first.payload["detection_ids"], sorted(d.pk for d in self.detections)) + + def test_merging_an_occurrence_keeps_its_reviews(self): + other, _ = self._make_track(1, captures=self._make_captures_after(1)) + self.verify(other) + self.client.force_authenticate(user=self.curator) + response = self.client.post( + f"/api/v2/occurrences/{self.occurrence.pk}/merge/", {"occurrence_ids": [other.pk]}, format="json" + ) + self.assertEqual(response.status_code, 200, response.data) + self.assertEqual(self.reviews().count(), 1) diff --git a/ami/ml/post_processing/class_masking.py b/ami/ml/post_processing/class_masking.py index 2da2001b7..7060b8ec5 100644 --- a/ami/ml/post_processing/class_masking.py +++ b/ami/ml/post_processing/class_masking.py @@ -1,4 +1,5 @@ import logging +import typing from collections.abc import Callable import numpy as np @@ -7,10 +8,14 @@ from django.db.models import QuerySet from django.utils import timezone -from ami.main.models import Classification, Occurrence, SourceImageCollection, TaxaList +from ami.main.models import Classification, Occurrence, OccurrenceHistoryRecord, SourceImageCollection, TaxaList +from ami.main.schemas import ClassMaskingResultPayload from ami.ml.models.algorithm import Algorithm, AlgorithmTaskType from ami.ml.post_processing.base import BasePostProcessingTask +if typing.TYPE_CHECKING: + from ami.jobs.models import Job + logger = logging.getLogger(__name__) @@ -52,6 +57,7 @@ def make_classifications_filtered_by_taxa_list( task_logger: logging.Logger = logger, on_setup: Callable[[int], None] | None = None, on_batch: Callable[[dict], None] | None = None, + job: "Job | None" = None, ) -> dict[str, int]: """Re-score ``classifications`` by masking out classes absent from ``taxa_list``. @@ -74,6 +80,9 @@ def make_classifications_filtered_by_taxa_list( ``occurrences_updated`` counts only occurrences whose determination actually changed (not just any occurrence touched), matching the size-filter convention. + Every occurrence with a re-scored classification gets one history record, written + once the run is done so an occurrence spanning batches is recorded once. + Returns final counters (checked / masked / occurrences updated) for stage metrics. """ taxa_in_list = set(taxa_list.taxa.all()) @@ -113,6 +122,9 @@ def make_classifications_filtered_by_taxa_list( occurrences_to_update: set[Occurrence] = set() # Tracks occurrences whose determination actually changed across all batches. changed_occurrence_ids: set[int] = set() + # Occurrence id -> (determination before the run, re-scored detection ids). + history: dict[int, tuple[int | None, set[int]]] = {} + determinations_after: dict[int, int | None] = {} timestamp = timezone.now() masked_count = 0 @@ -189,7 +201,10 @@ def make_classifications_filtered_by_taxa_list( detection = classification.detection if detection is not None and detection.occurrence is not None: - occurrences_to_update.add(detection.occurrence) + occurrence = detection.occurrence + occurrences_to_update.add(occurrence) + _, rescored = history.setdefault(occurrence.pk, (occurrence.determination_id, set())) + rescored.add(detection.pk) # Flush every batch_size items and at the final item. The flush fires even # when nothing was accumulated so the job health-check sees a heartbeat during @@ -209,6 +224,7 @@ def make_classifications_filtered_by_taxa_list( occurrence.save(update_determination=True) if occurrence.pk is not None and occurrence.determination_id != prev: changed_occurrence_ids.add(occurrence.pk) + determinations_after[occurrence.pk] = occurrence.determination_id classifications_to_demote.clear() classifications_to_add.clear() @@ -224,6 +240,24 @@ def make_classifications_filtered_by_taxa_list( } ) + OccurrenceHistoryRecord.objects.bulk_create( + OccurrenceHistoryRecord.build( + occurrence_id=occurrence_id, + kind=OccurrenceHistoryRecord.Kind.ALGORITHM_RESULT, + subtype="class_masking", + payload=ClassMaskingResultPayload( + taxa_list_id=taxa_list.pk, + source_algorithm_id=algorithm.pk, + detection_ids=sorted(detection_ids), + taxon_before_id=taxon_before_id, + taxon_after_id=determinations_after.get(occurrence_id), + ), + timestamp=timestamp, + job=job, + algorithm=new_algorithm, + ) + for occurrence_id, (taxon_before_id, detection_ids) in history.items() + ) task_logger.info( f"Re-scored {masked_count} of {total} classifications; updated {len(changed_occurrence_ids)} occurrences." ) @@ -368,6 +402,7 @@ def _on_batch(m: dict) -> None: task_logger=self.logger, on_setup=_on_setup, on_batch=_on_batch, + job=self.job, ) self.report_stage_metrics(metrics) self.logger.info(f"=== Completed {self.name} ===") diff --git a/ami/ml/post_processing/small_size_filter.py b/ami/ml/post_processing/small_size_filter.py index 6a39af780..ac0d94fa7 100644 --- a/ami/ml/post_processing/small_size_filter.py +++ b/ami/ml/post_processing/small_size_filter.py @@ -2,7 +2,16 @@ from django.db.models import QuerySet from django.utils import timezone -from ami.main.models import Classification, Detection, Occurrence, SourceImageCollection, Taxon, TaxonRank +from ami.main.models import ( + Classification, + Detection, + Occurrence, + OccurrenceHistoryRecord, + SourceImageCollection, + Taxon, + TaxonRank, +) +from ami.main.schemas import SizeFilterResultPayload from ami.ml.post_processing.base import BasePostProcessingTask from ami.ml.schemas import BoundingBox @@ -90,6 +99,10 @@ def run(self) -> None: updated_occurrence_ids: set[int] = set() modified_occurrences = 0 checked = 0 + # Occurrence id -> (determination before the run, flagged detection ids), written + # as one history record per occurrence once the run is done. + history: dict[int, tuple[int | None, list[int]]] = {} + determinations_after: dict[int, int | None] = {} for i, det in enumerate(detections.iterator(), start=1): bbox = det.get_bbox() @@ -131,6 +144,8 @@ def run(self) -> None: detections_to_update.add(det) if det.occurrence is not None: occcurrences_to_update.add(det.occurrence) + _, flagged = history.setdefault(det.occurrence.pk, (det.occurrence.determination_id, [])) + flagged.append(det.pk) self.logger.debug(f"Marking detection {det.pk} as {not_identifiable_taxon.name}") # Update progress every 100 detections @@ -159,6 +174,7 @@ def run(self) -> None: occ.save(update_determination=True) if occ.pk is not None and occ.determination_id != prev_determination_id: updated_occurrence_ids.add(occ.pk) + determinations_after[occ.pk] = occ.determination_id modified_occurrences = len(updated_occurrence_ids) occcurrences_to_update.clear() @@ -172,4 +188,22 @@ def run(self) -> None: } ) + OccurrenceHistoryRecord.objects.bulk_create( + OccurrenceHistoryRecord.build( + occurrence_id=occurrence_id, + kind=OccurrenceHistoryRecord.Kind.ALGORITHM_RESULT, + subtype="size_filter", + payload=SizeFilterResultPayload( + size_threshold=threshold, + detection_ids=sorted(detection_ids), + taxon_before_id=taxon_before_id, + taxon_after_id=determinations_after.get(occurrence_id), + ), + job=self.job, + algorithm=self.algorithm, + ) + for occurrence_id, (taxon_before_id, detection_ids) in history.items() + # A detection flagged in a batch that never flushed was not saved, so it gets no record. + if occurrence_id in determinations_after + ) self.logger.info(f"=== Completed {self.name}: {modified_detections} of {total} detections modified ===") diff --git a/ami/ml/post_processing/tests/test_class_masking.py b/ami/ml/post_processing/tests/test_class_masking.py index bf38decc9..fe8510dcb 100644 --- a/ami/ml/post_processing/tests/test_class_masking.py +++ b/ami/ml/post_processing/tests/test_class_masking.py @@ -19,6 +19,7 @@ Classification, Detection, Occurrence, + OccurrenceHistoryRecord, SourceImage, SourceImageCollection, TaxaList, @@ -359,6 +360,36 @@ def test_task_run_occurrence_scope(self): self.assertIsNotNone(new_clf) self.assertEqual(new_clf.taxon, self.species_taxa[0]) + def test_a_rescored_occurrence_gets_one_history_record_and_a_rerun_none(self): + logits = [2.0, 1.0, 5.0] + taxa_list = TaxaList.objects.create(name="History list") + taxa_list.taxa.set(self.species_taxa[:2]) + det, occ = self._detection_with_occurrence() + self._create_classification_with_logits(det, self.species_taxa[2], _softmax(logits), logits) + occ.save() + + for _ in range(2): + ClassMaskingTask(occurrence_id=occ.pk, taxa_list_id=taxa_list.pk, algorithm_id=self.algorithm.pk).run() + + record = OccurrenceHistoryRecord.objects.get(occurrence=occ, subtype="class_masking") + self.assertTrue(record.algorithm.key.startswith(f"{self.algorithm.key}_filtered_by_taxa_list_{taxa_list.pk}")) + self.assertEqual(record.payload["detection_ids"], [det.pk]) + self.assertEqual(record.payload["taxa_list_id"], taxa_list.pk) + self.assertEqual(record.payload["source_algorithm_id"], self.algorithm.pk) + self.assertEqual(record.payload["taxon_before_id"], self.species_taxa[2].pk) + self.assertEqual(record.payload["taxon_after_id"], self.species_taxa[0].pk) + + def test_an_occurrence_masking_does_not_change_gets_no_history_record(self): + logits = [2.0, 1.0, 5.0] + taxa_list = TaxaList.objects.create(name="Keeps everything") + taxa_list.taxa.set(self.species_taxa) + det, occ = self._detection_with_occurrence() + self._create_classification_with_logits(det, self.species_taxa[2], _softmax(logits), logits) + + ClassMaskingTask(occurrence_id=occ.pk, taxa_list_id=taxa_list.pk, algorithm_id=self.algorithm.pk).run() + + self.assertFalse(OccurrenceHistoryRecord.objects.filter(occurrence=occ).exists()) + # ----- batched commit + heartbeat ------------------------------------- def test_batched_commit_flushes_multiple_times(self): diff --git a/ami/ml/post_processing/tests/test_occurrence_history_writers.py b/ami/ml/post_processing/tests/test_occurrence_history_writers.py new file mode 100644 index 000000000..7c3213717 --- /dev/null +++ b/ami/ml/post_processing/tests/test_occurrence_history_writers.py @@ -0,0 +1,126 @@ +"""Tracking and the small size filter leave one history record per occurrence they change, and none otherwise. + +Class masking's records are covered in test_class_masking.py, next to its other tests. +""" + +import logging + +from django.test import TestCase + +from ami.main.models import Detection, Occurrence, OccurrenceHistoryRecord, SourceImage, Taxon +from ami.ml.post_processing.small_size_filter import SmallSizeFilterTask +from ami.ml.post_processing.tracking_task import TrackingTask +from ami.tests.fixtures.main import create_captures, create_taxa, setup_test_project + +logger = logging.getLogger(__name__) + + +class HistoryWriterFixture(TestCase): + """Three consecutive captures in one session, each with dimensions, and a project taxon.""" + + def setUp(self) -> None: + self.project, self.deployment = setup_test_project(reuse=False) + create_taxa(project=self.project) + create_captures(deployment=self.deployment, num_nights=1, images_per_night=3, interval_minutes=1) + self.captures = list(SourceImage.objects.filter(deployment=self.deployment).order_by("timestamp")) + SourceImage.objects.filter(pk__in=[c.pk for c in self.captures]).update(width=1000, height=1000) + for capture in self.captures: + capture.width = capture.height = 1000 + self.event = self.captures[0].event + self.taxon = Taxon.objects.filter(projects=self.project).order_by("pk").first() + + def _singleton(self, capture: SourceImage, bbox: list[int], score: float = 0.9) -> Occurrence: + """One detection in an occurrence of its own, classified as the fixture taxon.""" + occurrence = Occurrence.objects.create(event=self.event, deployment=self.deployment, project=self.project) + detection = Detection.objects.create( + source_image=capture, bbox=bbox, timestamp=capture.timestamp, occurrence=occurrence + ) + detection.classifications.create(taxon=self.taxon, score=score, timestamp=capture.timestamp) + occurrence.save() + return occurrence + + def records(self, subtype: str): + return OccurrenceHistoryRecord.objects.filter( + kind=OccurrenceHistoryRecord.Kind.ALGORITHM_RESULT, subtype=subtype + ).order_by("pk") + + +class TrackingHistoryTestCase(HistoryWriterFixture): + def test_a_merged_track_gets_one_record_and_an_untouched_occurrence_none(self): + track = [self._singleton(capture, [100, 100, 150, 150]) for capture in self.captures] + loner = self._singleton(self.captures[1], [800, 800, 820, 820]) + + TrackingTask(logger=logger, event_ids=[self.event.pk], require_features=False, cost_threshold=0.5).run() + + record = self.records("tracking").get() + keeper = Occurrence.objects.get(detections__source_image=self.captures[0]) + self.assertEqual(record.occurrence_id, keeper.pk) + self.assertNotEqual(record.occurrence_id, loner.pk) + self.assertEqual(record.algorithm.key, "tracking") + payload = record.payload + self.assertEqual(payload["detections_count"], 3) + self.assertEqual(payload["frames_linked"], 2) + self.assertEqual(payload["occurrences_merged"], sorted(o.pk for o in track if o.pk != keeper.pk)) + self.assertIsNone(payload["feature_algorithm_id"]) + self.assertEqual(payload["settings"]["cost_threshold"], 0.5) + self.assertNotIn("event_ids", payload["settings"]) + self.assertIsNotNone(payload["cost_mean"]) + self.assertLessEqual(payload["cost_mean"], payload["cost_max"]) + self.assertEqual(payload["taxon_before_id"], self.taxon.pk) + self.assertEqual(payload["taxon_after_id"], self.taxon.pk) + + def test_a_run_that_changes_nothing_writes_nothing(self): + for capture in self.captures: + self._singleton(capture, [100, 100, 150, 150]) + TrackingTask(logger=logger, event_ids=[self.event.pk], require_features=False, cost_threshold=0.5).run() + self.assertEqual(self.records("tracking").count(), 1) + + TrackingTask( + logger=logger, + event_ids=[self.event.pk], + require_features=False, + require_fresh_event=False, + skip_if_human_identifications=False, + cost_threshold=0.5, + ).run() + self.assertEqual(self.records("tracking").count(), 1) + + def test_a_merged_occurrence_hands_its_history_to_the_keeper(self): + first, second = (self._singleton(capture, [100, 100, 150, 150]) for capture in self.captures[:2]) + earlier = OccurrenceHistoryRecord.objects.create( + occurrence=second, + kind=OccurrenceHistoryRecord.Kind.ALGORITHM_RESULT, + subtype="size_filter", + payload={"size_threshold": 0.001, "detection_ids": []}, + timestamp=second.created_at, + ) + TrackingTask(logger=logger, event_ids=[self.event.pk], require_features=False, cost_threshold=0.5).run() + + earlier.refresh_from_db() + self.assertEqual(earlier.occurrence_id, first.pk) + + +class SizeFilterHistoryTestCase(HistoryWriterFixture): + def test_an_occurrence_with_a_flagged_detection_gets_one_record(self): + occurrence = self._singleton(self.captures[0], [0, 0, 10, 10]) + Detection.objects.create( + source_image=self.captures[1], + bbox=[0, 0, 12, 12], + timestamp=self.captures[1].timestamp, + occurrence=occurrence, + ) + + SmallSizeFilterTask(logger=logger, occurrence_id=occurrence.pk, size_threshold=0.01).run() + + record = self.records("size_filter").get() + self.assertEqual(record.occurrence_id, occurrence.pk) + self.assertEqual(record.algorithm.key, "small_size_filter") + self.assertEqual(record.payload["detection_ids"], sorted(occurrence.detections.values_list("pk", flat=True))) + self.assertEqual(record.payload["size_threshold"], 0.01) + self.assertEqual(record.payload["taxon_before_id"], self.taxon.pk) + self.assertEqual(record.payload["taxon_after_id"], Taxon.objects.get(name="Not identifiable").pk) + + def test_an_occurrence_with_nothing_flagged_gets_no_record(self): + occurrence = self._singleton(self.captures[0], [0, 0, 500, 500]) + SmallSizeFilterTask(logger=logger, occurrence_id=occurrence.pk, size_threshold=0.01).run() + self.assertFalse(self.records("size_filter").exists()) diff --git a/ami/ml/post_processing/tracking_task.py b/ami/ml/post_processing/tracking_task.py index 557102936..583faab02 100644 --- a/ami/ml/post_processing/tracking_task.py +++ b/ami/ml/post_processing/tracking_task.py @@ -1,4 +1,5 @@ import collections +import dataclasses import logging import math import typing @@ -16,6 +17,7 @@ Event, Identification, Occurrence, + OccurrenceHistoryRecord, SourceImage, SourceImageCollection, update_calculated_fields_for_sessions_and_stations, @@ -23,9 +25,13 @@ from ami.main.models_future.embeddings import algorithm_ids_with_vectors, vectors_for_detections from ami.main.models_future.track_stats import refresh_track_stats_for_ids from ami.main.models_future.tracks import clear_grouping_verification +from ami.main.schemas import TrackingResultPayload from ami.ml.models import Algorithm from ami.ml.post_processing.base import BasePostProcessingTask +if typing.TYPE_CHECKING: + from ami.jobs.models import Job + class TrackingConfig(pydantic.BaseModel): """Scope and tunables for a tracking run. @@ -244,8 +250,47 @@ def record_tracking_determination(occurrence: Occurrence, algorithm: Algorithm) ) +@dataclasses.dataclass +class TrackingHistory: + """What a tracking run records in the history of each occurrence it changes.""" + + settings: dict + feature_algorithm_id: int | None = None + job: "Job | None" = None + algorithm: Algorithm | None = None + # Detection id -> cost of the link this run made from it to its next detection. + link_costs: dict[int, float] = dataclasses.field(default_factory=dict) + + def record( + self, occurrence: Occurrence, chain: list[Detection], merged: Iterable[int], taxon_before_id: int | None + ) -> OccurrenceHistoryRecord: + costs = [self.link_costs[d.pk] for d in chain[:-1] if d.pk in self.link_costs] + payload = TrackingResultPayload( + settings=self.settings, + feature_algorithm_id=self.feature_algorithm_id, + detections_count=len(chain), + frames_linked=len(costs), + occurrences_merged=sorted(merged), + cost_mean=sum(costs) / len(costs) if costs else None, + cost_max=max(costs, default=None), + taxon_before_id=taxon_before_id, + taxon_after_id=occurrence.determination_id, + ) + return OccurrenceHistoryRecord.build( + occurrence_id=occurrence.pk, + kind=OccurrenceHistoryRecord.Kind.ALGORITHM_RESULT, + subtype="tracking", + payload=payload, + job=self.job, + algorithm=self.algorithm, + ) + + def assign_occurrences_from_detection_chains( - source_images: list[SourceImage], logger: logging.Logger, record_as: Algorithm | None = None + source_images: list[SourceImage], + logger: logging.Logger, + record_as: Algorithm | None = None, + history: TrackingHistory | None = None, ) -> dict[str, int]: """ Walk chains via ``Detection.next_detection`` and consolidate each chain into @@ -263,6 +308,8 @@ def assign_occurrences_from_detection_chains( an occurrence the chains leave as it was keeps its mark. - Store the track statistics of every occurrence the chains settle on, so the list can sort by them (see ``track_stats.refresh_track_stats_for_ids``). + - With ``history`` set, leave one history record on each occurrence a chain changed; + a chain that was already one occurrence gets none. Designed for fresh-event input (1:1 detection/occurrence). v2 incremental tracking can reuse this primitive for prepend/append: keeper survives, new detections fold in. @@ -275,6 +322,7 @@ def assign_occurrences_from_detection_chains( merged = 0 identifications_moved = 0 determinations_recorded = 0 + history_records: list[OccurrenceHistoryRecord] = [] existing = Occurrence.objects.filter(detections__source_image__in=source_images).distinct().count() # A chain ends at a session boundary. A regroup that splits a track keeps the link @@ -353,6 +401,7 @@ def session_of(detection: Detection) -> int | None: identifications_moved += Identification.objects.filter(occurrence_id__in=doomed).update( occurrence=keeper ) + OccurrenceHistoryRecord.objects.filter(occurrence_id__in=doomed).update(occurrence=keeper) undeleted: list[int] = [] for occ_id in doomed: try: @@ -373,8 +422,12 @@ def session_of(detection: Detection) -> int | None: if record_as is not None and keeper.determination_id != previous_determination_id: if record_tracking_determination(keeper, record_as) is not None: determinations_recorded += 1 + if history is not None: + history_records.append(history.record(keeper, chain, doomed, previous_determination_id)) settled.add(keeper.pk) + OccurrenceHistoryRecord.objects.bulk_create(history_records) + # Stored once every determination is settled, since id_agreement is measured against # it, and in batches for the whole event rather than three queries per chain. stats_stored = refresh_track_stats_for_ids(settled) @@ -584,6 +637,7 @@ def assign_occurrences_by_tracking_images( config: TrackingConfig, progress_cb: typing.Callable[[float], None] | None = None, record_as: Algorithm | None = None, + history: TrackingHistory | None = None, ) -> dict[str, int]: source_images = list(event.captures.order_by("timestamp")) if len(source_images) < 2: @@ -603,6 +657,8 @@ def assign_occurrences_by_tracking_images( else: save_links(proposed, logger) links += len(proposed) + if history is not None: + history.link_costs.update((det.pk, cost) for det, _, cost in proposed) if progress_cb: progress_cb((i + 1) / transitions) @@ -612,7 +668,9 @@ def assign_occurrences_by_tracking_images( "due to missing image dimensions." ) - counters = assign_occurrences_from_detection_chains(source_images, logger, record_as=record_as) + counters = assign_occurrences_from_detection_chains( + source_images, logger, record_as=record_as, history=history + ) counters["links_created"] = links return counters @@ -743,6 +801,12 @@ def _stage_progress(p: float, _idx=idx, _total=total) -> None: config=self.config, record_as=self.algorithm, progress_cb=_stage_progress, + history=TrackingHistory( + settings=self.config.dict(exclude={"source_image_collection_id", "event_ids"}), + feature_algorithm_id=algorithm.pk if algorithm is not None else None, + job=self.job, + algorithm=self.algorithm, + ), ) totals["events_tracked"] += 1 tracked_event_ids.append(event.pk) From dea7a5eaf66e7e25d8cf69d0a9c730f1b342bd21 Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Wed, 23 Sep 2026 02:36:42 -0700 Subject: [PATCH 08/44] feat(occurrences): show everything that happened to an occurrence at /occurrences/{id}/history/ Add a read-only history action on the occurrence viewset that returns the occurrence's history records, identifications and predictions merged into one list, newest first. Each entry names its source table in a type field and carries its timestamp, subtype, algorithm, job, taxon (and the taxon before, for an algorithm result), score and a payload. People appear by name and picture only. A prediction made by an algorithm that also left a history record on the occurrence is left out, since the record already stands for that change. The endpoint is visible to exactly the people who can open the occurrence itself and uses the same default filters as the detail view. It costs a fixed number of queries however many entries there are, which a test pins on a multi-row fixture, alongside a member, non-member, anonymous and superuser matrix on a public and a draft project. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01C7Xf6VPbwWtTumhjjF15g8 --- ami/main/api/serializers.py | 46 +++++++++++ ami/main/api/views.py | 24 +++++- ami/main/models_future/history.py | 102 ++++++++++++++++++++++++- ami/main/test_occurrence_history.py | 113 +++++++++++++++++++++++++++- 4 files changed, 279 insertions(+), 6 deletions(-) diff --git a/ami/main/api/serializers.py b/ami/main/api/serializers.py index bf2de2dc8..a714bdc64 100644 --- a/ami/main/api/serializers.py +++ b/ami/main/api/serializers.py @@ -2223,6 +2223,52 @@ class OccurrenceGroupingSerializer(serializers.Serializer): grouping_verified_by = serializers.CharField(allow_null=True) +class HistoryUserSerializer(serializers.Serializer): + """A person in an occurrence's history: name and picture only, never an email address.""" + + id = serializers.IntegerField() + name = serializers.CharField() + image = serializers.ImageField(allow_null=True) + + +class HistoryAlgorithmSerializer(serializers.Serializer): + id = serializers.IntegerField() + name = serializers.CharField() + key = serializers.CharField() + + +class HistoryJobSerializer(serializers.Serializer): + id = serializers.IntegerField() + name = serializers.CharField() + + +class HistoryTaxonSerializer(serializers.Serializer): + id = serializers.IntegerField() + name = serializers.CharField() + rank = serializers.CharField() + + +class OccurrenceHistoryEntrySerializer(serializers.Serializer): + """One entry of an occurrence's history, newest first. ``type`` says which table it came from.""" + + type = serializers.ChoiceField(choices=["algorithm_result", "review", "identification", "prediction"]) + id = serializers.IntegerField(help_text="Primary key of the row in the table ``type`` names.") + timestamp = serializers.DateTimeField() + subtype = serializers.CharField( + allow_null=True, + help_text="For algorithm results and reviews: tracking, class_masking, size_filter or track_complete.", + ) + user = HistoryUserSerializer(allow_null=True) + algorithm = HistoryAlgorithmSerializer(allow_null=True) + job = HistoryJobSerializer(allow_null=True) + taxon = HistoryTaxonSerializer( + allow_null=True, help_text="The identified or predicted taxon, or the determination after a result." + ) + taxon_before = HistoryTaxonSerializer(allow_null=True, help_text="The determination before a result.") + score = serializers.FloatField(allow_null=True) + payload = serializers.JSONField(help_text="Details that depend on the type and subtype.") + + class OccurrencePathCaptureSerializer(serializers.Serializer): """The capture one frame of a path was measured against.""" diff --git a/ami/main/api/views.py b/ami/main/api/views.py index 350660bdd..5b3310826 100644 --- a/ami/main/api/views.py +++ b/ami/main/api/views.py @@ -34,6 +34,7 @@ from ami.base.views import ProjectMixin from ami.main.api.schemas import limit_doc_param, project_id_doc_param from ami.main.api.serializers import TagSerializer +from ami.main.models_future.history import occurrence_timeline from ami.main.models_future.identifications import create_identifications_batch, resolve_occurrences from ami.main.models_future.merge_candidates import ( DEFAULT_ADJACENT_CAPTURES, @@ -109,6 +110,7 @@ ModelAgreementSerializer, OccurrenceAddDetectionsSerializer, OccurrenceGroupingSerializer, + OccurrenceHistoryEntrySerializer, OccurrenceListSerializer, OccurrenceMergeSerializer, OccurrencePathFrameSerializer, @@ -1630,7 +1632,7 @@ def get_serializer_class(self): ) # Actions that open one occurrence. They drop the determination requirement; see # get_queryset. - SINGLE_OCCURRENCE_ACTIONS = ("retrieve", "path") + SINGLE_OCCURRENCE_ACTIONS = ("retrieve", "path", "history") # Actions the project's default filters never hide an occurrence from. The session # view selects occurrences with those filters off and draws their paths. UNFILTERED_ACTIONS = (*TRACK_EDIT_ACTIONS, "path") @@ -1658,15 +1660,17 @@ def get_queryset(self) -> QuerySet["Occurrence"]: "event", ) qs = qs.with_detections_count().with_timestamps() # type: ignore - qs = qs.with_identifications() # type: ignore + if self.action != "history": + # The history reads identifications itself, with the fields its entries need. + qs = qs.with_identifications() # type: ignore if self.action not in self.UNFILTERED_ACTIONS: qs = qs.apply_default_filters( # type: ignore project, self.request, include_undetermined=allow_undetermined ) if self.action == "list": qs = qs.with_list_prefetches() # type: ignore - elif self.action not in ("path", "merge_candidates", "capture_matches"): - # `path` and the track-edit pickers build their own values() queries and never + elif self.action not in ("path", "merge_candidates", "capture_matches", "history"): + # `path`, `history` and the track-edit pickers build their own queries and never # serialize the occurrence, so the detail prefetch would only be waste: # measured at 249ms/4 queries against 6ms/2 for the same object without # it, on a 37-detection occurrence. @@ -1728,6 +1732,18 @@ def path(self, request: Request, pk=None) -> Response: occurrence = self.get_object() return Response(OccurrencePathFrameSerializer(occurrence_path(occurrence), many=True).data) + @extend_schema(parameters=[project_id_doc_param], responses=OccurrenceHistoryEntrySerializer(many=True)) + @action(detail=True, methods=["get"], name="history", pagination_class=None) + def history(self, request: Request, pk=None) -> Response: + """Everything that happened to this occurrence, newest first. + + Merges algorithm results, reviews of its grouping, identifications and predictions + into one list. Visible to whoever can open the occurrence itself. + """ + occurrence = self.get_object() + entries = occurrence_timeline(occurrence) + return Response(OccurrenceHistoryEntrySerializer(entries, many=True, context={"request": request}).data) + def get_permissions(self): # The viewset as a whole is staff-only for writes. Track edits are the # exception: they are a curation tool, gated per object by diff --git a/ami/main/models_future/history.py b/ami/main/models_future/history.py index 62944794e..436fdc48f 100644 --- a/ami/main/models_future/history.py +++ b/ami/main/models_future/history.py @@ -7,13 +7,19 @@ from __future__ import annotations +import dataclasses import datetime +import typing from django.utils import timezone -from ami.main.models import Detection, Occurrence, OccurrenceHistoryRecord, User +from ami.main.models import Detection, Identification, Occurrence, OccurrenceHistoryRecord, Taxon, User from ami.main.schemas import TrackCompleteReviewPayload +if typing.TYPE_CHECKING: + from ami.jobs.models import Job + from ami.ml.models import Algorithm + TRACK_COMPLETE = "track_complete" @@ -66,3 +72,97 @@ def record_track_complete_review( ) record.save() return record + + +@dataclasses.dataclass +class TimelineEntry: + """One entry of the merged history, in the shape ``OccurrenceTimelineEntrySerializer`` reads.""" + + type: str + id: int + timestamp: datetime.datetime + subtype: str | None = None + user: User | None = None + algorithm: Algorithm | None = None + job: Job | None = None + taxon: Taxon | None = None + taxon_before: Taxon | None = None + score: float | None = None + payload: dict = dataclasses.field(default_factory=dict) + + +def occurrence_timeline(occurrence: Occurrence) -> list[TimelineEntry]: + """History records, identifications and predictions of one occurrence, merged newest first. + + A prediction made by an algorithm that also left a history record here is left out: the + record already stands for that change and names the taxon before and after. Costs four + queries whatever the number of entries. + """ + records = list( + OccurrenceHistoryRecord.objects.filter(occurrence=occurrence) + .select_related("user", "algorithm", "job") + .order_by("-timestamp", "-pk") + ) + taxon_ids = { + record.payload.get(key) + for record in records + for key in ("taxon_before_id", "taxon_after_id") + if record.payload.get(key) is not None + } + taxa = {taxon.pk: taxon for taxon in Taxon.objects.filter(pk__in=taxon_ids)} if taxon_ids else {} + + entries = [ + TimelineEntry( + type=record.kind, + id=record.pk, + timestamp=record.timestamp, + subtype=record.subtype, + user=record.user, + algorithm=record.algorithm, + job=record.job, + taxon=taxa.get(record.payload.get("taxon_after_id")), + taxon_before=taxa.get(record.payload.get("taxon_before_id")), + payload=record.payload, + ) + for record in records + ] + + identifications = Identification.objects.filter(occurrence=occurrence).select_related("user", "taxon") + entries.extend( + TimelineEntry( + type="identification", + id=identification.pk, + timestamp=identification.created_at, + user=identification.user, + taxon=identification.taxon, + payload={ + "comment": identification.comment, + "withdrawn": identification.withdrawn, + "agreed_with_identification_id": identification.agreed_with_identification_id, + "agreed_with_prediction_id": identification.agreed_with_prediction_id, + }, + ) + for identification in identifications + ) + + folded = {record.algorithm_id for record in records if record.algorithm_id is not None} + entries.extend( + TimelineEntry( + type="prediction", + id=prediction.pk, + timestamp=prediction.created_at, + algorithm=prediction.algorithm, + taxon=prediction.taxon, + score=prediction.score, + payload={ + "detection_id": prediction.detection_id, + "terminal": prediction.terminal, + "applied_to_id": prediction.applied_to_id, + }, + ) + for prediction in occurrence.predictions() + if prediction.algorithm_id not in folded + ) + + entries.sort(key=lambda entry: (entry.timestamp, entry.id), reverse=True) + return entries diff --git a/ami/main/test_occurrence_history.py b/ami/main/test_occurrence_history.py index aa68e7d34..9f1d34c43 100644 --- a/ami/main/test_occurrence_history.py +++ b/ami/main/test_occurrence_history.py @@ -1,10 +1,18 @@ """An occurrence's history: algorithm results and reviews, and the endpoint that reads them.""" +import datetime + from django.test import TestCase from ami.main import tests as main_tests -from ami.main.models import Occurrence, OccurrenceHistoryRecord +from ami.main.models import Classification, Identification, Occurrence, OccurrenceHistoryRecord, Taxon +from ami.ml.models import Algorithm from ami.tests.fixtures.main import setup_test_project +from ami.users.models import User + +# Measured: two savepoints, the project, its default-filter taxa (2), the occurrence, then +# history records, their taxa, identifications and predictions. +HISTORY_QUERIES = 10 class OccurrenceHistoryPayloadTestCase(TestCase): @@ -107,3 +115,106 @@ def test_merging_an_occurrence_keeps_its_reviews(self): ) self.assertEqual(response.status_code, 200, response.data) self.assertEqual(self.reviews().count(), 1) + + +class OccurrenceHistoryEndpointTestCase(main_tests.TrackFixtureTestCase): + """GET /occurrences/{id}/history/ merges records, identifications and predictions, newest first.""" + + def setUp(self) -> None: + super().setUp() + self.tracking = Algorithm.objects.create(name="Occurrence Tracking", key="tracking-history-test") + self.other_taxon = Taxon.objects.filter(projects=self.project).exclude(pk=self.taxon.pk).first() + self.superuser = User.objects.create_superuser(email="history-super@insectai.org") # type: ignore + self.outsider = User.objects.create_user(email="history-outsider@insectai.org") # type: ignore + + def url(self, occurrence: Occurrence | None = None) -> str: + return f"/api/v2/occurrences/{(occurrence or self.occurrence).pk}/history/?project_id={self.project.pk}" + + def _add_history(self, start: datetime.datetime, rounds: int = 1) -> None: + """Per round: a tracking result, a folded tracking prediction, an identification and a review.""" + for i in range(rounds): + at = start + datetime.timedelta(hours=4 * i) + OccurrenceHistoryRecord.build( + occurrence_id=self.occurrence.pk, + kind=OccurrenceHistoryRecord.Kind.ALGORITHM_RESULT, + subtype="tracking", + payload={ + "detections_count": 4, + "frames_linked": 3, + "taxon_before_id": self.other_taxon.pk, + "taxon_after_id": self.taxon.pk, + }, + timestamp=at, + algorithm=self.tracking, + ).save() + Classification.objects.create( + detection=self.detections[0], taxon=self.taxon, score=0.1, algorithm=self.tracking, timestamp=at + ) + identification = Identification.objects.create( + occurrence=self.occurrence, user=self.reader, taxon=self.other_taxon, comment=f"round {i}" + ) + Identification.objects.filter(pk=identification.pk).update(created_at=at + datetime.timedelta(hours=1)) + OccurrenceHistoryRecord.build( + occurrence_id=self.occurrence.pk, + kind=OccurrenceHistoryRecord.Kind.REVIEW, + subtype="track_complete", + payload={"detection_ids": [d.pk for d in self.detections], "frames_count": 4}, + timestamp=at + datetime.timedelta(hours=2), + user=self.curator, + ).save() + + def test_entries_are_merged_newest_first_and_a_folded_prediction_is_left_out(self): + # Predictions are stamped with created_at (now), so the history rows sit in the past. + self._add_history(datetime.datetime.now() - datetime.timedelta(days=2)) + self.client.force_authenticate(user=self.reader) + response = self.client.get(self.url()) + self.assertEqual(response.status_code, 200, response.data) + + types = [entry["type"] for entry in response.data] + self.assertEqual(types[-3:], ["review", "identification", "algorithm_result"]) + self.assertEqual(set(types[:-3]), {"prediction"}) + self.assertEqual(len(types[:-3]), len(self.detections)) + self.assertTrue(all(entry["algorithm"] is None for entry in response.data[:-3])) + timestamps = [entry["timestamp"] for entry in response.data] + self.assertEqual(timestamps, sorted(timestamps, reverse=True)) + + review, identification, result = response.data[-3:] + self.assertEqual(set(review["user"]), {"id", "name", "image"}) + self.assertEqual(review["user"]["id"], self.curator.pk) + self.assertEqual(identification["user"]["id"], self.reader.pk) + self.assertEqual(identification["taxon"]["id"], self.other_taxon.pk) + self.assertEqual(identification["payload"]["comment"], "round 0") + self.assertEqual(result["subtype"], "tracking") + self.assertEqual(result["algorithm"]["key"], self.tracking.key) + self.assertEqual(result["taxon"]["id"], self.taxon.pk) + self.assertEqual(result["taxon_before"]["id"], self.other_taxon.pk) + self.assertNotIn("email", str(response.data)) + + def test_query_count_does_not_grow_with_the_number_of_entries(self): + self.client.force_authenticate(user=self.reader) + now = datetime.datetime.now() + for rounds, total in ((1, 1), (2, 3)): + self._add_history(now - datetime.timedelta(days=3 * total), rounds=rounds) + with self.subTest(rounds=total), main_tests.cachalot_disabled(): + with self.assertNumQueries(HISTORY_QUERIES): + response = self.client.get(self.url()) + self.assertEqual(response.status_code, 200) + self.assertEqual(len(response.data), 3 * total + len(self.detections)) + + def test_visible_to_whoever_can_open_the_occurrence(self): + """Member, non-member, anonymous and superuser, on a public and on a draft project.""" + expected = { + False: {"member": 200, "non-member": 200, "anonymous": 200, "superuser": 200}, + True: {"member": 200, "non-member": 404, "anonymous": 404, "superuser": 200}, + } + users = {"member": self.reader, "non-member": self.outsider, "anonymous": None, "superuser": self.superuser} + for draft, codes in expected.items(): + self.project.draft = draft + self.project.save() + for label, user in users.items(): + with self.subTest(draft=draft, user=label): + self.client.force_authenticate(user=user) + detail = self.client.get(f"/api/v2/occurrences/{self.occurrence.pk}/?project_id={self.project.pk}") + history = self.client.get(self.url()) + self.assertEqual(history.status_code, codes[label]) + self.assertEqual(history.status_code, detail.status_code) From 1350b8197734be62c5f4ff93d125185c640ff27b Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Wed, 23 Sep 2026 02:42:30 -0700 Subject: [PATCH 09/44] fix(post-processing): save the size filter's last batch when its final detection is skipped The batch flush sat at the end of the loop body, after the checks that skip a detection with no box, no image dimensions or an invalid box. When the last detection in scope was skipped, the final partial batch was never written: its Not identifiable classifications were dropped and its occurrences kept their old determination. Each full batch is now written before the next begins and the remainder once the loop ends, so the history writer no longer needs to guard against occurrences that were never saved. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01C7Xf6VPbwWtTumhjjF15g8 --- ami/ml/post_processing/small_size_filter.py | 86 ++++++++++--------- .../tests/test_occurrence_history_writers.py | 13 +++ 2 files changed, 58 insertions(+), 41 deletions(-) diff --git a/ami/ml/post_processing/small_size_filter.py b/ami/ml/post_processing/small_size_filter.py index ac0d94fa7..feb79ddb6 100644 --- a/ami/ml/post_processing/small_size_filter.py +++ b/ami/ml/post_processing/small_size_filter.py @@ -104,7 +104,51 @@ def run(self) -> None: history: dict[int, tuple[int | None, list[int]]] = {} determinations_after: dict[int, int | None] = {} + def flush(i: int) -> None: + nonlocal checked, modified_detections, modified_occurrences + checked = i + modified_detections += len(detections_to_update) + + # with transaction.atomic(): + self.logger.info(f"Creating {len(classifications_to_create)} new classifications") + Classification.objects.bulk_create(classifications_to_create) + classifications_to_create.clear() + + self.logger.info(f"Marking {len(detections_to_update)} detections as {not_identifiable_taxon.name}") + for det in detections_to_update: + det.updated_at = timezone.now() + Detection.objects.bulk_update(detections_to_update, ["updated_at"]) + detections_to_update.clear() + + self.logger.info(f"Updating {len(occcurrences_to_update)} occurrences") + for occ in occcurrences_to_update: + # Count an occurrence only when flagging its detection actually + # changes the determination. Re-saving recomputes it in place, so + # an occurrence pinned to a human identification keeps its taxon + # and must not inflate the metric. + prev_determination_id = occ.determination_id + occ.save(update_determination=True) + if occ.pk is not None and occ.determination_id != prev_determination_id: + updated_occurrence_ids.add(occ.pk) + determinations_after[occ.pk] = occ.determination_id + modified_occurrences = len(updated_occurrence_ids) + occcurrences_to_update.clear() + + progress = i / total if total > 0 else 1.0 + self.update_progress(progress) + self.report_stage_metrics( + { + "detections_checked": checked, + "detections_flagged": modified_detections, + "occurrences_updated": modified_occurrences, + } + ) + + i = 0 for i, det in enumerate(detections.iterator(), start=1): + # Write each full batch of 100 before the next begins, whether or not its rows were skipped. + if i > 1 and (i - 1) % 100 == 0: + flush(i - 1) bbox = det.get_bbox() if not bbox: self.logger.debug(f"Detection {det.pk}: no bbox, skipping") @@ -148,45 +192,7 @@ def run(self) -> None: flagged.append(det.pk) self.logger.debug(f"Marking detection {det.pk} as {not_identifiable_taxon.name}") - # Update progress every 100 detections - if i % 100 == 0 or i == total: - checked = i - modified_detections += len(detections_to_update) - - # with transaction.atomic(): - self.logger.info(f"Creating {len(classifications_to_create)} new classifications") - Classification.objects.bulk_create(classifications_to_create) - classifications_to_create.clear() - - self.logger.info(f"Marking {len(detections_to_update)} detections as {not_identifiable_taxon.name}") - for det in detections_to_update: - det.updated_at = timezone.now() - Detection.objects.bulk_update(detections_to_update, ["updated_at"]) - detections_to_update.clear() - - self.logger.info(f"Updating {len(occcurrences_to_update)} occurrences") - for occ in occcurrences_to_update: - # Count an occurrence only when flagging its detection actually - # changes the determination. Re-saving recomputes it in place, so - # an occurrence pinned to a human identification keeps its taxon - # and must not inflate the metric. - prev_determination_id = occ.determination_id - occ.save(update_determination=True) - if occ.pk is not None and occ.determination_id != prev_determination_id: - updated_occurrence_ids.add(occ.pk) - determinations_after[occ.pk] = occ.determination_id - modified_occurrences = len(updated_occurrence_ids) - occcurrences_to_update.clear() - - progress = i / total if total > 0 else 1.0 - self.update_progress(progress) - self.report_stage_metrics( - { - "detections_checked": checked, - "detections_flagged": modified_detections, - "occurrences_updated": modified_occurrences, - } - ) + flush(i) OccurrenceHistoryRecord.objects.bulk_create( OccurrenceHistoryRecord.build( @@ -203,7 +209,5 @@ def run(self) -> None: algorithm=self.algorithm, ) for occurrence_id, (taxon_before_id, detection_ids) in history.items() - # A detection flagged in a batch that never flushed was not saved, so it gets no record. - if occurrence_id in determinations_after ) self.logger.info(f"=== Completed {self.name}: {modified_detections} of {total} detections modified ===") diff --git a/ami/ml/post_processing/tests/test_occurrence_history_writers.py b/ami/ml/post_processing/tests/test_occurrence_history_writers.py index 7c3213717..385c0c151 100644 --- a/ami/ml/post_processing/tests/test_occurrence_history_writers.py +++ b/ami/ml/post_processing/tests/test_occurrence_history_writers.py @@ -124,3 +124,16 @@ def test_an_occurrence_with_nothing_flagged_gets_no_record(self): occurrence = self._singleton(self.captures[0], [0, 0, 500, 500]) SmallSizeFilterTask(logger=logger, occurrence_id=occurrence.pk, size_threshold=0.01).run() self.assertFalse(self.records("size_filter").exists()) + + def test_a_skipped_last_detection_still_saves_the_batch_before_it(self): + occurrence = self._singleton(self.captures[0], [0, 0, 10, 10]) + # Inserted last, so the scan reaches it last; it has no box and is skipped. + Detection.objects.create( + source_image=self.captures[1], bbox=None, timestamp=self.captures[1].timestamp, occurrence=occurrence + ) + + SmallSizeFilterTask(logger=logger, occurrence_id=occurrence.pk, size_threshold=0.01).run() + + occurrence.refresh_from_db() + self.assertEqual(occurrence.determination.name, "Not identifiable") + self.assertEqual(self.records("size_filter").get().payload["taxon_after_id"], occurrence.determination_id) From f0d1f577a056cb1ef9b0ea9a36c6ec5181ef0613 Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Wed, 23 Sep 2026 02:43:49 -0700 Subject: [PATCH 10/44] fix(occurrences): open the history of an occurrence the default filters hide The session view lists occurrences with the project's default filters off and opens their paths without them, and the history timeline is meant to sit beside it. The history endpoint still applied the score threshold and taxa filters, so an occurrence a reviewer could see and edit there returned 404. History now skips the default filters like the path and track-edit actions do, and visibility still goes through the usual object lookup. Skipping the filters also drops two queries. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01C7Xf6VPbwWtTumhjjF15g8 --- ami/main/api/views.py | 4 ++-- ami/main/test_occurrence_history.py | 15 ++++++++++++--- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/ami/main/api/views.py b/ami/main/api/views.py index 5b3310826..def37e92b 100644 --- a/ami/main/api/views.py +++ b/ami/main/api/views.py @@ -1634,8 +1634,8 @@ def get_serializer_class(self): # get_queryset. SINGLE_OCCURRENCE_ACTIONS = ("retrieve", "path", "history") # Actions the project's default filters never hide an occurrence from. The session - # view selects occurrences with those filters off and draws their paths. - UNFILTERED_ACTIONS = (*TRACK_EDIT_ACTIONS, "path") + # view selects occurrences with those filters off, then draws their paths and histories. + UNFILTERED_ACTIONS = (*TRACK_EDIT_ACTIONS, "path", "history") def get_queryset(self) -> QuerySet["Occurrence"]: """Occurrences this request may see, which is wider outside the list. diff --git a/ami/main/test_occurrence_history.py b/ami/main/test_occurrence_history.py index 9f1d34c43..f29971bcb 100644 --- a/ami/main/test_occurrence_history.py +++ b/ami/main/test_occurrence_history.py @@ -10,9 +10,9 @@ from ami.tests.fixtures.main import setup_test_project from ami.users.models import User -# Measured: two savepoints, the project, its default-filter taxa (2), the occurrence, then -# history records, their taxa, identifications and predictions. -HISTORY_QUERIES = 10 +# Measured: two savepoints, the project, the occurrence, then history records, their taxa, +# identifications and predictions. +HISTORY_QUERIES = 8 class OccurrenceHistoryPayloadTestCase(TestCase): @@ -218,3 +218,12 @@ def test_visible_to_whoever_can_open_the_occurrence(self): history = self.client.get(self.url()) self.assertEqual(history.status_code, codes[label]) self.assertEqual(history.status_code, detail.status_code) + + def test_an_occurrence_the_default_filters_hide_still_has_a_history(self): + """The session view lists occurrences with the default filters off, so their history must open too.""" + self.project.default_filters_score_threshold = 0.95 + self.project.save() + self.client.force_authenticate(user=self.reader) + detail = self.client.get(f"/api/v2/occurrences/{self.occurrence.pk}/?project_id={self.project.pk}") + self.assertEqual(detail.status_code, 404) + self.assertEqual(self.client.get(self.url()).status_code, 200) From dc5345cd17615a45b88e4921731f8c258d7a41df Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Wed, 23 Sep 2026 02:46:55 -0700 Subject: [PATCH 11/44] fix(occurrences): keep track reviews true to who confirmed which detections Three cases left the track review history out of step with the confirmation. A re-confirmation of an unchanged set was skipped even after the confirmation had been withdrawn or when a different person made it, so the timeline kept showing the old reviewer. Reviews moved onto an occurrence by a merge were compared with its next confirmation, so it reported the occurrence's own detections as added. A session split copied the confirmation onto each piece without any review, and the earliest piece's last review still listed the other pieces' detections. A review is now skipped only when the same person re-confirms a set that is still confirmed. Each review records the occurrence it was written for, and only those reviews are compared with a later confirmation. A split restates the confirmation as a review of each piece's own detections, with the original reviewer and time. The model docstrings now describe the two confirmation fields as the current confirmation rather than a cache of the latest review. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01C7Xf6VPbwWtTumhjjF15g8 --- ami/main/models.py | 3 +- ami/main/models_future/history.py | 74 ++++++++++++++++++++++------- ami/main/models_future/tracks.py | 13 +++-- ami/main/schemas.py | 6 +++ ami/main/test_occurrence_history.py | 38 +++++++++++++-- ami/main/tests.py | 20 ++++++++ 6 files changed, 128 insertions(+), 26 deletions(-) diff --git a/ami/main/models.py b/ami/main/models.py index 26eaa352a..f3553df61 100644 --- a/ami/main/models.py +++ b/ami/main/models.py @@ -4133,7 +4133,8 @@ class OccurrenceHistoryRecord(BaseModel): Identifications and predictions keep their own tables; the history endpoint merges all three. An algorithm result stands for the prediction change a run made, and a run writes at most one record per occurrence, none when it changed nothing about it. Reviews are - append-only; ``Occurrence.grouping_verified_at`` and ``_by`` cache the latest one. + append-only; ``Occurrence.grouping_verified_at`` and ``_by`` hold the current confirmation, + which unverifying or editing the track clears while its reviews stay. The payload is validated against the schema for its kind and subtype (ami/main/schemas.py). """ diff --git a/ami/main/models_future/history.py b/ami/main/models_future/history.py index 436fdc48f..2286c5656 100644 --- a/ami/main/models_future/history.py +++ b/ami/main/models_future/history.py @@ -11,8 +11,6 @@ import datetime import typing -from django.utils import timezone - from ami.main.models import Detection, Identification, Occurrence, OccurrenceHistoryRecord, Taxon, User from ami.main.schemas import TrackCompleteReviewPayload @@ -24,9 +22,13 @@ def latest_track_complete_review(occurrence: Occurrence) -> OccurrenceHistoryRecord | None: + """The latest review written for this occurrence, ignoring reviews a merge brought in.""" return ( OccurrenceHistoryRecord.objects.filter( - occurrence=occurrence, kind=OccurrenceHistoryRecord.Kind.REVIEW, subtype=TRACK_COMPLETE + occurrence=occurrence, + kind=OccurrenceHistoryRecord.Kind.REVIEW, + subtype=TRACK_COMPLETE, + payload__occurrence_id=occurrence.pk, ) .order_by("-timestamp", "-pk") .first() @@ -34,13 +36,54 @@ def latest_track_complete_review(occurrence: Occurrence) -> OccurrenceHistoryRec def record_track_complete_review( - occurrence: Occurrence, user: User, timestamp: datetime.datetime | None = None + occurrence: Occurrence, user: User, timestamp: datetime.datetime, was_confirmed: bool ) -> OccurrenceHistoryRecord | None: - """Record that ``user`` confirmed this occurrence's detections, unless they are the set last confirmed. + """Record that ``user`` confirmed this occurrence's detections. + + Nothing is written when the same person re-confirms a still-confirmed, unchanged set, + so the review list shows each distinct confirmation once. + """ + previous = latest_track_complete_review(occurrence) + record = _build_track_complete_review(occurrence, user.pk, timestamp, previous) + if ( + was_confirmed + and previous is not None + and previous.user_id == user.pk + and sorted(previous.payload["detection_ids"]) == record.payload["detection_ids"] + ): + return None + record.save() + return record + + +def carry_confirmation_over_split(occurrence: Occurrence, pieces: list[Occurrence]) -> None: + """Restate a confirmed occurrence's review for each piece a session split left. - Re-confirming an unchanged track adds nothing to the history, so the review list shows - only the sets a person actually looked at. The first review always posts. + Each review keeps the original reviewer and time but lists only its piece's + detections, so a later re-confirmation of a piece compares against what it holds. """ + if occurrence.grouping_verified_at is None: + return + OccurrenceHistoryRecord.objects.bulk_create( + _build_track_complete_review( + piece, + occurrence.grouping_verified_by_id, + occurrence.grouping_verified_at, + previous=None, + split_from_occurrence_id=occurrence.pk, + ) + for piece in [occurrence, *pieces] + ) + + +def _build_track_complete_review( + occurrence: Occurrence, + user_id: int | None, + timestamp: datetime.datetime, + previous: OccurrenceHistoryRecord | None, + split_from_occurrence_id: int | None = None, +) -> OccurrenceHistoryRecord: + """An unsaved review of the occurrence's current detections, with the change since ``previous``.""" frames = list( Detection.objects.valid() .filter(occurrence=occurrence) @@ -48,29 +91,26 @@ def record_track_complete_review( .values_list("pk", "source_image_id", "source_image__timestamp") ) detection_ids = sorted(pk for pk, _, _ in frames) - previous = latest_track_complete_review(occurrence) - previous_ids = sorted(previous.payload.get("detection_ids", [])) if previous else None - if previous_ids == detection_ids: - return None - + previous_ids = set(previous.payload["detection_ids"]) if previous else set(detection_ids) capture_times = [captured for _, _, captured in frames if captured is not None] payload = TrackCompleteReviewPayload( detection_ids=detection_ids, frames_count=len({capture_id for _, capture_id, _ in frames}), first_timestamp=min(capture_times, default=None), last_timestamp=max(capture_times, default=None), - detections_added=sorted(set(detection_ids) - set(previous_ids)) if previous_ids is not None else [], - detections_removed=sorted(set(previous_ids) - set(detection_ids)) if previous_ids is not None else [], + detections_added=sorted(set(detection_ids) - previous_ids), + detections_removed=sorted(previous_ids - set(detection_ids)), + occurrence_id=occurrence.pk, + split_from_occurrence_id=split_from_occurrence_id, ) record = OccurrenceHistoryRecord.build( occurrence_id=occurrence.pk, kind=OccurrenceHistoryRecord.Kind.REVIEW, subtype=TRACK_COMPLETE, payload=payload, - timestamp=timestamp or timezone.now(), - user=user, + timestamp=timestamp, ) - record.save() + record.user_id = user_id return record diff --git a/ami/main/models_future/tracks.py b/ami/main/models_future/tracks.py index 96815fd3d..66d0bfa99 100644 --- a/ami/main/models_future/tracks.py +++ b/ami/main/models_future/tracks.py @@ -52,7 +52,7 @@ update_calculated_fields_for_sessions_and_stations, update_occurrence_determination, ) -from ami.main.models_future.history import record_track_complete_review +from ami.main.models_future.history import carry_confirmation_over_split, record_track_complete_review from ami.main.models_future.track_stats import refresh_track_stats # Frame order within a track: capture time, then capture, then detection. The edits, @@ -159,7 +159,8 @@ def split_at_session_boundaries(occurrence: Occurrence) -> list[Occurrence]: keeps this occurrence and its identifications; each later piece is a new occurrence holding copies of them. Unlike a manual split, every piece keeps the grouping confirmation and the chain link to the next piece, since each piece is still the - whole track within its session and the link records that they are one animal. + whole track within its session and the link records that they are one animal. The + confirmation is restated as a review of each piece's own detections. Returns the new occurrences in time order, or an empty list when nothing was split. """ detections = occurrence.detections.select_related("source_image").order_by(*CAPTURE_ORDER) @@ -186,6 +187,7 @@ def split_at_session_boundaries(occurrence: Occurrence) -> list[Occurrence]: grouping_verified_at=occurrence.grouping_verified_at, grouping_verified_by_id=occurrence.grouping_verified_by_id, ) + carry_confirmation_over_split(occurrence, pieces) _copy_identifications(occurrence, pieces) for piece in [occurrence, *pieces]: @@ -540,13 +542,14 @@ def verify_grouping(occurrence: Occurrence, user: User) -> Occurrence: """Record that a person confirmed this occurrence holds the right detections. This is the label the tracking methods are scored against, so it is deliberately - an explicit act — no operation in this module sets it as a side effect. The review - goes into the occurrence's history; the two fields here cache the latest one. + an explicit act — no operation in this module sets it as a side effect. The two + fields hold the current confirmation; the history keeps every review. """ + was_confirmed = occurrence.grouping_verified_at is not None occurrence.grouping_verified_at = timezone.now() occurrence.grouping_verified_by = user occurrence.save(update_fields=["grouping_verified_at", "grouping_verified_by"]) - record_track_complete_review(occurrence, user, timestamp=occurrence.grouping_verified_at) + record_track_complete_review(occurrence, user, occurrence.grouping_verified_at, was_confirmed) return occurrence diff --git a/ami/main/schemas.py b/ami/main/schemas.py index 27e843a3a..478cbef31 100644 --- a/ami/main/schemas.py +++ b/ami/main/schemas.py @@ -51,6 +51,12 @@ class TrackCompleteReviewPayload(HistoryPayload): # Compared with the previous track_complete review; the first review lists none. detections_added: list[int] = [] detections_removed: list[int] = [] + # The occurrence reviewed. A merge moves reviews onto the surviving occurrence, and + # only its own reviews are compared with a later confirmation of it. + occurrence_id: int | None = None + # Set when regrouping split a confirmed occurrence at a session boundary: the + # confirmation carries over to each piece, restated for the piece's own detections. + split_from_occurrence_id: int | None = None # Keyed by (kind, subtype). A subtype missing here cannot be written. diff --git a/ami/main/test_occurrence_history.py b/ami/main/test_occurrence_history.py index f29971bcb..6eeb18bca 100644 --- a/ami/main/test_occurrence_history.py +++ b/ami/main/test_occurrence_history.py @@ -9,6 +9,7 @@ from ami.ml.models import Algorithm from ami.tests.fixtures.main import setup_test_project from ami.users.models import User +from ami.users.roles import MLDataManager # Measured: two savepoints, the project, the occurrence, then history records, their taxa, # identifications and predictions. @@ -64,10 +65,10 @@ def test_save_validates_too_and_an_unknown_subtype_is_refused(self): class TrackCompleteReviewTestCase(main_tests.TrackFixtureTestCase): - """Confirming a track posts a review only when the confirmed set of detections differs from the last one.""" + """Confirming a track posts a review unless the same person re-confirms a still-confirmed, unchanged set.""" - def verify(self, occurrence: Occurrence | None = None): - self.client.force_authenticate(user=self.curator) + def verify(self, occurrence: Occurrence | None = None, user: User | None = None): + self.client.force_authenticate(user=user or self.curator) occurrence = occurrence or self.occurrence response = self.client.post(f"/api/v2/occurrences/{occurrence.pk}/verify-grouping/", format="json") self.assertEqual(response.status_code, 200, response.data) @@ -106,6 +107,37 @@ def test_a_confirmation_after_an_edit_records_what_changed(self): self.assertEqual(second.payload["frames_count"], len(self.captures) - 1) self.assertEqual(first.payload["detection_ids"], sorted(d.pk for d in self.detections)) + def test_a_confirmation_after_it_was_withdrawn_or_by_someone_else_posts_a_review(self): + self.verify() + self.client.force_authenticate(user=self.curator) + response = self.client.post(f"/api/v2/occurrences/{self.occurrence.pk}/unverify-grouping/", format="json") + self.assertEqual(response.status_code, 200, response.data) + self.verify() + other_curator = User.objects.create_user(email="second-curator@insectai.org") # type: ignore + MLDataManager.assign_user(other_curator, self.project) + self.verify(user=other_curator) + + reviews = list(self.reviews()) + self.assertEqual([r.user for r in reviews], [self.curator, self.curator, other_curator]) + self.assertTrue(all(r.payload["detections_added"] == [] for r in reviews)) + self.occurrence.refresh_from_db() + self.assertEqual(self.occurrence.grouping_verified_at, reviews[-1].timestamp) + + def test_a_confirmation_after_a_merge_compares_with_this_occurrences_own_review(self): + other, other_detections = self._make_track(1, captures=self._make_captures_after(1)) + self.verify() + self.verify(other) + self.client.force_authenticate(user=self.curator) + response = self.client.post( + f"/api/v2/occurrences/{self.occurrence.pk}/merge/", {"occurrence_ids": [other.pk]}, format="json" + ) + self.assertEqual(response.status_code, 200, response.data) + self.verify() + + latest = self.reviews().last() + self.assertEqual(latest.payload["detections_added"], [d.pk for d in other_detections]) + self.assertEqual(latest.payload["detections_removed"], []) + def test_merging_an_occurrence_keeps_its_reviews(self): other, _ = self._make_track(1, captures=self._make_captures_after(1)) self.verify(other) diff --git a/ami/main/tests.py b/ami/main/tests.py index 2181cce06..dcc57a87e 100644 --- a/ami/main/tests.py +++ b/ami/main/tests.py @@ -1537,6 +1537,26 @@ def test_every_piece_keeps_the_grouping_confirmation(self): self.assertEqual(piece.grouping_verified_at, verified_at) self.assertEqual(piece.grouping_verified_by_id, self.user.pk) + def test_each_piece_gets_a_review_of_its_own_detections(self): + from ami.main.models_future.history import latest_track_complete_review + from ami.main.models_future.tracks import verify_grouping + + self._group(gap_hours=6) + occurrence, _ = self._make_track(self.captures) + verify_grouping(occurrence, self.user) + verified_at = Occurrence.objects.get(pk=occurrence.pk).grouping_verified_at + + self._group(gap_hours=2) + + for piece in Occurrence.objects.filter(deployment=self.deployment): + review = latest_track_complete_review(piece) + self.assertEqual(review.payload["detection_ids"], sorted(self._detection_ids(piece))) + self.assertEqual((review.user_id, review.timestamp), (self.user.pk, verified_at)) + self.assertEqual(review.payload["split_from_occurrence_id"], occurrence.pk) + # Re-confirming the piece as it stands changes nothing, so it records nothing. + verify_grouping(piece, self.user) + self.assertEqual(latest_track_complete_review(piece).pk, review.pk) + def test_identifications_are_copied_to_every_piece(self): self._group(gap_hours=6) occurrence, _ = self._make_track(self.captures) From 655a7c9b6ebc4acfb872a0c035a8939b7a4f3ad1 Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Wed, 23 Sep 2026 03:11:54 -0700 Subject: [PATCH 12/44] feat(ui): read an occurrence's history as one list of timeline cards Adds types for the occurrence history endpoint, a query hook keyed under the occurrences prefix so existing identification and track mutations refresh it, and helpers that map each history entry to the card that shows it and tell whether a track changed since it was last marked complete. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01C7Xf6VPbwWtTumhjjF15g8 --- .../hooks/occurrences/useOccurrenceHistory.ts | 33 ++ .../models/occurrence-history.test.ts | 271 ++++++++++++++ .../models/occurrence-history.ts | 333 ++++++++++++++++++ 3 files changed, 637 insertions(+) create mode 100644 ui/src/data-services/hooks/occurrences/useOccurrenceHistory.ts create mode 100644 ui/src/data-services/models/occurrence-history.test.ts create mode 100644 ui/src/data-services/models/occurrence-history.ts diff --git a/ui/src/data-services/hooks/occurrences/useOccurrenceHistory.ts b/ui/src/data-services/hooks/occurrences/useOccurrenceHistory.ts new file mode 100644 index 000000000..9e3a2cdc7 --- /dev/null +++ b/ui/src/data-services/hooks/occurrences/useOccurrenceHistory.ts @@ -0,0 +1,33 @@ +import { API_ROUTES, API_URL } from 'data-services/constants' +import { ServerOccurrenceHistoryEntry } from 'data-services/models/occurrence-history' +import { useAuthorizedQuery } from '../auth/useAuthorizedQuery' + +// Under the occurrences prefix, so every mutation that invalidates occurrences refreshes it. +export const getOccurrenceHistoryQueryKey = (occurrenceId: string) => [ + API_ROUTES.OCCURRENCES, + occurrenceId, + 'history', +] + +export const useOccurrenceHistory = ({ + occurrenceId, + projectId, +}: { + occurrenceId: string + projectId?: string +}): { + entries?: ServerOccurrenceHistoryEntry[] + isLoading: boolean + error?: unknown +} => { + const params = new URLSearchParams(projectId ? { project_id: projectId } : {}) + const { data, isLoading, error } = useAuthorizedQuery< + ServerOccurrenceHistoryEntry[] + >({ + enabled: !!occurrenceId, + queryKey: getOccurrenceHistoryQueryKey(occurrenceId), + url: `${API_URL}/${API_ROUTES.OCCURRENCES}/${occurrenceId}/history/?${params}`, + }) + + return { entries: data, isLoading, error } +} diff --git a/ui/src/data-services/models/occurrence-history.test.ts b/ui/src/data-services/models/occurrence-history.test.ts new file mode 100644 index 000000000..56e14ecfd --- /dev/null +++ b/ui/src/data-services/models/occurrence-history.test.ts @@ -0,0 +1,271 @@ +import { getOccurrenceHistoryQueryKey } from 'data-services/hooks/occurrences/useOccurrenceHistory' +import { API_ROUTES } from 'data-services/constants' +import { Algorithm } from './algorithm' +import { HumanIdentification, MachinePrediction } from './occurrence-details' +import { + getFallbackTimelineItems, + getFoldedPrediction, + getTimelineItems, + isEditedSinceComplete, + ServerOccurrenceHistoryEntry, + TrackCompleteReviewEntry, +} from './occurrence-history' +import { Taxon } from './taxa' +import { UserPermission } from 'utils/user/types' + +const NOCTUA = { id: 3, name: 'Noctua pronuba', rank: 'SPECIES' } +const XESTIA = { id: 4, name: 'Xestia c-nigrum', rank: 'SPECIES' } + +const base = { + algorithm: null, + job: null, + score: null, + subtype: null, + taxon: null, + taxon_before: null, + timestamp: '2026-04-29T22:00:00', + user: null, +} + +const identificationEntry = (id: number): ServerOccurrenceHistoryEntry => ({ + ...base, + id, + payload: { + agreed_with_identification_id: null, + agreed_with_prediction_id: null, + comment: 'Looks right', + withdrawn: false, + }, + taxon: NOCTUA, + type: 'identification', + user: { id: 9, image: null, name: '' }, +}) + +const predictionEntry = ( + id: number, + taxon: typeof NOCTUA | null = XESTIA +): ServerOccurrenceHistoryEntry => ({ + ...base, + algorithm: { id: 7, key: 'classifier', name: 'Classifier' }, + id, + payload: { applied_to_id: null, detection_id: 1, terminal: true }, + score: 0.8, + taxon, + type: 'prediction', +}) + +const review = ( + id: number, + detectionIds: number[], + occurrenceId: number | null = 100 +): TrackCompleteReviewEntry => ({ + ...base, + id, + payload: { + detection_ids: detectionIds, + detections_added: [], + detections_removed: [], + first_timestamp: null, + frames_count: detectionIds.length, + last_timestamp: null, + occurrence_id: occurrenceId, + }, + subtype: 'track_complete', + type: 'review', +}) + +const classMasking: ServerOccurrenceHistoryEntry = { + ...base, + algorithm: { id: 12, key: 'mask', name: 'Masked classifier' }, + id: 5, + payload: { + detection_ids: [1], + source_algorithm_id: 7, + taxa_list_id: 2, + taxon_after_id: 3, + taxon_before_id: 4, + }, + subtype: 'class_masking', + taxon: NOCTUA, + taxon_before: XESTIA, + type: 'algorithm_result', +} + +const ownIdentification: HumanIdentification = { + comment: '', + createdAt: '2026-04-29T22:00:00', + id: '1', + user: { name: 'Someone' }, + userPermissions: [UserPermission.Delete], +} + +const ownPrediction = ( + id: string, + algorithmId: number, + taxon = NOCTUA, + createdAt = '2026-04-29T21:00:00' +): MachinePrediction => ({ + algorithm: new Algorithm({ id: algorithmId }), + createdAt, + id, + score: 0.9, + taxon: new Taxon({ ...taxon, id: `${taxon.id}`, cover_image_url: null }), + terminal: true, + userPermissions: [UserPermission.Update], +}) + +describe('getTimelineItems', () => { + test('maps each entry type to its card, keeping the server order', () => { + const items = getTimelineItems({ + entries: [ + review(8, [1]), + classMasking, + identificationEntry(2), + predictionEntry(6), + ], + identifications: [], + predictions: [], + }) + + expect(items.map((item) => item.type)).toEqual([ + 'review', + 'algorithm_result', + 'identification', + 'prediction', + ]) + }) + + test("reuses the occurrence's own records, which carry the viewer's permissions", () => { + const prediction = ownPrediction('6', 7) + const items = getTimelineItems({ + entries: [identificationEntry(1), predictionEntry(6)], + identifications: [ownIdentification], + predictions: [prediction], + }) + + expect(items[0]).toMatchObject({ identification: ownIdentification }) + expect(items[1]).toMatchObject({ prediction }) + }) + + test('builds a read-only card when the occurrence lacks the record', () => { + const [item] = getTimelineItems({ + determinationTaxonId: '3', + entries: [identificationEntry(2)], + identifications: [], + predictions: [], + }) + + expect(item.type === 'identification' && item.identification).toMatchObject( + { + applied: true, + comment: 'Looks right', + user: { id: '9', name: 'Anonymous user' }, + userPermissions: [], + } + ) + }) + + test('drops unknown subtypes and predictions without a taxon', () => { + const unknown = { + ...classMasking, + subtype: 'something_new', + } as unknown as ServerOccurrenceHistoryEntry + + expect( + getTimelineItems({ + entries: [unknown, predictionEntry(6, null)], + identifications: [], + predictions: [], + }) + ).toEqual([]) + }) +}) + +describe('getFallbackTimelineItems', () => { + test('merges identifications and predictions newest first', () => { + const items = getFallbackTimelineItems({ + identifications: [ownIdentification], + predictions: [ + ownPrediction('6', 7, NOCTUA, '2026-04-29T23:00:00'), + ownPrediction('7', 7, NOCTUA, '2026-04-29T20:00:00'), + ], + }) + + expect(items.map((item) => item.id)).toEqual([ + 'prediction-6', + 'identification-1', + 'prediction-7', + ]) + }) +}) + +describe('getFoldedPrediction', () => { + test('finds the prediction behind an algorithm result by algorithm and taxon', () => { + const folded = ownPrediction('6', 12) + + expect( + getFoldedPrediction(classMasking as never, [ + ownPrediction('5', 7), + ownPrediction('4', 12, XESTIA), + folded, + ]) + ).toBe(folded) + }) +}) + +describe('isEditedSinceComplete', () => { + test('false without a review', () => { + expect( + isEditedSinceComplete({ + detectionIds: ['1'], + entries: [identificationEntry(1)], + occurrenceId: '100', + }) + ).toBe(false) + }) + + test('false when the detections match the latest review in any order', () => { + expect( + isEditedSinceComplete({ + detectionIds: ['2', '1'], + entries: [review(9, [1, 2]), review(8, [1])], + occurrenceId: '100', + }) + ).toBe(false) + }) + + test('true when a detection was added or removed since the latest review', () => { + const entries = [review(9, [1, 2])] + + expect( + isEditedSinceComplete({ + detectionIds: ['1', '2', '3'], + entries, + occurrenceId: '100', + }) + ).toBe(true) + expect( + isEditedSinceComplete({ + detectionIds: ['1'], + entries, + occurrenceId: '100', + }) + ).toBe(true) + }) + + test('ignores reviews a merge brought in from another occurrence', () => { + expect( + isEditedSinceComplete({ + detectionIds: ['1', '2', '5'], + entries: [review(9, [5], 200), review(8, [1, 2, 5])], + occurrenceId: '100', + }) + ).toBe(false) + }) +}) + +describe('getOccurrenceHistoryQueryKey', () => { + test('sits under the occurrences key that mutations invalidate', () => { + expect(getOccurrenceHistoryQueryKey('100')[0]).toBe(API_ROUTES.OCCURRENCES) + }) +}) diff --git a/ui/src/data-services/models/occurrence-history.ts b/ui/src/data-services/models/occurrence-history.ts new file mode 100644 index 000000000..c207e5924 --- /dev/null +++ b/ui/src/data-services/models/occurrence-history.ts @@ -0,0 +1,333 @@ +import { STRING, translate } from 'utils/language' +import { Algorithm } from './algorithm' +import { HumanIdentification, MachinePrediction } from './occurrence-details' +import { Taxon } from './taxa' + +export interface ServerHistoryTaxon { + id: number + name: string + rank: string +} + +export interface ServerHistoryUser { + id: number + image: string | null + name: string +} + +export interface ServerHistoryAlgorithm { + id: number + key: string + name: string +} + +export interface ServerHistoryJob { + id: number + name: string +} + +interface ServerDeterminationChangePayload { + taxon_after_id: number | null + taxon_before_id: number | null +} + +export interface ServerTrackingPayload + extends ServerDeterminationChangePayload { + cost_max: number | null + cost_mean: number | null + detections_count: number + feature_algorithm_id: number | null + frames_linked: number + occurrences_merged: number[] + settings: Record +} + +export interface ServerClassMaskingPayload + extends ServerDeterminationChangePayload { + detection_ids: number[] + source_algorithm_id: number + taxa_list_id: number +} + +export interface ServerSizeFilterPayload + extends ServerDeterminationChangePayload { + detection_ids: number[] + size_threshold: number +} + +export interface ServerTrackCompletePayload { + detection_ids: number[] + detections_added: number[] + detections_removed: number[] + first_timestamp: string | null + frames_count: number + last_timestamp: string | null + occurrence_id?: number | null + split_from_occurrence_id?: number | null +} + +export interface ServerIdentificationPayload { + agreed_with_identification_id: number | null + agreed_with_prediction_id: number | null + comment: string + withdrawn: boolean +} + +export interface ServerPredictionPayload { + applied_to_id: number | null + detection_id: number | null + terminal: boolean | null +} + +interface ServerHistoryEntryBase { + algorithm: ServerHistoryAlgorithm | null + id: number + job: ServerHistoryJob | null + payload: Payload + score: number | null + subtype: Subtype + taxon: ServerHistoryTaxon | null + taxon_before: ServerHistoryTaxon | null + timestamp: string + type: Type + user: ServerHistoryUser | null +} + +export type TrackingResultEntry = ServerHistoryEntryBase< + 'algorithm_result', + 'tracking', + ServerTrackingPayload +> +export type ClassMaskingResultEntry = ServerHistoryEntryBase< + 'algorithm_result', + 'class_masking', + ServerClassMaskingPayload +> +export type SizeFilterResultEntry = ServerHistoryEntryBase< + 'algorithm_result', + 'size_filter', + ServerSizeFilterPayload +> +export type AlgorithmResultEntry = + | TrackingResultEntry + | ClassMaskingResultEntry + | SizeFilterResultEntry +export type TrackCompleteReviewEntry = ServerHistoryEntryBase< + 'review', + 'track_complete', + ServerTrackCompletePayload +> +export type IdentificationEntry = ServerHistoryEntryBase< + 'identification', + null, + ServerIdentificationPayload +> +export type PredictionEntry = ServerHistoryEntryBase< + 'prediction', + null, + ServerPredictionPayload +> + +export type ServerOccurrenceHistoryEntry = + | AlgorithmResultEntry + | TrackCompleteReviewEntry + | IdentificationEntry + | PredictionEntry + +export type TimelineItem = + | { + type: 'identification' + id: string + identification: HumanIdentification + } + | { type: 'prediction'; id: string; prediction: MachinePrediction } + | { type: 'algorithm_result'; id: string; entry: AlgorithmResultEntry } + | { type: 'review'; id: string; entry: TrackCompleteReviewEntry } + +const ALGORITHM_RESULT_SUBTYPES = ['tracking', 'class_masking', 'size_filter'] + +export const convertHistoryTaxon = (taxon: ServerHistoryTaxon) => + new Taxon({ ...taxon, id: `${taxon.id}`, cover_image_url: null }) + +const toIdentification = ( + entry: IdentificationEntry, + determinationTaxonId?: string +): HumanIdentification => { + const taxon = entry.taxon ? convertHistoryTaxon(entry.taxon) : undefined + + return { + applied: !!taxon && taxon.id === determinationTaxonId, + comment: entry.payload.comment, + createdAt: entry.timestamp, + id: `${entry.id}`, + overridden: entry.payload.withdrawn, + taxon, + user: entry.user + ? { + id: `${entry.user.id}`, + image: entry.user.image ?? undefined, + name: entry.user.name?.length + ? entry.user.name + : translate(STRING.ANONYMOUS_USER), + } + : { name: translate(STRING.ANONYMOUS_USER) }, + userPermissions: [], + } +} + +const toPrediction = ( + entry: PredictionEntry & { taxon: ServerHistoryTaxon }, + determinationTaxonId?: string +): MachinePrediction => { + const taxon = convertHistoryTaxon(entry.taxon) + + return { + algorithm: new Algorithm(entry.algorithm ?? {}), + applied: taxon.id === determinationTaxonId, + createdAt: entry.timestamp, + id: `${entry.id}`, + overridden: taxon.id !== determinationTaxonId, + score: entry.score ?? 0, + taxon, + terminal: !!entry.payload.terminal, + userPermissions: [], + } +} + +/** + * The history as cards to render, newest first. Identifications and predictions reuse the + * occurrence's own records when it has them, since only those carry the viewer's permissions. + */ +export const getTimelineItems = ({ + determinationTaxonId, + entries, + identifications, + predictions, +}: { + determinationTaxonId?: string + entries: ServerOccurrenceHistoryEntry[] + identifications: HumanIdentification[] + predictions: MachinePrediction[] +}): TimelineItem[] => + entries.flatMap((entry): TimelineItem[] => { + const id = `${entry.type}-${entry.id}` + + switch (entry.type) { + case 'identification': { + const identification = + identifications.find((i) => i.id === `${entry.id}`) ?? + toIdentification(entry, determinationTaxonId) + + return [{ type: 'identification', id, identification }] + } + case 'prediction': { + const { taxon } = entry + const prediction = + predictions.find((p) => p.id === `${entry.id}`) ?? + (taxon + ? toPrediction({ ...entry, taxon }, determinationTaxonId) + : undefined) + + return prediction ? [{ type: 'prediction', id, prediction }] : [] + } + case 'algorithm_result': + return ALGORITHM_RESULT_SUBTYPES.includes(entry.subtype) + ? [{ type: 'algorithm_result', id, entry }] + : [] + case 'review': + return entry.subtype === 'track_complete' + ? [{ type: 'review', id, entry }] + : [] + default: + return [] + } + }) + +/** Identifications and predictions merged newest first, for when the history is unavailable. */ +export const getFallbackTimelineItems = ({ + identifications, + predictions, +}: { + identifications: HumanIdentification[] + predictions: MachinePrediction[] +}): TimelineItem[] => + [ + ...identifications.map( + (identification): TimelineItem => ({ + type: 'identification', + id: `identification-${identification.id}`, + identification, + }) + ), + ...predictions.map( + (prediction): TimelineItem => ({ + type: 'prediction', + id: `prediction-${prediction.id}`, + prediction, + }) + ), + ] + .map((item) => ({ item, time: new Date(getCreatedAt(item)).getTime() })) + .sort((a, b) => b.time - a.time) + .map(({ item }) => item) + +const getCreatedAt = (item: TimelineItem) => { + switch (item.type) { + case 'identification': + return item.identification.createdAt + case 'prediction': + return item.prediction.createdAt + default: + return item.entry.timestamp + } +} + +/** The prediction an algorithm result stands in for, so its taxon can still be agreed with. */ +export const getFoldedPrediction = ( + entry: AlgorithmResultEntry, + predictions: MachinePrediction[] +) => + entry.algorithm && entry.taxon + ? predictions.find( + (p) => + `${p.algorithm?.id}` === `${entry.algorithm?.id}` && + p.taxon.id === `${entry.taxon?.id}` + ) + : undefined + +/** Reviews a merge brought in from another occurrence are not reviews of this one. */ +export const getLatestTrackCompleteReview = ( + entries: ServerOccurrenceHistoryEntry[], + occurrenceId: string +) => + entries.find( + (entry): entry is TrackCompleteReviewEntry => + entry.type === 'review' && + entry.subtype === 'track_complete' && + (entry.payload.occurrence_id == null || + `${entry.payload.occurrence_id}` === occurrenceId) + ) + +export const isEditedSinceComplete = ({ + detectionIds, + entries, + occurrenceId, +}: { + detectionIds: string[] + entries: ServerOccurrenceHistoryEntry[] + occurrenceId: string +}) => { + const review = getLatestTrackCompleteReview(entries, occurrenceId) + + if (!review) { + return false + } + + const reviewed = new Set(review.payload.detection_ids.map((id) => `${id}`)) + const current = new Set(detectionIds) + + return ( + reviewed.size !== current.size || + Array.from(current).some((id) => !reviewed.has(id)) + ) +} From f01bfdfaca4e5d76d72cda12a25f6f143cf144dd Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Wed, 23 Sep 2026 03:11:54 -0700 Subject: [PATCH 13/44] feat(ui): show identifications, predictions, algorithm results and reviews in one timeline The identification tab now lists everything that happened to an occurrence newest first, from the history endpoint. Identification and prediction cards keep their existing actions; tracking, class masking and size filter runs get a compact result card, and each "Mark complete" review gets a card with its frames, time span and the change since the previous review. The track panel says when a track was edited after it was marked complete. When the history cannot load, the tab falls back to identifications and predictions. See #1433. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01C7Xf6VPbwWtTumhjjF15g8 --- .../identification-card/algorithm-result.tsx | 175 ++++++++++++++++++ .../grouping-confirmation.tsx | 38 ---- .../identification-card/history-stats.tsx | 30 +++ .../occurrence-timeline.tsx | 98 ++++++++++ .../identification-card/track-review.tsx | 90 +++++++++ .../occurrence-details/occurrence-details.tsx | 47 +++-- .../track/grouping-actions.tsx | 7 + ui/src/utils/language.ts | 44 ++++- 8 files changed, 465 insertions(+), 64 deletions(-) create mode 100644 ui/src/pages/occurrence-details/identification-card/algorithm-result.tsx delete mode 100644 ui/src/pages/occurrence-details/identification-card/grouping-confirmation.tsx create mode 100644 ui/src/pages/occurrence-details/identification-card/history-stats.tsx create mode 100644 ui/src/pages/occurrence-details/identification-card/occurrence-timeline.tsx create mode 100644 ui/src/pages/occurrence-details/identification-card/track-review.tsx diff --git a/ui/src/pages/occurrence-details/identification-card/algorithm-result.tsx b/ui/src/pages/occurrence-details/identification-card/algorithm-result.tsx new file mode 100644 index 000000000..7773c893d --- /dev/null +++ b/ui/src/pages/occurrence-details/identification-card/algorithm-result.tsx @@ -0,0 +1,175 @@ +import { + AlgorithmResultEntry, + getFoldedPrediction, + ServerHistoryTaxon, +} from 'data-services/models/occurrence-history' +import { OccurrenceDetails as Occurrence } from 'data-services/models/occurrence-details' +import { FilterIcon, RouteIcon, RulerIcon } from 'lucide-react' +import { IdentificationCard } from 'nova-ui-kit' +import { Link, useParams } from 'react-router-dom' +import { APP_ROUTES } from 'utils/constants' +import { getFormatedDateTimeString } from 'utils/date/getFormatedDateTimeString/getFormatedDateTimeString' +import { STRING, translate } from 'utils/language' +import { UserInfo, UserPermission } from 'utils/user/types' +import { Agree } from '../agree/agree' +import { + HistoryStat, + HistoryStats, + HistoryTime, + HistoryTypeBadge, +} from './history-stats' + +const SUBTYPES = { + class_masking: { icon: FilterIcon, label: STRING.HISTORY_CLASS_MASKING }, + size_filter: { icon: RulerIcon, label: STRING.HISTORY_SIZE_FILTER }, + tracking: { icon: RouteIcon, label: STRING.HISTORY_TRACKING }, +} + +const getDeterminationLabel = ( + before: ServerHistoryTaxon | null, + after: ServerHistoryTaxon | null +) => { + const notAvailable = translate(STRING.VALUE_NOT_AVAILABLE) + + if (before?.id === after?.id) { + return translate(STRING.HISTORY_DETERMINATION_UNCHANGED, { + name: after?.name ?? notAvailable, + }) + } + + return `${before?.name ?? notAvailable} → ${after?.name ?? notAvailable}` +} + +export const AlgorithmResult = ({ + currentUser, + entry, + occurrence, +}: { + currentUser?: UserInfo + entry: AlgorithmResultEntry + occurrence: Occurrence +}) => { + const { projectId } = useParams() + const { icon: Icon, label } = SUBTYPES[entry.subtype] + const foldedPrediction = getFoldedPrediction( + entry, + occurrence.machinePredictions + ) + const canAgree = + !!foldedPrediction && + occurrence.userPermissions.includes(UserPermission.Update) + + const stats: HistoryStat[] = [] + switch (entry.subtype) { + case 'tracking': + stats.push( + { + label: translate(STRING.HISTORY_FRAMES_LINKED), + value: entry.payload.frames_linked, + }, + { + label: translate(STRING.HISTORY_OCCURRENCES_MERGED), + value: entry.payload.occurrences_merged.length, + } + ) + break + case 'class_masking': + stats.push( + { + label: translate(STRING.HISTORY_DETERMINATION), + value: getDeterminationLabel(entry.taxon_before, entry.taxon), + }, + { + label: translate(STRING.HISTORY_SPECIES_LIST), + value: ( + + {translate(STRING.HISTORY_SPECIES_LIST_ID, { + id: `${entry.payload.taxa_list_id}`, + })} + + ), + }, + { + label: translate(STRING.HISTORY_DETECTIONS_AFFECTED), + value: entry.payload.detection_ids.length, + } + ) + break + case 'size_filter': + stats.push( + { + label: translate(STRING.HISTORY_DETERMINATION), + value: getDeterminationLabel(entry.taxon_before, entry.taxon), + }, + { + label: translate(STRING.HISTORY_SIZE_THRESHOLD), + value: entry.payload.size_threshold, + }, + { + label: translate(STRING.HISTORY_DETECTIONS_AFFECTED), + value: entry.payload.detection_ids.length, + } + ) + break + } + if (entry.job) { + stats.push({ + label: translate(STRING.FIELD_LABEL_JOB), + value: ( + + {entry.job.name} + + ), + }) + } + + return ( +
+ + } + subTitle={entry.algorithm ? translate(label) : undefined} + title={entry.algorithm?.name ?? translate(label)} + titleAddon={ + + } + > + + {canAgree && foldedPrediction ? ( +
+ +
+ ) : null} + +
+ ) +} diff --git a/ui/src/pages/occurrence-details/identification-card/grouping-confirmation.tsx b/ui/src/pages/occurrence-details/identification-card/grouping-confirmation.tsx deleted file mode 100644 index 6c5a660c5..000000000 --- a/ui/src/pages/occurrence-details/identification-card/grouping-confirmation.tsx +++ /dev/null @@ -1,38 +0,0 @@ -import { OccurrenceDetails as Occurrence } from 'data-services/models/occurrence-details' -import { UserIcon } from 'lucide-react' -import { IdentificationCard } from 'nova-ui-kit' -import { getFormatedDateTimeString } from 'utils/date/getFormatedDateTimeString/getFormatedDateTimeString' -import { STRING, translate } from 'utils/language' - -export const GroupingConfirmation = ({ - occurrence, -}: { - occurrence: Occurrence -}) => { - const user = occurrence.groupingVerifiedBy - const formattedTime = occurrence.groupingVerifiedAt - ? getFormatedDateTimeString({ date: occurrence.groupingVerifiedAt }) - : translate(STRING.VALUE_NOT_AVAILABLE) - - return ( -
- - {formattedTime} - - - ) : ( - - ) - } - subTitle={translate(STRING.TRACK_GROUPING_CONFIRMED_BY, { - date: formattedTime, - name: user?.name ?? translate(STRING.ANONYMOUS_USER), - })} - title={translate(STRING.TRACK_GROUPING_CONFIRMED)} - /> -
- ) -} diff --git a/ui/src/pages/occurrence-details/identification-card/history-stats.tsx b/ui/src/pages/occurrence-details/identification-card/history-stats.tsx new file mode 100644 index 000000000..bdbdb7a82 --- /dev/null +++ b/ui/src/pages/occurrence-details/identification-card/history-stats.tsx @@ -0,0 +1,30 @@ +import { Fragment, ReactNode } from 'react' + +export interface HistoryStat { + label: string + value: ReactNode +} + +export const HistoryStats = ({ stats }: { stats: HistoryStat[] }) => + stats.length ? ( +
+ {stats.map(({ label, value }) => ( + + {label} + {value} + + ))} +
+ ) : null + +export const HistoryTime = ({ label }: { label: string }) => ( + + {label} + +) + +export const HistoryTypeBadge = ({ label }: { label: string }) => ( + + {label} + +) diff --git a/ui/src/pages/occurrence-details/identification-card/occurrence-timeline.tsx b/ui/src/pages/occurrence-details/identification-card/occurrence-timeline.tsx new file mode 100644 index 000000000..bf7b051a4 --- /dev/null +++ b/ui/src/pages/occurrence-details/identification-card/occurrence-timeline.tsx @@ -0,0 +1,98 @@ +import { OccurrenceDetails as Occurrence } from 'data-services/models/occurrence-details' +import { + getFallbackTimelineItems, + getTimelineItems, + ServerOccurrenceHistoryEntry, +} from 'data-services/models/occurrence-history' +import { Loader2Icon } from 'lucide-react' +import { useMemo } from 'react' +import { STRING, translate } from 'utils/language' +import { UserInfo } from 'utils/user/types' +import { AlgorithmResult } from './algorithm-result' +import { HumanIdentification } from './human-identification' +import { MachinePrediction } from './machine-prediction' +import { TrackReview } from './track-review' + +export const OccurrenceTimeline = ({ + currentUser, + entries, + error, + isLoading, + occurrence, +}: { + currentUser?: UserInfo + entries?: ServerOccurrenceHistoryEntry[] + error?: unknown + isLoading: boolean + occurrence: Occurrence +}) => { + const items = useMemo( + () => + entries + ? getTimelineItems({ + determinationTaxonId: occurrence.determinationTaxon?.id, + entries, + identifications: occurrence.humanIdentifications, + predictions: occurrence.machinePredictions, + }) + : getFallbackTimelineItems({ + identifications: occurrence.humanIdentifications, + predictions: occurrence.machinePredictions, + }), + [entries, occurrence] + ) + + return ( + <> + {isLoading ? ( +
+ +
+ ) : null} + {error ? ( +

+ {translate(STRING.HISTORY_LOAD_ERROR)} +

+ ) : null} + {!isLoading && !items.length ? ( +

+ {translate(STRING.HISTORY_EMPTY)} +

+ ) : null} + {items.map((item) => { + switch (item.type) { + case 'identification': + return ( + + ) + case 'prediction': + return ( + + ) + case 'algorithm_result': + return ( + + ) + case 'review': + return + } + })} + + ) +} diff --git a/ui/src/pages/occurrence-details/identification-card/track-review.tsx b/ui/src/pages/occurrence-details/identification-card/track-review.tsx new file mode 100644 index 000000000..34f39ae11 --- /dev/null +++ b/ui/src/pages/occurrence-details/identification-card/track-review.tsx @@ -0,0 +1,90 @@ +import { TrackCompleteReviewEntry } from 'data-services/models/occurrence-history' +import { UserIcon } from 'lucide-react' +import { IdentificationCard } from 'nova-ui-kit' +import { Link, useParams } from 'react-router-dom' +import { APP_ROUTES } from 'utils/constants' +import { getCompactTimespanString } from 'utils/date/getCompactTimespanString/getCompactTimespanString' +import { getFormatedDateTimeString } from 'utils/date/getFormatedDateTimeString/getFormatedDateTimeString' +import { STRING, translate } from 'utils/language' +import { + HistoryStat, + HistoryStats, + HistoryTime, + HistoryTypeBadge, +} from './history-stats' + +export const TrackReview = ({ entry }: { entry: TrackCompleteReviewEntry }) => { + const { projectId } = useParams() + const { payload, user } = entry + const added = payload.detections_added.length + const removed = payload.detections_removed.length + + const stats: HistoryStat[] = [ + { + label: translate(STRING.TRACK_SUMMARY_FRAMES), + value: + payload.frames_count === 1 + ? translate(STRING.TRACK_FRAMES_ONE) + : translate(STRING.TRACK_FRAMES_COUNT, { + count: payload.frames_count, + }), + }, + ] + if (payload.first_timestamp && payload.last_timestamp) { + stats.push({ + label: translate(STRING.HISTORY_TIME_SPAN), + value: getCompactTimespanString({ + date1: new Date(payload.first_timestamp), + date2: new Date(payload.last_timestamp), + options: { second: true }, + }), + }) + } + + return ( +
+ + + ) : ( + + ) + } + subTitle={ + added || removed + ? translate(STRING.HISTORY_REVIEW_CHANGES, { added, removed }) + : undefined + } + title={translate(STRING.HISTORY_TRACK_COMPLETE_BY, { + name: user?.name?.length + ? user.name + : translate(STRING.ANONYMOUS_USER), + })} + titleAddon={ + + } + > + + {payload.split_from_occurrence_id ? ( +
+ + {translate(STRING.HISTORY_SPLIT_FROM, { + id: `${payload.split_from_occurrence_id}`, + })} + +
+ ) : null} + +
+ ) +} diff --git a/ui/src/pages/occurrence-details/occurrence-details.tsx b/ui/src/pages/occurrence-details/occurrence-details.tsx index 7a3827ee1..6eff9da60 100644 --- a/ui/src/pages/occurrence-details/occurrence-details.tsx +++ b/ui/src/pages/occurrence-details/occurrence-details.tsx @@ -4,10 +4,12 @@ import { } from 'components/blueprint-collection/blueprint-collection' import { CopyLinkButton } from 'components/copy-link-button/copy-link-button' import { TaxonDetails } from 'components/taxon-details/taxon-details' +import { useOccurrenceHistory } from 'data-services/hooks/occurrences/useOccurrenceHistory' import { FrameLabel, OccurrenceDetails as Occurrence, } from 'data-services/models/occurrence-details' +import { isEditedSinceComplete } from 'data-services/models/occurrence-history' import { SearchIcon } from 'lucide-react' import { BasicTooltip, @@ -33,10 +35,8 @@ import { useUser } from 'utils/user/userContext' import { useUserInfo } from 'utils/user/userInfoContext' import { Agree } from './agree/agree' import { IdQuickActions } from './id-quick-actions/id-quick-actions' -import { GroupingConfirmation } from './identification-card/grouping-confirmation' import { GroupingSummary } from './identification-card/grouping-summary' -import { HumanIdentification } from './identification-card/human-identification' -import { MachinePrediction } from './identification-card/machine-prediction' +import { OccurrenceTimeline } from './identification-card/occurrence-timeline' import styles from './occurrence-details.module.scss' import { StatusLabel } from './status-label/status-label' import { SuggestId } from './suggest-id/suggest-id' @@ -107,6 +107,17 @@ export const OccurrenceDetails = ({ occurrence.userPermissions ) const trackingEnabled = useProjectFeature('tracking') + const history = useOccurrenceHistory({ + occurrenceId: occurrence.id, + projectId, + }) + const editedSinceComplete = history.entries + ? isEditedSinceComplete({ + detectionIds: occurrence.detections, + entries: history.entries, + occurrenceId: occurrence.id, + }) + : false const sessionRoute = occurrence.sessionId ? APP_ROUTES.SESSION_DETAILS({ @@ -348,10 +359,6 @@ export const OccurrenceDetails = ({ )} - {trackingEnabled && occurrence.groupingVerifiedAt ? ( - - ) : null} - {trackingEnabled && occurrence.groupingSummary ? ( ) : null} - {occurrence.humanIdentifications.map((i) => ( - - ))} - - {occurrence.machinePredictions.map((p) => ( - - ))} + @@ -399,6 +395,7 @@ export const OccurrenceDetails = ({ )} diff --git a/ui/src/pages/occurrence-details/track/grouping-actions.tsx b/ui/src/pages/occurrence-details/track/grouping-actions.tsx index b66e5a55b..2187efd92 100644 --- a/ui/src/pages/occurrence-details/track/grouping-actions.tsx +++ b/ui/src/pages/occurrence-details/track/grouping-actions.tsx @@ -28,10 +28,13 @@ const DEFAULT_SCOPE: MergeScopeKey = 'next' export const GroupingActions = ({ canRestructure, canVerify, + editedSinceComplete, occurrence, }: { canRestructure: boolean canVerify: boolean + /** The detections differ from those in the latest "Mark complete" review. */ + editedSinceComplete?: boolean occurrence: OccurrenceDetails }) => { const { projectId } = useParams() @@ -121,6 +124,10 @@ export const GroupingActions = ({ })} + ) : editedSinceComplete ? ( + + {translate(STRING.TRACK_EDITED_SINCE_COMPLETE)} + ) : ( {translate(STRING.TRACK_GROUPING_NOT_CONFIRMED)} diff --git a/ui/src/utils/language.ts b/ui/src/utils/language.ts index dc66a1b3f..e0fcd13b5 100644 --- a/ui/src/utils/language.ts +++ b/ui/src/utils/language.ts @@ -397,6 +397,26 @@ export enum STRING { TRACK_BOX_EXTEND_LABEL, TRACK_BOX_LABEL, TRACK_BOX_LABEL_VERIFIED, + HISTORY_ALGORITHM_RESULT, + HISTORY_CLASS_MASKING, + HISTORY_DETECTIONS_AFFECTED, + HISTORY_DETERMINATION, + HISTORY_DETERMINATION_UNCHANGED, + HISTORY_EMPTY, + HISTORY_FRAMES_LINKED, + HISTORY_LOAD_ERROR, + HISTORY_OCCURRENCES_MERGED, + HISTORY_REVIEW, + HISTORY_REVIEW_CHANGES, + HISTORY_SIZE_FILTER, + HISTORY_SIZE_THRESHOLD, + HISTORY_SPECIES_LIST, + HISTORY_SPECIES_LIST_ID, + HISTORY_SPLIT_FROM, + HISTORY_TIME_SPAN, + HISTORY_TRACK_COMPLETE_BY, + HISTORY_TRACKING, + TRACK_EDITED_SINCE_COMPLETE, TRACK_CANDIDATE_IN_GAP, TRACK_CLEARS_CONFIRMATION, TRACK_COLUMN_COST, @@ -1058,6 +1078,28 @@ const ENGLISH_STRINGS: { [key in STRING]: string } = { [STRING.TRACK_COLUMN_MATCH]: 'Match', [STRING.TRACK_COLUMN_MATCH_HELP]: 'The same cost read on a 0 to 100 scale. A pair with no feature vector is scored on one term fewer, so its percentage is not comparable with a pair that has one. Sort by Cost for the order tracking itself uses.', + [STRING.HISTORY_ALGORITHM_RESULT]: 'Algorithm result', + [STRING.HISTORY_CLASS_MASKING]: 'Class masking', + [STRING.HISTORY_DETECTIONS_AFFECTED]: 'Detections changed', + [STRING.HISTORY_DETERMINATION]: 'Determination', + [STRING.HISTORY_DETERMINATION_UNCHANGED]: '{{name}} (unchanged)', + [STRING.HISTORY_EMPTY]: 'Nothing has happened to this occurrence yet.', + [STRING.HISTORY_FRAMES_LINKED]: 'Frames linked', + [STRING.HISTORY_LOAD_ERROR]: + 'Could not load the full history. Showing identifications and predictions only.', + [STRING.HISTORY_OCCURRENCES_MERGED]: 'Occurrences merged', + [STRING.HISTORY_REVIEW]: 'Review', + [STRING.HISTORY_REVIEW_CHANGES]: + '+{{added}} added, −{{removed}} removed since the previous review', + [STRING.HISTORY_SIZE_FILTER]: 'Size filter', + [STRING.HISTORY_SIZE_THRESHOLD]: 'Size threshold', + [STRING.HISTORY_SPECIES_LIST]: 'Species list', + [STRING.HISTORY_SPECIES_LIST_ID]: 'Species list #{{id}}', + [STRING.HISTORY_SPLIT_FROM]: 'Split from occurrence #{{id}}', + [STRING.HISTORY_TIME_SPAN]: 'Time span', + [STRING.HISTORY_TRACK_COMPLETE_BY]: 'Track marked complete by {{name}}', + [STRING.HISTORY_TRACKING]: 'Tracking', + [STRING.TRACK_EDITED_SINCE_COMPLETE]: 'Edited since marked complete', [STRING.TRACK_COLUMN_OVERLAP]: 'Overlap', [STRING.TRACK_COLUMN_SIMILARITY]: 'Similarity', [STRING.TRACK_COLUMN_SIZE]: 'Size match', @@ -1068,7 +1110,7 @@ const ENGLISH_STRINGS: { [key in STRING]: string } = { [STRING.TRACK_CONFIRM_GROUPING_DESCRIPTION]: 'Records that you checked this track holds every frame of this animal, and no frames of any other. This is a separate judgement from the species it was identified as, and any later change to its frames clears it.', [STRING.TRACK_CONFIRM_GROUPING_RESULT]: 'Track marked complete and accurate.', - [STRING.TRACK_CONFIRM_GROUPING]: 'Mark track as complete and accurate', + [STRING.TRACK_CONFIRM_GROUPING]: 'Mark complete', [STRING.TRACK_GROUPING_FILTER_NO]: 'Not marked', [STRING.TRACK_GROUPING_FILTER_TOOLTIP]: 'Whether a person has marked this track complete and accurate: it holds every frame of the animal and no frames of any other. A separate judgement from the species it was identified as.', From dd92ce55ac4acf368ec389af756fc71cbb8571b5 Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Wed, 23 Sep 2026 03:22:35 -0700 Subject: [PATCH 14/44] fix(ui): keep a masked classifier's prediction agreeable when the determination differs An algorithm result's taxon is the occurrence's determination after the run, not the taxon the algorithm predicted, so matching the folded prediction on both dropped it whenever a person or another algorithm set a different determination. The server folds every prediction of an algorithm that wrote a record, and keeps one prediction per algorithm, so the card now matches on the algorithm alone and shows that prediction's taxon and score. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01C7Xf6VPbwWtTumhjjF15g8 --- .../models/occurrence-history.test.ts | 14 ++++++++++++-- ui/src/data-services/models/occurrence-history.ts | 13 ++++++------- .../identification-card/algorithm-result.tsx | 11 ++++++++++- ui/src/utils/language.ts | 2 ++ 4 files changed, 30 insertions(+), 10 deletions(-) diff --git a/ui/src/data-services/models/occurrence-history.test.ts b/ui/src/data-services/models/occurrence-history.test.ts index 56e14ecfd..cad886685 100644 --- a/ui/src/data-services/models/occurrence-history.test.ts +++ b/ui/src/data-services/models/occurrence-history.test.ts @@ -200,13 +200,23 @@ describe('getFallbackTimelineItems', () => { }) describe('getFoldedPrediction', () => { - test('finds the prediction behind an algorithm result by algorithm and taxon', () => { + test('finds the prediction behind an algorithm result by its algorithm', () => { const folded = ownPrediction('6', 12) expect( getFoldedPrediction(classMasking as never, [ ownPrediction('5', 7), - ownPrediction('4', 12, XESTIA), + folded, + ]) + ).toBe(folded) + }) + + test('finds it when the determination differs from the predicted taxon', () => { + const folded = ownPrediction('6', 12, XESTIA) + + expect( + getFoldedPrediction(classMasking as never, [ + ownPrediction('5', 7), folded, ]) ).toBe(folded) diff --git a/ui/src/data-services/models/occurrence-history.ts b/ui/src/data-services/models/occurrence-history.ts index c207e5924..3f41efc7a 100644 --- a/ui/src/data-services/models/occurrence-history.ts +++ b/ui/src/data-services/models/occurrence-history.ts @@ -282,17 +282,16 @@ const getCreatedAt = (item: TimelineItem) => { } } -/** The prediction an algorithm result stands in for, so its taxon can still be agreed with. */ +/** + * The prediction an algorithm result stands in for, so it can still be agreed with. Matched on + * algorithm alone: the result's taxon is the determination after the run, not what was predicted. + */ export const getFoldedPrediction = ( entry: AlgorithmResultEntry, predictions: MachinePrediction[] ) => - entry.algorithm && entry.taxon - ? predictions.find( - (p) => - `${p.algorithm?.id}` === `${entry.algorithm?.id}` && - p.taxon.id === `${entry.taxon?.id}` - ) + entry.algorithm + ? predictions.find((p) => `${p.algorithm?.id}` === `${entry.algorithm?.id}`) : undefined /** Reviews a merge brought in from another occurrence are not reviews of this one. */ diff --git a/ui/src/pages/occurrence-details/identification-card/algorithm-result.tsx b/ui/src/pages/occurrence-details/identification-card/algorithm-result.tsx index 7773c893d..56d05b1f7 100644 --- a/ui/src/pages/occurrence-details/identification-card/algorithm-result.tsx +++ b/ui/src/pages/occurrence-details/identification-card/algorithm-result.tsx @@ -59,7 +59,16 @@ export const AlgorithmResult = ({ !!foldedPrediction && occurrence.userPermissions.includes(UserPermission.Update) - const stats: HistoryStat[] = [] + const stats: HistoryStat[] = foldedPrediction + ? [ + { + label: translate(STRING.HISTORY_PREDICTION), + value: `${ + foldedPrediction.taxon.name + } (${foldedPrediction.score.toFixed(2)})`, + }, + ] + : [] switch (entry.subtype) { case 'tracking': stats.push( diff --git a/ui/src/utils/language.ts b/ui/src/utils/language.ts index e0fcd13b5..32ff8e135 100644 --- a/ui/src/utils/language.ts +++ b/ui/src/utils/language.ts @@ -406,6 +406,7 @@ export enum STRING { HISTORY_FRAMES_LINKED, HISTORY_LOAD_ERROR, HISTORY_OCCURRENCES_MERGED, + HISTORY_PREDICTION, HISTORY_REVIEW, HISTORY_REVIEW_CHANGES, HISTORY_SIZE_FILTER, @@ -1088,6 +1089,7 @@ const ENGLISH_STRINGS: { [key in STRING]: string } = { [STRING.HISTORY_LOAD_ERROR]: 'Could not load the full history. Showing identifications and predictions only.', [STRING.HISTORY_OCCURRENCES_MERGED]: 'Occurrences merged', + [STRING.HISTORY_PREDICTION]: 'Prediction', [STRING.HISTORY_REVIEW]: 'Review', [STRING.HISTORY_REVIEW_CHANGES]: '+{{added}} added, −{{removed}} removed since the previous review', From 94a684302d7df61a9f786f4cf8a53ac1debdccaa Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Wed, 23 Sep 2026 03:22:43 -0700 Subject: [PATCH 15/44] fix(ui): stop a prediction without an algorithm from linking to a blank algorithm page A prediction read from the history with no algorithm was given an empty Algorithm object, which the card treated as present, so clicking its title opened the algorithm page with no id. The algorithm is now left undefined, and the card only links when it has one. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01C7Xf6VPbwWtTumhjjF15g8 --- ui/src/data-services/models/occurrence-details.ts | 1 - .../models/occurrence-history.test.ts | 14 ++++++++++++++ ui/src/data-services/models/occurrence-history.ts | 2 +- .../identification-card/machine-prediction.tsx | 9 ++++----- 4 files changed, 19 insertions(+), 7 deletions(-) diff --git a/ui/src/data-services/models/occurrence-details.ts b/ui/src/data-services/models/occurrence-details.ts index 7d8430ca8..2ea49246f 100644 --- a/ui/src/data-services/models/occurrence-details.ts +++ b/ui/src/data-services/models/occurrence-details.ts @@ -60,7 +60,6 @@ export interface HumanIdentification extends Identification { } export interface MachinePrediction extends Identification { - algorithm: Algorithm /** Whether a feature embedding was stored; null when the API did not say. */ hasFeatures?: boolean | null score: number diff --git a/ui/src/data-services/models/occurrence-history.test.ts b/ui/src/data-services/models/occurrence-history.test.ts index cad886685..b64c86f57 100644 --- a/ui/src/data-services/models/occurrence-history.test.ts +++ b/ui/src/data-services/models/occurrence-history.test.ts @@ -181,6 +181,20 @@ describe('getTimelineItems', () => { }) }) +describe('prediction cards built from the history', () => { + test('have no algorithm when the server names none', () => { + const [item] = getTimelineItems({ + entries: [{ ...predictionEntry(6), algorithm: null }], + identifications: [], + predictions: [], + }) + + expect(item.type === 'prediction' && item.prediction.algorithm).toBe( + undefined + ) + }) +}) + describe('getFallbackTimelineItems', () => { test('merges identifications and predictions newest first', () => { const items = getFallbackTimelineItems({ diff --git a/ui/src/data-services/models/occurrence-history.ts b/ui/src/data-services/models/occurrence-history.ts index 3f41efc7a..c99567ca2 100644 --- a/ui/src/data-services/models/occurrence-history.ts +++ b/ui/src/data-services/models/occurrence-history.ts @@ -182,7 +182,7 @@ const toPrediction = ( const taxon = convertHistoryTaxon(entry.taxon) return { - algorithm: new Algorithm(entry.algorithm ?? {}), + algorithm: entry.algorithm ? new Algorithm(entry.algorithm) : undefined, applied: taxon.id === determinationTaxonId, createdAt: entry.timestamp, id: `${entry.id}`, diff --git a/ui/src/pages/occurrence-details/identification-card/machine-prediction.tsx b/ui/src/pages/occurrence-details/identification-card/machine-prediction.tsx index 467e36993..14b8fc5e6 100644 --- a/ui/src/pages/occurrence-details/identification-card/machine-prediction.tsx +++ b/ui/src/pages/occurrence-details/identification-card/machine-prediction.tsx @@ -47,6 +47,7 @@ export const MachinePrediction = ({ date: new Date(identification.createdAt), }) const showAgree = occurrence.userPermissions.includes(UserPermission.Update) + const { algorithm } = identification return (
@@ -66,16 +67,14 @@ export const MachinePrediction = ({ ? translate(STRING.TERMINAL_CLASSIFICATION) : translate(STRING.INTERMEDIATE_CLASSIFICATION) } - title={ - identification.algorithm?.name ?? translate(STRING.MACHINE_SUGGESTION) - } + title={algorithm?.name ?? translate(STRING.MACHINE_SUGGESTION)} onTitleClick={ - identification.algorithm + algorithm ? () => navigate( APP_ROUTES.ALGORITHM_DETAILS({ projectId: projectId as string, - algorithmId: identification.algorithm?.id, + algorithmId: algorithm.id, }) ) : undefined From d8c2b13c507d0f48ac6f70e2f595d70efe6ad3d4 Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Wed, 23 Sep 2026 03:37:16 -0700 Subject: [PATCH 16/44] fix(occurrences): let the server say whether a track changed since it was confirmed The panel compared the latest review's detections with the occurrence's detections on the client. The review lists only valid detections, while the occurrence detail does not filter them, and the client also accepted a review with no occurrence id as its own where the server does not. Either difference could show 'Edited since marked complete' on a track nobody edited. The occurrence detail now returns grouping_edited_since_verified, computed with the same review lookup and detection queryset the server uses when it records a review, and the panel reads that field. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01C7Xf6VPbwWtTumhjjF15g8 --- ami/main/api/serializers.py | 11 ++++ ami/main/models_future/history.py | 9 ++++ ami/main/test_occurrence_history.py | 22 +++++++- .../models/occurrence-details.ts | 4 ++ .../models/occurrence-history.test.ts | 52 ------------------- .../models/occurrence-history.ts | 37 ------------- .../occurrence-details/occurrence-details.tsx | 10 +--- 7 files changed, 46 insertions(+), 99 deletions(-) diff --git a/ami/main/api/serializers.py b/ami/main/api/serializers.py index a714bdc64..255156073 100644 --- a/ami/main/api/serializers.py +++ b/ami/main/api/serializers.py @@ -1714,6 +1714,10 @@ class OccurrenceSerializer(OccurrenceListSerializer): event = EventNestedSerializer(read_only=True) grouping_verified_by = UserNestedSerializer(read_only=True) grouping_summary = serializers.SerializerMethodField() + grouping_edited_since_verified = serializers.SerializerMethodField( + help_text="Whether the detections changed since the grouping was last confirmed. " + "False while it is confirmed, and when it never was." + ) # first_appearance = TaxonSourceImageNestedSerializer(read_only=True) class Meta: @@ -1729,12 +1733,19 @@ class Meta: "grouping_verified", "grouping_verified_at", "grouping_verified_by", + "grouping_edited_since_verified", "grouping_summary", ] read_only_fields = [ "determination_score", ] + def get_grouping_edited_since_verified(self, obj: Occurrence) -> bool: + from ami.main.models_future.history import edited_since_track_complete_review + + # Editing the detections withdraws the confirmation, so a confirmed grouping is unchanged. + return obj.grouping_verified_at is None and edited_since_track_complete_review(obj) + @extend_schema_field(GroupingSummarySerializer()) def get_grouping_summary(self, obj: Occurrence) -> dict: from ami.main.models_future.track_stats import grouping_summary_from_prefetch, tracking_algorithm_summary diff --git a/ami/main/models_future/history.py b/ami/main/models_future/history.py index 2286c5656..dfe36ab72 100644 --- a/ami/main/models_future/history.py +++ b/ami/main/models_future/history.py @@ -35,6 +35,15 @@ def latest_track_complete_review(occurrence: Occurrence) -> OccurrenceHistoryRec ) +def edited_since_track_complete_review(occurrence: Occurrence) -> bool: + """Whether the occurrence's detections differ from those its latest own review confirmed.""" + review = latest_track_complete_review(occurrence) + if review is None: + return False + current = set(Detection.objects.valid().filter(occurrence=occurrence).values_list("pk", flat=True)) + return current != set(review.payload["detection_ids"]) + + def record_track_complete_review( occurrence: Occurrence, user: User, timestamp: datetime.datetime, was_confirmed: bool ) -> OccurrenceHistoryRecord | None: diff --git a/ami/main/test_occurrence_history.py b/ami/main/test_occurrence_history.py index 6eeb18bca..7fb2e454d 100644 --- a/ami/main/test_occurrence_history.py +++ b/ami/main/test_occurrence_history.py @@ -5,7 +5,7 @@ from django.test import TestCase from ami.main import tests as main_tests -from ami.main.models import Classification, Identification, Occurrence, OccurrenceHistoryRecord, Taxon +from ami.main.models import Classification, Detection, Identification, Occurrence, OccurrenceHistoryRecord, Taxon from ami.ml.models import Algorithm from ami.tests.fixtures.main import setup_test_project from ami.users.models import User @@ -148,6 +148,26 @@ def test_merging_an_occurrence_keeps_its_reviews(self): self.assertEqual(response.status_code, 200, response.data) self.assertEqual(self.reviews().count(), 1) + def edited_since_verified(self) -> bool: + self.client.force_authenticate(user=self.curator) + response = self.client.get(f"/api/v2/occurrences/{self.occurrence.pk}/?project_id={self.project.pk}") + self.assertEqual(response.status_code, 200, response.data) + return response.data["grouping_edited_since_verified"] + + def test_the_detail_says_whether_the_detections_changed_since_the_last_confirmation(self): + self.assertFalse(self.edited_since_verified()) + self.verify() + self.assertFalse(self.edited_since_verified()) + response = self.post("remove-detection", self.detections[-1], user=self.curator) + self.assertEqual(response.status_code, 200, response.data) + self.assertTrue(self.edited_since_verified()) + + def test_a_null_marker_on_the_occurrence_is_not_an_edit(self): + self.verify() + self.client.post(f"/api/v2/occurrences/{self.occurrence.pk}/unverify-grouping/", format="json") + Detection.objects.create(source_image=self.captures[0], occurrence=self.occurrence, bbox=None) + self.assertFalse(self.edited_since_verified()) + class OccurrenceHistoryEndpointTestCase(main_tests.TrackFixtureTestCase): """GET /occurrences/{id}/history/ merges records, identifications and predictions, newest first.""" diff --git a/ui/src/data-services/models/occurrence-details.ts b/ui/src/data-services/models/occurrence-details.ts index 2ea49246f..1f1abc79b 100644 --- a/ui/src/data-services/models/occurrence-details.ts +++ b/ui/src/data-services/models/occurrence-details.ts @@ -300,6 +300,10 @@ export class OccurrenceDetails extends Occurrence { return !!this._occurrence.grouping_verified } + get groupingEditedSinceVerified(): boolean { + return !!this._occurrence.grouping_edited_since_verified + } + get groupingVerifiedAt(): Date | undefined { return this._occurrence.grouping_verified_at ? new Date(this._occurrence.grouping_verified_at) diff --git a/ui/src/data-services/models/occurrence-history.test.ts b/ui/src/data-services/models/occurrence-history.test.ts index b64c86f57..e1aae7c06 100644 --- a/ui/src/data-services/models/occurrence-history.test.ts +++ b/ui/src/data-services/models/occurrence-history.test.ts @@ -6,7 +6,6 @@ import { getFallbackTimelineItems, getFoldedPrediction, getTimelineItems, - isEditedSinceComplete, ServerOccurrenceHistoryEntry, TrackCompleteReviewEntry, } from './occurrence-history' @@ -237,57 +236,6 @@ describe('getFoldedPrediction', () => { }) }) -describe('isEditedSinceComplete', () => { - test('false without a review', () => { - expect( - isEditedSinceComplete({ - detectionIds: ['1'], - entries: [identificationEntry(1)], - occurrenceId: '100', - }) - ).toBe(false) - }) - - test('false when the detections match the latest review in any order', () => { - expect( - isEditedSinceComplete({ - detectionIds: ['2', '1'], - entries: [review(9, [1, 2]), review(8, [1])], - occurrenceId: '100', - }) - ).toBe(false) - }) - - test('true when a detection was added or removed since the latest review', () => { - const entries = [review(9, [1, 2])] - - expect( - isEditedSinceComplete({ - detectionIds: ['1', '2', '3'], - entries, - occurrenceId: '100', - }) - ).toBe(true) - expect( - isEditedSinceComplete({ - detectionIds: ['1'], - entries, - occurrenceId: '100', - }) - ).toBe(true) - }) - - test('ignores reviews a merge brought in from another occurrence', () => { - expect( - isEditedSinceComplete({ - detectionIds: ['1', '2', '5'], - entries: [review(9, [5], 200), review(8, [1, 2, 5])], - occurrenceId: '100', - }) - ).toBe(false) - }) -}) - describe('getOccurrenceHistoryQueryKey', () => { test('sits under the occurrences key that mutations invalidate', () => { expect(getOccurrenceHistoryQueryKey('100')[0]).toBe(API_ROUTES.OCCURRENCES) diff --git a/ui/src/data-services/models/occurrence-history.ts b/ui/src/data-services/models/occurrence-history.ts index c99567ca2..9f10ceb59 100644 --- a/ui/src/data-services/models/occurrence-history.ts +++ b/ui/src/data-services/models/occurrence-history.ts @@ -293,40 +293,3 @@ export const getFoldedPrediction = ( entry.algorithm ? predictions.find((p) => `${p.algorithm?.id}` === `${entry.algorithm?.id}`) : undefined - -/** Reviews a merge brought in from another occurrence are not reviews of this one. */ -export const getLatestTrackCompleteReview = ( - entries: ServerOccurrenceHistoryEntry[], - occurrenceId: string -) => - entries.find( - (entry): entry is TrackCompleteReviewEntry => - entry.type === 'review' && - entry.subtype === 'track_complete' && - (entry.payload.occurrence_id == null || - `${entry.payload.occurrence_id}` === occurrenceId) - ) - -export const isEditedSinceComplete = ({ - detectionIds, - entries, - occurrenceId, -}: { - detectionIds: string[] - entries: ServerOccurrenceHistoryEntry[] - occurrenceId: string -}) => { - const review = getLatestTrackCompleteReview(entries, occurrenceId) - - if (!review) { - return false - } - - const reviewed = new Set(review.payload.detection_ids.map((id) => `${id}`)) - const current = new Set(detectionIds) - - return ( - reviewed.size !== current.size || - Array.from(current).some((id) => !reviewed.has(id)) - ) -} diff --git a/ui/src/pages/occurrence-details/occurrence-details.tsx b/ui/src/pages/occurrence-details/occurrence-details.tsx index 6eff9da60..ceb76301c 100644 --- a/ui/src/pages/occurrence-details/occurrence-details.tsx +++ b/ui/src/pages/occurrence-details/occurrence-details.tsx @@ -9,7 +9,6 @@ import { FrameLabel, OccurrenceDetails as Occurrence, } from 'data-services/models/occurrence-details' -import { isEditedSinceComplete } from 'data-services/models/occurrence-history' import { SearchIcon } from 'lucide-react' import { BasicTooltip, @@ -111,13 +110,6 @@ export const OccurrenceDetails = ({ occurrenceId: occurrence.id, projectId, }) - const editedSinceComplete = history.entries - ? isEditedSinceComplete({ - detectionIds: occurrence.detections, - entries: history.entries, - occurrenceId: occurrence.id, - }) - : false const sessionRoute = occurrence.sessionId ? APP_ROUTES.SESSION_DETAILS({ @@ -395,7 +387,7 @@ export const OccurrenceDetails = ({ )} From 6f2b642fc3f81b91dcc4036e98f29eb1f9fb05ff Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Wed, 23 Sep 2026 05:00:25 -0700 Subject: [PATCH 17/44] fix(ui): advance to the next occurrence after confirming from a timeline card Confirming an identification from a prediction, algorithm result or human identification card in the occurrence timeline now calls the same callback as the header Confirm button. When the dialog is opened from the occurrences list, this moves the reviewer on to the next occurrence instead of leaving them on the one they just confirmed. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01C7Xf6VPbwWtTumhjjF15g8 --- .../identification-card/algorithm-result.tsx | 3 +++ .../identification-card/human-identification.tsx | 3 +++ .../identification-card/machine-prediction.tsx | 4 ++++ .../identification-card/occurrence-timeline.tsx | 5 +++++ ui/src/pages/occurrence-details/occurrence-details.tsx | 1 + 5 files changed, 16 insertions(+) diff --git a/ui/src/pages/occurrence-details/identification-card/algorithm-result.tsx b/ui/src/pages/occurrence-details/identification-card/algorithm-result.tsx index 56d05b1f7..6a0f61420 100644 --- a/ui/src/pages/occurrence-details/identification-card/algorithm-result.tsx +++ b/ui/src/pages/occurrence-details/identification-card/algorithm-result.tsx @@ -44,10 +44,12 @@ export const AlgorithmResult = ({ currentUser, entry, occurrence, + onConfirmed, }: { currentUser?: UserInfo entry: AlgorithmResultEntry occurrence: Occurrence + onConfirmed?: (occurrenceId: string) => void }) => { const { projectId } = useParams() const { icon: Icon, label } = SUBTYPES[entry.subtype] @@ -174,6 +176,7 @@ export const AlgorithmResult = ({ agreeWith={{ predictionId: foldedPrediction.id }} applied={foldedPrediction.applied} occurrenceId={occurrence.id} + onSuccess={onConfirmed} taxonId={foldedPrediction.taxon.id} />
diff --git a/ui/src/pages/occurrence-details/identification-card/human-identification.tsx b/ui/src/pages/occurrence-details/identification-card/human-identification.tsx index 7023f33ca..a6bac5626 100644 --- a/ui/src/pages/occurrence-details/identification-card/human-identification.tsx +++ b/ui/src/pages/occurrence-details/identification-card/human-identification.tsx @@ -25,11 +25,13 @@ export const HumanIdentification = ({ currentUser, identification, occurrence, + onConfirmed, user, }: { currentUser?: UserInfo identification: Identification occurrence: Occurrence + onConfirmed?: (occurrenceId: string) => void user: { id?: string image?: string @@ -107,6 +109,7 @@ export const HumanIdentification = ({ agreeWith={{ identificationId: identification.id }} applied={identification.applied} occurrenceId={occurrence.id} + onSuccess={onConfirmed} taxonId={identification.taxon.id} /> )} diff --git a/ui/src/pages/occurrence-details/identification-card/machine-prediction.tsx b/ui/src/pages/occurrence-details/identification-card/machine-prediction.tsx index 14b8fc5e6..ec106740d 100644 --- a/ui/src/pages/occurrence-details/identification-card/machine-prediction.tsx +++ b/ui/src/pages/occurrence-details/identification-card/machine-prediction.tsx @@ -28,10 +28,12 @@ export const MachinePrediction = ({ currentUser, identification, occurrence, + onConfirmed, }: { currentUser?: UserInfo identification: Identification occurrence: Occurrence + onConfirmed?: (occurrenceId: string) => void }) => { const [open, setOpen] = useState(false) const { classification, error, isLoading } = useClassificationDetails( @@ -98,6 +100,7 @@ export const MachinePrediction = ({ agreeWith={{ predictionId: identification.id }} applied={identification.applied} occurrenceId={occurrence.id} + onSuccess={onConfirmed} taxonId={identification.taxon.id} /> )} @@ -129,6 +132,7 @@ export const MachinePrediction = ({ agreeWith={{ predictionId: identification.id }} applied={applied} occurrenceId={occurrence.id} + onSuccess={onConfirmed} taxonId={taxon.id} /> )} diff --git a/ui/src/pages/occurrence-details/identification-card/occurrence-timeline.tsx b/ui/src/pages/occurrence-details/identification-card/occurrence-timeline.tsx index bf7b051a4..eeaa53b59 100644 --- a/ui/src/pages/occurrence-details/identification-card/occurrence-timeline.tsx +++ b/ui/src/pages/occurrence-details/identification-card/occurrence-timeline.tsx @@ -19,12 +19,14 @@ export const OccurrenceTimeline = ({ error, isLoading, occurrence, + onConfirmed, }: { currentUser?: UserInfo entries?: ServerOccurrenceHistoryEntry[] error?: unknown isLoading: boolean occurrence: Occurrence + onConfirmed?: (occurrenceId: string) => void }) => { const items = useMemo( () => @@ -68,6 +70,7 @@ export const OccurrenceTimeline = ({ currentUser={currentUser} identification={item.identification} occurrence={occurrence} + onConfirmed={onConfirmed} user={item.identification.user} /> ) @@ -78,6 +81,7 @@ export const OccurrenceTimeline = ({ currentUser={currentUser} identification={item.prediction} occurrence={occurrence} + onConfirmed={onConfirmed} /> ) case 'algorithm_result': @@ -87,6 +91,7 @@ export const OccurrenceTimeline = ({ currentUser={currentUser} entry={item.entry} occurrence={occurrence} + onConfirmed={onConfirmed} /> ) case 'review': diff --git a/ui/src/pages/occurrence-details/occurrence-details.tsx b/ui/src/pages/occurrence-details/occurrence-details.tsx index ceb76301c..e4970bc95 100644 --- a/ui/src/pages/occurrence-details/occurrence-details.tsx +++ b/ui/src/pages/occurrence-details/occurrence-details.tsx @@ -364,6 +364,7 @@ export const OccurrenceDetails = ({ error={history.error} isLoading={history.isLoading} occurrence={occurrence} + onConfirmed={onConfirmed} /> From cf02f55e0cd81fb5e367414d841e26385997b8ff Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Wed, 23 Sep 2026 05:01:56 -0700 Subject: [PATCH 18/44] fix(ui): show the occurrence timeline without a spinner while its history loads The identification and prediction cards the occurrence already carries are shown on their own while the full history loads, and the spinner appears only when there is nothing else to show. Those cards now break timestamp ties on id, newest first, the same way the server orders the history, so they keep their place when the history arrives. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01C7Xf6VPbwWtTumhjjF15g8 --- .../models/occurrence-history.test.ts | 12 ++++++++++ .../models/occurrence-history.ts | 24 ++++++++++++++++--- .../occurrence-timeline.tsx | 3 ++- 3 files changed, 35 insertions(+), 4 deletions(-) diff --git a/ui/src/data-services/models/occurrence-history.test.ts b/ui/src/data-services/models/occurrence-history.test.ts index e1aae7c06..b89fa8a58 100644 --- a/ui/src/data-services/models/occurrence-history.test.ts +++ b/ui/src/data-services/models/occurrence-history.test.ts @@ -210,6 +210,18 @@ describe('getFallbackTimelineItems', () => { 'prediction-7', ]) }) + + test('breaks timestamp ties on id, newest first, as the server does', () => { + const items = getFallbackTimelineItems({ + identifications: [ownIdentification], + predictions: [ownPrediction('6', 7, NOCTUA, ownIdentification.createdAt)], + }) + + expect(items.map((item) => item.id)).toEqual([ + 'prediction-6', + 'identification-1', + ]) + }) }) describe('getFoldedPrediction', () => { diff --git a/ui/src/data-services/models/occurrence-history.ts b/ui/src/data-services/models/occurrence-history.ts index 9f10ceb59..05754326b 100644 --- a/ui/src/data-services/models/occurrence-history.ts +++ b/ui/src/data-services/models/occurrence-history.ts @@ -243,7 +243,10 @@ export const getTimelineItems = ({ } }) -/** Identifications and predictions merged newest first, for when the history is unavailable. */ +/** + * Identifications and predictions merged newest first, for while the history loads or when it is + * unavailable. Ties break on id like the server does, so cards keep their place once it arrives. + */ export const getFallbackTimelineItems = ({ identifications, predictions, @@ -267,10 +270,25 @@ export const getFallbackTimelineItems = ({ }) ), ] - .map((item) => ({ item, time: new Date(getCreatedAt(item)).getTime() })) - .sort((a, b) => b.time - a.time) + .map((item) => ({ + item, + time: new Date(getCreatedAt(item)).getTime(), + id: Number(getSourceId(item)), + })) + .sort((a, b) => b.time - a.time || b.id - a.id) .map(({ item }) => item) +const getSourceId = (item: TimelineItem) => { + switch (item.type) { + case 'identification': + return item.identification.id + case 'prediction': + return item.prediction.id + default: + return item.entry.id + } +} + const getCreatedAt = (item: TimelineItem) => { switch (item.type) { case 'identification': diff --git a/ui/src/pages/occurrence-details/identification-card/occurrence-timeline.tsx b/ui/src/pages/occurrence-details/identification-card/occurrence-timeline.tsx index eeaa53b59..70af38fde 100644 --- a/ui/src/pages/occurrence-details/identification-card/occurrence-timeline.tsx +++ b/ui/src/pages/occurrence-details/identification-card/occurrence-timeline.tsx @@ -46,7 +46,8 @@ export const OccurrenceTimeline = ({ return ( <> - {isLoading ? ( + {/* The fallback cards stand in while the history loads, so the spinner only fills an empty list. */} + {isLoading && !items.length ? (
From 8d672f2c74486643628213c4eb44928e0503c57a Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Wed, 23 Sep 2026 05:04:19 -0700 Subject: [PATCH 19/44] fix(ui): stop labelling signed-in users without a display name as anonymous Timeline cards, the track review card, the "Track marked complete by" line and the verified-by tooltip now name the viewer's own actions "You" when they have not set a display name, and label other users without one "Unnamed user". "Anonymous user" is kept for actions that have no user at all. A shared helper decides the label so these places stay consistent. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01C7Xf6VPbwWtTumhjjF15g8 --- .../models/occurrence-details.ts | 9 +++--- .../models/occurrence-history.test.ts | 2 +- .../models/occurrence-history.ts | 8 ++--- ui/src/data-services/models/occurrence.ts | 7 ++--- .../human-identification.tsx | 3 +- .../occurrence-timeline.tsx | 8 ++++- .../identification-card/track-review.tsx | 14 ++++++--- .../occurrence-details/occurrence-details.tsx | 6 +++- .../track/grouping-actions.tsx | 5 +++- ui/src/utils/language.ts | 2 ++ ui/src/utils/user/getUserLabel.test.ts | 30 +++++++++++++++++++ ui/src/utils/user/getUserLabel.ts | 24 +++++++++++++++ 12 files changed, 95 insertions(+), 23 deletions(-) create mode 100644 ui/src/utils/user/getUserLabel.test.ts create mode 100644 ui/src/utils/user/getUserLabel.ts diff --git a/ui/src/data-services/models/occurrence-details.ts b/ui/src/data-services/models/occurrence-details.ts index 1f1abc79b..2b4f12065 100644 --- a/ui/src/data-services/models/occurrence-details.ts +++ b/ui/src/data-services/models/occurrence-details.ts @@ -1,5 +1,6 @@ import { getFormatedTimeString } from 'utils/date/getFormatedTimeString/getFormatedTimeString' import { STRING, translate } from 'utils/language' +import { getUserLabel } from 'utils/user/getUserLabel' import { UserPermission } from 'utils/user/types' import { Algorithm } from './algorithm' import { Occurrence, ServerOccurrence } from './occurrence' @@ -240,12 +241,10 @@ export class OccurrenceDetails extends Occurrence { user: i.user ? { id: `${i.user.id}`, - name: i.user.name?.length - ? i.user.name - : translate(STRING.ANONYMOUS_USER), + name: getUserLabel(i.user), image: i.user.image, } - : { name: translate(STRING.ANONYMOUS_USER) }, + : { name: getUserLabel(i.user) }, comment: i.comment, userPermissions: i.user_permissions, createdAt: i.created_at, @@ -322,7 +321,7 @@ export class OccurrenceDetails extends Occurrence { return { id: `${user.id}`, image: user.image ?? undefined, - name: user.name?.length ? user.name : translate(STRING.ANONYMOUS_USER), + name: getUserLabel(user), } } diff --git a/ui/src/data-services/models/occurrence-history.test.ts b/ui/src/data-services/models/occurrence-history.test.ts index b89fa8a58..5d6f07f2d 100644 --- a/ui/src/data-services/models/occurrence-history.test.ts +++ b/ui/src/data-services/models/occurrence-history.test.ts @@ -158,7 +158,7 @@ describe('getTimelineItems', () => { { applied: true, comment: 'Looks right', - user: { id: '9', name: 'Anonymous user' }, + user: { id: '9', name: 'Unnamed user' }, userPermissions: [], } ) diff --git a/ui/src/data-services/models/occurrence-history.ts b/ui/src/data-services/models/occurrence-history.ts index 05754326b..07fa9903c 100644 --- a/ui/src/data-services/models/occurrence-history.ts +++ b/ui/src/data-services/models/occurrence-history.ts @@ -1,4 +1,4 @@ -import { STRING, translate } from 'utils/language' +import { getUserLabel } from 'utils/user/getUserLabel' import { Algorithm } from './algorithm' import { HumanIdentification, MachinePrediction } from './occurrence-details' import { Taxon } from './taxa' @@ -166,11 +166,9 @@ const toIdentification = ( ? { id: `${entry.user.id}`, image: entry.user.image ?? undefined, - name: entry.user.name?.length - ? entry.user.name - : translate(STRING.ANONYMOUS_USER), + name: getUserLabel(entry.user), } - : { name: translate(STRING.ANONYMOUS_USER) }, + : { name: getUserLabel(entry.user) }, userPermissions: [], } } diff --git a/ui/src/data-services/models/occurrence.ts b/ui/src/data-services/models/occurrence.ts index 64750712f..d4e65ad69 100644 --- a/ui/src/data-services/models/occurrence.ts +++ b/ui/src/data-services/models/occurrence.ts @@ -1,6 +1,7 @@ import { getFormatedDateString } from 'utils/date/getFormatedDateString/getFormatedDateString' import { getFormatedTimeString } from 'utils/date/getFormatedTimeString/getFormatedTimeString' import { STRING, translate } from 'utils/language' +import { getUserLabel } from 'utils/user/getUserLabel' import { UserPermission } from 'utils/user/types' import { Taxon } from './taxa' import { @@ -115,11 +116,9 @@ export class Occurrence { return verifiedBy ? { id: `${verifiedBy.id}`, - name: verifiedBy.name?.length - ? verifiedBy.name - : translate(STRING.ANONYMOUS_USER), + name: getUserLabel(verifiedBy), } - : { name: translate(STRING.ANONYMOUS_USER) } + : { name: getUserLabel(verifiedBy) } } get durationLabel(): string | undefined { diff --git a/ui/src/pages/occurrence-details/identification-card/human-identification.tsx b/ui/src/pages/occurrence-details/identification-card/human-identification.tsx index a6bac5626..769062b1e 100644 --- a/ui/src/pages/occurrence-details/identification-card/human-identification.tsx +++ b/ui/src/pages/occurrence-details/identification-card/human-identification.tsx @@ -18,6 +18,7 @@ import { APP_ROUTES } from 'utils/constants' import { getFormatedDateTimeString } from 'utils/date/getFormatedDateTimeString/getFormatedDateTimeString' import { getAppRoute } from 'utils/getAppRoute' import { STRING, translate } from 'utils/language' +import { getUserLabel } from 'utils/user/getUserLabel' import { UserInfo, UserPermission } from 'utils/user/types' import { Agree } from '../agree/agree' @@ -70,7 +71,7 @@ export const HumanIdentification = ({ ) } - title={user.name} + title={getUserLabel(user, currentUser)} > ) case 'review': - return + return ( + + ) } })} diff --git a/ui/src/pages/occurrence-details/identification-card/track-review.tsx b/ui/src/pages/occurrence-details/identification-card/track-review.tsx index 34f39ae11..7460a73b8 100644 --- a/ui/src/pages/occurrence-details/identification-card/track-review.tsx +++ b/ui/src/pages/occurrence-details/identification-card/track-review.tsx @@ -6,6 +6,8 @@ import { APP_ROUTES } from 'utils/constants' import { getCompactTimespanString } from 'utils/date/getCompactTimespanString/getCompactTimespanString' import { getFormatedDateTimeString } from 'utils/date/getFormatedDateTimeString/getFormatedDateTimeString' import { STRING, translate } from 'utils/language' +import { getUserLabel } from 'utils/user/getUserLabel' +import { UserInfo } from 'utils/user/types' import { HistoryStat, HistoryStats, @@ -13,7 +15,13 @@ import { HistoryTypeBadge, } from './history-stats' -export const TrackReview = ({ entry }: { entry: TrackCompleteReviewEntry }) => { +export const TrackReview = ({ + currentUser, + entry, +}: { + currentUser?: UserInfo + entry: TrackCompleteReviewEntry +}) => { const { projectId } = useParams() const { payload, user } = entry const added = payload.detections_added.length @@ -60,9 +68,7 @@ export const TrackReview = ({ entry }: { entry: TrackCompleteReviewEntry }) => { : undefined } title={translate(STRING.HISTORY_TRACK_COMPLETE_BY, { - name: user?.name?.length - ? user.name - : translate(STRING.ANONYMOUS_USER), + name: getUserLabel(user, currentUser), })} titleAddon={ diff --git a/ui/src/pages/occurrence-details/occurrence-details.tsx b/ui/src/pages/occurrence-details/occurrence-details.tsx index e4970bc95..66f070015 100644 --- a/ui/src/pages/occurrence-details/occurrence-details.tsx +++ b/ui/src/pages/occurrence-details/occurrence-details.tsx @@ -31,6 +31,7 @@ import { STRING, translate } from 'utils/language' import { UserPermission } from 'utils/user/types' import { useProjectFeature } from 'utils/project-features/useProjectFeature' import { useUser } from 'utils/user/userContext' +import { getUserLabel } from 'utils/user/getUserLabel' import { useUserInfo } from 'utils/user/userInfoContext' import { Agree } from './agree/agree' import { IdQuickActions } from './id-quick-actions/id-quick-actions' @@ -232,7 +233,10 @@ export const OccurrenceDetails = ({ content={ occurrence.determinationVerified ? translate(STRING.VERIFIED_BY, { - name: occurrence.determinationVerifiedBy?.name, + name: getUserLabel( + occurrence.determinationVerifiedBy, + userInfo + ), }) : translate(STRING.MACHINE_PREDICTION_SCORE, { score: `${occurrence.determinationScore}`, diff --git a/ui/src/pages/occurrence-details/track/grouping-actions.tsx b/ui/src/pages/occurrence-details/track/grouping-actions.tsx index 2187efd92..dc6a65e61 100644 --- a/ui/src/pages/occurrence-details/track/grouping-actions.tsx +++ b/ui/src/pages/occurrence-details/track/grouping-actions.tsx @@ -18,6 +18,8 @@ import { getFormatedDateTimeString } from 'utils/date/getFormatedDateTimeString/ import { getAppRoute } from 'utils/getAppRoute' import { STRING, translate } from 'utils/language' import { parseServerError } from 'utils/parseServerError/parseServerError' +import { getUserLabel } from 'utils/user/getUserLabel' +import { useUserInfo } from 'utils/user/userInfoContext' import { getCandidateSessionRoute } from 'components/track/candidate-session-route' import { OccurrencePicker } from 'components/track/occurrence-picker' import { TrackEditDialog } from 'components/track/track-edit-dialog' @@ -98,6 +100,7 @@ export const GroupingActions = ({ }) : undefined + const { userInfo } = useUserInfo() const verifiedBy = occurrence.groupingVerifiedBy const verifiedAt = occurrence.groupingVerifiedAt const verifyErrorMessage = verifyError @@ -120,7 +123,7 @@ export const GroupingActions = ({ date: verifiedAt ? getFormatedDateTimeString({ date: verifiedAt }) : translate(STRING.VALUE_NOT_AVAILABLE), - name: verifiedBy?.name ?? translate(STRING.ANONYMOUS_USER), + name: getUserLabel(verifiedBy, userInfo), })} diff --git a/ui/src/utils/language.ts b/ui/src/utils/language.ts index 32ff8e135..90561e818 100644 --- a/ui/src/utils/language.ts +++ b/ui/src/utils/language.ts @@ -601,6 +601,7 @@ export enum STRING { UNIDENTIFIED, UNKNOWN_ERROR, UNKNOWN, + UNNAMED_USER, UPDATING_DATA, UPLOAD_CAPTURES, USER_INFO, @@ -1288,6 +1289,7 @@ const ENGLISH_STRINGS: { [key in STRING]: string } = { [STRING.UNIDENTIFIED]: 'Unidentified', [STRING.UNKNOWN_ERROR]: 'Unknown error', [STRING.UNKNOWN]: 'Unknown', + [STRING.UNNAMED_USER]: 'Unnamed user', [STRING.UPDATING_DATA]: 'Updating data', [STRING.TRACK_WHEN_EARLIER]: '{{time}} earlier', [STRING.TRACK_WHEN_LATER]: '{{time}} later', diff --git a/ui/src/utils/user/getUserLabel.test.ts b/ui/src/utils/user/getUserLabel.test.ts new file mode 100644 index 000000000..0e6c6225d --- /dev/null +++ b/ui/src/utils/user/getUserLabel.test.ts @@ -0,0 +1,30 @@ +import { getUserLabel } from './getUserLabel' + +describe('getUserLabel', () => { + test('names the current user "You" when they have no display name', () => { + expect(getUserLabel({ id: 4, name: '' }, { id: '4', name: '' })).toBe('You') + expect( + getUserLabel( + { id: 4, name: 'Unnamed user' }, + { id: '4', name: undefined } + ) + ).toBe('You') + }) + + test('uses the display name of a user who has one, including the current user', () => { + expect(getUserLabel({ id: 4, name: 'Ada' }, { id: '4', name: 'Ada' })).toBe( + 'Ada' + ) + expect(getUserLabel({ id: 5, name: 'Ada' }, { id: '4' })).toBe('Ada') + }) + + test('labels another user without a display name as unnamed, not anonymous', () => { + expect(getUserLabel({ id: 5, name: '' }, { id: '4' })).toBe('Unnamed user') + expect(getUserLabel({ id: 5, name: null })).toBe('Unnamed user') + }) + + test('keeps "Anonymous user" for actions with no user', () => { + expect(getUserLabel(undefined, { id: '4' })).toBe('Anonymous user') + expect(getUserLabel(null)).toBe('Anonymous user') + }) +}) diff --git a/ui/src/utils/user/getUserLabel.ts b/ui/src/utils/user/getUserLabel.ts new file mode 100644 index 000000000..ed1ac3196 --- /dev/null +++ b/ui/src/utils/user/getUserLabel.ts @@ -0,0 +1,24 @@ +import { STRING, translate } from 'utils/language' + +/** + * The name to show for the author of an action. "Anonymous" is kept for actions with no user, so + * a signed-in user who has not set a display name reads as "You" to themselves and as unnamed to + * others. The viewer's own name is read from their profile, since `user.name` may already be a label. + */ +export const getUserLabel = ( + user?: { id?: string | number; name?: string | null } | null, + currentUser?: { id: string; name?: string } +) => { + if (!user) { + return translate(STRING.ANONYMOUS_USER) + } + + const isCurrentUser = + !!currentUser && user.id !== undefined && `${user.id}` === currentUser.id + + if (isCurrentUser && !currentUser.name?.length) { + return translate(STRING.YOU) + } + + return user.name?.length ? user.name : translate(STRING.UNNAMED_USER) +} From 370ab688b6167c451a7572769983260731581eb4 Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Wed, 23 Sep 2026 05:07:15 -0700 Subject: [PATCH 20/44] fix(ui): translate the determination change on algorithm result cards The before and after determination on class masking and size filter cards is built from a translated string, and the row is left out when there was no determination before or after the run, rather than reading "N/A (unchanged)". Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01C7Xf6VPbwWtTumhjjF15g8 --- .../identification-card/algorithm-result.tsx | 36 ++++++++++--------- ui/src/utils/language.ts | 2 ++ 2 files changed, 21 insertions(+), 17 deletions(-) diff --git a/ui/src/pages/occurrence-details/identification-card/algorithm-result.tsx b/ui/src/pages/occurrence-details/identification-card/algorithm-result.tsx index 6a0f61420..1c6f58cc3 100644 --- a/ui/src/pages/occurrence-details/identification-card/algorithm-result.tsx +++ b/ui/src/pages/occurrence-details/identification-card/algorithm-result.tsx @@ -25,19 +25,27 @@ const SUBTYPES = { tracking: { icon: RouteIcon, label: STRING.HISTORY_TRACKING }, } -const getDeterminationLabel = ( +/** The determination row, left out when there was no determination before or after. */ +const getDeterminationStats = ( before: ServerHistoryTaxon | null, after: ServerHistoryTaxon | null -) => { - const notAvailable = translate(STRING.VALUE_NOT_AVAILABLE) - - if (before?.id === after?.id) { - return translate(STRING.HISTORY_DETERMINATION_UNCHANGED, { - name: after?.name ?? notAvailable, - }) +): HistoryStat[] => { + if (!before && !after) { + return [] } - return `${before?.name ?? notAvailable} → ${after?.name ?? notAvailable}` + const notAvailable = translate(STRING.VALUE_NOT_AVAILABLE) + const value = + before?.id === after?.id + ? translate(STRING.HISTORY_DETERMINATION_UNCHANGED, { + name: after?.name ?? notAvailable, + }) + : translate(STRING.HISTORY_DETERMINATION_CHANGED, { + after: after?.name ?? notAvailable, + before: before?.name ?? notAvailable, + }) + + return [{ label: translate(STRING.HISTORY_DETERMINATION), value }] } export const AlgorithmResult = ({ @@ -86,10 +94,7 @@ export const AlgorithmResult = ({ break case 'class_masking': stats.push( - { - label: translate(STRING.HISTORY_DETERMINATION), - value: getDeterminationLabel(entry.taxon_before, entry.taxon), - }, + ...getDeterminationStats(entry.taxon_before, entry.taxon), { label: translate(STRING.HISTORY_SPECIES_LIST), value: ( @@ -114,10 +119,7 @@ export const AlgorithmResult = ({ break case 'size_filter': stats.push( - { - label: translate(STRING.HISTORY_DETERMINATION), - value: getDeterminationLabel(entry.taxon_before, entry.taxon), - }, + ...getDeterminationStats(entry.taxon_before, entry.taxon), { label: translate(STRING.HISTORY_SIZE_THRESHOLD), value: entry.payload.size_threshold, diff --git a/ui/src/utils/language.ts b/ui/src/utils/language.ts index 90561e818..3c75bb9d9 100644 --- a/ui/src/utils/language.ts +++ b/ui/src/utils/language.ts @@ -401,6 +401,7 @@ export enum STRING { HISTORY_CLASS_MASKING, HISTORY_DETECTIONS_AFFECTED, HISTORY_DETERMINATION, + HISTORY_DETERMINATION_CHANGED, HISTORY_DETERMINATION_UNCHANGED, HISTORY_EMPTY, HISTORY_FRAMES_LINKED, @@ -1084,6 +1085,7 @@ const ENGLISH_STRINGS: { [key in STRING]: string } = { [STRING.HISTORY_CLASS_MASKING]: 'Class masking', [STRING.HISTORY_DETECTIONS_AFFECTED]: 'Detections changed', [STRING.HISTORY_DETERMINATION]: 'Determination', + [STRING.HISTORY_DETERMINATION_CHANGED]: '{{before}} → {{after}}', [STRING.HISTORY_DETERMINATION_UNCHANGED]: '{{name}} (unchanged)', [STRING.HISTORY_EMPTY]: 'Nothing has happened to this occurrence yet.', [STRING.HISTORY_FRAMES_LINKED]: 'Frames linked', From a3e31dcbfaa7e26346d8d24a5fb24553bdf76f4d Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Wed, 23 Sep 2026 05:17:49 -0700 Subject: [PATCH 21/44] fix(occurrences): post a track review only when the detections or the confirming person changed Undoing a confirmation and marking the same unchanged track complete again wrote a duplicate track_complete review every time. A new review is now written only when the detection set differs from the latest review or a different person confirms it; the same person confirming the same set only refreshes the current confirmation fields. Reviews carried over a session split still apply per piece. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01C7Xf6VPbwWtTumhjjF15g8 --- ami/main/models_future/history.py | 10 +++++----- ami/main/models_future/tracks.py | 3 +-- ami/main/test_occurrence_history.py | 27 +++++++++++++++++++++------ 3 files changed, 27 insertions(+), 13 deletions(-) diff --git a/ami/main/models_future/history.py b/ami/main/models_future/history.py index dfe36ab72..8c395b4f7 100644 --- a/ami/main/models_future/history.py +++ b/ami/main/models_future/history.py @@ -45,18 +45,18 @@ def edited_since_track_complete_review(occurrence: Occurrence) -> bool: def record_track_complete_review( - occurrence: Occurrence, user: User, timestamp: datetime.datetime, was_confirmed: bool + occurrence: Occurrence, user: User, timestamp: datetime.datetime ) -> OccurrenceHistoryRecord | None: """Record that ``user`` confirmed this occurrence's detections. - Nothing is written when the same person re-confirms a still-confirmed, unchanged set, - so the review list shows each distinct confirmation once. + A review is written only when the detections differ from the latest review or a + different person confirms them, so withdrawing and re-confirming an unchanged track + does not repeat the same review. """ previous = latest_track_complete_review(occurrence) record = _build_track_complete_review(occurrence, user.pk, timestamp, previous) if ( - was_confirmed - and previous is not None + previous is not None and previous.user_id == user.pk and sorted(previous.payload["detection_ids"]) == record.payload["detection_ids"] ): diff --git a/ami/main/models_future/tracks.py b/ami/main/models_future/tracks.py index 66d0bfa99..78b1c40ee 100644 --- a/ami/main/models_future/tracks.py +++ b/ami/main/models_future/tracks.py @@ -545,11 +545,10 @@ def verify_grouping(occurrence: Occurrence, user: User) -> Occurrence: an explicit act — no operation in this module sets it as a side effect. The two fields hold the current confirmation; the history keeps every review. """ - was_confirmed = occurrence.grouping_verified_at is not None occurrence.grouping_verified_at = timezone.now() occurrence.grouping_verified_by = user occurrence.save(update_fields=["grouping_verified_at", "grouping_verified_by"]) - record_track_complete_review(occurrence, user, occurrence.grouping_verified_at, was_confirmed) + record_track_complete_review(occurrence, user, occurrence.grouping_verified_at) return occurrence diff --git a/ami/main/test_occurrence_history.py b/ami/main/test_occurrence_history.py index 7fb2e454d..a7391bac9 100644 --- a/ami/main/test_occurrence_history.py +++ b/ami/main/test_occurrence_history.py @@ -65,7 +65,7 @@ def test_save_validates_too_and_an_unknown_subtype_is_refused(self): class TrackCompleteReviewTestCase(main_tests.TrackFixtureTestCase): - """Confirming a track posts a review unless the same person re-confirms a still-confirmed, unchanged set.""" + """Confirming a track posts a review only when its detections or its confirming person changed.""" def verify(self, occurrence: Occurrence | None = None, user: User | None = None): self.client.force_authenticate(user=user or self.curator) @@ -95,8 +95,14 @@ def test_the_first_confirmation_posts_a_review_and_an_unchanged_one_does_not(sel self.assertEqual(self.occurrence.grouping_verified_by, self.curator) self.assertGreaterEqual(self.occurrence.grouping_verified_at, review.timestamp) + def unverify(self): + self.client.force_authenticate(user=self.curator) + response = self.client.post(f"/api/v2/occurrences/{self.occurrence.pk}/unverify-grouping/", format="json") + self.assertEqual(response.status_code, 200, response.data) + def test_a_confirmation_after_an_edit_records_what_changed(self): self.verify() + self.unverify() response = self.post("remove-detection", self.detections[-1], user=self.curator) self.assertEqual(response.status_code, 200, response.data) self.verify() @@ -107,18 +113,27 @@ def test_a_confirmation_after_an_edit_records_what_changed(self): self.assertEqual(second.payload["frames_count"], len(self.captures) - 1) self.assertEqual(first.payload["detection_ids"], sorted(d.pk for d in self.detections)) - def test_a_confirmation_after_it_was_withdrawn_or_by_someone_else_posts_a_review(self): + def test_withdrawing_and_reconfirming_an_unchanged_track_posts_no_new_review(self): self.verify() - self.client.force_authenticate(user=self.curator) - response = self.client.post(f"/api/v2/occurrences/{self.occurrence.pk}/unverify-grouping/", format="json") - self.assertEqual(response.status_code, 200, response.data) + review = self.reviews().get() + for _ in range(2): + self.unverify() + self.verify() + + self.assertEqual(list(self.reviews()), [review]) + self.occurrence.refresh_from_db() + self.assertEqual(self.occurrence.grouping_verified_by, self.curator) + self.assertGreater(self.occurrence.grouping_verified_at, review.timestamp) + + def test_a_confirmation_by_someone_else_posts_a_review(self): self.verify() other_curator = User.objects.create_user(email="second-curator@insectai.org") # type: ignore MLDataManager.assign_user(other_curator, self.project) + self.unverify() self.verify(user=other_curator) reviews = list(self.reviews()) - self.assertEqual([r.user for r in reviews], [self.curator, self.curator, other_curator]) + self.assertEqual([r.user for r in reviews], [self.curator, other_curator]) self.assertTrue(all(r.payload["detections_added"] == [] for r in reviews)) self.occurrence.refresh_from_db() self.assertEqual(self.occurrence.grouping_verified_at, reviews[-1].timestamp) From b52eb9797d3bb7c0adb76ccfbf9338aa48e0e6ba Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Wed, 23 Sep 2026 05:17:49 -0700 Subject: [PATCH 22/44] fix(occurrences): show one prediction per algorithm in the occurrence history When an algorithm's classifications tied for the top score across a track's detections, the history timeline showed one identical prediction entry per detection. The timeline now keeps one prediction per algorithm, preferring the highest score, then a terminal prediction, then the most recent. This changes the history response: tied predictions collapse into a single entry. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01C7Xf6VPbwWtTumhjjF15g8 --- ami/main/models_future/history.py | 30 +++++++++++++++++++---- ami/main/test_occurrence_history.py | 38 ++++++++++++++++++++++++++--- 2 files changed, 60 insertions(+), 8 deletions(-) diff --git a/ami/main/models_future/history.py b/ami/main/models_future/history.py index 8c395b4f7..4f9f51501 100644 --- a/ami/main/models_future/history.py +++ b/ami/main/models_future/history.py @@ -11,7 +11,7 @@ import datetime import typing -from ami.main.models import Detection, Identification, Occurrence, OccurrenceHistoryRecord, Taxon, User +from ami.main.models import Classification, Detection, Identification, Occurrence, OccurrenceHistoryRecord, Taxon, User from ami.main.schemas import TrackCompleteReviewPayload if typing.TYPE_CHECKING: @@ -143,9 +143,10 @@ class TimelineEntry: def occurrence_timeline(occurrence: Occurrence) -> list[TimelineEntry]: """History records, identifications and predictions of one occurrence, merged newest first. - A prediction made by an algorithm that also left a history record here is left out: the - record already stands for that change and names the taxon before and after. Costs four - queries whatever the number of entries. + Each algorithm contributes one prediction, its best. A prediction made by an algorithm + that also left a history record here is left out: the record already stands for that + change and names the taxon before and after. Costs four queries whatever the number of + entries. """ records = list( OccurrenceHistoryRecord.objects.filter(occurrence=occurrence) @@ -209,9 +210,28 @@ def occurrence_timeline(occurrence: Occurrence) -> list[TimelineEntry]: "applied_to_id": prediction.applied_to_id, }, ) - for prediction in occurrence.predictions() + for prediction in _one_prediction_per_algorithm(occurrence) if prediction.algorithm_id not in folded ) entries.sort(key=lambda entry: (entry.timestamp, entry.id), reverse=True) return entries + + +def _one_prediction_per_algorithm(occurrence: Occurrence) -> list[Classification]: + """The best prediction of each algorithm: highest score, then terminal, then latest. + + ``Occurrence.predictions()`` keeps every classification tied for an algorithm's top + score, which would show the same prediction once per detection. + """ + + def rank(prediction: Classification) -> tuple: + score = prediction.score if prediction.score is not None else float("-inf") + return (score, prediction.terminal, prediction.created_at, prediction.pk) + + best: dict[int | None, Classification] = {} + for prediction in occurrence.predictions(): + current = best.get(prediction.algorithm_id) + if current is None or rank(prediction) > rank(current): + best[prediction.algorithm_id] = prediction + return list(best.values()) diff --git a/ami/main/test_occurrence_history.py b/ami/main/test_occurrence_history.py index a7391bac9..3959ca722 100644 --- a/ami/main/test_occurrence_history.py +++ b/ami/main/test_occurrence_history.py @@ -240,8 +240,9 @@ def test_entries_are_merged_newest_first_and_a_folded_prediction_is_left_out(sel types = [entry["type"] for entry in response.data] self.assertEqual(types[-3:], ["review", "identification", "algorithm_result"]) self.assertEqual(set(types[:-3]), {"prediction"}) - self.assertEqual(len(types[:-3]), len(self.detections)) - self.assertTrue(all(entry["algorithm"] is None for entry in response.data[:-3])) + # The fixture's detections share one tied top prediction, so it is shown once. + self.assertEqual(len(types[:-3]), 1) + self.assertIsNone(response.data[0]["algorithm"]) timestamps = [entry["timestamp"] for entry in response.data] self.assertEqual(timestamps, sorted(timestamps, reverse=True)) @@ -257,6 +258,37 @@ def test_entries_are_merged_newest_first_and_a_folded_prediction_is_left_out(sel self.assertEqual(result["taxon_before"]["id"], self.other_taxon.pk) self.assertNotIn("email", str(response.data)) + def test_each_algorithm_shows_one_prediction_preferring_terminal_then_latest(self): + """Tied top scores would otherwise show the same prediction once per detection.""" + classifier = Algorithm.objects.create(name="Tied classifier", key="tied-classifier-history-test") + other = Algorithm.objects.create(name="Second tied classifier", key="second-tied-classifier-history-test") + now = datetime.datetime.now() + + def classify(detection, algorithm, terminal, created_at): + classification = Classification.objects.create( + detection=detection, taxon=self.taxon, score=0.8, algorithm=algorithm, terminal=terminal, timestamp=now + ) + Classification.objects.filter(pk=classification.pk).update(created_at=created_at) + return classification + + classify(self.detections[0], classifier, False, now) + terminal = classify(self.detections[1], classifier, True, now - datetime.timedelta(hours=1)) + classify(self.detections[2], other, True, now - datetime.timedelta(hours=1)) + latest = classify(self.detections[3], other, True, now) + + self.client.force_authenticate(user=self.reader) + response = self.client.get(self.url()) + self.assertEqual(response.status_code, 200, response.data) + + predictions = [entry for entry in response.data if entry["type"] == "prediction"] + by_algorithm = { + entry["algorithm"]["key"] if entry["algorithm"] else None: entry["id"] for entry in predictions + } + self.assertEqual(len(predictions), 3) + self.assertEqual(by_algorithm[classifier.key], terminal.pk) + self.assertEqual(by_algorithm[other.key], latest.pk) + self.assertIn(None, by_algorithm) + def test_query_count_does_not_grow_with_the_number_of_entries(self): self.client.force_authenticate(user=self.reader) now = datetime.datetime.now() @@ -266,7 +298,7 @@ def test_query_count_does_not_grow_with_the_number_of_entries(self): with self.assertNumQueries(HISTORY_QUERIES): response = self.client.get(self.url()) self.assertEqual(response.status_code, 200) - self.assertEqual(len(response.data), 3 * total + len(self.detections)) + self.assertEqual(len(response.data), 3 * total + 1) def test_visible_to_whoever_can_open_the_occurrence(self): """Member, non-member, anonymous and superuser, on a public and on a draft project.""" From 271515ca4de2a796ba4b22e694a0820f30671014 Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Wed, 23 Sep 2026 05:19:57 -0700 Subject: [PATCH 23/44] fix(ui): advance to the next occurrence only when a timeline card confirms the current ID A timeline card's button reads "Confirm" when its taxon is the current determination and "Apply ID" when it would change the determination to a different taxon. Only the confirm case now moves the reviewer on to the next occurrence, matching the header Confirm button. Applying a different ID keeps the reviewer on the occurrence so they can see the result of the change. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01C7Xf6VPbwWtTumhjjF15g8 --- .../identification-card/algorithm-result.tsx | 2 +- .../identification-card/human-identification.tsx | 2 +- .../identification-card/machine-prediction.tsx | 4 ++-- .../identification-card/occurrence-timeline.tsx | 1 + 4 files changed, 5 insertions(+), 4 deletions(-) diff --git a/ui/src/pages/occurrence-details/identification-card/algorithm-result.tsx b/ui/src/pages/occurrence-details/identification-card/algorithm-result.tsx index 1c6f58cc3..ab858cd4d 100644 --- a/ui/src/pages/occurrence-details/identification-card/algorithm-result.tsx +++ b/ui/src/pages/occurrence-details/identification-card/algorithm-result.tsx @@ -178,7 +178,7 @@ export const AlgorithmResult = ({ agreeWith={{ predictionId: foldedPrediction.id }} applied={foldedPrediction.applied} occurrenceId={occurrence.id} - onSuccess={onConfirmed} + onSuccess={foldedPrediction.applied ? onConfirmed : undefined} taxonId={foldedPrediction.taxon.id} /> diff --git a/ui/src/pages/occurrence-details/identification-card/human-identification.tsx b/ui/src/pages/occurrence-details/identification-card/human-identification.tsx index 769062b1e..68ead0f5d 100644 --- a/ui/src/pages/occurrence-details/identification-card/human-identification.tsx +++ b/ui/src/pages/occurrence-details/identification-card/human-identification.tsx @@ -110,7 +110,7 @@ export const HumanIdentification = ({ agreeWith={{ identificationId: identification.id }} applied={identification.applied} occurrenceId={occurrence.id} - onSuccess={onConfirmed} + onSuccess={identification.applied ? onConfirmed : undefined} taxonId={identification.taxon.id} /> )} diff --git a/ui/src/pages/occurrence-details/identification-card/machine-prediction.tsx b/ui/src/pages/occurrence-details/identification-card/machine-prediction.tsx index ec106740d..a5c1efc0d 100644 --- a/ui/src/pages/occurrence-details/identification-card/machine-prediction.tsx +++ b/ui/src/pages/occurrence-details/identification-card/machine-prediction.tsx @@ -100,7 +100,7 @@ export const MachinePrediction = ({ agreeWith={{ predictionId: identification.id }} applied={identification.applied} occurrenceId={occurrence.id} - onSuccess={onConfirmed} + onSuccess={identification.applied ? onConfirmed : undefined} taxonId={identification.taxon.id} /> )} @@ -132,7 +132,7 @@ export const MachinePrediction = ({ agreeWith={{ predictionId: identification.id }} applied={applied} occurrenceId={occurrence.id} - onSuccess={onConfirmed} + onSuccess={applied ? onConfirmed : undefined} taxonId={taxon.id} /> )} diff --git a/ui/src/pages/occurrence-details/identification-card/occurrence-timeline.tsx b/ui/src/pages/occurrence-details/identification-card/occurrence-timeline.tsx index 9749cee88..18111eb22 100644 --- a/ui/src/pages/occurrence-details/identification-card/occurrence-timeline.tsx +++ b/ui/src/pages/occurrence-details/identification-card/occurrence-timeline.tsx @@ -26,6 +26,7 @@ export const OccurrenceTimeline = ({ error?: unknown isLoading: boolean occurrence: Occurrence + /** Runs only when a card confirms the current ID, so applying a different ID keeps the reviewer here to see the result. */ onConfirmed?: (occurrenceId: string) => void }) => { const items = useMemo( From 1e35f58b55e77060261596cb92b1dc699aae5555 Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Wed, 23 Sep 2026 05:21:00 -0700 Subject: [PATCH 24/44] fix(ui): label unnamed users the same way in the session viewer and project summary The "Track confirmed by" line in the session capture toolbar and the top identifiers list on the project summary now use the shared user label, so a signed-in user without a display name reads as "Unnamed user" there too, matching the occurrence details. "Anonymous user" stays for a confirmation with no user attached. The capture model keeps the server's empty name instead of folding it into null, since the two cases are labelled differently. The toolbar only receives a name, not a user id, so it cannot say "You". Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01C7Xf6VPbwWtTumhjjF15g8 --- ui/src/data-services/models/capture.ts | 4 ++-- ui/src/pages/project/summary/summary.tsx | 5 ++--- .../pages/session-details/capture/occurrence-toolbar.tsx | 9 ++++++--- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/ui/src/data-services/models/capture.ts b/ui/src/data-services/models/capture.ts index a226f05be..e95f1dd4e 100644 --- a/ui/src/data-services/models/capture.ts +++ b/ui/src/data-services/models/capture.ts @@ -88,9 +88,9 @@ export class Capture { groupingVerifiedAt: detection.occurrence?.grouping_verified_at ? new Date(detection.occurrence.grouping_verified_at) : null, - // Accounts without a display name serialize as an empty string. + // An empty string is a user without a display name; null is no user. groupingVerifiedBy: - detection.occurrence?.grouping_verified_by || null, + detection.occurrence?.grouping_verified_by ?? null, id: `${detection.id}`, label: getDetectionLabel(detection), occurrenceId: detection.occurrence diff --git a/ui/src/pages/project/summary/summary.tsx b/ui/src/pages/project/summary/summary.tsx index 1214f1a6d..71b1f9351 100644 --- a/ui/src/pages/project/summary/summary.tsx +++ b/ui/src/pages/project/summary/summary.tsx @@ -14,6 +14,7 @@ import { useState } from 'react' import { Link, useOutletContext } from 'react-router-dom' import { APP_ROUTES } from 'utils/constants' import { STRING, translate } from 'utils/language' +import { getUserLabel } from 'utils/user/getUserLabel' import { UserPermission } from 'utils/user/types' import { DeploymentsMap } from './deployments-map' import { ListItem } from './list-item' @@ -176,9 +177,7 @@ const MostIdentifications = ({ projectId }: { projectId: string }) => { diff --git a/ui/src/pages/session-details/capture/occurrence-toolbar.tsx b/ui/src/pages/session-details/capture/occurrence-toolbar.tsx index 974a2068c..0d5d13d5a 100644 --- a/ui/src/pages/session-details/capture/occurrence-toolbar.tsx +++ b/ui/src/pages/session-details/capture/occurrence-toolbar.tsx @@ -9,6 +9,7 @@ import { ReactNode } from 'react' import { getFormatedDateTimeString } from 'utils/date/getFormatedDateTimeString/getFormatedDateTimeString' import { getFormatedTimeString } from 'utils/date/getFormatedTimeString/getFormatedTimeString' import { STRING, translate } from 'utils/language' +import { getUserLabel } from 'utils/user/getUserLabel' import { buildDetectionLink } from '../hooks/useActiveDetection' export interface ToolbarOccurrence { @@ -260,9 +261,11 @@ export const OccurrenceToolbar = ({ date: occurrence.groupingVerifiedAt, }) : translate(STRING.VALUE_NOT_AVAILABLE), - name: - occurrence.groupingVerifiedBy ?? - translate(STRING.ANONYMOUS_USER), + name: getUserLabel( + occurrence.groupingVerifiedBy === null + ? null + : { name: occurrence.groupingVerifiedBy } + ), })} ) : null} From 8c892444020aca0a846110157ed73802e2fbf090 Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Wed, 23 Sep 2026 06:08:29 -0700 Subject: [PATCH 25/44] test: count the occurrence history queries in the multi-frame merge budget On a branch that keeps occurrence history, merging two tracks also moves the merged track's history records to the target and cascade-deletes the rest when the merged occurrence is removed. Those are two fixed queries per merge, not per frame, so the budget rises from 29 to 31 and the test still guards against a per-frame query. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01C7Xf6VPbwWtTumhjjF15g8 --- ami/main/tests.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ami/main/tests.py b/ami/main/tests.py index dcc57a87e..cd024ec6c 100644 --- a/ami/main/tests.py +++ b/ami/main/tests.py @@ -9855,7 +9855,8 @@ def test_relinking_a_merge_costs_the_same_queries_however_many_frames_moved(self def test_a_multi_frame_merge_does_not_query_per_frame(self): other, _ = self._make_track(3, captures=self._make_captures_after(3)) - with self.assertNumQueries(29): + # Two of these move and cascade-delete the merged track's history records, once per merge. + with self.assertNumQueries(31): merge_occurrences(self.occurrence, [other]) self.assertFullyLinked(self.occurrence) From 847b839ebd7c544fb1c433e5c4ba7bbae33a0d68 Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Wed, 23 Sep 2026 06:59:47 -0700 Subject: [PATCH 26/44] fix(ui): show one prediction per algorithm in the timeline before its history loads When several frames tie for an algorithm's top score, the occurrence lists each of them, so the timeline drew two identical prediction cards until the history arrived and dropped one. The fallback list now keeps the best prediction of each algorithm by the same rule the server's history uses (highest score, then terminal, then most recent, then id), so the cards stay the same when the history loads. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01C7Xf6VPbwWtTumhjjF15g8 --- .../models/occurrence-history.test.ts | 37 ++++++++++++++++++- .../models/occurrence-history.ts | 33 ++++++++++++++++- 2 files changed, 67 insertions(+), 3 deletions(-) diff --git a/ui/src/data-services/models/occurrence-history.test.ts b/ui/src/data-services/models/occurrence-history.test.ts index 5d6f07f2d..b4bcc0be9 100644 --- a/ui/src/data-services/models/occurrence-history.test.ts +++ b/ui/src/data-services/models/occurrence-history.test.ts @@ -200,7 +200,7 @@ describe('getFallbackTimelineItems', () => { identifications: [ownIdentification], predictions: [ ownPrediction('6', 7, NOCTUA, '2026-04-29T23:00:00'), - ownPrediction('7', 7, NOCTUA, '2026-04-29T20:00:00'), + ownPrediction('7', 12, NOCTUA, '2026-04-29T20:00:00'), ], }) @@ -224,6 +224,41 @@ describe('getFallbackTimelineItems', () => { }) }) +describe('fallback predictions', () => { + const tied = (id: string, terminal: boolean, createdAt: string) => ({ + ...ownPrediction(id, 7, NOCTUA, createdAt), + terminal, + }) + + test('keep one per algorithm when frames tie for its top score', () => { + const items = getFallbackTimelineItems({ + identifications: [], + predictions: [ + tied('6', true, '2026-04-29T21:00:00'), + tied('7', true, '2026-04-29T22:00:00'), + ownPrediction('8', 12), + ], + }) + + expect(items.map((item) => item.id)).toEqual([ + 'prediction-7', + 'prediction-8', + ]) + }) + + test('prefer a terminal prediction over a later intermediate one', () => { + const items = getFallbackTimelineItems({ + identifications: [], + predictions: [ + tied('6', true, '2026-04-29T21:00:00'), + tied('7', false, '2026-04-29T22:00:00'), + ], + }) + + expect(items.map((item) => item.id)).toEqual(['prediction-6']) + }) +}) + describe('getFoldedPrediction', () => { test('finds the prediction behind an algorithm result by its algorithm', () => { const folded = ownPrediction('6', 12) diff --git a/ui/src/data-services/models/occurrence-history.ts b/ui/src/data-services/models/occurrence-history.ts index 07fa9903c..f31456ec3 100644 --- a/ui/src/data-services/models/occurrence-history.ts +++ b/ui/src/data-services/models/occurrence-history.ts @@ -241,9 +241,38 @@ export const getTimelineItems = ({ } }) +/** + * The best prediction of each algorithm: highest score, then terminal, then latest, then id. + * The occurrence lists every frame tied for an algorithm's top score, and the history keeps one. + */ +const onePredictionPerAlgorithm = (predictions: MachinePrediction[]) => { + const rank = (p: MachinePrediction) => [ + p.score ?? -Infinity, + p.terminal ? 1 : 0, + new Date(p.createdAt).getTime() || 0, + Number(p.id) || 0, + ] + const isBetter = (a: MachinePrediction, b: MachinePrediction) => { + const [rankA, rankB] = [rank(a), rank(b)] + const index = rankA.findIndex((value, i) => value !== rankB[i]) + + return index !== -1 && rankA[index] > rankB[index] + } + const best = new Map() + predictions.forEach((prediction) => { + const key = `${prediction.algorithm?.id}` + const current = best.get(key) + if (!current || isBetter(prediction, current)) { + best.set(key, prediction) + } + }) + + return [...best.values()] +} + /** * Identifications and predictions merged newest first, for while the history loads or when it is - * unavailable. Ties break on id like the server does, so cards keep their place once it arrives. + * unavailable. Predictions and tie breaks follow the server, so cards keep their place once it arrives. */ export const getFallbackTimelineItems = ({ identifications, @@ -260,7 +289,7 @@ export const getFallbackTimelineItems = ({ identification, }) ), - ...predictions.map( + ...onePredictionPerAlgorithm(predictions).map( (prediction): TimelineItem => ({ type: 'prediction', id: `prediction-${prediction.id}`, From 6f14d0c8c234bc12ff64ba898dd24c16b272918d Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Mon, 28 Sep 2026 12:40:44 -0700 Subject: [PATCH 27/44] style: collapse the tracking task import in the capture matcher onto one line isort wants the shorter import on one line now that the embeddings reader replaced one of the names it imported. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01C7Xf6VPbwWtTumhjjF15g8 --- ami/main/models_future/merge_candidates.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/ami/main/models_future/merge_candidates.py b/ami/main/models_future/merge_candidates.py index cfd587440..4279db8c3 100644 --- a/ami/main/models_future/merge_candidates.py +++ b/ami/main/models_future/merge_candidates.py @@ -398,11 +398,7 @@ def match_capture_detections(occurrence: Occurrence, capture: SourceImage) -> di """ from ami.main.models import Detection, SourceImage from ami.ml.models import Algorithm - from ami.ml.post_processing.tracking_task import ( - image_diagonal, - resolve_feature_algorithm, - select_links, - ) + from ami.ml.post_processing.tracking_task import image_diagonal, resolve_feature_algorithm, select_links config = tracking_config_for(occurrence) reference, relation = _reference_frame(occurrence.pk, capture) if capture.timestamp else (None, None) From e2fc8067c9dc6c9addd306244e9a4cc25f0cf6a7 Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Mon, 28 Sep 2026 13:03:48 -0700 Subject: [PATCH 28/44] test: read stored embedding vectors as plain lists pgvector 0.5 returns a vector field as a Python list rather than a numpy array, so the embedding tests' .tolist() calls raised AttributeError. list() works for both. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01C7Xf6VPbwWtTumhjjF15g8 --- ami/ml/tests.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ami/ml/tests.py b/ami/ml/tests.py index aad4786dd..4628cd97a 100644 --- a/ami/ml/tests.py +++ b/ami/ml/tests.py @@ -2464,7 +2464,7 @@ def _stored(image: SourceImage) -> dict[tuple[float, str], list[float]]: rows = DetectionEmbedding.objects.filter(detection__source_image=image).select_related( "detection", "algorithm" ) - return {(row.detection.bbox[0], row.algorithm.key): row.vector.tolist() for row in rows} + return {(row.detection.bbox[0], row.algorithm.key): list(row.vector) for row in rows} def test_every_detection_stores_one_vector_per_algorithm(self): """Including the rejected crop, which has no species classification that could carry one.""" @@ -2515,7 +2515,7 @@ def test_each_vector_records_the_job_that_last_stored_it_and_outlives_that_job(s second.delete() embedding = DetectionEmbedding.objects.get(detection__source_image=image) self.assertIsNone(embedding.job_id) - self.assertEqual(embedding.vector.tolist(), self.HIGH) + self.assertEqual(list(embedding.vector), self.HIGH) def test_a_vector_lands_on_its_own_detection_when_some_detections_already_exist(self): """Detection creation returns existing detections ahead of new ones, so pairing responses From f8b3eb8aedf8d7d65fd02b99a2fe519e966e3edb Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Mon, 28 Sep 2026 12:48:06 -0700 Subject: [PATCH 29/44] feat(embeddings): store vectors of any length and fill them in for existing detections Feature extractors differ in vector length (2048 for the moth classifier backbones, 1024 for BioCLIP), so the detection embedding column no longer fixes a dimension. Each algorithm records its own length (Algorithm.embedding_dimensions, from /info or the first vector stored) and a vector of another length is refused, so vectors that cannot be compared never reach the table under one algorithm. A pipeline whose algorithms all have a feature-extraction task type ("embedding" or "feature_extraction") is now run on the detections Antenna already has: an ML job with such a pipeline sends only real detections still missing a vector from it (sync requests and NATS tasks alike), and saving its results matches the returned boxes to existing detections and writes embeddings only, creating no detection, classification or occurrence and changing no determination. The processing service may send the vector under "features" or "vector". Existing embedding tests read vectors as lists, which is what this pgvector version returns, and the schema test reflects that any non-empty length parses. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01C7Xf6VPbwWtTumhjjF15g8 --- .../migrations/0102_detection_embedding.py | 7 +- ami/main/models.py | 4 +- .../0029_algorithm_embedding_dimensions.py | 55 +++++ ami/ml/models/algorithm.py | 15 ++ ami/ml/models/pipeline.py | 227 +++++++++++++++++- ami/ml/orchestration/jobs.py | 23 ++ ami/ml/schemas.py | 32 ++- ami/ml/serializers.py | 1 + 8 files changed, 342 insertions(+), 22 deletions(-) create mode 100644 ami/ml/migrations/0029_algorithm_embedding_dimensions.py diff --git a/ami/main/migrations/0102_detection_embedding.py b/ami/main/migrations/0102_detection_embedding.py index ba95853a5..39ff62b4f 100644 --- a/ami/main/migrations/0102_detection_embedding.py +++ b/ami/main/migrations/0102_detection_embedding.py @@ -1,7 +1,8 @@ # Generated by Django 4.2.10 on 2026-09-15 02:01 # -# Additive: creates an empty table. The unique constraint's index, which leads with -# detection_id, also serves reads of the vectors for a set of detections. See #1417. +# Additive: creates an empty table. The vector column has no fixed length because +# extractors differ; each algorithm records its own length. The unique constraint's index, +# which leads with detection_id, also serves reads of the vectors for a set of detections. See #1417. from django.db import migrations, models import django.db.models.deletion @@ -25,7 +26,7 @@ class Migration(migrations.Migration): ( "vector", pgvector.django.vector.VectorField( - dimensions=2048, help_text="Feature embedding from the model backbone" + help_text="Feature embedding from the model backbone. Its length is the algorithm's own." ), ), ( diff --git a/ami/main/models.py b/ami/main/models.py index f3553df61..d447dcca8 100644 --- a/ami/main/models.py +++ b/ami/main/models.py @@ -3517,9 +3517,9 @@ class DetectionEmbedding(BaseModel): # No separate index: the unique constraint's index leads with detection_id. detection = models.ForeignKey(Detection, on_delete=models.CASCADE, related_name="embeddings", db_index=False) algorithm = models.ForeignKey("ml.Algorithm", on_delete=models.CASCADE, related_name="detection_embeddings") + # No fixed length: extractors differ. Each algorithm keeps one (Algorithm.embedding_dimensions). vector = pgvector.django.VectorField( - dimensions=2048, - help_text="Feature embedding from the model backbone", + help_text="Feature embedding from the model backbone. Its length is the algorithm's own.", ) # The job whose results stored this vector; kept when the job is deleted, since the vector stays valid. job = models.ForeignKey("jobs.Job", on_delete=models.SET_NULL, null=True, blank=True, related_name="+") diff --git a/ami/ml/migrations/0029_algorithm_embedding_dimensions.py b/ami/ml/migrations/0029_algorithm_embedding_dimensions.py new file mode 100644 index 000000000..96dd5dd90 --- /dev/null +++ b/ami/ml/migrations/0029_algorithm_embedding_dimensions.py @@ -0,0 +1,55 @@ +# Additive: a nullable column, and a new task type choice (no schema change for the choice). + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("ml", "0028_normalize_empty_endpoint_url_to_null"), + ] + + operations = [ + migrations.AddField( + model_name="algorithm", + name="embedding_dimensions", + field=models.PositiveIntegerField( + blank=True, + help_text=( + "For a feature extractor, the length of every vector it produces. Set from the processing " + "service's /info or from the first vector stored; vectors of any other length are refused." + ), + null=True, + ), + ), + migrations.AlterField( + model_name="algorithm", + name="task_type", + field=models.CharField( + choices=[ + ("detection", "Detection"), + ("localization", "Localization"), + ("segmentation", "Segmentation"), + ("classification", "Classification"), + ("embedding", "Embedding"), + ("feature_extraction", "Feature Extraction"), + ("tracking", "Tracking"), + ("tagging", "Tagging"), + ("regression", "Regression"), + ("captioning", "Captioning"), + ("generation", "Generation"), + ("translation", "Translation"), + ("summarization", "Summarization"), + ("question_answering", "Question Answering"), + ("depth_estimation", "Depth Estimation"), + ("pose_estimation", "Pose Estimation"), + ("size_estimation", "Size Estimation"), + ("post_processing", "Post Processing"), + ("other", "Other"), + ("unknown", "Unknown"), + ], + default="unknown", + max_length=255, + null=True, + ), + ), + ] diff --git a/ami/ml/models/algorithm.py b/ami/ml/models/algorithm.py index 605f90861..12297a315 100644 --- a/ami/ml/models/algorithm.py +++ b/ami/ml/models/algorithm.py @@ -205,6 +205,7 @@ class AlgorithmTaskType(str, enum.Enum): SEGMENTATION = "segmentation" CLASSIFICATION = "classification" EMBEDDING = "embedding" + FEATURE_EXTRACTION = "feature_extraction" TRACKING = "tracking" TAGGING = "tagging" REGRESSION = "regression" @@ -249,6 +250,15 @@ class Algorithm(BaseModel): help_text=("A URI to the weights or model details. Could be a public web URL or object store path."), ) + embedding_dimensions = models.PositiveIntegerField( + null=True, + blank=True, + help_text=( + "For a feature extractor, the length of every vector it produces. Set from the processing " + "service's /info or from the first vector stored; vectors of any other length are refused." + ), + ) + category_map = models.ForeignKey( AlgorithmCategoryMap, on_delete=models.CASCADE, @@ -275,6 +285,11 @@ class Algorithm(BaseModel): AlgorithmTaskType.CLASSIFICATION, AlgorithmTaskType.TAGGING, ] + # A pipeline made only of these returns vectors for existing detections and nothing else. + feature_extraction_task_types = [ + AlgorithmTaskType.EMBEDDING, + AlgorithmTaskType.FEATURE_EXTRACTION, + ] def __str__(self): return f'#{self.pk} "{self.name}" ({self.key}) v{self.version}' diff --git a/ami/ml/models/pipeline.py b/ami/ml/models/pipeline.py index d4c0aad0e..4bc70fff1 100644 --- a/ami/ml/models/pipeline.py +++ b/ami/ml/models/pipeline.py @@ -107,6 +107,10 @@ def filter_processed_images( pipeline_algorithms = list(pipeline.algorithms.all()) pipeline_algorithm_ids = [a.id for a in pipeline_algorithms] + if feature_extraction_only(pipeline_algorithms): + yield from filter_images_missing_features(images, pipeline_algorithm_ids, batch_size=batch_size) + return + detection_type_keys = set(Algorithm.detection_task_types) has_detection_algorithm = any(a.task_type in detection_type_keys for a in pipeline_algorithms) if not has_detection_algorithm: @@ -222,6 +226,84 @@ def filter_processed_images( last_progress_save_monotonic = now_monotonic +def feature_extraction_only(algorithms: list[Algorithm]) -> bool: + """Whether a pipeline made of these algorithms only extracts features (see ``Pipeline.is_feature_only``).""" + feature_types = set(Algorithm.feature_extraction_task_types) + return bool(algorithms) and all(algorithm.task_type in feature_types for algorithm in algorithms) + + +def detections_missing_features(detections: models.QuerySet, algorithm_ids: list[int]) -> models.QuerySet: + """Real detections without a stored vector from at least one of the algorithms. + + Only ``DetectionEmbedding`` rows count: they are what a feature-only pipeline writes. + """ + missing = models.Q() + for algorithm_id in algorithm_ids: + missing |= ~models.Exists( + DetectionEmbedding.objects.filter(detection_id=models.OuterRef("pk"), algorithm_id=algorithm_id) + ) + return detections.valid().filter(missing) + + +def filter_images_missing_features( + images: typing.Iterable[SourceImage], + algorithm_ids: list[int], + batch_size: int = FILTER_PROCESSED_BATCH_SIZE, +) -> typing.Iterable[SourceImage]: + """The images with at least one real detection that lacks a vector from the algorithms. + + Images without detections are skipped: a feature-only pipeline has nothing to run on there. + """ + image_iter = iter(images) + while True: + batch = list(itertools.islice(image_iter, batch_size)) + if not batch: + return + needing = set( + detections_missing_features( + Detection.objects.filter(source_image_id__in=[image.pk for image in batch]), algorithm_ids + ) + .order_by() + .values_list("source_image_id", flat=True) + .distinct() + ) + yield from (image for image in batch if image.pk in needing) + + +def collect_detections_for_features( + source_image_requests: list[SourceImageRequest], + algorithm_ids: list[int], + include_existing_vectors: bool = False, +) -> list[DetectionRequest]: + """The existing detections a feature-only pipeline should embed, for all the images in one query. + + Detections that already have a vector from every one of the algorithms are left out + unless ``include_existing_vectors`` is set. + """ + request_by_image_id = {int(request.id): request for request in source_image_requests} + detections = Detection.objects.filter(source_image_id__in=list(request_by_image_id)) + detections = ( + detections.valid() if include_existing_vectors else detections_missing_features(detections, algorithm_ids) + ) + detection_requests: list[DetectionRequest] = [] + for detection in detections.select_related("detection_algorithm").order_by("source_image_id", "pk"): + bbox = detection.get_bbox() + if bbox is None or detection.detection_algorithm is None: + continue + detection_requests.append( + DetectionRequest( + source_image=request_by_image_id[detection.source_image_id], + bbox=bbox, + crop_image_url=detection.url(), + algorithm=AlgorithmReference( + name=detection.detection_algorithm.name, + key=detection.detection_algorithm.key, + ), + ) + ) + return detection_requests + + def collect_images( collection: SourceImageCollection | None = None, source_images: list[SourceImage] | None = None, @@ -349,6 +431,8 @@ def process_images( if pipeline_config.reprocess_existing_detections: reprocess_existing_detections = True + feature_algorithm_ids = [algorithm.pk for algorithm in pipeline.feature_extraction_algorithms()] + for source_image, url in zip(images, urls): if url: source_image_request = SourceImageRequest( @@ -357,10 +441,20 @@ def process_images( ) source_image_requests.append(source_image_request) - if reprocess_existing_detections: + if reprocess_existing_detections and not feature_algorithm_ids: detection_requests += collect_detections(source_image, source_image_request) - if reprocess_existing_detections: + if feature_algorithm_ids: + # A feature-only pipeline only ever runs on the boxes Antenna already has. + detection_requests = collect_detections_for_features( + source_image_requests, feature_algorithm_ids, include_existing_vectors=reprocess_all_images + ) + images_with_detections = {request.source_image.id for request in detection_requests} + source_image_requests = [request for request in source_image_requests if request.id in images_with_detections] + task_logger.info(f"Sending {len(detection_requests)} existing detections for feature extraction.") + if not source_image_requests: + return PipelineResultsResponse(pipeline=pipeline.slug, source_images=[], detections=[], total_time=0) + elif reprocess_existing_detections: task_logger.info(f"Found {len(detection_requests)} existing detections to reprocess.") else: task_logger.info("Reprocessing of existing detections is disabled, sending images without detections.") @@ -513,6 +607,17 @@ def get_or_create_algorithm_and_category_map( setattr(algo, field, new_value) algo_fields_updated.append(field) + if algorithm_config.embedding_dimensions and algo.embedding_dimensions != algorithm_config.embedding_dimensions: + if algo.embedding_dimensions is None: + algo.embedding_dimensions = algorithm_config.embedding_dimensions + algo_fields_updated.append("embedding_dimensions") + else: + # Stored vectors keep their length; a new length needs a new algorithm version. + logger.warning( + f"Algorithm {algo} reports {algorithm_config.embedding_dimensions}-dimension vectors but " + f"stores {algo.embedding_dimensions}; keeping {algo.embedding_dimensions}." + ) + if algo_fields_updated: algo.save(update_fields=algo_fields_updated) @@ -683,6 +788,41 @@ def create_detections( # A vector is roughly 40 KB of SQL text (estimate), so this keeps each INSERT to a few MB. EMBEDDING_BATCH_SIZE = 200 +# Boxes are matched to responses at this precision, so a service that re-serialises the +# coordinates it was sent still lands its vectors on the same detections. +BOX_MATCH_DECIMALS = 3 + + +def _box_key(source_image_id, coordinates) -> tuple: + return (str(source_image_id), tuple(round(float(value), BOX_MATCH_DECIMALS) for value in coordinates)) + + +class EmbeddingDimensionMismatch(PipelineNotConfigured): + """A vector's length differs from the length its algorithm has produced before.""" + + +def _check_embedding_dimensions(algorithm: Algorithm, lengths: set[int]) -> None: + """Refuse vectors whose length differs from the algorithm's, recording it on first sight. + + Vectors of different lengths can never be compared, so each algorithm keeps one length. + """ + if algorithm.embedding_dimensions is None: + if len(lengths) > 1: + raise EmbeddingDimensionMismatch( + f"Algorithm {algorithm.key} sent vectors of several lengths in one batch: {sorted(lengths)}" + ) + (length,) = lengths + # Conditional update: the first batch to record a length wins over a concurrent one. + Algorithm.objects.filter(pk=algorithm.pk, embedding_dimensions__isnull=True).update( + embedding_dimensions=length + ) + algorithm.refresh_from_db(fields=["embedding_dimensions"]) + wrong = lengths - {algorithm.embedding_dimensions} + if wrong: + raise EmbeddingDimensionMismatch( + f"Algorithm {algorithm.key} produces {algorithm.embedding_dimensions}-dimension vectors; " + f"refusing vectors of length {sorted(wrong)}." + ) def create_detection_embeddings( @@ -699,23 +839,26 @@ def create_detection_embeddings( changes nothing. Only ``DetectionEmbedding`` rows are written, never a classification, so no determination can change. - Responses are matched to detections by image and box, the key ``get_or_create_detection`` - reuses detections by, because ``create_detections`` does not return them in response order. - An algorithm key the pipeline has not registered raises ``PipelineNotConfigured``, as it - does for classifications. ``job_id`` records the job whose results stored each vector. + Responses are matched to ``detections`` by image and box (see ``BOX_MATCH_DECIMALS``), + the key ``get_or_create_detection`` reuses detections by; a response with no match is + skipped. An algorithm key the pipeline has not registered raises ``PipelineNotConfigured``, + as it does for classifications, and a vector whose length differs from its algorithm's + raises ``EmbeddingDimensionMismatch``. ``job_id`` records the job whose results stored each vector. """ by_box = { - (str(detection.source_image_id), tuple(detection.bbox)): detection + _box_key(detection.source_image_id, detection.bbox): detection for detection in detections if detection.bbox is not None } embeddings: dict[tuple[int, int], DetectionEmbedding] = {} + lengths_by_algorithm: dict[str, set[int]] = collections.defaultdict(set) + unmatched = 0 for detection_resp in detection_responses: if not detection_resp.embeddings or detection_resp.bbox is None: continue - detection = by_box.get((detection_resp.source_image_id, tuple(detection_resp.bbox.dict().values()))) + detection = by_box.get(_box_key(detection_resp.source_image_id, detection_resp.bbox.dict().values())) if detection is None: - # Its source image was not found; create_detections has logged that. + unmatched += 1 continue for embedding_resp in detection_resp.embeddings: try: @@ -726,10 +869,16 @@ def create_detection_embeddings( "The processing service must declare it in the /info endpoint. " f"Known algorithms: {list(algorithms_known.keys())}" ) from err + lengths_by_algorithm[algorithm.key].add(len(embedding_resp.features)) embeddings[(detection.pk, algorithm.pk)] = DetectionEmbedding( detection=detection, algorithm=algorithm, vector=embedding_resp.features, job_id=job_id ) + for key, lengths in lengths_by_algorithm.items(): + _check_embedding_dimensions(algorithms_known[key], lengths) + + if unmatched: + logger.warning(f"Skipped the vectors of {unmatched} detections that match no stored detection.") DetectionEmbedding.objects.bulk_create( list(embeddings.values()), update_conflicts=True, @@ -741,6 +890,34 @@ def create_detection_embeddings( return list(embeddings.values()) +def save_features_for_existing_detections( + results: PipelineResultsResponse, + algorithms_known: dict[str, Algorithm], + logger: logging.Logger = logger, + job_id: int | None = None, +) -> list[DetectionEmbedding]: + """Store the vectors a feature-only pipeline returned for detections Antenna already has. + + Writes ``DetectionEmbedding`` rows and nothing else: a box that matches no stored + detection is skipped rather than created, classifications in the response are ignored, + and no occurrence, determination or null marker is touched. + """ + ignored = sum(len(detection.classifications) for detection in results.detections) + if ignored: + logger.warning(f"Ignored {ignored} classifications returned by a feature-only pipeline.") + image_ids = {int(detection.source_image_id) for detection in results.detections if detection.bbox is not None} + detections = list( + Detection.objects.valid().filter(source_image_id__in=image_ids).only("pk", "source_image_id", "bbox") + ) + return create_detection_embeddings( + detections=detections, + detection_responses=results.detections, + algorithms_known=algorithms_known, + logger=logger, + job_id=job_id, + ) + + def create_category_map_for_classification( classification_resp: ClassificationResponse, logger: logging.Logger = logger, @@ -1121,6 +1298,26 @@ def save_results( ) algorithms_known: dict[str, Algorithm] = {algo.key: algo for algo in pipeline.algorithms.all()} + if feature_extraction_only(list(algorithms_known.values())): + job_logger.info(f"Pipeline {pipeline} only extracts features; storing vectors for existing detections.") + embeddings = save_features_for_existing_detections( + results, algorithms_known, logger=job_logger, job_id=job.pk if job else None + ) + total_time = time.time() - start_time + job_logger.info( + f"Saved {len(embeddings)} feature vectors from pipeline {pipeline} in {total_time:.2f} seconds" + ) + if return_created: + return PipelineSaveResults( + pipeline=pipeline, + source_images=list(source_images), + detections=[], + classifications=[], + algorithms={}, + total_time=total_time, + ) + return None + try: detection_algorithm = pipeline.algorithms.get(task_type__in=Algorithm.detection_task_types) except Algorithm.DoesNotExist: @@ -1309,6 +1506,18 @@ class Meta: def __str__(self): return f'#{self.pk} "{self.name}" ({self.slug}) v{self.version}' + def feature_extraction_algorithms(self) -> list[Algorithm]: + """The pipeline's algorithms when every one of them extracts features, otherwise none. + + Such a pipeline is run on existing detections and only stores their vectors: it + creates no detection, classification or occurrence. + """ + algorithms = list(self.algorithms.all()) + return algorithms if feature_extraction_only(algorithms) else [] + + def is_feature_only(self) -> bool: + return bool(self.feature_extraction_algorithms()) + def get_config(self, project_id: int | None = None) -> PipelineRequestConfigParameters: """ Get the configuration for the pipeline request. diff --git a/ami/ml/orchestration/jobs.py b/ami/ml/orchestration/jobs.py index a965ed7c9..8a459959e 100644 --- a/ami/ml/orchestration/jobs.py +++ b/ami/ml/orchestration/jobs.py @@ -81,6 +81,27 @@ async def cleanup(): return redis_success and nats_success +def _attach_detections_for_feature_pipeline(job: "Job", tasks: list[PipelineProcessingTask]) -> None: + """Give each task the existing boxes a feature-only pipeline should embed, in batches of images.""" + from ami.ml.models.pipeline import FILTER_PROCESSED_BATCH_SIZE, collect_detections_for_features + from ami.ml.schemas import SourceImageRequest + + algorithm_ids = ( + [algorithm.pk for algorithm in job.pipeline.feature_extraction_algorithms()] if job.pipeline else [] + ) + if not algorithm_ids: + return + include_existing = job.project.feature_flags.reprocess_all_images + for start in range(0, len(tasks), FILTER_PROCESSED_BATCH_SIZE): + batch = tasks[start : start + FILTER_PROCESSED_BATCH_SIZE] # noqa: E203 + requests = [SourceImageRequest(id=task.image_id, url=task.image_url) for task in batch] + by_image: dict[str, list] = {task.image_id: [] for task in batch} + for detection_request in collect_detections_for_features(requests, algorithm_ids, include_existing): + by_image[detection_request.source_image.id].append(detection_request) + for task in batch: + task.detections = by_image[task.image_id] + + def queue_images_to_nats(job: "Job", images: list[SourceImage]): """ Queue all images for a job to a NATS JetStream stream for the job. @@ -117,6 +138,8 @@ def queue_images_to_nats(job: "Job", images: list[SourceImage]): ) tasks.append((image.pk, task)) + _attach_detections_for_feature_pipeline(job, [task for _, task in tasks]) + # Store all image IDs in Redis for progress tracking state_manager = AsyncJobStateManager(job.pk) state_manager.initialize_job(image_ids) diff --git a/ami/ml/schemas.py b/ami/ml/schemas.py index 0224dc3f9..a7097367d 100644 --- a/ami/ml/schemas.py +++ b/ami/ml/schemas.py @@ -116,6 +116,11 @@ class AlgorithmConfigResponse(pydantic.BaseModel): description="A URI to the weight or model details, could be a public web URL or object store path.", ) category_map: AlgorithmCategoryMapResponse | None = None + embedding_dimensions: int | None = pydantic.Field( + default=None, + ge=1, + description="For a feature extractor, the length of every vector it returns.", + ) class Config: extra = "ignore" @@ -186,18 +191,25 @@ class EmbeddingResponse(pydantic.BaseModel): """A feature vector for one detection and the algorithm whose backbone produced it. Carried on the detection rather than on a classification, so storing it can never - add a prediction that competes for the occurrence's determination. + add a prediction that competes for the occurrence's determination. Its length is the + algorithm's own (extractors differ), and only vectors from one algorithm are comparable. """ features: list[float] = pydantic.Field( - description="The feature vector. Must be exactly 2048 floats.", + description="The feature vector. Also accepted under the key 'vector'.", ) algorithm: AlgorithmReference + @pydantic.root_validator(pre=True) + def _accept_vector_key(cls, values): + if isinstance(values, dict) and "features" not in values and "vector" in values: + values = {**values, "features": values["vector"]} + return values + @pydantic.validator("features") - def _features_length(cls, v): - if len(v) != 2048: - raise ValueError(f"features must be length 2048, got {len(v)}") + def _features_not_empty(cls, v): + if not v: + raise ValueError("features must contain at least one value") return v @@ -319,9 +331,13 @@ class PipelineProcessingTask(pydantic.BaseModel): image_id: str image_url: str reply_subject: str | None = None # The NATS subject to send the result to - # TODO: Do we need these? - # detections: list[DetectionRequest] | None = None - # config: PipelineRequestConfigParameters | dict | None = None + detections: list[DetectionRequest] | None = pydantic.Field( + default=None, + description=( + "Existing detections on the image to run the pipeline on instead of detecting anew. " + "Sent for feature-only pipelines, which return these boxes with an embedding each." + ), + ) class ProcessingServiceClientInfo(pydantic.BaseModel): diff --git a/ami/ml/serializers.py b/ami/ml/serializers.py index e7e9e6aaf..07306a36b 100644 --- a/ami/ml/serializers.py +++ b/ami/ml/serializers.py @@ -43,6 +43,7 @@ class Meta: "version", "version_name", "task_type", + "embedding_dimensions", "category_map", "category_count", "created_at", From 0f644ae27a581c8ba7a977f3f01f598504033f03 Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Mon, 28 Sep 2026 12:48:06 -0700 Subject: [PATCH 30/44] feat(tracking): compare the project's default feature extractor when a session has several A session can now carry vectors from more than one extractor, for example the moth classifier's backbone and a BioCLIP pass run afterwards. Tracking and the merge previews used to give up on such a session; when no extractor is chosen they now take the project's default: a feature-extraction algorithm in a pipeline enabled for the project, else the extractor whose vector was stored most recently in the session, else the newest algorithm. Vectors are still only ever compared within one extractor, the reader keeps a single vector length per algorithm, and cosine similarity refuses vectors of different lengths. require_features keeps its meaning. GET /api/v2/events/{id}/feature-extractors/ lists the extractors with vectors for a session, how many detections each covers, and which one is the default, for the tracking form. The admin tracking form also offers feature-extraction algorithms. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01C7Xf6VPbwWtTumhjjF15g8 --- ami/main/api/serializers.py | 17 + ami/main/api/views.py | 13 + .../management/commands/evaluate_tracking.py | 2 +- ami/main/models_future/embeddings.py | 97 ++++- ami/main/tests.py | 12 +- ami/ml/post_processing/admin/tracking_form.py | 8 +- ami/ml/post_processing/admin_forms.py | 6 +- ami/ml/post_processing/tracking_task.py | 44 ++- ami/ml/test_feature_extraction.py | 340 ++++++++++++++++++ 9 files changed, 507 insertions(+), 32 deletions(-) create mode 100644 ami/ml/test_feature_extraction.py diff --git a/ami/main/api/serializers.py b/ami/main/api/serializers.py index 255156073..ac551db2b 100644 --- a/ami/main/api/serializers.py +++ b/ami/main/api/serializers.py @@ -2477,3 +2477,20 @@ class CaptureMatchesResponseSerializer(serializers.Serializer): "and then every score is null.", ) detections = CaptureMatchSerializer(many=True) + + +class SessionFeatureExtractorSerializer(serializers.Serializer): + """A feature extractor with vectors stored for a session's detections.""" + + id = serializers.IntegerField(source="algorithm.pk") + name = serializers.CharField(source="algorithm.name") + key = serializers.CharField(source="algorithm.key") + task_type = serializers.CharField(source="algorithm.task_type", allow_null=True) + embedding_dimensions = serializers.IntegerField(source="algorithm.embedding_dimensions", allow_null=True) + embeddings_count = serializers.IntegerField(help_text="Detections with a stored embedding from it.") + classification_vectors_count = serializers.IntegerField( + help_text="Classifications from it that carry a vector (data processed before embeddings were stored)." + ) + is_default = serializers.BooleanField( + help_text="Whether tracking compares this extractor's vectors when none is chosen." + ) diff --git a/ami/main/api/views.py b/ami/main/api/views.py index def37e92b..6f0cf7634 100644 --- a/ami/main/api/views.py +++ b/ami/main/api/views.py @@ -34,6 +34,7 @@ from ami.base.views import ProjectMixin from ami.main.api.schemas import limit_doc_param, project_id_doc_param from ami.main.api.serializers import TagSerializer +from ami.main.models_future.embeddings import feature_extractors_with_vectors from ami.main.models_future.history import occurrence_timeline from ami.main.models_future.identifications import create_identifications_batch, resolve_occurrences from ami.main.models_future.merge_candidates import ( @@ -119,6 +120,7 @@ PageSerializer, ProjectListSerializer, ProjectSerializer, + SessionFeatureExtractorSerializer, SiteSerializer, SourceImageCollectionNestedSerializer, SourceImageCollectionSerializer, @@ -540,6 +542,17 @@ def get_queryset(self) -> QuerySet: return qs + @extend_schema(parameters=[project_id_doc_param], responses=SessionFeatureExtractorSerializer(many=True)) + @action(detail=True, methods=["get"], name="feature-extractors", url_path="feature-extractors") + def feature_extractors(self, request, pk=None): + """The feature extractors with vectors for this session's detections, the default one marked. + + Tracking compares vectors from one extractor only; this lists the choices. + """ + event = self.get_object() + rows = feature_extractors_with_vectors(event.project_id, source_image__event=event) + return Response(SessionFeatureExtractorSerializer(rows, many=True).data) + @action(detail=True, methods=["get"], name="timeline") def timeline(self, request, pk=None): """ diff --git a/ami/main/management/commands/evaluate_tracking.py b/ami/main/management/commands/evaluate_tracking.py index baf303c50..21d2ed419 100644 --- a/ami/main/management/commands/evaluate_tracking.py +++ b/ami/main/management/commands/evaluate_tracking.py @@ -41,7 +41,7 @@ def add_arguments(self, parser): "--feature-extraction-algorithm", type=int, default=None, - help="Algorithm ID whose embeddings to compare. Default: the only one in the session.", + help="Algorithm ID whose embeddings to compare. Default: as tracking picks it.", ) parser.add_argument("--format", choices=["text", "json"], default="text") parser.add_argument("--per-track", action="store_true", help="Include per-track scores in JSON output.") diff --git a/ami/main/models_future/embeddings.py b/ami/main/models_future/embeddings.py index d82991e2f..0901eb712 100644 --- a/ami/main/models_future/embeddings.py +++ b/ami/main/models_future/embeddings.py @@ -10,10 +10,14 @@ from __future__ import annotations +import collections +import logging from collections.abc import Iterable from typing import Any -from django.db.models import F, IntegerField, QuerySet, Value +from django.db.models import Count, F, IntegerField, QuerySet, Value + +logger = logging.getLogger(__name__) _PREFER_EMBEDDING = 0 _PREFER_CLASSIFICATION = 1 @@ -63,10 +67,21 @@ def latest_vectors( def vectors_for_detections(detection_ids: Iterable[int], algorithm_id: int) -> dict[int, Any]: - """Each detection's vector from one algorithm, in one query. Detections without one are absent.""" - return { + """Each detection's vector from one algorithm, in one query. Detections without one are absent. + + All the vectors returned have one length. Should an algorithm have stored two lengths + (an embedding and an older classification vector), only the more common length is + kept, so no caller can compare vectors of different sizes. + """ + vectors = { detection_id: vector for (detection_id, _), vector in latest_vectors(detection_ids, [algorithm_id]).items() } + lengths = collections.Counter(len(vector) for vector in vectors.values()) + if len(lengths) > 1: + keep = max(lengths, key=lambda length: (lengths[length], length)) + logger.warning(f"Algorithm {algorithm_id} has vectors of lengths {dict(lengths)}; using only length {keep}.") + vectors = {detection_id: vector for detection_id, vector in vectors.items() if len(vector) == keep} + return vectors def algorithm_ids_with_vectors(**detection_lookups: Any) -> set[int]: @@ -84,3 +99,79 @@ def algorithm_ids_with_vectors(**detection_lookups: Any) -> set[int]: .values_list("algorithm_id", flat=True) ) return set(embedded.union(classified)) + + +def default_feature_algorithm_id(project: Any, algorithm_ids: Iterable[int], **detection_lookups: Any) -> int | None: + """The extractor to compare when the caller chose none, among ``algorithm_ids``. + + One the project runs (a feature-extraction algorithm in a pipeline enabled for it) wins; + among several, or when there is none, the one whose vector was stored most recently for + the detections matching the lookups; failing that, the newest algorithm. At most 2 queries. + """ + from ami.main.models import DetectionEmbedding + from ami.ml.models import Algorithm, Pipeline + + algorithm_ids = sorted(set(algorithm_ids)) + if len(algorithm_ids) <= 1: + return algorithm_ids[0] if algorithm_ids else None + configured = sorted( + Algorithm.objects.filter( + pk__in=algorithm_ids, + task_type__in=Algorithm.feature_extraction_task_types, + pipelines__in=Pipeline.objects.all().enabled(project).values("pk"), + ) + .order_by() + .values_list("pk", flat=True) + .distinct() + ) + if len(configured) == 1: + return configured[0] + choices = configured or algorithm_ids + lookups = {f"detection__{key}": value for key, value in detection_lookups.items()} + latest = ( + DetectionEmbedding.objects.filter(algorithm_id__in=choices, **lookups) + .order_by("-updated_at", "-pk") + .values_list("algorithm_id", flat=True) + .first() + ) + return latest if latest is not None else max(choices) + + +def feature_extractors_with_vectors(project: Any, **detection_lookups: Any) -> list[dict[str, Any]]: + """The algorithms with vectors for the detections matching the lookups, and how many each has. + + Each row: the algorithm, ``embeddings_count`` (``DetectionEmbedding`` rows), + ``classification_vectors_count`` (classifications carrying a vector) and ``is_default``, + the one tracking compares when no extractor is chosen. Newest algorithm first. At most 5 queries. + """ + from ami.main.models import Classification, DetectionEmbedding + from ami.ml.models import Algorithm + + lookups = {f"detection__{key}": value for key, value in detection_lookups.items()} + embedded = dict( + DetectionEmbedding.objects.filter(**lookups) + .order_by() + .values("algorithm_id") + .annotate(n=Count("pk")) + .values_list("algorithm_id", "n") + ) + classified = dict( + Classification.objects.filter(**lookups, features_2048__isnull=False, algorithm_id__isnull=False) + .order_by() + .values("algorithm_id") + .annotate(n=Count("pk")) + .values_list("algorithm_id", "n") + ) + algorithm_ids = set(embedded) | set(classified) + if not algorithm_ids: + return [] + default_id = default_feature_algorithm_id(project, algorithm_ids, **detection_lookups) + return [ + { + "algorithm": algorithm, + "embeddings_count": embedded.get(algorithm.pk, 0), + "classification_vectors_count": classified.get(algorithm.pk, 0), + "is_default": algorithm.pk == default_id, + } + for algorithm in Algorithm.objects.filter(pk__in=sorted(algorithm_ids)).order_by("-pk") + ] diff --git a/ami/main/tests.py b/ami/main/tests.py index cd024ec6c..9a416b0df 100644 --- a/ami/main/tests.py +++ b/ami/main/tests.py @@ -10601,19 +10601,17 @@ def test_a_detector_only_session_links_nothing(self): self.assertEqual((row["skipped_reason"], row["would_link"]), ("no_vector", False)) self.assertIsNotNone(row["cost"], "The geometry is still scored") - def test_two_feature_extractors_link_nothing(self): - """Embeddings from two extractors leave tracking no single one to compare, so it skips the - captures while it requires features, as it does a session with none.""" + def test_two_feature_extractors_are_never_compared_with_each_other(self): + """With embeddings from two extractors, tracking compares the default one only (here the + one stored most recently), so a box whose track frame has no vector from it does not link.""" other_extractor = Algorithm.objects.create(name="Other extractor", key="other-extractor") track = self._track(self.captures[1:3], vector=self.VECTOR) self._box(self.captures[3], self.NEAR_BOX, vector=self.VECTOR, algorithm=other_extractor) data = self.get_matches(track, self.captures[3].pk).data - self.assertIsNone(data["feature_algorithm_id"]) - self.assertEqual( - (data["detections"][0]["skipped_reason"], data["detections"][0]["would_link"]), ("no_vector", False) - ) + self.assertEqual(data["feature_algorithm_id"], other_extractor.pk) + self.assertFalse(data["detections"][0]["would_link"]) def test_the_reference_is_the_nearest_track_frame_on_another_capture(self): """A track on the second and fifth of six captures a minute apart. Each capture is diff --git a/ami/ml/post_processing/admin/tracking_form.py b/ami/ml/post_processing/admin/tracking_form.py index 214731a72..203bfb044 100644 --- a/ami/ml/post_processing/admin/tracking_form.py +++ b/ami/ml/post_processing/admin/tracking_form.py @@ -37,12 +37,14 @@ class TrackingActionForm(BasePostProcessingActionForm): ), ) feature_extraction_algorithm_id = forms.ModelChoiceField( - queryset=Algorithm.objects.filter(task_type=AlgorithmTaskType.CLASSIFICATION.value).order_by("name"), + queryset=Algorithm.objects.filter( + task_type__in=[AlgorithmTaskType.CLASSIFICATION.value, *Algorithm.feature_extraction_task_types] + ).order_by("name"), required=False, label="Feature extractor", help_text=( - "Whose embeddings to compare. Leave blank to detect it automatically; set it when " - "more than one classifier has run on the same session." + "Whose embeddings to compare. Leave blank for the session's only extractor or, " + "when there are several, the project's default one." ), ) skip_if_human_identifications = forms.BooleanField( diff --git a/ami/ml/post_processing/admin_forms.py b/ami/ml/post_processing/admin_forms.py index 72494365d..013593968 100644 --- a/ami/ml/post_processing/admin_forms.py +++ b/ami/ml/post_processing/admin_forms.py @@ -77,9 +77,9 @@ class TrackingActionForm(forms.Form): required=False, help_text=( "Override the algorithm whose embeddings are used for matching. Leave " - "blank to auto-detect (works when only one feature-extracting algorithm " - "ran on the event). Required when multiple algorithms have produced " - "embeddings on the same event." + "blank to use the only one with vectors on the event, or, when several " + "have them, the project's default: one it runs as a feature extractor, " + "else the one stored most recently." ), ) diff --git a/ami/ml/post_processing/tracking_task.py b/ami/ml/post_processing/tracking_task.py index 583faab02..403f3c5ae 100644 --- a/ami/ml/post_processing/tracking_task.py +++ b/ami/ml/post_processing/tracking_task.py @@ -22,7 +22,11 @@ SourceImageCollection, update_calculated_fields_for_sessions_and_stations, ) -from ami.main.models_future.embeddings import algorithm_ids_with_vectors, vectors_for_detections +from ami.main.models_future.embeddings import ( + algorithm_ids_with_vectors, + default_feature_algorithm_id, + vectors_for_detections, +) from ami.main.models_future.track_stats import refresh_track_stats_for_ids from ami.main.models_future.tracks import clear_grouping_verification from ami.main.schemas import TrackingResultPayload @@ -66,8 +70,8 @@ class TrackingConfig(pydantic.BaseModel): # data is a v2 concern (see #1272 for the incremental append/prepend plan). require_fresh_event: bool = True - # Which feature extractor's embeddings to compare. Left unset, the task infers it - # when exactly one algorithm produced embeddings for the event. + # Which feature extractor's embeddings to compare. Left unset: the event's only one, + # or the project's default among several (see resolve_feature_algorithm). feature_extraction_algorithm_id: int | None = None @pydantic.root_validator(skip_on_failure=True) @@ -84,6 +88,9 @@ class Config: def cosine_similarity(v1: Iterable[float], v2: Iterable[float]) -> float: a = np.array(v1) b = np.array(v2) + if a.shape != b.shape: + # Vectors of different lengths come from different extractors and are not comparable. + raise ValueError(f"Cannot compare vectors of shapes {a.shape} and {b.shape}") sim = np.dot(a, b) / (np.linalg.norm(a) * np.linalg.norm(b)) return float(np.clip(sim, 0.0, 1.0)) @@ -138,9 +145,8 @@ def get_unique_feature_algorithm_for_event(event: Event) -> tuple[Algorithm | No If exactly one feature-extraction algorithm stored vectors (embeddings or classification ``features_2048``) for this event, returns that algorithm and a - single-element list. Otherwise returns ``(None, candidates)`` so the caller can - either skip with a warning or require the operator to pass an explicit - ``feature_extraction_algorithm_id``. + single-element list. Otherwise returns ``(None, candidates)`` and the caller picks + one (``resolve_feature_algorithm``). """ algo_ids = algorithm_ids_with_vectors(source_image__event=event) candidates = list(Algorithm.objects.filter(pk__in=algo_ids)) @@ -155,9 +161,11 @@ def resolve_feature_algorithm( """The feature extractor a tracking run compares embeddings from, and whether it tracks the event. Returns ``(algorithm, should_track, note)``. ``algorithm`` is None when the run falls - back to geometry alone, and ``note`` says why a run falls back or skips; it is empty - when one extractor was configured or found. ``candidates`` are the extractors that - produced embeddings: every one in the event unless the caller passes a narrower set. + back to geometry alone. With vectors from several extractors and none configured, the + project's default is taken (see ``default_feature_algorithm_id``). ``note`` says which + extractor was picked among several, or why a run falls back or skips; it is empty when + one extractor was configured or found. ``candidates`` are the extractors that produced + embeddings: every one in the event unless the caller passes a narrower set. """ if config.feature_extraction_algorithm_id is not None: algorithm = Algorithm.objects.filter(pk=config.feature_extraction_algorithm_id).first() @@ -176,14 +184,20 @@ def resolve_feature_algorithm( return candidates[0], True, "" if candidates: + # Vectors from several extractors: compare the project's default one, never a mix. + default_id = default_feature_algorithm_id( + event.project_id, [a.pk for a in candidates], source_image__event=event + ) + algorithm = next(a for a in candidates if a.pk == default_id) candidate_names = [f"#{a.pk} {a.name}" for a in candidates] - message = ( - f"Event {event.pk}: detections classified by {len(candidates)} different " - f"feature-extraction algorithms ({candidate_names}). Pass " - "feature_extraction_algorithm_id in the job config to disambiguate." + return ( + algorithm, + True, + f"Event {event.pk}: vectors from {len(candidates)} feature extractors ({candidate_names}); " + f"comparing #{algorithm.pk} {algorithm.name}. Pass feature_extraction_algorithm_id to choose another.", ) - else: - message = f"Event {event.pk}: no detections carry feature embeddings." + + message = f"Event {event.pk}: no detections carry feature embeddings." if config.require_features: return None, False, f"{message} Skipping." diff --git a/ami/ml/test_feature_extraction.py b/ami/ml/test_feature_extraction.py new file mode 100644 index 000000000..907529878 --- /dev/null +++ b/ami/ml/test_feature_extraction.py @@ -0,0 +1,340 @@ +"""Feature-only pipelines: vectors for detections Antenna already has, and nothing else. + +A feature extractor (e.g. a BioCLIP backbone) is run on existing detections so tracking +can compare every detection by appearance. These tests pin that such a run writes only +``DetectionEmbedding`` rows, skips detections that already have a vector, keeps one vector +length per algorithm, and that readers never compare vectors of different lengths. +""" + +import datetime +from unittest import mock + +import pydantic +from django.test import TestCase +from rest_framework.test import APITestCase + +from ami.main.models import ( + Classification, + Deployment, + Detection, + DetectionEmbedding, + Event, + Occurrence, + Project, + SourceImage, + SourceImageCollection, + group_images_into_events, +) +from ami.main.models_future.embeddings import ( + default_feature_algorithm_id, + feature_extractors_with_vectors, + vectors_for_detections, +) +from ami.ml.models import Algorithm, Pipeline, ProcessingService +from ami.ml.models.pipeline import ( + EmbeddingDimensionMismatch, + collect_detections_for_features, + collect_images, + process_images, + save_results, +) +from ami.ml.models.project_pipeline_config import ProjectPipelineConfig +from ami.ml.post_processing.tracking_task import TrackingConfig, cosine_similarity, resolve_feature_algorithm +from ami.ml.schemas import EmbeddingResponse, PipelineResultsResponse, SourceImageRequest +from ami.users.models import User + +BIOCLIP_DIMENSIONS = 1024 + + +def _box(offset: float) -> list[float]: + return [offset, offset, offset + 10.5, offset + 12.25] + + +class FeatureOnlyFixture: + """A project with detected, classified and determined captures, plus a feature-only pipeline.""" + + def _set_up_project(self, images: int = 3, boxes_per_image: int = 2) -> None: + self.project = Project.objects.create(name="Feature extraction") + self.deployment = Deployment.objects.create(name="Station", project=self.project) + self.detector = Algorithm.objects.create(name="Detector", key="test-detector", task_type="localization") + self.classifier = Algorithm.objects.create( + name="Classifier", key="test-classifier", task_type="classification" + ) + self.extractor = Algorithm.objects.create(name="Backbone", key="test-backbone", task_type="embedding") + self.pipeline = Pipeline.objects.create(name="Features only", slug="features-only") + self.pipeline.algorithms.set([self.extractor]) + + start = datetime.datetime(2024, 6, 1, 22, 0) + self.images = [ + SourceImage.objects.create( + deployment=self.deployment, + project=self.project, + path=f"feat/{i}.jpg", + timestamp=start + datetime.timedelta(minutes=i), + ) + for i in range(images) + ] + group_images_into_events(self.deployment) + self.images = list(SourceImage.objects.filter(pk__in=[i.pk for i in self.images]).order_by("timestamp")) + self.collection = SourceImageCollection.objects.create(project=self.project, name="Scope") + self.collection.images.set(self.images) + for image in self.images: + for n in range(boxes_per_image): + occurrence = Occurrence.objects.create( + project=self.project, deployment=self.deployment, event=image.event + ) + detection = Detection.objects.create( + source_image=image, + bbox=_box(20.0 * n), + detection_algorithm=self.detector, + occurrence=occurrence, + timestamp=image.timestamp, + ) + Classification.objects.create( + detection=detection, algorithm=self.classifier, score=0.9, timestamp=image.timestamp + ) + # A null marker must never be sent or embedded. + Detection.objects.create(source_image=self.images[0], bbox=None, detection_algorithm=self.detector) + + def _embed(self, detections, algorithm: Algorithm, length: int = BIOCLIP_DIMENSIONS) -> None: + DetectionEmbedding.objects.bulk_create( + [DetectionEmbedding(detection=d, algorithm=algorithm, vector=[0.5] * length) for d in detections] + ) + + def _response(self, boxes: list[tuple[SourceImage, list[float]]], length: int = BIOCLIP_DIMENSIONS, **extra): + """What a feature-only pipeline sends back: the boxes it was given, each with a vector.""" + return PipelineResultsResponse( + pipeline=self.pipeline.slug, + total_time=0.1, + source_images=[{"id": str(image.pk), "url": "x"} for image in {image for image, _ in boxes}], + detections=[ + { + "source_image_id": str(image.pk), + "bbox": dict(zip(["x1", "y1", "x2", "y2"], box)), + "algorithm": {"name": self.detector.name, "key": self.detector.key}, + "timestamp": datetime.datetime.now().isoformat(), + "embeddings": [ + { + "algorithm": {"name": self.extractor.name, "key": self.extractor.key}, + "vector": [0.1] * length, + } + ], + **extra, + } + for image, box in boxes + ], + ) + + def _counts(self) -> tuple: + return ( + Detection.objects.count(), + Classification.objects.count(), + Occurrence.objects.count(), + sorted(Occurrence.objects.values_list("pk", "determination_id", "determination_score")), + ) + + +class TestEmbeddingSchema(TestCase): + def test_a_vector_of_any_length_is_accepted_under_either_key(self): + algorithm = {"name": "Backbone", "key": "test-backbone"} + for key in ("features", "vector"): + parsed = EmbeddingResponse.parse_obj({"algorithm": algorithm, key: [0.1] * BIOCLIP_DIMENSIONS}) + self.assertEqual(len(parsed.features), BIOCLIP_DIMENSIONS) + with self.assertRaises(pydantic.ValidationError): + EmbeddingResponse.parse_obj({"algorithm": algorithm, "vector": []}) + + +class TestFeatureOnlySave(FeatureOnlyFixture, TestCase): + def setUp(self) -> None: + self._set_up_project() + + def test_an_embeddings_only_response_writes_only_vectors(self): + """No detection, classification or occurrence is created and no determination moves, even + for a box Antenna does not have or classifications the service sent anyway.""" + before = self._counts() + known = [(image, _box(0.0)) for image in self.images] + unknown = [(self.images[0], _box(500.0))] + stray = { + "classifications": [ + { + "classification": "Moth", + "scores": [0.99], + "algorithm": {"name": self.extractor.name, "key": self.extractor.key}, + "timestamp": datetime.datetime.now().isoformat(), + } + ] + } + save_results(self._response(known + unknown, **stray)) + + self.assertEqual(self._counts(), before) + stored = DetectionEmbedding.objects.filter(algorithm=self.extractor) + self.assertEqual( + sorted(stored.values_list("detection__source_image_id", flat=True)), sorted(i.pk for i in self.images) + ) + self.extractor.refresh_from_db() + self.assertEqual(self.extractor.embedding_dimensions, BIOCLIP_DIMENSIONS) + + def test_a_vector_of_another_length_is_refused_and_nothing_is_stored(self): + save_results(self._response([(self.images[0], _box(0.0))])) + with self.assertRaises(EmbeddingDimensionMismatch): + save_results(self._response([(self.images[1], _box(0.0))], length=512)) + self.assertEqual(DetectionEmbedding.objects.count(), 1) + + +class TestExtractFeaturesScope(FeatureOnlyFixture, TestCase): + """What an extract-features run sends: only real detections still missing a vector.""" + + def setUp(self) -> None: + self._set_up_project() + # The first image is done; one box on the second is done. + self._embed(self.images[0].detections.valid(), self.extractor) + self._embed(self.images[1].detections.valid().filter(bbox=_box(0.0)), self.extractor) + + def test_images_whose_detections_all_have_vectors_are_skipped(self): + collected = collect_images(collection=self.collection, pipeline=self.pipeline) + self.assertEqual([image.pk for image in collected], [self.images[1].pk, self.images[2].pk]) + + def test_a_vector_from_another_algorithm_does_not_count(self): + other = Algorithm.objects.create(name="Other", key="other-backbone", task_type="embedding") + self._embed(self.images[2].detections.valid(), other) + collected = collect_images(collection=self.collection, pipeline=self.pipeline) + self.assertIn(self.images[2].pk, [image.pk for image in collected]) + + def test_the_request_carries_only_the_missing_boxes_in_one_query(self): + requests = [SourceImageRequest(id=str(image.pk), url="x") for image in self.images] + with self.assertNumQueries(1): + detections = collect_detections_for_features(requests, [self.extractor.pk]) + self.assertEqual( + sorted((int(d.source_image.id), d.bbox.x1) for d in detections), + [(self.images[1].pk, 20.0), (self.images[2].pk, 0.0), (self.images[2].pk, 20.0)], + ) + self.assertEqual({d.algorithm.key for d in detections}, {self.detector.key}) + + def test_process_images_sends_existing_boxes_and_skips_done_images(self): + sent = {} + + def post(url, json): + sent.update(json) + return mock.Mock( + ok=True, + json=lambda: {"pipeline": json["pipeline"], "total_time": 0, "source_images": [], "detections": []}, + ) + + with mock.patch.object(SourceImage, "public_url", return_value="http://example.org/i.jpg"), mock.patch( + "ami.ml.models.pipeline.create_session", return_value=mock.Mock(post=post) + ): + process_images(self.pipeline, "http://example.org/process", self.images, project_id=self.project.pk) + + self.assertEqual(sorted(int(i["id"]) for i in sent["source_images"]), [self.images[1].pk, self.images[2].pk]) + self.assertEqual(len(sent["detections"]), 3) + + +class TestReadersNeverMixLengths(FeatureOnlyFixture, TestCase): + def setUp(self) -> None: + self._set_up_project(images=2, boxes_per_image=2) + + def test_one_algorithm_with_two_lengths_returns_one_length(self): + detections = list(Detection.objects.valid().order_by("pk")) + self._embed(detections[:3], self.extractor, length=BIOCLIP_DIMENSIONS) + Classification.objects.filter(detection=detections[3]).update( + algorithm=self.extractor, features_2048=[0.5] * 2048 + ) + vectors = vectors_for_detections([d.pk for d in detections], self.extractor.pk) + self.assertEqual({len(v) for v in vectors.values()}, {BIOCLIP_DIMENSIONS}) + self.assertEqual(len(vectors), 3) + + def test_cosine_similarity_refuses_vectors_of_different_lengths(self): + with self.assertRaises(ValueError): + cosine_similarity([1.0] * 1024, [1.0] * 2048) + + +class TestDefaultFeatureExtractor(FeatureOnlyFixture, TestCase): + """With vectors from several extractors and none chosen, tracking compares the default one.""" + + def setUp(self) -> None: + self._set_up_project(images=2, boxes_per_image=2) + self.event = Event.objects.get(pk=self.images[0].event_id) + self.older = Algorithm.objects.create(name="Older backbone", key="older-backbone", task_type="embedding") + detections = Detection.objects.valid() + self._embed(detections, self.older, length=2048) + self._embed(detections, self.extractor) # stored last + + def test_the_most_recently_stored_extractor_is_the_default(self): + algorithm, should_track, note = resolve_feature_algorithm( + self.event, TrackingConfig(event_ids=[self.event.pk], require_features=True) + ) + self.assertEqual((algorithm, should_track), (self.extractor, True)) + self.assertIn("2 feature extractors", note) + + def test_an_extractor_the_project_runs_wins_over_a_more_recent_one(self): + service = ProcessingService.objects.create(name="Service", endpoint_url=None) + service.projects.add(self.project) + older_pipeline = Pipeline.objects.create(name="Older features", slug="older-features") + older_pipeline.algorithms.set([self.older]) + service.pipelines.add(older_pipeline) + ProjectPipelineConfig.objects.create(project=self.project, pipeline=older_pipeline, enabled=True) + + algorithm_ids = [self.older.pk, self.extractor.pk] + self.assertEqual( + default_feature_algorithm_id(self.project.pk, algorithm_ids, source_image__event=self.event), self.older.pk + ) + + def test_a_chosen_extractor_is_kept(self): + config = TrackingConfig( + event_ids=[self.event.pk], require_features=True, feature_extraction_algorithm_id=self.older.pk + ) + self.assertEqual(resolve_feature_algorithm(self.event, config)[0], self.older) + + def test_listing_a_sessions_extractors_takes_a_fixed_number_of_queries(self): + with self.assertNumQueries(5): + rows = feature_extractors_with_vectors(self.project.pk, source_image__event=self.event) + self.assertEqual( + [(row["algorithm"].pk, row["embeddings_count"], row["is_default"]) for row in rows], + [(self.older.pk, 4, False), (self.extractor.pk, 4, True)], # newest algorithm first + ) + + +class TestSessionFeatureExtractorsEndpoint(FeatureOnlyFixture, APITestCase): + """Who can list a session's feature extractors: whoever can open the session.""" + + def setUp(self) -> None: + self._set_up_project(images=2, boxes_per_image=2) + self._embed(Detection.objects.valid(), self.extractor) + self.owner = User.objects.create_user(email="feat-owner@insectai.org", password="x") + self.member = User.objects.create_user(email="feat-member@insectai.org", password="x") + self.outsider = User.objects.create_user(email="feat-outsider@insectai.org", password="x") + self.superuser = User.objects.create_superuser(email="feat-admin@insectai.org", password="x") + self.project.owner = self.owner + self.project.draft = True + self.project.save() + self.project.members.add(self.member) + self.url = f"/api/v2/events/{self.images[0].event_id}/feature-extractors/?project_id={self.project.pk}" + + def _status(self, user) -> int: + self.client.force_authenticate(user) + return self.client.get(self.url).status_code + + def test_permission_matrix_on_a_draft_project(self): + self.assertEqual(self._status(self.member), 200) + self.assertEqual(self._status(self.superuser), 200) + self.assertIn(self._status(self.outsider), (403, 404)) + self.assertIn(self._status(None), (401, 403, 404)) + + def test_the_response_names_each_extractor_and_the_default(self): + self.client.force_authenticate(self.member) + body = self.client.get(self.url).json() + self.assertEqual( + body, + [ + { + "id": self.extractor.pk, + "name": self.extractor.name, + "key": self.extractor.key, + "task_type": "embedding", + "embedding_dimensions": None, + "embeddings_count": 4, + "classification_vectors_count": 0, + "is_default": True, + } + ], + ) From 2b802541be83a5f0fdd943d93c2d3f7af3323f31 Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Mon, 28 Sep 2026 12:48:06 -0700 Subject: [PATCH 31/44] docs: describe how detection embeddings are stored, filled in and chosen for tracking [skip ci] Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01C7Xf6VPbwWtTumhjjF15g8 --- docs/claude/reference/detection-embeddings.md | 93 +++++++++++++++++++ 1 file changed, 93 insertions(+) create mode 100644 docs/claude/reference/detection-embeddings.md diff --git a/docs/claude/reference/detection-embeddings.md b/docs/claude/reference/detection-embeddings.md new file mode 100644 index 000000000..3c3aa4c70 --- /dev/null +++ b/docs/claude/reference/detection-embeddings.md @@ -0,0 +1,93 @@ +# Detection embeddings (feature vectors) + +Reference for agents. How Antenna stores a feature vector for every detection, how a +feature-only pipeline fills them in for existing detections, and how tracking picks which +extractor's vectors to compare. See #1417 for the original design. + +## Storage + +- `DetectionEmbedding` (`ami/main/models.py`, migration `main/0102_detection_embedding.py`): + one row per (detection, algorithm), unique constraint `unique_detection_embedding_per_algorithm` + (its index leads with `detection_id` and serves reads). `vector` is a pgvector column with + **no fixed dimension**: extractors differ (2048 for the moth classifier backbones, 1024 for BioCLIP). + `job` FK (SET_NULL) records the job that last stored the vector. +- `Algorithm.embedding_dimensions` (`ami/ml/models/algorithm.py`, migration `ml/0029`): the one + length an algorithm's vectors have. Set from `/info` (`AlgorithmConfigResponse.embedding_dimensions`, + optional) in `get_or_create_algorithm_and_category_map`, or from the first vector stored + (`_check_embedding_dimensions` in `ami/ml/models/pipeline.py`, conditional UPDATE so concurrent + batches agree). A vector of another length raises `EmbeddingDimensionMismatch` (a + `PipelineNotConfigured`) and the batch stores nothing. +- Legacy: `Classification.features_2048` (vector(2048)) from classifiers run with `include_features`. + Readers fall back to it (see below). +- No ANN index exists; one would need a fixed dimension per index (a partial index per algorithm). + +## Contract with the processing service (`ami/ml/schemas.py`) + +- `DetectionResponse.embeddings: list[EmbeddingResponse] | None`, each + `{"algorithm": {"name", "key"}, "features": [float, ...]}`. The key `vector` is accepted as an + alias (root validator). Any non-empty length; the per-algorithm length is enforced at save time. +- The algorithm key must be declared in the pipeline's `/info`, else `PipelineNotConfigured`. +- **Feature-only pipeline**: every algorithm in the pipeline has `task_type` in + `Algorithm.feature_extraction_task_types` = `embedding` or `feature_extraction` + (`feature_extraction_only()` / `Pipeline.is_feature_only()`). +- Request for a feature-only run (sync, `process_images`): `PipelineRequest` with + `source_images` = only the images that still have a detection to embed, and `detections` = + those detections as `DetectionRequest{source_image, bbox, crop_image_url, algorithm=}` (`collect_detections_for_features`, one query). Async (NATS): the same list on + `PipelineProcessingTask.detections` (`_attach_detections_for_feature_pipeline` in + `ami/ml/orchestration/jobs.py`). The service must return the same boxes, each with embeddings, + and no new boxes. + +## Write path + +- `save_results` (`ami/ml/models/pipeline.py`): if the pipeline is feature-only it calls + `save_features_for_existing_detections` and returns. That path matches response boxes to + existing real detections by `(source_image_id, bbox rounded to BOX_MATCH_DECIMALS=3)`, skips + unmatched boxes (warning), ignores classifications (warning), and writes only + `DetectionEmbedding` rows: no detection, classification, occurrence, determination, null + marker or calculated-field update. +- Regular pipelines: `create_detection_embeddings` runs after `create_detections` and before + classifications, same matching key. Both use `bulk_create(update_conflicts=True)` on + (detection, algorithm), so re-delivery is idempotent and the newest vector wins. + +## The extract-features job + +No new job type: an ordinary ML job (`MLJob`, key `ml`) whose pipeline is feature-only. Same +permission as any ML job. Scope is the job's capture set, deployment or single capture. +`filter_processed_images` delegates to `filter_images_missing_features`: an image is sent only +when a real detection on it lacks a `DetectionEmbedding` from one of the pipeline's algorithms +(`detections_missing_features`; classification vectors do not count). The project flag +`reprocess_all_images` sends every real detection again. + +## Read path + +`ami/main/models_future/embeddings.py`: +- `latest_vectors` / `vectors_for_detections(ids, algorithm_id)`: one UNION ALL query over + embeddings and classification vectors, embedding preferred. Always one algorithm. + `vectors_for_detections` also keeps only the most common vector length, so an algorithm with + both a 1024 embedding and a 2048 classification vector never mixes them. +- `cosine_similarity` (`ami/ml/post_processing/tracking_task.py`) raises on a shape mismatch. +- `algorithm_ids_with_vectors(**lookups)`: extractors with vectors for a set of detections. +- `feature_extractors_with_vectors(project, **lookups)`: per extractor, `embeddings_count`, + `classification_vectors_count`, `is_default`; exposed as + `GET /api/v2/events/{id}/feature-extractors/?project_id=` (`EventViewSet.feature_extractors`, + `SessionFeatureExtractorSerializer`) for the tracking form. + +## How tracking picks the extractor + +`resolve_feature_algorithm(event, config)`: +1. `config.feature_extraction_algorithm_id` if given. +2. The only extractor with vectors in the session. +3. Several: `default_feature_algorithm_id(project, ids, source_image__event=event)`: a + feature-extraction algorithm in a pipeline enabled for the project, else the extractor whose + `DetectionEmbedding` was stored most recently in the session, else the newest algorithm id. + The run logs which one it compares. +4. None: `require_features=True` skips the session; otherwise geometry-only matching. + +Merge candidates use the same resolution over the two captures being compared. + +## Tests + +`ami/ml/test_feature_extraction.py` (feature-only save, dimension checks, extract scope, default +extractor, endpoint permissions and query count); `ami/ml/tests.py` `TestDetectionEmbeddings` +(regular-pipeline embeddings). From a190dd7013c29a356da3015e228d5d0b1a304e52 Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Mon, 28 Sep 2026 12:48:18 -0700 Subject: [PATCH 32/44] docs: index the detection embeddings reference [skip ci] Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01C7Xf6VPbwWtTumhjjF15g8 --- docs/claude/INDEX.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/claude/INDEX.md b/docs/claude/INDEX.md index b606cc2b1..2171d6d53 100644 --- a/docs/claude/INDEX.md +++ b/docs/claude/INDEX.md @@ -19,6 +19,7 @@ archived. | `reference/hierarchical-rollup-query-performance.md` | Per-taxon rollup counts on `GET /api/v2/taxa/` — query patterns and pitfalls. Keywords: taxa, rollup, counts | | `reference/occurrence-tracking.md` | How occurrences are populated by tracking: data model (`next_detection`, `features_2048`), cost function, config knobs, the adjacent-processed-captures constraint, how to run it, where results show up, all six editing endpoints (split/remove/merge/add/verify/unverify), the chain-boundary and clear-on-edit invariants, permission traps, and how to generate a demo session with known ground truth (`create_demo_project`). Keywords: tracking, occurrences, pgvector, embeddings, abundance, grouping verification, test set, demo data | | `reference/tracking-evaluation.md` | Scoring tracking against human-confirmed tracks (`grouping_verified_at`): metrics over detections in confirmed tracks only (pairwise and link precision/recall, fragments, completeness, purity, merges), the `evaluate_tracking` command (re-links raw detections read-only via `propose_event_links`, rolled back), the no-Django CSV mode, limits and next steps. Keywords: tracking evaluation, benchmark, ground truth, confirmed tracks, metrics | +| `reference/detection-embeddings.md` | Feature vectors per detection (2026-09-28): `DetectionEmbedding` with no fixed dimension, `Algorithm.embedding_dimensions` enforced on save, the `features`/`vector` response contract, feature-only pipelines (task type `embedding`/`feature_extraction`) that embed existing detections via an ordinary ML job, the embeddings-only save path, one-algorithm readers, how tracking picks the default extractor, and `GET /events/{id}/feature-extractors/`. Keywords: embeddings, BioCLIP, feature extraction, pgvector, extractor, tracking appearance | | `planning/idempotent-incremental-tracking.md` | Design for tracking that is always safe to re-run: the unit of work becomes a pair of adjacent processed captures rather than a whole session, with the successor marker recorded on `SourceImage`. Covers moving adjacency, bookkeeping without a new table, identification migration on merge, surviving human edits, concurrency, cost, and the trigger. Keywords: idempotent, incremental, tracking, adjacency, re-run, sampled captures | ## Architecture notes From d86a402cec128652ae223f9c1c98f2db1c236289 Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Mon, 28 Sep 2026 12:53:50 -0700 Subject: [PATCH 33/44] fix(embeddings): drop the fixed vector length on databases that ran an earlier draft of the embeddings table A database that applied an earlier draft of migration 0102 has a vector(2048) column, and Django cannot see the difference because its recorded state already has no length. Storing a vector of any other length then fails. This migration alters the column to an unbounded vector, which is a no-op where the column already has no length. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01C7Xf6VPbwWtTumhjjF15g8 --- ...04_detection_embedding_vector_any_length.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 ami/main/migrations/0104_detection_embedding_vector_any_length.py diff --git a/ami/main/migrations/0104_detection_embedding_vector_any_length.py b/ami/main/migrations/0104_detection_embedding_vector_any_length.py new file mode 100644 index 000000000..1f2938a02 --- /dev/null +++ b/ami/main/migrations/0104_detection_embedding_vector_any_length.py @@ -0,0 +1,18 @@ +# A database that applied an earlier draft of 0102 has a vector(2048) column, which rejects +# vectors of any other length. Dropping the length is a no-op where the column already has none. +# Django's state already matches, so this only touches the database. See #1417. + +from django.db import migrations + + +class Migration(migrations.Migration): + dependencies = [ + ("main", "0103_occurrence_history"), + ] + + operations = [ + migrations.RunSQL( + sql="ALTER TABLE main_detectionembedding ALTER COLUMN vector TYPE vector", + reverse_sql=migrations.RunSQL.noop, + ), + ] From 001485cf4604c2fa2b52b85d11ce1c2adeba70c9 Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Mon, 28 Sep 2026 13:00:22 -0700 Subject: [PATCH 34/44] fix(tracking): default to the feature extractor that covers the most detections in a session When a session has vectors from several extractors and none is chosen, the default was the one stored most recently. A feature-extraction run still in progress, or a one-off run on a single capture, then became the default, so tracking compared an extractor most boxes had no vector from, and the same session could track differently depending on which job wrote last. The default is now the extractor with vectors for the most detections in the session, counted once per detection across both stores. An extractor the project runs still wins once it covers at least 90% of that, and ties go to the newest algorithm. The Exists terms of the count are annotations rather than a condition inside the Count, so the query cache sees their tables and a newly stored vector refreshes the count. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01C7Xf6VPbwWtTumhjjF15g8 --- ami/main/models_future/embeddings.py | 69 ++++++++++++++++++++-------- ami/main/tests.py | 6 +-- ami/ml/test_feature_extraction.py | 39 ++++++++-------- 3 files changed, 74 insertions(+), 40 deletions(-) diff --git a/ami/main/models_future/embeddings.py b/ami/main/models_future/embeddings.py index 0901eb712..121e54dbb 100644 --- a/ami/main/models_future/embeddings.py +++ b/ami/main/models_future/embeddings.py @@ -15,7 +15,7 @@ from collections.abc import Iterable from typing import Any -from django.db.models import Count, F, IntegerField, QuerySet, Value +from django.db.models import Count, Exists, F, IntegerField, OuterRef, Q, QuerySet, Value logger = logging.getLogger(__name__) @@ -101,20 +101,53 @@ def algorithm_ids_with_vectors(**detection_lookups: Any) -> set[int]: return set(embedded.union(classified)) +# A configured extractor is preferred only once it covers nearly as many detections as the best +# one, so a feature-extraction run still in progress does not become the default for the session. +CONFIGURED_EXTRACTOR_MIN_COVERAGE = 0.9 + + +def detections_covered(algorithm_ids: Iterable[int], **detection_lookups: Any) -> dict[int, int]: + """How many detections matching the lookups have a vector from each algorithm, in one query. + + A detection counts once per algorithm whichever store (embedding or classification) holds its vector. + """ + from ami.main.models import Classification, Detection, DetectionEmbedding + + algorithm_ids = list(algorithm_ids) + if not algorithm_ids: + return {} + # The Exists terms are annotations, not a Q inside the Count, so cachalot sees their tables + # and a newly stored vector invalidates the cached count. + flags, counts = {}, {} + for algorithm_id in algorithm_ids: + embedded, classified = f"embedded_{algorithm_id}", f"classified_{algorithm_id}" + flags[embedded] = Exists( + DetectionEmbedding.objects.filter(detection_id=OuterRef("pk"), algorithm_id=algorithm_id) + ) + flags[classified] = Exists( + Classification.objects.filter( + detection_id=OuterRef("pk"), algorithm_id=algorithm_id, features_2048__isnull=False + ) + ) + counts[f"algorithm_{algorithm_id}"] = Count("pk", filter=Q(**{embedded: True}) | Q(**{classified: True})) + totals = Detection.objects.filter(**detection_lookups).order_by().annotate(**flags).aggregate(**counts) + return {algorithm_id: totals[f"algorithm_{algorithm_id}"] for algorithm_id in algorithm_ids} + + def default_feature_algorithm_id(project: Any, algorithm_ids: Iterable[int], **detection_lookups: Any) -> int | None: """The extractor to compare when the caller chose none, among ``algorithm_ids``. - One the project runs (a feature-extraction algorithm in a pipeline enabled for it) wins; - among several, or when there is none, the one whose vector was stored most recently for - the detections matching the lookups; failing that, the newest algorithm. At most 2 queries. + The one with vectors for the most detections matching the lookups, so the choice does not + depend on which job wrote last. An extractor the project runs (a feature-extraction algorithm + in a pipeline enabled for it) wins when it covers at least CONFIGURED_EXTRACTOR_MIN_COVERAGE + of that. Ties go to the newest algorithm. At most 2 queries. """ - from ami.main.models import DetectionEmbedding from ami.ml.models import Algorithm, Pipeline algorithm_ids = sorted(set(algorithm_ids)) if len(algorithm_ids) <= 1: return algorithm_ids[0] if algorithm_ids else None - configured = sorted( + configured = set( Algorithm.objects.filter( pk__in=algorithm_ids, task_type__in=Algorithm.feature_extraction_task_types, @@ -122,19 +155,19 @@ def default_feature_algorithm_id(project: Any, algorithm_ids: Iterable[int], **d ) .order_by() .values_list("pk", flat=True) - .distinct() - ) - if len(configured) == 1: - return configured[0] - choices = configured or algorithm_ids - lookups = {f"detection__{key}": value for key, value in detection_lookups.items()} - latest = ( - DetectionEmbedding.objects.filter(algorithm_id__in=choices, **lookups) - .order_by("-updated_at", "-pk") - .values_list("algorithm_id", flat=True) - .first() ) - return latest if latest is not None else max(choices) + coverage = detections_covered(algorithm_ids, **detection_lookups) + + def rank(algorithm_id: int) -> tuple[int, int]: + return coverage[algorithm_id], algorithm_id + + best = max(algorithm_ids, key=rank) + eligible = [ + algorithm_id + for algorithm_id in configured + if coverage[algorithm_id] >= CONFIGURED_EXTRACTOR_MIN_COVERAGE * coverage[best] + ] + return max(eligible, key=rank) if eligible else best def feature_extractors_with_vectors(project: Any, **detection_lookups: Any) -> list[dict[str, Any]]: diff --git a/ami/main/tests.py b/ami/main/tests.py index 9a416b0df..c005c7541 100644 --- a/ami/main/tests.py +++ b/ami/main/tests.py @@ -10602,15 +10602,15 @@ def test_a_detector_only_session_links_nothing(self): self.assertIsNotNone(row["cost"], "The geometry is still scored") def test_two_feature_extractors_are_never_compared_with_each_other(self): - """With embeddings from two extractors, tracking compares the default one only (here the - one stored most recently), so a box whose track frame has no vector from it does not link.""" + """With embeddings from two extractors, tracking compares the default one only (the one + covering the most boxes), so a box whose only vector is from the other does not link.""" other_extractor = Algorithm.objects.create(name="Other extractor", key="other-extractor") track = self._track(self.captures[1:3], vector=self.VECTOR) self._box(self.captures[3], self.NEAR_BOX, vector=self.VECTOR, algorithm=other_extractor) data = self.get_matches(track, self.captures[3].pk).data - self.assertEqual(data["feature_algorithm_id"], other_extractor.pk) + self.assertEqual(data["feature_algorithm_id"], self.extractor.pk) self.assertFalse(data["detections"][0]["would_link"]) def test_the_reference_is_the_nearest_track_frame_on_another_capture(self): diff --git a/ami/ml/test_feature_extraction.py b/ami/ml/test_feature_extraction.py index 907529878..124296ca6 100644 --- a/ami/ml/test_feature_extraction.py +++ b/ami/ml/test_feature_extraction.py @@ -249,48 +249,49 @@ def test_cosine_similarity_refuses_vectors_of_different_lengths(self): class TestDefaultFeatureExtractor(FeatureOnlyFixture, TestCase): - """With vectors from several extractors and none chosen, tracking compares the default one.""" + """With vectors from several extractors and none chosen, tracking compares the one covering most boxes.""" def setUp(self) -> None: self._set_up_project(images=2, boxes_per_image=2) self.event = Event.objects.get(pk=self.images[0].event_id) - self.older = Algorithm.objects.create(name="Older backbone", key="older-backbone", task_type="embedding") - detections = Detection.objects.valid() - self._embed(detections, self.older, length=2048) - self._embed(detections, self.extractor) # stored last + self.other = Algorithm.objects.create(name="Other backbone", key="other-backbone", task_type="embedding") + self.detections = list(Detection.objects.valid().filter(source_image__event=self.event).order_by("pk")) + self._embed(self.detections, self.other, length=2048) + self._embed(self.detections[:1], self.extractor) # stored last, on one box only - def test_the_most_recently_stored_extractor_is_the_default(self): + def test_the_extractor_covering_most_detections_wins_over_a_more_recent_one(self): algorithm, should_track, note = resolve_feature_algorithm( self.event, TrackingConfig(event_ids=[self.event.pk], require_features=True) ) - self.assertEqual((algorithm, should_track), (self.extractor, True)) + self.assertEqual((algorithm, should_track), (self.other, True)) self.assertIn("2 feature extractors", note) - def test_an_extractor_the_project_runs_wins_over_a_more_recent_one(self): + def test_an_extractor_the_project_runs_wins_only_once_it_covers_nearly_as_many(self): service = ProcessingService.objects.create(name="Service", endpoint_url=None) service.projects.add(self.project) - older_pipeline = Pipeline.objects.create(name="Older features", slug="older-features") - older_pipeline.algorithms.set([self.older]) - service.pipelines.add(older_pipeline) - ProjectPipelineConfig.objects.create(project=self.project, pipeline=older_pipeline, enabled=True) + service.pipelines.add(self.pipeline) + ProjectPipelineConfig.objects.create(project=self.project, pipeline=self.pipeline, enabled=True) + algorithm_ids = [self.other.pk, self.extractor.pk] - algorithm_ids = [self.older.pk, self.extractor.pk] - self.assertEqual( - default_feature_algorithm_id(self.project.pk, algorithm_ids, source_image__event=self.event), self.older.pk - ) + def default() -> int | None: + return default_feature_algorithm_id(self.project.pk, algorithm_ids, source_image__event=self.event) + + self.assertEqual(default(), self.other.pk) # 1 of 4 boxes: a run still in progress + self._embed(self.detections[1:], self.extractor) + self.assertEqual(default(), self.extractor.pk) def test_a_chosen_extractor_is_kept(self): config = TrackingConfig( - event_ids=[self.event.pk], require_features=True, feature_extraction_algorithm_id=self.older.pk + event_ids=[self.event.pk], require_features=True, feature_extraction_algorithm_id=self.extractor.pk ) - self.assertEqual(resolve_feature_algorithm(self.event, config)[0], self.older) + self.assertEqual(resolve_feature_algorithm(self.event, config)[0], self.extractor) def test_listing_a_sessions_extractors_takes_a_fixed_number_of_queries(self): with self.assertNumQueries(5): rows = feature_extractors_with_vectors(self.project.pk, source_image__event=self.event) self.assertEqual( [(row["algorithm"].pk, row["embeddings_count"], row["is_default"]) for row in rows], - [(self.older.pk, 4, False), (self.extractor.pk, 4, True)], # newest algorithm first + [(self.other.pk, 4, True), (self.extractor.pk, 1, False)], # newest algorithm first ) From fac78717bbdaa3933af1c6b088582076ad3fbae4 Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Mon, 28 Sep 2026 13:00:46 -0700 Subject: [PATCH 35/44] fix(ml): keep the Collect stage heartbeat when a feature-only pipeline filters images For a feature-only pipeline, filter_processed_images returned the result of filter_images_missing_features directly and dropped the job and total arguments, so a large capture set was filtered without the throttled collect progress saves that keep the stale-job reaper from revoking the job. The throttled save now lives in a small helper that both loops tick once per batch. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01C7Xf6VPbwWtTumhjjF15g8 --- ami/ml/models/pipeline.py | 60 +++++++++++++++++++++++---------------- 1 file changed, 35 insertions(+), 25 deletions(-) diff --git a/ami/ml/models/pipeline.py b/ami/ml/models/pipeline.py index 4bc70fff1..8fab94a7c 100644 --- a/ami/ml/models/pipeline.py +++ b/ami/ml/models/pipeline.py @@ -70,6 +70,33 @@ COLLECT_PROGRESS_MAX_FRACTION = 0.99 +class _CollectHeartbeat: + """Throttled ``collect`` progress saves, so the reaper sees a long Collect stage moving. + + Silent unless both ``job`` and a non-zero ``total`` are given. Capped at + COLLECT_PROGRESS_MAX_FRACTION so the caller's final SUCCESS flip owns the terminal value. + ``updated_at`` is saved explicitly: auto_now only fires for fields in update_fields, + and the reaper keys off it (see ``ami/jobs/tasks.py``). + """ + + def __init__(self, job: Job | None, total: int | None) -> None: + self.job, self.total = job, total + self.processed = 0 + self.last_save = time.monotonic() + + def tick(self, batch_size: int) -> None: + self.processed += batch_size + if self.job is None or not self.total: + return + now = time.monotonic() + if now - self.last_save >= COLLECT_PROGRESS_SAVE_INTERVAL_SECONDS: + self.job.progress.update_stage( + "collect", progress=min(self.processed / self.total, COLLECT_PROGRESS_MAX_FRACTION) + ) + self.job.save(update_fields=["progress", "updated_at"]) + self.last_save = now + + def filter_processed_images( images: typing.Iterable[SourceImage], pipeline: Pipeline, @@ -108,7 +135,9 @@ def filter_processed_images( pipeline_algorithm_ids = [a.id for a in pipeline_algorithms] if feature_extraction_only(pipeline_algorithms): - yield from filter_images_missing_features(images, pipeline_algorithm_ids, batch_size=batch_size) + yield from filter_images_missing_features( + images, pipeline_algorithm_ids, batch_size=batch_size, heartbeat=_CollectHeartbeat(job, total) + ) return detection_type_keys = set(Algorithm.detection_task_types) @@ -125,11 +154,7 @@ def filter_processed_images( return image_iter = iter(images) - # Track how many of the input images we've inspected so far so we can emit - # a fractional `collect` progress to the Job row. Only used when both - # `job` and `total` are passed by the caller; legacy callers stay silent. - processed_count = 0 - last_progress_save_monotonic = time.monotonic() + heartbeat = _CollectHeartbeat(job, total) while True: batch = list(itertools.islice(image_iter, batch_size)) if not batch: @@ -205,25 +230,7 @@ def filter_processed_images( f"Image {image} has existing detections classified by the pipeline: {pipeline}, skipping!" ) - # Throttled progress emit. Save only when both `job` and `total` are - # provided, the total is non-zero, and at least - # COLLECT_PROGRESS_SAVE_INTERVAL_SECONDS of wall time have passed since - # the last save. Capped at COLLECT_PROGRESS_MAX_FRACTION so the caller's - # final status=SUCCESS, progress=1 flip still owns the terminal value. - # - # `updated_at` is included in update_fields explicitly: Django only fires - # auto_now's pre_save hook for fields listed in update_fields, so without - # it the reaper's `Job.updated_at < cutoff` heuristic - # (`ami/jobs/tasks.py:929-944`) would not see this heartbeat and could - # still revoke the job mid-Collect. - processed_count += len(batch) - if job is not None and total: - now_monotonic = time.monotonic() - if now_monotonic - last_progress_save_monotonic >= COLLECT_PROGRESS_SAVE_INTERVAL_SECONDS: - fraction = min(processed_count / total, COLLECT_PROGRESS_MAX_FRACTION) - job.progress.update_stage("collect", progress=fraction) - job.save(update_fields=["progress", "updated_at"]) - last_progress_save_monotonic = now_monotonic + heartbeat.tick(len(batch)) def feature_extraction_only(algorithms: list[Algorithm]) -> bool: @@ -249,6 +256,7 @@ def filter_images_missing_features( images: typing.Iterable[SourceImage], algorithm_ids: list[int], batch_size: int = FILTER_PROCESSED_BATCH_SIZE, + heartbeat: _CollectHeartbeat | None = None, ) -> typing.Iterable[SourceImage]: """The images with at least one real detection that lacks a vector from the algorithms. @@ -268,6 +276,8 @@ def filter_images_missing_features( .distinct() ) yield from (image for image in batch if image.pk in needing) + if heartbeat is not None: + heartbeat.tick(len(batch)) def collect_detections_for_features( From 9831afec44e3265d8b2a1dfd7f476cc7c8e6b655 Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Mon, 28 Sep 2026 13:00:46 -0700 Subject: [PATCH 36/44] fix(ml): never queue a feature-only task without boxes to embed The image filter counted any real detection missing a vector, but the request builder skipped detections with no known detector, so an image could be queued on the async path with an empty detection list. A worker reads an empty list as no boxes given and runs its own detector, wasting GPU time, and the image is queued again on every run. Both now go through one definition of an embeddable detection, and the async path also drops any task still left with no boxes (for example when every image is reprocessed). Tests cover the per-task boxes on the async path, the skipped image and the collect heartbeat of a feature-only pipeline. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01C7Xf6VPbwWtTumhjjF15g8 --- ami/ml/models/pipeline.py | 17 +++++++-- ami/ml/orchestration/jobs.py | 17 +++++++-- ami/ml/test_feature_extraction.py | 63 +++++++++++++++++++++++++++++++ 3 files changed, 90 insertions(+), 7 deletions(-) diff --git a/ami/ml/models/pipeline.py b/ami/ml/models/pipeline.py index 8fab94a7c..db3636146 100644 --- a/ami/ml/models/pipeline.py +++ b/ami/ml/models/pipeline.py @@ -239,8 +239,17 @@ def feature_extraction_only(algorithms: list[Algorithm]) -> bool: return bool(algorithms) and all(algorithm.task_type in feature_types for algorithm in algorithms) +def embeddable_detections(detections: models.QuerySet) -> models.QuerySet: + """The detections a feature-only pipeline can be sent: real boxes whose detector is known. + + Choosing images and building the request both go through this, so an image is never + queued with no boxes to embed (a worker would then run its own detector on it). + """ + return detections.valid().filter(detection_algorithm__isnull=False) + + def detections_missing_features(detections: models.QuerySet, algorithm_ids: list[int]) -> models.QuerySet: - """Real detections without a stored vector from at least one of the algorithms. + """Embeddable detections without a stored vector from at least one of the algorithms. Only ``DetectionEmbedding`` rows count: they are what a feature-only pipeline writes. """ @@ -249,7 +258,7 @@ def detections_missing_features(detections: models.QuerySet, algorithm_ids: list missing |= ~models.Exists( DetectionEmbedding.objects.filter(detection_id=models.OuterRef("pk"), algorithm_id=algorithm_id) ) - return detections.valid().filter(missing) + return embeddable_detections(detections).filter(missing) def filter_images_missing_features( @@ -293,7 +302,9 @@ def collect_detections_for_features( request_by_image_id = {int(request.id): request for request in source_image_requests} detections = Detection.objects.filter(source_image_id__in=list(request_by_image_id)) detections = ( - detections.valid() if include_existing_vectors else detections_missing_features(detections, algorithm_ids) + embeddable_detections(detections) + if include_existing_vectors + else detections_missing_features(detections, algorithm_ids) ) detection_requests: list[DetectionRequest] = [] for detection in detections.select_related("detection_algorithm").order_by("source_image_id", "pk"): diff --git a/ami/ml/orchestration/jobs.py b/ami/ml/orchestration/jobs.py index 8a459959e..db833576c 100644 --- a/ami/ml/orchestration/jobs.py +++ b/ami/ml/orchestration/jobs.py @@ -81,8 +81,12 @@ async def cleanup(): return redis_success and nats_success -def _attach_detections_for_feature_pipeline(job: "Job", tasks: list[PipelineProcessingTask]) -> None: - """Give each task the existing boxes a feature-only pipeline should embed, in batches of images.""" +def _attach_detections_for_feature_pipeline(job: "Job", tasks: list[PipelineProcessingTask]) -> set[str]: + """Give each task the existing boxes a feature-only pipeline should embed, in batches of images. + + Returns the image ids left with no boxes. Those must not be queued: a worker given an + empty list runs its own detector, and boxes that match nothing are discarded. + """ from ami.ml.models.pipeline import FILTER_PROCESSED_BATCH_SIZE, collect_detections_for_features from ami.ml.schemas import SourceImageRequest @@ -90,7 +94,7 @@ def _attach_detections_for_feature_pipeline(job: "Job", tasks: list[PipelineProc [algorithm.pk for algorithm in job.pipeline.feature_extraction_algorithms()] if job.pipeline else [] ) if not algorithm_ids: - return + return set() include_existing = job.project.feature_flags.reprocess_all_images for start in range(0, len(tasks), FILTER_PROCESSED_BATCH_SIZE): batch = tasks[start : start + FILTER_PROCESSED_BATCH_SIZE] # noqa: E203 @@ -100,6 +104,7 @@ def _attach_detections_for_feature_pipeline(job: "Job", tasks: list[PipelineProc by_image[detection_request.source_image.id].append(detection_request) for task in batch: task.detections = by_image[task.image_id] + return {task.image_id for task in tasks if not task.detections} def queue_images_to_nats(job: "Job", images: list[SourceImage]): @@ -138,7 +143,11 @@ def queue_images_to_nats(job: "Job", images: list[SourceImage]): ) tasks.append((image.pk, task)) - _attach_detections_for_feature_pipeline(job, [task for _, task in tasks]) + nothing_to_embed = _attach_detections_for_feature_pipeline(job, [task for _, task in tasks]) + if nothing_to_embed: + job.logger.info(f"Not queuing {len(nothing_to_embed)} images that have no detections to embed") + tasks = [(pk, task) for pk, task in tasks if task.image_id not in nothing_to_embed] + image_ids = [image_id for image_id in image_ids if image_id not in nothing_to_embed] # Store all image IDs in Redis for progress tracking state_manager = AsyncJobStateManager(job.pk) diff --git a/ami/ml/test_feature_extraction.py b/ami/ml/test_feature_extraction.py index 124296ca6..54c750aac 100644 --- a/ami/ml/test_feature_extraction.py +++ b/ami/ml/test_feature_extraction.py @@ -13,6 +13,7 @@ from django.test import TestCase from rest_framework.test import APITestCase +from ami.jobs.models import Job, JobDispatchMode, MLJob from ami.main.models import ( Classification, Deployment, @@ -32,13 +33,17 @@ ) from ami.ml.models import Algorithm, Pipeline, ProcessingService from ami.ml.models.pipeline import ( + COLLECT_PROGRESS_MAX_FRACTION, + COLLECT_PROGRESS_SAVE_INTERVAL_SECONDS, EmbeddingDimensionMismatch, collect_detections_for_features, collect_images, + filter_processed_images, process_images, save_results, ) from ami.ml.models.project_pipeline_config import ProjectPipelineConfig +from ami.ml.orchestration.jobs import queue_images_to_nats from ami.ml.post_processing.tracking_task import TrackingConfig, cosine_similarity, resolve_feature_algorithm from ami.ml.schemas import EmbeddingResponse, PipelineResultsResponse, SourceImageRequest from ami.users.models import User @@ -229,6 +234,64 @@ def post(url, json): self.assertEqual(len(sent["detections"]), 3) +class TestExtractFeaturesQueue(FeatureOnlyFixture, TestCase): + """The async path: each queued task carries its missing boxes, and no task is queued empty.""" + + def setUp(self) -> None: + self._set_up_project() + self._embed(self.images[0].detections.valid(), self.extractor) + self._embed(self.images[1].detections.valid().filter(bbox=_box(0.0)), self.extractor) + # A box whose detector is unknown cannot be sent, so it must not make its image count as pending. + Detection.objects.create(source_image=self.images[0], bbox=_box(40.0), detection_algorithm=None) + self.job = Job.objects.create( + name="Extract features", + job_type_key=MLJob.key, + project=self.project, + pipeline=self.pipeline, + source_image_collection=self.collection, + dispatch_mode=JobDispatchMode.ASYNC_API, + ) + + @mock.patch("ami.ml.orchestration.jobs.AsyncJobStateManager") + @mock.patch("ami.ml.orchestration.jobs.TaskQueueManager") + def test_each_task_carries_its_missing_boxes_and_images_with_none_are_not_queued(self, manager_cls, state_cls): + manager = manager_cls.return_value + manager.__aenter__ = mock.AsyncMock(return_value=manager) + manager.__aexit__ = mock.AsyncMock(return_value=False) + manager.ensure_job_resources = mock.AsyncMock() + manager.publish_task = mock.AsyncMock(return_value=True) + + with mock.patch.object(SourceImage, "url", return_value="http://example.org/i.jpg"): + self.assertTrue(queue_images_to_nats(self.job, self.images)) + + published = { + int(call.kwargs["data"].image_id): sorted(d.bbox.x1 for d in call.kwargs["data"].detections) + for call in manager.publish_task.await_args_list + } + self.assertEqual(published, {self.images[1].pk: [20.0], self.images[2].pk: [0.0, 20.0]}) + state_cls.return_value.initialize_job.assert_called_once_with([str(self.images[1].pk), str(self.images[2].pk)]) + + def test_images_whose_only_pending_box_has_no_detector_are_skipped(self): + collected = collect_images(collection=self.collection, pipeline=self.pipeline) + self.assertEqual([image.pk for image in collected], [self.images[1].pk, self.images[2].pk]) + + def test_the_collect_stage_reports_progress_while_filtering(self): + clock = {"t": 0.0} + + def fake_monotonic() -> float: + clock["t"] += COLLECT_PROGRESS_SAVE_INTERVAL_SECONDS + return clock["t"] + + with mock.patch("ami.ml.models.pipeline.time.monotonic", side_effect=fake_monotonic), mock.patch.object( + Job, "save", autospec=True + ) as save: + list(filter_processed_images(self.images, self.pipeline, batch_size=1, job=self.job, total=3)) + + self.assertEqual(save.call_count, 3) + self.assertEqual(save.call_args.kwargs["update_fields"], ["progress", "updated_at"]) + self.assertEqual(self.job.progress.get_stage("collect").progress, COLLECT_PROGRESS_MAX_FRACTION) + + class TestReadersNeverMixLengths(FeatureOnlyFixture, TestCase): def setUp(self) -> None: self._set_up_project(images=2, boxes_per_image=2) From dee996e939c4efb7616ad88c072d58ef4e3157fb Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Mon, 28 Sep 2026 13:00:46 -0700 Subject: [PATCH 37/44] docs: describe the coverage-based default extractor, the vector-length repair migration and the feature-only queue rules [skip ci] Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01C7Xf6VPbwWtTumhjjF15g8 --- docs/claude/reference/detection-embeddings.md | 25 +++++++++++++------ 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/docs/claude/reference/detection-embeddings.md b/docs/claude/reference/detection-embeddings.md index 3c3aa4c70..08df70048 100644 --- a/docs/claude/reference/detection-embeddings.md +++ b/docs/claude/reference/detection-embeddings.md @@ -6,7 +6,9 @@ extractor's vectors to compare. See #1417 for the original design. ## Storage -- `DetectionEmbedding` (`ami/main/models.py`, migration `main/0102_detection_embedding.py`): +- `DetectionEmbedding` (`ami/main/models.py`, migration `main/0102_detection_embedding.py`; + `main/0104` re-runs `ALTER COLUMN vector TYPE vector` for databases that applied an earlier + 0102 draft with `vector(2048)`, which Django state cannot see): one row per (detection, algorithm), unique constraint `unique_detection_embedding_per_algorithm` (its index leads with `detection_id` and serves reads). `vector` is a pgvector column with **no fixed dimension**: extractors differ (2048 for the moth classifier backbones, 1024 for BioCLIP). @@ -56,8 +58,13 @@ No new job type: an ordinary ML job (`MLJob`, key `ml`) whose pipeline is featur permission as any ML job. Scope is the job's capture set, deployment or single capture. `filter_processed_images` delegates to `filter_images_missing_features`: an image is sent only when a real detection on it lacks a `DetectionEmbedding` from one of the pipeline's algorithms -(`detections_missing_features`; classification vectors do not count). The project flag -`reprocess_all_images` sends every real detection again. +(`detections_missing_features`; classification vectors do not count). Both the image filter +and the request go through `embeddable_detections` (real box and known detector), so no image +is chosen without boxes to send. The async path also drops any task left with no boxes +(`_attach_detections_for_feature_pipeline` returns them), because a worker given an empty list +runs its own detector. The filter emits the same throttled `collect` heartbeat as regular +pipelines (`_CollectHeartbeat`). The project flag `reprocess_all_images` sends every real +detection again. ## Read path @@ -78,10 +85,14 @@ when a real detection on it lacks a `DetectionEmbedding` from one of the pipelin `resolve_feature_algorithm(event, config)`: 1. `config.feature_extraction_algorithm_id` if given. 2. The only extractor with vectors in the session. -3. Several: `default_feature_algorithm_id(project, ids, source_image__event=event)`: a - feature-extraction algorithm in a pipeline enabled for the project, else the extractor whose - `DetectionEmbedding` was stored most recently in the session, else the newest algorithm id. - The run logs which one it compares. +3. Several: `default_feature_algorithm_id(project, ids, source_image__event=event)`: the + extractor with vectors for the most detections in the session (`detections_covered`, one + query, either store counts once); a configured one (feature-extraction algorithm in a + pipeline enabled for the project) wins if it covers at least + `CONFIGURED_EXTRACTOR_MIN_COVERAGE` (0.9) of the best; ties go to the newest algorithm id. + A half-finished backfill therefore never becomes the default. The run logs which one it compares. + Gotcha: the Exists terms in `detections_covered` must be annotations, not a `Q` inside + `Count(filter=...)`, or cachalot misses their tables and serves a stale count. 4. None: `require_features=True` skips the session; otherwise geometry-only matching. Merge candidates use the same resolution over the two captures being compared. From 91b9f62e66e32a49ecf1977f49c714c74aeb840b Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Mon, 28 Sep 2026 14:38:59 -0700 Subject: [PATCH 38/44] fix(ml): treat a pipeline as feature-only when it lists a detector beside its feature extractor A processing service that embeds existing boxes lists the detector whose boxes it echoes back alongside the feature extractor, so requiring every algorithm to be a feature extractor meant such a pipeline was run as a full detection pipeline. A pipeline is now feature-only when it has at least one feature extractor and no classifier, and only the extractors decide which detections still need a vector. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01C7Xf6VPbwWtTumhjjF15g8 --- ami/ml/models/algorithm.py | 2 +- ami/ml/models/pipeline.py | 29 ++++++++++++++----- ami/ml/test_feature_extraction.py | 12 +++++++- docs/claude/reference/detection-embeddings.md | 8 +++-- 4 files changed, 39 insertions(+), 12 deletions(-) diff --git a/ami/ml/models/algorithm.py b/ami/ml/models/algorithm.py index 12297a315..ab96eb8f7 100644 --- a/ami/ml/models/algorithm.py +++ b/ami/ml/models/algorithm.py @@ -285,7 +285,7 @@ class Algorithm(BaseModel): AlgorithmTaskType.CLASSIFICATION, AlgorithmTaskType.TAGGING, ] - # A pipeline made only of these returns vectors for existing detections and nothing else. + # A pipeline with one of these and no classifier returns vectors for existing detections only. feature_extraction_task_types = [ AlgorithmTaskType.EMBEDDING, AlgorithmTaskType.FEATURE_EXTRACTION, diff --git a/ami/ml/models/pipeline.py b/ami/ml/models/pipeline.py index db3636146..599a15db6 100644 --- a/ami/ml/models/pipeline.py +++ b/ami/ml/models/pipeline.py @@ -134,9 +134,13 @@ def filter_processed_images( pipeline_algorithms = list(pipeline.algorithms.all()) pipeline_algorithm_ids = [a.id for a in pipeline_algorithms] - if feature_extraction_only(pipeline_algorithms): + feature_extractors = feature_extractors_if_feature_only(pipeline_algorithms) + if feature_extractors: yield from filter_images_missing_features( - images, pipeline_algorithm_ids, batch_size=batch_size, heartbeat=_CollectHeartbeat(job, total) + images, + [algorithm.pk for algorithm in feature_extractors], + batch_size=batch_size, + heartbeat=_CollectHeartbeat(job, total), ) return @@ -233,10 +237,22 @@ def filter_processed_images( heartbeat.tick(len(batch)) +def feature_extractors_if_feature_only(algorithms: list[Algorithm]) -> list[Algorithm]: + """The feature extractors of a pipeline that only extracts features, otherwise none. + + Such a pipeline has a feature extractor and no classifier. A detector may be listed too: + the service names the detector of the boxes it echoes back, but detects nothing new. + """ + feature_types = set(Algorithm.feature_extraction_task_types) + classification_types = set(Algorithm.classification_task_types) + if any(algorithm.task_type in classification_types for algorithm in algorithms): + return [] + return [algorithm for algorithm in algorithms if algorithm.task_type in feature_types] + + def feature_extraction_only(algorithms: list[Algorithm]) -> bool: """Whether a pipeline made of these algorithms only extracts features (see ``Pipeline.is_feature_only``).""" - feature_types = set(Algorithm.feature_extraction_task_types) - return bool(algorithms) and all(algorithm.task_type in feature_types for algorithm in algorithms) + return bool(feature_extractors_if_feature_only(algorithms)) def embeddable_detections(detections: models.QuerySet) -> models.QuerySet: @@ -1528,13 +1544,12 @@ def __str__(self): return f'#{self.pk} "{self.name}" ({self.slug}) v{self.version}' def feature_extraction_algorithms(self) -> list[Algorithm]: - """The pipeline's algorithms when every one of them extracts features, otherwise none. + """The pipeline's feature extractors when it has no classifier, otherwise none. Such a pipeline is run on existing detections and only stores their vectors: it creates no detection, classification or occurrence. """ - algorithms = list(self.algorithms.all()) - return algorithms if feature_extraction_only(algorithms) else [] + return feature_extractors_if_feature_only(list(self.algorithms.all())) def is_feature_only(self) -> bool: return bool(self.feature_extraction_algorithms()) diff --git a/ami/ml/test_feature_extraction.py b/ami/ml/test_feature_extraction.py index 54c750aac..a8950d44d 100644 --- a/ami/ml/test_feature_extraction.py +++ b/ami/ml/test_feature_extraction.py @@ -67,7 +67,8 @@ def _set_up_project(self, images: int = 3, boxes_per_image: int = 2) -> None: ) self.extractor = Algorithm.objects.create(name="Backbone", key="test-backbone", task_type="embedding") self.pipeline = Pipeline.objects.create(name="Features only", slug="features-only") - self.pipeline.algorithms.set([self.extractor]) + # A feature-only service also lists the detector whose boxes it echoes back. + self.pipeline.algorithms.set([self.detector, self.extractor]) start = datetime.datetime(2024, 6, 1, 22, 0) self.images = [ @@ -153,6 +154,15 @@ class TestFeatureOnlySave(FeatureOnlyFixture, TestCase): def setUp(self) -> None: self._set_up_project() + def test_a_pipeline_is_feature_only_until_it_lists_a_classifier(self): + """The detector does not count against it; a classifier does, since its pipeline + detects and classifies anew and may embed as well.""" + self.assertEqual(self.pipeline.feature_extraction_algorithms(), [self.extractor]) + self.pipeline.algorithms.add(self.classifier) + self.assertFalse(self.pipeline.is_feature_only()) + self.pipeline.algorithms.set([self.detector]) + self.assertFalse(self.pipeline.is_feature_only()) + def test_an_embeddings_only_response_writes_only_vectors(self): """No detection, classification or occurrence is created and no determination moves, even for a box Antenna does not have or classifications the service sent anyway.""" diff --git a/docs/claude/reference/detection-embeddings.md b/docs/claude/reference/detection-embeddings.md index 08df70048..a34f36d5a 100644 --- a/docs/claude/reference/detection-embeddings.md +++ b/docs/claude/reference/detection-embeddings.md @@ -29,9 +29,11 @@ extractor's vectors to compare. See #1417 for the original design. `{"algorithm": {"name", "key"}, "features": [float, ...]}`. The key `vector` is accepted as an alias (root validator). Any non-empty length; the per-algorithm length is enforced at save time. - The algorithm key must be declared in the pipeline's `/info`, else `PipelineNotConfigured`. -- **Feature-only pipeline**: every algorithm in the pipeline has `task_type` in - `Algorithm.feature_extraction_task_types` = `embedding` or `feature_extraction` - (`feature_extraction_only()` / `Pipeline.is_feature_only()`). +- **Feature-only pipeline**: at least one algorithm has `task_type` in + `Algorithm.feature_extraction_task_types` (`embedding` or `feature_extraction`) and none is a + classifier (`classification` / `tagging`). Detector algorithms are allowed because the service + lists the detector whose boxes it echoes back (`feature_extractors_if_feature_only()` in + `ami/ml/models/pipeline.py`, `Pipeline.feature_extraction_algorithms()` returns only the extractors). - Request for a feature-only run (sync, `process_images`): `PipelineRequest` with `source_images` = only the images that still have a detection to embed, and `detections` = those detections as `DetectionRequest{source_image, bbox, crop_image_url, algorithm= Date: Mon, 28 Sep 2026 14:41:32 -0700 Subject: [PATCH 39/44] fix(ml): stop reprocessing classified captures when a classifier pipeline also embeds When a pipeline lists a feature extractor beside its detector and classifiers, the check for already processed captures counted the extractor as a classifier. The extractor never writes a classification, so every capture looked unprocessed and was sent again, adding a second set of classifications. Feature extractors are now left out of that check. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01C7Xf6VPbwWtTumhjjF15g8 --- ami/ml/models/pipeline.py | 4 +++- ami/ml/test_feature_extraction.py | 7 +++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/ami/ml/models/pipeline.py b/ami/ml/models/pipeline.py index 599a15db6..0c9e06381 100644 --- a/ami/ml/models/pipeline.py +++ b/ami/ml/models/pipeline.py @@ -148,7 +148,9 @@ def filter_processed_images( has_detection_algorithm = any(a.task_type in detection_type_keys for a in pipeline_algorithms) if not has_detection_algorithm: task_logger.warning(f"Pipeline {pipeline} has no detection algorithms saved. Will reprocess all images.") - pipeline_classifier_ids = {a.id for a in pipeline_algorithms if a.task_type not in detection_type_keys} + # A feature extractor never classifies, so it cannot mark an image as processed. + not_classifier_keys = detection_type_keys | set(Algorithm.feature_extraction_task_types) + pipeline_classifier_ids = {a.id for a in pipeline_algorithms if a.task_type not in not_classifier_keys} if not pipeline_classifier_ids: task_logger.warning(f"Pipeline {pipeline} has no classification algorithms saved. Will reprocess all images.") # set().issubset(anything) is vacuously True, so without this short-circuit diff --git a/ami/ml/test_feature_extraction.py b/ami/ml/test_feature_extraction.py index a8950d44d..3b8d91ef8 100644 --- a/ami/ml/test_feature_extraction.py +++ b/ami/ml/test_feature_extraction.py @@ -205,6 +205,13 @@ def setUp(self) -> None: self._embed(self.images[0].detections.valid(), self.extractor) self._embed(self.images[1].detections.valid().filter(bbox=_box(0.0)), self.extractor) + def test_a_classifier_pipeline_that_also_embeds_skips_classified_images(self): + """Its extractor writes no classification, so counting it as a classifier would + reprocess every image and add a second set of classifications.""" + pipeline = Pipeline.objects.create(name="Classify and embed", slug="classify-and-embed") + pipeline.algorithms.set([self.detector, self.classifier, self.extractor]) + self.assertEqual(list(collect_images(collection=self.collection, pipeline=pipeline)), []) + def test_images_whose_detections_all_have_vectors_are_skipped(self): collected = collect_images(collection=self.collection, pipeline=self.pipeline) self.assertEqual([image.pk for image in collected], [self.images[1].pk, self.images[2].pk]) From 9b2ae7ce88737bdbe66aec1263202e3d6013982a Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Mon, 28 Sep 2026 14:59:37 -0700 Subject: [PATCH 40/44] test: accept embedding vectors of any length in the results schema The results schema no longer fixes the vector length, since extractors differ and each algorithm's length is checked when vectors are saved. The schema test now pins that any non-empty vector parses and an empty one is refused. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01C7Xf6VPbwWtTumhjjF15g8 --- ami/ml/tests.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/ami/ml/tests.py b/ami/ml/tests.py index 4628cd97a..6c8dfd4c9 100644 --- a/ami/ml/tests.py +++ b/ami/ml/tests.py @@ -2379,10 +2379,14 @@ def test_a_detection_carries_each_vector_with_its_algorithm(self): ) self.assertIsNone(DetectionResponse.parse_obj(self._detection()).embeddings) - def test_a_vector_of_another_length_is_refused(self): - """The column holds 2048 floats, so a shorter vector must fail validation rather than the insert.""" + def test_an_empty_vector_is_refused(self): + """Any length parses (extractors differ; each algorithm's length is checked on save), but not none.""" + self.assertEqual( + len(DetectionResponse.parse_obj(self._detection(embeddings=_embedding_payload([0.5] * 512))).embeddings), + 1, + ) with self.assertRaises(pydantic.ValidationError): - DetectionResponse.parse_obj(self._detection(embeddings=_embedding_payload([0.5] * 512))) + DetectionResponse.parse_obj(self._detection(embeddings=_embedding_payload([]))) class TestDetectionEmbeddings(TestCase): From 23b44ca8b9dd089d38e94a118006eb6abd1eaa4d Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Mon, 28 Sep 2026 19:43:36 -0700 Subject: [PATCH 41/44] fix(ml): only treat a pipeline as feature-only when every other algorithm is a detector A pipeline counted as feature-only unless one of its algorithms was typed as a classifier. A classifier registered with a blank or unknown task type therefore made its pipeline feature-only whenever it also listed a feature extractor, and saving its results would have kept only the vectors and dropped every new detection and classification. The rule now fails closed: besides its feature extractors, a feature-only pipeline may list detectors and nothing else. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01C7Xf6VPbwWtTumhjjF15g8 --- ami/ml/models/pipeline.py | 9 +++++---- ami/ml/test_feature_extraction.py | 10 ++++++++++ 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/ami/ml/models/pipeline.py b/ami/ml/models/pipeline.py index 0c9e06381..5bb42ed0f 100644 --- a/ami/ml/models/pipeline.py +++ b/ami/ml/models/pipeline.py @@ -242,12 +242,13 @@ def filter_processed_images( def feature_extractors_if_feature_only(algorithms: list[Algorithm]) -> list[Algorithm]: """The feature extractors of a pipeline that only extracts features, otherwise none. - Such a pipeline has a feature extractor and no classifier. A detector may be listed too: - the service names the detector of the boxes it echoes back, but detects nothing new. + Such a pipeline has a feature extractor and otherwise only detectors: the service names the + detector of the boxes it echoes back, but detects nothing new. Any other task type, including + a blank or unknown one, rules it out, because a feature-only save drops new detections. """ feature_types = set(Algorithm.feature_extraction_task_types) - classification_types = set(Algorithm.classification_task_types) - if any(algorithm.task_type in classification_types for algorithm in algorithms): + allowed_types = feature_types | set(Algorithm.detection_task_types) + if any(algorithm.task_type not in allowed_types for algorithm in algorithms): return [] return [algorithm for algorithm in algorithms if algorithm.task_type in feature_types] diff --git a/ami/ml/test_feature_extraction.py b/ami/ml/test_feature_extraction.py index 3b8d91ef8..54171b358 100644 --- a/ami/ml/test_feature_extraction.py +++ b/ami/ml/test_feature_extraction.py @@ -163,6 +163,16 @@ def test_a_pipeline_is_feature_only_until_it_lists_a_classifier(self): self.pipeline.algorithms.set([self.detector]) self.assertFalse(self.pipeline.is_feature_only()) + def test_an_algorithm_of_unknown_task_type_rules_out_feature_only(self): + """A classifier registered without a task type must not turn its pipeline feature-only, + or saving its results would drop every new detection and classification.""" + for task_type in ("", "unknown"): + untyped = Algorithm.objects.create( + name=f"Untyped {task_type!r}", key=f"untyped-{task_type}", task_type=task_type + ) + self.pipeline.algorithms.set([self.detector, self.extractor, untyped]) + self.assertFalse(self.pipeline.is_feature_only(), task_type) + def test_an_embeddings_only_response_writes_only_vectors(self): """No detection, classification or occurrence is created and no determination moves, even for a box Antenna does not have or classifications the service sent anyway.""" From f5aae46df6b75b4fb8d4ebb1e0dd08ff0cb9abdd Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Mon, 28 Sep 2026 23:08:09 -0700 Subject: [PATCH 42/44] fix(ml): read embedding vectors only from the "features" key The processing service sends each detection's vector under "features" (ami-data-companion#175), so the fallback that also accepted "vector" is removed. One key keeps the contract with the service unambiguous. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01C7Xf6VPbwWtTumhjjF15g8 --- ami/ml/schemas.py | 10 +--------- ami/ml/test_feature_extraction.py | 15 ++++++++------- 2 files changed, 9 insertions(+), 16 deletions(-) diff --git a/ami/ml/schemas.py b/ami/ml/schemas.py index a7097367d..fa3a9d36c 100644 --- a/ami/ml/schemas.py +++ b/ami/ml/schemas.py @@ -195,17 +195,9 @@ class EmbeddingResponse(pydantic.BaseModel): algorithm's own (extractors differ), and only vectors from one algorithm are comparable. """ - features: list[float] = pydantic.Field( - description="The feature vector. Also accepted under the key 'vector'.", - ) + features: list[float] = pydantic.Field(description="The feature vector.") algorithm: AlgorithmReference - @pydantic.root_validator(pre=True) - def _accept_vector_key(cls, values): - if isinstance(values, dict) and "features" not in values and "vector" in values: - values = {**values, "features": values["vector"]} - return values - @pydantic.validator("features") def _features_not_empty(cls, v): if not v: diff --git a/ami/ml/test_feature_extraction.py b/ami/ml/test_feature_extraction.py index 54171b358..835660638 100644 --- a/ami/ml/test_feature_extraction.py +++ b/ami/ml/test_feature_extraction.py @@ -122,7 +122,7 @@ def _response(self, boxes: list[tuple[SourceImage, list[float]]], length: int = "embeddings": [ { "algorithm": {"name": self.extractor.name, "key": self.extractor.key}, - "vector": [0.1] * length, + "features": [0.1] * length, } ], **extra, @@ -141,13 +141,14 @@ def _counts(self) -> tuple: class TestEmbeddingSchema(TestCase): - def test_a_vector_of_any_length_is_accepted_under_either_key(self): + def test_a_vector_is_read_from_the_features_key_only(self): + """The processing service sends ``features``; an empty vector or any other key is refused.""" algorithm = {"name": "Backbone", "key": "test-backbone"} - for key in ("features", "vector"): - parsed = EmbeddingResponse.parse_obj({"algorithm": algorithm, key: [0.1] * BIOCLIP_DIMENSIONS}) - self.assertEqual(len(parsed.features), BIOCLIP_DIMENSIONS) - with self.assertRaises(pydantic.ValidationError): - EmbeddingResponse.parse_obj({"algorithm": algorithm, "vector": []}) + parsed = EmbeddingResponse.parse_obj({"algorithm": algorithm, "features": [0.1] * BIOCLIP_DIMENSIONS}) + self.assertEqual(len(parsed.features), BIOCLIP_DIMENSIONS) + for payload in ({"features": []}, {"vector": [0.1] * BIOCLIP_DIMENSIONS}): + with self.assertRaises(pydantic.ValidationError): + EmbeddingResponse.parse_obj({"algorithm": algorithm, **payload}) class TestFeatureOnlySave(FeatureOnlyFixture, TestCase): From 5b95f02bfc3d9b1c7493a053fa14a3a6c0be6fc3 Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Mon, 28 Sep 2026 23:15:21 -0700 Subject: [PATCH 43/44] fix(ml): fail a feature-only job whose returned boxes match no detection A feature-only pipeline returns the boxes Antenna sent it, each with a vector. A returned box that matches no stored detection was skipped with a warning, so if a whole batch matched nothing the job still succeeded, no vector was stored, and the next run sent the same detections again. Now the job progress counts the returned boxes that matched no detection ("unmatched" on the results stage), the log names the first few, and a batch in which no box matches raises FeatureResultsMatchNoDetections. In the async path that exception acks the message and fails the job instead of leaving it to be redelivered. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01C7Xf6VPbwWtTumhjjF15g8 --- ami/jobs/tasks.py | 30 +++++++++++- ami/jobs/tests/test_tasks.py | 64 ++++++++++++++++++++++++++ ami/ml/exceptions.py | 12 +++++ ami/ml/models/pipeline.py | 76 +++++++++++++++++++++++-------- ami/ml/test_feature_extraction.py | 15 ++++++ ami/ml/tests.py | 2 +- 6 files changed, 177 insertions(+), 22 deletions(-) diff --git a/ami/jobs/tasks.py b/ami/jobs/tasks.py index b00213ebb..ff03d10f3 100644 --- a/ami/jobs/tasks.py +++ b/ami/jobs/tasks.py @@ -13,6 +13,7 @@ from redis.exceptions import RedisError from ami.main.checks.schemas import IntegrityCheckResult +from ami.ml.exceptions import FeatureResultsMatchNoDetections from ami.ml.orchestration.async_job_state import AsyncJobStateManager from ami.ml.orchestration.nats_queue import ConsumerState, TaskQueueManager from ami.ml.schemas import PipelineResultsError, PipelineResultsResponse @@ -323,10 +324,16 @@ def process_nats_pipeline_result(self, job_id: int, result_data: dict, reply_sub try: # Save to database (this is the slow operation) detections_count, classifications_count, captures_count = 0, 0, 0 + unmatched_count: int | None = None if pipeline_result: # should never happen since otherwise we could not be processing results here assert job.pipeline is not None, "Job pipeline is None" - job.pipeline.save_results(results=pipeline_result, job_id=job.pk) + # Only a feature-only save reports boxes that matched no detection; asking the + # full save for its created rows would load every detection's algorithm. + feature_only = job.pipeline.is_feature_only() + saved = job.pipeline.save_results(results=pipeline_result, job_id=job.pk, return_created=feature_only) + if feature_only and saved: + unmatched_count = saved.unmatched_detections job.logger.info(f"Successfully saved results for job {job_id}") _, t = t( @@ -385,6 +392,9 @@ def process_nats_pipeline_result(self, job_id: int, result_data: dict, reply_sub counts_to_apply = ( (detections_count, classifications_count, captures_count) if is_first_processing else (0, 0, 0) ) + extra_counts = {} + if unmatched_count is not None: + extra_counts["unmatched"] = unmatched_count if is_first_processing else 0 _update_job_progress( job_id, "results", @@ -393,6 +403,7 @@ def process_nats_pipeline_result(self, job_id: int, result_data: dict, reply_sub detections=counts_to_apply[0], classifications=counts_to_apply[1], captures=counts_to_apply[2], + **extra_counts, ) # Ack LAST — only after the results-stage SREM and progress commit are @@ -408,6 +419,12 @@ def process_nats_pipeline_result(self, job_id: int, result_data: dict, reply_sub # Celery's autoretry_for handles the transient rather than this broad # except swallowing it. raise + except FeatureResultsMatchNoDetections as e: + # Redelivering would return the same boxes, and a later job would send the same + # detections again, so record the count and fail the job instead of retrying. + _update_job_progress(job_id, "results", 0, complete_state=JobState.FAILURE, unmatched=e.unmatched) + _ack_task_via_nats(reply_subject, job.logger) + _fail_job(job_id, str(e)) except Exception as e: error = f"Error processing pipeline result for job {job_id}: {e}" if not acked: @@ -502,6 +519,15 @@ async def ack_task(): return False +def _get_stage_param(job, stage: str, key: str) -> int: + """The integer value of one stage parameter, or 0 when the stage or parameter is absent.""" + try: + stage_obj = job.progress.get_stage(stage) + except ValueError: + return 0 + return next((param.value or 0 for param in stage_obj.params if param.key == key), 0) + + def _get_current_counts_from_job_progress(job, stage: str) -> tuple[int, int, int]: """ Get current detections, classifications, and captures counts from job progress. @@ -628,6 +654,8 @@ def _update_job_progress( state_params["detections"] = current_detections + new_detections state_params["classifications"] = current_classifications + new_classifications state_params["captures"] = current_captures + new_captures + if "unmatched" in state_params: + state_params["unmatched"] = _get_stage_param(job, stage, "unmatched") + state_params["unmatched"] # Don't overwrite a stage with a stale progress value. # This guards against the race where a slower worker calls _update_job_progress diff --git a/ami/jobs/tests/test_tasks.py b/ami/jobs/tests/test_tasks.py index c2a6829a3..c9b4c3277 100644 --- a/ami/jobs/tests/test_tasks.py +++ b/ami/jobs/tests/test_tasks.py @@ -421,6 +421,70 @@ def fail_on_results_stage(self, processed_image_ids, stage, failed_image_ids=Non self.assertEqual(process_progress.processed, 1) self.assertEqual(results_progress.processed, 0) + def _feature_only_result(self, image: SourceImage, boxes: list[list[float]]) -> dict: + """A feature-only result: the given boxes echoed back on one image, each with a vector.""" + return PipelineResultsResponse( + pipeline="test-pipeline", + total_time=1.0, + source_images=[SourceImageResponse(id=str(image.pk), url="http://example.com/x.jpg")], + detections=[ + { + "source_image_id": str(image.pk), + "bbox": dict(zip(["x1", "y1", "x2", "y2"], box)), + "algorithm": {"name": self.detector.name, "key": self.detector.key}, + "timestamp": datetime.datetime.now(), + "embeddings": [ + {"algorithm": {"name": self.extractor.name, "key": self.extractor.key}, "features": [0.1] * 8} + ], + } + for box in boxes + ], + ).dict() + + def _make_feature_only(self) -> None: + self.detector = Algorithm.objects.create( + name="feature-detector", key="feature-detector", task_type=AlgorithmTaskType.LOCALIZATION + ) + self.extractor = Algorithm.objects.create( + name="feature-backbone", key="feature-backbone", task_type=AlgorithmTaskType.EMBEDDING + ) + self.pipeline.algorithms.set([self.detector, self.extractor]) + for image in self.images: + Detection.objects.create(source_image=image, bbox=[0, 0, 10, 10], detection_algorithm=self.detector) + + def _results_param(self, key: str): + self.job.refresh_from_db() + stage = self.job.progress.get_stage("results") + return next((param.value for param in stage.params if param.key == key), None) + + @patch("ami.jobs.tasks.TaskQueueManager") + def test_feature_only_results_count_boxes_that_match_no_detection(self, mock_manager_class): + self._setup_mock_nats(mock_manager_class) + self._make_feature_only() + for image in self.images[:2]: + process_nats_pipeline_result( + job_id=self.job.pk, + result_data=self._feature_only_result(image, [[0, 0, 10, 10], [50, 50, 60, 60]]), + reply_subject=f"reply.features.{image.pk}", + ) + self.assertEqual(self._results_param("unmatched"), 2) + self.assertNotEqual(self.job.status, JobState.FAILURE.value) + + @patch("ami.jobs.tasks._ack_task_via_nats") + @patch("ami.jobs.tasks.TaskQueueManager") + def test_feature_only_batch_matching_no_detection_fails_the_job_and_acks(self, mock_manager_class, mock_ack): + """Redelivering it would return the same boxes, so the message is acked and the job fails.""" + self._setup_mock_nats(mock_manager_class) + self._make_feature_only() + process_nats_pipeline_result( + job_id=self.job.pk, + result_data=self._feature_only_result(self.images[0], [[50, 50, 60, 60], [70, 70, 80, 80]]), + reply_subject="reply.features.none", + ) + mock_ack.assert_called_once() + self.assertEqual(self._results_param("unmatched"), 2) + self.assertEqual(self.job.status, JobState.FAILURE.value) + @patch("ami.jobs.tasks.TaskQueueManager") def test_results_counter_does_not_inflate_on_replay(self, mock_manager_class): """ diff --git a/ami/ml/exceptions.py b/ami/ml/exceptions.py index ef94e1a59..d8fa5cbf8 100644 --- a/ami/ml/exceptions.py +++ b/ami/ml/exceptions.py @@ -1,2 +1,14 @@ class PipelineNotConfigured(ValueError): pass + + +class FeatureResultsMatchNoDetections(Exception): + """A feature-only batch returned boxes, and none of them is a detection Antenna has. + + Nothing can be stored for such a batch, and the same detections would be sent again on + every run, so the job fails instead of skipping it. + """ + + def __init__(self, message: str = "", unmatched: int = 0): + super().__init__(message) + self.unmatched = unmatched diff --git a/ami/ml/models/pipeline.py b/ami/ml/models/pipeline.py index 5bb42ed0f..abe289de4 100644 --- a/ami/ml/models/pipeline.py +++ b/ami/ml/models/pipeline.py @@ -40,7 +40,7 @@ update_calculated_fields_for_events, update_occurrence_determination, ) -from ami.ml.exceptions import PipelineNotConfigured +from ami.ml.exceptions import FeatureResultsMatchNoDetections, PipelineNotConfigured from ami.ml.models.algorithm import Algorithm, AlgorithmCategoryMap from ami.ml.schemas import ( AlgorithmConfigResponse, @@ -865,13 +865,32 @@ def _check_embedding_dimensions(algorithm: Algorithm, lengths: set[int]) -> None ) +# How many boxes that match no stored detection are named in the job log per batch. +UNMATCHED_BOXES_TO_LOG = 5 + + +@dataclasses.dataclass +class StoredEmbeddings: + """The vectors stored from one batch, and the returned boxes that matched no detection.""" + + embeddings: list[DetectionEmbedding] + unmatched: list[DetectionResponse] + + +def _describe_boxes(detection_responses: list[DetectionResponse]) -> str: + return "; ".join( + f"image {response.source_image_id} box {tuple(response.bbox.dict().values()) if response.bbox else None}" + for response in detection_responses[:UNMATCHED_BOXES_TO_LOG] + ) + + def create_detection_embeddings( detections: list[Detection], detection_responses: list[DetectionResponse], algorithms_known: dict[str, Algorithm], logger: logging.Logger = logger, job_id: int | None = None, -) -> list[DetectionEmbedding]: +) -> StoredEmbeddings: """ Store the feature vectors sent with each detection, one row per (detection, algorithm). @@ -880,10 +899,11 @@ def create_detection_embeddings( so no determination can change. Responses are matched to ``detections`` by image and box (see ``BOX_MATCH_DECIMALS``), - the key ``get_or_create_detection`` reuses detections by; a response with no match is - skipped. An algorithm key the pipeline has not registered raises ``PipelineNotConfigured``, - as it does for classifications, and a vector whose length differs from its algorithm's - raises ``EmbeddingDimensionMismatch``. ``job_id`` records the job whose results stored each vector. + the key ``get_or_create_detection`` reuses detections by. A returned box with no match is + skipped and listed in the result's ``unmatched``. An algorithm key the pipeline has not + registered raises ``PipelineNotConfigured``, as it does for classifications, and a vector + whose length differs from its algorithm's raises ``EmbeddingDimensionMismatch``. + ``job_id`` records the job whose results stored each vector. """ by_box = { _box_key(detection.source_image_id, detection.bbox): detection @@ -892,15 +912,15 @@ def create_detection_embeddings( } embeddings: dict[tuple[int, int], DetectionEmbedding] = {} lengths_by_algorithm: dict[str, set[int]] = collections.defaultdict(set) - unmatched = 0 + unmatched: list[DetectionResponse] = [] for detection_resp in detection_responses: - if not detection_resp.embeddings or detection_resp.bbox is None: + if detection_resp.bbox is None: continue detection = by_box.get(_box_key(detection_resp.source_image_id, detection_resp.bbox.dict().values())) if detection is None: - unmatched += 1 + unmatched.append(detection_resp) continue - for embedding_resp in detection_resp.embeddings: + for embedding_resp in detection_resp.embeddings or []: try: algorithm = algorithms_known[embedding_resp.algorithm.key] except KeyError as err: @@ -918,7 +938,10 @@ def create_detection_embeddings( _check_embedding_dimensions(algorithms_known[key], lengths) if unmatched: - logger.warning(f"Skipped the vectors of {unmatched} detections that match no stored detection.") + logger.warning( + f"Skipped {len(unmatched)} returned boxes that match no stored detection, " + f"for example: {_describe_boxes(unmatched)}" + ) DetectionEmbedding.objects.bulk_create( list(embeddings.values()), update_conflicts=True, @@ -927,7 +950,7 @@ def create_detection_embeddings( batch_size=EMBEDDING_BATCH_SIZE, ) logger.info(f"Stored {len(embeddings)} detection embeddings for {len(detections)} detections.") - return list(embeddings.values()) + return StoredEmbeddings(embeddings=list(embeddings.values()), unmatched=unmatched) def save_features_for_existing_detections( @@ -935,27 +958,37 @@ def save_features_for_existing_detections( algorithms_known: dict[str, Algorithm], logger: logging.Logger = logger, job_id: int | None = None, -) -> list[DetectionEmbedding]: +) -> StoredEmbeddings: """Store the vectors a feature-only pipeline returned for detections Antenna already has. Writes ``DetectionEmbedding`` rows and nothing else: a box that matches no stored detection is skipped rather than created, classifications in the response are ignored, - and no occurrence, determination or null marker is touched. + and no occurrence, determination or null marker is touched. A batch whose boxes all + match nothing raises ``FeatureResultsMatchNoDetections``, because its detections would + otherwise be sent again on every run. """ ignored = sum(len(detection.classifications) for detection in results.detections) if ignored: logger.warning(f"Ignored {ignored} classifications returned by a feature-only pipeline.") - image_ids = {int(detection.source_image_id) for detection in results.detections if detection.bbox is not None} + returned_boxes = [detection for detection in results.detections if detection.bbox is not None] + image_ids = {int(detection.source_image_id) for detection in returned_boxes} detections = list( Detection.objects.valid().filter(source_image_id__in=image_ids).only("pk", "source_image_id", "bbox") ) - return create_detection_embeddings( + stored = create_detection_embeddings( detections=detections, detection_responses=results.detections, algorithms_known=algorithms_known, logger=logger, job_id=job_id, ) + if returned_boxes and len(stored.unmatched) == len(returned_boxes): + raise FeatureResultsMatchNoDetections( + f"None of the {len(returned_boxes)} boxes returned by the feature-only pipeline matches a " + f"stored detection, so no vectors were saved. First boxes: {_describe_boxes(returned_boxes)}", + unmatched=len(returned_boxes), + ) + return stored def create_category_map_for_classification( @@ -1258,6 +1291,8 @@ class PipelineSaveResults: classifications: list[Classification] algorithms: dict[str, Algorithm] total_time: float + # Returned boxes that matched no stored detection; only a feature-only save reports it. + unmatched_detections: int | None = None def create_null_detections_for_undetected_images( @@ -1340,12 +1375,12 @@ def save_results( algorithms_known: dict[str, Algorithm] = {algo.key: algo for algo in pipeline.algorithms.all()} if feature_extraction_only(list(algorithms_known.values())): job_logger.info(f"Pipeline {pipeline} only extracts features; storing vectors for existing detections.") - embeddings = save_features_for_existing_detections( + stored = save_features_for_existing_detections( results, algorithms_known, logger=job_logger, job_id=job.pk if job else None ) total_time = time.time() - start_time job_logger.info( - f"Saved {len(embeddings)} feature vectors from pipeline {pipeline} in {total_time:.2f} seconds" + f"Saved {len(stored.embeddings)} feature vectors from pipeline {pipeline} in {total_time:.2f} seconds" ) if return_created: return PipelineSaveResults( @@ -1355,6 +1390,7 @@ def save_results( classifications=[], algorithms={}, total_time=total_time, + unmatched_detections=len(stored.unmatched), ) return None @@ -1669,8 +1705,8 @@ def process_images( reprocess_all_images=reprocess_all_images, ) - def save_results(self, results: PipelineResultsResponse, job_id: int | None = None): - return save_results(results=results, job_id=job_id) + def save_results(self, results: PipelineResultsResponse, job_id: int | None = None, return_created=False): + return save_results(results=results, job_id=job_id, return_created=return_created) def save_results_async(self, results: PipelineResultsResponse, job_id: int | None = None): # Returns an AsyncResult diff --git a/ami/ml/test_feature_extraction.py b/ami/ml/test_feature_extraction.py index 835660638..0009f743e 100644 --- a/ami/ml/test_feature_extraction.py +++ b/ami/ml/test_feature_extraction.py @@ -31,6 +31,7 @@ feature_extractors_with_vectors, vectors_for_detections, ) +from ami.ml.exceptions import FeatureResultsMatchNoDetections from ami.ml.models import Algorithm, Pipeline, ProcessingService from ami.ml.models.pipeline import ( COLLECT_PROGRESS_MAX_FRACTION, @@ -200,6 +201,20 @@ def test_an_embeddings_only_response_writes_only_vectors(self): self.extractor.refresh_from_db() self.assertEqual(self.extractor.embedding_dimensions, BIOCLIP_DIMENSIONS) + def test_boxes_that_match_no_detection_are_counted(self): + known = [(image, _box(0.0)) for image in self.images] + unknown = [(self.images[0], _box(500.0)), (self.images[1], _box(600.0))] + saved = save_results(self._response(known + unknown), return_created=True) + self.assertEqual(saved.unmatched_detections, 2) + self.assertEqual(DetectionEmbedding.objects.count(), len(self.images)) + + def test_a_batch_whose_boxes_all_match_nothing_fails_and_stores_nothing(self): + """Skipping it would leave the same detections without vectors, to be sent again on every run.""" + with self.assertRaises(FeatureResultsMatchNoDetections) as raised: + save_results(self._response([(self.images[0], _box(500.0)), (self.images[1], _box(600.0))])) + self.assertEqual(raised.exception.unmatched, 2) + self.assertFalse(DetectionEmbedding.objects.exists()) + def test_a_vector_of_another_length_is_refused_and_nothing_is_stored(self): save_results(self._response([(self.images[0], _box(0.0))])) with self.assertRaises(EmbeddingDimensionMismatch): diff --git a/ami/ml/tests.py b/ami/ml/tests.py index 6c8dfd4c9..6caf3e09c 100644 --- a/ami/ml/tests.py +++ b/ami/ml/tests.py @@ -2555,4 +2555,4 @@ def test_storing_vectors_takes_one_query_however_many_detections(self): with self.assertNumQueries(1): stored = create_detection_embeddings(detections, parsed, algorithms_known) - self.assertEqual(len(stored), 5) + self.assertEqual(len(stored.embeddings), 5) From 32da6cfbfd55a1b01bbe77731050df2ca23297ba Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Mon, 28 Sep 2026 23:57:55 -0700 Subject: [PATCH 44/44] fix(ml): fail a feature-only batch that stores no vector, and count detections left without one A feature-only batch could store nothing and still end in success in two ways the previous guard missed: the service echoed the boxes without vectors, or it returned fewer boxes than it was sent (including none). Either way the same detections were queued again on every run. After each feature-only save, the detections on the batch's images that still have no vector are counted and reported on the results stage as "without_vector". A batch that stores no vector while such detections remain, or whose returned boxes all match no detection, raises FeatureResultsStoredNothing (renamed from FeatureResultsMatchNoDetections, which no longer described every case). A response that reports an error is left to the existing error handling. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01C7Xf6VPbwWtTumhjjF15g8 --- ami/jobs/tasks.py | 38 ++++++++++++++++--------- ami/jobs/tests/test_tasks.py | 2 ++ ami/ml/exceptions.py | 12 ++++---- ami/ml/models/pipeline.py | 47 +++++++++++++++++++++++-------- ami/ml/test_feature_extraction.py | 24 ++++++++++++++-- 5 files changed, 91 insertions(+), 32 deletions(-) diff --git a/ami/jobs/tasks.py b/ami/jobs/tasks.py index ff03d10f3..40ce3704e 100644 --- a/ami/jobs/tasks.py +++ b/ami/jobs/tasks.py @@ -13,7 +13,7 @@ from redis.exceptions import RedisError from ami.main.checks.schemas import IntegrityCheckResult -from ami.ml.exceptions import FeatureResultsMatchNoDetections +from ami.ml.exceptions import FeatureResultsStoredNothing from ami.ml.orchestration.async_job_state import AsyncJobStateManager from ami.ml.orchestration.nats_queue import ConsumerState, TaskQueueManager from ami.ml.schemas import PipelineResultsError, PipelineResultsResponse @@ -324,16 +324,20 @@ def process_nats_pipeline_result(self, job_id: int, result_data: dict, reply_sub try: # Save to database (this is the slow operation) detections_count, classifications_count, captures_count = 0, 0, 0 - unmatched_count: int | None = None + feature_counts: dict[str, int] = {} if pipeline_result: # should never happen since otherwise we could not be processing results here assert job.pipeline is not None, "Job pipeline is None" - # Only a feature-only save reports boxes that matched no detection; asking the - # full save for its created rows would load every detection's algorithm. + # Only a feature-only save reports boxes that matched no detection and detections + # left without a vector; asking the full save for its created rows would load + # every detection's algorithm. feature_only = job.pipeline.is_feature_only() saved = job.pipeline.save_results(results=pipeline_result, job_id=job.pk, return_created=feature_only) if feature_only and saved: - unmatched_count = saved.unmatched_detections + feature_counts = { + "unmatched": saved.unmatched_detections or 0, + "without_vector": saved.detections_without_vector or 0, + } job.logger.info(f"Successfully saved results for job {job_id}") _, t = t( @@ -392,9 +396,7 @@ def process_nats_pipeline_result(self, job_id: int, result_data: dict, reply_sub counts_to_apply = ( (detections_count, classifications_count, captures_count) if is_first_processing else (0, 0, 0) ) - extra_counts = {} - if unmatched_count is not None: - extra_counts["unmatched"] = unmatched_count if is_first_processing else 0 + extra_counts = {key: count if is_first_processing else 0 for key, count in feature_counts.items()} _update_job_progress( job_id, "results", @@ -419,10 +421,17 @@ def process_nats_pipeline_result(self, job_id: int, result_data: dict, reply_sub # Celery's autoretry_for handles the transient rather than this broad # except swallowing it. raise - except FeatureResultsMatchNoDetections as e: - # Redelivering would return the same boxes, and a later job would send the same - # detections again, so record the count and fail the job instead of retrying. - _update_job_progress(job_id, "results", 0, complete_state=JobState.FAILURE, unmatched=e.unmatched) + except FeatureResultsStoredNothing as e: + # Redelivering would return the same response, and a later job would send the same + # detections again, so record the counts and fail the job instead of retrying. + _update_job_progress( + job_id, + "results", + 0, + complete_state=JobState.FAILURE, + unmatched=e.unmatched, + without_vector=e.without_vector, + ) _ack_task_via_nats(reply_subject, job.logger) _fail_job(job_id, str(e)) except Exception as e: @@ -654,8 +663,9 @@ def _update_job_progress( state_params["detections"] = current_detections + new_detections state_params["classifications"] = current_classifications + new_classifications state_params["captures"] = current_captures + new_captures - if "unmatched" in state_params: - state_params["unmatched"] = _get_stage_param(job, stage, "unmatched") + state_params["unmatched"] + for key in ("unmatched", "without_vector"): + if key in state_params: + state_params[key] = _get_stage_param(job, stage, key) + state_params[key] # Don't overwrite a stage with a stale progress value. # This guards against the race where a slower worker calls _update_job_progress diff --git a/ami/jobs/tests/test_tasks.py b/ami/jobs/tests/test_tasks.py index c9b4c3277..553af81e4 100644 --- a/ami/jobs/tests/test_tasks.py +++ b/ami/jobs/tests/test_tasks.py @@ -468,6 +468,7 @@ def test_feature_only_results_count_boxes_that_match_no_detection(self, mock_man reply_subject=f"reply.features.{image.pk}", ) self.assertEqual(self._results_param("unmatched"), 2) + self.assertEqual(self._results_param("without_vector"), 0) self.assertNotEqual(self.job.status, JobState.FAILURE.value) @patch("ami.jobs.tasks._ack_task_via_nats") @@ -483,6 +484,7 @@ def test_feature_only_batch_matching_no_detection_fails_the_job_and_acks(self, m ) mock_ack.assert_called_once() self.assertEqual(self._results_param("unmatched"), 2) + self.assertEqual(self._results_param("without_vector"), 1) self.assertEqual(self.job.status, JobState.FAILURE.value) @patch("ami.jobs.tasks.TaskQueueManager") diff --git a/ami/ml/exceptions.py b/ami/ml/exceptions.py index d8fa5cbf8..eee96c3c7 100644 --- a/ami/ml/exceptions.py +++ b/ami/ml/exceptions.py @@ -2,13 +2,15 @@ class PipelineNotConfigured(ValueError): pass -class FeatureResultsMatchNoDetections(Exception): - """A feature-only batch returned boxes, and none of them is a detection Antenna has. +class FeatureResultsStoredNothing(Exception): + """A feature-only batch stored no vector while its images still have detections without one. - Nothing can be stored for such a batch, and the same detections would be sent again on - every run, so the job fails instead of skipping it. + This happens when no returned box matches a stored detection, when the boxes come back + without vectors, or when the service returns no boxes at all. The same detections would + be sent again on every run, so the job fails instead of skipping the batch. """ - def __init__(self, message: str = "", unmatched: int = 0): + def __init__(self, message: str = "", unmatched: int = 0, without_vector: int = 0): super().__init__(message) self.unmatched = unmatched + self.without_vector = without_vector diff --git a/ami/ml/models/pipeline.py b/ami/ml/models/pipeline.py index abe289de4..b8aadc7bb 100644 --- a/ami/ml/models/pipeline.py +++ b/ami/ml/models/pipeline.py @@ -40,7 +40,7 @@ update_calculated_fields_for_events, update_occurrence_determination, ) -from ami.ml.exceptions import FeatureResultsMatchNoDetections, PipelineNotConfigured +from ami.ml.exceptions import FeatureResultsStoredNothing, PipelineNotConfigured from ami.ml.models.algorithm import Algorithm, AlgorithmCategoryMap from ami.ml.schemas import ( AlgorithmConfigResponse, @@ -871,10 +871,15 @@ def _check_embedding_dimensions(algorithm: Algorithm, lengths: set[int]) -> None @dataclasses.dataclass class StoredEmbeddings: - """The vectors stored from one batch, and the returned boxes that matched no detection.""" + """The vectors stored from one batch, and the returned boxes that matched no detection. + + ``without_vector`` counts the detections on the batch's images that still lack a vector + after the save: boxes the service did not return, or returned without a vector. + """ embeddings: list[DetectionEmbedding] unmatched: list[DetectionResponse] + without_vector: int = 0 def _describe_boxes(detection_responses: list[DetectionResponse]) -> str: @@ -963,9 +968,11 @@ def save_features_for_existing_detections( Writes ``DetectionEmbedding`` rows and nothing else: a box that matches no stored detection is skipped rather than created, classifications in the response are ignored, - and no occurrence, determination or null marker is touched. A batch whose boxes all - match nothing raises ``FeatureResultsMatchNoDetections``, because its detections would - otherwise be sent again on every run. + and no occurrence, determination or null marker is touched. Detections on the batch's + images that still lack a vector afterwards are counted in ``without_vector``. A batch + that stores no vector at all while such detections remain raises + ``FeatureResultsStoredNothing``, because they would otherwise be sent again on every run. + A response that reports an error is left to the error handling of the caller. """ ignored = sum(len(detection.classifications) for detection in results.detections) if ignored: @@ -982,11 +989,26 @@ def save_features_for_existing_detections( logger=logger, job_id=job_id, ) - if returned_boxes and len(stored.unmatched) == len(returned_boxes): - raise FeatureResultsMatchNoDetections( - f"None of the {len(returned_boxes)} boxes returned by the feature-only pipeline matches a " - f"stored detection, so no vectors were saved. First boxes: {_describe_boxes(returned_boxes)}", - unmatched=len(returned_boxes), + batch_image_ids = image_ids | {int(image.id) for image in results.source_images} + extractor_ids = [algorithm.pk for algorithm in feature_extractors_if_feature_only(list(algorithms_known.values()))] + if batch_image_ids and extractor_ids: + stored.without_vector = detections_missing_features( + Detection.objects.filter(source_image_id__in=batch_image_ids), extractor_ids + ).count() + if stored.without_vector: + logger.warning( + f"{stored.without_vector} detections on the {len(batch_image_ids)} images of this batch " + "still have no feature vector after saving it." + ) + all_unmatched = bool(returned_boxes) and len(stored.unmatched) == len(returned_boxes) + if not stored.embeddings and (stored.without_vector or all_unmatched) and not results.errors: + raise FeatureResultsStoredNothing( + f"The feature-only pipeline returned {len(returned_boxes)} boxes and no vector was saved, " + f"while {stored.without_vector} detections on these images still have none " + f"({len(stored.unmatched)} returned boxes match no stored detection). " + f"First boxes: {_describe_boxes(returned_boxes)}", + unmatched=len(stored.unmatched), + without_vector=stored.without_vector, ) return stored @@ -1291,8 +1313,10 @@ class PipelineSaveResults: classifications: list[Classification] algorithms: dict[str, Algorithm] total_time: float - # Returned boxes that matched no stored detection; only a feature-only save reports it. + # Only a feature-only save reports these: returned boxes that matched no stored detection, + # and detections on the saved images that still have no vector. unmatched_detections: int | None = None + detections_without_vector: int | None = None def create_null_detections_for_undetected_images( @@ -1391,6 +1415,7 @@ def save_results( algorithms={}, total_time=total_time, unmatched_detections=len(stored.unmatched), + detections_without_vector=stored.without_vector, ) return None diff --git a/ami/ml/test_feature_extraction.py b/ami/ml/test_feature_extraction.py index 0009f743e..23f65d5d2 100644 --- a/ami/ml/test_feature_extraction.py +++ b/ami/ml/test_feature_extraction.py @@ -31,7 +31,7 @@ feature_extractors_with_vectors, vectors_for_detections, ) -from ami.ml.exceptions import FeatureResultsMatchNoDetections +from ami.ml.exceptions import FeatureResultsStoredNothing from ami.ml.models import Algorithm, Pipeline, ProcessingService from ami.ml.models.pipeline import ( COLLECT_PROGRESS_MAX_FRACTION, @@ -210,11 +210,31 @@ def test_boxes_that_match_no_detection_are_counted(self): def test_a_batch_whose_boxes_all_match_nothing_fails_and_stores_nothing(self): """Skipping it would leave the same detections without vectors, to be sent again on every run.""" - with self.assertRaises(FeatureResultsMatchNoDetections) as raised: + with self.assertRaises(FeatureResultsStoredNothing) as raised: save_results(self._response([(self.images[0], _box(500.0)), (self.images[1], _box(600.0))])) self.assertEqual(raised.exception.unmatched, 2) self.assertFalse(DetectionEmbedding.objects.exists()) + def test_boxes_returned_without_vectors_fail_the_batch(self): + """Matching boxes that carry no vector store nothing, so the batch must not pass as a success.""" + image = self.images[0] + with self.assertRaises(FeatureResultsStoredNothing) as raised: + save_results(self._response([(image, _box(0.0)), (image, _box(20.0))], embeddings=[])) + self.assertEqual((raised.exception.unmatched, raised.exception.without_vector), (0, 2)) + self.assertFalse(DetectionEmbedding.objects.exists()) + + def test_a_response_with_no_boxes_fails_the_batch(self): + response = self._response([]) + response.source_images = [{"id": str(self.images[0].pk), "url": "x"}] + with self.assertRaises(FeatureResultsStoredNothing) as raised: + save_results(PipelineResultsResponse.parse_obj(response.dict())) + self.assertEqual(raised.exception.without_vector, 2) + + def test_detections_the_service_did_not_return_are_counted(self): + saved = save_results(self._response([(self.images[0], _box(0.0))]), return_created=True) + self.assertEqual(saved.detections_without_vector, 1) + self.assertEqual(DetectionEmbedding.objects.count(), 1) + def test_a_vector_of_another_length_is_refused_and_nothing_is_stored(self): save_results(self._response([(self.images[0], _box(0.0))])) with self.assertRaises(EmbeddingDimensionMismatch):